diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index 531ed060fd3866..0fe754bb071ea3 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -57,7 +57,7 @@ jobs: variables: testRunTitle: '$(build.sourceBranchName)-linux' testRunPlatform: linux - openssl_version: 1.1.1g + openssl_version: 1.1.1k steps: - template: ./posix-steps.yml @@ -83,7 +83,7 @@ jobs: variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.1g + openssl_version: 1.1.1k steps: - template: ./posix-steps.yml @@ -104,7 +104,7 @@ jobs: matrix: win32: arch: win32 - buildOpt: + buildOpt: '-p Win32' testRunTitle: '$(Build.SourceBranchName)-win32' testRunPlatform: win32 win64: diff --git a/.azure-pipelines/docs-steps.yml b/.azure-pipelines/docs-steps.yml index eaf4ad01927aeb..647daff7a033a8 100644 --- a/.azure-pipelines/docs-steps.yml +++ b/.azure-pipelines/docs-steps.yml @@ -12,11 +12,12 @@ steps: inputs: versionSpec: '>=3.6' -- script: python -m pip install sphinx==2.2.0 blurb python-docs-theme +- script: python -m pip install -r requirements.txt + workingDirectory: '$(build.sourcesDirectory)/Doc' displayName: 'Install build dependencies' - ${{ if ne(parameters.latex, 'true') }}: - - script: make check suspicious html PYTHON=python + - script: make check html PYTHON=python workingDirectory: '$(build.sourcesDirectory)/Doc' displayName: 'Build documentation' @@ -31,7 +32,7 @@ steps: - ${{ if eq(parameters.upload, 'true') }}: - task: PublishBuildArtifacts@1 displayName: 'Publish docs' - + inputs: PathToPublish: '$(build.sourcesDirectory)/Doc/build' ArtifactName: docs diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index 1ffe0a97a2465f..2d32e6d49bcc0e 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -57,7 +57,7 @@ jobs: variables: testRunTitle: '$(system.pullRequest.TargetBranch)-linux' testRunPlatform: linux - openssl_version: 1.1.1g + openssl_version: 1.1.1k steps: - template: ./posix-steps.yml @@ -83,7 +83,7 @@ jobs: variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.1g + openssl_version: 1.1.1k steps: - template: ./posix-steps.yml @@ -104,7 +104,7 @@ jobs: matrix: win32: arch: win32 - buildOpt: + buildOpt: '-p Win32' testRunTitle: '$(System.PullRequest.TargetBranch)-win32' testRunPlatform: win32 win64: diff --git a/.azure-pipelines/windows-release/stage-pack-msix.yml b/.azure-pipelines/windows-release/stage-pack-msix.yml index 26a5712e845ca9..f967cfdbe326f8 100644 --- a/.azure-pipelines/windows-release/stage-pack-msix.yml +++ b/.azure-pipelines/windows-release/stage-pack-msix.yml @@ -120,10 +120,11 @@ jobs: artifactName: unsigned_msix downloadPath: $(Build.BinariesDirectory) + # MSIX must be signed and timestamped simultaneously - powershell: | $failed = $true foreach ($retry in 1..3) { - signtool sign /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "$(SigningDescription)" (gi *.msix) + signtool sign /a /n "$(SigningCertificate)" /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d "$(SigningDescription)" (gi *.msix) if ($?) { $failed = $false break diff --git a/.azure-pipelines/windows-release/stage-pack-nuget.yml b/.azure-pipelines/windows-release/stage-pack-nuget.yml index b100364820d95b..8dfea382c35622 100644 --- a/.azure-pipelines/windows-release/stage-pack-nuget.yml +++ b/.azure-pipelines/windows-release/stage-pack-nuget.yml @@ -4,7 +4,7 @@ jobs: condition: and(succeeded(), eq(variables['DoNuget'], 'true')) pool: - vmImage: windows-2019 + name: 'Windows Release' workspace: clean: all @@ -36,6 +36,14 @@ jobs: nuget pack "$(Build.BinariesDirectory)\layout\python.nuspec" -OutputDirectory $(Build.ArtifactStagingDirectory) -NoPackageAnalysis -NonInteractive displayName: 'Create nuget package' + - powershell: | + gci *.nupkg | %{ + nuget sign "$_" -CertificateSubjectName "$(SigningCertificate)" -Timestamper http://timestamp.digicert.com/ -Overwrite + } + displayName: 'Sign nuget package' + workingDirectory: $(Build.ArtifactStagingDirectory) + condition: and(succeeded(), variables['SigningCertificate']) + - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact: nuget' inputs: diff --git a/.azure-pipelines/windows-release/stage-sign.yml b/.azure-pipelines/windows-release/stage-sign.yml index 584772af8b428e..c21e1c9f2b0f9b 100644 --- a/.azure-pipelines/windows-release/stage-sign.yml +++ b/.azure-pipelines/windows-release/stage-sign.yml @@ -57,7 +57,7 @@ jobs: $files = (gi ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }}) $failed = $true foreach ($retry in 1..10) { - signtool timestamp /t http://timestamp.verisign.com/scripts/timestamp.dll $files + signtool timestamp /tr http://timestamp.digicert.com/ /td sha256 $files if ($?) { $failed = $false break diff --git a/.gitattributes b/.gitattributes index 3827d09709d4a7..c66e765266382f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -47,7 +47,7 @@ PC/clinic/*.h linguist-generated=true Python/clinic/*.h linguist-generated=true Python/importlib.h linguist-generated=true Python/importlib_external.h linguist-generated=true -Include/Python-ast.h linguist-generated=true +Include/internal/pycore_ast.h linguist-generated=true Python/Python-ast.c linguist-generated=true Include/opcode.h linguist-generated=true Python/opcode_targets.h linguist-generated=true diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8c76f54af5504a..5b86d39dc9c259 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -19,6 +19,7 @@ Objects/frameobject.c @markshannon Objects/call.c @markshannon Python/ceval.c @markshannon Python/compile.c @markshannon +Python/ast_opt.c @isidentical # Hashing **/*hashlib* @python/crypto-team @tiran @@ -84,6 +85,16 @@ Include/pytime.h @pganssle @abalkin /Lib/test/test_peg_generator/ @pablogsal @lysnikolaou /Grammar/python.gram @pablogsal @lysnikolaou +# AST +Python/ast.c @isidentical +Parser/asdl.py @isidentical +Parser/asdl_c.py @isidentical +Lib/ast.py @isidentical + +# Mock +/Lib/unittest/mock.py @cjw296 +/Lib/unittest/test/testmock/* @cjw296 + # SQLite 3 **/*sqlite* @berkerpeksag diff --git a/.github/codecov.yml b/.github/codecov.yml deleted file mode 100644 index ea504f48672eac..00000000000000 --- a/.github/codecov.yml +++ /dev/null @@ -1,30 +0,0 @@ -codecov: - strict_yaml_branch: master - notify: - require_ci_to_pass: true -comment: off -ignore: - - "Doc/**/*" - - "Misc/**/*" - - "Mac/**/*" - - "PC/**/*" - - "PCbuild/**/*" - - "Tools/**/*" - - "Grammar/*" -coverage: - precision: 2 - range: 70...90 - round: down - status: - changes: off - project: off - patch: off -parsers: - gcov: - branch_detection: - conditional: true - loop: true - macro: false - method: false - javascript: - enable_partials: false diff --git a/.github/problem-matchers/msvc.json b/.github/problem-matchers/msvc.json new file mode 100644 index 00000000000000..303a36b1c585e6 --- /dev/null +++ b/.github/problem-matchers/msvc.json @@ -0,0 +1,19 @@ +{ + "__comment": "Taken from vscode's vs/workbench/contrib/tasks/common/problemMatcher.ts msCompile rule", + "problemMatcher": [ + { + "owner": "msvc-problem-matcher", + "pattern": [ + { + "regexp": "^(?:\\s+\\d+\\>)?([^\\s].*)\\((\\d+),?(\\d+)?(?:,\\d+,\\d+)?\\)\\s*:\\s+(error|warning|info)\\s+(\\w{1,2}\\d+)\\s*:\\s*(.*)$", + "file": 1, + "line": 2, + "column": 3, + "severity": 4, + "code": 5, + "message": 6 + } + ] + } + ] + } \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d00c227182f61a..41200bb957e47c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,13 +23,15 @@ jobs: runs-on: ubuntu-latest outputs: run_tests: ${{ steps.check.outputs.run_tests }} + run_ssl_tests: ${{ steps.check.outputs.run_ssl_tests }} steps: - uses: actions/checkout@v2 - name: Check for source changes id: check run: | - if [ -z "GITHUB_BASE_REF" ]; then + if [ -z "$GITHUB_BASE_REF" ]; then echo '::set-output name=run_tests::true' + echo '::set-output name=run_ssl_tests::true' else git fetch origin $GITHUB_BASE_REF --depth=1 # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more @@ -46,7 +48,40 @@ jobs: # # https://github.com/python/core-workflow/issues/373 git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo '::set-output name=run_tests::true' || true + git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE '(ssl|hashlib|hmac|^.github)' && echo '::set-output name=run_ssl_tests::true' || true fi + + check_generated_files: + name: 'Check if generated files are up to date' + runs-on: ubuntu-latest + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Build CPython + run: | + # Build Python with the libpython dynamic library + ./configure --with-pydebug --enable-shared + make -j4 regen-all + make regen-stdlib-module-names + - name: Check for changes + run: | + changes=$(git status --porcelain) + # Check for changes in regenerated files + if ! test -z "$changes" + then + echo "Generated files not up to date. Perhaps you forgot to run make regen-all ;)" + echo "$changes" + exit 1 + fi + - name: Check exported libpython symbols + run: make smelly + - name: Check limited ABI symbols + run: make check-limited-abi + build_win32: name: 'Windows (x86)' runs-on: windows-latest @@ -59,7 +94,7 @@ jobs: - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests - run: .\PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + run: .\PCbuild\rt.bat -p Win32 -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 build_win_amd64: name: 'Windows (x64)' @@ -68,12 +103,14 @@ jobs: if: needs.check_source.outputs.run_tests == 'true' steps: - uses: actions/checkout@v2 + - name: Register MSVC problem matcher + run: echo "::add-matcher::.github/problem-matchers/msvc.json" - name: Build CPython run: .\PCbuild\build.bat -e -p x64 - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests - run: .\PCbuild\rt.bat -x64 -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + run: .\PCbuild\rt.bat -p x64 -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 build_macos: name: 'macOS' @@ -93,31 +130,89 @@ jobs: build_ubuntu: name: 'Ubuntu' - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1f + OPENSSL_VER: 1.1.1k steps: - uses: actions/checkout@v2 - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v2.1.2 + uses: actions/cache@v2.1.4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1 - name: Configure CPython - run: ./configure --with-pydebug --with-openssl=$PWD/multissl/openssl/$OPENSSL_VER + run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR - name: Build CPython run: make -j4 - name: Display build info run: make pythoninfo - name: Tests run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + + build_ubuntu_ssltests: + name: 'Ubuntu SSL tests with OpenSSL' + runs-on: ubuntu-20.04 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_ssl_tests == 'true' + strategy: + fail-fast: false + matrix: + openssl_ver: [1.1.1k, 3.0.0-alpha15] + env: + OPENSSL_VER: ${{ matrix.openssl_ver }} + MULTISSL_DIR: ${{ github.workspace }}/multissl + OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }} + LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib + steps: + - uses: actions/checkout@v2 + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v2.1.4 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1 + - name: Configure CPython + run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: SSL tests + run: ./python Lib/test/ssltests.py diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml deleted file mode 100644 index ed71a012395d4f..00000000000000 --- a/.github/workflows/coverage.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: Coverage - -on: - push: - branches: - - master - - 3.9 - - 3.8 - - 3.7 - paths-ignore: - - 'Doc/**' - - 'Misc/**' - #pull_request: - # branches: - # - master - # - 3.9 - # - 3.8 - # - 3.7 - # paths-ignore: - # - 'Doc/**' - # - 'Misc/**' - -jobs: - coverage_ubuntu: - name: 'Ubuntu (Coverage)' - runs-on: ubuntu-latest - env: - OPENSSL_VER: 1.1.1f - steps: - - uses: actions/checkout@v2 - - name: Install Dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v2.1.2 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $OPENSSL_VER --system Linux - - name: Configure CPython - run: ./configure --with-openssl=$PWD/multissl/openssl/$OPENSSL_VER - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: 'Coverage Preparation' - run: | - ./python -m venv .venv - source ./.venv/bin/activate - python -m pip install -U coverage - python -m pip install -r Misc/requirements-test.txt - python -m test.pythoninfo - export PYTHONPATH=`find .venv -name fullcoverage` - - name: 'Tests with coverage' - run: > - source ./.venv/bin/activate && - xvfb-run python -m coverage - run --branch --pylib - -m test - --fail-env-changed - -uall,-cpu - -x test_multiprocessing_fork - -x test_multiprocessing_forkserver - -x test_multiprocessing_spawn - -x test_concurrent_futures - || true - - name: 'Publish code coverage results' - run: | - export PYTHONPATH= - source ./.venv/bin/activate - bash <(curl -s https://codecov.io/bash) -y .github/codecov.yml - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - - c_coverage_ubuntu: - name: 'Ubuntu (C Coverage)' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Install Dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure CPython - run: ./configure - - name: 'Build CPython and measure coverage' - run: xvfb-run make -j4 coverage-report - - name: 'Publish code coverage results' - if: always() - run: | - make pythoninfo - bash <(curl -s https://codecov.io/bash) -y .github/codecov.yml - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 7c6012eed4f2d2..8924fc992e25e6 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -36,9 +36,9 @@ jobs: - name: 'Install build dependencies' run: make -C Doc/ PYTHON=../python venv - name: 'Build documentation' - run: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W --keep-going -j4" doctest suspicious html + run: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W --keep-going -j4" doctest html suspicious - name: 'Upload' - uses: actions/upload-artifact@v2.2.0 + uses: actions/upload-artifact@v2.2.2 with: name: doc-html path: Doc/build/html diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index 2b879d32f8150d..56cc70edf60001 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -3,19 +3,20 @@ apt-get update apt-get -yq install \ build-essential \ - zlib1g-dev \ + ccache \ + gdb \ + lcov \ libbz2-dev \ + libffi-dev \ + libgdbm-dev \ liblzma-dev \ libncurses5-dev \ libreadline6-dev \ libsqlite3-dev \ libssl-dev \ - libgdbm-dev \ - tk-dev \ lzma \ lzma-dev \ - liblzma-dev \ - libffi-dev \ + tk-dev \ uuid-dev \ xvfb \ - lcov + zlib1g-dev diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 00000000000000..26806fad814f1a --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,22 @@ +name: Mark stale pull requests + +on: + schedule: + - cron: "0 0 * * *" + +permissions: + pull-requests: write + +jobs: + stale: + + runs-on: ubuntu-latest + + steps: + - uses: actions/stale@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.' + stale-pr-label: 'stale' + days-before-stale: 30 + days-before-close: -1 diff --git a/.travis.yml b/.travis.yml index 254a4ea35ab944..1112a0b266227b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ cache: env: global: - - OPENSSL=1.1.1f + - OPENSSL=1.1.1k - OPENSSL_DIR="$HOME/multissl/openssl/${OPENSSL}" - PATH="${OPENSSL_DIR}/bin:$PATH" - CFLAGS="-I${OPENSSL_DIR}/include" @@ -53,7 +53,7 @@ matrix: - cd Doc - make venv PYTHON=python script: - - make check suspicious html SPHINXOPTS="-q -W -j4" + - make check html suspicious SPHINXOPTS="-q -W -j4" - name: "Documentation tests" os: linux language: c @@ -69,61 +69,6 @@ matrix: - make -C Doc/ PYTHON=../python venv script: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W -j4" doctest - - name: "Test code coverage (Python)" - os: linux - language: c - compiler: gcc - env: OPTIONAL=true - addons: - apt: - packages: - - xvfb - before_script: - - | - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]] - then - echo "Don't run Python coverage on pull requests." - exit - fi - - ./configure - - make -j4 - # Need a venv that can parse covered code. - - ./python -m venv venv - - ./venv/bin/python -m pip install -U coverage - - ./venv/bin/python -m pip install -r Misc/requirements-test.txt - - ./venv/bin/python -m test.pythoninfo - - export PYTHONPATH=`find venv -name fullcoverage` - script: - # Skip tests that re-run the entire test suite. - - xvfb-run ./venv/bin/python -m coverage run --branch --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures || true - after_script: # Probably should be after_success once test suite updated to run under coverage.py. - # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files. - - export PYTHONPATH= - - source ./venv/bin/activate - - bash <(curl -s https://codecov.io/bash) -y .github/codecov.yml - - name: "Test code coverage (C)" - os: linux - language: c - compiler: gcc - env: OPTIONAL=true - addons: - apt: - packages: - - lcov - - xvfb - before_script: - - | - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]] - then - echo "Don't run C coverage on pull requests." - exit - fi - - ./configure - script: - - xvfb-run make -j4 coverage-report - after_script: # Probably should be after_success once test suite updated to run under coverage.py. - - make pythoninfo - - bash <(curl -s https://codecov.io/bash) -y .github/codecov.yml before_install: @@ -172,6 +117,7 @@ before_script: - eval "$(pyenv init -)" - pyenv global 3.8 - PYTHON_FOR_REGEN=python3.8 make -j4 regen-all + - make regen-stdlib-module-names - changes=`git status --porcelain` - | # Check for changes in regenerated files @@ -192,6 +138,8 @@ script: - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./python Tools/scripts/patchcheck.py --travis $TRAVIS_PULL_REQUEST; fi # Check that all symbols exported by libpython start with "Py" or "_Py" - make smelly + # Check that all symbols in the limited abi are present + - make check-limited-abi # `-r -w` implicitly provided through `make buildbottest`. - | if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then diff --git a/Doc/Makefile b/Doc/Makefile index f653d70674eb1c..f113dd06539869 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -202,6 +202,7 @@ dist: check: $(PYTHON) tools/rstlint.py -i tools -i $(VENVDIR) -i README.rst + $(PYTHON) tools/rstlint.py ../Misc/NEWS.d/next/ serve: $(PYTHON) ../Tools/scripts/serve.py build/html @@ -215,12 +216,11 @@ serve: # for development releases: always build autobuild-dev: - make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A switchers=1' - -make suspicious + make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' # for quick rebuilds (HTML only) autobuild-dev-html: - make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A switchers=1' + make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' # for stable releases: only build if not in pre-release stage (alpha, beta) # release candidate downloads are okay, since the stable tree can be in that stage diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index bdaae44e240a02..1d93b35dc1c884 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -149,7 +149,7 @@ which disallows mutable objects such as :class:`bytearray`. Previously, :exc:`TypeError` was raised when embedded null code points were encountered in the Python string. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsWideCharString`. @@ -158,7 +158,7 @@ which disallows mutable objects such as :class:`bytearray`. Unicode data buffer, the second one its length. This variant allows null code points. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsWideCharString`. @@ -166,7 +166,7 @@ which disallows mutable objects such as :class:`bytearray`. Like ``u``, but the Python object may also be ``None``, in which case the :c:type:`Py_UNICODE` pointer is set to ``NULL``. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsWideCharString`. @@ -174,7 +174,7 @@ which disallows mutable objects such as :class:`bytearray`. Like ``u#``, but the Python object may also be ``None``, in which case the :c:type:`Py_UNICODE` pointer is set to ``NULL``. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsWideCharString`. @@ -482,7 +482,8 @@ API Functions *min* and no more than *max*; *min* and *max* may be equal. Additional arguments must be passed to the function, each of which should be a pointer to a :c:type:`PyObject*` variable; these will be filled in with the values from - *args*; they will contain borrowed references. The variables which correspond + *args*; they will contain :term:`borrowed references `. + The variables which correspond to optional parameters not given by *args* will not be filled in; these should be initialized by the caller. This function returns true on success and false if *args* is not a tuple or contains the wrong number of elements; an exception diff --git a/Doc/c-api/bool.rst b/Doc/c-api/bool.rst index ce8de6e22f44ca..c197d447e9618c 100644 --- a/Doc/c-api/bool.rst +++ b/Doc/c-api/bool.rst @@ -13,7 +13,8 @@ are available, however. .. c:function:: int PyBool_Check(PyObject *o) - Return true if *o* is of type :c:data:`PyBool_Type`. + Return true if *o* is of type :c:data:`PyBool_Type`. This function always + succeeds. .. c:var:: PyObject* Py_False diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index b2f409c15abb7a..30bcfc7cf9f500 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -25,13 +25,13 @@ Type check macros .. c:function:: int PyByteArray_Check(PyObject *o) Return true if the object *o* is a bytearray object or an instance of a - subtype of the bytearray type. + subtype of the bytearray type. This function always succeeds. .. c:function:: int PyByteArray_CheckExact(PyObject *o) Return true if the object *o* is a bytearray object, but not an instance of a - subtype of the bytearray type. + subtype of the bytearray type. This function always succeeds. Direct API functions diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 0e33ed2c7c9403..de65701037a7c1 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -25,13 +25,13 @@ called with a non-bytes parameter. .. c:function:: int PyBytes_Check(PyObject *o) Return true if the object *o* is a bytes object or an instance of a subtype - of the bytes type. + of the bytes type. This function always succeeds. .. c:function:: int PyBytes_CheckExact(PyObject *o) Return true if the object *o* is a bytes object, but not an instance of a - subtype of the bytes type. + subtype of the bytes type. This function always succeeds. .. c:function:: PyObject* PyBytes_FromString(const char *v) diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst index 5eb313c89bfd59..908e92653dd483 100644 --- a/Doc/c-api/capsule.rst +++ b/Doc/c-api/capsule.rst @@ -34,7 +34,8 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:function:: int PyCapsule_CheckExact(PyObject *p) - Return true if its argument is a :c:type:`PyCapsule`. + Return true if its argument is a :c:type:`PyCapsule`. This function always + succeeds. .. c:function:: PyObject* PyCapsule_New(void *pointer, const char *name, PyCapsule_Destructor destructor) diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index 8408f7e398db7b..ac4ef5adc5cc20 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -27,7 +27,8 @@ Cell objects are not likely to be useful elsewhere. .. c:function:: int PyCell_Check(ob) - Return true if *ob* is a cell object; *ob* must not be ``NULL``. + Return true if *ob* is a cell object; *ob* must not be ``NULL``. This + function always succeeds. .. c:function:: PyObject* PyCell_New(PyObject *ob) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 6f8c41ccbf6e85..6e18a4225e8f43 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -27,7 +27,7 @@ bound into a function. .. c:function:: int PyCode_Check(PyObject *co) - Return true if *co* is a :class:`code` object. + Return true if *co* is a :class:`code` object. This function always succeeds. .. c:function:: int PyCode_GetNumFree(PyCodeObject *co) @@ -51,3 +51,11 @@ bound into a function. Return a new empty code object with the specified filename, function name, and first line number. It is illegal to :func:`exec` or :func:`eval` the resulting code object. + +.. c:function:: int PyCode_Addr2Line(PyCodeObject *co, int byte_offset) + + Return the line number of the instruction that occurs on or before ``byte_offset`` and ends after it. + If you just need the line number of a frame, use :c:func:`PyFrame_GetLineNumber` instead. + + For efficiently iterating over the line numbers in a code object, use `the API described in PEP 626 + `_. diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 06dbb2572725ee..e2ea766b3a32a7 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -94,13 +94,13 @@ Complex Numbers as Python Objects .. c:function:: int PyComplex_Check(PyObject *p) Return true if its argument is a :c:type:`PyComplexObject` or a subtype of - :c:type:`PyComplexObject`. + :c:type:`PyComplexObject`. This function always succeeds. .. c:function:: int PyComplex_CheckExact(PyObject *p) Return true if its argument is a :c:type:`PyComplexObject`, but not a subtype of - :c:type:`PyComplexObject`. + :c:type:`PyComplexObject`. This function always succeeds. .. c:function:: PyObject* PyComplex_FromCComplex(Py_complex v) diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index bf263d6e4c2641..c1d9fa1b41a3fe 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -115,4 +115,3 @@ Other Objects coro.rst contextvars.rst datetime.rst - decimal.rst diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst index 9c088814314a81..d970f5443b1df5 100644 --- a/Doc/c-api/contextvars.rst +++ b/Doc/c-api/contextvars.rst @@ -107,9 +107,9 @@ Context variable functions: .. c:function:: PyObject *PyContextVar_New(const char *name, PyObject *def) Create a new ``ContextVar`` object. The *name* parameter is used - for introspection and debug purposes. The *def* parameter may optionally - specify the default value for the context variable. If an error has - occurred, this function returns ``NULL``. + for introspection and debug purposes. The *def* parameter specifies + a default value for the context variable, or ``NULL`` for no default. + If an error has occurred, this function returns ``NULL``. .. c:function:: int PyContextVar_Get(PyObject *var, PyObject *default_value, PyObject **value) @@ -124,13 +124,12 @@ Context variable functions: - the default value of *var*, if not ``NULL``; - ``NULL`` - If the value was found, the function will create a new reference to it. + Except for ``NULL``, the function returns a new reference. .. c:function:: PyObject *PyContextVar_Set(PyObject *var, PyObject *value) - Set the value of *var* to *value* in the current context. Returns a - pointer to a :c:type:`PyObject` object, or ``NULL`` if an error - has occurred. + Set the value of *var* to *value* in the current context. Returns + a new token object for this change, or ``NULL`` if an error has occurred. .. c:function:: int PyContextVar_Reset(PyObject *var, PyObject *token) diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index efbaa52e2dc466..7b4cc1cacdd4ab 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -11,21 +11,21 @@ Functions for number conversion and formatted string output. .. c:function:: int PyOS_snprintf(char *str, size_t size, const char *format, ...) Output not more than *size* bytes to *str* according to the format string - *format* and the extra arguments. See the Unix man page :manpage:`snprintf(2)`. + *format* and the extra arguments. See the Unix man page :manpage:`snprintf(3)`. .. c:function:: int PyOS_vsnprintf(char *str, size_t size, const char *format, va_list va) Output not more than *size* bytes to *str* according to the format string *format* and the variable argument list *va*. Unix man page - :manpage:`vsnprintf(2)`. + :manpage:`vsnprintf(3)`. :c:func:`PyOS_snprintf` and :c:func:`PyOS_vsnprintf` wrap the Standard C library functions :c:func:`snprintf` and :c:func:`vsnprintf`. Their purpose is to guarantee consistent behavior in corner cases, which the Standard C functions do not. -The wrappers ensure that *str*[*size*-1] is always ``'\0'`` upon return. They +The wrappers ensure that ``str[size-1]`` is always ``'\0'`` upon return. They never write more than *size* bytes (including the trailing ``'\0'``) into str. Both functions require that ``str != NULL``, ``size > 0``, ``format != NULL`` and ``size < INT_MAX``. @@ -34,13 +34,13 @@ The return value (*rv*) for these functions should be interpreted as follows: * When ``0 <= rv < size``, the output conversion was successful and *rv* characters were written to *str* (excluding the trailing ``'\0'`` byte at - *str*[*rv*]). + ``str[rv]``). * When ``rv >= size``, the output conversion was truncated and a buffer with - ``rv + 1`` bytes would have been needed to succeed. *str*[*size*-1] is ``'\0'`` + ``rv + 1`` bytes would have been needed to succeed. ``str[size-1]`` is ``'\0'`` in this case. -* When ``rv < 0``, "something bad happened." *str*[*size*-1] is ``'\0'`` in +* When ``rv < 0``, "something bad happened." ``str[size-1]`` is ``'\0'`` in this case too, but the rest of *str* is undefined. The exact cause of the error depends on the underlying platform. diff --git a/Doc/c-api/coro.rst b/Doc/c-api/coro.rst index 2260944a9a93a9..caa855a10d20ca 100644 --- a/Doc/c-api/coro.rst +++ b/Doc/c-api/coro.rst @@ -24,6 +24,7 @@ return. .. c:function:: int PyCoro_CheckExact(PyObject *ob) Return true if *ob*'s type is :c:type:`PyCoro_Type`; *ob* must not be ``NULL``. + This function always succeeds. .. c:function:: PyObject* PyCoro_New(PyFrameObject *frame, PyObject *name, PyObject *qualname) diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index 415ce4cac67f15..4c4e4bcfa63335 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -28,61 +28,66 @@ Type-check macros: .. c:function:: int PyDate_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateType` or a subtype of - :c:data:`PyDateTime_DateType`. *ob* must not be ``NULL``. + :c:data:`PyDateTime_DateType`. *ob* must not be ``NULL``. This function always + succeeds. .. c:function:: int PyDate_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateType`. *ob* must not be - ``NULL``. + ``NULL``. This function always succeeds. .. c:function:: int PyDateTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType` or a subtype of - :c:data:`PyDateTime_DateTimeType`. *ob* must not be ``NULL``. + :c:data:`PyDateTime_DateTimeType`. *ob* must not be ``NULL``. This function always + succeeds. .. c:function:: int PyDateTime_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType`. *ob* must not - be ``NULL``. + be ``NULL``. This function always succeeds. .. c:function:: int PyTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TimeType` or a subtype of - :c:data:`PyDateTime_TimeType`. *ob* must not be ``NULL``. + :c:data:`PyDateTime_TimeType`. *ob* must not be ``NULL``. This function always + succeeds. .. c:function:: int PyTime_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TimeType`. *ob* must not be - ``NULL``. + ``NULL``. This function always succeeds. .. c:function:: int PyDelta_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DeltaType` or a subtype of - :c:data:`PyDateTime_DeltaType`. *ob* must not be ``NULL``. + :c:data:`PyDateTime_DeltaType`. *ob* must not be ``NULL``. This function always + succeeds. .. c:function:: int PyDelta_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DeltaType`. *ob* must not be - ``NULL``. + ``NULL``. This function always succeeds. .. c:function:: int PyTZInfo_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType` or a subtype of - :c:data:`PyDateTime_TZInfoType`. *ob* must not be ``NULL``. + :c:data:`PyDateTime_TZInfoType`. *ob* must not be ``NULL``. This function always + succeeds. .. c:function:: int PyTZInfo_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType`. *ob* must not be - ``NULL``. + ``NULL``. This function always succeeds. Macros to create objects: diff --git a/Doc/c-api/decimal.rst b/Doc/c-api/decimal.rst deleted file mode 100644 index 94cc4a7b845791..00000000000000 --- a/Doc/c-api/decimal.rst +++ /dev/null @@ -1,231 +0,0 @@ -.. sectionauthor:: Stefan Krah - -.. highlight:: c - - -Decimal capsule API -=================== - -Capsule API functions can be used in the same manner as regular library -functions, provided that the API has been initialized. - - -Initialize ----------- - -Typically, a C extension module that uses the decimal API will do these -steps in its init function: - -.. code-block:: c - - #include "pydecimal.h" - - static int decimal_initialized = 0; - if (!decimal_initialized) { - if (import_decimal() < 0) { - return NULL; - } - - decimal_initialized = 1; - } - - -Type checking, predicates, accessors ------------------------------------- - -.. c:function:: int PyDec_TypeCheck(const PyObject *dec) - - Return 1 if ``dec`` is a Decimal, 0 otherwise. This function does not set - any exceptions. - - -.. c:function:: int PyDec_IsSpecial(const PyObject *dec) - - Return 1 if ``dec`` is ``NaN``, ``sNaN`` or ``Infinity``, 0 otherwise. - - Set TypeError and return -1 if ``dec`` is not a Decimal. It is guaranteed that - this is the only failure mode, so if ``dec`` has already been type-checked, no - errors can occur and the function can be treated as a simple predicate. - - -.. c:function:: int PyDec_IsNaN(const PyObject *dec) - - Return 1 if ``dec`` is ``NaN`` or ``sNaN``, 0 otherwise. - - Set TypeError and return -1 if ``dec`` is not a Decimal. It is guaranteed that - this is the only failure mode, so if ``dec`` has already been type-checked, no - errors can occur and the function can be treated as a simple predicate. - - -.. c:function:: int PyDec_IsInfinite(const PyObject *dec) - - Return 1 if ``dec`` is ``Infinity``, 0 otherwise. - - Set TypeError and return -1 if ``dec`` is not a Decimal. It is guaranteed that - this is the only failure mode, so if ``dec`` has already been type-checked, no - errors can occur and the function can be treated as a simple predicate. - - -.. c:function:: int64_t PyDec_GetDigits(const PyObject *dec) - - Return the number of digits in the coefficient. For ``Infinity``, the - number of digits is always zero. Typically, the same applies to ``NaN`` - and ``sNaN``, but both of these can have a payload that is equivalent to - a coefficient. Therefore, ``NaNs`` can have a nonzero return value. - - Set TypeError and return -1 if ``dec`` is not a Decimal. It is guaranteed that - this is the only failure mode, so if ``dec`` has already been type-checked, no - errors can occur and the function can be treated as a simple accessor. - - -Exact conversions between decimals and primitive C types --------------------------------------------------------- - -This API supports conversions for decimals with a coefficient up to 38 digits. - -Data structures -~~~~~~~~~~~~~~~ - -The conversion functions use the following status codes and data structures: - -.. code-block:: c - - /* status cases for getting a triple */ - enum mpd_triple_class { - MPD_TRIPLE_NORMAL, - MPD_TRIPLE_INF, - MPD_TRIPLE_QNAN, - MPD_TRIPLE_SNAN, - MPD_TRIPLE_ERROR, - }; - - typedef struct { - enum mpd_triple_class tag; - uint8_t sign; - uint64_t hi; - uint64_t lo; - int64_t exp; - } mpd_uint128_triple_t; - -The status cases are explained below. ``sign`` is 0 for positive and 1 for negative. -``((uint128_t)hi << 64) + lo`` is the coefficient, ``exp`` is the exponent. - -The data structure is called "triple" because the decimal triple (sign, coeff, exp) -is an established term and (``hi``, ``lo``) represents a single ``uint128_t`` coefficient. - - -Functions -~~~~~~~~~ - -.. c:function:: mpd_uint128_triple_t PyDec_AsUint128Triple(const PyObject *dec) - - Convert a decimal to a triple. As above, it is guaranteed that the only - Python failure mode is a TypeError, checks can be omitted if the type is - known. - - For simplicity, the usage of the function and all special cases are - explained in code form and comments: - -.. code-block:: c - - triple = PyDec_AsUint128Triple(dec); - switch (triple.tag) { - case MPD_TRIPLE_QNAN: - /* - * Success: handle a quiet NaN. - * 1) triple.sign is 0 or 1. - * 2) triple.exp is always 0. - * 3) If triple.hi or triple.lo are nonzero, the NaN has a payload. - */ - break; - - case MPD_TRIPLE_SNAN: - /* - * Success: handle a signaling NaN. - * 1) triple.sign is 0 or 1. - * 2) triple.exp is always 0. - * 3) If triple.hi or triple.lo are nonzero, the sNaN has a payload. - */ - break; - - case MPD_TRIPLE_INF: - /* - * Success: handle Infinity. - * 1) triple.sign is 0 or 1. - * 2) triple.exp is always 0. - * 3) triple.hi and triple.lo are always zero. - */ - break; - - case MPD_TRIPLE_NORMAL: - /* Success: handle a finite value. */ - break; - - case MPD_TRIPLE_ERROR: - /* TypeError check: can be omitted if the type of dec is known. */ - if (PyErr_Occurred()) { - return NULL; - } - - /* Too large for conversion. PyDec_AsUint128Triple() does not set an - exception so applications can choose themselves. Typically this - would be a ValueError. */ - PyErr_SetString(PyExc_ValueError, - "value out of bounds for a uint128 triple"); - return NULL; - } - -.. c:function:: PyObject *PyDec_FromUint128Triple(const mpd_uint128_triple_t *triple) - - Create a decimal from a triple. The following rules must be observed for - initializing the triple: - - 1) ``triple.sign`` must always be 0 (for positive) or 1 (for negative). - - 2) ``MPD_TRIPLE_QNAN``: ``triple.exp`` must be 0. If ``triple.hi`` or ``triple.lo`` - are nonzero, create a ``NaN`` with a payload. - - 3) ``MPD_TRIPLE_SNAN``: ``triple.exp`` must be 0. If ``triple.hi`` or ``triple.lo`` - are nonzero, create an ``sNaN`` with a payload. - - 4) ``MPD_TRIPLE_INF``: ``triple.exp``, ``triple.hi`` and ``triple.lo`` must be zero. - - 5) ``MPD_TRIPLE_NORMAL``: ``MPD_MIN_ETINY + 38 < triple.exp < MPD_MAX_EMAX - 38``. - ``triple.hi`` and ``triple.lo`` can be chosen freely. - - 6) ``MPD_TRIPLE_ERROR``: It is always an error to set this tag. - - - If one of the above conditions is not met, the function returns ``NaN`` if - the ``InvalidOperation`` trap is not set in the thread local context. Otherwise, - it sets the ``InvalidOperation`` exception and returns NULL. - - Additionally, though extremely unlikely give the small allocation sizes, - the function can set ``MemoryError`` and return ``NULL``. - - -Advanced API ------------- - -This API enables the use of ``libmpdec`` functions. Since Python is compiled with -hidden symbols, the API requires an external libmpdec and the ``mpdecimal.h`` -header. - - -Functions -~~~~~~~~~ - -.. c:function:: PyObject *PyDec_Alloc(void) - - Return a new decimal that can be used in the ``result`` position of ``libmpdec`` - functions. - -.. c:function:: mpd_t *PyDec_Get(PyObject *v) - - Get a pointer to the internal ``mpd_t`` of the decimal. Decimals are immutable, - so this function must only be used on a new Decimal that has been created by - PyDec_Alloc(). - -.. c:function:: const mpd_t *PyDec_GetConst(const PyObject *v) - - Get a pointer to the constant internal ``mpd_t`` of the decimal. diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index 1005140c7acb3a..b32c113e5f0457 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -32,8 +32,8 @@ found in the dictionary of type objects. .. c:function:: int PyDescr_IsData(PyObject *descr) - Return true if the descriptor objects *descr* describes a data attribute, or - false if it describes a method. *descr* must be a descriptor object; there is + Return non-zero if the descriptor objects *descr* describes a data attribute, or + ``0`` if it describes a method. *descr* must be a descriptor object; there is no error checking. diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 8e0d6845463490..d257c9b5f763d1 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -22,13 +22,13 @@ Dictionary Objects .. c:function:: int PyDict_Check(PyObject *p) Return true if *p* is a dict object or an instance of a subtype of the dict - type. + type. This function always succeeds. .. c:function:: int PyDict_CheckExact(PyObject *p) Return true if *p* is a dict object, but not an instance of a subtype of - the dict type. + the dict type. This function always succeeds. .. c:function:: PyObject* PyDict_New() diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 247b6d68eceae9..21c508dc30ae22 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -182,8 +182,8 @@ For convenience, some of these functions will always return a .. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename) Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but the filename - is given as a C string. *filename* is decoded from the filesystem encoding - (:func:`os.fsdecode`). + is given as a C string. *filename* is decoded from the :term:`filesystem + encoding and error handler`. .. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) @@ -266,7 +266,7 @@ For convenience, some of these functions will always return a .. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string - decoded from the filesystem encoding (:func:`os.fsdecode`). + decoded from the :term:`filesystem encoding and error handler`. .. versionadded:: 3.2 @@ -343,7 +343,7 @@ an error value). Similar to :c:func:`PyErr_WarnExplicitObject` except that *message* and *module* are UTF-8 encoded strings, and *filename* is decoded from the - filesystem encoding (:func:`os.fsdecode`). + :term:`filesystem encoding and error handler`. .. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) @@ -505,29 +505,73 @@ Signal Handling single: SIGINT single: KeyboardInterrupt (built-in exception) - This function interacts with Python's signal handling. It checks whether a - signal has been sent to the processes and if so, invokes the corresponding - signal handler. If the :mod:`signal` module is supported, this can invoke a - signal handler written in Python. In all cases, the default effect for - :const:`SIGINT` is to raise the :exc:`KeyboardInterrupt` exception. If an - exception is raised the error indicator is set and the function returns ``-1``; - otherwise the function returns ``0``. The error indicator may or may not be - cleared if it was previously set. + This function interacts with Python's signal handling. + + If the function is called from the main thread and under the main Python + interpreter, it checks whether a signal has been sent to the processes + and if so, invokes the corresponding signal handler. If the :mod:`signal` + module is supported, this can invoke a signal handler written in Python. + + The function attempts to handle all pending signals, and then returns ``0``. + However, if a Python signal handler raises an exception, the error + indicator is set and the function returns ``-1`` immediately (such that + other pending signals may not have been handled yet: they will be on the + next :c:func:`PyErr_CheckSignals()` invocation). + + If the function is called from a non-main thread, or under a non-main + Python interpreter, it does nothing and returns ``0``. + + This function can be called by long-running C code that wants to + be interruptible by user requests (such as by pressing Ctrl-C). + + .. note:: + The default Python signal handler for :const:`SIGINT` raises the + :exc:`KeyboardInterrupt` exception. .. c:function:: void PyErr_SetInterrupt() .. index:: + module: signal single: SIGINT single: KeyboardInterrupt (built-in exception) - Simulate the effect of a :const:`SIGINT` signal arriving. The next time + Simulate the effect of a :const:`SIGINT` signal arriving. + This is equivalent to ``PyErr_SetInterruptEx(SIGINT)``. + + .. note:: + This function is async-signal-safe. It can be called without + the :term:`GIL` and from a C signal handler. + + +.. c:function:: int PyErr_SetInterruptEx(int signum) + + .. index:: + module: signal + single: KeyboardInterrupt (built-in exception) + + Simulate the effect of a signal arriving. The next time :c:func:`PyErr_CheckSignals` is called, the Python signal handler for - :const:`SIGINT` will be called. + the given signal number will be called. + + This function can be called by C code that sets up its own signal handling + and wants Python signal handlers to be invoked as expected when an + interruption is requested (for example when the user presses Ctrl-C + to interrupt an operation). + + If the given signal isn't handled by Python (it was set to + :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), it will be ignored. + + If *signum* is outside of the allowed range of signal numbers, ``-1`` + is returned. Otherwise, ``0`` is returned. The error indicator is + never changed by this function. + + .. note:: + This function is async-signal-safe. It can be called without + the :term:`GIL` and from a C signal handler. + + .. versionadded:: 3.10 - If :const:`SIGINT` isn't handled by Python (it was set to - :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does - nothing. .. c:function:: int PySignal_SetWakeupFd(int fd) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index b29937dbecdcfd..c107243a88dfc6 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -22,13 +22,13 @@ Floating Point Objects .. c:function:: int PyFloat_Check(PyObject *p) Return true if its argument is a :c:type:`PyFloatObject` or a subtype of - :c:type:`PyFloatObject`. + :c:type:`PyFloatObject`. This function always succeeds. .. c:function:: int PyFloat_CheckExact(PyObject *p) Return true if its argument is a :c:type:`PyFloatObject`, but not a subtype of - :c:type:`PyFloatObject`. + :c:type:`PyFloatObject`. This function always succeeds. .. c:function:: PyObject* PyFloat_FromString(PyObject *str) diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index bb416f4bb63aa2..ad008425f38116 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -26,7 +26,7 @@ There are a few functions specific to Python functions. .. c:function:: int PyFunction_Check(PyObject *o) Return true if *o* is a function object (has type :c:data:`PyFunction_Type`). - The parameter must not be ``NULL``. + The parameter must not be ``NULL``. This function always succeeds. .. c:function:: PyObject* PyFunction_New(PyObject *code, PyObject *globals) @@ -61,9 +61,11 @@ There are a few functions specific to Python functions. .. c:function:: PyObject* PyFunction_GetModule(PyObject *op) - Return the *__module__* attribute of the function object *op*. This is normally - a string containing the module name, but can be set to any other object by - Python code. + Return a :term:`borrowed reference` to the *__module__* attribute of the + function object *op*. It can be *NULL*. + + This is normally a string containing the module name, but can be set to any + other object by Python code. .. c:function:: PyObject* PyFunction_GetDefaults(PyObject *op) diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index eee114c19d5904..55ed9d4f7fad48 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -173,3 +173,46 @@ if the object is immutable. this method (don't just call :c:func:`Py_DECREF` on a reference). The collector will call this method if it detects that this object is involved in a reference cycle. + + +Controlling the Garbage Collector State +--------------------------------------- + +The C-API provides the following functions for controlling +garbage collection runs. + +.. c:function:: Py_ssize_t PyGC_Collect(void) + + Perform a full garbage collection, if the garbage collector is enabled. + (Note that :func:`gc.collect` runs it unconditionally.) + + Returns the number of collected + unreachable objects which cannot + be collected. + If the garbage collector is disabled or already collecting, + returns ``0`` immediately. + Errors during garbage collection are passed to :data:`sys.unraisablehook`. + This function does not raise exceptions. + + +.. c:function:: int PyGC_Enable(void) + + Enable the garbage collector: similar to :func:`gc.enable`. + Returns the previous state, 0 for disabled and 1 for enabled. + + .. versionadded:: 3.10 + + +.. c:function:: int PyGC_Disable(void) + + Disable the garbage collector: similar to :func:`gc.disable`. + Returns the previous state, 0 for disabled and 1 for enabled. + + .. versionadded:: 3.10 + + +.. c:function:: int PyGC_IsEnabled(void) + + Query the state of the garbage collector: similar to :func:`gc.isenabled`. + Returns the current state, 0 for disabled and 1 for enabled. + + .. versionadded:: 3.10 diff --git a/Doc/c-api/gen.rst b/Doc/c-api/gen.rst index 74410927bfde10..0eb5922f6da75f 100644 --- a/Doc/c-api/gen.rst +++ b/Doc/c-api/gen.rst @@ -22,12 +22,14 @@ than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`. .. c:function:: int PyGen_Check(PyObject *ob) - Return true if *ob* is a generator object; *ob* must not be ``NULL``. + Return true if *ob* is a generator object; *ob* must not be ``NULL``. This + function always succeeds. .. c:function:: int PyGen_CheckExact(PyObject *ob) - Return true if *ob*'s type is :c:type:`PyGen_Type`; *ob* must not be ``NULL``. + Return true if *ob*'s type is :c:type:`PyGen_Type`; *ob* must not be + ``NULL``. This function always succeeds. .. c:function:: PyObject* PyGen_New(PyFrameObject *frame) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 7f06648bcb4572..0f759732b35ea8 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -151,8 +151,9 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. c:var:: int Py_LegacyWindowsFSEncodingFlag - If the flag is non-zero, use the ``mbcs`` encoding instead of the UTF-8 - encoding for the filesystem encoding. + If the flag is non-zero, use the ``mbcs`` encoding with ``replace`` error + handler, instead of the UTF-8 encoding with ``surrogatepass`` error handler, + for the :term:`filesystem encoding and error handler`. Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable is set to a non-empty string. @@ -375,6 +376,12 @@ Process-wide parameters The returned string points into static storage; the caller should not modify its value. + This function should not be called before :c:func:`Py_Initialize`, otherwise + it returns ``NULL``. + + .. versionchanged:: 3.10 + It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + .. c:function:: wchar_t* Py_GetPrefix() @@ -388,6 +395,12 @@ Process-wide parameters script at build time. The value is available to Python code as ``sys.prefix``. It is only useful on Unix. See also the next function. + This function should not be called before :c:func:`Py_Initialize`, otherwise + it returns ``NULL``. + + .. versionchanged:: 3.10 + It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + .. c:function:: wchar_t* Py_GetExecPrefix() @@ -423,6 +436,12 @@ Process-wide parameters while having :file:`/usr/local/plat` be a different filesystem for each platform. + This function should not be called before :c:func:`Py_Initialize`, otherwise + it returns ``NULL``. + + .. versionchanged:: 3.10 + It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + .. c:function:: wchar_t* Py_GetProgramFullPath() @@ -436,6 +455,12 @@ Process-wide parameters static storage; the caller should not modify its value. The value is available to Python code as ``sys.executable``. + This function should not be called before :c:func:`Py_Initialize`, otherwise + it returns ``NULL``. + + .. versionchanged:: 3.10 + It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + .. c:function:: wchar_t* Py_GetPath() @@ -454,8 +479,14 @@ Process-wide parameters can be (and usually is) modified later to change the search path for loading modules. + This function should not be called before :c:func:`Py_Initialize`, otherwise + it returns ``NULL``. + .. XXX should give the exact rules + .. versionchanged:: 3.10 + It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + .. c:function:: void Py_SetPath(const wchar_t *) @@ -637,6 +668,12 @@ Process-wide parameters :c:func:`Py_SetPythonHome`, or the value of the :envvar:`PYTHONHOME` environment variable if it is set. + This function should not be called before :c:func:`Py_Initialize`, otherwise + it returns ``NULL``. + + .. versionchanged:: 3.10 + It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + .. _threads: @@ -1076,7 +1113,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. Get the current frame of the Python thread state *tstate*. - Return a strong reference. Return ``NULL`` if no frame is currently + Return a :term:`strong reference`. Return ``NULL`` if no frame is currently executing. See also :c:func:`PyEval_GetFrame`. @@ -1155,7 +1192,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. versionadded:: 3.9 -.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame); +.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) Set the frame evaluation function. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 37f5b9f880bf14..de85029481185a 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -8,46 +8,24 @@ Python Initialization Configuration .. versionadded:: 3.8 -Structures: - -* :c:type:`PyConfig` -* :c:type:`PyPreConfig` -* :c:type:`PyStatus` -* :c:type:`PyWideStringList` - -Functions: - -* :c:func:`PyConfig_Clear` -* :c:func:`PyConfig_InitIsolatedConfig` -* :c:func:`PyConfig_InitPythonConfig` -* :c:func:`PyConfig_Read` -* :c:func:`PyConfig_SetArgv` -* :c:func:`PyConfig_SetBytesArgv` -* :c:func:`PyConfig_SetBytesString` -* :c:func:`PyConfig_SetString` -* :c:func:`PyConfig_SetWideStringList` -* :c:func:`PyPreConfig_InitIsolatedConfig` -* :c:func:`PyPreConfig_InitPythonConfig` -* :c:func:`PyStatus_Error` -* :c:func:`PyStatus_Exception` -* :c:func:`PyStatus_Exit` -* :c:func:`PyStatus_IsError` -* :c:func:`PyStatus_IsExit` -* :c:func:`PyStatus_NoMemory` -* :c:func:`PyStatus_Ok` -* :c:func:`PyWideStringList_Append` -* :c:func:`PyWideStringList_Insert` -* :c:func:`Py_ExitStatusException` -* :c:func:`Py_InitializeFromConfig` -* :c:func:`Py_PreInitialize` -* :c:func:`Py_PreInitializeFromArgs` -* :c:func:`Py_PreInitializeFromBytesArgs` -* :c:func:`Py_RunMain` -* :c:func:`Py_GetArgcArgv` - -The preconfiguration (``PyPreConfig`` type) is stored in -``_PyRuntime.preconfig`` and the configuration (``PyConfig`` type) is stored in -``PyInterpreterState.config``. +Python can be initialized with :c:func:`Py_InitializeFromConfig` and the +:c:type:`PyConfig` structure. It can be preinitialized with +:c:func:`Py_PreInitialize` and the :c:type:`PyPreConfig` structure. + +There are two kinds of configuration: + +* The :ref:`Python Configuration ` can be used to build a + customized Python which behaves as the regular Python. For example, + environments variables and command line arguments are used to configure + Python. + +* The :ref:`Isolated Configuration ` can be used to embed + Python into an application. It isolates Python from the system. For example, + environments variables are ignored, the LC_CTYPE locale is left unchanged and + no signal handler is registred. + +The :c:func:`Py_RunMain` function can be used to write a customized Python +program. See also :ref:`Initialization, Finalization, and Threads `. @@ -55,8 +33,47 @@ See also :ref:`Initialization, Finalization, and Threads `. :pep:`587` "Python Initialization Configuration". +Example +======= + +Example of customized Python always running in isolated mode:: + + int main(int argc, char **argv) + { + PyStatus status; + + PyConfig config; + PyConfig_InitPythonConfig(&config); + config.isolated = 1; + + /* Decode command line arguments. + Implicitly preinitialize Python (in isolated mode). */ + status = PyConfig_SetBytesArgv(&config, argc, argv); + if (PyStatus_Exception(status)) { + goto exception; + } + + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + goto exception; + } + PyConfig_Clear(&config); + + return Py_RunMain(); + + exception: + PyConfig_Clear(&config); + if (PyStatus_IsExit(status)) { + return status.exitcode; + } + /* Display the error message and exit the process with + non-zero exit code */ + Py_ExitStatusException(status); + } + + PyWideStringList ----------------- +================ .. c:type:: PyWideStringList @@ -95,7 +112,7 @@ PyWideStringList List items. PyStatus --------- +======== .. c:type:: PyStatus @@ -128,6 +145,8 @@ PyStatus Initialization error with a message. + *err_msg* must not be ``NULL``. + .. c:function:: PyStatus PyStatus_NoMemory(void) Memory allocation failure (out of memory). @@ -185,15 +204,11 @@ Example:: PyPreConfig ------------ +=========== .. c:type:: PyPreConfig - Structure used to preinitialize Python: - - * Set the Python memory allocator - * Configure the LC_CTYPE locale - * Set the UTF-8 mode + Structure used to preinitialize Python. Function to initialize a preconfiguration: @@ -211,56 +226,89 @@ PyPreConfig .. c:member:: int allocator - Name of the memory allocator: + Name of the Python memory allocators: * ``PYMEM_ALLOCATOR_NOT_SET`` (``0``): don't change memory allocators - (use defaults) - * ``PYMEM_ALLOCATOR_DEFAULT`` (``1``): default memory allocators - * ``PYMEM_ALLOCATOR_DEBUG`` (``2``): default memory allocators with - debug hooks - * ``PYMEM_ALLOCATOR_MALLOC`` (``3``): force usage of ``malloc()`` + (use defaults). + * ``PYMEM_ALLOCATOR_DEFAULT`` (``1``): :ref:`default memory allocators + `. + * ``PYMEM_ALLOCATOR_DEBUG`` (``2``): :ref:`default memory allocators + ` with :ref:`debug hooks + `. + * ``PYMEM_ALLOCATOR_MALLOC`` (``3``): use ``malloc()`` of the C library. * ``PYMEM_ALLOCATOR_MALLOC_DEBUG`` (``4``): force usage of - ``malloc()`` with debug hooks + ``malloc()`` with :ref:`debug hooks `. * ``PYMEM_ALLOCATOR_PYMALLOC`` (``5``): :ref:`Python pymalloc memory - allocator ` + allocator `. * ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` (``6``): :ref:`Python pymalloc - memory allocator ` with debug hooks + memory allocator ` with :ref:`debug hooks + `. - ``PYMEM_ALLOCATOR_PYMALLOC`` and ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` - are not supported if Python is configured using ``--without-pymalloc`` + ``PYMEM_ALLOCATOR_PYMALLOC`` and ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` are + not supported if Python is :option:`configured using --without-pymalloc + <--without-pymalloc>`. See :ref:`Memory Management `. + Default: ``PYMEM_ALLOCATOR_NOT_SET``. + .. c:member:: int configure_locale - Set the LC_CTYPE locale to the user preferred locale? If equals to 0, set - :c:member:`coerce_c_locale` and :c:member:`coerce_c_locale_warn` to 0. + Set the LC_CTYPE locale to the user preferred locale? + + If equals to 0, set :c:member:`~PyPreConfig.coerce_c_locale` and + :c:member:`~PyPreConfig.coerce_c_locale_warn` members to 0. + + See the :term:`locale encoding`. + + Default: ``1`` in Python config, ``0`` in isolated config. .. c:member:: int coerce_c_locale - If equals to 2, coerce the C locale; if equals to 1, read the LC_CTYPE - locale to decide if it should be coerced. + If equals to 2, coerce the C locale. + + If equals to 1, read the LC_CTYPE locale to decide if it should be + coerced. + + See the :term:`locale encoding`. + + Default: ``-1`` in Python config, ``0`` in isolated config. .. c:member:: int coerce_c_locale_warn If non-zero, emit a warning if the C locale is coerced. + Default: ``-1`` in Python config, ``0`` in isolated config. + .. c:member:: int dev_mode - See :c:member:`PyConfig.dev_mode`. + If non-zero, enables the :ref:`Python Development Mode `: + see :c:member:`PyConfig.dev_mode`. + + Default: ``-1`` in Python mode, ``0`` in isolated mode. .. c:member:: int isolated - See :c:member:`PyConfig.isolated`. + Isolated mode: see :c:member:`PyConfig.isolated`. + + Default: ``0`` in Python mode, ``1`` in isolated mode. + + .. c:member:: int legacy_windows_fs_encoding - .. c:member:: int legacy_windows_fs_encoding (Windows only) + If non-zero: - If non-zero, disable UTF-8 Mode, set the Python filesystem encoding to - ``mbcs``, set the filesystem error handler to ``replace``. + * Set :c:member:`PyPreConfig.utf8_mode` to ``0``, + * Set :c:member:`PyConfig.filesystem_encoding` to ``"mbcs"``, + * Set :c:member:`PyConfig.filesystem_errors` to ``"replace"``. + + Initialized the from :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment + variable value. Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for Windows specific code. + Default: ``0``. + .. c:member:: int parse_argv If non-zero, :c:func:`Py_PreInitializeFromArgs` and @@ -268,16 +316,39 @@ PyPreConfig same way the regular Python parses command line arguments: see :ref:`Command Line Arguments `. + Default: ``1`` in Python config, ``0`` in isolated config. + .. c:member:: int use_environment - See :c:member:`PyConfig.use_environment`. + Use :ref:`environment variables `? See + :c:member:`PyConfig.use_environment`. + + Default: ``1`` in Python config and ``0`` in isolated config. .. c:member:: int utf8_mode - If non-zero, enable the UTF-8 mode. + If non-zero, enable the :ref:`Python UTF-8 Mode `. + + Set by the :option:`-X utf8 <-X>` command line option and the + :envvar:`PYTHONUTF8` environment variable. + + Default: ``-1`` in Python config and ``0`` in isolated config. + + +.. _c-preinit: + +Preinitialize Python with PyPreConfig +===================================== + +The preinitialization of Python: + +* Set the Python memory allocators (:c:member:`PyPreConfig.allocator`) +* Configure the LC_CTYPE locale (:term:`locale encoding`) +* Set the :ref:`Python UTF-8 Mode ` + (:c:member:`PyPreConfig.utf8_mode`) -Preinitialization with PyPreConfig ----------------------------------- +The current preconfiguration (``PyPreConfig`` type) is stored in +``_PyRuntime.preconfig``. Functions to preinitialize Python: @@ -285,15 +356,25 @@ Functions to preinitialize Python: Preinitialize Python from *preconfig* preconfiguration. + *preconfig* must not be ``NULL``. + .. c:function:: PyStatus Py_PreInitializeFromBytesArgs(const PyPreConfig *preconfig, int argc, char * const *argv) - Preinitialize Python from *preconfig* preconfiguration and command line - arguments (bytes strings). + Preinitialize Python from *preconfig* preconfiguration. + + Parse *argv* command line arguments (bytes strings) if + :c:member:`~PyPreConfig.parse_argv` of *preconfig* is non-zero. + + *preconfig* must not be ``NULL``. .. c:function:: PyStatus Py_PreInitializeFromArgs(const PyPreConfig *preconfig, int argc, wchar_t * const * argv) - Preinitialize Python from *preconfig* preconfiguration and command line - arguments (wide strings). + Preinitialize Python from *preconfig* preconfiguration. + + Parse *argv* command line arguments (wide strings) if + :c:member:`~PyPreConfig.parse_argv` of *preconfig* is non-zero. + + *preconfig* must not be ``NULL``. The caller is responsible to handle exceptions (error or exit) using :c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. @@ -303,7 +384,7 @@ For :ref:`Python Configuration ` command line arguments, the command line arguments must also be passed to preinitialize Python, since they have an effect on the pre-configuration like encodings. For example, the :option:`-X utf8 <-X>` command line option -enables the UTF-8 Mode. +enables the :ref:`Python UTF-8 Mode `. ``PyMem_SetAllocator()`` can be called after :c:func:`Py_PreInitialize` and before :c:func:`Py_InitializeFromConfig` to install a custom memory allocator. @@ -311,11 +392,12 @@ It can be called before :c:func:`Py_PreInitialize` if :c:member:`PyPreConfig.allocator` is set to ``PYMEM_ALLOCATOR_NOT_SET``. Python memory allocation functions like :c:func:`PyMem_RawMalloc` must not be -used before Python preinitialization, whereas calling directly ``malloc()`` and -``free()`` is always safe. :c:func:`Py_DecodeLocale` must not be called before -the preinitialization. +used before the Python preinitialization, whereas calling directly ``malloc()`` +and ``free()`` is always safe. :c:func:`Py_DecodeLocale` must not be called +before the Python preinitialization. -Example using the preinitialization to enable the UTF-8 Mode:: +Example using the preinitialization to enable +the :ref:`Python UTF-8 Mode `:: PyStatus status; PyPreConfig preconfig; @@ -328,7 +410,7 @@ Example using the preinitialization to enable the UTF-8 Mode:: Py_ExitStatusException(status); } - /* at this point, Python will speak UTF-8 */ + /* at this point, Python speaks UTF-8 */ Py_Initialize(); /* ... use Python API here ... */ @@ -336,53 +418,60 @@ Example using the preinitialization to enable the UTF-8 Mode:: PyConfig --------- +======== .. c:type:: PyConfig Structure containing most parameters to configure Python. + When done, the :c:func:`PyConfig_Clear` function must be used to release the + configuration memory. + Structure methods: .. c:function:: void PyConfig_InitPythonConfig(PyConfig *config) - Initialize configuration with :ref:`Python Configuration + Initialize configuration with the :ref:`Python Configuration `. .. c:function:: void PyConfig_InitIsolatedConfig(PyConfig *config) - Initialize configuration with :ref:`Isolated Configuration + Initialize configuration with the :ref:`Isolated Configuration `. .. c:function:: PyStatus PyConfig_SetString(PyConfig *config, wchar_t * const *config_str, const wchar_t *str) Copy the wide character string *str* into ``*config_str``. - Preinitialize Python if needed. + :ref:`Preinitialize Python ` if needed. .. c:function:: PyStatus PyConfig_SetBytesString(PyConfig *config, wchar_t * const *config_str, const char *str) - Decode *str* using ``Py_DecodeLocale()`` and set the result into ``*config_str``. + Decode *str* using :c:func:`Py_DecodeLocale` and set the result into + ``*config_str``. - Preinitialize Python if needed. + :ref:`Preinitialize Python ` if needed. .. c:function:: PyStatus PyConfig_SetArgv(PyConfig *config, int argc, wchar_t * const *argv) - Set command line arguments from wide character strings. + Set command line arguments (:c:member:`~PyConfig.argv` member of + *config*) from the *argv* list of wide character strings. - Preinitialize Python if needed. + :ref:`Preinitialize Python ` if needed. .. c:function:: PyStatus PyConfig_SetBytesArgv(PyConfig *config, int argc, char * const *argv) - Set command line arguments: decode bytes using :c:func:`Py_DecodeLocale`. + Set command line arguments (:c:member:`~PyConfig.argv` member of + *config*) from the *argv* list of bytes strings. Decode bytes using + :c:func:`Py_DecodeLocale`. - Preinitialize Python if needed. + :ref:`Preinitialize Python ` if needed. .. c:function:: PyStatus PyConfig_SetWideStringList(PyConfig *config, PyWideStringList *list, Py_ssize_t length, wchar_t **items) Set the list of wide strings *list* to *length* and *items*. - Preinitialize Python if needed. + :ref:`Preinitialize Python ` if needed. .. c:function:: PyStatus PyConfig_Read(PyConfig *config) @@ -390,24 +479,37 @@ PyConfig Fields which are already initialized are left unchanged. - Preinitialize Python if needed. + The :c:func:`PyConfig_Read` function only parses + :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` + is set to ``2`` after arguments are parsed. Since Python arguments are + strippped from :c:member:`PyConfig.argv`, parsing arguments twice would + parse the application options as Python options. + + :ref:`Preinitialize Python ` if needed. + + .. versionchanged:: 3.10 + The :c:member:`PyConfig.argv` arguments are now only parsed once, + :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments are + parsed, and arguments are only parsed if + :c:member:`PyConfig.parse_argv` equals ``1``. .. c:function:: void PyConfig_Clear(PyConfig *config) Release configuration memory. - Most ``PyConfig`` methods preinitialize Python if needed. In that case, the - Python preinitialization configuration in based on the :c:type:`PyConfig`. - If configuration fields which are in common with :c:type:`PyPreConfig` are - tuned, they must be set before calling a :c:type:`PyConfig` method: + Most ``PyConfig`` methods :ref:`preinitialize Python ` if needed. + In that case, the Python preinitialization configuration + (:c:type:`PyPreConfig`) in based on the :c:type:`PyConfig`. If configuration + fields which are in common with :c:type:`PyPreConfig` are tuned, they must + be set before calling a :c:type:`PyConfig` method: - * :c:member:`~PyConfig.dev_mode` - * :c:member:`~PyConfig.isolated` - * :c:member:`~PyConfig.parse_argv` - * :c:member:`~PyConfig.use_environment` + * :c:member:`PyConfig.dev_mode` + * :c:member:`PyConfig.isolated` + * :c:member:`PyConfig.parse_argv` + * :c:member:`PyConfig.use_environment` Moreover, if :c:func:`PyConfig_SetArgv` or :c:func:`PyConfig_SetBytesArgv` - is used, this method must be called first, before other methods, since the + is used, this method must be called before other methods, since the preinitialization configuration depends on command line arguments (if :c:member:`parse_argv` is non-zero). @@ -418,11 +520,17 @@ PyConfig .. c:member:: PyWideStringList argv - Command line arguments, :data:`sys.argv`. See - :c:member:`~PyConfig.parse_argv` to parse :c:member:`~PyConfig.argv` the - same way the regular Python parses Python command line arguments. If - :c:member:`~PyConfig.argv` is empty, an empty string is added to ensure - that :data:`sys.argv` always exists and is never empty. + Command line arguments: :data:`sys.argv`. + + Set :c:member:`~PyConfig.parse_argv` to ``1`` to parse + :c:member:`~PyConfig.argv` the same way the regular Python parses Python + command line arguments and then to strip Python arguments from + :c:member:`~PyConfig.argv`. + + If :c:member:`~PyConfig.argv` is empty, an empty string is added to + ensure that :data:`sys.argv` always exists and is never empty. + + Default: ``NULL``. See also the :c:member:`~PyConfig.orig_argv` member. @@ -430,80 +538,187 @@ PyConfig :data:`sys.base_exec_prefix`. + Default: ``NULL``. + + Part of the :ref:`Python Path Configuration ` output. + .. c:member:: wchar_t* base_executable - :data:`sys._base_executable`: ``__PYVENV_LAUNCHER__`` environment - variable value, or copy of :c:member:`PyConfig.executable`. + Python base executable: :data:`sys._base_executable`. + + Set by the :envvar:`__PYVENV_LAUNCHER__` environment variable. + + Set from :c:member:`PyConfig.executable` if ``NULL``. + + Default: ``NULL``. + + Part of the :ref:`Python Path Configuration ` output. .. c:member:: wchar_t* base_prefix :data:`sys.base_prefix`. - .. c:member:: wchar_t* platlibdir - - :data:`sys.platlibdir`: platform library directory name, set at configure time - by ``--with-platlibdir``, overrideable by the ``PYTHONPLATLIBDIR`` - environment variable. + Default: ``NULL``. - .. versionadded:: 3.9 + Part of the :ref:`Python Path Configuration ` output. .. c:member:: int buffered_stdio - If equals to 0, enable unbuffered mode, making the stdout and stderr - streams unbuffered. + If equals to 0 and :c:member:`~PyConfig.configure_c_stdio` is non-zero, + disable buffering on the C streams stdout and stderr. + + Set to 0 by the :option:`-u` command line option and the + :envvar:`PYTHONUNBUFFERED` environment variable. stdin is always opened in buffered mode. + Default: ``1``. + .. c:member:: int bytes_warning If equals to 1, issue a warning when comparing :class:`bytes` or :class:`bytearray` with :class:`str`, or comparing :class:`bytes` with - :class:`int`. If equal or greater to 2, raise a :exc:`BytesWarning` - exception. + :class:`int`. + + If equal or greater to 2, raise a :exc:`BytesWarning` exception in these + cases. + + Incremented by the :option:`-b` command line option. + + Default: ``0``. + + .. c:member:: int warn_default_encoding + + If non-zero, emit a :exc:`EncodingWarning` warning when :class:`io.TextIOWrapper` + uses its default encoding. See :ref:`io-encoding-warning` for details. + + Default: ``0``. + + .. versionadded:: 3.10 .. c:member:: wchar_t* check_hash_pycs_mode - Control the validation behavior of hash-based ``.pyc`` files (see - :pep:`552`): :option:`--check-hash-based-pycs` command line option value. + Control the validation behavior of hash-based ``.pyc`` files: + value of the :option:`--check-hash-based-pycs` command line option. + + Valid values: - Valid values: ``always``, ``never`` and ``default``. + - ``L"always"``: Hash the source file for invalidation regardless of + value of the 'check_source' flag. + - ``L"never"``: Assume that hash-based pycs always are valid. + - ``L"default"``: The 'check_source' flag in hash-based pycs + determines invalidation. - The default value is: ``default``. + Default: ``L"default"``. + + See also :pep:`552` "Deterministic pycs". .. c:member:: int configure_c_stdio - If non-zero, configure C standard streams (``stdio``, ``stdout``, - ``stdout``). For example, set their mode to ``O_BINARY`` on Windows. + If non-zero, configure C standard streams: + + * On Windows, set the binary mode (``O_BINARY``) on stdin, stdout and + stderr. + * If :c:member:`~PyConfig.buffered_stdio` equals zero, disable buffering + of stdin, stdout and stderr streams. + * If :c:member:`~PyConfig.interactive` is non-zero, enable stream + buffering on stdin and stdout (only stdout on Windows). + + Default: ``1`` in Python config, ``0`` in isolated config. .. c:member:: int dev_mode If non-zero, enable the :ref:`Python Development Mode `. + Default: ``-1`` in Python mode, ``0`` in isolated mode. + .. c:member:: int dump_refs + Dump Python refererences? + If non-zero, dump all objects which are still alive at exit. - ``Py_TRACE_REFS`` macro must be defined in build. + Set to ``1`` by the :envvar:`PYTHONDUMPREFS` environment variable. + + Need a special build of Python with the ``Py_TRACE_REFS`` macro defined: + see the :option:`configure --with-trace-refs option <--with-trace-refs>`. + + Default: ``0``. .. c:member:: wchar_t* exec_prefix - :data:`sys.exec_prefix`. + The site-specific directory prefix where the platform-dependent Python + files are installed: :data:`sys.exec_prefix`. + + Default: ``NULL``. + + Part of the :ref:`Python Path Configuration ` output. .. c:member:: wchar_t* executable + The absolute path of the executable binary for the Python interpreter: :data:`sys.executable`. + Default: ``NULL``. + + Part of the :ref:`Python Path Configuration ` output. + .. c:member:: int faulthandler + Enable faulthandler? + If non-zero, call :func:`faulthandler.enable` at startup. + Set to ``1`` by :option:`-X faulthandler <-X>` and the + :envvar:`PYTHONFAULTHANDLER` environment variable. + + Default: ``-1`` in Python mode, ``0`` in isolated mode. + .. c:member:: wchar_t* filesystem_encoding - Filesystem encoding, :func:`sys.getfilesystemencoding`. + :term:`Filesystem encoding `: + :func:`sys.getfilesystemencoding`. + + On macOS, Android and VxWorks: use ``"utf-8"`` by default. + + On Windows: use ``"utf-8"`` by default, or ``"mbcs"`` if + :c:member:`~PyPreConfig.legacy_windows_fs_encoding` of + :c:type:`PyPreConfig` is non-zero. + + Default encoding on other platforms: + + * ``"utf-8"`` if :c:member:`PyPreConfig.utf8_mode` is non-zero. + * ``"ascii"`` if Python detects that ``nl_langinfo(CODESET)`` announces + the ASCII encoding (or Roman8 encoding on HP-UX), whereas the + ``mbstowcs()`` function decodes from a different encoding (usually + Latin1). + * ``"utf-8"`` if ``nl_langinfo(CODESET)`` returns an empty string. + * Otherwise, use the :term:`locale encoding`: + ``nl_langinfo(CODESET)`` result. + + At Python statup, the encoding name is normalized to the Python codec + name. For example, ``"ANSI_X3.4-1968"`` is replaced with ``"ascii"``. + + See also the :c:member:`~PyConfig.filesystem_errors` member. .. c:member:: wchar_t* filesystem_errors - Filesystem encoding errors, :func:`sys.getfilesystemencodeerrors`. + :term:`Filesystem error handler `: + :func:`sys.getfilesystemencodeerrors`. + + On Windows: use ``"surrogatepass"`` by default, or ``"replace"`` if + :c:member:`~PyPreConfig.legacy_windows_fs_encoding` of + :c:type:`PyPreConfig` is non-zero. + + On other platforms: use ``"surrogateescape"`` by default. + + Supported error handlers: + + * ``"strict"`` + * ``"surrogateescape"`` + * ``"surrogatepass"`` (only supported with the UTF-8 encoding) + + See also the :c:member:`~PyConfig.filesystem_encoding` member. .. c:member:: unsigned long hash_seed .. c:member:: int use_hash_seed @@ -511,30 +726,62 @@ PyConfig Randomized hash function seed. If :c:member:`~PyConfig.use_hash_seed` is zero, a seed is chosen randomly - at Pythonstartup, and :c:member:`~PyConfig.hash_seed` is ignored. + at Python startup, and :c:member:`~PyConfig.hash_seed` is ignored. + + Set by the :envvar:`PYTHONHASHSEED` environment variable. + + Default *use_hash_seed* value: ``-1`` in Python mode, ``0`` in isolated + mode. .. c:member:: wchar_t* home Python home directory. - Initialized from :envvar:`PYTHONHOME` environment variable value by - default. + If :c:func:`Py_SetPythonHome` has been called, use its argument if it is + not ``NULL``. + + Set by the :envvar:`PYTHONHOME` environment variable. + + Default: ``NULL``. + + Part of the :ref:`Python Path Configuration ` input. .. c:member:: int import_time If non-zero, profile import time. + Set the ``1`` by the :option:`-X importtime <-X>` option and the + :envvar:`PYTHONPROFILEIMPORTTIME` environment variable. + + Default: ``0``. + .. c:member:: int inspect Enter interactive mode after executing a script or a command. + If greater than 0, enable inspect: when a script is passed as first + argument or the -c option is used, enter interactive mode after executing + the script or the command, even when :data:`sys.stdin` does not appear to + be a terminal. + + Incremented by the :option:`-i` command line option. Set to ``1`` if the + :envvar:`PYTHONINSPECT` environment variable is non-empty. + + Default: ``0``. + .. c:member:: int install_signal_handlers - Install signal handlers? + Install Python signal handlers? + + Default: ``1`` in Python mode, ``0`` in isolated mode. .. c:member:: int interactive - Interactive mode. + If greater than 0, enable the interactive mode (REPL). + + Incremented by the :option:`-i` command line option. + + Default: ``0``. .. c:member:: int isolated @@ -548,54 +795,99 @@ PyConfig * Set :c:member:`~PyConfig.use_environment` and :c:member:`~PyConfig.user_site_directory` to 0. + Default: ``0`` in Python mode, ``1`` in isolated mode. + + See also :c:member:`PyPreConfig.isolated`. + .. c:member:: int legacy_windows_stdio If non-zero, use :class:`io.FileIO` instead of :class:`io.WindowsConsoleIO` for :data:`sys.stdin`, :data:`sys.stdout` and :data:`sys.stderr`. + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment + variable is set to a non-empty string. + Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for Windows specific code. + Default: ``0``. + + See also the :pep:`528` (Change Windows console encoding to UTF-8). + .. c:member:: int malloc_stats If non-zero, dump statistics on :ref:`Python pymalloc memory allocator ` at exit. - The option is ignored if Python is built using ``--without-pymalloc``. + Set to ``1`` by the :envvar:`PYTHONMALLOCSTATS` environment variable. + + The option is ignored if Python is :option:`configured using + the --without-pymalloc option <--without-pymalloc>`. + + Default: ``0``. + + .. c:member:: wchar_t* platlibdir + + Platform library directory name: :data:`sys.platlibdir`. + + Set by the :envvar:`PYTHONPLATLIBDIR` environment variable. + + Default: value of the ``PLATLIBDIR`` macro which is set by the + :option:`configure --with-platlibdir option <--with-platlibdir>` + (default: ``"lib"``). + + Part of the :ref:`Python Path Configuration ` input. + + .. versionadded:: 3.9 .. c:member:: wchar_t* pythonpath_env - Module search paths as a string separated by ``DELIM`` + Module search paths (:data:`sys.path`) as a string separated by ``DELIM`` (:data:`os.path.pathsep`). - Initialized from :envvar:`PYTHONPATH` environment variable value by - default. + Set by the :envvar:`PYTHONPATH` environment variable. + + Default: ``NULL``. + + Part of the :ref:`Python Path Configuration ` input. .. c:member:: PyWideStringList module_search_paths .. c:member:: int module_search_paths_set - :data:`sys.path`. If :c:member:`~PyConfig.module_search_paths_set` is - equal to 0, the :c:member:`~PyConfig.module_search_paths` is overridden - by the function calculating the :ref:`Path Configuration - `. + Module search paths: :data:`sys.path`. + + If :c:member:`~PyConfig.module_search_paths_set` is equal to 0, the + function calculating the :ref:`Python Path Configuration ` + overrides the :c:member:`~PyConfig.module_search_paths` and sets + :c:member:`~PyConfig.module_search_paths_set` to ``1``. + + Default: empty list (``module_search_paths``) and ``0`` + (``module_search_paths_set``). + + Part of the :ref:`Python Path Configuration ` output. .. c:member:: int optimization_level Compilation optimization level: - * 0: Peephole optimizer (and ``__debug__`` is set to ``True``) - * 1: Remove assertions, set ``__debug__`` to ``False`` - * 2: Strip docstrings + * ``0``: Peephole optimizer, set ``__debug__`` to ``True``. + * ``1``: Level 0, remove assertions, set ``__debug__`` to ``False``. + * ``2``: Level 1, strip docstrings. + + Incremented by the :option:`-O` command line option. Set to the + :envvar:`PYTHONOPTIMIZE` environment variable value. + + Default: ``0``. .. c:member:: PyWideStringList orig_argv The list of the original command line arguments passed to the Python - executable. + executable: :data:`sys.orig_argv`. If :c:member:`~PyConfig.orig_argv` list is empty and :c:member:`~PyConfig.argv` is not a list only containing an empty - string, :c:func:`PyConfig_Read()` copies :c:member:`~PyConfig.argv` into + string, :c:func:`PyConfig_Read` copies :c:member:`~PyConfig.argv` into :c:member:`~PyConfig.orig_argv` before modifying :c:member:`~PyConfig.argv` (if :c:member:`~PyConfig.parse_argv` is non-zero). @@ -603,57 +895,127 @@ PyConfig See also the :c:member:`~PyConfig.argv` member and the :c:func:`Py_GetArgcArgv` function. + Default: empty list. + .. versionadded:: 3.10 .. c:member:: int parse_argv - If non-zero, parse :c:member:`~PyConfig.argv` the same way the regular - Python command line arguments, and strip Python arguments from - :c:member:`~PyConfig.argv`: see :ref:`Command Line Arguments - `. + Parse command line arguments? + + If equals to ``1``, parse :c:member:`~PyConfig.argv` the same way the regular + Python parses :ref:`command line arguments `, and strip + Python arguments from :c:member:`~PyConfig.argv`. + + The :c:func:`PyConfig_Read` function only parses + :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` + is set to ``2`` after arguments are parsed. Since Python arguments are + strippped from :c:member:`PyConfig.argv`, parsing arguments twice would + parse the application options as Python options. + + Default: ``1`` in Python mode, ``0`` in isolated mode. + + .. versionchanged:: 3.10 + The :c:member:`PyConfig.argv` arguments are now only parsed if + :c:member:`PyConfig.parse_argv` equals to ``1``. .. c:member:: int parser_debug - If non-zero, turn on parser debugging output (for expert only, depending + Parser debug mode. If greater than 0, turn on parser debugging output (for expert only, depending on compilation options). + Incremented by the :option:`-d` command line option. Set to the + :envvar:`PYTHONDEBUG` environment variable value. + + Default: ``0``. + .. c:member:: int pathconfig_warnings - If equal to 0, suppress warnings when calculating the :ref:`Path - Configuration ` (Unix only, Windows does not log any - warning). Otherwise, warnings are written into ``stderr``. + On Unix, if non-zero, calculating the :ref:`Python Path Configuration + ` can log warnings into ``stderr``. If equals to 0, + suppress these warnings. + + It has no effect on Windows. + + Default: ``1`` in Python mode, ``0`` in isolated mode. + + Part of the :ref:`Python Path Configuration ` input. .. c:member:: wchar_t* prefix - :data:`sys.prefix`. + The site-specific directory prefix where the platform independent Python + files are installed: :data:`sys.prefix`. + + Default: ``NULL``. + + Part of the :ref:`Python Path Configuration ` output. .. c:member:: wchar_t* program_name - Program name. Used to initialize :c:member:`~PyConfig.executable`, and in - early error messages. + Program name used to initialize :c:member:`~PyConfig.executable` and in + early error messages during Python initialization. + + * If :func:`Py_SetProgramName` has been called, use its argument. + * On macOS, use :envvar:`PYTHONEXECUTABLE` environment variable if set. + * If the ``WITH_NEXT_FRAMEWORK`` macro is defined, use + :envvar:`__PYVENV_LAUNCHER__` environment variable if set. + * Use ``argv[0]`` of :c:member:`~PyConfig.argv` if available and + non-empty. + * Otherwise, use ``L"python"`` on Windows, or ``L"python3"`` on other + platforms. + + Default: ``NULL``. + + Part of the :ref:`Python Path Configuration ` input. .. c:member:: wchar_t* pycache_prefix - :data:`sys.pycache_prefix`: ``.pyc`` cache prefix. + Directory where cached ``.pyc`` files are written: + :data:`sys.pycache_prefix`. + + Set by the :option:`-X pycache_prefix=PATH <-X>` command line option and + the :envvar:`PYTHONPYCACHEPREFIX` environment variable. If ``NULL``, :data:`sys.pycache_prefix` is set to ``None``. + Default: ``NULL``. + .. c:member:: int quiet - Quiet mode. For example, don't display the copyright and version messages - in interactive mode. + Quiet mode. If greater than 0, don't display the copyright and version at + Python startup in interactive mode. + + Incremented by the :option:`-q` command line option. + + Default: ``0``. .. c:member:: wchar_t* run_command - ``python3 -c COMMAND`` argument. Used by :c:func:`Py_RunMain`. + Value of the :option:`-c` command line option. + + Used by :c:func:`Py_RunMain`. + + Default: ``NULL``. .. c:member:: wchar_t* run_filename - ``python3 FILENAME`` argument. Used by :c:func:`Py_RunMain`. + Filename passed on the command line: trailing command line argument + without :option:`-c` or :option:`-m`. + + For example, it is set to ``script.py`` by the ``python3 script.py arg`` + command. + + Used by :c:func:`Py_RunMain`. + + Default: ``NULL``. .. c:member:: wchar_t* run_module - ``python3 -m MODULE`` argument. Used by :c:func:`Py_RunMain`. + Value of the :option:`-m` command line option. + + Used by :c:func:`Py_RunMain`. + + Default: ``NULL``. .. c:member:: int show_ref_count @@ -661,66 +1023,156 @@ PyConfig Set to 1 by :option:`-X showrefcount <-X>` command line option. - Need a debug build of Python (``Py_REF_DEBUG`` macro must be defined). + Need a :ref:`debug build of Python ` (the ``Py_REF_DEBUG`` + macro must be defined). + + Default: ``0``. .. c:member:: int site_import Import the :mod:`site` module at startup? + If equal to zero, disable the import of the module site and the + site-dependent manipulations of :data:`sys.path` that it entails. + + Also disable these manipulations if the :mod:`site` module is explicitly + imported later (call :func:`site.main` if you want them to be triggered). + + Set to ``0`` by the :option:`-S` command line option. + + :data:`sys.flags.no_site` is set to the inverted value of + :c:member:`~PyConfig.site_import`. + + Default: ``1``. + .. c:member:: int skip_source_first_line - Skip the first line of the source? + If non-zero, skip the first line of the :c:member:`PyConfig.run_filename` + source. + + It allows the usage of non-Unix forms of ``#!cmd``. This is intended for + a DOS specific hack only. + + Set to ``1`` by the :option:`-x` command line option. + + Default: ``0``. .. c:member:: wchar_t* stdio_encoding .. c:member:: wchar_t* stdio_errors Encoding and encoding errors of :data:`sys.stdin`, :data:`sys.stdout` and - :data:`sys.stderr`. + :data:`sys.stderr` (but :data:`sys.stderr` always uses + ``"backslashreplace"`` error handler). + + If :c:func:`Py_SetStandardStreamEncoding` has been called, use its + *error* and *errors* arguments if they are not ``NULL``. + + Use the :envvar:`PYTHONIOENCODING` environment variable if it is + non-empty. + + Default encoding: + + * ``"UTF-8"`` if :c:member:`PyPreConfig.utf8_mode` is non-zero. + * Otherwise, use the :term:`locale encoding`. + + Default error handler: + + * On Windows: use ``"surrogateescape"``. + * ``"surrogateescape"`` if :c:member:`PyPreConfig.utf8_mode` is non-zero, + or if the LC_CTYPE locale is "C" or "POSIX". + * ``"strict"`` otherwise. .. c:member:: int tracemalloc + Enable tracemalloc? + If non-zero, call :func:`tracemalloc.start` at startup. + Set by :option:`-X tracemalloc=N <-X>` command line option and by the + :envvar:`PYTHONTRACEMALLOC` environment variable. + + Default: ``-1`` in Python mode, ``0`` in isolated mode. + .. c:member:: int use_environment - If greater than 0, use :ref:`environment variables `. + Use :ref:`environment variables `? + + If equals to zero, ignore the :ref:`environment variables + `. + + Default: ``1`` in Python config and ``0`` in isolated config. .. c:member:: int user_site_directory - If non-zero, add user site directory to :data:`sys.path`. + If non-zero, add the user site directory to :data:`sys.path`. + + Set to ``0`` by the :option:`-s` and :option:`-I` command line options. + + Set to ``0`` by the :envvar:`PYTHONNOUSERSITE` environment variable. + + Default: ``1`` in Python mode, ``0`` in isolated mode. .. c:member:: int verbose - If non-zero, enable verbose mode. + Verbose mode. If greater than 0, print a message each time a module is + imported, showing the place (filename or built-in module) from which + it is loaded. + + If greater or equal to 2, print a message for each file that is checked + for when searching for a module. Also provides information on module + cleanup at exit. + + Incremented by the :option:`-v` command line option. + + Set to the :envvar:`PYTHONVERBOSE` environment variable value. + + Default: ``0``. .. c:member:: PyWideStringList warnoptions - :data:`sys.warnoptions`: options of the :mod:`warnings` module to build - warnings filters: lowest to highest priority. + Options of the :mod:`warnings` module to build warnings filters, lowest + to highest priority: :data:`sys.warnoptions`. The :mod:`warnings` module adds :data:`sys.warnoptions` in the reverse order: the last :c:member:`PyConfig.warnoptions` item becomes the first item of :data:`warnings.filters` which is checked first (highest priority). + The :option:`-W` command line options adds its value to + :c:member:`~PyConfig.warnoptions`, it can be used multiple times. + + The :envvar:`PYTHONWARNINGS` environment variable can also be used to add + warning options. Multiple options can be specified, separated by commas + (``,``). + + Default: empty list. + .. c:member:: int write_bytecode - If non-zero, write ``.pyc`` files. + If equal to 0, Python won't try to write ``.pyc`` files on the import of + source modules. + + Set to ``0`` by the :option:`-B` command line option and the + :envvar:`PYTHONDONTWRITEBYTECODE` environment variable. :data:`sys.dont_write_bytecode` is initialized to the inverted value of :c:member:`~PyConfig.write_bytecode`. + Default: ``1``. + .. c:member:: PyWideStringList xoptions - :data:`sys._xoptions`. + Values of the :option:`-X` command line options: :data:`sys._xoptions`. + + Default: empty list. -If ``parse_argv`` is non-zero, ``argv`` arguments are parsed the same -way the regular Python parses command line arguments, and Python -arguments are stripped from ``argv``: see :ref:`Command Line Arguments -`. +If :c:member:`~PyConfig.parse_argv` is non-zero, :c:member:`~PyConfig.argv` +arguments are parsed the same way the regular Python parses :ref:`command line +arguments `, and Python arguments are stripped from +:c:member:`~PyConfig.argv`. -The ``xoptions`` options are parsed to set other options: see :option:`-X` -option. +The :c:member:`~PyConfig.xoptions` options are parsed to set other options: see +the :option:`-X` command line option. .. versionchanged:: 3.9 @@ -728,7 +1180,7 @@ option. Initialization with PyConfig ----------------------------- +============================ Function to initialize Python: @@ -739,9 +1191,12 @@ Function to initialize Python: The caller is responsible to handle exceptions (error or exit) using :c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. -If ``PyImport_FrozenModules``, ``PyImport_AppendInittab()`` or -``PyImport_ExtendInittab()`` are used, they must be set or called after Python -preinitialization and before the Python initialization. +If :c:func:`PyImport_FrozenModules`, :c:func:`PyImport_AppendInittab` or +:c:func:`PyImport_ExtendInittab` are used, they must be set or called after +Python preinitialization and before the Python initialization. + +The current configuration (``PyConfig`` type) is stored in +``PyInterpreterState.config``. Example setting the program name:: @@ -756,17 +1211,17 @@ Example setting the program name:: status = PyConfig_SetString(&config, &config.program_name, L"/path/to/my_program"); if (PyStatus_Exception(status)) { - goto fail; + goto exception; } status = Py_InitializeFromConfig(&config); if (PyStatus_Exception(status)) { - goto fail; + goto exception; } PyConfig_Clear(&config); return; - fail: + exception: PyConfig_Clear(&config); Py_ExitStatusException(status); } @@ -786,7 +1241,7 @@ configuration, and then override some parameters:: Implicitly preinitialize Python. */ status = PyConfig_SetBytesString(&config, &config.program_name, - program_name); + program_name); if (PyStatus_Exception(status)) { goto done; } @@ -822,7 +1277,7 @@ configuration, and then override some parameters:: .. _init-isolated-conf: Isolated Configuration ----------------------- +====================== :c:func:`PyPreConfig_InitIsolatedConfig` and :c:func:`PyConfig_InitIsolatedConfig` functions create a configuration to @@ -835,7 +1290,7 @@ and user site directory. The C standard streams (ex: ``stdout``) and the LC_CTYPE locale are left unchanged. Signal handlers are not installed. Configuration files are still used with this configuration. Set the -:ref:`Path Configuration ` ("output fields") to ignore these +:ref:`Python Path Configuration ` ("output fields") to ignore these configuration files and avoid the function computing the default path configuration. @@ -843,7 +1298,7 @@ configuration. .. _init-python-config: Python Configuration --------------------- +==================== :c:func:`PyPreConfig_InitPythonConfig` and :c:func:`PyConfig_InitPythonConfig` functions create a configuration to build a customized Python which behaves as @@ -852,50 +1307,16 @@ the regular Python. Environments variables and command line arguments are used to configure Python, whereas global configuration variables are ignored. -This function enables C locale coercion (:pep:`538`) and UTF-8 Mode +This function enables C locale coercion (:pep:`538`) +and :ref:`Python UTF-8 Mode ` (:pep:`540`) depending on the LC_CTYPE locale, :envvar:`PYTHONUTF8` and :envvar:`PYTHONCOERCECLOCALE` environment variables. -Example of customized Python always running in isolated mode:: - - int main(int argc, char **argv) - { - PyStatus status; - - PyConfig config; - PyConfig_InitPythonConfig(&config); - config.isolated = 1; - - /* Decode command line arguments. - Implicitly preinitialize Python (in isolated mode). */ - status = PyConfig_SetBytesArgv(&config, argc, argv); - if (PyStatus_Exception(status)) { - goto fail; - } - - status = Py_InitializeFromConfig(&config); - if (PyStatus_Exception(status)) { - goto fail; - } - PyConfig_Clear(&config); - - return Py_RunMain(); - - fail: - PyConfig_Clear(&config); - if (PyStatus_IsExit(status)) { - return status.exitcode; - } - /* Display the error message and exit the process with - non-zero exit code */ - Py_ExitStatusException(status); - } - .. _init-path-config: -Path Configuration ------------------- +Python Path Configuration +========================= :c:type:`PyConfig` contains multiple fields for the path configuration: @@ -975,7 +1396,7 @@ The ``__PYVENV_LAUNCHER__`` environment variable is used to set Py_RunMain() ------------- +============ .. c:function:: int Py_RunMain(void) @@ -995,7 +1416,7 @@ customized Python always running in isolated mode using Py_GetArgcArgv() ----------------- +================ .. c:function:: void Py_GetArgcArgv(int *argc, wchar_t ***argv) @@ -1005,7 +1426,7 @@ Py_GetArgcArgv() Multi-Phase Initialization Private Provisional API --------------------------------------------------- +================================================== This section is a private provisional API introducing multi-phase initialization, the core feature of the :pep:`432`: diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 7ca8693afab79c..2d85d30702df9c 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -326,7 +326,7 @@ when it's no longer needed---or passing on this responsibility (usually to its caller). When a function passes ownership of a reference on to its caller, the caller is said to receive a *new* reference. When no ownership is transferred, the caller is said to *borrow* the reference. Nothing needs to be done for a -borrowed reference. +:term:`borrowed reference`. Conversely, when a calling function passes in a reference to an object, there are two possibilities: the function *steals* a reference to the object, or it @@ -742,43 +742,22 @@ allocator, or low-level profiling of the main interpreter loop. Only the most frequently-used builds will be described in the remainder of this section. Compiling the interpreter with the :c:macro:`Py_DEBUG` macro defined produces -what is generally meant by "a debug build" of Python. :c:macro:`Py_DEBUG` is -enabled in the Unix build by adding ``--with-pydebug`` to the -:file:`./configure` command. It is also implied by the presence of the +what is generally meant by :ref:`a debug build of Python `. +:c:macro:`Py_DEBUG` is enabled in the Unix build by adding +:option:`--with-pydebug` to the :file:`./configure` command. +It is also implied by the presence of the not-Python-specific :c:macro:`_DEBUG` macro. When :c:macro:`Py_DEBUG` is enabled in the Unix build, compiler optimization is disabled. -In addition to the reference count debugging described below, the following -extra checks are performed: +In addition to the reference count debugging described below, extra checks are +performed, see :ref:`Python Debug Build `. -* Extra checks are added to the object allocator. - -* Extra checks are added to the parser and compiler. - -* Downcasts from wide types to narrow types are checked for loss of information. - -* A number of assertions are added to the dictionary and set implementations. - In addition, the set object acquires a :meth:`test_c_api` method. - -* Sanity checks of the input arguments are added to frame creation. - -* The storage for ints is initialized with a known invalid pattern to catch - reference to uninitialized digits. - -* Low-level tracing and extra exception checking are added to the runtime - virtual machine. - -* Extra checks are added to the memory arena implementation. - -* Extra debugging is added to the thread module. - -There may be additional checks not mentioned here. - -Defining :c:macro:`Py_TRACE_REFS` enables reference tracing. When defined, a -circular doubly linked list of active objects is maintained by adding two extra +Defining :c:macro:`Py_TRACE_REFS` enables reference tracing +(see the :option:`configure --with-trace-refs option <--with-trace-refs>`). +When defined, a circular doubly linked list of active objects is maintained by adding two extra fields to every :c:type:`PyObject`. Total allocations are tracked as well. Upon exit, all existing references are printed. (In interactive mode this happens -after every statement run by the interpreter.) Implied by :c:macro:`Py_DEBUG`. +after every statement run by the interpreter.) Please refer to :file:`Misc/SpecialBuilds.txt` in the Python source distribution for more detailed information. diff --git a/Doc/c-api/iter.rst b/Doc/c-api/iter.rst index 68df6f6e89f513..63290e0a7f0bf7 100644 --- a/Doc/c-api/iter.rst +++ b/Doc/c-api/iter.rst @@ -9,8 +9,15 @@ There are two functions specifically for working with iterators. .. c:function:: int PyIter_Check(PyObject *o) - Return true if the object *o* supports the iterator protocol. + Return non-zero if the object *o* supports the iterator protocol, and ``0`` + otherwise. This function always succeeds. +.. c:function:: int PyAiter_Check(PyObject *o) + + Returns non-zero if the object 'obj' provides :class:`AsyncIterator` + protocols, and ``0`` otherwise. This function always succeeds. + + .. versionadded:: 3.10 .. c:function:: PyObject* PyIter_Next(PyObject *o) diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst index 4d91e4a224fe82..3fcf099134d4dd 100644 --- a/Doc/c-api/iterator.rst +++ b/Doc/c-api/iterator.rst @@ -21,7 +21,8 @@ sentinel value is returned. .. c:function:: int PySeqIter_Check(op) - Return true if the type of *op* is :c:data:`PySeqIter_Type`. + Return true if the type of *op* is :c:data:`PySeqIter_Type`. This function + always succeeds. .. c:function:: PyObject* PySeqIter_New(PyObject *seq) @@ -39,7 +40,8 @@ sentinel value is returned. .. c:function:: int PyCallIter_Check(op) - Return true if the type of *op* is :c:data:`PyCallIter_Type`. + Return true if the type of *op* is :c:data:`PyCallIter_Type`. This + function always succeeds. .. c:function:: PyObject* PyCallIter_New(PyObject *callable, PyObject *sentinel) diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 0bc0785f200d4c..f338e2ae066895 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -22,13 +22,13 @@ List Objects .. c:function:: int PyList_Check(PyObject *p) Return true if *p* is a list object or an instance of a subtype of the list - type. + type. This function always succeeds. .. c:function:: int PyList_CheckExact(PyObject *p) Return true if *p* is a list object, but not an instance of a subtype of - the list type. + the list type. This function always succeeds. .. c:function:: PyObject* PyList_New(Py_ssize_t len) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 4d859fb641df3d..4201490286b82f 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -27,13 +27,13 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: int PyLong_Check(PyObject *p) Return true if its argument is a :c:type:`PyLongObject` or a subtype of - :c:type:`PyLongObject`. + :c:type:`PyLongObject`. This function always succeeds. .. c:function:: int PyLong_CheckExact(PyObject *p) Return true if its argument is a :c:type:`PyLongObject`, but not a subtype of - :c:type:`PyLongObject`. + :c:type:`PyLongObject`. This function always succeeds. .. c:function:: PyObject* PyLong_FromLong(long v) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 87425bcf1e71f2..efddc6f7be5e71 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -92,6 +92,38 @@ for the I/O buffer escapes completely the Python memory manager. statistics of the :ref:`pymalloc memory allocator ` every time a new pymalloc object arena is created, and on shutdown. +Allocator Domains +================= + +All allocating functions belong to one of three different "domains" (see also +:c:type:`PyMemAllocatorDomain`). These domains represent different allocation +strategies and are optimized for different purposes. The specific details on +how every domain allocates memory or what internal functions each domain calls +is considered an implementation detail, but for debugging purposes a simplified +table can be found at :ref:`here `. There is no hard +requirement to use the memory returned by the allocation functions belonging to +a given domain for only the purposes hinted by that domain (although this is the +recommended practice). For example, one could use the memory returned by +:c:func:`PyMem_RawMalloc` for allocating Python objects or the memory returned +by :c:func:`PyObject_Malloc` for allocating memory for buffers. + +The three allocation domains are: + +* Raw domain: intended for allocating memory for general-purpose memory + buffers where the allocation *must* go to the system allocator or where the + allocator can operate without the :term:`GIL`. The memory is requested directly + to the system. + +* "Mem" domain: intended for allocating memory for Python buffers and + general-purpose memory buffers where the allocation must be performed with + the :term:`GIL` held. The memory is taken from the Python private heap. + +* Object domain: intended for allocating memory belonging to Python objects. The + memory is taken from the Python private heap. + +When freeing memory previously allocated by the allocating functions belonging to a +given domain,the matching specific deallocating functions must be used. For example, +:c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. Raw Memory Interface ==================== @@ -272,6 +304,12 @@ The following function sets, modeled after the ANSI C standard, but specifying behavior when requesting zero bytes, are available for allocating and releasing memory from the Python heap. +.. note:: + There is no guarantee that the memory returned by these allocators can be + successfully casted to a Python object when intercepting the allocating + functions in this domain by the methods described in + the :ref:`Customize Memory Allocators ` section. + The :ref:`default object allocator ` uses the :ref:`pymalloc memory allocator `. @@ -347,12 +385,15 @@ Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``mal Legend: -* Name: value for :envvar:`PYTHONMALLOC` environment variable +* Name: value for :envvar:`PYTHONMALLOC` environment variable. * ``malloc``: system allocators from the standard C library, C functions: - :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free` -* ``pymalloc``: :ref:`pymalloc memory allocator ` -* "+ debug": with debug hooks installed by :c:func:`PyMem_SetupDebugHooks` + :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`. +* ``pymalloc``: :ref:`pymalloc memory allocator `. +* "+ debug": with :ref:`debug hooks on the Python memory allocators + `. +* "Debug build": :ref:`Python build in debug mode `. +.. _customize-memory-allocators: Customize Memory Allocators =========================== @@ -438,45 +479,113 @@ Customize Memory Allocators .. c:function:: void PyMem_SetupDebugHooks(void) - Setup hooks to detect bugs in the Python memory allocator functions. + Setup :ref:`debug hooks in the Python memory allocators ` + to detect memory errors. + + +.. _pymem-debug-hooks: + +Debug hooks on the Python memory allocators +=========================================== + +When :ref:`Python is built is debug mode `, the +:c:func:`PyMem_SetupDebugHooks` function is called at the :ref:`Python +preinitialization ` to setup debug hooks on Python memory allocators +to detect memory errors. + +The :envvar:`PYTHONMALLOC` environment variable can be used to install debug +hooks on a Python compiled in release mode (ex: ``PYTHONMALLOC=debug``). + +The :c:func:`PyMem_SetupDebugHooks` function can be used to set debug hooks +after calling :c:func:`PyMem_SetAllocator`. + +These debug hooks fill dynamically allocated memory blocks with special, +recognizable bit patterns. Newly allocated memory is filled with the byte +``0xCD`` (``PYMEM_CLEANBYTE``), freed memory is filled with the byte ``0xDD`` +(``PYMEM_DEADBYTE``). Memory blocks are surrounded by "forbidden bytes" +filled with the byte ``0xFD`` (``PYMEM_FORBIDDENBYTE``). Strings of these bytes +are unlikely to be valid addresses, floats, or ASCII strings. + +Runtime checks: + +- Detect API violations. For example, detect if :c:func:`PyObject_Free` is + called on a memory block allocated by :c:func:`PyMem_Malloc`. +- Detect write before the start of the buffer (buffer underflow). +- Detect write after the end of the buffer (buffer overflow). +- Check that the :term:`GIL ` is held when + allocator functions of :c:data:`PYMEM_DOMAIN_OBJ` (ex: + :c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_MEM` (ex: + :c:func:`PyMem_Malloc`) domains are called. + +On error, the debug hooks use the :mod:`tracemalloc` module to get the +traceback where a memory block was allocated. The traceback is only displayed +if :mod:`tracemalloc` is tracing Python memory allocations and the memory block +was traced. + +Let *S* = ``sizeof(size_t)``. ``2*S`` bytes are added at each end of each block +of *N* bytes requested. The memory layout is like so, where p represents the +address returned by a malloc-like or realloc-like function (``p[i:j]`` means +the slice of bytes from ``*(p+i)`` inclusive up to ``*(p+j)`` exclusive; note +that the treatment of negative indices differs from a Python slice): + +``p[-2*S:-S]`` + Number of bytes originally asked for. This is a size_t, big-endian (easier + to read in a memory dump). +``p[-S]`` + API identifier (ASCII character): + + * ``'r'`` for :c:data:`PYMEM_DOMAIN_RAW`. + * ``'m'`` for :c:data:`PYMEM_DOMAIN_MEM`. + * ``'o'`` for :c:data:`PYMEM_DOMAIN_OBJ`. + +``p[-S+1:0]`` + Copies of PYMEM_FORBIDDENBYTE. Used to catch under- writes and reads. + +``p[0:N]`` + The requested memory, filled with copies of PYMEM_CLEANBYTE, used to catch + reference to uninitialized memory. When a realloc-like function is called + requesting a larger memory block, the new excess bytes are also filled with + PYMEM_CLEANBYTE. When a free-like function is called, these are + overwritten with PYMEM_DEADBYTE, to catch reference to freed memory. When + a realloc- like function is called requesting a smaller memory block, the + excess old bytes are also filled with PYMEM_DEADBYTE. + +``p[N:N+S]`` + Copies of PYMEM_FORBIDDENBYTE. Used to catch over- writes and reads. + +``p[N+S:N+2*S]`` + Only used if the ``PYMEM_DEBUG_SERIALNO`` macro is defined (not defined by + default). + + A serial number, incremented by 1 on each call to a malloc-like or + realloc-like function. Big-endian ``size_t``. If "bad memory" is detected + later, the serial number gives an excellent way to set a breakpoint on the + next run, to capture the instant at which this block was passed out. The + static function bumpserialno() in obmalloc.c is the only place the serial + number is incremented, and exists so you can set such a breakpoint easily. + +A realloc-like or free-like function first checks that the PYMEM_FORBIDDENBYTE +bytes at each end are intact. If they've been altered, diagnostic output is +written to stderr, and the program is aborted via Py_FatalError(). The other +main failure mode is provoking a memory error when a program reads up one of +the special bit patterns and tries to use it as an address. If you get in a +debugger then and look at the object, you're likely to see that it's entirely +filled with PYMEM_DEADBYTE (meaning freed memory is getting used) or +PYMEM_CLEANBYTE (meaning uninitialized memory is getting used). - Newly allocated memory is filled with the byte ``0xCD`` (``CLEANBYTE``), - freed memory is filled with the byte ``0xDD`` (``DEADBYTE``). Memory blocks - are surrounded by "forbidden bytes" (``FORBIDDENBYTE``: byte ``0xFD``). - - Runtime checks: - - - Detect API violations, ex: :c:func:`PyObject_Free` called on a buffer - allocated by :c:func:`PyMem_Malloc` - - Detect write before the start of the buffer (buffer underflow) - - Detect write after the end of the buffer (buffer overflow) - - Check that the :term:`GIL ` is held when - allocator functions of :c:data:`PYMEM_DOMAIN_OBJ` (ex: - :c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_MEM` (ex: - :c:func:`PyMem_Malloc`) domains are called - - On error, the debug hooks use the :mod:`tracemalloc` module to get the - traceback where a memory block was allocated. The traceback is only - displayed if :mod:`tracemalloc` is tracing Python memory allocations and the - memory block was traced. - - These hooks are :ref:`installed by default ` if - Python is compiled in debug - mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install - debug hooks on a Python compiled in release mode. - - .. versionchanged:: 3.6 - This function now also works on Python compiled in release mode. - On error, the debug hooks now use :mod:`tracemalloc` to get the traceback - where a memory block was allocated. The debug hooks now also check - if the GIL is held when functions of :c:data:`PYMEM_DOMAIN_OBJ` and - :c:data:`PYMEM_DOMAIN_MEM` domains are called. +.. versionchanged:: 3.6 + The :c:func:`PyMem_SetupDebugHooks` function now also works on Python + compiled in release mode. On error, the debug hooks now use + :mod:`tracemalloc` to get the traceback where a memory block was allocated. + The debug hooks now also check if the GIL is held when functions of + :c:data:`PYMEM_DOMAIN_OBJ` and :c:data:`PYMEM_DOMAIN_MEM` domains are + called. - .. versionchanged:: 3.8 - Byte patterns ``0xCB`` (``CLEANBYTE``), ``0xDB`` (``DEADBYTE``) and - ``0xFB`` (``FORBIDDENBYTE``) have been replaced with ``0xCD``, ``0xDD`` - and ``0xFD`` to use the same values than Windows CRT debug ``malloc()`` - and ``free()``. +.. versionchanged:: 3.8 + Byte patterns ``0xCB`` (``PYMEM_CLEANBYTE``), ``0xDB`` (``PYMEM_DEADBYTE``) + and ``0xFB`` (``PYMEM_FORBIDDENBYTE``) have been replaced with ``0xCD``, + ``0xDD`` and ``0xFD`` to use the same values than Windows CRT debug + ``malloc()`` and ``free()``. .. _pymalloc: @@ -499,6 +608,10 @@ The arena allocator uses the following functions: * :c:func:`mmap` and :c:func:`munmap` if available, * :c:func:`malloc` and :c:func:`free` otherwise. +This allocator is disabled if Python is configured with the +:option:`--without-pymalloc` option. It can also be disabled at runtime using +the :envvar:`PYTHONMALLOC` environment variable (ex: ``PYTHONMALLOC=malloc``). + Customize pymalloc Arena Allocator ---------------------------------- @@ -516,7 +629,7 @@ Customize pymalloc Arena Allocator +--------------------------------------------------+---------------------------------------+ | ``void* alloc(void *ctx, size_t size)`` | allocate an arena of size bytes | +--------------------------------------------------+---------------------------------------+ - | ``void free(void *ctx, size_t size, void *ptr)`` | free an arena | + | ``void free(void *ctx, void *ptr, size_t size)`` | free an arena | +--------------------------------------------------+---------------------------------------+ .. c:function:: void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) @@ -601,4 +714,3 @@ heap, objects in Python are allocated and released with :c:func:`PyObject_New`, These will be explained in the next chapter on defining and implementing new object types in C. - diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index de429e5c11dc76..24f8c935302e8e 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -45,7 +45,8 @@ any other object. .. c:function:: int PyMemoryView_Check(PyObject *obj) Return true if the object *obj* is a memoryview object. It is not - currently allowed to create subclasses of :class:`memoryview`. + currently allowed to create subclasses of :class:`memoryview`. This + function always succeeds. .. c:function:: Py_buffer *PyMemoryView_GET_BUFFER(PyObject *mview) diff --git a/Doc/c-api/method.rst b/Doc/c-api/method.rst index 0a5341cbbdf152..23852251dfe020 100644 --- a/Doc/c-api/method.rst +++ b/Doc/c-api/method.rst @@ -22,6 +22,7 @@ to bind a :c:data:`PyCFunction` to a class object. It replaces the former call Return true if *o* is an instance method object (has type :c:data:`PyInstanceMethod_Type`). The parameter must not be ``NULL``. + This function always succeeds. .. c:function:: PyObject* PyInstanceMethod_New(PyObject *func) @@ -64,7 +65,7 @@ no longer available. .. c:function:: int PyMethod_Check(PyObject *o) Return true if *o* is a method object (has type :c:data:`PyMethod_Type`). The - parameter must not be ``NULL``. + parameter must not be ``NULL``. This function always succeeds. .. c:function:: PyObject* PyMethod_New(PyObject *func, PyObject *self) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 6e9474bfa40eba..a2541afb685c30 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -19,12 +19,13 @@ Module Objects .. c:function:: int PyModule_Check(PyObject *p) Return true if *p* is a module object, or a subtype of a module object. + This function always succeeds. .. c:function:: int PyModule_CheckExact(PyObject *p) Return true if *p* is a module object, but not a subtype of - :c:data:`PyModule_Type`. + :c:data:`PyModule_Type`. This function always succeeds. .. c:function:: PyObject* PyModule_NewObject(PyObject *name) @@ -264,7 +265,7 @@ of the following two module creation functions: instead; only use this if you are sure you need it. Before it is returned from in the initialization function, the resulting module -object is typically populated using functions like :c:func:`PyModule_AddObject`. +object is typically populated using functions like :c:func:`PyModule_AddObjectRef`. .. _multi-phase-initialization: @@ -437,26 +438,102 @@ a function called from a module execution slot (if using multi-phase initialization), can use the following functions to help initialize the module state: +.. c:function:: int PyModule_AddObjectRef(PyObject *module, const char *name, PyObject *value) + + Add an object to *module* as *name*. This is a convenience function which + can be used from the module's initialization function. + + On success, return ``0``. On error, raise an exception and return ``-1``. + + Return ``NULL`` if *value* is ``NULL``. It must be called with an exception + raised in this case. + + Example usage:: + + static int + add_spam(PyObject *module, int value) + { + PyObject *obj = PyLong_FromLong(value); + if (obj == NULL) { + return -1; + } + int res = PyModule_AddObjectRef(module, "spam", obj); + Py_DECREF(obj); + return res; + } + + The example can also be written without checking explicitly if *obj* is + ``NULL``:: + + static int + add_spam(PyObject *module, int value) + { + PyObject *obj = PyLong_FromLong(value); + int res = PyModule_AddObjectRef(module, "spam", obj); + Py_XDECREF(obj); + return res; + } + + Note that ``Py_XDECREF()`` should be used instead of ``Py_DECREF()`` in + this case, since *obj* can be ``NULL``. + + .. versionadded:: 3.10 + + .. c:function:: int PyModule_AddObject(PyObject *module, const char *name, PyObject *value) - Add an object to *module* as *name*. This is a convenience function which can - be used from the module's initialization function. This steals a reference to - *value* on success. Return ``-1`` on error, ``0`` on success. + Similar to :c:func:`PyModule_AddObjectRef`, but steals a reference to + *value* on success (if it returns ``0``). + + The new :c:func:`PyModule_AddObjectRef` function is recommended, since it is + easy to introduce reference leaks by misusing the + :c:func:`PyModule_AddObject` function. .. note:: - Unlike other functions that steal references, ``PyModule_AddObject()`` only - decrements the reference count of *value* **on success**. + Unlike other functions that steal references, ``PyModule_AddObject()`` + only decrements the reference count of *value* **on success**. This means that its return value must be checked, and calling code must - :c:func:`Py_DECREF` *value* manually on error. Example usage:: - - Py_INCREF(spam); - if (PyModule_AddObject(module, "spam", spam) < 0) { - Py_DECREF(module); - Py_DECREF(spam); - return NULL; - } + :c:func:`Py_DECREF` *value* manually on error. + + Example usage:: + + static int + add_spam(PyObject *module, int value) + { + PyObject *obj = PyLong_FromLong(value); + if (obj == NULL) { + return -1; + } + if (PyModule_AddObject(module, "spam", obj) < 0) { + Py_DECREF(obj); + return -1; + } + // PyModule_AddObject() stole a reference to obj: + // Py_DECREF(obj) is not needed here + return 0; + } + + The example can also be written without checking explicitly if *obj* is + ``NULL``:: + + static int + add_spam(PyObject *module, int value) + { + PyObject *obj = PyLong_FromLong(value); + if (PyModule_AddObject(module, "spam", obj) < 0) { + Py_XDECREF(obj); + return -1; + } + // PyModule_AddObject() stole a reference to obj: + // Py_DECREF(obj) is not needed here + return 0; + } + + Note that ``Py_XDECREF()`` should be used instead of ``Py_DECREF()`` in + this case, since *obj* can be ``NULL``. + .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index a387b4a2df1342..42e3340acb79a0 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -297,8 +297,8 @@ Object Protocol .. c:function:: int PyObject_TypeCheck(PyObject *o, PyTypeObject *type) - Return true if the object *o* is of type *type* or a subtype of *type*. Both - parameters must be non-``NULL``. + Return non-zero if the object *o* is of type *type* or a subtype of *type*, and + ``0`` otherwise. Both parameters must be non-``NULL``. .. c:function:: Py_ssize_t PyObject_Size(PyObject *o) @@ -356,3 +356,14 @@ Object Protocol iterator for the object argument, or the object itself if the object is already an iterator. Raises :exc:`TypeError` and returns ``NULL`` if the object cannot be iterated. + + +.. c:function:: PyObject* PyObject_GetAiter(PyObject *o) + + This is the equivalent to the Python expression ``aiter(o)``. Takes an + :class:`AsyncIterable` object and returns an :class:`AsyncIterator` for it. + This is typically a new iterator but if the argument is an + :class:`AsyncIterator`, this returns itself. Raises :exc:`TypeError` and + returns ``NULL`` if the object cannot be iterated. + + .. versionadded:: 3.10 diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 0df12c67f40bc3..391907c8c2976a 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -13,8 +13,14 @@ objects. .. c:function:: void Py_INCREF(PyObject *o) - Increment the reference count for object *o*. The object must not be ``NULL``; if - you aren't sure that it isn't ``NULL``, use :c:func:`Py_XINCREF`. + Increment the reference count for object *o*. + + This function is usually used to convert a :term:`borrowed reference` to a + :term:`strong reference` in-place. The :c:func:`Py_NewRef` function can be + used to create a new :term:`strong reference`. + + The object must not be ``NULL``; if you aren't sure that it isn't + ``NULL``, use :c:func:`Py_XINCREF`. .. c:function:: void Py_XINCREF(PyObject *o) @@ -22,13 +28,55 @@ objects. Increment the reference count for object *o*. The object may be ``NULL``, in which case the macro has no effect. + See also :c:func:`Py_XNewRef`. + + +.. c:function:: PyObject* Py_NewRef(PyObject *o) + + Create a new :term:`strong reference` to an object: increment the reference + count of the object *o* and return the object *o*. + + When the :term:`strong reference` is no longer needed, :c:func:`Py_DECREF` + should be called on it to decrement the object reference count. + + The object *o* must not be ``NULL``; use :c:func:`Py_XNewRef` if *o* can be + ``NULL``. + + For example:: + + Py_INCREF(obj); + self->attr = obj; + + can be written as:: + + self->attr = Py_NewRef(obj); + + See also :c:func:`Py_INCREF`. + + .. versionadded:: 3.10 + + +.. c:function:: PyObject* Py_XNewRef(PyObject *o) + + Similar to :c:func:`Py_NewRef`, but the object *o* can be NULL. + + If the object *o* is ``NULL``, the function just returns ``NULL``. + + .. versionadded:: 3.10 + .. c:function:: void Py_DECREF(PyObject *o) - Decrement the reference count for object *o*. The object must not be ``NULL``; if - you aren't sure that it isn't ``NULL``, use :c:func:`Py_XDECREF`. If the reference - count reaches zero, the object's type's deallocation function (which must not be - ``NULL``) is invoked. + Decrement the reference count for object *o*. + + If the reference count reaches zero, the object's type's deallocation + function (which must not be ``NULL``) is invoked. + + This function is usually used to delete a :term:`strong reference` before + exiting its scope. + + The object must not be ``NULL``; if you aren't sure that it isn't ``NULL``, + use :c:func:`Py_XDECREF`. .. warning:: diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst index 9207d86012c8b3..64ce4d1d0c34df 100644 --- a/Doc/c-api/reflection.rst +++ b/Doc/c-api/reflection.rst @@ -35,7 +35,7 @@ Reflection Get the *frame* next outer frame. - Return a strong reference, or ``NULL`` if *frame* has no outer frame. + Return a :term:`strong reference`, or ``NULL`` if *frame* has no outer frame. *frame* must not be ``NULL``. @@ -46,7 +46,7 @@ Reflection Get the *frame* code. - Return a strong reference. + Return a :term:`strong reference`. *frame* must not be ``NULL``. The result (frame code) cannot be ``NULL``. diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index 879f394d966cd0..eca19c4d816474 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -53,28 +53,35 @@ the constructor functions work with any iterable Python object. .. c:function:: int PySet_Check(PyObject *p) Return true if *p* is a :class:`set` object or an instance of a subtype. + This function always succeeds. .. c:function:: int PyFrozenSet_Check(PyObject *p) Return true if *p* is a :class:`frozenset` object or an instance of a - subtype. + subtype. This function always succeeds. .. c:function:: int PyAnySet_Check(PyObject *p) Return true if *p* is a :class:`set` object, a :class:`frozenset` object, or an - instance of a subtype. + instance of a subtype. This function always succeeds. + +.. c:function:: int PySet_CheckExact(PyObject *p) + Return true if *p* is a :class:`set` object but not an instance of a + subtype. This function always succeeds. + + .. versionadded:: 3.10 .. c:function:: int PyAnySet_CheckExact(PyObject *p) Return true if *p* is a :class:`set` object or a :class:`frozenset` object but - not an instance of a subtype. + not an instance of a subtype. This function always succeeds. .. c:function:: int PyFrozenSet_CheckExact(PyObject *p) Return true if *p* is a :class:`frozenset` object but not an instance of a - subtype. + subtype. This function always succeeds. .. c:function:: PyObject* PySet_New(PyObject *iterable) diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst index 48a58c6c6f7e36..8271d9acfb645e 100644 --- a/Doc/c-api/slice.rst +++ b/Doc/c-api/slice.rst @@ -14,7 +14,8 @@ Slice Objects .. c:function:: int PySlice_Check(PyObject *ob) - Return true if *ob* is a slice object; *ob* must not be ``NULL``. + Return true if *ob* is a slice object; *ob* must not be ``NULL``. This + function always succeeds. .. c:function:: PyObject* PySlice_New(PyObject *start, PyObject *stop, PyObject *step) diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index a9e1c6fbcc3f94..20d5485d5544c2 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -62,15 +62,44 @@ the definition of all other Python objects. See documentation of :c:type:`PyVarObject` above. +.. c:function:: int Py_Is(const PyObject *x, const PyObject *y) + + Test if the *x* object is the *y* object, the same as ``x is y`` in Python. + + .. versionadded:: 3.10 + + +.. c:function:: int Py_IsNone(const PyObject *x) + + Test if an object is the ``None`` singleton, + the same as ``x is None`` in Python. + + .. versionadded:: 3.10 + + +.. c:function:: int Py_IsTrue(const PyObject *x) + + Test if an object is the ``True`` singleton, + the same as ``x is True`` in Python. + + .. versionadded:: 3.10 + + +.. c:function:: int Py_IsFalse(const PyObject *x) + + Test if an object is the ``False`` singleton, + the same as ``x is False`` in Python. + + .. versionadded:: 3.10 + + .. c:function:: PyTypeObject* Py_TYPE(const PyObject *o) Get the type of the Python object *o*. - Return a borrowed reference. + Return a :term:`borrowed reference`. - .. versionchanged:: 3.10 - :c:func:`Py_TYPE()` is changed to the inline static function. - Use :c:func:`Py_SET_TYPE()` to set an object type. + The :c:func:`Py_SET_TYPE` function must be used to set an object type. .. c:function:: int Py_IS_TYPE(PyObject *o, PyTypeObject *type) @@ -108,9 +137,7 @@ the definition of all other Python objects. Get the size of the Python object *o*. - .. versionchanged:: 3.10 - :c:func:`Py_SIZE()` is changed to the inline static function. - Use :c:func:`Py_SET_SIZE()` to set an object size. + The :c:func:`Py_SET_SIZE` function must be used to set an object size. .. c:function:: void Py_SET_SIZE(PyVarObject *o, Py_ssize_t size) @@ -263,10 +290,12 @@ There are these calling conventions: of :c:type:`PyObject*` values indicating the arguments and the third parameter is the number of arguments (the length of the array). - This is not part of the :ref:`limited API `. - .. versionadded:: 3.7 + .. versionchanged:: 3.10 + + ``METH_FASTCALL`` is now part of the stable ABI. + .. data:: METH_FASTCALL | METH_KEYWORDS diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 9ac91790978926..97717f5fc19230 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -118,22 +118,21 @@ Operating System Utilities .. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size) - Decode a byte string from the locale encoding with the :ref:`surrogateescape - error handler `: undecodable bytes are decoded as - characters in range U+DC80..U+DCFF. If a byte sequence can be decoded as a - surrogate character, escape the bytes using the surrogateescape error - handler instead of decoding them. - - Encoding, highest priority to lowest priority: - - * ``UTF-8`` on macOS, Android, and VxWorks; - * ``UTF-8`` on Windows if :c:data:`Py_LegacyWindowsFSEncodingFlag` is zero; - * ``UTF-8`` if the Python UTF-8 mode is enabled; - * ``ASCII`` if the ``LC_CTYPE`` locale is ``"C"``, - ``nl_langinfo(CODESET)`` returns the ``ASCII`` encoding (or an alias), - and :c:func:`mbstowcs` and :c:func:`wcstombs` functions uses the - ``ISO-8859-1`` encoding. - * the current locale encoding. + .. warning:: + This function should not be called directly: use the :c:type:`PyConfig` + API with the :c:func:`PyConfig_SetBytesString` function which ensures + that :ref:`Python is preinitialized `. + + This function must not be called before :ref:`Python is preinitialized + ` and so that the LC_CTYPE locale is properly configured: see + the :c:func:`Py_PreInitialize` function. + + Decode a byte string from the :term:`filesystem encoding and error handler`. + If the error handler is :ref:`surrogateescape error handler + `, undecodable bytes are decoded as characters in range + U+DC80..U+DCFF; and if a byte sequence can be decoded as a surrogate + character, the bytes are escaped using the surrogateescape error handler + instead of decoding them. Return a pointer to a newly allocated wide character string, use :c:func:`PyMem_RawFree` to free the memory. If size is not ``NULL``, write @@ -143,6 +142,10 @@ Operating System Utilities not ``NULL``, ``*size`` is set to ``(size_t)-1`` on memory error or set to ``(size_t)-2`` on decoding error. + The :term:`filesystem encoding and error handler` are selected by + :c:func:`PyConfig_Read`: see :c:member:`~PyConfig.filesystem_encoding` and + :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`. + Decoding errors should never happen, unless there is a bug in the C library. @@ -157,7 +160,8 @@ Operating System Utilities .. versionadded:: 3.5 .. versionchanged:: 3.7 - The function now uses the UTF-8 encoding in the UTF-8 mode. + The function now uses the UTF-8 encoding in the :ref:`Python UTF-8 Mode + `. .. versionchanged:: 3.8 The function now uses the UTF-8 encoding on Windows if @@ -166,22 +170,10 @@ Operating System Utilities .. c:function:: char* Py_EncodeLocale(const wchar_t *text, size_t *error_pos) - Encode a wide character string to the locale encoding with the - :ref:`surrogateescape error handler `: surrogate characters - in the range U+DC80..U+DCFF are converted to bytes 0x80..0xFF. - - Encoding, highest priority to lowest priority: - - * ``UTF-8`` on macOS, Android, and VxWorks; - * ``UTF-8`` on Windows if :c:data:`Py_LegacyWindowsFSEncodingFlag` is zero; - * ``UTF-8`` if the Python UTF-8 mode is enabled; - * ``ASCII`` if the ``LC_CTYPE`` locale is ``"C"``, - ``nl_langinfo(CODESET)`` returns the ``ASCII`` encoding (or an alias), - and :c:func:`mbstowcs` and :c:func:`wcstombs` functions uses the - ``ISO-8859-1`` encoding. - * the current locale encoding. - - The function uses the UTF-8 encoding in the Python UTF-8 mode. + Encode a wide character string to the :term:`filesystem encoding and error + handler`. If the error handler is :ref:`surrogateescape error handler + `, surrogate characters in the range U+DC80..U+DCFF are + converted to bytes 0x80..0xFF. Return a pointer to a newly allocated byte string, use :c:func:`PyMem_Free` to free the memory. Return ``NULL`` on encoding error or memory allocation @@ -190,9 +182,18 @@ Operating System Utilities If error_pos is not ``NULL``, ``*error_pos`` is set to ``(size_t)-1`` on success, or set to the index of the invalid character on encoding error. + The :term:`filesystem encoding and error handler` are selected by + :c:func:`PyConfig_Read`: see :c:member:`~PyConfig.filesystem_encoding` and + :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`. + Use the :c:func:`Py_DecodeLocale` function to decode the bytes string back to a wide character string. + .. warning:: + This function must not be called before :ref:`Python is preinitialized + ` and so that the LC_CTYPE locale is properly configured: see + the :c:func:`Py_PreInitialize` function. + .. seealso:: The :c:func:`PyUnicode_EncodeFSDefault` and @@ -201,7 +202,8 @@ Operating System Utilities .. versionadded:: 3.5 .. versionchanged:: 3.7 - The function now uses the UTF-8 encoding in the UTF-8 mode. + The function now uses the UTF-8 encoding in the :ref:`Python UTF-8 Mode + `. .. versionchanged:: 3.8 The function now uses the UTF-8 encoding on Windows if diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index bf751e44acde09..6919e61022788f 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -21,14 +21,14 @@ Tuple Objects .. c:function:: int PyTuple_Check(PyObject *p) - Return true if *p* is a tuple object or an instance of a subtype of the tuple - type. + Return true if *p* is a tuple object or an instance of a subtype of the + tuple type. This function always succeeds. .. c:function:: int PyTuple_CheckExact(PyObject *p) Return true if *p* is a tuple object, but not an instance of a subtype of the - tuple type. + tuple type. This function always succeeds. .. c:function:: PyObject* PyTuple_New(Py_ssize_t len) @@ -57,7 +57,7 @@ Tuple Objects .. c:function:: PyObject* PyTuple_GetItem(PyObject *p, Py_ssize_t pos) Return the object at position *pos* in the tuple pointed to by *p*. If *pos* is - out of bounds, return ``NULL`` and set an :exc:`IndexError` exception. + negative or out of bounds, return ``NULL`` and set an :exc:`IndexError` exception. .. c:function:: PyObject* PyTuple_GET_ITEM(PyObject *p, Py_ssize_t pos) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 73f26875d8194a..bdb636dff326f2 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -23,12 +23,14 @@ Type Objects Return non-zero if the object *o* is a type object, including instances of types derived from the standard type object. Return 0 in all other cases. + This function always succeeds. .. c:function:: int PyType_CheckExact(PyObject *o) - Return non-zero if the object *o* is a type object, but not a subtype of the - standard type object. Return 0 in all other cases. + Return non-zero if the object *o* is a type object, but not a subtype of + the standard type object. Return 0 in all other cases. This function + always succeeds. .. c:function:: unsigned int PyType_ClearCache() @@ -105,10 +107,12 @@ Type Objects See :c:member:`PyType_Slot.slot` for possible values of the *slot* argument. - An exception is raised if *type* is not a heap type. - .. versionadded:: 3.4 + .. versionchanged:: 3.10 + :c:func:`PyType_GetSlot` can now accept all types. + Previously, it was limited to :ref:`heap types `. + .. c:function:: PyObject* PyType_GetModule(PyTypeObject *type) Return the module object associated with the given type when the type was @@ -149,13 +153,13 @@ The following functions and structs are used to create .. c:function:: PyObject* PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) - Creates and returns a heap type object from the *spec* + Creates and returns a :ref:`heap type ` from the *spec* (:const:`Py_TPFLAGS_HEAPTYPE`). - If *bases* is a tuple, the created heap type contains all types contained - in it as base types. - - If *bases* is ``NULL``, the *Py_tp_base* slot is used instead. + The *bases* argument can be used to specify base classes; it can either + be only one class or a tuple of classes. + If *bases* is ``NULL``, the *Py_tp_bases* slot is used instead. + If that also is ``NULL``, the *Py_tp_base* slot is used instead. If that also is ``NULL``, the new type derives from :class:`object`. The *module* argument can be used to record the module in which the new @@ -169,6 +173,11 @@ The following functions and structs are used to create .. versionadded:: 3.9 + .. versionchanged:: 3.10 + + The function now accepts a single class as the *bases* argument and + ``NULL`` as the ``tp_doc`` slot. + .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromModuleAndSpec(NULL, spec, bases)``. @@ -247,7 +256,8 @@ The following functions and structs are used to create * :c:member:`~PyBufferProcs.bf_getbuffer` * :c:member:`~PyBufferProcs.bf_releasebuffer` - Setting :c:data:`Py_tp_bases` may be problematic on some platforms. + Setting :c:data:`Py_tp_bases` or :c:data:`Py_tp_base` may be + problematic on some platforms. To avoid issues, use the *bases* argument of :py:func:`PyType_FromSpecWithBases` instead. @@ -260,4 +270,4 @@ The following functions and structs are used to create The desired value of the slot. In most cases, this is a pointer to a function. - May not be ``NULL``. + Slots other than ``Py_tp_doc`` may not be ``NULL``. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index ddcb8ae3d0950c..82f2ab53451166 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -199,6 +199,8 @@ sub-slots +---------------------------------------------------------+-----------------------------------+--------------+ | :c:member:`~PyAsyncMethods.am_anext` | :c:type:`unaryfunc` | __anext__ | +---------------------------------------------------------+-----------------------------------+--------------+ + | :c:member:`~PyAsyncMethods.am_send` | :c:type:`sendfunc` | | + +---------------------------------------------------------+-----------------------------------+--------------+ | | +---------------------------------------------------------+-----------------------------------+--------------+ | :c:member:`~PyNumberMethods.nb_add` | :c:type:`binaryfunc` | __add__ | @@ -480,30 +482,14 @@ metatype) initializes :c:member:`~PyTypeObject.tp_itemsize`, which means that it type objects) *must* have the :attr:`ob_size` field. -.. c:member:: PyObject* PyObject._ob_next - PyObject* PyObject._ob_prev - - These fields are only present when the macro ``Py_TRACE_REFS`` is defined. - Their initialization to ``NULL`` is taken care of by the ``PyObject_HEAD_INIT`` - macro. For statically allocated objects, these fields always remain ``NULL``. - For dynamically allocated objects, these two fields are used to link the object - into a doubly-linked list of *all* live objects on the heap. This could be used - for various debugging purposes; currently the only use is to print the objects - that are still alive at the end of a run when the environment variable - :envvar:`PYTHONDUMPREFS` is set. - - **Inheritance:** - - These fields are not inherited by subtypes. - - .. c:member:: Py_ssize_t PyObject.ob_refcnt This is the type object's reference count, initialized to ``1`` by the - ``PyObject_HEAD_INIT`` macro. Note that for statically allocated type objects, - the type's instances (objects whose :attr:`ob_type` points back to the type) do - *not* count as references. But for dynamically allocated type objects, the - instances *do* count as references. + ``PyObject_HEAD_INIT`` macro. Note that for :ref:`statically allocated type + objects `, the type's instances (objects whose :attr:`ob_type` + points back to the type) do *not* count as references. But for + :ref:`dynamically allocated type objects `, the instances *do* + count as references. **Inheritance:** @@ -533,13 +519,36 @@ type objects) *must* have the :attr:`ob_size` field. This field is inherited by subtypes. +.. c:member:: PyObject* PyObject._ob_next + PyObject* PyObject._ob_prev + + These fields are only present when the macro ``Py_TRACE_REFS`` is defined + (see the :option:`configure --with-trace-refs option <--with-trace-refs>`). + + Their initialization to ``NULL`` is taken care of by the + ``PyObject_HEAD_INIT`` macro. For :ref:`statically allocated objects + `, these fields always remain ``NULL``. For :ref:`dynamically + allocated objects `, these two fields are used to link the + object into a doubly-linked list of *all* live objects on the heap. + + This could be used for various debugging purposes; currently the only uses + are the :func:`sys.getobjects` function and to print the objects that are + still alive at the end of a run when the environment variable + :envvar:`PYTHONDUMPREFS` is set. + + **Inheritance:** + + These fields are not inherited by subtypes. + + PyVarObject Slots ----------------- .. c:member:: Py_ssize_t PyVarObject.ob_size - For statically allocated type objects, this should be initialized to zero. For - dynamically allocated type objects, this field has a special internal meaning. + For :ref:`statically allocated type objects `, this should be + initialized to zero. For :ref:`dynamically allocated type objects + `, this field has a special internal meaning. **Inheritance:** @@ -564,11 +573,13 @@ and :c:type:`PyType_Type` effectively act as defaults.) :class:`T` defined in module :mod:`M` in subpackage :mod:`Q` in package :mod:`P` should have the :c:member:`~PyTypeObject.tp_name` initializer ``"P.Q.M.T"``. - For dynamically allocated type objects, this should just be the type name, and + For :ref:`dynamically allocated type objects `, + this should just be the type name, and the module name explicitly stored in the type dict as the value for key ``'__module__'``. - For statically allocated type objects, the tp_name field should contain a dot. + For :ref:`statically allocated type objects `, + the *tp_name* field should contain a dot. Everything before the last dot is made accessible as the :attr:`__module__` attribute, and everything after the last dot is made accessible as the :attr:`~definition.__name__` attribute. @@ -723,7 +734,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) always inherited. If it's not, then the subclass won't use :ref:`vectorcall `, except when :c:func:`PyVectorcall_Call` is explicitly called. - This is in particular the case for `heap types`_ + This is in particular the case for :ref:`heap types ` (including subclasses defined in Python). @@ -1114,7 +1125,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Inheritance:** - This flag is never inherited by heap types. + This flag is never inherited by :ref:`heap types `. For extension types, it is inherited whenever :c:member:`~PyTypeObject.tp_descr_get` is inherited. @@ -1161,13 +1172,97 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Inheritance:** - This bit is inherited for *static* subtypes if + This bit is inherited for :ref:`static subtypes ` if :c:member:`~PyTypeObject.tp_call` is also inherited. - `Heap types`_ do not inherit ``Py_TPFLAGS_HAVE_VECTORCALL``. + :ref:`Heap types ` do not inherit ``Py_TPFLAGS_HAVE_VECTORCALL``. .. versionadded:: 3.9 + .. data:: Py_TPFLAGS_HAVE_AM_SEND + + This bit is set when the :c:member:`~PyAsyncMethods.am_send` entry is present in the + :c:member:`~PyTypeObject.tp_as_async` slot of type structure. + + .. versionadded:: 3.10 + + .. data:: Py_TPFLAGS_IMMUTABLETYPE + + This bit is set for type objects that are immutable: type attributes cannot be set nor deleted. + + :c:func:`PyType_Ready` automatically applies this flag to + :ref:`static types `. + + **Inheritance:** + + This flag is not inherited. + + .. versionadded:: 3.10 + + .. data:: Py_TPFLAGS_DISALLOW_INSTANTIATION + + Disallow creating instances of the type: set + :c:member:`~PyTypeObject.tp_new` to NULL and don't create the ``__new__`` + key in the type dictionary. + + The flag must be set before creating the type, not after. For example, it + must be set before :c:func:`PyType_Ready` is called on the type. + + The flag is set automatically on :ref:`static types ` if + :c:member:`~PyTypeObject.tp_base` is NULL or ``&PyBaseObject_Type`` and + :c:member:`~PyTypeObject.tp_new` is NULL. + + **Inheritance:** + + This flag is not inherited. + + .. versionadded:: 3.10 + + + .. data:: Py_TPFLAGS_MAPPING + + This bit indicates that instances of the class may match mapping patterns + when used as the subject of a :keyword:`match` block. It is automatically + set when registering or subclassing :class:`collections.abc.Mapping`, and + unset when registering :class:`collections.abc.Sequence`. + + .. note:: + + :const:`Py_TPFLAGS_MAPPING` and :const:`Py_TPFLAGS_SEQUENCE` are + mutually exclusive; it is an error enable both flags simultaneously. + + **Inheritance:** + + This flag is inherited by types that do not already set + :const:`Py_TPFLAGS_SEQUENCE`. + + .. seealso:: :pep:`634` -- Structural Pattern Matching: Specification + + .. versionadded:: 3.10 + + + .. data:: Py_TPFLAGS_SEQUENCE + + This bit indicates that instances of the class may match sequence patterns + when used as the subject of a :keyword:`match` block. It is automatically + set when registering or subclassing :class:`collections.abc.Sequence`, and + unset when registering :class:`collections.abc.Mapping`. + + .. note:: + + :const:`Py_TPFLAGS_MAPPING` and :const:`Py_TPFLAGS_SEQUENCE` are + mutually exclusive; it is an error enable both flags simultaneously. + + **Inheritance:** + + This flag is inherited by types that do not already set + :const:`Py_TPFLAGS_MAPPING`. + + .. seealso:: :pep:`634` -- Structural Pattern Matching: Specification + + .. versionadded:: 3.10 + + .. c:member:: const char* PyTypeObject.tp_doc An optional pointer to a NUL-terminated C string giving the docstring for this @@ -1213,8 +1308,9 @@ and :c:type:`PyType_Type` effectively act as defaults.) :func:`~gc.get_referents` function will include it. .. warning:: - When implementing :c:member:`~PyTypeObject.tp_traverse`, only the members - that the instance *owns* (by having strong references to them) must be + When implementing :c:member:`~PyTypeObject.tp_traverse`, only the + members that the instance *owns* (by having :term:`strong references + ` to them) must be visited. For instance, if an object supports weak references via the :c:member:`~PyTypeObject.tp_weaklist` slot, the pointer supporting the linked list (what *tp_weaklist* points to) must **not** be @@ -1227,9 +1323,8 @@ and :c:type:`PyType_Type` effectively act as defaults.) :c:func:`local_traverse` to have these specific names; don't name them just anything. - Heap-allocated types (:const:`Py_TPFLAGS_HEAPTYPE`, such as those created - with :c:func:`PyType_FromSpec` and similar APIs) hold a reference to their - type. Their traversal function must therefore either visit + Instances of :ref:`heap-allocated types ` hold a reference to + their type. Their traversal function must therefore either visit :c:func:`Py_TYPE(self) `, or delegate this responsibility by calling ``tp_traverse`` of another heap-allocated type (such as a heap-allocated superclass). @@ -1644,8 +1739,8 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Default:** - This slot has no default. For static types, if the field is - ``NULL`` then no :attr:`__dict__` gets created for instances. + This slot has no default. For :ref:`static types `, if the + field is ``NULL`` then no :attr:`__dict__` gets created for instances. .. c:member:: initproc PyTypeObject.tp_init @@ -1680,7 +1775,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) **Default:** - For static types this field does not have a default. + For :ref:`static types ` this field does not have a default. .. c:member:: allocfunc PyTypeObject.tp_alloc @@ -1729,16 +1824,20 @@ and :c:type:`PyType_Type` effectively act as defaults.) in :c:member:`~PyTypeObject.tp_new`, while for mutable types, most initialization should be deferred to :c:member:`~PyTypeObject.tp_init`. + Set the :const:`Py_TPFLAGS_DISALLOW_INSTANTIATION` flag to disallow creating + instances of the type in Python. + **Inheritance:** - This field is inherited by subtypes, except it is not inherited by static types - whose :c:member:`~PyTypeObject.tp_base` is ``NULL`` or ``&PyBaseObject_Type``. + This field is inherited by subtypes, except it is not inherited by + :ref:`static types ` whose :c:member:`~PyTypeObject.tp_base` + is ``NULL`` or ``&PyBaseObject_Type``. **Default:** - For static types this field has no default. This means if the - slot is defined as ``NULL``, the type cannot be called to create new - instances; presumably there is some other way to create + For :ref:`static types ` this field has no default. + This means if the slot is defined as ``NULL``, the type cannot be called + to create new instances; presumably there is some other way to create instances, like a factory function. @@ -1780,7 +1879,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) (The only example of this are types themselves. The metatype, :c:data:`PyType_Type`, defines this function to distinguish between statically - and dynamically allocated types.) + and :ref:`dynamically allocated types `.) **Inheritance:** @@ -1926,10 +2025,10 @@ objects on the thread which called tp_dealloc will not violate any assumptions of the library. -.. _heap-types: +.. _static-types: -Heap Types ----------- +Static Types +------------ Traditionally, types defined in C code are *static*, that is, a static :c:type:`PyTypeObject` structure is defined directly in code @@ -1949,12 +2048,20 @@ Also, since :c:type:`PyTypeObject` is not part of the :ref:`stable ABI ` any extension modules using static types must be compiled for a specific Python minor version. -An alternative to static types is *heap-allocated types*, or *heap types* -for short, which correspond closely to classes created by Python's -``class`` statement. + +.. _heap-types: + +Heap Types +---------- + +An alternative to :ref:`static types ` is *heap-allocated types*, +or *heap types* for short, which correspond closely to classes created by +Python's ``class`` statement. Heap types have the :const:`Py_TPFLAGS_HEAPTYPE` +flag set. This is done by filling a :c:type:`PyType_Spec` structure and calling -:c:func:`PyType_FromSpecWithBases`. +:c:func:`PyType_FromSpec`, :c:func:`PyType_FromSpecWithBases`, +or :c:func:`PyType_FromModuleAndSpec`. .. _number-structs: @@ -2303,6 +2410,7 @@ Async Object Structures unaryfunc am_await; unaryfunc am_aiter; unaryfunc am_anext; + sendfunc am_send; } PyAsyncMethods; .. c:member:: unaryfunc PyAsyncMethods.am_await @@ -2336,6 +2444,15 @@ Async Object Structures Must return an :term:`awaitable` object. See :meth:`__anext__` for details. This slot may be set to ``NULL``. +.. c:member:: sendfunc PyAsyncMethods.am_send + + The signature of this function is:: + + PySendResult am_send(PyObject *self, PyObject *arg, PyObject **result); + + See :c:func:`PyIter_Send` for details. + This slot may be set to ``NULL``. + .. _slot-typedefs: @@ -2431,6 +2548,10 @@ Slot Type typedefs .. c:type:: PyObject *(*binaryfunc)(PyObject *, PyObject *) +.. c:type:: PySendResult (*sendfunc)(PyObject *, PyObject *, PyObject **) + + See :c:member:`~PyAsyncMethods.am_send`. + .. c:type:: PyObject *(*ternaryfunc)(PyObject *, PyObject *, PyObject *) .. c:type:: PyObject *(*ssizeargfunc)(PyObject *, Py_ssize_t) @@ -2452,7 +2573,7 @@ include common usage you may encounter. Some demonstrate tricky corner cases. For more examples, practical info, and a tutorial, see :ref:`defining-new-types` and :ref:`new-types-topics`. -A basic static type:: +A basic :ref:`static type `:: typedef struct { PyObject_HEAD @@ -2541,7 +2662,8 @@ A type that supports weakrefs, instance dicts, and hashing:: }; A str subclass that cannot be subclassed and cannot be called -to create instances (e.g. uses a separate factory func):: +to create instances (e.g. uses a separate factory func) using +:c:data:`Py_TPFLAGS_DISALLOW_INSTANTIATION` flag:: typedef struct { PyUnicodeObject raw; @@ -2554,12 +2676,11 @@ to create instances (e.g. uses a separate factory func):: .tp_basicsize = sizeof(MyStr), .tp_base = NULL, // set to &PyUnicode_Type in module init .tp_doc = "my custom str", - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_new = NULL, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, .tp_repr = (reprfunc)myobj_repr, }; -The simplest static type (with fixed-length instances):: +The simplest :ref:`static type ` with fixed-length instances:: typedef struct { PyObject_HEAD @@ -2570,7 +2691,7 @@ The simplest static type (with fixed-length instances):: .tp_name = "mymod.MyObject", }; -The simplest static type (with variable-length instances):: +The simplest :ref:`static type ` with variable-length instances:: typedef struct { PyObject_VAR_HEAD diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 54bd0a3cbb6556..8d4eef87db9fb6 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -19,8 +19,7 @@ points must be below 1114112 (which is the full Unicode range). :c:type:`Py_UNICODE*` and UTF-8 representations are created on demand and cached in the Unicode object. The :c:type:`Py_UNICODE*` representation is deprecated -and inefficient; it should be avoided in performance- or memory-sensitive -situations. +and inefficient. Due to the transition between the old APIs and the new APIs, Unicode objects can internally be in two states depending on how they were created: @@ -91,13 +90,13 @@ access internal read-only data of Unicode objects: .. c:function:: int PyUnicode_Check(PyObject *o) Return true if the object *o* is a Unicode object or an instance of a Unicode - subtype. + subtype. This function always succeeds. .. c:function:: int PyUnicode_CheckExact(PyObject *o) Return true if the object *o* is a Unicode object, but not an instance of a - subtype. + subtype. This function always succeeds. .. c:function:: int PyUnicode_READY(PyObject *o) @@ -434,7 +433,7 @@ APIs: If *u* is ``NULL``, this function behaves like :c:func:`PyUnicode_FromUnicode` with the buffer set to ``NULL``. This usage is deprecated in favor of - :c:func:`PyUnicode_New`. + :c:func:`PyUnicode_New`, and will be removed in Python 3.12. .. c:function:: PyObject *PyUnicode_FromString(const char *u) @@ -676,7 +675,7 @@ APIs: Deprecated Py_UNICODE APIs """""""""""""""""""""""""" -.. deprecated-removed:: 3.3 4.0 +.. deprecated-removed:: 3.3 3.12 These API functions are deprecated with the implementation of :pep:`393`. Extension modules can continue using them, as they will not be removed in Python @@ -726,6 +725,10 @@ Extension modules can continue using them, as they will not be removed in Python :c:type:`Py_UNICODE` buffer of the given *size* by ASCII digits 0--9 according to their decimal value. Return ``NULL`` if an exception occurs. + .. deprecated-removed:: 3.3 3.11 + Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using + :c:func:`Py_UNICODE_TODECIMAL`. + .. c:function:: Py_UNICODE* PyUnicode_AsUnicodeAndSize(PyObject *unicode, Py_ssize_t *size) @@ -783,7 +786,7 @@ system. :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at Python startup). - This function ignores the Python UTF-8 mode. + This function ignores the :ref:`Python UTF-8 Mode `. .. seealso:: @@ -819,7 +822,7 @@ system. :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at Python startup). - This function ignores the Python UTF-8 mode. + This function ignores the :ref:`Python UTF-8 Mode `. .. seealso:: @@ -878,8 +881,7 @@ conversion function: .. c:function:: PyObject* PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) - Decode a string using :c:data:`Py_FileSystemDefaultEncoding` and the - :c:data:`Py_FileSystemDefaultEncodeErrors` error handler. + Decode a string from the :term:`filesystem encoding and error handler`. If :c:data:`Py_FileSystemDefaultEncoding` is not set, fall back to the locale encoding. @@ -899,8 +901,8 @@ conversion function: .. c:function:: PyObject* PyUnicode_DecodeFSDefault(const char *s) - Decode a null-terminated string using :c:data:`Py_FileSystemDefaultEncoding` - and the :c:data:`Py_FileSystemDefaultEncodeErrors` error handler. + Decode a null-terminated string from the :term:`filesystem encoding and + error handler`. If :c:data:`Py_FileSystemDefaultEncoding` is not set, fall back to the locale encoding. @@ -1045,7 +1047,7 @@ These are the generic codec APIs: to be used is looked up using the Python codec registry. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsEncodedString`. @@ -1118,7 +1120,7 @@ These are the UTF-8 codec APIs: return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsUTF8String`, :c:func:`PyUnicode_AsUTF8AndSize` or :c:func:`PyUnicode_AsEncodedString`. @@ -1192,7 +1194,7 @@ These are the UTF-32 codec APIs: Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsUTF32String` or :c:func:`PyUnicode_AsEncodedString`. @@ -1267,7 +1269,7 @@ These are the UTF-16 codec APIs: Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsUTF16String` or :c:func:`PyUnicode_AsEncodedString`. @@ -1305,7 +1307,7 @@ These are the UTF-7 codec APIs: nonzero, whitespace will be encoded in base-64. Both are set to zero for the Python "utf-7" codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsEncodedString`. @@ -1335,7 +1337,7 @@ These are the "Unicode Escape" codec APIs: Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Unicode-Escape and return a bytes object. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsUnicodeEscapeString`. @@ -1366,7 +1368,7 @@ These are the "Raw Unicode Escape" codec APIs: Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Raw-Unicode-Escape and return a bytes object. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsRawUnicodeEscapeString` or :c:func:`PyUnicode_AsEncodedString`. @@ -1398,7 +1400,7 @@ ordinals and only these are accepted by the codecs during encoding. return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsLatin1String` or :c:func:`PyUnicode_AsEncodedString`. @@ -1430,7 +1432,7 @@ codes generate errors. return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsASCIIString` or :c:func:`PyUnicode_AsEncodedString`. @@ -1482,7 +1484,7 @@ These are the mapping codec APIs: *mapping* object and return the result as a bytes object. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsCharmapString` or :c:func:`PyUnicode_AsEncodedString`. @@ -1514,7 +1516,7 @@ The following codec API is special in that maps Unicode to Unicode. character *mapping* table to it and return the resulting Unicode object. Return ``NULL`` when an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_Translate`. or :ref:`generic codec based API ` diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index b908cb8354f73f..0f760eaa7ad578 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -112,9 +112,9 @@ the same library that the Python runtime is using. Similar to :c:func:`PyRun_SimpleStringFlags`, but the Python source code is read from *fp* instead of an in-memory string. *filename* should be the name of - the file, it is decoded from the filesystem encoding - (:func:`sys.getfilesystemencoding`). If *closeit* is true, the file is - closed before PyRun_SimpleFileExFlags returns. + the file, it is decoded from :term:`filesystem encoding and error handler`. + If *closeit* is true, the file is closed before + ``PyRun_SimpleFileExFlags()`` returns. .. note:: On Windows, *fp* should be opened as binary mode (e.g. ``fopen(filename, "rb")``). @@ -132,7 +132,7 @@ the same library that the Python runtime is using. Read and execute a single statement from a file associated with an interactive device according to the *flags* argument. The user will be prompted using ``sys.ps1`` and ``sys.ps2``. *filename* is decoded from the - filesystem encoding (:func:`sys.getfilesystemencoding`). + :term:`filesystem encoding and error handler`. Returns ``0`` when the input was executed successfully, ``-1`` if there was an exception, or an error code @@ -151,9 +151,8 @@ the same library that the Python runtime is using. Read and execute statements from a file associated with an interactive device until EOF is reached. The user will be prompted using ``sys.ps1`` and - ``sys.ps2``. *filename* is decoded from the filesystem encoding - (:func:`sys.getfilesystemencoding`). Returns ``0`` at EOF or a negative - number upon failure. + ``sys.ps2``. *filename* is decoded from the :term:`filesystem encoding and + error handler`. Returns ``0`` at EOF or a negative number upon failure. .. c:var:: int (*PyOS_InputHook)(void) @@ -206,8 +205,8 @@ the same library that the Python runtime is using. Parse Python source code from *str* using the start token *start* according to the *flags* argument. The result can be used to create a code object which can be evaluated efficiently. This is useful if a code fragment must be evaluated - many times. *filename* is decoded from the filesystem encoding - (:func:`sys.getfilesystemencoding`). + many times. *filename* is decoded from the :term:`filesystem encoding and + error handler`. .. c:function:: struct _node* PyParser_SimpleParseFile(FILE *fp, const char *filename, int start) @@ -262,7 +261,7 @@ the same library that the Python runtime is using. Similar to :c:func:`PyRun_StringFlags`, but the Python source code is read from *fp* instead of an in-memory string. *filename* should be the name of the file, - it is decoded from the filesystem encoding (:func:`sys.getfilesystemencoding`). + it is decoded from the :term:`filesystem encoding and error handler`. If *closeit* is true, the file is closed before :c:func:`PyRun_FileExFlags` returns. @@ -301,7 +300,7 @@ the same library that the Python runtime is using. .. c:function:: PyObject* Py_CompileStringExFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags, int optimize) Like :c:func:`Py_CompileStringObject`, but *filename* is a byte string - decoded from the filesystem encoding (:func:`os.fsdecode`). + decoded from the :term:`filesystem encoding and error handler`. .. versionadded:: 3.2 diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index e3a9bda54d671a..98ebe711adaeb4 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -13,17 +13,18 @@ as much as it can. .. c:function:: int PyWeakref_Check(ob) - Return true if *ob* is either a reference or proxy object. + Return true if *ob* is either a reference or proxy object. This function + always succeeds. .. c:function:: int PyWeakref_CheckRef(ob) - Return true if *ob* is a reference object. + Return true if *ob* is a reference object. This function always succeeds. .. c:function:: int PyWeakref_CheckProxy(ob) - Return true if *ob* is a proxy object. + Return true if *ob* is a proxy object. This function always succeeds. .. c:function:: PyObject* PyWeakref_NewRef(PyObject *ob, PyObject *callback) @@ -57,10 +58,10 @@ as much as it can. .. note:: - This function returns a **borrowed reference** to the referenced object. + This function returns a :term:`borrowed reference` to the referenced object. This means that you should always call :c:func:`Py_INCREF` on the object - except if you know that it cannot be destroyed while you are still - using it. + except when it cannot be destroyed before the last usage of the borrowed + reference. .. c:function:: PyObject* PyWeakref_GET_OBJECT(PyObject *ref) diff --git a/Doc/conf.py b/Doc/conf.py index 079d17717f381c..cf250981f58752 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -15,8 +15,7 @@ extensions = ['sphinx.ext.coverage', 'sphinx.ext.doctest', 'pyspecific', 'c_annotations', 'escape4chm', - 'asdl_highlight', 'peg_highlight'] - + 'asdl_highlight', 'peg_highlight', 'glossary_search'] doctest_global_setup = ''' try: @@ -238,3 +237,5 @@ # bpo-40204: Disable warnings on Sphinx 2 syntax of the C domain since the # documentation is built with -W (warnings treated as errors). c_warn_on_allowed_pre_v3 = False + +strip_signature_backslash = True diff --git a/Doc/copyright.rst b/Doc/copyright.rst index 1b90d9f172c992..4191c0bb63a2c1 100644 --- a/Doc/copyright.rst +++ b/Doc/copyright.rst @@ -4,7 +4,7 @@ Copyright Python and this documentation is: -Copyright © 2001-2020 Python Software Foundation. All rights reserved. +Copyright © 2001-2021 Python Software Foundation. All rights reserved. Copyright © 2000 BeOpen.com. All rights reserved. diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index d01e99ca5e3191..505f1203dd1bdd 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -1073,6 +1073,9 @@ PyInterpreterState_New:PyInterpreterState*::: PyIter_Check:int::: PyIter_Check:PyObject*:o:0: +PyAiter_Check:int::: +PyAiter_Check:PyObject*:o:0: + PyIter_Next:PyObject*::+1: PyIter_Next:PyObject*:o:0: @@ -1679,6 +1682,9 @@ PyObject_GetItem:PyObject*:key:0: PyObject_GetIter:PyObject*::+1: PyObject_GetIter:PyObject*:o:0: +PyObject_GetAiter:PyObject*::+1: +PyObject_GetAiter:PyObject*:o:0: + PyObject_HasAttr:int::: PyObject_HasAttr:PyObject*:o:0: PyObject_HasAttr:PyObject*:attr_name:0: @@ -3007,6 +3013,9 @@ Py_GetVersion:const char*::: Py_INCREF:void::: Py_INCREF:PyObject*:o:+1: +Py_NewRef:void::: +Py_NewRef:PyObject*:o:+1: + Py_Initialize:void::: Py_IsInitialized:int::: @@ -3028,6 +3037,9 @@ Py_XDECREF:PyObject*:o:-1:if o is not NULL Py_XINCREF:void::: Py_XINCREF:PyObject*:o:+1:if o is not NULL +Py_XNewRef:void::: +Py_XNewRef:PyObject*:o:+1:if o is not NULL + _PyImport_Fini:void::: _PyObject_New:PyObject*::+1: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat new file mode 100644 index 00000000000000..833228f7fd755d --- /dev/null +++ b/Doc/data/stable_abi.dat @@ -0,0 +1,953 @@ +# Generated by Tools/scripts/stable_abi.py + +METH_CLASS +METH_COEXIST +METH_FASTCALL +METH_METHOD +METH_NOARGS +METH_O +METH_STATIC +METH_VARARGS +PyAiter_Check +PyArg_Parse +PyArg_ParseTuple +PyArg_ParseTupleAndKeywords +PyArg_UnpackTuple +PyArg_VaParse +PyArg_VaParseTupleAndKeywords +PyArg_ValidateKeywordArguments +PyBaseObject_Type +PyBool_FromLong +PyBool_Type +PyByteArrayIter_Type +PyByteArray_AsString +PyByteArray_Concat +PyByteArray_FromObject +PyByteArray_FromStringAndSize +PyByteArray_Resize +PyByteArray_Size +PyByteArray_Type +PyBytesIter_Type +PyBytes_AsString +PyBytes_AsStringAndSize +PyBytes_Concat +PyBytes_ConcatAndDel +PyBytes_DecodeEscape +PyBytes_FromFormat +PyBytes_FromFormatV +PyBytes_FromObject +PyBytes_FromString +PyBytes_FromStringAndSize +PyBytes_Repr +PyBytes_Size +PyBytes_Type +PyCFunction +PyCFunctionWithKeywords +PyCFunction_Call +PyCFunction_GetFlags +PyCFunction_GetFunction +PyCFunction_GetSelf +PyCFunction_New +PyCFunction_NewEx +PyCFunction_Type +PyCMethod_New +PyCallIter_New +PyCallIter_Type +PyCallable_Check +PyCapsule_Destructor +PyCapsule_GetContext +PyCapsule_GetDestructor +PyCapsule_GetName +PyCapsule_GetPointer +PyCapsule_Import +PyCapsule_IsValid +PyCapsule_New +PyCapsule_SetContext +PyCapsule_SetDestructor +PyCapsule_SetName +PyCapsule_SetPointer +PyCapsule_Type +PyClassMethodDescr_Type +PyCodec_BackslashReplaceErrors +PyCodec_Decode +PyCodec_Decoder +PyCodec_Encode +PyCodec_Encoder +PyCodec_IgnoreErrors +PyCodec_IncrementalDecoder +PyCodec_IncrementalEncoder +PyCodec_KnownEncoding +PyCodec_LookupError +PyCodec_NameReplaceErrors +PyCodec_Register +PyCodec_RegisterError +PyCodec_ReplaceErrors +PyCodec_StreamReader +PyCodec_StreamWriter +PyCodec_StrictErrors +PyCodec_Unregister +PyCodec_XMLCharRefReplaceErrors +PyComplex_FromDoubles +PyComplex_ImagAsDouble +PyComplex_RealAsDouble +PyComplex_Type +PyDescr_NewClassMethod +PyDescr_NewGetSet +PyDescr_NewMember +PyDescr_NewMethod +PyDictItems_Type +PyDictIterItem_Type +PyDictIterKey_Type +PyDictIterValue_Type +PyDictKeys_Type +PyDictProxy_New +PyDictProxy_Type +PyDictRevIterItem_Type +PyDictRevIterKey_Type +PyDictRevIterValue_Type +PyDictValues_Type +PyDict_Clear +PyDict_Contains +PyDict_Copy +PyDict_DelItem +PyDict_DelItemString +PyDict_GetItem +PyDict_GetItemString +PyDict_GetItemWithError +PyDict_Items +PyDict_Keys +PyDict_Merge +PyDict_MergeFromSeq2 +PyDict_New +PyDict_Next +PyDict_SetItem +PyDict_SetItemString +PyDict_Size +PyDict_Type +PyDict_Update +PyDict_Values +PyEllipsis_Type +PyEnum_Type +PyErr_BadArgument +PyErr_BadInternalCall +PyErr_CheckSignals +PyErr_Clear +PyErr_Display +PyErr_ExceptionMatches +PyErr_Fetch +PyErr_Format +PyErr_FormatV +PyErr_GetExcInfo +PyErr_GivenExceptionMatches +PyErr_NewException +PyErr_NewExceptionWithDoc +PyErr_NoMemory +PyErr_NormalizeException +PyErr_Occurred +PyErr_Print +PyErr_PrintEx +PyErr_ProgramText +PyErr_ResourceWarning +PyErr_Restore +PyErr_SetExcFromWindowsErr +PyErr_SetExcFromWindowsErrWithFilename +PyErr_SetExcFromWindowsErrWithFilenameObject +PyErr_SetExcFromWindowsErrWithFilenameObjects +PyErr_SetExcInfo +PyErr_SetFromErrno +PyErr_SetFromErrnoWithFilename +PyErr_SetFromErrnoWithFilenameObject +PyErr_SetFromErrnoWithFilenameObjects +PyErr_SetFromWindowsErr +PyErr_SetFromWindowsErrWithFilename +PyErr_SetImportError +PyErr_SetImportErrorSubclass +PyErr_SetInterrupt +PyErr_SetInterruptEx +PyErr_SetNone +PyErr_SetObject +PyErr_SetString +PyErr_SyntaxLocation +PyErr_SyntaxLocationEx +PyErr_WarnEx +PyErr_WarnExplicit +PyErr_WarnFormat +PyErr_WriteUnraisable +PyEval_AcquireLock +PyEval_AcquireThread +PyEval_CallFunction +PyEval_CallMethod +PyEval_CallObjectWithKeywords +PyEval_EvalCode +PyEval_EvalCodeEx +PyEval_EvalFrame +PyEval_EvalFrameEx +PyEval_GetBuiltins +PyEval_GetFrame +PyEval_GetFuncDesc +PyEval_GetFuncName +PyEval_GetGlobals +PyEval_GetLocals +PyEval_InitThreads +PyEval_ReleaseLock +PyEval_ReleaseThread +PyEval_RestoreThread +PyEval_SaveThread +PyEval_ThreadsInitialized +PyExc_ArithmeticError +PyExc_AssertionError +PyExc_AttributeError +PyExc_BaseException +PyExc_BlockingIOError +PyExc_BrokenPipeError +PyExc_BufferError +PyExc_BytesWarning +PyExc_ChildProcessError +PyExc_ConnectionAbortedError +PyExc_ConnectionError +PyExc_ConnectionRefusedError +PyExc_ConnectionResetError +PyExc_DeprecationWarning +PyExc_EOFError +PyExc_EncodingWarning +PyExc_EnvironmentError +PyExc_Exception +PyExc_FileExistsError +PyExc_FileNotFoundError +PyExc_FloatingPointError +PyExc_FutureWarning +PyExc_GeneratorExit +PyExc_IOError +PyExc_ImportError +PyExc_ImportWarning +PyExc_IndentationError +PyExc_IndexError +PyExc_InterruptedError +PyExc_IsADirectoryError +PyExc_KeyError +PyExc_KeyboardInterrupt +PyExc_LookupError +PyExc_MemoryError +PyExc_ModuleNotFoundError +PyExc_NameError +PyExc_NotADirectoryError +PyExc_NotImplementedError +PyExc_OSError +PyExc_OverflowError +PyExc_PendingDeprecationWarning +PyExc_PermissionError +PyExc_ProcessLookupError +PyExc_RecursionError +PyExc_ReferenceError +PyExc_ResourceWarning +PyExc_RuntimeError +PyExc_RuntimeWarning +PyExc_StopAsyncIteration +PyExc_StopIteration +PyExc_SyntaxError +PyExc_SyntaxWarning +PyExc_SystemError +PyExc_SystemExit +PyExc_TabError +PyExc_TimeoutError +PyExc_TypeError +PyExc_UnboundLocalError +PyExc_UnicodeDecodeError +PyExc_UnicodeEncodeError +PyExc_UnicodeError +PyExc_UnicodeTranslateError +PyExc_UnicodeWarning +PyExc_UserWarning +PyExc_ValueError +PyExc_Warning +PyExc_WindowsError +PyExc_ZeroDivisionError +PyExceptionClass_Name +PyException_GetCause +PyException_GetContext +PyException_GetTraceback +PyException_SetCause +PyException_SetContext +PyException_SetTraceback +PyFile_FromFd +PyFile_GetLine +PyFile_WriteObject +PyFile_WriteString +PyFilter_Type +PyFloat_AsDouble +PyFloat_FromDouble +PyFloat_FromString +PyFloat_GetInfo +PyFloat_GetMax +PyFloat_GetMin +PyFloat_Type +PyFrame_GetCode +PyFrame_GetLineNumber +PyFrozenSet_New +PyFrozenSet_Type +PyGC_Collect +PyGC_Disable +PyGC_Enable +PyGC_IsEnabled +PyGILState_Ensure +PyGILState_GetThisThreadState +PyGILState_Release +PyGILState_STATE +PyGetSetDef +PyGetSetDescr_Type +PyImport_AddModule +PyImport_AddModuleObject +PyImport_AppendInittab +PyImport_ExecCodeModule +PyImport_ExecCodeModuleEx +PyImport_ExecCodeModuleObject +PyImport_ExecCodeModuleWithPathnames +PyImport_GetImporter +PyImport_GetMagicNumber +PyImport_GetMagicTag +PyImport_GetModule +PyImport_GetModuleDict +PyImport_Import +PyImport_ImportFrozenModule +PyImport_ImportFrozenModuleObject +PyImport_ImportModule +PyImport_ImportModuleLevel +PyImport_ImportModuleLevelObject +PyImport_ImportModuleNoBlock +PyImport_ReloadModule +PyIndex_Check +PyInterpreterState +PyInterpreterState_Clear +PyInterpreterState_Delete +PyInterpreterState_Get +PyInterpreterState_GetDict +PyInterpreterState_GetID +PyInterpreterState_New +PyIter_Check +PyIter_Next +PyIter_Send +PyListIter_Type +PyListRevIter_Type +PyList_Append +PyList_AsTuple +PyList_GetItem +PyList_GetSlice +PyList_Insert +PyList_New +PyList_Reverse +PyList_SetItem +PyList_SetSlice +PyList_Size +PyList_Sort +PyList_Type +PyLongObject +PyLongRangeIter_Type +PyLong_AsDouble +PyLong_AsLong +PyLong_AsLongAndOverflow +PyLong_AsLongLong +PyLong_AsLongLongAndOverflow +PyLong_AsSize_t +PyLong_AsSsize_t +PyLong_AsUnsignedLong +PyLong_AsUnsignedLongLong +PyLong_AsUnsignedLongLongMask +PyLong_AsUnsignedLongMask +PyLong_AsVoidPtr +PyLong_FromDouble +PyLong_FromLong +PyLong_FromLongLong +PyLong_FromSize_t +PyLong_FromSsize_t +PyLong_FromString +PyLong_FromUnsignedLong +PyLong_FromUnsignedLongLong +PyLong_FromVoidPtr +PyLong_GetInfo +PyLong_Type +PyMap_Type +PyMapping_Check +PyMapping_GetItemString +PyMapping_HasKey +PyMapping_HasKeyString +PyMapping_Items +PyMapping_Keys +PyMapping_Length +PyMapping_SetItemString +PyMapping_Size +PyMapping_Values +PyMem_Calloc +PyMem_Free +PyMem_Malloc +PyMem_Realloc +PyMemberDef +PyMemberDescr_Type +PyMemoryView_FromMemory +PyMemoryView_FromObject +PyMemoryView_GetContiguous +PyMemoryView_Type +PyMethodDef +PyMethodDescr_Type +PyModuleDef +PyModuleDef_Base +PyModuleDef_Init +PyModuleDef_Type +PyModule_AddFunctions +PyModule_AddIntConstant +PyModule_AddObject +PyModule_AddObjectRef +PyModule_AddStringConstant +PyModule_AddType +PyModule_Create2 +PyModule_ExecDef +PyModule_FromDefAndSpec2 +PyModule_GetDef +PyModule_GetDict +PyModule_GetFilename +PyModule_GetFilenameObject +PyModule_GetName +PyModule_GetNameObject +PyModule_GetState +PyModule_New +PyModule_NewObject +PyModule_SetDocString +PyModule_Type +PyNumber_Absolute +PyNumber_Add +PyNumber_And +PyNumber_AsSsize_t +PyNumber_Check +PyNumber_Divmod +PyNumber_Float +PyNumber_FloorDivide +PyNumber_InPlaceAdd +PyNumber_InPlaceAnd +PyNumber_InPlaceFloorDivide +PyNumber_InPlaceLshift +PyNumber_InPlaceMatrixMultiply +PyNumber_InPlaceMultiply +PyNumber_InPlaceOr +PyNumber_InPlacePower +PyNumber_InPlaceRemainder +PyNumber_InPlaceRshift +PyNumber_InPlaceSubtract +PyNumber_InPlaceTrueDivide +PyNumber_InPlaceXor +PyNumber_Index +PyNumber_Invert +PyNumber_Long +PyNumber_Lshift +PyNumber_MatrixMultiply +PyNumber_Multiply +PyNumber_Negative +PyNumber_Or +PyNumber_Positive +PyNumber_Power +PyNumber_Remainder +PyNumber_Rshift +PyNumber_Subtract +PyNumber_ToBase +PyNumber_TrueDivide +PyNumber_Xor +PyOS_AfterFork +PyOS_AfterFork_Child +PyOS_AfterFork_Parent +PyOS_BeforeFork +PyOS_CheckStack +PyOS_FSPath +PyOS_InputHook +PyOS_InterruptOccurred +PyOS_double_to_string +PyOS_getsig +PyOS_mystricmp +PyOS_mystrnicmp +PyOS_setsig +PyOS_sighandler_t +PyOS_snprintf +PyOS_string_to_double +PyOS_strtol +PyOS_strtoul +PyOS_vsnprintf +PyObject +PyObject_ASCII +PyObject_AsFileDescriptor +PyObject_Bytes +PyObject_Call +PyObject_CallFunction +PyObject_CallFunctionObjArgs +PyObject_CallMethod +PyObject_CallMethodObjArgs +PyObject_CallNoArgs +PyObject_CallObject +PyObject_Calloc +PyObject_ClearWeakRefs +PyObject_DelItem +PyObject_DelItemString +PyObject_Dir +PyObject_Format +PyObject_Free +PyObject_GC_Del +PyObject_GC_IsFinalized +PyObject_GC_IsTracked +PyObject_GC_Track +PyObject_GC_UnTrack +PyObject_GenericGetAttr +PyObject_GenericGetDict +PyObject_GenericSetAttr +PyObject_GenericSetDict +PyObject_GetAiter +PyObject_GetAttr +PyObject_GetAttrString +PyObject_GetItem +PyObject_GetIter +PyObject_HasAttr +PyObject_HasAttrString +PyObject_Hash +PyObject_HashNotImplemented +PyObject_Init +PyObject_InitVar +PyObject_IsInstance +PyObject_IsSubclass +PyObject_IsTrue +PyObject_Length +PyObject_Malloc +PyObject_Not +PyObject_Realloc +PyObject_Repr +PyObject_RichCompare +PyObject_RichCompareBool +PyObject_SelfIter +PyObject_SetAttr +PyObject_SetAttrString +PyObject_SetItem +PyObject_Size +PyObject_Str +PyObject_Type +PyProperty_Type +PyRangeIter_Type +PyRange_Type +PyReversed_Type +PySeqIter_New +PySeqIter_Type +PySequence_Check +PySequence_Concat +PySequence_Contains +PySequence_Count +PySequence_DelItem +PySequence_DelSlice +PySequence_Fast +PySequence_GetItem +PySequence_GetSlice +PySequence_In +PySequence_InPlaceConcat +PySequence_InPlaceRepeat +PySequence_Index +PySequence_Length +PySequence_List +PySequence_Repeat +PySequence_SetItem +PySequence_SetSlice +PySequence_Size +PySequence_Tuple +PySetIter_Type +PySet_Add +PySet_Clear +PySet_Contains +PySet_Discard +PySet_New +PySet_Pop +PySet_Size +PySet_Type +PySlice_AdjustIndices +PySlice_GetIndices +PySlice_GetIndicesEx +PySlice_New +PySlice_Type +PySlice_Unpack +PyState_AddModule +PyState_FindModule +PyState_RemoveModule +PyStructSequence_Desc +PyStructSequence_Field +PyStructSequence_GetItem +PyStructSequence_New +PyStructSequence_NewType +PyStructSequence_SetItem +PySuper_Type +PySys_AddWarnOption +PySys_AddWarnOptionUnicode +PySys_AddXOption +PySys_FormatStderr +PySys_FormatStdout +PySys_GetObject +PySys_GetXOptions +PySys_HasWarnOptions +PySys_ResetWarnOptions +PySys_SetArgv +PySys_SetArgvEx +PySys_SetObject +PySys_SetPath +PySys_WriteStderr +PySys_WriteStdout +PyThreadState +PyThreadState_Clear +PyThreadState_Delete +PyThreadState_Get +PyThreadState_GetDict +PyThreadState_GetFrame +PyThreadState_GetID +PyThreadState_GetInterpreter +PyThreadState_New +PyThreadState_SetAsyncExc +PyThreadState_Swap +PyThread_GetInfo +PyThread_ReInitTLS +PyThread_acquire_lock +PyThread_acquire_lock_timed +PyThread_allocate_lock +PyThread_create_key +PyThread_delete_key +PyThread_delete_key_value +PyThread_exit_thread +PyThread_free_lock +PyThread_get_key_value +PyThread_get_stacksize +PyThread_get_thread_ident +PyThread_get_thread_native_id +PyThread_init_thread +PyThread_release_lock +PyThread_set_key_value +PyThread_set_stacksize +PyThread_start_new_thread +PyThread_tss_alloc +PyThread_tss_create +PyThread_tss_delete +PyThread_tss_free +PyThread_tss_get +PyThread_tss_is_created +PyThread_tss_set +PyTraceBack_Here +PyTraceBack_Print +PyTraceBack_Type +PyTupleIter_Type +PyTuple_GetItem +PyTuple_GetSlice +PyTuple_New +PyTuple_Pack +PyTuple_SetItem +PyTuple_Size +PyTuple_Type +PyTypeObject +PyType_ClearCache +PyType_FromModuleAndSpec +PyType_FromSpec +PyType_FromSpecWithBases +PyType_GenericAlloc +PyType_GenericNew +PyType_GetFlags +PyType_GetModule +PyType_GetModuleState +PyType_GetSlot +PyType_IsSubtype +PyType_Modified +PyType_Ready +PyType_Slot +PyType_Spec +PyType_Type +PyUnicodeDecodeError_Create +PyUnicodeDecodeError_GetEncoding +PyUnicodeDecodeError_GetEnd +PyUnicodeDecodeError_GetObject +PyUnicodeDecodeError_GetReason +PyUnicodeDecodeError_GetStart +PyUnicodeDecodeError_SetEnd +PyUnicodeDecodeError_SetReason +PyUnicodeDecodeError_SetStart +PyUnicodeEncodeError_GetEncoding +PyUnicodeEncodeError_GetEnd +PyUnicodeEncodeError_GetObject +PyUnicodeEncodeError_GetReason +PyUnicodeEncodeError_GetStart +PyUnicodeEncodeError_SetEnd +PyUnicodeEncodeError_SetReason +PyUnicodeEncodeError_SetStart +PyUnicodeIter_Type +PyUnicodeTranslateError_GetEnd +PyUnicodeTranslateError_GetObject +PyUnicodeTranslateError_GetReason +PyUnicodeTranslateError_GetStart +PyUnicodeTranslateError_SetEnd +PyUnicodeTranslateError_SetReason +PyUnicodeTranslateError_SetStart +PyUnicode_Append +PyUnicode_AppendAndDel +PyUnicode_AsASCIIString +PyUnicode_AsCharmapString +PyUnicode_AsDecodedObject +PyUnicode_AsDecodedUnicode +PyUnicode_AsEncodedObject +PyUnicode_AsEncodedString +PyUnicode_AsEncodedUnicode +PyUnicode_AsLatin1String +PyUnicode_AsMBCSString +PyUnicode_AsRawUnicodeEscapeString +PyUnicode_AsUCS4 +PyUnicode_AsUCS4Copy +PyUnicode_AsUTF16String +PyUnicode_AsUTF32String +PyUnicode_AsUTF8AndSize +PyUnicode_AsUTF8String +PyUnicode_AsUnicodeEscapeString +PyUnicode_AsWideChar +PyUnicode_AsWideCharString +PyUnicode_BuildEncodingMap +PyUnicode_Compare +PyUnicode_CompareWithASCIIString +PyUnicode_Concat +PyUnicode_Contains +PyUnicode_Count +PyUnicode_Decode +PyUnicode_DecodeASCII +PyUnicode_DecodeCharmap +PyUnicode_DecodeCodePageStateful +PyUnicode_DecodeFSDefault +PyUnicode_DecodeFSDefaultAndSize +PyUnicode_DecodeLatin1 +PyUnicode_DecodeLocale +PyUnicode_DecodeLocaleAndSize +PyUnicode_DecodeMBCS +PyUnicode_DecodeMBCSStateful +PyUnicode_DecodeRawUnicodeEscape +PyUnicode_DecodeUTF16 +PyUnicode_DecodeUTF16Stateful +PyUnicode_DecodeUTF32 +PyUnicode_DecodeUTF32Stateful +PyUnicode_DecodeUTF7 +PyUnicode_DecodeUTF7Stateful +PyUnicode_DecodeUTF8 +PyUnicode_DecodeUTF8Stateful +PyUnicode_DecodeUnicodeEscape +PyUnicode_EncodeCodePage +PyUnicode_EncodeFSDefault +PyUnicode_EncodeLocale +PyUnicode_FSConverter +PyUnicode_FSDecoder +PyUnicode_Find +PyUnicode_FindChar +PyUnicode_Format +PyUnicode_FromEncodedObject +PyUnicode_FromFormat +PyUnicode_FromFormatV +PyUnicode_FromObject +PyUnicode_FromOrdinal +PyUnicode_FromString +PyUnicode_FromStringAndSize +PyUnicode_FromWideChar +PyUnicode_GetDefaultEncoding +PyUnicode_GetLength +PyUnicode_GetSize +PyUnicode_InternFromString +PyUnicode_InternImmortal +PyUnicode_InternInPlace +PyUnicode_IsIdentifier +PyUnicode_Join +PyUnicode_Partition +PyUnicode_RPartition +PyUnicode_RSplit +PyUnicode_ReadChar +PyUnicode_Replace +PyUnicode_Resize +PyUnicode_RichCompare +PyUnicode_Split +PyUnicode_Splitlines +PyUnicode_Substring +PyUnicode_Tailmatch +PyUnicode_Translate +PyUnicode_Type +PyUnicode_WriteChar +PyVarObject +PyWeakReference +PyWeakref_GetObject +PyWeakref_NewProxy +PyWeakref_NewRef +PyWrapperDescr_Type +PyWrapper_New +PyZip_Type +Py_AddPendingCall +Py_AtExit +Py_BEGIN_ALLOW_THREADS +Py_BLOCK_THREADS +Py_BuildValue +Py_BytesMain +Py_CompileString +Py_DecRef +Py_DecodeLocale +Py_END_ALLOW_THREADS +Py_EncodeLocale +Py_EndInterpreter +Py_EnterRecursiveCall +Py_Exit +Py_FatalError +Py_FileSystemDefaultEncodeErrors +Py_FileSystemDefaultEncoding +Py_Finalize +Py_FinalizeEx +Py_FrozenMain +Py_GenericAlias +Py_GenericAliasType +Py_GetBuildInfo +Py_GetCompiler +Py_GetCopyright +Py_GetExecPrefix +Py_GetPath +Py_GetPlatform +Py_GetPrefix +Py_GetProgramFullPath +Py_GetProgramName +Py_GetPythonHome +Py_GetRecursionLimit +Py_GetVersion +Py_HasFileSystemDefaultEncoding +Py_IncRef +Py_Initialize +Py_InitializeEx +Py_Is +Py_IsFalse +Py_IsInitialized +Py_IsNone +Py_IsTrue +Py_LeaveRecursiveCall +Py_Main +Py_MakePendingCalls +Py_NewInterpreter +Py_NewRef +Py_ReprEnter +Py_ReprLeave +Py_SetPath +Py_SetProgramName +Py_SetPythonHome +Py_SetRecursionLimit +Py_TPFLAGS_BASETYPE +Py_TPFLAGS_DEFAULT +Py_TPFLAGS_HAVE_GC +Py_TPFLAGS_METHOD_DESCRIPTOR +Py_UCS4 +Py_UNBLOCK_THREADS +Py_UTF8Mode +Py_VaBuildValue +Py_XNewRef +Py_am_aiter +Py_am_anext +Py_am_await +Py_am_send +Py_intptr_t +Py_mp_ass_subscript +Py_mp_length +Py_mp_subscript +Py_nb_absolute +Py_nb_add +Py_nb_and +Py_nb_bool +Py_nb_divmod +Py_nb_float +Py_nb_floor_divide +Py_nb_index +Py_nb_inplace_add +Py_nb_inplace_and +Py_nb_inplace_floor_divide +Py_nb_inplace_lshift +Py_nb_inplace_matrix_multiply +Py_nb_inplace_multiply +Py_nb_inplace_or +Py_nb_inplace_power +Py_nb_inplace_remainder +Py_nb_inplace_rshift +Py_nb_inplace_subtract +Py_nb_inplace_true_divide +Py_nb_inplace_xor +Py_nb_int +Py_nb_invert +Py_nb_lshift +Py_nb_matrix_multiply +Py_nb_multiply +Py_nb_negative +Py_nb_or +Py_nb_positive +Py_nb_power +Py_nb_remainder +Py_nb_rshift +Py_nb_subtract +Py_nb_true_divide +Py_nb_xor +Py_sq_ass_item +Py_sq_concat +Py_sq_contains +Py_sq_inplace_concat +Py_sq_inplace_repeat +Py_sq_item +Py_sq_length +Py_sq_repeat +Py_ssize_t +Py_tp_alloc +Py_tp_base +Py_tp_bases +Py_tp_call +Py_tp_clear +Py_tp_dealloc +Py_tp_del +Py_tp_descr_get +Py_tp_descr_set +Py_tp_doc +Py_tp_finalize +Py_tp_free +Py_tp_getattr +Py_tp_getattro +Py_tp_getset +Py_tp_hash +Py_tp_init +Py_tp_is_gc +Py_tp_iter +Py_tp_iternext +Py_tp_members +Py_tp_methods +Py_tp_new +Py_tp_repr +Py_tp_richcompare +Py_tp_setattr +Py_tp_setattro +Py_tp_str +Py_tp_traverse +Py_uintptr_t +_frame +_node +allocfunc +binaryfunc +descrgetfunc +descrsetfunc +destructor +getattrfunc +getattrofunc +getiterfunc +getter +hashfunc +initproc +inquiry +iternextfunc +lenfunc +newfunc +objobjargproc +objobjproc +reprfunc +richcmpfunc +setattrfunc +setattrofunc +setter +ssizeargfunc +ssizeobjargproc +ssizessizeargfunc +ssizessizeobjargproc +symtable +ternaryfunc +traverseproc +unaryfunc +visitproc diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst index b14197c2f94dba..e4437f4106b519 100644 --- a/Doc/distutils/apiref.rst +++ b/Doc/distutils/apiref.rst @@ -1452,6 +1452,8 @@ name. .. module:: distutils.sysconfig :synopsis: Low-level access to configuration information of the Python interpreter. +.. deprecated:: 3.10 + :mod:`distutils.sysconfig` has been merged into :mod:`sysconfig`. .. moduleauthor:: Fred L. Drake, Jr. .. moduleauthor:: Greg Ward .. sectionauthor:: Fred L. Drake, Jr. @@ -1510,6 +1512,9 @@ for other parts of the :mod:`distutils` package. meaning for other platforms will vary. The file is a platform-specific text file, if it exists. This function is only useful on POSIX platforms. +The following functions are deprecated together with this module and they +have no direct replacement. + .. function:: get_python_inc([plat_specific[, prefix]]) @@ -1862,11 +1867,6 @@ Subclasses of :class:`Command` must define the following methods. .. _Windows Installer: https://msdn.microsoft.com/en-us/library/cc185688(VS.85).aspx - In most cases, the ``bdist_msi`` installer is a better choice than the - ``bdist_wininst`` installer, because it provides better support for - Win64 platforms, allows administrators to perform non-interactive - installations, and allows installation through group policies. - :mod:`distutils.command.bdist_rpm` --- Build a binary distribution as a Redhat RPM and SRPM =========================================================================================== @@ -1878,19 +1878,6 @@ Subclasses of :class:`Command` must define the following methods. .. % todo -:mod:`distutils.command.bdist_wininst` --- Build a Windows installer -==================================================================== - -.. module:: distutils.command.bdist_wininst - :synopsis: Build a Windows installer - -.. deprecated:: 3.8 - Use bdist_wheel (wheel packages) instead. - - -.. % todo - - :mod:`distutils.command.sdist` --- Build a source distribution ============================================================== diff --git a/Doc/distutils/builtdist.rst b/Doc/distutils/builtdist.rst index e032c03e229a5c..c4409aca284f44 100644 --- a/Doc/distutils/builtdist.rst +++ b/Doc/distutils/builtdist.rst @@ -96,9 +96,6 @@ The available formats for built distributions are: +-------------+------------------------------+---------+ | ``sdux`` | HP-UX :program:`swinstall` | | +-------------+------------------------------+---------+ -| ``wininst`` | self-extracting ZIP file for | \(4) | -| | Windows | | -+-------------+------------------------------+---------+ | ``msi`` | Microsoft Installer. | | +-------------+------------------------------+---------+ @@ -141,14 +138,9 @@ generated by each, are: +--------------------------+-------------------------------------+ | :command:`bdist_rpm` | rpm, srpm | +--------------------------+-------------------------------------+ -| :command:`bdist_wininst` | wininst | -+--------------------------+-------------------------------------+ | :command:`bdist_msi` | msi | +--------------------------+-------------------------------------+ -.. note:: - bdist_wininst is deprecated since Python 3.8. - .. note:: bdist_msi is deprecated since Python 3.9. @@ -190,8 +182,7 @@ The former allows you to specify RPM-specific options; the latter allows you to easily specify multiple formats in one run. If you need to do both, you can explicitly specify multiple :command:`bdist_\*` commands and their options:: - python setup.py bdist_rpm --packager="John Doe " \ - bdist_wininst --target-version="2.0" + python setup.py bdist_rpm --packager="John Doe " Creating RPM packages is driven by a :file:`.spec` file, much as using the Distutils is driven by the setup script. To make your life easier, the @@ -299,61 +290,6 @@ file winds up deep in the "build tree," in a temporary directory created by .. % to the \file{.spec} file.) -.. _creating-wininst: - -Creating Windows Installers -=========================== - -.. warning:: - bdist_wininst is deprecated since Python 3.8. - -.. warning:: - bdist_msi is deprecated since Python 3.9. - -Executable installers are the natural format for binary distributions on -Windows. They display a nice graphical user interface, display some information -about the module distribution to be installed taken from the metadata in the -setup script, let the user select a few options, and start or cancel the -installation. - -Since the metadata is taken from the setup script, creating Windows installers -is usually as easy as running:: - - python setup.py bdist_wininst - -or the :command:`bdist` command with the :option:`!--formats` option:: - - python setup.py bdist --formats=wininst - -If you have a pure module distribution (only containing pure Python modules and -packages), the resulting installer will be version independent and have a name -like :file:`foo-1.0.win32.exe`. Note that creating ``wininst`` binary -distributions in only supported on Windows systems. - -If you have a non-pure distribution, the extensions can only be created on a -Windows platform, and will be Python version dependent. The installer filename -will reflect this and now has the form :file:`foo-1.0.win32-py2.0.exe`. You -have to create a separate installer for every Python version you want to -support. - -The installer will try to compile pure modules into :term:`bytecode` after installation -on the target system in normal and optimizing mode. If you don't want this to -happen for some reason, you can run the :command:`bdist_wininst` command with -the :option:`!--no-target-compile` and/or the :option:`!--no-target-optimize` -option. - -By default the installer will display the cool "Python Powered" logo when it is -run, but you can also supply your own 152x261 bitmap which must be a Windows -:file:`.bmp` file with the :option:`!--bitmap` option. - -The installer will also display a large title on the desktop background window -when it is run, which is constructed from the name of your distribution and the -version number. This can be changed to another text by using the -:option:`!--title` option. - -The installer file will be written to the "distribution directory" --- normally -:file:`dist/`, but customizable with the :option:`!--dist-dir` option. - .. _cross-compile-windows: Cross-compiling on Windows @@ -370,10 +306,7 @@ For example, on a 32bit version of Windows, you could execute:: python setup.py build --plat-name=win-amd64 -to build a 64bit version of your extension. The Windows Installers also -support this option, so the command:: - - python setup.py build --plat-name=win-amd64 bdist_wininst +to build a 64bit version of your extension. would create a 64bit installation executable on your 32bit version of Windows. @@ -462,18 +395,3 @@ built-in functions in the installation script. and *iconindex* is the index of the icon in the file *iconpath*. Again, for details consult the Microsoft documentation for the :class:`IShellLink` interface. - - -Vista User Access Control (UAC) -=============================== - -Starting with Python 2.6, bdist_wininst supports a :option:`!--user-access-control` -option. The default is 'none' (meaning no UAC handling is done), and other -valid values are 'auto' (meaning prompt for UAC elevation if Python was -installed for all users) and 'force' (meaning always prompt for elevation). - -.. note:: - bdist_wininst is deprecated since Python 3.8. - -.. note:: - bdist_msi is deprecated since Python 3.9. diff --git a/Doc/distutils/commandref.rst b/Doc/distutils/commandref.rst index 0f6fe2aba865f8..3e247e68d3a05f 100644 --- a/Doc/distutils/commandref.rst +++ b/Doc/distutils/commandref.rst @@ -101,6 +101,5 @@ anything except backslash or colon. .. % \subsection{\protect\command{bdist}} .. % \subsection{\protect\command{bdist\_dumb}} .. % \subsection{\protect\command{bdist\_rpm}} -.. % \subsection{\protect\command{bdist\_wininst}} diff --git a/Doc/distutils/index.rst b/Doc/distutils/index.rst index 1f72a25542494a..2ccddc38b5f26f 100644 --- a/Doc/distutils/index.rst +++ b/Doc/distutils/index.rst @@ -12,6 +12,13 @@ :ref:`distributing-index` The up to date module distribution documentations +.. note:: + + The entire ``distutils`` package has been deprecated and will be + removed in Python 3.12. This documentation is retained as a + reference only, and will be removed with the package. See the + :ref:`What's New ` entry for more information. + .. include:: ./_setuptools_disclaimer.rst .. note:: diff --git a/Doc/distutils/introduction.rst b/Doc/distutils/introduction.rst index 1f8a560e138616..87ed178e52bd45 100644 --- a/Doc/distutils/introduction.rst +++ b/Doc/distutils/introduction.rst @@ -112,17 +112,6 @@ the setup script. The difference is which Distutils *commands* they use: the :command:`install` is more often for installers (although most developers will want to install their own code occasionally). -If you want to make things really easy for your users, you can create one or -more built distributions for them. For instance, if you are running on a -Windows machine, and want to make things easy for other Windows users, you can -create an executable installer (the most appropriate type of built distribution -for this platform) with the :command:`bdist_wininst` command. For example:: - - python setup.py bdist_wininst - -will create an executable installer, :file:`foo-1.0.win32.exe`, in the current -directory. - Other useful built distribution formats are RPM, implemented by the :command:`bdist_rpm` command, Solaris :program:`pkgtool` (:command:`bdist_pkgtool`), and HP-UX :program:`swinstall` diff --git a/Doc/extending/building.rst b/Doc/extending/building.rst index 753b5511ed9d63..69dffbd56abf11 100644 --- a/Doc/extending/building.rst +++ b/Doc/extending/building.rst @@ -162,6 +162,5 @@ If the source distribution has been built successfully, maintainers can also create binary distributions. Depending on the platform, one of the following commands can be used to do so. :: - python setup.py bdist_wininst python setup.py bdist_rpm python setup.py bdist_dumb diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 25dc2934d29ef6..561d1c616cc10e 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -410,7 +410,7 @@ optionally followed by an import of the module:: /* Optionally import the module; alternatively, import can be deferred until the embedded script imports it. */ - pmodule = PyImport_ImportModule("spam"); + PyObject *pmodule = PyImport_ImportModule("spam"); if (!pmodule) { PyErr_Print(); fprintf(stderr, "Error: could not import module 'spam'\n"); @@ -911,12 +911,7 @@ the cycle itself. The cycle detector is able to detect garbage cycles and can reclaim them. The :mod:`gc` module exposes a way to run the detector (the :func:`~gc.collect` function), as well as configuration -interfaces and the ability to disable the detector at runtime. The cycle -detector is considered an optional component; though it is included by default, -it can be disabled at build time using the :option:`!--without-cycle-gc` option -to the :program:`configure` script on Unix platforms (including Mac OS X). If -the cycle detector is disabled in this way, the :mod:`gc` module will not be -available. +interfaces and the ability to disable the detector at runtime. .. _refcountsinpython: diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index d9023709fddc86..6e17897ed2c805 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -12,7 +12,7 @@ This section aims to give a quick fly-by on the various type methods you can implement and what they do. Here is the definition of :c:type:`PyTypeObject`, with some fields only used in -debug builds omitted: +:ref:`debug builds ` omitted: .. literalinclude:: ../includes/typestruct.h @@ -287,18 +287,23 @@ combined using bitwise-OR. +===========================+==============================================+ | :const:`READONLY` | Never writable. | +---------------------------+----------------------------------------------+ -| :const:`READ_RESTRICTED` | Not readable in restricted mode. | -+---------------------------+----------------------------------------------+ -| :const:`WRITE_RESTRICTED` | Not writable in restricted mode. | -+---------------------------+----------------------------------------------+ -| :const:`RESTRICTED` | Not readable or writable in restricted mode. | +| :const:`PY_AUDIT_READ` | Emit an ``object.__getattr__`` | +| | :ref:`audit events ` before | +| | reading. | +---------------------------+----------------------------------------------+ +.. versionchanged:: 3.10 + :const:`RESTRICTED`, :const:`READ_RESTRICTED` and :const:`WRITE_RESTRICTED` + are deprecated. However, :const:`READ_RESTRICTED` is an alias for + :const:`PY_AUDIT_READ`, so fields that specify either :const:`RESTRICTED` + or :const:`READ_RESTRICTED` will also raise an audit event. + .. index:: single: READONLY single: READ_RESTRICTED single: WRITE_RESTRICTED single: RESTRICTED + single: PY_AUDIT_READ An interesting advantage of using the :c:member:`~PyTypeObject.tp_members` table to build descriptors that are used at runtime is that any attribute defined this way can diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index 4da77e797d222c..530e2c4d35f848 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -69,7 +69,8 @@ at the start of each object struct and defines a field called ``ob_base`` of type :c:type:`PyObject`, containing a pointer to a type object and a reference count (these can be accessed using the macros :c:macro:`Py_REFCNT` and :c:macro:`Py_TYPE` respectively). The reason for the macro is to -abstract away the layout and to enable additional fields in debug builds. +abstract away the layout and to enable additional fields in :ref:`debug builds +`. .. note:: There is no semicolon above after the :c:macro:`PyObject_HEAD` macro. diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 8cf271c3024084..68570b33e2f626 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -259,9 +259,8 @@ Why isn't there a switch or case statement in Python? ----------------------------------------------------- You can do this easily enough with a sequence of ``if... elif... elif... else``. -There have been some proposals for switch statement syntax, but there is no -consensus (yet) on whether and how to do range tests. See :pep:`275` for -complete details and the current status. +For literal values, or constants within a namespace, you can also use a +``match ... case`` statement. For cases where you need to choose from a very large number of possibilities, you can create a dictionary mapping case values to functions to call. For @@ -601,7 +600,15 @@ test cases at all. Why is there no goto? --------------------- -You can use exceptions to provide a "structured goto" that even works across +In the 1970s people realized that unrestricted goto could lead +to messy "spaghetti" code that was hard to understand and revise. +In a high-level language, it is also unneeded as long as there +are ways to branch (in Python, with ``if`` statements and ``or``, +``and``, and ``if-else`` expressions) and loop (with ``while`` +and ``for`` statements, possibly containing ``continue`` and ``break``). + +One can also use exceptions to provide a "structured goto" +that works even across function calls. Many feel that exceptions can conveniently emulate all reasonable uses of the "go" or "goto" constructs of C, Fortran, and other languages. For example:: diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index 97058b5806a35c..4d27abd4351ec0 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -319,11 +319,11 @@ Here's a trivial example:: try: arg = q.get(block=False) except queue.Empty: - print('Worker', threading.currentThread(), end=' ') + print('Worker', threading.current_thread(), end=' ') print('queue empty') break else: - print('Worker', threading.currentThread(), end=' ') + print('Worker', threading.current_thread(), end=' ') print('running with argument', arg) time.sleep(0.5) diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index b75c60a17a2698..ed03c494388805 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -29,26 +29,26 @@ Python distribution (normally available as Tools/scripts/idle), includes a graphical debugger. PythonWin is a Python IDE that includes a GUI debugger based on pdb. The -Pythonwin debugger colors breakpoints and has quite a few cool features such as -debugging non-Pythonwin programs. Pythonwin is available as part of the `Python -for Windows Extensions `__ project and -as a part of the ActivePython distribution (see -https://www.activestate.com/activepython\ ). +PythonWin debugger colors breakpoints and has quite a few cool features such as +debugging non-PythonWin programs. PythonWin is available as part of +`pywin32 `_ project and +as a part of the +`ActivePython `_ distribution. `Eric `_ is an IDE built on PyQt and the Scintilla editing component. -Pydb is a version of the standard Python debugger pdb, modified for use with DDD -(Data Display Debugger), a popular graphical debugger front end. Pydb can be -found at http://bashdb.sourceforge.net/pydb/ and DDD can be found at -https://www.gnu.org/software/ddd. +`trepan3k `_ is a gdb-like debugger. + +`Visual Studio Code `_ is an IDE with debugging +tools that integrates with version-control software. There are a number of commercial Python IDEs that include graphical debuggers. They include: -* Wing IDE (https://wingware.com/) -* Komodo IDE (https://komodoide.com/) -* PyCharm (https://www.jetbrains.com/pycharm/) +* `Wing IDE `_ +* `Komodo IDE `_ +* `PyCharm `_ Are there tools to help find bugs or perform static analysis? @@ -90,11 +90,12 @@ then compiles the generated C code and links it with the rest of the Python interpreter to form a self-contained binary which acts exactly like your script. Obviously, freeze requires a C compiler. There are several other utilities -which don't. One is Thomas Heller's py2exe (Windows only) at - - http://www.py2exe.org/ +which don't: -Another tool is Anthony Tuininga's `cx_Freeze `_. +* `py2exe `_ for Windows binaries +* `py2app `_ for Mac OS X binaries +* `cx_Freeze `_ for cross-platform + binaries Are there coding standards or a style guide for Python programs? @@ -1420,6 +1421,41 @@ single class, e.g. ``isinstance(obj, (class1, class2, ...))``, and can also check whether an object is one of Python's built-in types, e.g. ``isinstance(obj, str)`` or ``isinstance(obj, (int, float, complex))``. +Note that :func:`isinstance` also checks for virtual inheritance from an +:term:`abstract base class`. So, the test will return ``True`` for a +registered class even if hasn't directly or indirectly inherited from it. To +test for "true inheritance", scan the :term:`MRO` of the class: + +.. testcode:: + + from collections.abc import Mapping + + class P: + pass + + class C(P): + pass + + Mapping.register(P) + +.. doctest:: + + >>> c = C() + >>> isinstance(c, C) # direct + True + >>> isinstance(c, P) # indirect + True + >>> isinstance(c, Mapping) # virtual + True + + # Actual inheritance chain + >>> type(c).__mro__ + (, , ) + + # Test for "true inheritance" + >>> Mapping in type(c).__mro__ + False + Note that most programs do not use :func:`isinstance` on user-defined classes very often. If you are developing the classes yourself, a more proper object-oriented style is to define methods on the classes that encapsulate a @@ -1701,6 +1737,93 @@ to the object: 13891296 +When can I rely on identity tests with the *is* operator? +--------------------------------------------------------- + +The ``is`` operator tests for object identity. The test ``a is b`` is +equivalent to ``id(a) == id(b)``. + +The most important property of an identity test is that an object is always +identical to itself, ``a is a`` always returns ``True``. Identity tests are +usually faster than equality tests. And unlike equality tests, identity tests +are guaranteed to return a boolean ``True`` or ``False``. + +However, identity tests can *only* be substituted for equality tests when +object identity is assured. Generally, there are three circumstances where +identity is guaranteed: + +1) Assignments create new names but do not change object identity. After the +assignment ``new = old``, it is guaranteed that ``new is old``. + +2) Putting an object in a container that stores object references does not +change object identity. After the list assignment ``s[0] = x``, it is +guaranteed that ``s[0] is x``. + +3) If an object is a singleton, it means that only one instance of that object +can exist. After the assignments ``a = None`` and ``b = None``, it is +guaranteed that ``a is b`` because ``None`` is a singleton. + +In most other circumstances, identity tests are inadvisable and equality tests +are preferred. In particular, identity tests should not be used to check +constants such as :class:`int` and :class:`str` which aren't guaranteed to be +singletons:: + + >>> a = 1000 + >>> b = 500 + >>> c = b + 500 + >>> a is c + False + + >>> a = 'Python' + >>> b = 'Py' + >>> c = b + 'thon' + >>> a is c + False + +Likewise, new instances of mutable containers are never identical:: + + >>> a = [] + >>> b = [] + >>> a is b + False + +In the standard library code, you will see several common patterns for +correctly using identity tests: + +1) As recommended by :pep:`8`, an identity test is the preferred way to check +for ``None``. This reads like plain English in code and avoids confusion with +other objects that may have boolean values that evaluate to false. + +2) Detecting optional arguments can be tricky when ``None`` is a valid input +value. In those situations, you can create an singleton sentinel object +guaranteed to be distinct from other objects. For example, here is how +to implement a method that behaves like :meth:`dict.pop`:: + + _sentinel = object() + + def pop(self, key, default=_sentinel): + if key in self: + value = self[key] + del self[key] + return value + if default is _sentinel: + raise KeyError(key) + return default + +3) Container implementations sometimes need to augment equality tests with +identity tests. This prevents the code from being confused by objects such as +``float('NaN')`` that are not equal to themselves. + +For example, here is the implementation of +:meth:`collections.abc.Sequence.__contains__`:: + + def __contains__(self, value): + for v in self: + if v is value or v == value: + return True + return False + + Modules ======= @@ -1776,26 +1899,26 @@ How can I have modules that mutually import each other? Suppose you have the following modules: -foo.py:: +:file:`foo.py`:: from bar import bar_var foo_var = 1 -bar.py:: +:file:`bar.py`:: from foo import foo_var bar_var = 2 The problem is that the interpreter will perform the following steps: -* main imports foo -* Empty globals for foo are created -* foo is compiled and starts executing -* foo imports bar -* Empty globals for bar are created -* bar is compiled and starts executing -* bar imports foo (which is a no-op since there already is a module named foo) -* bar.foo_var = foo.foo_var +* main imports ``foo`` +* Empty globals for ``foo`` are created +* ``foo`` is compiled and starts executing +* ``foo`` imports ``bar`` +* Empty globals for ``bar`` are created +* ``bar`` is compiled and starts executing +* ``bar`` imports ``foo`` (which is a no-op since there already is a module named ``foo``) +* The import mechanism tries to read ``foo_var`` from ``foo`` globals, to set ``bar.foo_var = foo.foo_var`` The last step fails, because Python isn't done with interpreting ``foo`` yet and the global symbol dictionary for ``foo`` is still empty. diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index c8e9c5fb670301..186dac2e255b37 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -140,11 +140,9 @@ offender. How do I make an executable from a Python script? ------------------------------------------------- -See `cx_Freeze `_ for a distutils extension -that allows you to create console and GUI executables from Python code. -`py2exe `_, the most popular extension for building -Python 2.x-based executables, does not yet support Python 3 but a version that -does is in development. +See `cx_Freeze `_ and +`py2exe `_, both are distutils extensions +that allow you to create console and GUI executables from Python code. Is a ``*.pyd`` file the same as a DLL? diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 847500e556056e..29c68ed72c6d70 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -57,6 +57,8 @@ Glossary See :term:`variable annotation`, :term:`function annotation`, :pep:`484` and :pep:`526`, which describe this functionality. + Also see :ref:`annotations-howto` + for best practices on working with annotations. argument A value passed to a :term:`function` (or :term:`method`) when calling the @@ -158,6 +160,18 @@ Glossary See also :term:`text file` for a file object able to read and write :class:`str` objects. + borrowed reference + In Python's C API, a borrowed reference is a reference to an object. + It does not modify the object reference count. It becomes a dangling + pointer if the object is destroyed. For example, a garbage collection can + remove the last :term:`strong reference` to the object and so destroy it. + + Calling :c:func:`Py_INCREF` on the :term:`borrowed reference` is + recommended to convert it to a :term:`strong reference` in-place, except + when the object cannot be destroyed before the last usage of the borrowed + reference. The :c:func:`Py_NewRef` function can be used to create a new + :term:`strong reference`. + bytes-like object An object that supports the :ref:`bufferobjects` and can export a C-:term:`contiguous` buffer. This includes all :class:`bytes`, @@ -386,6 +400,25 @@ Glossary file-like object A synonym for :term:`file object`. + filesystem encoding and error handler + Encoding and error handler used by Python to decode bytes from the + operating system and encode Unicode to the operating system. + + The filesystem encoding must guarantee to successfully decode all bytes + below 128. If the file system encoding fails to provide this guarantee, + API functions can raise :exc:`UnicodeError`. + + The :func:`sys.getfilesystemencoding` and + :func:`sys.getfilesystemencodeerrors` functions can be used to get the + filesystem encoding and error handler. + + The :term:`filesystem encoding and error handler` are configured at + Python startup by the :c:func:`PyConfig_Read` function: see + :c:member:`~PyConfig.filesystem_encoding` and + :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`. + + See also the :term:`locale encoding`. + finder An object that tries to find the :term:`loader` for a module that is being imported. @@ -424,6 +457,8 @@ Glossary See :term:`variable annotation` and :pep:`484`, which describe this functionality. + Also see :ref:`annotations-howto` + for best practices on working with annotations. __future__ A pseudo-module which programmers can use to enable new language features @@ -483,6 +518,13 @@ Glossary See also the :term:`single dispatch` glossary entry, the :func:`functools.singledispatch` decorator, and :pep:`443`. + generic type + A :term:`type` that can be parameterized; typically a container like + :class:`list`. Used for :term:`type hints ` and + :term:`annotations `. + + See :pep:`483` for more details, and :mod:`typing` or + :ref:`generic alias type ` for its uses. GIL See :term:`global interpreter lock`. @@ -666,6 +708,18 @@ Glossary thread removes *key* from *mapping* after the test, but before the lookup. This issue can be solved with locks or by using the EAFP approach. + locale encoding + On Unix, it is the encoding of the LC_CTYPE locale. It can be set with + ``locale.setlocale(locale.LC_CTYPE, new_locale)``. + + On Windows, it is is the ANSI code page (ex: ``cp1252``). + + ``locale.getpreferredencoding(False)`` can be used to get the locale + encoding. + + Python uses the :term:`filesystem encoding and error handler` to convert + between Unicode filenames and bytes filenames. + list A built-in Python :term:`sequence`. Despite its name it is more akin to an array in other languages than to a linked list since access to @@ -1062,6 +1116,18 @@ Glossary an :term:`expression` or one of several constructs with a keyword, such as :keyword:`if`, :keyword:`while` or :keyword:`for`. + strong reference + In Python's C API, a strong reference is a reference to an object + which increments the object's reference count when it is created and + decrements the object's reference count when it is deleted. + + The :c:func:`Py_NewRef` function can be used to create a strong reference + to an object. Usually, the :c:func:`Py_DECREF` function must be called on + the strong reference before exiting the scope of the strong reference, to + avoid leaking one reference. + + See also :term:`borrowed reference`. + text encoding A codec which encodes Unicode strings to bytes. @@ -1149,6 +1215,8 @@ Glossary See :term:`function annotation`, :pep:`484` and :pep:`526`, which describe this functionality. + Also see :ref:`annotations-howto` + for best practices on working with annotations. virtual environment A cooperatively isolated runtime environment that allows Python users diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst new file mode 100644 index 00000000000000..3e61103e99c9a6 --- /dev/null +++ b/Doc/howto/annotations.rst @@ -0,0 +1,226 @@ +.. _annotations-howto: + +************************** +Annotations Best Practices +************************** + +:author: Larry Hastings + +.. topic:: Abstract + + This document is designed to encapsulate the best practices + for working with annotations dicts. If you write Python code + that examines ``__annotations__`` on Python objects, we + encourage you to follow the guidelines described below. + + The document is organized into four sections: + best practices for accessing the annotations of an object + in Python versions 3.10 and newer, + best practices for accessing the annotations of an object + in Python versions 3.9 and older, + other best practices + for ``__annotations__`` that apply to any Python version, + and + quirks of ``__annotations__``. + + Note that this document is specifically about working with + ``__annotations__``, not uses *for* annotations. + If you're looking for information on how to use "type hints" + in your code, please see the :mod:`typing` module. + + +Accessing The Annotations Dict Of An Object In Python 3.10 And Newer +==================================================================== + + Python 3.10 adds a new function to the standard library: + :func:`inspect.get_annotations`. In Python versions 3.10 + and newer, calling this function is the best practice for + accessing the annotations dict of any object that supports + annotations. This function can also "un-stringize" + stringized annotations for you. + + If for some reason :func:`inspect.get_annotations` isn't + viable for your use case, you may access the + ``__annotations__`` data member manually. Best practice + for this changed in Python 3.10 as well: as of Python 3.10, + ``o.__annotations__`` is guaranteed to *always* work + on Python functions, classes, and modules. If you're + certain the object you're examining is one of these three + *specific* objects, you may simply use ``o.__annotations__`` + to get at the object's annotations dict. + + However, other types of callables--for example, + callables created by :func:`functools.partial`--may + not have an ``__annotations__`` attribute defined. When + accessing the ``__annotations__`` of a possibly unknown + object, best practice in Python versions 3.10 and + newer is to call :func:`getattr` with three arguments, + for example ``getattr(o, '__annotations__', None)``. + + +Accessing The Annotations Dict Of An Object In Python 3.9 And Older +=================================================================== + + In Python 3.9 and older, accessing the annotations dict + of an object is much more complicated than in newer versions. + The problem is a design flaw in these older versions of Python, + specifically to do with class annotations. + + Best practice for accessing the annotations dict of other + objects--functions, other callables, and modules--is the same + as best practice for 3.10, assuming you aren't calling + :func:`inspect.get_annotations`: you should use three-argument + :func:`getattr` to access the object's ``__annotations__`` + attribute. + + Unfortunately, this isn't best practice for classes. The problem + is that, since ``__annotations__`` is optional on classes, and + because classes can inherit attributes from their base classes, + accessing the ``__annotations__`` attribute of a class may + inadvertently return the annotations dict of a *base class.* + As an example:: + + class Base: + a: int = 3 + b: str = 'abc' + + class Derived(Base): + pass + + print(Derived.__annotations__) + + This will print the annotations dict from ``Base``, not + ``Derived``. + + Your code will have to have a separate code path if the object + you're examining is a class (``isinstance(o, type)``). + In that case, best practice relies on an implementation detail + of Python 3.9 and before: if a class has annotations defined, + they are stored in the class's ``__dict__`` dictionary. Since + the class may or may not have annotations defined, best practice + is to call the ``get`` method on the class dict. + + To put it all together, here is some sample code that safely + accesses the ``__annotations__`` attribute on an arbitrary + object in Python 3.9 and before:: + + if isinstance(o, type): + ann = o.__dict__.get('__annotations__', None) + else: + ann = getattr(o, '__annotations__', None) + + After running this code, ``ann`` should be either a + dictionary or ``None``. You're encouraged to double-check + the type of ``ann`` using :func:`isinstance` before further + examination. + + Note that some exotic or malformed type objects may not have + a ``__dict__`` attribute, so for extra safety you may also wish + to use :func:`getattr` to access ``__dict__``. + + +Manually Un-Stringizing Stringized Annotations +============================================== + + In situations where some annotations may be "stringized", + and you wish to evaluate those strings to produce the + Python values they represent, it really is best to + call :func:`inspect.get_annotations` to do this work + for you. + + If you're using Python 3.9 or older, or if for some reason + you can't use :func:`inspect.get_annotations`, you'll need + to duplicate its logic. You're encouraged to examine the + implementation of :func:`inspect.get_annotations` in the + current Python version and follow a similar approach. + + In a nutshell, if you wish to evaluate a stringized annotation + on an arbitrary object ``o``: + + * If ``o`` is a module, use ``o.__dict__`` as the + ``globals`` when calling :func:`eval`. + * If ``o`` is a class, use ``sys.modules[o.__module__].__dict__`` + as the ``globals``, and ``dict(vars(o))`` as the ``locals``, + when calling :func:`eval`. + * If ``o`` is a wrapped callable using :func:`functools.update_wrapper`, + :func:`functools.wraps`, or :func:`functools.partial`, iteratively + unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as + appropriate, until you have found the root unwrapped function. + * If ``o`` is a callable (but not a class), use + ``o.__globals__`` as the globals when calling :func:`eval`. + + However, not all string values used as annotations can + be successfully turned into Python values by :func:`eval`. + String values could theoretically contain any valid string, + and in practice there are valid use cases for type hints that + require annotating with string values that specifically + *can't* be evaluated. For example: + + * :pep:`604` union types using `|`, before support for this + was added to Python 3.10. + * Definitions that aren't needed at runtime, only imported + when :const:`typing.TYPE_CHECKING` is true. + + If :func:`eval` attempts to evaluate such values, it will + fail and raise an exception. So, when designing a library + API that works with annotations, it's recommended to only + attempt to evaluate string values when explicitly requested + to by the caller. + + +Best Practices For ``__annotations__`` In Any Python Version +============================================================ + + * You should avoid assigning to the ``__annotations__`` member + of objects directly. Let Python manage setting ``__annotations__``. + + * If you do assign directly to the ``__annotations__`` member + of an object, you should always set it to a ``dict`` object. + + * If you directly access the ``__annotations__`` member + of an object, you should ensure that it's a + dictionary before attempting to examine its contents. + + * You should avoid modifying ``__annotations__`` dicts. + + * You should avoid deleting the ``__annotations__`` attribute + of an object. + + +``__annotations__`` Quirks +========================== + + In all versions of Python 3, function + objects lazy-create an annotations dict if no annotations + are defined on that object. You can delete the ``__annotations__`` + attribute using ``del fn.__annotations__``, but if you then + access ``fn.__annotations__`` the object will create a new empty dict + that it will store and return as its annotations. Deleting the + annotations on a function before it has lazily created its annotations + dict will throw an ``AttributeError``; using ``del fn.__annotations__`` + twice in a row is guaranteed to always throw an ``AttributeError``. + + Everything in the above paragraph also applies to class and module + objects in Python 3.10 and newer. + + In all versions of Python 3, you can set ``__annotations__`` + on a function object to ``None``. However, subsequently + accessing the annotations on that object using ``fn.__annotations__`` + will lazy-create an empty dictionary as per the first paragraph of + this section. This is *not* true of modules and classes, in any Python + version; those objects permit setting ``__annotations__`` to any + Python value, and will retain whatever value is set. + + If Python stringizes your annotations for you + (using ``from __future__ import annotations``), and you + specify a string as an annotation, the string will + itself be quoted. In effect the annotation is quoted + *twice.* For example:: + + from __future__ import annotations + def foo(a: "str"): pass + + print(foo.__annotations__) + + This prints ``{'a': "'str'"}``. This shouldn't really be considered + a "quirk"; it's mentioned here simply because it might be surprising. diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 76d8e6be42935f..a97d10cfe6bb69 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -83,7 +83,7 @@ Following is a result of running the code: $ python3 prog.py --help usage: prog.py [-h] - optional arguments: + options: -h, --help show this help message and exit $ python3 prog.py --verbose usage: prog.py [-h] @@ -130,7 +130,7 @@ And running the code: positional arguments: echo - optional arguments: + options: -h, --help show this help message and exit $ python3 prog.py foo foo @@ -172,7 +172,7 @@ And we get: positional arguments: echo echo the string you use here - optional arguments: + options: -h, --help show this help message and exit Now, how about doing something even more useful:: @@ -241,7 +241,7 @@ And the output: $ python3 prog.py --help usage: prog.py [-h] [--verbosity VERBOSITY] - optional arguments: + options: -h, --help show this help message and exit --verbosity VERBOSITY increase output verbosity @@ -289,7 +289,7 @@ And the output: $ python3 prog.py --help usage: prog.py [-h] [--verbose] - optional arguments: + options: -h, --help show this help message and exit --verbose increase output verbosity @@ -332,7 +332,7 @@ And here goes: $ python3 prog.py --help usage: prog.py [-h] [-v] - optional arguments: + options: -h, --help show this help message and exit -v, --verbose increase output verbosity @@ -440,7 +440,7 @@ And the output: positional arguments: square display a square of a given number - optional arguments: + options: -h, --help show this help message and exit -v {0,1,2}, --verbosity {0,1,2} increase output verbosity @@ -468,7 +468,8 @@ verbosity argument (check the output of ``python --help``):: print(answer) We have introduced another action, "count", -to count the number of occurrences of a specific optional arguments: +to count the number of occurrences of specific options. + .. code-block:: shell-session @@ -489,7 +490,7 @@ to count the number of occurrences of a specific optional arguments: positional arguments: square display a square of a given number - optional arguments: + options: -h, --help show this help message and exit -v, --verbosity increase output verbosity $ python3 prog.py 4 -vvv @@ -626,7 +627,7 @@ Output: x the base y the exponent - optional arguments: + options: -h, --help show this help message and exit -v, --verbosity $ python3 prog.py 4 2 -v @@ -750,7 +751,7 @@ but not both at the same time: x the base y the exponent - optional arguments: + options: -h, --help show this help message and exit -v, --verbose -q, --quiet diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 50041829b8c388..3a3653a5ee3a3e 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1206,6 +1206,68 @@ type for ``self``, it's best to create your own converter, subclassing [clinic start generated code]*/ +Using a "defining class" converter +---------------------------------- + +Argument Clinic facilitates gaining access to the defining class of a method. +This is useful for :ref:`heap type ` methods that need to fetch +module level state. Use :c:func:`PyType_FromModuleAndSpec` to associate a new +heap type with a module. You can now use :c:func:`PyType_GetModuleState` on +the defining class to fetch the module state, for example from a module method. + +Example from ``Modules/zlibmodule.c``. First, ``defining_class`` is added to +the clinic input:: + + /*[clinic input] + zlib.Compress.compress + + cls: defining_class + data: Py_buffer + Binary data to be compressed. + / + + +After running the Argument Clinic tool, the following function signature is +generated:: + + /*[clinic start generated code]*/ + static PyObject * + zlib_Compress_compress_impl(compobject *self, PyTypeObject *cls, + Py_buffer *data) + /*[clinic end generated code: output=6731b3f0ff357ca6 input=04d00f65ab01d260]*/ + + +The following code can now use ``PyType_GetModuleState(cls)`` to fetch the +module state:: + + zlibstate *state = PyType_GetModuleState(cls); + + +Each method may only have one argument using this converter, and it must appear +after ``self``, or, if ``self`` is not used, as the first argument. The argument +will be of type ``PyTypeObject *``. The argument will not appear in the +``__text_signature__``. + +The ``defining_class`` converter is not compatible with ``__init__`` and ``__new__`` +methods, which cannot use the ``METH_METHOD`` convention. + +It is not possible to use ``defining_class`` with slot methods. In order to +fetch the module state from such methods, use ``_PyType_GetModuleByDef`` to +look up the module and then :c:func:`PyModule_GetState` to fetch the module +state. Example from the ``setattro`` slot method in +``Modules/_threadmodule.c``:: + + static int + local_setattro(localobject *self, PyObject *name, PyObject *v) + { + PyObject *module = _PyType_GetModuleByDef(Py_TYPE(self), &thread_module); + thread_module_state *state = get_thread_state(module); + ... + } + + +See also :pep:`573`. + Writing a custom converter -------------------------- diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index f1d1ab1d1d6101..575caeb720f3d0 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -16,7 +16,7 @@ storage, and deletion. This guide has four major sections: 1) The "primer" gives a basic overview, moving gently from simple examples, - adding one feature at a time. It is a great place to start. + adding one feature at a time. Start here if you're new to descriptors. 2) The second section shows a complete, practical descriptor example. If you already know the basics, start there. @@ -27,8 +27,9 @@ This guide has four major sections: 4) The last section has pure Python equivalents for built-in descriptors that are written in C. Read this if you're curious about how functions turn - into bound methods or about how to implement common tools like - :func:`classmethod`, :func:`staticmethod`, and :func:`property`. + into bound methods or about the implementation of common tools like + :func:`classmethod`, :func:`staticmethod`, :func:`property`, and + :term:`__slots__`. Primer @@ -41,21 +42,27 @@ add new capabilities one by one. Simple example: A descriptor that returns a constant ---------------------------------------------------- -The :class:`Ten` class is a descriptor that always returns the constant ``10``:: +The :class:`Ten` class is a descriptor whose :meth:`__get__` method always +returns the constant ``10``: +.. testcode:: class Ten: def __get__(self, obj, objtype=None): return 10 -To use the descriptor, it must be stored as a class variable in another class:: +To use the descriptor, it must be stored as a class variable in another class: + +.. testcode:: class A: x = 5 # Regular class attribute - y = Ten() # Descriptor + y = Ten() # Descriptor instance An interactive session shows the difference between normal attribute lookup -and descriptor lookup:: +and descriptor lookup: + +.. doctest:: >>> a = A() # Make an instance of class A >>> a.x # Normal attribute lookup @@ -63,9 +70,11 @@ and descriptor lookup:: >>> a.y # Descriptor lookup 10 -In the ``a.x`` attribute lookup, the dot operator finds the value ``5`` stored -in the class dictionary. In the ``a.y`` descriptor lookup, the dot operator -calls the descriptor's :meth:`__get__()` method. That method returns ``10``. +In the ``a.x`` attribute lookup, the dot operator finds ``'x': 5`` +in the class dictionary. In the ``a.y`` lookup, the dot operator +finds a descriptor instance, recognized by its ``__get__`` method. +Calling that method returns ``10``. + Note that the value ``10`` is not stored in either the class dictionary or the instance dictionary. Instead, the value ``10`` is computed on demand. @@ -78,8 +87,10 @@ In the next section, we'll create something more useful, a dynamic lookup. Dynamic lookups --------------- -Interesting descriptors typically run computations instead of doing lookups:: +Interesting descriptors typically run computations instead of returning +constants: +.. testcode:: import os @@ -90,7 +101,7 @@ Interesting descriptors typically run computations instead of doing lookups:: class Directory: - size = DirectorySize() # Descriptor + size = DirectorySize() # Descriptor instance def __init__(self, dirname): self.dirname = dirname # Regular instance attribute @@ -98,21 +109,20 @@ Interesting descriptors typically run computations instead of doing lookups:: An interactive session shows that the lookup is dynamic — it computes different, updated answers each time:: - >>> g = Directory('games') >>> s = Directory('songs') - >>> g.size # The games directory has three files - 3 - >>> os.system('touch games/newfile') # Add a fourth file to the directory - 0 - >>> g.size # Automatically updated - 4 + >>> g = Directory('games') >>> s.size # The songs directory has twenty files 20 + >>> g.size # The games directory has three files + 3 + >>> os.remove('games/chess') # Delete a game + >>> g.size # File count is automatically updated + 2 Besides showing how descriptors can run computations, this example also reveals the purpose of the parameters to :meth:`__get__`. The *self* parameter is *size*, an instance of *DirectorySize*. The *obj* parameter is -either *g* or *s*, an instance of *Directory*. It is *obj* parameter that +either *g* or *s*, an instance of *Directory*. It is the *obj* parameter that lets the :meth:`__get__` method learn the target directory. The *objtype* parameter is the class *Directory*. @@ -128,7 +138,9 @@ the public attribute is accessed. In the following example, *age* is the public attribute and *_age* is the private attribute. When the public attribute is accessed, the descriptor logs -the lookup or update:: +the lookup or update: + +.. testcode:: import logging @@ -147,18 +159,26 @@ the lookup or update:: class Person: - age = LoggedAgeAccess() # Descriptor + age = LoggedAgeAccess() # Descriptor instance def __init__(self, name, age): self.name = name # Regular instance attribute - self.age = age # Calls the descriptor + self.age = age # Calls __set__() def birthday(self): self.age += 1 # Calls both __get__() and __set__() An interactive session shows that all access to the managed attribute *age* is -logged, but that the regular attribute *name* is not logged:: +logged, but that the regular attribute *name* is not logged: + +.. testcode:: + :hide: + + import logging, sys + logging.basicConfig(level=logging.INFO, stream=sys.stdout, force=True) + +.. doctest:: >>> mary = Person('Mary M', 30) # The initial age update is logged INFO:root:Updating 'age' to 30 @@ -183,22 +203,24 @@ logged, but that the regular attribute *name* is not logged:: INFO:root:Accessing 'age' giving 40 40 -One major issue with this example is the private name *_age* is hardwired in +One major issue with this example is that the private name *_age* is hardwired in the *LoggedAgeAccess* class. That means that each instance can only have one logged attribute and that its name is unchangeable. In the next example, we'll fix that problem. -Customized Names +Customized names ---------------- -When a class uses descriptors, it can inform each descriptor about what +When a class uses descriptors, it can inform each descriptor about which variable name was used. In this example, the :class:`Person` class has two descriptor instances, *name* and *age*. When the :class:`Person` class is defined, it makes a callback to :meth:`__set_name__` in *LoggedAccess* so that the field names can -be recorded, giving each descriptor its own *public_name* and *private_name*:: +be recorded, giving each descriptor its own *public_name* and *private_name*: + +.. testcode:: import logging @@ -208,7 +230,7 @@ be recorded, giving each descriptor its own *public_name* and *private_name*:: def __set_name__(self, owner, name): self.public_name = name - self.private_name = f'_{name}' + self.private_name = '_' + name def __get__(self, obj, objtype=None): value = getattr(obj, self.private_name) @@ -221,8 +243,8 @@ be recorded, giving each descriptor its own *public_name* and *private_name*:: class Person: - name = LoggedAccess() # First descriptor - age = LoggedAccess() # Second descriptor + name = LoggedAccess() # First descriptor instance + age = LoggedAccess() # Second descriptor instance def __init__(self, name, age): self.name = name # Calls the first descriptor @@ -233,14 +255,24 @@ be recorded, giving each descriptor its own *public_name* and *private_name*:: An interactive session shows that the :class:`Person` class has called :meth:`__set_name__` so that the field names would be recorded. Here -we call :func:`vars` to lookup the descriptor without triggering it:: +we call :func:`vars` to look up the descriptor without triggering it: + +.. doctest:: >>> vars(vars(Person)['name']) {'public_name': 'name', 'private_name': '_name'} >>> vars(vars(Person)['age']) {'public_name': 'age', 'private_name': '_age'} -The new class now logs access to both *name* and *age*:: +The new class now logs access to both *name* and *age*: + +.. testcode:: + :hide: + + import logging, sys + logging.basicConfig(level=logging.INFO, stream=sys.stdout, force=True) + +.. doctest:: >>> pete = Person('Peter P', 10) INFO:root:Updating 'name' to 'Peter P' @@ -249,7 +281,9 @@ The new class now logs access to both *name* and *age*:: INFO:root:Updating 'name' to 'Catherine C' INFO:root:Updating 'age' to 20 -The two *Person* instances contain only the private names:: +The two *Person* instances contain only the private names: + +.. doctest:: >>> vars(pete) {'_name': 'Peter P', '_age': 10} @@ -265,7 +299,8 @@ A :term:`descriptor` is what we call any object that defines :meth:`__get__`, Optionally, descriptors can have a :meth:`__set_name__` method. This is only used in cases where a descriptor needs to know either the class where it was -created or the name of class variable it was assigned to. +created or the name of class variable it was assigned to. (This method, if +present, is called even if the class is not a descriptor.) Descriptors get invoked by the dot operator during attribute lookup. If a descriptor is accessed indirectly with ``vars(some_class)[descriptor_name]``, @@ -275,7 +310,7 @@ Descriptors only work when used as class variables. When put in instances, they have no effect. The main motivation for descriptors is to provide a hook allowing objects -stored in class variables to control what happens during dotted lookup. +stored in class variables to control what happens during attribute lookup. Traditionally, the calling class controls what happens during lookup. Descriptors invert that relationship and allow the data being looked-up to @@ -303,14 +338,16 @@ restrictions. If those restrictions aren't met, it raises an exception to prevent data corruption at its source. This :class:`Validator` class is both an :term:`abstract base class` and a -managed attribute descriptor:: +managed attribute descriptor: + +.. testcode:: from abc import ABC, abstractmethod class Validator(ABC): def __set_name__(self, owner, name): - self.private_name = f'_{name}' + self.private_name = '_' + name def __get__(self, obj, objtype=None): return getattr(obj, self.private_name) @@ -343,7 +380,7 @@ Here are three practical data validation utilities: user-defined `predicate `_ as well. -:: +.. testcode:: class OneOf(Validator): @@ -396,10 +433,12 @@ Here are three practical data validation utilities: ) -Practical use -------------- +Practical application +--------------------- + +Here's how the data validators can be used in a real class: -Here's how the data validators can be used in a real class:: +.. testcode:: class Component: @@ -412,13 +451,30 @@ Here's how the data validators can be used in a real class:: self.kind = kind self.quantity = quantity -The descriptors prevent invalid instances from being created:: +The descriptors prevent invalid instances from being created: + +.. doctest:: - Component('WIDGET', 'metal', 5) # Allowed. - Component('Widget', 'metal', 5) # Blocked: 'Widget' is not all uppercase - Component('WIDGET', 'metle', 5) # Blocked: 'metle' is misspelled - Component('WIDGET', 'metal', -5) # Blocked: -5 is negative - Component('WIDGET', 'metal', 'V') # Blocked: 'V' isn't a number + >>> Component('Widget', 'metal', 5) # Blocked: 'Widget' is not all uppercase + Traceback (most recent call last): + ... + ValueError: Expected to be true for 'Widget' + + >>> Component('WIDGET', 'metle', 5) # Blocked: 'metle' is misspelled + Traceback (most recent call last): + ... + ValueError: Expected 'metle' to be one of {'metal', 'plastic', 'wood'} + + >>> Component('WIDGET', 'metal', -5) # Blocked: -5 is negative + Traceback (most recent call last): + ... + ValueError: Expected -5 to be at least 0 + >>> Component('WIDGET', 'metal', 'V') # Blocked: 'V' isn't a number + Traceback (most recent call last): + ... + TypeError: Expected 'V' to be an int or float + + >>> c = Component('WIDGET', 'metal', 5) # Allowed: The inputs are valid Technical Tutorial @@ -435,23 +491,21 @@ Defines descriptors, summarizes the protocol, and shows how descriptors are called. Provides an example showing how object relational mappings work. Learning about descriptors not only provides access to a larger toolset, it -creates a deeper understanding of how Python works and an appreciation for the -elegance of its design. +creates a deeper understanding of how Python works. -Definition and Introduction +Definition and introduction --------------------------- -In general, a descriptor is an object attribute with "binding behavior", one -whose attribute access has been overridden by methods in the descriptor -protocol. Those methods are :meth:`__get__`, :meth:`__set__`, and -:meth:`__delete__`. If any of those methods are defined for an object, it is -said to be a :term:`descriptor`. +In general, a descriptor is an attribute value that has one of the methods in +the descriptor protocol. Those methods are :meth:`__get__`, :meth:`__set__`, +and :meth:`__delete__`. If any of those methods are defined for an +attribute, it is said to be a :term:`descriptor`. The default behavior for attribute access is to get, set, or delete the attribute from an object's dictionary. For instance, ``a.x`` has a lookup chain starting with ``a.__dict__['x']``, then ``type(a).__dict__['x']``, and -continuing through the base classes of ``type(a)``. If the +continuing through the method resolution order of ``type(a)``. If the looked-up value is an object defining one of the descriptor methods, then Python may override the default behavior and invoke the descriptor method instead. Where this occurs in the precedence chain depends on which descriptor methods @@ -464,7 +518,7 @@ simplify the underlying C code and offer a flexible set of new tools for everyday Python programs. -Descriptor Protocol +Descriptor protocol ------------------- ``descr.__get__(self, obj, type=None) -> value`` @@ -479,7 +533,7 @@ as an attribute. If an object defines :meth:`__set__` or :meth:`__delete__`, it is considered a data descriptor. Descriptors that only define :meth:`__get__` are called -non-data descriptors (they are typically used for methods but other uses are +non-data descriptors (they are often used for methods but other uses are possible). Data and non-data descriptors differ in how overrides are calculated with @@ -494,56 +548,245 @@ called. Defining the :meth:`__set__` method with an exception raising placeholder is enough to make it a data descriptor. -Invoking Descriptors --------------------- +Overview of descriptor invocation +--------------------------------- -A descriptor can be called directly by its method name. For example, -``d.__get__(obj)``. +A descriptor can be called directly with ``desc.__get__(obj)`` or +``desc.__get__(None, cls)``. But it is more common for a descriptor to be invoked automatically from -attribute access. The expression ``obj.d`` looks up ``d`` in the dictionary of -``obj``. If ``d`` defines the method :meth:`__get__`, then ``d.__get__(obj)`` -is invoked according to the precedence rules listed below. +attribute access. + +The expression ``obj.x`` looks up the attribute ``x`` in the chain of +namespaces for ``obj``. If the search finds a descriptor outside of the +instance ``__dict__``, its :meth:`__get__` method is invoked according to the +precedence rules listed below. The details of invocation depend on whether ``obj`` is an object, class, or instance of super. -**Objects**: The machinery is in :meth:`object.__getattribute__`. -It transforms ``b.x`` into ``type(b).__dict__['x'].__get__(b, type(b))``. +Invocation from an instance +--------------------------- + +Instance lookup scans through a chain of namespaces giving data descriptors +the highest priority, followed by instance variables, then non-data +descriptors, then class variables, and lastly :meth:`__getattr__` if it is +provided. + +If a descriptor is found for ``a.x``, then it is invoked with: +``desc.__get__(a, type(a))``. + +The logic for a dotted lookup is in :meth:`object.__getattribute__`. Here is +a pure Python equivalent: + +.. testcode:: + + def object_getattribute(obj, name): + "Emulate PyObject_GenericGetAttr() in Objects/object.c" + null = object() + objtype = type(obj) + cls_var = getattr(objtype, name, null) + descr_get = getattr(type(cls_var), '__get__', null) + if descr_get is not null: + if (hasattr(type(cls_var), '__set__') + or hasattr(type(cls_var), '__delete__')): + return descr_get(cls_var, obj, objtype) # data descriptor + if hasattr(obj, '__dict__') and name in vars(obj): + return vars(obj)[name] # instance variable + if descr_get is not null: + return descr_get(cls_var, obj, objtype) # non-data descriptor + if cls_var is not null: + return cls_var # class variable + raise AttributeError(name) + + +.. testcode:: + :hide: + + # Test the fidelity of object_getattribute() by comparing it with the + # normal object.__getattribute__(). The former will be accessed by + # square brackets and the latter by the dot operator. + + class Object: + + def __getitem__(obj, name): + try: + return object_getattribute(obj, name) + except AttributeError: + if not hasattr(type(obj), '__getattr__'): + raise + return type(obj).__getattr__(obj, name) # __getattr__ + + class DualOperator(Object): + + x = 10 + + def __init__(self, z): + self.z = z + + @property + def p2(self): + return 2 * self.x + + @property + def p3(self): + return 3 * self.x + + def m5(self, y): + return 5 * y + + def m7(self, y): + return 7 * y + + def __getattr__(self, name): + return ('getattr_hook', self, name) + + class DualOperatorWithSlots: + + __getitem__ = Object.__getitem__ + + __slots__ = ['z'] + + x = 15 + + def __init__(self, z): + self.z = z + + @property + def p2(self): + return 2 * self.x + + def m5(self, y): + return 5 * y + + def __getattr__(self, name): + return ('getattr_hook', self, name) + + +.. doctest:: + :hide: + + >>> a = DualOperator(11) + >>> vars(a).update(p3 = '_p3', m7 = '_m7') + >>> a.x == a['x'] == 10 + True + >>> a.z == a['z'] == 11 + True + >>> a.p2 == a['p2'] == 20 + True + >>> a.p3 == a['p3'] == 30 + True + >>> a.m5(100) == a.m5(100) == 500 + True + >>> a.m7 == a['m7'] == '_m7' + True + >>> a.g == a['g'] == ('getattr_hook', a, 'g') + True + + >>> b = DualOperatorWithSlots(22) + >>> b.x == b['x'] == 15 + True + >>> b.z == b['z'] == 22 + True + >>> b.p2 == b['p2'] == 30 + True + >>> b.m5(200) == b['m5'](200) == 1000 + True + >>> b.g == b['g'] == ('getattr_hook', b, 'g') + True + + +Interestingly, attribute lookup doesn't call :meth:`object.__getattribute__` +directly. Instead, both the dot operator and the :func:`getattr` function +perform attribute lookup by way of a helper function: + +.. testcode:: + + def getattr_hook(obj, name): + "Emulate slot_tp_getattr_hook() in Objects/typeobject.c" + try: + return obj.__getattribute__(name) + except AttributeError: + if not hasattr(type(obj), '__getattr__'): + raise + return type(obj).__getattr__(obj, name) # __getattr__ + +.. doctest:: + :hide: + + + >>> class ClassWithGetAttr: + ... x = 123 + ... def __getattr__(self, attr): + ... return attr.upper() + ... + >>> cw = ClassWithGetAttr() + >>> cw.y = 456 + >>> getattr_hook(cw, 'x') + 123 + >>> getattr_hook(cw, 'y') + 456 + >>> getattr_hook(cw, 'z') + 'Z' + + >>> class ClassWithoutGetAttr: + ... x = 123 + ... + >>> cwo = ClassWithoutGetAttr() + >>> cwo.y = 456 + >>> getattr_hook(cwo, 'x') + 123 + >>> getattr_hook(cwo, 'y') + 456 + >>> getattr_hook(cwo, 'z') + Traceback (most recent call last): + ... + AttributeError: 'ClassWithoutGetAttr' object has no attribute 'z' + +So if :meth:`__getattr__` exists, it is called whenever :meth:`__getattribute__` +raises :exc:`AttributeError` (either directly or in one of the descriptor calls). + +Also, if a user calls :meth:`object.__getattribute__` directly, the +:meth:`__getattr__` hook is bypassed entirely. -The implementation works through a precedence chain that gives data descriptors -priority over instance variables, instance variables priority over non-data -descriptors, and assigns lowest priority to :meth:`__getattr__` if provided. -The full C implementation can be found in :c:func:`PyObject_GenericGetAttr()` in -:source:`Objects/object.c`. +Invocation from a class +----------------------- -**Classes**: The machinery is in :meth:`type.__getattribute__`. +The logic for a dotted lookup such as ``A.x`` is in +:meth:`type.__getattribute__`. The steps are similar to those for +:meth:`object.__getattribute__` but the instance dictionary lookup is replaced +by a search through the class's :term:`method resolution order`. -It transforms ``A.x`` into ``A.__dict__['x'].__get__(None, A)``. +If a descriptor is found, it is invoked with ``desc.__get__(None, A)``. -The full C implementation can be found in :c:func:`type_getattro()` in -:source:`Objects/typeobject.c`. +The full C implementation can be found in :c:func:`type_getattro()` and +:c:func:`_PyType_Lookup()` in :source:`Objects/typeobject.c`. + + +Invocation from super +--------------------- -**Super**: The machinery is in the custom :meth:`__getattribute__` method for +The logic for super's dotted lookup is in the :meth:`__getattribute__` method for object returned by :class:`super()`. -The attribute lookup ``super(A, obj).m`` searches ``obj.__class__.__mro__`` for -the base class ``B`` immediately following ``A`` and then returns +A dotted lookup such as ``super(A, obj).m`` searches ``obj.__class__.__mro__`` +for the base class ``B`` immediately following ``A`` and then returns ``B.__dict__['m'].__get__(obj, A)``. If not a descriptor, ``m`` is returned -unchanged. If not in the dictionary, ``m`` reverts to a search using -:meth:`object.__getattribute__`. +unchanged. -The implementation details are in :c:func:`super_getattro()` in +The full C implementation can be found in :c:func:`super_getattro()` in :source:`Objects/typeobject.c`. A pure Python equivalent can be found in -`Guido's Tutorial`_. +`Guido's Tutorial +`_. -.. _`Guido's Tutorial`: https://www.python.org/download/releases/2.2.3/descrintro/#cooperation -**Summary**: The mechanism for descriptors is embedded in the -:meth:`__getattribute__()` methods for :class:`object`, :class:`type`, and -:func:`super`. +Summary of invocation logic +--------------------------- + +The mechanism for descriptors is embedded in the :meth:`__getattribute__()` +methods for :class:`object`, :class:`type`, and :func:`super`. The important points to remember are: @@ -565,15 +808,15 @@ The important points to remember are: * Non-data descriptors may be overridden by instance dictionaries. -Automatic Name Notification +Automatic name notification --------------------------- Sometimes it is desirable for a descriptor to know what class variable name it was assigned to. When a new class is created, the :class:`type` metaclass scans the dictionary of the new class. If any of the entries are descriptors and if they define :meth:`__set_name__`, that method is called with two -arguments. The *owner* is the class where the descriptor is used, the *name* -is class variable the descriptor was assigned to. +arguments. The *owner* is the class where the descriptor is used, and the +*name* is the class variable the descriptor was assigned to. The implementation details are in :c:func:`type_new()` and :c:func:`set_names()` in :source:`Objects/typeobject.c`. @@ -583,7 +826,7 @@ place at the time of class creation. If descriptors are added to the class afterwards, :meth:`__set_name__` will need to be called manually. -ORM Example +ORM example ----------- The following code is simplified skeleton showing how data descriptors could @@ -592,7 +835,9 @@ be used to implement an `object relational mapping The essential idea is that the data is stored in an external database. The Python instances only hold keys to the database's tables. Descriptors take -care of lookups or updates:: +care of lookups or updates: + +.. testcode:: class Field: @@ -607,8 +852,11 @@ care of lookups or updates:: conn.execute(self.store, [value, obj.key]) conn.commit() -We can use the :class:`Field` class to define "models" that describe the schema -for each table in a database:: +We can use the :class:`Field` class to define `models +`_ that describe the schema for +each table in a database: + +.. testcode:: class Movie: table = 'Movies' # Table name @@ -629,12 +877,41 @@ for each table in a database:: def __init__(self, key): self.key = key -An interactive session shows how data is retrieved from the database and how -it can be updated:: +To use the models, first connect to the database:: >>> import sqlite3 >>> conn = sqlite3.connect('entertainment.db') +An interactive session shows how data is retrieved from the database and how +it can be updated: + +.. testsetup:: + + song_data = [ + ('Country Roads', 'John Denver', 1972), + ('Me and Bobby McGee', 'Janice Joplin', 1971), + ('Coal Miners Daughter', 'Loretta Lynn', 1970), + ] + + movie_data = [ + ('Star Wars', 'George Lucas', 1977), + ('Jaws', 'Steven Spielberg', 1975), + ('Aliens', 'James Cameron', 1986), + ] + + import sqlite3 + + conn = sqlite3.connect(':memory:') + conn.execute('CREATE TABLE Music (title text, artist text, year integer);') + conn.execute('CREATE INDEX MusicNdx ON Music (title);') + conn.executemany('INSERT INTO Music VALUES (?, ?, ?);', song_data) + conn.execute('CREATE TABLE Movies (title text, director text, year integer);') + conn.execute('CREATE INDEX MovieNdx ON Music (title);') + conn.executemany('INSERT INTO Movies VALUES (?, ?, ?);', movie_data) + conn.commit() + +.. doctest:: + >>> Movie('Star Wars').director 'George Lucas' >>> jaws = Movie('Jaws') @@ -648,24 +925,27 @@ it can be updated:: >>> Movie('Star Wars').director 'J.J. Abrams' + Pure Python Equivalents ^^^^^^^^^^^^^^^^^^^^^^^ The descriptor protocol is simple and offers exciting possibilities. Several -use cases are so common that they have been prepackaged into builtin tools. -Properties, bound methods, static methods, and class methods are all based on -the descriptor protocol. +use cases are so common that they have been prepackaged into built-in tools. +Properties, bound methods, static methods, class methods, and \_\_slots\_\_ are +all based on the descriptor protocol. Properties ---------- Calling :func:`property` is a succinct way of building a data descriptor that -triggers function calls upon access to an attribute. Its signature is:: +triggers a function call upon access to an attribute. Its signature is:: property(fget=None, fset=None, fdel=None, doc=None) -> property -The documentation shows a typical use to define a managed attribute ``x``:: +The documentation shows a typical use to define a managed attribute ``x``: + +.. testcode:: class C: def getx(self): return self.__x @@ -673,8 +953,24 @@ The documentation shows a typical use to define a managed attribute ``x``:: def delx(self): del self.__x x = property(getx, setx, delx, "I'm the 'x' property.") +.. doctest:: + :hide: + + >>> C.x.__doc__ + "I'm the 'x' property." + >>> c.x = 2.71828 + >>> c.x + 2.71828 + >>> del c.x + >>> c.x + Traceback (most recent call last): + ... + AttributeError: 'C' object has no attribute '_C__x' + To see how :func:`property` is implemented in terms of the descriptor protocol, -here is a pure Python equivalent:: +here is a pure Python equivalent: + +.. testcode:: class Property: "Emulate PyProperty_Type() in Objects/descrobject.c" @@ -686,32 +982,93 @@ here is a pure Python equivalent:: if doc is None and fget is not None: doc = fget.__doc__ self.__doc__ = doc + self._name = '' + + def __set_name__(self, owner, name): + self._name = name def __get__(self, obj, objtype=None): if obj is None: return self if self.fget is None: - raise AttributeError("unreadable attribute") + raise AttributeError(f'unreadable attribute {self._name}') return self.fget(obj) def __set__(self, obj, value): if self.fset is None: - raise AttributeError("can't set attribute") + raise AttributeError(f"can't set attribute {self._name}") self.fset(obj, value) def __delete__(self, obj): if self.fdel is None: - raise AttributeError("can't delete attribute") + raise AttributeError(f"can't delete attribute {self._name}") self.fdel(obj) def getter(self, fget): - return type(self)(fget, self.fset, self.fdel, self.__doc__) + prop = type(self)(fget, self.fset, self.fdel, self.__doc__) + prop._name = self._name + return prop def setter(self, fset): - return type(self)(self.fget, fset, self.fdel, self.__doc__) + prop = type(self)(self.fget, fset, self.fdel, self.__doc__) + prop._name = self._name + return prop def deleter(self, fdel): - return type(self)(self.fget, self.fset, fdel, self.__doc__) + prop = type(self)(self.fget, self.fset, fdel, self.__doc__) + prop._name = self._name + return prop + +.. testcode:: + :hide: + + # Verify the Property() emulation + + class CC: + def getx(self): + return self.__x + def setx(self, value): + self.__x = value + def delx(self): + del self.__x + x = Property(getx, setx, delx, "I'm the 'x' property.") + + # Now do it again but use the decorator style + + class CCC: + @Property + def x(self): + return self.__x + @x.setter + def x(self, value): + self.__x = value + @x.deleter + def x(self): + del self.__x + + +.. doctest:: + :hide: + + >>> cc = CC() + >>> hasattr(cc, 'x') + False + >>> cc.x = 33 + >>> cc.x + 33 + >>> del cc.x + >>> hasattr(cc, 'x') + False + + >>> ccc = CCC() + >>> hasattr(ccc, 'x') + False + >>> ccc.x = 333 + >>> ccc.x == 333 + True + >>> del ccc.x + >>> hasattr(ccc, 'x') + False The :func:`property` builtin helps whenever a user interface has granted attribute access and then subsequent changes require the intervention of a @@ -721,7 +1078,9 @@ For instance, a spreadsheet class may grant access to a cell value through ``Cell('b10').value``. Subsequent improvements to the program require the cell to be recalculated on every access; however, the programmer does not want to affect existing client code accessing the attribute directly. The solution is -to wrap access to the value attribute in a property data descriptor:: +to wrap access to the value attribute in a property data descriptor: + +.. testcode:: class Cell: ... @@ -732,8 +1091,11 @@ to wrap access to the value attribute in a property data descriptor:: self.recalc() return self._value +Either the built-in :func:`property` or our :func:`Property` equivalent would +work in this example. -Functions and Methods + +Functions and methods --------------------- Python's object oriented features are built upon a function based environment. @@ -745,10 +1107,12 @@ prepended to the other arguments. By convention, the instance is called *self* but could be called *this* or any other variable name. Methods can be created manually with :class:`types.MethodType` which is -roughly equivalent to:: +roughly equivalent to: + +.. testcode:: class MethodType: - "Emulate Py_MethodType in Objects/classobject.c" + "Emulate PyMethod_Type in Objects/classobject.c" def __init__(self, func, obj): self.__func__ = func @@ -761,8 +1125,10 @@ roughly equivalent to:: To support automatic creation of methods, functions include the :meth:`__get__` method for binding methods during attribute access. This -means that functions are non-data descriptors which return bound methods -during dotted lookup from an instance. Here's how it works:: +means that functions are non-data descriptors that return bound methods +during dotted lookup from an instance. Here's how it works: + +.. testcode:: class Function: ... @@ -774,13 +1140,17 @@ during dotted lookup from an instance. Here's how it works:: return MethodType(self, obj) Running the following class in the interpreter shows how the function -descriptor works in practice:: +descriptor works in practice: + +.. testcode:: class D: def f(self, x): return x -The function has a :term:`qualified name` attribute to support introspection:: +The function has a :term:`qualified name` attribute to support introspection: + +.. doctest:: >>> D.f.__qualname__ 'D.f' @@ -808,7 +1178,7 @@ Internally, the bound method stores the underlying function and the bound instance:: >>> d.f.__func__ - + >>> d.f.__self__ <__main__.D object at 0x1012e1f98> @@ -817,8 +1187,8 @@ If you have ever wondered where *self* comes from in regular methods or where *cls* comes from in class methods, this is it! -Static Methods --------------- +Kinds of methods +---------------- Non-data descriptors provide a simple mechanism for variations on the usual patterns of binding functions into methods. @@ -841,6 +1211,10 @@ This chart summarizes the binding and its two most useful variants: | classmethod | f(type(obj), \*args) | f(cls, \*args) | +-----------------+----------------------+------------------+ + +Static methods +-------------- + Static methods return the underlying function without changes. Calling either ``c.f`` or ``C.f`` is the equivalent of a direct lookup into ``object.__getattribute__(c, "f")`` or ``object.__getattribute__(C, "f")``. As a @@ -860,20 +1234,26 @@ It can be called either from an object or the class: ``s.erf(1.5) --> .9332`` o ``Sample.erf(1.5) --> .9332``. Since static methods return the underlying function with no changes, the -example calls are unexciting:: +example calls are unexciting: + +.. testcode:: class E: @staticmethod def f(x): - print(x) + return x * 10 + +.. doctest:: >>> E.f(3) - 3 + 30 >>> E().f(3) - 3 + 30 Using the non-data descriptor protocol, a pure Python version of -:func:`staticmethod` would look like this:: +:func:`staticmethod` would look like this: + +.. testcode:: class StaticMethod: "Emulate PyStaticMethod_Type() in Objects/funcobject.c" @@ -884,33 +1264,53 @@ Using the non-data descriptor protocol, a pure Python version of def __get__(self, obj, objtype=None): return self.f +.. testcode:: + :hide: + + class E_sim: + @StaticMethod + def f(x): + return x * 10 + +.. doctest:: + :hide: -Class Methods + >>> E_sim.f(3) + 30 + >>> E_sim().f(3) + 30 + + +Class methods ------------- Unlike static methods, class methods prepend the class reference to the argument list before calling the function. This format is the same -for whether the caller is an object or a class:: +for whether the caller is an object or a class: + +.. testcode:: class F: @classmethod def f(cls, x): return cls.__name__, x - >>> print(F.f(3)) +.. doctest:: + + >>> F.f(3) ('F', 3) - >>> print(F().f(3)) + >>> F().f(3) ('F', 3) This behavior is useful whenever the method only needs to have a class -reference and does rely on data stored in a specific instance. One use for +reference and does not rely on data stored in a specific instance. One use for class methods is to create alternate class constructors. For example, the classmethod :func:`dict.fromkeys` creates a new dictionary from a list of -keys. The pure Python equivalent is:: +keys. The pure Python equivalent is: - class Dict: - ... +.. testcode:: + class Dict(dict): @classmethod def fromkeys(cls, iterable, value=None): "Emulate dict_fromkeys() in Objects/dictobject.c" @@ -919,13 +1319,20 @@ keys. The pure Python equivalent is:: d[key] = value return d -Now a new dictionary of unique keys can be constructed like this:: +Now a new dictionary of unique keys can be constructed like this: + +.. doctest:: - >>> Dict.fromkeys('abracadabra') - {'a': None, 'r': None, 'b': None, 'c': None, 'd': None} + >>> d = Dict.fromkeys('abracadabra') + >>> type(d) is Dict + True + >>> d + {'a': None, 'b': None, 'r': None, 'c': None, 'd': None} Using the non-data descriptor protocol, a pure Python version of -:func:`classmethod` would look like this:: +:func:`classmethod` would look like this: + +.. testcode:: class ClassMethod: "Emulate PyClassMethod_Type() in Objects/funcobject.c" @@ -936,16 +1343,313 @@ Using the non-data descriptor protocol, a pure Python version of def __get__(self, obj, cls=None): if cls is None: cls = type(obj) - if hasattr(obj, '__get__'): + if hasattr(type(self.f), '__get__'): return self.f.__get__(cls) return MethodType(self.f, cls) -The code path for ``hasattr(obj, '__get__')`` was added in Python 3.9 and -makes it possible for :func:`classmethod` to support chained decorators. -For example, a classmethod and property could be chained together:: +.. testcode:: + :hide: + + # Verify the emulation works + class T: + @ClassMethod + def cm(cls, x, y): + return (cls, x, y) + + @ClassMethod + @property + def __doc__(cls): + return f'A doc for {cls.__name__!r}' + + +.. doctest:: + :hide: + + >>> T.cm(11, 22) + (, 11, 22) + + # Also call it from an instance + >>> t = T() + >>> t.cm(11, 22) + (, 11, 22) + + # Check the alternate path for chained descriptors + >>> T.__doc__ + "A doc for 'T'" + + +The code path for ``hasattr(type(self.f), '__get__')`` was added in +Python 3.9 and makes it possible for :func:`classmethod` to support +chained decorators. For example, a classmethod and property could be +chained together: + +.. testcode:: class G: @classmethod @property def __doc__(cls): return f'A doc for {cls.__name__!r}' + +.. doctest:: + + >>> G.__doc__ + "A doc for 'G'" + + +Member objects and __slots__ +---------------------------- + +When a class defines ``__slots__``, it replaces instance dictionaries with a +fixed-length array of slot values. From a user point of view that has +several effects: + +1. Provides immediate detection of bugs due to misspelled attribute +assignments. Only attribute names specified in ``__slots__`` are allowed: + +.. testcode:: + + class Vehicle: + __slots__ = ('id_number', 'make', 'model') + +.. doctest:: + + >>> auto = Vehicle() + >>> auto.id_nubmer = 'VYE483814LQEX' + Traceback (most recent call last): + ... + AttributeError: 'Vehicle' object has no attribute 'id_nubmer' + +2. Helps create immutable objects where descriptors manage access to private +attributes stored in ``__slots__``: + +.. testcode:: + + class Immutable: + + __slots__ = ('_dept', '_name') # Replace the instance dictionary + + def __init__(self, dept, name): + self._dept = dept # Store to private attribute + self._name = name # Store to private attribute + + @property # Read-only descriptor + def dept(self): + return self._dept + + @property + def name(self): # Read-only descriptor + return self._name + +.. doctest:: + + >>> mark = Immutable('Botany', 'Mark Watney') + >>> mark.dept + 'Botany' + >>> mark.dept = 'Space Pirate' + Traceback (most recent call last): + ... + AttributeError: can't set attribute + >>> mark.location = 'Mars' + Traceback (most recent call last): + ... + AttributeError: 'Immutable' object has no attribute 'location' + +3. Saves memory. On a 64-bit Linux build, an instance with two attributes +takes 48 bytes with ``__slots__`` and 152 bytes without. This `flyweight +design pattern `_ likely only +matters when a large number of instances are going to be created. + +4. Improves speed. Reading instance variables is 35% faster with +``__slots__`` (as measured with Python 3.10 on an Apple M1 processor). + +5. Blocks tools like :func:`functools.cached_property` which require an +instance dictionary to function correctly: + +.. testcode:: + + from functools import cached_property + + class CP: + __slots__ = () # Eliminates the instance dict + + @cached_property # Requires an instance dict + def pi(self): + return 4 * sum((-1.0)**n / (2.0*n + 1.0) + for n in reversed(range(100_000))) + +.. doctest:: + + >>> CP().pi + Traceback (most recent call last): + ... + TypeError: No '__dict__' attribute on 'CP' instance to cache 'pi' property. + +It is not possible to create an exact drop-in pure Python version of +``__slots__`` because it requires direct access to C structures and control +over object memory allocation. However, we can build a mostly faithful +simulation where the actual C structure for slots is emulated by a private +``_slotvalues`` list. Reads and writes to that private structure are managed +by member descriptors: + +.. testcode:: + + null = object() + + class Member: + + def __init__(self, name, clsname, offset): + 'Emulate PyMemberDef in Include/structmember.h' + # Also see descr_new() in Objects/descrobject.c + self.name = name + self.clsname = clsname + self.offset = offset + + def __get__(self, obj, objtype=None): + 'Emulate member_get() in Objects/descrobject.c' + # Also see PyMember_GetOne() in Python/structmember.c + value = obj._slotvalues[self.offset] + if value is null: + raise AttributeError(self.name) + return value + + def __set__(self, obj, value): + 'Emulate member_set() in Objects/descrobject.c' + obj._slotvalues[self.offset] = value + + def __delete__(self, obj): + 'Emulate member_delete() in Objects/descrobject.c' + value = obj._slotvalues[self.offset] + if value is null: + raise AttributeError(self.name) + obj._slotvalues[self.offset] = null + + def __repr__(self): + 'Emulate member_repr() in Objects/descrobject.c' + return f'' + +The :meth:`type.__new__` method takes care of adding member objects to class +variables: + +.. testcode:: + + class Type(type): + 'Simulate how the type metaclass adds member objects for slots' + + def __new__(mcls, clsname, bases, mapping): + 'Emuluate type_new() in Objects/typeobject.c' + # type_new() calls PyTypeReady() which calls add_methods() + slot_names = mapping.get('slot_names', []) + for offset, name in enumerate(slot_names): + mapping[name] = Member(name, clsname, offset) + return type.__new__(mcls, clsname, bases, mapping) + +The :meth:`object.__new__` method takes care of creating instances that have +slots instead of an instance dictionary. Here is a rough simulation in pure +Python: + +.. testcode:: + + class Object: + 'Simulate how object.__new__() allocates memory for __slots__' + + def __new__(cls, *args): + 'Emulate object_new() in Objects/typeobject.c' + inst = super().__new__(cls) + if hasattr(cls, 'slot_names'): + empty_slots = [null] * len(cls.slot_names) + object.__setattr__(inst, '_slotvalues', empty_slots) + return inst + + def __setattr__(self, name, value): + 'Emulate _PyObject_GenericSetAttrWithDict() Objects/object.c' + cls = type(self) + if hasattr(cls, 'slot_names') and name not in cls.slot_names: + raise AttributeError( + f'{type(self).__name__!r} object has no attribute {name!r}' + ) + super().__setattr__(name, value) + + def __delattr__(self, name): + 'Emulate _PyObject_GenericSetAttrWithDict() Objects/object.c' + cls = type(self) + if hasattr(cls, 'slot_names') and name not in cls.slot_names: + raise AttributeError( + f'{type(self).__name__!r} object has no attribute {name!r}' + ) + super().__delattr__(name) + +To use the simulation in a real class, just inherit from :class:`Object` and +set the :term:`metaclass` to :class:`Type`: + +.. testcode:: + + class H(Object, metaclass=Type): + 'Instance variables stored in slots' + + slot_names = ['x', 'y'] + + def __init__(self, x, y): + self.x = x + self.y = y + +At this point, the metaclass has loaded member objects for *x* and *y*:: + + >>> from pprint import pp + >>> pp(dict(vars(H))) + {'__module__': '__main__', + '__doc__': 'Instance variables stored in slots', + 'slot_names': ['x', 'y'], + '__init__': , + 'x': , + 'y': } + +.. doctest:: + :hide: + + # We test this separately because the preceding section is not + # doctestable due to the hex memory address for the __init__ function + >>> isinstance(vars(H)['x'], Member) + True + >>> isinstance(vars(H)['y'], Member) + True + +When instances are created, they have a ``slot_values`` list where the +attributes are stored: + +.. doctest:: + + >>> h = H(10, 20) + >>> vars(h) + {'_slotvalues': [10, 20]} + >>> h.x = 55 + >>> vars(h) + {'_slotvalues': [55, 20]} + +Misspelled or unassigned attributes will raise an exception: + +.. doctest:: + + >>> h.xz + Traceback (most recent call last): + ... + AttributeError: 'H' object has no attribute 'xz' + +.. doctest:: + :hide: + + # Examples for deleted attributes are not shown because this section + # is already a bit lengthy. We still test that code here. + >>> del h.x + >>> hasattr(h, 'x') + False + + # Also test the code for uninitialized slots + >>> class HU(Object, metaclass=Type): + ... slot_names = ['x', 'y'] + ... + >>> hu = HU() + >>> hasattr(hu, 'x') + False + >>> hasattr(hu, 'y') + False diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst new file mode 100644 index 00000000000000..9ece93e660504f --- /dev/null +++ b/Doc/howto/enum.rst @@ -0,0 +1,1416 @@ +========== +Enum HOWTO +========== + +:Author: Ethan Furman + +.. _enum-basic-tutorial: + +.. currentmodule:: enum + +Basic Enum Tutorial +------------------- + +An :class:`Enum` is a set of symbolic names bound to unique values. They are +similar to global variables, but they offer a more useful :func:`repr()`, +grouping, type-safety, and a few other features. + +They are most useful when you have a variable that can take one of a limited +selection of values. For example, the days of the week:: + + >>> from enum import Enum + >>> class Weekday(Enum): + ... MONDAY = 1 + ... TUESDAY = 2 + ... WEDNESDAY = 3 + ... THURSDAY = 4 + ... FRIDAY = 5 + ... SATURDAY = 6 + ... SUNDAY = 7 + +As you can see, creating an :class:`Enum` is as simple as writing a class that +inherits from :class:`Enum` itself. + +.. note:: Case of Enum Members + + Because Enums are used to represent constants we recommend using + UPPER_CASE names for members, and will be using that style in our examples. + +Depending on the nature of the enum a member's value may or may not be +important, but either way that value can be used to get the corresponding +member:: + + >>> Weekday(3) + Weekday.WEDNESDAY + +As you can see, the ``repr()`` of a member shows the enum name and the +member name. The ``str()`` on a member shows only its name:: + + >>> print(Weekday.THURSDAY) + THURSDAY + +The *type* of an enumeration member is the enum it belongs to:: + + >>> type(Weekday.MONDAY) + + >>> isinstance(Weekday.FRIDAY, Weekday) + True + +Enum members have an attribute that contains just their :attr:`name`:: + + >>> print(Weekday.TUESDAY.name) + TUESDAY + +Likewise, they have an attribute for their :attr:`value`:: + + + >>> Weekday.WEDNESDAY.value + 3 + +Unlike many languages that treat enumerations solely as name/value pairs, +Python Enums can have behavior added. For example, :class:`datetime.date` +has two methods for returning the weekday: :meth:`weekday` and :meth:`isoweekday`. +The difference is that one of them counts from 0-6 and the other from 1-7. +Rather than keep track of that ourselves we can add a method to the :class:`Weekday` +enum to extract the day from the :class:`date` instance and return the matching +enum member:: + + @classmethod + def from_date(cls, date): + return cls(date.isoweekday()) + +The complete :class:`Weekday` enum now looks like this:: + + >>> class Weekday(Enum): + ... MONDAY = 1 + ... TUESDAY = 2 + ... WEDNESDAY = 3 + ... THURSDAY = 4 + ... FRIDAY = 5 + ... SATURDAY = 6 + ... SUNDAY = 7 + ... # + ... @classmethod + ... def from_date(cls, date): + ... return cls(date.isoweekday()) + +Now we can find out what today is! Observe:: + + >>> from datetime import date + >>> Weekday.from_date(date.today()) + Weekday.TUESDAY + +Of course, if you're reading this on some other day, you'll see that day instead. + +This :class:`Weekday` enum is great if our variable only needs one day, but +what if we need several? Maybe we're writing a function to plot chores during +a week, and don't want to use a :class:`list` -- we could use a different type +of :class:`Enum`:: + + >>> from enum import Flag + >>> class Weekday(Flag): + ... MONDAY = 1 + ... TUESDAY = 2 + ... WEDNESDAY = 4 + ... THURSDAY = 8 + ... FRIDAY = 16 + ... SATURDAY = 32 + ... SUNDAY = 64 + +We've changed two things: we're inherited from :class:`Flag`, and the values are +all powers of 2. + +Just like the original :class:`Weekday` enum above, we can have a single selection:: + + >>> first_week_day = Weekday.MONDAY + >>> first_week_day + Weekday.MONDAY + +But :class:`Flag` also allows us to combine several members into a single +variable:: + + >>> weekend = Weekday.SATURDAY | Weekday.SUNDAY + >>> weekend + Weekday.SATURDAY|Weekday.SUNDAY + +You can even iterate over a :class:`Flag` variable:: + + >>> for day in weekend: + ... print(day) + SATURDAY + SUNDAY + +Okay, let's get some chores set up:: + + >>> chores_for_ethan = { + ... 'feed the cat': Weekday.MONDAY | Weekday.WEDNESDAY | Weekday.FRIDAY, + ... 'do the dishes': Weekday.TUESDAY | Weekday.THURSDAY, + ... 'answer SO questions': Weekday.SATURDAY, + ... } + +And a function to display the chores for a given day:: + + >>> def show_chores(chores, day): + ... for chore, days in chores.items(): + ... if day in days: + ... print(chore) + >>> show_chores(chores_for_ethan, Weekday.SATURDAY) + answer SO questions + +In cases where the actual values of the members do not matter, you can save +yourself some work and use :func:`auto()` for the values:: + + >>> from enum import auto + >>> class Weekday(Flag): + ... MONDAY = auto() + ... TUESDAY = auto() + ... WEDNESDAY = auto() + ... THURSDAY = auto() + ... FRIDAY = auto() + ... SATURDAY = auto() + ... SUNDAY = auto() + + +.. _enum-advanced-tutorial: + +Programmatic access to enumeration members and their attributes +--------------------------------------------------------------- + +Sometimes it's useful to access members in enumerations programmatically (i.e. +situations where ``Color.RED`` won't do because the exact color is not known +at program-writing time). ``Enum`` allows such access:: + + >>> Color(1) + Color.RED + >>> Color(3) + Color.BLUE + +If you want to access enum members by *name*, use item access:: + + >>> Color['RED'] + Color.RED + >>> Color['GREEN'] + Color.GREEN + +If you have an enum member and need its :attr:`name` or :attr:`value`:: + + >>> member = Color.RED + >>> member.name + 'RED' + >>> member.value + 1 + + +Duplicating enum members and values +----------------------------------- + +Having two enum members with the same name is invalid:: + + >>> class Shape(Enum): + ... SQUARE = 2 + ... SQUARE = 3 + ... + Traceback (most recent call last): + ... + TypeError: 'SQUARE' already defined as: 2 + +However, an enum member can have other names associated with it. Given two +entries ``A`` and ``B`` with the same value (and ``A`` defined first), ``B`` +is an alias for the member ``A``. By-value lookup of the value of ``A`` will +return the member ``A``. By-name lookup of ``A`` will return the member ``A``. +By-name lookup of ``B`` will also return the member ``A``:: + + >>> class Shape(Enum): + ... SQUARE = 2 + ... DIAMOND = 1 + ... CIRCLE = 3 + ... ALIAS_FOR_SQUARE = 2 + ... + >>> Shape.SQUARE + Shape.SQUARE + >>> Shape.ALIAS_FOR_SQUARE + Shape.SQUARE + >>> Shape(2) + Shape.SQUARE + +.. note:: + + Attempting to create a member with the same name as an already + defined attribute (another member, a method, etc.) or attempting to create + an attribute with the same name as a member is not allowed. + + +Ensuring unique enumeration values +---------------------------------- + +By default, enumerations allow multiple names as aliases for the same value. +When this behavior isn't desired, you can use the :func:`unique` decorator:: + + >>> from enum import Enum, unique + >>> @unique + ... class Mistake(Enum): + ... ONE = 1 + ... TWO = 2 + ... THREE = 3 + ... FOUR = 3 + ... + Traceback (most recent call last): + ... + ValueError: duplicate values found in : FOUR -> THREE + + +Using automatic values +---------------------- + +If the exact value is unimportant you can use :class:`auto`:: + + >>> from enum import Enum, auto + >>> class Color(Enum): + ... RED = auto() + ... BLUE = auto() + ... GREEN = auto() + ... + >>> [member.value for member in Color] + [1, 2, 3] + +The values are chosen by :func:`_generate_next_value_`, which can be +overridden:: + + >>> class AutoName(Enum): + ... def _generate_next_value_(name, start, count, last_values): + ... return name + ... + >>> class Ordinal(AutoName): + ... NORTH = auto() + ... SOUTH = auto() + ... EAST = auto() + ... WEST = auto() + ... + >>> [member.value for member in Color] + ['NORTH', 'SOUTH', 'EAST', 'WEST'] + +.. note:: + + The :meth:`_generate_next_value_` method must be defined before any members. + +Iteration +--------- + +Iterating over the members of an enum does not provide the aliases:: + + >>> list(Shape) + [Shape.SQUARE, Shape.DIAMOND, Shape.CIRCLE] + +The special attribute ``__members__`` is a read-only ordered mapping of names +to members. It includes all names defined in the enumeration, including the +aliases:: + + >>> for name, member in Shape.__members__.items(): + ... name, member + ... + ('SQUARE', Shape.SQUARE) + ('DIAMOND', Shape.DIAMOND) + ('CIRCLE', Shape.CIRCLE) + ('ALIAS_FOR_SQUARE', Shape.SQUARE) + +The ``__members__`` attribute can be used for detailed programmatic access to +the enumeration members. For example, finding all the aliases:: + + >>> [name for name, member in Shape.__members__.items() if member.name != name] + ['ALIAS_FOR_SQUARE'] + + +Comparisons +----------- + +Enumeration members are compared by identity:: + + >>> Color.RED is Color.RED + True + >>> Color.RED is Color.BLUE + False + >>> Color.RED is not Color.BLUE + True + +Ordered comparisons between enumeration values are *not* supported. Enum +members are not integers (but see `IntEnum`_ below):: + + >>> Color.RED < Color.BLUE + Traceback (most recent call last): + File "", line 1, in + TypeError: '<' not supported between instances of 'Color' and 'Color' + +Equality comparisons are defined though:: + + >>> Color.BLUE == Color.RED + False + >>> Color.BLUE != Color.RED + True + >>> Color.BLUE == Color.BLUE + True + +Comparisons against non-enumeration values will always compare not equal +(again, :class:`IntEnum` was explicitly designed to behave differently, see +below):: + + >>> Color.BLUE == 2 + False + + +Allowed members and attributes of enumerations +---------------------------------------------- + +Most of the examples above use integers for enumeration values. Using integers is +short and handy (and provided by default by the `Functional API`_), but not +strictly enforced. In the vast majority of use-cases, one doesn't care what +the actual value of an enumeration is. But if the value *is* important, +enumerations can have arbitrary values. + +Enumerations are Python classes, and can have methods and special methods as +usual. If we have this enumeration:: + + >>> class Mood(Enum): + ... FUNKY = 1 + ... HAPPY = 3 + ... + ... def describe(self): + ... # self is the member here + ... return self.name, self.value + ... + ... def __str__(self): + ... return 'my custom str! {0}'.format(self.value) + ... + ... @classmethod + ... def favorite_mood(cls): + ... # cls here is the enumeration + ... return cls.HAPPY + ... + +Then:: + + >>> Mood.favorite_mood() + Mood.HAPPY + >>> Mood.HAPPY.describe() + ('HAPPY', 3) + >>> str(Mood.FUNKY) + 'my custom str! 1' + +The rules for what is allowed are as follows: names that start and end with +a single underscore are reserved by enum and cannot be used; all other +attributes defined within an enumeration will become members of this +enumeration, with the exception of special methods (:meth:`__str__`, +:meth:`__add__`, etc.), descriptors (methods are also descriptors), and +variable names listed in :attr:`_ignore_`. + +Note: if your enumeration defines :meth:`__new__` and/or :meth:`__init__` then +any value(s) given to the enum member will be passed into those methods. +See `Planet`_ for an example. + + +Restricted Enum subclassing +--------------------------- + +A new :class:`Enum` class must have one base enum class, up to one concrete +data type, and as many :class:`object`-based mixin classes as needed. The +order of these base classes is:: + + class EnumName([mix-in, ...,] [data-type,] base-enum): + pass + +Also, subclassing an enumeration is allowed only if the enumeration does not define +any members. So this is forbidden:: + + >>> class MoreColor(Color): + ... PINK = 17 + ... + Traceback (most recent call last): + ... + TypeError: MoreColor: cannot extend enumeration 'Color' + +But this is allowed:: + + >>> class Foo(Enum): + ... def some_behavior(self): + ... pass + ... + >>> class Bar(Foo): + ... HAPPY = 1 + ... SAD = 2 + ... + +Allowing subclassing of enums that define members would lead to a violation of +some important invariants of types and instances. On the other hand, it makes +sense to allow sharing some common behavior between a group of enumerations. +(See `OrderedEnum`_ for an example.) + + +Pickling +-------- + +Enumerations can be pickled and unpickled:: + + >>> from test.test_enum import Fruit + >>> from pickle import dumps, loads + >>> Fruit.TOMATO is loads(dumps(Fruit.TOMATO)) + True + +The usual restrictions for pickling apply: picklable enums must be defined in +the top level of a module, since unpickling requires them to be importable +from that module. + +.. note:: + + With pickle protocol version 4 it is possible to easily pickle enums + nested in other classes. + +It is possible to modify how enum members are pickled/unpickled by defining +:meth:`__reduce_ex__` in the enumeration class. + + +Functional API +-------------- + +The :class:`Enum` class is callable, providing the following functional API:: + + >>> Animal = Enum('Animal', 'ANT BEE CAT DOG') + >>> Animal + + >>> Animal.ANT + Animal.ANT + >>> Animal.ANT.value + 1 + >>> list(Animal) + [Animal.ANT, Animal.BEE, Animal.CAT, Animal.DOG] + +The semantics of this API resemble :class:`~collections.namedtuple`. The first +argument of the call to :class:`Enum` is the name of the enumeration. + +The second argument is the *source* of enumeration member names. It can be a +whitespace-separated string of names, a sequence of names, a sequence of +2-tuples with key/value pairs, or a mapping (e.g. dictionary) of names to +values. The last two options enable assigning arbitrary values to +enumerations; the others auto-assign increasing integers starting with 1 (use +the ``start`` parameter to specify a different starting value). A +new class derived from :class:`Enum` is returned. In other words, the above +assignment to :class:`Animal` is equivalent to:: + + >>> class Animal(Enum): + ... ANT = 1 + ... BEE = 2 + ... CAT = 3 + ... DOG = 4 + ... + +The reason for defaulting to ``1`` as the starting number and not ``0`` is +that ``0`` is ``False`` in a boolean sense, but by default enum members all +evaluate to ``True``. + +Pickling enums created with the functional API can be tricky as frame stack +implementation details are used to try and figure out which module the +enumeration is being created in (e.g. it will fail if you use a utility +function in separate module, and also may not work on IronPython or Jython). +The solution is to specify the module name explicitly as follows:: + + >>> Animal = Enum('Animal', 'ANT BEE CAT DOG', module=__name__) + +.. warning:: + + If ``module`` is not supplied, and Enum cannot determine what it is, + the new Enum members will not be unpicklable; to keep errors closer to + the source, pickling will be disabled. + +The new pickle protocol 4 also, in some circumstances, relies on +:attr:`~definition.__qualname__` being set to the location where pickle will be able +to find the class. For example, if the class was made available in class +SomeData in the global scope:: + + >>> Animal = Enum('Animal', 'ANT BEE CAT DOG', qualname='SomeData.Animal') + +The complete signature is:: + + Enum( + value='NewEnumName', + names=<...>, + *, + module='...', + qualname='...', + type=, + start=1, + ) + +:value: What the new enum class will record as its name. + +:names: The enum members. This can be a whitespace or comma separated string + (values will start at 1 unless otherwise specified):: + + 'RED GREEN BLUE' | 'RED,GREEN,BLUE' | 'RED, GREEN, BLUE' + + or an iterator of names:: + + ['RED', 'GREEN', 'BLUE'] + + or an iterator of (name, value) pairs:: + + [('CYAN', 4), ('MAGENTA', 5), ('YELLOW', 6)] + + or a mapping:: + + {'CHARTREUSE': 7, 'SEA_GREEN': 11, 'ROSEMARY': 42} + +:module: name of module where new enum class can be found. + +:qualname: where in module new enum class can be found. + +:type: type to mix in to new enum class. + +:start: number to start counting at if only names are passed in. + +.. versionchanged:: 3.5 + The *start* parameter was added. + + +Derived Enumerations +-------------------- + +IntEnum +^^^^^^^ + +The first variation of :class:`Enum` that is provided is also a subclass of +:class:`int`. Members of an :class:`IntEnum` can be compared to integers; +by extension, integer enumerations of different types can also be compared +to each other:: + + >>> from enum import IntEnum + >>> class Shape(IntEnum): + ... CIRCLE = 1 + ... SQUARE = 2 + ... + >>> class Request(IntEnum): + ... POST = 1 + ... GET = 2 + ... + >>> Shape == 1 + False + >>> Shape.CIRCLE == 1 + True + >>> Shape.CIRCLE == Request.POST + True + +However, they still can't be compared to standard :class:`Enum` enumerations:: + + >>> class Shape(IntEnum): + ... CIRCLE = 1 + ... SQUARE = 2 + ... + >>> class Color(Enum): + ... RED = 1 + ... GREEN = 2 + ... + >>> Shape.CIRCLE == Color.RED + False + +:class:`IntEnum` values behave like integers in other ways you'd expect:: + + >>> int(Shape.CIRCLE) + 1 + >>> ['a', 'b', 'c'][Shape.CIRCLE] + 'b' + >>> [i for i in range(Shape.SQUARE)] + [0, 1] + + +StrEnum +^^^^^^^ + +The second variation of :class:`Enum` that is provided is also a subclass of +:class:`str`. Members of a :class:`StrEnum` can be compared to strings; +by extension, string enumerations of different types can also be compared +to each other. :class:`StrEnum` exists to help avoid the problem of getting +an incorrect member:: + + >>> from enum import StrEnum + >>> class Directions(StrEnum): + ... NORTH = 'north', # notice the trailing comma + ... SOUTH = 'south' + +Before :class:`StrEnum`, ``Directions.NORTH`` would have been the :class:`tuple` +``('north',)``. + +.. versionadded:: 3.10 + + +IntFlag +^^^^^^^ + +The next variation of :class:`Enum` provided, :class:`IntFlag`, is also based +on :class:`int`. The difference being :class:`IntFlag` members can be combined +using the bitwise operators (&, \|, ^, ~) and the result is still an +:class:`IntFlag` member, if possible. However, as the name implies, :class:`IntFlag` +members also subclass :class:`int` and can be used wherever an :class:`int` is +used. + +.. note:: + + Any operation on an :class:`IntFlag` member besides the bit-wise operations will + lose the :class:`IntFlag` membership. + + Bit-wise operations that result in invalid :class:`IntFlag` values will lose the + :class:`IntFlag` membership. See :class:`FlagBoundary` for + details. + +.. versionadded:: 3.6 +.. versionchanged:: 3.10 + +Sample :class:`IntFlag` class:: + + >>> from enum import IntFlag + >>> class Perm(IntFlag): + ... R = 4 + ... W = 2 + ... X = 1 + ... + >>> Perm.R | Perm.W + Perm.R|Perm.W + >>> Perm.R + Perm.W + 6 + >>> RW = Perm.R | Perm.W + >>> Perm.R in RW + True + +It is also possible to name the combinations:: + + >>> class Perm(IntFlag): + ... R = 4 + ... W = 2 + ... X = 1 + ... RWX = 7 + >>> Perm.RWX + Perm.RWX + >>> ~Perm.RWX + Perm(0) + >>> Perm(7) + Perm.RWX + +.. note:: + + Named combinations are considered aliases. Aliases do not show up during + iteration, but can be returned from by-value lookups. + +.. versionchanged:: 3.10 + +Another important difference between :class:`IntFlag` and :class:`Enum` is that +if no flags are set (the value is 0), its boolean evaluation is :data:`False`:: + + >>> Perm.R & Perm.X + Perm(0) + >>> bool(Perm.R & Perm.X) + False + +Because :class:`IntFlag` members are also subclasses of :class:`int` they can +be combined with them (but may lose :class:`IntFlag` membership:: + + >>> Perm.X | 4 + Perm.R|Perm.X + + >>> Perm.X | 8 + 9 + +.. note:: + + The negation operator, ``~``, always returns an :class:`IntFlag` member with a + positive value:: + + >>> (~Perm.X).value == (Perm.R|Perm.W).value == 6 + True + +:class:`IntFlag` members can also be iterated over:: + + >>> list(RW) + [Perm.R, Perm.W] + +.. versionadded:: 3.10 + + +Flag +^^^^ + +The last variation is :class:`Flag`. Like :class:`IntFlag`, :class:`Flag` +members can be combined using the bitwise operators (&, \|, ^, ~). Unlike +:class:`IntFlag`, they cannot be combined with, nor compared against, any +other :class:`Flag` enumeration, nor :class:`int`. While it is possible to +specify the values directly it is recommended to use :class:`auto` as the +value and let :class:`Flag` select an appropriate value. + +.. versionadded:: 3.6 + +Like :class:`IntFlag`, if a combination of :class:`Flag` members results in no +flags being set, the boolean evaluation is :data:`False`:: + + >>> from enum import Flag, auto + >>> class Color(Flag): + ... RED = auto() + ... BLUE = auto() + ... GREEN = auto() + ... + >>> Color.RED & Color.GREEN + Color(0) + >>> bool(Color.RED & Color.GREEN) + False + +Individual flags should have values that are powers of two (1, 2, 4, 8, ...), +while combinations of flags won't:: + + >>> class Color(Flag): + ... RED = auto() + ... BLUE = auto() + ... GREEN = auto() + ... WHITE = RED | BLUE | GREEN + ... + >>> Color.WHITE + Color.WHITE + +Giving a name to the "no flags set" condition does not change its boolean +value:: + + >>> class Color(Flag): + ... BLACK = 0 + ... RED = auto() + ... BLUE = auto() + ... GREEN = auto() + ... + >>> Color.BLACK + Color.BLACK + >>> bool(Color.BLACK) + False + +:class:`Flag` members can also be iterated over:: + + >>> purple = Color.RED | Color.BLUE + >>> list(purple) + [Color.RED, Color.BLUE] + +.. versionadded:: 3.10 + +.. note:: + + For the majority of new code, :class:`Enum` and :class:`Flag` are strongly + recommended, since :class:`IntEnum` and :class:`IntFlag` break some + semantic promises of an enumeration (by being comparable to integers, and + thus by transitivity to other unrelated enumerations). :class:`IntEnum` + and :class:`IntFlag` should be used only in cases where :class:`Enum` and + :class:`Flag` will not do; for example, when integer constants are replaced + with enumerations, or for interoperability with other systems. + + +Others +^^^^^^ + +While :class:`IntEnum` is part of the :mod:`enum` module, it would be very +simple to implement independently:: + + class IntEnum(int, Enum): + pass + +This demonstrates how similar derived enumerations can be defined; for example +a :class:`StrEnum` that mixes in :class:`str` instead of :class:`int`. + +Some rules: + +1. When subclassing :class:`Enum`, mix-in types must appear before + :class:`Enum` itself in the sequence of bases, as in the :class:`IntEnum` + example above. +2. While :class:`Enum` can have members of any type, once you mix in an + additional type, all the members must have values of that type, e.g. + :class:`int` above. This restriction does not apply to mix-ins which only + add methods and don't specify another type. +3. When another data type is mixed in, the :attr:`value` attribute is *not the + same* as the enum member itself, although it is equivalent and will compare + equal. +4. %-style formatting: `%s` and `%r` call the :class:`Enum` class's + :meth:`__str__` and :meth:`__repr__` respectively; other codes (such as + `%i` or `%h` for IntEnum) treat the enum member as its mixed-in type. +5. :ref:`Formatted string literals `, :meth:`str.format`, + and :func:`format` will use the mixed-in type's :meth:`__format__` + unless :meth:`__str__` or :meth:`__format__` is overridden in the subclass, + in which case the overridden methods or :class:`Enum` methods will be used. + Use the !s and !r format codes to force usage of the :class:`Enum` class's + :meth:`__str__` and :meth:`__repr__` methods. + +When to use :meth:`__new__` vs. :meth:`__init__` +------------------------------------------------ + +:meth:`__new__` must be used whenever you want to customize the actual value of +the :class:`Enum` member. Any other modifications may go in either +:meth:`__new__` or :meth:`__init__`, with :meth:`__init__` being preferred. + +For example, if you want to pass several items to the constructor, but only +want one of them to be the value:: + + >>> class Coordinate(bytes, Enum): + ... """ + ... Coordinate with binary codes that can be indexed by the int code. + ... """ + ... def __new__(cls, value, label, unit): + ... obj = bytes.__new__(cls, [value]) + ... obj._value_ = value + ... obj.label = label + ... obj.unit = unit + ... return obj + ... PX = (0, 'P.X', 'km') + ... PY = (1, 'P.Y', 'km') + ... VX = (2, 'V.X', 'km/s') + ... VY = (3, 'V.Y', 'km/s') + ... + + >>> print(Coordinate['PY']) + PY + + >>> print(Coordinate(3)) + VY + + +Finer Points +^^^^^^^^^^^^ + +Supported ``__dunder__`` names +"""""""""""""""""""""""""""""" + +:attr:`__members__` is a read-only ordered mapping of ``member_name``:``member`` +items. It is only available on the class. + +:meth:`__new__`, if specified, must create and return the enum members; it is +also a very good idea to set the member's :attr:`_value_` appropriately. Once +all the members are created it is no longer used. + + +Supported ``_sunder_`` names +"""""""""""""""""""""""""""" + +- ``_name_`` -- name of the member +- ``_value_`` -- value of the member; can be set / modified in ``__new__`` + +- ``_missing_`` -- a lookup function used when a value is not found; may be + overridden +- ``_ignore_`` -- a list of names, either as a :class:`list` or a :class:`str`, + that will not be transformed into members, and will be removed from the final + class +- ``_order_`` -- used in Python 2/3 code to ensure member order is consistent + (class attribute, removed during class creation) +- ``_generate_next_value_`` -- used by the `Functional API`_ and by + :class:`auto` to get an appropriate value for an enum member; may be + overridden + +.. note:: + + For standard :class:`Enum` classes the next value chosen is the last value seen + incremented by one. + + For :class:`Flag` classes the next value chosen will be the next highest + power-of-two, regardless of the last value seen. + +.. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` +.. versionadded:: 3.7 ``_ignore_`` + +To help keep Python 2 / Python 3 code in sync an :attr:`_order_` attribute can +be provided. It will be checked against the actual order of the enumeration +and raise an error if the two do not match:: + + >>> class Color(Enum): + ... _order_ = 'RED GREEN BLUE' + ... RED = 1 + ... BLUE = 3 + ... GREEN = 2 + ... + Traceback (most recent call last): + ... + TypeError: member order does not match _order_: + ['RED', 'BLUE', 'GREEN'] + ['RED', 'GREEN', 'BLUE'] + +.. note:: + + In Python 2 code the :attr:`_order_` attribute is necessary as definition + order is lost before it can be recorded. + + +_Private__names +""""""""""""""" + +Private names are not converted to enum members, but remain normal attributes. + +.. versionchanged:: 3.10 + + +``Enum`` member type +"""""""""""""""""""" + +Enum members are instances of their enum class, and are normally accessed as +``EnumClass.member``. In Python versions ``3.5`` to ``3.9`` you could access +members from other members -- this practice was discouraged, and in ``3.12`` +:class:`Enum` will return to not allowing it, while in ``3.10`` and ``3.11`` +it will raise a :exc:`DeprecationWarning`:: + + >>> class FieldTypes(Enum): + ... name = 0 + ... value = 1 + ... size = 2 + ... + >>> FieldTypes.value.size # doctest: +SKIP + DeprecationWarning: accessing one member from another is not supported, + and will be disabled in 3.12 + + +.. versionchanged:: 3.5 +.. versionchanged:: 3.10 + + +Creating members that are mixed with other data types +""""""""""""""""""""""""""""""""""""""""""""""""""""" + +When subclassing other data types, such as :class:`int` or :class:`str`, with +an :class:`Enum`, all values after the `=` are passed to that data type's +constructor. For example:: + + >>> class MyEnum(IntEnum): + ... example = '11', 16 # '11' will be interpreted as a hexadecimal + ... # number + >>> MyEnum.example.value + 17 + + +Boolean value of ``Enum`` classes and members +""""""""""""""""""""""""""""""""""""""""""""" + +Enum classes that are mixed with non-:class:`Enum` types (such as +:class:`int`, :class:`str`, etc.) are evaluated according to the mixed-in +type's rules; otherwise, all members evaluate as :data:`True`. To make your +own enum's boolean evaluation depend on the member's value add the following to +your class:: + + def __bool__(self): + return bool(self.value) + +Plain :class:`Enum` classes always evaluate as :data:`True`. + + +``Enum`` classes with methods +""""""""""""""""""""""""""""" + +If you give your enum subclass extra methods, like the `Planet`_ +class above, those methods will show up in a :func:`dir` of the member, +but not of the class:: + + >>> dir(Planet) + ['EARTH', 'JUPITER', 'MARS', 'MERCURY', 'NEPTUNE', 'SATURN', 'URANUS', 'VENUS', '__class__', '__doc__', '__members__', '__module__'] + >>> dir(Planet.EARTH) + ['__class__', '__doc__', '__module__', 'mass', 'name', 'radius', 'surface_gravity', 'value'] + + +Combining members of ``Flag`` +""""""""""""""""""""""""""""" + +Iterating over a combination of :class:`Flag` members will only return the members that +are comprised of a single bit:: + + >>> class Color(Flag): + ... RED = auto() + ... GREEN = auto() + ... BLUE = auto() + ... MAGENTA = RED | BLUE + ... YELLOW = RED | GREEN + ... CYAN = GREEN | BLUE + ... + >>> Color(3) # named combination + Color.YELLOW + >>> Color(7) # not named combination + Color.RED|Color.GREEN|Color.BLUE + +``StrEnum`` and :meth:`str.__str__` +""""""""""""""""""""""""""""""""""" + +An important difference between :class:`StrEnum` and other Enums is the +:meth:`__str__` method; because :class:`StrEnum` members are strings, some +parts of Python will read the string data directly, while others will call +:meth:`str()`. To make those two operations have the same result, +:meth:`StrEnum.__str__` will be the same as :meth:`str.__str__` so that +``str(StrEnum.member) == StrEnum.member`` is true. + +``Flag`` and ``IntFlag`` minutia +"""""""""""""""""""""""""""""""" + +Using the following snippet for our examples:: + + >>> class Color(IntFlag): + ... BLACK = 0 + ... RED = 1 + ... GREEN = 2 + ... BLUE = 4 + ... PURPLE = RED | BLUE + ... WHITE = RED | GREEN | BLUE + ... + +the following are true: + +- single-bit flags are canonical +- multi-bit and zero-bit flags are aliases +- only canonical flags are returned during iteration:: + + >>> list(Color.WHITE) + [Color.RED, Color.GREEN, Color.BLUE] + +- negating a flag or flag set returns a new flag/flag set with the + corresponding positive integer value:: + + >>> Color.BLUE + Color.BLUE + + >>> ~Color.BLUE + Color.RED|Color.GREEN + +- names of pseudo-flags are constructed from their members' names:: + + >>> (Color.RED | Color.GREEN).name + 'RED|GREEN' + +- multi-bit flags, aka aliases, can be returned from operations:: + + >>> Color.RED | Color.BLUE + Color.PURPLE + + >>> Color(7) # or Color(-1) + Color.WHITE + + >>> Color(0) + Color.BLACK + +- membership / containment checking has changed slightly -- zero valued flags + are never considered to be contained:: + + >>> Color.BLACK in Color.WHITE + False + + otherwise, if all bits of one flag are in the other flag, True is returned:: + + >>> Color.PURPLE in Color.WHITE + True + +There is a new boundary mechanism that controls how out-of-range / invalid +bits are handled: ``STRICT``, ``CONFORM``, ``EJECT``, and ``KEEP``: + + * STRICT --> raises an exception when presented with invalid values + * CONFORM --> discards any invalid bits + * EJECT --> lose Flag status and become a normal int with the given value + * KEEP --> keep the extra bits + - keeps Flag status and extra bits + - extra bits do not show up in iteration + - extra bits do show up in repr() and str() + +The default for Flag is ``STRICT``, the default for ``IntFlag`` is ``EJECT``, +and the default for ``_convert_`` is ``KEEP`` (see ``ssl.Options`` for an +example of when ``KEEP`` is needed). + + +.. _enum-class-differences: + +How are Enums different? +------------------------ + +Enums have a custom metaclass that affects many aspects of both derived :class:`Enum` +classes and their instances (members). + + +Enum Classes +^^^^^^^^^^^^ + +The :class:`EnumType` metaclass is responsible for providing the +:meth:`__contains__`, :meth:`__dir__`, :meth:`__iter__` and other methods that +allow one to do things with an :class:`Enum` class that fail on a typical +class, such as `list(Color)` or `some_enum_var in Color`. :class:`EnumType` is +responsible for ensuring that various other methods on the final :class:`Enum` +class are correct (such as :meth:`__new__`, :meth:`__getnewargs__`, +:meth:`__str__` and :meth:`__repr__`). + + +Enum Members (aka instances) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The most interesting thing about enum members is that they are singletons. +:class:`EnumType` creates them all while it is creating the enum class itself, +and then puts a custom :meth:`__new__` in place to ensure that no new ones are +ever instantiated by returning only the existing member instances. + + +.. _enum-cookbook: + + +While :class:`Enum`, :class:`IntEnum`, :class:`StrEnum`, :class:`Flag`, and +:class:`IntFlag` are expected to cover the majority of use-cases, they cannot +cover them all. Here are recipes for some different types of enumerations +that can be used directly, or as examples for creating one's own. + + +Omitting values +^^^^^^^^^^^^^^^ + +In many use-cases one doesn't care what the actual value of an enumeration +is. There are several ways to define this type of simple enumeration: + +- use instances of :class:`auto` for the value +- use instances of :class:`object` as the value +- use a descriptive string as the value +- use a tuple as the value and a custom :meth:`__new__` to replace the + tuple with an :class:`int` value + +Using any of these methods signifies to the user that these values are not +important, and also enables one to add, remove, or reorder members without +having to renumber the remaining members. + + +Using :class:`auto` +""""""""""""""""""" + +Using :class:`auto` would look like:: + + >>> class Color(Enum): + ... RED = auto() + ... BLUE = auto() + ... GREEN = auto() + ... + >>> Color.GREEN + + + +Using :class:`object` +""""""""""""""""""""" + +Using :class:`object` would look like:: + + >>> class Color(Enum): + ... RED = object() + ... GREEN = object() + ... BLUE = object() + ... + >>> Color.GREEN + + + +Using a descriptive string +"""""""""""""""""""""""""" + +Using a string as the value would look like:: + + >>> class Color(Enum): + ... RED = 'stop' + ... GREEN = 'go' + ... BLUE = 'too fast!' + ... + >>> Color.GREEN + + >>> Color.GREEN.value + 'go' + + +Using a custom :meth:`__new__` +"""""""""""""""""""""""""""""" + +Using an auto-numbering :meth:`__new__` would look like:: + + >>> class AutoNumber(Enum): + ... def __new__(cls): + ... value = len(cls.__members__) + 1 + ... obj = object.__new__(cls) + ... obj._value_ = value + ... return obj + ... + >>> class Color(AutoNumber): + ... RED = () + ... GREEN = () + ... BLUE = () + ... + >>> Color.GREEN + + >>> Color.GREEN.value + 2 + +To make a more general purpose ``AutoNumber``, add ``*args`` to the signature:: + + >>> class AutoNumber(Enum): + ... def __new__(cls, *args): # this is the only change from above + ... value = len(cls.__members__) + 1 + ... obj = object.__new__(cls) + ... obj._value_ = value + ... return obj + ... + +Then when you inherit from ``AutoNumber`` you can write your own ``__init__`` +to handle any extra arguments:: + + >>> class Swatch(AutoNumber): + ... def __init__(self, pantone='unknown'): + ... self.pantone = pantone + ... AUBURN = '3497' + ... SEA_GREEN = '1246' + ... BLEACHED_CORAL = () # New color, no Pantone code yet! + ... + >>> Swatch.SEA_GREEN + + >>> Swatch.SEA_GREEN.pantone + '1246' + >>> Swatch.BLEACHED_CORAL.pantone + 'unknown' + +.. note:: + + The :meth:`__new__` method, if defined, is used during creation of the Enum + members; it is then replaced by Enum's :meth:`__new__` which is used after + class creation for lookup of existing members. + + +OrderedEnum +^^^^^^^^^^^ + +An ordered enumeration that is not based on :class:`IntEnum` and so maintains +the normal :class:`Enum` invariants (such as not being comparable to other +enumerations):: + + >>> class OrderedEnum(Enum): + ... def __ge__(self, other): + ... if self.__class__ is other.__class__: + ... return self.value >= other.value + ... return NotImplemented + ... def __gt__(self, other): + ... if self.__class__ is other.__class__: + ... return self.value > other.value + ... return NotImplemented + ... def __le__(self, other): + ... if self.__class__ is other.__class__: + ... return self.value <= other.value + ... return NotImplemented + ... def __lt__(self, other): + ... if self.__class__ is other.__class__: + ... return self.value < other.value + ... return NotImplemented + ... + >>> class Grade(OrderedEnum): + ... A = 5 + ... B = 4 + ... C = 3 + ... D = 2 + ... F = 1 + ... + >>> Grade.C < Grade.A + True + + +DuplicateFreeEnum +^^^^^^^^^^^^^^^^^ + +Raises an error if a duplicate member name is found instead of creating an +alias:: + + >>> class DuplicateFreeEnum(Enum): + ... def __init__(self, *args): + ... cls = self.__class__ + ... if any(self.value == e.value for e in cls): + ... a = self.name + ... e = cls(self.value).name + ... raise ValueError( + ... "aliases not allowed in DuplicateFreeEnum: %r --> %r" + ... % (a, e)) + ... + >>> class Color(DuplicateFreeEnum): + ... RED = 1 + ... GREEN = 2 + ... BLUE = 3 + ... GRENE = 2 + ... + Traceback (most recent call last): + ... + ValueError: aliases not allowed in DuplicateFreeEnum: 'GRENE' --> 'GREEN' + +.. note:: + + This is a useful example for subclassing Enum to add or change other + behaviors as well as disallowing aliases. If the only desired change is + disallowing aliases, the :func:`unique` decorator can be used instead. + + +Planet +^^^^^^ + +If :meth:`__new__` or :meth:`__init__` is defined the value of the enum member +will be passed to those methods:: + + >>> class Planet(Enum): + ... MERCURY = (3.303e+23, 2.4397e6) + ... VENUS = (4.869e+24, 6.0518e6) + ... EARTH = (5.976e+24, 6.37814e6) + ... MARS = (6.421e+23, 3.3972e6) + ... JUPITER = (1.9e+27, 7.1492e7) + ... SATURN = (5.688e+26, 6.0268e7) + ... URANUS = (8.686e+25, 2.5559e7) + ... NEPTUNE = (1.024e+26, 2.4746e7) + ... def __init__(self, mass, radius): + ... self.mass = mass # in kilograms + ... self.radius = radius # in meters + ... @property + ... def surface_gravity(self): + ... # universal gravitational constant (m3 kg-1 s-2) + ... G = 6.67300E-11 + ... return G * self.mass / (self.radius * self.radius) + ... + >>> Planet.EARTH.value + (5.976e+24, 6378140.0) + >>> Planet.EARTH.surface_gravity + 9.802652743337129 + +.. _enum-time-period: + +TimePeriod +^^^^^^^^^^ + +An example to show the :attr:`_ignore_` attribute in use:: + + >>> from datetime import timedelta + >>> class Period(timedelta, Enum): + ... "different lengths of time" + ... _ignore_ = 'Period i' + ... Period = vars() + ... for i in range(367): + ... Period['day_%d' % i] = i + ... + >>> list(Period)[:2] + [Period.day_0, Period.day_1] + >>> list(Period)[-2:] + [Period.day_365, Period.day_366] + + +Conforming input to Flag +^^^^^^^^^^^^^^^^^^^^^^^^ + +Creating a :class:`Flag` enum that is more resilient out-of-bounds results to +mathematical operations, you can use the :attr:`FlagBoundary.CONFORM` setting:: + + >>> from enum import Flag, CONFORM, auto + >>> class Weekday(Flag, boundary=CONFORM): + ... MONDAY = auto() + ... TUESDAY = auto() + ... WEDNESDAY = auto() + ... THURSDAY = auto() + ... FRIDAY = auto() + ... SATURDAY = auto() + ... SUNDAY = auto() + >>> today = Weekday.TUESDAY + >>> Weekday(today + 22) # what day is three weeks from tomorrow? + >>> Weekday.WEDNESDAY + + +.. _enumtype-examples: + +Subclassing EnumType +-------------------- + +While most enum needs can be met by customizing :class:`Enum` subclasses, +either with class decorators or custom functions, :class:`EnumType` can be +subclassed to provide a different Enum experience. + diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index 593341cc2b8a1e..eae8f143ee206f 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -17,6 +17,7 @@ Currently, the HOWTOs are: cporting.rst curses.rst descriptor.rst + enum.rst functional.rst logging.rst logging-cookbook.rst @@ -29,4 +30,5 @@ Currently, the HOWTOs are: ipaddress.rst clinic.rst instrumentation.rst + annotations.rst diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst index f0081e4ec28905..4a59ae82f96e2d 100644 --- a/Doc/howto/instrumentation.rst +++ b/Doc/howto/instrumentation.rst @@ -46,7 +46,8 @@ or:: $ sudo apt-get install systemtap-sdt-dev -CPython must then be configured ``--with-dtrace``: +CPython must then be :option:`configured with the --with-dtrace option +<--with-dtrace>`: .. code-block:: none @@ -77,7 +78,8 @@ the built binary by seeing if it contains a ".note.stapsdt" section. $ readelf -S ./python | grep .note.stapsdt [30] .note.stapsdt NOTE 0000000000000000 00308d78 -If you've built Python as a shared library (with --enable-shared), you +If you've built Python as a shared library +(with the :option:`--enable-shared` configure option), you need to look instead within the shared library. For example:: $ readelf -S libpython3.3dm.so.1.0 | grep .note.stapsdt @@ -252,7 +254,7 @@ where the columns are: and the remainder indicates the call/return hierarchy as the script executes. -For a `--enable-shared` build of CPython, the markers are contained within the +For a :option:`--enable-shared` build of CPython, the markers are contained within the libpython shared library, and the probe's dotted path needs to reflect this. For example, this line from the above example: @@ -266,7 +268,7 @@ should instead read: probe process("python").library("libpython3.6dm.so.1.0").mark("function__entry") { -(assuming a debug build of CPython 3.6) +(assuming a :ref:`debug build ` of CPython 3.6) Available static markers diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index de0f834551f5dd..5777a4c5031f85 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -1188,7 +1188,7 @@ to the above, as in the following example:: class StyleAdapter(logging.LoggerAdapter): def __init__(self, logger, extra=None): - super(StyleAdapter, self).__init__(logger, extra or {}) + super().__init__(logger, extra or {}) def log(self, level, msg, /, *args, **kwargs): if self.isEnabledFor(level): @@ -1783,7 +1783,7 @@ as in the following complete example:: return tuple(o) elif isinstance(o, unicode): return o.encode('unicode_escape').decode('ascii') - return super(Encoder, self).default(o) + return super().default(o) class StructuredMessage: def __init__(self, message, /, **kwargs): @@ -2175,11 +2175,11 @@ class, as shown in the following example:: """ Format an exception so that it prints on a single line. """ - result = super(OneLineExceptionFormatter, self).formatException(exc_info) + result = super().formatException(exc_info) return repr(result) # or format into one line however you want to def format(self, record): - s = super(OneLineExceptionFormatter, self).format(record) + s = super().format(record) if record.exc_text: s = s.replace('\n', '') + '|' return s @@ -2813,7 +2813,7 @@ refer to the comments in the code snippet for more detailed information. # class QtHandler(logging.Handler): def __init__(self, slotfunc, *args, **kwargs): - super(QtHandler, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.signaller = Signaller() self.signaller.signal.connect(slotfunc) @@ -2883,7 +2883,7 @@ refer to the comments in the code snippet for more detailed information. } def __init__(self, app): - super(Window, self).__init__() + super().__init__() self.app = app self.textedit = te = QtWidgets.QPlainTextEdit(self) # Set whatever the default monospace font is for the platform diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index b4dd9206c9d524..fcc6bec7688002 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -684,7 +684,6 @@ Here is the logging.conf file: [formatter_simpleFormatter] format=%(asctime)s - %(name)s - %(levelname)s - %(message)s - datefmt= The output is nearly identical to that of the non-config-file-based example: diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst index a8efe65353d6e1..37328c82dff270 100644 --- a/Doc/howto/sorting.rst +++ b/Doc/howto/sorting.rst @@ -18,7 +18,9 @@ Sorting Basics ============== A simple ascending sort is very easy: just call the :func:`sorted` function. It -returns a new sorted list:: +returns a new sorted list: + +.. doctest:: >>> sorted([5, 2, 3, 1, 4]) [1, 2, 3, 4, 5] @@ -28,6 +30,8 @@ in-place (and returns ``None`` to avoid confusion). Usually it's less convenient than :func:`sorted` - but if you don't need the original list, it's slightly more efficient. +.. doctest:: + >>> a = [5, 2, 3, 1, 4] >>> a.sort() >>> a @@ -36,6 +40,8 @@ more efficient. Another difference is that the :meth:`list.sort` method is only defined for lists. In contrast, the :func:`sorted` function accepts any iterable. +.. doctest:: + >>> sorted({1: 'D', 2: 'B', 3: 'B', 4: 'E', 5: 'A'}) [1, 2, 3, 4, 5] @@ -48,6 +54,8 @@ comparisons. For example, here's a case-insensitive string comparison: +.. doctest:: + >>> sorted("This is a test string from Andrew".split(), key=str.lower) ['a', 'Andrew', 'from', 'is', 'string', 'test', 'This'] @@ -59,6 +67,8 @@ input record. A common pattern is to sort complex objects using some of the object's indices as keys. For example: +.. doctest:: + >>> student_tuples = [ ... ('john', 'A', 15), ... ('jane', 'B', 12), @@ -69,6 +79,8 @@ as keys. For example: The same technique works for objects with named attributes. For example: +.. doctest:: + >>> class Student: ... def __init__(self, name, grade, age): ... self.name = name @@ -95,6 +107,8 @@ convenience functions to make accessor functions easier and faster. The Using those functions, the above examples become simpler and faster: +.. doctest:: + >>> from operator import itemgetter, attrgetter >>> sorted(student_tuples, key=itemgetter(2)) @@ -106,6 +120,8 @@ Using those functions, the above examples become simpler and faster: The operator module functions allow multiple levels of sorting. For example, to sort by *grade* then by *age*: +.. doctest:: + >>> sorted(student_tuples, key=itemgetter(1,2)) [('john', 'A', 15), ('dave', 'B', 10), ('jane', 'B', 12)] @@ -119,6 +135,8 @@ Both :meth:`list.sort` and :func:`sorted` accept a *reverse* parameter with a boolean value. This is used to flag descending sorts. For example, to get the student data in reverse *age* order: +.. doctest:: + >>> sorted(student_tuples, key=itemgetter(2), reverse=True) [('john', 'A', 15), ('jane', 'B', 12), ('dave', 'B', 10)] @@ -132,6 +150,8 @@ Sorts are guaranteed to be `stable `_\. That means that when multiple records have the same key, their original order is preserved. +.. doctest:: + >>> data = [('red', 1), ('blue', 1), ('red', 2), ('blue', 2)] >>> sorted(data, key=itemgetter(0)) [('blue', 1), ('blue', 2), ('red', 1), ('red', 2)] @@ -143,6 +163,8 @@ This wonderful property lets you build complex sorts in a series of sorting steps. For example, to sort the student data by descending *grade* and then ascending *age*, do the *age* sort first and then sort again using *grade*: +.. doctest:: + >>> s = sorted(student_objects, key=attrgetter('age')) # sort on secondary key >>> sorted(s, key=attrgetter('grade'), reverse=True) # now sort on primary key, descending [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] @@ -150,6 +172,8 @@ ascending *age*, do the *age* sort first and then sort again using *grade*: This can be abstracted out into a wrapper function that can take a list and tuples of field and order to sort them on multiple passes. +.. doctest:: + >>> def multisort(xs, specs): ... for key, reverse in reversed(specs): ... xs.sort(key=attrgetter(key), reverse=reverse) @@ -220,6 +244,8 @@ comparisons. That function should take two arguments to be compared and then return a negative value for less-than, return zero if they are equal, or return a positive value for greater-than. For example, we can do: +.. doctest:: + >>> def numeric_compare(x, y): ... return x - y >>> sorted([5, 2, 4, 1, 3], cmp=numeric_compare) # doctest: +SKIP @@ -227,6 +253,8 @@ a positive value for greater-than. For example, we can do: Or you can reverse the order of comparison with: +.. doctest:: + >>> def reverse_numeric(x, y): ... return y - x >>> sorted([5, 2, 4, 1, 3], cmp=reverse_numeric) # doctest: +SKIP @@ -234,7 +262,9 @@ Or you can reverse the order of comparison with: When porting code from Python 2.x to 3.x, the situation can arise when you have the user supplying a comparison function and you need to convert that to a key -function. The following wrapper makes that easy to do:: +function. The following wrapper makes that easy to do: + +.. testcode:: def cmp_to_key(mycmp): 'Convert a cmp= function into a key= function' @@ -255,6 +285,12 @@ function. The following wrapper makes that easy to do:: return mycmp(self.obj, other.obj) != 0 return K +.. doctest:: + :hide: + + >>> sorted([5, 2, 4, 1, 3], key=cmp_to_key(reverse_numeric)) + [5, 4, 3, 2, 1] + To convert to a key function, just wrap the old comparison function: .. testsetup:: @@ -280,6 +316,8 @@ Odd and Ends simulated without the parameter by using the builtin :func:`reversed` function twice: + .. doctest:: + >>> data = [('red', 1), ('blue', 1), ('red', 2), ('blue', 2)] >>> standard_way = sorted(data, key=itemgetter(0), reverse=True) >>> double_reversed = list(reversed(sorted(reversed(data), key=itemgetter(0)))) @@ -289,7 +327,9 @@ Odd and Ends * The sort routines are guaranteed to use :meth:`__lt__` when making comparisons between two objects. So, it is easy to add a standard sort order to a class by - defining an :meth:`__lt__` method:: + defining an :meth:`__lt__` method: + + .. doctest:: >>> Student.__lt__ = lambda self, other: self.age < other.age >>> sorted(student_objects) @@ -300,6 +340,8 @@ Odd and Ends are stored in a dictionary, they can be used to sort a separate list of student names: + .. doctest:: + >>> students = ['dave', 'john', 'jane'] >>> newgrades = {'john': 'F', 'jane':'A', 'dave': 'C'} >>> sorted(students, key=newgrades.__getitem__) diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index e948c1e3c662d6..535b21bd4a54f5 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -609,9 +609,9 @@ implemented by converting the Unicode string into some encoding that varies depending on the system. Today Python is converging on using UTF-8: Python on MacOS has used UTF-8 for several versions, and Python 3.6 switched to using UTF-8 on Windows as well. On Unix systems, -there will only be a filesystem encoding if you've set the ``LANG`` or -``LC_CTYPE`` environment variables; if you haven't, the default -encoding is again UTF-8. +there will only be a :term:`filesystem encoding `. if you've set the ``LANG`` or ``LC_CTYPE`` environment variables; if +you haven't, the default encoding is again UTF-8. The :func:`sys.getfilesystemencoding` function returns the encoding to use on your current system, in case you want to do the encoding manually, but there's @@ -633,8 +633,8 @@ provided the directory path as bytes or a Unicode string. If you pass a Unicode string as the path, filenames will be decoded using the filesystem's encoding and a list of Unicode strings will be returned, while passing a byte path will return the filenames as bytes. For example, -assuming the default filesystem encoding is UTF-8, running the following -program:: +assuming the default :term:`filesystem encoding ` is UTF-8, running the following program:: fn = 'filename\u4500abc' f = open(fn, 'w') diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 046a88af62f0b3..12d525771ddc28 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -97,7 +97,7 @@ schemes. For example, you can make an FTP request like so:: In the case of HTTP, there are two extra things that Request objects allow you to do: First, you can pass data to be sent to the server. Second, you can pass -extra information ("metadata") *about* the data or the about request itself, to +extra information ("metadata") *about* the data or about the request itself, to the server - this information is sent as HTTP "headers". Let's look at each of these in turn. diff --git a/Doc/includes/sqlite3/execute_1.py b/Doc/includes/sqlite3/execute_1.py index 3466b1265a5bf2..42aad4d5839f06 100644 --- a/Doc/includes/sqlite3/execute_1.py +++ b/Doc/includes/sqlite3/execute_1.py @@ -2,17 +2,22 @@ con = sqlite3.connect(":memory:") cur = con.cursor() -cur.execute("create table people (name_last, age)") - -who = "Yeltsin" -age = 72 +cur.execute("create table lang (lang_name, lang_age)") # This is the qmark style: -cur.execute("insert into people values (?, ?)", (who, age)) +cur.execute("insert into lang values (?, ?)", ("C", 49)) -# And this is the named style: -cur.execute("select * from people where name_last=:who and age=:age", {"who": who, "age": age}) +# The qmark style used with executemany(): +lang_list = [ + ("Fortran", 64), + ("Python", 30), + ("Go", 11), +] +cur.executemany("insert into lang values (?, ?)", lang_list) -print(cur.fetchone()) +# And this is the named style: +cur.execute("select * from lang where lang_name=:name and lang_age=:age", + {"name": "C", "age": 49}) +print(cur.fetchall()) con.close() diff --git a/Doc/includes/typestruct.h b/Doc/includes/typestruct.h index 9ada03cfc4a4cb..02f8ccfe4438a5 100644 --- a/Doc/includes/typestruct.h +++ b/Doc/includes/typestruct.h @@ -35,12 +35,14 @@ typedef struct _typeobject { const char *tp_doc; /* Documentation string */ + /* Assigned meaning in release 2.0 */ /* call function for all accessible objects */ traverseproc tp_traverse; /* delete references to contained objects */ inquiry tp_clear; + /* Assigned meaning in release 2.1 */ /* rich comparisons */ richcmpfunc tp_richcompare; @@ -55,6 +57,7 @@ typedef struct _typeobject { struct PyMethodDef *tp_methods; struct PyMemberDef *tp_members; struct PyGetSetDef *tp_getset; + // Strong reference on a heap type, borrowed reference on a static type struct _typeobject *tp_base; PyObject *tp_dict; descrgetfunc tp_descr_get; @@ -76,5 +79,5 @@ typedef struct _typeobject { unsigned int tp_version_tag; destructor tp_finalize; - + vectorcallfunc tp_vectorcall; } PyTypeObject; diff --git a/Doc/install/index.rst b/Doc/install/index.rst index e6d5a3e6ebde60..48c6e76a682a46 100644 --- a/Doc/install/index.rst +++ b/Doc/install/index.rst @@ -10,6 +10,13 @@ .. TODO: Fill in XXX comments +.. note:: + + The entire ``distutils`` package has been deprecated and will be + removed in Python 3.12. This documentation is retained as a + reference only, and will be removed with the package. See the + :ref:`What's New ` entry for more information. + .. seealso:: :ref:`installing-index` @@ -1064,8 +1071,7 @@ normal libraries do. .. [#] This also means you could replace all existing COFF-libraries with OMF-libraries of the same name. -.. [#] Check https://www.sourceware.org/cygwin/ and http://www.mingw.org/ for more - information +.. [#] Check https://www.sourceware.org/cygwin/ for more information .. [#] Then you have no POSIX emulation available, but you also don't need :file:`cygwin1.dll`. diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index 41399942d30308..24bbd90d02cf76 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -90,7 +90,7 @@ language using this mechanism: | generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | | | | | *StopIteration handling inside generators* | +------------------+-------------+--------------+---------------------------------------------+ -| annotations | 3.7.0b1 | 3.10 | :pep:`563`: | +| annotations | 3.7.0b1 | 3.11 | :pep:`563`: | | | | | *Postponed evaluation of annotations* | +------------------+-------------+--------------+---------------------------------------------+ diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index bd653ab32bb9c4..1e6452b7b826fd 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -61,15 +61,27 @@ This module defines the following constants and functions: :func:`sys.unraisablehook` is now used to handle unhandled exceptions. -.. function:: interrupt_main() +.. function:: interrupt_main(signum=signal.SIGINT, /) - Simulate the effect of a :data:`signal.SIGINT` signal arriving in the main - thread. A thread can use this function to interrupt the main thread. + Simulate the effect of a signal arriving in the main thread. + A thread can use this function to interrupt the main thread, though + there is no guarantee that the interruption will happen immediately. - If :data:`signal.SIGINT` isn't handled by Python (it was set to + If given, *signum* is the number of the signal to simulate. + If *signum* is not given, :data:`signal.SIGINT` is simulated. + + If the given signal isn't handled by Python (it was set to :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does nothing. + .. versionchanged:: 3.10 + The *signum* argument is added to customize the signal number. + + .. note:: + This does not emit the corresponding signal but schedules a call to + the associated handler (if it exists). + If you want to truly emit the signal, use :func:`signal.raise_signal`. + .. function:: exit() diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 3a7414d7358e7a..1a6ed474ff21da 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -336,6 +336,7 @@ The :mod:`abc` module also provides the following functions: .. versionadded:: 3.4 .. function:: update_abstractmethods(cls) + A function to recalculate an abstract class's abstraction status. This function should be called if a class's abstract methods have been implemented or changed after it was created. Usually, this function should @@ -343,7 +344,7 @@ The :mod:`abc` module also provides the following functions: Returns *cls*, to allow usage as a class decorator. - If *cls* is not an instance of ABCMeta, does nothing. + If *cls* is not an instance of :class:`ABCMeta`, does nothing. .. note:: diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 7a7a4cf94979a1..b2eb9eff914c69 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -57,7 +57,7 @@ be run at the command line and provides useful help messages: positional arguments: N an integer for the accumulator - optional arguments: + options: -h, --help show this help message and exit --sum sum the integers (default: find the max) @@ -217,14 +217,14 @@ The help for this program will display ``myprogram.py`` as the program name $ python myprogram.py --help usage: myprogram.py [-h] [--foo FOO] - optional arguments: + options: -h, --help show this help message and exit --foo FOO foo help $ cd .. $ python subdir/myprogram.py --help usage: myprogram.py [-h] [--foo FOO] - optional arguments: + options: -h, --help show this help message and exit --foo FOO foo help @@ -235,7 +235,7 @@ To change this default behavior, another value can be supplied using the >>> parser.print_help() usage: myprogram [-h] - optional arguments: + options: -h, --help show this help message and exit Note that the program name, whether determined from ``sys.argv[0]`` or from the @@ -249,7 +249,7 @@ specifier. >>> parser.print_help() usage: myprogram [-h] [--foo FOO] - optional arguments: + options: -h, --help show this help message and exit --foo FOO foo of the myprogram program @@ -269,7 +269,7 @@ arguments it contains:: positional arguments: bar bar help - optional arguments: + options: -h, --help show this help message and exit --foo [FOO] foo help @@ -284,7 +284,7 @@ The default message can be overridden with the ``usage=`` keyword argument:: positional arguments: bar bar help - optional arguments: + options: -h, --help show this help message and exit --foo [FOO] foo help @@ -307,7 +307,7 @@ various arguments:: A foo that bars - optional arguments: + options: -h, --help show this help message and exit By default, the description will be line-wrapped so that it fits within the @@ -329,7 +329,7 @@ argument to :class:`ArgumentParser`:: A foo that bars - optional arguments: + options: -h, --help show this help message and exit And that's how you'd foo a bar @@ -403,7 +403,7 @@ epilog_ texts in command-line help messages:: this description was indented weird but that is okay - optional arguments: + options: -h, --help show this help message and exit likewise for this epilog whose whitespace will be cleaned up and whose words @@ -432,7 +432,7 @@ should not be line-wrapped:: exactly the way I want it - optional arguments: + options: -h, --help show this help message and exit :class:`RawTextHelpFormatter` maintains whitespace for all sorts of help text, @@ -454,7 +454,7 @@ default values to each of the argument help messages:: positional arguments: bar BAR! (default: [1, 2, 3]) - optional arguments: + options: -h, --help show this help message and exit --foo FOO FOO! (default: 42) @@ -473,7 +473,7 @@ as the regular formatter does):: positional arguments: float - optional arguments: + options: -h, --help show this help message and exit --foo int @@ -592,7 +592,7 @@ older arguments with the same option string. To get this behavior, the value >>> parser.print_help() usage: PROG [-h] [-f FOO] [--foo FOO] - optional arguments: + options: -h, --help show this help message and exit -f FOO old foo help --foo FOO new foo help @@ -623,7 +623,7 @@ help will be printed: $ python myprogram.py --help usage: myprogram.py [-h] [--foo FOO] - optional arguments: + options: -h, --help show this help message and exit --foo FOO foo help @@ -636,7 +636,7 @@ This can be achieved by passing ``False`` as the ``add_help=`` argument to >>> parser.print_help() usage: PROG [--foo FOO] - optional arguments: + options: --foo FOO foo help The help option is typically ``-h/--help``. The exception to this is @@ -649,7 +649,7 @@ the help options:: >>> parser.print_help() usage: PROG [+h] - optional arguments: + options: +h, ++help show this help message and exit @@ -659,7 +659,7 @@ exit_on_error Normally, when you pass an invalid argument list to the :meth:`~ArgumentParser.parse_args` method of an :class:`ArgumentParser`, it will exit with error info. -If the user would like catch errors manually, the feature can be enable by setting +If the user would like to catch errors manually, the feature can be enabled by setting ``exit_on_error`` to ``False``:: >>> parser = argparse.ArgumentParser(exit_on_error=False) @@ -696,7 +696,7 @@ The add_argument() method * const_ - A constant value required by some action_ and nargs_ selections. * default_ - The value produced if the argument is absent from the - command line. + command line and if it is absent from the namespace object. * type_ - The type to which the command-line argument should be converted. @@ -863,7 +863,7 @@ An example of a custom action:: ... def __init__(self, option_strings, dest, nargs=None, **kwargs): ... if nargs is not None: ... raise ValueError("nargs not allowed") - ... super(FooAction, self).__init__(option_strings, dest, **kwargs) + ... super().__init__(option_strings, dest, **kwargs) ... def __call__(self, parser, namespace, values, option_string=None): ... print('%r %r %r' % (namespace, values, option_string)) ... setattr(namespace, self.dest, values) @@ -1006,6 +1006,14 @@ was not present at the command line:: >>> parser.parse_args([]) Namespace(foo=42) +If the target namespace already has an attribute set, the action *default* +will not over write it:: + + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo', default=42) + >>> parser.parse_args([], namespace=argparse.Namespace(foo=101)) + Namespace(foo=101) + If the ``default`` value is a string, the parser parses the value as if it were a command-line argument. In particular, the parser applies any type_ conversion argument, if provided, before setting the attribute on the @@ -1042,63 +1050,70 @@ command-line argument was not present:: type ^^^^ -By default, :class:`ArgumentParser` objects read command-line arguments in as simple +By default, the parser reads command-line arguments in as simple strings. However, quite often the command-line string should instead be -interpreted as another type, like a :class:`float` or :class:`int`. The -``type`` keyword argument of :meth:`~ArgumentParser.add_argument` allows any -necessary type-checking and type conversions to be performed. Common built-in -types and functions can be used directly as the value of the ``type`` argument:: +interpreted as another type, such as a :class:`float` or :class:`int`. The +``type`` keyword for :meth:`~ArgumentParser.add_argument` allows any +necessary type-checking and type conversions to be performed. - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('foo', type=int) - >>> parser.add_argument('bar', type=open) - >>> parser.parse_args('2 temp.txt'.split()) - Namespace(bar=<_io.TextIOWrapper name='temp.txt' encoding='UTF-8'>, foo=2) +If the type_ keyword is used with the default_ keyword, the type converter +is only applied if the default is a string. -See the section on the default_ keyword argument for information on when the -``type`` argument is applied to default arguments. +The argument to ``type`` can be any callable that accepts a single string. +If the function raises :exc:`ArgumentTypeError`, :exc:`TypeError`, or +:exc:`ValueError`, the exception is caught and a nicely formatted error +message is displayed. No other exception types are handled. -To ease the use of various types of files, the argparse module provides the -factory FileType which takes the ``mode=``, ``bufsize=``, ``encoding=`` and -``errors=`` arguments of the :func:`open` function. For example, -``FileType('w')`` can be used to create a writable file:: +Common built-in types and functions can be used as type converters: - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('bar', type=argparse.FileType('w')) - >>> parser.parse_args(['out.txt']) - Namespace(bar=<_io.TextIOWrapper name='out.txt' encoding='UTF-8'>) - -``type=`` can take any callable that takes a single string argument and returns -the converted value:: - - >>> def perfect_square(string): - ... value = int(string) - ... sqrt = math.sqrt(value) - ... if sqrt != int(sqrt): - ... msg = "%r is not a perfect square" % string - ... raise argparse.ArgumentTypeError(msg) - ... return value +.. testcode:: + + import argparse + import pathlib + + parser = argparse.ArgumentParser() + parser.add_argument('count', type=int) + parser.add_argument('distance', type=float) + parser.add_argument('street', type=ascii) + parser.add_argument('code_point', type=ord) + parser.add_argument('source_file', type=open) + parser.add_argument('dest_file', type=argparse.FileType('w', encoding='latin-1')) + parser.add_argument('datapath', type=pathlib.Path) + +User defined functions can be used as well: + +.. doctest:: + + >>> def hyphenated(string): + ... return '-'.join([word[:4] for word in string.casefold().split()]) ... - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('foo', type=perfect_square) - >>> parser.parse_args(['9']) - Namespace(foo=9) - >>> parser.parse_args(['7']) - usage: PROG [-h] foo - PROG: error: argument foo: '7' is not a perfect square + >>> parser = argparse.ArgumentParser() + >>> _ = parser.add_argument('short_title', type=hyphenated) + >>> parser.parse_args(['"The Tale of Two Cities"']) + Namespace(short_title='"the-tale-of-two-citi') -The choices_ keyword argument may be more convenient for type checkers that -simply check against a range of values:: +The :func:`bool` function is not recommended as a type converter. All it does +is convert empty strings to ``False`` and non-empty strings to ``True``. +This is usually not what is desired. - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('foo', type=int, choices=range(5, 10)) - >>> parser.parse_args(['7']) - Namespace(foo=7) - >>> parser.parse_args(['11']) - usage: PROG [-h] {5,6,7,8,9} - PROG: error: argument foo: invalid choice: 11 (choose from 5, 6, 7, 8, 9) +In general, the ``type`` keyword is a convenience that should only be used for +simple conversions that can only raise one of the three supported exceptions. +Anything with more interesting error-handling or resource management should be +done downstream after the arguments are parsed. + +For example, JSON or YAML conversions have complex error cases that require +better reporting than can be given by the ``type`` keyword. An +:exc:`~json.JSONDecodeError` would not be well formatted and a +:exc:`FileNotFound` exception would not be handled at all. + +Even :class:`~argparse.FileType` has its limitations for use with the ``type`` +keyword. If one argument uses *FileType* and then a subsequent argument fails, +an error is reported but the file is not automatically closed. In this case, it +would be better to wait until after the parser has run and then use the +:keyword:`with`-statement to manage the files. -See the choices_ section for more details. +For type checkers that simply check against a fixed set of values, consider +using the choices_ keyword instead. choices @@ -1133,20 +1148,14 @@ container should match the type_ specified:: Any container can be passed as the *choices* value, so :class:`list` objects, :class:`set` objects, and custom containers are all supported. -This includes :class:`enum.Enum`, which could be used to restrain -argument's choices; if we reuse previous rock/paper/scissors game example, -this could be as follows:: - - >>> from enum import Enum - >>> class GameMove(Enum): - ... ROCK = 'rock' - ... PAPER = 'paper' - ... SCISSORS = 'scissors' - ... - >>> parser = argparse.ArgumentParser(prog='game.py') - >>> parser.add_argument('move', type=GameMove, choices=GameMove) - >>> parser.parse_args(['rock']) - Namespace(move=) + +Use of :class:`enum.Enum` is not recommended because it is difficult to +control its appearance in usage, help, and error messages. + +Formatted choices overrides the default *metavar* which is normally derived +from *dest*. This is usually what you want because the user never sees the +*dest* parameter. If this display isn't desirable (perhaps because there are +many choices), just specify an explicit metavar_. required @@ -1194,7 +1203,7 @@ argument:: positional arguments: bar one of the bars to be frobbled - optional arguments: + options: -h, --help show this help message and exit --foo foo the bars before frobbling @@ -1212,7 +1221,7 @@ specifiers include the program name, ``%(prog)s`` and most keyword arguments to positional arguments: bar the bar to frobble (default: 42) - optional arguments: + options: -h, --help show this help message and exit As the help string supports %-formatting, if you want a literal ``%`` to appear @@ -1226,7 +1235,7 @@ setting the ``help`` value to ``argparse.SUPPRESS``:: >>> parser.print_help() usage: frobble [-h] - optional arguments: + options: -h, --help show this help message and exit @@ -1253,7 +1262,7 @@ will be referred to as ``FOO``. An example:: positional arguments: bar - optional arguments: + options: -h, --help show this help message and exit --foo FOO @@ -1270,7 +1279,7 @@ An alternative name can be specified with ``metavar``:: positional arguments: XXX - optional arguments: + options: -h, --help show this help message and exit --foo YYY @@ -1288,7 +1297,7 @@ arguments:: >>> parser.print_help() usage: PROG [-h] [-x X X] [--foo bar baz] - optional arguments: + options: -h, --help show this help message and exit -x X X --foo bar baz @@ -1692,7 +1701,7 @@ Sub-commands a a help b b help - optional arguments: + options: -h, --help show this help message and exit --foo foo help @@ -1702,13 +1711,13 @@ Sub-commands positional arguments: bar bar help - optional arguments: + options: -h, --help show this help message and exit >>> parser.parse_args(['b', '--help']) usage: PROG b [-h] [--baz {X,Y,Z}] - optional arguments: + options: -h, --help show this help message and exit --baz {X,Y,Z} baz help @@ -1725,7 +1734,7 @@ Sub-commands >>> parser.parse_args(['-h']) usage: [-h] {foo,bar} ... - optional arguments: + options: -h, --help show this help message and exit subcommands: diff --git a/Doc/library/array.rst b/Doc/library/array.rst index ff3ec6b1fd723b..c7f137d15b4b86 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -160,8 +160,7 @@ The following data items and methods are also supported: Read *n* items (as machine values) from the :term:`file object` *f* and append them to the end of the array. If less than *n* items are available, :exc:`EOFError` is raised, but the items that were available are still - inserted into the array. *f* must be a real built-in file object; something - else with a :meth:`read` method won't do. + inserted into the array. .. method:: array.fromlist(list) @@ -178,11 +177,15 @@ The following data items and methods are also supported: array of some other type. -.. method:: array.index(x) +.. method:: array.index(x[, start[, stop]]) Return the smallest *i* such that *i* is the index of the first occurrence of - *x* in the array. + *x* in the array. The optional arguments *start* and *stop* can be + specified to search for *x* within a subsection of the array. Raise + :exc:`ValueError` if *x* is not found. + .. versionchanged:: 3.10 + Added optional *start* and *stop* parameters. .. method:: array.insert(i, x) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index b8688ae61a4876..c7074c40f280c6 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -80,12 +80,13 @@ Node classes end_col_offset Instances of :class:`ast.expr` and :class:`ast.stmt` subclasses have - :attr:`lineno`, :attr:`col_offset`, :attr:`lineno`, and :attr:`col_offset` - attributes. The :attr:`lineno` and :attr:`end_lineno` are the first and - last line numbers of source text span (1-indexed so the first line is line 1) - and the :attr:`col_offset` and :attr:`end_col_offset` are the corresponding - UTF-8 byte offsets of the first and last tokens that generated the node. - The UTF-8 offset is recorded because the parser uses UTF-8 internally. + :attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, and + :attr:`end_col_offset` attributes. The :attr:`lineno` and :attr:`end_lineno` + are the first and last line numbers of source text span (1-indexed so the + first line is line 1) and the :attr:`col_offset` and :attr:`end_col_offset` + are the corresponding UTF-8 byte offsets of the first and last tokens that + generated the node. The UTF-8 offset is recorded because the parser uses + UTF-8 internally. Note that the end positions are not required by the compiler and are therefore optional. The end offset is *after* the last symbol, for example @@ -139,6 +140,11 @@ Node classes In the meantime, instantiating them will return an instance of a different class. +.. note:: + The descriptions of the specific node classes displayed here + were initially adapted from the fantastic `Green Tree + Snakes `__ project and + all its contributors. Literals ^^^^^^^^ @@ -832,7 +838,7 @@ Statements context), ``op`` is :class:`Add`, and ``value`` is a :class:`Constant` with value for 1. - The ``target`` attribute connot be of class :class:`Tuple` or :class:`List`, + The ``target`` attribute cannot be of class :class:`Tuple` or :class:`List`, unlike the targets of :class:`Assign`. .. doctest:: @@ -1239,6 +1245,374 @@ Control flow type_ignores=[]) +Pattern matching +^^^^^^^^^^^^^^^^ + + +.. class:: Match(subject, cases) + + A ``match`` statement. ``subject`` holds the subject of the match (the object + that is being matched against the cases) and ``cases`` contains an iterable of + :class:`match_case` nodes with the different cases. + +.. class:: match_case(pattern, guard, body) + + A single case pattern in a ``match`` statement. ``pattern`` contains the + match pattern that the subject will be matched against. Note that the + :class:`AST` nodes produced for patterns differ from those produced for + expressions, even when they share the same syntax. + + The ``guard`` attribute contains an expression that will be evaluated if + the pattern matches the subject. + + ``body`` contains a list of nodes to execute if the pattern matches and + the result of evaluating the guard expression is truthy. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case [x] if x>0: + ... ... + ... case tuple(): + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchSequence( + patterns=[ + MatchAs(name='x')]), + guard=Compare( + left=Name(id='x', ctx=Load()), + ops=[ + Gt()], + comparators=[ + Constant(value=0)]), + body=[ + Expr( + value=Constant(value=Ellipsis))]), + match_case( + pattern=MatchClass( + cls=Name(id='tuple', ctx=Load()), + patterns=[], + kwd_attrs=[], + kwd_patterns=[]), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + +.. class:: MatchValue(value) + + A match literal or value pattern that compares by equality. ``value`` is + an expression node. Permitted value nodes are restricted as described in + the match statement documentation. This pattern succeeds if the match + subject is equal to the evaluated value. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case "Relevant": + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchValue( + value=Constant(value='Relevant')), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + +.. class:: MatchSingleton(value) + + A match literal pattern that compares by identity. ``value`` is the + singleton to be compared against: ``None``, ``True``, or ``False``. This + pattern succeeds if the match subject is the given constant. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case None: + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchSingleton(value=None), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + +.. class:: MatchSequence(patterns) + + A match sequence pattern. ``patterns`` contains the patterns to be matched + against the subject elements if the subject is a sequence. Matches a variable + length sequence if one of the subpatterns is a ``MatchStar`` node, otherwise + matches a fixed length sequence. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case [1, 2]: + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchSequence( + patterns=[ + MatchValue( + value=Constant(value=1)), + MatchValue( + value=Constant(value=2))]), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + +.. class:: MatchStar(name) + + Matches the rest of the sequence in a variable length match sequence pattern. + If ``name`` is not ``None``, a list containing the remaining sequence + elements is bound to that name if the overall sequence pattern is successful. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case [1, 2, *rest]: + ... ... + ... case [*_]: + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchSequence( + patterns=[ + MatchValue( + value=Constant(value=1)), + MatchValue( + value=Constant(value=2)), + MatchStar(name='rest')]), + body=[ + Expr( + value=Constant(value=Ellipsis))]), + match_case( + pattern=MatchSequence( + patterns=[ + MatchStar()]), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + +.. class:: MatchMapping(keys, patterns, rest) + + A match mapping pattern. ``keys`` is a sequence of expression nodes. + ``patterns`` is a corresponding sequence of pattern nodes. ``rest`` is an + optional name that can be specified to capture the remaining mapping elements. + Permitted key expressions are restricted as described in the match statement + documentation. + + This pattern succeeds if the subject is a mapping, all evaluated key + expressions are present in the mapping, and the value corresponding to each + key matches the corresponding subpattern. If ``rest`` is not ``None``, a dict + containing the remaining mapping elements is bound to that name if the overall + mapping pattern is successful. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case {1: _, 2: _}: + ... ... + ... case {**rest}: + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchMapping( + keys=[ + Constant(value=1), + Constant(value=2)], + patterns=[ + MatchAs(), + MatchAs()]), + body=[ + Expr( + value=Constant(value=Ellipsis))]), + match_case( + pattern=MatchMapping(keys=[], patterns=[], rest='rest'), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + +.. class:: MatchClass(cls, patterns, kwd_attrs, kwd_patterns) + + A match class pattern. ``cls`` is an expression giving the nominal class to + be matched. ``patterns`` is a sequence of pattern nodes to be matched against + the class defined sequence of pattern matching attributes. ``kwd_attrs`` is a + sequence of additional attributes to be matched (specified as keyword arguments + in the class pattern), ``kwd_patterns`` are the corresponding patterns + (specified as keyword values in the class pattern). + + This pattern succeeds if the subject is an instance of the nominated class, + all positional patterns match the corresponding class-defined attributes, and + any specified keyword attributes match their corresponding pattern. + + Note: classes may define a property that returns self in order to match a + pattern node against the instance being matched. Several builtin types are + also matched that way, as described in the match statement documentation. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case Point2D(0, 0): + ... ... + ... case Point3D(x=0, y=0, z=0): + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchClass( + cls=Name(id='Point2D', ctx=Load()), + patterns=[ + MatchValue( + value=Constant(value=0)), + MatchValue( + value=Constant(value=0))], + kwd_attrs=[], + kwd_patterns=[]), + body=[ + Expr( + value=Constant(value=Ellipsis))]), + match_case( + pattern=MatchClass( + cls=Name(id='Point3D', ctx=Load()), + patterns=[], + kwd_attrs=[ + 'x', + 'y', + 'z'], + kwd_patterns=[ + MatchValue( + value=Constant(value=0)), + MatchValue( + value=Constant(value=0)), + MatchValue( + value=Constant(value=0))]), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + +.. class:: MatchAs(pattern, name) + + A match "as-pattern", capture pattern or wildcard pattern. ``pattern`` + contains the match pattern that the subject will be matched against. + If the pattern is ``None``, the node represents a capture pattern (i.e a + bare name) and will always succeed. + + The ``name`` attribute contains the name that will be bound if the pattern + is successful. If ``name`` is ``None``, ``pattern`` must also be ``None`` + and the node represents the wildcard pattern. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case [x] as y: + ... ... + ... case _: + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchAs( + pattern=MatchSequence( + patterns=[ + MatchAs(name='x')]), + name='y'), + body=[ + Expr( + value=Constant(value=Ellipsis))]), + match_case( + pattern=MatchAs(), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + +.. class:: MatchOr(patterns) + + A match "or-pattern". An or-pattern matches each of its subpatterns in turn + to the subject, until one succeeds. The or-pattern is then deemed to + succeed. If none of the subpatterns succeed the or-pattern fails. The + ``patterns`` attribute contains a list of match pattern nodes that will be + matched against the subject. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case [x] | (y): + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchOr( + patterns=[ + MatchSequence( + patterns=[ + MatchAs(name='x')]), + MatchAs(name='y')]), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + + Function and class definitions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1575,7 +1949,7 @@ and classes for traversing abstract syntax trees: Safely evaluate an expression node or a string containing a Python literal or container display. The string or node provided may only consist of the following Python literal structures: strings, bytes, numbers, tuples, lists, - dicts, sets, booleans, and ``None``. + dicts, sets, booleans, ``None`` and ``Ellipsis``. This can be used for safely evaluating strings containing Python values from untrusted sources without the need to parse the values oneself. It is not @@ -1587,6 +1961,10 @@ and classes for traversing abstract syntax trees: sufficiently large/complex string due to stack depth limitations in Python's AST compiler. + It can raise :exc:`ValueError`, :exc:`TypeError`, :exc:`SyntaxError`, + :exc:`MemoryError` and :exc:`RecursionError` depending on the malformed + input. + .. versionchanged:: 3.2 Now allows bytes and set literals. diff --git a/Doc/library/asyncio-api-index.rst b/Doc/library/asyncio-api-index.rst index 047e5bbc58ccad..f558724d4a3ff6 100644 --- a/Doc/library/asyncio-api-index.rst +++ b/Doc/library/asyncio-api-index.rst @@ -49,7 +49,7 @@ await on multiple things with timeouts. - Task object. * - :func:`to_thread` - - Asychronously run a function in a separate OS thread. + - Asynchronously run a function in a separate OS thread. * - :func:`run_coroutine_threadsafe` - Schedule a coroutine from another OS thread. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index b1e73189a7a4cf..ca91efec260db2 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -53,6 +53,11 @@ an event loop: Consider also using the :func:`asyncio.run` function instead of using lower level functions to manually create and close an event loop. + .. deprecated:: 3.10 + Deprecation warning is emitted if there is no running event loop. + If future Python releases this function will be an alias of + :func:`get_running_loop`. + .. function:: set_event_loop(loop) Set *loop* as a current event loop for the current OS thread. @@ -321,7 +326,7 @@ Creating Futures and Tasks .. versionadded:: 3.5.2 -.. method:: loop.create_task(coro, \*, name=None) +.. method:: loop.create_task(coro, *, name=None) Schedule the execution of a :ref:`coroutine`. Return a :class:`Task` object. @@ -356,7 +361,7 @@ Opening network connections ^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. coroutinemethod:: loop.create_connection(protocol_factory, \ - host=None, port=None, \*, ssl=None, \ + host=None, port=None, *, ssl=None, \ family=0, proto=0, flags=0, sock=None, \ local_addr=None, server_hostname=None, \ ssl_handshake_timeout=None, \ @@ -482,7 +487,7 @@ Opening network connections that can be used directly in async/await code. .. coroutinemethod:: loop.create_datagram_endpoint(protocol_factory, \ - local_addr=None, remote_addr=None, \*, \ + local_addr=None, remote_addr=None, *, \ family=0, proto=0, flags=0, \ reuse_address=None, reuse_port=None, \ allow_broadcast=None, sock=None) @@ -559,7 +564,7 @@ Opening network connections Added support for Windows. .. coroutinemethod:: loop.create_unix_connection(protocol_factory, \ - path=None, \*, ssl=None, sock=None, \ + path=None, *, ssl=None, sock=None, \ server_hostname=None, ssl_handshake_timeout=None) Create a Unix connection. @@ -592,7 +597,7 @@ Creating network servers ^^^^^^^^^^^^^^^^^^^^^^^^ .. coroutinemethod:: loop.create_server(protocol_factory, \ - host=None, port=None, \*, \ + host=None, port=None, *, \ family=socket.AF_UNSPEC, \ flags=socket.AI_PASSIVE, \ sock=None, backlog=100, ssl=None, \ @@ -683,7 +688,7 @@ Creating network servers .. coroutinemethod:: loop.create_unix_server(protocol_factory, path=None, \ - \*, sock=None, backlog=100, ssl=None, \ + *, sock=None, backlog=100, ssl=None, \ ssl_handshake_timeout=None, start_serving=True) Similar to :meth:`loop.create_server` but works with the @@ -708,7 +713,7 @@ Creating network servers The *path* parameter can now be a :class:`~pathlib.Path` object. .. coroutinemethod:: loop.connect_accepted_socket(protocol_factory, \ - sock, \*, ssl=None, ssl_handshake_timeout=None) + sock, *, ssl=None, ssl_handshake_timeout=None) Wrap an already accepted connection into a transport/protocol pair. @@ -773,7 +778,7 @@ TLS Upgrade ^^^^^^^^^^^ .. coroutinemethod:: loop.start_tls(transport, protocol, \ - sslcontext, \*, server_side=False, \ + sslcontext, *, server_side=False, \ server_hostname=None, ssl_handshake_timeout=None) Upgrade an existing transport-based connection to TLS. @@ -806,7 +811,7 @@ TLS Upgrade Watching file descriptors ^^^^^^^^^^^^^^^^^^^^^^^^^ -.. method:: loop.add_reader(fd, callback, \*args) +.. method:: loop.add_reader(fd, callback, *args) Start monitoring the *fd* file descriptor for read availability and invoke *callback* with the specified arguments once *fd* is available for @@ -816,7 +821,7 @@ Watching file descriptors Stop monitoring the *fd* file descriptor for read availability. -.. method:: loop.add_writer(fd, callback, \*args) +.. method:: loop.add_writer(fd, callback, *args) Start monitoring the *fd* file descriptor for write availability and invoke *callback* with the specified arguments once *fd* is available for @@ -930,7 +935,7 @@ convenient. :meth:`loop.create_server` and :func:`start_server`. .. coroutinemethod:: loop.sock_sendfile(sock, file, offset=0, count=None, \ - \*, fallback=True) + *, fallback=True) Send a file using high-performance :mod:`os.sendfile` if possible. Return the total number of bytes sent. @@ -964,7 +969,7 @@ convenient. DNS ^^^ -.. coroutinemethod:: loop.getaddrinfo(host, port, \*, family=0, \ +.. coroutinemethod:: loop.getaddrinfo(host, port, *, family=0, \ type=0, proto=0, flags=0) Asynchronous version of :meth:`socket.getaddrinfo`. @@ -1029,7 +1034,7 @@ Working with pipes Unix signals ^^^^^^^^^^^^ -.. method:: loop.add_signal_handler(signum, callback, \*args) +.. method:: loop.add_signal_handler(signum, callback, *args) Set *callback* as the handler for the *signum* signal. @@ -1064,7 +1069,7 @@ Unix signals Executing code in thread or process pools ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. awaitablemethod:: loop.run_in_executor(executor, func, \*args) +.. awaitablemethod:: loop.run_in_executor(executor, func, *args) Arrange for *func* to be called in the specified executor. @@ -1184,10 +1189,13 @@ Allows customizing how exceptions are handled in the event loop. * 'message': Error message; * 'exception' (optional): Exception object; * 'future' (optional): :class:`asyncio.Future` instance; + * 'task' (optional): :class:`asyncio.Task` instance; * 'handle' (optional): :class:`asyncio.Handle` instance; * 'protocol' (optional): :ref:`Protocol ` instance; * 'transport' (optional): :ref:`Transport ` instance; - * 'socket' (optional): :class:`socket.socket` instance. + * 'socket' (optional): :class:`socket.socket` instance; + * 'asyncgen' (optional): Asynchronous generator that caused + the exception. .. note:: @@ -1234,9 +1242,9 @@ async/await code consider using the high-level subprocesses. See :ref:`Subprocess Support on Windows ` for details. -.. coroutinemethod:: loop.subprocess_exec(protocol_factory, \*args, \ +.. coroutinemethod:: loop.subprocess_exec(protocol_factory, *args, \ stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ - stderr=subprocess.PIPE, \*\*kwargs) + stderr=subprocess.PIPE, **kwargs) Create a subprocess from one or more string arguments specified by *args*. @@ -1316,9 +1324,9 @@ async/await code consider using the high-level conforms to the :class:`asyncio.SubprocessTransport` base class and *protocol* is an object instantiated by the *protocol_factory*. -.. coroutinemethod:: loop.subprocess_shell(protocol_factory, cmd, \*, \ +.. coroutinemethod:: loop.subprocess_shell(protocol_factory, cmd, *, \ stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ - stderr=subprocess.PIPE, \*\*kwargs) + stderr=subprocess.PIPE, **kwargs) Create a subprocess from *cmd*, which can be a :class:`str` or a :class:`bytes` string encoded to the diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index e1ac18eaf09882..ef496a23f5cd4c 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -31,7 +31,7 @@ Future Functions .. versionadded:: 3.5 -.. function:: ensure_future(obj, \*, loop=None) +.. function:: ensure_future(obj, *, loop=None) Return: @@ -57,17 +57,25 @@ Future Functions .. versionchanged:: 3.5.1 The function accepts any :term:`awaitable` object. + .. deprecated:: 3.10 + Deprecation warning is emitted if *obj* is not a Future-like object + and *loop* is not specified and there is no running event loop. -.. function:: wrap_future(future, \*, loop=None) + +.. function:: wrap_future(future, *, loop=None) Wrap a :class:`concurrent.futures.Future` object in a :class:`asyncio.Future` object. + .. deprecated:: 3.10 + Deprecation warning is emitted if *future* is not a Future-like object + and *loop* is not specified and there is no running event loop. + Future Object ============= -.. class:: Future(\*, loop=None) +.. class:: Future(*, loop=None) A Future represents an eventual result of an asynchronous operation. Not thread-safe. @@ -90,6 +98,10 @@ Future Object .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. + .. deprecated:: 3.10 + Deprecation warning is emitted if *loop* is not specified + and there is no running event loop. + .. method:: result() Return the result of the Future. diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index 88e69ceff9adc1..ef6a0588506b52 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -159,7 +159,7 @@ implementation used by the asyncio event loop: .. class:: AbstractChildWatcher - .. method:: add_child_handler(pid, callback, \*args) + .. method:: add_child_handler(pid, callback, *args) Register a new child handler. @@ -209,7 +209,7 @@ implementation used by the asyncio event loop: It works reliably even when the asyncio event loop is run in a non-main OS thread. There is no noticeable overhead when handling a big number of children (*O(1)* each - time a child terminates), but stating a thread per process requires extra memory. + time a child terminates), but starting a thread per process requires extra memory. This watcher is used by default. diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index 524560b691d720..289ad1b014c356 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -23,7 +23,7 @@ See also the `Examples`_ section below. Queue ===== -.. class:: Queue(maxsize=0, \*, loop=None) +.. class:: Queue(maxsize=0) A first in, first out (FIFO) queue. @@ -36,9 +36,6 @@ Queue the queue is always known and can be returned by calling the :meth:`qsize` method. - .. deprecated-removed:: 3.8 3.10 - The *loop* parameter. - This class is :ref:`not thread safe `. diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index b76ed379c7f4c8..ad3c7442ad56cd 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -48,9 +48,9 @@ The following top-level asyncio functions can be used to create and work with streams: -.. coroutinefunction:: open_connection(host=None, port=None, \*, \ - loop=None, limit=None, ssl=None, family=0, \ - proto=0, flags=0, sock=None, local_addr=None, \ +.. coroutinefunction:: open_connection(host=None, port=None, *, \ + limit=None, ssl=None, family=0, proto=0, \ + flags=0, sock=None, local_addr=None, \ server_hostname=None, ssl_handshake_timeout=None) Establish a network connection and return a pair of @@ -59,9 +59,6 @@ and work with streams: The returned *reader* and *writer* objects are instances of :class:`StreamReader` and :class:`StreamWriter` classes. - The *loop* argument is optional and can always be determined - automatically when this function is awaited from a coroutine. - *limit* determines the buffer size limit used by the returned :class:`StreamReader` instance. By default the *limit* is set to 64 KiB. @@ -74,7 +71,7 @@ and work with streams: The *ssl_handshake_timeout* parameter. .. coroutinefunction:: start_server(client_connected_cb, host=None, \ - port=None, \*, loop=None, limit=None, \ + port=None, *, limit=None, \ family=socket.AF_UNSPEC, \ flags=socket.AI_PASSIVE, sock=None, \ backlog=100, ssl=None, reuse_address=None, \ @@ -92,9 +89,6 @@ and work with streams: :ref:`coroutine function `; if it is a coroutine function, it will be automatically scheduled as a :class:`Task`. - The *loop* argument is optional and can always be determined - automatically when this method is awaited from a coroutine. - *limit* determines the buffer size limit used by the returned :class:`StreamReader` instance. By default the *limit* is set to 64 KiB. @@ -109,9 +103,9 @@ and work with streams: .. rubric:: Unix Sockets -.. coroutinefunction:: open_unix_connection(path=None, \*, loop=None, \ - limit=None, ssl=None, sock=None, \ - server_hostname=None, ssl_handshake_timeout=None) +.. coroutinefunction:: open_unix_connection(path=None, *, limit=None, \ + ssl=None, sock=None, server_hostname=None, \ + ssl_handshake_timeout=None) Establish a Unix socket connection and return a pair of ``(reader, writer)``. @@ -132,9 +126,8 @@ and work with streams: .. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \ - \*, loop=None, limit=None, sock=None, \ - backlog=100, ssl=None, ssl_handshake_timeout=None, \ - start_serving=True) + *, limit=None, sock=None, backlog=100, ssl=None, \ + ssl_handshake_timeout=None, start_serving=True) Start a Unix socket server. diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index b0330349dfb651..ef4d9bcc434c9f 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -61,9 +61,8 @@ See also the `Examples`_ subsection. Creating Subprocesses ===================== -.. coroutinefunction:: create_subprocess_exec(program, \*args, stdin=None, \ - stdout=None, stderr=None, loop=None, \ - limit=None, \*\*kwds) +.. coroutinefunction:: create_subprocess_exec(program, *args, stdin=None, \ + stdout=None, stderr=None, limit=None, **kwds) Create a subprocess. @@ -76,13 +75,9 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_exec` for other parameters. - .. deprecated-removed:: 3.8 3.10 - - The *loop* parameter. .. coroutinefunction:: create_subprocess_shell(cmd, stdin=None, \ - stdout=None, stderr=None, loop=None, \ - limit=None, \*\*kwds) + stdout=None, stderr=None, limit=None, **kwds) Run the *cmd* shell command. @@ -104,10 +99,6 @@ Creating Subprocesses escape whitespace and special shell characters in strings that are going to be used to construct shell commands. - .. deprecated-removed:: 3.8 3.10 - - The *loop* parameter. - .. note:: Subprocesses are available for Windows if a :class:`ProactorEventLoop` is @@ -251,7 +242,7 @@ their completion. .. method:: kill() - Kill the child. + Kill the child process. On POSIX systems this method sends :py:data:`SIGKILL` to the child process. diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 84a52cb2d57571..d12630afc6a326 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -36,7 +36,7 @@ asyncio has the following basic synchronization primitives: Lock ==== -.. class:: Lock(\*, loop=None) +.. class:: Lock() Implements a mutex lock for asyncio tasks. Not thread-safe. @@ -63,9 +63,6 @@ Lock finally: lock.release() - .. deprecated-removed:: 3.8 3.10 - The *loop* parameter. - .. coroutinemethod:: acquire() Acquire the lock. @@ -96,7 +93,7 @@ Lock Event ===== -.. class:: Event(\*, loop=None) +.. class:: Event() An event object. Not thread-safe. @@ -104,14 +101,10 @@ Event that some event has happened. An Event object manages an internal flag that can be set to *true* - with the :meth:`set` method and reset to *false* with the - :meth:`clear` method. The :meth:`wait` method blocks until the + with the :meth:`~Event.set` method and reset to *false* with the + :meth:`clear` method. The :meth:`~Event.wait` method blocks until the flag is set to *true*. The flag is set to *false* initially. - - .. deprecated-removed:: 3.8 3.10 - The *loop* parameter. - .. _asyncio_example_sync_event: Example:: @@ -142,7 +135,7 @@ Event Wait until the event is set. If the event is set, return ``True`` immediately. - Otherwise block until another task calls :meth:`set`. + Otherwise block until another task calls :meth:`~Event.set`. .. method:: set() @@ -155,8 +148,8 @@ Event Clear (unset) the event. - Tasks awaiting on :meth:`wait` will now block until the - :meth:`set` method is called again. + Tasks awaiting on :meth:`~Event.wait` will now block until the + :meth:`~Event.set` method is called again. .. method:: is_set() @@ -166,7 +159,7 @@ Event Condition ========= -.. class:: Condition(lock=None, \*, loop=None) +.. class:: Condition(lock=None) A Condition object. Not thread-safe. @@ -184,10 +177,6 @@ Condition ``None``. In the latter case a new Lock object is created automatically. - - .. deprecated-removed:: 3.8 3.10 - The *loop* parameter. - The preferred way to use a Condition is an :keyword:`async with` statement:: @@ -270,7 +259,7 @@ Condition Semaphore ========= -.. class:: Semaphore(value=1, \*, loop=None) +.. class:: Semaphore(value=1) A Semaphore object. Not thread-safe. @@ -284,10 +273,6 @@ Semaphore internal counter (``1`` by default). If the given value is less than ``0`` a :exc:`ValueError` is raised. - - .. deprecated-removed:: 3.8 3.10 - The *loop* parameter. - The preferred way to use a Semaphore is an :keyword:`async with` statement:: @@ -332,7 +317,7 @@ Semaphore BoundedSemaphore ================ -.. class:: BoundedSemaphore(value=1, \*, loop=None) +.. class:: BoundedSemaphore(value=1) A bounded semaphore object. Not thread-safe. @@ -340,10 +325,6 @@ BoundedSemaphore a :exc:`ValueError` in :meth:`~Semaphore.release` if it increases the internal counter above the initial *value*. - - .. deprecated-removed:: 3.8 3.10 - The *loop* parameter. - --------- diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 99f012540d989b..3f54ecb08efc1f 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -210,7 +210,7 @@ is :meth:`loop.run_in_executor`. Running an asyncio Program ========================== -.. function:: run(coro, \*, debug=False) +.. function:: run(coro, *, debug=False) Execute the :term:`coroutine` *coro* and return the result. @@ -247,7 +247,7 @@ Running an asyncio Program Creating Tasks ============== -.. function:: create_task(coro, \*, name=None) +.. function:: create_task(coro, *, name=None) Wrap the *coro* :ref:`coroutine ` into a :class:`Task` and schedule its execution. Return the Task object. @@ -283,7 +283,7 @@ Creating Tasks Sleeping ======== -.. coroutinefunction:: sleep(delay, result=None, \*, loop=None) +.. coroutinefunction:: sleep(delay, result=None) Block for *delay* seconds. @@ -293,8 +293,9 @@ Sleeping ``sleep()`` always suspends the current task, allowing other tasks to run. - .. deprecated-removed:: 3.8 3.10 - The *loop* parameter. + Setting the delay to 0 provides an optimized path to allow other + tasks to run. This can be used by long-running functions to avoid + blocking the event loop for the full duration of the function call. .. _asyncio_example_sleep: @@ -319,7 +320,7 @@ Sleeping Running Tasks Concurrently ========================== -.. awaitablefunction:: gather(\*aws, loop=None, return_exceptions=False) +.. awaitablefunction:: gather(*aws, return_exceptions=False) Run :ref:`awaitable objects ` in the *aws* sequence *concurrently*. @@ -348,9 +349,6 @@ Running Tasks Concurrently cancellation of one submitted Task/Future to cause other Tasks/Futures to be cancelled. - .. deprecated-removed:: 3.8 3.10 - The *loop* parameter. - .. _asyncio_example_gather: Example:: @@ -399,11 +397,16 @@ Running Tasks Concurrently If the *gather* itself is cancelled, the cancellation is propagated regardless of *return_exceptions*. + .. deprecated:: 3.10 + Deprecation warning is emitted if no positional arguments are provided + or not all positional arguments are Future-like objects + and there is no running event loop. + Shielding From Cancellation =========================== -.. awaitablefunction:: shield(aw, \*, loop=None) +.. awaitablefunction:: shield(aw) Protect an :ref:`awaitable object ` from being :meth:`cancelled `. @@ -436,14 +439,15 @@ Shielding From Cancellation except CancelledError: res = None - .. deprecated-removed:: 3.8 3.10 - The *loop* parameter. + .. deprecated:: 3.10 + Deprecation warning is emitted if *aw* is not Future-like object + and there is no running event loop. Timeouts ======== -.. coroutinefunction:: wait_for(aw, timeout, \*, loop=None) +.. coroutinefunction:: wait_for(aw, timeout) Wait for the *aw* :ref:`awaitable ` to complete with a timeout. @@ -466,9 +470,6 @@ Timeouts If the wait is cancelled, the future *aw* is also cancelled. - .. deprecated-removed:: 3.8 3.10 - The *loop* parameter. - .. _asyncio_example_waitfor: Example:: @@ -500,14 +501,13 @@ Timeouts Waiting Primitives ================== -.. coroutinefunction:: wait(aws, \*, loop=None, timeout=None,\ - return_when=ALL_COMPLETED) +.. coroutinefunction:: wait(aws, *, timeout=None, return_when=ALL_COMPLETED) Run :ref:`awaitable objects ` in the *aws* - set concurrently and block until the condition specified + iterable concurrently and block until the condition specified by *return_when*. - The *aws* set must not be empty. + The *aws* iterable must not be empty. Returns two sets of Tasks/Futures: ``(done, pending)``. @@ -553,10 +553,6 @@ Waiting Primitives ``wait()`` directly is deprecated as it leads to :ref:`confusing behavior `. - .. deprecated-removed:: 3.8 3.10 - - The *loop* parameter. - .. _asyncio_example_wait_coroutine: .. note:: @@ -590,35 +586,36 @@ Waiting Primitives deprecated. -.. function:: as_completed(aws, \*, loop=None, timeout=None) +.. function:: as_completed(aws, *, timeout=None) Run :ref:`awaitable objects ` in the *aws* - set concurrently. Return an iterator of coroutines. + iterable concurrently. Return an iterator of coroutines. Each coroutine returned can be awaited to get the earliest next - result from the set of the remaining awaitables. + result from the iterable of the remaining awaitables. Raises :exc:`asyncio.TimeoutError` if the timeout occurs before all Futures are done. - .. deprecated-removed:: 3.8 3.10 - The *loop* parameter. - Example:: for coro in as_completed(aws): earliest_result = await coro # ... + .. deprecated:: 3.10 + Deprecation warning is emitted if not all awaitable objects in the *aws* + iterable are Future-like objects and there is no running event loop. + Running in Threads ================== -.. coroutinefunction:: to_thread(func, /, \*args, \*\*kwargs) +.. coroutinefunction:: to_thread(func, /, *args, **kwargs) Asynchronously run function *func* in a separate thread. Any \*args and \*\*kwargs supplied for this function are directly passed - to *func*. Also, the current :class:`contextvars.Context` is propogated, + to *func*. Also, the current :class:`contextvars.Context` is propagated, allowing context variables from the event loop thread to be accessed in the separate thread. @@ -743,7 +740,7 @@ Introspection Task Object =========== -.. class:: Task(coro, \*, loop=None, name=None) +.. class:: Task(coro, *, loop=None, name=None) A :class:`Future-like ` object that runs a Python :ref:`coroutine `. Not thread-safe. @@ -791,6 +788,10 @@ Task Object .. deprecated-removed:: 3.8 3.10 The *loop* parameter. + .. deprecated:: 3.10 + Deprecation warning is emitted if *loop* is not specified + and there is no running event loop. + .. method:: cancel(msg=None) Request the Task to be cancelled. @@ -909,7 +910,7 @@ Task Object See the documentation of :meth:`Future.remove_done_callback` for more details. - .. method:: get_stack(\*, limit=None) + .. method:: get_stack(*, limit=None) Return the list of stack frames for this Task. @@ -930,7 +931,7 @@ Task Object stack are returned, but the oldest frames of a traceback are returned. (This matches the behavior of the traceback module.) - .. method:: print_stack(\*, limit=None, file=None) + .. method:: print_stack(*, limit=None, file=None) Print the stack or traceback for this Task. diff --git a/Doc/library/audit_events.rst b/Doc/library/audit_events.rst index 367d56e4e37359..8227a7955bef81 100644 --- a/Doc/library/audit_events.rst +++ b/Doc/library/audit_events.rst @@ -7,7 +7,7 @@ Audit events table This table contains all events raised by :func:`sys.audit` or :c:func:`PySys_Audit` calls throughout the CPython runtime and the -standard library. These calls were added in 3.8.0 or later. +standard library. These calls were added in 3.8.0 or later (see :pep:`578`). See :func:`sys.addaudithook` and :c:func:`PySys_AddAuditHook` for information on handling these events. diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index 85cdc16a7d78d4..f6787ab120ed57 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -25,8 +25,6 @@ The :mod:`bz2` module contains: * The :func:`compress` and :func:`decompress` functions for one-shot (de)compression. -All of the classes in this module may safely be accessed from multiple threads. - (De)compression of files ------------------------ @@ -140,6 +138,11 @@ All of the classes in this module may safely be accessed from multiple threads. The *compresslevel* parameter became keyword-only. + .. versionchanged:: 3.10 + This class is thread unsafe in the face of multiple simultaneous + readers or writers, just like its equivalent classes in :mod:`gzip` and + :mod:`lzma` have always been. + Incremental (de)compression --------------------------- @@ -266,7 +269,6 @@ Below are some examples of typical usage of the :mod:`bz2` module. Using :func:`compress` and :func:`decompress` to demonstrate round-trip compression: >>> import bz2 - >>> data = b"""\ ... Donec rhoncus quis sapien sit amet molestie. Fusce scelerisque vel augue ... nec ullamcorper. Nam rutrum pretium placerat. Aliquam vel tristique lorem, @@ -275,11 +277,9 @@ Using :func:`compress` and :func:`decompress` to demonstrate round-trip compress ... Aliquam pharetra lacus non risus vehicula rutrum. Maecenas aliquam leo ... felis. Pellentesque semper nunc sit amet nibh ullamcorper, ac elementum ... dolor luctus. Curabitur lacinia mi ornare consectetur vestibulum.""" - >>> c = bz2.compress(data) >>> len(data) / len(c) # Data compression ratio 1.513595166163142 - >>> d = bz2.decompress(c) >>> data == d # Check equality to original object after round-trip True @@ -287,7 +287,6 @@ Using :func:`compress` and :func:`decompress` to demonstrate round-trip compress Using :class:`BZ2Compressor` for incremental compression: >>> import bz2 - >>> def gen_data(chunks=10, chunksize=1000): ... """Yield incremental blocks of chunksize bytes.""" ... for _ in range(chunks): @@ -310,7 +309,6 @@ while ordered, repetitive data usually yields a high compression ratio. Writing and reading a bzip2-compressed file in binary mode: >>> import bz2 - >>> data = b"""\ ... Donec rhoncus quis sapien sit amet molestie. Fusce scelerisque vel augue ... nec ullamcorper. Nam rutrum pretium placerat. Aliquam vel tristique lorem, @@ -319,14 +317,11 @@ Writing and reading a bzip2-compressed file in binary mode: ... Aliquam pharetra lacus non risus vehicula rutrum. Maecenas aliquam leo ... felis. Pellentesque semper nunc sit amet nibh ullamcorper, ac elementum ... dolor luctus. Curabitur lacinia mi ornare consectetur vestibulum.""" - >>> with bz2.open("myfile.bz2", "wb") as f: ... # Write compressed data to file ... unused = f.write(data) - >>> with bz2.open("myfile.bz2", "rb") as f: ... # Decompress data from file ... content = f.read() - >>> content == data # Check equality to original object after round-trip True diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst index 4048592e7361f7..05d9cdf424073f 100644 --- a/Doc/library/cgi.rst +++ b/Doc/library/cgi.rst @@ -277,14 +277,14 @@ These are useful if you want more control, or if you want to employ some of the algorithms implemented in this module in other circumstances. -.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False) +.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False, separator="&") Parse a query in the environment or from a file (the file defaults to - ``sys.stdin``). The *keep_blank_values* and *strict_parsing* parameters are + ``sys.stdin``). The *keep_blank_values*, *strict_parsing* and *separator* parameters are passed to :func:`urllib.parse.parse_qs` unchanged. -.. function:: parse_multipart(fp, pdict, encoding="utf-8", errors="replace") +.. function:: parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator="&") Parse input of type :mimetype:`multipart/form-data` (for file uploads). Arguments are *fp* for the input file, *pdict* for a dictionary containing @@ -303,6 +303,9 @@ algorithms implemented in this module in other circumstances. Added the *encoding* and *errors* parameters. For non-file fields, the value is now a list of strings, not bytes. + .. versionchanged:: 3.10 + Added the *separator* parameter. + .. function:: parse_header(string) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index a0265135205901..0dcd88f9fd5b7f 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -159,9 +159,13 @@ function: .. function:: register(search_function) Register a codec search function. Search functions are expected to take one - argument, being the encoding name in all lower case letters, and return a - :class:`CodecInfo` object. In case a search function cannot find - a given encoding, it should return ``None``. + argument, being the encoding name in all lower case letters with hyphens + and spaces converted to underscores, and return a :class:`CodecInfo` object. + In case a search function cannot find a given encoding, it should return + ``None``. + + .. versionchanged:: 3.9 + Hyphens and spaces are converted to underscore. .. function:: unregister(search_function) @@ -697,7 +701,7 @@ compatible with the Python codec registry. .. method:: reset() - Flushes and resets the codec buffers used for keeping state. + Resets the codec buffers used for keeping internal state. Calling this method should ensure that the data on the output is put into a clean state that allows appending of new fresh data without having to @@ -792,7 +796,7 @@ compatible with the Python codec registry. .. method:: reset() - Resets the codec buffers used for keeping state. + Resets the codec buffers used for keeping internal state. Note that no stream repositioning should take place. This method is primarily intended to be able to recover from decoding errors. @@ -1417,6 +1421,9 @@ Applications) and :rfc:`3492` (Nameprep: A Stringprep Profile for Internationalized Domain Names (IDN)). It builds upon the ``punycode`` encoding and :mod:`stringprep`. +If you need the IDNA 2008 standard from :rfc:`5891` and :rfc:`5895`, use the +third-party `idna module _`. + These RFCs together define a protocol to support non-ASCII characters in domain names. A domain name containing non-ASCII characters (such as ``www.Alliancefrançaise.nu``) is converted into an ASCII-compatible encoding diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index db0e25bb0772eb..2345e78a17e4f5 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -291,7 +291,7 @@ Notes on using :class:`Set` and :class:`MutableSet` as a mixin: :meth:`_from_iterable` which calls ``cls(iterable)`` to produce a new set. If the :class:`Set` mixin is being used in a class with a different constructor signature, you will need to override :meth:`_from_iterable` - with a classmethod that can construct new instances from + with a classmethod or regular method that can construct new instances from an iterable argument. (2) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index f538da5e1c9faa..94166ec6c754a7 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -33,11 +33,6 @@ Python's general purpose built-in containers, :class:`dict`, :class:`list`, :class:`UserString` wrapper around string objects for easier string subclassing ===================== ==================================================================== -.. deprecated-removed:: 3.3 3.10 - Moved :ref:`collections-abstract-base-classes` to the :mod:`collections.abc` module. - For backwards compatibility, they continue to be visible in this module through - Python 3.9. - :class:`ChainMap` objects ------------------------- @@ -77,19 +72,23 @@ The class can be used to simulate nested scopes and is useful in templating. be modified to change which mappings are searched. The list should always contain at least one mapping. - .. method:: new_child(m=None) + .. method:: new_child(m=None, **kwargs) Returns a new :class:`ChainMap` containing a new map followed by all of the maps in the current instance. If ``m`` is specified, it becomes the new map at the front of the list of mappings; if not specified, an empty dict is used, so that a call to ``d.new_child()`` - is equivalent to: ``ChainMap({}, *d.maps)``. This method is used for - creating subcontexts that can be updated without altering values in any - of the parent mappings. + is equivalent to: ``ChainMap({}, *d.maps)``. If any keyword arguments + are specified, they update passed map or new empty dict. This method + is used for creating subcontexts that can be updated without altering + values in any of the parent mappings. .. versionchanged:: 3.4 The optional ``m`` parameter was added. + .. versionchanged:: 3.10 + Keyword arguments support was added. + .. attribute:: parents Property returning a new :class:`ChainMap` containing all of the maps in @@ -128,7 +127,7 @@ The class can be used to simulate nested scopes and is useful in templating. writing to any mapping in the chain. * Django's `Context class - `_ + `_ for templating is a read-only chain of mappings. It also features pushing and popping of contexts similar to the :meth:`~collections.ChainMap.new_child` method and the @@ -314,6 +313,16 @@ For example:: .. versionadded:: 3.2 + .. method:: total() + + Compute the sum of the counts. + + >>> c = Counter(a=10, b=5, c=0) + >>> c.total() + 15 + + .. versionadded:: 3.10 + The usual dictionary methods are available for :class:`Counter` objects except for two which work differently for counters. @@ -343,7 +352,7 @@ All of those tests treat missing elements as having zero counts so that Common patterns for working with :class:`Counter` objects:: - sum(c.values()) # total of all counts + c.total() # total of all counts c.clear() # reset all counts list(c) # list unique elements set(c) # convert to a set @@ -862,6 +871,9 @@ they add the ability to access fields by name instead of position index. Named tuple instances do not have per-instance dictionaries, so they are lightweight and require no more memory than regular tuples. + To support pickling, the named tuple class should be assigned to a variable + that matches *typename*. + .. versionchanged:: 3.1 Added support for *rename*. diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index 9b914b1f0d9c6d..5c6e68f9304753 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -148,7 +148,7 @@ runtime. Public functions ---------------- -.. function:: compile_dir(dir, maxlevels=sys.getrecursionlimit(), ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None, \*, stripdir=None, prependdir=None, limit_sl_dest=None, hardlink_dupes=False) +.. function:: compile_dir(dir, maxlevels=sys.getrecursionlimit(), ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None, *, stripdir=None, prependdir=None, limit_sl_dest=None, hardlink_dupes=False) Recursively descend the directory tree named by *dir*, compiling all :file:`.py` files along the way. Return a true value if all the files compiled successfully, @@ -231,7 +231,7 @@ Public functions Added *stripdir*, *prependdir*, *limit_sl_dest* and *hardlink_dupes* arguments. Default value of *maxlevels* was changed from ``10`` to ``sys.getrecursionlimit()`` -.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None, \*, stripdir=None, prependdir=None, limit_sl_dest=None, hardlink_dupes=False) +.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None, *, stripdir=None, prependdir=None, limit_sl_dest=None, hardlink_dupes=False) Compile the file with path *fullname*. Return a true value if the file compiled successfully, and a false value otherwise. diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst index b150990b83b75b..5be1a1106b09a0 100644 --- a/Doc/library/concurrency.rst +++ b/Doc/library/concurrency.rst @@ -21,6 +21,7 @@ multitasking). Here's an overview: subprocess.rst sched.rst queue.rst + contextvars.rst The following are support modules for some of the above services: diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 675a9ffdd0711a..d57f8ce23d12c4 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -67,7 +67,7 @@ Executor Objects .. versionchanged:: 3.5 Added the *chunksize* argument. - .. method:: shutdown(wait=True, \*, cancel_futures=False) + .. method:: shutdown(wait=True, *, cancel_futures=False) Signal the executor that it should free any resources that it is using when the currently pending futures are done executing. Calls to @@ -236,9 +236,9 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. An :class:`Executor` subclass that executes calls asynchronously using a pool of at most *max_workers* processes. If *max_workers* is ``None`` or not given, it will default to the number of processors on the machine. - If *max_workers* is lower or equal to ``0``, then a :exc:`ValueError` + If *max_workers* is less than or equal to ``0``, then a :exc:`ValueError` will be raised. - On Windows, *max_workers* must be equal or lower than ``61``. If it is not + On Windows, *max_workers* must be less than or equal to ``61``. If it is not then :exc:`ValueError` will be raised. If *max_workers* is ``None``, then the default chosen will be at most ``61``, even if more processors are available. @@ -250,7 +250,7 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. each worker process; *initargs* is a tuple of arguments passed to the initializer. Should *initializer* raise an exception, all currently pending jobs will raise a :exc:`~concurrent.futures.process.BrokenProcessPool`, - as well any attempt to submit more jobs to the pool. + as well as any attempt to submit more jobs to the pool. .. versionchanged:: 3.3 When one of the worker processes terminates abruptly, a diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 0aa4ad76523480..b92f703509fcac 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -126,6 +126,31 @@ Functions and classes provided: .. versionadded:: 3.7 + Context managers defined with :func:`asynccontextmanager` can be used + either as decorators or with :keyword:`async with` statements:: + + import time + + async def timeit(): + now = time.monotonic() + try: + yield + finally: + print(f'it took {time.monotonic() - now}s to run') + + @timeit() + async def main(): + # ... async code ... + + When used as a decorator, a new generator instance is implicitly created on + each function call. This allows the otherwise "one-shot" context managers + created by :func:`asynccontextmanager` to meet the requirement that context + managers support multiple invocations in order to be used as decorators. + + .. versionchanged:: 3.10 + Async context managers created with :func:`asynccontextmanager` can + be used as decorators. + .. function:: closing(thing) @@ -154,6 +179,39 @@ Functions and classes provided: ``page.close()`` will be called when the :keyword:`with` block is exited. +.. class:: aclosing(thing) + + Return an async context manager that calls the ``aclose()`` method of *thing* + upon completion of the block. This is basically equivalent to:: + + from contextlib import asynccontextmanager + + @asynccontextmanager + async def aclosing(thing): + try: + yield thing + finally: + await thing.aclose() + + Significantly, ``aclosing()`` supports deterministic cleanup of async + generators when they happen to exit early by :keyword:`break` or an + exception. For example:: + + from contextlib import aclosing + + async with aclosing(my_generator()) as values: + async for value in values: + if value == 42: + break + + This pattern ensures that the generator's async exit code is executed in + the same context as its iterations (so that exceptions and context + variables work as expected, and the exit code isn't run after the + lifetime of some task it depends on). + + .. versionadded:: 3.10 + + .. _simplifying-support-for-single-optional-context-managers: .. function:: nullcontext(enter_result=None) @@ -185,8 +243,26 @@ Functions and classes provided: with cm as file: # Perform processing on the file + It can also be used as a stand-in for + :ref:`asynchronous context managers `:: + + async def send_http(session=None): + if not session: + # If no http session, create it with aiohttp + cm = aiohttp.ClientSession() + else: + # Caller is responsible for closing the session + cm = nullcontext(session) + + async with cm as session: + # Send http requests with session + .. versionadded:: 3.7 + .. versionchanged:: 3.10 + :term:`asynchronous context manager` support was added. + + .. function:: suppress(*exceptions) @@ -351,6 +427,45 @@ Functions and classes provided: .. versionadded:: 3.2 +.. class:: AsyncContextDecorator + + Similar to :class:`ContextDecorator` but only for asynchronous functions. + + Example of ``AsyncContextDecorator``:: + + from asyncio import run + from contextlib import AsyncContextDecorator + + class mycontext(AsyncContextDecorator): + async def __aenter__(self): + print('Starting') + return self + + async def __aexit__(self, *exc): + print('Finishing') + return False + + >>> @mycontext() + ... async def function(): + ... print('The bit in the middle') + ... + >>> run(function()) + Starting + The bit in the middle + Finishing + + >>> async def function(): + ... async with mycontext(): + ... print('The bit in the middle') + ... + >>> run(function()) + Starting + The bit in the middle + Finishing + + .. versionadded:: 3.10 + + .. class:: ExitStack() A context manager that is designed to make it easy to programmatically @@ -638,7 +753,7 @@ even further by means of a small helper class:: class Callback(ExitStack): def __init__(self, callback, /, *args, **kwds): - super(Callback, self).__init__() + super().__init__() self.callback(callback, *args, **kwds) def cancel(self): diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 8805661c456edb..14ac47f4c9eb16 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -26,7 +26,7 @@ See also :pep:`567` for additional details. Context Variables ----------------- -.. class:: ContextVar(name, [\*, default]) +.. class:: ContextVar(name, [*, default]) This class is used to declare a new Context Variable, e.g.:: @@ -146,7 +146,7 @@ Manual Context Management Context implements the :class:`collections.abc.Mapping` interface. - .. method:: run(callable, \*args, \*\*kwargs) + .. method:: run(callable, *args, **kwargs) Execute ``callable(*args, **kwargs)`` code in the context object the *run* method is called on. Return the result of the execution diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 176e01db6f9faf..01ebf198d7c501 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -27,7 +27,7 @@ Interface summary: Return a deep copy of *x*. -.. exception:: error +.. exception:: Error Raised for module specific errors. diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index bf32d3e549b480..fd6422cc8c06c5 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -919,9 +919,9 @@ Let's try it. We create two instances of ``cell``, and let them point to each other, and finally follow the pointer chain a few times:: >>> c1 = cell() - >>> c1.name = "foo" + >>> c1.name = b"foo" >>> c2 = cell() - >>> c2.name = "bar" + >>> c2.name = b"bar" >>> c1.next = pointer(c2) >>> c2.next = pointer(c1) >>> p = c1 @@ -2508,7 +2508,7 @@ other data types containing pointer type fields. Arrays and pointers ^^^^^^^^^^^^^^^^^^^ -.. class:: Array(\*args) +.. class:: Array(*args) Abstract base class for arrays. diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 3684d54d4ecade..efbece437af2dd 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -112,14 +112,15 @@ The module :mod:`curses` defines the following functions: .. function:: color_content(color_number) Return the intensity of the red, green, and blue (RGB) components in the color - *color_number*, which must be between ``0`` and :const:`COLORS`. Return a 3-tuple, + *color_number*, which must be between ``0`` and ``COLORS - 1``. Return a 3-tuple, containing the R,G,B values for the given color, which will be between ``0`` (no component) and ``1000`` (maximum amount of component). -.. function:: color_pair(color_number) +.. function:: color_pair(pair_number) - Return the attribute value for displaying text in the specified color. This + Return the attribute value for displaying text in the specified color pair. + Only the first 256 color pairs are supported. This attribute value can be combined with :const:`A_STANDOUT`, :const:`A_REVERSE`, and the other :const:`A_\*` attributes. :func:`pair_number` is the counterpart to this function. @@ -219,11 +220,15 @@ The module :mod:`curses` defines the following functions: multiple devices, and *x*, *y*, *z* are the event's coordinates. (*z* is currently unused.) *bstate* is an integer value whose bits will be set to indicate the type of event, and will be the bitwise OR of one or more of the - following constants, where *n* is the button number from 1 to 4: + following constants, where *n* is the button number from 1 to 5: :const:`BUTTONn_PRESSED`, :const:`BUTTONn_RELEASED`, :const:`BUTTONn_CLICKED`, :const:`BUTTONn_DOUBLE_CLICKED`, :const:`BUTTONn_TRIPLE_CLICKED`, :const:`BUTTON_SHIFT`, :const:`BUTTON_CTRL`, :const:`BUTTON_ALT`. + .. versionchanged:: 3.10 + The ``BUTTON5_*`` constants are now exposed if they are provided by the + underlying curses library. + .. function:: getsyx() @@ -287,7 +292,7 @@ The module :mod:`curses` defines the following functions: Change the definition of a color, taking the number of the color to be changed followed by three RGB values (for the amounts of red, green, and blue components). The value of *color_number* must be between ``0`` and - :const:`COLORS`. Each of *r*, *g*, *b*, must be a value between ``0`` and + `COLORS - 1`. Each of *r*, *g*, *b*, must be a value between ``0`` and ``1000``. When :func:`init_color` is used, all occurrences of that color on the screen immediately change to the new definition. This function is a no-op on most terminals; it is active only if :func:`can_change_color` returns ``True``. @@ -300,7 +305,8 @@ The module :mod:`curses` defines the following functions: color number. The value of *pair_number* must be between ``1`` and ``COLOR_PAIRS - 1`` (the ``0`` color pair is wired to white on black and cannot be changed). The value of *fg* and *bg* arguments must be between ``0`` and - :const:`COLORS`. If the color-pair was previously initialized, the screen is + ``COLORS - 1``, or, after calling :func:`use_default_colors`, ``-1``. + If the color-pair was previously initialized, the screen is refreshed and all occurrences of that color-pair are changed to the new definition. @@ -450,7 +456,7 @@ The module :mod:`curses` defines the following functions: .. function:: pair_content(pair_number) Return a tuple ``(fg, bg)`` containing the colors for the requested color pair. - The value of *pair_number* must be between ``1`` and ``COLOR_PAIRS - 1``. + The value of *pair_number* must be between ``0`` and ``COLOR_PAIRS - 1``. .. function:: pair_number(attr) @@ -909,6 +915,9 @@ the following methods and attributes: determining what subset of the screen windows enclose the location of a mouse event. + .. versionchanged:: 3.10 + Previously it returned ``1`` or ``0`` instead of ``True`` or ``False``. + .. attribute:: window.encoding diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index e706f7fcc566d8..64540b3e963b52 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -33,7 +33,7 @@ using :pep:`526` type annotations. For example this code:: Will add, among other things, a :meth:`__init__` that looks like:: - def __init__(self, name: str, unit_price: float, quantity_on_hand: int=0): + def __init__(self, name: str, unit_price: float, quantity_on_hand: int = 0): self.name = name self.unit_price = unit_price self.quantity_on_hand = quantity_on_hand @@ -43,10 +43,10 @@ directly specified in the ``InventoryItem`` definition shown above. .. versionadded:: 3.7 -Module-level decorators, classes, and functions ------------------------------------------------ +Module contents +--------------- -.. decorator:: dataclass(*, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) +.. decorator:: dataclass(*, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False) This function is a :term:`decorator` that is used to add generated :term:`special method`\s to classes, as described below. @@ -79,7 +79,7 @@ Module-level decorators, classes, and functions class C: ... - @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) + @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False) class C: ... @@ -136,7 +136,7 @@ Module-level decorators, classes, and functions attribute ``__hash__ = None`` has a specific meaning to Python, as described in the :meth:`__hash__` documentation. - If :meth:`__hash__` is not explicit defined, or if it is set to ``None``, + If :meth:`__hash__` is not explicitly defined, or if it is set to ``None``, then :func:`dataclass` *may* add an implicit :meth:`__hash__` method. Although not recommended, you can force :func:`dataclass` to create a :meth:`__hash__` method with ``unsafe_hash=True``. This might be the case @@ -161,6 +161,33 @@ Module-level decorators, classes, and functions :meth:`__setattr__` or :meth:`__delattr__` is defined in the class, then :exc:`TypeError` is raised. See the discussion below. + - ``match_args``: If true (the default is ``True``), the + ``__match_args__`` tuple will be created from the list of + parameters to the generated :meth:`__init__` method (even if + :meth:`__init__` is not generated, see above). If false, or if + ``__match_args__`` is already defined in the class, then + ``__match_args__`` will not be generated. + + .. versionadded:: 3.10 + + - ``kw_only``: If true (the default value is ``False``), then all + fields will be marked as keyword-only. If a field is marked as + keyword-only, then the only affect is that the :meth:`__init__` + parameter generated from a keyword-only field must be specified + with a keyword when :meth:`__init__` is called. There is no + effect on any other aspect of dataclasses. See the + :term:`parameter` glossary entry for details. Also see the + :const:`KW_ONLY` section. + + .. versionadded:: 3.10 + + - ``slots``: If true (the default is ``False``), :attr:`__slots__` attribute + will be generated and new class will be returned instead of the original one. + If :attr:`__slots__` is already defined in the class, then :exc:`TypeError` + is raised. + + .. versionadded:: 3.10 + ``field``\s may optionally specify a default value, using normal Python syntax:: @@ -178,7 +205,7 @@ Module-level decorators, classes, and functions follows a field with a default value. This is true either when this occurs in a single class, or as a result of class inheritance. -.. function:: field(*, default=MISSING, default_factory=MISSING, repr=True, hash=None, init=True, compare=True, metadata=None) +.. function:: field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, hash=None, compare=True, metadata=None, kw_only=MISSING): For common and simple use cases, no other functionality is required. There are, however, some dataclass features that @@ -193,10 +220,10 @@ Module-level decorators, classes, and functions c = C() c.mylist += [1, 2, 3] - As shown above, the ``MISSING`` value is a sentinel object used to + As shown above, the :const:`MISSING` value is a sentinel object used to detect if the ``default`` and ``default_factory`` parameters are provided. This sentinel is used because ``None`` is a valid value - for ``default``. No code should directly use the ``MISSING`` + for ``default``. No code should directly use the :const:`MISSING` value. The parameters to :func:`field` are: @@ -217,10 +244,6 @@ Module-level decorators, classes, and functions - ``repr``: If true (the default), this field is included in the string returned by the generated :meth:`__repr__` method. - - ``compare``: If true (the default), this field is included in the - generated equality and comparison methods (:meth:`__eq__`, - :meth:`__gt__`, et al.). - - ``hash``: This can be a bool or ``None``. If true, this field is included in the generated :meth:`__hash__` method. If ``None`` (the default), use the value of ``compare``: this would normally be @@ -234,6 +257,10 @@ Module-level decorators, classes, and functions fields that contribute to the type's hash value. Even if a field is excluded from the hash, it will still be used for comparisons. + - ``compare``: If true (the default), this field is included in the + generated equality and comparison methods (:meth:`__eq__`, + :meth:`__gt__`, et al.). + - ``metadata``: This can be a mapping or None. None is treated as an empty dict. This value is wrapped in :func:`~types.MappingProxyType` to make it read-only, and exposed @@ -242,6 +269,12 @@ Module-level decorators, classes, and functions Multiple third-parties can each have their own key, to use as a namespace in the metadata. + - ``kw_only``: If true, this field will be marked as keyword-only. + This is used when the generated :meth:`__init__` method's + parameters are computed. + + .. versionadded:: 3.10 + If the default value of a field is specified by a call to :func:`field()`, then the class attribute for this field will be replaced by the specified ``default`` value. If no ``default`` is @@ -274,8 +307,8 @@ Module-level decorators, classes, and functions - ``type``: The type of the field. - ``default``, ``default_factory``, ``init``, ``repr``, ``hash``, - ``compare``, and ``metadata`` have the identical meaning and - values as they do in the :func:`field` declaration. + ``compare``, ``metadata``, and ``kw_only`` have the identical + meaning and values as they do in the :func:`field` declaration. Other attributes may exist, but they are private and must not be inspected or relied on. @@ -325,7 +358,7 @@ Module-level decorators, classes, and functions Raises :exc:`TypeError` if ``instance`` is not a dataclass instance. -.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) +.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False) Creates a new dataclass with name ``cls_name``, fields as defined in ``fields``, base classes as given in ``bases``, and initialized @@ -333,8 +366,9 @@ Module-level decorators, classes, and functions iterable whose elements are each either ``name``, ``(name, type)``, or ``(name, type, Field)``. If just ``name`` is supplied, ``typing.Any`` is used for ``type``. The values of ``init``, - ``repr``, ``eq``, ``order``, ``unsafe_hash``, and ``frozen`` have - the same meaning as they do in :func:`dataclass`. + ``repr``, ``eq``, ``order``, ``unsafe_hash``, ``frozen``, + ``match_args``, ``kw_only``, and ``slots`` have the same meaning as + they do in :func:`dataclass`. This function is not strictly required, because any Python mechanism for creating a new class with ``__annotations__`` can @@ -399,6 +433,41 @@ Module-level decorators, classes, and functions def is_dataclass_instance(obj): return is_dataclass(obj) and not isinstance(obj, type) +.. data:: MISSING + + A sentinel value signifying a missing default or default_factory. + +.. data:: KW_ONLY + + A sentinel value used as a type annotation. Any fields after a + pseudo-field with the type of :const:`KW_ONLY` are marked as + keyword-only fields. Note that a pseudo-field of type + :const:`KW_ONLY` is otherwise completely ignored. This includes the + name of such a field. By convention, a name of ``_`` is used for a + :const:`KW_ONLY` field. Keyword-only fields signify + :meth:`__init__` parameters that must be specified as keywords when + the class is instantiated. + + In this example, the fields ``y`` and ``z`` will be marked as keyword-only fields:: + + @dataclass + class Point: + x: float + _: KW_ONLY + y: float + z: float + + p = Point(0, y=1.5, z=2.0) + + In a single dataclass, it is an error to specify more than one + field whose type is :const:`KW_ONLY`. + +.. exception:: FrozenInstanceError + + Raised when an implicitly defined :meth:`__setattr__` or + :meth:`__delattr__` is called on a dataclass which was defined with + ``frozen=True``. It is a subclass of :exc:`AttributeError`. + Post-init processing -------------------- @@ -511,6 +580,42 @@ The generated :meth:`__init__` method for ``C`` will look like:: def __init__(self, x: int = 15, y: int = 0, z: int = 10): +Re-ordering of keyword-only parameters in :meth:`__init__` +---------------------------------------------------------- + +After the parameters needed for :meth:`__init__` are computed, any +keyword-only parameters are moved to come after all regular +(non-keyword-only) parameters. This is a requirement of how +keyword-only parameters are implemented in Python: they must come +after non-keyword-only parameters. + +In this example, ``Base.y``, ``Base.w``, and ``D.t`` are keyword-only +fields, and ``Base.x`` and ``D.z`` are regular fields:: + + @dataclass + class Base: + x: Any = 15.0 + _: KW_ONLY + y: int = 0 + w: int = 1 + + @dataclass + class D(Base): + z: int = 10 + t: int = field(kw_only=True, default=0) + +The generated :meth:`__init__` method for ``D`` will look like:: + + def __init__(self, x: Any = 15.0, z: int = 10, *, y: int = 0, w: int = 1, t: int = 0): + +Note that the parameters have been re-ordered from how they appear in +the list of fields: parameters derived from regular fields are +followed by parameters derived from keyword-only fields. + +The relative ordering of keyword-only parameters is maintained in the +re-ordered :meth:`__init__` parameter list. + + Default factory functions ------------------------- @@ -567,14 +672,15 @@ Mutable default values assert D().x is D().x This has the same issue as the original example using class ``C``. - That is, two instances of class ``D`` that do not specify a value for - ``x`` when creating a class instance will share the same copy of - ``x``. Because dataclasses just use normal Python class creation - they also share this behavior. There is no general way for Data - Classes to detect this condition. Instead, dataclasses will raise a - :exc:`TypeError` if it detects a default parameter of type ``list``, - ``dict``, or ``set``. This is a partial solution, but it does protect - against many common errors. + That is, two instances of class ``D`` that do not specify a value + for ``x`` when creating a class instance will share the same copy + of ``x``. Because dataclasses just use normal Python class + creation they also share this behavior. There is no general way + for Data Classes to detect this condition. Instead, the + :func:`dataclass` decorator will raise a :exc:`TypeError` if it + detects a default parameter of type ``list``, ``dict``, or ``set``. + This is a partial solution, but it does protect against many common + errors. Using default factory functions is a way to create new instances of mutable types as default values for fields:: @@ -584,12 +690,3 @@ Mutable default values x: list = field(default_factory=list) assert D().x is not D().x - -Exceptions ----------- - -.. exception:: FrozenInstanceError - - Raised when an implicitly defined :meth:`__setattr__` or - :meth:`__delattr__` is called on a dataclass which was defined with - ``frozen=True``. diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 508bc88e7f4b8a..dae0dd7aa55898 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1219,7 +1219,7 @@ Instance methods: .. method:: datetime.replace(year=self.year, month=self.month, day=self.day, \ hour=self.hour, minute=self.minute, second=self.second, microsecond=self.microsecond, \ - tzinfo=self.tzinfo, * fold=0) + tzinfo=self.tzinfo, *, fold=0) Return a datetime with the same attributes, except for those attributes given new values by whichever keyword arguments are specified. Note that @@ -1783,7 +1783,7 @@ Other constructor: Instance methods: .. method:: time.replace(hour=self.hour, minute=self.minute, second=self.second, \ - microsecond=self.microsecond, tzinfo=self.tzinfo, * fold=0) + microsecond=self.microsecond, tzinfo=self.tzinfo, *, fold=0) Return a :class:`.time` with the same value, except for those attributes given new values by whichever keyword arguments are specified. Note that diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index e194649e30d85c..e759c5cf23b9e7 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -1484,7 +1484,8 @@ are also included in the pure Python version for compatibility. .. data:: HAVE_CONTEXTVAR - The default value is ``True``. If Python is compiled ``--without-decimal-contextvar``, + The default value is ``True``. If Python is :option:`configured using + the --without-decimal-contextvar option <--without-decimal-contextvar>`, the C version uses a thread-local rather than a coroutine-local context and the value is ``False``. This is slightly faster in some nested context scenarios. diff --git a/Doc/library/devmode.rst b/Doc/library/devmode.rst index d5a40cdeeac5ef..44e7d4f541d817 100644 --- a/Doc/library/devmode.rst +++ b/Doc/library/devmode.rst @@ -13,6 +13,8 @@ detected. It can be enabled using the :option:`-X dev <-X>` command line option or by setting the :envvar:`PYTHONDEVMODE` environment variable to ``1``. +See also :ref:`Python debug build `. + Effects of the Python Development Mode ====================================== @@ -93,6 +95,9 @@ The Python Development Mode does not prevent the :option:`-O` command line option from removing :keyword:`assert` statements nor from setting :const:`__debug__` to ``False``. +The Python Development Mode can only be enabled at the Python startup. Its +value can be read from :data:`sys.flags.dev_mode `. + .. versionchanged:: 3.8 The :class:`io.IOBase` destructor now logs ``close()`` exceptions. diff --git a/Doc/library/dialog.rst b/Doc/library/dialog.rst index dc82a974ce095d..53f98c1018988f 100644 --- a/Doc/library/dialog.rst +++ b/Doc/library/dialog.rst @@ -198,7 +198,7 @@ These do not emulate the native look-and-feel of the platform. A subclass of FileDialog that creates a dialog window for selecting a destination file. - .. method:: ok_command() + .. method:: ok_command() Test whether or not the selection points to a valid file that is not a directory. Confirmation is required if an already existing file is diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index d0307bd8e9f3d0..0491a9d4a072bf 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -346,7 +346,7 @@ The Python compiler currently generates the following bytecode instructions. .. opcode:: ROT_FOUR - Lifts second, third and forth stack items one position up, moves top down + Lifts second, third and fourth stack items one position up, moves top down to position four. .. versionadded:: 3.8 @@ -708,7 +708,8 @@ iterations of the loop. .. opcode:: RERAISE - Re-raises the exception currently on top of the stack. + Re-raises the exception currently on top of the stack. If oparg is non-zero, + restores ``f_lasti`` of the current frame to its value when the exception was raised. .. versionadded:: 3.9 @@ -741,7 +742,7 @@ iterations of the loop. This opcode performs several operations before a with block starts. First, it loads :meth:`~object.__exit__` from the context manager and pushes it onto - the stack for later use by :opcode:`WITH_CLEANUP_START`. Then, + the stack for later use by :opcode:`WITH_EXCEPT_START`. Then, :meth:`~object.__enter__` is called, and a finally block pointing to *delta* is pushed. Finally, the result of calling the ``__enter__()`` method is pushed onto the stack. The next opcode will either ignore it (:opcode:`POP_TOP`), or @@ -751,6 +752,49 @@ iterations of the loop. .. versionadded:: 3.2 +.. opcode:: COPY_DICT_WITHOUT_KEYS + + TOS is a tuple of mapping keys, and TOS1 is the match subject. Replace TOS + with a :class:`dict` formed from the items of TOS1, but without any of the + keys in TOS. + + .. versionadded:: 3.10 + + +.. opcode:: GET_LEN + + Push ``len(TOS)`` onto the stack. + + .. versionadded:: 3.10 + + +.. opcode:: MATCH_MAPPING + + If TOS is an instance of :class:`collections.abc.Mapping`, push ``True`` onto + the stack. Otherwise, push ``False``. + + .. versionadded:: 3.10 + + +.. opcode:: MATCH_SEQUENCE + + If TOS is an instance of :class:`collections.abc.Sequence` and is *not* an + instance of :class:`str`/:class:`bytes`/:class:`bytearray`, push ``True`` + onto the stack. Otherwise, push ``False``. + + .. versionadded:: 3.10 + + +.. opcode:: MATCH_KEYS + + TOS is a tuple of mapping keys, and TOS1 is the match subject. If TOS1 + contains all of the keys in TOS, push a :class:`tuple` containing the + corresponding values, followed by ``True``. Otherwise, push ``None``, + followed by ``False``. + + .. versionadded:: 3.10 + + All of the following opcodes use their arguments. .. opcode:: STORE_NAME (namei) @@ -861,7 +905,7 @@ All of the following opcodes use their arguments. .. opcode:: LIST_TO_TUPLE - Pops a list from the stack and pushes a tuple containing the same values. + Pops a list from the stack and pushes a tuple containing the same values. .. versionadded:: 3.9 @@ -889,7 +933,7 @@ All of the following opcodes use their arguments. .. opcode:: DICT_MERGE - Like :opcode:`DICT_UPDATE` but raises an exception for duplicate keys. + Like :opcode:`DICT_UPDATE` but raises an exception for duplicate keys. .. versionadded:: 3.9 @@ -907,14 +951,14 @@ All of the following opcodes use their arguments. .. opcode:: IS_OP (invert) - Performs ``is`` comparison, or ``is not`` if ``invert`` is 1. + Performs ``is`` comparison, or ``is not`` if ``invert`` is 1. .. versionadded:: 3.9 .. opcode:: CONTAINS_OP (invert) - Performs ``in`` comparison, or ``not in`` if ``invert`` is 1. + Performs ``in`` comparison, or ``not in`` if ``invert`` is 1. .. versionadded:: 3.9 @@ -955,8 +999,8 @@ All of the following opcodes use their arguments. .. opcode:: JUMP_IF_NOT_EXC_MATCH (target) - Tests whether the second value on the stack is an exception matching TOS, - and jumps if it is not. Pops two values from the stack. + Tests whether the second value on the stack is an exception matching TOS, + and jumps if it is not. Pops two values from the stack. .. versionadded:: 3.9 @@ -1144,11 +1188,13 @@ All of the following opcodes use their arguments. * ``0x01`` a tuple of default values for positional-only and positional-or-keyword parameters in positional order * ``0x02`` a dictionary of keyword-only parameters' default values - * ``0x04`` an annotation dictionary + * ``0x04`` a tuple of strings containing parameters' annotations * ``0x08`` a tuple containing cells for free variables, making a closure * the code associated with the function (at TOS1) * the :term:`qualified name` of the function (at TOS) + .. versionchanged:: 3.10 + Flag value ``0x04`` is a tuple of strings instead of dictionary .. opcode:: BUILD_SLICE (argc) @@ -1189,6 +1235,36 @@ All of the following opcodes use their arguments. .. versionadded:: 3.6 +.. opcode:: MATCH_CLASS (count) + + TOS is a tuple of keyword attribute names, TOS1 is the class being matched + against, and TOS2 is the match subject. *count* is the number of positional + sub-patterns. + + Pop TOS. If TOS2 is an instance of TOS1 and has the positional and keyword + attributes required by *count* and TOS, set TOS to ``True`` and TOS1 to a + tuple of extracted attributes. Otherwise, set TOS to ``False``. + + .. versionadded:: 3.10 + +.. opcode:: GEN_START (kind) + + Pops TOS. If TOS was not ``None``, raises an exception. The ``kind`` + operand corresponds to the type of generator or coroutine and determines + the error message. The legal kinds are 0 for generator, 1 for coroutine, + and 2 for async generator. + + .. versionadded:: 3.10 + + +.. opcode:: ROT_N (count) + + Lift the top *count* stack items one position up, and move TOS down to + position *count*. + + .. versionadded:: 3.10 + + .. opcode:: HAVE_ARGUMENT This is not really an opcode. It identifies the dividing line between diff --git a/Doc/library/distutils.rst b/Doc/library/distutils.rst index 62abc85ac397a4..31c4ae5b23906b 100644 --- a/Doc/library/distutils.rst +++ b/Doc/library/distutils.rst @@ -9,6 +9,11 @@ -------------- +:mod:`distutils` is deprecated with removal planned for Python 3.12. +See the :ref:`What's New ` entry for more information. + +-------------- + The :mod:`distutils` package provides support for building and installing additional modules into a Python installation. The new modules may be either 100%-pure Python, or may be extension modules written in C, or may be diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index 745b3a6a3ad42a..c68e773b1688aa 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -23,7 +23,7 @@ policy :attr:`~email.policy.Compat32`. If you are going to use another policy, you should be using the :class:`~email.message.EmailMessage` class instead. An email message consists of *headers* and a *payload*. Headers must be -:rfc:`5233` style names and values, where the field name and value are +:rfc:`5322` style names and values, where the field name and value are separated by a colon. The colon is not part of either the field name or the field value. The payload may be a simple text message, or a binary object, or a structured sequence of sub-messages each with their own set of headers and diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst index e09c7c0e402bbc..918fc55677e723 100644 --- a/Doc/library/email.contentmanager.rst +++ b/Doc/library/email.contentmanager.rst @@ -116,7 +116,7 @@ Currently the email package provides only one concrete content manager, decoding the payload to unicode. The default error handler is ``replace``. - .. method:: set_content(msg, <'str'>, subtype="plain", charset='utf-8' \ + .. method:: set_content(msg, <'str'>, subtype="plain", charset='utf-8', \ cte=None, \ disposition=None, filename=None, cid=None, \ params=None, headers=None) diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index 9376da2b8d39ce..3e1d97a03264b2 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -289,7 +289,7 @@ variant, :attr:`~.BaseHeader.max_count` is set to 1. A :class:`ParameterizedMIMEHeader` class that handles the :mailheader:`Content-Disposition` header. - .. attribute:: content-disposition + .. attribute:: content_disposition ``inline`` and ``attachment`` are the only valid values in common use. diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index 8e7076259810f5..bf53b9520fc723 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -210,7 +210,7 @@ added matters. To illustrate:: :meth:`register_defect` method. - .. attribute:: mangle_from\_ + .. attribute:: mangle_from_ If :const:`True`, lines starting with *"From "* in the body are escaped by putting a ``>`` in front of them. This parameter is used when diff --git a/Doc/library/email.rst b/Doc/library/email.rst index fae99cf3e6abbe..5eebcd9e896d93 100644 --- a/Doc/library/email.rst +++ b/Doc/library/email.rst @@ -16,7 +16,7 @@ The :mod:`email` package is a library for managing email messages. It is specifically *not* designed to do any sending of email messages to SMTP (:rfc:`2821`), NNTP, or other servers; those are functions of modules such as :mod:`smtplib` and :mod:`nntplib`. The :mod:`email` package attempts to be as -RFC-compliant as possible, supporting :rfc:`5233` and :rfc:`6532`, as well as +RFC-compliant as possible, supporting :rfc:`5322` and :rfc:`6532`, as well as such MIME-related RFCs as :rfc:`2045`, :rfc:`2046`, :rfc:`2047`, :rfc:`2183`, and :rfc:`2231`. diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index a5221250c40486..fa1b42cf484094 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -48,7 +48,7 @@ The simplest possible invocation is:: This invocation will install ``pip`` if it is not already installed, but otherwise does nothing. To ensure the installed version of ``pip`` -is at least as recent as the one bundled with ``ensurepip``, pass the +is at least as recent as the one available in ``ensurepip``, pass the ``--upgrade`` option:: python -m ensurepip --upgrade @@ -86,7 +86,7 @@ Module API .. function:: version() - Returns a string specifying the bundled version of pip that will be + Returns a string specifying the available version of pip that will be installed when bootstrapping an environment. .. function:: bootstrap(root=None, upgrade=False, user=False, \ @@ -100,7 +100,7 @@ Module API for the current environment. *upgrade* indicates whether or not to upgrade an existing installation - of an earlier version of ``pip`` to the bundled version. + of an earlier version of ``pip`` to the available version. *user* indicates whether to use the user scheme rather than installing globally. diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 118002bef19f85..b5f9c2f08b1873 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -13,1248 +13,617 @@ **Source code:** :source:`Lib/enum.py` ----------------- +.. sidebar:: Important -An enumeration is a set of symbolic names (members) bound to unique, -constant values. Within an enumeration, the members can be compared -by identity, and the enumeration itself can be iterated over. + This page contains the API reference information. For tutorial + information and discussion of more advanced topics, see -.. note:: Case of Enum Members + * :ref:`Basic Tutorial ` + * :ref:`Advanced Tutorial ` + * :ref:`Enum Cookbook ` - Because Enums are used to represent constants we recommend using - UPPER_CASE names for enum members, and will be using that style - in our examples. +---------------- +An enumeration: -Module Contents ---------------- +* is a set of symbolic names (members) bound to unique values +* can be iterated over to return its members in definition order +* uses :meth:`call` syntax to return members by value +* uses :meth:`index` syntax to return members by name -This module defines four enumeration classes that can be used to define unique -sets of names and values: :class:`Enum`, :class:`IntEnum`, :class:`Flag`, and -:class:`IntFlag`. It also defines one decorator, :func:`unique`, and one -helper, :class:`auto`. +Enumerations are created either by using the :keyword:`class` syntax, or by +using function-call syntax:: -.. class:: Enum + >>> from enum import Enum - Base class for creating enumerated constants. See section - `Functional API`_ for an alternate construction syntax. + >>> # class syntax + >>> class Color(Enum): + ... RED = 1 + ... GREEN = 2 + ... BLUE = 3 -.. class:: IntEnum + >>> # functional syntax + >>> Color = Enum('Color', ['RED', 'GREEN', 'BLUE']) - Base class for creating enumerated constants that are also - subclasses of :class:`int`. +Even though we can use the :keyword:`class` syntax to create Enums, Enums +are not normal Python classes. See +:ref:`How are Enums different? ` for more details. -.. class:: StrEnum +.. note:: Nomenclature - Base class for creating enumerated constants that are also - subclasses of :class:`str`. + - The class :class:`Color` is an *enumeration* (or *enum*) + - The attributes :attr:`Color.RED`, :attr:`Color.GREEN`, etc., are + *enumeration members* (or *enum members*) and are functionally constants. + - The enum members have *names* and *values* (the name of + :attr:`Color.RED` is ``RED``, the value of :attr:`Color.BLUE` is + ``3``, etc.) -.. class:: IntFlag - Base class for creating enumerated constants that can be combined using - the bitwise operators without losing their :class:`IntFlag` membership. - :class:`IntFlag` members are also subclasses of :class:`int`. - -.. class:: Flag +Module Contents +--------------- - Base class for creating enumerated constants that can be combined using - the bitwise operations without losing their :class:`Flag` membership. + :class:`EnumType` -.. function:: unique - :noindex: + The ``type`` for Enum and its subclasses. - Enum class decorator that ensures only one name is bound to any one value. + :class:`Enum` -.. class:: auto + Base class for creating enumerated constants. - Instances are replaced with an appropriate value for Enum members. By default, the initial value starts at 1. - -.. versionadded:: 3.6 ``Flag``, ``IntFlag``, ``auto`` + :class:`IntEnum` + Base class for creating enumerated constants that are also + subclasses of :class:`int`. -Creating an Enum ----------------- + :class:`StrEnum` -Enumerations are created using the :keyword:`class` syntax, which makes them -easy to read and write. An alternative creation method is described in -`Functional API`_. To define an enumeration, subclass :class:`Enum` as -follows:: + Base class for creating enumerated constants that are also + subclasses of :class:`str`. - >>> from enum import Enum - >>> class Color(Enum): - ... RED = 1 - ... GREEN = 2 - ... BLUE = 3 - ... + :class:`Flag` -.. note:: Enum member values + Base class for creating enumerated constants that can be combined using + the bitwise operations without losing their :class:`Flag` membership. - Member values can be anything: :class:`int`, :class:`str`, etc.. If - the exact value is unimportant you may use :class:`auto` instances and an - appropriate value will be chosen for you. Care must be taken if you mix - :class:`auto` with other values. + :class:`IntFlag` -.. note:: Nomenclature + Base class for creating enumerated constants that can be combined using + the bitwise operators without losing their :class:`IntFlag` membership. + :class:`IntFlag` members are also subclasses of :class:`int`. - - The class :class:`Color` is an *enumeration* (or *enum*) - - The attributes :attr:`Color.RED`, :attr:`Color.GREEN`, etc., are - *enumeration members* (or *enum members*) and are functionally constants. - - The enum members have *names* and *values* (the name of - :attr:`Color.RED` is ``RED``, the value of :attr:`Color.BLUE` is - ``3``, etc.) + :class:`FlagBoundary` -.. note:: + An enumeration with the values ``STRICT``, ``CONFORM``, ``EJECT``, and + ``KEEP`` which allows for more fine-grained control over how invalid values + are dealt with in an enumeration. - Even though we use the :keyword:`class` syntax to create Enums, Enums - are not normal Python classes. See `How are Enums different?`_ for - more details. + :class:`auto` -Enumeration members have human readable string representations:: + Instances are replaced with an appropriate value for Enum members. + :class:`StrEnum` defaults to the lower-cased version of the member name, + while other Enums default to 1 and increase from there. - >>> print(Color.RED) - Color.RED + :func:`global_enum` -...while their ``repr`` has more information:: + :class:`Enum` class decorator to apply the appropriate global `__repr__`, + and export its members into the global name space. - >>> print(repr(Color.RED)) - + :func:`property` -The *type* of an enumeration member is the enumeration it belongs to:: + Allows :class:`Enum` members to have attributes without conflicting with + other members' names. - >>> type(Color.RED) - - >>> isinstance(Color.GREEN, Color) - True + :func:`unique` -Enum members also have a property that contains just their item name:: + Enum class decorator that ensures only one name is bound to any one value. - >>> print(Color.RED.name) - RED -Enumerations support iteration, in definition order:: +.. versionadded:: 3.6 ``Flag``, ``IntFlag``, ``auto`` +.. versionadded:: 3.10 ``StrEnum`` - >>> class Shake(Enum): - ... VANILLA = 7 - ... CHOCOLATE = 4 - ... COOKIES = 9 - ... MINT = 3 - ... - >>> for shake in Shake: - ... print(shake) - ... - Shake.VANILLA - Shake.CHOCOLATE - Shake.COOKIES - Shake.MINT -Enumeration members are hashable, so they can be used in dictionaries and sets:: +Data Types +---------- - >>> apples = {} - >>> apples[Color.RED] = 'red delicious' - >>> apples[Color.GREEN] = 'granny smith' - >>> apples == {Color.RED: 'red delicious', Color.GREEN: 'granny smith'} - True +.. class:: EnumType -Programmatic access to enumeration members and their attributes ---------------------------------------------------------------- + *EnumType* is the :term:`metaclass` for *enum* enumerations. It is possible + to subclass *EnumType* -- see :ref:`Subclassing EnumType ` + for details. -Sometimes it's useful to access members in enumerations programmatically (i.e. -situations where ``Color.RED`` won't do because the exact color is not known -at program-writing time). ``Enum`` allows such access:: + .. method:: EnumType.__contains__(cls, member) - >>> Color(1) - - >>> Color(3) - + Returns ``True`` if member belongs to the ``cls``:: -If you want to access enum members by *name*, use item access:: + >>> some_var = Color.RED + >>> some_var in Color + True - >>> Color['RED'] - - >>> Color['GREEN'] - + .. note:: -If you have an enum member and need its :attr:`name` or :attr:`value`:: + In Python 3.12 it will be possible to check for member values and not + just members; until then, a ``TypeError`` will be raised if a + non-Enum-member is used in a containment check. - >>> member = Color.RED - >>> member.name - 'RED' - >>> member.value - 1 + .. method:: EnumType.__dir__(cls) + Returns ``['__class__', '__doc__', '__members__', '__module__']`` and the + names of the members in *cls*:: -Duplicating enum members and values ------------------------------------ + >>> dir(Color) + ['BLUE', 'GREEN', 'RED', '__class__', '__doc__', '__members__', '__module__'] -Having two enum members with the same name is invalid:: + .. method:: EnumType.__getattr__(cls, name) - >>> class Shape(Enum): - ... SQUARE = 2 - ... SQUARE = 3 - ... - Traceback (most recent call last): - ... - TypeError: Attempted to reuse key: 'SQUARE' + Returns the Enum member in *cls* matching *name*, or raises an :exc:`AttributeError`:: -However, two enum members are allowed to have the same value. Given two members -A and B with the same value (and A defined first), B is an alias to A. By-value -lookup of the value of A and B will return A. By-name lookup of B will also -return A:: + >>> Color.GREEN + Color.GREEN - >>> class Shape(Enum): - ... SQUARE = 2 - ... DIAMOND = 1 - ... CIRCLE = 3 - ... ALIAS_FOR_SQUARE = 2 - ... - >>> Shape.SQUARE - - >>> Shape.ALIAS_FOR_SQUARE - - >>> Shape(2) - + .. method:: EnumType.__getitem__(cls, name) -.. note:: + Returns the Enum member in *cls* matching *name*, or raises an :exc:`KeyError`:: - Attempting to create a member with the same name as an already - defined attribute (another member, a method, etc.) or attempting to create - an attribute with the same name as a member is not allowed. + >>> Color['BLUE'] + Color.BLUE + .. method:: EnumType.__iter__(cls) -Ensuring unique enumeration values ----------------------------------- + Returns each member in *cls* in definition order:: -By default, enumerations allow multiple names as aliases for the same value. -When this behavior isn't desired, the following decorator can be used to -ensure each value is used only once in the enumeration: + >>> list(Color) + [Color.RED, Color.GREEN, Color.BLUE] -.. decorator:: unique + .. method:: EnumType.__len__(cls) -A :keyword:`class` decorator specifically for enumerations. It searches an -enumeration's :attr:`__members__` gathering any aliases it finds; if any are -found :exc:`ValueError` is raised with the details:: - - >>> from enum import Enum, unique - >>> @unique - ... class Mistake(Enum): - ... ONE = 1 - ... TWO = 2 - ... THREE = 3 - ... FOUR = 3 - ... - Traceback (most recent call last): - ... - ValueError: duplicate values found in : FOUR -> THREE - - -Using automatic values ----------------------- - -If the exact value is unimportant you can use :class:`auto`:: - - >>> from enum import Enum, auto - >>> class Color(Enum): - ... RED = auto() - ... BLUE = auto() - ... GREEN = auto() - ... - >>> list(Color) - [, , ] - -The values are chosen by :func:`_generate_next_value_`, which can be -overridden:: - - >>> class AutoName(Enum): - ... def _generate_next_value_(name, start, count, last_values): - ... return name - ... - >>> class Ordinal(AutoName): - ... NORTH = auto() - ... SOUTH = auto() - ... EAST = auto() - ... WEST = auto() - ... - >>> list(Ordinal) - [, , , ] + Returns the number of member in *cls*:: -.. note:: + >>> len(Color) + 3 - The goal of the default :meth:`_generate_next_value_` methods is to provide - the next :class:`int` in sequence with the last :class:`int` provided, but - the way it does this is an implementation detail and may change. + .. method:: EnumType.__reversed__(cls) -.. note:: + Returns each member in *cls* in reverse definition order:: - The :meth:`_generate_next_value_` method must be defined before any members. + >>> list(reversed(Color)) + [Color.BLUE, Color.GREEN, Color.RED] -Iteration ---------- -Iterating over the members of an enum does not provide the aliases:: - - >>> list(Shape) - [, , ] - -The special attribute ``__members__`` is a read-only ordered mapping of names -to members. It includes all names defined in the enumeration, including the -aliases:: - - >>> for name, member in Shape.__members__.items(): - ... name, member - ... - ('SQUARE', ) - ('DIAMOND', ) - ('CIRCLE', ) - ('ALIAS_FOR_SQUARE', ) +.. class:: Enum -The ``__members__`` attribute can be used for detailed programmatic access to -the enumeration members. For example, finding all the aliases:: + *Enum* is the base class for all *enum* enumerations. - >>> [name for name, member in Shape.__members__.items() if member.name != name] - ['ALIAS_FOR_SQUARE'] + .. attribute:: Enum.name + The name used to define the ``Enum`` member:: -Comparisons ------------ + >>> Color.BLUE.name + 'BLUE' -Enumeration members are compared by identity:: + .. attribute:: Enum.value - >>> Color.RED is Color.RED - True - >>> Color.RED is Color.BLUE - False - >>> Color.RED is not Color.BLUE - True + The value given to the ``Enum`` member:: -Ordered comparisons between enumeration values are *not* supported. Enum -members are not integers (but see `IntEnum`_ below):: - - >>> Color.RED < Color.BLUE - Traceback (most recent call last): - File "", line 1, in - TypeError: '<' not supported between instances of 'Color' and 'Color' - -Equality comparisons are defined though:: - - >>> Color.BLUE == Color.RED - False - >>> Color.BLUE != Color.RED - True - >>> Color.BLUE == Color.BLUE - True + >>> Color.RED.value + 1 -Comparisons against non-enumeration values will always compare not equal -(again, :class:`IntEnum` was explicitly designed to behave differently, see -below):: + .. note:: Enum member values - >>> Color.BLUE == 2 - False + Member values can be anything: :class:`int`, :class:`str`, etc.. If + the exact value is unimportant you may use :class:`auto` instances and an + appropriate value will be chosen for you. Care must be taken if you mix + :class:`auto` with other values. + .. attribute:: Enum._ignore_ -Allowed members and attributes of enumerations ----------------------------------------------- + ``_ignore_`` is only used during creation and is removed from the + enumeration once that is complete. -The examples above use integers for enumeration values. Using integers is -short and handy (and provided by default by the `Functional API`_), but not -strictly enforced. In the vast majority of use-cases, one doesn't care what -the actual value of an enumeration is. But if the value *is* important, -enumerations can have arbitrary values. + ``_ignore_`` is a list of names that will not become members, and whose + names will also be removed from the completed enumeration. See + :ref:`TimePeriod ` for an example. -Enumerations are Python classes, and can have methods and special methods as -usual. If we have this enumeration:: + .. method:: Enum.__call__(cls, value, names=None, \*, module=None, qualname=None, type=None, start=1, boundary=None) - >>> class Mood(Enum): - ... FUNKY = 1 - ... HAPPY = 3 - ... - ... def describe(self): - ... # self is the member here - ... return self.name, self.value - ... - ... def __str__(self): - ... return 'my custom str! {0}'.format(self.value) - ... - ... @classmethod - ... def favorite_mood(cls): - ... # cls here is the enumeration - ... return cls.HAPPY - ... + This method is called in two different ways: -Then:: + * to look up an existing member: - >>> Mood.favorite_mood() - - >>> Mood.HAPPY.describe() - ('HAPPY', 3) - >>> str(Mood.FUNKY) - 'my custom str! 1' - -The rules for what is allowed are as follows: names that start and end with -a single underscore are reserved by enum and cannot be used; all other -attributes defined within an enumeration will become members of this -enumeration, with the exception of special methods (:meth:`__str__`, -:meth:`__add__`, etc.), descriptors (methods are also descriptors), and -variable names listed in :attr:`_ignore_`. - -Note: if your enumeration defines :meth:`__new__` and/or :meth:`__init__` then -any value(s) given to the enum member will be passed into those methods. -See `Planet`_ for an example. - - -Restricted Enum subclassing ---------------------------- - -A new :class:`Enum` class must have one base Enum class, up to one concrete -data type, and as many :class:`object`-based mixin classes as needed. The -order of these base classes is:: - - class EnumName([mix-in, ...,] [data-type,] base-enum): - pass - -Also, subclassing an enumeration is allowed only if the enumeration does not define -any members. So this is forbidden:: - - >>> class MoreColor(Color): - ... PINK = 17 - ... - Traceback (most recent call last): - ... - TypeError: Cannot extend enumerations - -But this is allowed:: - - >>> class Foo(Enum): - ... def some_behavior(self): - ... pass - ... - >>> class Bar(Foo): - ... HAPPY = 1 - ... SAD = 2 - ... - -Allowing subclassing of enums that define members would lead to a violation of -some important invariants of types and instances. On the other hand, it makes -sense to allow sharing some common behavior between a group of enumerations. -(See `OrderedEnum`_ for an example.) - - -Pickling --------- - -Enumerations can be pickled and unpickled:: - - >>> from test.test_enum import Fruit - >>> from pickle import dumps, loads - >>> Fruit.TOMATO is loads(dumps(Fruit.TOMATO)) - True - -The usual restrictions for pickling apply: picklable enums must be defined in -the top level of a module, since unpickling requires them to be importable -from that module. + :cls: The enum class being called. + :value: The value to lookup. + + * to use the ``cls`` enum to create a new enum: + + :cls: The enum class being called. + :value: The name of the new Enum to create. + :names: The names/values of the members for the new Enum. + :module: The name of the module the new Enum is created in. + :qualname: The actual location in the module where this Enum can be found. + :type: A mix-in type for the new Enum. + :start: The first integer value for the Enum (used by :class:`auto`) + :boundary: How to handle out-of-range values from bit operations (:class:`Flag` only) + + .. method:: Enum.__dir__(self) + + Returns ``['__class__', '__doc__', '__module__', 'name', 'value']`` and + any public methods defined on *self.__class__*:: + + >>> from datetime import date + >>> class Weekday(Enum): + ... MONDAY = 1 + ... TUESDAY = 2 + ... WEDNESDAY = 3 + ... THURSDAY = 4 + ... FRIDAY = 5 + ... SATURDAY = 6 + ... SUNDAY = 7 + ... @classmethod + ... def today(cls): + ... print('today is %s' % cls(date.today.isoweekday).naem) + >>> dir(Weekday.SATURDAY) + ['__class__', '__doc__', '__module__', 'name', 'today', 'value'] + + .. method:: Enum._generate_next_value_(name, start, count, last_values) + + :name: The name of the member being defined (e.g. 'RED'). + :start: The start value for the Enum; the default is 1. + :count: The number of members currently defined, not including this one. + :last_values: A list of the previous values. + + A *staticmethod* that is used to determine the next value returned by + :class:`auto`:: + + >>> from enum import auto + >>> class PowersOfThree(Enum): + ... @staticmethod + ... def _generate_next_value_(name, start, count, last_values): + ... return (count + 1) * 3 + ... FIRST = auto() + ... SECOND = auto() + >>> PowersOfThree.SECOND.value + 6 + + .. method:: Enum._missing_(cls, value) + + A *classmethod* for looking up values not found in *cls*. By default it + does nothing, but can be overridden to implement custom search behavior:: + + >>> from enum import StrEnum + >>> class Build(StrEnum): + ... DEBUG = auto() + ... OPTIMIZED = auto() + ... @classmethod + ... def _missing_(cls, value): + ... value = value.lower() + ... for member in cls: + ... if member.value == value: + ... return member + ... return None + >>> Build.DEBUG.value + 'debug' + >>> Build('deBUG') + Build.DEBUG + + .. method:: Enum.__repr__(self) + + Returns the string used for *repr()* calls. By default, returns the + *Enum* name and the member name, but can be overridden:: + + >>> class OldStyle(Enum): + ... RETRO = auto() + ... OLD_SCHOOl = auto() + ... YESTERYEAR = auto() + ... def __repr__(self): + ... cls_name = self.__class__.__name__ + ... return f'<{cls_name}.{self.name}: {self.value}>' + >>> OldStyle.RETRO + + + .. method:: Enum.__str__(self) + + Returns the string used for *str()* calls. By default, returns the + member name, but can be overridden:: + + >>> class OldStyle(Enum): + ... RETRO = auto() + ... OLD_SCHOOl = auto() + ... YESTERYEAR = auto() + ... def __str__(self): + ... cls_name = self.__class__.__name__ + ... return f'{cls_name}.{self.name}' + >>> OldStyle.RETRO + OldStyle.RETRO .. note:: - With pickle protocol version 4 it is possible to easily pickle enums - nested in other classes. - -It is possible to modify how Enum members are pickled/unpickled by defining -:meth:`__reduce_ex__` in the enumeration class. - + Using :class:`auto` with :class:`Enum` results in integers of increasing value, + starting with ``1``. -Functional API --------------- -The :class:`Enum` class is callable, providing the following functional API:: +.. class:: IntEnum - >>> Animal = Enum('Animal', 'ANT BEE CAT DOG') - >>> Animal - - >>> Animal.ANT - - >>> Animal.ANT.value - 1 - >>> list(Animal) - [, , , ] + *IntEnum* is the same as *Enum*, but its members are also integers and can be + used anywhere that an integer can be used. If any integer operation is performed + with an *IntEnum* member, the resulting value loses its enumeration status. + + >>> from enum import IntEnum + >>> class Numbers(IntEnum): + ... ONE = 1 + ... TWO = 2 + ... THREE = 3 + >>> Numbers.THREE + Numbers.THREE + >>> Numbers.ONE + Numbers.TWO + 3 + >>> Numbers.THREE + 5 + 8 + >>> Numbers.THREE == 3 + True -The semantics of this API resemble :class:`~collections.namedtuple`. The first -argument of the call to :class:`Enum` is the name of the enumeration. +.. note:: -The second argument is the *source* of enumeration member names. It can be a -whitespace-separated string of names, a sequence of names, a sequence of -2-tuples with key/value pairs, or a mapping (e.g. dictionary) of names to -values. The last two options enable assigning arbitrary values to -enumerations; the others auto-assign increasing integers starting with 1 (use -the ``start`` parameter to specify a different starting value). A -new class derived from :class:`Enum` is returned. In other words, the above -assignment to :class:`Animal` is equivalent to:: + Using :class:`auto` with :class:`IntEnum` results in integers of increasing value, + starting with ``1``. - >>> class Animal(Enum): - ... ANT = 1 - ... BEE = 2 - ... CAT = 3 - ... DOG = 4 - ... -The reason for defaulting to ``1`` as the starting number and not ``0`` is -that ``0`` is ``False`` in a boolean sense, but enum members all evaluate -to ``True``. +.. class:: StrEnum -Pickling enums created with the functional API can be tricky as frame stack -implementation details are used to try and figure out which module the -enumeration is being created in (e.g. it will fail if you use a utility -function in separate module, and also may not work on IronPython or Jython). -The solution is to specify the module name explicitly as follows:: + *StrEnum* is the same as *Enum*, but its members are also strings and can be used + in most of the same places that a string can be used. The result of any string + operation performed on or with a *StrEnum* member is not part of the enumeration. - >>> Animal = Enum('Animal', 'ANT BEE CAT DOG', module=__name__) + .. note:: There are places in the stdlib that check for an exact :class:`str` + instead of a :class:`str` subclass (i.e. ``type(unknown) == str`` + instead of ``isinstance(str, unknown)``), and in those locations you + will need to use ``str(StrEnum.member)``. -.. warning:: - If ``module`` is not supplied, and Enum cannot determine what it is, - the new Enum members will not be unpicklable; to keep errors closer to - the source, pickling will be disabled. +.. note:: -The new pickle protocol 4 also, in some circumstances, relies on -:attr:`~definition.__qualname__` being set to the location where pickle will be able -to find the class. For example, if the class was made available in class -SomeData in the global scope:: + Using :class:`auto` with :class:`StrEnum` results in values of the member name, + lower-cased. - >>> Animal = Enum('Animal', 'ANT BEE CAT DOG', qualname='SomeData.Animal') -The complete signature is:: +.. class:: Flag - Enum(value='NewEnumName', names=<...>, *, module='...', qualname='...', type=, start=1) + *Flag* members support the bitwise operators ``&`` (*AND*), ``|`` (*OR*), + ``^`` (*XOR*), and ``~`` (*INVERT*); the results of those operators are members + of the enumeration. -:value: What the new Enum class will record as its name. + .. method:: __contains__(self, value) -:names: The Enum members. This can be a whitespace or comma separated string - (values will start at 1 unless otherwise specified):: + Returns *True* if value is in self:: - 'RED GREEN BLUE' | 'RED,GREEN,BLUE' | 'RED, GREEN, BLUE' + >>> from enum import Flag, auto + >>> class Color(Flag): + ... RED = auto() + ... GREEN = auto() + ... BLUE = auto() + >>> purple = Color.RED | Color.BLUE + >>> white = Color.RED | Color.GREEN | Color.BLUE + >>> Color.GREEN in purple + False + >>> Color.GREEN in white + True + >>> purple in white + True + >>> white in purple + False - or an iterator of names:: + .. method:: __iter__(self): - ['RED', 'GREEN', 'BLUE'] + Returns all contained members:: - or an iterator of (name, value) pairs:: + >>> list(Color.RED) + [Color.RED] + >>> list(purple) + [Color.RED, Color.BLUE] - [('CYAN', 4), ('MAGENTA', 5), ('YELLOW', 6)] + .. method:: __len__(self): - or a mapping:: + Returns number of members in flag:: - {'CHARTREUSE': 7, 'SEA_GREEN': 11, 'ROSEMARY': 42} + >>> len(Color.GREEN) + 1 + >>> len(white) + 3 -:module: name of module where new Enum class can be found. + .. method:: __bool__(self): -:qualname: where in module new Enum class can be found. + Returns *True* if any members in flag, *False* otherwise:: -:type: type to mix in to new Enum class. + >>> bool(Color.GREEN) + True + >>> bool(white) + True + >>> black = Color(0) + >>> bool(black) + False -:start: number to start counting at if only names are passed in. + .. method:: __or__(self, other) -.. versionchanged:: 3.5 - The *start* parameter was added. + Returns current flag binary or'ed with other:: + >>> Color.RED | Color.GREEN + Color.RED|Color.GREEN -Derived Enumerations --------------------- + .. method:: __and__(self, other) -IntEnum -^^^^^^^ + Returns current flag binary and'ed with other:: -The first variation of :class:`Enum` that is provided is also a subclass of -:class:`int`. Members of an :class:`IntEnum` can be compared to integers; -by extension, integer enumerations of different types can also be compared -to each other:: + >>> purple & white + Color.RED|Color.BLUE + >>> purple & Color.GREEN + 0x0 - >>> from enum import IntEnum - >>> class Shape(IntEnum): - ... CIRCLE = 1 - ... SQUARE = 2 - ... - >>> class Request(IntEnum): - ... POST = 1 - ... GET = 2 - ... - >>> Shape == 1 - False - >>> Shape.CIRCLE == 1 - True - >>> Shape.CIRCLE == Request.POST - True + .. method:: __xor__(self, other) -However, they still can't be compared to standard :class:`Enum` enumerations:: + Returns current flag binary xor'ed with other:: - >>> class Shape(IntEnum): - ... CIRCLE = 1 - ... SQUARE = 2 - ... - >>> class Color(Enum): - ... RED = 1 - ... GREEN = 2 - ... - >>> Shape.CIRCLE == Color.RED - False + >>> purple ^ white + Color.GREEN + >>> purple ^ Color.GREEN + Color.RED|Color.GREEN|Color.BLUE -:class:`IntEnum` values behave like integers in other ways you'd expect:: + .. method:: __invert__(self): - >>> int(Shape.CIRCLE) - 1 - >>> ['a', 'b', 'c'][Shape.CIRCLE] - 'b' - >>> [i for i in range(Shape.SQUARE)] - [0, 1] + Returns all the flags in *type(self)* that are not in self:: + >>> ~white + 0x0 + >>> ~purple + Color.GREEN + >>> ~Color.RED + Color.GREEN|Color.BLUE -StrEnum -^^^^^^^ +.. note:: -The second variation of :class:`Enum` that is provided is also a subclass of -:class:`str`. Members of a :class:`StrEnum` can be compared to strings; -by extension, string enumerations of different types can also be compared -to each other. :class:`StrEnum` exists to help avoid the problem of getting -an incorrect member:: + Using :class:`auto` with :class:`Flag` results in integers that are powers + of two, starting with ``1``. - >>> class Directions(StrEnum): - ... NORTH = 'north', # notice the trailing comma - ... SOUTH = 'south' -Before :class:`StrEnum`, ``Directions.NORTH`` would have been the :class:`tuple` -``('north',)``. +.. class:: IntFlag -.. note:: + *IntFlag* is the same as *Flag*, but its members are also integers and can be + used anywhere that an integer can be used. - Unlike other Enum's, ``str(StrEnum.member)`` will return the value of the - member instead of the usual ``"EnumClass.member"``. - -.. versionadded:: 3.10 - - -IntFlag -^^^^^^^ - -The next variation of :class:`Enum` provided, :class:`IntFlag`, is also based -on :class:`int`. The difference being :class:`IntFlag` members can be combined -using the bitwise operators (&, \|, ^, ~) and the result is still an -:class:`IntFlag` member. However, as the name implies, :class:`IntFlag` -members also subclass :class:`int` and can be used wherever an :class:`int` is -used. Any operation on an :class:`IntFlag` member besides the bit-wise -operations will lose the :class:`IntFlag` membership. - -.. versionadded:: 3.6 - -Sample :class:`IntFlag` class:: - - >>> from enum import IntFlag - >>> class Perm(IntFlag): - ... R = 4 - ... W = 2 - ... X = 1 - ... - >>> Perm.R | Perm.W - - >>> Perm.R + Perm.W - 6 - >>> RW = Perm.R | Perm.W - >>> Perm.R in RW - True - -It is also possible to name the combinations:: - - >>> class Perm(IntFlag): - ... R = 4 - ... W = 2 - ... X = 1 - ... RWX = 7 - >>> Perm.RWX - - >>> ~Perm.RWX - - -Another important difference between :class:`IntFlag` and :class:`Enum` is that -if no flags are set (the value is 0), its boolean evaluation is :data:`False`:: - - >>> Perm.R & Perm.X - - >>> bool(Perm.R & Perm.X) - False - -Because :class:`IntFlag` members are also subclasses of :class:`int` they can -be combined with them:: - - >>> Perm.X | 8 - - -:class:`IntFlag` members can also be iterated over:: - - >>> list(RW) - [, ] - -.. versionadded:: 3.10 - - -Flag -^^^^ - -The last variation is :class:`Flag`. Like :class:`IntFlag`, :class:`Flag` -members can be combined using the bitwise operators (&, \|, ^, ~). Unlike -:class:`IntFlag`, they cannot be combined with, nor compared against, any -other :class:`Flag` enumeration, nor :class:`int`. While it is possible to -specify the values directly it is recommended to use :class:`auto` as the -value and let :class:`Flag` select an appropriate value. - -.. versionadded:: 3.6 - -Like :class:`IntFlag`, if a combination of :class:`Flag` members results in no -flags being set, the boolean evaluation is :data:`False`:: - - >>> from enum import Flag, auto - >>> class Color(Flag): - ... RED = auto() - ... BLUE = auto() - ... GREEN = auto() - ... - >>> Color.RED & Color.GREEN - - >>> bool(Color.RED & Color.GREEN) - False - -Individual flags should have values that are powers of two (1, 2, 4, 8, ...), -while combinations of flags won't:: - - >>> class Color(Flag): - ... RED = auto() - ... BLUE = auto() - ... GREEN = auto() - ... WHITE = RED | BLUE | GREEN - ... - >>> Color.WHITE - - -Giving a name to the "no flags set" condition does not change its boolean -value:: - - >>> class Color(Flag): - ... BLACK = 0 - ... RED = auto() - ... BLUE = auto() - ... GREEN = auto() - ... - >>> Color.BLACK - - >>> bool(Color.BLACK) - False - -:class:`Flag` members can also be iterated over:: - - >>> purple = Color.RED | Color.BLUE - >>> list(purple) - [, ] - -.. versionadded:: 3.10 + >>> from enum import IntFlag, auto + >>> class Color(IntFlag): + ... RED = auto() + ... GREEN = auto() + ... BLUE = auto() + >>> Color.RED & 2 + 0x0 + >>> Color.RED | 2 + Color.RED|Color.GREEN -.. note:: + If any integer operation is performed with an *IntFlag* member, the result is + not an *IntFlag*:: - For the majority of new code, :class:`Enum` and :class:`Flag` are strongly - recommended, since :class:`IntEnum` and :class:`IntFlag` break some - semantic promises of an enumeration (by being comparable to integers, and - thus by transitivity to other unrelated enumerations). :class:`IntEnum` - and :class:`IntFlag` should be used only in cases where :class:`Enum` and - :class:`Flag` will not do; for example, when integer constants are replaced - with enumerations, or for interoperability with other systems. - - -Others -^^^^^^ - -While :class:`IntEnum` is part of the :mod:`enum` module, it would be very -simple to implement independently:: - - class IntEnum(int, Enum): - pass - -This demonstrates how similar derived enumerations can be defined; for example -a :class:`StrEnum` that mixes in :class:`str` instead of :class:`int`. - -Some rules: - -1. When subclassing :class:`Enum`, mix-in types must appear before - :class:`Enum` itself in the sequence of bases, as in the :class:`IntEnum` - example above. -2. While :class:`Enum` can have members of any type, once you mix in an - additional type, all the members must have values of that type, e.g. - :class:`int` above. This restriction does not apply to mix-ins which only - add methods and don't specify another type. -3. When another data type is mixed in, the :attr:`value` attribute is *not the - same* as the enum member itself, although it is equivalent and will compare - equal. -4. %-style formatting: `%s` and `%r` call the :class:`Enum` class's - :meth:`__str__` and :meth:`__repr__` respectively; other codes (such as - `%i` or `%h` for IntEnum) treat the enum member as its mixed-in type. -5. :ref:`Formatted string literals `, :meth:`str.format`, - and :func:`format` will use the mixed-in type's :meth:`__format__` - unless :meth:`__str__` or :meth:`__format__` is overridden in the subclass, - in which case the overridden methods or :class:`Enum` methods will be used. - Use the !s and !r format codes to force usage of the :class:`Enum` class's - :meth:`__str__` and :meth:`__repr__` methods. - -When to use :meth:`__new__` vs. :meth:`__init__` ------------------------------------------------- - -:meth:`__new__` must be used whenever you want to customize the actual value of -the :class:`Enum` member. Any other modifications may go in either -:meth:`__new__` or :meth:`__init__`, with :meth:`__init__` being preferred. - -For example, if you want to pass several items to the constructor, but only -want one of them to be the value:: - - >>> class Coordinate(bytes, Enum): - ... """ - ... Coordinate with binary codes that can be indexed by the int code. - ... """ - ... def __new__(cls, value, label, unit): - ... obj = bytes.__new__(cls, [value]) - ... obj._value_ = value - ... obj.label = label - ... obj.unit = unit - ... return obj - ... PX = (0, 'P.X', 'km') - ... PY = (1, 'P.Y', 'km') - ... VX = (2, 'V.X', 'km/s') - ... VY = (3, 'V.Y', 'km/s') - ... - - >>> print(Coordinate['PY']) - Coordinate.PY - - >>> print(Coordinate(3)) - Coordinate.VY - -Interesting examples --------------------- - -While :class:`Enum`, :class:`IntEnum`, :class:`IntFlag`, and :class:`Flag` are -expected to cover the majority of use-cases, they cannot cover them all. Here -are recipes for some different types of enumerations that can be used directly, -or as examples for creating one's own. - - -Omitting values -^^^^^^^^^^^^^^^ - -In many use-cases one doesn't care what the actual value of an enumeration -is. There are several ways to define this type of simple enumeration: - -- use instances of :class:`auto` for the value -- use instances of :class:`object` as the value -- use a descriptive string as the value -- use a tuple as the value and a custom :meth:`__new__` to replace the - tuple with an :class:`int` value - -Using any of these methods signifies to the user that these values are not -important, and also enables one to add, remove, or reorder members without -having to renumber the remaining members. - -Whichever method you choose, you should provide a :meth:`repr` that also hides -the (unimportant) value:: - - >>> class NoValue(Enum): - ... def __repr__(self): - ... return '<%s.%s>' % (self.__class__.__name__, self.name) - ... - - -Using :class:`auto` -""""""""""""""""""" - -Using :class:`auto` would look like:: - - >>> class Color(NoValue): - ... RED = auto() - ... BLUE = auto() - ... GREEN = auto() - ... - >>> Color.GREEN - - - -Using :class:`object` -""""""""""""""""""""" - -Using :class:`object` would look like:: - - >>> class Color(NoValue): - ... RED = object() - ... GREEN = object() - ... BLUE = object() - ... - >>> Color.GREEN - - - -Using a descriptive string -"""""""""""""""""""""""""" - -Using a string as the value would look like:: - - >>> class Color(NoValue): - ... RED = 'stop' - ... GREEN = 'go' - ... BLUE = 'too fast!' - ... - >>> Color.GREEN - - >>> Color.GREEN.value - 'go' - - -Using a custom :meth:`__new__` -"""""""""""""""""""""""""""""" - -Using an auto-numbering :meth:`__new__` would look like:: - - >>> class AutoNumber(NoValue): - ... def __new__(cls): - ... value = len(cls.__members__) + 1 - ... obj = object.__new__(cls) - ... obj._value_ = value - ... return obj - ... - >>> class Color(AutoNumber): - ... RED = () - ... GREEN = () - ... BLUE = () - ... - >>> Color.GREEN - - >>> Color.GREEN.value - 2 - -To make a more general purpose ``AutoNumber``, add ``*args`` to the signature:: - - >>> class AutoNumber(NoValue): - ... def __new__(cls, *args): # this is the only change from above - ... value = len(cls.__members__) + 1 - ... obj = object.__new__(cls) - ... obj._value_ = value - ... return obj - ... - -Then when you inherit from ``AutoNumber`` you can write your own ``__init__`` -to handle any extra arguments:: - - >>> class Swatch(AutoNumber): - ... def __init__(self, pantone='unknown'): - ... self.pantone = pantone - ... AUBURN = '3497' - ... SEA_GREEN = '1246' - ... BLEACHED_CORAL = () # New color, no Pantone code yet! - ... - >>> Swatch.SEA_GREEN - - >>> Swatch.SEA_GREEN.pantone - '1246' - >>> Swatch.BLEACHED_CORAL.pantone - 'unknown' + >>> Color.RED + 2 + 3 -.. note:: + If a *Flag* operation is performed with an *IntFlag* member and: - The :meth:`__new__` method, if defined, is used during creation of the Enum - members; it is then replaced by Enum's :meth:`__new__` which is used after - class creation for lookup of existing members. - - -OrderedEnum -^^^^^^^^^^^ - -An ordered enumeration that is not based on :class:`IntEnum` and so maintains -the normal :class:`Enum` invariants (such as not being comparable to other -enumerations):: - - >>> class OrderedEnum(Enum): - ... def __ge__(self, other): - ... if self.__class__ is other.__class__: - ... return self.value >= other.value - ... return NotImplemented - ... def __gt__(self, other): - ... if self.__class__ is other.__class__: - ... return self.value > other.value - ... return NotImplemented - ... def __le__(self, other): - ... if self.__class__ is other.__class__: - ... return self.value <= other.value - ... return NotImplemented - ... def __lt__(self, other): - ... if self.__class__ is other.__class__: - ... return self.value < other.value - ... return NotImplemented - ... - >>> class Grade(OrderedEnum): - ... A = 5 - ... B = 4 - ... C = 3 - ... D = 2 - ... F = 1 - ... - >>> Grade.C < Grade.A - True - - -DuplicateFreeEnum -^^^^^^^^^^^^^^^^^ - -Raises an error if a duplicate member name is found instead of creating an -alias:: - - >>> class DuplicateFreeEnum(Enum): - ... def __init__(self, *args): - ... cls = self.__class__ - ... if any(self.value == e.value for e in cls): - ... a = self.name - ... e = cls(self.value).name - ... raise ValueError( - ... "aliases not allowed in DuplicateFreeEnum: %r --> %r" - ... % (a, e)) - ... - >>> class Color(DuplicateFreeEnum): - ... RED = 1 - ... GREEN = 2 - ... BLUE = 3 - ... GRENE = 2 - ... - Traceback (most recent call last): - ... - ValueError: aliases not allowed in DuplicateFreeEnum: 'GRENE' --> 'GREEN' + * the result is a valid *IntFlag*: an *IntFlag* is returned + * the result is not a valid *IntFlag*: the result depends on the *FlagBoundary* setting .. note:: - This is a useful example for subclassing Enum to add or change other - behaviors as well as disallowing aliases. If the only desired change is - disallowing aliases, the :func:`unique` decorator can be used instead. + Using :class:`auto` with :class:`IntFlag` results in integers that are powers + of two, starting with ``1``. +.. class:: FlagBoundary -Planet -^^^^^^ + *FlagBoundary* controls how out-of-range values are handled in *Flag* and its + subclasses. -If :meth:`__new__` or :meth:`__init__` is defined the value of the enum member -will be passed to those methods:: - - >>> class Planet(Enum): - ... MERCURY = (3.303e+23, 2.4397e6) - ... VENUS = (4.869e+24, 6.0518e6) - ... EARTH = (5.976e+24, 6.37814e6) - ... MARS = (6.421e+23, 3.3972e6) - ... JUPITER = (1.9e+27, 7.1492e7) - ... SATURN = (5.688e+26, 6.0268e7) - ... URANUS = (8.686e+25, 2.5559e7) - ... NEPTUNE = (1.024e+26, 2.4746e7) - ... def __init__(self, mass, radius): - ... self.mass = mass # in kilograms - ... self.radius = radius # in meters - ... @property - ... def surface_gravity(self): - ... # universal gravitational constant (m3 kg-1 s-2) - ... G = 6.67300E-11 - ... return G * self.mass / (self.radius * self.radius) - ... - >>> Planet.EARTH.value - (5.976e+24, 6378140.0) - >>> Planet.EARTH.surface_gravity - 9.802652743337129 - - -TimePeriod -^^^^^^^^^^ - -An example to show the :attr:`_ignore_` attribute in use:: - - >>> from datetime import timedelta - >>> class Period(timedelta, Enum): - ... "different lengths of time" - ... _ignore_ = 'Period i' - ... Period = vars() - ... for i in range(367): - ... Period['day_%d' % i] = i - ... - >>> list(Period)[:2] - [, ] - >>> list(Period)[-2:] - [, ] - - -How are Enums different? ------------------------- - -Enums have a custom metaclass that affects many aspects of both derived Enum -classes and their instances (members). - - -Enum Classes -^^^^^^^^^^^^ - -The :class:`EnumMeta` metaclass is responsible for providing the -:meth:`__contains__`, :meth:`__dir__`, :meth:`__iter__` and other methods that -allow one to do things with an :class:`Enum` class that fail on a typical -class, such as `list(Color)` or `some_enum_var in Color`. :class:`EnumMeta` is -responsible for ensuring that various other methods on the final :class:`Enum` -class are correct (such as :meth:`__new__`, :meth:`__getnewargs__`, -:meth:`__str__` and :meth:`__repr__`). - - -Enum Members (aka instances) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The most interesting thing about Enum members is that they are singletons. -:class:`EnumMeta` creates them all while it is creating the :class:`Enum` -class itself, and then puts a custom :meth:`__new__` in place to ensure -that no new ones are ever instantiated by returning only the existing -member instances. - - -Finer Points -^^^^^^^^^^^^ - -Supported ``__dunder__`` names -"""""""""""""""""""""""""""""" - -:attr:`__members__` is a read-only ordered mapping of ``member_name``:``member`` -items. It is only available on the class. - -:meth:`__new__`, if specified, must create and return the enum members; it is -also a very good idea to set the member's :attr:`_value_` appropriately. Once -all the members are created it is no longer used. - - -Supported ``_sunder_`` names -"""""""""""""""""""""""""""" - -- ``_name_`` -- name of the member -- ``_value_`` -- value of the member; can be set / modified in ``__new__`` - -- ``_missing_`` -- a lookup function used when a value is not found; may be - overridden -- ``_ignore_`` -- a list of names, either as a :class:`list` or a :class:`str`, - that will not be transformed into members, and will be removed from the final - class -- ``_order_`` -- used in Python 2/3 code to ensure member order is consistent - (class attribute, removed during class creation) -- ``_generate_next_value_`` -- used by the `Functional API`_ and by - :class:`auto` to get an appropriate value for an enum member; may be - overridden - -.. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` -.. versionadded:: 3.7 ``_ignore_`` - -To help keep Python 2 / Python 3 code in sync an :attr:`_order_` attribute can -be provided. It will be checked against the actual order of the enumeration -and raise an error if the two do not match:: - - >>> class Color(Enum): - ... _order_ = 'RED GREEN BLUE' - ... RED = 1 - ... BLUE = 3 - ... GREEN = 2 - ... - Traceback (most recent call last): - ... - TypeError: member order does not match _order_ + .. attribute:: STRICT -.. note:: - - In Python 2 code the :attr:`_order_` attribute is necessary as definition - order is lost before it can be recorded. - -``Enum`` member type -"""""""""""""""""""" + Out-of-range values cause a :exc:`ValueError` to be raised. This is the + default for :class:`Flag`:: -:class:`Enum` members are instances of their :class:`Enum` class, and are -normally accessed as ``EnumClass.member``. Under certain circumstances they -can also be accessed as ``EnumClass.member.member``, but you should never do -this as that lookup may fail or, worse, return something besides the -:class:`Enum` member you are looking for (this is another good reason to use -all-uppercase names for members):: + >>> from enum import Flag, STRICT + >>> class StrictFlag(Flag, boundary=STRICT): + ... RED = auto() + ... GREEN = auto() + ... BLUE = auto() + >>> StrictFlag(2**2 + 2**4) + Traceback (most recent call last): + ... + ValueError: StrictFlag: invalid value: 20 + given 0b0 10100 + allowed 0b0 00111 - >>> class FieldTypes(Enum): - ... name = 0 - ... value = 1 - ... size = 2 - ... - >>> FieldTypes.value.size - - >>> FieldTypes.size.value - 2 + .. attribute:: CONFORM -.. versionchanged:: 3.5 + Out-of-range values have invalid values removed, leaving a valid *Flag* + value:: + >>> from enum import Flag, CONFORM + >>> class ConformFlag(Flag, boundary=CONFORM): + ... RED = auto() + ... GREEN = auto() + ... BLUE = auto() + >>> ConformFlag(2**2 + 2**4) + ConformFlag.BLUE -Creating members that are mixed with other data types -""""""""""""""""""""""""""""""""""""""""""""""""""""" + .. attribute:: EJECT -When subclassing other data types, such as :class:`int` or :class:`str`, with -an :class:`Enum`, all values after the `=` are passed to that data type's -constructor. For example:: + Out-of-range values lose their *Flag* membership and revert to :class:`int`. + This is the default for :class:`IntFlag`:: - >>> class MyEnum(IntEnum): - ... example = '11', 16 # '11' will be interpreted as a hexadecimal - ... # number - >>> MyEnum.example - + >>> from enum import Flag, EJECT + >>> class EjectFlag(Flag, boundary=EJECT): + ... RED = auto() + ... GREEN = auto() + ... BLUE = auto() + >>> EjectFlag(2**2 + 2**4) + 20 + .. attribute:: KEEP -Boolean value of ``Enum`` classes and members -""""""""""""""""""""""""""""""""""""""""""""" + Out-of-range values are kept, and the *Flag* membership is kept. This is + used for some stdlib flags: -:class:`Enum` members that are mixed with non-:class:`Enum` types (such as -:class:`int`, :class:`str`, etc.) are evaluated according to the mixed-in -type's rules; otherwise, all members evaluate as :data:`True`. To make your -own Enum's boolean evaluation depend on the member's value add the following to -your class:: + >>> from enum import Flag, KEEP + >>> class KeepFlag(Flag, boundary=KEEP): + ... RED = auto() + ... GREEN = auto() + ... BLUE = auto() + >>> KeepFlag(2**2 + 2**4) + KeepFlag.BLUE|0x10 - def __bool__(self): - return bool(self.value) -:class:`Enum` classes always evaluate as :data:`True`. +Utilites and Decorators +----------------------- +.. class:: auto -``Enum`` classes with methods -""""""""""""""""""""""""""""" + *auto* can be used in place of a value. If used, the *Enum* machinery will + call an *Enum*'s :meth:`_generate_next_value_` to get an appropriate value. + For *Enum* and *IntEnum* that appropriate value will be the last value plus + one; for *Flag* and *IntFlag* it will be the first power-of-two greater + than the last value; for *StrEnum* it will be the lower-cased version of the + member's name. -If you give your :class:`Enum` subclass extra methods, like the `Planet`_ -class above, those methods will show up in a :func:`dir` of the member, -but not of the class:: + ``_generate_next_value_`` can be overridden to customize the values used by + *auto*. - >>> dir(Planet) - ['EARTH', 'JUPITER', 'MARS', 'MERCURY', 'NEPTUNE', 'SATURN', 'URANUS', 'VENUS', '__class__', '__doc__', '__members__', '__module__'] - >>> dir(Planet.EARTH) - ['__class__', '__doc__', '__module__', 'name', 'surface_gravity', 'value'] +.. decorator:: global_enum + A :keyword:`class` decorator specifically for enumerations. It replaces the + :meth:`__repr__` method with one that shows *module_name*.*member_name*. It + also injects the members, and their aliases, into the the global namespace + they were defined in. -Combining members of ``Flag`` -""""""""""""""""""""""""""""" -If a combination of Flag members is not named, the :func:`repr` will include -all named flags and all named combinations of flags that are in the value:: +.. decorator:: property - >>> class Color(Flag): - ... RED = auto() - ... GREEN = auto() - ... BLUE = auto() - ... MAGENTA = RED | BLUE - ... YELLOW = RED | GREEN - ... CYAN = GREEN | BLUE - ... - >>> Color(3) # named combination - - >>> Color(7) # not named combination - + A decorator similar to the built-in *property*, but specifically for + enumerations. It allows member attributes to have the same names as members + themselves. -``StrEnum`` and :meth:`str.__str__` -""""""""""""""""""""""""""""""""""" + .. note:: the *property* and the member must be defined in separate classes; + for example, the *value* and *name* attributes are defined in the + *Enum* class, and *Enum* subclasses can define members with the + names ``value`` and ``name``. -An important difference between :class:`StrEnum` and other Enums is the -:meth:`__str__` method; because :class:`StrEnum` members are strings, some -parts of Python will read the string data directly, while others will call -:meth:`str()`. To make those two operations have the same result, -:meth:`StrEnum.__str__` will be the same as :meth:`str.__str__` so that -``str(StrEnum.member) == StrEnum.member`` is true. +.. decorator:: unique + A :keyword:`class` decorator specifically for enumerations. It searches an + enumeration's :attr:`__members__`, gathering any aliases it finds; if any are + found :exc:`ValueError` is raised with the details:: + + >>> from enum import Enum, unique + >>> @unique + ... class Mistake(Enum): + ... ONE = 1 + ... TWO = 2 + ... THREE = 3 + ... FOUR = 3 + ... + Traceback (most recent call last): + ... + ValueError: duplicate values found in : FOUR -> THREE diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index df2cda9d67ad15..c83daae302c19d 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -42,12 +42,12 @@ include the originating exception(s) and the final exception. When raising a new exception (rather than using a bare ``raise`` to re-raise the exception currently being handled), the implicit exception context can be -supplemented with an explicit cause by using :keyword:`from` with +supplemented with an explicit cause by using :keyword:`from` with :keyword:`raise`:: raise new_exc from original_exc -The expression following :keyword:`from` must be an exception or ``None``. It +The expression following :keyword:`from` must be an exception or ``None``. It will be set as :attr:`__cause__` on the raised exception. Setting :attr:`__cause__` also implicitly sets the :attr:`__suppress_context__` attribute to ``True``, so that using ``raise new_exc from None`` @@ -90,8 +90,13 @@ The following exceptions are used mostly as base classes for other exceptions. .. method:: with_traceback(tb) This method sets *tb* as the new traceback for the exception and returns - the exception object. It is usually used in exception handling code like - this:: + the exception object. It was more commonly used before the exception + chaining features of :pep:`3134` became available. The following example + shows how we can convert an instance of ``SomeException`` into an + instance of ``OtherException`` while preserving the traceback. Once + raised, the current frame is pushed onto the traceback of the + ``OtherException``, as would have happened to the traceback of the + original ``SomeException`` had we allowed it to propagate to the caller. try: ... @@ -144,6 +149,13 @@ The following exceptions are the exceptions that are usually raised. assignment fails. (When an object does not support attribute references or attribute assignments at all, :exc:`TypeError` is raised.) + The :attr:`name` and :attr:`obj` attributes can be set using keyword-only + arguments to the constructor. When set they represent the name of the attribute + that was attempted to be accessed and the object that was accessed for said + attribute, respectively. + + .. versionchanged:: 3.10 + Added the :attr:`name` and :attr:`obj` attributes. .. exception:: EOFError @@ -230,6 +242,13 @@ The following exceptions are the exceptions that are usually raised. unqualified names. The associated value is an error message that includes the name that could not be found. + The :attr:`name` attribute can be set using a keyword-only argument to the + constructor. When set it represent the name of the variable that was attempted + to be accessed. + + .. versionchanged:: 3.10 + Added the :attr:`name` attribute. + .. exception:: NotImplementedError @@ -313,8 +332,8 @@ The following exceptions are the exceptions that are usually raised. .. versionchanged:: 3.4 The :attr:`filename` attribute is now the original file name passed to the function, instead of the name encoded to or decoded from the - filesystem encoding. Also, the *filename2* constructor argument and - attribute was added. + :term:`filesystem encoding and error handler`. Also, the *filename2* + constructor argument and attribute was added. .. exception:: OverflowError @@ -397,10 +416,38 @@ The following exceptions are the exceptions that are usually raised. or :func:`eval`, or when reading the initial script or standard input (also interactively). - Instances of this class have attributes :attr:`filename`, :attr:`lineno`, - :attr:`offset` and :attr:`text` for easier access to the details. :func:`str` - of the exception instance returns only the message. + The :func:`str` of the exception instance returns only the error message. + + .. attribute:: filename + + The name of the file the syntax error occurred in. + + .. attribute:: lineno + + Which line number in the file the error occurred in. This is + 1-indexed: the first line in the file has a ``lineno`` of 1. + + .. attribute:: offset + + The column in the line where the error occurred. This is + 1-indexed: the first character in the line has an ``offset`` of 1. + + .. attribute:: text + The source code text involved in the error. + + .. attribute:: end_lineno + + Which line number in the file the error occurred ends in. This is + 1-indexed: the first line in the file has a ``lineno`` of 1. + + .. attribute:: end_offset + + The column in the end line where the error occurred finishes. This is + 1-indexed: the first character in the line has an ``offset`` of 1. + + .. versionchanged:: 3.10 + Added the :attr:`end_lineno` and :attr:`end_offset` attributes. .. exception:: IndentationError @@ -736,6 +783,15 @@ The following exceptions are used as warning categories; see the Base class for warnings related to Unicode. +.. exception:: EncodingWarning + + Base class for warnings related to encodings. + + See :ref:`io-encoding-warning` for details. + + .. versionadded:: 3.10 + + .. exception:: BytesWarning Base class for warnings related to :class:`bytes` and :class:`bytearray`. diff --git a/Doc/library/filecmp.rst b/Doc/library/filecmp.rst index 31b9b4afab9348..c60603b30a6d7d 100644 --- a/Doc/library/filecmp.rst +++ b/Doc/library/filecmp.rst @@ -173,7 +173,13 @@ The :class:`dircmp` class .. attribute:: subdirs A dictionary mapping names in :attr:`common_dirs` to :class:`dircmp` - objects. + instances (or MyDirCmp instances if this instance is of type MyDirCmp, a + subclass of :class:`dircmp`). + + .. versionchanged:: 3.10 + Previously entries were always :class:`dircmp` instances. Now entries + are the same type as *self*, if *self* is a subclass of + :class:`dircmp`. .. attribute:: DEFAULT_IGNORES diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index cc4039a30e38ae..819640045cadf7 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -18,7 +18,7 @@ write one file see :func:`open`. The typical use is:: import fileinput - for line in fileinput.input(): + for line in fileinput.input(encoding="utf-8"): process(line) This iterates over the lines of all files listed in ``sys.argv[1:]``, defaulting @@ -49,13 +49,14 @@ a file may not have one. You can control how files are opened by providing an opening hook via the *openhook* parameter to :func:`fileinput.input` or :class:`FileInput()`. The hook must be a function that takes two arguments, *filename* and *mode*, and -returns an accordingly opened file-like object. Two useful hooks are already -provided by this module. +returns an accordingly opened file-like object. If *encoding* and/or *errors* +are specified, they will be passed to the hook as aditional keyword arguments. +This module provides a :func:`hook_encoded` to support compressed files. The following function is the primary interface of this module: -.. function:: input(files=None, inplace=False, backup='', *, mode='r', openhook=None) +.. function:: input(files=None, inplace=False, backup='', *, mode='r', openhook=None, encoding=None, errors=None) Create an instance of the :class:`FileInput` class. The instance will be used as global state for the functions of this module, and is also returned to use @@ -66,7 +67,7 @@ The following function is the primary interface of this module: :keyword:`with` statement. In this example, *input* is closed after the :keyword:`!with` statement is exited, even if an exception occurs:: - with fileinput.input(files=('spam.txt', 'eggs.txt')) as f: + with fileinput.input(files=('spam.txt', 'eggs.txt'), encoding="utf-8") as f: for line in f: process(line) @@ -76,6 +77,9 @@ The following function is the primary interface of this module: .. versionchanged:: 3.8 The keyword parameters *mode* and *openhook* are now keyword-only. + .. versionchanged:: 3.10 + The keyword-only parameter *encoding* and *errors* are added. + The following functions use the global state created by :func:`fileinput.input`; if there is no active state, :exc:`RuntimeError` is raised. @@ -137,7 +141,7 @@ The class which implements the sequence behavior provided by the module is available for subclassing as well: -.. class:: FileInput(files=None, inplace=False, backup='', *, mode='r', openhook=None) +.. class:: FileInput(files=None, inplace=False, backup='', *, mode='r', openhook=None, encoding=None, errors=None) Class :class:`FileInput` is the implementation; its methods :meth:`filename`, :meth:`fileno`, :meth:`lineno`, :meth:`filelineno`, :meth:`isfirstline`, @@ -155,6 +159,8 @@ available for subclassing as well: *filename* and *mode*, and returns an accordingly opened file-like object. You cannot use *inplace* and *openhook* together. + You can specify *encoding* and *errors* that is passed to :func:`open` or *openhook*. + A :class:`FileInput` instance can be used as a context manager in the :keyword:`with` statement. In this example, *input* is closed after the :keyword:`!with` statement is exited, even if an exception occurs:: @@ -162,7 +168,6 @@ available for subclassing as well: with FileInput(files=('spam.txt', 'eggs.txt')) as input: process(input) - .. versionchanged:: 3.2 Can be used as a context manager. @@ -175,6 +180,8 @@ available for subclassing as well: .. versionchanged:: 3.8 The keyword parameter *mode* and *openhook* are now keyword-only. + .. versionchanged:: 3.10 + The keyword-only parameter *encoding* and *errors* are added. **Optional in-place filtering:** if the keyword argument ``inplace=True`` is @@ -191,14 +198,20 @@ when standard input is read. The two following opening hooks are provided by this module: -.. function:: hook_compressed(filename, mode) +.. function:: hook_compressed(filename, mode, *, encoding=None, errors=None) Transparently opens files compressed with gzip and bzip2 (recognized by the extensions ``'.gz'`` and ``'.bz2'``) using the :mod:`gzip` and :mod:`bz2` modules. If the filename extension is not ``'.gz'`` or ``'.bz2'``, the file is opened normally (ie, using :func:`open` without any decompression). - Usage example: ``fi = fileinput.FileInput(openhook=fileinput.hook_compressed)`` + The *encoding* and *errors* values are passed to to :class:`io.TextIOWrapper` + for compressed files and open for normal files. + + Usage example: ``fi = fileinput.FileInput(openhook=fileinput.hook_compressed, encoding="utf-8")`` + + .. versionchanged:: 3.10 + The keyword-only parameter *encoding* and *errors* are added. .. function:: hook_encoded(encoding, errors=None) @@ -212,3 +225,7 @@ The two following opening hooks are provided by this module: .. versionchanged:: 3.6 Added the optional *errors* parameter. + + .. deprecated:: 3.10 + This function is deprecated since :func:`input` and :class:`FileInput` + now have *encoding* and *errors* parameters. diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index ce07d326b395d8..925f08e914685e 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -75,7 +75,7 @@ patterns. .. function:: filter(names, pattern) - Return the subset of the list of *names* that match *pattern*. It is the same as + Construct a list from those elements of the iterable *names* that match *pattern*. It is the same as ``[n for n in names if fnmatch(n, pattern)]``, but implemented more efficiently. diff --git a/Doc/library/formatter.rst b/Doc/library/formatter.rst deleted file mode 100644 index 6c10ac6fab50e4..00000000000000 --- a/Doc/library/formatter.rst +++ /dev/null @@ -1,351 +0,0 @@ -:mod:`formatter` --- Generic output formatting -============================================== - -.. module:: formatter - :synopsis: Generic output formatter and device interface. - :deprecated: - -.. deprecated:: 3.4 - Due to lack of usage, the formatter module has been deprecated. - --------------- - -This module supports two interface definitions, each with multiple -implementations: The *formatter* interface, and the *writer* interface which is -required by the formatter interface. - -Formatter objects transform an abstract flow of formatting events into specific -output events on writer objects. Formatters manage several stack structures to -allow various properties of a writer object to be changed and restored; writers -need not be able to handle relative changes nor any sort of "change back" -operation. Specific writer properties which may be controlled via formatter -objects are horizontal alignment, font, and left margin indentations. A -mechanism is provided which supports providing arbitrary, non-exclusive style -settings to a writer as well. Additional interfaces facilitate formatting -events which are not reversible, such as paragraph separation. - -Writer objects encapsulate device interfaces. Abstract devices, such as file -formats, are supported as well as physical devices. The provided -implementations all work with abstract devices. The interface makes available -mechanisms for setting the properties which formatter objects manage and -inserting data into the output. - - -.. _formatter-interface: - -The Formatter Interface ------------------------ - -Interfaces to create formatters are dependent on the specific formatter class -being instantiated. The interfaces described below are the required interfaces -which all formatters must support once initialized. - -One data element is defined at the module level: - - -.. data:: AS_IS - - Value which can be used in the font specification passed to the ``push_font()`` - method described below, or as the new value to any other ``push_property()`` - method. Pushing the ``AS_IS`` value allows the corresponding ``pop_property()`` - method to be called without having to track whether the property was changed. - -The following attributes are defined for formatter instance objects: - - -.. attribute:: formatter.writer - - The writer instance with which the formatter interacts. - - -.. method:: formatter.end_paragraph(blanklines) - - Close any open paragraphs and insert at least *blanklines* before the next - paragraph. - - -.. method:: formatter.add_line_break() - - Add a hard line break if one does not already exist. This does not break the - logical paragraph. - - -.. method:: formatter.add_hor_rule(*args, **kw) - - Insert a horizontal rule in the output. A hard break is inserted if there is - data in the current paragraph, but the logical paragraph is not broken. The - arguments and keywords are passed on to the writer's :meth:`send_line_break` - method. - - -.. method:: formatter.add_flowing_data(data) - - Provide data which should be formatted with collapsed whitespace. Whitespace - from preceding and successive calls to :meth:`add_flowing_data` is considered as - well when the whitespace collapse is performed. The data which is passed to - this method is expected to be word-wrapped by the output device. Note that any - word-wrapping still must be performed by the writer object due to the need to - rely on device and font information. - - -.. method:: formatter.add_literal_data(data) - - Provide data which should be passed to the writer unchanged. Whitespace, - including newline and tab characters, are considered legal in the value of - *data*. - - -.. method:: formatter.add_label_data(format, counter) - - Insert a label which should be placed to the left of the current left margin. - This should be used for constructing bulleted or numbered lists. If the - *format* value is a string, it is interpreted as a format specification for - *counter*, which should be an integer. The result of this formatting becomes the - value of the label; if *format* is not a string it is used as the label value - directly. The label value is passed as the only argument to the writer's - :meth:`send_label_data` method. Interpretation of non-string label values is - dependent on the associated writer. - - Format specifications are strings which, in combination with a counter value, - are used to compute label values. Each character in the format string is copied - to the label value, with some characters recognized to indicate a transform on - the counter value. Specifically, the character ``'1'`` represents the counter - value formatter as an Arabic number, the characters ``'A'`` and ``'a'`` - represent alphabetic representations of the counter value in upper and lower - case, respectively, and ``'I'`` and ``'i'`` represent the counter value in Roman - numerals, in upper and lower case. Note that the alphabetic and roman - transforms require that the counter value be greater than zero. - - -.. method:: formatter.flush_softspace() - - Send any pending whitespace buffered from a previous call to - :meth:`add_flowing_data` to the associated writer object. This should be called - before any direct manipulation of the writer object. - - -.. method:: formatter.push_alignment(align) - - Push a new alignment setting onto the alignment stack. This may be - :const:`AS_IS` if no change is desired. If the alignment value is changed from - the previous setting, the writer's :meth:`new_alignment` method is called with - the *align* value. - - -.. method:: formatter.pop_alignment() - - Restore the previous alignment. - - -.. method:: formatter.push_font((size, italic, bold, teletype)) - - Change some or all font properties of the writer object. Properties which are - not set to :const:`AS_IS` are set to the values passed in while others are - maintained at their current settings. The writer's :meth:`new_font` method is - called with the fully resolved font specification. - - -.. method:: formatter.pop_font() - - Restore the previous font. - - -.. method:: formatter.push_margin(margin) - - Increase the number of left margin indentations by one, associating the logical - tag *margin* with the new indentation. The initial margin level is ``0``. - Changed values of the logical tag must be true values; false values other than - :const:`AS_IS` are not sufficient to change the margin. - - -.. method:: formatter.pop_margin() - - Restore the previous margin. - - -.. method:: formatter.push_style(*styles) - - Push any number of arbitrary style specifications. All styles are pushed onto - the styles stack in order. A tuple representing the entire stack, including - :const:`AS_IS` values, is passed to the writer's :meth:`new_styles` method. - - -.. method:: formatter.pop_style(n=1) - - Pop the last *n* style specifications passed to :meth:`push_style`. A tuple - representing the revised stack, including :const:`AS_IS` values, is passed to - the writer's :meth:`new_styles` method. - - -.. method:: formatter.set_spacing(spacing) - - Set the spacing style for the writer. - - -.. method:: formatter.assert_line_data(flag=1) - - Inform the formatter that data has been added to the current paragraph - out-of-band. This should be used when the writer has been manipulated - directly. The optional *flag* argument can be set to false if the writer - manipulations produced a hard line break at the end of the output. - - -.. _formatter-impls: - -Formatter Implementations -------------------------- - -Two implementations of formatter objects are provided by this module. Most -applications may use one of these classes without modification or subclassing. - - -.. class:: NullFormatter(writer=None) - - A formatter which does nothing. If *writer* is omitted, a :class:`NullWriter` - instance is created. No methods of the writer are called by - :class:`NullFormatter` instances. Implementations should inherit from this - class if implementing a writer interface but don't need to inherit any - implementation. - - -.. class:: AbstractFormatter(writer) - - The standard formatter. This implementation has demonstrated wide applicability - to many writers, and may be used directly in most circumstances. It has been - used to implement a full-featured World Wide Web browser. - - -.. _writer-interface: - -The Writer Interface --------------------- - -Interfaces to create writers are dependent on the specific writer class being -instantiated. The interfaces described below are the required interfaces which -all writers must support once initialized. Note that while most applications can -use the :class:`AbstractFormatter` class as a formatter, the writer must -typically be provided by the application. - - -.. method:: writer.flush() - - Flush any buffered output or device control events. - - -.. method:: writer.new_alignment(align) - - Set the alignment style. The *align* value can be any object, but by convention - is a string or ``None``, where ``None`` indicates that the writer's "preferred" - alignment should be used. Conventional *align* values are ``'left'``, - ``'center'``, ``'right'``, and ``'justify'``. - - -.. method:: writer.new_font(font) - - Set the font style. The value of *font* will be ``None``, indicating that the - device's default font should be used, or a tuple of the form ``(size, - italic, bold, teletype)``. Size will be a string indicating the size of - font that should be used; specific strings and their interpretation must be - defined by the application. The *italic*, *bold*, and *teletype* values are - Boolean values specifying which of those font attributes should be used. - - -.. method:: writer.new_margin(margin, level) - - Set the margin level to the integer *level* and the logical tag to *margin*. - Interpretation of the logical tag is at the writer's discretion; the only - restriction on the value of the logical tag is that it not be a false value for - non-zero values of *level*. - - -.. method:: writer.new_spacing(spacing) - - Set the spacing style to *spacing*. - - -.. method:: writer.new_styles(styles) - - Set additional styles. The *styles* value is a tuple of arbitrary values; the - value :const:`AS_IS` should be ignored. The *styles* tuple may be interpreted - either as a set or as a stack depending on the requirements of the application - and writer implementation. - - -.. method:: writer.send_line_break() - - Break the current line. - - -.. method:: writer.send_paragraph(blankline) - - Produce a paragraph separation of at least *blankline* blank lines, or the - equivalent. The *blankline* value will be an integer. Note that the - implementation will receive a call to :meth:`send_line_break` before this call - if a line break is needed; this method should not include ending the last line - of the paragraph. It is only responsible for vertical spacing between - paragraphs. - - -.. method:: writer.send_hor_rule(*args, **kw) - - Display a horizontal rule on the output device. The arguments to this method - are entirely application- and writer-specific, and should be interpreted with - care. The method implementation may assume that a line break has already been - issued via :meth:`send_line_break`. - - -.. method:: writer.send_flowing_data(data) - - Output character data which may be word-wrapped and re-flowed as needed. Within - any sequence of calls to this method, the writer may assume that spans of - multiple whitespace characters have been collapsed to single space characters. - - -.. method:: writer.send_literal_data(data) - - Output character data which has already been formatted for display. Generally, - this should be interpreted to mean that line breaks indicated by newline - characters should be preserved and no new line breaks should be introduced. The - data may contain embedded newline and tab characters, unlike data provided to - the :meth:`send_formatted_data` interface. - - -.. method:: writer.send_label_data(data) - - Set *data* to the left of the current left margin, if possible. The value of - *data* is not restricted; treatment of non-string values is entirely - application- and writer-dependent. This method will only be called at the - beginning of a line. - - -.. _writer-impls: - -Writer Implementations ----------------------- - -Three implementations of the writer object interface are provided as examples by -this module. Most applications will need to derive new writer classes from the -:class:`NullWriter` class. - - -.. class:: NullWriter() - - A writer which only provides the interface definition; no actions are taken on - any methods. This should be the base class for all writers which do not need to - inherit any implementation methods. - - -.. class:: AbstractWriter() - - A writer which can be used in debugging formatters, but not much else. Each - method simply announces itself by printing its name and arguments on standard - output. - - -.. class:: DumbWriter(file=None, maxcol=72) - - Simple writer class which writes output on the :term:`file object` passed - in as *file* or, if *file* is omitted, on standard output. The output is - simply word-wrapped to the number of columns specified by *maxcol*. This - class is suitable for reflowing a sequence of paragraphs. - diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 52a1512b5884fe..7d8a669f9e16d4 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -12,31 +12,31 @@ are always available. They are listed here in alphabetical order. +=========================+=======================+=======================+=========================+ | | **A** | | **E** | | **L** | | **R** | | | :func:`abs` | | :func:`enumerate` | | :func:`len` | | |func-range|_ | -| | :func:`all` | | :func:`eval` | | |func-list|_ | | :func:`repr` | -| | :func:`any` | | :func:`exec` | | :func:`locals` | | :func:`reversed` | -| | :func:`ascii` | | | | | | :func:`round` | -| | | | **F** | | **M** | | | -| | **B** | | :func:`filter` | | :func:`map` | | **S** | -| | :func:`bin` | | :func:`float` | | :func:`max` | | |func-set|_ | -| | :func:`bool` | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | -| | :func:`breakpoint` | | |func-frozenset|_ | | :func:`min` | | :func:`slice` | -| | |func-bytearray|_ | | | | | | :func:`sorted` | -| | |func-bytes|_ | | **G** | | **N** | | :func:`staticmethod` | -| | | | :func:`getattr` | | :func:`next` | | |func-str|_ | -| | **C** | | :func:`globals` | | | | :func:`sum` | -| | :func:`callable` | | | | **O** | | :func:`super` | -| | :func:`chr` | | **H** | | :func:`object` | | | -| | :func:`classmethod` | | :func:`hasattr` | | :func:`oct` | | **T** | -| | :func:`compile` | | :func:`hash` | | :func:`open` | | |func-tuple|_ | -| | :func:`complex` | | :func:`help` | | :func:`ord` | | :func:`type` | -| | | | :func:`hex` | | | | | -| | **D** | | | | **P** | | **V** | -| | :func:`delattr` | | **I** | | :func:`pow` | | :func:`vars` | -| | |func-dict|_ | | :func:`id` | | :func:`print` | | | -| | :func:`dir` | | :func:`input` | | :func:`property` | | **Z** | -| | :func:`divmod` | | :func:`int` | | | | :func:`zip` | -| | | | :func:`isinstance` | | | | | -| | | | :func:`issubclass` | | | | **_** | +| | :func:`aiter` | | :func:`eval` | | |func-list|_ | | :func:`repr` | +| | :func:`all` | | :func:`exec` | | :func:`locals` | | :func:`reversed` | +| | :func:`any` | | | | | | :func:`round` | +| | :func:`anext` | | **F** | | **M** | | | +| | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** | +| | | | :func:`float` | | :func:`max` | | |func-set|_ | +| | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | +| | :func:`bin` | | |func-frozenset|_ | | :func:`min` | | :func:`slice` | +| | :func:`bool` | | | | | | :func:`sorted` | +| | :func:`breakpoint` | | **G** | | **N** | | :func:`staticmethod` | +| | |func-bytearray|_ | | :func:`getattr` | | :func:`next` | | |func-str|_ | +| | |func-bytes|_ | | :func:`globals` | | | | :func:`sum` | +| | | | | | **O** | | :func:`super` | +| | **C** | | **H** | | :func:`object` | | | +| | :func:`callable` | | :func:`hasattr` | | :func:`oct` | | **T** | +| | :func:`chr` | | :func:`hash` | | :func:`open` | | |func-tuple|_ | +| | :func:`classmethod` | | :func:`help` | | :func:`ord` | | :func:`type` | +| | :func:`compile` | | :func:`hex` | | | | | +| | :func:`complex` | | | | **P** | | **V** | +| | | | **I** | | :func:`pow` | | :func:`vars` | +| | **D** | | :func:`id` | | :func:`print` | | | +| | :func:`delattr` | | :func:`input` | | :func:`property` | | **Z** | +| | |func-dict|_ | | :func:`int` | | | | :func:`zip` | +| | :func:`dir` | | :func:`isinstance` | | | | | +| | :func:`divmod` | | :func:`issubclass` | | | | **_** | | | | | :func:`iter` | | | | :func:`__import__` | +-------------------------+-----------------------+-----------------------+-------------------------+ @@ -61,6 +61,18 @@ are always available. They are listed here in alphabetical order. If the argument is a complex number, its magnitude is returned. +.. function:: aiter(async_iterable) + + Return an :term:`asynchronous iterator` for an :term:`asynchronous iterable`. + Equivalent to calling ``x.__aiter__()``. + + ``aiter(x)`` itself has an ``__aiter__()`` method that returns ``x``, + so ``aiter(aiter(x))`` is the same as ``aiter(x)``. + + Note: Unlike :func:`iter`, :func:`aiter` has no 2-argument variant. + + .. versionadded:: 3.10 + .. function:: all(iterable) Return ``True`` if all elements of the *iterable* are true (or if the iterable @@ -73,6 +85,21 @@ are always available. They are listed here in alphabetical order. return True +.. awaitablefunction:: anext(async_iterator[, default]) + + When awaited, return the next item from the given :term:`asynchronous + iterator`, or *default* if given and the iterator is exhausted. + + This is the async variant of the :func:`next` builtin, and behaves + similarly. + + This calls the :meth:`~object.__anext__` method of *async_iterator*, + returning an :term:`awaitable`. Awaiting this returns the next value of the + iterator. If *default* is given, it is returned if the iterator is exhausted, + otherwise :exc:`StopAsyncIteration` is raised. + + .. versionadded:: 3.10 + .. function:: any(iterable) Return ``True`` if any element of the *iterable* is true. If the iterable @@ -242,6 +269,11 @@ are always available. They are listed here in alphabetical order. Class methods can now wrap other :term:`descriptors ` such as :func:`property`. + .. versionchanged:: 3.10 + Class methods now inherit the method attributes (``__module__``, + ``__name__``, ``__qualname__``, ``__doc__`` and ``__annotations__``) and + have a new ``__wrapped__`` attribute. + .. function:: compile(source, filename, mode, flags=0, dont_inherit=False, optimize=-1) Compile the *source* into a code or AST object. Code objects can be executed @@ -259,7 +291,7 @@ are always available. They are listed here in alphabetical order. interactive statement (in the latter case, expression statements that evaluate to something other than ``None`` will be printed). - The optional argument *flags* and *dont_inherit* controls which + The optional arguments *flags* and *dont_inherit* control which :ref:`compiler options ` should be activated and which :ref:`future features ` should be allowed. If neither is present (or both are zero) the code is compiled with the same flags that @@ -478,14 +510,15 @@ are always available. They are listed here in alphabetical order. dictionaries as global and local namespace. If the *globals* dictionary is present and does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module :mod:`builtins` is - inserted under that key before *expression* is parsed. This means that - *expression* normally has full access to the standard :mod:`builtins` - module and restricted environments are propagated. If the *locals* - dictionary is omitted it defaults to the *globals* dictionary. If both - dictionaries are omitted, the expression is executed with the *globals* and - *locals* in the environment where :func:`eval` is called. Note, *eval()* - does not have access to the :term:`nested scopes ` (non-locals) in the - enclosing environment. + inserted under that key before *expression* is parsed. That way you can + control what builtins are available to the executed code by inserting your + own ``__builtins__`` dictionary into *globals* before passing it to + :func:`eval`. If the *locals* dictionary is omitted it defaults to the + *globals* dictionary. If both dictionaries are omitted, the expression is + executed with the *globals* and *locals* in the environment where + :func:`eval` is called. Note, *eval()* does not have access to the + :term:`nested scopes ` (non-locals) in the enclosing + environment. The return value is the result of the evaluated expression. Syntax errors are reported as exceptions. Example: @@ -525,7 +558,8 @@ are always available. They are listed here in alphabetical order. occurs). [#]_ If it is a code object, it is simply executed. In all cases, the code that's executed is expected to be valid as file input (see the section "File input" in the Reference Manual). Be aware that the - :keyword:`return` and :keyword:`yield` statements may not be used outside of + :keyword:`nonlocal`, :keyword:`yield`, and :keyword:`return` + statements may not be used outside of function definitions even within the context of code passed to the :func:`exec` function. The return value is ``None``. @@ -692,6 +726,13 @@ are always available. They are listed here in alphabetical order. ``x.foobar``. If the named attribute does not exist, *default* is returned if provided, otherwise :exc:`AttributeError` is raised. + .. note:: + + Since :ref:`private name mangling ` happens at + compilation time, one must manually mangle a private attribute's + (attributes with two leading underscores) name in order to retrieve it with + :func:`getattr`. + .. function:: globals() @@ -861,19 +902,27 @@ are always available. They are listed here in alphabetical order. class>`) subclass thereof. If *object* is not an object of the given type, the function always returns ``False``. If *classinfo* is a tuple of type objects (or recursively, other such - tuples), return ``True`` if *object* is an instance of any of the types. + tuples) or a :ref:`types-union` of multiple types, return ``True`` if + *object* is an instance of any of the types. If *classinfo* is not a type or tuple of types and such tuples, a :exc:`TypeError` exception is raised. + .. versionchanged:: 3.10 + *classinfo* can be a :ref:`types-union`. + .. function:: issubclass(class, classinfo) Return ``True`` if *class* is a subclass (direct, indirect or :term:`virtual `) of *classinfo*. A class is considered a subclass of itself. *classinfo* may be a tuple of class - objects, in which case every entry in *classinfo* will be checked. In any other + objects or a :ref:`types-union`, in which case every entry in *classinfo* + will be checked. In any other case, a :exc:`TypeError` exception is raised. + .. versionchanged:: 3.10 + *classinfo* can be a :ref:`types-union`. + .. function:: iter(object[, sentinel]) @@ -1511,6 +1560,13 @@ are always available. They are listed here in alphabetical order. object allows it. For example, ``setattr(x, 'foobar', 123)`` is equivalent to ``x.foobar = 123``. + .. note:: + + Since :ref:`private name mangling ` happens at + compilation time, one must manually mangle a private attribute's + (attributes with two leading underscores) name in order to set it with + :func:`setattr`. + .. class:: slice(stop) slice(start, stop[, step]) @@ -1563,8 +1619,9 @@ are always available. They are listed here in alphabetical order. The ``@staticmethod`` form is a function :term:`decorator` -- see :ref:`function` for details. - A static method can be called either on the class (such as ``C.f()``) or on an instance (such - as ``C().f()``). + A static method can be called either on the class (such as ``C.f()``) or on + an instance (such as ``C().f()``). Moreover, they can be called as regular + functions (such as ``f()``). Static methods in Python are similar to those found in Java or C++. Also see :func:`classmethod` for a variant that is useful for creating alternate class @@ -1576,11 +1633,20 @@ are always available. They are listed here in alphabetical order. body and you want to avoid the automatic transformation to instance method. For these cases, use this idiom:: + def regular_function(): + ... + class C: - builtin_open = staticmethod(open) + method = staticmethod(regular_function) For more information on static methods, see :ref:`types`. + .. versionchanged:: 3.10 + Static methods now inherit the method attributes (``__module__``, + ``__name__``, ``__qualname__``, ``__doc__`` and ``__annotations__``), + have a new ``__wrapped__`` attribute, and are now callable as regular + functions. + .. index:: single: string; str() (built-in function) @@ -1611,7 +1677,7 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.8 The *start* parameter can be specified as a keyword argument. -.. function:: super([type[, object-or-type]]) +.. class:: super([type[, object-or-type]]) Return a proxy object that delegates method calls to a parent or sibling class of *type*. This is useful for accessing inherited methods that have @@ -1645,7 +1711,7 @@ are always available. They are listed here in alphabetical order. not found in statically compiled languages or languages that only support single inheritance. This makes it possible to implement "diamond diagrams" where multiple base classes implement the same method. Good design dictates - that this method have the same calling signature in every case (because the + that such implementations have the same calling signature in every case (because the order of calls is determined at runtime, because that order adapts to changes in the class hierarchy, and because that order can include sibling classes that are unknown prior to runtime). @@ -1689,7 +1755,7 @@ are always available. They are listed here in alphabetical order. .. class:: type(object) - type(name, bases, dict) + type(name, bases, dict, **kwds) .. index:: object: type @@ -1702,21 +1768,29 @@ are always available. They are listed here in alphabetical order. With three arguments, return a new type object. This is essentially a - dynamic form of the :keyword:`class` statement. The *name* string is the - class name and becomes the :attr:`~definition.__name__` attribute; the *bases* - tuple itemizes the base classes and becomes the :attr:`~class.__bases__` - attribute; and the *dict* dictionary is the namespace containing definitions - for class body and is copied to a standard dictionary to become the - :attr:`~object.__dict__` attribute. For example, the following two - statements create identical :class:`type` objects: + dynamic form of the :keyword:`class` statement. The *name* string is + the class name and becomes the :attr:`~definition.__name__` attribute. + The *bases* tuple contains the base classes and becomes the + :attr:`~class.__bases__` attribute; if empty, :class:`object`, the + ultimate base of all classes, is added. The *dict* dictionary contains + attribute and method definitions for the class body; it may be copied + or wrapped before becoming the :attr:`~object.__dict__` attribute. + The following two statements create identical :class:`type` objects: >>> class X: ... a = 1 ... - >>> X = type('X', (object,), dict(a=1)) + >>> X = type('X', (), dict(a=1)) See also :ref:`bltin-type-objects`. + Keyword arguments provided to the three argument form are passed to the + appropriate metaclass machinery (usually :meth:`~object.__init_subclass__`) + in the same way that keywords in a class + definition (besides *metaclass*) would. + + See also :ref:`class-customization`. + .. versionchanged:: 3.6 Subclasses of :class:`type` which don't override ``type.__new__`` may no longer use the one-argument form to get the type of an object. diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 75c9d41b43acd0..e981bcdf6f2573 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -62,16 +62,26 @@ The :mod:`functools` module defines the following functions: Example:: class DataSet: + def __init__(self, sequence_of_numbers): - self._data = sequence_of_numbers + self._data = tuple(sequence_of_numbers) @cached_property def stdev(self): return statistics.stdev(self._data) - @cached_property - def variance(self): - return statistics.variance(self._data) + The mechanics of :func:`cached_property` are somewhat different from + :func:`property`. A regular property blocks attribute writes unless a + setter is defined. In contrast, a *cached_property* allows writes. + + The *cached_property* decorator only runs on lookups and only when an + attribute of the same name doesn't exist. When it does run, the + *cached_property* writes to the attribute with the same name. Subsequent + attribute reads and writes take precedence over the *cached_property* + method and it works like a normal attribute. + + The cached value can be cleared by deleting the attribute. This + allows the *cached_property* method to run again. Note, this decorator interferes with the operation of :pep:`412` key-sharing dictionaries. This means that instance dictionaries diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 2d85cd3431711a..69a1a8313b7593 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -72,6 +72,8 @@ The :mod:`gc` module provides the following functions: .. versionchanged:: 3.8 New *generation* parameter. + .. audit-event:: gc.get_objects generation gc.get_objects + .. function:: get_stats() Return a list of three per-generation dictionaries containing collection @@ -135,10 +137,13 @@ The :mod:`gc` module provides the following functions: resulting referrers. To get only currently live objects, call :func:`collect` before calling :func:`get_referrers`. - Care must be taken when using objects returned by :func:`get_referrers` because - some of them could still be under construction and hence in a temporarily - invalid state. Avoid using :func:`get_referrers` for any purpose other than - debugging. + .. warning:: + Care must be taken when using objects returned by :func:`get_referrers` because + some of them could still be under construction and hence in a temporarily + invalid state. Avoid using :func:`get_referrers` for any purpose other than + debugging. + + .. audit-event:: gc.get_referrers objs gc.get_referrers .. function:: get_referents(*objs) @@ -151,6 +156,7 @@ The :mod:`gc` module provides the following functions: be involved in a cycle. So, for example, if an integer is directly reachable from an argument, that integer object may or may not appear in the result list. + .. audit-event:: gc.get_referents objs gc.get_referents .. function:: is_tracked(obj) diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 3fdba6937c1de0..215f60d328c76a 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -65,6 +65,7 @@ For example, ``'[?]'`` matches the character ``'?'``. match. .. audit-event:: glob.glob pathname,recursive glob.glob + .. audit-event:: glob.glob/2 pathname,recursive,root_dir,dir_fd glob.glob .. note:: Using the "``**``" pattern in large directory trees may consume @@ -83,6 +84,13 @@ For example, ``'[?]'`` matches the character ``'?'``. without actually storing them all simultaneously. .. audit-event:: glob.glob pathname,recursive glob.iglob + .. audit-event:: glob.glob/2 pathname,recursive,root_dir,dir_fd glob.iglob + + .. versionchanged:: 3.5 + Support for recursive globs using "``**``". + + .. versionchanged:: 3.10 + Added the *root_dir* and *dir_fd* parameters. .. function:: escape(pathname) @@ -128,4 +136,3 @@ default. For example, consider a directory containing :file:`card.gif` and Module :mod:`fnmatch` Shell-style filename (not path) expansion - diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index d644974e660984..d77a2e5912d5e6 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -118,10 +118,10 @@ More condensed: Using :func:`new` with an algorithm provided by OpenSSL: - >>> h = hashlib.new('ripemd160') + >>> h = hashlib.new('sha512_256') >>> h.update(b"Nobody inspects the spammish repetition") >>> h.hexdigest() - 'cc4a5ce1b3df48aec5d22d1f16b894a0b894eccc' + '19197dc4d03829df858011c6c87600f994a858103bbc19005f20987aa19a97e2' Hashlib provides the following constant attributes: @@ -266,6 +266,12 @@ include a `salt `_. Python implementation uses an inline version of :mod:`hmac`. It is about three times slower and doesn't release the GIL. + .. deprecated:: 3.10 + + Slow Python implementation of *pbkdf2_hmac* is deprecated. In the + future the function will only be available when Python is compiled + with OpenSSL. + .. function:: scrypt(password, *, salt, n, r, p, maxmem=0, dklen=64) The function provides scrypt password-based key derivation function as @@ -280,8 +286,6 @@ include a `salt `_. factor and *maxmem* limits memory (OpenSSL 1.1.0 defaults to 32 MiB). *dklen* is the length of the derived key. - .. availability:: OpenSSL 1.1+. - .. versionadded:: 3.6 @@ -746,5 +750,5 @@ Domain Dedication 1.0 Universal: Wikipedia article with information on which algorithms have known issues and what that means regarding their use. - https://www.ietf.org/rfc/rfc2898.txt - PKCS #5: Password-Based Cryptography Specification Version 2.0 + https://www.ietf.org/rfc/rfc8018.txt + PKCS #5: Password-Based Cryptography Specification Version 2.1 diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 35997db2a9d27d..e605f7b8b14172 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -99,6 +99,11 @@ The module provides the following classes: :attr:`ssl.SSLContext.post_handshake_auth` for the default *context* or when *cert_file* is passed with a custom *context*. + .. versionchanged:: 3.10 + This class now sends an ALPN extension with protocol indicator + ``http/1.1`` when no *context* is given. Custom *context* should set + ALPN protocols with :meth:`~ssl.SSLContext.set_alpn_protocol`. + .. deprecated:: 3.6 *key_file* and *cert_file* are deprecated in favor of *context*. @@ -363,6 +368,8 @@ HTTPConnection Objects this is called automatically when making a request if the client does not already have a connection. + .. audit-event:: http.client.connect self,host,port http.client.HTTPConnection.connect + .. method:: HTTPConnection.close() @@ -432,6 +439,8 @@ also send your request step by step, by using the four functions below. :meth:`endheaders` method has been called and before :meth:`getresponse` is called. + .. audit-event:: http.client.send self,data http.client.HTTPConnection.send + .. _httpresponse-objects: diff --git a/Doc/library/http.rst b/Doc/library/http.rst index 14ee73363e62e0..1569d504c7f92a 100644 --- a/Doc/library/http.rst +++ b/Doc/library/http.rst @@ -35,7 +35,7 @@ associated messages through the :class:`http.HTTPStatus` enum: >>> from http import HTTPStatus >>> HTTPStatus.OK - + HTTPStatus.OK >>> HTTPStatus.OK == 200 True >>> HTTPStatus.OK.value @@ -45,7 +45,7 @@ associated messages through the :class:`http.HTTPStatus` enum: >>> HTTPStatus.OK.description 'Request fulfilled, document follows' >>> list(HTTPStatus) - [, , ...] + [HTTPStatus.CONTINUE, HTTPStatus.SWITCHING_PROTOCOLS, ...] .. _http-status-codes: diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 478a5b31475cfd..729d7e37137a81 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -98,7 +98,9 @@ provides three different variants: .. attribute:: path - Contains the request path. + Contains the request path. If query component of the URL is present, + then ``path`` includes the query. Using the terminology of :rfc:`3986`, + ``path`` here includes ``hier-part`` and the ``query``. .. attribute:: request_version diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index a59a5d3a465703..3c302115b5f408 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -250,7 +250,7 @@ View Last Restart Scroll the shell window to the last Shell restart. Restart Shell - Restart the shell to clean the environment. + Restart the shell to clean the environment and reset display and exception handling. Previous History Cycle through earlier commands in history which match the current entry. @@ -670,8 +670,16 @@ IDLE uses a socket to communicate between the IDLE GUI process and the user code execution process. A connection must be established whenever the Shell starts or restarts. (The latter is indicated by a divider line that says 'RESTART'). If the user process fails to connect to the GUI process, it -displays a ``Tk`` error box with a 'cannot connect' message that directs the -user here. It then exits. +usually displays a ``Tk`` error box with a 'cannot connect' message +that directs the user here. It then exits. + +One specific connection failure on Unix systems results from +misconfigured masquerading rules somewhere in a system's network setup. +When IDLE is started from a terminal, one will see a message starting +with ``** Invalid host:``. +The valid value is ``127.0.0.1 (idlelib.rpc.LOCALHOST)``. +One can diagnose with ``tcpconnect -irv 127.0.0.1 6543`` in one +terminal window and ``tcplisten `` in another. A common cause of failure is a user-written file with the same name as a standard library module, such as *random.py* and *tkinter.py*. When such a @@ -709,6 +717,13 @@ If IDLE quits with no message, and it was not started from a console, try starting it from a console or terminal (``python -m idlelib``) and see if this results in an error message. +On Unix-based systems with tcl/tk older than ``8.6.11`` (see +``About IDLE``) certain characters of certain fonts can cause +a tk failure with a message to the terminal. This can happen either +if one starts IDLE to edit a file with such a character or later +when entering such a character. If one cannot upgrade tcl/tk, +then re-configure IDLE to use a font that works better. + Running user code ^^^^^^^^^^^^^^^^^ @@ -717,7 +732,7 @@ intended to be the same as executing the same code by the default method, directly with Python in a text-mode system console or terminal window. However, the different interface and operation occasionally affect visible results. For instance, ``sys.modules`` starts with more entries, -and ``threading.activeCount()`` returns 2 instead of 1. +and ``threading.active_count()`` returns 2 instead of 1. By default, IDLE runs user code in a separate OS process rather than in the user interface process that runs the shell and editor. In the execution @@ -726,28 +741,38 @@ with objects that get input from and send output to the Shell window. The original values stored in ``sys.__stdin__``, ``sys.__stdout__``, and ``sys.__stderr__`` are not touched, but may be ``None``. -When Shell has the focus, it controls the keyboard and screen. This is -normally transparent, but functions that directly access the keyboard -and screen will not work. These include system-specific functions that -determine whether a key has been pressed and if so, which. +Sending print output from one process to a text widget in another is +slower than printing to a system terminal in the same process. +This has the most effect when printing multiple arguments, as the string +for each argument, each separator, the newline are sent separately. +For development, this is usually not a problem, but if one wants to +print faster in IDLE, format and join together everything one wants +displayed together and then print a single string. Both format strings +and :meth:`str.join` can help combine fields and lines. IDLE's standard stream replacements are not inherited by subprocesses -created in the execution process, whether directly by user code or by modules -such as multiprocessing. If such subprocess use ``input`` from sys.stdin -or ``print`` or ``write`` to sys.stdout or sys.stderr, +created in the execution process, whether directly by user code or by +modules such as multiprocessing. If such subprocess use ``input`` from +sys.stdin or ``print`` or ``write`` to sys.stdout or sys.stderr, IDLE should be started in a command line window. The secondary subprocess will then be attached to that window for input and output. -The IDLE code running in the execution process adds frames to the call stack -that would not be there otherwise. IDLE wraps ``sys.getrecursionlimit`` and -``sys.setrecursionlimit`` to reduce the effect of the additional stack frames. - If ``sys`` is reset by user code, such as with ``importlib.reload(sys)``, IDLE's changes are lost and input from the keyboard and output to the screen will not work correctly. -When user code raises SystemExit either directly or by calling sys.exit, IDLE -returns to a Shell prompt instead of exiting. +When Shell has the focus, it controls the keyboard and screen. This is +normally transparent, but functions that directly access the keyboard +and screen will not work. These include system-specific functions that +determine whether a key has been pressed and if so, which. + +The IDLE code running in the execution process adds frames to the call stack +that would not be there otherwise. IDLE wraps ``sys.getrecursionlimit`` and +``sys.setrecursionlimit`` to reduce the effect of the additional stack +frames. + +When user code raises SystemExit either directly or by calling sys.exit, +IDLE returns to a Shell prompt instead of exiting. User output in Shell ^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 02ecfd95d43767..65681ec093598c 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -174,9 +174,9 @@ example of usage. .. seealso:: - Documents describing the protocol, and sources and binaries for servers - implementing it, can all be found at the University of Washington's *IMAP - Information Center* (https://www.washington.edu/imap/). + Documents describing the protocol, sources for servers + implementing it, by the University of Washington's IMAP Information Center + can all be found at (**Source Code**) https://github.com/uw-imap/imap (**Not Maintained**). .. _imap4-objects: diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 21da143f3bebf9..9bedee5af28f69 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -4,6 +4,13 @@ Using :mod:`!importlib.metadata` ================================= +.. module:: importlib.metadata + :synopsis: The implementation of the importlib metadata. + +.. versionadded:: 3.8 + +**Source code:** :source:`Lib/importlib/metadata.py` + .. note:: This functionality is provisional and may deviate from the usual version semantics of the standard library. @@ -74,18 +81,48 @@ This package provides the following functionality via its public API. Entry points ------------ -The ``entry_points()`` function returns a dictionary of all entry points, -keyed by group. Entry points are represented by ``EntryPoint`` instances; +The ``entry_points()`` function returns a collection of entry points. +Entry points are represented by ``EntryPoint`` instances; each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and a ``.load()`` method to resolve the value. There are also ``.module``, ``.attr``, and ``.extras`` attributes for getting the components of the -``.value`` attribute:: +``.value`` attribute. + +Query all entry points:: >>> eps = entry_points() # doctest: +SKIP - >>> list(eps) # doctest: +SKIP + +The ``entry_points()`` function returns an ``EntryPoints`` object, +a sequence of all ``EntryPoint`` objects with ``names`` and ``groups`` +attributes for convenience:: + + >>> sorted(eps.groups) # doctest: +SKIP ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] - >>> scripts = eps['console_scripts'] # doctest: +SKIP - >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0] # doctest: +SKIP + +``EntryPoints`` has a ``select`` method to select entry points +matching specific properties. Select entry points in the +``console_scripts`` group:: + + >>> scripts = eps.select(group='console_scripts') # doctest: +SKIP + +Equivalently, since ``entry_points`` passes keyword arguments +through to select:: + + >>> scripts = entry_points(group='console_scripts') # doctest: +SKIP + +Pick out a specific script named "wheel" (found in the wheel project):: + + >>> 'wheel' in scripts.names # doctest: +SKIP + True + >>> wheel = scripts['wheel'] # doctest: +SKIP + +Equivalently, query for that entry point during selection:: + + >>> (wheel,) = entry_points(group='console_scripts', name='wheel') # doctest: +SKIP + >>> (wheel,) = entry_points().select(group='console_scripts', name='wheel') # doctest: +SKIP + +Inspect the resolved entry point:: + >>> wheel # doctest: +SKIP EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') >>> wheel.module # doctest: +SKIP @@ -104,6 +141,17 @@ group. Read `the setuptools docs `_ for more information on entry points, their definition, and usage. +*Compatibility Note* + +The "selectable" entry points were introduced in ``importlib_metadata`` +3.6 and Python 3.10. Prior to those changes, ``entry_points`` accepted +no parameters and always returned a dictionary of entry points, keyed +by group. For compatibility, if no parameters are passed to entry_points, +a ``SelectableGroups`` object is returned, implementing that dict +interface. In the future, calling ``entry_points`` with no parameters +will return an ``EntryPoints`` object. Users should rely on the selection +interface to retrieve entry points by group. + .. _metadata: @@ -115,12 +163,26 @@ Every distribution includes some metadata, which you can extract using the >>> wheel_metadata = metadata('wheel') # doctest: +SKIP -The keys of the returned data structure [#f1]_ name the metadata keywords, and -their values are returned unparsed from the distribution metadata:: +The keys of the returned data structure, a ``PackageMetadata``, +name the metadata keywords, and +the values are returned unparsed from the distribution metadata:: >>> wheel_metadata['Requires-Python'] # doctest: +SKIP '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' +``PackageMetadata`` also presents a ``json`` attribute that returns +all the metadata in a JSON-compatible form per :PEP:`566`:: + + >>> wheel_metadata.json['requires_python'] + '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' + +.. versionchanged:: 3.10 + The ``Description`` is now included in the metadata when presented + through the payload. Line continuation characters have been removed. + +.. versionadded:: 3.10 + The ``json`` attribute was added. + .. _version: @@ -142,7 +204,7 @@ Distribution files You can also get the full set of files contained within a distribution. The ``files()`` function takes a distribution package name and returns all of the files installed by this distribution. Each file object returned is a -``PackagePath``, a :class:`pathlib.Path` derived object with additional ``dist``, +``PackagePath``, a :class:`pathlib.PurePath` derived object with additional ``dist``, ``size``, and ``hash`` properties as indicated by the metadata. For example:: >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] # doctest: +SKIP @@ -166,6 +228,12 @@ Once you have the file, you can also read its contents:: return s.encode('utf-8') return s +You can also use the ``locate`` method to get a the absolute path to the +file:: + + >>> util.locate() # doctest: +SKIP + PosixPath('/home/gustav/example/lib/site-packages/wheel/util.py') + In the case where the metadata file listing files (RECORD or SOURCES.txt) is missing, ``files()`` will return ``None``. The caller may wish to wrap calls to @@ -186,6 +254,19 @@ function:: ["pytest (>=3.0.0) ; extra == 'test'", "pytest-cov ; extra == 'test'"] +Package distributions +--------------------- + +A convience method to resolve the distribution or +distributions (in the case of a namespace package) for top-level +Python packages or modules:: + + >>> packages_distributions() + {'importlib_metadata': ['importlib-metadata'], 'yaml': ['PyYAML'], 'jaraco': ['jaraco.classes', 'jaraco.functools'], ...} + +.. versionadded:: 3.10 + + Distributions ============= @@ -206,9 +287,9 @@ Thus, an alternative way to get the version number is through the There are all kinds of additional metadata available on the ``Distribution`` instance:: - >>> d.metadata['Requires-Python'] # doctest: +SKIP + >>> dist.metadata['Requires-Python'] # doctest: +SKIP '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' - >>> d.metadata['License'] # doctest: +SKIP + >>> dist.metadata['License'] # doctest: +SKIP 'MIT' The full set of available metadata is not described here. See :pep:`566` @@ -259,9 +340,3 @@ a custom finder, return instances of this derived ``Distribution`` in the .. rubric:: Footnotes - -.. [#f1] Technically, the returned distribution metadata object is an - :class:`email.message.EmailMessage` - instance, but this is an implementation detail, and not part of the - stable API. You should only use dictionary-like methods and syntax - to access the metadata contents. diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 45b62aec9ef537..50297e0d73e7c8 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -208,7 +208,7 @@ Functions .. versionadded:: 3.4 .. versionchanged:: 3.7 :exc:`ModuleNotFoundError` is raised when the module being reloaded lacks - a :class:`ModuleSpec`. + a :class:`~importlib.machinery.ModuleSpec`. :mod:`importlib.abc` -- Abstract base classes related to import @@ -257,6 +257,10 @@ ABC hierarchy:: Returns ``None`` when called instead of raising :exc:`NotImplementedError`. + .. deprecated:: 3.10 + Implement :meth:`MetaPathFinder.find_spec` or + :meth:`PathEntryFinder.find_spec` instead. + .. class:: MetaPathFinder @@ -265,6 +269,9 @@ ABC hierarchy:: .. versionadded:: 3.3 + .. versionchanged:: 3.10 + No longer a subclass of :class:`Finder`. + .. method:: find_spec(fullname, path, target=None) An abstract method for finding a :term:`spec ` for @@ -313,11 +320,13 @@ ABC hierarchy:: An abstract base class representing a :term:`path entry finder`. Though it bears some similarities to :class:`MetaPathFinder`, ``PathEntryFinder`` is meant for use only within the path-based import subsystem provided - by :class:`PathFinder`. This ABC is a subclass of :class:`Finder` for - compatibility reasons only. + by :class:`importlib.machinery.PathFinder`. .. versionadded:: 3.3 + .. versionchanged:: 3.10 + No longer a subclass of :class:`Finder`. + .. method:: find_spec(fullname, target=None) An abstract method for finding a :term:`spec ` for @@ -363,7 +372,8 @@ ABC hierarchy:: .. method:: invalidate_caches() An optional method which, when called, should invalidate any internal - cache used by the finder. Used by :meth:`PathFinder.invalidate_caches` + cache used by the finder. Used by + :meth:`importlib.machinery.PathFinder.invalidate_caches` when invalidating the caches of all cached finders. @@ -891,6 +901,22 @@ The following functions are available. .. versionadded:: 3.9 +.. function:: as_file(traversable) + + Given a :class:`importlib.resources.abc.Traversable` object representing + a file, typically from :func:`importlib.resources.files`, return + a context manager for use in a :keyword:`with` statement. + The context manager provides a :class:`pathlib.Path` object. + + Exiting the context manager cleans up any temporary file created when the + resource was extracted from e.g. a zip file. + + Use ``as_file`` when the Traversable methods + (``read_text``, etc) are insufficient and an actual file on + the file system is required. + + .. versionadded:: 3.9 + .. function:: open_binary(package, resource) Open for binary reading the *resource* within *package*. @@ -1138,7 +1164,7 @@ find and load modules. directory for ``''`` (i.e. the empty string). -.. class:: FileFinder(path, \*loader_details) +.. class:: FileFinder(path, *loader_details) A concrete implementation of :class:`importlib.abc.PathEntryFinder` which caches results from the file system. @@ -1177,11 +1203,14 @@ find and load modules. Attempt to find the loader to handle *fullname* within :attr:`path`. + .. deprecated:: 3.10 + Use :meth:`find_spec` instead. + .. method:: invalidate_caches() Clear out the internal cache. - .. classmethod:: path_hook(\*loader_details) + .. classmethod:: path_hook(*loader_details) A class method which returns a closure for use on :attr:`sys.path_hooks`. An instance of :class:`FileFinder` is returned by the closure using the @@ -1476,7 +1505,7 @@ an :term:`importer`. If **name** has no leading dots, then **name** is simply returned. This allows for usage such as - ``importlib.util.resolve_name('sys', __package__)`` without doing a + ``importlib.util.resolve_name('sys', __spec__.parent)`` without doing a check to see if the **package** argument is needed. :exc:`ImportError` is raised if **name** is a relative module name but @@ -1591,9 +1620,9 @@ an :term:`importer`. .. function:: spec_from_loader(name, loader, *, origin=None, is_package=None) - A factory function for creating a :class:`ModuleSpec` instance based - on a loader. The parameters have the same meaning as they do for - ModuleSpec. The function uses available :term:`loader` APIs, such as + A factory function for creating a :class:`~importlib.machinery.ModuleSpec` + instance based on a loader. The parameters have the same meaning as they do + for ModuleSpec. The function uses available :term:`loader` APIs, such as :meth:`InspectLoader.is_package`, to fill in any missing information on the spec. @@ -1601,9 +1630,9 @@ an :term:`importer`. .. function:: spec_from_file_location(name, location, *, loader=None, submodule_search_locations=None) - A factory function for creating a :class:`ModuleSpec` instance based - on the path to a file. Missing information will be filled in on the - spec by making use of loader APIs and by the implication that the + A factory function for creating a :class:`~importlib.machinery.ModuleSpec` + instance based on the path to a file. Missing information will be filled in + on the spec by making use of loader APIs and by the implication that the module will be file-based. .. versionadded:: 3.4 diff --git a/Doc/library/index.rst b/Doc/library/index.rst index bebf7429b0e63e..1a9e52ec5ccfba 100644 --- a/Doc/library/index.rst +++ b/Doc/library/index.rst @@ -56,7 +56,6 @@ the `Python Package Index `_. crypto.rst allos.rst concurrency.rst - contextvars.rst ipc.rst netdata.rst markup.rst @@ -72,7 +71,6 @@ the `Python Package Index `_. custominterp.rst modules.rst language.rst - misc.rst windows.rst unix.rst superseded.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index d00a30ff004063..b9e8be1234e259 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -95,6 +95,8 @@ attributes: | | __globals__ | global namespace in which | | | | this function was defined | +-----------+-------------------+---------------------------+ +| | __builtins__ | builtins namespace | ++-----------+-------------------+---------------------------+ | | __annotations__ | mapping of parameters | | | | names to annotations; | | | | ``"return"`` key is | @@ -251,6 +253,10 @@ attributes: Add ``cr_origin`` attribute to coroutines. +.. versionchanged:: 3.10 + + Add ``__builtins__`` attribute to functions. + .. function:: getmembers(object[, predicate]) Return all the members of an object in a list of ``(name, value)`` @@ -556,7 +562,7 @@ The Signature object represents the call signature of a callable object and its return annotation. To retrieve a Signature object, use the :func:`signature` function. -.. function:: signature(callable, \*, follow_wrapped=True) +.. function:: signature(callable, *, follow_wrapped=True, globals=None, locals=None, eval_str=False) Return a :class:`Signature` object for the given ``callable``:: @@ -578,8 +584,20 @@ function. Accepts a wide range of Python callables, from plain functions and classes to :func:`functools.partial` objects. + For objects defined in modules using stringized annotations + (``from __future__ import annotations``), :func:`signature` will + attempt to automatically un-stringize the annotations using + :func:`inspect.get_annotations()`. The + ``global``, ``locals``, and ``eval_str`` parameters are passed + into :func:`inspect.get_annotations()` when resolving the + annotations; see the documentation for :func:`inspect.get_annotations()` + for instructions on how to use these parameters. + Raises :exc:`ValueError` if no signature can be provided, and - :exc:`TypeError` if that type of object is not supported. + :exc:`TypeError` if that type of object is not supported. Also, + if the annotations are stringized, and ``eval_str`` is not false, + the ``eval()`` call(s) to un-stringize the annotations could + potentially raise any kind of exception. A slash(/) in the signature of a function denotes that the parameters prior to it are positional-only. For more info, see @@ -590,6 +608,9 @@ function. ``callable`` specifically (``callable.__wrapped__`` will not be used to unwrap decorated callables.) + .. versionadded:: 3.10 + ``globals``, ``locals``, and ``eval_str`` parameters. + .. note:: Some callables may not be introspectable in certain implementations of @@ -597,7 +618,7 @@ function. C provide no metadata about their arguments. -.. class:: Signature(parameters=None, \*, return_annotation=Signature.empty) +.. class:: Signature(parameters=None, *, return_annotation=Signature.empty) A Signature object represents the call signature of a function and its return annotation. For each parameter accepted by the function it stores a @@ -668,11 +689,12 @@ function. >>> str(new_sig) "(a, b) -> 'new return anno'" - .. classmethod:: Signature.from_callable(obj, \*, follow_wrapped=True) + .. classmethod:: Signature.from_callable(obj, *, follow_wrapped=True, globalns=None, localns=None) Return a :class:`Signature` (or its subclass) object for a given callable ``obj``. Pass ``follow_wrapped=False`` to get a signature of ``obj`` - without unwrapping its ``__wrapped__`` chain. + without unwrapping its ``__wrapped__`` chain. ``globalns`` and + ``localns`` will be used as the namespaces when resolving annotations. This method simplifies subclassing of :class:`Signature`:: @@ -683,8 +705,11 @@ function. .. versionadded:: 3.5 + .. versionadded:: 3.10 + ``globalns`` and ``localns`` parameters. + -.. class:: Parameter(name, kind, \*, default=Parameter.empty, annotation=Parameter.empty) +.. class:: Parameter(name, kind, *, default=Parameter.empty, annotation=Parameter.empty) Parameter objects are *immutable*. Instead of modifying a Parameter object, you can use :meth:`Parameter.replace` to create a modified copy. @@ -1093,6 +1118,62 @@ Classes and functions .. versionadded:: 3.4 +.. function:: get_annotations(obj, *, globals=None, locals=None, eval_str=False) + + Compute the annotations dict for an object. + + ``obj`` may be a callable, class, or module. + Passing in an object of any other type raises :exc:`TypeError`. + + Returns a dict. ``get_annotations()`` returns a new dict every time + it's called; calling it twice on the same object will return two + different but equivalent dicts. + + This function handles several details for you: + + * If ``eval_str`` is true, values of type ``str`` will + be un-stringized using :func:`eval()`. This is intended + for use with stringized annotations + (``from __future__ import annotations``). + * If ``obj`` doesn't have an annotations dict, returns an + empty dict. (Functions and methods always have an + annotations dict; classes, modules, and other types of + callables may not.) + * Ignores inherited annotations on classes. If a class + doesn't have its own annotations dict, returns an empty dict. + * All accesses to object members and dict values are done + using ``getattr()`` and ``dict.get()`` for safety. + * Always, always, always returns a freshly-created dict. + + ``eval_str`` controls whether or not values of type ``str`` are replaced + with the result of calling :func:`eval()` on those values: + + * If eval_str is true, :func:`eval()` is called on values of type ``str``. + (Note that ``get_annotations`` doesn't catch exceptions; if :func:`eval()` + raises an exception, it will unwind the stack past the ``get_annotations`` + call.) + * If eval_str is false (the default), values of type ``str`` are unchanged. + + ``globals`` and ``locals`` are passed in to :func:`eval()`; see the documentation + for :func:`eval()` for more information. If ``globals`` or ``locals`` + is ``None``, this function may replace that value with a context-specific + default, contingent on ``type(obj)``: + + * If ``obj`` is a module, ``globals`` defaults to ``obj.__dict__``. + * If ``obj`` is a class, ``globals`` defaults to + ``sys.modules[obj.__module__].__dict__`` and ``locals`` defaults + to the ``obj`` class namespace. + * If ``obj`` is a callable, ``globals`` defaults to ``obj.__globals__``, + although if ``obj`` is a wrapped function (using + ``functools.update_wrapper()``) it is first unwrapped. + + Calling ``get_annotations`` is best practice for accessing the + annotations dict of any object. See :ref:`annotations-howto` for + more information on annotations best practices. + + .. versionadded:: 3.10 + + .. _inspect-stack: The interpreter stack diff --git a/Doc/library/io.rst b/Doc/library/io.rst index aecbec56866d73..f9ffc19fac489d 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -106,6 +106,56 @@ stream by opening a file in binary mode with buffering disabled:: The raw stream API is described in detail in the docs of :class:`RawIOBase`. +.. _io-text-encoding: + +Text Encoding +------------- + +The default encoding of :class:`TextIOWrapper` and :func:`open` is +locale-specific (:func:`locale.getpreferredencoding(False) `). + +However, many developers forget to specify the encoding when opening text files +encoded in UTF-8 (e.g. JSON, TOML, Markdown, etc...) since most Unix +platforms use UTF-8 locale by default. This causes bugs because the locale +encoding is not UTF-8 for most Windows users. For example:: + + # May not work on Windows when non-ASCII characters in the file. + with open("README.md") as f: + long_description = f.read() + +Additionally, while there is no concrete plan as of yet, Python may change +the default text file encoding to UTF-8 in the future. + +Accordingly, it is highly recommended that you specify the encoding +explicitly when opening text files. If you want to use UTF-8, pass +``encoding="utf-8"``. To use the current locale encoding, +``encoding="locale"`` is supported in Python 3.10. + +When you need to run existing code on Windows that attempts to opens +UTF-8 files using the default locale encoding, you can enable the UTF-8 +mode. See :ref:`UTF-8 mode on Windows `. + +.. _io-encoding-warning: + +Opt-in EncodingWarning +^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 3.10 + See :pep:`597` for more details. + +To find where the default locale encoding is used, you can enable +the ``-X warn_default_encoding`` command line option or set the +:envvar:`PYTHONWARNDEFAULTENCODING` environment variable, which will +emit an :exc:`EncodingWarning` when the default encoding is used. + +If you are providing an API that uses :func:`open` or +:class:`TextIOWrapper` and passes ``encoding=None`` as a parameter, you +can use :func:`text_encoding` so that callers of the API will emit an +:exc:`EncodingWarning` if they don't pass an ``encoding``. However, +please consider using UTF-8 by default (i.e. ``encoding="utf-8"``) for +new APIs. + + High-level Module Interface --------------------------- @@ -143,6 +193,32 @@ High-level Module Interface .. versionadded:: 3.8 +.. function:: text_encoding(encoding, stacklevel=2) + + This is a helper function for callables that use :func:`open` or + :class:`TextIOWrapper` and have an ``encoding=None`` parameter. + + This function returns *encoding* if it is not ``None`` and ``"locale"`` if + *encoding* is ``None``. + + This function emits an :class:`EncodingWarning` if + :data:`sys.flags.warn_default_encoding ` is true and *encoding* + is None. *stacklevel* specifies where the warning is emitted. + For example:: + + def read_text(path, encoding=None): + encoding = io.text_encoding(encoding) # stacklevel=2 + with open(path, encoding) as f: + return f.read() + + In this example, an :class:`EncodingWarning` is emitted for the caller of + ``read_text()``. + + See :ref:`io-text-encoding` for more information. + + .. versionadded:: 3.10 + + .. exception:: BlockingIOError This is a compatibility alias for the builtin :exc:`BlockingIOError` @@ -155,16 +231,6 @@ High-level Module Interface when an unsupported operation is called on a stream. -In-memory streams -^^^^^^^^^^^^^^^^^ - -It is also possible to use a :class:`str` or :term:`bytes-like object` as a -file for both reading and writing. For strings :class:`StringIO` can be used -like a file opened in text mode. :class:`BytesIO` can be used like a file -opened in binary mode. Both provide full read-write capabilities with random -access. - - .. seealso:: :mod:`sys` @@ -879,6 +945,8 @@ Text I/O *encoding* gives the name of the encoding that the stream will be decoded or encoded with. It defaults to :func:`locale.getpreferredencoding(False) `. + ``encoding="locale"`` can be used to specify the current locale's encoding + explicitly. See :ref:`io-text-encoding` for more information. *errors* is an optional string that specifies how encoding and decoding errors are to be handled. Pass ``'strict'`` to raise a :exc:`ValueError` @@ -930,6 +998,9 @@ Text I/O locale encoding using :func:`locale.setlocale`, use the current locale encoding instead of the user preferred encoding. + .. versionchanged:: 3.10 + The *encoding* argument now supports the ``"locale"`` dummy encoding name. + :class:`TextIOWrapper` provides these data attributes and methods in addition to those from :class:`TextIOBase` and :class:`IOBase`: diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 5f5e66412da477..1c2263b128a8fe 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -104,8 +104,7 @@ write code that handles both IP versions correctly. Address objects are 1. A string in decimal-dot notation, consisting of four decimal integers in the inclusive range 0--255, separated by dots (e.g. ``192.168.0.1``). Each integer represents an octet (byte) in the address. Leading zeroes are - tolerated only for values less than 8 (as there is no ambiguity - between the decimal and octal interpretations of such strings). + not tolerated to prevent confusion with octal notation. 2. An integer that fits into 32 bits. 3. An integer packed into a :class:`bytes` object of length 4 (most significant octet first). @@ -117,6 +116,22 @@ write code that handles both IP versions correctly. Address objects are >>> ipaddress.IPv4Address(b'\xC0\xA8\x00\x01') IPv4Address('192.168.0.1') + .. versionchanged:: 3.8 + + Leading zeros are tolerated, even in ambiguous cases that look like + octal notation. + + .. versionchanged:: 3.10 + + Leading zeros are no longer tolerated and are treated as an error. + IPv4 address strings are now parsed as strict as glibc + :func:`~socket.inet_pton`. + + .. versionchanged:: 3.9.5 + + The above change was also included in Python 3.9 starting with + version 3.9.5. + .. attribute:: version The appropriate version number: ``4`` for IPv4, ``6`` for IPv6. @@ -202,6 +217,32 @@ write code that handles both IP versions correctly. Address objects are .. _iana-ipv4-special-registry: https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml .. _iana-ipv6-special-registry: https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml +.. method:: IPv4Address.__format__(fmt) + + Returns a string representation of the IP address, controlled by + an explicit format string. + *fmt* can be one of the following: ``'s'``, the default option, + equivalent to :func:`str`, ``'b'`` for a zero-padded binary string, + ``'X'`` or ``'x'`` for an uppercase or lowercase hexadecimal + representation, or ``'n'``, which is equivalent to ``'b'`` for IPv4 + addresses and ``'x'`` for IPv6. For binary and hexadecimal + representations, the form specifier ``'#'`` and the grouping option + ``'_'`` are available. ``__format__`` is used by ``format``, ``str.format`` + and f-strings. + + >>> format(ipaddress.IPv4Address('192.168.0.1')) + '192.168.0.1' + >>> '{:#b}'.format(ipaddress.IPv4Address('192.168.0.1')) + '0b11000000101010000000000000000001' + >>> f'{ipaddress.IPv6Address("2001:db8::1000"):s}' + '2001:db8::1000' + >>> format(ipaddress.IPv6Address('2001:db8::1000'), '_X') + '2001_0DB8_0000_0000_0000_0000_0000_1000' + >>> '{:#_n}'.format(ipaddress.IPv6Address('2001:db8::1000')) + '0x2001_0db8_0000_0000_0000_0000_0000_1000' + + .. versionadded:: 3.9 + .. class:: IPv6Address(address) @@ -246,8 +287,8 @@ write code that handles both IP versions correctly. Address objects are groups consisting entirely of zeroes included. - For the following attributes, see the corresponding documentation of the - :class:`IPv4Address` class: + For the following attributes and methods, see the corresponding + documentation of the :class:`IPv4Address` class: .. attribute:: packed .. attribute:: reverse_pointer @@ -297,6 +338,12 @@ write code that handles both IP versions correctly. Address objects are the embedded ``(server, client)`` IP address pair. For any other address, this property will be ``None``. +.. method:: IPv6Address.__format__(fmt) + + Refer to the corresponding method documentation in + :class:`IPv4Address`. + + .. versionadded:: 3.9 Conversion to Strings and Integers ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 107bc515a67785..fd77f99a88f577 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -55,6 +55,7 @@ Iterator Arguments Results :func:`filterfalse` pred, seq elements of seq where pred(elem) is false ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8`` :func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) :func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) --> C D E F G`` +:func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') --> AB BC CD DE EF FG`` :func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000`` :func:`takewhile` pred, seq seq[0], seq[1], until pred fails ``takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4`` :func:`tee` it, n it1, it2, ... itn splits one iterator into n @@ -475,6 +476,22 @@ loops that truncate the stream. If *start* is ``None``, then iteration starts at zero. If *step* is ``None``, then the step defaults to one. +.. function:: pairwise(iterable) + + Return successive overlapping pairs taken from the input *iterable*. + + The number of 2-tuples in the output iterator will be one fewer than the + number of inputs. It will be empty if the input iterable has fewer than + two values. + + Roughly equivalent to:: + + def pairwise(iterable): + # pairwise('ABCDEFG') --> AB BC CD DE EF FG + a, b = tee(iterable) + next(b, None) + return zip(a, b) + .. function:: permutations(iterable, r=None) @@ -565,7 +582,7 @@ loops that truncate the stream. Before :func:`product` runs, it completely consumes the input iterables, keeping pools of values in memory to generate the products. Accordingly, - it only useful with finite inputs. + it is only useful with finite inputs. .. function:: repeat(object[, times]) @@ -755,7 +772,7 @@ which incur interpreter overhead. "Count how many times the predicate is true" return sum(map(pred, iterable)) - def padnone(iterable): + def pad_none(iterable): """Returns the sequence elements and then returns None indefinitely. Useful for emulating the behavior of the built-in map() function. @@ -769,6 +786,18 @@ which incur interpreter overhead. def dotproduct(vec1, vec2): return sum(map(operator.mul, vec1, vec2)) + def convolve(signal, kernel): + # See: https://betterexplained.com/articles/intuitive-convolution/ + # convolve(data, [0.25, 0.25, 0.25, 0.25]) --> Moving average (blur) + # convolve(data, [1, -1]) --> 1st finite difference (1st derivative) + # convolve(data, [1, -2, 1]) --> 2nd finite difference (2nd derivative) + kernel = tuple(kernel)[::-1] + n = len(kernel) + window = collections.deque([0], maxlen=n) * n + for x in chain(signal, repeat(0, n-1)): + window.append(x) + yield sum(map(operator.mul, kernel, window)) + def flatten(list_of_lists): "Flatten one level of nesting" return chain.from_iterable(list_of_lists) @@ -782,12 +811,6 @@ which incur interpreter overhead. return starmap(func, repeat(args)) return starmap(func, repeat(args, times)) - def pairwise(iterable): - "s -> (s0,s1), (s1,s2), (s2, s3), ..." - a, b = tee(iterable) - next(b, None) - return zip(a, b) - def grouper(iterable, n, fillvalue=None): "Collect data into fixed-length chunks or blocks" # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx" @@ -809,7 +832,7 @@ which incur interpreter overhead. nexts = cycle(islice(nexts, num_active)) def partition(pred, iterable): - 'Use a predicate to partition entries into false entries and true entries' + "Use a predicate to partition entries into false entries and true entries" # partition(is_odd, range(10)) --> 0 2 4 6 8 and 1 3 5 7 9 t1, t2 = tee(iterable) return filterfalse(pred, t1), filter(pred, t2) @@ -881,7 +904,7 @@ which incur interpreter overhead. def random_product(*args, repeat=1): "Random selection from itertools.product(*args, **kwds)" pools = [tuple(pool) for pool in args] * repeat - return tuple(random.choice(pool) for pool in pools) + return tuple(map(random.choice, pools)) def random_permutation(iterable, r=None): "Random selection from itertools.permutations(iterable, r)" @@ -900,11 +923,11 @@ which incur interpreter overhead. "Random selection from itertools.combinations_with_replacement(iterable, r)" pool = tuple(iterable) n = len(pool) - indices = sorted(random.randrange(n) for i in range(r)) + indices = sorted(random.choices(range(n), k=r)) return tuple(pool[i] for i in indices) def nth_combination(iterable, r, index): - 'Equivalent to list(combinations(iterable, r))[index]' + "Equivalent to list(combinations(iterable, r))[index]" pool = tuple(iterable) n = len(pool) if r < 0 or r > n: diff --git a/Doc/library/json.rst b/Doc/library/json.rst index e1a246aad6190b..c8184da80fe43e 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -333,7 +333,7 @@ Encoders and Decoders *object_hook*, if specified, will be called with the result of every JSON object decoded and its return value will be used in place of the given :class:`dict`. This can be used to provide custom deserializations (e.g. to - support JSON-RPC class hinting). + support `JSON-RPC `_ class hinting). *object_pairs_hook*, if specified will be called with the result of every JSON object decoded with an ordered list of pairs. The return value of @@ -422,10 +422,9 @@ Encoders and Decoders for ``o`` if possible, otherwise it should call the superclass implementation (to raise :exc:`TypeError`). - If *skipkeys* is false (the default), then it is a :exc:`TypeError` to - attempt encoding of keys that are not :class:`str`, :class:`int`, - :class:`float` or ``None``. If *skipkeys* is true, such items are simply - skipped. + If *skipkeys* is false (the default), a :exc:`TypeError` will be raised when + trying to encode keys that are not :class:`str`, :class:`int`, :class:`float` + or ``None``. If *skipkeys* is true, such items are simply skipped. If *ensure_ascii* is true (the default), the output is guaranteed to have all incoming non-ASCII characters escaped. If *ensure_ascii* is @@ -479,8 +478,8 @@ Encoders and Decoders object for *o*, or calls the base implementation (to raise a :exc:`TypeError`). - For example, to support arbitrary iterators, you could implement default - like this:: + For example, to support arbitrary iterators, you could implement + :meth:`default` like this:: def default(self, o): try: diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index 678148a0dda294..60d0c59d017c73 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -315,21 +315,25 @@ The :mod:`locale` module defines the following exception and functions: .. function:: getpreferredencoding(do_setlocale=True) - Return the encoding used for text data, according to user preferences. User - preferences are expressed differently on different systems, and might not be - available programmatically on some systems, so this function only returns a - guess. + Return the :term:`locale encoding` used for text data, according to user + preferences. User preferences are expressed differently on different + systems, and might not be available programmatically on some systems, so + this function only returns a guess. - On some systems, it is necessary to invoke :func:`setlocale` to obtain the user - preferences, so this function is not thread-safe. If invoking setlocale is not - necessary or desired, *do_setlocale* should be set to ``False``. + On some systems, it is necessary to invoke :func:`setlocale` to obtain the + user preferences, so this function is not thread-safe. If invoking setlocale + is not necessary or desired, *do_setlocale* should be set to ``False``. - On Android or in the UTF-8 mode (:option:`-X` ``utf8`` option), always - return ``'UTF-8'``, the locale and the *do_setlocale* argument are ignored. + On Android or if the :ref:`Python UTF-8 Mode ` is enabled, always + return ``'UTF-8'``, the :term:`locale encoding` and the *do_setlocale* + argument are ignored. + + The :ref:`Python preinitialization ` configures the LC_CTYPE + locale. See also the :term:`filesystem encoding and error handler`. .. versionchanged:: 3.7 - The function now always returns ``UTF-8`` on Android or if the UTF-8 mode - is enabled. + The function now always returns ``UTF-8`` on Android or if the + :ref:`Python UTF-8 Mode ` is enabled. .. function:: normalize(localename) @@ -423,6 +427,14 @@ The :mod:`locale` module defines the following exception and functions: .. versionadded:: 3.5 +.. function:: localize(string, grouping=False, monetary=False) + + Converts a normalized number string into a formatted string following the + :const:`LC_NUMERIC` settings. + + .. versionadded:: 3.10 + + .. function:: atof(string) Converts a string to a floating point number, following the :const:`LC_NUMERIC` diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 683d6ed5e8ba52..f833bcdd1fcf5b 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -35,49 +35,49 @@ in :mod:`logging` itself) and defining handlers which are declared either in .. function:: dictConfig(config) - Takes the logging configuration from a dictionary. The contents of - this dictionary are described in :ref:`logging-config-dictschema` - below. - - If an error is encountered during configuration, this function will - raise a :exc:`ValueError`, :exc:`TypeError`, :exc:`AttributeError` - or :exc:`ImportError` with a suitably descriptive message. The - following is a (possibly incomplete) list of conditions which will - raise an error: - - * A ``level`` which is not a string or which is a string not - corresponding to an actual logging level. - * A ``propagate`` value which is not a boolean. - * An id which does not have a corresponding destination. - * A non-existent handler id found during an incremental call. - * An invalid logger name. - * Inability to resolve to an internal or external object. - - Parsing is performed by the :class:`DictConfigurator` class, whose - constructor is passed the dictionary used for configuration, and - has a :meth:`configure` method. The :mod:`logging.config` module - has a callable attribute :attr:`dictConfigClass` - which is initially set to :class:`DictConfigurator`. - You can replace the value of :attr:`dictConfigClass` with a - suitable implementation of your own. - - :func:`dictConfig` calls :attr:`dictConfigClass` passing - the specified dictionary, and then calls the :meth:`configure` method on - the returned object to put the configuration into effect:: - - def dictConfig(config): - dictConfigClass(config).configure() - - For example, a subclass of :class:`DictConfigurator` could call - ``DictConfigurator.__init__()`` in its own :meth:`__init__()`, then - set up custom prefixes which would be usable in the subsequent - :meth:`configure` call. :attr:`dictConfigClass` would be bound to - this new subclass, and then :func:`dictConfig` could be called exactly as - in the default, uncustomized state. + Takes the logging configuration from a dictionary. The contents of + this dictionary are described in :ref:`logging-config-dictschema` + below. + + If an error is encountered during configuration, this function will + raise a :exc:`ValueError`, :exc:`TypeError`, :exc:`AttributeError` + or :exc:`ImportError` with a suitably descriptive message. The + following is a (possibly incomplete) list of conditions which will + raise an error: + + * A ``level`` which is not a string or which is a string not + corresponding to an actual logging level. + * A ``propagate`` value which is not a boolean. + * An id which does not have a corresponding destination. + * A non-existent handler id found during an incremental call. + * An invalid logger name. + * Inability to resolve to an internal or external object. + + Parsing is performed by the :class:`DictConfigurator` class, whose + constructor is passed the dictionary used for configuration, and + has a :meth:`configure` method. The :mod:`logging.config` module + has a callable attribute :attr:`dictConfigClass` + which is initially set to :class:`DictConfigurator`. + You can replace the value of :attr:`dictConfigClass` with a + suitable implementation of your own. + + :func:`dictConfig` calls :attr:`dictConfigClass` passing + the specified dictionary, and then calls the :meth:`configure` method on + the returned object to put the configuration into effect:: + + def dictConfig(config): + dictConfigClass(config).configure() + + For example, a subclass of :class:`DictConfigurator` could call + ``DictConfigurator.__init__()`` in its own :meth:`__init__()`, then + set up custom prefixes which would be usable in the subsequent + :meth:`configure` call. :attr:`dictConfigClass` would be bound to + this new subclass, and then :func:`dictConfig` could be called exactly as + in the default, uncustomized state. .. versionadded:: 3.2 -.. function:: fileConfig(fname, defaults=None, disable_existing_loggers=True) +.. function:: fileConfig(fname, defaults=None, disable_existing_loggers=True, encoding=None) Reads the logging configuration from a :mod:`configparser`\-format file. The format of the file should be as described in @@ -111,6 +111,8 @@ in :mod:`logging` itself) and defining handlers which are declared either in they or their ancestors are explicitly named in the logging configuration. + :param encoding: The encoding used to open file when *fname* is filename. + .. versionchanged:: 3.4 An instance of a subclass of :class:`~configparser.RawConfigParser` is now accepted as a value for ``fname``. This facilitates: @@ -121,6 +123,9 @@ in :mod:`logging` itself) and defining handlers which are declared either in application (e.g. based on command-line parameters or other aspects of the runtime environment) before being passed to ``fileConfig``. + .. versionadded:: 3.10 + The *encoding* parameter is added. + .. function:: listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None) Starts up a socket server on the specified port, and listens for new @@ -218,18 +223,29 @@ custom instantiation is required. If so, the mechanism described in :ref:`logging-config-dict-userdef` below is used to create an instance; otherwise, the context is used to determine what to instantiate. +.. _logging-config-dictschema-formatters: + * *formatters* - the corresponding value will be a dict in which each key is a formatter id and each value is a dict describing how to configure the corresponding :class:`~logging.Formatter` instance. - The configuring dict is searched for keys ``format`` and ``datefmt`` - (with defaults of ``None``) and these are used to construct a - :class:`~logging.Formatter` instance. + The configuring dict is searched for the following optional keys + which correspond to the arguments passed to create a + :class:`~logging.Formatter` object: + + * ``format`` + * ``datefmt`` + * ``style`` + * ``validate`` (since version >=3.8) - .. versionchanged:: 3.8 - a ``validate`` key (with default of ``True``) can be added into - the ``formatters`` section of the configuring dict, this is to - validate the format. + An optional ``class`` key indicates the name of the formatter's + class (as a dotted module and class name). The instantiation + arguments are as for :class:`~logging.Formatter`, thus this key is + most useful for instantiating a customised subclass of + :class:`~logging.Formatter`. For example, the alternative class + might present exception tracebacks in an expanded or condensed + format. If your formatter requires different or extra configuration + keys, you should use :ref:`logging-config-dict-userdef`. * *filters* - the corresponding value will be a dict in which each key is a filter id and each value is a dict describing how to configure @@ -577,7 +593,7 @@ in the configuration, the string ``'cfg://handlers'`` would resolve to the dict with key ``handlers``, the string ``'cfg://handlers.email`` would resolve to the dict with key ``email`` in the ``handlers`` dict, and so on. The string ``'cfg://handlers.email.toaddrs[1]`` would -resolve to ``'dev_team.domain.tld'`` and the string +resolve to ``'dev_team@domain.tld'`` and the string ``'cfg://handlers.email.toaddrs[0]'`` would resolve to the value ``'support_team@domain.tld'``. The ``subject`` value could be accessed using either ``'cfg://handlers.email.subject'`` or, equivalently, @@ -791,20 +807,13 @@ Sections which specify formatter configuration are typified by the following. [formatter_form01] format=F1 %(asctime)s %(levelname)s %(message)s datefmt= + style='%' + validate=True class=logging.Formatter -The ``format`` entry is the overall format string, and the ``datefmt`` entry is -the :func:`strftime`\ -compatible date/time format string. If empty, the -package substitutes something which is almost equivalent to specifying the date -format string ``'%Y-%m-%d %H:%M:%S'``. This format also specifies milliseconds, -which are appended to the result of using the above format string, with a comma -separator. An example time in this format is ``2003-01-23 00:29:50,411``. - -The ``class`` entry is optional. It indicates the name of the formatter's class -(as a dotted module and class name.) This option is useful for instantiating a -:class:`~logging.Formatter` subclass. Subclasses of -:class:`~logging.Formatter` can present exception tracebacks in an expanded or -condensed format. +The arguments for the formatter configuration are the same as the keys +in the dictionary schema :ref:`formatters section +`. .. note:: diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index fb8ea705b0469f..70a703dde18a03 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -1096,8 +1096,8 @@ functions. suitable value. .. versionchanged:: 3.7 - The *level* parameter was defaulted to level ``CRITICAL``. See Issue - #28524 for more information about this change. + The *level* parameter was defaulted to level ``CRITICAL``. See + :issue:`28524` for more information about this change. .. function:: addLevelName(level, levelName) @@ -1113,18 +1113,27 @@ functions. .. function:: getLevelName(level) - Returns the textual representation of logging level *level*. If the level is one - of the predefined levels :const:`CRITICAL`, :const:`ERROR`, :const:`WARNING`, - :const:`INFO` or :const:`DEBUG` then you get the corresponding string. If you - have associated levels with names using :func:`addLevelName` then the name you - have associated with *level* is returned. If a numeric value corresponding to one - of the defined levels is passed in, the corresponding string representation is - returned. Otherwise, the string 'Level %s' % level is returned. + Returns the textual or numeric representation of logging level *level*. + + If *level* is one of the predefined levels :const:`CRITICAL`, :const:`ERROR`, + :const:`WARNING`, :const:`INFO` or :const:`DEBUG` then you get the + corresponding string. If you have associated levels with names using + :func:`addLevelName` then the name you have associated with *level* is + returned. If a numeric value corresponding to one of the defined levels is + passed in, the corresponding string representation is returned. + + The *level* parameter also accepts a string representation of the level such + as 'INFO'. In such cases, this functions returns the corresponding numeric + value of the level. + + If no matching numeric or string value is passed in, the string + 'Level %s' % level is returned. .. note:: Levels are internally integers (as they need to be compared in the logging logic). This function is used to convert between an integer level and the level name displayed in the formatted log output by means of the - ``%(levelname)s`` format specifier (see :ref:`logrecord-attributes`). + ``%(levelname)s`` format specifier (see :ref:`logrecord-attributes`), and + vice versa. .. versionchanged:: 3.4 In Python versions earlier than 3.4, this function could also be passed a @@ -1174,7 +1183,9 @@ functions. | | to ``'a'``. | +--------------+---------------------------------------------+ | *format* | Use the specified format string for the | - | | handler. | + | | handler. Defaults to attributes | + | | ``levelname``, ``name`` and ``message`` | + | | separated by colons. | +--------------+---------------------------------------------+ | *datefmt* | Use the specified date/time format, as | | | accepted by :func:`time.strftime`. | diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 4bfff9c6147ed4..21092645366da7 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -19,8 +19,8 @@ interface supporting the ``.xz`` and legacy ``.lzma`` file formats used by the :program:`xz` utility, as well as raw compressed streams. The interface provided by this module is very similar to that of the :mod:`bz2` -module. However, note that :class:`LZMAFile` is *not* thread-safe, unlike -:class:`bz2.BZ2File`, so if you need to use a single :class:`LZMAFile` instance +module. Note that :class:`LZMAFile` and :class:`bz2.BZ2File` are *not* +thread-safe, so if you need to use a single :class:`LZMAFile` instance from multiple threads, it is necessary to protect it with a lock. @@ -33,7 +33,7 @@ from multiple threads, it is necessary to protect it with a lock. Reading and writing compressed files ------------------------------------ -.. function:: open(filename, mode="rb", \*, format=None, check=-1, preset=None, filters=None, encoding=None, errors=None, newline=None) +.. function:: open(filename, mode="rb", *, format=None, check=-1, preset=None, filters=None, encoding=None, errors=None, newline=None) Open an LZMA-compressed file in binary or text mode, returning a :term:`file object`. @@ -69,7 +69,7 @@ Reading and writing compressed files Accepts a :term:`path-like object`. -.. class:: LZMAFile(filename=None, mode="r", \*, format=None, check=-1, preset=None, filters=None) +.. class:: LZMAFile(filename=None, mode="r", *, format=None, check=-1, preset=None, filters=None) Open an LZMA-compressed file in binary mode. diff --git a/Doc/library/misc.rst b/Doc/library/misc.rst deleted file mode 100644 index 0943235246a91b..00000000000000 --- a/Doc/library/misc.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. _misc: - -********************** -Miscellaneous Services -********************** - -The modules described in this chapter provide miscellaneous services that are -available in all Python versions. Here's an overview: - - -.. toctree:: - - formatter.rst diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 698c17653786b3..d9825b47c71333 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -337,6 +337,8 @@ MADV_* Constants MADV_NOCORE MADV_CORE MADV_PROTECT + MADV_FREE_REUSABLE + MADV_FREE_REUSE These options can be passed to :meth:`mmap.madvise`. Not every option will be present on every system. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index f8e7556f25b4e0..ae4f7bf3b9621d 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -98,7 +98,7 @@ to start a process. These *start methods* are *spawn* The parent process starts a fresh python interpreter process. The child process will only inherit those resources necessary to run - the process objects :meth:`~Process.run` method. In particular, + the process object's :meth:`~Process.run` method. In particular, unnecessary file descriptors and handles from the parent process will not be inherited. Starting a process using this method is rather slow compared to using *fork* or *forkserver*. @@ -1926,7 +1926,7 @@ client to access it remotely:: >>> class Worker(Process): ... def __init__(self, q): ... self.q = q - ... super(Worker, self).__init__() + ... super().__init__() ... def run(self): ... self.q.put('local hello') ... @@ -2569,9 +2569,9 @@ Address Formats filesystem. * An ``'AF_PIPE'`` address is a string of the form - :samp:`r'\\\\.\\pipe\\{PipeName}'`. To use :func:`Client` to connect to a named - pipe on a remote computer called *ServerName* one should use an address of the - form :samp:`r'\\\\{ServerName}\\pipe\\{PipeName}'` instead. + :samp:`r'\\\\.\\pipe\\{PipeName}'`. To use :func:`Client` to connect to a named + pipe on a remote computer called *ServerName* one should use an address of the + form :samp:`r'\\\\{ServerName}\\pipe\\{PipeName}'` instead. Note that any string beginning with two backslashes is assumed by default to be an ``'AF_PIPE'`` address rather than an ``'AF_UNIX'`` address. @@ -2661,6 +2661,46 @@ The :mod:`multiprocessing.dummy` module :mod:`multiprocessing.dummy` replicates the API of :mod:`multiprocessing` but is no more than a wrapper around the :mod:`threading` module. +.. currentmodule:: multiprocessing.pool + +In particular, the ``Pool`` function provided by :mod:`multiprocessing.dummy` +returns an instance of :class:`ThreadPool`, which is a subclass of +:class:`Pool` that supports all the same method calls but uses a pool of +worker threads rather than worker processes. + + +.. class:: ThreadPool([processes[, initializer[, initargs]]]) + + A thread pool object which controls a pool of worker threads to which jobs + can be submitted. :class:`ThreadPool` instances are fully interface + compatible with :class:`Pool` instances, and their resources must also be + properly managed, either by using the pool as a context manager or by + calling :meth:`~multiprocessing.pool.Pool.close` and + :meth:`~multiprocessing.pool.Pool.terminate` manually. + + *processes* is the number of worker threads to use. If *processes* is + ``None`` then the number returned by :func:`os.cpu_count` is used. + + If *initializer* is not ``None`` then each worker process will call + ``initializer(*initargs)`` when it starts. + + Unlike :class:`Pool`, *maxtasksperchild* and *context* cannot be provided. + + .. note:: + + A :class:`ThreadPool` shares the same interface as :class:`Pool`, which + is designed around a pool of processes and predates the introduction of + the :class:`concurrent.futures` module. As such, it inherits some + operations that don't make sense for a pool backed by threads, and it + has its own type for representing the status of asynchronous jobs, + :class:`AsyncResult`, that is not understood by any other libraries. + + Users should generally prefer to use + :class:`concurrent.futures.ThreadPoolExecutor`, which has a simpler + interface that was designed around threads from the start, and which + returns :class:`concurrent.futures.Future` instances that are + compatible with many other libraries, including :mod:`asyncio`. + .. _multiprocessing-programming: diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index 3d29ac49b9191a..4bf7de67c1d03e 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -38,6 +38,10 @@ the Unix :program:`ftp` program and other FTP clients. :func:`os.path.expanduser` is used to find the location of the :file:`.netrc` file when *file* is not passed as argument. + .. versionchanged:: 3.10 + :class:`netrc` try UTF-8 encoding before using locale specific + encoding. + .. exception:: NetrcParseError diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 36c53556c2685e..0cdba68f3770ed 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -319,15 +319,12 @@ expect a function argument. method. Dictionaries accept any hashable value. Lists, tuples, and strings accept an index or a slice: - >>> itemgetter('name')({'name': 'tu', 'age': 18}) - 'tu' >>> itemgetter(1)('ABCDEFG') 'B' - >>> itemgetter(1,3,5)('ABCDEFG') + >>> itemgetter(1, 3, 5)('ABCDEFG') ('B', 'D', 'F') - >>> itemgetter(slice(2,None))('ABCDEFG') + >>> itemgetter(slice(2, None))('ABCDEFG') 'CDEFG' - >>> soldier = dict(rank='captain', name='dotterbart') >>> itemgetter('rank')(soldier) 'captain' diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index a5abacf02144a7..d06d9ce8c9e3d7 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -175,8 +175,8 @@ the :mod:`glob` module.) On Windows, :envvar:`USERPROFILE` will be used if set, otherwise a combination of :envvar:`HOMEPATH` and :envvar:`HOMEDRIVE` will be used. An initial - ``~user`` is handled by stripping the last directory component from the created - user path derived above. + ``~user`` is handled by checking that the last directory component of the current + user's home directory matches :envvar:`USERNAME`, and replacing it if so. If the expansion fails or if the path does not begin with a tilde, the path is returned unchanged. @@ -306,11 +306,10 @@ the :mod:`glob` module.) Join one or more path components intelligently. The return value is the concatenation of *path* and any members of *\*paths* with exactly one - directory separator (``os.sep``) following each non-empty part except the - last, meaning that the result will only end in a separator if the last - part is empty. If a component is an absolute path, all previous - components are thrown away and joining continues from the absolute path - component. + directory separator following each non-empty part except the last, meaning + that the result will only end in a separator if the last part is empty. If + a component is an absolute path, all previous components are thrown away + and joining continues from the absolute path component. On Windows, the drive letter is not reset when an absolute path component (e.g., ``r'\foo'``) is encountered. If a component contains a drive @@ -345,15 +344,24 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: realpath(path) +.. function:: realpath(path, *, strict=False) Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path (if they are supported by the operating system). + If a path doesn't exist or a symlink loop is encountered, and *strict* is + ``True``, :exc:`OSError` is raised. If *strict* is ``False``, the path is + resolved as far as possible and any remainder is appended without checking + whether it exists. + .. note:: - When symbolic link cycles occur, the returned path will be one member of - the cycle, but no guarantee is made about which member that will be. + This function emulates the operating system's procedure for making a path + canonical, which differs slightly between Windows and UNIX with respect + to how links and subsequent path components interact. + + Operating system APIs make paths canonical as needed, so it's not + normally necessary to call this function. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -361,13 +369,17 @@ the :mod:`glob` module.) .. versionchanged:: 3.8 Symbolic links and junctions are now resolved on Windows. + .. versionchanged:: 3.10 + The *strict* parameter was added. + .. function:: relpath(path, start=os.curdir) Return a relative filepath to *path* either from the current directory or from an optional *start* directory. This is a path computation: the filesystem is not accessed to confirm the existence or nature of *path* or - *start*. + *start*. On Windows, :exc:`ValueError` is raised when *path* and *start* + are on different drives. *start* defaults to :attr:`os.curdir`. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 718d98138d2679..6f9f3219481c2b 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -32,7 +32,7 @@ Notes on the availability of these functions: objects, and result in an object of the same type, if a path or file name is returned. -* On VxWorks, os.fork, os.execv and os.spawn*p* are not supported. +* On VxWorks, os.popen, os.fork, os.execv and os.spawn*p* are not supported. .. note:: @@ -68,8 +68,13 @@ File Names, Command Line Arguments, and Environment Variables In Python, file names, command line arguments, and environment variables are represented using the string type. On some systems, decoding these strings to and from bytes is necessary before passing them to the operating system. Python -uses the file system encoding to perform this conversion (see -:func:`sys.getfilesystemencoding`). +uses the :term:`filesystem encoding and error handler` to perform this +conversion (see :func:`sys.getfilesystemencoding`). + +The :term:`filesystem encoding and error handler` are configured at Python +startup by the :c:func:`PyConfig_Read` function: see +:c:member:`~PyConfig.filesystem_encoding` and +:c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`. .. versionchanged:: 3.1 On some systems, conversion using the file system encoding may fail. In this @@ -79,9 +84,72 @@ uses the file system encoding to perform this conversion (see original byte on encoding. -The file system encoding must guarantee to successfully decode all bytes -below 128. If the file system encoding fails to provide this guarantee, API -functions may raise UnicodeErrors. +The :term:`file system encoding ` must +guarantee to successfully decode all bytes below 128. If the file system +encoding fails to provide this guarantee, API functions can raise +:exc:`UnicodeError`. + +See also the :term:`locale encoding`. + + +.. _utf8-mode: + +Python UTF-8 Mode +----------------- + +.. versionadded:: 3.7 + See :pep:`540` for more details. + +The Python UTF-8 Mode ignores the :term:`locale encoding` and forces the usage +of the UTF-8 encoding: + +* Use UTF-8 as the :term:`filesystem encoding `. +* :func:`sys.getfilesystemencoding()` returns ``'UTF-8'``. +* :func:`locale.getpreferredencoding()` returns ``'UTF-8'`` (the *do_setlocale* + argument has no effect). +* :data:`sys.stdin`, :data:`sys.stdout`, and :data:`sys.stderr` all use + UTF-8 as their text encoding, with the ``surrogateescape`` + :ref:`error handler ` being enabled for :data:`sys.stdin` + and :data:`sys.stdout` (:data:`sys.stderr` continues to use + ``backslashreplace`` as it does in the default locale-aware mode) +* On Unix, :func:`os.device_encoding` returns ``'UTF-8'``. rather than the + device encoding. + +Note that the standard stream settings in UTF-8 mode can be overridden by +:envvar:`PYTHONIOENCODING` (just as they can be in the default locale-aware +mode). + +As a consequence of the changes in those lower level APIs, other higher +level APIs also exhibit different default behaviours: + +* Command line arguments, environment variables and filenames are decoded + to text using the UTF-8 encoding. +* :func:`os.fsdecode()` and :func:`os.fsencode()` use the UTF-8 encoding. +* :func:`open()`, :func:`io.open()`, and :func:`codecs.open()` use the UTF-8 + encoding by default. However, they still use the strict error handler by + default so that attempting to open a binary file in text mode is likely + to raise an exception rather than producing nonsense data. + +The :ref:`Python UTF-8 Mode ` is enabled if the LC_CTYPE locale is +``C`` or ``POSIX`` at Python startup (see the :c:func:`PyConfig_Read` +function). + +It can be enabled or disabled using the :option:`-X utf8 <-X>` command line +option and the :envvar:`PYTHONUTF8` environment variable. + +If the :envvar:`PYTHONUTF8` environment variable is not set at all, then the +interpreter defaults to using the current locale settings, *unless* the current +locale is identified as a legacy ASCII-based locale (as described for +:envvar:`PYTHONCOERCECLOCALE`), and locale coercion is either disabled or +fails. In such legacy locales, the interpreter will default to enabling UTF-8 +mode unless explicitly instructed not to do so. + +The Python UTF-8 Mode can only be enabled at the Python startup. Its value +can be read from :data:`sys.flags.utf8_mode `. + +See also the :ref:`UTF-8 mode on Windows ` +and the :term:`filesystem encoding and error handler`. .. _os-procinfo: @@ -165,9 +233,9 @@ process and user. .. function:: fsencode(filename) - Encode :term:`path-like ` *filename* to the filesystem - encoding with ``'surrogateescape'`` error handler, or ``'strict'`` on - Windows; return :class:`bytes` unchanged. + Encode :term:`path-like ` *filename* to the + :term:`filesystem encoding and error handler`; return :class:`bytes` + unchanged. :func:`fsdecode` is the reverse function. @@ -181,8 +249,8 @@ process and user. .. function:: fsdecode(filename) Decode the :term:`path-like ` *filename* from the - filesystem encoding with ``'surrogateescape'`` error handler, or ``'strict'`` - on Windows; return :class:`str` unchanged. + :term:`filesystem encoding and error handler`; return :class:`str` + unchanged. :func:`fsencode` is the reverse function. @@ -742,6 +810,12 @@ as internal buffering of data. Return a string describing the encoding of the device associated with *fd* if it is connected to a terminal; else return :const:`None`. + On Unix, if the :ref:`Python UTF-8 Mode ` is enabled, return + ``'UTF-8'`` rather than the device encoding. + + .. versionchanged:: 3.10 + On Unix, the function now implements the Python UTF-8 Mode. + .. function:: dup(fd) @@ -1016,6 +1090,16 @@ or `the MSDN `_ on Windo The above constants are only available on Windows. +.. data:: O_EVTONLY + O_FSYNC + O_SYMLINK + O_NOFOLLOW_ANY + + The above constants are only available on macOS. + + .. versionchanged:: 3.10 + Add :data:`O_EVTONLY`, :data:`O_FSYNC`, :data:`O_SYMLINK` + and :data:`O_NOFOLLOW_ANY` constants. .. data:: O_ASYNC O_DIRECT @@ -1249,12 +1333,12 @@ or `the MSDN `_ on Windo .. data:: RWF_APPEND - Provide a per-write equivalent of the :data:`O_APPEND` :func:`os.open` - flag. This flag is meaningful only for :func:`os.pwritev`, and its - effect applies only to the data range written by the system call. The - *offset* argument does not affect the write operation; the data is always - appended to the end of the file. However, if the *offset* argument is - ``-1``, the current file *offset* is updated. + Provide a per-write equivalent of the :data:`O_APPEND` :func:`os.open` + flag. This flag is meaningful only for :func:`os.pwritev`, and its + effect applies only to the data range written by the system call. The + *offset* argument does not affect the write operation; the data is always + appended to the end of the file. However, if the *offset* argument is + ``-1``, the current file *offset* is updated. .. availability:: Linux 4.16 and newer. @@ -1345,6 +1429,39 @@ or `the MSDN `_ on Windo .. versionadded:: 3.3 +.. function:: splice(src, dst, count, offset_src=None, offset_dst=None) + + Transfer *count* bytes from file descriptor *src*, starting from offset + *offset_src*, to file descriptor *dst*, starting from offset *offset_dst*. + At least one of the file descriptors must refer to a pipe. If *offset_src* + is None, then *src* is read from the current position; respectively for + *offset_dst*. The offset associated to the file descriptor that refers to a + pipe must be ``None``. The files pointed by *src* and *dst* must reside in + the same filesystem, otherwise an :exc:`OSError` is raised with + :attr:`~OSError.errno` set to :data:`errno.EXDEV`. + + This copy is done without the additional cost of transferring data + from the kernel to user space and then back into the kernel. Additionally, + some filesystems could implement extra optimizations. The copy is done as if + both files are opened as binary. + + Upon successful completion, returns the number of bytes spliced to or from + the pipe. A return value of 0 means end of input. If *src* refers to a + pipe, then this means that there was no data to transfer, and it would not + make sense to block because there are no writers connected to the write end + of the pipe. + + .. availability:: Linux kernel >= 2.6.17 and glibc >= 2.5 + + .. versionadded:: 3.10 + + +.. data:: SPLICE_F_MOVE + SPLICE_F_NONBLOCK + SPLICE_F_MORE + + .. versionadded:: 3.10 + .. function:: readv(fd, buffers) Read from a file descriptor *fd* into a number of mutable :term:`bytes-like @@ -1884,7 +2001,7 @@ features: Accepts a :term:`path-like object`. -.. function:: lstat(path, \*, dir_fd=None) +.. function:: lstat(path, *, dir_fd=None) Perform the equivalent of an :c:func:`lstat` system call on the given path. Similar to :func:`~os.stat`, but does not follow symbolic links. Return a @@ -1910,7 +2027,7 @@ features: Added the *dir_fd* parameter. .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *src* and *dst*. + Accepts a :term:`path-like object`. .. versionchanged:: 3.8 On Windows, now opens reparse points that represent another path @@ -2130,6 +2247,7 @@ features: Remove (delete) the file *path*. If *path* is a directory, an :exc:`IsADirectoryError` is raised. Use :func:`rmdir` to remove directories. + If the file does not exist, a :exc:`FileNotFoundError` is raised. This function can support :ref:`paths relative to directory descriptors `. @@ -2391,7 +2509,7 @@ features: On the first, uncached call, a system call is required on Windows but not on Unix. - .. method:: is_dir(\*, follow_symlinks=True) + .. method:: is_dir(*, follow_symlinks=True) Return ``True`` if this entry is a directory or a symbolic link pointing to a directory; return ``False`` if the entry is or points to any other @@ -2415,7 +2533,7 @@ features: This method can raise :exc:`OSError`, such as :exc:`PermissionError`, but :exc:`FileNotFoundError` is caught and not raised. - .. method:: is_file(\*, follow_symlinks=True) + .. method:: is_file(*, follow_symlinks=True) Return ``True`` if this entry is a file or a symbolic link pointing to a file; return ``False`` if the entry is or points to a directory or other @@ -2445,7 +2563,7 @@ features: This method can raise :exc:`OSError`, such as :exc:`PermissionError`, but :exc:`FileNotFoundError` is caught and not raised. - .. method:: stat(\*, follow_symlinks=True) + .. method:: stat(*, follow_symlinks=True) Return a :class:`stat_result` object for this entry. This method follows symbolic links by default; to stat a symbolic link add the @@ -2477,7 +2595,7 @@ features: for :class:`bytes` paths on Windows. -.. function:: stat(path, \*, dir_fd=None, follow_symlinks=True) +.. function:: stat(path, *, dir_fd=None, follow_symlinks=True) Get the status of a file or a file descriptor. Perform the equivalent of a :c:func:`stat` system call on the given path. *path* may be specified as @@ -3202,6 +3320,102 @@ features: .. versionadded:: 3.8 +.. function:: eventfd(initval[, flags=os.EFD_CLOEXEC]) + + Create and return an event file descriptor. The file descriptors supports + raw :func:`read` and :func:`write` with a buffer size of 8, + :func:`~select.select`, :func:`~select.poll` and similar. See man page + :manpage:`eventfd(2)` for more information. By default, the + new file descriptor is :ref:`non-inheritable `. + + *initval* is the initial value of the event counter. The initial value + must be an 32 bit unsigned integer. Please note that the initial value is + limited to a 32 bit unsigned int although the event counter is an unsigned + 64 bit integer with a maximum value of 2\ :sup:`64`\ -\ 2. + + *flags* can be constructed from :const:`EFD_CLOEXEC`, + :const:`EFD_NONBLOCK`, and :const:`EFD_SEMAPHORE`. + + If :const:`EFD_SEMAPHORE` is specified and the event counter is non-zero, + :func:`eventfd_read` returns 1 and decrements the counter by one. + + If :const:`EFD_SEMAPHORE` is not specified and the event counter is + non-zero, :func:`eventfd_read` returns the current event counter value and + resets the counter to zero. + + If the event counter is zero and :const:`EFD_NONBLOCK` is not + specified, :func:`eventfd_read` blocks. + + :func:`eventfd_write` increments the event counter. Write blocks if the + write operation would increment the counter to a value larger than + 2\ :sup:`64`\ -\ 2. + + Example:: + + import os + + # semaphore with start value '1' + fd = os.eventfd(1, os.EFD_SEMAPHORE | os.EFC_CLOEXEC) + try: + # acquire semaphore + v = os.eventfd_read(fd) + try: + do_work() + finally: + # release semaphore + os.eventfd_write(fd, v) + finally: + os.close(fd) + + .. availability:: Linux 2.6.27 or newer with glibc 2.8 or newer. + + .. versionadded:: 3.10 + +.. function:: eventfd_read(fd) + + Read value from an :func:`eventfd` file descriptor and return a 64 bit + unsigned int. The function does not verify that *fd* is an :func:`eventfd`. + + .. availability:: See :func:`eventfd` + + .. versionadded:: 3.10 + +.. function:: eventfd_write(fd, value) + + Add value to an :func:`eventfd` file descriptor. *value* must be a 64 bit + unsigned int. The function does not verify that *fd* is an :func:`eventfd`. + + .. availability:: See :func:`eventfd` + + .. versionadded:: 3.10 + +.. data:: EFD_CLOEXEC + + Set close-on-exec flag for new :func:`eventfd` file descriptor. + + .. availability:: See :func:`eventfd` + + .. versionadded:: 3.10 + +.. data:: EFD_NONBLOCK + + Set :const:`O_NONBLOCK` status flag for new :func:`eventfd` file + descriptor. + + .. availability:: See :func:`eventfd` + + .. versionadded:: 3.10 + +.. data:: EFD_SEMAPHORE + + Provide semaphore-like semantics for reads from a :func:`eventfd` file + descriptor. On read the internal counter is decremented by one. + + .. availability:: Linux 2.6.30 or newer with glibc 2.8 or newer. + + .. versionadded:: 3.10 + + Linux extended attributes ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -3246,7 +3460,7 @@ These functions are all available on Linux only. Removes the extended filesystem attribute *attribute* from *path*. *attribute* should be bytes or str (directly or indirectly through the :class:`PathLike` interface). If it is a string, it is encoded - with the filesystem encoding. + with the :term:`filesystem encoding and error handler`. This function can support :ref:`specifying a file descriptor ` and :ref:`not following symlinks `. @@ -3262,11 +3476,11 @@ These functions are all available on Linux only. Set the extended filesystem attribute *attribute* on *path* to *value*. *attribute* must be a bytes or str with no embedded NULs (directly or indirectly through the :class:`PathLike` interface). If it is a str, - it is encoded with the filesystem encoding. *flags* may be + it is encoded with the :term:`filesystem encoding and error handler`. *flags* may be :data:`XATTR_REPLACE` or :data:`XATTR_CREATE`. If :data:`XATTR_REPLACE` is - given and the attribute does not exist, ``EEXISTS`` will be raised. + given and the attribute does not exist, ``ENODATA`` will be raised. If :data:`XATTR_CREATE` is given and the attribute already exists, the - attribute will not be created and ``ENODATA`` will be raised. + attribute will not be created and ``EEXISTS`` will be raised. This function can support :ref:`specifying a file descriptor ` and :ref:`not following symlinks `. @@ -3942,7 +4156,7 @@ written in Python, such as a mail server's external command delivery program. .. availability:: Windows. -.. function:: startfile(path[, operation]) +.. function:: startfile(path, [operation], [arguments], [cwd], [show_cmd]) Start a file with its associated application. @@ -3956,13 +4170,25 @@ written in Python, such as a mail server's external command delivery program. ``'print'`` and ``'edit'`` (to be used on files) as well as ``'explore'`` and ``'find'`` (to be used on directories). + When launching an application, specify *arguments* to be passed as a single + string. This argument may have no effect when using this function to launch a + document. + + The default working directory is inherited, but may be overridden by the *cwd* + argument. This should be an absolute path. A relative *path* will be resolved + against this argument. + + Use *show_cmd* to override the default window style. Whether this has any + effect will depend on the application being launched. Values are integers as + supported by the Win32 :c:func:`ShellExecute` function. + :func:`startfile` returns as soon as the associated application is launched. There is no option to wait for the application to close, and no way to retrieve the application's exit status. The *path* parameter is relative to the current - directory. If you want to use an absolute path, make sure the first character - is not a slash (``'/'``); the underlying Win32 :c:func:`ShellExecute` function - doesn't work if it is. Use the :func:`os.path.normpath` function to ensure that - the path is properly encoded for Win32. + directory or *cwd*. If you want to use an absolute path, make sure the first + character is not a slash (``'/'``) Use :mod:`pathlib` or the + :func:`os.path.normpath` function to ensure that paths are properly encoded for + Win32. To reduce interpreter startup overhead, the Win32 :c:func:`ShellExecute` function is not resolved until this function is first called. If the function @@ -3970,8 +4196,14 @@ written in Python, such as a mail server's external command delivery program. .. audit-event:: os.startfile path,operation os.startfile + .. audit-event:: os.startfile/2 path,operation,arguments,cwd,show_cmd os.startfile + .. availability:: Windows. + .. versionchanged:: 3.10 + Added the *arguments*, *cwd* and *show_cmd* arguments, and the + ``os.startfile/2`` audit event. + .. function:: system(command) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 9de72bb725c758..b6507eb4d6fa2c 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -336,6 +336,8 @@ Pure paths provide the following methods and properties: >>> p.parents[2] PureWindowsPath('c:/') + .. versionchanged:: 3.10 + The parents sequence now supports :term:`slices ` and negative index values. .. data:: PurePath.parent @@ -703,7 +705,10 @@ call fails (for example because the path doesn't exist). .. classmethod:: Path.home() Return a new path object representing the user's home directory (as - returned by :func:`os.path.expanduser` with ``~`` construct):: + returned by :func:`os.path.expanduser` with ``~`` construct). If the home + directory can't be resolved, :exc:`RuntimeError` is raised. + + :: >>> Path.home() PosixPath('/home/antoine') @@ -711,11 +716,14 @@ call fails (for example because the path doesn't exist). .. versionadded:: 3.5 -.. method:: Path.stat() +.. method:: Path.stat(*, follow_symlinks=True) Return a :class:`os.stat_result` object containing information about this path, like :func:`os.stat`. The result is looked up at each call to this method. + This method normally follows symlinks; to stat a symlink add the argument + ``follow_symlinks=False``, or use :meth:`~Path.lstat`. + :: >>> p = Path('setup.py') @@ -724,10 +732,18 @@ call fails (for example because the path doesn't exist). >>> p.stat().st_mtime 1327883547.852554 + .. versionchanged:: 3.10 + The *follow_symlinks* parameter was added. -.. method:: Path.chmod(mode) +.. method:: Path.chmod(mode, *, follow_symlinks=True) - Change the file mode and permissions, like :func:`os.chmod`:: + Change the file mode and permissions, like :func:`os.chmod`. + + This method normally follows symlinks. Some Unix flavours support changing + permissions on the symlink itself; on these platforms you may add the + argument ``follow_symlinks=False``, or use :meth:`~Path.lchmod`. + + :: >>> p = Path('setup.py') >>> p.stat().st_mode @@ -736,6 +752,8 @@ call fails (for example because the path doesn't exist). >>> p.stat().st_mode 33060 + .. versionchanged:: 3.10 + The *follow_symlinks* parameter was added. .. method:: Path.exists() @@ -758,7 +776,10 @@ call fails (for example because the path doesn't exist). .. method:: Path.expanduser() Return a new path with expanded ``~`` and ``~user`` constructs, - as returned by :meth:`os.path.expanduser`:: + as returned by :meth:`os.path.expanduser`. If a home directory can't be + resolved, :exc:`RuntimeError` is raised. + + :: >>> p = PosixPath('~/films/Monty Python') >>> p.expanduser() @@ -777,8 +798,9 @@ call fails (for example because the path doesn't exist). >>> sorted(Path('.').glob('*/*.py')) [PosixPath('docs/conf.py')] - The "``**``" pattern means "this directory and all subdirectories, - recursively". In other words, it enables recursive globbing:: + Patterns are the same as for :mod:`fnmatch`, with the addition of "``**``" + which means "this directory and all subdirectories, recursively". In other + words, it enables recursive globbing:: >>> sorted(Path('.').glob('**/*.py')) [PosixPath('build/lib/pathlib.py'), @@ -1118,6 +1140,35 @@ call fails (for example because the path doesn't exist). The order of arguments (link, target) is the reverse of :func:`os.symlink`'s. +.. method:: Path.hardlink_to(target) + + Make this path a hard link to the same file as *target*. + + .. note:: + The order of arguments (link, target) is the reverse + of :func:`os.link`'s. + + .. versionadded:: 3.10 + +.. method:: Path.link_to(target) + + Make *target* a hard link to this path. + + .. warning:: + + This function does not make this path a hard link to *target*, despite + the implication of the function and argument names. The argument order + (target, link) is the reverse of :func:`Path.symlink_to` and + :func:`Path.hardlink_to`, but matches that of :func:`os.link`. + + .. versionadded:: 3.8 + + .. deprecated:: 3.10 + + This method is deprecated in favor of :meth:`Path.hardlink_to`, as the + argument order of :meth:`Path.link_to` does not match that of + :meth:`Path.symlink_to`. + .. method:: Path.touch(mode=0o666, exist_ok=True) @@ -1143,13 +1194,6 @@ call fails (for example because the path doesn't exist). The *missing_ok* parameter was added. -.. method:: Path.link_to(target) - - Create a hard link pointing to a path named *target*. - - .. versionadded:: 3.8 - - .. method:: Path.write_bytes(data) Open the file pointed to in bytes mode, write *data* to it, and close the @@ -1193,14 +1237,15 @@ Below is a table mapping various :mod:`os` functions to their corresponding .. note:: - Although :func:`os.path.relpath` and :meth:`PurePath.relative_to` have some - overlapping use-cases, their semantics differ enough to warrant not - considering them equivalent. + Not all pairs of functions/methods below are equivalent. Some of them, + despite having some overlapping use-cases, have different semantics. They + include :func:`os.path.abspath` and :meth:`Path.resolve`, + :func:`os.path.relpath` and :meth:`PurePath.relative_to`. ==================================== ============================== -os and os.path pathlib +:mod:`os` and :mod:`os.path` :mod:`pathlib` ==================================== ============================== -:func:`os.path.abspath` :meth:`Path.resolve` +:func:`os.path.abspath` :meth:`Path.resolve` [#]_ :func:`os.chmod` :meth:`Path.chmod` :func:`os.mkdir` :meth:`Path.mkdir` :func:`os.makedirs` :meth:`Path.mkdir` @@ -1216,9 +1261,10 @@ os and os.path pathlib :func:`os.path.isdir` :meth:`Path.is_dir` :func:`os.path.isfile` :meth:`Path.is_file` :func:`os.path.islink` :meth:`Path.is_symlink` -:func:`os.link` :meth:`Path.link_to` +:func:`os.link` :meth:`Path.hardlink_to` :func:`os.symlink` :meth:`Path.symlink_to` :func:`os.readlink` :meth:`Path.readlink` +:func:`os.path.relpath` :meth:`Path.relative_to` [#]_ :func:`os.stat` :meth:`Path.stat`, :meth:`Path.owner`, :meth:`Path.group` @@ -1229,3 +1275,8 @@ os and os.path pathlib :func:`os.path.samefile` :meth:`Path.samefile` :func:`os.path.splitext` :data:`PurePath.suffix` ==================================== ============================== + +.. rubric:: Footnotes + +.. [#] :func:`os.path.abspath` does not resolve symbolic links while :meth:`Path.resolve` does. +.. [#] :meth:`Path.relative_to` requires ``self`` to be the subpath of the argument, but :func:`os.path.relpath` does not. diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index b7c34527719486..be48561ed10ac8 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -213,7 +213,7 @@ The :mod:`pickle` module provides the following constants: The :mod:`pickle` module provides the following functions to make the pickling process more convenient: -.. function:: dump(obj, file, protocol=None, \*, fix_imports=True, buffer_callback=None) +.. function:: dump(obj, file, protocol=None, *, fix_imports=True, buffer_callback=None) Write the pickled representation of the object *obj* to the open :term:`file object` *file*. This is equivalent to @@ -225,7 +225,7 @@ process more convenient: .. versionchanged:: 3.8 The *buffer_callback* argument was added. -.. function:: dumps(obj, protocol=None, \*, fix_imports=True, buffer_callback=None) +.. function:: dumps(obj, protocol=None, *, fix_imports=True, buffer_callback=None) Return the pickled representation of the object *obj* as a :class:`bytes` object, instead of writing it to a file. @@ -236,7 +236,7 @@ process more convenient: .. versionchanged:: 3.8 The *buffer_callback* argument was added. -.. function:: load(file, \*, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) +.. function:: load(file, *, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) Read the pickled representation of an object from the open :term:`file object` *file* and return the reconstituted object hierarchy specified therein. @@ -252,7 +252,7 @@ process more convenient: .. versionchanged:: 3.8 The *buffers* argument was added. -.. function:: loads(data, /, \*, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) +.. function:: loads(data, /, *, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) Return the reconstituted object hierarchy of the pickled representation *data* of an object. *data* must be a :term:`bytes-like object`. @@ -296,7 +296,7 @@ The :mod:`pickle` module defines three exceptions: The :mod:`pickle` module exports three classes, :class:`Pickler`, :class:`Unpickler` and :class:`PickleBuffer`: -.. class:: Pickler(file, protocol=None, \*, fix_imports=True, buffer_callback=None) +.. class:: Pickler(file, protocol=None, *, fix_imports=True, buffer_callback=None) This takes a binary file for writing a pickle data stream. @@ -391,7 +391,7 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, Use :func:`pickletools.optimize` if you need more compact pickles. -.. class:: Unpickler(file, \*, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) +.. class:: Unpickler(file, *, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) This takes a binary file for reading a pickle data stream. diff --git a/Doc/library/pipes.rst b/Doc/library/pipes.rst index 0a22da1f555bc2..57e27a6acf4b66 100644 --- a/Doc/library/pipes.rst +++ b/Doc/library/pipes.rst @@ -17,6 +17,8 @@ The :mod:`pipes` module defines a class to abstract the concept of a *pipeline* Because the module uses :program:`/bin/sh` command lines, a POSIX or compatible shell for :func:`os.system` and :func:`os.popen` is required. +.. availability:: Unix. Not available on VxWorks. + The :mod:`pipes` module defines the following class: diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 8e8e3775aaff4a..be86e568c180ba 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -209,13 +209,6 @@ Windows Platform which means the OS version uses debugging code, i.e. code that checks arguments, ranges, etc. - .. note:: - - This function works best with Mark Hammond's - :mod:`win32all` package installed, but also on Python 2.3 and - later (support for this was added in Python 2.6). It obviously - only runs on Win32 compatible platforms. - .. function:: win32_edition() Returns a string representing the current Windows edition. Possible @@ -260,3 +253,41 @@ Unix Platforms using :program:`gcc`. The file is read and scanned in chunks of *chunksize* bytes. + + +Linux Platforms +--------------- + +.. function:: freedesktop_os_release() + + Get operating system identification from ``os-release`` file and return + it as a dict. The ``os-release`` file is a `freedesktop.org standard + `_ and + is available in most Linux distributions. A noticeable exception is + Android and Android-based distributions. + + Raises :exc:`OSError` or subclass when neither ``/etc/os-release`` nor + ``/usr/lib/os-release`` can be read. + + On success, the function returns a dictionary where keys and values are + strings. Values have their special characters like ``"`` and ``$`` + unquoted. The fields ``NAME``, ``ID``, and ``PRETTY_NAME`` are always + defined according to the standard. All other fields are optional. Vendors + may include additional fields. + + Note that fields like ``NAME``, ``VERSION``, and ``VARIANT`` are strings + suitable for presentation to users. Programs should use fields like + ``ID``, ``ID_LIKE``, ``VERSION_ID``, or ``VARIANT_ID`` to identify + Linux distributions. + + Example:: + + def get_like_distro(): + info = platform.freedesktop_os_release() + ids = [info["ID"]] + if "ID_LIKE" in info: + # ids are space separated and ordered by precedence + ids.extend(info["ID_LIKE"].split()) + return ids + + .. versionadded:: 3.10 diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 6def72b3736b91..ce6d4a85bf5e9d 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -52,7 +52,7 @@ or :class:`datetime.datetime` objects. This module defines the following functions: -.. function:: load(fp, \*, fmt=None, dict_type=dict) +.. function:: load(fp, *, fmt=None, dict_type=dict) Read a plist file. *fp* should be a readable and binary file object. Return the unpacked root object (which usually is a @@ -80,7 +80,7 @@ This module defines the following functions: .. versionadded:: 3.4 -.. function:: loads(data, \*, fmt=None, dict_type=dict) +.. function:: loads(data, *, fmt=None, dict_type=dict) Load a plist from a bytes object. See :func:`load` for an explanation of the keyword arguments. @@ -88,7 +88,7 @@ This module defines the following functions: .. versionadded:: 3.4 -.. function:: dump(value, fp, \*, fmt=FMT_XML, sort_keys=True, skipkeys=False) +.. function:: dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False) Write *value* to a plist file. *Fp* should be a writable, binary file object. @@ -116,7 +116,7 @@ This module defines the following functions: .. versionadded:: 3.4 -.. function:: dumps(value, \*, fmt=FMT_XML, sort_keys=True, skipkeys=False) +.. function:: dumps(value, *, fmt=FMT_XML, sort_keys=True, skipkeys=False) Return *value* as a plist-formatted bytes object. See the documentation for :func:`dump` for an explanation of the keyword diff --git a/Doc/library/posix.rst b/Doc/library/posix.rst index 2105af9215f769..ad417a17879c1f 100644 --- a/Doc/library/posix.rst +++ b/Doc/library/posix.rst @@ -37,7 +37,7 @@ Large File Support .. sectionauthor:: Steve Clift -Several operating systems (including AIX, HP-UX, Irix and Solaris) provide +Several operating systems (including AIX, HP-UX and Solaris) provide support for files that are larger than 2 GiB from a C programming model where :c:type:`int` and :c:type:`long` are 32-bit values. This is typically accomplished by defining the relevant size and offset types as 64-bit values. Such files are @@ -47,8 +47,8 @@ Large file support is enabled in Python when the size of an :c:type:`off_t` is larger than a :c:type:`long` and the :c:type:`long long` is at least as large as an :c:type:`off_t`. It may be necessary to configure and compile Python with certain compiler flags -to enable this mode. For example, it is enabled by default with recent versions -of Irix, but with Solaris 2.6 and 2.7 you need to do something like:: +to enable this mode. For example, with Solaris 2.6 and 2.7 you need to do +something like:: CFLAGS="`getconf LFS_CFLAGS`" OPT="-g -O2 $CFLAGS" \ ./configure diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index 16256c549208fc..f45c66fd9f46fb 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -28,6 +28,9 @@ Dictionaries are sorted by key before the display is computed. .. versionchanged:: 3.9 Added support for pretty-printing :class:`types.SimpleNamespace`. +.. versionchanged:: 3.10 + Added support for pretty-printing :class:`dataclasses.dataclass`. + The :mod:`pprint` module defines one class: .. First the implementation class: @@ -36,7 +39,7 @@ The :mod:`pprint` module defines one class: .. index:: single: ...; placeholder .. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None, *, \ - compact=False, sort_dicts=True) + compact=False, sort_dicts=True, underscore_numbers=False) Construct a :class:`PrettyPrinter` instance. This constructor understands several keyword parameters. An output stream may be set using the *stream* @@ -55,7 +58,10 @@ The :mod:`pprint` module defines one class: will be formatted on a separate line. If *compact* is true, as many items as will fit within the *width* will be formatted on each output line. If *sort_dicts* is true (the default), dictionaries will be formatted with their - keys sorted, otherwise they will display in insertion order. + keys sorted, otherwise they will display in insertion order. If + *underscore_numbers* is true, integers will be formatted with the + ``_`` character for a thousands separator, otherwise underscores are not + displayed (the default). .. versionchanged:: 3.4 Added the *compact* parameter. @@ -63,6 +69,8 @@ The :mod:`pprint` module defines one class: .. versionchanged:: 3.8 Added the *sort_dicts* parameter. + .. versionchanged:: 3.10 + Added the *underscore_numbers* parameter. >>> import pprint >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] @@ -91,10 +99,10 @@ The :mod:`pprint` module defines one class: The :mod:`pprint` module also provides several shortcut functions: .. function:: pformat(object, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=True) + compact=False, sort_dicts=True, underscore_numbers=False) Return the formatted representation of *object* as a string. *indent*, - *width*, *depth*, *compact* and *sort_dicts* will be passed to the + *width*, *depth*, *compact*, *sort_dicts* and *underscore_numbers* will be passed to the :class:`PrettyPrinter` constructor as formatting parameters. .. versionchanged:: 3.4 @@ -103,6 +111,9 @@ The :mod:`pprint` module also provides several shortcut functions: .. versionchanged:: 3.8 Added the *sort_dicts* parameter. + .. versionchanged:: 3.10 + Added the *underscore_numbers* parameter. + .. function:: pp(object, *args, sort_dicts=False, **kwargs) @@ -116,13 +127,13 @@ The :mod:`pprint` module also provides several shortcut functions: .. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=True) + compact=False, sort_dicts=True, underscore_numbers=False) Prints the formatted representation of *object* on *stream*, followed by a newline. If *stream* is ``None``, ``sys.stdout`` is used. This may be used in the interactive interpreter instead of the :func:`print` function for inspecting values (you can even reassign ``print = pprint.pprint`` for use - within a scope). *indent*, *width*, *depth*, *compact* and *sort_dicts* will + within a scope). *indent*, *width*, *depth*, *compact*, *sort_dicts* and *underscore_numbers* will be passed to the :class:`PrettyPrinter` constructor as formatting parameters. .. versionchanged:: 3.4 @@ -131,6 +142,9 @@ The :mod:`pprint` module also provides several shortcut functions: .. versionchanged:: 3.8 Added the *sort_dicts* parameter. + .. versionchanged:: 3.10 + Added the *underscore_numbers* parameter. + >>> import pprint >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] >>> stuff.insert(0, stuff) diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 34525a96f55c43..774d46d0e96247 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -525,16 +525,16 @@ Analysis of the profiler data is done using the :class:`~pstats.Stats` class. ordering are identical to the :meth:`~pstats.Stats.print_callers` method. - .. method:: get_stats_profile() + .. method:: get_stats_profile() This method returns an instance of StatsProfile, which contains a mapping of function names to instances of FunctionProfile. Each FunctionProfile instance holds information related to the function's profile such as how long the function took to run, how many times it was called, etc... - .. versionadded:: 3.9 - Added the following dataclasses: StatsProfile, FunctionProfile. - Added the following function: get_stats_profile. + .. versionadded:: 3.9 + Added the following dataclasses: StatsProfile, FunctionProfile. + Added the following function: get_stats_profile. .. _deterministic-profiling: diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index 36e83e85c23141..1c40ba4838ca75 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -97,6 +97,13 @@ statements. They have the following attributes: .. versionadded:: 3.7 +.. attribute:: Function.is_async + + ``True`` for functions that are defined with the ``async`` prefix, ``False`` otherwise. + + .. versionadded:: 3.10 + + .. _pyclbr-class-objects: Class Objects diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index e43b9aecd86835..034e579315de00 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -665,14 +665,14 @@ The ``errors`` module has the following attributes: .. data:: codes - A dictionary mapping numeric error codes to their string descriptions. + A dictionary mapping string descriptions to their error codes. .. versionadded:: 3.2 .. data:: messages - A dictionary mapping string descriptions to their error codes. + A dictionary mapping numeric error codes to their string descriptions. .. versionadded:: 3.2 diff --git a/Doc/library/random.rst b/Doc/library/random.rst index c243aced986e56..e924127d8b828d 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -135,6 +135,16 @@ Functions for integers values. Formerly it used a style like ``int(random()*n)`` which could produce slightly uneven distributions. + .. deprecated:: 3.10 + The automatic conversion of non-integer types to equivalent integers is + deprecated. Currently ``randrange(10.0)`` is losslessly converted to + ``randrange(10)``. In the future, this will raise a :exc:`TypeError`. + + .. deprecated:: 3.10 + The exception raised for non-integral values such as ``randrange(10.5)`` + or ``randrange('10')`` will be changed from :exc:`ValueError` to + :exc:`TypeError`. + .. function:: randint(a, b) Return a random integer *N* such that ``a <= N <= b``. Alias for @@ -142,10 +152,11 @@ Functions for integers .. function:: getrandbits(k) - Returns a Python integer with *k* random bits. This method is supplied with - the MersenneTwister generator and some other generators may also provide it - as an optional part of the API. When available, :meth:`getrandbits` enables - :meth:`randrange` to handle arbitrarily large ranges. + Returns a non-negative Python integer with *k* random bits. This method + is supplied with the MersenneTwister generator and some other generators + may also provide it as an optional part of the API. When available, + :meth:`getrandbits` enables :meth:`randrange` to handle arbitrarily large + ranges. .. versionchanged:: 3.9 This method now accepts zero for *k*. @@ -315,9 +326,9 @@ be found in any statistics text. .. function:: gauss(mu, sigma) - Gaussian distribution. *mu* is the mean, and *sigma* is the standard - deviation. This is slightly faster than the :func:`normalvariate` function - defined below. + Normal distribution, also called the Gaussian distribution. *mu* is the mean, + and *sigma* is the standard deviation. This is slightly faster than + the :func:`normalvariate` function defined below. Multithreading note: When two threads call this function simultaneously, it is possible that they will receive the @@ -408,7 +419,7 @@ Basic examples:: >>> random() # Random float: 0.0 <= x < 1.0 0.37444887175646646 - >>> uniform(2.5, 10.0) # Random float: 2.5 <= x < 10.0 + >>> uniform(2.5, 10.0) # Random float: 2.5 <= x <= 10.0 3.1800146073117523 >>> expovariate(1 / 5) # Interval between arrivals averaging 5 seconds diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index eae0a6df45f30e..4d485d25b54020 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -258,7 +258,9 @@ with a custom completer, a different set of word delimiters should be set. Get the beginning or ending index of the completion scope. These indexes are the *start* and *end* arguments passed to the :c:data:`rl_attempted_completion_function` callback of the - underlying library. + underlying library. The values may be different in the same + input editing scenario based on the underlying C readline implementation. + Ex: libedit is known to behave differently than libreadline. .. function:: set_completer_delims(string) diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index e4eac43642d14d..67e9b44fe48c46 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -241,7 +241,9 @@ platform. The maximum size (in bytes) of the swap space that may be reserved or used by all of this user id's processes. This limit is enforced only if bit 1 of the vm.overcommit sysctl is set. - Please see :manpage:`tuning(7)` for a complete description of this sysctl. + Please see + `tuning(7) `__ + for a complete description of this sysctl. .. availability:: FreeBSD 9 or later. @@ -255,6 +257,14 @@ platform. .. versionadded:: 3.4 +.. data:: RLIMIT_KQUEUES + + The maximum number of kqueues this user id is allowed to create. + + .. availability:: FreeBSD 11 or later. + + .. versionadded:: 3.10 + Resource Usage -------------- diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index f08c58179a2f9f..07caf91d5b7d99 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -25,8 +25,9 @@ lots of shared sub-objects. The keys are ordinary strings. database file is opened for reading and writing. The optional *flag* parameter has the same interpretation as the *flag* parameter of :func:`dbm.open`. - By default, version 3 pickles are used to serialize values. The version of the - pickle protocol can be specified with the *protocol* parameter. + By default, pickles created with :data:`pickle.DEFAULT_PROTOCOL` are used + to serialize values. The version of the pickle protocol can be specified + with the *protocol* parameter. Because of Python semantics, a shelf cannot know when a mutable persistent-dictionary entry is modified. By default modified objects are @@ -40,6 +41,10 @@ lots of shared sub-objects. The keys are ordinary strings. determine which accessed entries are mutable, nor which ones were actually mutated). + .. versionchanged:: 3.10 + :data:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle + protocol. + .. note:: Do not rely on the shelf being closed automatically; always call @@ -108,9 +113,10 @@ Restrictions A subclass of :class:`collections.abc.MutableMapping` which stores pickled values in the *dict* object. - By default, version 3 pickles are used to serialize values. The version of the - pickle protocol can be specified with the *protocol* parameter. See the - :mod:`pickle` documentation for a discussion of the pickle protocols. + By default, pickles created with :data:`pickle.DEFAULT_PROTOCOL` are used + to serialize values. The version of the pickle protocol can be specified + with the *protocol* parameter. See the :mod:`pickle` documentation for a + discussion of the pickle protocols. If the *writeback* parameter is ``True``, the object will hold a cache of all entries accessed and write them back to the *dict* at sync and close times. @@ -130,6 +136,10 @@ Restrictions .. versionchanged:: 3.4 Added context manager support. + .. versionchanged:: 3.10 + :data:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle + protocol. + .. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8') diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index 7f7f0c7f124ac5..aab6a543792096 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -61,6 +61,20 @@ The :mod:`shlex` module defines the following functions: string that can safely be used as one token in a shell command line, for cases where you cannot use a list. + .. _shlex-quote-warning: + + .. warning:: + + The ``shlex`` module is **only designed for Unix shells**. + + The :func:`quote` function is not guaranteed to be correct on non-POSIX + compliant shells or shells from other operating systems such as Windows. + Executing commands quoted by this module on such shells can open up the + possibility of a command injection vulnerability. + + Consider using functions that pass command arguments with lists such as + :func:`subprocess.run` with ``shell=False``. + This idiom would be unsafe: >>> filename = 'somefile; rm -rf ~' diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 3f5122760ee16f..d5080da15bba41 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -218,7 +218,7 @@ Directory and files operations copy the file more efficiently. See :ref:`shutil-platform-dependent-efficient-copy-operations` section. -.. function:: ignore_patterns(\*patterns) +.. function:: ignore_patterns(*patterns) This factory function creates a function that can be used as a callable for :func:`copytree`\'s *ignore* argument, ignoring files and directories that @@ -443,8 +443,9 @@ Directory and files operations Platform-dependent efficient copy operations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Starting from Python 3.8 all functions involving a file copy (:func:`copyfile`, -:func:`copy`, :func:`copy2`, :func:`copytree`, and :func:`move`) may use +Starting from Python 3.8, all functions involving a file copy +(:func:`copyfile`, :func:`~shutil.copy`, :func:`copy2`, +:func:`copytree`, and :func:`move`) may use platform-specific "fast-copy" syscalls in order to copy the file more efficiently (see :issue:`33671`). "fast-copy" means that the copying operation occurs within the kernel, avoiding diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 00a730b6b9ca6a..84a569d03eb293 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -117,7 +117,7 @@ The variables defined in the :mod:`signal` module are: Child process stopped or terminated. - .. availability:: Windows. + .. availability:: Unix. .. data:: SIGCLD diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index b3cc60357f554a..52220f7a7f30ac 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -32,7 +32,7 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). than a success code, an :exc:`SMTPConnectError` is raised. The optional *timeout* parameter specifies a timeout in seconds for blocking operations like the connection attempt (if not specified, the global default timeout - setting will be used). If the timeout expires, :exc:`socket.timeout` is + setting will be used). If the timeout expires, :exc:`TimeoutError` is raised. The optional source_address parameter allows binding to some specific source address in a machine with multiple network interfaces, and/or to some specific source TCP port. It takes a 2-tuple @@ -115,7 +115,7 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). If the *timeout* parameter is set to be zero, it will raise a :class:`ValueError` to prevent the creation of a non-blocking socket -.. class:: LMTP(host='', port=LMTP_PORT, local_hostname=None, +.. class:: LMTP(host='', port=LMTP_PORT, local_hostname=None, \ source_address=None[, timeout]) The LMTP protocol, which is very similar to ESMTP, is heavily based on the diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index faf8a76251420e..8a1fc96e0362e2 100755 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -56,12 +56,12 @@ created. Socket addresses are represented as follows: bytes-like object can be used for either type of address when passing it as an argument. - .. versionchanged:: 3.3 - Previously, :const:`AF_UNIX` socket paths were assumed to use UTF-8 - encoding. + .. versionchanged:: 3.3 + Previously, :const:`AF_UNIX` socket paths were assumed to use UTF-8 + encoding. - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. + .. versionchanged:: 3.5 + Writable :term:`bytes-like object` is now accepted. .. _host_port: @@ -283,6 +283,8 @@ Exceptions .. exception:: timeout + A deprecated alias of :exc:`TimeoutError`. + A subclass of :exc:`OSError`, this exception is raised when a timeout occurs on a socket which has had timeouts enabled via a prior call to :meth:`~socket.settimeout` (or implicitly through @@ -292,6 +294,9 @@ Exceptions .. versionchanged:: 3.3 This class was made a subclass of :exc:`OSError`. + .. versionchanged:: 3.10 + This class was made an alias of :exc:`TimeoutError`. + Constants ^^^^^^^^^ @@ -605,6 +610,9 @@ The following functions all create :ref:`socket objects `. .. versionchanged:: 3.9 The CAN_J1939 protocol was added. + .. versionchanged:: 3.10 + The IPPROTO_MPTCP protocol was added. + .. function:: socketpair([family[, type[, proto]]]) Build a pair of connected socket objects using the given address family, socket @@ -777,9 +785,9 @@ The :mod:`socket` module also offers various network-related services: system if IPv6 isn't enabled):: >>> socket.getaddrinfo("example.org", 80, proto=socket.IPPROTO_TCP) - [(, , + [(socket.AF_INET6, socket.SOCK_STREAM, 6, '', ('2606:2800:220:1:248:1893:25c8:1946', 80, 0, 0)), - (, , + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('93.184.216.34', 80))] .. versionchanged:: 3.2 @@ -902,11 +910,9 @@ The :mod:`socket` module also offers various network-related services: where the host byte order is the same as network byte order, this is a no-op; otherwise, it performs a 2-byte swap operation. - .. deprecated:: 3.7 - In case *x* does not fit in 16-bit unsigned integer, but does fit in a - positive C int, it is silently truncated to 16-bit unsigned integer. - This silent truncation feature is deprecated, and will raise an - exception in future versions of Python. + .. versionchanged:: 3.10 + Raises :exc:`OverflowError` if *x* does not fit in a 16-bit unsigned + integer. .. function:: htonl(x) @@ -922,11 +928,9 @@ The :mod:`socket` module also offers various network-related services: where the host byte order is the same as network byte order, this is a no-op; otherwise, it performs a 2-byte swap operation. - .. deprecated:: 3.7 - In case *x* does not fit in 16-bit unsigned integer, but does fit in a - positive C int, it is silently truncated to 16-bit unsigned integer. - This silent truncation feature is deprecated, and will raise an - exception in future versions of Python. + .. versionchanged:: 3.10 + Raises :exc:`OverflowError` if *x* does not fit in a 16-bit unsigned + integer. .. function:: inet_aton(ip_string) @@ -1139,6 +1143,32 @@ The :mod:`socket` module also offers various network-related services: "Interface name" is a name as documented in :func:`if_nameindex`. +.. function:: send_fds(sock, buffers, fds[, flags[, address]]) + + Send the list of file descriptors *fds* over an :const:`AF_UNIX` socket *sock*. + The *fds* parameter is a sequence of file descriptors. + Consult :meth:`sendmsg` for the documentation of these parameters. + + .. availability:: Unix supporting :meth:`~socket.sendmsg` and :const:`SCM_RIGHTS` mechanism. + + .. versionadded:: 3.9 + + +.. function:: recv_fds(sock, bufsize, maxfds[, flags]) + + Receive up to *maxfds* file descriptors from an :const:`AF_UNIX` socket *sock*. + Return ``(msg, list(fds), flags, addr)``. + Consult :meth:`recvmsg` for the documentation of these parameters. + + .. availability:: Unix supporting :meth:`~socket.recvmsg` and :const:`SCM_RIGHTS` mechanism. + + .. versionadded:: 3.9 + + .. note:: + + Any truncated integers at the end of the list of file descriptors. + + .. _socket-objects: Socket Objects @@ -1208,7 +1238,7 @@ to sockets. address family --- see above.) If the connection is interrupted by a signal, the method waits until the - connection completes, or raise a :exc:`socket.timeout` on timeout, if the + connection completes, or raise a :exc:`TimeoutError` on timeout, if the signal handler doesn't raise an exception and the socket is blocking or has a timeout. For non-blocking sockets, the method raises an :exc:`InterruptedError` exception if the connection is interrupted by a @@ -1633,29 +1663,6 @@ to sockets. .. versionadded:: 3.6 -.. method:: socket.send_fds(sock, buffers, fds[, flags[, address]]) - - Send the list of file descriptors *fds* over an :const:`AF_UNIX` socket. - The *fds* parameter is a sequence of file descriptors. - Consult :meth:`sendmsg` for the documentation of these parameters. - - .. availability:: Unix supporting :meth:`~socket.sendmsg` and :const:`SCM_RIGHTS` mechanism. - - .. versionadded:: 3.9 - -.. method:: socket.recv_fds(sock, bufsize, maxfds[, flags]) - - Receive up to *maxfds* file descriptors. Return ``(msg, list(fds), flags, addr)``. Consult - :meth:`recvmsg` for the documentation of these parameters. - - .. availability:: Unix supporting :meth:`~socket.recvmsg` and :const:`SCM_RIGHTS` mechanism. - - .. versionadded:: 3.9 - - .. note:: - - Any truncated integers at the end of the list of file descriptors. - .. method:: socket.sendfile(file, offset=0, count=None) Send a file until EOF is reached by using high-performance diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 13aa8c512d0319..d0f28db12fda16 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -19,68 +19,41 @@ PostgreSQL or Oracle. The sqlite3 module was written by Gerhard Häring. It provides a SQL interface compliant with the DB-API 2.0 specification described by :pep:`249`, and -requires SQLite 3.7.3 or newer. +requires SQLite 3.7.15 or newer. To use the module, you must first create a :class:`Connection` object that represents the database. Here the data will be stored in the :file:`example.db` file:: import sqlite3 - conn = sqlite3.connect('example.db') + con = sqlite3.connect('example.db') You can also supply the special name ``:memory:`` to create a database in RAM. Once you have a :class:`Connection`, you can create a :class:`Cursor` object and call its :meth:`~Cursor.execute` method to perform SQL commands:: - c = conn.cursor() + cur = con.cursor() # Create table - c.execute('''CREATE TABLE stocks - (date text, trans text, symbol text, qty real, price real)''') + cur.execute('''CREATE TABLE stocks + (date text, trans text, symbol text, qty real, price real)''') # Insert a row of data - c.execute("INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)") + cur.execute("INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)") # Save (commit) the changes - conn.commit() + con.commit() # We can also close the connection if we are done with it. # Just be sure any changes have been committed or they will be lost. - conn.close() + con.close() The data you've saved is persistent and is available in subsequent sessions:: import sqlite3 - conn = sqlite3.connect('example.db') - c = conn.cursor() - -Usually your SQL operations will need to use values from Python variables. You -shouldn't assemble your query using Python's string operations because doing so -is insecure; it makes your program vulnerable to an SQL injection attack -(see https://xkcd.com/327/ for humorous example of what can go wrong). - -Instead, use the DB-API's parameter substitution. Put ``?`` as a placeholder -wherever you want to use a value, and then provide a tuple of values as the -second argument to the cursor's :meth:`~Cursor.execute` method. (Other database -modules may use a different placeholder, such as ``%s`` or ``:1``.) For -example:: - - # Never do this -- insecure! - symbol = 'RHAT' - c.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) - - # Do this instead - t = ('RHAT',) - c.execute('SELECT * FROM stocks WHERE symbol=?', t) - print(c.fetchone()) - - # Larger example that inserts many records at a time - purchases = [('2006-03-28', 'BUY', 'IBM', 1000, 45.00), - ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00), - ('2006-04-06', 'SELL', 'IBM', 500, 53.00), - ] - c.executemany('INSERT INTO stocks VALUES (?,?,?,?,?)', purchases) + con = sqlite3.connect('example.db') + cur = con.cursor() To retrieve data after executing a SELECT statement, you can either treat the cursor as an :term:`iterator`, call the cursor's :meth:`~Cursor.fetchone` method to @@ -89,7 +62,7 @@ matching rows. This example uses the iterator form:: - >>> for row in c.execute('SELECT * FROM stocks ORDER BY price'): + >>> for row in cur.execute('SELECT * FROM stocks ORDER BY price'): print(row) ('2006-01-05', 'BUY', 'RHAT', 100, 35.14) @@ -98,11 +71,34 @@ This example uses the iterator form:: ('2006-04-05', 'BUY', 'MSFT', 1000, 72.0) -.. seealso:: +.. _sqlite3-placeholders: + +Usually your SQL operations will need to use values from Python variables. You +shouldn't assemble your query using Python's string operations because doing so +is insecure; it makes your program vulnerable to an SQL injection attack +(see the `xkcd webcomic `_ for a humorous example of +what can go wrong):: - https://github.com/ghaering/pysqlite - The pysqlite web page -- sqlite3 is developed externally under the name - "pysqlite". + # Never do this -- insecure! + symbol = 'RHAT' + cur.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) + +Instead, use the DB-API's parameter substitution. Put a placeholder wherever +you want to use a value, and then provide a tuple of values as the second +argument to the cursor's :meth:`~Cursor.execute` method. An SQL statement may +use one of two kinds of placeholders: question marks (qmark style) or named +placeholders (named style). For the qmark style, ``parameters`` must be a +:term:`sequence `. For the named style, it can be either a +:term:`sequence ` or :class:`dict` instance. The length of the +:term:`sequence ` must match the number of placeholders, or a +:exc:`ProgrammingError` is raised. If a :class:`dict` is given, it must contain +keys for all named parameters. Any extra items are ignored. Here's an example +of both styles: + +.. literalinclude:: ../includes/sqlite3/execute_1.py + + +.. seealso:: https://www.sqlite.org The SQLite web page; the documentation describes the syntax and the @@ -198,7 +194,9 @@ Module functions and constants *detect_types* defaults to 0 (i. e. off, no type detection), you can set it to any combination of :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES` to turn - type detection on. + type detection on. Due to SQLite behaviour, types can't be detected for generated + fields (for example ``max(data)``), even when *detect_types* parameter is set. In + such case, the returned type is :class:`str`. By default, *check_same_thread* is :const:`True` and only the creating thread may use the connection. If set :const:`False`, the returned connection may be shared @@ -227,6 +225,7 @@ Module functions and constants be found in the `SQLite URI documentation `_. .. audit-event:: sqlite3.connect database sqlite3.connect + .. audit-event:: sqlite3.connect/handle connection_handle sqlite3.connect .. versionchanged:: 3.4 Added the *uri* parameter. @@ -234,6 +233,9 @@ Module functions and constants .. versionchanged:: 3.7 *database* can now also be a :term:`path-like object`, not only a string. + .. versionchanged:: 3.10 + Added the ``sqlite3.connect/handle`` auditing event. + .. function:: register_converter(typename, callable) @@ -469,8 +471,13 @@ Connection Objects Loadable extensions are disabled by default. See [#f1]_. + .. audit-event:: sqlite3.enable_load_extension connection,enabled sqlite3.enable_load_extension + .. versionadded:: 3.2 + .. versionchanged:: 3.10 + Added the ``sqlite3.enable_load_extension`` auditing event. + .. literalinclude:: ../includes/sqlite3/load_extension.py .. method:: load_extension(path) @@ -481,8 +488,13 @@ Connection Objects Loadable extensions are disabled by default. See [#f1]_. + .. audit-event:: sqlite3.load_extension connection,path sqlite3.load_extension + .. versionadded:: 3.2 + .. versionchanged:: 3.10 + Added the ``sqlite3.load_extension`` auditing event. + .. attribute:: row_factory You can change this attribute to a callable that accepts the cursor and the @@ -544,7 +556,7 @@ Connection Objects con.close() - .. method:: backup(target, *, pages=0, progress=None, name="main", sleep=0.250) + .. method:: backup(target, *, pages=-1, progress=None, name="main", sleep=0.250) This method makes a backup of a SQLite database even while it's being accessed by other clients, or concurrently by the same connection. The copy will be @@ -609,14 +621,8 @@ Cursor Objects .. method:: execute(sql[, parameters]) - Executes an SQL statement. The SQL statement may be parameterized (i. e. - placeholders instead of SQL literals). The :mod:`sqlite3` module supports two - kinds of placeholders: question marks (qmark style) and named placeholders - (named style). - - Here's an example of both styles: - - .. literalinclude:: ../includes/sqlite3/execute_1.py + Executes an SQL statement. Values may be bound to the statement using + :ref:`placeholders `. :meth:`execute` will only execute a single SQL statement. If you try to execute more than one statement with it, it will raise a :exc:`.Warning`. Use @@ -626,9 +632,10 @@ Cursor Objects .. method:: executemany(sql, seq_of_parameters) - Executes an SQL command against all parameter sequences or mappings found in - the sequence *seq_of_parameters*. The :mod:`sqlite3` module also allows - using an :term:`iterator` yielding parameters instead of a sequence. + Executes a :ref:`parameterized ` SQL command + against all parameter sequences or mappings found in the sequence + *seq_of_parameters*. The :mod:`sqlite3` module also allows using an + :term:`iterator` yielding parameters instead of a sequence. .. literalinclude:: ../includes/sqlite3/executemany_1.py @@ -766,23 +773,23 @@ Row Objects Let's assume we initialize a table as in the example given above:: - conn = sqlite3.connect(":memory:") - c = conn.cursor() - c.execute('''create table stocks + con = sqlite3.connect(":memory:") + cur = con.cursor() + cur.execute('''create table stocks (date text, trans text, symbol text, qty real, price real)''') - c.execute("""insert into stocks - values ('2006-01-05','BUY','RHAT',100,35.14)""") - conn.commit() - c.close() + cur.execute("""insert into stocks + values ('2006-01-05','BUY','RHAT',100,35.14)""") + con.commit() + cur.close() Now we plug :class:`Row` in:: - >>> conn.row_factory = sqlite3.Row - >>> c = conn.cursor() - >>> c.execute('select * from stocks') + >>> con.row_factory = sqlite3.Row + >>> cur = con.cursor() + >>> cur.execute('select * from stocks') - >>> r = c.fetchone() + >>> r = cur.fetchone() >>> type(r) >>> tuple(r) @@ -1087,23 +1094,10 @@ committed: .. literalinclude:: ../includes/sqlite3/ctx_manager.py -Common issues -------------- - -Multithreading -^^^^^^^^^^^^^^ - -Older SQLite versions had issues with sharing connections between threads. -That's why the Python module disallows sharing connections and cursors between -threads. If you still try to do so, you will get an exception at runtime. - -The only exception is calling the :meth:`~Connection.interrupt` method, which -only makes sense to call from a different thread. - .. rubric:: Footnotes .. [#f1] The sqlite3 module is not built with loadable extension support by default, because some platforms (notably Mac OS X) have SQLite libraries which are compiled without this feature. To get loadable - extension support, you must pass --enable-loadable-sqlite-extensions to - configure. + extension support, you must pass the + :option:`--enable-loadable-sqlite-extensions` option to configure. diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 1cfd165202d0ef..afa3d87f5767a1 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -25,8 +25,8 @@ probably additional platforms, as long as OpenSSL is installed on that platform. Some behavior may be platform dependent, since calls are made to the operating system socket APIs. The installed version of OpenSSL may also - cause variations in behavior. For example, TLSv1.1 and TLSv1.2 come with - openssl version 1.0.1. + cause variations in behavior. For example, TLSv1.3 with OpenSSL version + 1.1.1. .. warning:: Don't use this module without reading the :ref:`ssl-security`. Doing so @@ -58,6 +58,13 @@ by SSL sockets created through the :meth:`SSLContext.wrap_socket` method. In the future the ssl module will require at least OpenSSL 1.0.2 or 1.1.0. +.. versionchanged:: 3.10 + + :pep:`644` has been implemented. The ssl module requires OpenSSL 1.1.1 + or newer. + + Use of deprecated constants and functions result in deprecation warnings. + Functions, Constants, and Exceptions ------------------------------------ @@ -131,8 +138,9 @@ purposes. :const:`None`, this function can choose to trust the system's default CA certificates instead. - The settings are: :data:`PROTOCOL_TLS`, :data:`OP_NO_SSLv2`, and - :data:`OP_NO_SSLv3` with high encryption cipher suites without RC4 and + The settings are: :data:`PROTOCOL_TLS_CLIENT` or + :data:`PROTOCOL_TLS_SERVER`, :data:`OP_NO_SSLv2`, and :data:`OP_NO_SSLv3` + with high encryption cipher suites without RC4 and without unauthenticated cipher suites. Passing :data:`~Purpose.SERVER_AUTH` as *purpose* sets :data:`~SSLContext.verify_mode` to :data:`CERT_REQUIRED` and either loads CA certificates (when at least one of *cafile*, *capath* or @@ -180,6 +188,12 @@ purposes. Support for key logging to :envvar:`SSLKEYLOGFILE` was added. + .. versionchanged:: 3.10 + + The context now uses :data:`PROTOCOL_TLS_CLIENT` or + :data:`PROTOCOL_TLS_SERVER` protocol instead of generic + :data:`PROTOCOL_TLS`. + Exceptions ^^^^^^^^^^ @@ -325,19 +339,6 @@ Random generation :func:`ssl.RAND_egd` and :func:`ssl.RAND_add` to increase the randomness of the pseudo-random number generator. -.. function:: RAND_egd(path) - - If you are running an entropy-gathering daemon (EGD) somewhere, and *path* - is the pathname of a socket connection open to it, this will read 256 bytes - of randomness from the socket, and add it to the SSL pseudo-random number - generator to increase the security of generated secret keys. This is - typically only necessary on systems without better sources of randomness. - - See http://egd.sourceforge.net/ or http://prngd.sourceforge.net/ for sources - of entropy-gathering daemons. - - .. availability:: not available with LibreSSL and OpenSSL > 1.1.0. - .. function:: RAND_add(bytes, entropy) Mix the given *bytes* into the SSL pseudo-random number generator. The @@ -425,7 +426,8 @@ Certificate handling previously. Return an integer (no fractions of a second in the input format) -.. function:: get_server_certificate(addr, ssl_version=PROTOCOL_TLS, ca_certs=None) +.. function:: get_server_certificate(addr, ssl_version=PROTOCOL_TLS_CLIENT, \ + ca_certs=None[, timeout]) Given the address ``addr`` of an SSL-protected server, as a (*hostname*, *port-number*) pair, fetches the server's certificate, and returns it as a @@ -435,7 +437,8 @@ Certificate handling same format as used for the same parameter in :meth:`SSLContext.wrap_socket`. The call will attempt to validate the server certificate against that set of root certificates, and will fail - if the validation attempt fails. + if the validation attempt fails. A timeout can be specified with the + ``timeout`` parameter. .. versionchanged:: 3.3 This function is now IPv6-compatible. @@ -444,6 +447,9 @@ Certificate handling The default *ssl_version* is changed from :data:`PROTOCOL_SSLv3` to :data:`PROTOCOL_TLS` for maximum compatibility with modern servers. + .. versionchanged:: 3.10 + The *timeout* parameter was added. + .. function:: DER_cert_to_PEM_cert(DER_cert_bytes) Given a certificate as a DER-encoded blob of bytes, returns a PEM-encoded @@ -634,6 +640,13 @@ Constants .. versionadded:: 3.4 +.. data:: VERIFY_ALLOW_PROXY_CERTS + + Possible value for :attr:`SSLContext.verify_flags` to enables proxy + certificate verification. + + .. versionadded:: 3.10 + .. data:: VERIFY_X509_TRUSTED_FIRST Possible value for :attr:`SSLContext.verify_flags`. It instructs OpenSSL to @@ -642,6 +655,17 @@ Constants .. versionadded:: 3.4.4 +.. data:: VERIFY_X509_PARTIAL_CHAIN + + Possible value for :attr:`SSLContext.verify_flags`. It instructs OpenSSL to + accept intermediate CAs in the trust store to be treated as trust-anchors, + in the same way as the self-signed root CA certificates. This makes it + possible to trust certificates issued by an intermediate CA without having + to trust its ancestor root CA. + + .. versionadded:: 3.10 + + .. class:: VerifyFlags :class:`enum.IntFlag` collection of VERIFY_* constants. @@ -655,6 +679,8 @@ Constants .. versionadded:: 3.6 + .. deprecated:: 3.10 + .. data:: PROTOCOL_TLS_CLIENT Auto-negotiate the highest protocol version like :data:`PROTOCOL_TLS`, @@ -708,7 +734,10 @@ Constants .. deprecated:: 3.6 OpenSSL has deprecated all version specific protocols. Use the default - protocol :data:`PROTOCOL_TLS` with flags like :data:`OP_NO_SSLv3` instead. + protocol :data:`PROTOCOL_TLS_SERVER` or :data:`PROTOCOL_TLS_CLIENT` + with :attr:`SSLContext.minimum_version` and + :attr:`SSLContext.maximum_version` instead. + .. data:: PROTOCOL_TLSv1 @@ -716,8 +745,7 @@ Constants .. deprecated:: 3.6 - OpenSSL has deprecated all version specific protocols. Use the default - protocol :data:`PROTOCOL_TLS` with flags like :data:`OP_NO_SSLv3` instead. + OpenSSL has deprecated all version specific protocols. .. data:: PROTOCOL_TLSv1_1 @@ -728,21 +756,18 @@ Constants .. deprecated:: 3.6 - OpenSSL has deprecated all version specific protocols. Use the default - protocol :data:`PROTOCOL_TLS` with flags like :data:`OP_NO_SSLv3` instead. + OpenSSL has deprecated all version specific protocols. .. data:: PROTOCOL_TLSv1_2 - Selects TLS version 1.2 as the channel encryption protocol. This is the - most modern version, and probably the best choice for maximum protection, - if both sides can speak it. Available only with openssl version 1.0.1+. + Selects TLS version 1.2 as the channel encryption protocol. + Available only with openssl version 1.0.1+. .. versionadded:: 3.4 .. deprecated:: 3.6 - OpenSSL has deprecated all version specific protocols. Use the default - protocol :data:`PROTOCOL_TLS` with flags like :data:`OP_NO_SSLv3` instead. + OpenSSL has deprecated all version specific protocols. .. data:: OP_ALL @@ -764,7 +789,6 @@ Constants SSLv2 is deprecated - .. data:: OP_NO_SSLv3 Prevents an SSLv3 connection. This option is only applicable in @@ -872,8 +896,6 @@ Constants Disable compression on the SSL channel. This is useful if the application protocol supports its own compression scheme. - This option is only available with OpenSSL 1.0.0 and later. - .. versionadded:: 3.3 .. class:: Options @@ -886,6 +908,14 @@ Constants .. versionadded:: 3.6 +.. data:: OP_IGNORE_UNEXPECTED_EOF + + Ignore unexpected shutdown of TLS connections. + + This option is only available with OpenSSL 3.0.0 and later. + + .. versionadded:: 3.10 + .. data:: HAS_ALPN Whether the OpenSSL library has built-in support for the *Application-Layer @@ -1064,6 +1094,11 @@ Constants SSL 3.0 to TLS 1.3. + .. deprecated:: 3.10 + + All :class:`TLSVersion` members except :attr:`TLSVersion.TLSv1_2` and + :attr:`TLSVersion.TLSv1_3` are deprecated. + SSL Sockets ----------- @@ -1118,6 +1153,11 @@ SSL Sockets to create instances directly. This was never documented or officially supported. + .. versionchanged:: 3.10 + Python now uses ``SSL_read_ex`` and ``SSL_write_ex`` internally. The + functions support reading and writing of data larger than 2 GB. Writing + zero-length data no longer fails with a protocol violation error. + SSL sockets also have the following additional methods and attributes: .. method:: SSLSocket.read(len=1024, buffer=None) @@ -1318,6 +1358,10 @@ SSL sockets also have the following additional methods and attributes: .. versionadded:: 3.3 + .. deprecated:: 3.10 + + NPN has been superseded by ALPN + .. method:: SSLSocket.unwrap() Performs the SSL shutdown handshake, which removes the TLS layer from the @@ -1414,7 +1458,7 @@ such as SSL configuration options, certificate(s) and private key(s). It also manages a cache of SSL sessions for server-side sockets, in order to speed up repeated connections from the same clients. -.. class:: SSLContext(protocol=PROTOCOL_TLS) +.. class:: SSLContext(protocol=None) Create a new SSL context. You may pass *protocol* which must be one of the ``PROTOCOL_*`` constants defined in this module. The parameter @@ -1463,6 +1507,20 @@ to speed up repeated connections from the same clients. ciphers, no ``NULL`` ciphers and no ``MD5`` ciphers (except for :data:`PROTOCOL_SSLv2`). + .. deprecated:: 3.10 + + :class:`SSLContext` without protocol argument is deprecated. The + context class will either require :data:`PROTOCOL_TLS_CLIENT` or + :data:`PROTOCOL_TLS_SERVER` protocol in the future. + + .. versionchanged:: 3.10 + + The default cipher suites now include only secure AES and ChaCha20 + ciphers with forward secrecy and security level 2. RSA and DH keys with + less than 2048 bits and ECC keys with less than 224 bits are prohibited. + :data:`PROTOCOL_TLS`, :data:`PROTOCOL_TLS_CLIENT`, and + :data:`PROTOCOL_TLS_SERVER` use TLS 1.2 as minimum TLS version. + :class:`SSLContext` objects have the following methods and attributes: @@ -1578,25 +1636,7 @@ to speed up repeated connections from the same clients. >>> ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) >>> ctx.set_ciphers('ECDHE+AESGCM:!ECDSA') - >>> ctx.get_ciphers() # OpenSSL 1.0.x - [{'alg_bits': 256, - 'description': 'ECDHE-RSA-AES256-GCM-SHA384 TLSv1.2 Kx=ECDH Au=RSA ' - 'Enc=AESGCM(256) Mac=AEAD', - 'id': 50380848, - 'name': 'ECDHE-RSA-AES256-GCM-SHA384', - 'protocol': 'TLSv1/SSLv3', - 'strength_bits': 256}, - {'alg_bits': 128, - 'description': 'ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 Kx=ECDH Au=RSA ' - 'Enc=AESGCM(128) Mac=AEAD', - 'id': 50380847, - 'name': 'ECDHE-RSA-AES128-GCM-SHA256', - 'protocol': 'TLSv1/SSLv3', - 'strength_bits': 128}] - - On OpenSSL 1.1 and newer the cipher dict contains additional fields:: - - >>> ctx.get_ciphers() # OpenSSL 1.1+ + >>> ctx.get_ciphers() [{'aead': True, 'alg_bits': 256, 'auth': 'auth-rsa', @@ -1622,8 +1662,6 @@ to speed up repeated connections from the same clients. 'strength_bits': 128, 'symmetric': 'aes-128-gcm'}] - .. availability:: OpenSSL 1.0.2+. - .. versionadded:: 3.6 .. method:: SSLContext.set_default_verify_paths() @@ -1648,8 +1686,8 @@ to speed up repeated connections from the same clients. when connected, the :meth:`SSLSocket.cipher` method of SSL sockets will give the currently selected cipher. - OpenSSL 1.1.1 has TLS 1.3 cipher suites enabled by default. The suites - cannot be disabled with :meth:`~SSLContext.set_ciphers`. + TLS 1.3 cipher suites cannot be disabled with + :meth:`~SSLContext.set_ciphers`. .. method:: SSLContext.set_alpn_protocols(protocols) @@ -1663,10 +1701,6 @@ to speed up repeated connections from the same clients. This method will raise :exc:`NotImplementedError` if :data:`HAS_ALPN` is ``False``. - OpenSSL 1.1.0 to 1.1.0e will abort the handshake and raise :exc:`SSLError` - when both sides support ALPN but cannot agree on a protocol. 1.1.0f+ - behaves like 1.0.2, :meth:`SSLSocket.selected_alpn_protocol` returns None. - .. versionadded:: 3.5 .. method:: SSLContext.set_npn_protocols(protocols) @@ -1684,6 +1718,10 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.3 + .. deprecated:: 3.10 + + NPN has been superseded by ALPN + .. attribute:: SSLContext.sni_callback Register a callback function that will be called after the TLS Client Hello @@ -1868,7 +1906,7 @@ to speed up repeated connections from the same clients. .. method:: SSLContext.session_stats() Get statistics about the SSL sessions created or managed by this context. - A dictionary is returned which maps the names of each `piece of information `_ to their + A dictionary is returned which maps the names of each `piece of information `_ to their numeric values. For example, here is the total number of hits and misses in the session cache since the context was created:: @@ -1878,7 +1916,7 @@ to speed up repeated connections from the same clients. .. attribute:: SSLContext.check_hostname - Whether to match the peer cert's hostname with :func:`match_hostname` in + Whether to match the peer cert's hostname in :meth:`SSLSocket.do_handshake`. The context's :attr:`~SSLContext.verify_mode` must be set to :data:`CERT_OPTIONAL` or :data:`CERT_REQUIRED`, and you must pass *server_hostname* to @@ -1911,10 +1949,6 @@ to speed up repeated connections from the same clients. :attr:`~SSLContext.verify_mode` is :data:`CERT_NONE`. Previously the same operation would have failed with a :exc:`ValueError`. - .. note:: - - This features requires OpenSSL 0.9.8f or newer. - .. attribute:: SSLContext.keylog_filename Write TLS keys to a keylog file, whenever key material is generated or @@ -1925,10 +1959,6 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.8 - .. note:: - - This features requires OpenSSL 1.1.1 or newer. - .. attribute:: SSLContext.maximum_version A :class:`TLSVersion` enum member representing the highest supported @@ -1945,11 +1975,6 @@ to speed up repeated connections from the same clients. :attr:`~SSLContext.maximum_version` set to :attr:`TLSVersion.TLSv1_2` will not be able to establish a TLS 1.2 connection. - .. note:: - - This attribute is not available unless the ssl module is compiled - with OpenSSL 1.1.0g or newer. - .. versionadded:: 3.7 .. attribute:: SSLContext.minimum_version @@ -1957,24 +1982,14 @@ to speed up repeated connections from the same clients. Like :attr:`SSLContext.maximum_version` except it is the lowest supported version or :attr:`TLSVersion.MINIMUM_SUPPORTED`. - .. note:: - - This attribute is not available unless the ssl module is compiled - with OpenSSL 1.1.0g or newer. - .. versionadded:: 3.7 .. attribute:: SSLContext.num_tickets Control the number of TLS 1.3 session tickets of a - :attr:`TLS_PROTOCOL_SERVER` context. The setting has no impact on TLS + :attr:`PROTOCOL_TLS_SERVER` context. The setting has no impact on TLS 1.0 to 1.2 connections. - .. note:: - - This attribute is not available unless the ssl module is compiled - with OpenSSL 1.1.1 or newer. - .. versionadded:: 3.8 .. attribute:: SSLContext.options @@ -1983,17 +1998,18 @@ to speed up repeated connections from the same clients. The default value is :data:`OP_ALL`, but you can specify other options such as :data:`OP_NO_SSLv2` by ORing them together. - .. note:: - With versions of OpenSSL older than 0.9.8m, it is only possible - to set options, not to clear them. Attempting to clear an option - (by resetting the corresponding bits) will raise a :exc:`ValueError`. - .. versionchanged:: 3.6 :attr:`SSLContext.options` returns :class:`Options` flags: >>> ssl.create_default_context().options # doctest: +SKIP + .. deprecated:: 3.7 + + All ``OP_NO_SSL*`` and ``OP_NO_TLS*`` options have been deprecated since + Python 3.7. Use :attr:`SSLContext.minimum_version` and + :attr:`SSLContext.maximum_version` instead. + .. attribute:: SSLContext.post_handshake_auth Enable TLS 1.3 post-handshake client authentication. Post-handshake auth @@ -2010,10 +2026,6 @@ to speed up repeated connections from the same clients. :meth:`SSLSocket.verify_client_post_handshake` is called and some I/O is performed. - .. note:: - Only available with OpenSSL 1.1.1 and TLS 1.3 enabled. Without TLS 1.3 - support, the property value is None and can't be modified - .. versionadded:: 3.8 .. attribute:: SSLContext.protocol @@ -2027,19 +2039,19 @@ to speed up repeated connections from the same clients. subject common name in the absence of a subject alternative name extension (default: true). - .. note:: - Only writeable with OpenSSL 1.1.0 or higher. - .. versionadded:: 3.7 + .. versionchanged:: 3.10 + + The flag had no effect with OpenSSL before version 1.1.1k. Python 3.8.9, + 3.9.3, and 3.10 include workarounds for previous versions. + .. attribute:: SSLContext.security_level An integer representing the `security level `_ for the context. This attribute is read-only. - .. availability:: OpenSSL 1.1.0 or newer - .. versionadded:: 3.10 .. attribute:: SSLContext.verify_flags @@ -2047,7 +2059,6 @@ to speed up repeated connections from the same clients. The flags for certificate verification operations. You can set flags like :data:`VERIFY_CRL_CHECK_LEAF` by ORing them together. By default OpenSSL does neither require nor verify certificate revocation lists (CRLs). - Available only with openssl version 0.9.8+. .. versionadded:: 3.4 @@ -2055,7 +2066,7 @@ to speed up repeated connections from the same clients. :attr:`SSLContext.verify_flags` returns :class:`VerifyFlags` flags: >>> ssl.create_default_context().verify_flags # doctest: +SKIP - + ssl.VERIFY_X509_TRUSTED_FIRST .. attribute:: SSLContext.verify_mode @@ -2067,7 +2078,7 @@ to speed up repeated connections from the same clients. :attr:`SSLContext.verify_mode` returns :class:`VerifyMode` enum: >>> ssl.create_default_context().verify_mode - + ssl.CERT_REQUIRED .. index:: single: certificates @@ -2671,8 +2682,8 @@ disabled by default. :: >>> client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - >>> client_context.options |= ssl.OP_NO_TLSv1 - >>> client_context.options |= ssl.OP_NO_TLSv1_1 + >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + >>> client_context.maximum_version = ssl.TLSVersion.TLSv1_3 The SSL context created above will only allow TLSv1.2 and later (if @@ -2713,9 +2724,8 @@ TLS 1.3 .. versionadded:: 3.7 -Python has provisional and experimental support for TLS 1.3 with OpenSSL -1.1.1. The new protocol behaves slightly differently than previous version -of TLS/SSL. Some new TLS 1.3 features are not yet available. +The TLS 1.3 protocol behaves slightly differently than previous version +of TLS/SSL. Some new TLS 1.3 features are not yet available. - TLS 1.3 uses a disjunct set of cipher suites. All AES-GCM and ChaCha20 cipher suites are enabled by default. The method @@ -2732,23 +2742,6 @@ of TLS/SSL. Some new TLS 1.3 features are not yet available. signature algorithm configuration, and rekeying are not supported yet. -.. _ssl-libressl: - -LibreSSL support ----------------- - -LibreSSL is a fork of OpenSSL 1.0.1. The ssl module has limited support for -LibreSSL. Some features are not available when the ssl module is compiled -with LibreSSL. - -* LibreSSL >= 2.6.1 no longer supports NPN. The methods - :meth:`SSLContext.set_npn_protocols` and - :meth:`SSLSocket.selected_npn_protocol` are not available. -* :meth:`SSLContext.set_default_verify_paths` ignores the env vars - :envvar:`SSL_CERT_FILE` and :envvar:`SSL_CERT_PATH` although - :func:`get_default_verify_paths` still reports them. - - .. seealso:: Class :class:`socket.socket` diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 38a499ab37e890..117d2b63cbea15 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -51,7 +51,7 @@ or sample. :func:`median_high` High median of data. :func:`median_grouped` Median, or 50th percentile, of grouped data. :func:`mode` Single mode (most common value) of discrete or nominal data. -:func:`multimode` List of modes (most common values) of discrete or nomimal data. +:func:`multimode` List of modes (most common values) of discrete or nominal data. :func:`quantiles` Divide data into intervals with equal probability. ======================= =============================================================== @@ -68,6 +68,17 @@ tends to deviate from the typical or average values. :func:`variance` Sample variance of data. ======================= ============================================= +Statistics for relations between two inputs +------------------------------------------- + +These functions calculate statistics regarding relations between two inputs. + +========================= ===================================================== +:func:`covariance` Sample covariance for two variables. +:func:`correlation` Pearson's correlation coefficient for two variables. +:func:`linear_regression` Intercept and slope for simple linear regression. +========================= ===================================================== + Function details ---------------- @@ -156,20 +167,20 @@ However, for reading convenience, most of the examples show sorted sequences. .. versionadded:: 3.8 -.. function:: harmonic_mean(data) +.. function:: harmonic_mean(data, weights=None) Return the harmonic mean of *data*, a sequence or iterable of - real-valued numbers. + real-valued numbers. If *weights* is omitted or *None*, then + equal weighting is assumed. - The harmonic mean, sometimes called the subcontrary mean, is the - reciprocal of the arithmetic :func:`mean` of the reciprocals of the - data. For example, the harmonic mean of three values *a*, *b* and *c* - will be equivalent to ``3/(1/a + 1/b + 1/c)``. If one of the values - is zero, the result will be zero. + The harmonic mean is the reciprocal of the arithmetic :func:`mean` of the + reciprocals of the data. For example, the harmonic mean of three values *a*, + *b* and *c* will be equivalent to ``3/(1/a + 1/b + 1/c)``. If one of the + values is zero, the result will be zero. The harmonic mean is a type of average, a measure of the central location of the data. It is often appropriate when averaging - rates or ratios, for example speeds. + ratios or rates, for example speeds. Suppose a car travels 10 km at 40 km/hr, then another 10 km at 60 km/hr. What is the average speed? @@ -179,17 +190,17 @@ However, for reading convenience, most of the examples show sorted sequences. >>> harmonic_mean([40, 60]) 48.0 - Suppose an investor purchases an equal value of shares in each of - three companies, with P/E (price/earning) ratios of 2.5, 3 and 10. - What is the average P/E ratio for the investor's portfolio? + Suppose a car travels 40 km/hr for 5 km, and when traffic clears, + speeds-up to 60 km/hr for the remaining 30 km of the journey. What + is the average speed? .. doctest:: - >>> harmonic_mean([2.5, 3, 10]) # For an equal investment portfolio. - 3.6 + >>> harmonic_mean([40, 60], weights=[5, 30]) + 56.0 - :exc:`StatisticsError` is raised if *data* is empty, or any element - is less than zero. + :exc:`StatisticsError` is raised if *data* is empty, any element + is less than zero, or if the weighted sum isn't positive. The current algorithm has an early-out when it encounters a zero in the input. This means that the subsequent inputs are not tested @@ -197,6 +208,8 @@ However, for reading convenience, most of the examples show sorted sequences. .. versionadded:: 3.6 + .. versionchanged:: 3.10 + Added support for *weights*. .. function:: median(data) @@ -564,6 +577,98 @@ However, for reading convenience, most of the examples show sorted sequences. .. versionadded:: 3.8 +.. function:: covariance(x, y, /) + + Return the sample covariance of two inputs *x* and *y*. Covariance + is a measure of the joint variability of two inputs. + + Both inputs must be of the same length (no less than two), otherwise + :exc:`StatisticsError` is raised. + + Examples: + + .. doctest:: + + >>> x = [1, 2, 3, 4, 5, 6, 7, 8, 9] + >>> y = [1, 2, 3, 1, 2, 3, 1, 2, 3] + >>> covariance(x, y) + 0.75 + >>> z = [9, 8, 7, 6, 5, 4, 3, 2, 1] + >>> covariance(x, z) + -7.5 + >>> covariance(z, x) + -7.5 + + .. versionadded:: 3.10 + +.. function:: correlation(x, y, /) + + Return the `Pearson's correlation coefficient + `_ + for two inputs. Pearson's correlation coefficient *r* takes values + between -1 and +1. It measures the strength and direction of the linear + relationship, where +1 means very strong, positive linear relationship, + -1 very strong, negative linear relationship, and 0 no linear relationship. + + Both inputs must be of the same length (no less than two), and need + not to be constant, otherwise :exc:`StatisticsError` is raised. + + Examples: + + .. doctest:: + + >>> x = [1, 2, 3, 4, 5, 6, 7, 8, 9] + >>> y = [9, 8, 7, 6, 5, 4, 3, 2, 1] + >>> correlation(x, x) + 1.0 + >>> correlation(x, y) + -1.0 + + .. versionadded:: 3.10 + +.. function:: linear_regression(regressor, dependent_variable) + + Return the intercept and slope of `simple linear regression + `_ + parameters estimated using ordinary least squares. Simple linear + regression describes relationship between *regressor* and + *dependent variable* in terms of linear function: + + *dependent_variable = intercept + slope \* regressor + noise* + + where ``intercept`` and ``slope`` are the regression parameters that are + estimated, and noise term is an unobserved random variable, for the + variability of the data that was not explained by the linear regression + (it is equal to the difference between prediction and the actual values + of dependent variable). + + Both inputs must be of the same length (no less than two), and regressor + needs not to be constant, otherwise :exc:`StatisticsError` is raised. + + For example, if we took the data on the data on `release dates of the Monty + Python films `_, and used + it to predict the cumulative number of Monty Python films produced, we could + predict what would be the number of films they could have made till year + 2019, assuming that they kept the pace. + + .. doctest:: + + >>> year = [1971, 1975, 1979, 1982, 1983] + >>> films_total = [1, 2, 3, 4, 5] + >>> intercept, slope = linear_regression(year, films_total) + >>> round(intercept + slope * 2019) + 16 + + We could also use it to "predict" how many Monty Python films existed when + Brian Cohen was born. + + .. doctest:: + + >>> round(intercept + slope * 1) + -610 + + .. versionadded:: 3.10 + Exceptions ---------- diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index c74d1644636013..c4e6b4d1186d04 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -499,7 +499,7 @@ class`. In addition, it provides a few more methods: .. versionadded:: 3.10 -.. method:: int.to_bytes(length, byteorder, \*, signed=False) +.. method:: int.to_bytes(length, byteorder, *, signed=False) Return an array of bytes representing an integer. @@ -531,7 +531,7 @@ class`. In addition, it provides a few more methods: .. versionadded:: 3.2 -.. classmethod:: int.from_bytes(bytes, byteorder, \*, signed=False) +.. classmethod:: int.from_bytes(bytes, byteorder, *, signed=False) Return the integer represented by the given array of bytes. @@ -692,10 +692,9 @@ Here are the rules in detail: as ``-hash(-x)``. If the resulting hash is ``-1``, replace it with ``-2``. -- The particular values ``sys.hash_info.inf``, ``-sys.hash_info.inf`` - and ``sys.hash_info.nan`` are used as hash values for positive - infinity, negative infinity, or nans (respectively). (All hashable - nans have the same hash value.) +- The particular values ``sys.hash_info.inf`` and ``-sys.hash_info.inf`` + are used as hash values for positive + infinity or negative infinity (respectively). - For a :class:`complex` number ``z``, the hash values of the real and imaginary parts are combined by computing ``hash(z.real) + @@ -740,7 +739,7 @@ number, :class:`float`, or :class:`complex`:: """Compute the hash of a float x.""" if math.isnan(x): - return sys.hash_info.nan + return super().__hash__() elif math.isinf(x): return sys.hash_info.inf if x > 0 else -sys.hash_info.inf else: @@ -1581,13 +1580,15 @@ expression support in the :mod:`re` module). By default, the *errors* argument is not checked for best performances, but only used at the first encoding error. Enable the :ref:`Python Development - Mode `, or use a debug build to check *errors*. + Mode `, or use a :ref:`debug build ` to check + *errors*. .. versionchanged:: 3.1 Support for keyword arguments added. .. versionchanged:: 3.9 - The *errors* is now checked in development mode and in debug mode. + The *errors* is now checked in development mode and + in :ref:`debug mode `. .. method:: str.endswith(suffix[, start[, end]]) @@ -2710,7 +2711,7 @@ arbitrary binary data. By default, the *errors* argument is not checked for best performances, but only used at the first decoding error. Enable the :ref:`Python Development - Mode `, or use a debug build to check *errors*. + Mode `, or use a :ref:`debug build ` to check *errors*. .. note:: @@ -2722,7 +2723,8 @@ arbitrary binary data. Added support for keyword arguments. .. versionchanged:: 3.9 - The *errors* is now checked in development mode and in debug mode. + The *errors* is now checked in development mode and + in :ref:`debug mode `. .. method:: bytes.endswith(suffix[, start[, end]]) @@ -4763,25 +4765,236 @@ define these methods must provide them as a normal Python accessible method. Compared to the overhead of setting up the runtime context, the overhead of a single class dictionary lookup is negligible. + +Type Annotation Types --- :ref:`Generic Alias `, :ref:`Union ` +=============================================================================================== + +.. index:: + single: annotation; type annotation; type hint + +The core built-in types for :term:`type annotations ` are +:ref:`Generic Alias ` and :ref:`Union `. + + +.. _types-genericalias: + +Generic Alias Type +------------------ + +.. index:: + object: GenericAlias + pair: Generic; Alias + +``GenericAlias`` objects are created by subscripting a class (usually a +container), such as ``list[int]``. They are intended primarily for +:term:`type annotations `. + +Usually, the :ref:`subscription ` of container objects calls the +method :meth:`__getitem__` of the object. However, the subscription of some +containers' classes may call the classmethod :meth:`__class_getitem__` of the +class instead. The classmethod :meth:`__class_getitem__` should return a +``GenericAlias`` object. + +.. note:: + If the :meth:`__getitem__` of the class' metaclass is present, it will take + precedence over the :meth:`__class_getitem__` defined in the class (see + :pep:`560` for more details). + +The ``GenericAlias`` object acts as a proxy for :term:`generic types +`, implementing *parameterized generics* - a specific instance +of a generic which provides the types for container elements. + +The user-exposed type for the ``GenericAlias`` object can be accessed from +:class:`types.GenericAlias` and used for :func:`isinstance` checks. It can +also be used to create ``GenericAlias`` objects directly. + +.. describe:: T[X, Y, ...] + + Creates a ``GenericAlias`` representing a type ``T`` containing elements + of types *X*, *Y*, and more depending on the ``T`` used. + For example, a function expecting a :class:`list` containing + :class:`float` elements:: + + def average(values: list[float]) -> float: + return sum(values) / len(values) + + Another example for :term:`mapping` objects, using a :class:`dict`, which + is a generic type expecting two type parameters representing the key type + and the value type. In this example, the function expects a ``dict`` with + keys of type :class:`str` and values of type :class:`int`:: + + def send_post_request(url: str, body: dict[str, int]) -> None: + ... + +The builtin functions :func:`isinstance` and :func:`issubclass` do not accept +``GenericAlias`` types for their second argument:: + + >>> isinstance([1, 2], list[str]) + Traceback (most recent call last): + File "", line 1, in + TypeError: isinstance() argument 2 cannot be a parameterized generic + +The Python runtime does not enforce :term:`type annotations `. +This extends to generic types and their type parameters. When creating +an object from a ``GenericAlias``, container elements are not checked +against their type. For example, the following code is discouraged, but will +run without errors:: + + >>> t = list[str] + >>> t([1, 2, 3]) + [1, 2, 3] + +Furthermore, parameterized generics erase type parameters during object +creation:: + + >>> t = list[str] + >>> type(t) + + + >>> l = t() + >>> type(l) + + +Calling :func:`repr` or :func:`str` on a generic shows the parameterized type:: + + >>> repr(list[int]) + 'list[int]' + + >>> str(list[int]) + 'list[int]' + +The :meth:`__getitem__` method of generics will raise an exception to disallow +mistakes like ``dict[str][str]``:: + + >>> dict[str][str] + Traceback (most recent call last): + File "", line 1, in + TypeError: There are no type variables left in dict[str] + +However, such expressions are valid when :ref:`type variables ` are +used. The index must have as many elements as there are type variable items +in the ``GenericAlias`` object's :attr:`__args__ `. :: + + >>> from typing import TypeVar + >>> Y = TypeVar('Y') + >>> dict[str, Y][int] + dict[str, int] + + +Standard Generic Collections +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These standard library collections support parameterized generics. + +* :class:`tuple` +* :class:`list` +* :class:`dict` +* :class:`set` +* :class:`frozenset` +* :class:`type` +* :class:`collections.deque` +* :class:`collections.defaultdict` +* :class:`collections.OrderedDict` +* :class:`collections.Counter` +* :class:`collections.ChainMap` +* :class:`collections.abc.Awaitable` +* :class:`collections.abc.Coroutine` +* :class:`collections.abc.AsyncIterable` +* :class:`collections.abc.AsyncIterator` +* :class:`collections.abc.AsyncGenerator` +* :class:`collections.abc.Iterable` +* :class:`collections.abc.Iterator` +* :class:`collections.abc.Generator` +* :class:`collections.abc.Reversible` +* :class:`collections.abc.Container` +* :class:`collections.abc.Collection` +* :class:`collections.abc.Callable` +* :class:`collections.abc.Set` +* :class:`collections.abc.MutableSet` +* :class:`collections.abc.Mapping` +* :class:`collections.abc.MutableMapping` +* :class:`collections.abc.Sequence` +* :class:`collections.abc.MutableSequence` +* :class:`collections.abc.ByteString` +* :class:`collections.abc.MappingView` +* :class:`collections.abc.KeysView` +* :class:`collections.abc.ItemsView` +* :class:`collections.abc.ValuesView` +* :class:`contextlib.AbstractContextManager` +* :class:`contextlib.AbstractAsyncContextManager` +* :ref:`re.Pattern ` +* :ref:`re.Match ` + + +Special Attributes of Generic Alias +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +All parameterized generics implement special read-only attributes. + +.. attribute:: genericalias.__origin__ + + This attribute points at the non-parameterized generic class:: + + >>> list[int].__origin__ + + + +.. attribute:: genericalias.__args__ + + This attribute is a :class:`tuple` (possibly of length 1) of generic + types passed to the original :meth:`__class_getitem__` + of the generic container:: + + >>> dict[str, list[int]].__args__ + (, list[int]) + + +.. attribute:: genericalias.__parameters__ + + This attribute is a lazily computed tuple (possibly empty) of unique type + variables found in ``__args__``:: + + >>> from typing import TypeVar + + >>> T = TypeVar('T') + >>> list[T].__parameters__ + (~T,) + + + .. note:: + A ``GenericAlias`` object with :class:`typing.ParamSpec` parameters may not + have correct ``__parameters__`` after substitution because + :class:`typing.ParamSpec` is intended primarily for static type checking. + +.. seealso:: + + * :pep:`585` -- "Type Hinting Generics In Standard Collections" + * :meth:`__class_getitem__` -- Used to implement parameterized generics. + * :ref:`generics` -- Generics in the :mod:`typing` module. + +.. versionadded:: 3.9 + + .. _types-union: Union Type -========== +---------- .. index:: object: Union pair: union; type A union object holds the value of the ``|`` (bitwise or) operation on -multiple :ref:`type objects`. These types are intended -primarily for type annotations. The union type expression enables cleaner -type hinting syntax compared to :data:`typing.Union`. +multiple :ref:`type objects `. These types are intended +primarily for :term:`type annotations `. The union type expression +enables cleaner type hinting syntax compared to :data:`typing.Union`. .. describe:: X | Y | ... Defines a union object which holds types *X*, *Y*, and so forth. ``X | Y`` means either X or Y. It is equivalent to ``typing.Union[X, Y]``. - Example:: + For example, the following function expects an argument of type + :class:`int` or :class:`float`:: def square(number: int | float) -> int | float: return number ** 2 @@ -4790,15 +5003,15 @@ type hinting syntax compared to :data:`typing.Union`. Union objects can be tested for equality with other union objects. Details: - * Unions of unions are flattened, e.g.:: + * Unions of unions are flattened:: (int | str) | float == int | str | float - * Redundant types are removed, e.g.:: + * Redundant types are removed:: int | str | int == int | str - * When comparing unions, the order is ignored, e.g.:: + * When comparing unions, the order is ignored:: int | str == str | int @@ -4811,47 +5024,24 @@ type hinting syntax compared to :data:`typing.Union`. str | None == typing.Optional[str] .. describe:: isinstance(obj, union_object) +.. describe:: issubclass(obj, union_object) - Calls to :func:`isinstance` are also supported with a union object:: + Calls to :func:`isinstance` and :func:`issubclass` are also supported with a + union object:: >>> isinstance("", int | str) True - .. - At the time of writing this, there is no documentation for parameterized - generics or PEP 585. Thus the link currently points to PEP 585 itself. - Please change the link for parameterized generics to reference the correct - documentation once documentation for PEP 585 becomes available. - - However, union objects containing `parameterized generics - `_ cannot be used:: + However, union objects containing :ref:`parameterized generics + ` cannot be used:: >>> isinstance(1, int | list[int]) Traceback (most recent call last): File "", line 1, in TypeError: isinstance() argument 2 cannot contain a parameterized generic -.. describe:: issubclass(obj, union_object) - - Calls to :func:`issubclass` are also supported with a union object:: - - >>> issubclass(bool, int | str) - True - - .. - Once again, please change the link below for parameterized generics to - reference the correct documentation once documentation for PEP 585 - becomes available. - - However, union objects containing `parameterized generics - `_ cannot be used:: - - >>> issubclass(bool, bool | list[str]) - Traceback (most recent call last): - File "", line 1, in - TypeError: issubclass() argument 2 cannot contain a parameterized generic - -The type of a union object is :data:`types.Union`. An object cannot be +The user-exposed type for the union object can be accessed from +:data:`types.Union` and used for :func:`isinstance` checks. An object cannot be instantiated from the type:: >>> import types @@ -4869,7 +5059,7 @@ instantiated from the type:: >>> class M(type): ... def __or__(self, other): - ... return "Hello" + ... return "Hello" ... >>> class C(metaclass=M): ... pass @@ -5005,6 +5195,9 @@ environment. Code objects are returned by the built-in :func:`compile` function and can be extracted from function objects through their :attr:`__code__` attribute. See also the :mod:`code` module. +Accessing ``__code__`` raises an :ref:`auditing event ` +``object.__getattr__`` with arguments ``obj`` and ``"__code__"``. + .. index:: builtin: exec builtin: eval @@ -5157,8 +5350,8 @@ types, where they are relevant. Some of these are not reported by the .. method:: class.__subclasses__ Each class keeps a list of weak references to its immediate subclasses. This - method returns a list of all those references still alive. - Example:: + method returns a list of all those references still alive. The list is in + definition order. Example:: >>> int.__subclasses__() [] diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 91f43e9353d915..d935419f7b75e7 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -188,8 +188,8 @@ Format String Syntax The :meth:`str.format` method and the :class:`Formatter` class share the same syntax for format strings (although in the case of :class:`Formatter`, subclasses can define their own format string syntax). The syntax is -related to that of :ref:`formatted string literals `, but -there are differences. +related to that of :ref:`formatted string literals `, but it is +less sophisticated and, in particular, does not support arbitrary expressions. .. index:: single: {} (curly brackets); in string formatting @@ -347,8 +347,8 @@ The meaning of the various alignment options is as follows: | ``'='`` | Forces the padding to be placed after the sign (if any) | | | but before the digits. This is used for printing fields | | | in the form '+000000120'. This alignment option is only | - | | valid for numeric types. It becomes the default when '0'| - | | immediately precedes the field width. | + | | valid for numeric types. It becomes the default for | + | | numbers when '0' immediately precedes the field width. | +---------+----------------------------------------------------------+ | ``'^'`` | Forces the field to be centered within the available | | | space. | @@ -384,10 +384,10 @@ following: The ``'#'`` option causes the "alternate form" to be used for the conversion. The alternate form is defined differently for different -types. This option is only valid for integer, float, complex and -Decimal types. For integers, when binary, octal, or hexadecimal output +types. This option is only valid for integer, float and complex +types. For integers, when binary, octal, or hexadecimal output is used, this option adds the prefix respective ``'0b'``, ``'0o'``, or -``'0x'`` to the output value. For floats, complex and Decimal the +``'0x'`` to the output value. For float and complex the alternate form causes the result of the conversion to always contain a decimal-point character, even if no digits follow it. Normally, a decimal-point character appears in the result of these conversions @@ -424,6 +424,10 @@ When no explicit alignment is given, preceding the *width* field by a zero sign-aware zero-padding for numeric types. This is equivalent to a *fill* character of ``'0'`` with an *alignment* type of ``'='``. +.. versionchanged:: 3.10 + Preceding the *width* field by ``'0'`` no longer affects the default + alignment for strings. + The *precision* is a decimal number indicating how many digits should be displayed after the decimal point for a floating point value formatted with ``'f'`` and ``'F'``, or before and after the decimal point for a floating point @@ -476,20 +480,36 @@ with the floating point presentation types listed below (except ``'n'`` and ``None``). When doing so, :func:`float` is used to convert the integer to a floating point number before formatting. -The available presentation types for floating point and decimal values are: +The available presentation types for :class:`float` and +:class:`~decimal.Decimal` values are: +---------+----------------------------------------------------------+ | Type | Meaning | +=========+==========================================================+ - | ``'e'`` | Exponent notation. Prints the number in scientific | - | | notation using the letter 'e' to indicate the exponent. | - | | The default precision is ``6``. | + | ``'e'`` | Scientific notation. For a given precision ``p``, | + | | formats the number in scientific notation with the | + | | letter 'e' separating the coefficient from the exponent. | + | | The coefficient has one digit before and ``p`` digits | + | | after the decimal point, for a total of ``p + 1`` | + | | significant digits. With no precision given, uses a | + | | precision of ``6`` digits after the decimal point for | + | | :class:`float`, and shows all coefficient digits | + | | for :class:`~decimal.Decimal`. If no digits follow the | + | | decimal point, the decimal point is also removed unless | + | | the ``#`` option is used. | +---------+----------------------------------------------------------+ - | ``'E'`` | Exponent notation. Same as ``'e'`` except it uses an | - | | upper case 'E' as the separator character. | + | ``'E'`` | Scientific notation. Same as ``'e'`` except it uses | + | | an upper case 'E' as the separator character. | +---------+----------------------------------------------------------+ - | ``'f'`` | Fixed-point notation. Displays the number as a | - | | fixed-point number. The default precision is ``6``. | + | ``'f'`` | Fixed-point notation. For a given precision ``p``, | + | | formats the number as a decimal number with exactly | + | | ``p`` digits following the decimal point. With no | + | | precision given, uses a precision of ``6`` digits after | + | | the decimal point for :class:`float`, and uses a | + | | precision large enough to show all coefficient digits | + | | for :class:`~decimal.Decimal`. If no digits follow the | + | | decimal point, the decimal point is also removed unless | + | | the ``#`` option is used. | +---------+----------------------------------------------------------+ | ``'F'`` | Fixed-point notation. Same as ``'f'``, but converts | | | ``nan`` to ``NAN`` and ``inf`` to ``INF``. | @@ -498,6 +518,8 @@ The available presentation types for floating point and decimal values are: | | this rounds the number to ``p`` significant digits and | | | then formats the result in either fixed-point format | | | or in scientific notation, depending on its magnitude. | + | | A precision of ``0`` is treated as equivalent to a | + | | precision of ``1``. | | | | | | The precise rules are as follows: suppose that the | | | result formatted with presentation type ``'e'`` and | @@ -512,13 +534,19 @@ The available presentation types for floating point and decimal values are: | | removed if there are no remaining digits following it, | | | unless the ``'#'`` option is used. | | | | + | | With no precision given, uses a precision of ``6`` | + | | significant digits for :class:`float`. For | + | | :class:`~decimal.Decimal`, the coefficient of the result | + | | is formed from the coefficient digits of the value; | + | | scientific notation is used for values smaller than | + | | ``1e-6`` in absolute value and values where the place | + | | value of the least significant digit is larger than 1, | + | | and fixed-point notation is used otherwise. | + | | | | | Positive and negative infinity, positive and negative | | | zero, and nans, are formatted as ``inf``, ``-inf``, | | | ``0``, ``-0`` and ``nan`` respectively, regardless of | | | the precision. | - | | | - | | A precision of ``0`` is treated as equivalent to a | - | | precision of ``1``. The default precision is ``6``. | +---------+----------------------------------------------------------+ | ``'G'`` | General format. Same as ``'g'`` except switches to | | | ``'E'`` if the number gets too large. The | @@ -531,12 +559,18 @@ The available presentation types for floating point and decimal values are: | ``'%'`` | Percentage. Multiplies the number by 100 and displays | | | in fixed (``'f'``) format, followed by a percent sign. | +---------+----------------------------------------------------------+ - | None | Similar to ``'g'``, except that fixed-point notation, | - | | when used, has at least one digit past the decimal point.| - | | The default precision is as high as needed to represent | - | | the particular value. The overall effect is to match the | - | | output of :func:`str` as altered by the other format | - | | modifiers. | + | None | For :class:`float` this is the same as ``'g'``, except | + | | that when fixed-point notation is used to format the | + | | result, it always includes at least one digit past the | + | | decimal point. The precision used is as large as needed | + | | to represent the given value faithfully. | + | | | + | | For :class:`~decimal.Decimal`, this is the same as | + | | either ``'g'`` or ``'G'`` depending on the value of | + | | ``context.capitals`` for the current decimal context. | + | | | + | | The overall effect is to match the output of :func:`str` | + | | as altered by the other format modifiers. | +---------+----------------------------------------------------------+ diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 85d0f46624cead..b60db58d182e6c 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -339,7 +339,7 @@ functions. stderr=None, preexec_fn=None, close_fds=True, shell=False, \ cwd=None, env=None, universal_newlines=None, \ startupinfo=None, creationflags=0, restore_signals=True, \ - start_new_session=False, pass_fds=(), \*, group=None, \ + start_new_session=False, pass_fds=(), *, group=None, \ extra_groups=None, user=None, umask=-1, \ encoding=None, errors=None, text=None, pipesize=-1) @@ -543,7 +543,7 @@ functions. If *cwd* is not ``None``, the function changes the working directory to *cwd* before executing the child. *cwd* can be a string, bytes or - :term:`path-like ` object. In POSIX, the function + :term:`path-like ` object. On POSIX, the function looks for *executable* (or for the first item in *args*) relative to *cwd* if the executable path is a relative path. @@ -718,11 +718,8 @@ If the shell is invoked explicitly, via ``shell=True``, it is the application's responsibility to ensure that all whitespace and metacharacters are quoted appropriately to avoid `shell injection `_ -vulnerabilities. - -When using ``shell=True``, the :func:`shlex.quote` function can be -used to properly escape whitespace and shell metacharacters in strings -that are going to be used to construct shell commands. +vulnerabilities. On :ref:`some platforms `, it is possible +to use :func:`shlex.quote` for this escaping. Popen Objects @@ -1190,8 +1187,9 @@ calls these functions. The arguments shown above are merely some common ones. The full function signature is largely the same as that of :func:`run` - most arguments are passed directly through to that interface. - However, explicitly passing ``input=None`` to inherit the parent's - standard input file handle is not supported. + One API deviation from :func:`run` behavior exists: passing ``input=None`` + will behave the same as ``input=b''`` (or ``input=''``, depending on other + arguments) rather than using the parent's standard input file handle. By default, this function will return the data as encoded bytes. The actual encoding of the output data may depend on the command being invoked, so the @@ -1284,7 +1282,7 @@ be used directly: becomes:: - output=check_output("dmesg | grep hda", shell=True) + output = check_output("dmesg | grep hda", shell=True) Replacing :func:`os.system` diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index c9521d649b893e..e364232247c200 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -156,10 +156,12 @@ Examining Symbol Tables Return ``True`` if the symbol is local to its block. - .. method:: is_annotated() + .. method:: is_annotated() Return ``True`` if the symbol is annotated. + .. versionadded:: 3.6 + .. method:: is_free() Return ``True`` if the symbol is referenced in its block, but not assigned diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 468a30d326891b..ec12e02fb37d45 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -26,12 +26,12 @@ always available. .. function:: addaudithook(hook) Append the callable *hook* to the list of active auditing hooks for the - current interpreter. + current (sub)interpreter. When an auditing event is raised through the :func:`sys.audit` function, each hook will be called in the order it was added with the event name and the tuple of arguments. Native hooks added by :c:func:`PySys_AddAuditHook` are - called first, followed by hooks added in the current interpreter. Hooks + called first, followed by hooks added in the current (sub)interpreter. Hooks can then log the event, raise an exception to abort the operation, or terminate the process entirely. @@ -153,10 +153,12 @@ always available. .. data:: builtin_module_names - A tuple of strings giving the names of all modules that are compiled into this + A tuple of strings containing the names of all modules that are compiled into this Python interpreter. (This information is not available in any other way --- ``modules.keys()`` only lists the imported modules.) + See also the :attr:`sys.stdlib_module_names` list. + .. function:: call_tracing(func, args) @@ -196,6 +198,18 @@ always available. .. audit-event:: sys._current_frames "" sys._current_frames +.. function:: _current_exceptions() + + Return a dictionary mapping each thread's identifier to the topmost exception + currently active in that thread at the time the function is called. + If a thread is not currently handling an exception, it is not included in + the result dictionary. + + This is most useful for statistical profiling. + + This function should be used for internal and specialized purposes only. + + .. audit-event:: sys._current_exceptions "" sys._current_exceptions .. function:: breakpointhook() @@ -236,7 +250,8 @@ always available. Print low-level information to stderr about the state of CPython's memory allocator. - If Python is configured --with-pydebug, it also performs some expensive + If Python is `built in debug mode ` (:option:`configure + --with-pydebug option <--with-pydebug>`), it also performs some expensive internal consistency checks. .. versionadded:: 3.3 @@ -615,30 +630,24 @@ always available. .. function:: getfilesystemencoding() - Return the name of the encoding used to convert between Unicode - filenames and bytes filenames. For best compatibility, str should be - used for filenames in all cases, although representing filenames as bytes - is also supported. Functions accepting or returning filenames should support - either str or bytes and internally convert to the system's preferred - representation. + Get the :term:`filesystem encoding `: + the encoding used with the :term:`filesystem error handler ` to convert between Unicode filenames and bytes + filenames. The filesystem error handler is returned from + :func:`getfilesystemencoding`. - This encoding is always ASCII-compatible. + For best compatibility, str should be used for filenames in all cases, + although representing filenames as bytes is also supported. Functions + accepting or returning filenames should support either str or bytes and + internally convert to the system's preferred representation. :func:`os.fsencode` and :func:`os.fsdecode` should be used to ensure that the correct encoding and errors mode are used. - * In the UTF-8 mode, the encoding is ``utf-8`` on any platform. - - * On macOS, the encoding is ``'utf-8'``. - - * On Unix, the encoding is the locale encoding. - - * On Windows, the encoding may be ``'utf-8'`` or ``'mbcs'``, depending - on user configuration. - - * On Android, the encoding is ``'utf-8'``. - - * On VxWorks, the encoding is ``'utf-8'``. + The :term:`filesystem encoding and error handler` are configured at Python + startup by the :c:func:`PyConfig_Read` function: see + :c:member:`~PyConfig.filesystem_encoding` and + :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`. .. versionchanged:: 3.2 :func:`getfilesystemencoding` result cannot be ``None`` anymore. @@ -648,18 +657,26 @@ always available. and :func:`_enablelegacywindowsfsencoding` for more information. .. versionchanged:: 3.7 - Return 'utf-8' in the UTF-8 mode. + Return ``'utf-8'`` if the :ref:`Python UTF-8 Mode ` is + enabled. .. function:: getfilesystemencodeerrors() - Return the name of the error mode used to convert between Unicode filenames - and bytes filenames. The encoding name is returned from + Get the :term:`filesystem error handler `: the error handler used with the :term:`filesystem encoding + ` to convert between Unicode + filenames and bytes filenames. The filesystem encoding is returned from :func:`getfilesystemencoding`. :func:`os.fsencode` and :func:`os.fsdecode` should be used to ensure that the correct encoding and errors mode are used. + The :term:`filesystem encoding and error handler` are configured at Python + startup by the :c:func:`PyConfig_Read` function: see + :c:member:`~PyConfig.filesystem_encoding` and + :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`. + .. versionadded:: 3.6 .. function:: getrefcount(object) @@ -779,11 +796,16 @@ always available. Microsoft documentation on :c:func:`OSVERSIONINFOEX` for more information about these fields. - *platform_version* returns the accurate major version, minor version and + *platform_version* returns the major version, minor version and build number of the current operating system, rather than the version that is being emulated for the process. It is intended for use in logging rather than for feature detection. + .. note:: + *platform_version* derives the version from kernel32.dll which can be of a different + version than the OS version. Please use :mod:`platform` module for achieving accurate + OS version. + .. availability:: Windows. .. versionchanged:: 3.2 @@ -838,7 +860,7 @@ always available. +---------------------+--------------------------------------------------+ | :const:`inf` | hash value returned for a positive infinity | +---------------------+--------------------------------------------------+ - | :const:`nan` | hash value returned for a nan | + | :const:`nan` | (this attribute is no longer used) | +---------------------+--------------------------------------------------+ | :const:`imag` | multiplier used for the imaginary part of a | | | complex number | @@ -1451,12 +1473,16 @@ always available. .. function:: _enablelegacywindowsfsencoding() - Changes the default filesystem encoding and errors mode to 'mbcs' and - 'replace' respectively, for consistency with versions of Python prior to 3.6. + Changes the :term:`filesystem encoding and error handler` to 'mbcs' and + 'replace' respectively, for consistency with versions of Python prior to + 3.6. This is equivalent to defining the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable before launching Python. + See also :func:`sys.getfilesystemencoding` and + :func:`sys.getfilesystemencodeerrors`. + .. availability:: Windows. .. versionadded:: 3.6 @@ -1479,9 +1505,8 @@ always available. returned by the :func:`open` function. Their parameters are chosen as follows: - * The character encoding is platform-dependent. Non-Windows - platforms use the locale encoding (see - :meth:`locale.getpreferredencoding()`). + * The encoding and error handling are is initialized from + :c:member:`PyConfig.stdio_encoding` and :c:member:`PyConfig.stdio_errors`. On Windows, UTF-8 is used for the console device. Non-character devices such as disk files and pipes use the system locale @@ -1489,7 +1514,7 @@ always available. devices such as NUL (i.e. where ``isatty()`` returns ``True``) use the value of the console input and output codepages at startup, respectively for stdin and stdout/stderr. This defaults to the - system locale encoding if the process is not initially attached + system :term:`locale encoding` if the process is not initially attached to a console. The special behaviour of the console can be overridden @@ -1547,6 +1572,25 @@ always available. to a console and Python apps started with :program:`pythonw`. +.. data:: stdlib_module_names + + A frozenset of strings containing the names of standard library modules. + + It is the same on all platforms. Modules which are not available on + some platforms and modules disabled at Python build are also listed. + All module kinds are listed: pure Python, built-in, frozen and extension + modules. Test modules are excluded. + + For packages, only the main package is listed: sub-packages and sub-modules + are not listed. For example, the ``email`` package is listed, but the + ``email.mime`` sub-package and the ``email.message`` sub-module are not + listed. + + See also the :attr:`sys.builtin_module_names` list. + + .. versionadded:: 3.10 + + .. data:: thread_info A :term:`named tuple` holding information about the thread diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 78a1dfce9ae05c..bb8e2d88c0fc95 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -32,7 +32,7 @@ can be accessed using :func:`get_config_vars` or :func:`get_config_var`. Notice that on Windows, it's a much smaller set. -.. function:: get_config_vars(\*args) +.. function:: get_config_vars(*args) With no arguments, return a dictionary of all configuration variables relevant for the current platform. @@ -107,6 +107,43 @@ identifier. Python currently uses eight paths: :mod:`sysconfig`. +.. function:: get_default_scheme() + + Return the default scheme name for the current platform. + + .. versionchanged:: 3.10 + This function was previously named ``_get_default_scheme()`` and + considered an implementation detail. + + +.. function:: get_preferred_scheme(key) + + Return a preferred scheme name for an installation layout specified by *key*. + + *key* must be either ``"prefix"``, ``"home"``, or ``"user"``. + + The return value is a scheme name listed in :func:`get_scheme_names`. It + can be passed to :mod:`sysconfig` functions that take a *scheme* argument, + such as :func:`get_paths`. + + .. versionadded:: 3.10 + + +.. function:: _get_preferred_schemes() + + Return a dict containing preferred scheme names on the current platform. + Python implementers and redistributors may add their preferred schemes to + the ``_INSTALL_SCHEMES`` module-level global value, and modify this function + to return those scheme names, to e.g. provide different schemes for system + and language package managers to use, so packages installed by either do not + mix with those by the other. + + End users should not use this function, but :func:`get_default_scheme` and + :func:`get_preferred_scheme()` instead. + + .. versionadded:: 3.10 + + .. function:: get_path_names() Return a tuple containing all path names currently supported in @@ -224,6 +261,7 @@ Other functions Return the path of :file:`Makefile`. + Using :mod:`sysconfig` as a script ---------------------------------- diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 7a114fdf5d54b1..13088a10d77c57 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -37,7 +37,7 @@ Some facts and figures: Added support for :mod:`lzma` compression. -.. function:: open(name=None, mode='r', fileobj=None, bufsize=10240, \*\*kwargs) +.. function:: open(name=None, mode='r', fileobj=None, bufsize=10240, **kwargs) Return a :class:`TarFile` object for the pathname *name*. For detailed information on :class:`TarFile` objects and the keyword arguments that are diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index f9421da5fe7dfa..2970252036780d 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -118,12 +118,12 @@ The module defines the following user-callable items: Added *errors* parameter. -.. function:: TemporaryDirectory(suffix=None, prefix=None, dir=None) +.. function:: TemporaryDirectory(suffix=None, prefix=None, dir=None, ignore_cleanup_errors=False) This function securely creates a temporary directory using the same rules as :func:`mkdtemp`. The resulting object can be used as a context manager (see :ref:`tempfile-examples`). On completion of the context or destruction - of the temporary directory object the newly created temporary directory + of the temporary directory object, the newly created temporary directory and all its contents are removed from the filesystem. The directory name can be retrieved from the :attr:`name` attribute of the @@ -132,12 +132,21 @@ The module defines the following user-callable items: the :keyword:`with` statement, if there is one. The directory can be explicitly cleaned up by calling the - :func:`cleanup` method. + :func:`cleanup` method. If *ignore_cleanup_errors* is true, any unhandled + exceptions during explicit or implicit cleanup (such as a + :exc:`PermissionError` removing open files on Windows) will be ignored, + and the remaining removable items deleted on a "best-effort" basis. + Otherwise, errors will be raised in whatever context cleanup occurs + (the :func:`cleanup` call, exiting the context manager, when the object + is garbage-collected or during interpreter shutdown). .. audit-event:: tempfile.mkdtemp fullpath tempfile.TemporaryDirectory .. versionadded:: 3.2 + .. versionchanged:: 3.10 + Added *ignore_cleanup_errors* parameter. + .. function:: mkstemp(suffix=None, prefix=None, dir=None, text=False) @@ -248,6 +257,11 @@ The module defines the following user-callable items: The result of this search is cached, see the description of :data:`tempdir` below. + .. versionchanged:: 3.10 + + Always returns a str. Previously it would return any :data:`tempdir` + value regardless of type so long as it was not ``None``. + .. function:: gettempdirb() Same as :func:`gettempdir` but the return value is in bytes. @@ -269,18 +283,30 @@ The module uses a global variable to store the name of the directory used for temporary files returned by :func:`gettempdir`. It can be set directly to override the selection process, but this is discouraged. All functions in this module take a *dir* argument which can be used -to specify the directory and this is the recommended approach. +to specify the directory. This is the recommended approach that does +not surprise other unsuspecting code by changing global API behavior. .. data:: tempdir When set to a value other than ``None``, this variable defines the default value for the *dir* argument to the functions defined in this - module. + module, including its type, bytes or str. It cannot be a + :term:`path-like object`. If ``tempdir`` is ``None`` (the default) at any call to any of the above functions except :func:`gettempprefix` it is initialized following the algorithm described in :func:`gettempdir`. + .. note:: + + Beware that if you set ``tempdir`` to a bytes value, there is a + nasty side effect: The global default return type of + :func:`mkstemp` and :func:`mkdtemp` changes to bytes when no + explicit ``prefix``, ``suffix``, or ``dir`` arguments of type + str are supplied. Please do not write code expecting or + depending on this. This awkward behavior is maintained for + compatibility with the historical implementation. + .. _tempfile-examples: Examples diff --git a/Doc/library/test.rst b/Doc/library/test.rst index ce6b868458ea4f..e4f779bd83eb87 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -453,7 +453,7 @@ The :mod:`test.support` module defines the following functions: Define match test with regular expression *patterns*. -.. function:: run_unittest(\*classes) +.. function:: run_unittest(*classes) Execute :class:`unittest.TestCase` subclasses passed to the function. The function scans the classes for methods starting with the prefix ``test_`` @@ -1599,7 +1599,7 @@ The :mod:`test.support.warnings_helper` module provides support for warnings tes .. versionadded:: 3.8 -.. function:: check_warnings(\*filters, quiet=True) +.. function:: check_warnings(*filters, quiet=True) A convenience wrapper for :func:`warnings.catch_warnings()` that makes it easier to test that a warning was correctly raised. It is approximately diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 7eb12fe116bd2d..16f23c3a0c3548 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -16,9 +16,22 @@ level :mod:`_thread` module. See also the :mod:`queue` module. .. note:: - While they are not listed below, the ``camelCase`` names used for some - methods and functions in this module in the Python 2.x series are still - supported by this module. + In the Python 2.x series, this module contained ``camelCase`` names + for some methods and functions. These are deprecated as of Python 3.10, + but they are still supported for compatibility with Python 2.5 and lower. + + +.. impl-detail:: + + In CPython, due to the :term:`Global Interpreter Lock + `, only one thread + can execute Python code at once (even though certain performance-oriented + libraries might overcome this limitation). + If you want your application to make better use of the computational + resources of multi-core machines, you are advised to use + :mod:`multiprocessing` or :class:`concurrent.futures.ProcessPoolExecutor`. + However, threading is still an appropriate model if you want to run + multiple I/O-bound tasks simultaneously. This module defines the following functions: @@ -29,6 +42,8 @@ This module defines the following functions: Return the number of :class:`Thread` objects currently alive. The returned count is equal to the length of the list returned by :func:`.enumerate`. + The function ``activeCount`` is a deprecated alias for this function. + .. function:: current_thread() @@ -37,6 +52,8 @@ This module defines the following functions: :mod:`threading` module, a dummy thread object with limited functionality is returned. + The function ``currentThread`` is a deprecated alias for this function. + .. function:: excepthook(args, /) @@ -71,6 +88,13 @@ This module defines the following functions: .. versionadded:: 3.8 +.. data:: __excepthook__ + + Holds the original value of :func:`threading.excepthook`. It is saved so that the + original value can be restored in case they happen to get replaced with + broken or alternative objects. + + .. versionadded:: 3.10 .. function:: get_ident() @@ -121,6 +145,17 @@ This module defines the following functions: :meth:`~Thread.run` method is called. +.. function:: gettrace() + + .. index:: + single: trace function + single: debugger + + Get the trace function as set by :func:`settrace`. + + .. versionadded:: 3.10 + + .. function:: setprofile(func) .. index:: single: profile function @@ -130,6 +165,15 @@ This module defines the following functions: :meth:`~Thread.run` method is called. +.. function:: getprofile() + + .. index:: single: profile function + + Get the profiler function as set by :func:`setprofile`. + + .. versionadded:: 3.10 + + .. function:: stack_size([size]) Return the thread stack size used when creating new threads. The optional @@ -341,9 +385,11 @@ since it is impossible to detect the termination of alien threads. .. method:: getName() setName() - Old getter/setter API for :attr:`~Thread.name`; use it directly as a + Deprecated getter/setter API for :attr:`~Thread.name`; use it directly as a property instead. + .. deprecated:: 3.10 + .. attribute:: ident The 'thread identifier' of this thread or ``None`` if the thread has not @@ -393,21 +439,10 @@ since it is impossible to detect the termination of alien threads. .. method:: isDaemon() setDaemon() - Old getter/setter API for :attr:`~Thread.daemon`; use it directly as a + Deprecated getter/setter API for :attr:`~Thread.daemon`; use it directly as a property instead. - -.. impl-detail:: - - In CPython, due to the :term:`Global Interpreter Lock - `, only one thread - can execute Python code at once (even though certain performance-oriented - libraries might overcome this limitation). - If you want your application to make better use of the computational - resources of multi-core machines, you are advised to use - :mod:`multiprocessing` or :class:`concurrent.futures.ProcessPoolExecutor`. - However, threading is still an appropriate model if you want to run - multiple I/O-bound tasks simultaneously. + .. deprecated:: 3.10 .. _lock-objects: @@ -744,6 +779,8 @@ item to the buffer only needs to wake up one consumer thread. calling thread has not acquired the lock when this method is called, a :exc:`RuntimeError` is raised. + The method ``notifyAll`` is a deprecated alias for this method. + .. _semaphore-objects: @@ -881,6 +918,8 @@ method. The :meth:`~Event.wait` method blocks until the flag is true. Return ``True`` if and only if the internal flag is true. + The method ``isSet`` is a deprecated alias for this method. + .. method:: set() Set the internal flag to true. All threads waiting for it to become true diff --git a/Doc/library/time.rst b/Doc/library/time.rst index cff6320b526db5..cfd67e87501cd4 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -166,6 +166,9 @@ Functions Return the time of the specified clock *clk_id*. Refer to :ref:`time-clock-id-constants` for a list of accepted values for *clk_id*. + Use :func:`clock_gettime_ns` to avoid the precision loss caused by the + :class:`float` type. + .. availability:: Unix. .. versionadded:: 3.3 @@ -185,6 +188,9 @@ Functions Set the time of the specified clock *clk_id*. Currently, :data:`CLOCK_REALTIME` is the only accepted value for *clk_id*. + Use :func:`clock_settime_ns` to avoid the precision loss caused by the + :class:`float` type. + .. availability:: Unix. .. versionadded:: 3.3 @@ -271,12 +277,19 @@ Functions Return the value (in fractional seconds) of a monotonic clock, i.e. a clock that cannot go backwards. The clock is not affected by system clock updates. The reference point of the returned value is undefined, so that only the - difference between the results of consecutive calls is valid. + difference between the results of two calls is valid. + + Use :func:`monotonic_ns` to avoid the precision loss caused by the + :class:`float` type. .. versionadded:: 3.3 + .. versionchanged:: 3.5 The function is now always available and always system-wide. + .. versionchanged:: 3.10 + On macOS, the function is now system-wide. + .. function:: monotonic_ns() -> int @@ -293,10 +306,16 @@ Functions clock with the highest available resolution to measure a short duration. It does include time elapsed during sleep and is system-wide. The reference point of the returned value is undefined, so that only the difference between - the results of consecutive calls is valid. + the results of two calls is valid. + + Use :func:`perf_counter_ns` to avoid the precision loss caused by the + :class:`float` type. .. versionadded:: 3.3 + .. versionchanged:: 3.10 + On Windows, the function is now system-wide. + .. function:: perf_counter_ns() -> int Similar to :func:`perf_counter`, but return time as nanoseconds. @@ -315,7 +334,10 @@ Functions CPU time of the current process. It does not include time elapsed during sleep. It is process-wide by definition. The reference point of the returned value is undefined, so that only the difference between the results - of consecutive calls is valid. + of two calls is valid. + + Use :func:`process_time_ns` to avoid the precision loss caused by the + :class:`float` type. .. versionadded:: 3.3 @@ -581,6 +603,17 @@ Functions :class:`struct_time` object is returned, from which the components of the calendar date may be accessed as attributes. + Use :func:`time_ns` to avoid the precision loss caused by the :class:`float` + type. + + +.. function:: time_ns() -> int + + Similar to :func:`~time.time` but returns time as an integer number of nanoseconds + since the epoch_. + + .. versionadded:: 3.7 + .. function:: thread_time() -> float @@ -593,7 +626,10 @@ Functions CPU time of the current thread. It does not include time elapsed during sleep. It is thread-specific by definition. The reference point of the returned value is undefined, so that only the difference between the results - of consecutive calls in the same thread is valid. + of two calls in the same thread is valid. + + Use :func:`thread_time_ns` to avoid the precision loss caused by the + :class:`float` type. .. availability:: Windows, Linux, Unix systems supporting ``CLOCK_THREAD_CPUTIME_ID``. @@ -608,13 +644,6 @@ Functions .. versionadded:: 3.7 -.. function:: time_ns() -> int - - Similar to :func:`~time.time` but returns time as an integer number of nanoseconds - since the epoch_. - - .. versionadded:: 3.7 - .. function:: tzset() Reset the time conversion rules used by the library routines. The environment diff --git a/Doc/library/tkinter.font.rst b/Doc/library/tkinter.font.rst index b0f4505e9e3c69..c7c2b7b566cf8f 100644 --- a/Doc/library/tkinter.font.rst +++ b/Doc/library/tkinter.font.rst @@ -91,6 +91,9 @@ The different font weights and slants are: Return the names of defined fonts. -.. function:: nametofont(name) +.. function:: nametofont(name, root=None) - Return a :class:`Font` representation of a tk named font. \ No newline at end of file + Return a :class:`Font` representation of a tk named font. + + .. versionchanged:: 3.10 + The *root* parameter was added. diff --git a/Doc/library/token-list.inc b/Doc/library/token-list.inc index 877d39a432ed70..1a99f0518d1b47 100644 --- a/Doc/library/token-list.inc +++ b/Doc/library/token-list.inc @@ -211,6 +211,8 @@ .. data:: TYPE_COMMENT +.. data:: SOFT_KEYWORD + .. data:: ERRORTOKEN .. data:: N_TOKENS diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index c2732d900bc138..40cf198f1287d7 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -153,47 +153,47 @@ Programmatic Interface count information. *timing* enables a timestamp relative to when tracing was started to be displayed. - .. method:: run(cmd) + .. method:: run(cmd) - Execute the command and gather statistics from the execution with - the current tracing parameters. *cmd* must be a string or code object, - suitable for passing into :func:`exec`. + Execute the command and gather statistics from the execution with + the current tracing parameters. *cmd* must be a string or code object, + suitable for passing into :func:`exec`. - .. method:: runctx(cmd, globals=None, locals=None) + .. method:: runctx(cmd, globals=None, locals=None) - Execute the command and gather statistics from the execution with the - current tracing parameters, in the defined global and local - environments. If not defined, *globals* and *locals* default to empty - dictionaries. + Execute the command and gather statistics from the execution with the + current tracing parameters, in the defined global and local + environments. If not defined, *globals* and *locals* default to empty + dictionaries. - .. method:: runfunc(func, /, *args, **kwds) + .. method:: runfunc(func, /, *args, **kwds) - Call *func* with the given arguments under control of the :class:`Trace` - object with the current tracing parameters. + Call *func* with the given arguments under control of the :class:`Trace` + object with the current tracing parameters. - .. method:: results() + .. method:: results() - Return a :class:`CoverageResults` object that contains the cumulative - results of all previous calls to ``run``, ``runctx`` and ``runfunc`` - for the given :class:`Trace` instance. Does not reset the accumulated - trace results. + Return a :class:`CoverageResults` object that contains the cumulative + results of all previous calls to ``run``, ``runctx`` and ``runfunc`` + for the given :class:`Trace` instance. Does not reset the accumulated + trace results. .. class:: CoverageResults A container for coverage results, created by :meth:`Trace.results`. Should not be created directly by the user. - .. method:: update(other) + .. method:: update(other) - Merge in data from another :class:`CoverageResults` object. + Merge in data from another :class:`CoverageResults` object. - .. method:: write_results(show_missing=True, summary=False, coverdir=None) + .. method:: write_results(show_missing=True, summary=False, coverdir=None) - Write coverage results. Set *show_missing* to show lines that had no - hits. Set *summary* to include in the output the coverage summary per - module. *coverdir* specifies the directory into which the coverage - result files will be output. If ``None``, the results for each source - file are placed in its directory. + Write coverage results. Set *show_missing* to show lines that had no + hits. Set *summary* to include in the output the coverage summary per + module. *coverdir* specifies the directory into which the coverage + result files will be output. If ``None``, the results for each source + file are placed in its directory. A simple example demonstrating the use of the programmatic interface:: diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 462a6a5566e201..e938dd58b05312 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -36,7 +36,8 @@ The module defines the following functions: Added negative *limit* support. -.. function:: print_exception(etype, value, tb, limit=None, file=None, chain=True) +.. function:: print_exception(exc, /[, value, tb], limit=None, \ + file=None, chain=True) Print exception information and stack trace entries from traceback object *tb* to *file*. This differs from :func:`print_tb` in the following @@ -45,7 +46,7 @@ The module defines the following functions: * if *tb* is not ``None``, it prints a header ``Traceback (most recent call last):`` - * it prints the exception *etype* and *value* after the stack trace + * it prints the exception type and *value* after the stack trace .. index:: single: ^ (caret); marker @@ -53,6 +54,10 @@ The module defines the following functions: format, it prints the line where the syntax error occurred with a caret indicating the approximate position of the error. + Since Python 3.10, instead of passing *value* and *tb*, an exception object + can be passed as the first argument. If *value* and *tb* are provided, the + first argument is ignored in order to provide backwards compatibility. + The optional *limit* argument has the same meaning as for :func:`print_tb`. If *chain* is true (the default), then chained exceptions (the :attr:`__cause__` or :attr:`__context__` attributes of the exception) will be @@ -62,6 +67,10 @@ The module defines the following functions: .. versionchanged:: 3.5 The *etype* argument is ignored and inferred from the type of *value*. + .. versionchanged:: 3.10 + The *etype* parameter has been renamed to *exc* and is now + positional-only. + .. function:: print_exc(limit=None, file=None, chain=True) @@ -121,18 +130,26 @@ The module defines the following functions: text line is not ``None``. -.. function:: format_exception_only(etype, value) +.. function:: format_exception_only(exc, /[, value]) + + Format the exception part of a traceback using an exception value such as + given by ``sys.last_value``. The return value is a list of strings, each + ending in a newline. Normally, the list contains a single string; however, + for :exc:`SyntaxError` exceptions, it contains several lines that (when + printed) display detailed information about where the syntax error occurred. + The message indicating which exception occurred is the always last string in + the list. + + Since Python 3.10, instead of passing *value*, an exception object + can be passed as the first argument. If *value* is provided, the first + argument is ignored in order to provide backwards compatibility. - Format the exception part of a traceback. The arguments are the exception - type and value such as given by ``sys.last_type`` and ``sys.last_value``. - The return value is a list of strings, each ending in a newline. Normally, - the list contains a single string; however, for :exc:`SyntaxError` - exceptions, it contains several lines that (when printed) display detailed - information about where the syntax error occurred. The message indicating - which exception occurred is the always last string in the list. + .. versionchanged:: 3.10 + The *etype* parameter has been renamed to *exc* and is now + positional-only. -.. function:: format_exception(etype, value, tb, limit=None, chain=True) +.. function:: format_exception(exc, /[, value, tb], limit=None, chain=True) Format a stack trace and the exception information. The arguments have the same meaning as the corresponding arguments to :func:`print_exception`. The @@ -143,6 +160,10 @@ The module defines the following functions: .. versionchanged:: 3.5 The *etype* argument is ignored and inferred from the type of *value*. + .. versionchanged:: 3.10 + This function's behavior and signature were modified to match + :func:`print_exception`. + .. function:: format_exc(limit=None, chain=True) @@ -191,11 +212,16 @@ The module also defines the following classes: :class:`TracebackException` objects are created from actual exceptions to capture data for later printing in a lightweight fashion. -.. class:: TracebackException(exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False) +.. class:: TracebackException(exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False) Capture an exception for later rendering. *limit*, *lookup_lines* and *capture_locals* are as for the :class:`StackSummary` class. + If *compact* is true, only data that is required by :class:`TracebackException`'s + ``format`` method is saved in the class attributes. In particular, the + ``__context__`` field is calculated only if ``__cause__`` is ``None`` and + ``__suppress_context__`` is false. + Note that when locals are captured, they are also shown in the traceback. .. attribute:: __cause__ @@ -273,6 +299,9 @@ capture data for later printing in a lightweight fashion. The message indicating which exception occurred is always the last string in the output. + .. versionchanged:: 3.10 + Added the *compact* parameter. + :class:`StackSummary` Objects ----------------------------- diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index d3487537df99a9..6a9d61916ad1a5 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -662,7 +662,7 @@ Tell Turtle's state Return the angle between the line from turtle position to position specified by (x,y), the vector or the other turtle. This depends on the turtle's start - orientation which depends on the mode - "standard"/"world" or "logo"). + orientation which depends on the mode - "standard"/"world" or "logo". .. doctest:: :skipif: _tkinter is None @@ -913,8 +913,8 @@ Color control Set pencolor to the RGB color represented by *r*, *g*, and *b*. Each of *r*, *g*, and *b* must be in the range 0..colormode. - If turtleshape is a polygon, the outline of that polygon is drawn with the - newly set pencolor. + If turtleshape is a polygon, the outline of that polygon is drawn with the + newly set pencolor. .. doctest:: :skipif: _tkinter is None @@ -962,8 +962,8 @@ Color control Set fillcolor to the RGB color represented by *r*, *g*, and *b*. Each of *r*, *g*, and *b* must be in the range 0..colormode. - If turtleshape is a polygon, the interior of that polygon is drawn - with the newly set fillcolor. + If turtleshape is a polygon, the interior of that polygon is drawn + with the newly set fillcolor. .. doctest:: :skipif: _tkinter is None @@ -1001,8 +1001,8 @@ Color control Equivalent to ``pencolor(colorstring1)`` and ``fillcolor(colorstring2)`` and analogously if the other input format is used. - If turtleshape is a polygon, outline and interior of that polygon is drawn - with the newly set colors. + If turtleshape is a polygon, outline and interior of that polygon is drawn + with the newly set colors. .. doctest:: :skipif: _tkinter is None @@ -1105,7 +1105,7 @@ More drawing control :param font: a triple (fontname, fontsize, fonttype) Write text - the string representation of *arg* - at the current turtle - position according to *align* ("left", "center" or right") and with the given + position according to *align* ("left", "center" or "right") and with the given font. If *move* is true, the pen is moved to the bottom-right corner of the text. By default, *move* is ``False``. @@ -1192,7 +1192,7 @@ Appearance :func:`shapesize`. - "noresize": no adaption of the turtle's appearance takes place. - resizemode("user") is called by :func:`shapesize` when used with arguments. + ``resizemode("user")`` is called by :func:`shapesize` when used with arguments. .. doctest:: :skipif: _tkinter is None @@ -1330,7 +1330,7 @@ Appearance matrix as a tuple of 4 elements. Otherwise set the given elements and transform the turtleshape according to the matrix consisting of first row t11, t12 and - second row t21, 22. The determinant t11 * t22 - t12 * t21 must not be + second row t21, t22. The determinant t11 * t22 - t12 * t21 must not be zero, otherwise an error is raised. Modify stretchfactor, shearfactor and tiltangle according to the given matrix. @@ -1513,7 +1513,7 @@ Special Turtle methods :param size: an integer or ``None`` - Set or disable undobuffer. If *size* is an integer an empty undobuffer of + Set or disable undobuffer. If *size* is an integer, an empty undobuffer of given size is installed. *size* gives the maximum number of turtle actions that can be undone by the :func:`undo` method/function. If *size* is ``None``, the undobuffer is disabled. @@ -1821,7 +1821,7 @@ Using screen events existing bindings are removed. Example for a TurtleScreen instance named ``screen`` and a Turtle instance - named turtle: + named ``turtle``: .. doctest:: :skipif: _tkinter is None @@ -2048,7 +2048,7 @@ Methods specific to Screen, not inherited from TurtleScreen .. function:: exitonclick() - Bind bye() method to mouse clicks on the Screen. + Bind ``bye()`` method to mouse clicks on the Screen. If the value "using_IDLE" in the configuration dictionary is ``False`` diff --git a/Doc/library/types.rst b/Doc/library/types.rst index a5cf9ab1344a36..b3fac293ecc9e1 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -34,7 +34,7 @@ Dynamic Type Creation freshly created class namespace. It should accept the class namespace as its sole argument and update the namespace directly with the class contents. If no callback is provided, it has the same effect as passing - in ``lambda ns: ns``. + in ``lambda ns: None``. .. versionadded:: 3.3 @@ -223,7 +223,7 @@ Standard names are defined for the following types: .. class:: ModuleType(name, doc=None) - The type of :term:`modules `. Constructor takes the name of the + The type of :term:`modules `. The constructor takes the name of the module to be created and optionally its :term:`docstring`. .. note:: @@ -238,12 +238,23 @@ Standard names are defined for the following types: The :term:`loader` which loaded the module. Defaults to ``None``. + This attribute is to match :attr:`importlib.machinery.ModuleSpec.loader` + as stored in the attr:`__spec__` object. + + .. note:: + A future version of Python may stop setting this attribute by default. + To guard against this potential change, preferrably read from the + :attr:`__spec__` attribute instead or use + ``getattr(module, "__loader__", None)`` if you explicitly need to use + this attribute. + .. versionchanged:: 3.4 Defaults to ``None``. Previously the attribute was optional. .. attribute:: __name__ - The name of the module. + The name of the module. Expected to match + :attr:`importlib.machinery.ModuleSpec.name`. .. attribute:: __package__ @@ -252,9 +263,26 @@ Standard names are defined for the following types: to ``''``, else it should be set to the name of the package (which can be :attr:`__name__` if the module is a package itself). Defaults to ``None``. + This attribute is to match :attr:`importlib.machinery.ModuleSpec.parent` + as stored in the attr:`__spec__` object. + + .. note:: + A future version of Python may stop setting this attribute by default. + To guard against this potential change, preferrably read from the + :attr:`__spec__` attribute instead or use + ``getattr(module, "__package__", None)`` if you explicitly need to use + this attribute. + .. versionchanged:: 3.4 Defaults to ``None``. Previously the attribute was optional. + .. attribute:: __spec__ + + A record of the the module's import-system-related state. Expected to be + an instance of :class:`importlib.machinery.ModuleSpec`. + + .. versionadded:: 3.4 + .. data:: EllipsisType @@ -262,6 +290,28 @@ Standard names are defined for the following types: .. versionadded:: 3.10 +.. class:: GenericAlias(t_origin, t_args) + + The type of :ref:`parameterized generics ` such as + ``list[int]``. + + ``t_origin`` should be a non-parameterized generic class, such as ``list``, + ``tuple`` or ``dict``. ``t_args`` should be a :class:`tuple` (possibly of + length 1) of types which parameterize ``t_origin``:: + + >>> from types import GenericAlias + + >>> list[int] == GenericAlias(list, (int,)) + True + >>> dict[str, int] == GenericAlias(dict, (str, int)) + True + + .. versionadded:: 3.9 + + .. versionchanged:: 3.9.2 + This type can now be subclassed. + + .. data:: Union The type of :ref:`union type expressions`. @@ -391,7 +441,9 @@ Additional Utility Classes and Functions return "{}({})".format(type(self).__name__, ", ".join(items)) def __eq__(self, other): - return self.__dict__ == other.__dict__ + if isinstance(self, SimpleNamespace) and isinstance(other, SimpleNamespace): + return self.__dict__ == other.__dict__ + return NotImplemented ``SimpleNamespace`` may be useful as a replacement for ``class NS: pass``. However, for a structured record type use :func:`~collections.namedtuple` diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 402dd24fde6e08..8b1ce34d98ec66 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -18,7 +18,8 @@ -------------- This module provides runtime support for type hints as specified by -:pep:`484`, :pep:`526`, :pep:`544`, :pep:`586`, :pep:`589`, :pep:`591`, and :pep:`613`. +:pep:`484`, :pep:`526`, :pep:`544`, :pep:`586`, :pep:`589`, :pep:`591`, +:pep:`612` and :pep:`613`. The most fundamental support consists of the types :data:`Any`, :data:`Union`, :data:`Tuple`, :data:`Callable`, :class:`TypeVar`, and :class:`Generic`. For full specification please see :pep:`484`. For @@ -171,6 +172,22 @@ It is possible to declare the return type of a callable without specifying the call signature by substituting a literal ellipsis for the list of arguments in the type hint: ``Callable[..., ReturnType]``. +Callables which take other callables as arguments may indicate that their +parameter types are dependent on each other using :class:`ParamSpec`. +Additionally, if that callable adds or removes arguments from other +callables, the :data:`Concatenate` operator may be used. They +take the form ``Callable[ParamSpecVariable, ReturnType]`` and +``Callable[Concatenate[Arg1Type, Arg2Type, ..., ParamSpecVariable], ReturnType]`` +respectively. + +.. versionchanged:: 3.10 + ``Callable`` now supports :class:`ParamSpec` and :data:`Concatenate`. + See :pep:`612` for more information. + +.. seealso:: + The documentation for :class:`ParamSpec` and :class:`Concatenate` provide + examples of usage in ``Callable``. + .. _generics: Generics @@ -316,6 +333,43 @@ User defined generic type aliases are also supported. Examples:: .. versionchanged:: 3.7 :class:`Generic` no longer has a custom metaclass. +User-defined generics for parameter expressions are also supported via parameter +specification variables in the form ``Generic[P]``. The behavior is consistent +with type variables' described above as parameter specification variables are +treated by the typing module as a specialized type variable. The one exception +to this is that a list of types can be used to substitute a :class:`ParamSpec`:: + + >>> from typing import Generic, ParamSpec, TypeVar + + >>> T = TypeVar('T') + >>> P = ParamSpec('P') + + >>> class Z(Generic[T, P]): ... + ... + >>> Z[int, [dict, float]] + __main__.Z[int, (, )] + + +Furthermore, a generic with only one parameter specification variable will accept +parameter lists in the forms ``X[[Type1, Type2, ...]]`` and also +``X[Type1, Type2, ...]`` for aesthetic reasons. Internally, the latter is converted +to the former and are thus equivalent:: + + >>> class X(Generic[P]): ... + ... + >>> X[int, str] + __main__.X[(, )] + >>> X[[int, str]] + __main__.X[(, )] + +Do note that generics with :class:`ParamSpec` may not have correct +``__parameters__`` after substitution in some cases because they +are intended primarily for static type checking. + +.. versionchanged:: 3.10 + :class:`Generic` can now be parameterized over parameter expressions. + See :class:`ParamSpec` and :pep:`612` for more details. + A user-defined generic class can have ABCs as base classes without a metaclass conflict. Generic metaclasses are not supported. The outcome of parameterizing generics is cached, and most types in the typing module are hashable and @@ -524,7 +578,8 @@ These can be used as types in annotations using ``[]``, each having a unique syn is equivalent to ``Tuple[Any, ...]``, and in turn to :class:`tuple`. .. deprecated:: 3.9 - :class:`builtins.tuple ` now supports ``[]``. See :pep:`585`. + :class:`builtins.tuple ` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. .. data:: Union @@ -601,8 +656,79 @@ These can be used as types in annotations using ``[]``, each having a unique syn ``Callable[..., Any]``, and in turn to :class:`collections.abc.Callable`. + Callables which take other callables as arguments may indicate that their + parameter types are dependent on each other using :class:`ParamSpec`. + Additionally, if that callable adds or removes arguments from other + callables, the :data:`Concatenate` operator may be used. They + take the form ``Callable[ParamSpecVariable, ReturnType]`` and + ``Callable[Concatenate[Arg1Type, Arg2Type, ..., ParamSpecVariable], ReturnType]`` + respectively. + .. deprecated:: 3.9 - :class:`collections.abc.Callable` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.Callable` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. + + .. versionchanged:: 3.10 + ``Callable`` now supports :class:`ParamSpec` and :data:`Concatenate`. + See :pep:`612` for more information. + + .. seealso:: + The documentation for :class:`ParamSpec` and :class:`Concatenate` provide + examples of usage with ``Callable``. + +.. data:: Concatenate + + Used with :data:`Callable` and :class:`ParamSpec` to type annotate a higher + order callable which adds, removes, or transforms parameters of another + callable. Usage is in the form + ``Concatenate[Arg1Type, Arg2Type, ..., ParamSpecVariable]``. ``Concatenate`` + is currently only valid when used as the first argument to a :data:`Callable`. + The last parameter to ``Concatenate`` must be a :class:`ParamSpec`. + + For example, to annotate a decorator ``with_lock`` which provides a + :class:`threading.Lock` to the decorated function, ``Concatenate`` can be + used to indicate that ``with_lock`` expects a callable which takes in a + ``Lock`` as the first argument, and returns a callable with a different type + signature. In this case, the :class:`ParamSpec` indicates that the returned + callable's parameter types are dependent on the parameter types of the + callable being passed in:: + + from collections.abc import Callable + from threading import Lock + from typing import Any, Concatenate, ParamSpec, TypeVar + + P = ParamSpec('P') + R = TypeVar('R') + + # Use this lock to ensure that only one thread is executing a function + # at any time. + my_lock = Lock() + + def with_lock(f: Callable[Concatenate[Lock, P], R]) -> Callable[P, R]: + '''A type-safe decorator which provides a lock.''' + global my_lock + def inner(*args: P.args, **kwargs: P.kwargs) -> R: + # Provide the lock as the first argument. + return f(my_lock, *args, **kwargs) + return inner + + @with_lock + def sum_threadsafe(lock: Lock, numbers: list[float]) -> float: + '''Add a list of numbers together in a thread-safe manner.''' + with lock: + return sum(numbers) + + # We don't need to pass in the lock ourselves thanks to the decorator. + sum_threadsafe([1.1, 2.2, 3.3]) + +.. versionadded:: 3.10 + +.. seealso:: + + * :pep:`612` -- Parameter Specification Variables (the PEP which introduced + ``ParamSpec`` and ``Concatenate``). + * :class:`ParamSpec` and :class:`Callable`. + .. class:: Type(Generic[CT_co]) @@ -639,7 +765,7 @@ These can be used as types in annotations using ``[]``, each having a unique syn :ref:`type variables `, and unions of any of these types. For example:: - def new_non_team_user(user_class: Type[Union[BaseUser, ProUser]]): ... + def new_non_team_user(user_class: Type[Union[BasicUser, ProUser]]): ... ``Type[Any]`` is equivalent to ``Type`` which in turn is equivalent to ``type``, which is the root of Python's metaclass hierarchy. @@ -647,7 +773,8 @@ These can be used as types in annotations using ``[]``, each having a unique syn .. versionadded:: 3.5.2 .. deprecated:: 3.9 - :class:`builtins.type ` now supports ``[]``. See :pep:`585`. + :class:`builtins.type ` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. .. data:: Literal @@ -671,6 +798,12 @@ These can be used as types in annotations using ``[]``, each having a unique syn .. versionadded:: 3.8 + .. versionchanged:: 3.9.1 + ``Literal`` now de-duplicates parameters. Equality comparison of + ``Literal`` objects are no longer order dependent. ``Literal`` objects + will now raise a :exc:`TypeError` exception during equality comparisons + if one of their parameters are not :term:`immutable`. + .. data:: ClassVar Special type construct to mark class variables. @@ -800,6 +933,75 @@ These can be used as types in annotations using ``[]``, each having a unique syn .. versionadded:: 3.9 + +.. data:: TypeGuard + + Special typing form used to annotate the return type of a user-defined + type guard function. ``TypeGuard`` only accepts a single type argument. + At runtime, functions marked this way should return a boolean. + + ``TypeGuard`` aims to benefit *type narrowing* -- a technique used by static + type checkers to determine a more precise type of an expression within a + program's code flow. Usually type narrowing is done by analyzing + conditional code flow and applying the narrowing to a block of code. The + conditional expression here is sometimes referred to as a "type guard":: + + def is_str(val: Union[str, float]): + # "isinstance" type guard + if isinstance(val, str): + # Type of ``val`` is narrowed to ``str`` + ... + else: + # Else, type of ``val`` is narrowed to ``float``. + ... + + Sometimes it would be convenient to use a user-defined boolean function + as a type guard. Such a function should use ``TypeGuard[...]`` as its + return type to alert static type checkers to this intention. + + Using ``-> TypeGuard`` tells the static type checker that for a given + function: + + 1. The return value is a boolean. + 2. If the return value is ``True``, the type of its argument + is the type inside ``TypeGuard``. + + For example:: + + def is_str_list(val: List[object]) -> TypeGuard[List[str]]: + '''Determines whether all objects in the list are strings''' + return all(isinstance(x, str) for x in val) + + def func1(val: List[object]): + if is_str_list(val): + # Type of ``val`` is narrowed to ``List[str]``. + print(" ".join(val)) + else: + # Type of ``val`` remains as ``List[object]``. + print("Not a list of strings!") + + If ``is_str_list`` is a class or instance method, then the type in + ``TypeGuard`` maps to the type of the second parameter after ``cls`` or + ``self``. + + In short, the form ``def foo(arg: TypeA) -> TypeGuard[TypeB]: ...``, + means that if ``foo(arg)`` returns ``True``, then ``arg`` narrows from + ``TypeA`` to ``TypeB``. + + .. note:: + + ``TypeB`` need not be a narrower form of ``TypeA`` -- it can even be a + wider form. The main reason is to allow for things like + narrowing ``List[object]`` to ``List[str]`` even though the latter + is not a subtype of the former, since ``List`` is invariant. + The responsibility of writing type-safe type guards is left to the user. + + ``TypeGuard`` also works with type variables. For more information, see + :pep:`647` (User-Defined Type Guards). + + .. versionadded:: 3.10 + + Building generic types """""""""""""""""""""" @@ -867,6 +1069,104 @@ These are not used in annotations. They are building blocks for creating generic for the type variable must be a subclass of the boundary type, see :pep:`484`. +.. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False) + + Parameter specification variable. A specialized version of + :class:`type variables `. + + Usage:: + + P = ParamSpec('P') + + Parameter specification variables exist primarily for the benefit of static + type checkers. They are used to forward the parameter types of one + callable to another callable -- a pattern commonly found in higher order + functions and decorators. They are only valid when used in ``Concatenate``, + or as the first argument to ``Callable``, or as parameters for user-defined + Generics. See :class:`Generic` for more information on generic types. + + For example, to add basic logging to a function, one can create a decorator + ``add_logging`` to log function calls. The parameter specification variable + tells the type checker that the callable passed into the decorator and the + new callable returned by it have inter-dependent type parameters:: + + from collections.abc import Callable + from typing import TypeVar, ParamSpec + import logging + + T = TypeVar('T') + P = ParamSpec('P') + + def add_logging(f: Callable[P, T]) -> Callable[P, T]: + '''A type-safe decorator to add logging to a function.''' + def inner(*args: P.args, **kwargs: P.kwargs) -> T: + logging.info(f'{f.__name__} was called') + return f(*args, **kwargs) + return inner + + @add_logging + def add_two(x: float, y: float) -> float: + '''Add two numbers together.''' + return x + y + + Without ``ParamSpec``, the simplest way to annotate this previously was to + use a :class:`TypeVar` with bound ``Callable[..., Any]``. However this + causes two problems: + + 1. The type checker can't type check the ``inner`` function because + ``*args`` and ``**kwargs`` have to be typed :data:`Any`. + 2. :func:`~cast` may be required in the body of the ``add_logging`` + decorator when returning the ``inner`` function, or the static type + checker must be told to ignore the ``return inner``. + + .. attribute:: args + .. attribute:: kwargs + + Since ``ParamSpec`` captures both positional and keyword parameters, + ``P.args`` and ``P.kwargs`` can be used to split a ``ParamSpec`` into its + components. ``P.args`` represents the tuple of positional parameters in a + given call and should only be used to annotate ``*args``. ``P.kwargs`` + represents the mapping of keyword parameters to their values in a given call, + and should be only be used to annotate ``**kwargs``. Both + attributes require the annotated parameter to be in scope. At runtime, + ``P.args`` and ``P.kwargs`` are instances respectively of + :class:`ParamSpecArgs` and :class:`ParamSpecKwargs`. + + Parameter specification variables created with ``covariant=True`` or + ``contravariant=True`` can be used to declare covariant or contravariant + generic types. The ``bound`` argument is also accepted, similar to + :class:`TypeVar`. However the actual semantics of these keywords are yet to + be decided. + + .. versionadded:: 3.10 + + .. note:: + Only parameter specification variables defined in global scope can + be pickled. + + .. seealso:: + * :pep:`612` -- Parameter Specification Variables (the PEP which introduced + ``ParamSpec`` and ``Concatenate``). + * :class:`Callable` and :class:`Concatenate`. + +.. data:: ParamSpecArgs +.. data:: ParamSpecKwargs + + Arguments and keyword arguments attributes of a :class:`ParamSpec`. The + ``P.args`` attribute of a ``ParamSpec`` is an instance of ``ParamSpecArgs``, + and ``P.kwargs`` is an instance of ``ParamSpecKwargs``. They are intended + for runtime introspection and have no special meaning to static type checkers. + + Calling :func:`get_origin` on either of these objects will return the + original ``ParamSpec``:: + + P = ParamSpec("P") + get_origin(P.args) # returns P + get_origin(P.kwargs) # returns P + + .. versionadded:: 3.10 + + .. data:: AnyStr ``AnyStr`` is a type variable defined as @@ -933,10 +1233,7 @@ These are not used in annotations. They are building blocks for creating generic .. note:: :func:`runtime_checkable` will check only the presence of the required methods, - not their type signatures! For example, :class:`builtins.complex ` - implements :func:`__float__`, therefore it passes an :func:`issubclass` check - against :class:`SupportsFloat`. However, the ``complex.__float__`` method - exists only to raise a :class:`TypeError` with a more informative message. + not their type signatures. .. versionadded:: 3.8 @@ -1036,26 +1333,28 @@ These are not used in annotations. They are building blocks for declaring types. assert Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') - The type info for introspection can be accessed via ``Point2D.__annotations__`` - and ``Point2D.__total__``. To allow using this feature with older versions - of Python that do not support :pep:`526`, ``TypedDict`` supports two additional - equivalent syntactic forms:: + The type info for introspection can be accessed via ``Point2D.__annotations__``, + ``Point2D.__total__``, ``Point2D.__required_keys__``, and + ``Point2D.__optional_keys__``. + To allow using this feature with older versions of Python that do not + support :pep:`526`, ``TypedDict`` supports two additional equivalent + syntactic forms:: Point2D = TypedDict('Point2D', x=int, y=int, label=str) Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) - By default, all keys must be present in a TypedDict. It is possible - to override this by specifying totality. + By default, all keys must be present in a ``TypedDict``. It is possible to + override this by specifying totality. Usage:: - class point2D(TypedDict, total=False): + class Point2D(TypedDict, total=False): x: int y: int - This means that a point2D TypedDict can have any of the keys omitted. A type - checker is only expected to support a literal False or True as the value of - the total argument. True is the default, and makes all items defined in the - class body be required. + This means that a ``Point2D`` ``TypedDict`` can have any of the keys + omitted. A type checker is only expected to support a literal ``False`` or + ``True`` as the value of the ``total`` argument. ``True`` is the default, + and makes all items defined in the class body required. See :pep:`589` for more examples and detailed rules of using ``TypedDict``. @@ -1079,7 +1378,8 @@ Corresponding to built-in types ... .. deprecated:: 3.9 - :class:`builtins.dict ` now supports ``[]``. See :pep:`585`. + :class:`builtins.dict ` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. .. class:: List(list, MutableSequence[T]) @@ -1099,7 +1399,8 @@ Corresponding to built-in types return [item for item in vector if item > 0] .. deprecated:: 3.9 - :class:`builtins.list ` now supports ``[]``. See :pep:`585`. + :class:`builtins.list ` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. .. class:: Set(set, MutableSet[T]) @@ -1108,14 +1409,16 @@ Corresponding to built-in types to use an abstract collection type such as :class:`AbstractSet`. .. deprecated:: 3.9 - :class:`builtins.set ` now supports ``[]``. See :pep:`585`. + :class:`builtins.set ` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. .. class:: FrozenSet(frozenset, AbstractSet[T_co]) A generic version of :class:`builtins.frozenset `. .. deprecated:: 3.9 - :class:`builtins.frozenset ` now supports ``[]``. See :pep:`585`. + :class:`builtins.frozenset ` now supports ``[]``. See + :pep:`585` and :ref:`types-genericalias`. .. note:: :data:`Tuple` is a special form. @@ -1129,7 +1432,8 @@ Corresponding to types in :mod:`collections` .. versionadded:: 3.5.2 .. deprecated:: 3.9 - :class:`collections.defaultdict` now supports ``[]``. See :pep:`585`. + :class:`collections.defaultdict` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. .. class:: OrderedDict(collections.OrderedDict, MutableMapping[KT, VT]) @@ -1138,7 +1442,8 @@ Corresponding to types in :mod:`collections` .. versionadded:: 3.7.2 .. deprecated:: 3.9 - :class:`collections.OrderedDict` now supports ``[]``. See :pep:`585`. + :class:`collections.OrderedDict` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. .. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) @@ -1148,7 +1453,8 @@ Corresponding to types in :mod:`collections` .. versionadded:: 3.6.1 .. deprecated:: 3.9 - :class:`collections.ChainMap` now supports ``[]``. See :pep:`585`. + :class:`collections.ChainMap` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. .. class:: Counter(collections.Counter, Dict[T, int]) @@ -1158,7 +1464,8 @@ Corresponding to types in :mod:`collections` .. versionadded:: 3.6.1 .. deprecated:: 3.9 - :class:`collections.Counter` now supports ``[]``. See :pep:`585`. + :class:`collections.Counter` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. .. class:: Deque(deque, MutableSequence[T]) @@ -1168,7 +1475,8 @@ Corresponding to types in :mod:`collections` .. versionadded:: 3.6.1 .. deprecated:: 3.9 - :class:`collections.deque` now supports ``[]``. See :pep:`585`. + :class:`collections.deque` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. Other concrete types """""""""""""""""""" @@ -1193,7 +1501,8 @@ Other concrete types ``Match[bytes]``. These types are also in the ``typing.re`` namespace. .. deprecated:: 3.9 - Classes ``Pattern`` and ``Match`` from :mod:`re` now support ``[]``. See :pep:`585`. + Classes ``Pattern`` and ``Match`` from :mod:`re` now support ``[]``. + See :pep:`585` and :ref:`types-genericalias`. .. class:: Text @@ -1220,7 +1529,8 @@ Corresponding to collections in :mod:`collections.abc` A generic version of :class:`collections.abc.Set`. .. deprecated:: 3.9 - :class:`collections.abc.Set` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.Set` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. .. class:: ByteString(Sequence[int]) @@ -1233,7 +1543,8 @@ Corresponding to collections in :mod:`collections.abc` annotate arguments of any of the types mentioned above. .. deprecated:: 3.9 - :class:`collections.abc.ByteString` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.ByteString` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. .. class:: Collection(Sized, Iterable[T_co], Container[T_co]) @@ -1242,28 +1553,32 @@ Corresponding to collections in :mod:`collections.abc` .. versionadded:: 3.6.0 .. deprecated:: 3.9 - :class:`collections.abc.Collection` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.Collection` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. .. class:: Container(Generic[T_co]) A generic version of :class:`collections.abc.Container`. .. deprecated:: 3.9 - :class:`collections.abc.Container` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.Container` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. .. class:: ItemsView(MappingView, Generic[KT_co, VT_co]) A generic version of :class:`collections.abc.ItemsView`. .. deprecated:: 3.9 - :class:`collections.abc.ItemsView` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.ItemsView` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. .. class:: KeysView(MappingView[KT_co], AbstractSet[KT_co]) A generic version of :class:`collections.abc.KeysView`. .. deprecated:: 3.9 - :class:`collections.abc.KeysView` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.KeysView` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. .. class:: Mapping(Sized, Collection[KT], Generic[VT_co]) @@ -1274,49 +1589,56 @@ Corresponding to collections in :mod:`collections.abc` return word_list[word] .. deprecated:: 3.9 - :class:`collections.abc.Mapping` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.Mapping` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. .. class:: MappingView(Sized, Iterable[T_co]) A generic version of :class:`collections.abc.MappingView`. .. deprecated:: 3.9 - :class:`collections.abc.MappingView` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.MappingView` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. .. class:: MutableMapping(Mapping[KT, VT]) A generic version of :class:`collections.abc.MutableMapping`. .. deprecated:: 3.9 - :class:`collections.abc.MutableMapping` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.MutableMapping` now supports ``[]``. See + :pep:`585` and :ref:`types-genericalias`. .. class:: MutableSequence(Sequence[T]) A generic version of :class:`collections.abc.MutableSequence`. .. deprecated:: 3.9 - :class:`collections.abc.MutableSequence` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.MutableSequence` now supports ``[]``. See + :pep:`585` and :ref:`types-genericalias`. .. class:: MutableSet(AbstractSet[T]) A generic version of :class:`collections.abc.MutableSet`. .. deprecated:: 3.9 - :class:`collections.abc.MutableSet` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.MutableSet` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. .. class:: Sequence(Reversible[T_co], Collection[T_co]) A generic version of :class:`collections.abc.Sequence`. .. deprecated:: 3.9 - :class:`collections.abc.Sequence` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.Sequence` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. .. class:: ValuesView(MappingView[VT_co]) A generic version of :class:`collections.abc.ValuesView`. .. deprecated:: 3.9 - :class:`collections.abc.ValuesView` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.ValuesView` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. Corresponding to other types in :mod:`collections.abc` """""""""""""""""""""""""""""""""""""""""""""""""""""" @@ -1326,14 +1648,16 @@ Corresponding to other types in :mod:`collections.abc` A generic version of :class:`collections.abc.Iterable`. .. deprecated:: 3.9 - :class:`collections.abc.Iterable` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.Iterable` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. .. class:: Iterator(Iterable[T_co]) A generic version of :class:`collections.abc.Iterator`. .. deprecated:: 3.9 - :class:`collections.abc.Iterator` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.Iterator` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. .. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co]) @@ -1367,7 +1691,8 @@ Corresponding to other types in :mod:`collections.abc` start += 1 .. deprecated:: 3.9 - :class:`collections.abc.Generator` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.Generator` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. .. class:: Hashable @@ -1378,7 +1703,8 @@ Corresponding to other types in :mod:`collections.abc` A generic version of :class:`collections.abc.Reversible`. .. deprecated:: 3.9 - :class:`collections.abc.Reversible` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.Reversible` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. .. class:: Sized @@ -1403,7 +1729,8 @@ Asynchronous programming .. versionadded:: 3.5.3 .. deprecated:: 3.9 - :class:`collections.abc.Coroutine` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.Coroutine` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. .. class:: AsyncGenerator(AsyncIterator[T_co], Generic[T_co, T_contra]) @@ -1439,7 +1766,8 @@ Asynchronous programming .. versionadded:: 3.6.1 .. deprecated:: 3.9 - :class:`collections.abc.AsyncGenerator` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.AsyncGenerator` now supports ``[]``. See + :pep:`585` and :ref:`types-genericalias`. .. class:: AsyncIterable(Generic[T_co]) @@ -1448,7 +1776,8 @@ Asynchronous programming .. versionadded:: 3.5.2 .. deprecated:: 3.9 - :class:`collections.abc.AsyncIterable` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.AsyncIterable` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. .. class:: AsyncIterator(AsyncIterable[T_co]) @@ -1457,7 +1786,8 @@ Asynchronous programming .. versionadded:: 3.5.2 .. deprecated:: 3.9 - :class:`collections.abc.AsyncIterator` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.AsyncIterator` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. .. class:: Awaitable(Generic[T_co]) @@ -1466,7 +1796,8 @@ Asynchronous programming .. versionadded:: 3.5.2 .. deprecated:: 3.9 - :class:`collections.abc.Awaitable` now supports ``[]``. See :pep:`585`. + :class:`collections.abc.Awaitable` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. Context manager types @@ -1480,7 +1811,8 @@ Context manager types .. versionadded:: 3.6.0 .. deprecated:: 3.9 - :class:`contextlib.AbstractContextManager` now supports ``[]``. See :pep:`585`. + :class:`contextlib.AbstractContextManager` now supports ``[]``. See + :pep:`585` and :ref:`types-genericalias`. .. class:: AsyncContextManager(Generic[T_co]) @@ -1490,7 +1822,8 @@ Context manager types .. versionadded:: 3.6.2 .. deprecated:: 3.9 - :class:`contextlib.AbstractAsyncContextManager` now supports ``[]``. See :pep:`585`. + :class:`contextlib.AbstractAsyncContextManager` now supports ``[]``. See + :pep:`585` and :ref:`types-genericalias`. Protocols --------- @@ -1669,6 +2002,9 @@ Introspection helpers For a typing object of the form ``X[Y, Z, ...]`` these functions return ``X`` and ``(Y, Z, ...)``. If ``X`` is a generic alias for a builtin or :mod:`collections` class, it gets normalized to the original class. + If ``X`` is a :class:`Union` or :class:`Literal` contained in another + generic type, the order of ``(Y, Z, ...)`` may be different from the order + of the original arguments ``[Y, Z, ...]`` due to type caching. For unsupported objects return ``None`` and ``()`` correspondingly. Examples:: @@ -1698,10 +2034,17 @@ Introspection helpers .. class:: ForwardRef A class used for internal typing representation of string forward references. - For example, ``list["SomeClass"]`` is implicitly transformed into - ``list[ForwardRef("SomeClass")]``. This class should not be instantiated by + For example, ``List["SomeClass"]`` is implicitly transformed into + ``List[ForwardRef("SomeClass")]``. This class should not be instantiated by a user, but may be used by introspection tools. + .. note:: + :pep:`585` generic types such as ``list["SomeClass"]`` will not be + implicitly transformed into ``list[ForwardRef("SomeClass")]`` and thus + will not automatically resolve to ``list[SomeClass]``. + + .. versionadded:: 3.7.4 + Constant -------- @@ -1730,4 +2073,3 @@ Constant (see :pep:`563`). .. versionadded:: 3.5.2 - diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index e650bb1e23e03e..24a18c68484686 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -893,7 +893,7 @@ Here's an example implementation: ... def __call__(self, /, *args, **kwargs): ... args = deepcopy(args) ... kwargs = deepcopy(kwargs) - ... return super(CopyingMock, self).__call__(*args, **kwargs) + ... return super().__call__(*args, **kwargs) ... >>> c = CopyingMock(return_value=None) >>> arg = set() diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index c5360f91f518d6..1604731b99e3fe 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -30,7 +30,7 @@ module and class level attributes within the scope of a test, along with some examples of how to use :class:`Mock`, :class:`MagicMock` and :func:`patch`. -Mock is very easy to use and is designed for use with :mod:`unittest`. Mock +Mock is designed for use with :mod:`unittest` and is based on the 'action -> assertion' pattern instead of 'record -> replay' used by many mocking frameworks. @@ -262,9 +262,10 @@ the *new_callable* argument to :func:`patch`. this is a new Mock (created on first access). See the :attr:`return_value` attribute. - * *unsafe*: By default if any attribute starts with *assert* or - *assret* will raise an :exc:`AttributeError`. Passing ``unsafe=True`` - will allow access to these attributes. + * *unsafe*: By default, accessing any attribute with name starting with + *assert*, *assret*, *asert*, *aseert* or *assrt* will raise an + :exc:`AttributeError`. Passing ``unsafe=True`` will allow access to + these attributes. .. versionadded:: 3.5 diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 51e10119d3e8d0..0c29408cb19214 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -330,7 +330,10 @@ Test modules and packages can customize test loading and discovery by through the `load_tests protocol`_. .. versionchanged:: 3.4 - Test discovery supports :term:`namespace packages `. + Test discovery supports :term:`namespace packages ` + for the start directory. Note that you need to specify the top level + directory too (e.g. + ``python -m unittest discover -s root/namespace -t root``). .. _organizing-tests: @@ -897,8 +900,7 @@ Test cases .. method:: assertIs(first, second, msg=None) assertIsNot(first, second, msg=None) - Test that *first* and *second* evaluate (or don't evaluate) to the - same object. + Test that *first* and *second* are (or are not) the same object. .. versionadded:: 3.1 @@ -1499,11 +1501,11 @@ Test cases after :meth:`setUpClass` if :meth:`setUpClass` raises an exception. It is responsible for calling all the cleanup functions added by - :meth:`addCleanupClass`. If you need cleanup functions to be called + :meth:`addClassCleanup`. If you need cleanup functions to be called *prior* to :meth:`tearDownClass` then you can call - :meth:`doCleanupsClass` yourself. + :meth:`doClassCleanups` yourself. - :meth:`doCleanupsClass` pops methods off the stack of cleanup + :meth:`doClassCleanups` pops methods off the stack of cleanup functions one at a time, so it can be called at any time. .. versionadded:: 3.8 @@ -1871,11 +1873,15 @@ Loading and running tests .. versionchanged:: 3.4 Modules that raise :exc:`SkipTest` on import are recorded as skips, - not errors. - Discovery works for :term:`namespace packages `. - Paths are sorted before being imported so that execution order is - the same even if the underlying file system's ordering is not - dependent on file name. + not errors. + + .. versionchanged:: 3.4 + *start_dir* can be a :term:`namespace packages `. + + .. versionchanged:: 3.4 + Paths are sorted before being imported so that execution order is the + same even if the underlying file system's ordering is not dependent + on file name. .. versionchanged:: 3.5 Found packages are now checked for ``load_tests`` regardless of diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index f9c8ba7398f66f..0aaac562883f18 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -165,7 +165,7 @@ or on combining URL components into a URL string. now raise :exc:`ValueError`. -.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None) +.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator='&') Parse a query string given as a string argument (data of type :mimetype:`application/x-www-form-urlencoded`). Data are returned as a @@ -190,6 +190,9 @@ or on combining URL components into a URL string. read. If set, then throws a :exc:`ValueError` if there are more than *max_num_fields* fields read. + The optional argument *separator* is the symbol to use for separating the + query arguments. It defaults to ``&``. + Use the :func:`urllib.parse.urlencode` function (with the ``doseq`` parameter set to ``True``) to convert such dictionaries into query strings. @@ -201,8 +204,14 @@ or on combining URL components into a URL string. .. versionchanged:: 3.8 Added *max_num_fields* parameter. + .. versionchanged:: 3.10 + Added *separator* parameter with the default value of ``&``. Python + versions earlier than Python 3.10 allowed using both ``;`` and ``&`` as + query parameter separator. This has been changed to allow only a single + separator key, with ``&`` as the default separator. + -.. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None) +.. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator='&') Parse a query string given as a string argument (data of type :mimetype:`application/x-www-form-urlencoded`). Data are returned as a list of @@ -226,6 +235,9 @@ or on combining URL components into a URL string. read. If set, then throws a :exc:`ValueError` if there are more than *max_num_fields* fields read. + The optional argument *separator* is the symbol to use for separating the + query arguments. It defaults to ``&``. + Use the :func:`urllib.parse.urlencode` function to convert such lists of pairs into query strings. @@ -235,6 +247,12 @@ or on combining URL components into a URL string. .. versionchanged:: 3.8 Added *max_num_fields* parameter. + .. versionchanged:: 3.10 + Added *separator* parameter with the default value of ``&``. Python + versions earlier than Python 3.10 allowed using both ``;`` and ``&`` as + query parameter separator. This has been changed to allow only a single + separator key, with ``&`` as the default separator. + .. function:: urlunparse(parts) @@ -294,6 +312,9 @@ or on combining URL components into a URL string. ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is decomposed before parsing, no error will be raised. + Following the `WHATWG spec`_ that updates RFC 3986, ASCII newline + ``\n``, ``\r`` and tab ``\t`` characters are stripped from the URL. + .. versionchanged:: 3.6 Out-of-range port numbers now raise :exc:`ValueError`, instead of returning :const:`None`. @@ -302,6 +323,10 @@ or on combining URL components into a URL string. Characters that affect netloc parsing under NFKC normalization will now raise :exc:`ValueError`. + .. versionchanged:: 3.10 + ASCII newline and tab characters are stripped from the URL. + +.. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser .. function:: urlunsplit(parts) @@ -656,6 +681,10 @@ task isn't already covered by the URL parsing functions above. .. seealso:: + `WHATWG`_ - URL Living standard + Working Group for the URL Standard that defines URLs, domains, IP addresses, the + application/x-www-form-urlencoded format, and their API. + :rfc:`3986` - Uniform Resource Identifiers This is the current standard (STD66). Any changes to urllib.parse module should conform to this. Certain deviations could be observed, which are @@ -679,3 +708,5 @@ task isn't already covered by the URL parsing functions above. :rfc:`1738` - Uniform Resource Locators (URL) This specifies the formal syntax and semantics of absolute URLs. + +.. _WHATWG: https://url.spec.whatwg.org/ diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index b37f230feb6015..b4435a62ad43d9 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -109,6 +109,11 @@ The :mod:`urllib.request` module defines the following functions: .. versionchanged:: 3.4.3 *context* was added. + .. versionchanged:: 3.10 + HTTPS connection now send an ALPN extension with protocol indicator + ``http/1.1`` when no *context* is given. Custom *context* should set + ALPN protocols with :meth:`~ssl.SSLContext.set_alpn_protocol`. + .. deprecated:: 3.6 *cafile*, *capath* and *cadefault* are deprecated in favor of *context*. diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 5d4a36481f1dcc..2a4eede91a7c17 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -404,7 +404,7 @@ subclass which installs setuptools and pip into a created virtual environment:: :param context: The information for the virtual environment creation request being processed. """ - url = 'https://raw.github.com/pypa/pip/master/contrib/get-pip.py' + url = 'https://bootstrap.pypa.io/get-pip.py' self.install_script(context, 'pip', url) def main(args=None): diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index a481a3509d4ec8..fe11aabbcbdd6b 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -229,7 +229,7 @@ In regular release builds, the default warning filter has the following entries ignore::ImportWarning ignore::ResourceWarning -In debug builds, the list of default warning filters is empty. +In a :ref:`debug build `, the list of default warning filters is empty. .. versionchanged:: 3.2 :exc:`DeprecationWarning` is now ignored by default in addition to @@ -491,7 +491,7 @@ Available Functions Available Context Managers -------------------------- -.. class:: catch_warnings(\*, record=False, module=None) +.. class:: catch_warnings(*, record=False, module=None) A context manager that copies and, upon exit, restores the warnings filter and the :func:`showwarning` function. diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index d3c3a070f38af0..b88543e4453723 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -382,7 +382,7 @@ the referent is accessed:: class ExtendedRef(weakref.ref): def __init__(self, ob, callback=None, /, **annotations): - super(ExtendedRef, self).__init__(ob, callback) + super().__init__(ob, callback) self.__counter = 0 for k, v in annotations.items(): setattr(self, k, v) @@ -391,7 +391,7 @@ the referent is accessed:: """Return a pair containing the referent and the number of times the reference has been called. """ - ob = super(ExtendedRef, self).__call__() + ob = super().__call__() if ob is not None: self.__counter += 1 ob = (ob, self.__counter) diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst index dccb7db27e90cc..487856a3ac6c60 100644 --- a/Doc/library/winreg.rst +++ b/Doc/library/winreg.rst @@ -791,7 +791,7 @@ integer handle, and also disconnect the Windows handle from the handle object. .. method:: PyHKEY.__enter__() - PyHKEY.__exit__(\*exc_info) + PyHKEY.__exit__(*exc_info) The HKEY object implements :meth:`~object.__enter__` and :meth:`~object.__exit__` and thus supports the context protocol for the diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 2c78cd939243a8..bf72c46561b7c7 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -132,7 +132,7 @@ module documentation. This section lists the differences between the API and ... # Work with dom. -.. method:: Node.writexml(writer, indent="", addindent="", newl="", +.. method:: Node.writexml(writer, indent="", addindent="", newl="", \ encoding=None, standalone=None) Write XML to the writer object. The writer receives texts but not bytes as input, @@ -174,7 +174,7 @@ module documentation. This section lists the differences between the API and The :meth:`toxml` method now preserves the attribute order specified by the user. -.. method:: Node.toprettyxml(indent="\\t", newl="\\n", encoding=None, +.. method:: Node.toprettyxml(indent="\\t", newl="\\n", encoding=None, \ standalone=None) Return a pretty-printed version of the document. *indent* specifies the diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index f4bccf6609810e..87f4ee347d604d 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -455,6 +455,12 @@ Supported XPath syntax | | has the given value. The value cannot contain | | | quotes. | +-----------------------+------------------------------------------------------+ +| ``[@attrib!='value']``| Selects all elements for which the given attribute | +| | does not have the given value. The value cannot | +| | contain quotes. | +| | | +| | .. versionadded:: 3.10 | ++-----------------------+------------------------------------------------------+ | ``[tag]`` | Selects all elements that have a child named | | | ``tag``. Only immediate children are supported. | +-----------------------+------------------------------------------------------+ @@ -463,10 +469,22 @@ Supported XPath syntax | | | | | .. versionadded:: 3.7 | +-----------------------+------------------------------------------------------+ +| ``[.!='text']`` | Selects all elements whose complete text content, | +| | including descendants, does not equal the given | +| | ``text``. | +| | | +| | .. versionadded:: 3.10 | ++-----------------------+------------------------------------------------------+ | ``[tag='text']`` | Selects all elements that have a child named | | | ``tag`` whose complete text content, including | | | descendants, equals the given ``text``. | +-----------------------+------------------------------------------------------+ +| ``[tag!='text']`` | Selects all elements that have a child named | +| | ``tag`` whose complete text content, including | +| | descendants, does not equal the given ``text``. | +| | | +| | .. versionadded:: 3.10 | ++-----------------------+------------------------------------------------------+ | ``[position]`` | Selects all elements that are located at the given | | | position. The position can be either an integer | | | (1 is the first position), the expression ``last()`` | diff --git a/Doc/library/xml.sax.handler.rst b/Doc/library/xml.sax.handler.rst index 3746a58c9b9558..59d0d1b3b01175 100644 --- a/Doc/library/xml.sax.handler.rst +++ b/Doc/library/xml.sax.handler.rst @@ -50,7 +50,7 @@ module :mod:`xml.sax.handler`, so that all methods get default implementations. .. class:: LexicalHandler - Interface used by the parser to represent low freqency events which may not + Interface used by the parser to represent low frequency events which may not be of interest to many applications. In addition to these classes, :mod:`xml.sax.handler` provides symbolic constants diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index 32403819531aa8..51279b35fd71b7 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -378,7 +378,7 @@ Fault Objects .. attribute:: faultCode - A string indicating the fault type. + An int indicating the fault type. .. attribute:: faultString diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 7126d8bd703f62..3db55e646c47cc 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -483,7 +483,7 @@ Path Objects Path objects expose the following features of :mod:`pathlib.Path` objects: -Path objects are traversable using the ``/`` operator. +Path objects are traversable using the ``/`` operator or ``joinpath``. .. attribute:: Path.name @@ -532,6 +532,19 @@ Path objects are traversable using the ``/`` operator. Read the current file as bytes. +.. method:: Path.joinpath(*other) + + Return a new Path object with each of the *other* arguments + joined. The following are equivalent:: + + >>> Path(...).joinpath('child').joinpath('grandchild') + >>> Path(...).joinpath('child', 'grandchild') + >>> Path(...) / 'child' / 'grandchild' + + .. versionchanged:: 3.10 + Prior to 3.10, ``joinpath`` was undocumented and accepted + exactly one parameter. + .. _pyzipfile-objects: diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index 8ac3fb16bdb90d..cd7030fed01732 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -44,8 +44,9 @@ doesn't contain :file:`.pyc` files, importing may be rather slow. follows the specification in :pep:`273`, but uses an implementation written by Just van Rossum that uses the import hooks described in :pep:`302`. - :pep:`302` - New Import Hooks - The PEP to add the import hooks that help this module work. + :mod:`importlib` - The implementation of the import machinery + Package providing the relevant protocols for all importers to + implement. This module defines an exception: @@ -73,7 +74,31 @@ zipimporter Objects :exc:`ZipImportError` is raised if *archivepath* doesn't point to a valid ZIP archive. - .. method:: find_module(fullname[, path]) + .. method:: create_module(spec) + + Implementation of :meth:`importlib.abc.Loader.create_module` that returns + :const:`None` to explicitly request the default semantics. + + .. versionadded:: 3.10 + + + .. method:: exec_module(module) + + Implementation of :meth:`importlib.abc.Loader.exec_module`. + + .. versionadded:: 3.10 + + + .. method:: find_loader(fullname, path=None) + + An implementation of :meth:`importlib.abc.PathEntryFinder.find_loader`. + + .. deprecated:: 3.10 + + Use :meth:`find_spec` instead. + + + .. method:: find_module(fullname, path=None) Search for a module specified by *fullname*. *fullname* must be the fully qualified (dotted) module name. It returns the zipimporter instance itself @@ -81,11 +106,22 @@ zipimporter Objects *path* argument is ignored---it's there for compatibility with the importer protocol. + .. deprecated:: 3.10 + + Use :meth:`find_spec` instead. + + + .. method:: find_spec(fullname, target=None) + + An implementation of :meth:`importlib.abc.PathEntryFinder.find_spec`. + + .. versionadded:: 3.10 + .. method:: get_code(fullname) Return the code object for the specified module. Raise - :exc:`ZipImportError` if the module couldn't be found. + :exc:`ZipImportError` if the module couldn't be imported. .. method:: get_data(pathname) @@ -101,7 +137,7 @@ zipimporter Objects Return the value ``__file__`` would be set to if the specified module was imported. Raise :exc:`ZipImportError` if the module couldn't be - found. + imported. .. versionadded:: 3.1 @@ -123,8 +159,20 @@ zipimporter Objects .. method:: load_module(fullname) Load the module specified by *fullname*. *fullname* must be the fully - qualified (dotted) module name. It returns the imported module, or raises - :exc:`ZipImportError` if it wasn't found. + qualified (dotted) module name. Returns the imported module on success, + raises :exc:`ZipImportError` on failure. + + .. deprecated:: 3.10 + + Use :meth:`exec_module` instead. + + + .. method:: invalidate_caches() + + Clear out the internal cache of information about files found within + the ZIP archive. + + .. versionadded:: 3.10 .. attribute:: archive diff --git a/Doc/library/zoneinfo.rst b/Doc/library/zoneinfo.rst index 3a4c12a73acd72..1b2ba2af2ae6fb 100644 --- a/Doc/library/zoneinfo.rst +++ b/Doc/library/zoneinfo.rst @@ -124,8 +124,9 @@ time zone database (except on Windows, where there are no "well-known" locations for time zone data). On POSIX systems, downstream distributors and those building Python from source who know where their system time zone data is deployed may change the default time zone path by specifying -the compile-time option ``TZPATH`` (or, more likely, the ``configure`` flag -``--with-tzpath``), which should be a string delimited by :data:`os.pathsep`. +the compile-time option ``TZPATH`` (or, more likely, the :option:`configure +flag --with-tzpath <--with-tzpath>`), which should be a string delimited by +:data:`os.pathsep`. On all platforms, the configured value is available as the ``TZPATH`` key in :func:`sysconfig.get_config_var`. diff --git a/Doc/license.rst b/Doc/license.rst index 4030825bbd28ee..f487d98b2b43a3 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -100,7 +100,7 @@ PSF LICENSE AGREEMENT FOR PYTHON |release| analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python |release| alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of - copyright, i.e., "Copyright © 2001-2020 Python Software Foundation; All Rights + copyright, i.e., "Copyright © 2001-2021 Python Software Foundation; All Rights Reserved" are retained in Python |release| alone or in any derivative version prepared by Licensee. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 62986cb151964a..96bd9b028d5044 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -51,6 +51,7 @@ Summarizing: : | `for_stmt` : | `try_stmt` : | `with_stmt` + : | `match_stmt` : | `funcdef` : | `classdef` : | `async_with_stmt` @@ -254,7 +255,8 @@ present, must be last; it matches any exception. For an except clause with an expression, that expression is evaluated, and the clause matches the exception if the resulting object is "compatible" with the exception. An object is compatible with an exception if it is the class or a base class of the exception -object or a tuple containing an item compatible with the exception. +object, or a tuple containing an item that is the class or a base class of +the exception object. If no except clause matches the exception, the search for an exception handler continues in the surrounding code and on the invocation stack. [#]_ @@ -301,9 +303,27 @@ Before an except clause's suite is executed, details about the exception are stored in the :mod:`sys` module and can be accessed via :func:`sys.exc_info`. :func:`sys.exc_info` returns a 3-tuple consisting of the exception class, the exception instance and a traceback object (see section :ref:`types`) identifying -the point in the program where the exception occurred. :func:`sys.exc_info` -values are restored to their previous values (before the call) when returning -from a function that handled an exception. +the point in the program where the exception occurred. The details about the +exception accessed via :func:`sys.exc_info` are restored to their previous values +when leaving an exception handler:: + + >>> print(sys.exc_info()) + (None, None, None) + >>> try: + ... raise TypeError + ... except: + ... print(sys.exc_info()) + ... try: + ... raise ValueError + ... except: + ... print(sys.exc_info()) + ... print(sys.exc_info()) + ... + (, TypeError(), ) + (, ValueError(), ) + (, TypeError(), ) + >>> print(sys.exc_info()) + (None, None, None) .. index:: keyword: else @@ -392,7 +412,8 @@ This allows common :keyword:`try`...\ :keyword:`except`...\ :keyword:`finally` usage patterns to be encapsulated for convenient reuse. .. productionlist:: python-grammar - with_stmt: "with" `with_item` ("," `with_item`)* ":" `suite` + with_stmt: "with" ( "(" `with_stmt_contents` ","? ")" | `with_stmt_contents` ) ":" `suite` + with_stmt_contents: `with_item` ("," `with_item`)* with_item: `expression` ["as" `target`] The execution of the :keyword:`with` statement with one "item" proceeds as follows: @@ -469,15 +490,623 @@ is semantically equivalent to:: with B() as b: SUITE +You can also write multi-item context managers in multiple lines if +the items are surrounded by parentheses. For example:: + + with ( + A() as a, + B() as b, + ): + SUITE + .. versionchanged:: 3.1 Support for multiple context expressions. +.. versionchanged:: 3.10 + Support for using grouping parentheses to break the statement in multiple lines. + .. seealso:: :pep:`343` - The "with" statement The specification, background, and examples for the Python :keyword:`with` statement. +.. _match: + +The :keyword:`!match` statement +=============================== + +.. index:: + ! statement: match + ! keyword: case + ! single: pattern matching + keyword: if + keyword: as + pair: match; case + single: : (colon); compound statement + +.. versionadded:: 3.10 + +The match statement is used for pattern matching. Syntax: + +.. productionlist:: python-grammar + match_stmt: 'match' `subject_expr` ":" NEWLINE INDENT `case_block`+ DEDENT + subject_expr: `star_named_expression` "," `star_named_expressions`? + : | `named_expression` + case_block: 'case' `patterns` [`guard`] ":" `block` + +.. note:: + This section uses single quotes to denote + :ref:`soft keywords `. + +Pattern matching takes a pattern as input (following ``case``) and a subject +value (following ``match``). The pattern (which may contain subpatterns) is +matched against the subject value. The outcomes are: + +* A match success or failure (also termed a pattern success or failure). + +* Possible binding of matched values to a name. The prerequisites for this are + further discussed below. + +The ``match`` and ``case`` keywords are :ref:`soft keywords `. + +.. seealso:: + + * :pep:`634` -- Structural Pattern Matching: Specification + * :pep:`636` -- Structural Pattern Matching: Tutorial + + +Overview +-------- + +Here's an overview of the logical flow of a match statement: + + +#. The subject expression ``subject_expr`` is evaluated and a resulting subject + value obtained. If the subject expression contains a comma, a tuple is + constructed using :ref:`the standard rules `. + +#. Each pattern in a ``case_block`` is attempted to match with the subject value. The + specific rules for success or failure are described below. The match attempt can also + bind some or all of the standalone names within the pattern. The precise + pattern binding rules vary per pattern type and are + specified below. **Name bindings made during a successful pattern match + outlive the executed block and can be used after the match statement**. + + .. note:: + + During failed pattern matches, some subpatterns may succeed. Do not + rely on bindings being made for a failed match. Conversely, do not + rely on variables remaining unchanged after a failed match. The exact + behavior is dependent on implementation and may vary. This is an + intentional decision made to allow different implementations to add + optimizations. + +#. If the pattern succeeds, the corresponding guard (if present) is evaluated. In + this case all name bindings are guaranteed to have happened. + + * If the guard evaluates as truthy or missing, the ``block`` inside ``case_block`` is + executed. + + * Otherwise, the next ``case_block`` is attempted as described above. + + * If there are no further case blocks, the match statement is completed. + +.. note:: + + Users should generally never rely on a pattern being evaluated. Depending on + implementation, the interpreter may cache values or use other optimizations + which skip repeated evaluations. + +A sample match statement:: + + >>> flag = False + >>> match (100, 200): + ... case (100, 300): # Mismatch: 200 != 300 + ... print('Case 1') + ... case (100, 200) if flag: # Successful match, but guard fails + ... print('Case 2') + ... case (100, y): # Matches and binds y to 200 + ... print(f'Case 3, y: {y}') + ... case _: # Pattern not attempted + ... print('Case 4, I match anything!') + ... + Case 3, y: 200 + + +In this case, ``if flag`` is a guard. Read more about that in the next section. + +Guards +------ + +.. index:: ! guard + +.. productionlist:: python-grammar + guard: "if" `named_expression` + +A ``guard`` (which is part of the ``case``) must succeed for code inside +the ``case`` block to execute. It takes the form: :keyword:`if` followed by an +expression. + + +The logical flow of a ``case`` block with a ``guard`` follows: + +#. Check that the pattern in the ``case`` block succeeded. If the pattern + failed, the ``guard`` is not evaluated and the next ``case`` block is + checked. + +#. If the pattern succeeded, evaluate the ``guard``. + + * If the ``guard`` condition evaluates to "truthy", the case block is + selected. + + * If the ``guard`` condition evaluates to "falsy", the case block is not + selected. + + * If the ``guard`` raises an exception during evaluation, the exception + bubbles up. + +Guards are allowed to have side effects as they are expressions. Guard +evaluation must proceed from the first to the last case block, one at a time, +skipping case blocks whose pattern(s) don't all succeed. (I.e., +guard evaluation must happen in order.) Guard evaluation must stop once a case +block is selected. + + +.. _irrefutable_case: + +Irrefutable Case Blocks +----------------------- + +.. index:: irrefutable case block, case block + +An irrefutable case block is a match-all case block. A match statement may have +at most one irrefutable case block, and it must be last. + +A case block is considered irrefutable if it has no guard and its pattern is +irrefutable. A pattern is considered irrefutable if we can prove from its +syntax alone that it will always succeed. Only the following patterns are +irrefutable: + +* :ref:`as-patterns` whose left-hand side is irrefutable + +* :ref:`or-patterns` containing at least one irrefutable pattern + +* :ref:`capture-patterns` + +* :ref:`wildcard-patterns` + +* parenthesized irrefutable patterns + + +Patterns +-------- + +.. index:: + single: ! patterns + single: AS pattern, OR pattern, capture pattern, wildcard pattern + +.. note:: + This section uses grammar notations beyond standard EBNF: + + * the notation ``SEP.RULE+`` is shorthand for ``RULE (SEP RULE)*`` + + * the notation ``!RULE`` is shorthand for a negative lookahead assertion + + +The top-level syntax for ``patterns`` is: + +.. productionlist:: python-grammar + patterns: `open_sequence_pattern` | `pattern` + pattern: `as_pattern` | `or_pattern` + closed_pattern: | `literal_pattern` + : | `capture_pattern` + : | `wildcard_pattern` + : | `value_pattern` + : | `group_pattern` + : | `sequence_pattern` + : | `mapping_pattern` + : | `class_pattern` + +The descriptions below will include a description "in simple terms" of what a pattern +does for illustration purposes (credits to Raymond Hettinger for a document that +inspired most of the descriptions). Note that these descriptions are purely for +illustration purposes and **may not** reflect +the underlying implementation. Furthermore, they do not cover all valid forms. + + +.. _or-patterns: + +OR Patterns +^^^^^^^^^^^ + +An OR pattern is two or more patterns separated by vertical +bars ``|``. Syntax: + +.. productionlist:: python-grammar + or_pattern: "|".`closed_pattern`+ + +Only the final subpattern may be :ref:`irrefutable `, and each +subpattern must bind the same set of names to avoid ambiguity. + +An OR pattern matches each of its subpatterns in turn to the subject value, +until one succeeds. The OR pattern is then considered successful. Otherwise, +if none of the subpatterns succeed, the OR pattern fails. + +In simple terms, ``P1 | P2 | ...`` will try to match ``P1``, if it fails it will try to +match ``P2``, succeeding immediately if any succeeds, failing otherwise. + +.. _as-patterns: + +AS Patterns +^^^^^^^^^^^ + +An AS pattern matches an OR pattern on the left of the :keyword:`as` +keyword against a subject. Syntax: + +.. productionlist:: python-grammar + as_pattern: `or_pattern` "as" `capture_pattern` + +If the OR pattern fails, the AS pattern fails. Otherwise, the AS pattern binds +the subject to the name on the right of the as keyword and succeeds. +``capture_pattern`` cannot be a a ``_``. + +In simple terms ``P as NAME`` will match with ``P``, and on success it will +set ``NAME = ``. + + +.. _literal-patterns: + +Literal Patterns +^^^^^^^^^^^^^^^^ + +A literal pattern corresponds to most +:ref:`literals ` in Python. Syntax: + +.. productionlist:: python-grammar + literal_pattern: `signed_number` + : | `signed_number` "+" NUMBER + : | `signed_number` "-" NUMBER + : | `strings` + : | "None" + : | "True" + : | "False" + : | `signed_number`: NUMBER | "-" NUMBER + +The rule ``strings`` and the token ``NUMBER`` are defined in the +:doc:`standard Python grammar <./grammar>`. Triple-quoted strings are +supported. Raw strings and byte strings are supported. :ref:`f-strings` are +not supported. + +The forms ``signed_number '+' NUMBER`` and ``signed_number '-' NUMBER`` are +for expressing :ref:`complex numbers `; they require a real number +on the left and an imaginary number on the right. E.g. ``3 + 4j``. + +In simple terms, ``LITERAL`` will succeed only if `` == LITERAL``. For +the singletons ``None``, ``True`` and ``False``, the :keyword:`is` operator is used. + +.. _capture-patterns: + +Capture Patterns +^^^^^^^^^^^^^^^^ + +A capture pattern binds the subject value to a name. +Syntax: + +.. productionlist:: python-grammar + capture_pattern: !'_' NAME + +A single underscore ``_`` is not a capture pattern (this is what ``!'_'`` +expresses). And is instead treated as a :token:`wildcard_pattern`. + +In a given pattern, a given name can only be bound once. E.g. +``case x, x: ...`` is invalid while ``case [x] | x: ...`` is allowed. + +Capture patterns always succeed. The binding follows scoping rules +established by the assignment expression operator in :pep:`572`; the +name becomes a local variable in the closest containing function scope unless +there's an applicable :keyword:`global` or :keyword:`nonlocal` statement. + +In simple terms ``NAME`` will always succeed and it will set ``NAME = ``. + +.. _wildcard-patterns: + +Wildcard Patterns +^^^^^^^^^^^^^^^^^ + +A wildcard pattern always succeeds (matches anything) +and binds no name. Syntax: + +.. productionlist:: python-grammar + wildcard_pattern: '_' + +``_`` is a :ref:`soft keyword `. + +In simple terms, ``_`` will always succeed. + +.. _value-patterns: + +Value Patterns +^^^^^^^^^^^^^^ + +A value pattern represents a named value in Python. +Syntax: + +.. productionlist:: python-grammar + value_pattern: `attr` + attr: `name_or_attr` "." NAME + name_or_attr: `attr` | NAME + +The dotted name in the pattern is looked up using standard Python +:ref:`name resolution rules `. The pattern succeeds if the +value found compares equal to the subject value (using the ``==`` equality +operator). + +In simple terms ``NAME1.NAME2`` will succeed only if `` == NAME1.NAME2`` + +.. note:: + + If the same value occurs multiple times in the same match statement, the + interpreter may cache the first value found and reuse it rather than repeat + the same lookup. This cache is strictly tied to a given execution of a + given match statement. + +.. _group-patterns: + +Group Patterns +^^^^^^^^^^^^^^ + +A group pattern allows users to add parentheses around patterns to +emphasize the intended grouping. Otherwise, it has no additional syntax. +Syntax: + +.. productionlist:: python-grammar + group_pattern: "(" `pattern` ")" + +In simple terms ``(P)`` has the same effect as ``P``. + +.. _sequence-patterns: + +Sequence Patterns +^^^^^^^^^^^^^^^^^ + +A sequence pattern contains several subpatterns to be matched against sequence elements. +The syntax is similar to the unpacking of a list or tuple. + +.. productionlist:: python-grammar + sequence_pattern: "[" [`maybe_sequence_pattern`] "]" + : | "(" [`open_sequence_pattern`] ")" + open_sequence_pattern: `maybe_star_pattern` "," [`maybe_sequence_pattern`] + maybe_sequence_pattern: ",".`maybe_star_pattern`+ ","? + maybe_star_pattern: `star_pattern` | `pattern` + star_pattern: "*" (`capture_pattern` | `wildcard_pattern`) + +There is no difference if parentheses or square brackets +are used for sequence patterns (i.e. ``(...)`` vs ``[...]`` ). + +.. note:: + A single pattern enclosed in parentheses without a trailing comma + (e.g. ``(3 | 4)``) is a :ref:`group pattern `. + While a single pattern enclosed in square brackets (e.g. ``[3 | 4]``) is + still a sequence pattern. + +At most one star subpattern may be in a sequence pattern. The star subpattern +may occur in any position. If no star subpattern is present, the sequence +pattern is a fixed-length sequence pattern; otherwise it is a variable-length +sequence pattern. + +The following is the logical flow for matching a sequence pattern against a +subject value: + +#. If the subject value is not an instance of a + :class:`collections.abc.Sequence` the sequence pattern fails. + +#. If the subject value is an instance of ``str``, ``bytes`` or ``bytearray`` + the sequence pattern fails. + +#. The subsequent steps depend on whether the sequence pattern is fixed or + variable-length. + + If the sequence pattern is fixed-length: + + #. If the length of the subject sequence is not equal to the number of + subpatterns, the sequence pattern fails + + #. Subpatterns in the sequence pattern are matched to their corresponding + items in the subject sequence from left to right. Matching stops as soon + as a subpattern fails. If all subpatterns succeed in matching their + corresponding item, the sequence pattern succeeds. + + Otherwise, if the sequence pattern is variable-length: + + #. If the length of the subject sequence is less than the number of non-star + subpatterns, the sequence pattern fails. + + #. The leading non-star subpatterns are matched to their corresponding items + as for fixed-length sequences. + + #. If the previous step succeeds, the star subpattern matches a list formed + of the remaining subject items, excluding the remaining items + corresponding to non-star subpatterns following the star subpattern. + + #. Remaining non-star subpatterns are matched to their corresponding subject + items, as for a fixed-length sequence. + + .. note:: The length of the subject sequence is obtained via + :func:`len` (i.e. via the :meth:`__len__` protocol). This length may be + cached by the interpreter in a similar manner as + :ref:`value patterns `. + + +In simple terms ``[P1, P2, P3,`` ... ``, P]`` matches only if all the following +happens: + +* ``isinstance(, collections.abc.Sequence)`` +* ``len(subject) == `` +* ``P1`` matches ``[0]`` (note that this match can also bind names) +* ``P2`` matches ``[1]`` (note that this match can also bind names) +* ... and so on for the corresponding pattern/element. + +.. _mapping-patterns: + +Mapping Patterns +^^^^^^^^^^^^^^^^ + +A mapping pattern contains one or more key-value patterns. The syntax is +similar to the construction of a dictionary. +Syntax: + +.. productionlist:: python-grammar + mapping_pattern: "{" [`items_pattern`] "}" + items_pattern: ",".`key_value_pattern`+ ","? + key_value_pattern: (`literal_pattern` | `value_pattern`) ":" `pattern` + : | `double_star_pattern` + double_star_pattern: "**" `capture_pattern` + +At most one double star pattern may be in a mapping pattern. The double star +pattern must be the last subpattern in the mapping pattern. + +Duplicate key values in mapping patterns are disallowed. (If all key patterns +are literal patterns this is considered a syntax error; otherwise this is a +runtime error and will raise :exc:`ValueError`.) + +The following is the logical flow for matching a mapping pattern against a +subject value: + +#. If the subject value is not an instance of :class:`collections.abc.Mapping`, + the mapping pattern fails. + +#. If every key given in the mapping pattern is present in the subject mapping, + and the pattern for each key matches the corresponding item of the subject + mapping, the mapping pattern succeeds. + +#. If duplicate keys are detected in the mapping pattern, the pattern is + considered invalid and :exc:`ValueError` is raised. + +.. note:: Key-value pairs are matched using the two-argument form of the mapping + subject's ``get()`` method. Matched key-value pairs must already be present + in the mapping, and not created on-the-fly via :meth:`__missing__` or + :meth:`__getitem__`. + +In simple terms ``{KEY1: P1, KEY2: P2, ... }`` matches only if all the following +happens: + +* ``isinstance(, collections.abc.Mapping)`` +* ``KEY1 in `` +* ``P1`` matches ``[KEY1]`` +* ... and so on for the corresponding KEY/pattern pair. + + +.. _class-patterns: + +Class Patterns +^^^^^^^^^^^^^^ + +A class pattern represents a class and its positional and keyword arguments +(if any). Syntax: + +.. productionlist:: python-grammar + class_pattern: `name_or_attr` "(" [`pattern_arguments` ","?] ")" + pattern_arguments: `positional_patterns` ["," `keyword_patterns`] + : | `keyword_patterns` + positional_patterns: ",".`pattern`+ + keyword_patterns: ",".`keyword_pattern`+ + keyword_pattern: NAME "=" `pattern` + +The same keyword should not be repeated in class patterns. + +The following is the logical flow for matching a mapping pattern against a +subject value: + +#. If ``name_or_attr`` is not an instance of the builtin :class:`type` , raise + :exc:`TypeError`. + +#. If the subject value is not an instance of ``name_or_attr`` (tested via + :func:`isinstance`), the class pattern fails. + +#. If no pattern arguments are present, the pattern succeeds. Otherwise, + the subsequent steps depend on whether keyword or positional argument patterns + are present. + + For a number of built-in types (specified below), a single positional + subpattern is accepted which will match the entire subject; for these types + keyword patterns also work as for other types. + + If only keyword patterns are present, they are processed as follows, + one by one: + + I. The keyword is looked up as an attribute on the subject. + + * If this raises an exception other than :exc:`AttributeError`, the + exception bubbles up. + + * If this raises :exc:`AttributeError`, the class pattern has failed. + + * Else, the subpattern associated with the keyword pattern is matched + against the subject's attribute value. If this fails, the class + pattern fails; if this succeeds, the match proceeds to the next keyword. + + + II. If all keyword patterns succeed, the class pattern succeeds. + + If any positional patterns are present, they are converted to keyword + patterns using the :data:`~object.__match_args__` attribute on the class + ``name_or_attr`` before matching: + + I. The equivalent of ``getattr(cls, "__match_args__", ()))`` is called. + + * If this raises an exception, the exception bubbles up. + + * If the returned value is not a tuple, the conversion fails and + :exc:`TypeError` is raised. + + * If there are more positional patterns than ``len(cls.__match_args__)``, + :exc:`TypeError` is raised. + + * Otherwise, positional pattern ``i`` is converted to a keyword pattern + using ``__match_args__[i]`` as the keyword. ``__match_args__[i]`` must + be a string; if not :exc:`TypeError` is raised. + + * If there are duplicate keywords, :exc:`TypeError` is raised. + + .. seealso:: :ref:`class-pattern-matching` + + II. Once all positional patterns have been converted to keyword patterns, + the match proceeds as if there were only keyword patterns. + + For the following built-in types the handling of positional subpatterns is + different: + + * :class:`bool` + * :class:`bytearray` + * :class:`bytes` + * :class:`dict` + * :class:`float` + * :class:`frozenset` + * :class:`int` + * :class:`list` + * :class:`set` + * :class:`str` + * :class:`tuple` + + These classes accept a single positional argument, and the pattern there is matched + against the whole object rather than an attribute. For example ``int(0|1)`` matches + the value ``0``, but not the values ``0.0`` or ``False``. + +In simple terms ``CLS(P1, attr=P2)`` matches only if the following happens: + +* ``isinstance(, CLS)`` +* convert ``P1`` to a keyword pattern using ``CLS.__match_args__`` +* For each keyword argument ``attr=P2``: + * ``hasattr(, "attr")`` + * ``P2`` matches ``.attr`` +* ... and so on for the corresponding keyword argument/pattern pair. + +.. seealso:: + + * :pep:`634` -- Structural Pattern Matching: Specification + * :pep:`636` -- Structural Pattern Matching: Tutorial + .. index:: single: parameter; function definition @@ -509,7 +1138,6 @@ A function definition defines a user-defined function object (see section : ["->" `expression`] ":" `suite` decorators: `decorator`+ decorator: "@" `assignment_expression` NEWLINE - dotted_name: `identifier` ("." `identifier`)* parameter_list: `defparameter` ("," `defparameter`)* "," "/" ["," [`parameter_list_no_posonly`]] : | `parameter_list_no_posonly` parameter_list_no_posonly: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] @@ -586,19 +1214,25 @@ e.g.:: return penguin .. index:: + single: / (slash); function definition single: * (asterisk); function definition single: **; function definition Function call semantics are described in more detail in section :ref:`calls`. A function call always assigns values to all parameters mentioned in the parameter -list, either from position arguments, from keyword arguments, or from default +list, either from positional arguments, from keyword arguments, or from default values. If the form "``*identifier``" is present, it is initialized to a tuple receiving any excess positional parameters, defaulting to the empty tuple. If the form "``**identifier``" is present, it is initialized to a new ordered mapping receiving any excess keyword arguments, defaulting to a new empty mapping of the same type. Parameters after "``*``" or "``*identifier``" are keyword-only parameters and may only be passed -used keyword arguments. +by keyword arguments. Parameters before "``/``" are positional-only parameters +and may only be passed by positional arguments. + +.. versionchanged:: 3.8 + The ``/`` function parameter syntax may be used to indicate positional-only + parameters. See :pep:`570` for details. .. index:: pair: function; annotations @@ -610,9 +1244,13 @@ following the parameter name. Any parameter may have an annotation, even those ``*identifier`` or ``**identifier``. Functions may have "return" annotation of the form "``-> expression``" after the parameter list. These annotations can be any valid Python expression. The presence of annotations does not change the -semantics of a function. The annotation values are available as string values -in a dictionary keyed by the parameters' names in the :attr:`__annotations__` -attribute of the function object. +semantics of a function. The annotation values are available as values of +a dictionary keyed by the parameters' names in the :attr:`__annotations__` +attribute of the function object. If the ``annotations`` import from +:mod:`__future__` is used, annotations are preserved as strings at runtime which +enables postponed evaluation. Otherwise, they are evaluated when the function +definition is executed. In this case annotations may be evaluated in +a different order than they appear in the source code. .. index:: pair: lambda; expression @@ -796,12 +1434,12 @@ The :keyword:`!async for` statement .. productionlist:: python-grammar async_for_stmt: "async" `for_stmt` -An :term:`asynchronous iterable` is able to call asynchronous code in its -*iter* implementation, and :term:`asynchronous iterator` can call asynchronous -code in its *next* method. +An :term:`asynchronous iterable` provides an ``__aiter__`` method that directly +returns an :term:`asynchronous iterator`, which can call asynchronous code in +its ``__anext__`` method. The ``async for`` statement allows convenient iteration over asynchronous -iterators. +iterables. The following code:: diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index d92e19761a9654..eefdc3d5100b56 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -217,7 +217,6 @@ Ellipsis There are two types of integers: Integers (:class:`int`) - These represent numbers in an unlimited range, subject to available (virtual) memory only. For the purpose of shift and mask operations, a binary representation is assumed, and negative numbers are represented in a variant of @@ -554,7 +553,10 @@ Callable types | | the dict are the parameter | | | | names, and ``'return'`` for | | | | the return annotation, if | | - | | provided. | | + | | provided. For more | | + | | information on working with | | + | | this attribute, see | | + | | :ref:`annotations-howto`. | | +-------------------------+-------------------------------+-----------+ | :attr:`__kwdefaults__` | A dict containing defaults | Writable | | | for keyword-only parameters. | | @@ -749,16 +751,29 @@ Modules single: __annotations__ (module attribute) pair: module; namespace - Predefined (writable) attributes: :attr:`__name__` is the module's name; - :attr:`__doc__` is the module's documentation string, or ``None`` if - unavailable; :attr:`__annotations__` (optional) is a dictionary containing - :term:`variable annotations ` collected during module - body execution; :attr:`__file__` is the pathname of the file from which the - module was loaded, if it was loaded from a file. The :attr:`__file__` - attribute may be missing for certain types of modules, such as C modules - that are statically linked into the interpreter; for extension modules - loaded dynamically from a shared library, it is the pathname of the shared - library file. + Predefined (writable) attributes: + + :attr:`__name__` + The module's name. + + :attr:`__doc__` + The module's documentation string, or ``None`` if + unavailable. + + :attr:`__file__` + The pathname of the file from which the + module was loaded, if it was loaded from a file. + The :attr:`__file__` + attribute may be missing for certain types of modules, such as C modules + that are statically linked into the interpreter. For extension modules + loaded dynamically from a shared library, it's the pathname of the shared + library file. + + :attr:`__annotations__` + A dictionary containing + :term:`variable annotations ` collected during + module body execution. For best practices on working + with :attr:`__annotations__`, please see :ref:`annotations-howto`. .. index:: single: __dict__ (module attribute) @@ -822,14 +837,30 @@ Custom classes single: __doc__ (class attribute) single: __annotations__ (class attribute) - Special attributes: :attr:`~definition.__name__` is the class name; :attr:`__module__` is - the module name in which the class was defined; :attr:`~object.__dict__` is the - dictionary containing the class's namespace; :attr:`~class.__bases__` is a - tuple containing the base classes, in the order of their occurrence in the - base class list; :attr:`__doc__` is the class's documentation string, - or ``None`` if undefined; :attr:`__annotations__` (optional) is a dictionary - containing :term:`variable annotations ` collected during - class body execution. + Special attributes: + + :attr:`~definition.__name__` + The class name. + + :attr:`__module__` + The name of the module in which the class was defined. + + :attr:`~object.__dict__` + The dictionary containing the class's namespace. + + :attr:`~class.__bases__` + A tuple containing the base classes, in the order of + their occurrence in the base class list. + + :attr:`__doc__` + The class's documentation string, or ``None`` if undefined. + + :attr:`__annotations__` + A dictionary containing + :term:`variable annotations ` + collected during class body execution. For best practices on + working with :attr:`__annotations__`, please see + :ref:`annotations-howto`. Class instances .. index:: @@ -1006,6 +1037,9 @@ Internal types :attr:`f_lasti` gives the precise instruction (this is an index into the bytecode string of the code object). + Accessing ``f_code`` raises an :ref:`auditing event ` + ``object.__getattr__`` with arguments ``obj`` and ``"f_code"``. + .. index:: single: f_trace (frame attribute) single: f_trace_lines (frame attribute) @@ -1090,6 +1124,9 @@ Internal types :keyword:`try` statement with no matching except clause or with a finally clause. + Accessing ``tb_frame`` raises an :ref:`auditing event ` + ``object.__getattr__`` with arguments ``obj`` and ``"tb_frame"``. + .. index:: single: tb_next (traceback attribute) @@ -1133,9 +1170,8 @@ Internal types around any other object, usually a user-defined method object. When a static method object is retrieved from a class or a class instance, the object actually returned is the wrapped object, which is not subject to any further - transformation. Static method objects are not themselves callable, although the - objects they wrap usually are. Static method objects are created by the built-in - :func:`staticmethod` constructor. + transformation. Static method objects are also callable. Static method + objects are created by the built-in :func:`staticmethod` constructor. Class method objects A class method object, like a static method object, is a wrapper around another @@ -1774,7 +1810,7 @@ Super Binding immediately preceding ``B`` and then invokes the descriptor with the call: ``A.__dict__['m'].__get__(obj, obj.__class__)``. -For instance bindings, the precedence of descriptor invocation depends on the +For instance bindings, the precedence of descriptor invocation depends on which descriptor methods are defined. A descriptor can define any combination of :meth:`__get__`, :meth:`__set__` and :meth:`__delete__`. If it does not define :meth:`__get__`, then accessing the attribute will return the descriptor @@ -2168,7 +2204,7 @@ Emulating callable objects .. index:: pair: call; instance Called when the instance is "called" as a function; if this method is defined, - ``x(arg1, arg2, ...)`` is a shorthand for ``x.__call__(arg1, arg2, ...)``. + ``x(arg1, arg2, ...)`` roughly translates to ``type(x).__call__(x, arg1, ...)``. .. _sequence-types: @@ -2554,6 +2590,38 @@ For more information on context managers, see :ref:`typecontextmanager`. statement. +.. _class-pattern-matching: + +Customizing positional arguments in class pattern matching +---------------------------------------------------------- + +When using a class name in a pattern, positional arguments in the pattern are not +allowed by default, i.e. ``case MyClass(x, y)`` is typically invalid without special +support in ``MyClass``. To be able to use that kind of patterns, the class needs to +define a *__match_args__* attribute. + +.. data:: object.__match_args__ + + This class variable can be assigned a tuple of strings. When this class is + used in a class pattern with positional arguments, each positional argument will + be converted into a keyword argument, using the corresponding value in + *__match_args__* as the keyword. The absence of this attribute is equivalent to + setting it to ``()``. + +For example, if ``MyClass.__match_args__`` is ``("left", "center", "right")`` that means +that ``case MyClass(x, y)`` is equivalent to ``case MyClass(left=x, center=y)``. Note +that the number of arguments in the pattern must be smaller than or equal to the number +of elements in *__match_args__*; if it is larger, the pattern match attempt will raise +a :exc:`TypeError`. + +.. versionadded:: 3.10 + +.. seealso:: + + :pep:`634` - Structural Pattern Matching + The specification for the Python ``match`` statement. + + .. _special-lookup: Special method lookup diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 938a9732f5a250..5ad640ad35d615 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -77,6 +77,8 @@ When the name is bound to an object, evaluation of the atom yields that object. When a name is not bound, an attempt to evaluate it raises a :exc:`NameError` exception. +.. _private-name-mangling: + .. index:: pair: name; mangling pair: private; names @@ -183,7 +185,7 @@ Common syntax elements for comprehensions are: comprehension: `assignment_expression` `comp_for` comp_for: ["async"] "for" `target_list` "in" `or_test` [`comp_iter`] comp_iter: `comp_for` | `comp_if` - comp_if: "if" `expression_nocond` [`comp_iter`] + comp_if: "if" `or_test` [`comp_iter`] The comprehension consists of a single expression followed by at least one :keyword:`!for` clause and zero or more :keyword:`!for` or :keyword:`!if` clauses. @@ -476,8 +478,8 @@ allowing any pending :keyword:`finally` clauses to execute. .. index:: single: from; yield from expression -When ``yield from `` is used, it treats the supplied expression as -a subiterator. All values produced by that subiterator are passed directly +When ``yield from `` is used, the supplied expression must be an +iterable. The values produced by iterating that iterable are passed directly to the caller of the current generator's methods. Any values passed in with :meth:`~generator.send` and any exceptions passed in with :meth:`~generator.throw` are passed to the underlying iterator if it has the @@ -643,6 +645,16 @@ after resuming depends on the method which resumed the execution. If :meth:`~agen.asend` is used, then the result will be the value passed in to that method. +If an asynchronous generator happens to exit early by :keyword:`break`, the caller +task being cancelled, or other exceptions, the generator's async cleanup code +will run and possibly raise exceptions or access context variables in an +unexpected context--perhaps after the lifetime of tasks it depends, or +during the event loop shutdown when the async-generator garbage collection hook +is called. +To prevent this, the caller must explicitly close the async generator by calling +:meth:`~agen.aclose` method to finalize the generator and ultimately detach it +from the event loop. + In an asynchronous generator function, yield expressions are allowed anywhere in a :keyword:`try` construct. However, if an asynchronous generator is not resumed before it is finalized (by reaching a zero reference count or by @@ -654,9 +666,9 @@ generator-iterator's :meth:`~agen.aclose` method and run the resulting coroutine object, thus allowing any pending :keyword:`!finally` clauses to execute. -To take care of finalization, an event loop should define -a *finalizer* function which takes an asynchronous generator-iterator -and presumably calls :meth:`~agen.aclose` and executes the coroutine. +To take care of finalization upon event loop termination, an event loop should +define a *finalizer* function which takes an asynchronous generator-iterator and +presumably calls :meth:`~agen.aclose` and executes the coroutine. This *finalizer* may be registered by calling :func:`sys.set_asyncgen_hooks`. When first iterated over, an asynchronous generator-iterator will store the registered *finalizer* to be called upon finalization. For a reference example @@ -837,14 +849,8 @@ this method will need to explicitly add that support. A string's items are characters. A character is not a separate data type but a string of exactly one character. -.. - At the time of writing this, there is no documentation for generic alias - or PEP 585. Thus the link currently points to PEP 585 itself. - Please change the link for generic alias to reference the correct - documentation once documentation for PEP 585 becomes available. - Subscription of certain :term:`classes ` or :term:`types ` -creates a `generic alias `_. +creates a :ref:`generic alias `. In this case, user-defined classes can support subscription by providing a :meth:`__class_getitem__` classmethod. @@ -1701,7 +1707,6 @@ Conditional expressions .. productionlist:: python-grammar conditional_expression: `or_test` ["if" `or_test` "else" `expression`] expression: `conditional_expression` | `lambda_expr` - expression_nocond: `or_test` | `lambda_expr_nocond` Conditional expressions (sometimes called a "ternary operator") have the lowest priority of all Python operations. @@ -1727,7 +1732,6 @@ Lambdas .. productionlist:: python-grammar lambda_expr: "lambda" [`parameter_list`] ":" `expression` - lambda_expr_nocond: "lambda" [`parameter_list`] ":" `expression_nocond` Lambda expressions (sometimes called lambda forms) are used to create anonymous functions. The expression ``lambda parameters: expression`` yields a function @@ -1815,8 +1819,8 @@ Operator precedence .. index:: pair: operator; precedence -The following table summarizes the operator precedence in Python, from lowest -precedence (least binding) to highest precedence (most binding). Operators in +The following table summarizes the operator precedence in Python, from highest +precedence (most binding) to lowest precedence (least binding). Operators in the same box have the same precedence. Unless the syntax is explicitly given, operators are binary. Operators in the same box group left to right (except for exponentiation, which groups from right to left). @@ -1829,50 +1833,50 @@ precedence and have a left-to-right chaining feature as described in the +-----------------------------------------------+-------------------------------------+ | Operator | Description | +===============================================+=====================================+ -| ``:=`` | Assignment expression | -+-----------------------------------------------+-------------------------------------+ -| :keyword:`lambda` | Lambda expression | +| ``(expressions...)``, | Binding or parenthesized | +| | expression, | +| ``[expressions...]``, | list display, | +| ``{key: value...}``, | dictionary display, | +| ``{expressions...}`` | set display | +-----------------------------------------------+-------------------------------------+ -| :keyword:`if ` -- :keyword:`!else` | Conditional expression | +| ``x[index]``, ``x[index:index]``, | Subscription, slicing, | +| ``x(arguments...)``, ``x.attribute`` | call, attribute reference | +-----------------------------------------------+-------------------------------------+ -| :keyword:`or` | Boolean OR | +| :keyword:`await` ``x`` | Await expression | +-----------------------------------------------+-------------------------------------+ -| :keyword:`and` | Boolean AND | +| ``**`` | Exponentiation [#]_ | +-----------------------------------------------+-------------------------------------+ -| :keyword:`not` ``x`` | Boolean NOT | +| ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT | +-----------------------------------------------+-------------------------------------+ -| :keyword:`in`, :keyword:`not in`, | Comparisons, including membership | -| :keyword:`is`, :keyword:`is not`, ``<``, | tests and identity tests | -| ``<=``, ``>``, ``>=``, ``!=``, ``==`` | | +| ``*``, ``@``, ``/``, ``//``, ``%`` | Multiplication, matrix | +| | multiplication, division, floor | +| | division, remainder [#]_ | +-----------------------------------------------+-------------------------------------+ -| ``|`` | Bitwise OR | +| ``+``, ``-`` | Addition and subtraction | +-----------------------------------------------+-------------------------------------+ -| ``^`` | Bitwise XOR | +| ``<<``, ``>>`` | Shifts | +-----------------------------------------------+-------------------------------------+ | ``&`` | Bitwise AND | +-----------------------------------------------+-------------------------------------+ -| ``<<``, ``>>`` | Shifts | +| ``^`` | Bitwise XOR | +-----------------------------------------------+-------------------------------------+ -| ``+``, ``-`` | Addition and subtraction | +| ``|`` | Bitwise OR | +-----------------------------------------------+-------------------------------------+ -| ``*``, ``@``, ``/``, ``//``, ``%`` | Multiplication, matrix | -| | multiplication, division, floor | -| | division, remainder [#]_ | +| :keyword:`in`, :keyword:`not in`, | Comparisons, including membership | +| :keyword:`is`, :keyword:`is not`, ``<``, | tests and identity tests | +| ``<=``, ``>``, ``>=``, ``!=``, ``==`` | | +-----------------------------------------------+-------------------------------------+ -| ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT | +| :keyword:`not` ``x`` | Boolean NOT | +-----------------------------------------------+-------------------------------------+ -| ``**`` | Exponentiation [#]_ | +| :keyword:`and` | Boolean AND | +-----------------------------------------------+-------------------------------------+ -| :keyword:`await` ``x`` | Await expression | +| :keyword:`or` | Boolean OR | +-----------------------------------------------+-------------------------------------+ -| ``x[index]``, ``x[index:index]``, | Subscription, slicing, | -| ``x(arguments...)``, ``x.attribute`` | call, attribute reference | +| :keyword:`if ` -- :keyword:`!else` | Conditional expression | +-----------------------------------------------+-------------------------------------+ -| ``(expressions...)``, | Binding or parenthesized | -| | expression, | -| ``[expressions...]``, | list display, | -| ``{key: value...}``, | dictionary display, | -| ``{expressions...}`` | set display | +| :keyword:`lambda` | Lambda expression | ++-----------------------------------------------+-------------------------------------+ +| ``:=`` | Assignment expression | +-----------------------------------------------+-------------------------------------+ @@ -1916,8 +1920,8 @@ precedence and have a left-to-right chaining feature as described in the the :keyword:`is` operator, like those involving comparisons between instance methods, or constants. Check their documentation for more info. -.. [#] The ``%`` operator is also used for string formatting; the same - precedence applies. - .. [#] The power operator ``**`` binds less tightly than an arithmetic or bitwise unary operator on its right, that is, ``2**-1`` is ``0.5``. + +.. [#] The ``%`` operator is also used for string formatting; the same + precedence applies. diff --git a/Doc/reference/grammar.rst b/Doc/reference/grammar.rst index acf83765b5796c..59b45005836a76 100644 --- a/Doc/reference/grammar.rst +++ b/Doc/reference/grammar.rst @@ -13,7 +13,8 @@ In particular, ``&`` followed by a symbol, token or parenthesized group indicates a positive lookahead (i.e., is required to match but not consumed), while ``!`` indicates a negative lookahead (i.e., is required _not_ to match). We use the ``|`` separator to mean PEG's -"ordered choice" (written as ``/`` in traditional PEG grammars). +"ordered choice" (written as ``/`` in traditional PEG grammars). See +:pep:`617` for more details on the grammar's syntax. .. literalinclude:: ../../Grammar/python.gram :language: peg diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index c5952426fdcf8d..2ef5b901b930bf 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -329,6 +329,10 @@ modules, and one that knows how to import modules from an :term:`import path` import machinery will try it only if the finder does not implement ``find_spec()``. +.. versionchanged:: 3.10 + Use of :meth:`~importlib.abc.MetaPathFinder.find_module` by the import system + now raises :exc:`ImportWarning`. + Loading ======= @@ -470,6 +474,9 @@ import machinery will create the new module itself. An :exc:`ImportError` is raised when ``exec_module()`` is defined but ``create_module()`` is not. +.. versionchanged:: 3.10 + Use of ``load_module()`` will raise :exc:`ImportWarning`. + Submodules ---------- @@ -675,6 +682,13 @@ Here are the exact rules used: :meth:`~importlib.abc.Loader.module_repr` method, if defined, before trying either approach described above. However, the method is deprecated. +.. versionchanged:: 3.10 + + Calling :meth:`~importlib.abc.Loader.module_repr` now occurs after trying to + use a module's ``__spec__`` attribute but before falling back on + ``__file__``. Use of :meth:`~importlib.abc.Loader.module_repr` is slated to + stop in Python 3.12. + .. _pyc-invalidation: Cached bytecode invalidation @@ -889,6 +903,11 @@ a list containing the portion. exist on a path entry finder, the import system will always call ``find_loader()`` in preference to ``find_module()``. +.. versionchanged:: 3.10 + Calls to :meth:`~importlib.abc.PathEntryFinder.find_module` and + :meth:`~importlib.abc.PathEntryFinder.find_loader` by the import + system will raise :exc:`ImportWarning`. + Replacing the standard import system ==================================== diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 77e0578f5d89b6..4ad8f8be1e7ddf 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -351,6 +351,27 @@ exactly as written here: assert del global not with async elif if or yield + +.. _soft-keywords: + +Soft Keywords +------------- + +.. index:: soft keyword, keyword + +.. versionadded:: 3.10 + +Some identifiers are only reserved under specific contexts. These are known as +*soft keywords*. The identifiers ``match``, ``case`` and ``_`` can +syntactically act as keywords in contexts related to the pattern matching +statement, but this distinction is done at the parser level, not when +tokenizing. + +As soft keywords, their use with pattern matching is possible while still +preserving compatibility with existing code that uses ``match``, ``case`` and ``_`` as +identifier names. + + .. index:: single: _, identifiers single: __, identifiers diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index f8ab2e918c6a10..3aba2700048bc0 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -591,10 +591,13 @@ instance, with its traceback set to its argument), like so:: __context__ (exception attribute) The ``from`` clause is used for exception chaining: if given, the second -*expression* must be another exception class or instance, which will then be -attached to the raised exception as the :attr:`__cause__` attribute (which is -writable). If the raised exception is not handled, both exceptions will be -printed:: +*expression* must be another exception class or instance. If the second +expression is an exception instance, it will be attached to the raised +exception as the :attr:`__cause__` attribute (which is writable). If the +expression is an exception class, the class will be instantiated and the +resulting exception instance will be attached to the raised exception as the +:attr:`__cause__` attribute. If the raised exception is not handled, both +exceptions will be printed:: >>> try: ... print(1 / 0) @@ -950,7 +953,7 @@ Names listed in a :keyword:`global` statement must not be used in the same code block textually preceding that :keyword:`!global` statement. Names listed in a :keyword:`global` statement must not be defined as formal -parameters or in a :keyword:`for` loop control target, :keyword:`class` +parameters, or as targets in :keyword:`with` statements or :keyword:`except` clauses, or in a :keyword:`for` target list, :keyword:`class` definition, function definition, :keyword:`import` statement, or variable annotation. diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index fa8244a8fd318e..76c9d920cbe31f 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -79,9 +79,9 @@ def add_annotations(self, app, doctree): classes=['stableabi'])) if par['objtype'] != 'function': continue - if not par[0].has_key('names') or not par[0]['names']: + if not par[0].has_key('ids') or not par[0]['ids']: continue - name = par[0]['names'][0] + name = par[0]['ids'][0] if name.startswith("c."): name = name[2:] entry = self.get(name) diff --git a/Doc/tools/extensions/glossary_search.py b/Doc/tools/extensions/glossary_search.py new file mode 100644 index 00000000000000..59a6862ea3d3f4 --- /dev/null +++ b/Doc/tools/extensions/glossary_search.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +""" + glossary_search.py + ~~~~~~~~~~~~~~~~ + + Feature search results for glossary items prominently. + + :license: Python license. +""" +import json +import os.path +from docutils.nodes import definition_list_item +from sphinx.addnodes import glossary +from sphinx.util import logging + + +logger = logging.getLogger(__name__) +STATIC_DIR = '_static' +JSON = 'glossary.json' + + +def process_glossary_nodes(app, doctree, fromdocname): + if app.builder.format != 'html': + return + + terms = {} + + for node in doctree.traverse(glossary): + for glossary_item in node.traverse(definition_list_item): + term = glossary_item[0].astext().lower() + definition = glossary_item[1] + + rendered = app.builder.render_partial(definition) + terms[term] = { + 'title': glossary_item[0].astext(), + 'body': rendered['html_body'] + } + + if hasattr(app.env, 'glossary_terms'): + app.env.glossary_terms.update(terms) + else: + app.env.glossary_terms = terms + +def on_build_finish(app, exc): + if not hasattr(app.env, 'glossary_terms'): + return + if not app.env.glossary_terms: + return + + logger.info(f'Writing {JSON}', color='green') + + dest_dir = os.path.join(app.outdir, STATIC_DIR) + os.makedirs(dest_dir, exist_ok=True) + + with open(os.path.join(dest_dir, JSON), 'w') as f: + json.dump(app.env.glossary_terms, f) + + +def setup(app): + app.connect('doctree-resolved', process_glossary_nodes) + app.connect('build-finished', on_build_finish) + + return {'version': '0.1', 'parallel_read_safe': True} diff --git a/Doc/tools/extensions/peg_highlight.py b/Doc/tools/extensions/peg_highlight.py index 9a2acb7f320ba6..42101be10ea9b6 100644 --- a/Doc/tools/extensions/peg_highlight.py +++ b/Doc/tools/extensions/peg_highlight.py @@ -27,6 +27,12 @@ class PEGLexer(RegexLexer): tokens = { "ws": [(r"\n", Text), (r"\s+", Text), (r"#.*$", Comment.Singleline),], "lookaheads": [ + # Forced tokens + (r"(&&)(?=\w+\s?)", bygroups(None)), + (r"(&&)(?='.+'\s?)", bygroups(None)), + (r'(&&)(?=".+"\s?)', bygroups(None)), + (r"(&&)(?=\(.+\)\s?)", bygroups(None)), + (r"(?<=\|\s)(&\w+\s?)", bygroups(None)), (r"(?<=\|\s)(&'.+'\s?)", bygroups(None)), (r'(?<=\|\s)(&".+"\s?)', bygroups(None)), @@ -36,14 +42,19 @@ class PEGLexer(RegexLexer): (r"(@\w+ '''(.|\n)+?''')", bygroups(None)), (r"^(@.*)$", bygroups(None)), ], - "actions": [(r"{(.|\n)+?}", bygroups(None)),], + "actions": [ + (r"{(.|\n)+?}", bygroups(None)), + ], "strings": [ (r"'\w+?'", Keyword), (r'"\w+?"', Keyword), (r"'\W+?'", Text), (r'"\W+?"', Text), ], - "variables": [(_name + _text_ws + r"(\[.*\])?" + _text_ws + "(=)", bygroups(None, None, None, None, None),),], + "variables": [ + (_name + _text_ws + "(=)", bygroups(None, None, None),), + (_name + _text_ws + r"(\[[\w\d_\*]+?\])" + _text_ws + "(=)", bygroups(None, None, None, None, None),), + ], "invalids": [ (r"^(\s+\|\s+invalid_\w+\s*\n)", bygroups(None)), (r"^(\s+\|\s+incorrect_\w+\s*\n)", bygroups(None)), diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 80fbd96d56fdc0..bcd9d26b36846f 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -224,6 +224,7 @@ def run(self): info['source'].append((env.docname, target)) pnode = nodes.paragraph(text, classes=["audit-hook"], ids=ids) + pnode.line = self.lineno if self.content: self.state.nested_parse(self.content, self.content_offset, pnode) else: @@ -394,7 +395,12 @@ def run(self): translatable=False) node.append(para) env = self.state.document.settings.env - env.get_domain('changeset').note_changeset(node) + # deprecated pre-Sphinx-2 method + if hasattr(env, 'note_versionchange'): + env.note_versionchange('deprecated', version[0], node, self.lineno) + # new method + else: + env.get_domain('changeset').note_changeset(node) return [node] + messages diff --git a/Doc/tools/rstlint.py b/Doc/tools/rstlint.py index a3024d6734d25b..cbcb8eb801b135 100755 --- a/Doc/tools/rstlint.py +++ b/Doc/tools/rstlint.py @@ -13,6 +13,7 @@ import re import sys import getopt +from string import ascii_letters from os.path import join, splitext, abspath, exists from collections import defaultdict @@ -128,6 +129,81 @@ def check_leaked_markup(fn, lines): yield lno+1, 'possibly leaked markup: %r' % line +def hide_literal_blocks(lines): + """Tool to remove literal blocks from given lines. + + It yields empty lines in place of blocks, so line numbers are + still meaningful. + """ + in_block = False + for line in lines: + if line.endswith("::\n"): + in_block = True + elif in_block: + if line == "\n" or line.startswith(" "): + line = "\n" + else: + in_block = False + yield line + + +def type_of_explicit_markup(line): + if re.match(fr'\.\. {all_directives}::', line): + return 'directive' + if re.match(r'\.\. \[[0-9]+\] ', line): + return 'footnote' + if re.match(r'\.\. \[[^\]]+\] ', line): + return 'citation' + if re.match(r'\.\. _.*[^_]: ', line): + return 'target' + if re.match(r'\.\. \|[^\|]*\| ', line): + return 'substitution_definition' + return 'comment' + + +def hide_comments(lines): + """Tool to remove comments from given lines. + + It yields empty lines in place of comments, so line numbers are + still meaningfull. + """ + in_multiline_comment = False + for line in lines: + if line == "..\n": + in_multiline_comment = True + elif in_multiline_comment: + if line == "\n" or line.startswith(" "): + line = "\n" + else: + in_multiline_comment = False + if line.startswith(".. ") and type_of_explicit_markup(line) == 'comment': + line = "\n" + yield line + + + +@checker(".rst", severity=2) +def check_missing_surrogate_space_on_plural(fn, lines): + r"""Check for missing 'backslash-space' between a code sample a letter. + + Good: ``Point``\ s + Bad: ``Point``s + """ + in_code_sample = False + check_next_one = False + for lno, line in enumerate(hide_comments(hide_literal_blocks(lines))): + tokens = line.split("``") + for token_no, token in enumerate(tokens): + if check_next_one: + if token[0] in ascii_letters: + yield lno + 1, f"Missing backslash-space between code sample and {token!r}." + check_next_one = False + if token_no == len(tokens) - 1: + continue + if in_code_sample: + check_next_one = True + in_code_sample = not in_code_sample + def main(argv): usage = '''\ Usage: %s [-v] [-f] [-s sev] [-i path]* [path] diff --git a/Doc/tools/static/switchers.js b/Doc/tools/static/switchers.js deleted file mode 100644 index 1a1c7d0fa57e23..00000000000000 --- a/Doc/tools/static/switchers.js +++ /dev/null @@ -1,156 +0,0 @@ -(function() { - 'use strict'; - - // Parses versions in URL segments like: - // "3", "dev", "release/2.7" or "3.6rc2" - var version_regexs = [ - '(?:\\d)', - '(?:\\d\\.\\d[\\w\\d\\.]*)', - '(?:dev)', - '(?:release/\\d.\\d[\\x\\d\\.]*)']; - - var all_versions = { - '3.10': 'dev (3.10)', - '3.9': 'pre (3.9)', - '3.8': '3.8', - '3.7': '3.7', - '3.6': '3.6', - '2.7': '2.7', - }; - - var all_languages = { - 'en': 'English', - 'fr': 'French', - 'ja': 'Japanese', - 'ko': 'Korean', - 'pt-br': 'Brazilian Portuguese', - 'zh-cn': 'Simplified Chinese', - }; - - function build_version_select(current_version, current_release) { - var buf = [''); - return buf.join(''); - } - - function build_language_select(current_language) { - var buf = [''); - return buf.join(''); - } - - function navigate_to_first_existing(urls) { - // Navigate to the first existing URL in urls. - var url = urls.shift(); - if (urls.length == 0) { - window.location.href = url; - return; - } - $.ajax({ - url: url, - success: function() { - window.location.href = url; - }, - error: function() { - navigate_to_first_existing(urls); - } - }); - } - - function on_version_switch() { - var selected_version = $(this).children('option:selected').attr('value') + '/'; - var url = window.location.href; - var current_language = language_segment_from_url(url); - var current_version = version_segment_in_url(url); - var new_url = url.replace('.org/' + current_language + current_version, - '.org/' + current_language + selected_version); - if (new_url != url) { - navigate_to_first_existing([ - new_url, - url.replace('.org/' + current_language + current_version, - '.org/' + selected_version), - 'https://docs.python.org/' + current_language + selected_version, - 'https://docs.python.org/' + selected_version, - 'https://docs.python.org/' - ]); - } - } - - function on_language_switch() { - var selected_language = $(this).children('option:selected').attr('value') + '/'; - var url = window.location.href; - var current_language = language_segment_from_url(url); - var current_version = version_segment_in_url(url); - if (selected_language == 'en/') // Special 'default' case for english. - selected_language = ''; - var new_url = url.replace('.org/' + current_language + current_version, - '.org/' + selected_language + current_version); - if (new_url != url) { - navigate_to_first_existing([ - new_url, - 'https://docs.python.org/' - ]); - } - } - - // Returns the path segment of the language as a string, like 'fr/' - // or '' if not found. - function language_segment_from_url(url) { - var language_regexp = '\.org/([a-z]{2}(?:-[a-z]{2})?/)'; - var match = url.match(language_regexp); - if (match !== null) - return match[1]; - return ''; - } - - // Returns the path segment of the version as a string, like '3.6/' - // or '' if not found. - function version_segment_in_url(url) { - var language_segment = '(?:[a-z]{2}(?:-[a-z]{2})?/)'; - var version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; - var version_regexp = '\\.org/' + language_segment + '?(' + version_segment + ')'; - var match = url.match(version_regexp); - if (match !== null) - return match[1]; - return '' - } - - $(document).ready(function() { - var release = DOCUMENTATION_OPTIONS.VERSION; - var language_segment = language_segment_from_url(window.location.href); - var current_language = language_segment.replace(/\/+$/g, '') || 'en'; - var version = release.substr(0, 3); - var version_select = build_version_select(version, release); - - $('.version_switcher_placeholder').html(version_select); - $('.version_switcher_placeholder select').bind('change', on_version_switch); - - var language_select = build_language_select(current_language); - - $('.language_switcher_placeholder').html(language_select); - $('.language_switcher_placeholder select').bind('change', on_language_switch); - }); -})(); diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index b15fd32b357f03..d56a2b9fd0bfb9 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -5,7 +5,7 @@ c-api/sequence,,:i2,o[i1:i2] c-api/tuple,,:high,p[low:high] c-api/unicode,,:end,str[start:end] c-api/unicode,,:start,unicode[start:start+length] -distutils/examples,267,`,This is the description of the ``foobar`` package. +distutils/examples,,`,This is the description of the ``foobar`` package. distutils/setupscript,,::, extending/embedding,,:numargs,"if(!PyArg_ParseTuple(args, "":numargs""))" extending/extending,,:myfunction,"PyArg_ParseTuple(args, ""D:myfunction"", &c);" @@ -147,8 +147,10 @@ library/ipaddress,,:db8,IPv6Address('2001:db8::') library/ipaddress,,::,IPv6Address('2001:db8::') library/ipaddress,,:db8,>>> ipaddress.IPv6Address('2001:db8::1000') library/ipaddress,,::,>>> ipaddress.IPv6Address('2001:db8::1000') -library/ipaddress,,:db8,IPv6Address('2001:db8::1000') -library/ipaddress,,::,IPv6Address('2001:db8::1000') +library/ipaddress,,:db8,'2001:db8::1000' +library/ipaddress,,::,'2001:db8::1000' +library/ipaddress,,:db8,">>> f'{ipaddress.IPv6Address(""2001:db8::1000""):s}'" +library/ipaddress,,::,">>> f'{ipaddress.IPv6Address(""2001:db8::1000""):s}'" library/ipaddress,,::,IPv6Address('ff02::5678%1') library/ipaddress,,::,fe80::1234 library/ipaddress,,:db8,">>> ipaddress.ip_address(""2001:db8::1"").reverse_pointer" @@ -166,6 +168,7 @@ library/ipaddress,,::,2001:db00::0/24 library/ipaddress,,:db00,2001:db00::0/ffff:ff00:: library/ipaddress,,::,2001:db00::0/ffff:ff00:: library/itertools,,:step,elements from seq[start:stop:step] +library/itertools,,::,kernel = tuple(kernel)[::-1] library/itertools,,:stop,elements from seq[start:stop:step] library/logging.handlers,,:port,host:port library/mmap,,:i2,obj[i1:i2] @@ -206,9 +209,9 @@ library/smtplib,,:port,method must support that as well as a regular host:port library/socket,,::,'5aef:2b::8' library/socket,,:can,"return (can_id, can_dlc, data[:can_dlc])" library/socket,,:len,fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) -library/sqlite3,,:age,"cur.execute(""select * from people where name_last=:who and age=:age"", {""who"": who, ""age"": age})" +library/sqlite3,,:name,"cur.execute(""select * from lang where lang_name=:name and lang_age=:age""," +library/sqlite3,,:age,"cur.execute(""select * from lang where lang_name=:name and lang_age=:age""," library/sqlite3,,:memory, -library/sqlite3,,:who,"cur.execute(""select * from people where name_last=:who and age=:age"", {""who"": who, ""age"": age})" library/sqlite3,,:path,"db = sqlite3.connect('file:path/to/database?mode=ro', uri=True)" library/ssl,,:My,"Organizational Unit Name (eg, section) []:My Group" library/ssl,,:My,"Organization Name (eg, company) [Internet Widgits Pty Ltd]:My Organization, Inc." @@ -276,7 +279,11 @@ tutorial/stdlib2,,:start,"fields = struct.unpack('{% trans %}Download{% endtrans %}

{% trans %}Download these documents{% endtrans %}

{% trans %}Docs by version{% endtrans %}

diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 17592d74a4eb52..98ccf4224804b2 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -12,22 +12,14 @@ {% block rootrellink %} {{ super() }} -
  • - {%- if switchers is defined %} - {{ language or 'en' }} - {{ release }} - {% trans %}Documentation {% endtrans %}{{ reldelim1 }} - {%- else %} +
  • {{ shorttitle }}{{ reldelim1 }} - {%- endif %}
    • {% endblock %} {% block extrahead %} {% if builder != "htmlhelp" %} - {% if switchers is defined and not embedded %} - {% endif %} {% if pagename == 'whatsnew/changelog' and not embedded %} {% endif %} {% endif %} diff --git a/Doc/tools/templates/search.html b/Doc/tools/templates/search.html new file mode 100644 index 00000000000000..cf20c2e1d4ff83 --- /dev/null +++ b/Doc/tools/templates/search.html @@ -0,0 +1,48 @@ +{% extends "!search.html" %} +{% block extrahead %} + {{ super() }} + +{% endblock %} \ No newline at end of file diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index b8aec2b04f13fb..0b09c18170dd84 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -36,6 +36,9 @@ to avoid excessive indentation. An :keyword:`!if` ... :keyword:`!elif` ... :keyword:`!elif` ... sequence is a substitute for the ``switch`` or ``case`` statements found in other languages. +If you're comparing the same value to several constants, or checking for specific types or +attributes, you may also find the :keyword:`!match` statement useful. For more +details see :ref:`tut-match`. .. _tut-for: @@ -246,6 +249,172 @@ at a more abstract level. The :keyword:`!pass` is silently ignored:: ... pass # Remember to implement this! ... + +.. _tut-match: + +:keyword:`!match` Statements +============================ + +A match statement takes an expression and compares its value to successive +patterns given as one or more case blocks. This is superficially +similar to a switch statement in C, Java or JavaScript (and many +other languages), but it can also extract components (sequence elements or +object attributes) from the value into variables. + +The simplest form compares a subject value against one or more literals:: + + def http_error(status): + match status: + case 400: + return "Bad request" + case 404: + return "Not found" + case 418: + return "I'm a teapot" + case _: + return "Something's wrong with the Internet" + +Note the last block: the "variable name" ``_`` acts as a *wildcard* and +never fails to match. If no case matches, none of the branches is executed. + +You can combine several literals in a single pattern using ``|`` ("or"):: + + case 401 | 403 | 404: + return "Not allowed" + +Patterns can look like unpacking assignments, and can be used to bind +variables:: + + # point is an (x, y) tuple + match point: + case (0, 0): + print("Origin") + case (0, y): + print(f"Y={y}") + case (x, 0): + print(f"X={x}") + case (x, y): + print(f"X={x}, Y={y}") + case _: + raise ValueError("Not a point") + +Study that one carefully! The first pattern has two literals, and can +be thought of as an extension of the literal pattern shown above. But +the next two patterns combine a literal and a variable, and the +variable *binds* a value from the subject (``point``). The fourth +pattern captures two values, which makes it conceptually similar to +the unpacking assignment ``(x, y) = point``. + +If you are using classes to structure your data +you can use the class name followed by an argument list resembling a +constructor, but with the ability to capture attributes into variables:: + + class Point: + x: int + y: int + + def where_is(point): + match point: + case Point(x=0, y=0): + print("Origin") + case Point(x=0, y=y): + print(f"Y={y}") + case Point(x=x, y=0): + print(f"X={x}") + case Point(): + print("Somewhere else") + case _: + print("Not a point") + +You can use positional parameters with some builtin classes that provide an +ordering for their attributes (e.g. dataclasses). You can also define a specific +position for attributes in patterns by setting the ``__match_args__`` special +attribute in your classes. If it's set to ("x", "y"), the following patterns are all +equivalent (and all bind the ``y`` attribute to the ``var`` variable):: + + Point(1, var) + Point(1, y=var) + Point(x=1, y=var) + Point(y=var, x=1) + +A recommended way to read patterns is to look at them as an extended form of what you +would put on the left of an assignment, to understand which variables would be set to +what. +Only the standalone names (like ``var`` above) are assigned to by a match statement. +Dotted names (like ``foo.bar``), attribute names (the ``x=`` and ``y=`` above) or class names +(recognized by the "(...)" next to them like ``Point`` above) are never assigned to. + +Patterns can be arbitrarily nested. For example, if we have a short +list of points, we could match it like this:: + + match points: + case []: + print("No points") + case [Point(0, 0)]: + print("The origin") + case [Point(x, y)]: + print(f"Single point {x}, {y}") + case [Point(0, y1), Point(0, y2)]: + print(f"Two on the Y axis at {y1}, {y2}") + case _: + print("Something else") + +We can add an ``if`` clause to a pattern, known as a "guard". If the +guard is false, ``match`` goes on to try the next case block. Note +that value capture happens before the guard is evaluated:: + + match point: + case Point(x, y) if x == y: + print(f"Y=X at {x}") + case Point(x, y): + print(f"Not on the diagonal") + +Several other key features of this statement: + +- Like unpacking assignments, tuple and list patterns have exactly the + same meaning and actually match arbitrary sequences. An important + exception is that they don't match iterators or strings. + +- Sequence patterns support extended unpacking: ``[x, y, *rest]`` and ``(x, y, + *rest)`` work similar to unpacking assignments. The + name after ``*`` may also be ``_``, so ``(x, y, *_)`` matches a sequence + of at least two items without binding the remaining items. + +- Mapping patterns: ``{"bandwidth": b, "latency": l}`` captures the + ``"bandwidth"`` and ``"latency"`` values from a dictionary. Unlike sequence + patterns, extra keys are ignored. An unpacking like ``**rest`` is also + supported. (But ``**_`` would be redundant, so it not allowed.) + +- Subpatterns may be captured using the ``as`` keyword:: + + case (Point(x1, y1), Point(x2, y2) as p2): ... + + will capture the second element of the input as ``p2`` (as long as the input is + a sequence of two points) + +- Most literals are compared by equality, however the singletons ``True``, + ``False`` and ``None`` are compared by identity. + +- Patterns may use named constants. These must be dotted names + to prevent them from being interpreted as capture variable:: + + from enum import Enum + class Color(Enum): + RED = 0 + GREEN = 1 + BLUE = 2 + + match color: + case Color.RED: + print("I see red!") + case Color.GREEN: + print("Grass is green") + case Color.BLUE: + print("I'm feeling the blues :(") + +For a more detailed explanation and additional examples, you can look into +:pep:`636` which is written in a tutorial format. + .. _tut-functions: Defining Functions @@ -297,7 +466,8 @@ referenced. The actual parameters (arguments) to a function call are introduced in the local symbol table of the called function when it is called; thus, arguments are passed using *call by value* (where the *value* is always an object *reference*, -not the value of the object). [#]_ When a function calls another function, a new +not the value of the object). [#]_ When a function calls another function, +or calls itself recursively, a new local symbol table is created for that call. A function definition associates the function name with the function object in @@ -868,7 +1038,7 @@ function. Parameter annotations are defined by a colon after the parameter name by an expression evaluating to the value of the annotation. Return annotations are defined by a literal ``->``, followed by an expression, between the parameter list and the colon denoting the end of the :keyword:`def` statement. The -following example has a positional argument, a keyword argument, and the return +following example has a required argument, an optional argument, and the return value annotated:: >>> def f(ham: str, eggs: str = 'eggs') -> str: diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index ff4c797f66cd63..e42b380db3d23c 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -78,7 +78,7 @@ objects: Return the number of times *x* appears in the list. -.. method:: list.sort(key=None, reverse=False) +.. method:: list.sort(*, key=None, reverse=False) :noindex: Sort the items of the list in place (the arguments can be used for sort @@ -661,9 +661,8 @@ operators, not just comparisons. The comparison operators ``in`` and ``not in`` check whether a value occurs (does not occur) in a sequence. The operators ``is`` and ``is not`` compare -whether two objects are really the same object; this only matters for mutable -objects like lists. All comparison operators have the same priority, which is -lower than that of all numerical operators. +whether two objects are really the same object. All comparison operators have +the same priority, which is lower than that of all numerical operators. Comparisons can be chained. For example, ``a < b == c`` tests whether ``a`` is less than ``b`` and moreover ``b`` equals ``c``. diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 0ce96466e8c286..25bb4fc567b95b 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -101,29 +101,29 @@ The :keyword:`try` statement works as follows. * If no exception occurs, the *except clause* is skipped and execution of the :keyword:`try` statement is finished. -* If an exception occurs during execution of the try clause, the rest of the - clause is skipped. Then if its type matches the exception named after the - :keyword:`except` keyword, the except clause is executed, and then execution - continues after the :keyword:`try` statement. +* If an exception occurs during execution of the :keyword:`try` clause, the rest of the + clause is skipped. Then, if its type matches the exception named after the + :keyword:`except` keyword, the *except clause* is executed, and then execution + continues after the try/except block. -* If an exception occurs which does not match the exception named in the except - clause, it is passed on to outer :keyword:`try` statements; if no handler is +* If an exception occurs which does not match the exception named in the *except + clause*, it is passed on to outer :keyword:`try` statements; if no handler is found, it is an *unhandled exception* and execution stops with a message as shown above. -A :keyword:`try` statement may have more than one except clause, to specify +A :keyword:`try` statement may have more than one *except clause*, to specify handlers for different exceptions. At most one handler will be executed. -Handlers only handle exceptions that occur in the corresponding try clause, not -in other handlers of the same :keyword:`!try` statement. An except clause may -name multiple exceptions as a parenthesized tuple, for example:: +Handlers only handle exceptions that occur in the corresponding *try clause*, +not in other handlers of the same :keyword:`!try` statement. An *except clause* +may name multiple exceptions as a parenthesized tuple, for example:: ... except (RuntimeError, TypeError, NameError): ... pass A class in an :keyword:`except` clause is compatible with an exception if it is the same class or a base class thereof (but not the other way around --- an -except clause listing a derived class is not compatible with a base class). For -example, the following code will print B, C, D in that order:: +*except clause* listing a derived class is not compatible with a base class). +For example, the following code will print B, C, D in that order:: class B(Exception): pass @@ -144,10 +144,10 @@ example, the following code will print B, C, D in that order:: except B: print("B") -Note that if the except clauses were reversed (with ``except B`` first), it -would have printed B, B, B --- the first matching except clause is triggered. +Note that if the *except clauses* were reversed (with ``except B`` first), it +would have printed B, B, B --- the first matching *except clause* is triggered. -The last except clause may omit the exception name(s), to serve as a wildcard. +The last *except clause* may omit the exception name(s), to serve as a wildcard. Use this with extreme caution, since it is easy to mask a real programming error in this way! It can also be used to print an error message and then re-raise the exception (allowing a caller to handle the exception as well):: @@ -167,9 +167,9 @@ the exception (allowing a caller to handle the exception as well):: raise The :keyword:`try` ... :keyword:`except` statement has an optional *else -clause*, which, when present, must follow all except clauses. It is useful for -code that must be executed if the try clause does not raise an exception. For -example:: +clause*, which, when present, must follow all *except clauses*. It is useful +for code that must be executed if the *try clause* does not raise an exception. +For example:: for arg in sys.argv[1:]: try: @@ -189,7 +189,7 @@ When an exception occurs, it may have an associated value, also known as the exception's *argument*. The presence and type of the argument depend on the exception type. -The except clause may specify a variable after the exception name. The +The *except clause* may specify a variable after the exception name. The variable is bound to an exception instance with the arguments stored in ``instance.args``. For convenience, the exception instance defines :meth:`__str__` so the arguments can be printed directly without having to @@ -217,8 +217,8 @@ If an exception has arguments, they are printed as the last part ('detail') of the message for unhandled exceptions. Exception handlers don't just handle exceptions if they occur immediately in the -try clause, but also if they occur inside functions that are called (even -indirectly) in the try clause. For example:: +*try clause*, but also if they occur inside functions that are called (even +indirectly) in the *try clause*. For example:: >>> def this_fails(): ... x = 1/0 @@ -272,37 +272,36 @@ re-raise the exception:: Exception Chaining ================== -The :keyword:`raise` statement allows an optional :keyword:`from` which enables -chaining exceptions by setting the ``__cause__`` attribute of the raised -exception. For example:: +The :keyword:`raise` statement allows an optional :keyword:`from` which enables +chaining exceptions. For example:: - raise RuntimeError from OSError + # exc must be exception instance or None. + raise RuntimeError from exc This can be useful when you are transforming exceptions. For example:: >>> def func(): - ... raise IOError + ... raise ConnectionError ... >>> try: ... func() - ... except IOError as exc: + ... except ConnectionError as exc: ... raise RuntimeError('Failed to open database') from exc ... Traceback (most recent call last): File "", line 2, in File "", line 2, in func - OSError + ConnectionError The above exception was the direct cause of the following exception: Traceback (most recent call last): File "", line 4, in - RuntimeError + RuntimeError: Failed to open database -The expression following the :keyword:`from` must be either an exception or -``None``. Exception chaining happens automatically when an exception is raised -inside an exception handler or :keyword:`finally` section. Exception chaining -can be disabled by using ``from None`` idiom: +Exception chaining happens automatically when an exception is raised inside an +:keyword:`except` or :keyword:`finally` section. This can be +disabled by using ``from None`` idiom: >>> try: ... open('database.sqlite') @@ -313,6 +312,8 @@ can be disabled by using ``from None`` idiom: File "", line 4, in RuntimeError +For more information about chaining mechanics, see :ref:`bltin-exceptions`. + .. _tut-userexceptions: diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 366a532e817afa..4e27cff83ce59f 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -329,11 +329,16 @@ equivalent :keyword:`try`\ -\ :keyword:`finally` blocks:: If you're not using the :keyword:`with` keyword, then you should call ``f.close()`` to close the file and immediately free up any system -resources used by it. If you don't explicitly close a file, Python's -garbage collector will eventually destroy the object and close the -open file for you, but the file may stay open for a while. Another -risk is that different Python implementations will do this clean-up at -different times. +resources used by it. + +.. warning:: + Calling ``f.write()`` without using the :keyword:`!with` keyword or calling + ``f.close()`` **might** result in the arguments + of ``f.write()`` not being completely written to the disk, even if the + program exits successfully. + +.. + See also https://bugs.python.org/issue17852 After a file object is closed, either by a :keyword:`with` statement or by calling ``f.close()``, attempts to use the file object will diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index d2733a9968fb1d..d7f300d93b021b 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -10,13 +10,13 @@ Using the Python Interpreter Invoking the Interpreter ======================== -The Python interpreter is usually installed as :file:`/usr/local/bin/python3.10` +The Python interpreter is usually installed as :file:`/usr/local/bin/python3.11` on those machines where it is available; putting :file:`/usr/local/bin` in your Unix shell's search path makes it possible to start it by typing the command: .. code-block:: text - python3.10 + python3.11 to the shell. [#]_ Since the choice of the directory where the interpreter lives is an installation option, other places are possible; check with your local @@ -24,7 +24,7 @@ Python guru or system administrator. (E.g., :file:`/usr/local/python` is a popular alternative location.) On Windows machines where you have installed Python from the :ref:`Microsoft Store -`, the :file:`python3.10` command will be available. If you have +`, the :file:`python3.11` command will be available. If you have the :ref:`py.exe launcher ` installed, you can use the :file:`py` command. See :ref:`setting-envvars` for other ways to launch Python. @@ -97,9 +97,9 @@ before printing the first prompt: .. code-block:: shell-session - $ python3.10 - Python 3.10 (default, June 4 2019, 09:25:04) - [GCC 4.8.2] on linux + $ python3.11 + Python 3.11 (default, April 4 2021, 09:25:04) + [GCC 10.2.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index f33265cd2b0eb4..bad750424c8e85 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -15,7 +15,7 @@ operating system:: >>> import os >>> os.getcwd() # Return the current working directory - 'C:\\Python310' + 'C:\\Python311' >>> os.chdir('/server/accesslogs') # Change current working directory >>> os.system('mkdir today') # Run the command mkdir in the system shell 0 diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 298034d3b48440..69ec0d4727c0bb 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -278,7 +278,7 @@ applications include caching objects that are expensive to create:: Traceback (most recent call last): File "", line 1, in d['primary'] # entry was automatically removed - File "C:/python310/lib/weakref.py", line 46, in __getitem__ + File "C:/python311/lib/weakref.py", line 46, in __getitem__ o = self.data[key]() KeyError: 'primary' diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index f422146aae8a30..221c11ca74b4ef 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -48,7 +48,7 @@ place it, and run the :mod:`venv` module as a script with the directory path:: This will create the ``tutorial-env`` directory if it doesn't exist, and also create directories inside it containing a copy of the Python -interpreter, the standard library, and various supporting files. +interpreter and various supporting files. A common directory location for a virtual environment is ``.venv``. This name keeps the directory typically hidden in your shell and thus @@ -95,20 +95,9 @@ Managing Packages with pip You can install, upgrade, and remove packages using a program called :program:`pip`. By default ``pip`` will install packages from the Python Package Index, . You can browse the Python -Package Index by going to it in your web browser, or you can use ``pip``'s -limited search feature: +Package Index by going to it in your web browser. -.. code-block:: bash - - (tutorial-env) $ pip search astronomy - skyfield - Elegant astronomy for Python - gary - Galactic astronomy and gravitational dynamics. - novas - The United States Naval Observatory NOVAS astronomy library - astroobs - Provides astronomy ephemeris to plan telescope observations - PyAstronomy - A collection of astronomy related tools for Python. - ... - -``pip`` has a number of subcommands: "search", "install", "uninstall", +``pip`` has a number of subcommands: "install", "uninstall", "freeze", etc. (Consult the :ref:`installing-index` guide for complete documentation for ``pip``.) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 603b0e105fb43e..25e05d413de877 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -380,25 +380,8 @@ Miscellaneous options .. _using-on-warnings: .. cmdoption:: -W arg - Warning control. Python's warning machinery by default prints warning - messages to :data:`sys.stderr`. A typical warning message has the following - form: - - .. code-block:: none - - file:line: category: message - - By default, each warning is printed once for each source line where it - occurs. This option controls how often warnings are printed. - - Multiple :option:`-W` options may be given; when a warning matches more than - one option, the action for the last matching option is performed. Invalid - :option:`-W` options are ignored (though, a warning message is printed about - invalid options when the first warning is issued). - - Warnings can also be controlled using the :envvar:`PYTHONWARNINGS` - environment variable and from within a Python program using the - :mod:`warnings` module. + Warning control. Python's warning machinery by default prints warning + messages to :data:`sys.stderr`. The simplest settings apply a particular action unconditionally to all warnings emitted by a process (even those that are otherwise ignored by @@ -411,14 +394,48 @@ Miscellaneous options -Wonce # Warn once per Python process -Wignore # Never warn - The action names can be abbreviated as desired (e.g. ``-Wi``, ``-Wd``, - ``-Wa``, ``-We``) and the interpreter will resolve them to the appropriate - action name. + The action names can be abbreviated as desired and the interpreter will + resolve them to the appropriate action name. For example, ``-Wi`` is the + same as ``-Wignore``. + + The full form of argument is:: + + action:message:category:module:lineno + + Empty fields match all values; trailing empty fields may be omitted. For + example ``-W ignore::DeprecationWarning`` ignores all DeprecationWarning + warnings. + + The *action* field is as explained above but only applies to warnings that + match the remaining fields. + + The *message* field must match the whole warning message; this match is + case-insensitive. + + The *category* field matches the warning category + (ex: ``DeprecationWarning``). This must be a class name; the match test + whether the actual warning category of the message is a subclass of the + specified warning category. + + The *module* field matches the (fully-qualified) module name; this match is + case-sensitive. + + The *lineno* field matches the line number, where zero matches all line + numbers and is thus equivalent to an omitted line number. + + Multiple :option:`-W` options can be given; when a warning matches more than + one option, the action for the last matching option is performed. Invalid + :option:`-W` options are ignored (though, a warning message is printed about + invalid options when the first warning is issued). + + Warnings can also be controlled using the :envvar:`PYTHONWARNINGS` + environment variable and from within a Python program using the + :mod:`warnings` module. For example, the :func:`warnings.filterwarnings` + function can be used to use a regular expression on the warning message. See :ref:`warning-filter` and :ref:`describing-warning-filters` for more details. - .. cmdoption:: -x Skip the first line of the source, allowing use of non-Unix forms of @@ -433,7 +450,8 @@ Miscellaneous options * ``-X faulthandler`` to enable :mod:`faulthandler`; * ``-X showrefcount`` to output the total reference count and number of used memory blocks when the program finishes or after each statement in the - interactive interpreter. This only works on debug builds. + interactive interpreter. This only works on :ref:`debug builds + `. * ``-X tracemalloc`` to start tracing Python memory allocations using the :mod:`tracemalloc` module. By default, only the most recent frame is stored in a traceback of a trace. Use ``-X tracemalloc=NFRAME`` to start @@ -447,13 +465,15 @@ Miscellaneous options * ``-X dev``: enable :ref:`Python Development Mode `, introducing additional runtime checks that are too expensive to be enabled by default. - * ``-X utf8`` enables UTF-8 mode for operating system interfaces, overriding - the default locale-aware mode. ``-X utf8=0`` explicitly disables UTF-8 - mode (even when it would otherwise activate automatically). - See :envvar:`PYTHONUTF8` for more details. + * ``-X utf8`` enables the :ref:`Python UTF-8 Mode `. + ``-X utf8=0`` explicitly disables :ref:`Python UTF-8 Mode ` + (even when it would otherwise activate automatically). * ``-X pycache_prefix=PATH`` enables writing ``.pyc`` files to a parallel tree rooted at the given directory instead of to the code tree. See also :envvar:`PYTHONPYCACHEPREFIX`. + * ``-X warn_default_encoding`` issues a :class:`EncodingWarning` when the + locale-specific default encoding is used for opening files. + See also :envvar:`PYTHONWARNDEFAULTENCODING`. It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -483,6 +503,9 @@ Miscellaneous options The ``-X showalloccount`` option has been removed. + .. versionadded:: 3.10 + The ``-X warn_default_encoding`` option. + .. deprecated-removed:: 3.9 3.10 The ``-X oldparser`` option. @@ -776,17 +799,13 @@ conflict. :c:data:`PYMEM_DOMAIN_MEM` and :c:data:`PYMEM_DOMAIN_OBJ` domains and use the :c:func:`malloc` function for the :c:data:`PYMEM_DOMAIN_RAW` domain. - Install debug hooks: + Install :ref:`debug hooks `: * ``debug``: install debug hooks on top of the :ref:`default memory allocators `. * ``malloc_debug``: same as ``malloc`` but also install debug hooks. * ``pymalloc_debug``: same as ``pymalloc`` but also install debug hooks. - See the :ref:`default memory allocators ` and the - :c:func:`PyMem_SetupDebugHooks` function (install debug hooks on Python - memory allocators). - .. versionchanged:: 3.7 Added the ``"default"`` allocator. @@ -810,9 +829,10 @@ conflict. .. envvar:: PYTHONLEGACYWINDOWSFSENCODING - If set to a non-empty string, the default filesystem encoding and errors mode - will revert to their pre-3.6 values of 'mbcs' and 'replace', respectively. - Otherwise, the new defaults 'utf-8' and 'surrogatepass' are used. + If set to a non-empty string, the default :term:`filesystem encoding and + error handler` mode will revert to their pre-3.6 values of 'mbcs' and + 'replace', respectively. Otherwise, the new defaults 'utf-8' and + 'surrogatepass' are used. This may also be enabled at runtime with :func:`sys._enablelegacywindowsfsencoding()`. @@ -898,66 +918,33 @@ conflict. .. envvar:: PYTHONUTF8 - If set to ``1``, enables the interpreter's UTF-8 mode, where ``UTF-8`` is - used as the text encoding for system interfaces, regardless of the - current locale setting. - - This means that: + If set to ``1``, enable the :ref:`Python UTF-8 Mode `. - * :func:`sys.getfilesystemencoding()` returns ``'UTF-8'`` (the locale - encoding is ignored). - * :func:`locale.getpreferredencoding()` returns ``'UTF-8'`` (the locale - encoding is ignored, and the function's ``do_setlocale`` parameter has no - effect). - * :data:`sys.stdin`, :data:`sys.stdout`, and :data:`sys.stderr` all use - UTF-8 as their text encoding, with the ``surrogateescape`` - :ref:`error handler ` being enabled for :data:`sys.stdin` - and :data:`sys.stdout` (:data:`sys.stderr` continues to use - ``backslashreplace`` as it does in the default locale-aware mode) - - As a consequence of the changes in those lower level APIs, other higher - level APIs also exhibit different default behaviours: - - * Command line arguments, environment variables and filenames are decoded - to text using the UTF-8 encoding. - * :func:`os.fsdecode()` and :func:`os.fsencode()` use the UTF-8 encoding. - * :func:`open()`, :func:`io.open()`, and :func:`codecs.open()` use the UTF-8 - encoding by default. However, they still use the strict error handler by - default so that attempting to open a binary file in text mode is likely - to raise an exception rather than producing nonsense data. - - Note that the standard stream settings in UTF-8 mode can be overridden by - :envvar:`PYTHONIOENCODING` (just as they can be in the default locale-aware - mode). - - If set to ``0``, the interpreter runs in its default locale-aware mode. + If set to ``0``, disable the :ref:`Python UTF-8 Mode `. Setting any other non-empty string causes an error during interpreter initialisation. - If this environment variable is not set at all, then the interpreter defaults - to using the current locale settings, *unless* the current locale is - identified as a legacy ASCII-based locale - (as described for :envvar:`PYTHONCOERCECLOCALE`), and locale coercion is - either disabled or fails. In such legacy locales, the interpreter will - default to enabling UTF-8 mode unless explicitly instructed not to do so. + .. versionadded:: 3.7 - Also available as the :option:`-X` ``utf8`` option. +.. envvar:: PYTHONWARNDEFAULTENCODING - .. versionadded:: 3.7 - See :pep:`540` for more details. + If this environment variable is set to a non-empty string, issue a + :class:`EncodingWarning` when the locale-specific default encoding is used. + + See :ref:`io-encoding-warning` for details. + + .. versionadded:: 3.10 Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ -Setting these variables only has an effect in a debug build of Python. - .. envvar:: PYTHONTHREADDEBUG If set, Python will print threading debug info. - Need Python configured with the ``--with-pydebug`` build option. + Need a :ref:`debug build of Python `. .. envvar:: PYTHONDUMPREFS @@ -965,4 +952,4 @@ Setting these variables only has an effect in a debug build of Python. If set, Python will dump objects and reference counts still alive after shutting down the interpreter. - Need Python configured with the ``--with-trace-refs`` build option. + Need Python configured with the :option:`--with-trace-refs` build option. diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst new file mode 100644 index 00000000000000..c37540c7e031d6 --- /dev/null +++ b/Doc/using/configure.rst @@ -0,0 +1,816 @@ +**************** +Configure Python +**************** + +.. _configure-options: + +Configure Options +================= + +List all ``./configure`` script options using:: + + ./configure --help + +See also the :file:`Misc/SpecialBuilds.txt` in the Python source distribution. + +General Options +--------------- + +.. cmdoption:: --enable-loadable-sqlite-extensions + + Support loadable extensions in the :mod:`_sqlite` extension module (default + is no). + + See the :meth:`sqlite3.Connection.enable_load_extension` method of the + :mod:`sqlite3` module. + + .. versionadded:: 3.6 + +.. cmdoption:: --disable-ipv6 + + Disable IPv6 support (enabled by default if supported), see the + :mod:`socket` module. + +.. cmdoption:: --enable-big-digits=[15|30] + + Define the size in bits of Python :class:`int` digits: 15 or 30 bits. + + By default, the number of bits is selected depending on ``sizeof(void*)``: + 30 bits if ``void*`` size is 64-bit or larger, 15 bits otherwise. + + Define the ``PYLONG_BITS_IN_DIGIT`` to ``15`` or ``30``. + + See :data:`sys.int_info.bits_per_digit `. + +.. cmdoption:: --with-cxx-main +.. cmdoption:: --with-cxx-main=COMPILER + + Compile the Python ``main()`` function and link Python executable with C++ + compiler: ``$CXX``, or *COMPILER* if specified. + +.. cmdoption:: --with-suffix=SUFFIX + + Set the Python executable suffix to *SUFFIX*. + + The default suffix is ``.exe`` on Windows and macOS (``python.exe`` + executable), and an empty string on other platforms (``python`` executable). + +.. cmdoption:: --with-tzpath= + + Select the default time zone search path for :data:`zoneinfo.TZPATH`. + See the :ref:`Compile-time configuration + ` of the :mod:`zoneinfo` module. + + Default: ``/usr/share/zoneinfo:/usr/lib/zoneinfo:/usr/share/lib/zoneinfo:/etc/zoneinfo``. + + See :data:`os.pathsep` path separator. + + .. versionadded:: 3.9 + +.. cmdoption:: --without-decimal-contextvar + + Build the ``_decimal`` extension module using a thread-local context rather + than a coroutine-local context (default), see the :mod:`decimal` module. + + See :data:`decimal.HAVE_CONTEXTVAR` and the :mod:`contextvars` module. + + .. versionadded:: 3.9 + +.. cmdoption:: --with-dbmliborder=db1:db2:... + + Override order to check db backends for the :mod:`dbm` module + + A valid value is a colon (``:``) separated string with the backend names: + + * ``ndbm``; + * ``gdbm``; + * ``bdb``. + +.. cmdoption:: --without-c-locale-coercion + + Disable C locale coercion to a UTF-8 based locale (enabled by default). + + Don't define the ``PY_COERCE_C_LOCALE`` macro. + + See :envvar:`PYTHONCOERCECLOCALE` and the :pep:`538`. + +.. cmdoption:: --with-platlibdir=DIRNAME + + Python library directory name (default is ``lib``). + + Fedora and SuSE use ``lib64`` on 64-bit platforms. + + See :data:`sys.platlibdir`. + + .. versionadded:: 3.9 + +.. cmdoption:: --with-wheel-pkg-dir=PATH + + Directory of wheel packages used by the :mod:`ensurepip` module + (none by default). + + Some Linux distribution packaging policies recommend against bundling + dependencies. For example, Fedora installs wheel packages in the + ``/usr/share/python-wheels/`` directory and don't install the + :mod:`ensurepip._bundled` package. + + .. versionadded:: 3.10 + + +Install Options +--------------- + +.. cmdoption:: --disable-test-modules + + Don't build nor install test modules, like the :mod:`test` package or the + :mod:`_testcapi` extension module (built and installed by default). + + .. versionadded:: 3.10 + +.. cmdoption:: --with-ensurepip=[upgrade|install|no] + + Select the :mod:`ensurepip` command run on Python installation: + + * ``upgrade`` (default): run ``python -m ensurepip --altinstall --upgrade`` + command. + * ``install``: run ``python -m ensurepip --altinstall`` command; + * ``no``: don't run ensurepip; + + .. versionadded:: 3.6 + + +Performance options +------------------- + +Configuring Python using ``--enable-optimizations --with-lto`` (PGO + LTO) is +recommended for best performance. + +.. cmdoption:: --enable-optimizations + + Enable Profile Guided Optimization (PGO) using :envvar:`PROFILE_TASK` + (disabled by default). + + The C compiler Clang requires ``llvm-profdata`` program for PGO. On + macOS, GCC also requires it: GCC is just an alias to Clang on macOS. + + Disable also semantic interposition in libpython if ``--enable-shared`` and + GCC is used: add ``-fno-semantic-interposition`` to the compiler and linker + flags. + + .. versionadded:: 3.6 + + .. versionchanged:: 3.10 + Use ``-fno-semantic-interposition`` on GCC. + +.. envvar:: PROFILE_TASK + + Environment variable used in the Makefile: Python command line arguments for + the PGO generation task. + + Default: ``-m test --pgo --timeout=$(TESTTIMEOUT)``. + + .. versionadded:: 3.8 + +.. cmdoption:: --with-lto + + Enable Link Time Optimization (LTO) in any build (disabled by default). + + The C compiler Clang requires ``llvm-ar`` for LTO, as well as an LTO-aware + linker (``ld.gold`` or ``lld``). + + .. versionadded:: 3.6 + +.. cmdoption:: --with-computed-gotos + + Enable computed gotos in evaluation loop (enabled by default on supported + compilers). + +.. cmdoption:: --without-pymalloc + + Disable the specialized Python memory allocator :ref:`pymalloc ` + (enabled by default). + + See also :envvar:`PYTHONMALLOC` environment variable. + +.. cmdoption:: --without-doc-strings + + Disable static documentation strings to reduce the memory footprint (enabled + by default). Documentation strings defined in Python are not affected. + + Don't define the ``WITH_DOC_STRINGS`` macro. + + See the ``PyDoc_STRVAR()`` macro. + +.. cmdoption:: --enable-profiling + + Enable C-level code profiling with ``gprof`` (disabled by default). + + +.. _debug-build: + +Python Debug Build +------------------ + +A debug build is Python built with the :option:`--with-pydebug` configure +option. + +Effects of a debug build: + +* Display all warnings by default: the list of default warning filters is empty + in the :mod:`warnings` module. +* Add ``d`` to :data:`sys.abiflags`. +* Add :func:`sys.gettotalrefcount` function. +* Add :option:`-X showrefcount <-X>` command line option. +* Add :envvar:`PYTHONTHREADDEBUG` environment variable. +* Add support for the ``__ltrace__`` variable: enable low-level tracing in the + bytecode evaluation loop if the variable is defined. +* Install :ref:`debug hooks on memory allocators ` + to detect buffer overflow and other memory errors. +* Define ``Py_DEBUG`` and ``Py_REF_DEBUG`` macros. +* Add runtime checks: code surroundeded by ``#ifdef Py_DEBUG`` and ``#endif``. + Enable ``assert(...)`` and ``_PyObject_ASSERT(...)`` assertions: don't set + the ``NDEBUG`` macro (see also the :option:`--with-assertions` configure + option). Main runtime checks: + + * Add sanity checks on the function arguments. + * Unicode and int objects are created with their memory filled with a pattern + to detect usage of uninitialized objects. + * Ensure that functions which can clear or replace the current exception are + not called with an exception raised. + * The garbage collector (:func:`gc.collect` function) runs some basic checks + on objects consistency. + * The :c:macro:`Py_SAFE_DOWNCAST()` macro checks for integer underflow and + overflow when downcasting from wide types to narrow types. + +See also the :ref:`Python Development Mode ` and the +:option:`--with-trace-refs` configure option. + +.. versionchanged:: 3.8 + Release builds and debug builds are now ABI compatible: defining the + ``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro (see the + :option:`--with-trace-refs` option), which introduces the only ABI + incompatibility. + + +Debug options +------------- + +.. cmdoption:: --with-pydebug + + :ref:`Build Python in debug mode `: define the ``Py_DEBUG`` + macro (disabled by default). + +.. cmdoption:: --with-trace-refs + + Enable tracing references for debugging purpose (disabled by default). + + Effects: + + * Define the ``Py_TRACE_REFS`` macro. + * Add :func:`sys.getobjects` function. + * Add :envvar:`PYTHONDUMPREFS` environment variable. + + This build is not ABI compatible with release build (default build) or debug + build (``Py_DEBUG`` and ``Py_REF_DEBUG`` macros). + + .. versionadded:: 3.8 + +.. cmdoption:: --with-assertions + + Build with C assertions enabled (default is no): ``assert(...);`` and + ``_PyObject_ASSERT(...);``. + + If set, the ``NDEBUG`` macro is not defined in the :envvar:`OPT` compiler + variable. + + See also the :option:`--with-pydebug` option (:ref:`debug build + `) which also enables assertions. + + .. versionadded:: 3.6 + +.. cmdoption:: --with-valgrind + + Enable Valgrind support (default is no). + +.. cmdoption:: --with-dtrace + + Enable DTrace support (default is no). + + See :ref:`Instrumenting CPython with DTrace and SystemTap + `. + + .. versionadded:: 3.6 + +.. cmdoption:: --with-address-sanitizer + + Enable AddressSanitizer memory error detector, ``asan`` (default is no). + + .. versionadded:: 3.6 + +.. cmdoption:: --with-memory-sanitizer + + Enable MemorySanitizer allocation error detector, ``msan`` (default is no). + + .. versionadded:: 3.6 + +.. cmdoption:: --with-undefined-behavior-sanitizer + + Enable UndefinedBehaviorSanitizer undefined behaviour detector, ``ubsan`` + (default is no). + + .. versionadded:: 3.6 + + +Linker options +-------------- + +.. cmdoption:: --enable-shared + + Enable building a shared Python library: ``libpython`` (default is no). + +.. cmdoption:: --without-static-libpython + + Do not build ``libpythonMAJOR.MINOR.a`` and do not install ``python.o`` + (built and enabled by default). + + .. versionadded:: 3.10 + + +Libraries options +----------------- + +.. cmdoption:: --with-libs='lib1 ...' + + Link against additional libraries (default is no). + +.. cmdoption:: --with-system-expat + + Build the :mod:`pyexpat` module using an installed ``expat`` library + (default is no). + +.. cmdoption:: --with-system-ffi + + Build the :mod:`_ctypes` extension module using an installed ``ffi`` + library, see the :mod:`ctypes` module (default is system-dependent). + +.. cmdoption:: --with-system-libmpdec + + Build the ``_decimal`` extension module using an installed ``mpdec`` + library, see the :mod:`decimal` module (default is no). + + .. versionadded:: 3.3 + +.. cmdoption:: --with-readline=editline + + Use ``editline`` library for backend of the :mod:`readline` module. + + Define the ``WITH_EDITLINE`` macro. + + .. versionadded:: 3.10 + +.. cmdoption:: --without-readline + + Don't build the :mod:`readline` module (built by default). + + Don't define the ``HAVE_LIBREADLINE`` macro. + + .. versionadded:: 3.10 + +.. cmdoption:: --with-tcltk-includes='-I...' + + Override search for Tcl and Tk include files. + +.. cmdoption:: --with-tcltk-libs='-L...' + + Override search for Tcl and Tk libraries. + +.. cmdoption:: --with-libm=STRING + + Override ``libm`` math library to *STRING* (default is system-dependent). + +.. cmdoption:: --with-libc=STRING + + Override ``libc`` C library to *STRING* (default is system-dependent). + +.. cmdoption:: --with-openssl=DIR + + Root of the OpenSSL directory. + + .. versionadded:: 3.7 + +.. cmdoption:: --with-openssl-rpath=[no|auto|DIR] + + Set runtime library directory (rpath) for OpenSSL libraries: + + * ``no`` (default): don't set rpath; + * ``auto``: auto-detect rpath from :option:`--with-openssl` and + ``pkg-config``; + * *DIR*: set an explicit rpath. + + .. versionadded:: 3.10 + + +Security Options +---------------- + +.. cmdoption:: --with-hash-algorithm=[fnv|siphash24] + + Select hash algorithm for use in ``Python/pyhash.c``: + + * ``siphash24`` (default). + * ``fnv``; + + .. versionadded:: 3.4 + +.. cmdoption:: --with-builtin-hashlib-hashes=md5,sha1,sha256,sha512,sha3,blake2 + + Built-in hash modules: + + * ``md5``; + * ``sha1``; + * ``sha256``; + * ``sha512``; + * ``sha3`` (with shake); + * ``blake2``. + + .. versionadded:: 3.9 + +.. cmdoption:: --with-ssl-default-suites=[python|openssl|STRING] + + Override the OpenSSL default cipher suites string: + + * ``python`` (default): use Python's preferred selection; + * ``openssl``: leave OpenSSL's defaults untouched; + * *STRING*: use a custom string + + See the :mod:`ssl` module. + + .. versionadded:: 3.7 + + .. versionchanged:: 3.10 + + The settings ``python`` and *STRING* also set TLS 1.2 as minimum + protocol version. + +macOS Options +------------- + +See ``Mac/README.rst``. + +.. cmdoption:: --enable-universalsdk +.. cmdoption:: --enable-universalsdk=SDKDIR + + Create a universal binary build. *SDKDIR* specifies which macOS SDK should + be used to perform the build (default is no). + +.. cmdoption:: --enable-framework +.. cmdoption:: --enable-framework=INSTALLDIR + + Create a Python.framework rather than a traditional Unix install. Optional + *INSTALLDIR* specifies the installation path (default is no). + +.. cmdoption:: --with-universal-archs=ARCH + + Specify the kind of universal binary that should be created. This option is + only valid when :option:`--enable-universalsdk` is set. + + Options: + + * ``universal2``; + * ``32-bit``; + * ``64-bit``; + * ``3-way``; + * ``intel``; + * ``intel-32``; + * ``intel-64``; + * ``all``. + +.. cmdoption:: --with-framework-name=FRAMEWORK + + Specify the name for the python framework on macOS only valid when + :option:`--enable-framework` is set (default: ``Python``). + + +Python Build System +=================== + +Main files of the build system +------------------------------ + +* :file:`configure.ac` => :file:`configure`; +* :file:`Makefile.pre.in` => :file:`Makefile` (created by :file:`configure`); +* :file:`pyconfig.h` (created by :file:`configure`); +* :file:`Modules/Setup`: C extensions built by the Makefile using + :file:`Module/makesetup` shell script; +* :file:`setup.py`: C extensions built using the :mod:`distutils` module. + +Main build steps +---------------- + +* C files (``.c``) are built as object files (``.o``). +* A static ``libpython`` library (``.a``) is created from objects files. +* ``python.o`` and the static ``libpython`` library are linked into the + final ``python`` program. +* C extensions are built by the Makefile (see :file:`Modules/Setup`) + and ``python setup.py build``. + +Main Makefile targets +--------------------- + +* ``make``: Build Python with the standard library. +* ``make platform:``: build the ``python`` program, but don't build the + standard library extension modules. +* ``make profile-opt``: build Python using Profile Guided Optimization (PGO). + You can use the configure :option:`--enable-optimizations` option to make + this the default target of the ``make`` command (``make all`` or just + ``make``). +* ``make buildbottest``: Build Python and run the Python test suite, the same + way than buildbots test Python. Set ``TESTTIMEOUT`` variable (in seconds) + to change the test timeout (1200 by default: 20 minutes). +* ``make install``: Build and install Python. +* ``make regen-all``: Regenerate (almost) all generated files; + ``make regen-stdlib-module-names`` and ``autoconf`` must be run separately + for the remaining generated files. +* ``make clean``: Remove built files. +* ``make distclean``: Same than ``make clean``, but remove also files created + by the configure script. + +C extensions +------------ + +Some C extensions are built as built-in modules, like the ``sys`` module. +They are built with the ``Py_BUILD_CORE_BUILTIN`` macro defined. +Built-in modules have no ``__file__`` attribute:: + + >>> import sys + >>> sys + + >>> sys.__file__ + Traceback (most recent call last): + File "", line 1, in + AttributeError: module 'sys' has no attribute '__file__' + +Other C extensins are built as dynamic libraires, like the ``_asyncio`` module. +They are built with the ``Py_BUILD_CORE_MODULE`` macro defined. +Example on Linux x86-64:: + + >>> import _asyncio + >>> _asyncio + + >>> _asyncio.__file__ + '/usr/lib64/python3.9/lib-dynload/_asyncio.cpython-39-x86_64-linux-gnu.so' + +:file:`Modules/Setup` is used to generate Makefile targets to build C extensions. +At the beginning of the files, C extensions are built as built-in modules. +Extensions defined after the ``*shared*`` marker are built as dynamic libraries. + +The :file:`setup.py` script only builds C extensions as shared libraries using +the :mod:`distutils` module. + +The :c:macro:`PyAPI_FUNC()`, :c:macro:`PyAPI_API()` and +:c:macro:`PyMODINIT_FUNC()` macros of :file:`Include/pyport.h` are defined +differently depending if the ``Py_BUILD_CORE_MODULE`` macro is defined: + +* Use ``Py_EXPORTED_SYMBOL`` if the ``Py_BUILD_CORE_MODULE`` is defined +* Use ``Py_IMPORTED_SYMBOL`` otherwise. + +If the ``Py_BUILD_CORE_BUILTIN`` macro is used by mistake on a C extension +built as a shared library, its ``PyInit_xxx()`` function is not exported, +causing an :exc:`ImportError` on import. + + +Compiler and linker flags +========================= + +Options set by the ``./configure`` script and environment variables and used by +``Makefile``. + +Preprocessor flags +------------------ + +.. envvar:: CONFIGURE_CPPFLAGS + + Value of :envvar:`CPPFLAGS` variable passed to the ``./configure`` script. + + .. versionadded:: 3.6 + +.. envvar:: CPPFLAGS + + (Objective) C/C++ preprocessor flags, e.g. ``-I`` if you have + headers in a nonstandard directory ````. + + Both :envvar:`CPPFLAGS` and :envvar:`LDFLAGS` need to contain the shell's + value for setup.py to be able to build extension modules using the + directories specified in the environment variables. + +.. envvar:: BASECPPFLAGS + + .. versionadded:: 3.4 + +.. envvar:: PY_CPPFLAGS + + Extra preprocessor flags added for building the interpreter object files. + + Default: ``$(BASECPPFLAGS) -I. -I$(srcdir)/Include $(CONFIGURE_CPPFLAGS) $(CPPFLAGS)``. + + .. versionadded:: 3.2 + +Compiler flags +-------------- + +.. envvar:: CC + + C compiler command. + + Example: ``gcc -pthread``. + +.. envvar:: MAINCC + + C compiler command used to build the ``main()`` function of programs like + ``python``. + + Variable set by the :option:`--with-cxx-main` option of the configure + script. + + Default: ``$(CC)``. + +.. envvar:: CXX + + C++ compiler command. + + Used if the :option:`--with-cxx-main` option is used. + + Example: ``g++ -pthread``. + +.. envvar:: CFLAGS + + C compiler flags. + +.. envvar:: CFLAGS_NODIST + + :envvar:`CFLAGS_NODIST` is used for building the interpreter and stdlib C + extensions. Use it when a compiler flag should *not* be part of the + distutils :envvar:`CFLAGS` once Python is installed (:issue:`21121`). + + .. versionadded:: 3.5 + +.. envvar:: EXTRA_CFLAGS + + Extra C compiler flags. + +.. envvar:: CONFIGURE_CFLAGS + + Value of :envvar:`CFLAGS` variable passed to the ``./configure`` + script. + + .. versionadded:: 3.2 + +.. envvar:: CONFIGURE_CFLAGS_NODIST + + Value of :envvar:`CFLAGS_NODIST` variable passed to the ``./configure`` + script. + + .. versionadded:: 3.5 + +.. envvar:: BASECFLAGS + + Base compiler flags. + +.. envvar:: OPT + + Optimization flags. + +.. envvar:: CFLAGS_ALIASING + + Strict or non-strict aliasing flags used to compile ``Python/dtoa.c``. + + .. versionadded:: 3.7 + +.. envvar:: CCSHARED + + Compiler flags used to build a shared library. + + For example, ``-fPIC`` is used on Linux and on BSD. + +.. envvar:: CFLAGSFORSHARED + + Extra C flags added for building the interpreter object files. + + Default: ``$(CCSHARED)`` when :option:`--enable-shared` is used, or an empty + string otherwise. + +.. envvar:: PY_CFLAGS + + Default: ``$(BASECFLAGS) $(OPT) $(CONFIGURE_CFLAGS) $(CFLAGS) $(EXTRA_CFLAGS)``. + +.. envvar:: PY_CFLAGS_NODIST + + Default: ``$(CONFIGURE_CFLAGS_NODIST) $(CFLAGS_NODIST) -I$(srcdir)/Include/internal``. + + .. versionadded:: 3.5 + +.. envvar:: PY_STDMODULE_CFLAGS + + C flags used for building the interpreter object files. + + Default: ``$(PY_CFLAGS) $(PY_CFLAGS_NODIST) $(PY_CPPFLAGS) $(CFLAGSFORSHARED)``. + + .. versionadded:: 3.7 + +.. envvar:: PY_CORE_CFLAGS + + Default: ``$(PY_STDMODULE_CFLAGS) -DPy_BUILD_CORE``. + + .. versionadded:: 3.2 + +.. envvar:: PY_BUILTIN_MODULE_CFLAGS + + Compiler flags to build a standard library extension module as a built-in + module, like the :mod:`posix` module. + + Default: ``$(PY_STDMODULE_CFLAGS) -DPy_BUILD_CORE_BUILTIN``. + + .. versionadded:: 3.8 + +.. envvar:: PURIFY + + Purify command. Purify is a memory debugger program. + + Default: empty string (not used). + + +Linker flags +------------ + +.. envvar:: LINKCC + + Linker command used to build programs like ``python`` and ``_testembed``. + + Default: ``$(PURIFY) $(MAINCC)``. + +.. envvar:: CONFIGURE_LDFLAGS + + Value of :envvar:`LDFLAGS` variable passed to the ``./configure`` script. + + Avoid assigning :envvar:`CFLAGS`, :envvar:`LDFLAGS`, etc. so users can use + them on the command line to append to these values without stomping the + pre-set values. + + .. versionadded:: 3.2 + +.. envvar:: LDFLAGS_NODIST + + :envvar:`LDFLAGS_NODIST` is used in the same manner as + :envvar:`CFLAGS_NODIST`. Use it when a linker flag should *not* be part of + the distutils :envvar:`LDFLAGS` once Python is installed (:issue:`35257`). + +.. envvar:: CONFIGURE_LDFLAGS_NODIST + + Value of :envvar:`LDFLAGS_NODIST` variable passed to the ``./configure`` + script. + + .. versionadded:: 3.8 + +.. envvar:: LDFLAGS + + Linker flags, e.g. ``-L`` if you have libraries in a nonstandard + directory ````. + + Both :envvar:`CPPFLAGS` and :envvar:`LDFLAGS` need to contain the shell's + value for setup.py to be able to build extension modules using the + directories specified in the environment variables. + +.. envvar:: LIBS + + Linker flags to pass libraries to the linker when linking the Python + executable. + + Example: ``-lrt``. + +.. envvar:: LDSHARED + + Command to build a shared library. + + Default: ``@LDSHARED@ $(PY_LDFLAGS)``. + +.. envvar:: BLDSHARED + + Command to build ``libpython`` shared library. + + Default: ``@BLDSHARED@ $(PY_CORE_LDFLAGS)``. + +.. envvar:: PY_LDFLAGS + + Default: ``$(CONFIGURE_LDFLAGS) $(LDFLAGS)``. + +.. envvar:: PY_LDFLAGS_NODIST + + Default: ``$(CONFIGURE_LDFLAGS_NODIST) $(LDFLAGS_NODIST)``. + + .. versionadded:: 3.8 + +.. envvar:: PY_CORE_LDFLAGS + + Linker flags used for building the interpreter object files. + + .. versionadded:: 3.8 diff --git a/Doc/using/index.rst b/Doc/using/index.rst index 4a45121ac2eebd..e1a3111f36a44f 100644 --- a/Doc/using/index.rst +++ b/Doc/using/index.rst @@ -15,6 +15,7 @@ interpreter and things that make working with Python easier. cmdline.rst unix.rst + configure.rst windows.rst mac.rst editors.rst diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index c0a5643fc2c242..1d1fa8bd85d7ed 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -78,9 +78,9 @@ The build process consists of the usual commands:: make make install -Configuration options and caveats for specific Unix platforms are extensively -documented in the :source:`README.rst` file in the root of the Python source -tree. +:ref:`Configuration options ` and caveats for specific Unix +platforms are extensively documented in the :source:`README.rst` file in the +root of the Python source tree. .. warning:: @@ -134,3 +134,55 @@ some Unices may not have the :program:`env` command, so you may need to hardcode ``/usr/bin/python3`` as the interpreter path. To use shell commands in your Python scripts, look at the :mod:`subprocess` module. + +.. _unix_custom_openssl: + +Custom OpenSSL +============== + +1. To use your vendor's OpenSSL configuration and system trust store, locate + the directory with ``openssl.cnf`` file or symlink in ``/etc``. On most + distribution the file is either in ``/etc/ssl`` or ``/etc/pki/tls``. The + directory should also contain a ``cert.pem`` file and/or a ``certs`` + directory. + + .. code-block:: shell-session + + $ find /etc/ -name openssl.cnf -printf "%h\n" + /etc/ssl + +2. Download, build, and install OpenSSL. Make sure you use ``install_sw`` and + not ``install``. The ``install_sw`` target does not override + ``openssl.cnf``. + + .. code-block:: shell-session + + $ curl -O https://www.openssl.org/source/openssl-VERSION.tar.gz + $ tar xzf openssl-VERSION + $ pushd openssl-VERSION + $ ./config \ + --prefix=/usr/local/custom-openssl \ + --openssldir=/etc/ssl + $ make -j1 depend + $ make -j8 + $ make install_sw + $ popd + +3. Build Python with custom OpenSSL + (see the configure `--with-openssl` and `--with-openssl-rpath` options) + + .. code-block:: shell-session + + $ pushd python-3.x.x + $ ./configure -C \ + --with-openssl=/usr/local/custom-openssl \ + --with-openssl-rpath=auto \ + --prefix=/usr/local/python-3.x.x + $ make -j8 + $ make altinstall + +.. note:: + + Patch releases of OpenSSL have a backwards compatible ABI. You don't need + to recompile Python to update OpenSSL. It's sufficient to replace the + custom OpenSSL installation with a newer version. diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 275495bc6d1292..68ee09c565e21f 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -23,8 +23,8 @@ available for application-local distributions. As specified in :pep:`11`, a Python release only supports a Windows platform while Microsoft considers the platform under extended support. This means that -Python |version| supports Windows Vista and newer. If you require Windows XP -support then please install Python 3.4. +Python |version| supports Windows 8.1 and newer. If you require Windows 7 +support, please install Python 3.8. There are a number of different installers available for Windows, each with certain benefits and downsides. @@ -339,6 +339,11 @@ full write access to shared locations such as ``TEMP`` and the registry. Instead, it will write to a private copy. If your scripts must modify the shared locations, you will need to install the full installer. +For more detail on the technical basis for these limitations, please consult +Microsoft's documentation on packaged full-trust apps, currently available at +`docs.microsoft.com/en-us/windows/msix/desktop/desktop-to-uwp-behind-the-scenes +`_ + .. _windows-nuget: @@ -614,21 +619,14 @@ Page). Python uses it for the default encoding of text files (e.g. This may cause issues because UTF-8 is widely used on the internet and most Unix systems, including WSL (Windows Subsystem for Linux). -You can use UTF-8 mode to change the default text encoding to UTF-8. -You can enable UTF-8 mode via the ``-X utf8`` command line option, or -the ``PYTHONUTF8=1`` environment variable. See :envvar:`PYTHONUTF8` for -enabling UTF-8 mode, and :ref:`setting-envvars` for how to modify -environment variables. - -When UTF-8 mode is enabled: +You can use the :ref:`Python UTF-8 Mode ` to change the default text +encoding to UTF-8. You can enable the :ref:`Python UTF-8 Mode ` via +the ``-X utf8`` command line option, or the ``PYTHONUTF8=1`` environment +variable. See :envvar:`PYTHONUTF8` for enabling UTF-8 mode, and +:ref:`setting-envvars` for how to modify environment variables. -* :func:`locale.getpreferredencoding` returns ``'UTF-8'`` instead of - the system encoding. This function is used for the default text - encoding in many places, including :func:`open`, :class:`Popen`, - :meth:`Path.read_text`, etc. -* :data:`sys.stdin`, :data:`sys.stdout`, and :data:`sys.stderr` - all use UTF-8 as their text encoding. -* You can still use the system encoding via the "mbcs" codec. +When the :ref:`Python UTF-8 Mode ` is enabled, you can still use the +system encoding (the ANSI Code Page) via the "mbcs" codec. Note that adding ``PYTHONUTF8=1`` to the default environment variables will affect all Python 3.7+ applications on your system. @@ -641,7 +639,8 @@ temporarily or use the ``-X utf8`` command line option. on Windows for: * Console I/O including standard I/O (see :pep:`528` for details). - * The filesystem encoding (see :pep:`529` for details). + * The :term:`filesystem encoding ` + (see :pep:`529` for details). .. _launcher: @@ -1107,7 +1106,7 @@ shipped with PyWin32. It is an embeddable IDE with a built-in debugger. cx_Freeze --------- -`cx_Freeze `_ is a :mod:`distutils` +`cx_Freeze `_ is a :mod:`distutils` extension (see :ref:`extending-distutils`) which wraps Python scripts into executable Windows programs (:file:`{*}.exe` files). When you have done this, you can distribute your application without requiring your users to install @@ -1150,8 +1149,6 @@ For extension modules, consult :ref:`building-on-windows`. MinGW gcc under Windows" or "Installing Python extension with distutils and without Microsoft Visual C++" by Sébastien Sauvage, 2003 - `MingW -- Python extensions `_ - Other Platforms =============== diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index b2c6d10ba8deb7..eb452b07f55f61 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -65,6 +65,7 @@ Summary -- Release highlights .. PEP-sized items next. +* :pep:`644`, require OpenSSL 1.1.1 or newer New Features @@ -72,60 +73,605 @@ New Features .. _whatsnew310-pep563: -PEP 563: Postponed Evaluation of Annotations Becomes Default ------------------------------------------------------------- - -In Python 3.7, postponed evaluation of annotations was added, -to be enabled with a ``from __future__ import annotations`` -directive. In 3.10 this became the default behavior, even -without that future directive. With this being default, all -annotations stored in :attr:`__annotations__` will be strings. -If needed, annotations can be resolved at runtime using -:func:`typing.get_type_hints`. See :pep:`563` for a full -description. Also, the :func:`inspect.signature` will try to -resolve types from now on, and when it fails it will fall back to -showing the string annotations. (Contributed by Batuhan Taskaya -in :issue:`38605`.) +Parenthesized context managers +------------------------------ -* The :class:`int` type has a new method :meth:`int.bit_count`, returning the - number of ones in the binary expansion of a given integer, also known - as the population count. (Contributed by Niklas Fiekas in :issue:`29882`.) +Using enclosing parentheses for continuation across multiple lines +in context managers is now supported. This allows formatting a long +collection of context managers in multiple lines in a similar way +as it was previously possible with import statements. For instance, +all these examples are now valid: -* The views returned by :meth:`dict.keys`, :meth:`dict.values` and - :meth:`dict.items` now all have a ``mapping`` attribute that gives a - :class:`types.MappingProxyType` object wrapping the original - dictionary. (Contributed by Dennis Sweeney in :issue:`40890`.) +.. code-block:: python -* :pep:`618`: The :func:`zip` function now has an optional ``strict`` flag, used - to require that all the iterables have an equal length. + with (CtxManager() as example): + ... -PEP 613: TypeAlias Annotation ------------------------------ + with ( + CtxManager1(), + CtxManager2() + ): + ... -:pep:`484` introduced the concept of type aliases, only requiring them to be -top-level unannotated assignments. This simplicity sometimes made it difficult -for type checkers to distinguish between type aliases and ordinary assignments, -especially when forward references or invalid types were involved. Compare:: + with (CtxManager1() as example, + CtxManager2()): + ... - StrCache = 'Cache[str]' # a type alias - LOG_PREFIX = 'LOG[DEBUG]' # a module constant + with (CtxManager1(), + CtxManager2() as example): + ... -Now the :mod:`typing` module has a special annotation :data:`TypeAlias` to -declare type aliases more explicitly:: + with ( + CtxManager1() as example1, + CtxManager2() as example2 + ): + ... - StrCache: TypeAlias = 'Cache[str]' # a type alias - LOG_PREFIX = 'LOG[DEBUG]' # a module constant +it is also possible to use a trailing comma at the end of the +enclosed group: -See :pep:`613` for more details. +.. code-block:: python -(Contributed by Mikhail Golubev in :issue:`41923`.) + with ( + CtxManager1() as example1, + CtxManager2() as example2, + CtxManager3() as example3, + ): + ... + +This new syntax uses the non LL(1) capacities of the new parser. +Check :pep:`617` for more details. + +(Contributed by Guido van Rossum, Pablo Galindo and Lysandros Nikolaou +in :issue:`12782` and :issue:`40334`.) + + +Better error messages +--------------------- + +SyntaxErrors +~~~~~~~~~~~~ + +When parsing code that contains unclosed parentheses or brackets the interpreter +now includes the location of the unclosed bracket of parentheses instead of displaying +*SyntaxError: unexpected EOF while parsing* or pointing to some incorrect location. +For instance, consider the following code (notice the unclosed '{'): + +.. code-block:: python + + expected = {9: 1, 18: 2, 19: 2, 27: 3, 28: 3, 29: 3, 36: 4, 37: 4, + 38: 4, 39: 4, 45: 5, 46: 5, 47: 5, 48: 5, 49: 5, 54: 6, + some_other_code = foo() + +previous versions of the interpreter reported confusing places as the location of +the syntax error: + +.. code-block:: python + + File "example.py", line 3 + some_other_code = foo() + ^ + SyntaxError: invalid syntax + +but in Python3.10 a more informative error is emitted: + +.. code-block:: python + + File "example.py", line 1 + expected = {9: 1, 18: 2, 19: 2, 27: 3, 28: 3, 29: 3, 36: 4, 37: 4, + ^ + SyntaxError: '{' was never closed + + +In a similar way, errors involving unclosed string literals (single and triple +quoted) now point to the start of the string instead of reporting EOF/EOL. + +These improvements are inspired by previous work in the PyPy interpreter. + +(Contributed by Pablo Galindo in :issue:`42864` and Batuhan Taskaya in +:issue:`40176`.) + +:exc:`SyntaxError` exceptions raised by the intepreter will now highlight the +full error range of the expression that consistutes the syntax error itself, +instead of just where the problem is detected. In this way, instead of displaying +(before Python 3.10): + +.. code-block:: python + + >>> foo(x, z for z in range(10), t, w) + File "", line 1 + foo(x, z for z in range(10), t, w) + ^ + SyntaxError: Generator expression must be parenthesized + +now Python 3.10 will display the exception as: + +.. code-block:: python + + >>> foo(x, z for z in range(10), t, w) + File "", line 1 + foo(x, z for z in range(10), t, w) + ^^^^^^^^^^^^^^^^^^^^ + SyntaxError: Generator expression must be parenthesized + +This improvement has been contributed by Pablo Galindo in :issue:`43914`. + +A considerable amount of new specialized messages for :exc:`SyntaxError` exceptions +have been incorporated. Some of the most notable ones: + +* Missing ``:`` before blocks: + + .. code-block:: python + + >>> if rocket.position > event_horizon + File "", line 1 + if rocket.position > event_horizon + ^ + SyntaxError: expected ':' + + (Contributed by Pablo Galindo in :issue:`42997`) + +* Unparenthesised tuples in comprehensions targets: + + .. code-block:: python + + >>> {x,y for x,y in range(100)} + File "", line 1 + {x,y for x,y in range(100)} + ^ + SyntaxError: did you forget parentheses around the comprehension target? + + (Contributed by Pablo Galindo in :issue:`43017`) + +* Missing commas in collection literals and between expressions: + + .. code-block:: python + + >>> items = { + ... x: 1, + ... y: 2 + ... z: 3, + File "", line 3 + y: 2 + ^ + SyntaxError: invalid syntax. Perhaps you forgot a comma? + + (Contributed by Pablo Galindo in :issue:`43822`) + +* Exception groups without parentheses: + + .. code-block:: python + + >>> try: + ... build_dyson_sphere() + ... except NotEnoughScienceError, NotEnoughResourcesError: + File "", line 3 + except NotEnoughScienceError, NotEnoughResourcesError: + ^ + SyntaxError: exception group must be parenthesized + + (Contributed by Pablo Galindo in :issue:`43149`) + +* Missing ``:`` and values in dictionary literals: + + .. code-block:: python + + >>> values = { + ... x: 1, + ... y: 2, + ... z: + ... } + File "", line 4 + z: + ^ + SyntaxError: expression expected after dictionary key and ':' + + >>> values = {x:1, y:2, z w:3} + File "", line 1 + values = {x:1, y:2, z w:3} + ^ + SyntaxError: ':' expected after dictionary key + + (Contributed by Pablo Galindo in :issue:`43823`) + +* Usage of ``=`` instead of ``==`` in comparisons: + + .. code-block:: python + + >>> if rocket.position = event_horizon: + File "", line 1 + if rocket.position = event_horizon: + ^ + SyntaxError: cannot assign to attribute here. Maybe you meant '==' instead of '='? + + (Contributed by Pablo Galindo in :issue:`43797`) + +* Usage of ``*`` in f-strings: + + .. code-block:: python + + >>> f"Black holes {*all_black_holes} and revelations" + File "", line 1 + (*all_black_holes) + ^ + SyntaxError: f-string: cannot use starred expression here + + (Contributed by Pablo Galindo in :issue:`41064`) + +IndentationErrors +~~~~~~~~~~~~~~~~~ + +Many :exc:`IndentationError` exceptions now have more context regarding what kind of block +was expecting an indentation, including the location of the statement: + +.. code-block:: python + + >>> def foo(): + ... if lel: + ... x = 2 + File "", line 3 + x = 2 + ^ + IndentationError: expected an indented block after 'if' statement in line 2 + + +AttributeErrors +~~~~~~~~~~~~~~~ + +When printing :exc:`AttributeError`, :c:func:`PyErr_Display` will offer +suggestions of similar attribute names in the object that the exception was +raised from: + +.. code-block:: python -PEP604: New Type Union Operator -------------------------------- + >>> collections.namedtoplo + Traceback (most recent call last): + File "", line 1, in + AttributeError: module 'collections' has no attribute 'namedtoplo'. Did you mean: namedtuple? + +(Contributed by Pablo Galindo in :issue:`38530`.) + +NameErrors +~~~~~~~~~~ + +When printing :exc:`NameError` raised by the interpreter, :c:func:`PyErr_Display` +will offer suggestions of similar variable names in the function that the exception +was raised from: + +.. code-block:: python + + >>> schwarzschild_black_hole = None + >>> schwarschild_black_hole + Traceback (most recent call last): + File "", line 1, in + NameError: name 'schwarschild_black_hole' is not defined. Did you mean: schwarzschild_black_hole? + +(Contributed by Pablo Galindo in :issue:`38530`.) + +PEP 626: Precise line numbers for debugging and other tools +----------------------------------------------------------- + +PEP 626 brings more precise and reliable line numbers for debugging, profiling and coverage tools. +Tracing events, with the correct line number, are generated for all lines of code executed and only for lines of code that are executed. + +The ``f_lineno`` attribute of frame objects will always contain the expected line number. + +The ``co_lnotab`` attribute of code objects is deprecated and will be removed in 3.12. +Code that needs to convert from offset to line number should use the new ``co_lines()`` method instead. + +PEP 634: Structural Pattern Matching +------------------------------------ + +Structural pattern matching has been added in the form of a *match statement* +and *case statements* of patterns with associated actions. Patterns +consist of sequences, mappings, primitive data types as well as class instances. +Pattern matching enables programs to extract information from complex data types, +branch on the structure of data, and apply specific actions based on different +forms of data. + +Syntax and operations +~~~~~~~~~~~~~~~~~~~~~ + +The generic syntax of pattern matching is:: + + match subject: + case : + + case : + + case : + + case _: + + +A match statement takes an expression and compares its value to successive +patterns given as one or more case blocks. Specifically, pattern matching +operates by: + + 1. using data with type and shape (the ``subject``) + 2. evaluating the ``subject`` in the ``match`` statement + 3. comparing the subject with each pattern in a ``case`` statement + from top to bottom until a match is confirmed. + 4. executing the action associated with the pattern of the confirmed + match + 5. If an exact match is not confirmed, the last case, a wildcard ``_``, + if provided, will be used as the matching case. If an exact match is + not confirmed and a wildcard case does not exist, the entire match + block is a no-op. + +Declarative approach +~~~~~~~~~~~~~~~~~~~~ + +Readers may be aware of pattern matching through the simple example of matching +a subject (data object) to a literal (pattern) with the switch statement found +in C, Java or JavaScript (and many other languages). Often the switch statement +is used for comparison of an object/expression with case statements containing +literals. + +More powerful examples of pattern matching can be found in languages, such as +Scala and Elixir. With structural pattern matching, the approach is "declarative" and +explicitly states the conditions (the patterns) for data to match. + +While an "imperative" series of instructions using nested "if" statements +could be used to accomplish something similar to structural pattern matching, +it is less clear than the "declarative" approach. Instead the "declarative" +approach states the conditions to meet for a match and is more readable through +its explicit patterns. While structural pattern matching can be used in its +simplest form comparing a variable to a literal in a case statement, its +true value for Python lies in its handling of the subject's type and shape. + +Simple pattern: match to a literal +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Let's look at this example as pattern matching in its simplest form: a value, +the subject, being matched to several literals, the patterns. In the example +below, ``status`` is the subject of the match statement. The patterns are +each of the case statements, where literals represent request status codes. +The associated action to the case is executed after a match:: + + def http_error(status): + match status: + case 400: + return "Bad request" + case 404: + return "Not found" + case 418: + return "I'm a teapot" + case _: + return "Something's wrong with the Internet" + +If the above function is passed a ``status`` of 418, "I'm a teapot" is returned. +If the above function is passed a ``status`` of 500, the case statement with +``_`` will match as a wildcard, and "Something's wrong with the Internet" is +returned. +Note the last block: the variable name, ``_``, acts as a *wildcard* and insures +the subject will always match. The use of ``_`` is optional. + +You can combine several literals in a single pattern using ``|`` ("or"):: + + case 401 | 403 | 404: + return "Not allowed" + +Behavior without the wildcard +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If we modify the above example by removing the last case block, the example +becomes:: + + def http_error(status): + match status: + case 400: + return "Bad request" + case 404: + return "Not found" + case 418: + return "I'm a teapot" + +Without the use of ``_`` in a case statement, a match may not exist. If no +match exists, the behavior is a no-op. For example, if ``status`` of 500 is +passed, a no-op occurs. + +Patterns with a literal and variable +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Patterns can look like unpacking assignments, and a pattern may be used to bind +variables. In this example, a data point can be unpacked to its x-coordinate +and y-coordinate:: + + # point is an (x, y) tuple + match point: + case (0, 0): + print("Origin") + case (0, y): + print(f"Y={y}") + case (x, 0): + print(f"X={x}") + case (x, y): + print(f"X={x}, Y={y}") + case _: + raise ValueError("Not a point") + +The first pattern has two literals, ``(0, 0)``, and may be thought of as an +extension of the literal pattern shown above. The next two patterns combine a +literal and a variable, and the variable *binds* a value from the subject +(``point``). The fourth pattern captures two values, which makes it +conceptually similar to the unpacking assignment ``(x, y) = point``. + +Patterns and classes +~~~~~~~~~~~~~~~~~~~~ + +If you are using classes to structure your data, you can use as a pattern +the class name followed by an argument list resembling a constructor. This +pattern has the ability to capture class attributes into variables:: + + class Point: + x: int + y: int + + def location(point): + match point: + case Point(x=0, y=0): + print("Origin is the point's location.") + case Point(x=0, y=y): + print(f"Y={y} and the point is on the y-axis.") + case Point(x=x, y=0): + print(f"X={x} and the point is on the x-axis.") + case Point(): + print("The point is located somewhere else on the plane.") + case _: + print("Not a point") + +Patterns with positional parameters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can use positional parameters with some builtin classes that provide an +ordering for their attributes (e.g. dataclasses). You can also define a specific +position for attributes in patterns by setting the ``__match_args__`` special +attribute in your classes. If it's set to ("x", "y"), the following patterns +are all equivalent (and all bind the ``y`` attribute to the ``var`` variable):: + + Point(1, var) + Point(1, y=var) + Point(x=1, y=var) + Point(y=var, x=1) + +Nested patterns +~~~~~~~~~~~~~~~ + +Patterns can be arbitrarily nested. For example, if our data is a short +list of points, it could be matched like this:: + + match points: + case []: + print("No points in the list.") + case [Point(0, 0)]: + print("The origin is the only point in the list.") + case [Point(x, y)]: + print(f"A single point {x}, {y} is in the list.") + case [Point(0, y1), Point(0, y2)]: + print(f"Two points on the Y axis at {y1}, {y2} are in the list.") + case _: + print("Something else is found in the list.") + +Complex patterns and the wildcard +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To this point, the examples have used ``_`` alone in the last case statement. +A wildcard can be used in more complex patterns, such as ``('error', code, _)``. +For example:: + + match test_variable: + case ('warning', code, 40): + print("A warning has been received.") + case ('error', code, _): + print(f"An error {code} occurred.") + +In the above case, ``test_variable`` will match for ('error', code, 100) and +('error', code, 800). + +Guard +~~~~~ + +We can add an ``if`` clause to a pattern, known as a "guard". If the +guard is false, ``match`` goes on to try the next case block. Note +that value capture happens before the guard is evaluated:: + + match point: + case Point(x, y) if x == y: + print(f"The point is located on the diagonal Y=X at {x}.") + case Point(x, y): + print(f"Point is not on the diagonal.") + +Other Key Features +~~~~~~~~~~~~~~~~~~ + +Several other key features: + +- Like unpacking assignments, tuple and list patterns have exactly the + same meaning and actually match arbitrary sequences. Technically, + the subject must be an instance of ``collections.abc.Sequence``. + Therefore, an important exception is that patterns don't match iterators. + Also, to prevent a common mistake, sequence patterns don't match strings. + +- Sequence patterns support wildcards: ``[x, y, *rest]`` and ``(x, y, + *rest)`` work similar to wildcards in unpacking assignments. The + name after ``*`` may also be ``_``, so ``(x, y, *_)`` matches a sequence + of at least two items without binding the remaining items. + +- Mapping patterns: ``{"bandwidth": b, "latency": l}`` captures the + ``"bandwidth"`` and ``"latency"`` values from a dict. Unlike sequence + patterns, extra keys are ignored. A wildcard ``**rest`` is also + supported. (But ``**_`` would be redundant, so is not allowed.) + +- Subpatterns may be captured using the ``as`` keyword:: + + case (Point(x1, y1), Point(x2, y2) as p2): ... + + This binds x1, y1, x2, y2 like you would expect without the ``as`` clause, + and p2 to the entire second item of the subject. + +- Most literals are compared by equality. However, the singletons ``True``, + ``False`` and ``None`` are compared by identity. + +- Named constants may be used in patterns. These named constants must be + dotted names to prevent the constant from being interpreted as a capture + variable:: + + from enum import Enum + class Color(Enum): + RED = 0 + GREEN = 1 + BLUE = 2 + + match color: + case Color.RED: + print("I see red!") + case Color.GREEN: + print("Grass is green") + case Color.BLUE: + print("I'm feeling the blues :(") + +For the full specification see :pep:`634`. Motivation and rationale +are in :pep:`635`, and a longer tutorial is in :pep:`636`. + + +.. _whatsnew310-pep597: + +Optional ``EncodingWarning`` and ``encoding="locale"`` option +------------------------------------------------------------- + +The default encoding of :class:`TextIOWrapper` and :func:`open` is +platform and locale dependent. Since UTF-8 is used on most Unix +platforms, omitting ``encoding`` option when opening UTF-8 files +(e.g. JSON, YAML, TOML, Markdown) is a very common bug. For example:: + + # BUG: "rb" mode or encoding="utf-8" should be used. + with open("data.json") as f: + data = json.load(f) + +To find this type of bug, optional ``EncodingWarning`` is added. +It is emitted when :data:`sys.flags.warn_default_encoding ` +is true and locale-specific default encoding is used. + +``-X warn_default_encoding`` option and :envvar:`PYTHONWARNDEFAULTENCODING` +are added to enable the warning. + +See :ref:`io-text-encoding` for more information. + + +New Features Related to Type Hints +================================== + +This section covers major changes affecting :pep:`484` type hints and +the :mod:`typing` module. + + +PEP 604: New Type Union Operator +-------------------------------- A new type union operator was introduced which enables the syntax ``X | Y``. This provides a cleaner way of expressing 'either type X or type Y' instead of -using :data:`typing.Union`, especially in type hints (annotations). +using :data:`typing.Union`, especially in type hints. In previous versions of Python, to apply a type hint for functions accepting arguments of multiple types, :data:`typing.Union` was used:: @@ -134,19 +680,95 @@ arguments of multiple types, :data:`typing.Union` was used:: return number ** 2 -Now, type hints can be written in a more succinct manner:: +Type hints can now be written in a more succinct manner:: def square(number: int | float) -> int | float: return number ** 2 -See :pep:`604` for more details. +This new syntax is also accepted as the second argument to :func:`isinstance` +and :func:`issubclass`:: + + >>> isinstance(1, int | str) + True + +See :ref:`types-union` and :pep:`604` for more details. (Contributed by Maggie Moss and Philippe Prados in :issue:`41428`.) + +PEP 612: Parameter Specification Variables +------------------------------------------ + +Two new options to improve the information provided to static type checkers for +:pep:`484`\ 's ``Callable`` have been added to the :mod:`typing` module. + +The first is the parameter specification variable. They are used to forward the +parameter types of one callable to another callable -- a pattern commonly +found in higher order functions and decorators. Examples of usage can be found +in :class:`typing.ParamSpec`. Previously, there was no easy way to type annotate +dependency of parameter types in such a precise manner. + +The second option is the new ``Concatenate`` operator. It's used in conjunction +with parameter specification variables to type annotate a higher order callable +which adds or removes parameters of another callable. Examples of usage can +be found in :class:`typing.Concatenate`. + +See :class:`typing.Callable`, :class:`typing.ParamSpec`, +:class:`typing.Concatenate`, :class:`typing.ParamSpecArgs`, +:class:`typing.ParamSpecKwargs`, and :pep:`612` for more details. + +(Contributed by Ken Jin in :issue:`41559`, with minor enhancements by Jelle +Zijlstra in :issue:`43783`. PEP written by Mark Mendoza.) + + +PEP 613: TypeAlias +------------------ + +:pep:`484` introduced the concept of type aliases, only requiring them to be +top-level unannotated assignments. This simplicity sometimes made it difficult +for type checkers to distinguish between type aliases and ordinary assignments, +especially when forward references or invalid types were involved. Compare:: + + StrCache = 'Cache[str]' # a type alias + LOG_PREFIX = 'LOG[DEBUG]' # a module constant + +Now the :mod:`typing` module has a special value :data:`TypeAlias` +which lets you declare type aliases more explicitly:: + + StrCache: TypeAlias = 'Cache[str]' # a type alias + LOG_PREFIX = 'LOG[DEBUG]' # a module constant + +See :pep:`613` for more details. + +(Contributed by Mikhail Golubev in :issue:`41923`.) + +PEP 647: User-Defined Type Guards +--------------------------------- + +:data:`TypeGuard` has been added to the :mod:`typing` module to annotate +type guard functions and improve information provided to static type checkers +during type narrowing. For more information, please see :data:`TypeGuard`\ 's +documentation, and :pep:`647`. + +(Contributed by Ken Jin and Guido van Rossum in :issue:`43766`. +PEP written by Eric Traut.) + Other Language Changes ====================== +* The :class:`int` type has a new method :meth:`int.bit_count`, returning the + number of ones in the binary expansion of a given integer, also known + as the population count. (Contributed by Niklas Fiekas in :issue:`29882`.) + +* The views returned by :meth:`dict.keys`, :meth:`dict.values` and + :meth:`dict.items` now all have a ``mapping`` attribute that gives a + :class:`types.MappingProxyType` object wrapping the original + dictionary. (Contributed by Dennis Sweeney in :issue:`40890`.) + +* :pep:`618`: The :func:`zip` function now has an optional ``strict`` flag, used + to require that all the iterables have an equal length. + * Builtin and extension functions that take integer arguments no longer accept :class:`~decimal.Decimal`\ s, :class:`~fractions.Fraction`\ s and other objects that can be converted to integers only with a loss (e.g. that have @@ -154,6 +776,41 @@ Other Language Changes :meth:`~object.__index__` method). (Contributed by Serhiy Storchaka in :issue:`37999`.) +* If :func:`object.__ipow__` returns :const:`NotImplemented`, the operator will + correctly fall back to :func:`object.__pow__` and :func:`object.__rpow__` as expected. + (Contributed by Alex Shkop in :issue:`38302`.) + +* Assignment expressions can now be used unparenthesized within set literals + and set comprehensions, as well as in sequence indexes (but not slices). + +* Functions have a new ``__builtins__`` attribute which is used to look for + builtin symbols when a function is executed, instead of looking into + ``__globals__['__builtins__']``. The attribute is initialized from + ``__globals__["__builtins__"]`` if it exists, else from the current builtins. + (Contributed by Mark Shannon in :issue:`42990`.) + +* Two new builtin functions -- :func:`aiter` and :func:`anext` have been added + to provide asynchronous counterparts to :func:`iter` and :func:`next`, + respectively. + (Contributed by Joshua Bronson, Daniel Pope, and Justin Wang in :issue:`31861`.) + +* Static methods (:func:`@staticmethod `) and class methods + (:func:`@classmethod `) now inherit the method attributes + (``__module__``, ``__name__``, ``__qualname__``, ``__doc__``, + ``__annotations__``) and have a new ``__wrapped__`` attribute. + Moreover, static methods are now callable as regular functions. + (Contributed by Victor Stinner in :issue:`43682`.) + +* Annotations for complex targets (everything beside ``simple name`` targets + defined by :pep:`526`) no longer cause any runtime effects with ``from __future__ import annotations``. + (Contributed by Batuhan Taskaya in :issue:`42737`.) + +* Class and module objects now lazy-create empty annotations dicts on demand. + The annotations dicts are stored in the object’s ``__dict__`` for + backwards compatibility. This improves the best practices for working + with ``__annotations__``; for more information, please see + :ref:`annotations-howto`. + (Contributed by Larry Hastings in :issue:`43901`.) New Modules =========== @@ -164,18 +821,72 @@ New Modules Improved Modules ================ +asyncio +------- + +Added missing :meth:`~asyncio.events.AbstractEventLoop.connect_accepted_socket` +method. +(Contributed by Alex Grönholm in :issue:`41332`.) + +argparse +-------- + +Misleading phrase "optional arguments" was replaced with "options" in argparse help. Some tests might require adaptation if they rely on exact output match. +(Contributed by Raymond Hettinger in :issue:`9694`.) + +array +----- + +The :meth:`~array.array.index` method of :class:`array.array` now has +optional *start* and *stop* parameters. +(Contributed by Anders Lorentsen and Zackery Spytz in :issue:`31956`.) + base64 ------ Add :func:`base64.b32hexencode` and :func:`base64.b32hexdecode` to support the Base32 Encoding with Extended Hex Alphabet. +bdb +--- + +Add :meth:`~bdb.Breakpoint.clearBreakpoints` to reset all set breakpoints. +(Contributed by Irit Katriel in :issue:`24160`.) + codecs ------ Add a :func:`codecs.unregister` function to unregister a codec search function. (Contributed by Hai Shi in :issue:`41842`.) +collections.abc +--------------- + +The ``__args__`` of the :ref:`parameterized generic ` for +:class:`collections.abc.Callable` are now consistent with :data:`typing.Callable`. +:class:`collections.abc.Callable` generic now flattens type parameters, similar +to what :data:`typing.Callable` currently does. This means that +``collections.abc.Callable[[int, str], str]`` will have ``__args__`` of +``(int, str, str)``; previously this was ``([int, str], str)``. To allow this +change, :class:`types.GenericAlias` can now be subclassed, and a subclass will +be returned when subscripting the :class:`collections.abc.Callable` type. Note +that a :exc:`TypeError` may be raised for invalid forms of parameterizing +:class:`collections.abc.Callable` which may have passed silently in Python 3.9. +(Contributed by Ken Jin in :issue:`42195`.) + +contextlib +---------- + +Add a :func:`contextlib.aclosing` context manager to safely close async generators +and objects representing asynchronously released resources. +(Contributed by Joongi Kim and John Belmonte in :issue:`41229`.) + +Add asynchronous context manager support to :func:`contextlib.nullcontext`. +(Contributed by Tom Gringauz in :issue:`41543`.) + +Add :class:`AsyncContextDecorator`, for supporting usage of async context managers +as decorators. + curses ------ @@ -186,11 +897,76 @@ by :func:`curses.color_content`, :func:`curses.init_color`, support is provided by the underlying ncurses library. (Contributed by Jeffrey Kintscher and Hans Petter Jansson in :issue:`36982`.) +The ``BUTTON5_*`` constants are now exposed in the :mod:`curses` module if +they are provided by the underlying curses library. +(Contributed by Zackery Spytz in :issue:`39273`.) + +dataclasses +----------- + +Added ``slots`` parameter in :func:`dataclasses.dataclass` decorator. +(Contributed by Yurii Karabas in :issue:`42269`) + +.. _distutils-deprecated: + +distutils +--------- + +The entire ``distutils`` package is deprecated, to be removed in Python +3.12. Its functionality for specifying package builds has already been +completely replaced by third-party packages ``setuptools`` and +``packaging``, and most other commonly used APIs are available elsewhere +in the standard library (such as :mod:`platform`, :mod:`shutil`, +:mod:`subprocess` or :mod:`sysconfig`). There are no plans to migrate +any other functionality from ``distutils``, and applications that are +using other functions should plan to make private copies of the code. +Refer to :pep:`632` for discussion. + +The ``bdist_wininst`` command deprecated in Python 3.8 has been removed. +The ``bdist_wheel`` command is now recommended to distribute binary packages +on Windows. +(Contributed by Victor Stinner in :issue:`42802`.) + +doctest +------- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + encodings --------- + :func:`encodings.normalize_encoding` now ignores non-ASCII characters. (Contributed by Hai Shi in :issue:`39337`.) +enum +---- + +:class:`Enum` :func:`__repr__` now returns ``enum_name.member_name`` and +:func:`__str__` now returns ``member_name``. Stdlib enums available as +module constants have a :func:`repr` of ``module_name.member_name``. +(Contributed by Ethan Furman in :issue:`40066`.) + +Add :class:`enum.StrEnum` for enums where all members are strings. +(Contributed by Ethan Furman in :issue:`41816`.) + +fileinput +--------- + +Added *encoding* and *errors* parameters in :func:`fileinput.input` and +:class:`fileinput.FileInput`. +(Contributed by Inada Naoki in :issue:`43712`.) + +:func:`fileinput.hook_compressed` now returns :class:`TextIOWrapper` object +when *mode* is "r" and file is compressed, like uncompressed files. +(Contributed by Inada Naoki in :issue:`5758`.) + +gc +-- + +Added audit hooks for :func:`gc.get_objects`, :func:`gc.get_referrers` and +:func:`gc.get_referents`. (Contributed by Pablo Galindo in :issue:`43439`.) + glob ---- @@ -198,18 +974,240 @@ Added the *root_dir* and *dir_fd* parameters in :func:`~glob.glob` and :func:`~glob.iglob` which allow to specify the root directory for searching. (Contributed by Serhiy Storchaka in :issue:`38144`.) +hashlib +------- + +The hashlib module requires OpenSSL 1.1.1 or newer. +(Contributed by Christian Heimes in :pep:`644` and :issue:`43669`.) + +The hashlib module has preliminary support for OpenSSL 3.0.0. +(Contributed by Christian Heimes in :issue:`38820` and other issues.) + +The pure-Python fallback of :func:`~hashlib.pbkdf2_hmac` is deprecated. In +the future PBKDF2-HMAC will only be available when Python has been built with +OpenSSL support. +(Contributed by Christian Heimes in :issue:`43880`.) + +hmac +---- + +The hmac module now uses OpenSSL's HMAC implementation internally. +(Contributed by Christian Heimes in :issue:`40645`.) + +IDLE and idlelib +---------------- + +Make IDLE invoke :func:`sys.excepthook` (when started without '-n'). +User hooks were previously ignored. (Patch by Ken Hilton in +:issue:`43008`.) + +This change was backported to a 3.9 maintenance release. + +Add a Shell sidebar. Move the primary prompt ('>>>') to the sidebar. +Add secondary prompts ('...') to the sidebar. Left click and optional +drag selects one or more lines of text, as with the editor +line number sidebar. Right click after selecting text lines displays +a context menu with 'copy with prompts'. This zips together prompts +from the sidebar with lines from the selected text. This option also +appears on the context menu for the text. (Contributed by Tal Einat +in :issue:`37903`.) + +Use spaces instead of tabs to indent interactive code. This makes +interactive code entries 'look right'. Making this feasible was a +major motivation for adding the shell sidebar. Contributed by +Terry Jan Reedy in :issue:`37892`.) + +We expect to backport these shell changes to a future 3.9 maintenance +release. + +importlib.metadata +------------------ + +Feature parity with ``importlib_metadata`` 3.7. + +:func:`importlib.metadata.entry_points` now provides a nicer experience +for selecting entry points by group and name through a new +:class:`importlib.metadata.EntryPoints` class. + +Added :func:`importlib.metadata.packages_distributions` for resolving +top-level Python modules and packages to their +:class:`importlib.metadata.Distribution`. + +inspect +------- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + +Added :func:`inspect.get_annotations`, which safely computes the annotations +defined on an object. It works around the quirks of accessing the annotations +on various types of objects, and makes very few assumptions about the object +it examines. :func:`inspect.get_annotations` can also correctly un-stringize +stringized annotations. :func:`inspect.get_annotations` is now considered +best practice for accessing the annotations dict defined on any Python object; +for more information on best practices for working with annotations, please see +:ref:`annotations-howto`. +Relatedly, :func:`inspect.signature`, +:func:`inspect.Signature.from_callable`, and ``inspect.Signature.from_function`` +now call :func:`inspect.get_annotations` to retrieve annotations. This means +:func:`inspect.signature` and :func:`inspect.Signature.from_callable` can +also now un-stringize stringized annotations. +(Contributed by Larry Hastings in :issue:`43817`.) + +linecache +--------- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + os -- Added :func:`os.cpu_count()` support for VxWorks RTOS. (Contributed by Peixing Xin in :issue:`41440`.) +Added a new function :func:`os.eventfd` and related helpers to wrap the +``eventfd2`` syscall on Linux. +(Contributed by Christian Heimes in :issue:`41001`.) + +Added :func:`os.splice()` that allows to move data between two file +descriptors without copying between kernel address space and user +address space, where one of the file descriptors must refer to a +pipe. (Contributed by Pablo Galindo in :issue:`41625`.) + +Added :data:`~os.O_EVTONLY`, :data:`~os.O_FSYNC`, :data:`~os.O_SYMLINK` +and :data:`~os.O_NOFOLLOW_ANY` for macOS. +(Contributed by Dong-hee Na in :issue:`43106`.) + +pathlib +------- + +Added slice support to :attr:`PurePath.parents `. +(Contributed by Joshua Cannon in :issue:`35498`) + +Added negative indexing support to :attr:`PurePath.parents +`. +(Contributed by Yaroslav Pankovych in :issue:`21041`) + +Added :meth:`Path.hardlink_to ` method that +supersedes :meth:`~pathlib.Path.link_to`. The new method has the same argument +order as :meth:`~pathlib.Path.symlink_to`. +(Contributed by Barney Gale in :issue:`39950`.) + +platform +-------- + +Added :func:`platform.freedesktop_os_release()` to retrieve operation system +identification from `freedesktop.org os-release +`_ standard file. +(Contributed by Christian Heimes in :issue:`28468`) + +pprint +------ + +:mod:`pprint` can now pretty-print :class:`dataclasses.dataclass` instances. +(Contributed by Lewis Gaul in :issue:`43080`.) + py_compile ---------- Added ``--quiet`` option to command-line interface of :mod:`py_compile`. (Contributed by Gregory Schevchenko in :issue:`38731`.) +pyclbr +------ + +Added an ``end_lineno`` attribute to the ``Function`` and ``Class`` +objects in the tree returned by :func:`pyclbr.readline` and +:func:`pyclbr.readline_ex`. It matches the existing (start) ``lineno``. +(Contributed by Aviral Srivastava in :issue:`38307`.) + +shelve +------ + +The :mod:`shelve` module now uses :data:`pickle.DEFAULT_PROTOCOL` by default +instead of :mod:`pickle` protocol ``3`` when creating shelves. +(Contributed by Zackery Spytz in :issue:`34204`.) + +statistics +---------- + +Added :func:`~statistics.covariance`, Pearson's +:func:`~statistics.correlation`, and simple +:func:`~statistics.linear_regression` functions. +(Contributed by Tymoteusz WoÅ‚odźko in :issue:`38490`.) + +site +---- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + +socket +------ + +The exception :exc:`socket.timeout` is now an alias of :exc:`TimeoutError`. +(Contributed by Christian Heimes in :issue:`42413`.) + +Added option to create MPTCP sockets with ``IPPROTO_MPTCP`` +(Contributed by Rui Cunha in :issue:`43571`.) + +ssl +--- + +The ssl module requires OpenSSL 1.1.1 or newer. +(Contributed by Christian Heimes in :pep:`644` and :issue:`43669`.) + +The ssl module has preliminary support for OpenSSL 3.0.0 and new option +:data:`~ssl.OP_IGNORE_UNEXPECTED_EOF`. +(Contributed by Christian Heimes in :issue:`38820`, :issue:`43794`, +:issue:`43788`, :issue:`43791`, :issue:`43799`, :issue:`43920`, +:issue:`43789`, and :issue:`43811`.) + +Deprecated function and use of deprecated constants now result in +a :exc:`DeprecationWarning`. The following features have been deprecated +since Python 3.6, Python 3.7, or OpenSSL 1.1.0: +:data:`~ssl.OP_NO_SSLv2`, :data:`~ssl.OP_NO_SSLv3`, :data:`~ssl.OP_NO_TLSv1`, +:data:`~ssl.OP_NO_TLSv1_1`, :data:`~ssl.OP_NO_TLSv1_2`, +:data:`~ssl.OP_NO_TLSv1_3`, :data:`~ssl.PROTOCOL_SSLv2`, +:data:`~ssl.PROTOCOL_SSLv3`, :data:`~ssl.PROTOCOL_SSLv23`, +:data:`~ssl.PROTOCOL_TLSv1`, :data:`~ssl.PROTOCOL_TLSv1_1`, +:data:`~ssl.PROTOCOL_TLSv1_2`, :data:`~ssl.PROTOCOL_TLS`, +:func:`~ssl.wrap_socket`, :func:`~ssl.match_hostname`, +:func:`~ssl.RAND_pseudo_bytes`, :func:`~ssl.RAND_egd`, +:meth:`ssl.SSLSocket.selected_npn_protocol`, +:meth:`ssl.SSLContext.set_npn_protocols`. +(Contributed by Christian Heimes in :issue:`43880`.) + +The ssl module now has more secure default settings. Ciphers without forward +secrecy or SHA-1 MAC are disabled by default. Security level 2 prohibits +weak RSA, DH, and ECC keys with less than 112 bits of security. +:class:`~ssl.SSLContext` defaults to minimum protocol version TLS 1.2. +Settings are based on Hynek Schlawack's research. +(Contributed by Christian Heimes in :issue:`43998`.) + +The deprecated protocols SSL 3.0, TLS 1.0, and TLS 1.1 are no longer +officially supported. Python does not block them actively. However +OpenSSL build options, distro configurations, vendor patches, and cipher +suites may prevent a successful handshake. + +Add a *timeout* parameter to the :func:`ssl.get_server_certificate` function. +(Contributed by Zackery Spytz in :issue:`31870`.) + +The ssl module uses heap-types and multi-phase initialization. +(Contributed by Christian Heimes in :issue:`42333`.) + +A new verify flag :data:`~ssl.VERIFY_X509_PARTIAL_CHAIN` has been added. +(Contributed by l0x in :issue:`40849`.) + +sqlite3 +------- + +Add audit events for :func:`~sqlite3.connect/handle`, +:meth:`~sqlite3.Connection.enable_load_extension`, and +:meth:`~sqlite3.Connection.load_extension`. +(Contributed by Erlend E. Aasland in :issue:`43762`.) + sys --- @@ -217,6 +1215,39 @@ Add :data:`sys.orig_argv` attribute: the list of the original command line arguments passed to the Python executable. (Contributed by Victor Stinner in :issue:`23427`.) +Add :data:`sys.stdlib_module_names`, containing the list of the standard library +module names. +(Contributed by Victor Stinner in :issue:`42955`.) + +_thread +------- + +:func:`_thread.interrupt_main` now takes an optional signal number to +simulate (the default is still :data:`signal.SIGINT`). +(Contributed by Antoine Pitrou in :issue:`43356`.) + +threading +--------- + +Added :func:`threading.gettrace` and :func:`threading.getprofile` to +retrieve the functions set by :func:`threading.settrace` and +:func:`threading.setprofile` respectively. +(Contributed by Mario Corchero in :issue:`42251`.) + +Add :data:`threading.__excepthook__` to allow retrieving the original value +of :func:`threading.excepthook` in case it is set to a broken or a different +value. +(Contributed by Mario Corchero in :issue:`42308`.) + +traceback +--------- + +The :func:`~traceback.format_exception`, +:func:`~traceback.format_exception_only`, and +:func:`~traceback.print_exception` functions can now take an exception object +as a positional-only argument. +(Contributed by Zackery Spytz and Matthias Bussonnier in :issue:`26389`.) + types ----- @@ -225,6 +1256,38 @@ and :data:`types.NotImplementedType` classes, providing a new set of types readily interpretable by type checkers. (Contributed by Bas van Beek in :issue:`41810`.) +typing +------ + +For major changes, see `New Features Related to Type Hints`_. + +The behavior of :class:`typing.Literal` was changed to conform with :pep:`586` +and to match the behavior of static type checkers specified in the PEP. + +1. ``Literal`` now de-duplicates parameters. +2. Equality comparisons between ``Literal`` objects are now order independent. +3. ``Literal`` comparisons now respects types. For example, + ``Literal[0] == Literal[False]`` previously evaluated to ``True``. It is + now ``False``. To support this change, the internally used type cache now + supports differentiating types. +4. ``Literal`` objects will now raise a :exc:`TypeError` exception during + equality comparisons if one of their parameters are not :term:`immutable`. + Note that declaring ``Literal`` with mutable parameters will not throw + an error:: + + >>> from typing import Literal + >>> Literal[{0}] + >>> Literal[{0}] == Literal[{False}] + Traceback (most recent call last): + File "", line 1, in + TypeError: unhashable type: 'set' + +(Contributed by Yurii Karabas in :issue:`42345`.) + +Add new function :func:`typing.is_typeddict` to introspect if an annotation +is a :class:`typing.TypedDict`. +(Contributed by Patrick Reader in :issue:`41792`) + unittest -------- @@ -232,6 +1295,19 @@ Add new method :meth:`~unittest.TestCase.assertNoLogs` to complement the existing :meth:`~unittest.TestCase.assertLogs`. (Contributed by Kit Yan Choi in :issue:`39385`.) +urllib.parse +------------ + +Python versions earlier than Python 3.10 allowed using both ``;`` and ``&`` as +query parameter separators in :func:`urllib.parse.parse_qs` and +:func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with +newer W3C recommendations, this has been changed to allow only a single +separator key, with ``&`` as the default. This change also affects +:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected +functions internally. For more details, please see their respective +documentation. +(Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) + xml --- @@ -239,6 +1315,16 @@ Add a :class:`~xml.sax.handler.LexicalHandler` class to the :mod:`xml.sax.handler` module. (Contributed by Jonathan Gossage and Zackery Spytz in :issue:`35018`.) +zipimport +--------- +Add methods related to :pep:`451`: :meth:`~zipimport.zipimporter.find_spec`, +:meth:`zipimport.zipimporter.create_module`, and +:meth:`zipimport.zipimporter.exec_module`. +(Contributed by Brett Cannon in :issue:`42131`.) + +Add :meth:`~zipimport.zipimporter.invalidate_caches` method. +(Contributed by Desmond Cheong in :issue:`14678`.) + Optimizations ============= @@ -248,21 +1334,55 @@ Optimizations (Contributed by Serhiy Storchaka in :issue:`41334`.) * The :mod:`runpy` module now imports fewer modules. - The ``python3 -m module-name`` command startup time is 1.3x faster in - average. - (Contributed by Victor Stinner in :issue:`41006`.) - -* The ``LOAD_ATTR`` instruction now uses new "per opcode cache" mechanism. - It is about 36% faster now. (Contributed by Pablo Galindo and Yury Selivanov - in :issue:`42093`.) - -* When building Python with ``--enable-optimizations`` now + The ``python3 -m module-name`` command startup time is 1.4x faster in + average. On Linux, ``python3 -I -m module-name`` imports 69 modules on Python + 3.9, whereas it only imports 51 modules (-18) on Python 3.10. + (Contributed by Victor Stinner in :issue:`41006` and :issue:`41718`.) + +* The ``LOAD_ATTR`` instruction now uses new "per opcode cache" mechanism. It + is about 36% faster now for regular attributes and 44% faster for slots. + (Contributed by Pablo Galindo and Yury Selivanov in :issue:`42093` and Guido + van Rossum in :issue:`42927`, based on ideas implemented originally in PyPy + and MicroPython.) + +* When building Python with :option:`--enable-optimizations` now ``-fno-semantic-interposition`` is added to both the compile and link line. - This speeds builds of the Python interpreter created with ``--enable-shared`` + This speeds builds of the Python interpreter created with :option:`--enable-shared` with ``gcc`` by up to 30%. See `this article `_ for more details. (Contributed by Victor Stinner and Pablo Galindo in - :issue:`38980`) + :issue:`38980`.) + +* Use a new output buffer management code for :mod:`bz2` / :mod:`lzma` / + :mod:`zlib` modules, and add ``.readall()`` function to + ``_compression.DecompressReader`` class. bz2 decompression 1.09x ~ 1.17x + faster, lzma decompression 1.20x ~ 1.32x faster, ``GzipFile.read(-1)`` 1.11x + ~ 1.18x faster. (Contributed by Ma Lin, reviewed by Gregory P. Smith, in :issue:`41486`) + +* When using stringized annotations, annotations dicts for functions are no longer + created when the function is created. Instead, they're stored as a tuple of + strings, and the function object lazily converts this into the annotations dict + on demand. This optimization cuts the CPU time needed to define an annotated + function by half. + (Contributed by Yurii Karabas and Inada Naoki in :issue:`42202`) + +* Substring search functions such as ``str1 in str2`` and ``str2.find(str1)`` + now sometimes use Crochemore & Perrin's "Two-Way" string searching + algorithm to avoid quadratic behavior on long strings. (Contributed + by Dennis Sweeney in :issue:`41972`) + +* Added micro-optimizations to ``_PyType_Lookup()`` to improve type attribute cache lookup + performance in the common case of cache hits. This makes the interpreter 1.04 times faster + in average (Contributed by Dino Viehland in :issue:`43452`) + +* Following built-in functions now support the faster :pep:`590` vectorcall calling convention: + :func:`map`, :func:`filter`, :func:`reversed`, :func:`bool` and :func:`float`. + (Contributed by Dong-hee Na and Jeroen Demeyerin in :issue:`43575`, :issue:`43287`, :issue:`41922`, :issue:`41873` and :issue:`41870`) + +* :class:`BZ2File` performance is improved by removing internal ``RLock``. + This makes :class:`BZ2File` thread unsafe in the face of multiple simultaneous + readers or writers, just like its equivalent classes in :mod:`gzip` and + :mod:`lzma` have always been. (Contributed by Inada Naoki in :issue:`43785`). Deprecated ========== @@ -284,6 +1404,143 @@ Deprecated as appropriate to help identify code which needs updating during this transition. +* The entire ``distutils`` namespace is deprecated, to be removed in + Python 3.12. Refer to the :ref:`module changes ` + section for more information. + +* Non-integer arguments to :func:`random.randrange` are deprecated. + The :exc:`ValueError` is deprecated in favor of a :exc:`TypeError`. + (Contributed by Serhiy Storchaka and Raymond Hettinger in :issue:`37319`.) + +* The various ``load_module()`` methods of :mod:`importlib` have been + documented as deprecated since Python 3.6, but will now also trigger + a :exc:`DeprecationWarning`. Use + :meth:`~importlib.abc.Loader.exec_module` instead. + (Contributed by Brett Cannon in :issue:`26131`.) + +* :meth:`zimport.zipimporter.load_module` has been deprecated in + preference for :meth:`~zipimport.zipimporter.exec_module`. + (Contributed by Brett Cannon in :issue:`26131`.) + +* The use of :meth:`~importlib.abc.Loader.load_module` by the import + system now triggers an :exc:`ImportWarning` as + :meth:`~importlib.abc.Loader.exec_module` is preferred. + (Contributed by Brett Cannon in :issue:`26131`.) + +* The use of :meth:`importlib.abc.MetaPathFinder.find_module` and + :meth:`importlib.abc.PathEntryFinder.find_module` by the import system now + trigger an :exc:`ImportWarning` as + :meth:`importlib.abc.MetaPathFinder.find_spec` and + :meth:`importlib.abc.PathEntryFinder.find_spec` + are preferred, respectively. You can use + :func:`importlib.util.spec_from_loader` to help in porting. + (Contributed by Brett Cannon in :issue:`42134`.) + +* The use of :meth:`importlib.abc.PathEntryFinder.find_loader` by the import + system now triggers an :exc:`ImportWarning` as + :meth:`importlib.abc.PathEntryFinder.find_spec` is preferred. You can use + :func:`importlib.util.spec_from_loader` to help in porting. + (Contributed by Brett Cannon in :issue:`43672`.) + +* The various implementations of + :meth:`importlib.abc.MetaPathFinder.find_module` ( + :meth:`importlib.machinery.BuiltinImporter.find_module`, + :meth:`importlib.machinery.FrozenImporter.find_module`, + :meth:`importlib.machinery.WindowsRegistryFinder.find_module`, + :meth:`importlib.machinery.PathFinder.find_module`, + :meth:`importlib.abc.MetaPathFinder.find_module`), + :meth:`importlib.abc.PathEntryFinder.find_module` ( + :meth:`importlib.machinery.FileFinder.find_module`, + ), and + :meth:`importlib.abc.PathEntryFinder.find_loader` ( + :meth:`importlib.machinery.FileFinder.find_loader` + ) now raise :exc:`DeprecationWarning` and are slated for removal in + Python 3.12 (previously they were documented as deprecated in Python 3.4). + (Contributed by Brett Cannon in :issue:`42135`.) + +* :class:`importlib.abc.Finder` is deprecated (including its sole method, + :meth:`~importlib.abc.Finder.find_module`). Both + :class:`importlib.abc.MetaPathFinder` and :class:`importlib.abc.PathEntryFinder` + no longer inherit from the class. Users should inherit from one of these two + classes as appropriate instead. + (Contributed by Brett Cannon in :issue:`42135`.) + +* The deprecations of :mod:`imp`, :func:`importlib.find_loader`, + :func:`importlib.util.set_package_wrapper`, + :func:`importlib.util.set_loader_wrapper`, + :func:`importlib.util.module_for_loader`, + :class:`pkgutil.ImpImporter`, and + :class:`pkgutil.ImpLoader` have all been updated to list Python 3.12 as the + slated version of removal (they began raising :exc:`DeprecationWarning` in + previous versions of Python). + (Contributed by Brett Cannon in :issue:`43720`.) + +* The import system now uses the ``__spec__`` attribute on modules before + falling back on :meth:`~importlib.abc.Loader.module_repr` for a module's + ``__repr__()`` method. Removal of the use of ``module_repr()`` is scheduled + for Python 3.12. + (Contributed by Brett Cannon in :issue:`42137`.) + +* :meth:`importlib.abc.Loader.module_repr`, + :meth:`importlib.machinery.FrozenLoader.module_repr`, and + :meth:`importlib.machinery.BuiltinLoader.module_repr` are deprecated and + slated for removal in Python 3.12. + (Contributed by Brett Cannon in :issue:`42136`.) + +* ``sqlite3.OptimizedUnicode`` has been undocumented and obsolete since Python + 3.3, when it was made an alias to :class:`str`. It is now deprecated, + scheduled for removal in Python 3.12. + (Contributed by Erlend E. Aasland in :issue:`42264`.) + +* :func:`asyncio.get_event_loop` emits now a deprecation warning if there is + no running event loop. In future it will be an alias of + :func:`~asyncio.get_running_loop`. + :mod:`asyncio` functions which implicitly create a :class:`~asyncio.Future` + or :class:`~asyncio.Task` objects emit now + a deprecation warning if there is no running event loop and no explicit + *loop* argument is passed: :func:`~asyncio.ensure_future`, + :func:`~asyncio.wrap_future`, :func:`~asyncio.gather`, + :func:`~asyncio.shield`, :func:`~asyncio.as_completed` and constructors of + :class:`~asyncio.Future`, :class:`~asyncio.Task`, + :class:`~asyncio.StreamReader`, :class:`~asyncio.StreamReaderProtocol`. + (Contributed by Serhiy Storchaka in :issue:`39529`.) + +* The undocumented built-in function ``sqlite3.enable_shared_cache`` is now + deprecated, scheduled for removal in Python 3.12. Its use is strongly + discouraged by the SQLite3 documentation. See `the SQLite3 docs + `_ for more details. + If shared cache must be used, open the database in URI mode using the + ``cache=shared`` query parameter. + (Contributed by Erlend E. Aasland in :issue:`24464`.) + +* The following ``threading`` methods are now deprecated: + + * ``threading.currentThread`` => :func:`threading.current_thread` + + * ``threading.activeCount`` => :func:`threading.active_count` + + * ``threading.Condition.notifyAll`` => + :meth:`threading.Condition.notify_all` + + * ``threading.Event.isSet`` => :meth:`threading.Event.is_set` + + * ``threading.Thread.setName`` => :attr:`threading.Thread.name` + + * ``threading.thread.getName`` => :attr:`threading.Thread.name` + + * ``threading.Thread.isDaemon`` => :attr:`threading.Thread.daemon` + + * ``threading.Thread.setDaemon`` => :attr:`threading.Thread.daemon` + + (Contributed by Jelle Zijlstra in :issue:`21574`.) + +* :meth:`pathlib.Path.link_to` is deprecated and slated for removal in + Python 3.12. Use :meth:`pathlib.Path.hardlink_to` instead. + (Contributed by Barney Gale in :issue:`39950`.) + +* ``cgi.log()`` is deprecated and slated for for removal in Python 3.12. + (Contributed by Inada Naoki in :issue:`41139`.) + Removed ======= @@ -305,6 +1562,48 @@ Removed moved to the internal C API. (Contributed by Victor Stinner in :issue:`42157`.) +* Removed the ``parser`` module, which was deprecated in 3.9 due to the + switch to the new PEG parser, as well as all the C source and header files + that were only being used by the old parser, including ``node.h``, ``parser.h``, + ``graminit.h`` and ``grammar.h``. + +* Removed the Public C API functions :c:func:`PyParser_SimpleParseStringFlags`, + :c:func:`PyParser_SimpleParseStringFlagsFilename`, + :c:func:`PyParser_SimpleParseFileFlags` and :c:func:`PyNode_Compile` + that were deprecated in 3.9 due to the switch to the new PEG parser. + +* Removed the ``formatter`` module, which was deprecated in Python 3.4. + It is somewhat obsolete, little used, and not tested. It was originally + scheduled to be removed in Python 3.6, but such removals were delayed until + after Python 2.7 EOL. Existing users should copy whatever classes they use + into their code. + (Contributed by Dong-hee Na and Terry J. Reedy in :issue:`42299`.) + +* Removed the :c:func:`PyModule_GetWarningsModule` function that was useless + now due to the _warnings module was converted to a builtin module in 2.6. + (Contributed by Hai Shi in :issue:`42599`.) + +* Remove deprecated aliases to :ref:`collections-abstract-base-classes` from + the :mod:`collections` module. + (Contributed by Victor Stinner in :issue:`37324`.) + +* The ``loop`` parameter has been removed from most of :mod:`asyncio`\ 's + :doc:`high-level API <../library/asyncio-api-index>` following deprecation + in Python 3.8. The motivation behind this change is multifold: + + 1. This simplifies the high-level API. + 2. The functions in the high-level API have been implicitly getting the + current thread's running event loop since Python 3.7. There isn't a need to + pass the event loop to the API in most normal use cases. + 3. Event loop passing is error-prone especially when dealing with loops + running in different threads. + + Note that the low-level API will still accept ``loop``. + See `Changes in the Python API`_ for examples of how to replace existing code. + + (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley + in :issue:`42392`.) + Porting to Python 3.10 ====================== @@ -313,18 +1612,122 @@ This section lists previously described changes and other bugfixes that may require changes to your code. +Changes in the Python API +------------------------- + +* The *etype* parameters of the :func:`~traceback.format_exception`, + :func:`~traceback.format_exception_only`, and + :func:`~traceback.print_exception` functions in the :mod:`traceback` module + have been renamed to *exc*. + (Contributed by Zackery Spytz and Matthias Bussonnier in :issue:`26389`.) + +* :mod:`atexit`: At Python exit, if a callback registered with + :func:`atexit.register` fails, its exception is now logged. Previously, only + some exceptions were logged, and the last exception was always silently + ignored. + (Contributed by Victor Stinner in :issue:`42639`.) + +* :class:`collections.abc.Callable` generic now flattens type parameters, similar + to what :data:`typing.Callable` currently does. This means that + ``collections.abc.Callable[[int, str], str]`` will have ``__args__`` of + ``(int, str, str)``; previously this was ``([int, str], str)``. Code which + accesses the arguments via :func:`typing.get_args` or ``__args__`` need to account + for this change. Furthermore, :exc:`TypeError` may be raised for invalid forms + of parameterizing :class:`collections.abc.Callable` which may have passed + silently in Python 3.9. + (Contributed by Ken Jin in :issue:`42195`.) + +* :meth:`socket.htons` and :meth:`socket.ntohs` now raise :exc:`OverflowError` + instead of :exc:`DeprecationWarning` if the given parameter will not fit in + a 16-bit unsigned integer. + (Contributed by Erlend E. Aasland in :issue:`42393`.) + +* The ``loop`` parameter has been removed from most of :mod:`asyncio`\ 's + :doc:`high-level API <../library/asyncio-api-index>` following deprecation + in Python 3.8. + + A coroutine that currently look like this:: + + async def foo(loop): + await asyncio.sleep(1, loop=loop) + + Should be replaced with this:: + + async def foo(): + await asyncio.sleep(1) + + If ``foo()`` was specifically designed *not* to run in the current thread's + running event loop (e.g. running in another thread's event loop), consider + using :func:`asyncio.run_coroutine_threadsafe` instead. + + (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley + in :issue:`42392`.) + +* The :data:`types.FunctionType` constructor now inherits the current builtins + if the *globals* dictionary has no ``"__builtins__"`` key, rather than using + ``{"None": None}`` as builtins: same behavior as :func:`eval` and + :func:`exec` functions. Defining a function with ``def function(...): ...`` + in Python is not affected, globals cannot be overriden with this syntax: it + also inherits the current builtins. + (Contributed by Victor Stinner in :issue:`42990`.) + +CPython bytecode changes +======================== + +* The ``MAKE_FUNCTION`` instruction now accepts either a dict or a tuple of + strings as the function's annotations. + (Contributed by Yurii Karabas and Inada Naoki in :issue:`42202`) Build Changes ============= +* :pep:`644`: Python now requires OpenSSL 1.1.1 or newer. OpenSSL 1.0.2 is no + longer supported. + (Contributed by Christian Heimes in :issue:`43669`.) * The C99 functions :c:func:`snprintf` and :c:func:`vsnprintf` are now required to build Python. (Contributed by Victor Stinner in :issue:`36020`.) -* :mod:`sqlite3` requires SQLite 3.7.3 or higher. - (Contributed by Sergey Fedoseev and Erlend E. Aasland :issue:`40744`.) +* :mod:`sqlite3` requires SQLite 3.7.15 or higher. (Contributed by Sergey Fedoseev + and Erlend E. Aasland :issue:`40744` and :issue:`40810`.) + +* The :mod:`atexit` module must now always be built as a built-in module. + (Contributed by Victor Stinner in :issue:`42639`.) + +* Added :option:`--disable-test-modules` option to the ``configure`` script: + don't build nor install test modules. + (Contributed by Xavier de Gaye, Thomas Petazzoni and Peixing Xin in :issue:`27640`.) + +* Add :option:`--with-wheel-pkg-dir=PATH option <--with-wheel-pkg-dir>` + to the ``./configure`` script. If + specified, the :mod:`ensurepip` module looks for ``setuptools`` and ``pip`` + wheel packages in this directory: if both are present, these wheel packages + are used instead of ensurepip bundled wheel packages. + Some Linux distribution packaging policies recommend against bundling + dependencies. For example, Fedora installs wheel packages in the + ``/usr/share/python-wheels/`` directory and don't install the + ``ensurepip._bundled`` package. + + (Contributed by Victor Stinner in :issue:`42856`.) + +* Add a new :option:`configure --without-static-libpython option + <--without-static-libpython>` to not build the ``libpythonMAJOR.MINOR.a`` + static library and not install the ``python.o`` object file. + + (Contributed by Victor Stinner in :issue:`43103`.) + +* The ``configure`` script now uses the ``pkg-config`` utility, if available, + to detect the location of Tcl/Tk headers and libraries. As before, those + locations can be explicitly specified with the :option:`--with-tcltk-includes` + and :option:`--with-tcltk-libs` configuration options. + (Contributed by Manolis Stamatogiannakis in :issue:`42603`.) + +* Add :option:`--with-openssl-rpath` option to ``configure`` script. The option + simplifies building Python with a custom OpenSSL installation, e.g. + ``./configure --with-openssl=/path/to/openssl --with-openssl-rpath=auto``. + (Contributed by Christian Heimes in :issue:`43466`.) C API Changes @@ -359,6 +1762,66 @@ New Features * Added :c:func:`PyUnicode_AsUTF8AndSize` to the limited C API. (Contributed by Alex Gaynor in :issue:`41784`.) +* Added :c:func:`PyModule_AddObjectRef` function: similar to + :c:func:`PyModule_AddObject` but don't steal a reference to the value on + success. + (Contributed by Victor Stinner in :issue:`1635741`.) + +* Added :c:func:`Py_NewRef` and :c:func:`Py_XNewRef` functions to increment the + reference count of an object and return the object. + (Contributed by Victor Stinner in :issue:`42262`.) + +* The :c:func:`PyType_FromSpecWithBases` and :c:func:`PyType_FromModuleAndSpec` + functions now accept a single class as the *bases* argument. + (Contributed by Serhiy Storchaka in :issue:`42423`.) + +* The :c:func:`PyType_FromModuleAndSpec` function now accepts NULL ``tp_doc`` + slot. + (Contributed by Hai Shi in :issue:`41832`.) + +* The :c:func:`PyType_GetSlot` function can accept + :ref:`static types `. + (Contributed by Hai Shi and Petr Viktorin in :issue:`41073`.) + +* Add a new :c:func:`PySet_CheckExact` function to the C-API to check if an + object is an instance of :class:`set` but not an instance of a subtype. + (Contributed by Pablo Galindo in :issue:`43277`.) + +* Added :c:func:`PyErr_SetInterruptEx` which allows passing a signal number + to simulate. + (Contributed by Antoine Pitrou in :issue:`43356`.) + +* The limited C API is now supported if :ref:`Python is built in debug mode + ` (if the ``Py_DEBUG`` macro is defined). In the limited C API, + the :c:func:`Py_INCREF` and :c:func:`Py_DECREF` functions are now implemented + as opaque function + calls, rather than accessing directly the :c:member:`PyObject.ob_refcnt` + member, if Python is built in debug mode and the ``Py_LIMITED_API`` macro + targets Python 3.10 or newer. It became possible to support the limited C API + in debug mode because the :c:type:`PyObject` structure is the same in release + and debug mode since Python 3.8 (see :issue:`36465`). + + The limited C API is still not supported in the :option:`--with-trace-refs` + special build (``Py_TRACE_REFS`` macro). + (Contributed by Victor Stinner in :issue:`43688`.) + +* Add the :c:func:`Py_Is(x, y) ` function to test if the *x* object is + the *y* object, the same as ``x is y`` in Python. Add also the + :c:func:`Py_IsNone`, :c:func:`Py_IsTrue`, :c:func:`Py_IsFalse` functions to + test if an object is, respectively, the ``None`` singleton, the ``True`` + singleton or the ``False`` singleton. + (Contributed by Victor Stinner in :issue:`43753`.) + +* Add new functions to quickly control the garbage collector from C code: + :c:func:`PyGC_Enable()`, + :c:func:`PyGC_Disable()`, + :c:func:`PyGC_IsEnabled()`. + These functions allow to activate, deactivate and query the state of the garbage collector from C code without + having to import the :mod:`gc` module. + +* Add a new :c:data:`Py_TPFLAGS_DISALLOW_INSTANTIATION` type flag to disallow + creating type instances. + (Contributed by Victor Stinner in :issue:`43916`.) Porting to Python 3.10 ---------------------- @@ -370,17 +1833,6 @@ Porting to Python 3.10 ` and the :pep:`353`. (Contributed by Victor Stinner in :issue:`40943`.) -* Since :c:func:`Py_TYPE()` is changed to the inline static function, - ``Py_TYPE(obj) = new_type`` must be replaced with ``Py_SET_TYPE(obj, new_type)``: - see :c:func:`Py_SET_TYPE()` (available since Python 3.9). For backward - compatibility, this macro can be used:: - - #if PY_VERSION_HEX < 0x030900A4 - # define Py_SET_TYPE(obj, type) ((Py_TYPE(obj) = (type)), (void)0) - #endif - - (Contributed by Dong-hee Na in :issue:`39573`.) - * Since :c:func:`Py_REFCNT()` is changed to the inline static function, ``Py_REFCNT(obj) = new_refcnt`` must be replaced with ``Py_SET_REFCNT(obj, new_refcnt)``: see :c:func:`Py_SET_REFCNT()` (available since Python 3.9). For backward @@ -392,17 +1844,6 @@ Porting to Python 3.10 (Contributed by Victor Stinner in :issue:`39573`.) -* Since :c:func:`Py_SIZE()` is changed to the inline static function, - ``Py_SIZE(obj) = new_size`` must be replaced with ``Py_SET_SIZE(obj, new_size)``: - see :c:func:`Py_SET_SIZE()` (available since Python 3.9). For backward - compatibility, this macro can be used:: - - #if PY_VERSION_HEX < 0x030900A4 - # define Py_SET_SIZE(obj, size) ((Py_SIZE(obj) = (size)), (void)0) - #endif - - (Contributed by Victor Stinner in :issue:`39573`.) - * Calling :c:func:`PyDict_GetItem` without :term:`GIL` held had been allowed for historical reason. It is no longer allowed. (Contributed by Victor Stinner in :issue:`40839`.) @@ -416,6 +1857,29 @@ Porting to Python 3.10 ``unicodedata.ucnhash_CAPI`` has been moved to the internal C API. (Contributed by Victor Stinner in :issue:`42157`.) +* :c:func:`Py_GetPath`, :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, + :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome` and + :c:func:`Py_GetProgramName` functions now return ``NULL`` if called before + :c:func:`Py_Initialize` (before Python is initialized). Use the new + :ref:`Python Initialization Configuration API ` to get the + :ref:`Python Path Configuration. `. + (Contributed by Victor Stinner in :issue:`42260`.) + +* :c:func:`PyList_SET_ITEM`, :c:func:`PyTuple_SET_ITEM` and + :c:func:`PyCell_SET` macros can no longer be used as l-value or r-value. + For example, ``x = PyList_SET_ITEM(a, b, c)`` and + ``PyList_SET_ITEM(a, b, c) = x`` now fail with a compiler error. It prevents + bugs like ``if (PyList_SET_ITEM (a, b, c) < 0) ...`` test. + (Contributed by Zackery Spytz and Victor Stinner in :issue:`30459`.) + +* The non-limited API files ``odictobject.h``, ``parser_interface.h``, + ``picklebufobject.h``, ``pyarena.h``, ``pyctype.h``, ``pydebug.h``, + ``pyfpe.h``, and ``pytime.h`` have been moved to the ``Include/cpython`` + directory. These files must not be included directly, as they are already + included in ``Python.h``: :ref:`Include Files `. If they have + been included directly, consider including ``Python.h`` instead. + (Contributed by Nicholas Sim in :issue:`35134`) + Deprecated ---------- @@ -464,3 +1928,71 @@ Removed ``Py_END_ALLOW_RECURSION`` and the ``recursion_critical`` field of the :c:type:`PyInterpreterState` structure. (Contributed by Serhiy Storchaka in :issue:`41936`.) + +* Removed the undocumented ``PyOS_InitInterrupts()`` function. Initializing + Python already implicitly installs signal handlers: see + :c:member:`PyConfig.install_signal_handlers`. + (Contributed by Victor Stinner in :issue:`41713`.) + +* Remove the ``PyAST_Validate()`` function. It is no longer possible to build a + AST object (``mod_ty`` type) with the public C API. The function was already + excluded from the limited C API (:pep:`384`). + (Contributed by Victor Stinner in :issue:`43244`.) + +* Remove the ``symtable.h`` header file and the undocumented functions: + + * ``PyST_GetScope()`` + * ``PySymtable_Build()`` + * ``PySymtable_BuildObject()`` + * ``PySymtable_Free()`` + * ``Py_SymtableString()`` + * ``Py_SymtableStringObject()`` + + The ``Py_SymtableString()`` function was part the stable ABI by mistake but + it could not be used, because the ``symtable.h`` header file was excluded + from the limited C API. + + Use Python :mod:`symtable` module instead. + (Contributed by Victor Stinner in :issue:`43244`.) + +* Remove :c:func:`PyOS_ReadlineFunctionPointer` from the limited C API headers + and from ``python3.dll``, the library that provides the stable ABI on + Windows. Since the function takes a ``FILE*`` argument, its ABI stability + cannot be guaranteed. + (Contributed by Petr Viktorin in :issue:`43868`.) + +* Remove ``ast.h``, ``asdl.h``, and ``Python-ast.h`` header files. + These functions were undocumented and excluded from the limited C API. + Most names defined by these header files were not prefixed by ``Py`` and so + could create names conflicts. For example, ``Python-ast.h`` defined a + ``Yield`` macro which was conflict with the ``Yield`` name used by the + Windows ```` header. Use the Python :mod:`ast` module instead. + (Contributed by Victor Stinner in :issue:`43244`.) + +* Remove the compiler and parser functions using ``struct _mod`` type, because + the public AST C API was removed: + + * ``PyAST_Compile()`` + * ``PyAST_CompileEx()`` + * ``PyAST_CompileObject()`` + * ``PyFuture_FromAST()`` + * ``PyFuture_FromASTObject()`` + * ``PyParser_ASTFromFile()`` + * ``PyParser_ASTFromFileObject()`` + * ``PyParser_ASTFromFilename()`` + * ``PyParser_ASTFromString()`` + * ``PyParser_ASTFromStringObject()`` + + These functions were undocumented and excluded from the limited C API. + (Contributed by Victor Stinner in :issue:`43244`.) + +* Remove the ``pyarena.h`` header file with functions: + + * ``PyArena_New()`` + * ``PyArena_Free()`` + * ``PyArena_Malloc()`` + * ``PyArena_AddPyObject()`` + + These functions were undocumented, excluded from the limited C API, and were + only used internally by the compiler. + (Contributed by Victor Stinner in :issue:`43244`.) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 85a6657fdfbdac..d7884ea30d7d2b 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -722,8 +722,8 @@ for the following events in the interpreter: * line of code executed. This can be used to instrument running interpreters in production, -without the need to recompile specific debug builds or providing -application-specific profiling/debugging code. +without the need to recompile specific :ref:`debug builds ` or +providing application-specific profiling/debugging code. More details in :ref:`instrumentation`. @@ -2443,3 +2443,16 @@ because of the behavior of the socket option ``SO_REUSEADDR`` in UDP. For more details, see the documentation for ``loop.create_datagram_endpoint()``. (Contributed by Kyle Stanley, Antoine Pitrou, and Yury Selivanov in :issue:`37228`.) + +Notable changes in Python 3.6.13 +================================ + +Earlier Python versions allowed using both ``;`` and ``&`` as +query parameter separators in :func:`urllib.parse.parse_qs` and +:func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with +newer W3C recommendations, this has been changed to allow only a single +separator key, with ``&`` as the default. This change also affects +:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected +functions internally. For more details, please see their respective +documentation. +(Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 25b1e1e33e325c..1199535c84dcaa 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -232,7 +232,8 @@ PEP 540: Forced UTF-8 Runtime Mode ----------------------------------- The new :option:`-X` ``utf8`` command line option and :envvar:`PYTHONUTF8` -environment variable can be used to enable the CPython *UTF-8 mode*. +environment variable can be used to enable the :ref:`Python UTF-8 Mode +`. When in UTF-8 mode, CPython ignores the locale settings, and uses the UTF-8 encoding by default. The error handlers for :data:`sys.stdin` and @@ -1587,8 +1588,8 @@ The initialization of the default warnings filters has changed as follows: for the new ``-X dev`` mode) * any implicit filters defined directly by the warnings machinery -* in CPython debug builds, all warnings are now displayed by default (the - implicit filter list is empty) +* in :ref:`CPython debug builds `, all warnings are now displayed + by default (the implicit filter list is empty) (Contributed by Nick Coghlan and Victor Stinner in :issue:`20361`, :issue:`32043`, and :issue:`32230`.) @@ -2556,3 +2557,16 @@ because of the behavior of the socket option ``SO_REUSEADDR`` in UDP. For more details, see the documentation for ``loop.create_datagram_endpoint()``. (Contributed by Kyle Stanley, Antoine Pitrou, and Yury Selivanov in :issue:`37228`.) + +Notable changes in Python 3.7.10 +================================ + +Earlier Python versions allowed using both ``;`` and ``&`` as +query parameter separators in :func:`urllib.parse.parse_qs` and +:func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with +newer W3C recommendations, this has been changed to allow only a single +separator key, with ``&`` as the default. This change also affects +:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected +functions internally. For more details, please see their respective +documentation. +(Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 6a9fa341569465..f1a903624f4c2c 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -211,12 +211,12 @@ Python now uses the same ABI whether it's built in release or debug mode. On Unix, when Python is built in debug mode, it is now possible to load C extensions built in release mode and C extensions built using the stable ABI. -Release builds and debug builds are now ABI compatible: defining the +Release builds and :ref:`debug builds ` are now ABI compatible: defining the ``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro, which introduces the only ABI incompatibility. The ``Py_TRACE_REFS`` macro, which adds the :func:`sys.getobjects` function and the :envvar:`PYTHONDUMPREFS` -environment variable, can be set using the new ``./configure --with-trace-refs`` -build option. +environment variable, can be set using the new :option:`./configure +--with-trace-refs <--with-trace-refs>` build option. (Contributed by Victor Stinner in :issue:`36465`.) On Unix, C extensions are no longer linked to libpython except on Android @@ -352,7 +352,8 @@ PEP 590: Vectorcall: a fast calling protocol for CPython :ref:`vectorcall` is added to the Python/C API. It is meant to formalize existing optimizations which were already done for various classes. -Any static type implementing a callable can use this protocol. +Any :ref:`static type ` implementing a callable can use this +protocol. This is currently provisional. The aim is to make it fully public in Python 3.9. @@ -904,7 +905,7 @@ for :func:`property`, :func:`classmethod`, and :func:`staticmethod`:: io -- -In development mode (:option:`-X` ``env``) and in debug build, the +In development mode (:option:`-X` ``env``) and in :ref:`debug build `, the :class:`io.IOBase` finalizer now logs the exception if the ``close()`` method fails. The exception is ignored silently by default in release build. (Contributed by Victor Stinner in :issue:`18748`.) @@ -2040,7 +2041,7 @@ Changes in the C API This makes types created through :c:func:`PyType_FromSpec` behave like other classes in managed code. - Statically allocated types are not affected. + :ref:`Statically allocated types ` are not affected. For the vast majority of cases, there should be no side effect. However, types that manually increase the reference count after allocating @@ -2113,9 +2114,6 @@ Changes in the C API (Contributed by Antoine Pitrou in :issue:`32388`.) -* The :c:func:`PyCode_New` has a new parameter in the second position (*posonlyargcount*) - to support :pep:`570`, indicating the number of positional-only arguments. - * The functions :c:func:`PyNode_AddChild` and :c:func:`PyParser_AddToken` now accept two additional ``int`` arguments *end_lineno* and *end_col_offset*. @@ -2237,3 +2235,16 @@ because of the behavior of the socket option ``SO_REUSEADDR`` in UDP. For more details, see the documentation for ``loop.create_datagram_endpoint()``. (Contributed by Kyle Stanley, Antoine Pitrou, and Yury Selivanov in :issue:`37228`.) + +Notable changes in Python 3.8.8 +=============================== + +Earlier Python versions allowed using both ``;`` and ``&`` as +query parameter separators in :func:`urllib.parse.parse_qs` and +:func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with +newer W3C recommendations, this has been changed to allow only a single +separator key, with ``&`` as the default. This change also affects +:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected +functions internally. For more details, please see their respective +documentation. +(Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 1a37f16ea2b09a..772fb5a3fe7458 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -135,8 +135,8 @@ More generally, try to run your tests in the :ref:`Python Development Mode ` which helps to prepare your code to make it compatible with the next Python version. -Note: a number of pre-existing deprecatations were removed in this version -of Python as well. Consult the :ref:`removed-in-python-39` section. +Note: a number of pre-existing deprecations were removed in this version of +Python as well. Consult the :ref:`removed-in-python-39` section. New Features @@ -231,7 +231,7 @@ Other Language Changes absolute path for :mod:`__main__` module frames in this case. (Contributed by Victor Stinner in :issue:`20443`.) -* In the :ref:`Python Development Mode ` and in debug build, the +* In the :ref:`Python Development Mode ` and in :ref:`debug build `, the *encoding* and *errors* arguments are now checked for string encoding and decoding operations. Examples: :func:`open`, :meth:`str.encode` and :meth:`bytes.decode`. @@ -267,6 +267,10 @@ Other Language Changes :func:`~operator.countOf` of the :mod:`operator` module. (Contributed by Serhiy Storchaka in :issue:`40824`.) +* Unparenthesized lambda expressions can no longer be the expression part in an + ``if`` clause in comprehensions and generator expressions. See :issue:`41848` + and :issue:`43755` for details. + New Modules =========== @@ -533,6 +537,10 @@ Scoped IPv6 addresses can be parsed using :class:`ipaddress.IPv6Address`. If present, scope zone ID is available through the :attr:`~ipaddress.IPv6Address.scope_id` attribute. (Contributed by Oleksandr Pavliuk in :issue:`34788`.) +Starting with Python 3.9.5 the :mod:`ipaddress` module no longer +accepts any leading zeros in IPv4 address strings. +(Contributed by Christian Heimes in :issue:`36384`). + math ---- @@ -654,7 +662,7 @@ The socket module now supports the :data:`~socket.CAN_J1939` protocol on platforms that support it. (Contributed by Karl Ding in :issue:`40291`.) The socket module now has the :func:`socket.send_fds` and -:func:`socket.recv.fds` methods. (Contributed by Joannah Nanjekye, Shinya +:func:`socket.recv_fds` functions. (Contributed by Joannah Nanjekye, Shinya Okano and Victor Stinner in :issue:`28724`.) @@ -1110,11 +1118,25 @@ Changes in the Python API compatible classes that don't inherit from those mentioned types. (Contributed by Roger Aiudi in :issue:`34775`). +* Starting with Python 3.9.5 the :mod:`ipaddress` module no longer + accepts any leading zeros in IPv4 address strings. Leading zeros are + ambiguous and interpreted as octal notation by some libraries. For example + the legacy function :func:`socket.inet_aton` treats leading zeros as octal + notatation. glibc implementation of modern :func:`~socket.inet_pton` does + not accept any leading zeros. + (Contributed by Christian Heimes in :issue:`36384`). + +* :func:`codecs.lookup` now normalizes the encoding name the same way as + :func:`encodings.normalize_encoding`, except that :func:`codecs.lookup` also + converts the name to lower case. For example, ``"latex+latin1"`` encoding + name is now normalized to ``"latex_latin1"``. + (Contributed by Jordon Xu in :issue:`37751`.) + Changes in the C API -------------------- -* Instances of heap-allocated types (such as those created with +* Instances of :ref:`heap-allocated types ` (such as those created with :c:func:`PyType_FromSpec` and similar APIs) hold a reference to their type object since Python 3.8. As indicated in the "Changes in the C API" of Python 3.8, for the vast majority of cases, there should be no side effect but for @@ -1137,7 +1159,8 @@ Changes in the C API If your traverse function delegates to ``tp_traverse`` of its base class (or another type), ensure that ``Py_TYPE(self)`` is visited only once. - Note that only heap types are expected to visit the type in ``tp_traverse``. + Note that only :ref:`heap type ` are expected to visit the type + in ``tp_traverse``. For example, if your ``tp_traverse`` function includes: @@ -1150,7 +1173,7 @@ Changes in the C API .. code-block:: c #if PY_VERSION_HEX >= 0x03090000 - // This was not needed before Python 3.9 (Python issue 35810 and 40217) + // This was not needed before Python 3.9 (bpo-35810 and bpo-40217) if (base->tp_flags & Py_TPFLAGS_HEAPTYPE) { // a heap type's tp_traverse already visited Py_TYPE(self) } else { @@ -1207,9 +1230,9 @@ Build Changes non-system Tcl and Tk frameworks if they are installed in ``/Library/Frameworks``, as had been the case on older releases of macOS. If a macOS SDK is explicitly configured, by using - ``--enable-universalsdk=`` or ``-isysroot``, only the SDK itself is + :option:`--enable-universalsdk` or ``-isysroot``, only the SDK itself is searched. The default behavior can still be overridden with - ``--with-tcltk-includes`` and ``--with-tcltk-libs``. + :option:`--with-tcltk-includes` and :option:`--with-tcltk-libs`. (Contributed by Ned Deily in :issue:`34956`.) * Python can now be built for Windows 10 ARM64. @@ -1289,6 +1312,10 @@ New Features representation of a function-like object. (Patch by Jeroen Demeyer in :issue:`37645`.) +* Added :c:func:`PyObject_CallOneArg` for calling an object with one + positional argument + (Patch by Jeroen Demeyer in :issue:`37483`.) + Porting to Python 3.9 --------------------- @@ -1450,3 +1477,78 @@ Removed ``PyNullImporter_Type``, ``PyCmpWrapper_Type``, ``PySortWrapper_Type``, ``PyNoArgsFunction``. (Contributed by Pablo Galindo Salgado in :issue:`39372`.) + +Notable changes in Python 3.9.1 +=============================== + +typing +------ + +The behavior of :class:`typing.Literal` was changed to conform with :pep:`586` +and to match the behavior of static type checkers specified in the PEP. + +1. ``Literal`` now de-duplicates parameters. +2. Equality comparisons between ``Literal`` objects are now order independent. +3. ``Literal`` comparisons now respect types. For example, + ``Literal[0] == Literal[False]`` previously evaluated to ``True``. It is + now ``False``. To support this change, the internally used type cache now + supports differentiating types. +4. ``Literal`` objects will now raise a :exc:`TypeError` exception during + equality comparisons if one of their parameters are not :term:`immutable`. + Note that declaring ``Literal`` with mutable parameters will not throw + an error:: + + >>> from typing import Literal + >>> Literal[{0}] + >>> Literal[{0}] == Literal[{False}] + Traceback (most recent call last): + File "", line 1, in + TypeError: unhashable type: 'set' + +(Contributed by Yurii Karabas in :issue:`42345`.) + +macOS 11.0 (Big Sur) and Apple Silicon Mac support +-------------------------------------------------- + +As of 3.9.1, Python now fully supports building and running on macOS 11.0 +(Big Sur) and on Apple Silicon Macs (based on the ``ARM64`` architecture). +A new universal build variant, ``universal2``, is now available to natively +support both ``ARM64`` and ``Intel 64`` in one set of executables. Binaries +can also now be built on current versions of macOS to be deployed on a range +of older macOS versions (tested to 10.9) while making some newer OS +functions and options conditionally available based on the operating system +version in use at runtime ("weaklinking"). + +(Contributed by Ronald Oussoren and Lawrence D'Anna in :issue:`41100`.) + +Notable changes in Python 3.9.2 +=============================== + +collections.abc +--------------- + +:class:`collections.abc.Callable` generic now flattens type parameters, similar +to what :data:`typing.Callable` currently does. This means that +``collections.abc.Callable[[int, str], str]`` will have ``__args__`` of +``(int, str, str)``; previously this was ``([int, str], str)``. To allow this +change, :class:`types.GenericAlias` can now be subclassed, and a subclass will +be returned when subscripting the :class:`collections.abc.Callable` type. +Code which accesses the arguments via :func:`typing.get_args` or ``__args__`` +need to account for this change. A :exc:`DeprecationWarning` may be emitted for +invalid forms of parameterizing :class:`collections.abc.Callable` which may have +passed silently in Python 3.9.1. This :exc:`DeprecationWarning` will +become a :exc:`TypeError` in Python 3.10. +(Contributed by Ken Jin in :issue:`42195`.) + +urllib.parse +------------ + +Earlier Python versions allowed using both ``;`` and ``&`` as +query parameter separators in :func:`urllib.parse.parse_qs` and +:func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with +newer W3C recommendations, this has been changed to allow only a single +separator key, with ``&`` as the default. This change also affects +:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected +functions internally. For more details, please see their respective +documentation. +(Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) diff --git a/Grammar/Tokens b/Grammar/Tokens index 9de2da5d15fc3b..1f3e3b09913653 100644 --- a/Grammar/Tokens +++ b/Grammar/Tokens @@ -59,6 +59,7 @@ AWAIT ASYNC TYPE_IGNORE TYPE_COMMENT +SOFT_KEYWORD ERRORTOKEN # These aren't used by the C tokenizer but are needed for tokenize.py diff --git a/Grammar/python.gram b/Grammar/python.gram index 19c85accf8d9a1..2f553c6d390c71 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -7,6 +7,7 @@ _PyPegen_parse(Parser *p) // Initialize keywords p->keywords = reserved_keywords; p->n_keyword_lists = n_keyword_lists; + p->soft_keywords = soft_keywords; // Run parser void *result = NULL; @@ -28,9 +29,9 @@ _PyPegen_parse(Parser *p) // The end ''' file[mod_ty]: a=[statements] ENDMARKER { _PyPegen_make_module(p, a) } -interactive[mod_ty]: a=statement_newline { Interactive(a, p->arena) } -eval[mod_ty]: a=expressions NEWLINE* ENDMARKER { Expression(a, p->arena) } -func_type[mod_ty]: '(' a=[type_expressions] ')' '->' b=expression NEWLINE* ENDMARKER { FunctionType(a, b, p->arena) } +interactive[mod_ty]: a=statement_newline { _PyAST_Interactive(a, p->arena) } +eval[mod_ty]: a=expressions NEWLINE* ENDMARKER { _PyAST_Expression(a, p->arena) } +func_type[mod_ty]: '(' a=[type_expressions] ')' '->' b=expression NEWLINE* ENDMARKER { _PyAST_FunctionType(a, b, p->arena) } fstring[expr_ty]: star_expressions # type_expressions allow */** but ignore them @@ -52,29 +53,29 @@ type_expressions[asdl_expr_seq*]: | a[asdl_expr_seq*]=','.expression+ {a} statements[asdl_stmt_seq*]: a=statement+ { (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a) } -statement[asdl_stmt_seq*]: a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } | a[asdl_stmt_seq*]=simple_stmt { a } +statement[asdl_stmt_seq*]: a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } | a[asdl_stmt_seq*]=simple_stmts { a } statement_newline[asdl_stmt_seq*]: | a=compound_stmt NEWLINE { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } - | simple_stmt - | NEWLINE { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, CHECK(stmt_ty, _Py_Pass(EXTRA))) } + | simple_stmts + | NEWLINE { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, CHECK(stmt_ty, _PyAST_Pass(EXTRA))) } | ENDMARKER { _PyPegen_interactive_exit(p) } -simple_stmt[asdl_stmt_seq*]: - | a=small_stmt !';' NEWLINE { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } # Not needed, there for speedup - | a[asdl_stmt_seq*]=';'.small_stmt+ [';'] NEWLINE { a } +simple_stmts[asdl_stmt_seq*]: + | a=simple_stmt !';' NEWLINE { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } # Not needed, there for speedup + | a[asdl_stmt_seq*]=';'.simple_stmt+ [';'] NEWLINE { a } # NOTE: assignment MUST precede expression, else parsing a simple assignment # will throw a SyntaxError. -small_stmt[stmt_ty] (memo): +simple_stmt[stmt_ty] (memo): | assignment - | e=star_expressions { _Py_Expr(e, EXTRA) } + | e=star_expressions { _PyAST_Expr(e, EXTRA) } | &'return' return_stmt | &('import' | 'from') import_stmt | &'raise' raise_stmt - | 'pass' { _Py_Pass(EXTRA) } + | 'pass' { _PyAST_Pass(EXTRA) } | &'del' del_stmt | &'yield' yield_stmt | &'assert' assert_stmt - | 'break' { _Py_Break(EXTRA) } - | 'continue' { _Py_Continue(EXTRA) } + | 'break' { _PyAST_Break(EXTRA) } + | 'continue' { _PyAST_Continue(EXTRA) } | &'global' global_stmt | &'nonlocal' nonlocal_stmt compound_stmt[stmt_ty]: @@ -85,6 +86,7 @@ compound_stmt[stmt_ty]: | &('for' | ASYNC) for_stmt | &'try' try_stmt | &'while' while_stmt + | match_stmt # NOTE: annotated_rhs may start with 'yield'; yield_expr must start with 'yield' assignment[stmt_ty]: @@ -93,15 +95,15 @@ assignment[stmt_ty]: stmt_ty, 6, "Variable annotation syntax is", - _Py_AnnAssign(CHECK(expr_ty, _PyPegen_set_expr_context(p, a, Store)), b, c, 1, EXTRA) + _PyAST_AnnAssign(CHECK(expr_ty, _PyPegen_set_expr_context(p, a, Store)), b, c, 1, EXTRA) ) } | a=('(' b=single_target ')' { b } | single_subscript_attribute_target) ':' b=expression c=['=' d=annotated_rhs { d }] { - CHECK_VERSION(stmt_ty, 6, "Variable annotations syntax is", _Py_AnnAssign(a, b, c, 0, EXTRA)) } + CHECK_VERSION(stmt_ty, 6, "Variable annotations syntax is", _PyAST_AnnAssign(a, b, c, 0, EXTRA)) } | a[asdl_expr_seq*]=(z=star_targets '=' { z })+ b=(yield_expr | star_expressions) !'=' tc=[TYPE_COMMENT] { - _Py_Assign(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } + _PyAST_Assign(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } | a=single_target b=augassign ~ c=(yield_expr | star_expressions) { - _Py_AugAssign(a, b->kind, c, EXTRA) } + _PyAST_AugAssign(a, b->kind, c, EXTRA) } | invalid_assignment augassign[AugOperator*]: @@ -120,112 +122,293 @@ augassign[AugOperator*]: | '//=' { _PyPegen_augoperator(p, FloorDiv) } global_stmt[stmt_ty]: 'global' a[asdl_expr_seq*]=','.NAME+ { - _Py_Global(CHECK(asdl_identifier_seq*, _PyPegen_map_names_to_ids(p, a)), EXTRA) } + _PyAST_Global(CHECK(asdl_identifier_seq*, _PyPegen_map_names_to_ids(p, a)), EXTRA) } nonlocal_stmt[stmt_ty]: 'nonlocal' a[asdl_expr_seq*]=','.NAME+ { - _Py_Nonlocal(CHECK(asdl_identifier_seq*, _PyPegen_map_names_to_ids(p, a)), EXTRA) } + _PyAST_Nonlocal(CHECK(asdl_identifier_seq*, _PyPegen_map_names_to_ids(p, a)), EXTRA) } -yield_stmt[stmt_ty]: y=yield_expr { _Py_Expr(y, EXTRA) } +yield_stmt[stmt_ty]: y=yield_expr { _PyAST_Expr(y, EXTRA) } -assert_stmt[stmt_ty]: 'assert' a=expression b=[',' z=expression { z }] { _Py_Assert(a, b, EXTRA) } +assert_stmt[stmt_ty]: 'assert' a=expression b=[',' z=expression { z }] { _PyAST_Assert(a, b, EXTRA) } del_stmt[stmt_ty]: - | 'del' a=del_targets &(';' | NEWLINE) { _Py_Delete(a, EXTRA) } + | 'del' a=del_targets &(';' | NEWLINE) { _PyAST_Delete(a, EXTRA) } | invalid_del_stmt import_stmt[stmt_ty]: import_name | import_from -import_name[stmt_ty]: 'import' a=dotted_as_names { _Py_Import(a, EXTRA) } +import_name[stmt_ty]: 'import' a=dotted_as_names { _PyAST_Import(a, EXTRA) } # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS import_from[stmt_ty]: | 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets { - _Py_ImportFrom(b->v.Name.id, c, _PyPegen_seq_count_dots(a), EXTRA) } + _PyAST_ImportFrom(b->v.Name.id, c, _PyPegen_seq_count_dots(a), EXTRA) } | 'from' a=('.' | '...')+ 'import' b=import_from_targets { - _Py_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), EXTRA) } + _PyAST_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), EXTRA) } import_from_targets[asdl_alias_seq*]: | '(' a=import_from_as_names [','] ')' { a } | import_from_as_names !',' - | '*' { (asdl_alias_seq*)_PyPegen_singleton_seq(p, CHECK(alias_ty, _PyPegen_alias_for_star(p))) } + | '*' { (asdl_alias_seq*)_PyPegen_singleton_seq(p, CHECK(alias_ty, _PyPegen_alias_for_star(p, EXTRA))) } | invalid_import_from_targets import_from_as_names[asdl_alias_seq*]: | a[asdl_alias_seq*]=','.import_from_as_name+ { a } import_from_as_name[alias_ty]: - | a=NAME b=['as' z=NAME { z }] { _Py_alias(a->v.Name.id, + | a=NAME b=['as' z=NAME { z }] { _PyAST_alias(a->v.Name.id, (b) ? ((expr_ty) b)->v.Name.id : NULL, - p->arena) } + EXTRA) } dotted_as_names[asdl_alias_seq*]: | a[asdl_alias_seq*]=','.dotted_as_name+ { a } dotted_as_name[alias_ty]: - | a=dotted_name b=['as' z=NAME { z }] { _Py_alias(a->v.Name.id, + | a=dotted_name b=['as' z=NAME { z }] { _PyAST_alias(a->v.Name.id, (b) ? ((expr_ty) b)->v.Name.id : NULL, - p->arena) } + EXTRA) } dotted_name[expr_ty]: | a=dotted_name '.' b=NAME { _PyPegen_join_names_with_dot(p, a, b) } | NAME if_stmt[stmt_ty]: + | invalid_if_stmt | 'if' a=named_expression ':' b=block c=elif_stmt { - _Py_If(a, b, CHECK(asdl_stmt_seq*, _PyPegen_singleton_seq(p, c)), EXTRA) } - | 'if' a=named_expression ':' b=block c=[else_block] { _Py_If(a, b, c, EXTRA) } + _PyAST_If(a, b, CHECK(asdl_stmt_seq*, _PyPegen_singleton_seq(p, c)), EXTRA) } + | 'if' a=named_expression ':' b=block c=[else_block] { _PyAST_If(a, b, c, EXTRA) } elif_stmt[stmt_ty]: + | invalid_elif_stmt | 'elif' a=named_expression ':' b=block c=elif_stmt { - _Py_If(a, b, CHECK(asdl_stmt_seq*, _PyPegen_singleton_seq(p, c)), EXTRA) } - | 'elif' a=named_expression ':' b=block c=[else_block] { _Py_If(a, b, c, EXTRA) } -else_block[asdl_stmt_seq*]: 'else' ':' b=block { b } + _PyAST_If(a, b, CHECK(asdl_stmt_seq*, _PyPegen_singleton_seq(p, c)), EXTRA) } + | 'elif' a=named_expression ':' b=block c=[else_block] { _PyAST_If(a, b, c, EXTRA) } +else_block[asdl_stmt_seq*]: + | invalid_else_stmt + | 'else' &&':' b=block { b } while_stmt[stmt_ty]: - | 'while' a=named_expression ':' b=block c=[else_block] { _Py_While(a, b, c, EXTRA) } + | invalid_while_stmt + | 'while' a=named_expression ':' b=block c=[else_block] { _PyAST_While(a, b, c, EXTRA) } for_stmt[stmt_ty]: - | 'for' t=star_targets 'in' ~ ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] { - _Py_For(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA) } - | ASYNC 'for' t=star_targets 'in' ~ ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] { - CHECK_VERSION(stmt_ty, 5, "Async for loops are", _Py_AsyncFor(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA)) } + | invalid_for_stmt + | 'for' t=star_targets 'in' ~ ex=star_expressions &&':' tc=[TYPE_COMMENT] b=block el=[else_block] { + _PyAST_For(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA) } + | ASYNC 'for' t=star_targets 'in' ~ ex=star_expressions &&':' tc=[TYPE_COMMENT] b=block el=[else_block] { + CHECK_VERSION(stmt_ty, 5, "Async for loops are", _PyAST_AsyncFor(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA)) } | invalid_for_target with_stmt[stmt_ty]: + | invalid_with_stmt_indent | 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' b=block { - _Py_With(a, b, NULL, EXTRA) } + _PyAST_With(a, b, NULL, EXTRA) } | 'with' a[asdl_withitem_seq*]=','.with_item+ ':' tc=[TYPE_COMMENT] b=block { - _Py_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } + _PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } | ASYNC 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' b=block { - CHECK_VERSION(stmt_ty, 5, "Async with statements are", _Py_AsyncWith(a, b, NULL, EXTRA)) } + CHECK_VERSION(stmt_ty, 5, "Async with statements are", _PyAST_AsyncWith(a, b, NULL, EXTRA)) } | ASYNC 'with' a[asdl_withitem_seq*]=','.with_item+ ':' tc=[TYPE_COMMENT] b=block { - CHECK_VERSION(stmt_ty, 5, "Async with statements are", _Py_AsyncWith(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA)) } + CHECK_VERSION(stmt_ty, 5, "Async with statements are", _PyAST_AsyncWith(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA)) } + | invalid_with_stmt + with_item[withitem_ty]: - | e=expression 'as' t=star_target &(',' | ')' | ':') { _Py_withitem(e, t, p->arena) } + | e=expression 'as' t=star_target &(',' | ')' | ':') { _PyAST_withitem(e, t, p->arena) } | invalid_with_item - | e=expression { _Py_withitem(e, NULL, p->arena) } + | e=expression { _PyAST_withitem(e, NULL, p->arena) } try_stmt[stmt_ty]: - | 'try' ':' b=block f=finally_block { _Py_Try(b, NULL, NULL, f, EXTRA) } - | 'try' ':' b=block ex[asdl_excepthandler_seq*]=except_block+ el=[else_block] f=[finally_block] { _Py_Try(b, ex, el, f, EXTRA) } + | invalid_try_stmt + | 'try' &&':' b=block f=finally_block { _PyAST_Try(b, NULL, NULL, f, EXTRA) } + | 'try' &&':' b=block ex[asdl_excepthandler_seq*]=except_block+ el=[else_block] f=[finally_block] { _PyAST_Try(b, ex, el, f, EXTRA) } except_block[excepthandler_ty]: + | invalid_except_stmt_indent | 'except' e=expression t=['as' z=NAME { z }] ':' b=block { - _Py_ExceptHandler(e, (t) ? ((expr_ty) t)->v.Name.id : NULL, b, EXTRA) } - | 'except' ':' b=block { _Py_ExceptHandler(NULL, NULL, b, EXTRA) } -finally_block[asdl_stmt_seq*]: 'finally' ':' a=block { a } + _PyAST_ExceptHandler(e, (t) ? ((expr_ty) t)->v.Name.id : NULL, b, EXTRA) } + | 'except' ':' b=block { _PyAST_ExceptHandler(NULL, NULL, b, EXTRA) } + | invalid_except_stmt +finally_block[asdl_stmt_seq*]: + | invalid_finally_stmt + | 'finally' &&':' a=block { a } + +match_stmt[stmt_ty]: + | "match" subject=subject_expr ':' NEWLINE INDENT cases[asdl_match_case_seq*]=case_block+ DEDENT { + CHECK_VERSION(stmt_ty, 10, "Pattern matching is", _PyAST_Match(subject, cases, EXTRA)) } + | invalid_match_stmt +subject_expr[expr_ty]: + | value=star_named_expression ',' values=star_named_expressions? { + _PyAST_Tuple(CHECK(asdl_expr_seq*, _PyPegen_seq_insert_in_front(p, value, values)), Load, EXTRA) } + | named_expression +case_block[match_case_ty]: + | invalid_case_block + | "case" pattern=patterns guard=guard? ':' body=block { + _PyAST_match_case(pattern, guard, body, p->arena) } +guard[expr_ty]: 'if' guard=named_expression { guard } + +patterns[pattern_ty]: + | patterns[asdl_pattern_seq*]=open_sequence_pattern { + _PyAST_MatchSequence(patterns, EXTRA) } + | pattern +pattern[pattern_ty]: + | as_pattern + | or_pattern +as_pattern[pattern_ty]: + | pattern=or_pattern 'as' target=pattern_capture_target { + _PyAST_MatchAs(pattern, target->v.Name.id, EXTRA) } +or_pattern[pattern_ty]: + | patterns[asdl_pattern_seq*]='|'.closed_pattern+ { + asdl_seq_LEN(patterns) == 1 ? asdl_seq_GET(patterns, 0) : _PyAST_MatchOr(patterns, EXTRA) } +closed_pattern[pattern_ty]: + | literal_pattern + | capture_pattern + | wildcard_pattern + | value_pattern + | group_pattern + | sequence_pattern + | mapping_pattern + | class_pattern + +# Literal patterns are used for equality and identity constraints +literal_pattern[pattern_ty]: + | value=signed_number !('+' | '-') { _PyAST_MatchValue(value, EXTRA) } + | value=complex_number { _PyAST_MatchValue(value, EXTRA) } + | value=strings { _PyAST_MatchValue(value, EXTRA) } + | 'None' { _PyAST_MatchSingleton(Py_None, EXTRA) } + | 'True' { _PyAST_MatchSingleton(Py_True, EXTRA) } + | 'False' { _PyAST_MatchSingleton(Py_False, EXTRA) } + +# Literal expressions are used to restrict permitted mapping pattern keys +literal_expr[expr_ty]: + | signed_number !('+' | '-') + | complex_number + | strings + | 'None' { _PyAST_Constant(Py_None, NULL, EXTRA) } + | 'True' { _PyAST_Constant(Py_True, NULL, EXTRA) } + | 'False' { _PyAST_Constant(Py_False, NULL, EXTRA) } + +complex_number[expr_ty]: + | real=signed_real_number '+' imag=imaginary_number { + _PyAST_BinOp(real, Add, imag, EXTRA) } + | real=signed_real_number '-' imag=imaginary_number { + _PyAST_BinOp(real, Sub, imag, EXTRA) } + +signed_number[expr_ty]: + | NUMBER + | '-' number=NUMBER { _PyAST_UnaryOp(USub, number, EXTRA) } + +signed_real_number[expr_ty]: + | real_number + | '-' real=real_number { _PyAST_UnaryOp(USub, real, EXTRA) } + +real_number[expr_ty]: + | real=NUMBER { _PyPegen_ensure_real(p, real) } + +imaginary_number[expr_ty]: + | imag=NUMBER { _PyPegen_ensure_imaginary(p, imag) } + +capture_pattern[pattern_ty]: + | target=pattern_capture_target { _PyAST_MatchAs(NULL, target->v.Name.id, EXTRA) } + +pattern_capture_target[expr_ty]: + | !"_" name=NAME !('.' | '(' | '=') { + _PyPegen_set_expr_context(p, name, Store) } + +wildcard_pattern[pattern_ty]: + | "_" { _PyAST_MatchAs(NULL, NULL, EXTRA) } + +value_pattern[pattern_ty]: + | attr=attr !('.' | '(' | '=') { _PyAST_MatchValue(attr, EXTRA) } +attr[expr_ty]: + | value=name_or_attr '.' attr=NAME { + _PyAST_Attribute(value, attr->v.Name.id, Load, EXTRA) } +name_or_attr[expr_ty]: + | attr + | NAME + +group_pattern[pattern_ty]: + | '(' pattern=pattern ')' { pattern } + +sequence_pattern[pattern_ty]: + | '[' patterns=maybe_sequence_pattern? ']' { _PyAST_MatchSequence(patterns, EXTRA) } + | '(' patterns=open_sequence_pattern? ')' { _PyAST_MatchSequence(patterns, EXTRA) } +open_sequence_pattern[asdl_seq*]: + | pattern=maybe_star_pattern ',' patterns=maybe_sequence_pattern? { + _PyPegen_seq_insert_in_front(p, pattern, patterns) } +maybe_sequence_pattern[asdl_seq*]: + | patterns=','.maybe_star_pattern+ ','? { patterns } +maybe_star_pattern[pattern_ty]: + | star_pattern + | pattern +star_pattern[pattern_ty]: + | '*' target=pattern_capture_target { + _PyAST_MatchStar(target->v.Name.id, EXTRA) } + | '*' wildcard_pattern { + _PyAST_MatchStar(NULL, EXTRA) } + +mapping_pattern[pattern_ty]: + | '{' '}' { + _PyAST_MatchMapping(NULL, NULL, NULL, EXTRA) } + | '{' rest=double_star_pattern ','? '}' { + _PyAST_MatchMapping(NULL, NULL, rest->v.Name.id, EXTRA) } + | '{' items=items_pattern ',' rest=double_star_pattern ','? '}' { + _PyAST_MatchMapping( + CHECK(asdl_expr_seq*, _PyPegen_get_pattern_keys(p, items)), + CHECK(asdl_pattern_seq*, _PyPegen_get_patterns(p, items)), + rest->v.Name.id, + EXTRA) } + | '{' items=items_pattern ','? '}' { + _PyAST_MatchMapping( + CHECK(asdl_expr_seq*, _PyPegen_get_pattern_keys(p, items)), + CHECK(asdl_pattern_seq*, _PyPegen_get_patterns(p, items)), + NULL, + EXTRA) } +items_pattern[asdl_seq*]: + | ','.key_value_pattern+ +key_value_pattern[KeyPatternPair*]: + | key=(literal_expr | attr) ':' pattern=pattern { + _PyPegen_key_pattern_pair(p, key, pattern) } +double_star_pattern[expr_ty]: + | '**' target=pattern_capture_target { target } + +class_pattern[pattern_ty]: + | cls=name_or_attr '(' ')' { + _PyAST_MatchClass(cls, NULL, NULL, NULL, EXTRA) } + | cls=name_or_attr '(' patterns=positional_patterns ','? ')' { + _PyAST_MatchClass(cls, patterns, NULL, NULL, EXTRA) } + | cls=name_or_attr '(' keywords=keyword_patterns ','? ')' { + _PyAST_MatchClass( + cls, NULL, + CHECK(asdl_identifier_seq*, _PyPegen_map_names_to_ids(p, + CHECK(asdl_expr_seq*, _PyPegen_get_pattern_keys(p, keywords)))), + CHECK(asdl_pattern_seq*, _PyPegen_get_patterns(p, keywords)), + EXTRA) } + | cls=name_or_attr '(' patterns=positional_patterns ',' keywords=keyword_patterns ','? ')' { + _PyAST_MatchClass( + cls, + patterns, + CHECK(asdl_identifier_seq*, _PyPegen_map_names_to_ids(p, + CHECK(asdl_expr_seq*, _PyPegen_get_pattern_keys(p, keywords)))), + CHECK(asdl_pattern_seq*, _PyPegen_get_patterns(p, keywords)), + EXTRA) } +positional_patterns[asdl_pattern_seq*]: + | args[asdl_pattern_seq*]=','.pattern+ { args } +keyword_patterns[asdl_seq*]: + | ','.keyword_pattern+ +keyword_pattern[KeyPatternPair*]: + | arg=NAME '=' value=pattern { _PyPegen_key_pattern_pair(p, arg, value) } return_stmt[stmt_ty]: - | 'return' a=[star_expressions] { _Py_Return(a, EXTRA) } + | 'return' a=[star_expressions] { _PyAST_Return(a, EXTRA) } raise_stmt[stmt_ty]: - | 'raise' a=expression b=['from' z=expression { z }] { _Py_Raise(a, b, EXTRA) } - | 'raise' { _Py_Raise(NULL, NULL, EXTRA) } + | 'raise' a=expression b=['from' z=expression { z }] { _PyAST_Raise(a, b, EXTRA) } + | 'raise' { _PyAST_Raise(NULL, NULL, EXTRA) } function_def[stmt_ty]: | d=decorators f=function_def_raw { _PyPegen_function_def_decorators(p, d, f) } | function_def_raw function_def_raw[stmt_ty]: - | 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block { - _Py_FunctionDef(n->v.Name.id, + | invalid_def_raw + | 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { + _PyAST_FunctionDef(n->v.Name.id, (params) ? params : CHECK(arguments_ty, _PyPegen_empty_arguments(p)), b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA) } - | ASYNC 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block { + | ASYNC 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { CHECK_VERSION( stmt_ty, 5, "Async functions are", - _Py_AsyncFunctionDef(n->v.Name.id, + _PyAST_AsyncFunctionDef(n->v.Name.id, (params) ? params : CHECK(arguments_ty, _PyPegen_empty_arguments(p)), b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA) ) } @@ -289,7 +472,7 @@ param_with_default[NameDefaultPair*]: param_maybe_default[NameDefaultPair*]: | a=param c=default? ',' tc=TYPE_COMMENT? { _PyPegen_name_default_pair(p, a, c, tc) } | a=param c=default? tc=TYPE_COMMENT? &')' { _PyPegen_name_default_pair(p, a, c, tc) } -param[arg_ty]: a=NAME b=annotation? { _Py_arg(a->v.Name.id, b, NULL, EXTRA) } +param[arg_ty]: a=NAME b=annotation? { _PyAST_arg(a->v.Name.id, b, NULL, EXTRA) } annotation[expr_ty]: ':' a=expression { a } default[expr_ty]: '=' a=expression { a } @@ -300,51 +483,57 @@ class_def[stmt_ty]: | a=decorators b=class_def_raw { _PyPegen_class_def_decorators(p, a, b) } | class_def_raw class_def_raw[stmt_ty]: - | 'class' a=NAME b=['(' z=[arguments] ')' { z }] ':' c=block { - _Py_ClassDef(a->v.Name.id, + | invalid_class_def_raw + | 'class' a=NAME b=['(' z=[arguments] ')' { z }] &&':' c=block { + _PyAST_ClassDef(a->v.Name.id, (b) ? ((expr_ty) b)->v.Call.args : NULL, (b) ? ((expr_ty) b)->v.Call.keywords : NULL, c, NULL, EXTRA) } block[asdl_stmt_seq*] (memo): | NEWLINE INDENT a=statements DEDENT { a } - | simple_stmt + | simple_stmts | invalid_block -expressions_list[asdl_expr_seq*]: a[asdl_expr_seq*]=','.star_expression+ [','] { a } star_expressions[expr_ty]: | a=star_expression b=(',' c=star_expression { c })+ [','] { - _Py_Tuple(CHECK(asdl_expr_seq*, _PyPegen_seq_insert_in_front(p, a, b)), Load, EXTRA) } - | a=star_expression ',' { _Py_Tuple(CHECK(asdl_expr_seq*, _PyPegen_singleton_seq(p, a)), Load, EXTRA) } + _PyAST_Tuple(CHECK(asdl_expr_seq*, _PyPegen_seq_insert_in_front(p, a, b)), Load, EXTRA) } + | a=star_expression ',' { _PyAST_Tuple(CHECK(asdl_expr_seq*, _PyPegen_singleton_seq(p, a)), Load, EXTRA) } | star_expression star_expression[expr_ty] (memo): - | '*' a=bitwise_or { _Py_Starred(a, Load, EXTRA) } + | '*' a=bitwise_or { _PyAST_Starred(a, Load, EXTRA) } | expression star_named_expressions[asdl_expr_seq*]: a[asdl_expr_seq*]=','.star_named_expression+ [','] { a } star_named_expression[expr_ty]: - | '*' a=bitwise_or { _Py_Starred(a, Load, EXTRA) } + | '*' a=bitwise_or { _PyAST_Starred(a, Load, EXTRA) } | named_expression + named_expression[expr_ty]: - | a=NAME ':=' ~ b=expression { _Py_NamedExpr(CHECK(expr_ty, _PyPegen_set_expr_context(p, a, Store)), b, EXTRA) } - | expression !':=' + | a=NAME ':=' ~ b=expression { _PyAST_NamedExpr(CHECK(expr_ty, _PyPegen_set_expr_context(p, a, Store)), b, EXTRA) } | invalid_named_expression + | expression !':=' + +direct_named_expression[expr_ty]: + | a=NAME ':=' ~ b=expression { _PyAST_NamedExpr(CHECK(expr_ty, _PyPegen_set_expr_context(p, a, Store)), b, EXTRA) } + | expression !':=' annotated_rhs[expr_ty]: yield_expr | star_expressions expressions[expr_ty]: | a=expression b=(',' c=expression { c })+ [','] { - _Py_Tuple(CHECK(asdl_expr_seq*, _PyPegen_seq_insert_in_front(p, a, b)), Load, EXTRA) } - | a=expression ',' { _Py_Tuple(CHECK(asdl_expr_seq*, _PyPegen_singleton_seq(p, a)), Load, EXTRA) } + _PyAST_Tuple(CHECK(asdl_expr_seq*, _PyPegen_seq_insert_in_front(p, a, b)), Load, EXTRA) } + | a=expression ',' { _PyAST_Tuple(CHECK(asdl_expr_seq*, _PyPegen_singleton_seq(p, a)), Load, EXTRA) } | expression expression[expr_ty] (memo): - | a=disjunction 'if' b=disjunction 'else' c=expression { _Py_IfExp(b, a, c, EXTRA) } + | invalid_expression + | a=disjunction 'if' b=disjunction 'else' c=expression { _PyAST_IfExp(b, a, c, EXTRA) } | disjunction | lambdef lambdef[expr_ty]: | 'lambda' a=[lambda_params] ':' b=expression { - _Py_Lambda((a) ? a : CHECK(arguments_ty, _PyPegen_empty_arguments(p)), b, EXTRA) } + _PyAST_Lambda((a) ? a : CHECK(arguments_ty, _PyPegen_empty_arguments(p)), b, EXTRA) } lambda_params[arguments_ty]: | invalid_lambda_parameters @@ -390,26 +579,26 @@ lambda_param_with_default[NameDefaultPair*]: lambda_param_maybe_default[NameDefaultPair*]: | a=lambda_param c=default? ',' { _PyPegen_name_default_pair(p, a, c, NULL) } | a=lambda_param c=default? &':' { _PyPegen_name_default_pair(p, a, c, NULL) } -lambda_param[arg_ty]: a=NAME { _Py_arg(a->v.Name.id, NULL, NULL, EXTRA) } +lambda_param[arg_ty]: a=NAME { _PyAST_arg(a->v.Name.id, NULL, NULL, EXTRA) } disjunction[expr_ty] (memo): - | a=conjunction b=('or' c=conjunction { c })+ { _Py_BoolOp( + | a=conjunction b=('or' c=conjunction { c })+ { _PyAST_BoolOp( Or, CHECK(asdl_expr_seq*, _PyPegen_seq_insert_in_front(p, a, b)), EXTRA) } | conjunction conjunction[expr_ty] (memo): - | a=inversion b=('and' c=inversion { c })+ { _Py_BoolOp( + | a=inversion b=('and' c=inversion { c })+ { _PyAST_BoolOp( And, CHECK(asdl_expr_seq*, _PyPegen_seq_insert_in_front(p, a, b)), EXTRA) } | inversion inversion[expr_ty] (memo): - | 'not' a=inversion { _Py_UnaryOp(Not, a, EXTRA) } + | 'not' a=inversion { _PyAST_UnaryOp(Not, a, EXTRA) } | comparison comparison[expr_ty]: | a=bitwise_or b=compare_op_bitwise_or_pair+ { - _Py_Compare( + _PyAST_Compare( a, CHECK(asdl_int_seq*, _PyPegen_get_cmpops(p, b)), CHECK(asdl_expr_seq*, _PyPegen_get_exprs(p, b)), @@ -428,7 +617,7 @@ compare_op_bitwise_or_pair[CmpopExprPair*]: | is_bitwise_or eq_bitwise_or[CmpopExprPair*]: '==' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Eq, a) } noteq_bitwise_or[CmpopExprPair*]: - | (tok='!=' {_PyPegen_check_barry_as_flufl(p) ? NULL : tok}) a=bitwise_or {_PyPegen_cmpop_expr_pair(p, NotEq, a) } + | (tok='!=' { _PyPegen_check_barry_as_flufl(p, tok) ? NULL : tok}) a=bitwise_or {_PyPegen_cmpop_expr_pair(p, NotEq, a) } lte_bitwise_or[CmpopExprPair*]: '<=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, LtE, a) } lt_bitwise_or[CmpopExprPair*]: '<' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Lt, a) } gte_bitwise_or[CmpopExprPair*]: '>=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, GtE, a) } @@ -439,97 +628,100 @@ isnot_bitwise_or[CmpopExprPair*]: 'is' 'not' a=bitwise_or { _PyPegen_cmpop_expr_ is_bitwise_or[CmpopExprPair*]: 'is' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Is, a) } bitwise_or[expr_ty]: - | a=bitwise_or '|' b=bitwise_xor { _Py_BinOp(a, BitOr, b, EXTRA) } + | a=bitwise_or '|' b=bitwise_xor { _PyAST_BinOp(a, BitOr, b, EXTRA) } | bitwise_xor bitwise_xor[expr_ty]: - | a=bitwise_xor '^' b=bitwise_and { _Py_BinOp(a, BitXor, b, EXTRA) } + | a=bitwise_xor '^' b=bitwise_and { _PyAST_BinOp(a, BitXor, b, EXTRA) } | bitwise_and bitwise_and[expr_ty]: - | a=bitwise_and '&' b=shift_expr { _Py_BinOp(a, BitAnd, b, EXTRA) } + | a=bitwise_and '&' b=shift_expr { _PyAST_BinOp(a, BitAnd, b, EXTRA) } | shift_expr shift_expr[expr_ty]: - | a=shift_expr '<<' b=sum { _Py_BinOp(a, LShift, b, EXTRA) } - | a=shift_expr '>>' b=sum { _Py_BinOp(a, RShift, b, EXTRA) } + | a=shift_expr '<<' b=sum { _PyAST_BinOp(a, LShift, b, EXTRA) } + | a=shift_expr '>>' b=sum { _PyAST_BinOp(a, RShift, b, EXTRA) } | sum sum[expr_ty]: - | a=sum '+' b=term { _Py_BinOp(a, Add, b, EXTRA) } - | a=sum '-' b=term { _Py_BinOp(a, Sub, b, EXTRA) } + | a=sum '+' b=term { _PyAST_BinOp(a, Add, b, EXTRA) } + | a=sum '-' b=term { _PyAST_BinOp(a, Sub, b, EXTRA) } | term term[expr_ty]: - | a=term '*' b=factor { _Py_BinOp(a, Mult, b, EXTRA) } - | a=term '/' b=factor { _Py_BinOp(a, Div, b, EXTRA) } - | a=term '//' b=factor { _Py_BinOp(a, FloorDiv, b, EXTRA) } - | a=term '%' b=factor { _Py_BinOp(a, Mod, b, EXTRA) } - | a=term '@' b=factor { CHECK_VERSION(expr_ty, 5, "The '@' operator is", _Py_BinOp(a, MatMult, b, EXTRA)) } + | a=term '*' b=factor { _PyAST_BinOp(a, Mult, b, EXTRA) } + | a=term '/' b=factor { _PyAST_BinOp(a, Div, b, EXTRA) } + | a=term '//' b=factor { _PyAST_BinOp(a, FloorDiv, b, EXTRA) } + | a=term '%' b=factor { _PyAST_BinOp(a, Mod, b, EXTRA) } + | a=term '@' b=factor { CHECK_VERSION(expr_ty, 5, "The '@' operator is", _PyAST_BinOp(a, MatMult, b, EXTRA)) } | factor factor[expr_ty] (memo): - | '+' a=factor { _Py_UnaryOp(UAdd, a, EXTRA) } - | '-' a=factor { _Py_UnaryOp(USub, a, EXTRA) } - | '~' a=factor { _Py_UnaryOp(Invert, a, EXTRA) } + | '+' a=factor { _PyAST_UnaryOp(UAdd, a, EXTRA) } + | '-' a=factor { _PyAST_UnaryOp(USub, a, EXTRA) } + | '~' a=factor { _PyAST_UnaryOp(Invert, a, EXTRA) } | power power[expr_ty]: - | a=await_primary '**' b=factor { _Py_BinOp(a, Pow, b, EXTRA) } + | a=await_primary '**' b=factor { _PyAST_BinOp(a, Pow, b, EXTRA) } | await_primary await_primary[expr_ty] (memo): - | AWAIT a=primary { CHECK_VERSION(expr_ty, 5, "Await expressions are", _Py_Await(a, EXTRA)) } + | AWAIT a=primary { CHECK_VERSION(expr_ty, 5, "Await expressions are", _PyAST_Await(a, EXTRA)) } | primary primary[expr_ty]: - | a=primary '.' b=NAME { _Py_Attribute(a, b->v.Name.id, Load, EXTRA) } - | a=primary b=genexp { _Py_Call(a, CHECK(asdl_expr_seq*, (asdl_expr_seq*)_PyPegen_singleton_seq(p, b)), NULL, EXTRA) } + | invalid_primary # must be before 'primay genexp' because of invalid_genexp + | a=primary '.' b=NAME { _PyAST_Attribute(a, b->v.Name.id, Load, EXTRA) } + | a=primary b=genexp { _PyAST_Call(a, CHECK(asdl_expr_seq*, (asdl_expr_seq*)_PyPegen_singleton_seq(p, b)), NULL, EXTRA) } | a=primary '(' b=[arguments] ')' { - _Py_Call(a, + _PyAST_Call(a, (b) ? ((expr_ty) b)->v.Call.args : NULL, (b) ? ((expr_ty) b)->v.Call.keywords : NULL, EXTRA) } - | a=primary '[' b=slices ']' { _Py_Subscript(a, b, Load, EXTRA) } + | a=primary '[' b=slices ']' { _PyAST_Subscript(a, b, Load, EXTRA) } | atom slices[expr_ty]: | a=slice !',' { a } - | a[asdl_expr_seq*]=','.slice+ [','] { _Py_Tuple(a, Load, EXTRA) } + | a[asdl_expr_seq*]=','.slice+ [','] { _PyAST_Tuple(a, Load, EXTRA) } slice[expr_ty]: - | a=[expression] ':' b=[expression] c=[':' d=[expression] { d }] { _Py_Slice(a, b, c, EXTRA) } - | a=expression { a } + | a=[expression] ':' b=[expression] c=[':' d=[expression] { d }] { _PyAST_Slice(a, b, c, EXTRA) } + | a=named_expression { a } atom[expr_ty]: | NAME - | 'True' { _Py_Constant(Py_True, NULL, EXTRA) } - | 'False' { _Py_Constant(Py_False, NULL, EXTRA) } - | 'None' { _Py_Constant(Py_None, NULL, EXTRA) } + | 'True' { _PyAST_Constant(Py_True, NULL, EXTRA) } + | 'False' { _PyAST_Constant(Py_False, NULL, EXTRA) } + | 'None' { _PyAST_Constant(Py_None, NULL, EXTRA) } | &STRING strings | NUMBER | &'(' (tuple | group | genexp) | &'[' (list | listcomp) | &'{' (dict | set | dictcomp | setcomp) - | '...' { _Py_Constant(Py_Ellipsis, NULL, EXTRA) } + | '...' { _PyAST_Constant(Py_Ellipsis, NULL, EXTRA) } strings[expr_ty] (memo): a=STRING+ { _PyPegen_concatenate_strings(p, a) } list[expr_ty]: - | '[' a=[star_named_expressions] ']' { _Py_List(a, Load, EXTRA) } + | '[' a=[star_named_expressions] ']' { _PyAST_List(a, Load, EXTRA) } listcomp[expr_ty]: - | '[' a=named_expression ~ b=for_if_clauses ']' { _Py_ListComp(a, b, EXTRA) } + | '[' a=named_expression b=for_if_clauses ']' { _PyAST_ListComp(a, b, EXTRA) } | invalid_comprehension tuple[expr_ty]: | '(' a=[y=star_named_expression ',' z=[star_named_expressions] { _PyPegen_seq_insert_in_front(p, y, z) } ] ')' { - _Py_Tuple(a, Load, EXTRA) } + _PyAST_Tuple(a, Load, EXTRA) } group[expr_ty]: | '(' a=(yield_expr | named_expression) ')' { a } | invalid_group genexp[expr_ty]: - | '(' a=expression ~ b=for_if_clauses ')' { _Py_GeneratorExp(a, b, EXTRA) } + | '(' a=direct_named_expression b=for_if_clauses ')' { _PyAST_GeneratorExp(a, b, EXTRA) } | invalid_comprehension -set[expr_ty]: '{' a=expressions_list '}' { _Py_Set(a, EXTRA) } +set[expr_ty]: '{' a=star_named_expressions '}' { _PyAST_Set(a, EXTRA) } setcomp[expr_ty]: - | '{' a=expression ~ b=for_if_clauses '}' { _Py_SetComp(a, b, EXTRA) } + | '{' a=named_expression b=for_if_clauses '}' { _PyAST_SetComp(a, b, EXTRA) } | invalid_comprehension dict[expr_ty]: | '{' a=[double_starred_kvpairs] '}' { - _Py_Dict( + _PyAST_Dict( CHECK(asdl_expr_seq*, _PyPegen_get_keys(p, a)), CHECK(asdl_expr_seq*, _PyPegen_get_values(p, a)), EXTRA) } + | '{' invalid_double_starred_kvpairs '}' + dictcomp[expr_ty]: - | '{' a=kvpair b=for_if_clauses '}' { _Py_DictComp(a->key, a->value, b, EXTRA) } + | '{' a=kvpair b=for_if_clauses '}' { _PyAST_DictComp(a->key, a->value, b, EXTRA) } | invalid_dict_comprehension double_starred_kvpairs[asdl_seq*]: a=','.double_starred_kvpair+ [','] { a } double_starred_kvpair[KeyValuePair*]: @@ -540,21 +732,21 @@ for_if_clauses[asdl_comprehension_seq*]: | a[asdl_comprehension_seq*]=for_if_clause+ { a } for_if_clause[comprehension_ty]: | ASYNC 'for' a=star_targets 'in' ~ b=disjunction c[asdl_expr_seq*]=('if' z=disjunction { z })* { - CHECK_VERSION(comprehension_ty, 6, "Async comprehensions are", _Py_comprehension(a, b, c, 1, p->arena)) } + CHECK_VERSION(comprehension_ty, 6, "Async comprehensions are", _PyAST_comprehension(a, b, c, 1, p->arena)) } | 'for' a=star_targets 'in' ~ b=disjunction c[asdl_expr_seq*]=('if' z=disjunction { z })* { - _Py_comprehension(a, b, c, 0, p->arena) } + _PyAST_comprehension(a, b, c, 0, p->arena) } | invalid_for_target yield_expr[expr_ty]: - | 'yield' 'from' a=expression { _Py_YieldFrom(a, EXTRA) } - | 'yield' a=[star_expressions] { _Py_Yield(a, EXTRA) } + | 'yield' 'from' a=expression { _PyAST_YieldFrom(a, EXTRA) } + | 'yield' a=[star_expressions] { _PyAST_Yield(a, EXTRA) } arguments[expr_ty] (memo): | a=args [','] &')' { a } | invalid_arguments args[expr_ty]: - | a[asdl_expr_seq*]=','.(starred_expression | named_expression !'=')+ b=[',' k=kwargs {k}] { _PyPegen_collect_call_seqs(p, a, b, EXTRA) } - | a=kwargs { _Py_Call(_PyPegen_dummy_name(p), + | a[asdl_expr_seq*]=','.(starred_expression | direct_named_expression !'=')+ b=[',' k=kwargs {k}] { _PyPegen_collect_call_seqs(p, a, b, EXTRA) } + | a=kwargs { _PyAST_Call(_PyPegen_dummy_name(p), CHECK_NULL_ALLOWED(asdl_expr_seq*, _PyPegen_seq_extract_starred_exprs(p, a)), CHECK_NULL_ALLOWED(asdl_keyword_seq*, _PyPegen_seq_delete_starred_exprs(p, a)), EXTRA) } @@ -563,67 +755,72 @@ kwargs[asdl_seq*]: | ','.kwarg_or_starred+ | ','.kwarg_or_double_starred+ starred_expression[expr_ty]: - | '*' a=expression { _Py_Starred(a, Load, EXTRA) } + | '*' a=expression { _PyAST_Starred(a, Load, EXTRA) } kwarg_or_starred[KeywordOrStarred*]: | a=NAME '=' b=expression { - _PyPegen_keyword_or_starred(p, CHECK(keyword_ty, _Py_keyword(a->v.Name.id, b, EXTRA)), 1) } + _PyPegen_keyword_or_starred(p, CHECK(keyword_ty, _PyAST_keyword(a->v.Name.id, b, EXTRA)), 1) } | a=starred_expression { _PyPegen_keyword_or_starred(p, a, 0) } | invalid_kwarg kwarg_or_double_starred[KeywordOrStarred*]: | a=NAME '=' b=expression { - _PyPegen_keyword_or_starred(p, CHECK(keyword_ty, _Py_keyword(a->v.Name.id, b, EXTRA)), 1) } - | '**' a=expression { _PyPegen_keyword_or_starred(p, CHECK(keyword_ty, _Py_keyword(NULL, a, EXTRA)), 1) } + _PyPegen_keyword_or_starred(p, CHECK(keyword_ty, _PyAST_keyword(a->v.Name.id, b, EXTRA)), 1) } + | '**' a=expression { _PyPegen_keyword_or_starred(p, CHECK(keyword_ty, _PyAST_keyword(NULL, a, EXTRA)), 1) } | invalid_kwarg # NOTE: star_targets may contain *bitwise_or, targets may not. star_targets[expr_ty]: | a=star_target !',' { a } | a=star_target b=(',' c=star_target { c })* [','] { - _Py_Tuple(CHECK(asdl_expr_seq*, _PyPegen_seq_insert_in_front(p, a, b)), Store, EXTRA) } -star_targets_seq[asdl_expr_seq*]: a[asdl_expr_seq*]=','.star_target+ [','] { a } + _PyAST_Tuple(CHECK(asdl_expr_seq*, _PyPegen_seq_insert_in_front(p, a, b)), Store, EXTRA) } +star_targets_list_seq[asdl_expr_seq*]: a[asdl_expr_seq*]=','.star_target+ [','] { a } +star_targets_tuple_seq[asdl_expr_seq*]: + | a=star_target b=(',' c=star_target { c })+ [','] { (asdl_expr_seq*) _PyPegen_seq_insert_in_front(p, a, b) } + | a=star_target ',' { (asdl_expr_seq*) _PyPegen_singleton_seq(p, a) } star_target[expr_ty] (memo): | '*' a=(!'*' star_target) { - _Py_Starred(CHECK(expr_ty, _PyPegen_set_expr_context(p, a, Store)), Store, EXTRA) } - | a=t_primary '.' b=NAME !t_lookahead { _Py_Attribute(a, b->v.Name.id, Store, EXTRA) } - | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Store, EXTRA) } + _PyAST_Starred(CHECK(expr_ty, _PyPegen_set_expr_context(p, a, Store)), Store, EXTRA) } + | target_with_star_atom +target_with_star_atom[expr_ty] (memo): + | a=t_primary '.' b=NAME !t_lookahead { _PyAST_Attribute(a, b->v.Name.id, Store, EXTRA) } + | a=t_primary '[' b=slices ']' !t_lookahead { _PyAST_Subscript(a, b, Store, EXTRA) } | star_atom star_atom[expr_ty]: | a=NAME { _PyPegen_set_expr_context(p, a, Store) } - | '(' a=star_target ')' { _PyPegen_set_expr_context(p, a, Store) } - | '(' a=[star_targets_seq] ')' { _Py_Tuple(a, Store, EXTRA) } - | '[' a=[star_targets_seq] ']' { _Py_List(a, Store, EXTRA) } + | '(' a=target_with_star_atom ')' { _PyPegen_set_expr_context(p, a, Store) } + | '(' a=[star_targets_tuple_seq] ')' { _PyAST_Tuple(a, Store, EXTRA) } + | '[' a=[star_targets_list_seq] ']' { _PyAST_List(a, Store, EXTRA) } single_target[expr_ty]: | single_subscript_attribute_target | a=NAME { _PyPegen_set_expr_context(p, a, Store) } | '(' a=single_target ')' { a } single_subscript_attribute_target[expr_ty]: - | a=t_primary '.' b=NAME !t_lookahead { _Py_Attribute(a, b->v.Name.id, Store, EXTRA) } - | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Store, EXTRA) } + | a=t_primary '.' b=NAME !t_lookahead { _PyAST_Attribute(a, b->v.Name.id, Store, EXTRA) } + | a=t_primary '[' b=slices ']' !t_lookahead { _PyAST_Subscript(a, b, Store, EXTRA) } del_targets[asdl_expr_seq*]: a[asdl_expr_seq*]=','.del_target+ [','] { a } del_target[expr_ty] (memo): - | a=t_primary '.' b=NAME !t_lookahead { _Py_Attribute(a, b->v.Name.id, Del, EXTRA) } - | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Del, EXTRA) } + | a=t_primary '.' b=NAME !t_lookahead { _PyAST_Attribute(a, b->v.Name.id, Del, EXTRA) } + | a=t_primary '[' b=slices ']' !t_lookahead { _PyAST_Subscript(a, b, Del, EXTRA) } | del_t_atom del_t_atom[expr_ty]: | a=NAME { _PyPegen_set_expr_context(p, a, Del) } | '(' a=del_target ')' { _PyPegen_set_expr_context(p, a, Del) } - | '(' a=[del_targets] ')' { _Py_Tuple(a, Del, EXTRA) } - | '[' a=[del_targets] ']' { _Py_List(a, Del, EXTRA) } + | '(' a=[del_targets] ')' { _PyAST_Tuple(a, Del, EXTRA) } + | '[' a=[del_targets] ']' { _PyAST_List(a, Del, EXTRA) } targets[asdl_expr_seq*]: a[asdl_expr_seq*]=','.target+ [','] { a } target[expr_ty] (memo): - | a=t_primary '.' b=NAME !t_lookahead { _Py_Attribute(a, b->v.Name.id, Store, EXTRA) } - | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Store, EXTRA) } + | a=t_primary '.' b=NAME !t_lookahead { _PyAST_Attribute(a, b->v.Name.id, Store, EXTRA) } + | a=t_primary '[' b=slices ']' !t_lookahead { _PyAST_Subscript(a, b, Store, EXTRA) } | t_atom t_primary[expr_ty]: - | a=t_primary '.' b=NAME &t_lookahead { _Py_Attribute(a, b->v.Name.id, Load, EXTRA) } - | a=t_primary '[' b=slices ']' &t_lookahead { _Py_Subscript(a, b, Load, EXTRA) } + | a=t_primary '.' b=NAME &t_lookahead { _PyAST_Attribute(a, b->v.Name.id, Load, EXTRA) } + | a=t_primary '[' b=slices ']' &t_lookahead { _PyAST_Subscript(a, b, Load, EXTRA) } | a=t_primary b=genexp &t_lookahead { - _Py_Call(a, CHECK(asdl_expr_seq*, (asdl_expr_seq*)_PyPegen_singleton_seq(p, b)), NULL, EXTRA) } + _PyAST_Call(a, CHECK(asdl_expr_seq*, (asdl_expr_seq*)_PyPegen_singleton_seq(p, b)), NULL, EXTRA) } | a=t_primary '(' b=[arguments] ')' &t_lookahead { - _Py_Call(a, + _PyAST_Call(a, (b) ? ((expr_ty) b)->v.Call.args : NULL, (b) ? ((expr_ty) b)->v.Call.keywords : NULL, EXTRA) } @@ -632,27 +829,40 @@ t_lookahead: '(' | '[' | '.' t_atom[expr_ty]: | a=NAME { _PyPegen_set_expr_context(p, a, Store) } | '(' a=target ')' { _PyPegen_set_expr_context(p, a, Store) } - | '(' b=[targets] ')' { _Py_Tuple(b, Store, EXTRA) } - | '[' b=[targets] ']' { _Py_List(b, Store, EXTRA) } + | '(' b=[targets] ')' { _PyAST_Tuple(b, Store, EXTRA) } + | '[' b=[targets] ']' { _PyAST_List(b, Store, EXTRA) } # From here on, there are rules for invalid syntax with specialised error messages invalid_arguments: - | args ',' '*' { RAISE_SYNTAX_ERROR("iterable argument unpacking follows keyword argument unpacking") } - | a=expression for_if_clauses ',' [args | expression for_if_clauses] { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "Generator expression must be parenthesized") } + | a=args ',' '*' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "iterable argument unpacking follows keyword argument unpacking") } + | a=expression b=for_if_clauses ',' [args | expression for_if_clauses] { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, PyPegen_last_item(b, comprehension_ty)->target, "Generator expression must be parenthesized") } | a=args for_if_clauses { _PyPegen_nonparen_genexp_in_call(p, a) } - | args ',' a=expression for_if_clauses { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "Generator expression must be parenthesized") } + | args ',' a=expression b=for_if_clauses { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, asdl_seq_GET(b, b->size-1)->target, "Generator expression must be parenthesized") } | a=args ',' args { _PyPegen_arguments_parsing_error(p, a) } invalid_kwarg: - | a=expression '=' { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION( - a, "expression cannot contain assignment, perhaps you meant \"==\"?") } + | a=expression b='=' { + RAISE_SYNTAX_ERROR_KNOWN_RANGE( + a, b, "expression cannot contain assignment, perhaps you meant \"==\"?") } + +invalid_expression: + # !(NAME STRING) is not matched so we don't show this error with some invalid string prefixes like: kf"dsfsdf" + # Soft keywords need to also be ignored because they can be parsed as NAME NAME + | !(NAME STRING | SOFT_KEYWORD) a=disjunction b=expression { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Perhaps you forgot a comma?") } + invalid_named_expression: | a=expression ':=' expression { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( a, "cannot use assignment expressions with %s", _PyPegen_get_expr_name(a)) } + | a=NAME '=' b=bitwise_or !('='|':='|',') { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Maybe you meant '==' or ':=' instead of '='?") } + | !(list|tuple|genexp|'True'|'None'|'False') a=bitwise_or b='=' bitwise_or !('='|':='|',') { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot assign to %s here. Maybe you meant '==' instead of '='?", + _PyPegen_get_expr_name(a)) } + invalid_assignment: | a=invalid_ann_assign_target ':' expression { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( @@ -668,7 +878,7 @@ invalid_assignment: RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) } | (star_targets '=')* a=yield_expr '=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "assignment to yield expression not possible") } | a=star_expressions augassign (yield_expr | star_expressions) { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( a, "'%s' is an illegal expression for augmented assignment", _PyPegen_get_expr_name(a) @@ -682,20 +892,33 @@ invalid_del_stmt: RAISE_SYNTAX_ERROR_INVALID_TARGET(DEL_TARGETS, a) } invalid_block: | NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block") } +invalid_primary: + | primary a='{' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "invalid syntax") } invalid_comprehension: | ('[' | '(' | '{') a=starred_expression for_if_clauses { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "iterable unpacking cannot be used in comprehension") } + | ('[' | '{') a=star_named_expression ',' b=star_named_expressions for_if_clauses { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, PyPegen_last_item(b, expr_ty), + "did you forget parentheses around the comprehension target?") } + | ('[' | '{') a=star_named_expression b=',' for_if_clauses { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "did you forget parentheses around the comprehension target?") } invalid_dict_comprehension: | '{' a='**' bitwise_or for_if_clauses '}' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "dict unpacking cannot be used in dict comprehension") } invalid_parameters: - | param_no_default* (slash_with_default | param_with_default+) param_no_default { - RAISE_SYNTAX_ERROR("non-default argument follows default argument") } + | param_no_default* invalid_parameters_helper a=param_no_default { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "non-default argument follows default argument") } +invalid_parameters_helper: # This is only there to avoid type errors + | a=slash_with_default { _PyPegen_singleton_seq(p, a) } + | param_with_default+ invalid_lambda_parameters: - | lambda_param_no_default* (lambda_slash_with_default | lambda_param_with_default+) lambda_param_no_default { - RAISE_SYNTAX_ERROR("non-default argument follows default argument") } + | lambda_param_no_default* invalid_lambda_parameters_helper a=lambda_param_no_default { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "non-default argument follows default argument") } +invalid_lambda_parameters_helper: + | a=lambda_slash_with_default { _PyPegen_singleton_seq(p, a) } + | lambda_param_with_default+ invalid_star_etc: - | '*' (')' | ',' (')' | '**')) { RAISE_SYNTAX_ERROR("named arguments must follow bare *") } + | a='*' (')' | ',' (')' | '**')) { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "named arguments must follow bare *") } | '*' ',' TYPE_COMMENT { RAISE_SYNTAX_ERROR("bare * has associated type comment") } invalid_lambda_star_etc: | '*' (':' | ',' (':' | '**')) { RAISE_SYNTAX_ERROR("named arguments must follow bare *") } @@ -703,7 +926,7 @@ invalid_double_type_comments: | TYPE_COMMENT NEWLINE TYPE_COMMENT NEWLINE INDENT { RAISE_SYNTAX_ERROR("Cannot have two type comments on def") } invalid_with_item: - | expression 'as' a=expression { + | expression 'as' a=expression &(',' | ')' | ':') { RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) } invalid_for_target: @@ -712,7 +935,76 @@ invalid_for_target: invalid_group: | '(' a=starred_expression ')' { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "can't use starred expression here") } + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use starred expression here") } + | '(' a='**' expression ')' { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use double starred expression here") } invalid_import_from_targets: | import_from_as_names ',' { RAISE_SYNTAX_ERROR("trailing comma not allowed without surrounding parentheses") } + +invalid_with_stmt: + | [ASYNC] 'with' ','.(expression ['as' star_target])+ &&':' + | [ASYNC] 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' &&':' +invalid_with_stmt_indent: + | [ASYNC] a='with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT { + RAISE_INDENTATION_ERROR("expected an indented block after 'with' statement on line %d", a->lineno) } + | [ASYNC] a='with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT { + RAISE_INDENTATION_ERROR("expected an indented block after 'with' statement on line %d", a->lineno) } + +invalid_try_stmt: + | a='try' ':' NEWLINE !INDENT { + RAISE_INDENTATION_ERROR("expected an indented block after 'try' statement on line %d", a->lineno) } +invalid_except_stmt: + | 'except' a=expression ',' expressions ['as' NAME ] ':' { + RAISE_SYNTAX_ERROR_STARTING_FROM(a, "exception group must be parenthesized") } + | a='except' expression ['as' NAME ] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } + | a='except' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } +invalid_finally_stmt: + | a='finally' ':' NEWLINE !INDENT { + RAISE_INDENTATION_ERROR("expected an indented block after 'finally' statement on line %d", a->lineno) } +invalid_except_stmt_indent: + | a='except' expression ['as' NAME ] ':' NEWLINE !INDENT { + RAISE_INDENTATION_ERROR("expected an indented block after 'except' statement on line %d", a->lineno) } + | a='except' ':' NEWLINE !INDENT { RAISE_SYNTAX_ERROR("expected an indented block after except statement on line %d", a->lineno) } +invalid_match_stmt: + | "match" subject_expr !':' { CHECK_VERSION(void*, 10, "Pattern matching is", RAISE_SYNTAX_ERROR("expected ':'") ) } + | a="match" subject=subject_expr ':' NEWLINE !INDENT { + RAISE_INDENTATION_ERROR("expected an indented block after 'match' statement on line %d", a->lineno) } +invalid_case_block: + | "case" patterns guard? !':' { RAISE_SYNTAX_ERROR("expected ':'") } + | a="case" patterns guard? ':' NEWLINE !INDENT { + RAISE_INDENTATION_ERROR("expected an indented block after 'case' statement on line %d", a->lineno) } +invalid_if_stmt: + | 'if' named_expression NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } + | a='if' a=named_expression ':' NEWLINE !INDENT { + RAISE_INDENTATION_ERROR("expected an indented block after 'if' statement on line %d", a->lineno) } +invalid_elif_stmt: + | 'elif' named_expression NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } + | a='elif' named_expression ':' NEWLINE !INDENT { + RAISE_INDENTATION_ERROR("expected an indented block after 'elif' statement on line %d", a->lineno) } +invalid_else_stmt: + | a='else' ':' NEWLINE !INDENT { + RAISE_INDENTATION_ERROR("expected an indented block after 'else' statement on line %d", a->lineno) } +invalid_while_stmt: + | 'while' named_expression NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } + | a='while' named_expression ':' NEWLINE !INDENT { + RAISE_INDENTATION_ERROR("expected an indented block after 'while' statement on line %d", a->lineno) } +invalid_for_stmt: + | [ASYNC] a='for' star_targets 'in' star_expressions ':' NEWLINE !INDENT { + RAISE_INDENTATION_ERROR("expected an indented block after 'for' statement on line %d", a->lineno) } +invalid_def_raw: + | [ASYNC] a='def' NAME '(' [params] ')' ['->' expression] ':' NEWLINE !INDENT { + RAISE_INDENTATION_ERROR("expected an indented block after function definition on line %d", a->lineno) } +invalid_class_def_raw: + | a='class' NAME ['('[arguments] ')'] ':' NEWLINE !INDENT { + RAISE_INDENTATION_ERROR("expected an indented block after class definition on line %d", a->lineno) } + +invalid_double_starred_kvpairs: + | ','.double_starred_kvpair+ ',' invalid_kvpair + | expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use a starred expression in a dictionary value") } + | expression a=':' &('}'|',') { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") } +invalid_kvpair: + | a=expression !(':') { + RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, a->lineno, a->end_col_offset - 1, a->end_lineno, -1, "':' expected after dictionary key") } + | expression ':' a='*' bitwise_or { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "cannot use a starred expression in a dictionary value") } + | expression a=':' {RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") } diff --git a/Include/Python-ast.h b/Include/Python-ast.h deleted file mode 100644 index e14bab566fb5a2..00000000000000 --- a/Include/Python-ast.h +++ /dev/null @@ -1,783 +0,0 @@ -/* File automatically generated by Parser/asdl_c.py. */ - -#ifndef Py_PYTHON_AST_H -#define Py_PYTHON_AST_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -#include "asdl.h" - -#undef Yield /* undefine macro conflicting with */ - -typedef struct _mod *mod_ty; - -typedef struct _stmt *stmt_ty; - -typedef struct _expr *expr_ty; - -typedef enum _expr_context { Load=1, Store=2, Del=3 } expr_context_ty; - -typedef enum _boolop { And=1, Or=2 } boolop_ty; - -typedef enum _operator { Add=1, Sub=2, Mult=3, MatMult=4, Div=5, Mod=6, Pow=7, - LShift=8, RShift=9, BitOr=10, BitXor=11, BitAnd=12, - FloorDiv=13 } operator_ty; - -typedef enum _unaryop { Invert=1, Not=2, UAdd=3, USub=4 } unaryop_ty; - -typedef enum _cmpop { Eq=1, NotEq=2, Lt=3, LtE=4, Gt=5, GtE=6, Is=7, IsNot=8, - In=9, NotIn=10 } cmpop_ty; - -typedef struct _comprehension *comprehension_ty; - -typedef struct _excepthandler *excepthandler_ty; - -typedef struct _arguments *arguments_ty; - -typedef struct _arg *arg_ty; - -typedef struct _keyword *keyword_ty; - -typedef struct _alias *alias_ty; - -typedef struct _withitem *withitem_ty; - -typedef struct _type_ignore *type_ignore_ty; - - -typedef struct { - _ASDL_SEQ_HEAD - mod_ty typed_elements[1]; -} asdl_mod_seq; - -asdl_mod_seq *_Py_asdl_mod_seq_new(Py_ssize_t size, PyArena *arena); - -typedef struct { - _ASDL_SEQ_HEAD - stmt_ty typed_elements[1]; -} asdl_stmt_seq; - -asdl_stmt_seq *_Py_asdl_stmt_seq_new(Py_ssize_t size, PyArena *arena); - -typedef struct { - _ASDL_SEQ_HEAD - expr_ty typed_elements[1]; -} asdl_expr_seq; - -asdl_expr_seq *_Py_asdl_expr_seq_new(Py_ssize_t size, PyArena *arena); - -typedef struct { - _ASDL_SEQ_HEAD - comprehension_ty typed_elements[1]; -} asdl_comprehension_seq; - -asdl_comprehension_seq *_Py_asdl_comprehension_seq_new(Py_ssize_t size, PyArena - *arena); - -typedef struct { - _ASDL_SEQ_HEAD - excepthandler_ty typed_elements[1]; -} asdl_excepthandler_seq; - -asdl_excepthandler_seq *_Py_asdl_excepthandler_seq_new(Py_ssize_t size, PyArena - *arena); - -typedef struct { - _ASDL_SEQ_HEAD - arguments_ty typed_elements[1]; -} asdl_arguments_seq; - -asdl_arguments_seq *_Py_asdl_arguments_seq_new(Py_ssize_t size, PyArena *arena); - -typedef struct { - _ASDL_SEQ_HEAD - arg_ty typed_elements[1]; -} asdl_arg_seq; - -asdl_arg_seq *_Py_asdl_arg_seq_new(Py_ssize_t size, PyArena *arena); - -typedef struct { - _ASDL_SEQ_HEAD - keyword_ty typed_elements[1]; -} asdl_keyword_seq; - -asdl_keyword_seq *_Py_asdl_keyword_seq_new(Py_ssize_t size, PyArena *arena); - -typedef struct { - _ASDL_SEQ_HEAD - alias_ty typed_elements[1]; -} asdl_alias_seq; - -asdl_alias_seq *_Py_asdl_alias_seq_new(Py_ssize_t size, PyArena *arena); - -typedef struct { - _ASDL_SEQ_HEAD - withitem_ty typed_elements[1]; -} asdl_withitem_seq; - -asdl_withitem_seq *_Py_asdl_withitem_seq_new(Py_ssize_t size, PyArena *arena); - -typedef struct { - _ASDL_SEQ_HEAD - type_ignore_ty typed_elements[1]; -} asdl_type_ignore_seq; - -asdl_type_ignore_seq *_Py_asdl_type_ignore_seq_new(Py_ssize_t size, PyArena - *arena); - - -enum _mod_kind {Module_kind=1, Interactive_kind=2, Expression_kind=3, - FunctionType_kind=4}; -struct _mod { - enum _mod_kind kind; - union { - struct { - asdl_stmt_seq *body; - asdl_type_ignore_seq *type_ignores; - } Module; - - struct { - asdl_stmt_seq *body; - } Interactive; - - struct { - expr_ty body; - } Expression; - - struct { - asdl_expr_seq *argtypes; - expr_ty returns; - } FunctionType; - - } v; -}; - -enum _stmt_kind {FunctionDef_kind=1, AsyncFunctionDef_kind=2, ClassDef_kind=3, - Return_kind=4, Delete_kind=5, Assign_kind=6, - AugAssign_kind=7, AnnAssign_kind=8, For_kind=9, - AsyncFor_kind=10, While_kind=11, If_kind=12, With_kind=13, - AsyncWith_kind=14, Raise_kind=15, Try_kind=16, - Assert_kind=17, Import_kind=18, ImportFrom_kind=19, - Global_kind=20, Nonlocal_kind=21, Expr_kind=22, Pass_kind=23, - Break_kind=24, Continue_kind=25}; -struct _stmt { - enum _stmt_kind kind; - union { - struct { - identifier name; - arguments_ty args; - asdl_stmt_seq *body; - asdl_expr_seq *decorator_list; - expr_ty returns; - string type_comment; - } FunctionDef; - - struct { - identifier name; - arguments_ty args; - asdl_stmt_seq *body; - asdl_expr_seq *decorator_list; - expr_ty returns; - string type_comment; - } AsyncFunctionDef; - - struct { - identifier name; - asdl_expr_seq *bases; - asdl_keyword_seq *keywords; - asdl_stmt_seq *body; - asdl_expr_seq *decorator_list; - } ClassDef; - - struct { - expr_ty value; - } Return; - - struct { - asdl_expr_seq *targets; - } Delete; - - struct { - asdl_expr_seq *targets; - expr_ty value; - string type_comment; - } Assign; - - struct { - expr_ty target; - operator_ty op; - expr_ty value; - } AugAssign; - - struct { - expr_ty target; - expr_ty annotation; - expr_ty value; - int simple; - } AnnAssign; - - struct { - expr_ty target; - expr_ty iter; - asdl_stmt_seq *body; - asdl_stmt_seq *orelse; - string type_comment; - } For; - - struct { - expr_ty target; - expr_ty iter; - asdl_stmt_seq *body; - asdl_stmt_seq *orelse; - string type_comment; - } AsyncFor; - - struct { - expr_ty test; - asdl_stmt_seq *body; - asdl_stmt_seq *orelse; - } While; - - struct { - expr_ty test; - asdl_stmt_seq *body; - asdl_stmt_seq *orelse; - } If; - - struct { - asdl_withitem_seq *items; - asdl_stmt_seq *body; - string type_comment; - } With; - - struct { - asdl_withitem_seq *items; - asdl_stmt_seq *body; - string type_comment; - } AsyncWith; - - struct { - expr_ty exc; - expr_ty cause; - } Raise; - - struct { - asdl_stmt_seq *body; - asdl_excepthandler_seq *handlers; - asdl_stmt_seq *orelse; - asdl_stmt_seq *finalbody; - } Try; - - struct { - expr_ty test; - expr_ty msg; - } Assert; - - struct { - asdl_alias_seq *names; - } Import; - - struct { - identifier module; - asdl_alias_seq *names; - int level; - } ImportFrom; - - struct { - asdl_identifier_seq *names; - } Global; - - struct { - asdl_identifier_seq *names; - } Nonlocal; - - struct { - expr_ty value; - } Expr; - - } v; - int lineno; - int col_offset; - int end_lineno; - int end_col_offset; -}; - -enum _expr_kind {BoolOp_kind=1, NamedExpr_kind=2, BinOp_kind=3, UnaryOp_kind=4, - Lambda_kind=5, IfExp_kind=6, Dict_kind=7, Set_kind=8, - ListComp_kind=9, SetComp_kind=10, DictComp_kind=11, - GeneratorExp_kind=12, Await_kind=13, Yield_kind=14, - YieldFrom_kind=15, Compare_kind=16, Call_kind=17, - FormattedValue_kind=18, JoinedStr_kind=19, Constant_kind=20, - Attribute_kind=21, Subscript_kind=22, Starred_kind=23, - Name_kind=24, List_kind=25, Tuple_kind=26, Slice_kind=27}; -struct _expr { - enum _expr_kind kind; - union { - struct { - boolop_ty op; - asdl_expr_seq *values; - } BoolOp; - - struct { - expr_ty target; - expr_ty value; - } NamedExpr; - - struct { - expr_ty left; - operator_ty op; - expr_ty right; - } BinOp; - - struct { - unaryop_ty op; - expr_ty operand; - } UnaryOp; - - struct { - arguments_ty args; - expr_ty body; - } Lambda; - - struct { - expr_ty test; - expr_ty body; - expr_ty orelse; - } IfExp; - - struct { - asdl_expr_seq *keys; - asdl_expr_seq *values; - } Dict; - - struct { - asdl_expr_seq *elts; - } Set; - - struct { - expr_ty elt; - asdl_comprehension_seq *generators; - } ListComp; - - struct { - expr_ty elt; - asdl_comprehension_seq *generators; - } SetComp; - - struct { - expr_ty key; - expr_ty value; - asdl_comprehension_seq *generators; - } DictComp; - - struct { - expr_ty elt; - asdl_comprehension_seq *generators; - } GeneratorExp; - - struct { - expr_ty value; - } Await; - - struct { - expr_ty value; - } Yield; - - struct { - expr_ty value; - } YieldFrom; - - struct { - expr_ty left; - asdl_int_seq *ops; - asdl_expr_seq *comparators; - } Compare; - - struct { - expr_ty func; - asdl_expr_seq *args; - asdl_keyword_seq *keywords; - } Call; - - struct { - expr_ty value; - int conversion; - expr_ty format_spec; - } FormattedValue; - - struct { - asdl_expr_seq *values; - } JoinedStr; - - struct { - constant value; - string kind; - } Constant; - - struct { - expr_ty value; - identifier attr; - expr_context_ty ctx; - } Attribute; - - struct { - expr_ty value; - expr_ty slice; - expr_context_ty ctx; - } Subscript; - - struct { - expr_ty value; - expr_context_ty ctx; - } Starred; - - struct { - identifier id; - expr_context_ty ctx; - } Name; - - struct { - asdl_expr_seq *elts; - expr_context_ty ctx; - } List; - - struct { - asdl_expr_seq *elts; - expr_context_ty ctx; - } Tuple; - - struct { - expr_ty lower; - expr_ty upper; - expr_ty step; - } Slice; - - } v; - int lineno; - int col_offset; - int end_lineno; - int end_col_offset; -}; - -struct _comprehension { - expr_ty target; - expr_ty iter; - asdl_expr_seq *ifs; - int is_async; -}; - -enum _excepthandler_kind {ExceptHandler_kind=1}; -struct _excepthandler { - enum _excepthandler_kind kind; - union { - struct { - expr_ty type; - identifier name; - asdl_stmt_seq *body; - } ExceptHandler; - - } v; - int lineno; - int col_offset; - int end_lineno; - int end_col_offset; -}; - -struct _arguments { - asdl_arg_seq *posonlyargs; - asdl_arg_seq *args; - arg_ty vararg; - asdl_arg_seq *kwonlyargs; - asdl_expr_seq *kw_defaults; - arg_ty kwarg; - asdl_expr_seq *defaults; -}; - -struct _arg { - identifier arg; - expr_ty annotation; - string type_comment; - int lineno; - int col_offset; - int end_lineno; - int end_col_offset; -}; - -struct _keyword { - identifier arg; - expr_ty value; - int lineno; - int col_offset; - int end_lineno; - int end_col_offset; -}; - -struct _alias { - identifier name; - identifier asname; -}; - -struct _withitem { - expr_ty context_expr; - expr_ty optional_vars; -}; - -enum _type_ignore_kind {TypeIgnore_kind=1}; -struct _type_ignore { - enum _type_ignore_kind kind; - union { - struct { - int lineno; - string tag; - } TypeIgnore; - - } v; -}; - - -// Note: these macros affect function definitions, not only call sites. -#define Module(a0, a1, a2) _Py_Module(a0, a1, a2) -mod_ty _Py_Module(asdl_stmt_seq * body, asdl_type_ignore_seq * type_ignores, - PyArena *arena); -#define Interactive(a0, a1) _Py_Interactive(a0, a1) -mod_ty _Py_Interactive(asdl_stmt_seq * body, PyArena *arena); -#define Expression(a0, a1) _Py_Expression(a0, a1) -mod_ty _Py_Expression(expr_ty body, PyArena *arena); -#define FunctionType(a0, a1, a2) _Py_FunctionType(a0, a1, a2) -mod_ty _Py_FunctionType(asdl_expr_seq * argtypes, expr_ty returns, PyArena - *arena); -#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) -stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_stmt_seq * - body, asdl_expr_seq * decorator_list, expr_ty returns, - string type_comment, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); -#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) -stmt_ty _Py_AsyncFunctionDef(identifier name, arguments_ty args, asdl_stmt_seq - * body, asdl_expr_seq * decorator_list, expr_ty - returns, string type_comment, int lineno, int - col_offset, int end_lineno, int end_col_offset, - PyArena *arena); -#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) -stmt_ty _Py_ClassDef(identifier name, asdl_expr_seq * bases, asdl_keyword_seq * - keywords, asdl_stmt_seq * body, asdl_expr_seq * - decorator_list, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); -#define Return(a0, a1, a2, a3, a4, a5) _Py_Return(a0, a1, a2, a3, a4, a5) -stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, int end_lineno, - int end_col_offset, PyArena *arena); -#define Delete(a0, a1, a2, a3, a4, a5) _Py_Delete(a0, a1, a2, a3, a4, a5) -stmt_ty _Py_Delete(asdl_expr_seq * targets, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); -#define Assign(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Assign(a0, a1, a2, a3, a4, a5, a6, a7) -stmt_ty _Py_Assign(asdl_expr_seq * targets, expr_ty value, string type_comment, - int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); -#define AugAssign(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AugAssign(a0, a1, a2, a3, a4, a5, a6, a7) -stmt_ty _Py_AugAssign(expr_ty target, operator_ty op, expr_ty value, int - lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); -#define AnnAssign(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_AnnAssign(a0, a1, a2, a3, a4, a5, a6, a7, a8) -stmt_ty _Py_AnnAssign(expr_ty target, expr_ty annotation, expr_ty value, int - simple, int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); -#define For(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_For(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) -stmt_ty _Py_For(expr_ty target, expr_ty iter, asdl_stmt_seq * body, - asdl_stmt_seq * orelse, string type_comment, int lineno, int - col_offset, int end_lineno, int end_col_offset, PyArena *arena); -#define AsyncFor(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_AsyncFor(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) -stmt_ty _Py_AsyncFor(expr_ty target, expr_ty iter, asdl_stmt_seq * body, - asdl_stmt_seq * orelse, string type_comment, int lineno, - int col_offset, int end_lineno, int end_col_offset, - PyArena *arena); -#define While(a0, a1, a2, a3, a4, a5, a6, a7) _Py_While(a0, a1, a2, a3, a4, a5, a6, a7) -stmt_ty _Py_While(expr_ty test, asdl_stmt_seq * body, asdl_stmt_seq * orelse, - int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); -#define If(a0, a1, a2, a3, a4, a5, a6, a7) _Py_If(a0, a1, a2, a3, a4, a5, a6, a7) -stmt_ty _Py_If(expr_ty test, asdl_stmt_seq * body, asdl_stmt_seq * orelse, int - lineno, int col_offset, int end_lineno, int end_col_offset, - PyArena *arena); -#define With(a0, a1, a2, a3, a4, a5, a6, a7) _Py_With(a0, a1, a2, a3, a4, a5, a6, a7) -stmt_ty _Py_With(asdl_withitem_seq * items, asdl_stmt_seq * body, string - type_comment, int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); -#define AsyncWith(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AsyncWith(a0, a1, a2, a3, a4, a5, a6, a7) -stmt_ty _Py_AsyncWith(asdl_withitem_seq * items, asdl_stmt_seq * body, string - type_comment, int lineno, int col_offset, int end_lineno, - int end_col_offset, PyArena *arena); -#define Raise(a0, a1, a2, a3, a4, a5, a6) _Py_Raise(a0, a1, a2, a3, a4, a5, a6) -stmt_ty _Py_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); -#define Try(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_Try(a0, a1, a2, a3, a4, a5, a6, a7, a8) -stmt_ty _Py_Try(asdl_stmt_seq * body, asdl_excepthandler_seq * handlers, - asdl_stmt_seq * orelse, asdl_stmt_seq * finalbody, int lineno, - int col_offset, int end_lineno, int end_col_offset, PyArena - *arena); -#define Assert(a0, a1, a2, a3, a4, a5, a6) _Py_Assert(a0, a1, a2, a3, a4, a5, a6) -stmt_ty _Py_Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); -#define Import(a0, a1, a2, a3, a4, a5) _Py_Import(a0, a1, a2, a3, a4, a5) -stmt_ty _Py_Import(asdl_alias_seq * names, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); -#define ImportFrom(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ImportFrom(a0, a1, a2, a3, a4, a5, a6, a7) -stmt_ty _Py_ImportFrom(identifier module, asdl_alias_seq * names, int level, - int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); -#define Global(a0, a1, a2, a3, a4, a5) _Py_Global(a0, a1, a2, a3, a4, a5) -stmt_ty _Py_Global(asdl_identifier_seq * names, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); -#define Nonlocal(a0, a1, a2, a3, a4, a5) _Py_Nonlocal(a0, a1, a2, a3, a4, a5) -stmt_ty _Py_Nonlocal(asdl_identifier_seq * names, int lineno, int col_offset, - int end_lineno, int end_col_offset, PyArena *arena); -#define Expr(a0, a1, a2, a3, a4, a5) _Py_Expr(a0, a1, a2, a3, a4, a5) -stmt_ty _Py_Expr(expr_ty value, int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); -#define Pass(a0, a1, a2, a3, a4) _Py_Pass(a0, a1, a2, a3, a4) -stmt_ty _Py_Pass(int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); -#define Break(a0, a1, a2, a3, a4) _Py_Break(a0, a1, a2, a3, a4) -stmt_ty _Py_Break(int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); -#define Continue(a0, a1, a2, a3, a4) _Py_Continue(a0, a1, a2, a3, a4) -stmt_ty _Py_Continue(int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); -#define BoolOp(a0, a1, a2, a3, a4, a5, a6) _Py_BoolOp(a0, a1, a2, a3, a4, a5, a6) -expr_ty _Py_BoolOp(boolop_ty op, asdl_expr_seq * values, int lineno, int - col_offset, int end_lineno, int end_col_offset, PyArena - *arena); -#define NamedExpr(a0, a1, a2, a3, a4, a5, a6) _Py_NamedExpr(a0, a1, a2, a3, a4, a5, a6) -expr_ty _Py_NamedExpr(expr_ty target, expr_ty value, int lineno, int - col_offset, int end_lineno, int end_col_offset, PyArena - *arena); -#define BinOp(a0, a1, a2, a3, a4, a5, a6, a7) _Py_BinOp(a0, a1, a2, a3, a4, a5, a6, a7) -expr_ty _Py_BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, int - col_offset, int end_lineno, int end_col_offset, PyArena - *arena); -#define UnaryOp(a0, a1, a2, a3, a4, a5, a6) _Py_UnaryOp(a0, a1, a2, a3, a4, a5, a6) -expr_ty _Py_UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int col_offset, - int end_lineno, int end_col_offset, PyArena *arena); -#define Lambda(a0, a1, a2, a3, a4, a5, a6) _Py_Lambda(a0, a1, a2, a3, a4, a5, a6) -expr_ty _Py_Lambda(arguments_ty args, expr_ty body, int lineno, int col_offset, - int end_lineno, int end_col_offset, PyArena *arena); -#define IfExp(a0, a1, a2, a3, a4, a5, a6, a7) _Py_IfExp(a0, a1, a2, a3, a4, a5, a6, a7) -expr_ty _Py_IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, int - col_offset, int end_lineno, int end_col_offset, PyArena - *arena); -#define Dict(a0, a1, a2, a3, a4, a5, a6) _Py_Dict(a0, a1, a2, a3, a4, a5, a6) -expr_ty _Py_Dict(asdl_expr_seq * keys, asdl_expr_seq * values, int lineno, int - col_offset, int end_lineno, int end_col_offset, PyArena - *arena); -#define Set(a0, a1, a2, a3, a4, a5) _Py_Set(a0, a1, a2, a3, a4, a5) -expr_ty _Py_Set(asdl_expr_seq * elts, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); -#define ListComp(a0, a1, a2, a3, a4, a5, a6) _Py_ListComp(a0, a1, a2, a3, a4, a5, a6) -expr_ty _Py_ListComp(expr_ty elt, asdl_comprehension_seq * generators, int - lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); -#define SetComp(a0, a1, a2, a3, a4, a5, a6) _Py_SetComp(a0, a1, a2, a3, a4, a5, a6) -expr_ty _Py_SetComp(expr_ty elt, asdl_comprehension_seq * generators, int - lineno, int col_offset, int end_lineno, int end_col_offset, - PyArena *arena); -#define DictComp(a0, a1, a2, a3, a4, a5, a6, a7) _Py_DictComp(a0, a1, a2, a3, a4, a5, a6, a7) -expr_ty _Py_DictComp(expr_ty key, expr_ty value, asdl_comprehension_seq * - generators, int lineno, int col_offset, int end_lineno, - int end_col_offset, PyArena *arena); -#define GeneratorExp(a0, a1, a2, a3, a4, a5, a6) _Py_GeneratorExp(a0, a1, a2, a3, a4, a5, a6) -expr_ty _Py_GeneratorExp(expr_ty elt, asdl_comprehension_seq * generators, int - lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); -#define Await(a0, a1, a2, a3, a4, a5) _Py_Await(a0, a1, a2, a3, a4, a5) -expr_ty _Py_Await(expr_ty value, int lineno, int col_offset, int end_lineno, - int end_col_offset, PyArena *arena); -#define Yield(a0, a1, a2, a3, a4, a5) _Py_Yield(a0, a1, a2, a3, a4, a5) -expr_ty _Py_Yield(expr_ty value, int lineno, int col_offset, int end_lineno, - int end_col_offset, PyArena *arena); -#define YieldFrom(a0, a1, a2, a3, a4, a5) _Py_YieldFrom(a0, a1, a2, a3, a4, a5) -expr_ty _Py_YieldFrom(expr_ty value, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); -#define Compare(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Compare(a0, a1, a2, a3, a4, a5, a6, a7) -expr_ty _Py_Compare(expr_ty left, asdl_int_seq * ops, asdl_expr_seq * - comparators, int lineno, int col_offset, int end_lineno, - int end_col_offset, PyArena *arena); -#define Call(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Call(a0, a1, a2, a3, a4, a5, a6, a7) -expr_ty _Py_Call(expr_ty func, asdl_expr_seq * args, asdl_keyword_seq * - keywords, int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); -#define FormattedValue(a0, a1, a2, a3, a4, a5, a6, a7) _Py_FormattedValue(a0, a1, a2, a3, a4, a5, a6, a7) -expr_ty _Py_FormattedValue(expr_ty value, int conversion, expr_ty format_spec, - int lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); -#define JoinedStr(a0, a1, a2, a3, a4, a5) _Py_JoinedStr(a0, a1, a2, a3, a4, a5) -expr_ty _Py_JoinedStr(asdl_expr_seq * values, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); -#define Constant(a0, a1, a2, a3, a4, a5, a6) _Py_Constant(a0, a1, a2, a3, a4, a5, a6) -expr_ty _Py_Constant(constant value, string kind, int lineno, int col_offset, - int end_lineno, int end_col_offset, PyArena *arena); -#define Attribute(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Attribute(a0, a1, a2, a3, a4, a5, a6, a7) -expr_ty _Py_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int - lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); -#define Subscript(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Subscript(a0, a1, a2, a3, a4, a5, a6, a7) -expr_ty _Py_Subscript(expr_ty value, expr_ty slice, expr_context_ty ctx, int - lineno, int col_offset, int end_lineno, int - end_col_offset, PyArena *arena); -#define Starred(a0, a1, a2, a3, a4, a5, a6) _Py_Starred(a0, a1, a2, a3, a4, a5, a6) -expr_ty _Py_Starred(expr_ty value, expr_context_ty ctx, int lineno, int - col_offset, int end_lineno, int end_col_offset, PyArena - *arena); -#define Name(a0, a1, a2, a3, a4, a5, a6) _Py_Name(a0, a1, a2, a3, a4, a5, a6) -expr_ty _Py_Name(identifier id, expr_context_ty ctx, int lineno, int - col_offset, int end_lineno, int end_col_offset, PyArena - *arena); -#define List(a0, a1, a2, a3, a4, a5, a6) _Py_List(a0, a1, a2, a3, a4, a5, a6) -expr_ty _Py_List(asdl_expr_seq * elts, expr_context_ty ctx, int lineno, int - col_offset, int end_lineno, int end_col_offset, PyArena - *arena); -#define Tuple(a0, a1, a2, a3, a4, a5, a6) _Py_Tuple(a0, a1, a2, a3, a4, a5, a6) -expr_ty _Py_Tuple(asdl_expr_seq * elts, expr_context_ty ctx, int lineno, int - col_offset, int end_lineno, int end_col_offset, PyArena - *arena); -#define Slice(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Slice(a0, a1, a2, a3, a4, a5, a6, a7) -expr_ty _Py_Slice(expr_ty lower, expr_ty upper, expr_ty step, int lineno, int - col_offset, int end_lineno, int end_col_offset, PyArena - *arena); -#define comprehension(a0, a1, a2, a3, a4) _Py_comprehension(a0, a1, a2, a3, a4) -comprehension_ty _Py_comprehension(expr_ty target, expr_ty iter, asdl_expr_seq - * ifs, int is_async, PyArena *arena); -#define ExceptHandler(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ExceptHandler(a0, a1, a2, a3, a4, a5, a6, a7) -excepthandler_ty _Py_ExceptHandler(expr_ty type, identifier name, asdl_stmt_seq - * body, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena - *arena); -#define arguments(a0, a1, a2, a3, a4, a5, a6, a7) _Py_arguments(a0, a1, a2, a3, a4, a5, a6, a7) -arguments_ty _Py_arguments(asdl_arg_seq * posonlyargs, asdl_arg_seq * args, - arg_ty vararg, asdl_arg_seq * kwonlyargs, - asdl_expr_seq * kw_defaults, arg_ty kwarg, - asdl_expr_seq * defaults, PyArena *arena); -#define arg(a0, a1, a2, a3, a4, a5, a6, a7) _Py_arg(a0, a1, a2, a3, a4, a5, a6, a7) -arg_ty _Py_arg(identifier arg, expr_ty annotation, string type_comment, int - lineno, int col_offset, int end_lineno, int end_col_offset, - PyArena *arena); -#define keyword(a0, a1, a2, a3, a4, a5, a6) _Py_keyword(a0, a1, a2, a3, a4, a5, a6) -keyword_ty _Py_keyword(identifier arg, expr_ty value, int lineno, int - col_offset, int end_lineno, int end_col_offset, PyArena - *arena); -#define alias(a0, a1, a2) _Py_alias(a0, a1, a2) -alias_ty _Py_alias(identifier name, identifier asname, PyArena *arena); -#define withitem(a0, a1, a2) _Py_withitem(a0, a1, a2) -withitem_ty _Py_withitem(expr_ty context_expr, expr_ty optional_vars, PyArena - *arena); -#define TypeIgnore(a0, a1, a2) _Py_TypeIgnore(a0, a1, a2) -type_ignore_ty _Py_TypeIgnore(int lineno, string tag, PyArena *arena); - -PyObject* PyAST_mod2obj(mod_ty t); -mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode); -int PyAST_Check(PyObject* obj); -#endif /* !Py_LIMITED_API */ - -#ifdef __cplusplus -} -#endif -#endif /* !Py_PYTHON_AST_H */ diff --git a/Include/Python.h b/Include/Python.h index 57f71d41d8d477..4d0335d3c52c36 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -72,17 +72,7 @@ # endif #endif -/* Debug-mode build with pymalloc implies PYMALLOC_DEBUG. - * PYMALLOC_DEBUG is in error if pymalloc is not in use. - */ -#if defined(Py_DEBUG) && defined(WITH_PYMALLOC) && !defined(PYMALLOC_DEBUG) -#define PYMALLOC_DEBUG -#endif -#if defined(PYMALLOC_DEBUG) && !defined(WITH_PYMALLOC) -#error "PYMALLOC_DEBUG requires WITH_PYMALLOC" -#endif #include "pymath.h" -#include "pytime.h" #include "pymem.h" #include "object.h" @@ -90,7 +80,7 @@ #include "typeslots.h" #include "pyhash.h" -#include "pydebug.h" +#include "cpython/pydebug.h" #include "bytearrayobject.h" #include "bytesobject.h" @@ -105,7 +95,7 @@ #include "tupleobject.h" #include "listobject.h" #include "dictobject.h" -#include "odictobject.h" +#include "cpython/odictobject.h" #include "enumobject.h" #include "setobject.h" #include "methodobject.h" @@ -127,7 +117,8 @@ #include "weakrefobject.h" #include "structseq.h" #include "namespaceobject.h" -#include "picklebufobject.h" +#include "cpython/picklebufobject.h" +#include "cpython/pytime.h" #include "codecs.h" #include "pyerrors.h" @@ -137,11 +128,9 @@ #include "pystate.h" #include "context.h" -#include "pyarena.h" #include "modsupport.h" #include "compile.h" #include "pythonrun.h" -#include "parser_interface.h" #include "pylifecycle.h" #include "ceval.h" #include "sysmodule.h" @@ -154,11 +143,11 @@ #include "eval.h" -#include "pyctype.h" +#include "cpython/pyctype.h" #include "pystrtod.h" #include "pystrcmp.h" #include "fileutils.h" -#include "pyfpe.h" +#include "cpython/pyfpe.h" #include "tracemalloc.h" #endif /* !Py_PYTHON_H */ diff --git a/Include/README.rst b/Include/README.rst new file mode 100644 index 00000000000000..d2467ca22fa038 --- /dev/null +++ b/Include/README.rst @@ -0,0 +1,68 @@ +The Python C API +================ + +The C API is divided into three sections: + +1. ``Include/`` +2. ``Include/cpython/`` +3. ``Include/internal/`` + + +Include: Limited API +==================== + +``Include/``, excluding the ``cpython`` and ``internal`` subdirectories, +contains the public Limited API (Application Programming Interface). +The Limited API is a subset of the C API, designed to guarantee ABI +stability across Python 3 versions, and is defined in :pep:`384`. + +Guidelines for expanding the Limited API: + +- Functions *must not* steal references +- Functions *must not* return borrowed references +- Functions returning references *must* return a strong reference +- Macros should not expose implementation details +- Please start a public discussion before expanding the API +- Functions or macros with a ``_Py`` prefix do not belong in ``Include/``. + +It is possible to add a function or macro to the Limited API from a +given Python version. For example, to add a function to the Limited API +from Python 3.10 and onwards, wrap it with +``#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000``. + + +Include/cpython: CPython implementation details +=============================================== + +``Include/cpython/`` contains the public API that is excluded from the +Limited API and the Stable ABI. + +Guidelines for expanding the public API: + +- Functions *must not* steal references +- Functions *must not* return borrowed references +- Functions returning references *must* return a strong reference + + +Include/internal: The internal API +================================== + + +With PyAPI_FUNC or PyAPI_DATA +----------------------------- + +Functions or structures in ``Include/internal/`` defined with +``PyAPI_FUNC`` or ``PyAPI_DATA`` are internal functions which are +exposed only for specific use cases like debuggers and profilers. + + +With the extern keyword +----------------------- + +Functions in ``Include/internal/`` defined with the ``extern`` keyword +*must not and can not* be used outside the CPython code base. Only +built-in stdlib extensions (built with the ``Py_BUILD_CORE_BUILTIN`` +macro defined) can use such functions. + +When in doubt, new internal C functions should be defined in +``Include/internal`` using the ``extern`` keyword. diff --git a/Include/abstract.h b/Include/abstract.h index 28e576b92935f9..1af1487deec916 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -324,11 +324,21 @@ PyAPI_FUNC(PyObject *) PyObject_Format(PyObject *obj, returns itself. */ PyAPI_FUNC(PyObject *) PyObject_GetIter(PyObject *); -/* Returns 1 if the object 'obj' provides iterator protocols, and 0 otherwise. +/* Takes an AsyncIterable object and returns an AsyncIterator for it. + This is typically a new iterator but if the argument is an AsyncIterator, + this returns itself. */ +PyAPI_FUNC(PyObject *) PyObject_GetAiter(PyObject *); + +/* Returns non-zero if the object 'obj' provides iterator protocols, and 0 otherwise. This function always succeeds. */ PyAPI_FUNC(int) PyIter_Check(PyObject *); +/* Returns non-zero if the object 'obj' provides AsyncIterator protocols, and 0 otherwise. + + This function always succeeds. */ +PyAPI_FUNC(int) PyAiter_Check(PyObject *); + /* Takes an iterator object and calls its tp_iternext slot, returning the next value. @@ -339,11 +349,6 @@ PyAPI_FUNC(int) PyIter_Check(PyObject *); PyAPI_FUNC(PyObject *) PyIter_Next(PyObject *); #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 -typedef enum { - PYGEN_RETURN = 0, - PYGEN_ERROR = -1, - PYGEN_NEXT = 1, -} PySendResult; /* Takes generator, coroutine or iterator object and sends the value into it. Returns: diff --git a/Include/ast.h b/Include/ast.h deleted file mode 100644 index 434ee18dd91b2a..00000000000000 --- a/Include/ast.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef Py_LIMITED_API -#ifndef Py_AST_H -#define Py_AST_H -#ifdef __cplusplus -extern "C" { -#endif - -#include "Python-ast.h" /* mod_ty */ - -PyAPI_FUNC(int) PyAST_Validate(mod_ty); - -/* _PyAST_ExprAsUnicode is defined in ast_unparse.c */ -PyAPI_FUNC(PyObject *) _PyAST_ExprAsUnicode(expr_ty); - -/* Return the borrowed reference to the first literal string in the - sequence of statements or NULL if it doesn't start from a literal string. - Doesn't set exception. */ -PyAPI_FUNC(PyObject *) _PyAST_GetDocString(asdl_stmt_seq *); - -#ifdef __cplusplus -} -#endif -#endif /* !Py_AST_H */ -#endif /* !Py_LIMITED_API */ diff --git a/Include/boolobject.h b/Include/boolobject.h index bb8044a2b02cf6..cda6f89a99e9a2 100644 --- a/Include/boolobject.h +++ b/Include/boolobject.h @@ -15,15 +15,24 @@ PyAPI_DATA(PyTypeObject) PyBool_Type; Don't forget to apply Py_INCREF() when returning either!!! */ /* Don't use these directly */ -PyAPI_DATA(struct _longobject) _Py_FalseStruct, _Py_TrueStruct; +PyAPI_DATA(struct _longobject) _Py_FalseStruct; +PyAPI_DATA(struct _longobject) _Py_TrueStruct; /* Use these macros */ #define Py_False ((PyObject *) &_Py_FalseStruct) #define Py_True ((PyObject *) &_Py_TrueStruct) +// Test if an object is the True singleton, the same as "x is True" in Python. +PyAPI_FUNC(int) Py_IsTrue(PyObject *x); +#define Py_IsTrue(x) Py_Is((x), Py_True) + +// Test if an object is the False singleton, the same as "x is False" in Python. +PyAPI_FUNC(int) Py_IsFalse(PyObject *x); +#define Py_IsFalse(x) Py_Is((x), Py_False) + /* Macros for returning Py_True or Py_False, respectively */ -#define Py_RETURN_TRUE return Py_INCREF(Py_True), Py_True -#define Py_RETURN_FALSE return Py_INCREF(Py_False), Py_False +#define Py_RETURN_TRUE return Py_NewRef(Py_True) +#define Py_RETURN_FALSE return Py_NewRef(Py_False) /* Function to return a bool from a C long */ PyAPI_FUNC(PyObject *) PyBool_FromLong(long); diff --git a/Include/bytesobject.h b/Include/bytesobject.h index 5062d8d123ad3e..39c241a2dcf5f7 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -1,5 +1,5 @@ -/* Bytes (String) object interface */ +/* Bytes object interface */ #ifndef Py_BYTESOBJECT_H #define Py_BYTESOBJECT_H @@ -10,23 +10,20 @@ extern "C" { #include /* -Type PyBytesObject represents a character string. An extra zero byte is +Type PyBytesObject represents a byte string. An extra zero byte is reserved at the end to ensure it is zero-terminated, but a size is present so strings with null bytes in them can be represented. This is an immutable object type. -There are functions to create new string objects, to test -an object for string-ness, and to get the -string value. The latter function returns a null pointer +There are functions to create new bytes objects, to test +an object for bytes-ness, and to get the +byte string value. The latter function returns a null pointer if the object is not of the proper type. There is a variant that takes an explicit size as well as a variant that assumes a zero-terminated string. Note that none of the -functions should be applied to nil objects. +functions should be applied to NULL pointer. */ -/* Caching the hash (ob_shash) saves recalculation of a string's hash value. - This significantly speeds up dict lookups. */ - PyAPI_DATA(PyTypeObject) PyBytes_Type; PyAPI_DATA(PyTypeObject) PyBytesIter_Type; @@ -50,26 +47,16 @@ PyAPI_FUNC(PyObject *) PyBytes_DecodeEscape(const char *, Py_ssize_t, const char *, Py_ssize_t, const char *); -/* Provides access to the internal data buffer and size of a string - object or the default encoded version of a Unicode object. Passing - NULL as *len parameter will force the string buffer to be - 0-terminated (passing a string with embedded NULL characters will +/* Provides access to the internal data buffer and size of a bytes object. + Passing NULL as len parameter will force the string buffer to be + 0-terminated (passing a string with embedded NUL characters will cause an exception). */ PyAPI_FUNC(int) PyBytes_AsStringAndSize( - PyObject *obj, /* string or Unicode object */ + PyObject *obj, /* bytes object */ char **s, /* pointer to buffer variable */ - Py_ssize_t *len /* pointer to length variable or NULL - (only possible for 0-terminated - strings) */ + Py_ssize_t *len /* pointer to length variable or NULL */ ); -/* Flags used by string formatting */ -#define F_LJUST (1<<0) -#define F_SIGN (1<<1) -#define F_BLANK (1<<2) -#define F_ALT (1<<3) -#define F_ZERO (1<<4) - #ifndef Py_LIMITED_API # define Py_CPYTHON_BYTESOBJECT_H # include "cpython/bytesobject.h" diff --git a/Include/cellobject.h b/Include/cellobject.h index f12aa90a42a8fe..81bc784d36f3e0 100644 --- a/Include/cellobject.h +++ b/Include/cellobject.h @@ -20,7 +20,7 @@ PyAPI_FUNC(PyObject *) PyCell_Get(PyObject *); PyAPI_FUNC(int) PyCell_Set(PyObject *, PyObject *); #define PyCell_GET(op) (((PyCellObject *)(op))->ob_ref) -#define PyCell_SET(op, v) (((PyCellObject *)(op))->ob_ref = v) +#define PyCell_SET(op, v) ((void)(((PyCellObject *)(op))->ob_ref = v)) #ifdef __cplusplus } diff --git a/Include/compile.h b/Include/compile.h index 4dd5435ce71a96..3c5acd7209f763 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -1,102 +1,9 @@ #ifndef Py_COMPILE_H #define Py_COMPILE_H - -#ifndef Py_LIMITED_API - #ifdef __cplusplus extern "C" { #endif -/* Public interface */ -#define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | \ - CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | \ - CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \ - CO_FUTURE_GENERATOR_STOP | CO_FUTURE_ANNOTATIONS) -#define PyCF_MASK_OBSOLETE (CO_NESTED) - -/* bpo-39562: CO_FUTURE_ and PyCF_ constants must be kept unique. - PyCF_ constants can use bits from 0x0100 to 0x10000. - CO_FUTURE_ constants use bits starting at 0x20000. */ -#define PyCF_SOURCE_IS_UTF8 0x0100 -#define PyCF_DONT_IMPLY_DEDENT 0x0200 -#define PyCF_ONLY_AST 0x0400 -#define PyCF_IGNORE_COOKIE 0x0800 -#define PyCF_TYPE_COMMENTS 0x1000 -#define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000 -#define PyCF_COMPILE_MASK (PyCF_ONLY_AST | PyCF_ALLOW_TOP_LEVEL_AWAIT | \ - PyCF_TYPE_COMMENTS | PyCF_DONT_IMPLY_DEDENT) - -#ifndef Py_LIMITED_API -typedef struct { - int cf_flags; /* bitmask of CO_xxx flags relevant to future */ - int cf_feature_version; /* minor Python version (PyCF_ONLY_AST) */ -} PyCompilerFlags; - -#define _PyCompilerFlags_INIT \ - (PyCompilerFlags){.cf_flags = 0, .cf_feature_version = PY_MINOR_VERSION} -#endif - -/* Future feature support */ - -typedef struct { - int ff_features; /* flags set by future statements */ - int ff_lineno; /* line number of last future statement */ -} PyFutureFeatures; - -#define FUTURE_NESTED_SCOPES "nested_scopes" -#define FUTURE_GENERATORS "generators" -#define FUTURE_DIVISION "division" -#define FUTURE_ABSOLUTE_IMPORT "absolute_import" -#define FUTURE_WITH_STATEMENT "with_statement" -#define FUTURE_PRINT_FUNCTION "print_function" -#define FUTURE_UNICODE_LITERALS "unicode_literals" -#define FUTURE_BARRY_AS_BDFL "barry_as_FLUFL" -#define FUTURE_GENERATOR_STOP "generator_stop" -#define FUTURE_ANNOTATIONS "annotations" - -struct _mod; /* Declare the existence of this type */ -#define PyAST_Compile(mod, s, f, ar) PyAST_CompileEx(mod, s, f, -1, ar) -PyAPI_FUNC(PyCodeObject *) PyAST_CompileEx( - struct _mod *mod, - const char *filename, /* decoded from the filesystem encoding */ - PyCompilerFlags *flags, - int optimize, - PyArena *arena); -PyAPI_FUNC(PyCodeObject *) PyAST_CompileObject( - struct _mod *mod, - PyObject *filename, - PyCompilerFlags *flags, - int optimize, - PyArena *arena); -PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST( - struct _mod * mod, - const char *filename /* decoded from the filesystem encoding */ - ); -PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromASTObject( - struct _mod * mod, - PyObject *filename - ); - -/* _Py_Mangle is defined in compile.c */ -PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name); - -#define PY_INVALID_STACK_EFFECT INT_MAX -PyAPI_FUNC(int) PyCompile_OpcodeStackEffect(int opcode, int oparg); -PyAPI_FUNC(int) PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump); - -typedef struct { - int optimize; - int ff_features; -} _PyASTOptimizeState; - -PyAPI_FUNC(int) _PyAST_Optimize(struct _mod *, PyArena *arena, _PyASTOptimizeState *state); - -#ifdef __cplusplus -} -#endif - -#endif /* !Py_LIMITED_API */ - /* These definitions must match corresponding definitions in graminit.h. */ #define Py_single_input 256 #define Py_file_input 257 @@ -106,4 +13,13 @@ PyAPI_FUNC(int) _PyAST_Optimize(struct _mod *, PyArena *arena, _PyASTOptimizeSta /* This doesn't need to match anything */ #define Py_fstring_input 800 +#ifndef Py_LIMITED_API +# define Py_CPYTHON_COMPILE_H +# include "cpython/compile.h" +# undef Py_CPYTHON_COMPILE_H +#endif + +#ifdef __cplusplus +} +#endif #endif /* !Py_COMPILE_H */ diff --git a/Include/cpython/abstract.h b/Include/cpython/abstract.h index b5b6e4819788c5..db850219645281 100644 --- a/Include/cpython/abstract.h +++ b/Include/cpython/abstract.h @@ -63,7 +63,7 @@ PyVectorcall_Function(PyObject *callable) { PyTypeObject *tp; Py_ssize_t offset; - vectorcallfunc *ptr; + vectorcallfunc ptr; assert(callable != NULL); tp = Py_TYPE(callable); @@ -73,8 +73,8 @@ PyVectorcall_Function(PyObject *callable) assert(PyCallable_Check(callable)); offset = tp->tp_vectorcall_offset; assert(offset > 0); - ptr = (vectorcallfunc *)(((char *)callable) + offset); - return *ptr; + memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); + return ptr; } /* Call the callable object 'callable' with the "vectorcall" calling @@ -119,7 +119,7 @@ static inline PyObject * PyObject_Vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = PyThreadState_Get(); return _PyObject_VectorcallTstate(tstate, callable, args, nargsf, kwnames); } @@ -155,7 +155,7 @@ _PyObject_FastCallTstate(PyThreadState *tstate, PyObject *func, PyObject *const static inline PyObject * _PyObject_FastCall(PyObject *func, PyObject *const *args, Py_ssize_t nargs) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = PyThreadState_Get(); return _PyObject_FastCallTstate(tstate, func, args, nargs); } @@ -164,7 +164,7 @@ _PyObject_FastCall(PyObject *func, PyObject *const *args, Py_ssize_t nargs) PyObject_CallNoArgs(). */ static inline PyObject * _PyObject_CallNoArg(PyObject *func) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = PyThreadState_Get(); return _PyObject_VectorcallTstate(tstate, func, NULL, 0, NULL); } @@ -179,7 +179,7 @@ PyObject_CallOneArg(PyObject *func, PyObject *arg) assert(arg != NULL); args = _args + 1; // For PY_VECTORCALL_ARGUMENTS_OFFSET args[0] = arg; - tstate = PyThreadState_GET(); + tstate = PyThreadState_Get(); nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; return _PyObject_VectorcallTstate(tstate, func, args, nargsf, NULL); } @@ -325,12 +325,6 @@ PyAPI_FUNC(int) PyBuffer_FillInfo(Py_buffer *view, PyObject *o, void *buf, /* Releases a Py_buffer obtained from getbuffer ParseTuple's "s*". */ PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view); -/* ==== Iterators ================================================ */ - -#define PyIter_Check(obj) \ - (Py_TYPE(obj)->tp_iternext != NULL && \ - Py_TYPE(obj)->tp_iternext != &_PyObject_NextNotImplemented) - /* === Sequence protocol ================================================ */ /* Assume tp_as_sequence and sq_item exist and that 'i' does not @@ -376,4 +370,4 @@ PyAPI_FUNC(void) _Py_add_one_to_index_C(int nd, Py_ssize_t *index, PyAPI_FUNC(int) _Py_convert_optional_to_ssize_t(PyObject *, void *); /* Same as PyNumber_Index but can return an instance of a subclass of int. */ -PyAPI_FUNC(PyObject *) _PyNumber_Index(PyObject *o); \ No newline at end of file +PyAPI_FUNC(PyObject *) _PyNumber_Index(PyObject *o); diff --git a/Include/cpython/bytesobject.h b/Include/cpython/bytesobject.h index f284c5835df099..6b3f55224fc553 100644 --- a/Include/cpython/bytesobject.h +++ b/Include/cpython/bytesobject.h @@ -10,7 +10,7 @@ typedef struct { /* Invariants: * ob_sval contains space for 'ob_size+1' elements. * ob_sval[ob_size] == 0. - * ob_shash is the hash of the string or -1 if not computed yet. + * ob_shash is the hash of the byte string or -1 if not computed yet. */ } PyBytesObject; diff --git a/Include/cpython/code.h b/Include/cpython/code.h index cda28ac6ee9343..e810eb4f816f3a 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -38,7 +38,7 @@ struct PyCodeObject { Py_ssize_t *co_cell2arg; /* Maps cell vars which are arguments. */ PyObject *co_filename; /* unicode (where it was loaded from) */ PyObject *co_name; /* unicode (name, for reference) */ - PyObject *co_lnotab; /* string (encoding addr<->lineno mapping) See + PyObject *co_linetable; /* string (encoding addr<->lineno mapping) See Objects/lnotab_notes.txt for details. */ void *co_zombieframe; /* for optimization only (see frameobject.c) */ PyObject *co_weakreflist; /* to support weakrefs to code objects */ @@ -135,16 +135,23 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno); PyAPI_FUNC(int) PyCode_Addr2Line(PyCodeObject *, int); /* for internal use only */ -typedef struct _addr_pair { - int ap_lower; - int ap_upper; -} PyAddrPair; +struct _opaque { + int computed_line; + char *lo_next; + char *limit; +}; + +typedef struct _line_offsets { + int ar_start; + int ar_end; + int ar_line; + struct _opaque opaque; +} PyCodeAddressRange; /* Update *bounds to describe the first and one-past-the-last instructions in the same line as lasti. Return the number of that line. */ -PyAPI_FUNC(int) _PyCode_CheckLineNumber(PyCodeObject* co, - int lasti, PyAddrPair *bounds); +PyAPI_FUNC(int) _PyCode_CheckLineNumber(int lasti, PyCodeAddressRange *bounds); /* Create a comparable key used to compare constants taking in account the * object type. It is used to make sure types are not coerced (e.g., float and @@ -163,3 +170,15 @@ PyAPI_FUNC(int) _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra); PyAPI_FUNC(int) _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra); + +/** API for initializing the line number table. */ +int _PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds); + +/** Out of process API for initializing the line number table. */ +void PyLineTable_InitAddressRange(char *linetable, Py_ssize_t length, int firstlineno, PyCodeAddressRange *range); + +/** API for traversing the line number table. */ +int PyLineTable_NextAddressRange(PyCodeAddressRange *range); +int PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); + + diff --git a/Include/cpython/compile.h b/Include/cpython/compile.h new file mode 100644 index 00000000000000..a202c0b0e65508 --- /dev/null +++ b/Include/cpython/compile.h @@ -0,0 +1,52 @@ +#ifndef Py_CPYTHON_COMPILE_H +# error "this header file must not be included directly" +#endif + +/* Public interface */ +#define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | \ + CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | \ + CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \ + CO_FUTURE_GENERATOR_STOP | CO_FUTURE_ANNOTATIONS) +#define PyCF_MASK_OBSOLETE (CO_NESTED) + +/* bpo-39562: CO_FUTURE_ and PyCF_ constants must be kept unique. + PyCF_ constants can use bits from 0x0100 to 0x10000. + CO_FUTURE_ constants use bits starting at 0x20000. */ +#define PyCF_SOURCE_IS_UTF8 0x0100 +#define PyCF_DONT_IMPLY_DEDENT 0x0200 +#define PyCF_ONLY_AST 0x0400 +#define PyCF_IGNORE_COOKIE 0x0800 +#define PyCF_TYPE_COMMENTS 0x1000 +#define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000 +#define PyCF_COMPILE_MASK (PyCF_ONLY_AST | PyCF_ALLOW_TOP_LEVEL_AWAIT | \ + PyCF_TYPE_COMMENTS | PyCF_DONT_IMPLY_DEDENT) + +typedef struct { + int cf_flags; /* bitmask of CO_xxx flags relevant to future */ + int cf_feature_version; /* minor Python version (PyCF_ONLY_AST) */ +} PyCompilerFlags; + +#define _PyCompilerFlags_INIT \ + (PyCompilerFlags){.cf_flags = 0, .cf_feature_version = PY_MINOR_VERSION} + +/* Future feature support */ + +typedef struct { + int ff_features; /* flags set by future statements */ + int ff_lineno; /* line number of last future statement */ +} PyFutureFeatures; + +#define FUTURE_NESTED_SCOPES "nested_scopes" +#define FUTURE_GENERATORS "generators" +#define FUTURE_DIVISION "division" +#define FUTURE_ABSOLUTE_IMPORT "absolute_import" +#define FUTURE_WITH_STATEMENT "with_statement" +#define FUTURE_PRINT_FUNCTION "print_function" +#define FUTURE_UNICODE_LITERALS "unicode_literals" +#define FUTURE_BARRY_AS_BDFL "barry_as_FLUFL" +#define FUTURE_GENERATOR_STOP "generator_stop" +#define FUTURE_ANNOTATIONS "annotations" + +#define PY_INVALID_STACK_EFFECT INT_MAX +PyAPI_FUNC(int) PyCompile_OpcodeStackEffect(int opcode, int oparg); +PyAPI_FUNC(int) PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump); diff --git a/Include/cpython/fileutils.h b/Include/cpython/fileutils.h index e79d03e24f5771..ccf37e9468d615 100644 --- a/Include/cpython/fileutils.h +++ b/Include/cpython/fileutils.h @@ -32,6 +32,9 @@ PyAPI_FUNC(int) _Py_EncodeLocaleEx( int current_locale, _Py_error_handler errors); +PyAPI_FUNC(char*) _Py_EncodeLocaleRaw( + const wchar_t *text, + size_t *error_pos); PyAPI_FUNC(PyObject *) _Py_device_encoding(int); @@ -95,10 +98,6 @@ PyAPI_FUNC(FILE *) _Py_wfopen( const wchar_t *path, const wchar_t *mode); -PyAPI_FUNC(FILE*) _Py_fopen( - const char *pathname, - const char *mode); - PyAPI_FUNC(FILE*) _Py_fopen_obj( PyObject *path, const char *mode); @@ -162,4 +161,12 @@ PyAPI_FUNC(int) _Py_dup(int fd); PyAPI_FUNC(int) _Py_get_blocking(int fd); PyAPI_FUNC(int) _Py_set_blocking(int fd, int blocking); -#endif /* !MS_WINDOWS */ +#else /* MS_WINDOWS */ +PyAPI_FUNC(void*) _Py_get_osfhandle_noraise(int fd); + +PyAPI_FUNC(void*) _Py_get_osfhandle(int fd); + +PyAPI_FUNC(int) _Py_open_osfhandle_noraise(void *handle, int flags); + +PyAPI_FUNC(int) _Py_open_osfhandle(void *handle, int flags); +#endif /* MS_WINDOWS */ diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 63240b5b6d5ccb..5122ec41a3d22a 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -42,12 +42,7 @@ struct _frame { PyObject *f_gen; int f_lasti; /* Last instruction if called */ - /* Call PyFrame_GetLineNumber() instead of reading this field - directly. As of 2.3 f_lineno is only valid when tracing is - active (i.e. when f_trace is set). At other times we use - PyCode_Addr2Line to calculate the line from the current - bytecode index. */ - int f_lineno; /* Current line number */ + int f_lineno; /* Current line number. Only valid if non-zero */ int f_iblock; /* index in f_blockstack */ PyFrameState f_state; /* What state the frame is in */ PyTryBlock f_blockstack[CO_MAXBLOCKS]; /* for try and loop blocks */ @@ -76,8 +71,8 @@ PyAPI_FUNC(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *, PyObject *, PyObject *); /* only internal use */ -PyFrameObject* _PyFrame_New_NoTrack(PyThreadState *, PyCodeObject *, - PyObject *, PyObject *); +PyFrameObject* +_PyFrame_New_NoTrack(PyThreadState *, PyFrameConstructor *, PyObject *); /* The rest of the interface is specific for frame objects */ diff --git a/Include/cpython/import.h b/Include/cpython/import.h index 3b20a74c855dbf..bad68f0e0980d5 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -13,8 +13,6 @@ PyAPI_FUNC(int) _PyImport_SetModuleString(const char *name, PyObject* module); PyAPI_FUNC(void) _PyImport_AcquireLock(void); PyAPI_FUNC(int) _PyImport_ReleaseLock(void); -PyAPI_FUNC(PyObject *) _PyImport_FindExtensionObject(PyObject *, PyObject *); - PyAPI_FUNC(int) _PyImport_FixupBuiltin( PyObject *mod, const char *name, /* UTF-8 encoded string */ diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index bbe83876777157..09f9a2947efef3 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -41,7 +41,7 @@ PyAPI_FUNC(PyStatus) PyWideStringList_Insert(PyWideStringList *list, /* --- PyPreConfig ----------------------------------------------- */ -typedef struct { +typedef struct PyPreConfig { int _config_init; /* _PyConfigInitEnum value */ /* Parse Py_PreInitializeFromBytesArgs() arguments? @@ -127,296 +127,85 @@ PyAPI_FUNC(void) PyPreConfig_InitIsolatedConfig(PyPreConfig *config); /* --- PyConfig ---------------------------------------------- */ -typedef struct { +/* This structure is best documented in the Doc/c-api/init_config.rst file. */ +typedef struct PyConfig { int _config_init; /* _PyConfigInitEnum value */ - int isolated; /* Isolated mode? see PyPreConfig.isolated */ - int use_environment; /* Use environment variables? see PyPreConfig.use_environment */ - int dev_mode; /* Python Development Mode? See PyPreConfig.dev_mode */ - - /* Install signal handlers? Yes by default. */ + int isolated; + int use_environment; + int dev_mode; int install_signal_handlers; - - int use_hash_seed; /* PYTHONHASHSEED=x */ + int use_hash_seed; unsigned long hash_seed; - - /* Enable faulthandler? - Set to 1 by -X faulthandler and PYTHONFAULTHANDLER. -1 means unset. */ int faulthandler; - - /* Enable tracemalloc? - Set by -X tracemalloc=N and PYTHONTRACEMALLOC. -1 means unset */ int tracemalloc; - - int import_time; /* PYTHONPROFILEIMPORTTIME, -X importtime */ - int show_ref_count; /* -X showrefcount */ - int dump_refs; /* PYTHONDUMPREFS */ - int malloc_stats; /* PYTHONMALLOCSTATS */ - - /* Python filesystem encoding and error handler: - sys.getfilesystemencoding() and sys.getfilesystemencodeerrors(). - - Default encoding and error handler: - - * if Py_SetStandardStreamEncoding() has been called: they have the - highest priority; - * PYTHONIOENCODING environment variable; - * The UTF-8 Mode uses UTF-8/surrogateescape; - * If Python forces the usage of the ASCII encoding (ex: C locale - or POSIX locale on FreeBSD or HP-UX), use ASCII/surrogateescape; - * locale encoding: ANSI code page on Windows, UTF-8 on Android and - VxWorks, LC_CTYPE locale encoding on other platforms; - * On Windows, "surrogateescape" error handler; - * "surrogateescape" error handler if the LC_CTYPE locale is "C" or "POSIX"; - * "surrogateescape" error handler if the LC_CTYPE locale has been coerced - (PEP 538); - * "strict" error handler. - - Supported error handlers: "strict", "surrogateescape" and - "surrogatepass". The surrogatepass error handler is only supported - if Py_DecodeLocale() and Py_EncodeLocale() use directly the UTF-8 codec; - it's only used on Windows. - - initfsencoding() updates the encoding to the Python codec name. - For example, "ANSI_X3.4-1968" is replaced with "ascii". - - On Windows, sys._enablelegacywindowsfsencoding() sets the - encoding/errors to mbcs/replace at runtime. - - - See Py_FileSystemDefaultEncoding and Py_FileSystemDefaultEncodeErrors. - */ + int import_time; + int show_ref_count; + int dump_refs; + int malloc_stats; wchar_t *filesystem_encoding; wchar_t *filesystem_errors; - - wchar_t *pycache_prefix; /* PYTHONPYCACHEPREFIX, -X pycache_prefix=PATH */ - int parse_argv; /* Parse argv command line arguments? */ - - /* Command line arguments (sys.argv). - - Set parse_argv to 1 to parse argv as Python command line arguments - and then strip Python arguments from argv. - - If argv is empty, an empty string is added to ensure that sys.argv - always exists and is never empty. */ + wchar_t *pycache_prefix; + int parse_argv; + PyWideStringList orig_argv; PyWideStringList argv; - - /* Program name: - - - If Py_SetProgramName() was called, use its value. - - On macOS, use PYTHONEXECUTABLE environment variable if set. - - If WITH_NEXT_FRAMEWORK macro is defined, use __PYVENV_LAUNCHER__ - environment variable is set. - - Use argv[0] if available and non-empty. - - Use "python" on Windows, or "python3 on other platforms. */ - wchar_t *program_name; - - PyWideStringList xoptions; /* Command line -X options */ - - /* Warnings options: lowest to highest priority. warnings.filters - is built in the reverse order (highest to lowest priority). */ + PyWideStringList xoptions; PyWideStringList warnoptions; - - /* If equal to zero, disable the import of the module site and the - site-dependent manipulations of sys.path that it entails. Also disable - these manipulations if site is explicitly imported later (call - site.main() if you want them to be triggered). - - Set to 0 by the -S command line option. If set to -1 (default), it is - set to !Py_NoSiteFlag. */ int site_import; - - /* Bytes warnings: - - * If equal to 1, issue a warning when comparing bytes or bytearray with - str or bytes with int. - * If equal or greater to 2, issue an error. - - Incremented by the -b command line option. If set to -1 (default), inherit - Py_BytesWarningFlag value. */ int bytes_warning; - - /* If greater than 0, enable inspect: when a script is passed as first - argument or the -c option is used, enter interactive mode after - executing the script or the command, even when sys.stdin does not appear - to be a terminal. - - Incremented by the -i command line option. Set to 1 if the PYTHONINSPECT - environment variable is non-empty. If set to -1 (default), inherit - Py_InspectFlag value. */ + int warn_default_encoding; int inspect; - - /* If greater than 0: enable the interactive mode (REPL). - - Incremented by the -i command line option. If set to -1 (default), - inherit Py_InteractiveFlag value. */ int interactive; - - /* Optimization level. - - Incremented by the -O command line option. Set by the PYTHONOPTIMIZE - environment variable. If set to -1 (default), inherit Py_OptimizeFlag - value. */ int optimization_level; - - /* If greater than 0, enable the debug mode: turn on parser debugging - output (for expert only, depending on compilation options). - - Incremented by the -d command line option. Set by the PYTHONDEBUG - environment variable. If set to -1 (default), inherit Py_DebugFlag - value. */ int parser_debug; - - /* If equal to 0, Python won't try to write ``.pyc`` files on the - import of source modules. - - Set to 0 by the -B command line option and the PYTHONDONTWRITEBYTECODE - environment variable. If set to -1 (default), it is set to - !Py_DontWriteBytecodeFlag. */ int write_bytecode; - - /* If greater than 0, enable the verbose mode: print a message each time a - module is initialized, showing the place (filename or built-in module) - from which it is loaded. - - If greater or equal to 2, print a message for each file that is checked - for when searching for a module. Also provides information on module - cleanup at exit. - - Incremented by the -v option. Set by the PYTHONVERBOSE environment - variable. If set to -1 (default), inherit Py_VerboseFlag value. */ int verbose; - - /* If greater than 0, enable the quiet mode: Don't display the copyright - and version messages even in interactive mode. - - Incremented by the -q option. If set to -1 (default), inherit - Py_QuietFlag value. */ int quiet; - - /* If greater than 0, don't add the user site-packages directory to - sys.path. - - Set to 0 by the -s and -I command line options , and the PYTHONNOUSERSITE - environment variable. If set to -1 (default), it is set to - !Py_NoUserSiteDirectory. */ int user_site_directory; - - /* If non-zero, configure C standard steams (stdio, stdout, - stderr): - - - Set O_BINARY mode on Windows. - - If buffered_stdio is equal to zero, make streams unbuffered. - Otherwise, enable streams buffering if interactive is non-zero. */ int configure_c_stdio; - - /* If equal to 0, enable unbuffered mode: force the stdout and stderr - streams to be unbuffered. - - Set to 0 by the -u option. Set by the PYTHONUNBUFFERED environment - variable. - If set to -1 (default), it is set to !Py_UnbufferedStdioFlag. */ int buffered_stdio; - - /* Encoding of sys.stdin, sys.stdout and sys.stderr. - Value set from PYTHONIOENCODING environment variable and - Py_SetStandardStreamEncoding() function. - See also 'stdio_errors' attribute. */ wchar_t *stdio_encoding; - - /* Error handler of sys.stdin and sys.stdout. - Value set from PYTHONIOENCODING environment variable and - Py_SetStandardStreamEncoding() function. - See also 'stdio_encoding' attribute. */ wchar_t *stdio_errors; - #ifdef MS_WINDOWS - /* If greater than zero, use io.FileIO instead of WindowsConsoleIO for sys - standard streams. - - Set to 1 if the PYTHONLEGACYWINDOWSSTDIO environment variable is set to - a non-empty string. If set to -1 (default), inherit - Py_LegacyWindowsStdioFlag value. - - See PEP 528 for more details. */ int legacy_windows_stdio; #endif - - /* Value of the --check-hash-based-pycs command line option: - - - "default" means the 'check_source' flag in hash-based pycs - determines invalidation - - "always" causes the interpreter to hash the source file for - invalidation regardless of value of 'check_source' bit - - "never" causes the interpreter to always assume hash-based pycs are - valid - - The default value is "default". - - See PEP 552 "Deterministic pycs" for more details. */ wchar_t *check_hash_pycs_mode; /* --- Path configuration inputs ------------ */ - - /* If greater than 0, suppress _PyPathConfig_Calculate() warnings on Unix. - The parameter has no effect on Windows. - - If set to -1 (default), inherit !Py_FrozenFlag value. */ int pathconfig_warnings; - - wchar_t *pythonpath_env; /* PYTHONPATH environment variable */ - wchar_t *home; /* PYTHONHOME environment variable, - see also Py_SetPythonHome(). */ + wchar_t *program_name; + wchar_t *pythonpath_env; + wchar_t *home; + wchar_t *platlibdir; /* --- Path configuration outputs ----------- */ - - int module_search_paths_set; /* If non-zero, use module_search_paths */ - PyWideStringList module_search_paths; /* sys.path paths. Computed if - module_search_paths_set is equal - to zero. */ - - wchar_t *executable; /* sys.executable */ - wchar_t *base_executable; /* sys._base_executable */ - wchar_t *prefix; /* sys.prefix */ - wchar_t *base_prefix; /* sys.base_prefix */ - wchar_t *exec_prefix; /* sys.exec_prefix */ - wchar_t *base_exec_prefix; /* sys.base_exec_prefix */ - wchar_t *platlibdir; /* sys.platlibdir */ + int module_search_paths_set; + PyWideStringList module_search_paths; + wchar_t *executable; + wchar_t *base_executable; + wchar_t *prefix; + wchar_t *base_prefix; + wchar_t *exec_prefix; + wchar_t *base_exec_prefix; /* --- Parameter only used by Py_Main() ---------- */ - - /* Skip the first line of the source ('run_filename' parameter), allowing use of non-Unix forms of - "#!cmd". This is intended for a DOS specific hack only. - - Set by the -x command line option. */ int skip_source_first_line; - - wchar_t *run_command; /* -c command line argument */ - wchar_t *run_module; /* -m command line argument */ - wchar_t *run_filename; /* Trailing command line argument without -c or -m */ + wchar_t *run_command; + wchar_t *run_module; + wchar_t *run_filename; /* --- Private fields ---------------------------- */ - /* Install importlib? If set to 0, importlib is not initialized at all. - Needed by freeze_importlib. */ + // Install importlib? If equals to 0, importlib is not initialized at all. + // Needed by freeze_importlib. int _install_importlib; - /* If equal to 0, stop Python initialization before the "main" phase */ + // If equal to 0, stop Python initialization before the "main" phase. int _init_main; - /* If non-zero, disallow threads, subprocesses, and fork. - Default: 0. */ + // If non-zero, disallow threads, subprocesses, and fork. + // Default: 0. int _isolated_interpreter; - - /* The list of the original command line arguments passed to the Python - executable. - - If 'orig_argv' list is empty and 'argv' is not a list only containing an - empty string, PyConfig_Read() copies 'argv' into 'orig_argv' before - modifying 'argv' (if 'parse_argv is non-zero). - - _PyConfig_Write() initializes Py_GetArgcArgv() to this list. */ - PyWideStringList orig_argv; } PyConfig; PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config); diff --git a/Include/cpython/listobject.h b/Include/cpython/listobject.h index e1b9462d5b3612..e3239152c497c2 100644 --- a/Include/cpython/listobject.h +++ b/Include/cpython/listobject.h @@ -30,5 +30,5 @@ PyAPI_FUNC(void) _PyList_DebugMallocStats(FILE *out); #define _PyList_CAST(op) (assert(PyList_Check(op)), (PyListObject *)(op)) #define PyList_GET_ITEM(op, i) (_PyList_CAST(op)->ob_item[i]) -#define PyList_SET_ITEM(op, i, v) (_PyList_CAST(op)->ob_item[i] = (v)) +#define PyList_SET_ITEM(op, i, v) ((void)(_PyList_CAST(op)->ob_item[i] = (v))) #define PyList_GET_SIZE(op) Py_SIZE(_PyList_CAST(op)) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 875a600f79565a..84c60e55d5c9df 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -35,12 +35,13 @@ PyAPI_FUNC(Py_ssize_t) _Py_GetRefTotal(void); _PyObject_{Get,Set,Has}AttrId are __getattr__ versions using _Py_Identifier*. */ typedef struct _Py_Identifier { - struct _Py_Identifier *next; const char* string; - PyObject *object; + // Index in PyInterpreterState.unicode.ids.array. It is process-wide + // unique and must be initialized to -1. + Py_ssize_t index; } _Py_Identifier; -#define _Py_static_string_init(value) { .next = NULL, .string = value, .object = NULL } +#define _Py_static_string_init(value) { .string = value, .index = -1 } #define _Py_static_string(varname, value) static _Py_Identifier varname = _Py_static_string_init(value) #define _Py_IDENTIFIER(varname) _Py_static_string(PyId_##varname, #varname) @@ -167,10 +168,13 @@ typedef struct { objobjargproc mp_ass_subscript; } PyMappingMethods; +typedef PySendResult (*sendfunc)(PyObject *iter, PyObject *value, PyObject **result); + typedef struct { unaryfunc am_await; unaryfunc am_aiter; unaryfunc am_anext; + sendfunc am_send; } PyAsyncMethods; typedef struct { @@ -182,6 +186,8 @@ typedef struct { * backwards-compatibility */ typedef Py_ssize_t printfunc; +// If this structure is modified, Doc/includes/typestruct.h should be updated +// as well. struct _typeobject { PyObject_VAR_HEAD const char *tp_name; /* For printing, in format "." */ @@ -241,6 +247,7 @@ struct _typeobject { struct PyMethodDef *tp_methods; struct PyMemberDef *tp_members; struct PyGetSetDef *tp_getset; + // Strong reference on a heap type, borrowed reference on a static type struct _typeobject *tp_base; PyObject *tp_dict; descrgetfunc tp_descr_get; @@ -296,6 +303,8 @@ PyAPI_FUNC(PyObject *) _PyObject_LookupSpecial(PyObject *, _Py_Identifier *); PyAPI_FUNC(PyTypeObject *) _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); PyAPI_FUNC(PyObject *) _PyType_GetDocFromInternalDoc(const char *, const char *); PyAPI_FUNC(PyObject *) _PyType_GetTextSignatureFromInternalDoc(const char *, const char *); +struct PyModuleDef; +PyAPI_FUNC(PyObject *) _PyType_GetModuleByDef(PyTypeObject *, struct PyModuleDef *); struct _Py_Identifier; PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); @@ -510,6 +519,8 @@ struct _ts; /* Python 3.9 private API, invoked by the macros below. */ PyAPI_FUNC(int) _PyTrash_begin(struct _ts *tstate, PyObject *op); PyAPI_FUNC(void) _PyTrash_end(struct _ts *tstate); +/* Python 3.10 private API, invoked by the Py_TRASHCAN_BEGIN(). */ +PyAPI_FUNC(int) _PyTrash_cond(PyObject *op, destructor dealloc); #define PyTrash_UNWIND_LEVEL 50 @@ -519,7 +530,7 @@ PyAPI_FUNC(void) _PyTrash_end(struct _ts *tstate); /* If "cond" is false, then _tstate remains NULL and the deallocator \ * is run normally without involving the trashcan */ \ if (cond) { \ - _tstate = PyThreadState_GET(); \ + _tstate = PyThreadState_Get(); \ if (_PyTrash_begin(_tstate, _PyObject_CAST(op))) { \ break; \ } \ @@ -533,7 +544,7 @@ PyAPI_FUNC(void) _PyTrash_end(struct _ts *tstate); #define Py_TRASHCAN_BEGIN(op, dealloc) \ Py_TRASHCAN_BEGIN_CONDITION(op, \ - Py_TYPE(op)->tp_dealloc == (destructor)(dealloc)) + _PyTrash_cond(_PyObject_CAST(op), (destructor)dealloc)) /* For backwards compatibility, these macros enable the trashcan * unconditionally */ diff --git a/Include/cpython/objimpl.h b/Include/cpython/objimpl.h index 15999a239f7a9a..d83700e2a4647f 100644 --- a/Include/cpython/objimpl.h +++ b/Include/cpython/objimpl.h @@ -79,10 +79,6 @@ PyAPI_FUNC(void) PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator); PyAPI_FUNC(void) PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator); -PyAPI_FUNC(Py_ssize_t) _PyGC_CollectNoFail(void); -PyAPI_FUNC(Py_ssize_t) _PyGC_CollectIfEnabled(void); - - /* Test if an object implements the garbage collector protocol */ PyAPI_FUNC(int) PyObject_IS_GC(PyObject *obj); diff --git a/Include/odictobject.h b/Include/cpython/odictobject.h similarity index 100% rename from Include/odictobject.h rename to Include/cpython/odictobject.h diff --git a/Include/picklebufobject.h b/Include/cpython/picklebufobject.h similarity index 100% rename from Include/picklebufobject.h rename to Include/cpython/picklebufobject.h diff --git a/Include/pyctype.h b/Include/cpython/pyctype.h similarity index 95% rename from Include/pyctype.h rename to Include/cpython/pyctype.h index 6bce63eeb63b82..729d93275e6c53 100644 --- a/Include/pyctype.h +++ b/Include/cpython/pyctype.h @@ -1,6 +1,9 @@ #ifndef Py_LIMITED_API #ifndef PYCTYPE_H #define PYCTYPE_H +#ifdef __cplusplus +extern "C" { +#endif #define PY_CTF_LOWER 0x01 #define PY_CTF_UPPER 0x02 @@ -29,5 +32,8 @@ PyAPI_DATA(const unsigned char) _Py_ctype_toupper[256]; #define Py_TOLOWER(c) (_Py_ctype_tolower[Py_CHARMASK(c)]) #define Py_TOUPPER(c) (_Py_ctype_toupper[Py_CHARMASK(c)]) +#ifdef __cplusplus +} +#endif #endif /* !PYCTYPE_H */ #endif /* !Py_LIMITED_API */ diff --git a/Include/pydebug.h b/Include/cpython/pydebug.h similarity index 100% rename from Include/pydebug.h rename to Include/cpython/pydebug.h diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index c2500d927bf7f0..5e57129c3b82cf 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -20,6 +20,8 @@ typedef struct { PyObject *filename; PyObject *lineno; PyObject *offset; + PyObject *end_lineno; + PyObject *end_offset; PyObject *text; PyObject *print_file_and_line; } PySyntaxErrorObject; @@ -62,6 +64,17 @@ typedef struct { PyObject *value; } PyStopIterationObject; +typedef struct { + PyException_HEAD + PyObject *name; +} PyNameErrorObject; + +typedef struct { + PyException_HEAD + PyObject *obj; + PyObject *name; +} PyAttributeErrorObject; + /* Compatibility typedefs */ typedef PyOSErrorObject PyEnvironmentErrorObject; #ifdef MS_WINDOWS @@ -78,10 +91,6 @@ PyAPI_FUNC(void) _PyErr_GetExcInfo(PyThreadState *, PyObject **, PyObject **, Py PyAPI_FUNC(void) _PyErr_ChainExceptions(PyObject *, PyObject *, PyObject *); -/* */ - -#define PyExceptionClass_Name(x) (((PyTypeObject*)(x))->tp_name) - /* Convenience functions */ #ifdef MS_WINDOWS @@ -141,6 +150,13 @@ PyAPI_FUNC(void) PyErr_SyntaxLocationObject( int lineno, int col_offset); +PyAPI_FUNC(void) PyErr_RangedSyntaxLocationObject( + PyObject *filename, + int lineno, + int col_offset, + int end_lineno, + int end_col_offset); + PyAPI_FUNC(PyObject *) PyErr_ProgramTextObject( PyObject *filename, int lineno); diff --git a/Include/pyfpe.h b/Include/cpython/pyfpe.h similarity index 100% rename from Include/pyfpe.h rename to Include/cpython/pyfpe.h diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index f38ec5a4ae3990..13f7a26ba12d02 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -35,15 +35,11 @@ PyAPI_FUNC(int) Py_RunMain(void); PyAPI_FUNC(void) _Py_NO_RETURN Py_ExitStatusException(PyStatus err); -/* Py_PyAtExit is for the atexit module, Py_AtExit is for low-level - * exit functions. - */ -PyAPI_FUNC(void) _Py_PyAtExit(void (*func)(PyObject *), PyObject *); - /* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. */ PyAPI_FUNC(void) _Py_RestoreSignals(void); PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *); +PyAPI_FUNC(int) _Py_FdIsInteractive(FILE *fp, PyObject *filename); PyAPI_FUNC(void) _Py_SetProgramFullPath(const wchar_t *); diff --git a/Include/cpython/pymem.h b/Include/cpython/pymem.h index 61d719584584e8..d1054d76520b9a 100644 --- a/Include/cpython/pymem.h +++ b/Include/cpython/pymem.h @@ -10,8 +10,6 @@ PyAPI_FUNC(void) PyMem_RawFree(void *ptr); /* Try to get the allocators name set by _PyMem_SetupAllocators(). */ PyAPI_FUNC(const char*) _PyMem_GetCurrentAllocatorName(void); -PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize); - /* strdup() using PyMem_RawMalloc() */ PyAPI_FUNC(char *) _PyMem_RawStrdup(const char *str); diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 5d5e4e331978ac..e3ccc543560849 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -29,6 +29,21 @@ typedef int (*Py_tracefunc)(PyObject *, PyFrameObject *, int, PyObject *); #define PyTrace_OPCODE 7 +typedef struct _cframe { + /* This struct will be threaded through the C stack + * allowing fast access to per-thread state that needs + * to be accessed quickly by the interpreter, but can + * be modified outside of the interpreter. + * + * WARNING: This makes data on the C stack accessible from + * heap objects. Care must be taken to maintain stack + * discipline and make sure that instances of this struct cannot + * accessed outside of their lifetime. + */ + int use_tracing; + struct _cframe *previous; +} CFrame; + typedef struct _err_stackitem { /* This struct represents an entry on the exception stack, which is a * per-coroutine state. (Coroutine in the computer science sense, @@ -54,15 +69,17 @@ struct _ts { /* Borrowed reference to the current frame (it can be NULL) */ PyFrameObject *frame; int recursion_depth; - char overflowed; /* The stack has overflowed. Allow 50 more calls - to handle the runtime error. */ + int recursion_headroom; /* Allow 50 more calls to handle any errors. */ int stackcheck_counter; /* 'tracing' keeps track of the execution depth when tracing/profiling. This is to prevent the actual trace/profile code from being recorded in the trace/profile. */ int tracing; - int use_tracing; + + /* Pointer to current CFrame in the C stack frame of the currently, + * or most recently, executing _PyEval_EvalFrameDefault. */ + CFrame *cframe; Py_tracefunc c_profilefunc; Py_tracefunc c_tracefunc; @@ -130,6 +147,8 @@ struct _ts { /* Unique thread state id. */ uint64_t id; + CFrame root_cframe; + /* XXX signal handlers should also be here */ }; @@ -167,6 +186,11 @@ PyAPI_FUNC(PyInterpreterState *) _PyGILState_GetInterpreterStateUnsafe(void); */ PyAPI_FUNC(PyObject *) _PyThread_CurrentFrames(void); +/* The implementation of sys._current_exceptions() Returns a dict mapping + thread id to that thread's current exception. +*/ +PyAPI_FUNC(PyObject *) _PyThread_CurrentExceptions(void); + /* Routines for advanced debuggers, requested by David Beazley. Don't use unless you know what you are doing! */ PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Main(void); @@ -188,6 +212,36 @@ PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameFunc( PyAPI_FUNC(const PyConfig*) _PyInterpreterState_GetConfig(PyInterpreterState *interp); +/* Get a copy of the current interpreter configuration. + + Return 0 on success. Raise an exception and return -1 on error. + + The caller must initialize 'config', using PyConfig_InitPythonConfig() + for example. + + Python must be preinitialized to call this method. + The caller must hold the GIL. */ +PyAPI_FUNC(int) _PyInterpreterState_GetConfigCopy( + struct PyConfig *config); + +/* Set the configuration of the current interpreter. + + This function should be called during or just after the Python + initialization. + + Update the sys module with the new configuration. If the sys module was + modified directly after the Python initialization, these changes are lost. + + Some configuration like faulthandler or warnoptions can be updated in the + configuration, but don't reconfigure Python (don't enable/disable + faulthandler and don't reconfigure warnings filters). + + Return 0 on success. Raise an exception and return -1 on error. + + The configuration should come from _PyInterpreterState_GetConfigCopy(). */ +PyAPI_FUNC(int) _PyInterpreterState_SetConfig( + const struct PyConfig *config); + // Get the configuration of the currrent interpreter. // The caller must hold the GIL. PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); diff --git a/Include/cpython/pythonrun.h b/Include/cpython/pythonrun.h new file mode 100644 index 00000000000000..2e72d0820d34f5 --- /dev/null +++ b/Include/cpython/pythonrun.h @@ -0,0 +1,121 @@ +#ifndef Py_CPYTHON_PYTHONRUN_H +# error "this header file must not be included directly" +#endif + +PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *); +PyAPI_FUNC(int) _PyRun_SimpleFileObject( + FILE *fp, + PyObject *filename, + int closeit, + PyCompilerFlags *flags); +PyAPI_FUNC(int) PyRun_AnyFileExFlags( + FILE *fp, + const char *filename, /* decoded from the filesystem encoding */ + int closeit, + PyCompilerFlags *flags); +PyAPI_FUNC(int) _PyRun_AnyFileObject( + FILE *fp, + PyObject *filename, + int closeit, + PyCompilerFlags *flags); +PyAPI_FUNC(int) PyRun_SimpleFileExFlags( + FILE *fp, + const char *filename, /* decoded from the filesystem encoding */ + int closeit, + PyCompilerFlags *flags); +PyAPI_FUNC(int) PyRun_InteractiveOneFlags( + FILE *fp, + const char *filename, /* decoded from the filesystem encoding */ + PyCompilerFlags *flags); +PyAPI_FUNC(int) PyRun_InteractiveOneObject( + FILE *fp, + PyObject *filename, + PyCompilerFlags *flags); +PyAPI_FUNC(int) PyRun_InteractiveLoopFlags( + FILE *fp, + const char *filename, /* decoded from the filesystem encoding */ + PyCompilerFlags *flags); +PyAPI_FUNC(int) _PyRun_InteractiveLoopObject( + FILE *fp, + PyObject *filename, + PyCompilerFlags *flags); + + +PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *, + PyObject *, PyCompilerFlags *); + +PyAPI_FUNC(PyObject *) PyRun_FileExFlags( + FILE *fp, + const char *filename, /* decoded from the filesystem encoding */ + int start, + PyObject *globals, + PyObject *locals, + int closeit, + PyCompilerFlags *flags); + + +PyAPI_FUNC(PyObject *) Py_CompileStringExFlags( + const char *str, + const char *filename, /* decoded from the filesystem encoding */ + int start, + PyCompilerFlags *flags, + int optimize); +PyAPI_FUNC(PyObject *) Py_CompileStringObject( + const char *str, + PyObject *filename, int start, + PyCompilerFlags *flags, + int optimize); + +#define Py_CompileString(str, p, s) Py_CompileStringExFlags(str, p, s, NULL, -1) +#define Py_CompileStringFlags(str, p, s, f) Py_CompileStringExFlags(str, p, s, f, -1) + + +PyAPI_FUNC(const char *) _Py_SourceAsString( + PyObject *cmd, + const char *funcname, + const char *what, + PyCompilerFlags *cf, + PyObject **cmd_copy); + + +/* A function flavor is also exported by libpython. It is required when + libpython is accessed directly rather than using header files which defines + macros below. On Windows, for example, PyAPI_FUNC() uses dllexport to + export functions in pythonXX.dll. */ +PyAPI_FUNC(PyObject *) PyRun_String(const char *str, int s, PyObject *g, PyObject *l); +PyAPI_FUNC(int) PyRun_AnyFile(FILE *fp, const char *name); +PyAPI_FUNC(int) PyRun_AnyFileEx(FILE *fp, const char *name, int closeit); +PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, const char *, PyCompilerFlags *); +PyAPI_FUNC(int) PyRun_SimpleString(const char *s); +PyAPI_FUNC(int) PyRun_SimpleFile(FILE *f, const char *p); +PyAPI_FUNC(int) PyRun_SimpleFileEx(FILE *f, const char *p, int c); +PyAPI_FUNC(int) PyRun_InteractiveOne(FILE *f, const char *p); +PyAPI_FUNC(int) PyRun_InteractiveLoop(FILE *f, const char *p); +PyAPI_FUNC(PyObject *) PyRun_File(FILE *fp, const char *p, int s, PyObject *g, PyObject *l); +PyAPI_FUNC(PyObject *) PyRun_FileEx(FILE *fp, const char *p, int s, PyObject *g, PyObject *l, int c); +PyAPI_FUNC(PyObject *) PyRun_FileFlags(FILE *fp, const char *p, int s, PyObject *g, PyObject *l, PyCompilerFlags *flags); + +/* Use macros for a bunch of old variants */ +#define PyRun_String(str, s, g, l) PyRun_StringFlags(str, s, g, l, NULL) +#define PyRun_AnyFile(fp, name) PyRun_AnyFileExFlags(fp, name, 0, NULL) +#define PyRun_AnyFileEx(fp, name, closeit) \ + PyRun_AnyFileExFlags(fp, name, closeit, NULL) +#define PyRun_AnyFileFlags(fp, name, flags) \ + PyRun_AnyFileExFlags(fp, name, 0, flags) +#define PyRun_SimpleString(s) PyRun_SimpleStringFlags(s, NULL) +#define PyRun_SimpleFile(f, p) PyRun_SimpleFileExFlags(f, p, 0, NULL) +#define PyRun_SimpleFileEx(f, p, c) PyRun_SimpleFileExFlags(f, p, c, NULL) +#define PyRun_InteractiveOne(f, p) PyRun_InteractiveOneFlags(f, p, NULL) +#define PyRun_InteractiveLoop(f, p) PyRun_InteractiveLoopFlags(f, p, NULL) +#define PyRun_File(fp, p, s, g, l) \ + PyRun_FileExFlags(fp, p, s, g, l, 0, NULL) +#define PyRun_FileEx(fp, p, s, g, l, c) \ + PyRun_FileExFlags(fp, p, s, g, l, c, NULL) +#define PyRun_FileFlags(fp, p, s, g, l, flags) \ + PyRun_FileExFlags(fp, p, s, g, l, 0, flags) + + +/* Stuff with no proper home (yet) */ +PyAPI_FUNC(char *) PyOS_Readline(FILE *, FILE *, const char *); +PyAPI_DATA(PyThreadState*) _PyOS_ReadlineTState; +PyAPI_DATA(char) *(*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *); diff --git a/Include/pytime.h b/Include/cpython/pytime.h similarity index 90% rename from Include/pytime.h rename to Include/cpython/pytime.h index bdda1da2e6b8f2..56607d199ed542 100644 --- a/Include/pytime.h +++ b/Include/cpython/pytime.h @@ -2,9 +2,6 @@ #ifndef Py_PYTIME_H #define Py_PYTIME_H -#include "pyconfig.h" /* include for defines */ -#include "object.h" - /************************************************************************** Symbols and macros to supply platform-independent interfaces to time related functions and constants @@ -164,22 +161,6 @@ PyAPI_FUNC(_PyTime_t) _PyTime_MulDiv(_PyTime_t ticks, _PyTime_t mul, _PyTime_t div); -/* Get the current time from the system clock. - - The function cannot fail. _PyTime_Init() ensures that the system clock - works. */ -PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void); - -/* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. - The clock is not affected by system clock updates. The reference point of - the returned value is undefined, so that only the difference between the - results of consecutive calls is valid. - - The function cannot fail. _PyTime_Init() ensures that a monotonic clock - is available and works. */ -PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); - - /* Structure used by time.get_clock_info() */ typedef struct { const char *implementation; @@ -189,13 +170,34 @@ typedef struct { } _Py_clock_info_t; /* Get the current time from the system clock. - * Fill clock information if info is not NULL. - * Raise an exception and return -1 on error, return 0 on success. + + If the internal clock fails, silently ignore the error and return 0. + On integer overflow, silently ignore the overflow and truncated the clock to + _PyTime_MIN or _PyTime_MAX. + + Use _PyTime_GetSystemClockWithInfo() to check for failure. */ +PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void); + +/* Get the current time from the system clock. + * On success, set *t and *info (if not NULL), and return 0. + * On error, raise an exception and return -1. */ PyAPI_FUNC(int) _PyTime_GetSystemClockWithInfo( _PyTime_t *t, _Py_clock_info_t *info); +/* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. + The clock is not affected by system clock updates. The reference point of + the returned value is undefined, so that only the difference between the + results of consecutive calls is valid. + + If the internal clock fails, silently ignore the error and return 0. + On integer overflow, silently ignore the overflow and truncated the clock to + _PyTime_MIN or _PyTime_MAX. + + Use _PyTime_GetMonotonicClockWithInfo() to check for failure. */ +PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); + /* Get the time of a monotonic clock, i.e. a clock that cannot go backwards. The clock is not affected by system clock updates. The reference point of the returned value is undefined, so that only the difference between the @@ -209,10 +211,6 @@ PyAPI_FUNC(int) _PyTime_GetMonotonicClockWithInfo( _Py_clock_info_t *info); -/* Initialize time. - Return 0 on success, raise an exception and return -1 on error. */ -PyAPI_FUNC(int) _PyTime_Init(void); - /* Converts a timestamp to the Gregorian time, using the local time zone. Return 0 on success, raise an exception and return -1 on error. */ PyAPI_FUNC(int) _PyTime_localtime(time_t t, struct tm *tm); @@ -224,8 +222,11 @@ PyAPI_FUNC(int) _PyTime_gmtime(time_t t, struct tm *tm); /* Get the performance counter: clock with the highest available resolution to measure a short duration. - The function cannot fail. _PyTime_Init() ensures that the system clock - works. */ + If the internal clock fails, silently ignore the error and return 0. + On integer overflow, silently ignore the overflow and truncated the clock to + _PyTime_MIN or _PyTime_MAX. + + Use _PyTime_GetPerfCounterWithInfo() to check for failure. */ PyAPI_FUNC(_PyTime_t) _PyTime_GetPerfCounter(void); /* Get the performance counter: clock with the highest available resolution to diff --git a/Include/cpython/tupleobject.h b/Include/cpython/tupleobject.h index 51dcd4237be18c..7cada8848c49fc 100644 --- a/Include/cpython/tupleobject.h +++ b/Include/cpython/tupleobject.h @@ -23,6 +23,6 @@ PyAPI_FUNC(void) _PyTuple_MaybeUntrack(PyObject *); #define PyTuple_GET_ITEM(op, i) (_PyTuple_CAST(op)->ob_item[i]) /* Macro, *only* to be used to fill in brand new tuples */ -#define PyTuple_SET_ITEM(op, i, v) (_PyTuple_CAST(op)->ob_item[i] = v) +#define PyTuple_SET_ITEM(op, i, v) ((void)(_PyTuple_CAST(op)->ob_item[i] = v)) PyAPI_FUNC(void) _PyTuple_DebugMallocStats(FILE *out); diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index f1b44554e3078d..30bf994cda35be 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -22,7 +22,7 @@ */ #define Py_UNICODE_ISSPACE(ch) \ - ((ch) < 128U ? _Py_ascii_whitespace[(ch)] : _PyUnicode_IsWhitespace(ch)) + ((Py_UCS4)(ch) < 128U ? _Py_ascii_whitespace[(ch)] : _PyUnicode_IsWhitespace(ch)) #define Py_UNICODE_ISLOWER(ch) _PyUnicode_IsLowercase(ch) #define Py_UNICODE_ISUPPER(ch) _PyUnicode_IsUppercase(ch) @@ -737,13 +737,6 @@ PyAPI_FUNC(int) _PyUnicode_FormatAdvancedWriter( Use of this API is DEPRECATED since no size information can be extracted from the returned data. - - *** This API is for interpreter INTERNAL USE ONLY and will likely - *** be removed or changed for Python 3.1. - - *** If you need to access the Unicode object as UTF-8 bytes string, - *** please use PyUnicode_AsUTF8String() instead. - */ PyAPI_FUNC(const char *) PyUnicode_AsUTF8(PyObject *unicode); diff --git a/Include/descrobject.h b/Include/descrobject.h index ead269d1d2f796..703bc8fd6df213 100644 --- a/Include/descrobject.h +++ b/Include/descrobject.h @@ -93,7 +93,7 @@ PyAPI_FUNC(PyObject *) PyDescr_NewGetSet(PyTypeObject *, #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) PyDescr_NewWrapper(PyTypeObject *, struct wrapperbase *, void *); -#define PyDescr_IsData(d) (Py_TYPE(d)->tp_descr_set != NULL) +PyAPI_FUNC(int) PyDescr_IsData(PyObject *); #endif PyAPI_FUNC(PyObject *) PyDictProxy_New(PyObject *); diff --git a/Include/errcode.h b/Include/errcode.h index 790518b8b7730e..f2671d6c9b30b4 100644 --- a/Include/errcode.h +++ b/Include/errcode.h @@ -26,8 +26,6 @@ extern "C" { #define E_TOODEEP 20 /* Too many indentation levels */ #define E_DEDENT 21 /* No matching outer block for dedent */ #define E_DECODE 22 /* Error in decoding into Unicode */ -#define E_EOFS 23 /* EOF in triple-quoted string */ -#define E_EOLS 24 /* EOL in single-quoted string */ #define E_LINECONT 25 /* Unexpected characters after a line continuation */ #define E_BADSINGLE 27 /* Ill-formed single statement input */ diff --git a/Include/eval.h b/Include/eval.h index 2c1c2d0549a9a4..eda28df8f65281 100644 --- a/Include/eval.h +++ b/Include/eval.h @@ -18,16 +18,6 @@ PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyObject *co, PyObject *kwdefs, PyObject *closure); #ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyEval_EvalCodeWithName( - PyObject *co, - PyObject *globals, PyObject *locals, - PyObject *const *args, Py_ssize_t argcount, - PyObject *const *kwnames, PyObject *const *kwargs, - Py_ssize_t kwcount, int kwstep, - PyObject *const *defs, Py_ssize_t defcount, - PyObject *kwdefs, PyObject *closure, - PyObject *name, PyObject *qualname); - PyAPI_FUNC(PyObject *) _PyEval_CallTracing(PyObject *func, PyObject *args); #endif diff --git a/Include/fileutils.h b/Include/fileutils.h index 12bd071c49c04a..16f3b635deed89 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -12,10 +12,6 @@ PyAPI_FUNC(wchar_t *) Py_DecodeLocale( PyAPI_FUNC(char*) Py_EncodeLocale( const wchar_t *text, size_t *error_pos); - -PyAPI_FUNC(char*) _Py_EncodeLocaleRaw( - const wchar_t *text, - size_t *error_pos); #endif #ifndef Py_LIMITED_API diff --git a/Include/funcobject.h b/Include/funcobject.h index c5cc9d261a314e..d7acd18c6519e4 100644 --- a/Include/funcobject.h +++ b/Include/funcobject.h @@ -7,6 +7,21 @@ extern "C" { #endif + +#define COMMON_FIELDS(PREFIX) \ + PyObject *PREFIX ## globals; \ + PyObject *PREFIX ## builtins; \ + PyObject *PREFIX ## name; \ + PyObject *PREFIX ## qualname; \ + PyObject *PREFIX ## code; /* A code object, the __code__ attribute */ \ + PyObject *PREFIX ## defaults; /* NULL or a tuple */ \ + PyObject *PREFIX ## kwdefaults; /* NULL or a dict */ \ + PyObject *PREFIX ## closure; /* NULL or a tuple of cell objects */ + +typedef struct { + COMMON_FIELDS(fc_) +} PyFrameConstructor; + /* Function objects and code objects should not be confused with each other: * * Function objects are created by the execution of the 'def' statement. @@ -20,18 +35,12 @@ extern "C" { typedef struct { PyObject_HEAD - PyObject *func_code; /* A code object, the __code__ attribute */ - PyObject *func_globals; /* A dictionary (other mappings won't do) */ - PyObject *func_defaults; /* NULL or a tuple */ - PyObject *func_kwdefaults; /* NULL or a dict */ - PyObject *func_closure; /* NULL or a tuple of cell objects */ + COMMON_FIELDS(func_) PyObject *func_doc; /* The __doc__ attribute, can be anything */ - PyObject *func_name; /* The __name__ attribute, a string object */ PyObject *func_dict; /* The __dict__ attribute, a dict or NULL */ PyObject *func_weakreflist; /* List of weak references */ PyObject *func_module; /* The __module__ attribute, can be anything */ PyObject *func_annotations; /* Annotations, a dict or NULL */ - PyObject *func_qualname; /* The qualified name */ vectorcallfunc vectorcall; /* Invariant: @@ -84,6 +93,9 @@ PyAPI_FUNC(PyObject *) _PyFunction_Vectorcall( #define PyFunction_GET_ANNOTATIONS(func) \ (((PyFunctionObject *)func) -> func_annotations) +#define PyFunction_AS_FRAME_CONSTRUCTOR(func) \ + ((PyFrameConstructor *)&((PyFunctionObject *)(func))->func_globals) + /* The classmethod and staticmethod types lives here, too */ PyAPI_DATA(PyTypeObject) PyClassMethod_Type; PyAPI_DATA(PyTypeObject) PyStaticMethod_Type; diff --git a/Include/asdl.h b/Include/internal/pycore_asdl.h similarity index 81% rename from Include/asdl.h rename to Include/internal/pycore_asdl.h index 8b61e16c329ea9..c0b07c31810b99 100644 --- a/Include/asdl.h +++ b/Include/internal/pycore_asdl.h @@ -1,6 +1,14 @@ -#ifndef Py_LIMITED_API -#ifndef Py_ASDL_H -#define Py_ASDL_H +#ifndef Py_INTERNAL_ASDL_H +#define Py_INTERNAL_ASDL_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_pyarena.h" // _PyArena_Malloc() typedef PyObject * identifier; typedef PyObject * string; @@ -59,7 +67,7 @@ asdl_ ## NAME ## _seq *_Py_asdl_ ## NAME ## _seq_new(Py_ssize_t size, PyArena *a return NULL; \ } \ n += sizeof(asdl_ ## NAME ## _seq); \ - seq = (asdl_ ## NAME ## _seq *)PyArena_Malloc(arena, n); \ + seq = (asdl_ ## NAME ## _seq *)_PyArena_Malloc(arena, n); \ if (!seq) { \ PyErr_NoMemory(); \ return NULL; \ @@ -73,8 +81,9 @@ asdl_ ## NAME ## _seq *_Py_asdl_ ## NAME ## _seq_new(Py_ssize_t size, PyArena *a #define asdl_seq_GET_UNTYPED(S, I) (S)->elements[(I)] #define asdl_seq_GET(S, I) (S)->typed_elements[(I)] #define asdl_seq_LEN(S) ((S) == NULL ? 0 : (S)->size) + #ifdef Py_DEBUG -#define asdl_seq_SET(S, I, V) \ +# define asdl_seq_SET(S, I, V) \ do { \ Py_ssize_t _asdl_i = (I); \ assert((S) != NULL); \ @@ -82,11 +91,11 @@ asdl_ ## NAME ## _seq *_Py_asdl_ ## NAME ## _seq_new(Py_ssize_t size, PyArena *a (S)->typed_elements[_asdl_i] = (V); \ } while (0) #else -#define asdl_seq_SET(S, I, V) (S)->typed_elements[I] = (V) +# define asdl_seq_SET(S, I, V) (S)->typed_elements[I] = (V) #endif #ifdef Py_DEBUG -#define asdl_seq_SET_UNTYPED(S, I, V) \ +# define asdl_seq_SET_UNTYPED(S, I, V) \ do { \ Py_ssize_t _asdl_i = (I); \ assert((S) != NULL); \ @@ -94,8 +103,10 @@ asdl_ ## NAME ## _seq *_Py_asdl_ ## NAME ## _seq_new(Py_ssize_t size, PyArena *a (S)->elements[_asdl_i] = (V); \ } while (0) #else -#define asdl_seq_SET_UNTYPED(S, I, V) (S)->elements[I] = (V) +# define asdl_seq_SET_UNTYPED(S, I, V) (S)->elements[I] = (V) #endif -#endif /* !Py_ASDL_H */ -#endif /* Py_LIMITED_API */ +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_ASDL_H */ diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h new file mode 100644 index 00000000000000..ebb6a90087bb52 --- /dev/null +++ b/Include/internal/pycore_ast.h @@ -0,0 +1,855 @@ +// File automatically generated by Parser/asdl_c.py. + +#ifndef Py_INTERNAL_AST_H +#define Py_INTERNAL_AST_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_asdl.h" + +typedef struct _mod *mod_ty; + +typedef struct _stmt *stmt_ty; + +typedef struct _expr *expr_ty; + +typedef enum _expr_context { Load=1, Store=2, Del=3 } expr_context_ty; + +typedef enum _boolop { And=1, Or=2 } boolop_ty; + +typedef enum _operator { Add=1, Sub=2, Mult=3, MatMult=4, Div=5, Mod=6, Pow=7, + LShift=8, RShift=9, BitOr=10, BitXor=11, BitAnd=12, + FloorDiv=13 } operator_ty; + +typedef enum _unaryop { Invert=1, Not=2, UAdd=3, USub=4 } unaryop_ty; + +typedef enum _cmpop { Eq=1, NotEq=2, Lt=3, LtE=4, Gt=5, GtE=6, Is=7, IsNot=8, + In=9, NotIn=10 } cmpop_ty; + +typedef struct _comprehension *comprehension_ty; + +typedef struct _excepthandler *excepthandler_ty; + +typedef struct _arguments *arguments_ty; + +typedef struct _arg *arg_ty; + +typedef struct _keyword *keyword_ty; + +typedef struct _alias *alias_ty; + +typedef struct _withitem *withitem_ty; + +typedef struct _match_case *match_case_ty; + +typedef struct _pattern *pattern_ty; + +typedef struct _type_ignore *type_ignore_ty; + + +typedef struct { + _ASDL_SEQ_HEAD + mod_ty typed_elements[1]; +} asdl_mod_seq; + +asdl_mod_seq *_Py_asdl_mod_seq_new(Py_ssize_t size, PyArena *arena); + +typedef struct { + _ASDL_SEQ_HEAD + stmt_ty typed_elements[1]; +} asdl_stmt_seq; + +asdl_stmt_seq *_Py_asdl_stmt_seq_new(Py_ssize_t size, PyArena *arena); + +typedef struct { + _ASDL_SEQ_HEAD + expr_ty typed_elements[1]; +} asdl_expr_seq; + +asdl_expr_seq *_Py_asdl_expr_seq_new(Py_ssize_t size, PyArena *arena); + +typedef struct { + _ASDL_SEQ_HEAD + comprehension_ty typed_elements[1]; +} asdl_comprehension_seq; + +asdl_comprehension_seq *_Py_asdl_comprehension_seq_new(Py_ssize_t size, PyArena + *arena); + +typedef struct { + _ASDL_SEQ_HEAD + excepthandler_ty typed_elements[1]; +} asdl_excepthandler_seq; + +asdl_excepthandler_seq *_Py_asdl_excepthandler_seq_new(Py_ssize_t size, PyArena + *arena); + +typedef struct { + _ASDL_SEQ_HEAD + arguments_ty typed_elements[1]; +} asdl_arguments_seq; + +asdl_arguments_seq *_Py_asdl_arguments_seq_new(Py_ssize_t size, PyArena *arena); + +typedef struct { + _ASDL_SEQ_HEAD + arg_ty typed_elements[1]; +} asdl_arg_seq; + +asdl_arg_seq *_Py_asdl_arg_seq_new(Py_ssize_t size, PyArena *arena); + +typedef struct { + _ASDL_SEQ_HEAD + keyword_ty typed_elements[1]; +} asdl_keyword_seq; + +asdl_keyword_seq *_Py_asdl_keyword_seq_new(Py_ssize_t size, PyArena *arena); + +typedef struct { + _ASDL_SEQ_HEAD + alias_ty typed_elements[1]; +} asdl_alias_seq; + +asdl_alias_seq *_Py_asdl_alias_seq_new(Py_ssize_t size, PyArena *arena); + +typedef struct { + _ASDL_SEQ_HEAD + withitem_ty typed_elements[1]; +} asdl_withitem_seq; + +asdl_withitem_seq *_Py_asdl_withitem_seq_new(Py_ssize_t size, PyArena *arena); + +typedef struct { + _ASDL_SEQ_HEAD + match_case_ty typed_elements[1]; +} asdl_match_case_seq; + +asdl_match_case_seq *_Py_asdl_match_case_seq_new(Py_ssize_t size, PyArena + *arena); + +typedef struct { + _ASDL_SEQ_HEAD + pattern_ty typed_elements[1]; +} asdl_pattern_seq; + +asdl_pattern_seq *_Py_asdl_pattern_seq_new(Py_ssize_t size, PyArena *arena); + +typedef struct { + _ASDL_SEQ_HEAD + type_ignore_ty typed_elements[1]; +} asdl_type_ignore_seq; + +asdl_type_ignore_seq *_Py_asdl_type_ignore_seq_new(Py_ssize_t size, PyArena + *arena); + + +enum _mod_kind {Module_kind=1, Interactive_kind=2, Expression_kind=3, + FunctionType_kind=4}; +struct _mod { + enum _mod_kind kind; + union { + struct { + asdl_stmt_seq *body; + asdl_type_ignore_seq *type_ignores; + } Module; + + struct { + asdl_stmt_seq *body; + } Interactive; + + struct { + expr_ty body; + } Expression; + + struct { + asdl_expr_seq *argtypes; + expr_ty returns; + } FunctionType; + + } v; +}; + +enum _stmt_kind {FunctionDef_kind=1, AsyncFunctionDef_kind=2, ClassDef_kind=3, + Return_kind=4, Delete_kind=5, Assign_kind=6, + AugAssign_kind=7, AnnAssign_kind=8, For_kind=9, + AsyncFor_kind=10, While_kind=11, If_kind=12, With_kind=13, + AsyncWith_kind=14, Match_kind=15, Raise_kind=16, Try_kind=17, + Assert_kind=18, Import_kind=19, ImportFrom_kind=20, + Global_kind=21, Nonlocal_kind=22, Expr_kind=23, Pass_kind=24, + Break_kind=25, Continue_kind=26}; +struct _stmt { + enum _stmt_kind kind; + union { + struct { + identifier name; + arguments_ty args; + asdl_stmt_seq *body; + asdl_expr_seq *decorator_list; + expr_ty returns; + string type_comment; + } FunctionDef; + + struct { + identifier name; + arguments_ty args; + asdl_stmt_seq *body; + asdl_expr_seq *decorator_list; + expr_ty returns; + string type_comment; + } AsyncFunctionDef; + + struct { + identifier name; + asdl_expr_seq *bases; + asdl_keyword_seq *keywords; + asdl_stmt_seq *body; + asdl_expr_seq *decorator_list; + } ClassDef; + + struct { + expr_ty value; + } Return; + + struct { + asdl_expr_seq *targets; + } Delete; + + struct { + asdl_expr_seq *targets; + expr_ty value; + string type_comment; + } Assign; + + struct { + expr_ty target; + operator_ty op; + expr_ty value; + } AugAssign; + + struct { + expr_ty target; + expr_ty annotation; + expr_ty value; + int simple; + } AnnAssign; + + struct { + expr_ty target; + expr_ty iter; + asdl_stmt_seq *body; + asdl_stmt_seq *orelse; + string type_comment; + } For; + + struct { + expr_ty target; + expr_ty iter; + asdl_stmt_seq *body; + asdl_stmt_seq *orelse; + string type_comment; + } AsyncFor; + + struct { + expr_ty test; + asdl_stmt_seq *body; + asdl_stmt_seq *orelse; + } While; + + struct { + expr_ty test; + asdl_stmt_seq *body; + asdl_stmt_seq *orelse; + } If; + + struct { + asdl_withitem_seq *items; + asdl_stmt_seq *body; + string type_comment; + } With; + + struct { + asdl_withitem_seq *items; + asdl_stmt_seq *body; + string type_comment; + } AsyncWith; + + struct { + expr_ty subject; + asdl_match_case_seq *cases; + } Match; + + struct { + expr_ty exc; + expr_ty cause; + } Raise; + + struct { + asdl_stmt_seq *body; + asdl_excepthandler_seq *handlers; + asdl_stmt_seq *orelse; + asdl_stmt_seq *finalbody; + } Try; + + struct { + expr_ty test; + expr_ty msg; + } Assert; + + struct { + asdl_alias_seq *names; + } Import; + + struct { + identifier module; + asdl_alias_seq *names; + int level; + } ImportFrom; + + struct { + asdl_identifier_seq *names; + } Global; + + struct { + asdl_identifier_seq *names; + } Nonlocal; + + struct { + expr_ty value; + } Expr; + + } v; + int lineno; + int col_offset; + int end_lineno; + int end_col_offset; +}; + +enum _expr_kind {BoolOp_kind=1, NamedExpr_kind=2, BinOp_kind=3, UnaryOp_kind=4, + Lambda_kind=5, IfExp_kind=6, Dict_kind=7, Set_kind=8, + ListComp_kind=9, SetComp_kind=10, DictComp_kind=11, + GeneratorExp_kind=12, Await_kind=13, Yield_kind=14, + YieldFrom_kind=15, Compare_kind=16, Call_kind=17, + FormattedValue_kind=18, JoinedStr_kind=19, Constant_kind=20, + Attribute_kind=21, Subscript_kind=22, Starred_kind=23, + Name_kind=24, List_kind=25, Tuple_kind=26, Slice_kind=27}; +struct _expr { + enum _expr_kind kind; + union { + struct { + boolop_ty op; + asdl_expr_seq *values; + } BoolOp; + + struct { + expr_ty target; + expr_ty value; + } NamedExpr; + + struct { + expr_ty left; + operator_ty op; + expr_ty right; + } BinOp; + + struct { + unaryop_ty op; + expr_ty operand; + } UnaryOp; + + struct { + arguments_ty args; + expr_ty body; + } Lambda; + + struct { + expr_ty test; + expr_ty body; + expr_ty orelse; + } IfExp; + + struct { + asdl_expr_seq *keys; + asdl_expr_seq *values; + } Dict; + + struct { + asdl_expr_seq *elts; + } Set; + + struct { + expr_ty elt; + asdl_comprehension_seq *generators; + } ListComp; + + struct { + expr_ty elt; + asdl_comprehension_seq *generators; + } SetComp; + + struct { + expr_ty key; + expr_ty value; + asdl_comprehension_seq *generators; + } DictComp; + + struct { + expr_ty elt; + asdl_comprehension_seq *generators; + } GeneratorExp; + + struct { + expr_ty value; + } Await; + + struct { + expr_ty value; + } Yield; + + struct { + expr_ty value; + } YieldFrom; + + struct { + expr_ty left; + asdl_int_seq *ops; + asdl_expr_seq *comparators; + } Compare; + + struct { + expr_ty func; + asdl_expr_seq *args; + asdl_keyword_seq *keywords; + } Call; + + struct { + expr_ty value; + int conversion; + expr_ty format_spec; + } FormattedValue; + + struct { + asdl_expr_seq *values; + } JoinedStr; + + struct { + constant value; + string kind; + } Constant; + + struct { + expr_ty value; + identifier attr; + expr_context_ty ctx; + } Attribute; + + struct { + expr_ty value; + expr_ty slice; + expr_context_ty ctx; + } Subscript; + + struct { + expr_ty value; + expr_context_ty ctx; + } Starred; + + struct { + identifier id; + expr_context_ty ctx; + } Name; + + struct { + asdl_expr_seq *elts; + expr_context_ty ctx; + } List; + + struct { + asdl_expr_seq *elts; + expr_context_ty ctx; + } Tuple; + + struct { + expr_ty lower; + expr_ty upper; + expr_ty step; + } Slice; + + } v; + int lineno; + int col_offset; + int end_lineno; + int end_col_offset; +}; + +struct _comprehension { + expr_ty target; + expr_ty iter; + asdl_expr_seq *ifs; + int is_async; +}; + +enum _excepthandler_kind {ExceptHandler_kind=1}; +struct _excepthandler { + enum _excepthandler_kind kind; + union { + struct { + expr_ty type; + identifier name; + asdl_stmt_seq *body; + } ExceptHandler; + + } v; + int lineno; + int col_offset; + int end_lineno; + int end_col_offset; +}; + +struct _arguments { + asdl_arg_seq *posonlyargs; + asdl_arg_seq *args; + arg_ty vararg; + asdl_arg_seq *kwonlyargs; + asdl_expr_seq *kw_defaults; + arg_ty kwarg; + asdl_expr_seq *defaults; +}; + +struct _arg { + identifier arg; + expr_ty annotation; + string type_comment; + int lineno; + int col_offset; + int end_lineno; + int end_col_offset; +}; + +struct _keyword { + identifier arg; + expr_ty value; + int lineno; + int col_offset; + int end_lineno; + int end_col_offset; +}; + +struct _alias { + identifier name; + identifier asname; + int lineno; + int col_offset; + int end_lineno; + int end_col_offset; +}; + +struct _withitem { + expr_ty context_expr; + expr_ty optional_vars; +}; + +struct _match_case { + pattern_ty pattern; + expr_ty guard; + asdl_stmt_seq *body; +}; + +enum _pattern_kind {MatchValue_kind=1, MatchSingleton_kind=2, + MatchSequence_kind=3, MatchMapping_kind=4, + MatchClass_kind=5, MatchStar_kind=6, MatchAs_kind=7, + MatchOr_kind=8}; +struct _pattern { + enum _pattern_kind kind; + union { + struct { + expr_ty value; + } MatchValue; + + struct { + constant value; + } MatchSingleton; + + struct { + asdl_pattern_seq *patterns; + } MatchSequence; + + struct { + asdl_expr_seq *keys; + asdl_pattern_seq *patterns; + identifier rest; + } MatchMapping; + + struct { + expr_ty cls; + asdl_pattern_seq *patterns; + asdl_identifier_seq *kwd_attrs; + asdl_pattern_seq *kwd_patterns; + } MatchClass; + + struct { + identifier name; + } MatchStar; + + struct { + pattern_ty pattern; + identifier name; + } MatchAs; + + struct { + asdl_pattern_seq *patterns; + } MatchOr; + + } v; + int lineno; + int col_offset; + int end_lineno; + int end_col_offset; +}; + +enum _type_ignore_kind {TypeIgnore_kind=1}; +struct _type_ignore { + enum _type_ignore_kind kind; + union { + struct { + int lineno; + string tag; + } TypeIgnore; + + } v; +}; + + +// Note: these macros affect function definitions, not only call sites. +mod_ty _PyAST_Module(asdl_stmt_seq * body, asdl_type_ignore_seq * type_ignores, + PyArena *arena); +mod_ty _PyAST_Interactive(asdl_stmt_seq * body, PyArena *arena); +mod_ty _PyAST_Expression(expr_ty body, PyArena *arena); +mod_ty _PyAST_FunctionType(asdl_expr_seq * argtypes, expr_ty returns, PyArena + *arena); +stmt_ty _PyAST_FunctionDef(identifier name, arguments_ty args, asdl_stmt_seq * + body, asdl_expr_seq * decorator_list, expr_ty + returns, string type_comment, int lineno, int + col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +stmt_ty _PyAST_AsyncFunctionDef(identifier name, arguments_ty args, + asdl_stmt_seq * body, asdl_expr_seq * + decorator_list, expr_ty returns, string + type_comment, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +stmt_ty _PyAST_ClassDef(identifier name, asdl_expr_seq * bases, + asdl_keyword_seq * keywords, asdl_stmt_seq * body, + asdl_expr_seq * decorator_list, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +stmt_ty _PyAST_Return(expr_ty value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +stmt_ty _PyAST_Delete(asdl_expr_seq * targets, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +stmt_ty _PyAST_Assign(asdl_expr_seq * targets, expr_ty value, string + type_comment, int lineno, int col_offset, int end_lineno, + int end_col_offset, PyArena *arena); +stmt_ty _PyAST_AugAssign(expr_ty target, operator_ty op, expr_ty value, int + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +stmt_ty _PyAST_AnnAssign(expr_ty target, expr_ty annotation, expr_ty value, int + simple, int lineno, int col_offset, int end_lineno, + int end_col_offset, PyArena *arena); +stmt_ty _PyAST_For(expr_ty target, expr_ty iter, asdl_stmt_seq * body, + asdl_stmt_seq * orelse, string type_comment, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +stmt_ty _PyAST_AsyncFor(expr_ty target, expr_ty iter, asdl_stmt_seq * body, + asdl_stmt_seq * orelse, string type_comment, int + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +stmt_ty _PyAST_While(expr_ty test, asdl_stmt_seq * body, asdl_stmt_seq * + orelse, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +stmt_ty _PyAST_If(expr_ty test, asdl_stmt_seq * body, asdl_stmt_seq * orelse, + int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +stmt_ty _PyAST_With(asdl_withitem_seq * items, asdl_stmt_seq * body, string + type_comment, int lineno, int col_offset, int end_lineno, + int end_col_offset, PyArena *arena); +stmt_ty _PyAST_AsyncWith(asdl_withitem_seq * items, asdl_stmt_seq * body, + string type_comment, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +stmt_ty _PyAST_Match(expr_ty subject, asdl_match_case_seq * cases, int lineno, + int col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +stmt_ty _PyAST_Raise(expr_ty exc, expr_ty cause, int lineno, int col_offset, + int end_lineno, int end_col_offset, PyArena *arena); +stmt_ty _PyAST_Try(asdl_stmt_seq * body, asdl_excepthandler_seq * handlers, + asdl_stmt_seq * orelse, asdl_stmt_seq * finalbody, int + lineno, int col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +stmt_ty _PyAST_Assert(expr_ty test, expr_ty msg, int lineno, int col_offset, + int end_lineno, int end_col_offset, PyArena *arena); +stmt_ty _PyAST_Import(asdl_alias_seq * names, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +stmt_ty _PyAST_ImportFrom(identifier module, asdl_alias_seq * names, int level, + int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +stmt_ty _PyAST_Global(asdl_identifier_seq * names, int lineno, int col_offset, + int end_lineno, int end_col_offset, PyArena *arena); +stmt_ty _PyAST_Nonlocal(asdl_identifier_seq * names, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +stmt_ty _PyAST_Expr(expr_ty value, int lineno, int col_offset, int end_lineno, + int end_col_offset, PyArena *arena); +stmt_ty _PyAST_Pass(int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +stmt_ty _PyAST_Break(int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +stmt_ty _PyAST_Continue(int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +expr_ty _PyAST_BoolOp(boolop_ty op, asdl_expr_seq * values, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +expr_ty _PyAST_NamedExpr(expr_ty target, expr_ty value, int lineno, int + col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +expr_ty _PyAST_BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, + int col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +expr_ty _PyAST_UnaryOp(unaryop_ty op, expr_ty operand, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +expr_ty _PyAST_Lambda(arguments_ty args, expr_ty body, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +expr_ty _PyAST_IfExp(expr_ty test, expr_ty body, expr_ty orelse, int lineno, + int col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +expr_ty _PyAST_Dict(asdl_expr_seq * keys, asdl_expr_seq * values, int lineno, + int col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +expr_ty _PyAST_Set(asdl_expr_seq * elts, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +expr_ty _PyAST_ListComp(expr_ty elt, asdl_comprehension_seq * generators, int + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +expr_ty _PyAST_SetComp(expr_ty elt, asdl_comprehension_seq * generators, int + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +expr_ty _PyAST_DictComp(expr_ty key, expr_ty value, asdl_comprehension_seq * + generators, int lineno, int col_offset, int end_lineno, + int end_col_offset, PyArena *arena); +expr_ty _PyAST_GeneratorExp(expr_ty elt, asdl_comprehension_seq * generators, + int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +expr_ty _PyAST_Await(expr_ty value, int lineno, int col_offset, int end_lineno, + int end_col_offset, PyArena *arena); +expr_ty _PyAST_Yield(expr_ty value, int lineno, int col_offset, int end_lineno, + int end_col_offset, PyArena *arena); +expr_ty _PyAST_YieldFrom(expr_ty value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +expr_ty _PyAST_Compare(expr_ty left, asdl_int_seq * ops, asdl_expr_seq * + comparators, int lineno, int col_offset, int end_lineno, + int end_col_offset, PyArena *arena); +expr_ty _PyAST_Call(expr_ty func, asdl_expr_seq * args, asdl_keyword_seq * + keywords, int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +expr_ty _PyAST_FormattedValue(expr_ty value, int conversion, expr_ty + format_spec, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +expr_ty _PyAST_JoinedStr(asdl_expr_seq * values, int lineno, int col_offset, + int end_lineno, int end_col_offset, PyArena *arena); +expr_ty _PyAST_Constant(constant value, string kind, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +expr_ty _PyAST_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, + int lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +expr_ty _PyAST_Subscript(expr_ty value, expr_ty slice, expr_context_ty ctx, int + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +expr_ty _PyAST_Starred(expr_ty value, expr_context_ty ctx, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +expr_ty _PyAST_Name(identifier id, expr_context_ty ctx, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +expr_ty _PyAST_List(asdl_expr_seq * elts, expr_context_ty ctx, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +expr_ty _PyAST_Tuple(asdl_expr_seq * elts, expr_context_ty ctx, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +expr_ty _PyAST_Slice(expr_ty lower, expr_ty upper, expr_ty step, int lineno, + int col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +comprehension_ty _PyAST_comprehension(expr_ty target, expr_ty iter, + asdl_expr_seq * ifs, int is_async, + PyArena *arena); +excepthandler_ty _PyAST_ExceptHandler(expr_ty type, identifier name, + asdl_stmt_seq * body, int lineno, int + col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +arguments_ty _PyAST_arguments(asdl_arg_seq * posonlyargs, asdl_arg_seq * args, + arg_ty vararg, asdl_arg_seq * kwonlyargs, + asdl_expr_seq * kw_defaults, arg_ty kwarg, + asdl_expr_seq * defaults, PyArena *arena); +arg_ty _PyAST_arg(identifier arg, expr_ty annotation, string type_comment, int + lineno, int col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +keyword_ty _PyAST_keyword(identifier arg, expr_ty value, int lineno, int + col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +alias_ty _PyAST_alias(identifier name, identifier asname, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena); +withitem_ty _PyAST_withitem(expr_ty context_expr, expr_ty optional_vars, + PyArena *arena); +match_case_ty _PyAST_match_case(pattern_ty pattern, expr_ty guard, + asdl_stmt_seq * body, PyArena *arena); +pattern_ty _PyAST_MatchValue(expr_ty value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +pattern_ty _PyAST_MatchSingleton(constant value, int lineno, int col_offset, + int end_lineno, int end_col_offset, PyArena + *arena); +pattern_ty _PyAST_MatchSequence(asdl_pattern_seq * patterns, int lineno, int + col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +pattern_ty _PyAST_MatchMapping(asdl_expr_seq * keys, asdl_pattern_seq * + patterns, identifier rest, int lineno, int + col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +pattern_ty _PyAST_MatchClass(expr_ty cls, asdl_pattern_seq * patterns, + asdl_identifier_seq * kwd_attrs, asdl_pattern_seq + * kwd_patterns, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +pattern_ty _PyAST_MatchStar(identifier name, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +pattern_ty _PyAST_MatchAs(pattern_ty pattern, identifier name, int lineno, int + col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +pattern_ty _PyAST_MatchOr(asdl_pattern_seq * patterns, int lineno, int + col_offset, int end_lineno, int end_col_offset, + PyArena *arena); +type_ignore_ty _PyAST_TypeIgnore(int lineno, string tag, PyArena *arena); + + +PyObject* PyAST_mod2obj(mod_ty t); +mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode); +int PyAST_Check(PyObject* obj); + +extern int _PyAST_Validate(mod_ty); + +/* _PyAST_ExprAsUnicode is defined in ast_unparse.c */ +extern PyObject* _PyAST_ExprAsUnicode(expr_ty); + +/* Return the borrowed reference to the first literal string in the + sequence of statements or NULL if it doesn't start from a literal string. + Doesn't set exception. */ +extern PyObject* _PyAST_GetDocString(asdl_stmt_seq *); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_AST_H */ diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h new file mode 100644 index 00000000000000..882cd09c00628d --- /dev/null +++ b/Include/internal/pycore_ast_state.h @@ -0,0 +1,255 @@ +// File automatically generated by Parser/asdl_c.py. + +#ifndef Py_INTERNAL_AST_STATE_H +#define Py_INTERNAL_AST_STATE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +struct ast_state { + int initialized; + PyObject *AST_type; + PyObject *Add_singleton; + PyObject *Add_type; + PyObject *And_singleton; + PyObject *And_type; + PyObject *AnnAssign_type; + PyObject *Assert_type; + PyObject *Assign_type; + PyObject *AsyncFor_type; + PyObject *AsyncFunctionDef_type; + PyObject *AsyncWith_type; + PyObject *Attribute_type; + PyObject *AugAssign_type; + PyObject *Await_type; + PyObject *BinOp_type; + PyObject *BitAnd_singleton; + PyObject *BitAnd_type; + PyObject *BitOr_singleton; + PyObject *BitOr_type; + PyObject *BitXor_singleton; + PyObject *BitXor_type; + PyObject *BoolOp_type; + PyObject *Break_type; + PyObject *Call_type; + PyObject *ClassDef_type; + PyObject *Compare_type; + PyObject *Constant_type; + PyObject *Continue_type; + PyObject *Del_singleton; + PyObject *Del_type; + PyObject *Delete_type; + PyObject *DictComp_type; + PyObject *Dict_type; + PyObject *Div_singleton; + PyObject *Div_type; + PyObject *Eq_singleton; + PyObject *Eq_type; + PyObject *ExceptHandler_type; + PyObject *Expr_type; + PyObject *Expression_type; + PyObject *FloorDiv_singleton; + PyObject *FloorDiv_type; + PyObject *For_type; + PyObject *FormattedValue_type; + PyObject *FunctionDef_type; + PyObject *FunctionType_type; + PyObject *GeneratorExp_type; + PyObject *Global_type; + PyObject *GtE_singleton; + PyObject *GtE_type; + PyObject *Gt_singleton; + PyObject *Gt_type; + PyObject *IfExp_type; + PyObject *If_type; + PyObject *ImportFrom_type; + PyObject *Import_type; + PyObject *In_singleton; + PyObject *In_type; + PyObject *Interactive_type; + PyObject *Invert_singleton; + PyObject *Invert_type; + PyObject *IsNot_singleton; + PyObject *IsNot_type; + PyObject *Is_singleton; + PyObject *Is_type; + PyObject *JoinedStr_type; + PyObject *LShift_singleton; + PyObject *LShift_type; + PyObject *Lambda_type; + PyObject *ListComp_type; + PyObject *List_type; + PyObject *Load_singleton; + PyObject *Load_type; + PyObject *LtE_singleton; + PyObject *LtE_type; + PyObject *Lt_singleton; + PyObject *Lt_type; + PyObject *MatMult_singleton; + PyObject *MatMult_type; + PyObject *MatchAs_type; + PyObject *MatchClass_type; + PyObject *MatchMapping_type; + PyObject *MatchOr_type; + PyObject *MatchSequence_type; + PyObject *MatchSingleton_type; + PyObject *MatchStar_type; + PyObject *MatchValue_type; + PyObject *Match_type; + PyObject *Mod_singleton; + PyObject *Mod_type; + PyObject *Module_type; + PyObject *Mult_singleton; + PyObject *Mult_type; + PyObject *Name_type; + PyObject *NamedExpr_type; + PyObject *Nonlocal_type; + PyObject *NotEq_singleton; + PyObject *NotEq_type; + PyObject *NotIn_singleton; + PyObject *NotIn_type; + PyObject *Not_singleton; + PyObject *Not_type; + PyObject *Or_singleton; + PyObject *Or_type; + PyObject *Pass_type; + PyObject *Pow_singleton; + PyObject *Pow_type; + PyObject *RShift_singleton; + PyObject *RShift_type; + PyObject *Raise_type; + PyObject *Return_type; + PyObject *SetComp_type; + PyObject *Set_type; + PyObject *Slice_type; + PyObject *Starred_type; + PyObject *Store_singleton; + PyObject *Store_type; + PyObject *Sub_singleton; + PyObject *Sub_type; + PyObject *Subscript_type; + PyObject *Try_type; + PyObject *Tuple_type; + PyObject *TypeIgnore_type; + PyObject *UAdd_singleton; + PyObject *UAdd_type; + PyObject *USub_singleton; + PyObject *USub_type; + PyObject *UnaryOp_type; + PyObject *While_type; + PyObject *With_type; + PyObject *YieldFrom_type; + PyObject *Yield_type; + PyObject *__dict__; + PyObject *__doc__; + PyObject *__match_args__; + PyObject *__module__; + PyObject *_attributes; + PyObject *_fields; + PyObject *alias_type; + PyObject *annotation; + PyObject *arg; + PyObject *arg_type; + PyObject *args; + PyObject *argtypes; + PyObject *arguments_type; + PyObject *asname; + PyObject *ast; + PyObject *attr; + PyObject *bases; + PyObject *body; + PyObject *boolop_type; + PyObject *cases; + PyObject *cause; + PyObject *cls; + PyObject *cmpop_type; + PyObject *col_offset; + PyObject *comparators; + PyObject *comprehension_type; + PyObject *context_expr; + PyObject *conversion; + PyObject *ctx; + PyObject *decorator_list; + PyObject *defaults; + PyObject *elt; + PyObject *elts; + PyObject *end_col_offset; + PyObject *end_lineno; + PyObject *exc; + PyObject *excepthandler_type; + PyObject *expr_context_type; + PyObject *expr_type; + PyObject *finalbody; + PyObject *format_spec; + PyObject *func; + PyObject *generators; + PyObject *guard; + PyObject *handlers; + PyObject *id; + PyObject *ifs; + PyObject *is_async; + PyObject *items; + PyObject *iter; + PyObject *key; + PyObject *keys; + PyObject *keyword_type; + PyObject *keywords; + PyObject *kind; + PyObject *kw_defaults; + PyObject *kwarg; + PyObject *kwd_attrs; + PyObject *kwd_patterns; + PyObject *kwonlyargs; + PyObject *left; + PyObject *level; + PyObject *lineno; + PyObject *lower; + PyObject *match_case_type; + PyObject *mod_type; + PyObject *module; + PyObject *msg; + PyObject *name; + PyObject *names; + PyObject *op; + PyObject *operand; + PyObject *operator_type; + PyObject *ops; + PyObject *optional_vars; + PyObject *orelse; + PyObject *pattern; + PyObject *pattern_type; + PyObject *patterns; + PyObject *posonlyargs; + PyObject *rest; + PyObject *returns; + PyObject *right; + PyObject *simple; + PyObject *slice; + PyObject *step; + PyObject *stmt_type; + PyObject *subject; + PyObject *tag; + PyObject *target; + PyObject *targets; + PyObject *test; + PyObject *type; + PyObject *type_comment; + PyObject *type_ignore_type; + PyObject *type_ignores; + PyObject *unaryop_type; + PyObject *upper; + PyObject *value; + PyObject *values; + PyObject *vararg; + PyObject *withitem_type; +}; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_AST_STATE_H */ + diff --git a/Include/internal/pycore_atomic.h b/Include/internal/pycore_atomic.h index 1d5c5621677eb3..3d42e54464c4c7 100644 --- a/Include/internal/pycore_atomic.h +++ b/Include/internal/pycore_atomic.h @@ -11,8 +11,8 @@ extern "C" { #include "dynamic_annotations.h" /* _Py_ANNOTATE_MEMORY_ORDER */ #include "pyconfig.h" -#if defined(HAVE_STD_ATOMIC) -#include +#ifdef HAVE_STD_ATOMIC +# include #endif @@ -62,7 +62,7 @@ typedef struct _Py_atomic_int { #define _Py_atomic_load_explicit(ATOMIC_VAL, ORDER) \ atomic_load_explicit(&((ATOMIC_VAL)->_value), ORDER) -/* Use builtin atomic operations in GCC >= 4.7 */ +// Use builtin atomic operations in GCC >= 4.7 and clang #elif defined(HAVE_BUILTIN_ATOMIC) typedef enum _Py_memory_order { diff --git a/Include/internal/pycore_atomic_funcs.h b/Include/internal/pycore_atomic_funcs.h new file mode 100644 index 00000000000000..a708789cea733b --- /dev/null +++ b/Include/internal/pycore_atomic_funcs.h @@ -0,0 +1,94 @@ +/* Atomic functions: similar to pycore_atomic.h, but don't need + to declare variables as atomic. + + Py_ssize_t type: + + * value = _Py_atomic_size_get(&var) + * _Py_atomic_size_set(&var, value) + + Use sequentially-consistent ordering (__ATOMIC_SEQ_CST memory order): + enforce total ordering with all other atomic functions. +*/ +#ifndef Py_ATOMIC_FUNC_H +#define Py_ATOMIC_FUNC_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#if defined(_MSC_VER) +# include // _InterlockedExchange() +#endif + + +// Use builtin atomic operations in GCC >= 4.7 and clang +#ifdef HAVE_BUILTIN_ATOMIC + +static inline Py_ssize_t _Py_atomic_size_get(Py_ssize_t *var) +{ + return __atomic_load_n(var, __ATOMIC_SEQ_CST); +} + +static inline void _Py_atomic_size_set(Py_ssize_t *var, Py_ssize_t value) +{ + __atomic_store_n(var, value, __ATOMIC_SEQ_CST); +} + +#elif defined(_MSC_VER) + +static inline Py_ssize_t _Py_atomic_size_get(Py_ssize_t *var) +{ +#if SIZEOF_VOID_P == 8 + Py_BUILD_ASSERT(sizeof(__int64) == sizeof(*var)); + volatile __int64 *volatile_var = (volatile __int64 *)var; + __int64 old; + do { + old = *volatile_var; + } while(_InterlockedCompareExchange64(volatile_var, old, old) != old); +#else + Py_BUILD_ASSERT(sizeof(long) == sizeof(*var)); + volatile long *volatile_var = (volatile long *)var; + long old; + do { + old = *volatile_var; + } while(_InterlockedCompareExchange(volatile_var, old, old) != old); +#endif + return old; +} + +static inline void _Py_atomic_size_set(Py_ssize_t *var, Py_ssize_t value) +{ +#if SIZEOF_VOID_P == 8 + Py_BUILD_ASSERT(sizeof(__int64) == sizeof(*var)); + volatile __int64 *volatile_var = (volatile __int64 *)var; + _InterlockedExchange64(volatile_var, value); +#else + Py_BUILD_ASSERT(sizeof(long) == sizeof(*var)); + volatile long *volatile_var = (volatile long *)var; + _InterlockedExchange(volatile_var, value); +#endif +} + +#else +// Fallback implementation using volatile + +static inline Py_ssize_t _Py_atomic_size_get(Py_ssize_t *var) +{ + volatile Py_ssize_t *volatile_var = (volatile Py_ssize_t *)var; + return *volatile_var; +} + +static inline void _Py_atomic_size_set(Py_ssize_t *var, Py_ssize_t value) +{ + volatile Py_ssize_t *volatile_var = (volatile Py_ssize_t *)var; + *volatile_var = value; +} +#endif + +#ifdef __cplusplus +} +#endif +#endif /* Py_ATOMIC_FUNC_H */ diff --git a/Include/internal/pycore_bitutils.h b/Include/internal/pycore_bitutils.h index 1602fc68d94074..e4aa7a3d0d0567 100644 --- a/Include/internal/pycore_bitutils.h +++ b/Include/internal/pycore_bitutils.h @@ -17,12 +17,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#if ((defined(__GNUC__) \ - && ((__GNUC__ >= 5) || (__GNUC__ == 4) && (__GNUC_MINOR__ >= 8))) \ - || (defined(__clang__) \ - && (__clang_major__ >= 4 \ - || (__clang_major__ == 3 && __clang_minor__ >= 2)))) - /* __builtin_bswap16() is available since GCC 4.8 and clang 3.2, +#if defined(__GNUC__) \ + && ((__GNUC__ >= 5) || (__GNUC__ == 4) && (__GNUC_MINOR__ >= 8)) + /* __builtin_bswap16() is available since GCC 4.8, __builtin_bswap32() is available since GCC 4.3, __builtin_bswap64() is available since GCC 4.3. */ # define _PY_HAVE_BUILTIN_BSWAP @@ -36,7 +33,7 @@ extern "C" { static inline uint16_t _Py_bswap16(uint16_t word) { -#ifdef _PY_HAVE_BUILTIN_BSWAP +#if defined(_PY_HAVE_BUILTIN_BSWAP) || _Py__has_builtin(__builtin_bswap16) return __builtin_bswap16(word); #elif defined(_MSC_VER) Py_BUILD_ASSERT(sizeof(word) == sizeof(unsigned short)); @@ -51,7 +48,7 @@ _Py_bswap16(uint16_t word) static inline uint32_t _Py_bswap32(uint32_t word) { -#ifdef _PY_HAVE_BUILTIN_BSWAP +#if defined(_PY_HAVE_BUILTIN_BSWAP) || _Py__has_builtin(__builtin_bswap32) return __builtin_bswap32(word); #elif defined(_MSC_VER) Py_BUILD_ASSERT(sizeof(word) == sizeof(unsigned long)); @@ -68,7 +65,7 @@ _Py_bswap32(uint32_t word) static inline uint64_t _Py_bswap64(uint64_t word) { -#ifdef _PY_HAVE_BUILTIN_BSWAP +#if defined(_PY_HAVE_BUILTIN_BSWAP) || _Py__has_builtin(__builtin_bswap64) return __builtin_bswap64(word); #elif defined(_MSC_VER) return _byteswap_uint64(word); diff --git a/Include/internal/pycore_blocks_output_buffer.h b/Include/internal/pycore_blocks_output_buffer.h new file mode 100644 index 00000000000000..22546e9a32a80b --- /dev/null +++ b/Include/internal/pycore_blocks_output_buffer.h @@ -0,0 +1,317 @@ +/* + _BlocksOutputBuffer is used to maintain an output buffer + that has unpredictable size. Suitable for compression/decompression + API (bz2/lzma/zlib) that has stream->next_out and stream->avail_out: + + stream->next_out: point to the next output position. + stream->avail_out: the number of available bytes left in the buffer. + + It maintains a list of bytes object, so there is no overhead of resizing + the buffer. + + Usage: + + 1, Initialize the struct instance like this: + _BlocksOutputBuffer buffer = {.list = NULL}; + Set .list to NULL for _BlocksOutputBuffer_OnError() + + 2, Initialize the buffer use one of these functions: + _BlocksOutputBuffer_InitAndGrow() + _BlocksOutputBuffer_InitWithSize() + + 3, If (avail_out == 0), grow the buffer: + _BlocksOutputBuffer_Grow() + + 4, Get the current outputted data size: + _BlocksOutputBuffer_GetDataSize() + + 5, Finish the buffer, and return a bytes object: + _BlocksOutputBuffer_Finish() + + 6, Clean up the buffer when an error occurred: + _BlocksOutputBuffer_OnError() +*/ + +#ifndef Py_INTERNAL_BLOCKS_OUTPUT_BUFFER_H +#define Py_INTERNAL_BLOCKS_OUTPUT_BUFFER_H +#ifdef __cplusplus +extern "C" { +#endif + +#include "Python.h" + +typedef struct { + // List of bytes objects + PyObject *list; + // Number of whole allocated size + Py_ssize_t allocated; + // Max length of the buffer, negative number means unlimited length. + Py_ssize_t max_length; +} _BlocksOutputBuffer; + +static const char unable_allocate_msg[] = "Unable to allocate output buffer."; + +/* In 32-bit build, the max block size should <= INT32_MAX. */ +#define OUTPUT_BUFFER_MAX_BLOCK_SIZE (256*1024*1024) + +/* Block size sequence */ +#define KB (1024) +#define MB (1024*1024) +const Py_ssize_t BUFFER_BLOCK_SIZE[] = + { 32*KB, 64*KB, 256*KB, 1*MB, 4*MB, 8*MB, 16*MB, 16*MB, + 32*MB, 32*MB, 32*MB, 32*MB, 64*MB, 64*MB, 128*MB, 128*MB, + OUTPUT_BUFFER_MAX_BLOCK_SIZE }; +#undef KB +#undef MB + +/* According to the block sizes defined by BUFFER_BLOCK_SIZE, the whole + allocated size growth step is: + 1 32 KB +32 KB + 2 96 KB +64 KB + 3 352 KB +256 KB + 4 1.34 MB +1 MB + 5 5.34 MB +4 MB + 6 13.34 MB +8 MB + 7 29.34 MB +16 MB + 8 45.34 MB +16 MB + 9 77.34 MB +32 MB + 10 109.34 MB +32 MB + 11 141.34 MB +32 MB + 12 173.34 MB +32 MB + 13 237.34 MB +64 MB + 14 301.34 MB +64 MB + 15 429.34 MB +128 MB + 16 557.34 MB +128 MB + 17 813.34 MB +256 MB + 18 1069.34 MB +256 MB + 19 1325.34 MB +256 MB + 20 1581.34 MB +256 MB + 21 1837.34 MB +256 MB + 22 2093.34 MB +256 MB + ... +*/ + +/* Initialize the buffer, and grow the buffer. + + max_length: Max length of the buffer, -1 for unlimited length. + + On success, return allocated size (>=0) + On failure, return -1 +*/ +static inline Py_ssize_t +_BlocksOutputBuffer_InitAndGrow(_BlocksOutputBuffer *buffer, + const Py_ssize_t max_length, + void **next_out) +{ + PyObject *b; + Py_ssize_t block_size; + + // ensure .list was set to NULL + assert(buffer->list == NULL); + + // get block size + if (0 <= max_length && max_length < BUFFER_BLOCK_SIZE[0]) { + block_size = max_length; + } else { + block_size = BUFFER_BLOCK_SIZE[0]; + } + + // the first block + b = PyBytes_FromStringAndSize(NULL, block_size); + if (b == NULL) { + return -1; + } + + // create the list + buffer->list = PyList_New(1); + if (buffer->list == NULL) { + Py_DECREF(b); + return -1; + } + PyList_SET_ITEM(buffer->list, 0, b); + + // set variables + buffer->allocated = block_size; + buffer->max_length = max_length; + + *next_out = PyBytes_AS_STRING(b); + return block_size; +} + +/* Initialize the buffer, with an initial size. + + Check block size limit in the outer wrapper function. For example, some libs + accept UINT32_MAX as the maximum block size, then init_size should <= it. + + On success, return allocated size (>=0) + On failure, return -1 +*/ +static inline Py_ssize_t +_BlocksOutputBuffer_InitWithSize(_BlocksOutputBuffer *buffer, + const Py_ssize_t init_size, + void **next_out) +{ + PyObject *b; + + // ensure .list was set to NULL + assert(buffer->list == NULL); + + // the first block + b = PyBytes_FromStringAndSize(NULL, init_size); + if (b == NULL) { + PyErr_SetString(PyExc_MemoryError, unable_allocate_msg); + return -1; + } + + // create the list + buffer->list = PyList_New(1); + if (buffer->list == NULL) { + Py_DECREF(b); + return -1; + } + PyList_SET_ITEM(buffer->list, 0, b); + + // set variables + buffer->allocated = init_size; + buffer->max_length = -1; + + *next_out = PyBytes_AS_STRING(b); + return init_size; +} + +/* Grow the buffer. The avail_out must be 0, please check it before calling. + + On success, return allocated size (>=0) + On failure, return -1 +*/ +static inline Py_ssize_t +_BlocksOutputBuffer_Grow(_BlocksOutputBuffer *buffer, + void **next_out, + const Py_ssize_t avail_out) +{ + PyObject *b; + const Py_ssize_t list_len = Py_SIZE(buffer->list); + Py_ssize_t block_size; + + // ensure no gaps in the data + if (avail_out != 0) { + PyErr_SetString(PyExc_SystemError, + "avail_out is non-zero in _BlocksOutputBuffer_Grow()."); + return -1; + } + + // get block size + if (list_len < (Py_ssize_t) Py_ARRAY_LENGTH(BUFFER_BLOCK_SIZE)) { + block_size = BUFFER_BLOCK_SIZE[list_len]; + } else { + block_size = BUFFER_BLOCK_SIZE[Py_ARRAY_LENGTH(BUFFER_BLOCK_SIZE) - 1]; + } + + // check max_length + if (buffer->max_length >= 0) { + // if (rest == 0), should not grow the buffer. + Py_ssize_t rest = buffer->max_length - buffer->allocated; + assert(rest > 0); + + // block_size of the last block + if (block_size > rest) { + block_size = rest; + } + } + + // check buffer->allocated overflow + if (block_size > PY_SSIZE_T_MAX - buffer->allocated) { + PyErr_SetString(PyExc_MemoryError, unable_allocate_msg); + return -1; + } + + // create the block + b = PyBytes_FromStringAndSize(NULL, block_size); + if (b == NULL) { + PyErr_SetString(PyExc_MemoryError, unable_allocate_msg); + return -1; + } + if (PyList_Append(buffer->list, b) < 0) { + Py_DECREF(b); + return -1; + } + Py_DECREF(b); + + // set variables + buffer->allocated += block_size; + + *next_out = PyBytes_AS_STRING(b); + return block_size; +} + +/* Return the current outputted data size. */ +static inline Py_ssize_t +_BlocksOutputBuffer_GetDataSize(_BlocksOutputBuffer *buffer, + const Py_ssize_t avail_out) +{ + return buffer->allocated - avail_out; +} + +/* Finish the buffer. + + Return a bytes object on success + Return NULL on failure +*/ +static inline PyObject * +_BlocksOutputBuffer_Finish(_BlocksOutputBuffer *buffer, + const Py_ssize_t avail_out) +{ + PyObject *result, *block; + const Py_ssize_t list_len = Py_SIZE(buffer->list); + + // fast path for single block + if ((list_len == 1 && avail_out == 0) || + (list_len == 2 && Py_SIZE(PyList_GET_ITEM(buffer->list, 1)) == avail_out)) + { + block = PyList_GET_ITEM(buffer->list, 0); + Py_INCREF(block); + + Py_CLEAR(buffer->list); + return block; + } + + // final bytes object + result = PyBytes_FromStringAndSize(NULL, buffer->allocated - avail_out); + if (result == NULL) { + PyErr_SetString(PyExc_MemoryError, unable_allocate_msg); + return NULL; + } + + // memory copy + if (list_len > 0) { + char *posi = PyBytes_AS_STRING(result); + + // blocks except the last one + Py_ssize_t i = 0; + for (; i < list_len-1; i++) { + block = PyList_GET_ITEM(buffer->list, i); + memcpy(posi, PyBytes_AS_STRING(block), Py_SIZE(block)); + posi += Py_SIZE(block); + } + // the last block + block = PyList_GET_ITEM(buffer->list, i); + memcpy(posi, PyBytes_AS_STRING(block), Py_SIZE(block) - avail_out); + } else { + assert(Py_SIZE(result) == 0); + } + + Py_CLEAR(buffer->list); + return result; +} + +/* Clean up the buffer when an error occurred. */ +static inline void +_BlocksOutputBuffer_OnError(_BlocksOutputBuffer *buffer) +{ + Py_CLEAR(buffer->list); +} + +#ifdef __cplusplus +} +#endif +#endif /* Py_INTERNAL_BLOCKS_OUTPUT_BUFFER_H */ \ No newline at end of file diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index bbb667ea32d27e..f573c3e5086807 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -23,7 +23,7 @@ PyAPI_FUNC(int) _PyEval_AddPendingCall( PyInterpreterState *interp, int (*func)(void *), void *arg); -PyAPI_FUNC(void) _PyEval_SignalAsyncExc(PyThreadState *tstate); +PyAPI_FUNC(void) _PyEval_SignalAsyncExc(PyInterpreterState *interp); #ifdef HAVE_FORK extern PyStatus _PyEval_ReInitThreads(PyThreadState *tstate); #endif @@ -31,24 +31,26 @@ PyAPI_FUNC(void) _PyEval_SetCoroutineOriginTrackingDepth( PyThreadState *tstate, int new_depth); -/* Private function */ void _PyEval_Fini(void); + +extern PyObject* _PyEval_GetBuiltins(PyThreadState *tstate); +extern PyObject *_PyEval_BuiltinsFromGlobals( + PyThreadState *tstate, + PyObject *globals); + + static inline PyObject* _PyEval_EvalFrame(PyThreadState *tstate, PyFrameObject *f, int throwflag) { return tstate->interp->eval_frame(tstate, f, throwflag); } -extern PyObject *_PyEval_EvalCode( - PyThreadState *tstate, - PyObject *_co, PyObject *globals, PyObject *locals, - PyObject *const *args, Py_ssize_t argcount, - PyObject *const *kwnames, PyObject *const *kwargs, - Py_ssize_t kwcount, int kwstep, - PyObject *const *defs, Py_ssize_t defcount, - PyObject *kwdefs, PyObject *closure, - PyObject *name, PyObject *qualname); +extern PyObject * +_PyEval_Vector(PyThreadState *tstate, + PyFrameConstructor *desc, PyObject *locals, + PyObject* const* args, size_t argcount, + PyObject *kwnames); #ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS extern int _PyEval_ThreadsInitialized(PyInterpreterState *interp); @@ -56,10 +58,12 @@ extern int _PyEval_ThreadsInitialized(PyInterpreterState *interp); extern int _PyEval_ThreadsInitialized(struct pyruntimestate *runtime); #endif extern PyStatus _PyEval_InitGIL(PyThreadState *tstate); -extern void _PyEval_FiniGIL(PyThreadState *tstate); +extern void _PyEval_FiniGIL(PyInterpreterState *interp); extern void _PyEval_ReleaseLock(PyThreadState *tstate); +extern void _PyEval_DeactivateOpCache(void); + /* --- _Py_EnterRecursiveCall() ----------------------------------------- */ @@ -92,24 +96,8 @@ static inline int _Py_EnterRecursiveCall_inline(const char *where) { #define Py_EnterRecursiveCall(where) _Py_EnterRecursiveCall_inline(where) -/* Compute the "lower-water mark" for a recursion limit. When - * Py_LeaveRecursiveCall() is called with a recursion depth below this mark, - * the overflowed flag is reset to 0. */ -static inline int _Py_RecursionLimitLowerWaterMark(int limit) { - if (limit > 200) { - return (limit - 50); - } - else { - return (3 * (limit >> 2)); - } -} - static inline void _Py_LeaveRecursiveCall(PyThreadState *tstate) { tstate->recursion_depth--; - int limit = tstate->interp->ceval.recursion_limit; - if (tstate->recursion_depth < _Py_RecursionLimitLowerWaterMark(limit)) { - tstate->overflowed = 0; - } } static inline void _Py_LeaveRecursiveCall_inline(void) { diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h new file mode 100644 index 00000000000000..06a6082cddae6a --- /dev/null +++ b/Include/internal/pycore_compile.h @@ -0,0 +1,44 @@ +#ifndef Py_INTERNAL_COMPILE_H +#define Py_INTERNAL_COMPILE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +struct _arena; // Type defined in pycore_pyarena.h +struct _mod; // Type defined in pycore_ast.h + +// Export the symbol for test_peg_generator (built as a library) +PyAPI_FUNC(PyCodeObject*) _PyAST_Compile( + struct _mod *mod, + PyObject *filename, + PyCompilerFlags *flags, + int optimize, + struct _arena *arena); +extern PyFutureFeatures* _PyFuture_FromAST( + struct _mod * mod, + PyObject *filename + ); + +extern PyObject* _Py_Mangle(PyObject *p, PyObject *name); + +typedef struct { + int optimize; + int ff_features; + + int recursion_depth; /* current recursion depth */ + int recursion_limit; /* recursion limit */ +} _PyASTOptimizeState; + +extern int _PyAST_Optimize( + struct _mod *, + struct _arena *arena, + _PyASTOptimizeState *state); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_COMPILE_H */ diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index ea4b3c8ea738f0..a482dd42122872 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -37,6 +37,6 @@ struct _pycontexttokenobject { int _PyContext_Init(void); -void _PyContext_Fini(PyThreadState *tstate); +void _PyContext_Fini(PyInterpreterState *interp); #endif /* !Py_INTERNAL_CONTEXT_H */ diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 9cb5fc66ee2e03..c1c9244a1bc7c4 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -50,6 +50,21 @@ PyAPI_FUNC(int) _Py_GetLocaleconvNumeric( PyAPI_FUNC(void) _Py_closerange(int first, int last); +PyAPI_FUNC(wchar_t*) _Py_GetLocaleEncoding(void); +PyAPI_FUNC(PyObject*) _Py_GetLocaleEncodingObject(void); + +#ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION +extern int _Py_LocaleUsesNonUnicodeWchar(void); + +extern wchar_t* _Py_DecodeNonUnicodeWchar( + const wchar_t* native, + Py_ssize_t size); + +extern int _Py_EncodeNonUnicodeWchar_InPlace( + wchar_t* unicode, + Py_ssize_t size); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_format.h b/Include/internal/pycore_format.h new file mode 100644 index 00000000000000..1b8d57539ca505 --- /dev/null +++ b/Include/internal/pycore_format.h @@ -0,0 +1,27 @@ +#ifndef Py_INTERNAL_FORMAT_H +#define Py_INTERNAL_FORMAT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +/* Format codes + * F_LJUST '-' + * F_SIGN '+' + * F_BLANK ' ' + * F_ALT '#' + * F_ZERO '0' + */ +#define F_LJUST (1<<0) +#define F_SIGN (1<<1) +#define F_BLANK (1<<2) +#define F_ALT (1<<3) +#define F_ZERO (1<<4) + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_FORMAT_H */ diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index da202a1df532e4..9db4a4716fa588 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -161,17 +161,19 @@ struct _gc_runtime_state { Py_ssize_t long_lived_pending; }; -PyAPI_FUNC(void) _PyGC_InitState(struct _gc_runtime_state *); +extern void _PyGC_InitState(struct _gc_runtime_state *); + +extern Py_ssize_t _PyGC_CollectNoFail(PyThreadState *tstate); // Functions to clear types free lists -extern void _PyFrame_ClearFreeList(PyThreadState *tstate); -extern void _PyTuple_ClearFreeList(PyThreadState *tstate); -extern void _PyFloat_ClearFreeList(PyThreadState *tstate); -extern void _PyList_ClearFreeList(PyThreadState *tstate); -extern void _PyDict_ClearFreeList(PyThreadState *tstate); -extern void _PyAsyncGen_ClearFreeLists(PyThreadState *tstate); -extern void _PyContext_ClearFreeList(PyThreadState *tstate); +extern void _PyFrame_ClearFreeList(PyInterpreterState *interp); +extern void _PyTuple_ClearFreeList(PyInterpreterState *interp); +extern void _PyFloat_ClearFreeList(PyInterpreterState *interp); +extern void _PyList_ClearFreeList(PyInterpreterState *interp); +extern void _PyDict_ClearFreeList(PyInterpreterState *interp); +extern void _PyAsyncGen_ClearFreeLists(PyInterpreterState *interp); +extern void _PyContext_ClearFreeList(PyInterpreterState *interp); #ifdef __cplusplus } diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 35a67abebac6f7..e21ed0a7a06a2a 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -5,15 +5,10 @@ extern "C" { #endif -PyAPI_FUNC(PyObject *) _PyImport_FindBuiltin( - PyThreadState *tstate, - const char *name /* UTF-8 encoded string */ - ); - #ifdef HAVE_FORK extern PyStatus _PyImport_ReInitLock(void); #endif -extern void _PyImport_Cleanup(PyThreadState *tstate); +extern PyObject* _PyImport_BootstrapImp(PyThreadState *tstate); #ifdef __cplusplus } diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 457a005860b202..4b009e816b4927 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -44,6 +44,8 @@ struct pyruntimestate; #define _PyStatus_UPDATE_FUNC(err) \ do { err.func = _PyStatus_GET_FUNC(); } while (0) +PyObject* _PyErr_SetFromPyStatus(PyStatus status); + /* --- PyWideStringList ------------------------------------------------ */ #define _PyWideStringList_INIT (PyWideStringList){.length = 0, .items = NULL} @@ -100,6 +102,7 @@ typedef struct { int isolated; /* -I option */ int use_environment; /* -E option */ int dev_mode; /* -X dev and PYTHONDEVMODE */ + int warn_default_encoding; /* -X warn_default_encoding and PYTHONWARNDEFAULTENCODING */ } _PyPreCmdline; #define _PyPreCmdline_INIT \ @@ -149,13 +152,19 @@ PyAPI_FUNC(void) _PyConfig_InitCompatConfig(PyConfig *config); extern PyStatus _PyConfig_Copy( PyConfig *config, const PyConfig *config2); -extern PyStatus _PyConfig_InitPathConfig(PyConfig *config); +extern PyStatus _PyConfig_InitPathConfig( + PyConfig *config, + int compute_path_config); +extern PyStatus _PyConfig_Read(PyConfig *config, int compute_path_config); extern PyStatus _PyConfig_Write(const PyConfig *config, struct pyruntimestate *runtime); extern PyStatus _PyConfig_SetPyArgv( PyConfig *config, const _PyArgv *args); +PyAPI_FUNC(PyObject*) _PyConfig_AsDict(const PyConfig *config); +PyAPI_FUNC(int) _PyConfig_FromDict(PyConfig *config, PyObject *dict); + /* --- Function used for testing ---------------------------------- */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index eee369a44bfc72..bfd082b588256b 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -8,10 +8,11 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_atomic.h" /* _Py_atomic_address */ -#include "pycore_gil.h" /* struct _gil_runtime_state */ -#include "pycore_gc.h" /* struct _gc_runtime_state */ -#include "pycore_warnings.h" /* struct _warnings_runtime_state */ +#include "pycore_atomic.h" // _Py_atomic_address +#include "pycore_ast_state.h" // struct ast_state +#include "pycore_gil.h" // struct _gil_runtime_state +#include "pycore_gc.h" // struct _gc_runtime_state +#include "pycore_warnings.h" // struct _warnings_runtime_state struct _pending_calls { PyThread_type_lock lock; @@ -32,12 +33,6 @@ struct _pending_calls { struct _ceval_state { int recursion_limit; - /* Records whether tracing is on for any thread. Counts the number - of threads for which tstate->c_tracefunc is non-NULL, so if the - value is 0, we know we don't have to check this thread's - c_tracefunc. This speeds up the if statement in - _PyEval_EvalFrameDefault() after fast_next_opcode. */ - int tracing_possible; /* This single variable consolidates all requests to break out of the fast path in the eval loop. */ _Py_atomic_int eval_breaker; @@ -63,6 +58,11 @@ struct _Py_bytes_state { PyBytesObject *characters[256]; }; +struct _Py_unicode_ids { + Py_ssize_t size; + PyObject **array; +}; + struct _Py_unicode_state { // The empty Unicode object is a singleton to improve performance. PyObject *empty_string; @@ -70,6 +70,19 @@ struct _Py_unicode_state { shared as well. */ PyObject *latin1[256]; struct _Py_unicode_fs_codec fs_codec; + + /* This dictionary holds all interned unicode strings. Note that references + to strings in this dictionary are *not* counted in the string's ob_refcnt. + When the interned string reaches a refcnt of 0 the string deallocation + function will delete the reference from this dictionary. + + Another way to look at this is that to say that the actual reference + count of a string is: s->ob_refcnt + (s->state ? 2 : 0) + */ + PyObject *interned; + + // Unicode identifiers (_Py_Identifier): see _PyUnicode_FromId() + struct _Py_unicode_ids ids; }; struct _Py_float_state { @@ -158,6 +171,41 @@ struct _Py_exc_state { }; +// atexit state +typedef struct { + PyObject *func; + PyObject *args; + PyObject *kwargs; +} atexit_callback; + +struct atexit_state { + atexit_callback **callbacks; + int ncallbacks; + int callback_len; +}; + + +// Type attribute lookup cache: speed up attribute and method lookups, +// see _PyType_Lookup(). +struct type_cache_entry { + unsigned int version; // initialized from type->tp_version_tag + PyObject *name; // reference to exactly a str or None + PyObject *value; // borrowed reference or NULL +}; + +#define MCACHE_SIZE_EXP 12 +#define MCACHE_STATS 0 + +struct type_cache { + struct type_cache_entry hashtable[1 << MCACHE_SIZE_EXP]; +#if MCACHE_STATS + size_t hits; + size_t misses; + size_t collisions; +#endif +}; + + /* interpreter state */ #define _PY_NSMALLPOSINTS 257 @@ -189,10 +237,14 @@ struct _is { struct _ceval_state ceval; struct _gc_runtime_state gc; + // sys.modules dictionary PyObject *modules; PyObject *modules_by_index; + // Dictionary of the sys module PyObject *sysdict; + // Dictionary of the builtins module PyObject *builtins; + // importlib module PyObject *importlib; /* Used in Modules/_threadmodule.c. */ @@ -217,7 +269,7 @@ struct _is { PyObject *builtins_copy; PyObject *import_func; - /* Initialized to PyEval_EvalFrameDefault(). */ + // Initialized to _PyEval_EvalFrameDefault(). _PyFrameEvalFunction eval_frame; Py_ssize_t co_extra_user_count; @@ -228,13 +280,11 @@ struct _is { PyObject *after_forkers_parent; PyObject *after_forkers_child; #endif - /* AtExit module */ - void (*pyexitfunc)(PyObject *); - PyObject *pyexitmodule; uint64_t tstate_next_unique_id; struct _warnings_runtime_state warnings; + struct atexit_state atexit; PyObject *audit_hooks; @@ -258,15 +308,13 @@ struct _is { struct _Py_async_gen_state async_gen; struct _Py_context_state context; struct _Py_exc_state exc_state; + + struct ast_state ast; + struct type_cache type_cache; }; -/* Used by _PyImport_Cleanup() */ extern void _PyInterpreterState_ClearModules(PyInterpreterState *interp); - -extern PyStatus _PyInterpreterState_SetConfig( - PyInterpreterState *interp, - const PyConfig *config); - +extern void _PyInterpreterState_Clear(PyThreadState *tstate); /* cross-interpreter data registry */ @@ -286,11 +334,10 @@ struct _xidregitem { PyAPI_FUNC(struct _is*) _PyInterpreterState_LookUpID(int64_t); PyAPI_FUNC(int) _PyInterpreterState_IDInitref(struct _is *); -PyAPI_FUNC(void) _PyInterpreterState_IDIncref(struct _is *); +PyAPI_FUNC(int) _PyInterpreterState_IDIncref(struct _is *); PyAPI_FUNC(void) _PyInterpreterState_IDDecref(struct _is *); #ifdef __cplusplus } #endif #endif /* !Py_INTERNAL_INTERP_H */ - diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index ec95786531c850..2bea3a55ec8735 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -14,15 +14,12 @@ extern "C" { // Don't call this function but _PyLong_GetZero() and _PyLong_GetOne() static inline PyObject* __PyLong_GetSmallInt_internal(int value) { - PyThreadState *tstate = _PyThreadState_GET(); -#ifdef Py_DEBUG - _Py_EnsureTstateNotNULL(tstate); -#endif + PyInterpreterState *interp = _PyInterpreterState_GET(); assert(-_PY_NSMALLNEGINTS <= value && value < _PY_NSMALLPOSINTS); size_t index = _PY_NSMALLNEGINTS + value; - PyObject *obj = (PyObject*)tstate->interp->small_ints[index]; - // _PyLong_GetZero() and _PyLong_GetOne() must not be called - // before _PyLong_Init() nor after _PyLong_Fini() + PyObject *obj = (PyObject*)interp->small_ints[index]; + // _PyLong_GetZero(), _PyLong_GetOne() and get_small_int() must not be + // called before _PyLong_Init() nor after _PyLong_Fini(). assert(obj != NULL); return obj; } diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h new file mode 100644 index 00000000000000..e9978abd25b10a --- /dev/null +++ b/Include/internal/pycore_moduleobject.h @@ -0,0 +1,42 @@ +#ifndef Py_INTERNAL_MODULEOBJECT_H +#define Py_INTERNAL_MODULEOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +typedef struct { + PyObject_HEAD + PyObject *md_dict; + struct PyModuleDef *md_def; + void *md_state; + PyObject *md_weaklist; + // for logging purposes after md_dict is cleared + PyObject *md_name; +} PyModuleObject; + +static inline PyModuleDef* _PyModule_GetDef(PyObject *mod) { + assert(PyModule_Check(mod)); + return ((PyModuleObject *)mod)->md_def; +} + +static inline void* _PyModule_GetState(PyObject* mod) { + assert(PyModule_Check(mod)); + return ((PyModuleObject *)mod)->md_state; +} + +static inline PyObject* _PyModule_GetDict(PyObject *mod) { + assert(PyModule_Check(mod)); + PyObject *dict = ((PyModuleObject *)mod) -> md_dict; + // _PyModule_GetDict(mod) must not be used after calling module_clear(mod) + assert(dict != NULL); + return dict; +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_MODULEOBJECT_H */ diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index edd0031c3eff85..9dfc8c62babadb 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -8,9 +8,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() -#include "pycore_interp.h" // PyInterpreterState.gc -#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() +#include "pycore_interp.h" // PyInterpreterState.gc +#include "pycore_pystate.h" // _PyInterpreterState_GET() PyAPI_FUNC(int) _PyType_CheckConsistency(PyTypeObject *type); PyAPI_FUNC(int) _PyDict_CheckConsistency(PyObject *mp, int check_content); @@ -27,6 +27,9 @@ _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { return ((type->tp_flags & feature) != 0); } +extern void _PyType_InitCache(PyInterpreterState *interp); + + /* Inline functions trading binary compatibility for speed: _PyObject_Init() is the fast version of PyObject_Init(), and _PyObject_InitVar() is the fast version of PyObject_InitVar(). @@ -82,8 +85,8 @@ static inline void _PyObject_GC_TRACK( "object is in generation which is garbage collected", filename, lineno, __func__); - PyThreadState *tstate = _PyThreadState_GET(); - PyGC_Head *generation0 = tstate->interp->gc.generation0; + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyGC_Head *generation0 = interp->gc.generation0; PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); @@ -165,6 +168,16 @@ _PyObject_IS_GC(PyObject *obj) // Fast inlined version of PyType_IS_GC() #define _PyType_IS_GC(t) _PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) +// Usage: assert(_Py_CheckSlotResult(obj, "__getitem__", result != NULL))); +extern int _Py_CheckSlotResult( + PyObject *obj, + const char *slot_name, + int success); + +// PyType_Ready() must be called if _PyType_IsReady() is false. +// See also the Py_TPFLAGS_READY flag. +#define _PyType_IsReady(type) ((type)->tp_dict != NULL) + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h new file mode 100644 index 00000000000000..e2de24e2ca9734 --- /dev/null +++ b/Include/internal/pycore_parser.h @@ -0,0 +1,31 @@ +#ifndef Py_INTERNAL_PARSER_H +#define Py_INTERNAL_PARSER_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +extern struct _mod* _PyParser_ASTFromString( + const char *str, + PyObject* filename, + int mode, + PyCompilerFlags *flags, + PyArena *arena); +extern struct _mod* _PyParser_ASTFromFile( + FILE *fp, + PyObject *filename_ob, + const char *enc, + int mode, + const char *ps1, + const char *ps2, + PyCompilerFlags *flags, + int *errcode, + PyArena *arena); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_PARSER_H */ diff --git a/Include/internal/pycore_pyarena.h b/Include/internal/pycore_pyarena.h new file mode 100644 index 00000000000000..d78972a88ca238 --- /dev/null +++ b/Include/internal/pycore_pyarena.h @@ -0,0 +1,64 @@ +/* An arena-like memory interface for the compiler. + */ + +#ifndef Py_INTERNAL_PYARENA_H +#define Py_INTERNAL_PYARENA_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +typedef struct _arena PyArena; + +/* _PyArena_New() and _PyArena_Free() create a new arena and free it, + respectively. Once an arena has been created, it can be used + to allocate memory via _PyArena_Malloc(). Pointers to PyObject can + also be registered with the arena via _PyArena_AddPyObject(), and the + arena will ensure that the PyObjects stay alive at least until + _PyArena_Free() is called. When an arena is freed, all the memory it + allocated is freed, the arena releases internal references to registered + PyObject*, and none of its pointers are valid. + XXX (tim) What does "none of its pointers are valid" mean? Does it + XXX mean that pointers previously obtained via _PyArena_Malloc() are + XXX no longer valid? (That's clearly true, but not sure that's what + XXX the text is trying to say.) + + _PyArena_New() returns an arena pointer. On error, it + returns a negative number and sets an exception. + XXX (tim): Not true. On error, _PyArena_New() actually returns NULL, + XXX and looks like it may or may not set an exception (e.g., if the + XXX internal PyList_New(0) returns NULL, _PyArena_New() passes that on + XXX and an exception is set; OTOH, if the internal + XXX block_new(DEFAULT_BLOCK_SIZE) returns NULL, that's passed on but + XXX an exception is not set in that case). +*/ +PyAPI_FUNC(PyArena*) _PyArena_New(void); +PyAPI_FUNC(void) _PyArena_Free(PyArena *); + +/* Mostly like malloc(), return the address of a block of memory spanning + * `size` bytes, or return NULL (without setting an exception) if enough + * new memory can't be obtained. Unlike malloc(0), _PyArena_Malloc() with + * size=0 does not guarantee to return a unique pointer (the pointer + * returned may equal one or more other pointers obtained from + * _PyArena_Malloc()). + * Note that pointers obtained via _PyArena_Malloc() must never be passed to + * the system free() or realloc(), or to any of Python's similar memory- + * management functions. _PyArena_Malloc()-obtained pointers remain valid + * until _PyArena_Free(ar) is called, at which point all pointers obtained + * from the arena `ar` become invalid simultaneously. + */ +PyAPI_FUNC(void*) _PyArena_Malloc(PyArena *, size_t size); + +/* This routine isn't a proper arena allocation routine. It takes + * a PyObject* and records it so that it can be DECREFed when the + * arena is freed. + */ +PyAPI_FUNC(int) _PyArena_AddPyObject(PyArena *, PyObject *); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_PYARENA_H */ diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 2cf1160afc0149..a5e97fe23fb05e 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -84,6 +84,12 @@ PyAPI_FUNC(PyObject *) _PyErr_FormatFromCauseTstate( PyAPI_FUNC(int) _PyErr_CheckSignalsTstate(PyThreadState *tstate); +PyAPI_FUNC(void) _Py_DumpExtensionModules(int fd, PyInterpreterState *interp); + +extern PyObject* _Py_Offer_Suggestions(PyObject* exception); +PyAPI_FUNC(Py_ssize_t) _Py_UTF8_Edit_Cost(PyObject *str_a, PyObject *str_b, + Py_ssize_t max_cost); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 6d84e37232b305..524be9d4cbb940 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -8,6 +8,24 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#ifdef HAVE_SIGNAL_H +#include +#endif + +#include "pycore_runtime.h" // _PyRuntimeState + +#ifndef NSIG +# if defined(_NSIG) +# define NSIG _NSIG /* For BSD/SysV */ +# elif defined(_SIGMAX) +# define NSIG (_SIGMAX + 1) /* For QNX */ +# elif defined(SIGMAX) +# define NSIG (SIGMAX + 1) /* For djgpp */ +# else +# define NSIG 64 /* Use a reasonable default value */ +# endif +#endif + /* Forward declarations */ struct _PyArgv; struct pyruntimestate; @@ -31,65 +49,71 @@ PyAPI_FUNC(int) _Py_IsLocaleCoercionTarget(const char *ctype_loc); /* Various one-time initializers */ -extern PyStatus _PyUnicode_Init(PyThreadState *tstate); -extern PyStatus _PyBytes_Init(PyThreadState *tstate); +extern PyStatus _PyUnicode_Init(PyInterpreterState *interp); +extern PyStatus _PyUnicode_InitTypes(void); +extern PyStatus _PyBytes_Init(PyInterpreterState *interp); extern int _PyStructSequence_Init(void); -extern int _PyLong_Init(PyThreadState *tstate); -extern PyStatus _PyTuple_Init(PyThreadState *tstate); +extern int _PyLong_Init(PyInterpreterState *interp); +extern int _PyLong_InitTypes(void); +extern PyStatus _PyTuple_Init(PyInterpreterState *interp); extern PyStatus _PyFaulthandler_Init(int enable); extern int _PyTraceMalloc_Init(int enable); -extern PyObject * _PyBuiltin_Init(PyThreadState *tstate); +extern PyObject * _PyBuiltin_Init(PyInterpreterState *interp); extern PyStatus _PySys_Create( PyThreadState *tstate, PyObject **sysmod_p); extern PyStatus _PySys_ReadPreinitWarnOptions(PyWideStringList *options); extern PyStatus _PySys_ReadPreinitXOptions(PyConfig *config); -extern int _PySys_InitMain(PyThreadState *tstate); -extern PyStatus _PyExc_Init(PyThreadState *tstate); -extern PyStatus _PyErr_Init(void); +extern int _PySys_UpdateConfig(PyThreadState *tstate); +extern PyStatus _PyExc_Init(PyInterpreterState *interp); +extern PyStatus _PyErr_InitTypes(void); extern PyStatus _PyBuiltins_AddExceptions(PyObject * bltinmod); -extern PyStatus _PyImportHooks_Init(PyThreadState *tstate); -extern int _PyFloat_Init(void); +extern void _PyFloat_Init(void); +extern int _PyFloat_InitTypes(void); extern PyStatus _Py_HashRandomization_Init(const PyConfig *); extern PyStatus _PyTypes_Init(void); extern PyStatus _PyTypes_InitSlotDefs(void); extern PyStatus _PyImportZip_Init(PyThreadState *tstate); -extern PyStatus _PyGC_Init(PyThreadState *tstate); +extern PyStatus _PyGC_Init(PyInterpreterState *interp); +extern PyStatus _PyAtExit_Init(PyInterpreterState *interp); /* Various internal finalizers */ -extern void _PyFrame_Fini(PyThreadState *tstate); -extern void _PyDict_Fini(PyThreadState *tstate); -extern void _PyTuple_Fini(PyThreadState *tstate); -extern void _PyList_Fini(PyThreadState *tstate); -extern void _PyBytes_Fini(PyThreadState *tstate); -extern void _PyFloat_Fini(PyThreadState *tstate); -extern void _PySlice_Fini(PyThreadState *tstate); -extern void _PyAsyncGen_Fini(PyThreadState *tstate); +extern void _PyFrame_Fini(PyInterpreterState *interp); +extern void _PyDict_Fini(PyInterpreterState *interp); +extern void _PyTuple_Fini(PyInterpreterState *interp); +extern void _PyList_Fini(PyInterpreterState *interp); +extern void _PyBytes_Fini(PyInterpreterState *interp); +extern void _PyFloat_Fini(PyInterpreterState *interp); +extern void _PySlice_Fini(PyInterpreterState *interp); +extern void _PyAsyncGen_Fini(PyInterpreterState *interp); -extern void PyOS_FiniInterrupts(void); +extern int _PySignal_Init(int install_signal_handlers); +extern void _PySignal_Fini(void); -extern void _PyExc_Fini(PyThreadState *tstate); +extern void _PyExc_Fini(PyInterpreterState *interp); extern void _PyImport_Fini(void); extern void _PyImport_Fini2(void); -extern void _PyGC_Fini(PyThreadState *tstate); -extern void _PyType_Fini(void); +extern void _PyGC_Fini(PyInterpreterState *interp); +extern void _PyType_Fini(PyInterpreterState *interp); extern void _Py_HashRandomization_Fini(void); -extern void _PyUnicode_Fini(PyThreadState *tstate); -extern void _PyUnicode_ClearInterned(PyThreadState *tstate); -extern void _PyLong_Fini(PyThreadState *tstate); +extern void _PyUnicode_Fini(PyInterpreterState *interp); +extern void _PyUnicode_ClearInterned(PyInterpreterState *interp); +extern void _PyLong_Fini(PyInterpreterState *interp); extern void _PyFaulthandler_Fini(void); extern void _PyHash_Fini(void); extern void _PyTraceMalloc_Fini(void); extern void _PyWarnings_Fini(PyInterpreterState *interp); -extern void _PyAST_Fini(PyThreadState *tstate); +extern void _PyAST_Fini(PyInterpreterState *interp); +extern void _PyAtExit_Fini(PyInterpreterState *interp); -extern PyStatus _PyGILState_Init(PyThreadState *tstate); -extern void _PyGILState_Fini(PyThreadState *tstate); +extern PyStatus _PyGILState_Init(_PyRuntimeState *runtime); +extern PyStatus _PyGILState_SetTstate(PyThreadState *tstate); +extern void _PyGILState_Fini(PyInterpreterState *interp); -PyAPI_FUNC(void) _PyGC_DumpShutdownStats(PyThreadState *tstate); +PyAPI_FUNC(void) _PyGC_DumpShutdownStats(PyInterpreterState *interp); PyAPI_FUNC(PyStatus) _Py_PreInitializeFromPyArgv( const PyPreConfig *src_config, @@ -109,6 +133,8 @@ PyAPI_FUNC(void) _PyErr_Display(PyObject *file, PyObject *exception, PyAPI_FUNC(void) _PyThreadState_DeleteCurrent(PyThreadState *tstate); +extern void _PyAtExit_Call(PyInterpreterState *interp); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 0cd5550cfda5c4..4b894f3eff4967 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -22,11 +22,11 @@ _Py_IsMainThread(void) static inline int -_Py_IsMainInterpreter(PyThreadState* tstate) +_Py_IsMainInterpreter(PyInterpreterState *interp) { /* Use directly _PyRuntime rather than tstate->interp->runtime, since this function is used in performance critical code path (ceval) */ - return (tstate->interp == _PyRuntime.interpreters.main); + return (interp == _PyRuntime.interpreters.main); } diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 3a01d64e63d811..bcd710c4496bd7 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -49,6 +49,13 @@ typedef struct _Py_AuditHookEntry { void *userData; } _Py_AuditHookEntry; +struct _Py_unicode_runtime_ids { + PyThread_type_lock lock; + // next_index value must be preserved when Py_Initialize()/Py_Finalize() + // is called multiple times: see _PyUnicode_FromId() implementation. + Py_ssize_t next_index; +}; + /* Full Python runtime state */ typedef struct pyruntimestate { @@ -102,10 +109,14 @@ typedef struct pyruntimestate { PyPreConfig preconfig; + // Audit values must be preserved when Py_Initialize()/Py_Finalize() + // is called multiple times. Py_OpenCodeHookFunction open_code_hook; void *open_code_userdata; _Py_AuditHookEntry *audit_hook_head; + struct _Py_unicode_runtime_ids unicode_ids; + // XXX Consolidate globals found via the check-c-globals script. } _PyRuntimeState; diff --git a/Include/internal/pycore_structseq.h b/Include/internal/pycore_structseq.h new file mode 100644 index 00000000000000..84c8d477e0dac8 --- /dev/null +++ b/Include/internal/pycore_structseq.h @@ -0,0 +1,21 @@ +#ifndef Py_INTERNAL_STRUCTSEQ_H +#define Py_INTERNAL_STRUCTSEQ_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +PyAPI_FUNC(int) _PyStructSequence_InitType( + PyTypeObject *type, + PyStructSequence_Desc *desc, + unsigned long tp_flags); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_STRUCTSEQ_H */ diff --git a/Include/symtable.h b/Include/internal/pycore_symtable.h similarity index 86% rename from Include/symtable.h rename to Include/internal/pycore_symtable.h index abd19a7923e1ba..f3505f8949be4b 100644 --- a/Include/symtable.h +++ b/Include/internal/pycore_symtable.h @@ -1,17 +1,16 @@ -#ifndef Py_LIMITED_API -#ifndef Py_SYMTABLE_H -#define Py_SYMTABLE_H +#ifndef Py_INTERNAL_SYMTABLE_H +#define Py_INTERNAL_SYMTABLE_H #ifdef __cplusplus extern "C" { #endif -#include "Python-ast.h" /* mod_ty */ +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif -/* XXX(ncoghlan): This is a weird mix of public names and interpreter internal - * names. - */ +struct _mod; // Type defined in pycore_ast.h -typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock } +typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock, AnnotationBlock } _Py_block_ty; struct _symtable_entry; @@ -62,28 +61,26 @@ typedef struct _symtable_entry { int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */ int ste_lineno; /* first line of block */ int ste_col_offset; /* offset of first line of block */ + int ste_end_lineno; /* end line of block */ + int ste_end_col_offset; /* end offset of first line of block */ int ste_opt_lineno; /* lineno of last exec or import * */ int ste_opt_col_offset; /* offset of last exec or import * */ struct symtable *ste_table; } PySTEntryObject; -PyAPI_DATA(PyTypeObject) PySTEntry_Type; +extern PyTypeObject PySTEntry_Type; #define PySTEntry_Check(op) Py_IS_TYPE(op, &PySTEntry_Type) -PyAPI_FUNC(int) PyST_GetScope(PySTEntryObject *, PyObject *); +extern int _PyST_GetScope(PySTEntryObject *, PyObject *); -PyAPI_FUNC(struct symtable *) PySymtable_Build( - mod_ty mod, - const char *filename, /* decoded from the filesystem encoding */ - PyFutureFeatures *future); -PyAPI_FUNC(struct symtable *) PySymtable_BuildObject( - mod_ty mod, +extern struct symtable* _PySymtable_Build( + struct _mod *mod, PyObject *filename, PyFutureFeatures *future); PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *); -PyAPI_FUNC(void) PySymtable_Free(struct symtable *); +extern void _PySymtable_Free(struct symtable *); /* Flags for def-use information */ @@ -116,8 +113,14 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *); #define GENERATOR 1 #define GENERATOR_EXPRESSION 2 +// Used by symtablemodule.c +extern struct symtable* _Py_SymtableStringObjectFlags( + const char *str, + PyObject *filename, + int start, + PyCompilerFlags *flags); + #ifdef __cplusplus } #endif -#endif /* !Py_SYMTABLE_H */ -#endif /* !Py_LIMITED_API */ +#endif /* !Py_INTERNAL_SYMTABLE_H */ diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index 1f092411a72ba5..4d282308769dc8 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -74,17 +74,13 @@ PyAPI_FUNC(void) _Py_DumpASCII(int fd, PyObject *text); This function is signal safe. */ PyAPI_FUNC(void) _Py_DumpDecimal( int fd, - unsigned long value); + size_t value); -/* Format an integer as hexadecimal into the file descriptor fd with at least - width digits. - - The maximum width is sizeof(unsigned long)*2 digits. - - This function is signal safe. */ +/* Format an integer as hexadecimal with width digits into fd file descriptor. + The function is signal safe. */ PyAPI_FUNC(void) _Py_DumpHexadecimal( int fd, - unsigned long value, + uintptr_t value, Py_ssize_t width); PyAPI_FUNC(PyObject*) _PyTraceBack_FromFrame( diff --git a/Include/internal/pycore_unionobject.h b/Include/internal/pycore_unionobject.h index fa8ba6ed944c1a..4d82b6fbeae8b5 100644 --- a/Include/internal/pycore_unionobject.h +++ b/Include/internal/pycore_unionobject.h @@ -10,6 +10,7 @@ extern "C" { PyAPI_FUNC(PyObject *) _Py_Union(PyObject *args); PyAPI_DATA(PyTypeObject) _Py_UnionType; +PyAPI_FUNC(PyObject *) _Py_union_type_or(PyObject* self, PyObject* param); #ifdef __cplusplus } diff --git a/Include/internal/pycore_warnings.h b/Include/internal/pycore_warnings.h index cafe305edb0e02..f728ec3077b3c5 100644 --- a/Include/internal/pycore_warnings.h +++ b/Include/internal/pycore_warnings.h @@ -17,7 +17,7 @@ struct _warnings_runtime_state { long filters_version; }; -extern PyStatus _PyWarnings_InitState(PyThreadState *tstate); +extern int _PyWarnings_InitState(PyInterpreterState *interp); #ifdef __cplusplus } diff --git a/Include/intrcheck.h b/Include/intrcheck.h index 88f2a7076ce379..b8cc65601683cb 100644 --- a/Include/intrcheck.h +++ b/Include/intrcheck.h @@ -5,7 +5,6 @@ extern "C" { #endif PyAPI_FUNC(int) PyOS_InterruptOccurred(void); -PyAPI_FUNC(void) PyOS_InitInterrupts(void); #ifdef HAVE_FORK #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000 PyAPI_FUNC(void) PyOS_BeforeFork(void); diff --git a/Include/iterobject.h b/Include/iterobject.h index 51139bf1874088..6454611aebef8a 100644 --- a/Include/iterobject.h +++ b/Include/iterobject.h @@ -7,6 +7,9 @@ extern "C" { PyAPI_DATA(PyTypeObject) PySeqIter_Type; PyAPI_DATA(PyTypeObject) PyCallIter_Type; +#ifdef Py_BUILD_CORE +extern PyTypeObject _PyAnextAwaitable_Type; +#endif #define PySeqIter_Check(op) Py_IS_TYPE(op, &PySeqIter_Type) diff --git a/Include/longobject.h b/Include/longobject.h index 06e3e2490401e4..e2301d7abfcccf 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -210,9 +210,6 @@ PyAPI_FUNC(PyObject *) _PyLong_GCD(PyObject *, PyObject *); #endif /* !Py_LIMITED_API */ #ifndef Py_LIMITED_API -PyAPI_DATA(PyObject *) _PyLong_Zero; -PyAPI_DATA(PyObject *) _PyLong_One; - PyAPI_FUNC(PyObject *) _PyLong_Rshift(PyObject *, size_t); PyAPI_FUNC(PyObject *) _PyLong_Lshift(PyObject *, size_t); #endif diff --git a/Include/methodobject.h b/Include/methodobject.h index 12e049b4043ba5..9ffe8e1a3ddfcb 100644 --- a/Include/methodobject.h +++ b/Include/methodobject.h @@ -41,7 +41,13 @@ struct PyMethodDef { }; typedef struct PyMethodDef PyMethodDef; +/* PyCFunction_New is declared as a function for stable ABI (declaration is + * needed for e.g. GCC with -fvisibility=hidden), but redefined as a macro + * that calls PyCFunction_NewEx. */ +PyAPI_FUNC(PyObject *) PyCFunction_New(PyMethodDef *, PyObject *); #define PyCFunction_New(ML, SELF) PyCFunction_NewEx((ML), (SELF), NULL) + +/* PyCFunction_NewEx is similar: on 3.9+, this calls PyCMethod_New. */ PyAPI_FUNC(PyObject *) PyCFunction_NewEx(PyMethodDef *, PyObject *, PyObject *); @@ -73,15 +79,15 @@ PyAPI_FUNC(PyObject *) PyCMethod_New(PyMethodDef *, PyObject *, #define METH_COEXIST 0x0040 -#ifndef Py_LIMITED_API -#define METH_FASTCALL 0x0080 +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030a0000 +# define METH_FASTCALL 0x0080 #endif /* This bit is preserved for Stackless Python */ #ifdef STACKLESS -#define METH_STACKLESS 0x0100 +# define METH_STACKLESS 0x0100 #else -#define METH_STACKLESS 0x0000 +# define METH_STACKLESS 0x0000 #endif /* METH_METHOD means the function stores an diff --git a/Include/modsupport.h b/Include/modsupport.h index 4c4aab65bac103..f009d586bf6202 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -136,7 +136,15 @@ PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywords( void _PyArg_Fini(void); #endif /* Py_LIMITED_API */ -PyAPI_FUNC(int) PyModule_AddObject(PyObject *, const char *, PyObject *); +// Add an attribute with name 'name' and value 'obj' to the module 'mod. +// On success, return 0 on success. +// On error, raise an exception and return -1. +PyAPI_FUNC(int) PyModule_AddObjectRef(PyObject *mod, const char *name, PyObject *value); + +// Similar to PyModule_AddObjectRef() but steal a reference to 'obj' +// (Py_DECREF(obj)) on success (if it returns 0). +PyAPI_FUNC(int) PyModule_AddObject(PyObject *mod, const char *, PyObject *value); + PyAPI_FUNC(int) PyModule_AddIntConstant(PyObject *, const char *, long); PyAPI_FUNC(int) PyModule_AddStringConstant(PyObject *, const char *, const char *); #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03090000 diff --git a/Include/moduleobject.h b/Include/moduleobject.h index cf9ad40c0a17a0..49b116ca1c3587 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -84,6 +84,12 @@ typedef struct PyModuleDef{ freefunc m_free; } PyModuleDef; + +// Internal C API +#ifdef Py_BUILD_CORE +extern int _PyModule_IsExtension(PyObject *obj); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/object.h b/Include/object.h index 6ee4ee7848551e..4c069998574b4a 100644 --- a/Include/object.h +++ b/Include/object.h @@ -54,11 +54,11 @@ whose size is determined when the object is allocated. /* Py_DEBUG implies Py_REF_DEBUG. */ #if defined(Py_DEBUG) && !defined(Py_REF_DEBUG) -#define Py_REF_DEBUG +# define Py_REF_DEBUG #endif -#if defined(Py_LIMITED_API) && defined(Py_REF_DEBUG) -#error Py_LIMITED_API is incompatible with Py_DEBUG, Py_TRACE_REFS, and Py_REF_DEBUG +#if defined(Py_LIMITED_API) && defined(Py_TRACE_REFS) +# error Py_LIMITED_API is incompatible with Py_TRACE_REFS #endif /* PyTypeObject structure is defined in cpython/object.h. @@ -74,8 +74,8 @@ typedef struct _typeobject PyTypeObject; #define _PyObject_EXTRA_INIT 0, 0, #else -#define _PyObject_HEAD_EXTRA -#define _PyObject_EXTRA_INIT +# define _PyObject_HEAD_EXTRA +# define _PyObject_EXTRA_INIT #endif /* PyObject_HEAD defines the initial segment of every PyObject. */ @@ -122,22 +122,22 @@ typedef struct { #define _PyVarObject_CAST_CONST(op) ((const PyVarObject*)(op)) +// Test if the 'x' object is the 'y' object, the same as "x is y" in Python. +PyAPI_FUNC(int) Py_Is(PyObject *x, PyObject *y); +#define Py_Is(x, y) ((x) == (y)) + + static inline Py_ssize_t _Py_REFCNT(const PyObject *ob) { return ob->ob_refcnt; } #define Py_REFCNT(ob) _Py_REFCNT(_PyObject_CAST_CONST(ob)) -static inline Py_ssize_t _Py_SIZE(const PyVarObject *ob) { - return ob->ob_size; -} -#define Py_SIZE(ob) _Py_SIZE(_PyVarObject_CAST_CONST(ob)) +// bpo-39573: The Py_SET_TYPE() function must be used to set an object type. +#define Py_TYPE(ob) (_PyObject_CAST(ob)->ob_type) - -static inline PyTypeObject* _Py_TYPE(const PyObject *ob) { - return ob->ob_type; -} -#define Py_TYPE(ob) _Py_TYPE(_PyObject_CAST_CONST(ob)) +// bpo-39573: The Py_SET_SIZE() function must be used to set an object size. +#define Py_SIZE(ob) (_PyVarObject_CAST(ob)->ob_size) static inline int _Py_IS_TYPE(const PyObject *ob, const PyTypeObject *type) { @@ -240,8 +240,11 @@ PyAPI_FUNC(void *) PyType_GetModuleState(struct _typeobject *); /* Generic type check */ PyAPI_FUNC(int) PyType_IsSubtype(PyTypeObject *, PyTypeObject *); -#define PyObject_TypeCheck(ob, tp) \ - (Py_IS_TYPE(ob, tp) || PyType_IsSubtype(Py_TYPE(ob), (tp))) + +static inline int _PyObject_TypeCheck(PyObject *ob, PyTypeObject *type) { + return Py_IS_TYPE(ob, type) || PyType_IsSubtype(Py_TYPE(ob), type); +} +#define PyObject_TypeCheck(ob, type) _PyObject_TypeCheck(_PyObject_CAST(ob), type) PyAPI_DATA(PyTypeObject) PyType_Type; /* built-in 'type' */ PyAPI_DATA(PyTypeObject) PyBaseObject_Type; /* built-in 'object' */ @@ -317,6 +320,20 @@ Code can use PyType_HasFeature(type_ob, flag_value) to test whether the given type object has a specified feature. */ +#ifndef Py_LIMITED_API +/* Set if instances of the type object are treated as sequences for pattern matching */ +#define Py_TPFLAGS_SEQUENCE (1 << 5) +/* Set if instances of the type object are treated as mappings for pattern matching */ +#define Py_TPFLAGS_MAPPING (1 << 6) +#endif + +/* Disallow creating instances of the type: set tp_new to NULL and don't create + * the "__new__" key in the type dictionary. */ +#define Py_TPFLAGS_DISALLOW_INSTANTIATION (1UL << 7) + +/* Set if the type object is immutable: type attributes cannot be set nor deleted */ +#define Py_TPFLAGS_IMMUTABLETYPE (1UL << 8) + /* Set if the type object is dynamically allocated */ #define Py_TPFLAGS_HEAPTYPE (1UL << 9) @@ -356,6 +373,16 @@ given type object has a specified feature. /* Type is abstract and cannot be instantiated */ #define Py_TPFLAGS_IS_ABSTRACT (1UL << 20) +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 +/* Type has am_send entry in tp_as_async slot */ +#define Py_TPFLAGS_HAVE_AM_SEND (1UL << 21) +#endif + +// This undocumented flag gives certain built-ins their unique pattern-matching +// behavior, which allows a single positional subpattern to match against the +// subject itself (rather than a mapped attribute on it): +#define _Py_TPFLAGS_MATCH_SELF (1UL << 22) + /* These flags are used to determine if a type is a subclass. */ #define Py_TPFLAGS_LONG_SUBCLASS (1UL << 24) #define Py_TPFLAGS_LIST_SUBCLASS (1UL << 25) @@ -419,22 +446,46 @@ PyAPI_FUNC(void) _Py_NegativeRefcount(const char *filename, int lineno, PyAPI_FUNC(void) _Py_Dealloc(PyObject *); +/* +These are provided as conveniences to Python runtime embedders, so that +they can have object code that is not dependent on Python compilation flags. +*/ +PyAPI_FUNC(void) Py_IncRef(PyObject *); +PyAPI_FUNC(void) Py_DecRef(PyObject *); + +// Similar to Py_IncRef() and Py_DecRef() but the argument must be non-NULL. +// Private functions used by Py_INCREF() and Py_DECREF(). +PyAPI_FUNC(void) _Py_IncRef(PyObject *); +PyAPI_FUNC(void) _Py_DecRef(PyObject *); + static inline void _Py_INCREF(PyObject *op) { +#if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000 + // Stable ABI for Python 3.10 built in debug mode. + _Py_IncRef(op); +#else + // Non-limited C API and limited C API for Python 3.9 and older access + // directly PyObject.ob_refcnt. #ifdef Py_REF_DEBUG _Py_RefTotal++; #endif op->ob_refcnt++; +#endif } - #define Py_INCREF(op) _Py_INCREF(_PyObject_CAST(op)) static inline void _Py_DECREF( -#ifdef Py_REF_DEBUG +#if defined(Py_REF_DEBUG) && !(defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000) const char *filename, int lineno, #endif PyObject *op) { +#if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000 + // Stable ABI for Python 3.10 built in debug mode. + _Py_DecRef(op); +#else + // Non-limited C API and limited C API for Python 3.9 and older access + // directly PyObject.ob_refcnt. #ifdef Py_REF_DEBUG _Py_RefTotal--; #endif @@ -448,9 +499,9 @@ static inline void _Py_DECREF( else { _Py_Dealloc(op); } +#endif } - -#ifdef Py_REF_DEBUG +#if defined(Py_REF_DEBUG) && !(defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000) # define Py_DECREF(op) _Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) #else # define Py_DECREF(op) _Py_DECREF(_PyObject_CAST(op)) @@ -519,12 +570,31 @@ static inline void _Py_XDECREF(PyObject *op) #define Py_XDECREF(op) _Py_XDECREF(_PyObject_CAST(op)) -/* -These are provided as conveniences to Python runtime embedders, so that -they can have object code that is not dependent on Python compilation flags. -*/ -PyAPI_FUNC(void) Py_IncRef(PyObject *); -PyAPI_FUNC(void) Py_DecRef(PyObject *); +// Create a new strong reference to an object: +// increment the reference count of the object and return the object. +PyAPI_FUNC(PyObject*) Py_NewRef(PyObject *obj); + +// Similar to Py_NewRef(), but the object can be NULL. +PyAPI_FUNC(PyObject*) Py_XNewRef(PyObject *obj); + +static inline PyObject* _Py_NewRef(PyObject *obj) +{ + Py_INCREF(obj); + return obj; +} + +static inline PyObject* _Py_XNewRef(PyObject *obj) +{ + Py_XINCREF(obj); + return obj; +} + +// Py_NewRef() and Py_XNewRef() are exported as functions for the stable ABI. +// Names overriden with macros by static inline functions for best +// performances. +#define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj)) +#define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj)) + /* _Py_NoneStruct is an object of undefined type which can be used in contexts @@ -535,8 +605,12 @@ Don't forget to apply Py_INCREF() when returning this value!!! PyAPI_DATA(PyObject) _Py_NoneStruct; /* Don't use this directly */ #define Py_None (&_Py_NoneStruct) +// Test if an object is the None singleton, the same as "x is None" in Python. +PyAPI_FUNC(int) Py_IsNone(PyObject *x); +#define Py_IsNone(x) Py_Is((x), Py_None) + /* Macro for returning Py_None from a function */ -#define Py_RETURN_NONE return Py_INCREF(Py_None), Py_None +#define Py_RETURN_NONE return Py_NewRef(Py_None) /* Py_NotImplemented is a singleton used to signal that an operation is @@ -546,8 +620,7 @@ PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */ #define Py_NotImplemented (&_Py_NotImplementedStruct) /* Macro for returning Py_NotImplemented from a function */ -#define Py_RETURN_NOTIMPLEMENTED \ - return Py_INCREF(Py_NotImplemented), Py_NotImplemented +#define Py_RETURN_NOTIMPLEMENTED return Py_NewRef(Py_NotImplemented) /* Rich comparison opcodes */ #define Py_LT 0 @@ -557,6 +630,15 @@ PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */ #define Py_GT 4 #define Py_GE 5 +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 +/* Result of calling PyIter_Send */ +typedef enum { + PYGEN_RETURN = 0, + PYGEN_ERROR = -1, + PYGEN_NEXT = 1, +} PySendResult; +#endif + /* * Macro for implementing rich comparisons * diff --git a/Include/objimpl.h b/Include/objimpl.h index af537175bfed85..450befad679e70 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -38,7 +38,7 @@ Functions and macros for modules that implement new object types. object with room for n items. In addition to the refcount and type pointer fields, this also fills in the ob_size field. - - PyObject_Del(op) releases the memory allocated for an object. It does not + - PyObject_Free(op) releases the memory allocated for an object. It does not run a destructor -- it only frees the memory. PyObject_Free is identical. - PyObject_Init(op, typeobj) and PyObject_InitVar(op, typeobj, n) don't @@ -48,8 +48,8 @@ Functions and macros for modules that implement new object types. Note that objects created with PyObject_{New, NewVar} are allocated using the specialized Python allocator (implemented in obmalloc.c), if WITH_PYMALLOC is -enabled. In addition, a special debugging allocator is used if PYMALLOC_DEBUG -is also #defined. +enabled. In addition, a special debugging allocator is used if Py_DEBUG +macro is also defined. In case a specific form of memory management is needed (for example, if you must use the platform malloc heap(s), or shared memory, or C++ local storage or @@ -102,7 +102,9 @@ PyAPI_FUNC(void *) PyObject_Realloc(void *ptr, size_t new_size); PyAPI_FUNC(void) PyObject_Free(void *ptr); -/* Macros */ +// Deprecated aliases only kept for backward compatibility. +// PyObject_Del and PyObject_DEL are defined with no parameter to be able to +// use them as function pointers (ex: tp_free = PyObject_Del). #define PyObject_MALLOC PyObject_Malloc #define PyObject_REALLOC PyObject_Realloc #define PyObject_FREE PyObject_Free @@ -138,8 +140,8 @@ PyAPI_FUNC(PyVarObject *) _PyObject_NewVar(PyTypeObject *, Py_ssize_t); #define PyObject_NewVar(type, typeobj, n) \ ( (type *) _PyObject_NewVar((typeobj), (n)) ) -// Alias to PyObject_New(). In Python 3.8, PyObject_NEW() called directly -// PyObject_MALLOC() with _PyObject_VAR_SIZE(). +// Alias to PyObject_NewVar(). In Python 3.8, PyObject_NEW_VAR() called +// directly PyObject_MALLOC() with _PyObject_VAR_SIZE(). #define PyObject_NEW_VAR(type, typeobj, n) PyObject_NewVar(type, typeobj, n) @@ -148,8 +150,12 @@ PyAPI_FUNC(PyVarObject *) _PyObject_NewVar(PyTypeObject *, Py_ssize_t); * ========================== */ -/* C equivalent of gc.collect() which ignores the state of gc.enabled. */ +/* C equivalent of gc.collect(). */ PyAPI_FUNC(Py_ssize_t) PyGC_Collect(void); +/* C API for controlling the state of the garbage collector */ +PyAPI_FUNC(int) PyGC_Enable(void); +PyAPI_FUNC(int) PyGC_Disable(void); +PyAPI_FUNC(int) PyGC_IsEnabled(void); /* Test if a type has a GC head */ #define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) diff --git a/Include/opcode.h b/Include/opcode.h index 420c87aa0f24f3..52039754bd88ea 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -30,7 +30,11 @@ extern "C" { #define BINARY_TRUE_DIVIDE 27 #define INPLACE_FLOOR_DIVIDE 28 #define INPLACE_TRUE_DIVIDE 29 -#define RERAISE 48 +#define GET_LEN 30 +#define MATCH_MAPPING 31 +#define MATCH_SEQUENCE 32 +#define MATCH_KEYS 33 +#define COPY_DICT_WITHOUT_KEYS 34 #define WITH_EXCEPT_START 49 #define GET_AITER 50 #define GET_ANEXT 51 @@ -77,6 +81,7 @@ extern "C" { #define DELETE_ATTR 96 #define STORE_GLOBAL 97 #define DELETE_GLOBAL 98 +#define ROT_N 99 #define LOAD_CONST 100 #define LOAD_NAME 101 #define BUILD_TUPLE 102 @@ -96,11 +101,13 @@ extern "C" { #define LOAD_GLOBAL 116 #define IS_OP 117 #define CONTAINS_OP 118 +#define RERAISE 119 #define JUMP_IF_NOT_EXC_MATCH 121 #define SETUP_FINALLY 122 #define LOAD_FAST 124 #define STORE_FAST 125 #define DELETE_FAST 126 +#define GEN_START 129 #define RAISE_VARARGS 130 #define CALL_FUNCTION 131 #define MAKE_FUNCTION 132 @@ -117,6 +124,7 @@ extern "C" { #define SET_ADD 146 #define MAP_ADD 147 #define LOAD_CLASSDEREF 148 +#define MATCH_CLASS 152 #define SETUP_ASYNC_WITH 154 #define FORMAT_VALUE 155 #define BUILD_CONST_KEY_MAP 156 diff --git a/Include/parser_interface.h b/Include/parser_interface.h deleted file mode 100644 index 1c6576d926d8d3..00000000000000 --- a/Include/parser_interface.h +++ /dev/null @@ -1,52 +0,0 @@ -#ifndef Py_PEGENINTERFACE -#define Py_PEGENINTERFACE -#ifdef __cplusplus -extern "C" { -#endif - -#include "Python.h" - -#ifndef Py_LIMITED_API -PyAPI_FUNC(struct _mod *) PyParser_ASTFromString( - const char *str, - const char *filename, - int mode, - PyCompilerFlags *flags, - PyArena *arena); -PyAPI_FUNC(struct _mod *) PyParser_ASTFromStringObject( - const char *str, - PyObject* filename, - int mode, - PyCompilerFlags *flags, - PyArena *arena); -PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile( - FILE *fp, - const char *filename, - const char* enc, - int mode, - const char *ps1, - const char *ps2, - PyCompilerFlags *flags, - int *errcode, - PyArena *arena); -PyAPI_FUNC(struct _mod *) PyParser_ASTFromFileObject( - FILE *fp, - PyObject *filename_ob, - const char *enc, - int mode, - const char *ps1, - const char *ps2, - PyCompilerFlags *flags, - int *errcode, - PyArena *arena); -PyAPI_FUNC(struct _mod *) PyParser_ASTFromFilename( - const char *filename, - int mode, - PyCompilerFlags *flags, - PyArena *arena); -#endif /* !Py_LIMITED_API */ - -#ifdef __cplusplus -} -#endif -#endif /* !Py_PEGENINTERFACE */ diff --git a/Include/patchlevel.h b/Include/patchlevel.h index eeb23af9766700..f37c4d48e37607 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -17,13 +17,13 @@ /* Version parsed out into numeric values */ /*--start constants--*/ #define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 10 +#define PY_MINOR_VERSION 11 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.10.0a1+" +#define PY_VERSION "3.11.0a0" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Include/pyarena.h b/Include/pyarena.h deleted file mode 100644 index db3ad0188fe1cd..00000000000000 --- a/Include/pyarena.h +++ /dev/null @@ -1,64 +0,0 @@ -/* An arena-like memory interface for the compiler. - */ - -#ifndef Py_LIMITED_API -#ifndef Py_PYARENA_H -#define Py_PYARENA_H - -#ifdef __cplusplus -extern "C" { -#endif - - typedef struct _arena PyArena; - - /* PyArena_New() and PyArena_Free() create a new arena and free it, - respectively. Once an arena has been created, it can be used - to allocate memory via PyArena_Malloc(). Pointers to PyObject can - also be registered with the arena via PyArena_AddPyObject(), and the - arena will ensure that the PyObjects stay alive at least until - PyArena_Free() is called. When an arena is freed, all the memory it - allocated is freed, the arena releases internal references to registered - PyObject*, and none of its pointers are valid. - XXX (tim) What does "none of its pointers are valid" mean? Does it - XXX mean that pointers previously obtained via PyArena_Malloc() are - XXX no longer valid? (That's clearly true, but not sure that's what - XXX the text is trying to say.) - - PyArena_New() returns an arena pointer. On error, it - returns a negative number and sets an exception. - XXX (tim): Not true. On error, PyArena_New() actually returns NULL, - XXX and looks like it may or may not set an exception (e.g., if the - XXX internal PyList_New(0) returns NULL, PyArena_New() passes that on - XXX and an exception is set; OTOH, if the internal - XXX block_new(DEFAULT_BLOCK_SIZE) returns NULL, that's passed on but - XXX an exception is not set in that case). - */ - PyAPI_FUNC(PyArena *) PyArena_New(void); - PyAPI_FUNC(void) PyArena_Free(PyArena *); - - /* Mostly like malloc(), return the address of a block of memory spanning - * `size` bytes, or return NULL (without setting an exception) if enough - * new memory can't be obtained. Unlike malloc(0), PyArena_Malloc() with - * size=0 does not guarantee to return a unique pointer (the pointer - * returned may equal one or more other pointers obtained from - * PyArena_Malloc()). - * Note that pointers obtained via PyArena_Malloc() must never be passed to - * the system free() or realloc(), or to any of Python's similar memory- - * management functions. PyArena_Malloc()-obtained pointers remain valid - * until PyArena_Free(ar) is called, at which point all pointers obtained - * from the arena `ar` become invalid simultaneously. - */ - PyAPI_FUNC(void *) PyArena_Malloc(PyArena *, size_t size); - - /* This routine isn't a proper arena allocation routine. It takes - * a PyObject* and records it so that it can be DECREFed when the - * arena is freed. - */ - PyAPI_FUNC(int) PyArena_AddPyObject(PyArena *, PyObject *); - -#ifdef __cplusplus -} -#endif - -#endif /* !Py_PYARENA_H */ -#endif /* Py_LIMITED_API */ diff --git a/Include/pydecimal.h b/Include/pydecimal.h deleted file mode 100644 index 9b6440e1c2ab1c..00000000000000 --- a/Include/pydecimal.h +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright (c) 2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#ifndef CPYTHON_DECIMAL_H_ -#define CPYTHON_DECIMAL_H_ - - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/****************************************************************************/ -/* Libmpdec API */ -/****************************************************************************/ - -#ifndef LIBMPDEC_MPDECIMAL_H_ -struct mpd_t; /* ABI-stable in the libmpdec-2.x series */ - -/* status cases for getting a triple */ -enum mpd_triple_class { - MPD_TRIPLE_NORMAL, - MPD_TRIPLE_INF, - MPD_TRIPLE_QNAN, - MPD_TRIPLE_SNAN, - MPD_TRIPLE_ERROR, -}; - -typedef struct { - enum mpd_triple_class tag; - uint8_t sign; - uint64_t hi; - uint64_t lo; - int64_t exp; -} mpd_uint128_triple_t; -#endif - - -/****************************************************************************/ -/* Capsule API */ -/****************************************************************************/ - -/* Simple API */ -#define PyDec_TypeCheck_INDEX 0 -#define PyDec_TypeCheck_RETURN int -#define PyDec_TypeCheck_ARGS (const PyObject *) - -#define PyDec_IsSpecial_INDEX 1 -#define PyDec_IsSpecial_RETURN int -#define PyDec_IsSpecial_ARGS (const PyObject *) - -#define PyDec_IsNaN_INDEX 2 -#define PyDec_IsNaN_RETURN int -#define PyDec_IsNaN_ARGS (const PyObject *) - -#define PyDec_IsInfinite_INDEX 3 -#define PyDec_IsInfinite_RETURN int -#define PyDec_IsInfinite_ARGS (const PyObject *) - -#define PyDec_GetDigits_INDEX 4 -#define PyDec_GetDigits_RETURN int64_t -#define PyDec_GetDigits_ARGS (const PyObject *) - -#define PyDec_AsUint128Triple_INDEX 5 -#define PyDec_AsUint128Triple_RETURN mpd_uint128_triple_t -#define PyDec_AsUint128Triple_ARGS (const PyObject *) - -#define PyDec_FromUint128Triple_INDEX 6 -#define PyDec_FromUint128Triple_RETURN PyObject * -#define PyDec_FromUint128Triple_ARGS (const mpd_uint128_triple_t *triple) - -/* Advanced API */ -#define PyDec_Alloc_INDEX 7 -#define PyDec_Alloc_RETURN PyObject * -#define PyDec_Alloc_ARGS (void) - -#define PyDec_Get_INDEX 8 -#define PyDec_Get_RETURN mpd_t * -#define PyDec_Get_ARGS (PyObject *) - -#define PyDec_GetConst_INDEX 9 -#define PyDec_GetConst_RETURN const mpd_t * -#define PyDec_GetConst_ARGS (const PyObject *) - -#define CPYTHON_DECIMAL_MAX_API 10 - - -#ifdef CPYTHON_DECIMAL_MODULE -/* Simple API */ -static PyDec_TypeCheck_RETURN PyDec_TypeCheck PyDec_TypeCheck_ARGS; -static PyDec_IsSpecial_RETURN PyDec_IsSpecial PyDec_IsSpecial_ARGS; -static PyDec_IsNaN_RETURN PyDec_IsNaN PyDec_IsNaN_ARGS; -static PyDec_IsInfinite_RETURN PyDec_IsInfinite PyDec_IsInfinite_ARGS; -static PyDec_GetDigits_RETURN PyDec_GetDigits PyDec_GetDigits_ARGS; -static PyDec_AsUint128Triple_RETURN PyDec_AsUint128Triple PyDec_AsUint128Triple_ARGS; -static PyDec_FromUint128Triple_RETURN PyDec_FromUint128Triple PyDec_FromUint128Triple_ARGS; - -/* Advanced API */ -static PyDec_Alloc_RETURN PyDec_Alloc PyDec_Alloc_ARGS; -static PyDec_Get_RETURN PyDec_Get PyDec_Get_ARGS; -static PyDec_GetConst_RETURN PyDec_GetConst PyDec_GetConst_ARGS; -#else -static void **_decimal_api; - -/* Simple API */ -#define PyDec_TypeCheck \ - (*(PyDec_TypeCheck_RETURN (*)PyDec_TypeCheck_ARGS) _decimal_api[PyDec_TypeCheck_INDEX]) - -#define PyDec_IsSpecial \ - (*(PyDec_IsSpecial_RETURN (*)PyDec_IsSpecial_ARGS) _decimal_api[PyDec_IsSpecial_INDEX]) - -#define PyDec_IsNaN \ - (*(PyDec_IsNaN_RETURN (*)PyDec_IsNaN_ARGS) _decimal_api[PyDec_IsNaN_INDEX]) - -#define PyDec_IsInfinite \ - (*(PyDec_IsInfinite_RETURN (*)PyDec_IsInfinite_ARGS) _decimal_api[PyDec_IsInfinite_INDEX]) - -#define PyDec_GetDigits \ - (*(PyDec_GetDigits_RETURN (*)PyDec_GetDigits_ARGS) _decimal_api[PyDec_GetDigits_INDEX]) - -#define PyDec_AsUint128Triple \ - (*(PyDec_AsUint128Triple_RETURN (*)PyDec_AsUint128Triple_ARGS) _decimal_api[PyDec_AsUint128Triple_INDEX]) - -#define PyDec_FromUint128Triple \ - (*(PyDec_FromUint128Triple_RETURN (*)PyDec_FromUint128Triple_ARGS) _decimal_api[PyDec_FromUint128Triple_INDEX]) - -/* Advanced API */ -#define PyDec_Alloc \ - (*(PyDec_Alloc_RETURN (*)PyDec_Alloc_ARGS) _decimal_api[PyDec_Alloc_INDEX]) - -#define PyDec_Get \ - (*(PyDec_Get_RETURN (*)PyDec_Get_ARGS) _decimal_api[PyDec_Get_INDEX]) - -#define PyDec_GetConst \ - (*(PyDec_GetConst_RETURN (*)PyDec_GetConst_ARGS) _decimal_api[PyDec_GetConst_INDEX]) - - -static int -import_decimal(void) -{ - _decimal_api = (void **)PyCapsule_Import("_decimal._API", 0); - if (_decimal_api == NULL) { - return -1; - } - - return 0; -} -#endif - -#ifdef __cplusplus -} -#endif - -#endif /* CPYTHON_DECIMAL_H_ */ diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 979a26ba68a033..f5d1c711577186 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -30,12 +30,6 @@ PyAPI_FUNC(void) PyErr_SetExcInfo(PyObject *, PyObject *, PyObject *); macro is defined. */ PyAPI_FUNC(void) _Py_NO_RETURN Py_FatalError(const char *message); -#if defined(Py_DEBUG) || defined(Py_LIMITED_API) -#define _PyErr_OCCURRED() PyErr_Occurred() -#else -#define _PyErr_OCCURRED() (PyThreadState_GET()->curexc_type) -#endif - /* Error testing and normalization */ PyAPI_FUNC(int) PyErr_GivenExceptionMatches(PyObject *, PyObject *); PyAPI_FUNC(int) PyErr_ExceptionMatches(PyObject *); @@ -152,6 +146,7 @@ PyAPI_DATA(PyObject *) PyExc_FutureWarning; PyAPI_DATA(PyObject *) PyExc_ImportWarning; PyAPI_DATA(PyObject *) PyExc_UnicodeWarning; PyAPI_DATA(PyObject *) PyExc_BytesWarning; +PyAPI_DATA(PyObject *) PyExc_EncodingWarning; PyAPI_DATA(PyObject *) PyExc_ResourceWarning; @@ -230,6 +225,9 @@ PyAPI_FUNC(void) PyErr_WriteUnraisable(PyObject *); /* In signalmodule.c */ PyAPI_FUNC(int) PyErr_CheckSignals(void); PyAPI_FUNC(void) PyErr_SetInterrupt(void); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 +PyAPI_FUNC(int) PyErr_SetInterruptEx(int signum); +#endif /* Support for adding program text to SyntaxErrors */ PyAPI_FUNC(void) PyErr_SyntaxLocation( diff --git a/Include/pyhash.h b/Include/pyhash.h index 4437b870332bde..a314ea907b7fe2 100644 --- a/Include/pyhash.h +++ b/Include/pyhash.h @@ -7,7 +7,7 @@ extern "C" { /* Helpers for hash functions */ #ifndef Py_LIMITED_API -PyAPI_FUNC(Py_hash_t) _Py_HashDouble(double); +PyAPI_FUNC(Py_hash_t) _Py_HashDouble(PyObject *, double); PyAPI_FUNC(Py_hash_t) _Py_HashPointer(const void*); // Similar to _Py_HashPointer(), but don't replace -1 with -2 PyAPI_FUNC(Py_hash_t) _Py_HashPointerRaw(const void*); @@ -29,7 +29,6 @@ PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void*, Py_ssize_t); #define _PyHASH_MODULUS (((size_t)1 << _PyHASH_BITS) - 1) #define _PyHASH_INF 314159 -#define _PyHASH_NAN 0 #define _PyHASH_IMAG _PyHASH_MULTIPLIER @@ -77,7 +76,6 @@ typedef union { } expat; } _Py_HashSecret_t; PyAPI_DATA(_Py_HashSecret_t) _Py_HashSecret; -#endif #ifdef Py_DEBUG PyAPI_DATA(int) _Py_HashSecret_Initialized; @@ -85,7 +83,6 @@ PyAPI_DATA(int) _Py_HashSecret_Initialized; /* hash function definition */ -#ifndef Py_LIMITED_API typedef struct { Py_hash_t (*const hash)(const void *, Py_ssize_t); const char *name; diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index c5368b3c5edaa0..783fcb455eb528 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -32,6 +32,8 @@ PyAPI_FUNC(void) _Py_NO_RETURN Py_Exit(int); /* Bootstrap __main__ (defined in Modules/main.c) */ PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv); +PyAPI_FUNC(int) Py_FrozenMain(int argc, char **argv); + PyAPI_FUNC(int) Py_BytesMain(int argc, char **argv); /* In pathconfig.c */ diff --git a/Include/pymem.h b/Include/pymem.h index 607feb9484f24f..66cdb0d2973cd0 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -25,8 +25,8 @@ extern "C" { heap used by the Python DLL; it could be a disaster if you free()'ed that directly in your own extension. Using PyMem_Free instead ensures Python can return the memory to the proper heap. As another example, in - PYMALLOC_DEBUG mode, Python wraps all calls to all PyMem_ and PyObject_ - memory functions in special debugging wrappers that add additional + a debug build (Py_DEBUG macro), Python wraps all calls to all PyMem_ and + PyObject_ memory functions in special debugging wrappers that add additional debugging info to dynamic memory blocks. The system routines have no idea what to do with that stuff, and the Python wrappers have no idea what to do with raw blocks obtained directly by the system routines then. @@ -50,21 +50,10 @@ extern "C" { */ PyAPI_FUNC(void *) PyMem_Malloc(size_t size); +PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize); PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size); PyAPI_FUNC(void) PyMem_Free(void *ptr); -/* Macros. */ - -/* PyMem_MALLOC(0) means malloc(1). Some systems would return NULL - for malloc(0), which would be treated as an error. Some platforms - would return a pointer with no memory behind it, which would break - pymalloc. To solve these problems, allocate an extra byte. */ -/* Returns NULL to indicate error if a negative size or size larger than - Py_ssize_t can represent is supplied. Helps prevents security holes. */ -#define PyMem_MALLOC(n) PyMem_Malloc(n) -#define PyMem_REALLOC(p, n) PyMem_Realloc(p, n) -#define PyMem_FREE(p) PyMem_Free(p) - /* * Type-oriented memory interface * ============================== @@ -78,9 +67,6 @@ PyAPI_FUNC(void) PyMem_Free(void *ptr); #define PyMem_New(type, n) \ ( ((size_t)(n) > PY_SSIZE_T_MAX / sizeof(type)) ? NULL : \ ( (type *) PyMem_Malloc((n) * sizeof(type)) ) ) -#define PyMem_NEW(type, n) \ - ( ((size_t)(n) > PY_SSIZE_T_MAX / sizeof(type)) ? NULL : \ - ( (type *) PyMem_MALLOC((n) * sizeof(type)) ) ) /* * The value of (p) is always clobbered by this macro regardless of success. @@ -91,15 +77,18 @@ PyAPI_FUNC(void) PyMem_Free(void *ptr); #define PyMem_Resize(p, type, n) \ ( (p) = ((size_t)(n) > PY_SSIZE_T_MAX / sizeof(type)) ? NULL : \ (type *) PyMem_Realloc((p), (n) * sizeof(type)) ) -#define PyMem_RESIZE(p, type, n) \ - ( (p) = ((size_t)(n) > PY_SSIZE_T_MAX / sizeof(type)) ? NULL : \ - (type *) PyMem_REALLOC((p), (n) * sizeof(type)) ) -/* PyMem{Del,DEL} are left over from ancient days, and shouldn't be used - * anymore. They're just confusing aliases for PyMem_{Free,FREE} now. - */ -#define PyMem_Del PyMem_Free -#define PyMem_DEL PyMem_FREE + +// Deprecated aliases only kept for backward compatibility. +// PyMem_Del and PyMem_DEL are defined with no parameter to be able to use +// them as function pointers (ex: dealloc = PyMem_Del). +#define PyMem_MALLOC(n) PyMem_Malloc(n) +#define PyMem_NEW(type, n) PyMem_New(type, n) +#define PyMem_REALLOC(p, n) PyMem_Realloc(p, n) +#define PyMem_RESIZE(p, type, n) PyMem_Resize(p, type, n) +#define PyMem_FREE(p) PyMem_Free(p) +#define PyMem_Del PyMem_Free +#define PyMem_DEL PyMem_Free #ifndef Py_LIMITED_API diff --git a/Include/pyport.h b/Include/pyport.h index 7137006870bf01..6ab0ae4007922a 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -101,7 +101,9 @@ typedef intptr_t Py_intptr_t; * sizeof(size_t). C99 doesn't define such a thing directly (size_t is an * unsigned integral type). See PEP 353 for details. */ -#ifdef HAVE_SSIZE_T +#ifdef HAVE_PY_SSIZE_T + +#elif HAVE_SSIZE_T typedef ssize_t Py_ssize_t; #elif SIZEOF_VOID_P == SIZEOF_SIZE_T typedef Py_intptr_t Py_ssize_t; @@ -181,8 +183,9 @@ typedef int Py_ssize_clean_t; #if defined(_MSC_VER) # if defined(PY_LOCAL_AGGRESSIVE) - /* enable more aggressive optimization for visual studio */ -# pragma optimize("agtw", on) + /* enable more aggressive optimization for MSVC */ + /* active in both release and debug builds - see bpo-43271 */ +# pragma optimize("gt", on) #endif /* ignore warnings if the compiler decides not to inline a function */ # pragma warning(disable: 4710) @@ -841,12 +844,16 @@ extern _invalid_parameter_handler _Py_silent_invalid_parameter_handler; #endif #if defined(__ANDROID__) || defined(__VXWORKS__) - /* Ignore the locale encoding: force UTF-8 */ + // Use UTF-8 as the locale encoding, ignore the LC_CTYPE locale. + // See _Py_GetLocaleEncoding(), PyUnicode_DecodeLocale() + // and PyUnicode_EncodeLocale(). # define _Py_FORCE_UTF8_LOCALE #endif #if defined(_Py_FORCE_UTF8_LOCALE) || defined(__APPLE__) - /* Use UTF-8 as filesystem encoding */ + // Use UTF-8 as the filesystem encoding. + // See PyUnicode_DecodeFSDefaultAndSize(), PyUnicode_EncodeFSDefault(), + // Py_DecodeLocale() and Py_EncodeLocale(). # define _Py_FORCE_UTF8_FS_ENCODING #endif @@ -854,6 +861,7 @@ extern _invalid_parameter_handler _Py_silent_invalid_parameter_handler; PyAPI_FUNC(void) _Py_NO_RETURN PyThread_exit_thread(void); XLC support is intentionally omitted due to bpo-40244 */ +#ifndef _Py_NO_RETURN #if defined(__clang__) || \ (defined(__GNUC__) && \ ((__GNUC__ >= 3) || \ @@ -864,5 +872,18 @@ extern _invalid_parameter_handler _Py_silent_invalid_parameter_handler; #else # define _Py_NO_RETURN #endif +#endif + + +// Preprocessor check for a builtin preprocessor function. Always return 0 +// if __has_builtin() macro is not defined. +// +// __has_builtin() is available on clang and GCC 10. +#ifdef __has_builtin +# define _Py__has_builtin(x) __has_builtin(x) +#else +# define _Py__has_builtin(x) 0 +#endif + #endif /* Py_PYPORT_H */ diff --git a/Include/pythonrun.h b/Include/pythonrun.h index d43734b5a12ff0..b0a2fc3002d37d 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -7,137 +7,15 @@ extern "C" { #endif -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *); -PyAPI_FUNC(int) PyRun_AnyFileExFlags( - FILE *fp, - const char *filename, /* decoded from the filesystem encoding */ - int closeit, - PyCompilerFlags *flags); -PyAPI_FUNC(int) PyRun_SimpleFileExFlags( - FILE *fp, - const char *filename, /* decoded from the filesystem encoding */ - int closeit, - PyCompilerFlags *flags); -PyAPI_FUNC(int) PyRun_InteractiveOneFlags( - FILE *fp, - const char *filename, /* decoded from the filesystem encoding */ - PyCompilerFlags *flags); -PyAPI_FUNC(int) PyRun_InteractiveOneObject( - FILE *fp, - PyObject *filename, - PyCompilerFlags *flags); -PyAPI_FUNC(int) PyRun_InteractiveLoopFlags( - FILE *fp, - const char *filename, /* decoded from the filesystem encoding */ - PyCompilerFlags *flags); - - -PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *, - PyObject *, PyCompilerFlags *); - -PyAPI_FUNC(PyObject *) PyRun_FileExFlags( - FILE *fp, - const char *filename, /* decoded from the filesystem encoding */ - int start, - PyObject *globals, - PyObject *locals, - int closeit, - PyCompilerFlags *flags); -#endif - -#ifdef Py_LIMITED_API PyAPI_FUNC(PyObject *) Py_CompileString(const char *, const char *, int); -#else -#define Py_CompileString(str, p, s) Py_CompileStringExFlags(str, p, s, NULL, -1) -#define Py_CompileStringFlags(str, p, s, f) Py_CompileStringExFlags(str, p, s, f, -1) -PyAPI_FUNC(PyObject *) Py_CompileStringExFlags( - const char *str, - const char *filename, /* decoded from the filesystem encoding */ - int start, - PyCompilerFlags *flags, - int optimize); -PyAPI_FUNC(PyObject *) Py_CompileStringObject( - const char *str, - PyObject *filename, int start, - PyCompilerFlags *flags, - int optimize); -#endif -PyAPI_FUNC(struct symtable *) Py_SymtableString( - const char *str, - const char *filename, /* decoded from the filesystem encoding */ - int start); -#ifndef Py_LIMITED_API -PyAPI_FUNC(const char *) _Py_SourceAsString( - PyObject *cmd, - const char *funcname, - const char *what, - PyCompilerFlags *cf, - PyObject **cmd_copy); - -PyAPI_FUNC(struct symtable *) Py_SymtableStringObject( - const char *str, - PyObject *filename, - int start); - -PyAPI_FUNC(struct symtable *) _Py_SymtableStringObjectFlags( - const char *str, - PyObject *filename, - int start, - PyCompilerFlags *flags); -#endif PyAPI_FUNC(void) PyErr_Print(void); PyAPI_FUNC(void) PyErr_PrintEx(int); PyAPI_FUNC(void) PyErr_Display(PyObject *, PyObject *, PyObject *); -#ifndef Py_LIMITED_API -/* A function flavor is also exported by libpython. It is required when - libpython is accessed directly rather than using header files which defines - macros below. On Windows, for example, PyAPI_FUNC() uses dllexport to - export functions in pythonXX.dll. */ -PyAPI_FUNC(PyObject *) PyRun_String(const char *str, int s, PyObject *g, PyObject *l); -PyAPI_FUNC(int) PyRun_AnyFile(FILE *fp, const char *name); -PyAPI_FUNC(int) PyRun_AnyFileEx(FILE *fp, const char *name, int closeit); -PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, const char *, PyCompilerFlags *); -PyAPI_FUNC(int) PyRun_SimpleString(const char *s); -PyAPI_FUNC(int) PyRun_SimpleFile(FILE *f, const char *p); -PyAPI_FUNC(int) PyRun_SimpleFileEx(FILE *f, const char *p, int c); -PyAPI_FUNC(int) PyRun_InteractiveOne(FILE *f, const char *p); -PyAPI_FUNC(int) PyRun_InteractiveLoop(FILE *f, const char *p); -PyAPI_FUNC(PyObject *) PyRun_File(FILE *fp, const char *p, int s, PyObject *g, PyObject *l); -PyAPI_FUNC(PyObject *) PyRun_FileEx(FILE *fp, const char *p, int s, PyObject *g, PyObject *l, int c); -PyAPI_FUNC(PyObject *) PyRun_FileFlags(FILE *fp, const char *p, int s, PyObject *g, PyObject *l, PyCompilerFlags *flags); - -/* Use macros for a bunch of old variants */ -#define PyRun_String(str, s, g, l) PyRun_StringFlags(str, s, g, l, NULL) -#define PyRun_AnyFile(fp, name) PyRun_AnyFileExFlags(fp, name, 0, NULL) -#define PyRun_AnyFileEx(fp, name, closeit) \ - PyRun_AnyFileExFlags(fp, name, closeit, NULL) -#define PyRun_AnyFileFlags(fp, name, flags) \ - PyRun_AnyFileExFlags(fp, name, 0, flags) -#define PyRun_SimpleString(s) PyRun_SimpleStringFlags(s, NULL) -#define PyRun_SimpleFile(f, p) PyRun_SimpleFileExFlags(f, p, 0, NULL) -#define PyRun_SimpleFileEx(f, p, c) PyRun_SimpleFileExFlags(f, p, c, NULL) -#define PyRun_InteractiveOne(f, p) PyRun_InteractiveOneFlags(f, p, NULL) -#define PyRun_InteractiveLoop(f, p) PyRun_InteractiveLoopFlags(f, p, NULL) -#define PyRun_File(fp, p, s, g, l) \ - PyRun_FileExFlags(fp, p, s, g, l, 0, NULL) -#define PyRun_FileEx(fp, p, s, g, l, c) \ - PyRun_FileExFlags(fp, p, s, g, l, c, NULL) -#define PyRun_FileFlags(fp, p, s, g, l, flags) \ - PyRun_FileExFlags(fp, p, s, g, l, 0, flags) -#endif /* Stuff with no proper home (yet) */ -#ifndef Py_LIMITED_API -PyAPI_FUNC(char *) PyOS_Readline(FILE *, FILE *, const char *); -#endif PyAPI_DATA(int) (*PyOS_InputHook)(void); -PyAPI_DATA(char) *(*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *); -#ifndef Py_LIMITED_API -PyAPI_DATA(PyThreadState*) _PyOS_ReadlineTState; -#endif /* Stack size, in "pointers" (so we get extra safety margins on 64-bit platforms). On a 32-bit platform, this translates @@ -154,6 +32,12 @@ PyAPI_DATA(PyThreadState*) _PyOS_ReadlineTState; PyAPI_FUNC(int) PyOS_CheckStack(void); #endif +#ifndef Py_LIMITED_API +# define Py_CPYTHON_PYTHONRUN_H +# include "cpython/pythonrun.h" +# undef Py_CPYTHON_PYTHONRUN_H +#endif + #ifdef __cplusplus } #endif diff --git a/Include/setobject.h b/Include/setobject.h index 119619ebe72994..62516be5ab29be 100644 --- a/Include/setobject.h +++ b/Include/setobject.h @@ -88,18 +88,21 @@ PyAPI_FUNC(PyObject *) PySet_Pop(PyObject *set); PyAPI_FUNC(Py_ssize_t) PySet_Size(PyObject *anyset); #define PyFrozenSet_CheckExact(ob) Py_IS_TYPE(ob, &PyFrozenSet_Type) +#define PyFrozenSet_Check(ob) \ + (Py_IS_TYPE(ob, &PyFrozenSet_Type) || \ + PyType_IsSubtype(Py_TYPE(ob), &PyFrozenSet_Type)) + #define PyAnySet_CheckExact(ob) \ (Py_IS_TYPE(ob, &PySet_Type) || Py_IS_TYPE(ob, &PyFrozenSet_Type)) #define PyAnySet_Check(ob) \ (Py_IS_TYPE(ob, &PySet_Type) || Py_IS_TYPE(ob, &PyFrozenSet_Type) || \ PyType_IsSubtype(Py_TYPE(ob), &PySet_Type) || \ PyType_IsSubtype(Py_TYPE(ob), &PyFrozenSet_Type)) + +#define PySet_CheckExact(op) Py_IS_TYPE(op, &PySet_Type) #define PySet_Check(ob) \ (Py_IS_TYPE(ob, &PySet_Type) || \ PyType_IsSubtype(Py_TYPE(ob), &PySet_Type)) -#define PyFrozenSet_Check(ob) \ - (Py_IS_TYPE(ob, &PyFrozenSet_Type) || \ - PyType_IsSubtype(Py_TYPE(ob), &PyFrozenSet_Type)) #ifdef __cplusplus } diff --git a/Include/structmember.h b/Include/structmember.h index b54f7081f458dd..93b7aff8091848 100644 --- a/Include/structmember.h +++ b/Include/structmember.h @@ -62,6 +62,7 @@ typedef struct PyMemberDef { #define PY_WRITE_RESTRICTED 4 #define RESTRICTED (READ_RESTRICTED | PY_WRITE_RESTRICTED) +#define PY_AUDIT_READ READ_RESTRICTED /* Current API, use this */ PyAPI_FUNC(PyObject *) PyMember_GetOne(const char *, struct PyMemberDef *); diff --git a/Include/token.h b/Include/token.h index 9b8a3aae074674..eb1b9ea47b469a 100644 --- a/Include/token.h +++ b/Include/token.h @@ -69,8 +69,9 @@ extern "C" { #define ASYNC 56 #define TYPE_IGNORE 57 #define TYPE_COMMENT 58 -#define ERRORTOKEN 59 -#define N_TOKENS 63 +#define SOFT_KEYWORD 59 +#define ERRORTOKEN 60 +#define N_TOKENS 64 #define NT_OFFSET 256 /* Special definitions for cooperation with parser */ diff --git a/Include/typeslots.h b/Include/typeslots.h index 64f6fff5144493..5800d0158bc924 100644 --- a/Include/typeslots.h +++ b/Include/typeslots.h @@ -88,3 +88,7 @@ /* New in 3.5 */ #define Py_tp_finalize 80 #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 +/* New in 3.10 */ +#define Py_am_send 81 +#endif diff --git a/LICENSE b/LICENSE index f42f8adbed845d..473861da1be7c5 100644 --- a/LICENSE +++ b/LICENSE @@ -84,7 +84,7 @@ analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Python Software Foundation; +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Python Software Foundation; All Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee. diff --git a/Lib/__future__.py b/Lib/__future__.py index 0e7b5552343356..97dc90c6e4644a 100644 --- a/Lib/__future__.py +++ b/Lib/__future__.py @@ -42,7 +42,7 @@ argument to the builtin function compile() to enable the feature in dynamically compiled code. This flag is stored in the .compiler_flag attribute on _Future instances. These values must match the appropriate -#defines of CO_xxx flags in Include/compile.h. +#defines of CO_xxx flags in Include/cpython/compile.h. No feature line is ever to be deleted from this file. """ @@ -143,5 +143,5 @@ def __repr__(self): CO_FUTURE_GENERATOR_STOP) annotations = _Feature((3, 7, 0, "beta", 1), - (3, 10, 0, "alpha", 0), + (3, 11, 0, "alpha", 0), CO_FUTURE_ANNOTATIONS) diff --git a/Lib/_bootlocale.py b/Lib/_bootlocale.py deleted file mode 100644 index 3273a3b42252ba..00000000000000 --- a/Lib/_bootlocale.py +++ /dev/null @@ -1,46 +0,0 @@ -"""A minimal subset of the locale module used at interpreter startup -(imported by the _io module), in order to reduce startup time. - -Don't import directly from third-party code; use the `locale` module instead! -""" - -import sys -import _locale - -if sys.platform.startswith("win"): - def getpreferredencoding(do_setlocale=True): - if sys.flags.utf8_mode: - return 'UTF-8' - return _locale._getdefaultlocale()[1] -else: - try: - _locale.CODESET - except AttributeError: - if hasattr(sys, 'getandroidapilevel'): - # On Android langinfo.h and CODESET are missing, and UTF-8 is - # always used in mbstowcs() and wcstombs(). - def getpreferredencoding(do_setlocale=True): - return 'UTF-8' - else: - def getpreferredencoding(do_setlocale=True): - if sys.flags.utf8_mode: - return 'UTF-8' - # This path for legacy systems needs the more complex - # getdefaultlocale() function, import the full locale module. - import locale - return locale.getpreferredencoding(do_setlocale) - else: - def getpreferredencoding(do_setlocale=True): - assert not do_setlocale - if sys.flags.utf8_mode: - return 'UTF-8' - result = _locale.nl_langinfo(_locale.CODESET) - if not result and sys.platform == 'darwin': - # nl_langinfo can return an empty string - # when the setting has an invalid value. - # Default to UTF-8 in that case because - # UTF-8 is the default charset on OSX and - # returning nothing will crash the - # interpreter. - result = 'UTF-8' - return result diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 36cd9930003475..d92b848e9436fb 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -10,6 +10,10 @@ import sys GenericAlias = type(list[int]) +EllipsisType = type(...) +def _f(): pass +FunctionType = type(_f) +del _f __all__ = ["Awaitable", "Coroutine", "AsyncIterable", "AsyncIterator", "AsyncGenerator", @@ -409,6 +413,141 @@ def __subclasshook__(cls, C): return NotImplemented +class _CallableGenericAlias(GenericAlias): + """ Represent `Callable[argtypes, resulttype]`. + + This sets ``__args__`` to a tuple containing the flattened ``argtypes`` + followed by ``resulttype``. + + Example: ``Callable[[int, str], float]`` sets ``__args__`` to + ``(int, str, float)``. + """ + + __slots__ = () + + def __new__(cls, origin, args): + return cls.__create_ga(origin, args) + + @classmethod + def __create_ga(cls, origin, args): + if not isinstance(args, tuple) or len(args) != 2: + raise TypeError( + "Callable must be used as Callable[[arg, ...], result].") + t_args, t_result = args + if isinstance(t_args, (list, tuple)): + ga_args = tuple(t_args) + (t_result,) + # This relaxes what t_args can be on purpose to allow things like + # PEP 612 ParamSpec. Responsibility for whether a user is using + # Callable[...] properly is deferred to static type checkers. + else: + ga_args = args + return super().__new__(cls, origin, ga_args) + + @property + def __parameters__(self): + params = [] + for arg in self.__args__: + # Looks like a genericalias + if hasattr(arg, "__parameters__") and isinstance(arg.__parameters__, tuple): + params.extend(arg.__parameters__) + else: + if _is_typevarlike(arg): + params.append(arg) + return tuple(dict.fromkeys(params)) + + def __repr__(self): + if _has_special_args(self.__args__): + return super().__repr__() + return (f'collections.abc.Callable' + f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' + f'{_type_repr(self.__args__[-1])}]') + + def __reduce__(self): + args = self.__args__ + if not _has_special_args(args): + args = list(args[:-1]), args[-1] + return _CallableGenericAlias, (Callable, args) + + def __getitem__(self, item): + # Called during TypeVar substitution, returns the custom subclass + # rather than the default types.GenericAlias object. Most of the + # code is copied from typing's _GenericAlias and the builtin + # types.GenericAlias. + + # A special case in PEP 612 where if X = Callable[P, int], + # then X[int, str] == X[[int, str]]. + param_len = len(self.__parameters__) + if param_len == 0: + raise TypeError(f'There are no type or parameter specification' + f'variables left in {self}') + if (param_len == 1 + and isinstance(item, (tuple, list)) + and len(item) > 1) or not isinstance(item, tuple): + item = (item,) + item_len = len(item) + if item_len != param_len: + raise TypeError(f'Too {"many" if item_len > param_len else "few"}' + f' arguments for {self};' + f' actual {item_len}, expected {param_len}') + subst = dict(zip(self.__parameters__, item)) + new_args = [] + for arg in self.__args__: + if _is_typevarlike(arg): + arg = subst[arg] + # Looks like a GenericAlias + elif hasattr(arg, '__parameters__') and isinstance(arg.__parameters__, tuple): + subparams = arg.__parameters__ + if subparams: + subargs = tuple(subst[x] for x in subparams) + arg = arg[subargs] + new_args.append(arg) + + # args[0] occurs due to things like Z[[int, str, bool]] from PEP 612 + if not isinstance(new_args[0], (tuple, list)): + t_result = new_args[-1] + t_args = new_args[:-1] + new_args = (t_args, t_result) + return _CallableGenericAlias(Callable, tuple(new_args)) + +def _is_typevarlike(arg): + obj = type(arg) + # looks like a TypeVar/ParamSpec + return (obj.__module__ == 'typing' + and obj.__name__ in {'ParamSpec', 'TypeVar'}) + +def _has_special_args(args): + """Checks if args[0] matches either ``...``, ``ParamSpec`` or + ``_ConcatenateGenericAlias`` from typing.py + """ + if len(args) != 2: + return False + obj = args[0] + if obj is Ellipsis: + return True + obj = type(obj) + names = ('ParamSpec', '_ConcatenateGenericAlias') + return obj.__module__ == 'typing' and any(obj.__name__ == name for name in names) + + +def _type_repr(obj): + """Return the repr() of an object, special-casing types (internal helper). + + Copied from :mod:`typing` since collections.abc + shouldn't depend on that module. + """ + if isinstance(obj, GenericAlias): + return repr(obj) + if isinstance(obj, type): + if obj.__module__ == 'builtins': + return obj.__qualname__ + return f'{obj.__module__}.{obj.__qualname__}' + if obj is Ellipsis: + return '...' + if isinstance(obj, FunctionType): + return obj.__name__ + return repr(obj) + + class Callable(metaclass=ABCMeta): __slots__ = () @@ -423,14 +562,13 @@ def __subclasshook__(cls, C): return _check_methods(C, "__call__") return NotImplemented - __class_getitem__ = classmethod(GenericAlias) + __class_getitem__ = classmethod(_CallableGenericAlias) ### SETS ### class Set(Collection): - """A set is a finite, iterable container. This class provides concrete generic implementations of all @@ -655,19 +793,19 @@ def __isub__(self, it): ### MAPPINGS ### - class Mapping(Collection): - - __slots__ = () - """A Mapping is a generic container for associating key/value pairs. This class provides concrete generic implementations of all methods except for __getitem__, __iter__, and __len__. - """ + __slots__ = () + + # Tell ABCMeta.__new__ that this class should have TPFLAGS_MAPPING set. + __abc_tpflags__ = 1 << 6 # Py_TPFLAGS_MAPPING + @abstractmethod def __getitem__(self, key): raise KeyError @@ -706,7 +844,6 @@ def __eq__(self, other): __reversed__ = None - Mapping.register(mappingproxy) @@ -789,18 +926,16 @@ def __iter__(self): class MutableMapping(Mapping): - - __slots__ = () - """A MutableMapping is a generic container for associating key/value pairs. This class provides concrete generic implementations of all methods except for __getitem__, __setitem__, __delitem__, __iter__, and __len__. - """ + __slots__ = () + @abstractmethod def __setitem__(self, key, value): raise KeyError @@ -877,9 +1012,7 @@ def setdefault(self, key, default=None): ### SEQUENCES ### - class Sequence(Reversible, Collection): - """All the operations on a read-only sequence. Concrete subclasses must override __new__ or __init__, @@ -888,6 +1021,9 @@ class Sequence(Reversible, Collection): __slots__ = () + # Tell ABCMeta.__new__ that this class should have TPFLAGS_SEQUENCE set. + __abc_tpflags__ = 1 << 5 # Py_TPFLAGS_SEQUENCE + @abstractmethod def __getitem__(self, index): raise IndexError @@ -939,7 +1075,6 @@ def count(self, value): 'S.count(value) -> integer -- return number of occurrences of value' return sum(1 for v in self if v is value or v == value) - Sequence.register(tuple) Sequence.register(str) Sequence.register(range) @@ -947,7 +1082,6 @@ def count(self, value): class ByteString(Sequence): - """This unifies bytes and bytearray. XXX Should add all their methods. @@ -960,16 +1094,14 @@ class ByteString(Sequence): class MutableSequence(Sequence): - - __slots__ = () - """All the operations on a read-write sequence. Concrete subclasses must provide __new__ or __init__, __getitem__, __setitem__, __delitem__, __len__, and insert(). - """ + __slots__ = () + @abstractmethod def __setitem__(self, index, value): raise IndexError diff --git a/Lib/_compression.py b/Lib/_compression.py index b00f31b400c3f3..e8b70aa0a3e680 100644 --- a/Lib/_compression.py +++ b/Lib/_compression.py @@ -1,7 +1,7 @@ """Internal classes used by the gzip, lzma and bz2 modules""" import io - +import sys BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE # Compressed data read chunk size @@ -110,6 +110,16 @@ def read(self, size=-1): self._pos += len(data) return data + def readall(self): + chunks = [] + # sys.maxsize means the max length of output buffer is unlimited, + # so that the whole input buffer can be decompressed within one + # .decompress() call. + while data := self.read(sys.maxsize): + chunks.append(data) + + return b"".join(chunks) + # Rewind the file to the beginning of the data stream. def _rewind(self): self._fp.seek(0) diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index e9efce7d7ed5bd..b7145ee069fa32 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -52,7 +52,7 @@ def _find_executable(executable, path=None): return executable -def _read_output(commandstring): +def _read_output(commandstring, capture_stderr=False): """Output from successful command execution or None""" # Similar to os.popen(commandstring, "r").read(), # but without actually using os.popen because that @@ -67,7 +67,10 @@ def _read_output(commandstring): os.getpid(),), "w+b") with contextlib.closing(fp) as fp: - cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name) + if capture_stderr: + cmd = "%s >'%s' 2>&1" % (commandstring, fp.name) + else: + cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name) return fp.read().decode('utf-8').strip() if not os.system(cmd) else None @@ -93,7 +96,7 @@ def _get_system_version(): if _SYSTEM_VERSION is None: _SYSTEM_VERSION = '' try: - f = open('/System/Library/CoreServices/SystemVersion.plist') + f = open('/System/Library/CoreServices/SystemVersion.plist', encoding="utf-8") except OSError: # We're on a plain darwin box, fall back to the default # behaviour. @@ -110,6 +113,26 @@ def _get_system_version(): return _SYSTEM_VERSION +_SYSTEM_VERSION_TUPLE = None +def _get_system_version_tuple(): + """ + Return the macOS system version as a tuple + + The return value is safe to use to compare + two version numbers. + """ + global _SYSTEM_VERSION_TUPLE + if _SYSTEM_VERSION_TUPLE is None: + osx_version = _get_system_version() + if osx_version: + try: + _SYSTEM_VERSION_TUPLE = tuple(int(i) for i in osx_version.split('.')) + except ValueError: + _SYSTEM_VERSION_TUPLE = () + + return _SYSTEM_VERSION_TUPLE + + def _remove_original_values(_config_vars): """Remove original unmodified values for testing""" # This is needed for higher-level cross-platform tests of get_platform. @@ -125,6 +148,33 @@ def _save_modified_value(_config_vars, cv, newvalue): _config_vars[_INITPRE + cv] = oldvalue _config_vars[cv] = newvalue + +_cache_default_sysroot = None +def _default_sysroot(cc): + """ Returns the root of the default SDK for this system, or '/' """ + global _cache_default_sysroot + + if _cache_default_sysroot is not None: + return _cache_default_sysroot + + contents = _read_output('%s -c -E -v - "): + in_incdirs = True + elif line.startswith("End of search list"): + in_incdirs = False + elif in_incdirs: + line = line.strip() + if line == '/usr/include': + _cache_default_sysroot = '/' + elif line.endswith(".sdk/usr/include"): + _cache_default_sysroot = line[:-12] + if _cache_default_sysroot is None: + _cache_default_sysroot = '/' + + return _cache_default_sysroot + def _supports_universal_builds(): """Returns True if universal builds are supported on this system""" # As an approximation, we assume that if we are running on 10.4 or above, @@ -132,14 +182,18 @@ def _supports_universal_builds(): # builds, in particular -isysroot and -arch arguments to the compiler. This # is in support of allowing 10.4 universal builds to run on 10.3.x systems. - osx_version = _get_system_version() - if osx_version: - try: - osx_version = tuple(int(i) for i in osx_version.split('.')) - except ValueError: - osx_version = '' + osx_version = _get_system_version_tuple() return bool(osx_version >= (10, 4)) if osx_version else False +def _supports_arm64_builds(): + """Returns True if arm64 builds are supported on this system""" + # There are two sets of systems supporting macOS/arm64 builds: + # 1. macOS 11 and later, unconditionally + # 2. macOS 10.15 with Xcode 12.2 or later + # For now the second category is ignored. + osx_version = _get_system_version_tuple() + return osx_version >= (11, 0) if osx_version else False + def _find_appropriate_compiler(_config_vars): """Find appropriate C compiler for extension module builds""" @@ -331,6 +385,12 @@ def compiler_fixup(compiler_so, cc_args): except ValueError: break + elif not _supports_arm64_builds(): + # Look for "-arch arm64" and drop that + for idx in reversed(range(len(compiler_so))): + if compiler_so[idx] == '-arch' and compiler_so[idx+1] == "arm64": + del compiler_so[idx:idx+2] + if 'ARCHFLAGS' in os.environ and not stripArch: # User specified different -arch flags in the environ, # see also distutils.sysconfig @@ -465,10 +525,10 @@ def get_platform_osx(_config_vars, osname, release, machine): try: macrelease = tuple(int(i) for i in macrelease.split('.')[0:2]) except ValueError: - macrelease = (10, 0) + macrelease = (10, 3) else: # assume no universal support - macrelease = (10, 0) + macrelease = (10, 3) if (macrelease >= (10, 4)) and '-arch' in cflags.strip(): # The universal build will build fat binaries, but not on @@ -481,6 +541,8 @@ def get_platform_osx(_config_vars, osname, release, machine): if len(archs) == 1: machine = archs[0] + elif archs == ('arm64', 'x86_64'): + machine = 'universal2' elif archs == ('i386', 'ppc'): machine = 'fat' elif archs == ('i386', 'x86_64'): diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index ab989e5206a9e9..ff23322ed5603e 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -951,7 +951,7 @@ def __hash__(self): if self.is_snan(): raise TypeError('Cannot hash a signaling NaN value.') elif self.is_nan(): - return _PyHASH_NAN + return super().__hash__() else: if self._sign: return -_PyHASH_INF diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 4804ed27cd14d6..56e9a0cb33c5fd 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -40,6 +40,36 @@ _CHECK_ERRORS = _IOBASE_EMITS_UNRAISABLE +def text_encoding(encoding, stacklevel=2): + """ + A helper function to choose the text encoding. + + When encoding is not None, just return it. + Otherwise, return the default text encoding (i.e. "locale"). + + This function emits an EncodingWarning if *encoding* is None and + sys.flags.warn_default_encoding is true. + + This can be used in APIs with an encoding=None parameter + that pass it to TextIOWrapper or open. + However, please consider using encoding="utf-8" for new APIs. + """ + if encoding is None: + encoding = "locale" + if sys.flags.warn_default_encoding: + import warnings + warnings.warn("'encoding' argument not specified.", + EncodingWarning, stacklevel + 1) + return encoding + + +# Wrapper for builtins.open +# +# Trick so that open() won't become a bound method when stored +# as a class variable (as dbm.dumb does). +# +# See init_set_builtins_open() in Python/pylifecycle.c. +@staticmethod def open(file, mode="r", buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None): @@ -248,6 +278,7 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, result = buffer if binary: return result + encoding = text_encoding(encoding) text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering) result = text text.mode = mode @@ -280,27 +311,20 @@ def _open_code_with_warning(path): open_code = _open_code_with_warning -class DocDescriptor: - """Helper for builtins.open.__doc__ - """ - def __get__(self, obj, typ=None): - return ( - "open(file, mode='r', buffering=-1, encoding=None, " - "errors=None, newline=None, closefd=True)\n\n" + - open.__doc__) - -class OpenWrapper: - """Wrapper for builtins.open - - Trick so that open won't become a bound method when stored - as a class variable (as dbm.dumb does). - - See initstdio() in Python/pylifecycle.c. - """ - __doc__ = DocDescriptor() - - def __new__(cls, *args, **kwargs): - return open(*args, **kwargs) +def __getattr__(name): + if name == "OpenWrapper": + # bpo-43680: Until Python 3.9, _pyio.open was not a static method and + # builtins.open was set to OpenWrapper to not become a bound method + # when set to a class variable. _io.open is a built-in function whereas + # _pyio.open is a Python function. In Python 3.10, _pyio.open() is now + # a static method, and builtins.open() is now io.open(). + import warnings + warnings.warn('OpenWrapper is deprecated, use open instead', + DeprecationWarning, stacklevel=2) + global OpenWrapper + OpenWrapper = open + return OpenWrapper + raise AttributeError(name) # In normal operation, both `UnsupportedOperation`s should be bound to the @@ -2004,19 +2028,22 @@ class TextIOWrapper(TextIOBase): def __init__(self, buffer, encoding=None, errors=None, newline=None, line_buffering=False, write_through=False): self._check_newline(newline) - if encoding is None: + encoding = text_encoding(encoding) + + if encoding == "locale": try: - encoding = os.device_encoding(buffer.fileno()) + encoding = os.device_encoding(buffer.fileno()) or "locale" except (AttributeError, UnsupportedOperation): pass - if encoding is None: - try: - import locale - except ImportError: - # Importing locale may fail if Python is being built - encoding = "ascii" - else: - encoding = locale.getpreferredencoding(False) + + if encoding == "locale": + try: + import locale + except ImportError: + # Importing locale may fail if Python is being built + encoding = "utf-8" + else: + encoding = locale.getpreferredencoding(False) if not isinstance(encoding, str): raise ValueError("invalid encoding: %r" % encoding) diff --git a/Lib/_sitebuiltins.py b/Lib/_sitebuiltins.py index c29cf4bf8fe488..c66269a571967f 100644 --- a/Lib/_sitebuiltins.py +++ b/Lib/_sitebuiltins.py @@ -47,7 +47,7 @@ def __setup(self): data = None for filename in self.__filenames: try: - with open(filename, "r") as fp: + with open(filename, encoding='utf-8') as fp: data = fp.read() break except OSError: diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 5df37f5f4b89d5..b97dfcce1e8e4d 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -201,7 +201,7 @@ def __init__(self, locale_time=None): #XXX: Does 'Y' need to worry about having less or more than # 4 digits? 'Y': r"(?P\d\d\d\d)", - 'z': r"(?P[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|Z)", + 'z': r"(?P[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|(?-i:Z))", 'A': self.__seqToRE(self.locale_time.f_weekday, 'A'), 'a': self.__seqToRE(self.locale_time.a_weekday, 'a'), 'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'), diff --git a/Lib/argparse.py b/Lib/argparse.py index 2fb1da59f942cf..8a12dea7668799 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1719,7 +1719,7 @@ def __init__(self, add_group = self.add_argument_group self._positionals = add_group(_('positional arguments')) - self._optionals = add_group(_('optional arguments')) + self._optionals = add_group(_('options')) self._subparsers = None # register types diff --git a/Lib/ast.py b/Lib/ast.py index d8bd3373701dec..0c53e5c5712f5e 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -27,7 +27,7 @@ import sys from _ast import * from contextlib import contextmanager, nullcontext -from enum import IntEnum, auto +from enum import IntEnum, auto, _simple_enum def parse(source, filename='', mode='exec', *, @@ -63,7 +63,10 @@ def literal_eval(node_or_string): if isinstance(node_or_string, Expression): node_or_string = node_or_string.body def _raise_malformed_node(node): - raise ValueError(f'malformed node or string: {node!r}') + msg = "malformed node or string" + if lno := getattr(node, 'lineno', None): + msg += f' on line {lno}' + raise ValueError(msg + f': {node!r}') def _convert_num(node): if not isinstance(node, Constant) or type(node.value) not in (int, float, complex): _raise_malformed_node(node) @@ -633,7 +636,8 @@ class Param(expr_context): # We unparse those infinities to INFSTR. _INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) -class _Precedence(IntEnum): +@_simple_enum(IntEnum) +class _Precedence: """Precedence table that originated from python grammar.""" TUPLE = auto() @@ -662,17 +666,23 @@ def next(self): except ValueError: return self + +_SINGLE_QUOTES = ("'", '"') +_MULTI_QUOTES = ('"""', "'''") +_ALL_QUOTES = (*_SINGLE_QUOTES, *_MULTI_QUOTES) + class _Unparser(NodeVisitor): """Methods in this class recursively traverse an AST and output source code for the abstract syntax; original formatting is disregarded.""" - def __init__(self): + def __init__(self, *, _avoid_backslashes=False): self._source = [] self._buffer = [] self._precedences = {} self._type_ignores = {} self._indent = 0 + self._avoid_backslashes = _avoid_backslashes def interleave(self, inter, f, seq): """Call f on each item in seq, calling inter() in between.""" @@ -788,6 +798,9 @@ def traverse(self, node): else: super().visit(node) + # Note: as visit() resets the output text, do NOT rely on + # NodeVisitor.generic_visit to handle any nodes (as it calls back in to + # the subclass visit() method, which resets self._source to an empty list) def visit(self, node): """Outputs a source code string that, if converted back to an ast (using ast.parse) will generate an AST equivalent to *node*""" @@ -1067,15 +1080,85 @@ def visit_AsyncWith(self, node): with self.block(extra=self.get_type_comment(node)): self.traverse(node.body) + def _str_literal_helper( + self, string, *, quote_types=_ALL_QUOTES, escape_special_whitespace=False + ): + """Helper for writing string literals, minimizing escapes. + Returns the tuple (string literal to write, possible quote types). + """ + def escape_char(c): + # \n and \t are non-printable, but we only escape them if + # escape_special_whitespace is True + if not escape_special_whitespace and c in "\n\t": + return c + # Always escape backslashes and other non-printable characters + if c == "\\" or not c.isprintable(): + return c.encode("unicode_escape").decode("ascii") + return c + + escaped_string = "".join(map(escape_char, string)) + possible_quotes = quote_types + if "\n" in escaped_string: + possible_quotes = [q for q in possible_quotes if q in _MULTI_QUOTES] + possible_quotes = [q for q in possible_quotes if q not in escaped_string] + if not possible_quotes: + # If there aren't any possible_quotes, fallback to using repr + # on the original string. Try to use a quote from quote_types, + # e.g., so that we use triple quotes for docstrings. + string = repr(string) + quote = next((q for q in quote_types if string[0] in q), string[0]) + return string[1:-1], [quote] + if escaped_string: + # Sort so that we prefer '''"''' over """\"""" + possible_quotes.sort(key=lambda q: q[0] == escaped_string[-1]) + # If we're using triple quotes and we'd need to escape a final + # quote, escape it + if possible_quotes[0][0] == escaped_string[-1]: + assert len(possible_quotes[0]) == 3 + escaped_string = escaped_string[:-1] + "\\" + escaped_string[-1] + return escaped_string, possible_quotes + + def _write_str_avoiding_backslashes(self, string, *, quote_types=_ALL_QUOTES): + """Write string literal value with a best effort attempt to avoid backslashes.""" + string, quote_types = self._str_literal_helper(string, quote_types=quote_types) + quote_type = quote_types[0] + self.write(f"{quote_type}{string}{quote_type}") + def visit_JoinedStr(self, node): self.write("f") - self._fstring_JoinedStr(node, self.buffer_writer) - self.write(repr(self.buffer)) + if self._avoid_backslashes: + self._fstring_JoinedStr(node, self.buffer_writer) + self._write_str_avoiding_backslashes(self.buffer) + return + + # If we don't need to avoid backslashes globally (i.e., we only need + # to avoid them inside FormattedValues), it's cosmetically preferred + # to use escaped whitespace. That is, it's preferred to use backslashes + # for cases like: f"{x}\n". To accomplish this, we keep track of what + # in our buffer corresponds to FormattedValues and what corresponds to + # Constant parts of the f-string, and allow escapes accordingly. + buffer = [] + for value in node.values: + meth = getattr(self, "_fstring_" + type(value).__name__) + meth(value, self.buffer_writer) + buffer.append((self.buffer, isinstance(value, Constant))) + new_buffer = [] + quote_types = _ALL_QUOTES + for value, is_constant in buffer: + # Repeatedly narrow down the list of possible quote_types + value, quote_types = self._str_literal_helper( + value, quote_types=quote_types, + escape_special_whitespace=is_constant + ) + new_buffer.append(value) + value = "".join(new_buffer) + quote_type = quote_types[0] + self.write(f"{quote_type}{value}{quote_type}") def visit_FormattedValue(self, node): self.write("f") self._fstring_FormattedValue(node, self.buffer_writer) - self.write(repr(self.buffer)) + self._write_str_avoiding_backslashes(self.buffer) def _fstring_JoinedStr(self, node, write): for value in node.values: @@ -1090,11 +1173,13 @@ def _fstring_Constant(self, node, write): def _fstring_FormattedValue(self, node, write): write("{") - unparser = type(self)() + unparser = type(self)(_avoid_backslashes=True) unparser.set_precedence(_Precedence.TEST.next(), node.value) expr = unparser.visit(node.value) if expr.startswith("{"): write(" ") # Separate pair of opening brackets as "{ {" + if "\\" in expr: + raise ValueError("Unable to avoid backslash in f-string expression part") write(expr) if node.conversion != -1: conversion = chr(node.conversion) @@ -1111,33 +1196,22 @@ def visit_Name(self, node): self.write(node.id) def _write_docstring(self, node): - def esc_char(c): - if c in ("\n", "\t"): - # In the AST form, we don't know the author's intentation - # about how this should be displayed. We'll only escape - # \n and \t, because they are more likely to be unescaped - # in the source - return c - return c.encode('unicode_escape').decode('ascii') - self.fill() if node.kind == "u": self.write("u") - - value = node.value - if value: - # Preserve quotes in the docstring by escaping them - value = "".join(map(esc_char, value)) - if value[-1] == '"': - value = value.replace('"', '\\"', -1) - value = value.replace('"""', '""\\"') - - self.write(f'"""{value}"""') + self._write_str_avoiding_backslashes(node.value, quote_types=_MULTI_QUOTES) def _write_constant(self, value): if isinstance(value, (float, complex)): - # Substitute overflowing decimal literal for AST infinities. - self.write(repr(value).replace("inf", _INFSTR)) + # Substitute overflowing decimal literal for AST infinities, + # and inf - inf for NaNs. + self.write( + repr(value) + .replace("inf", _INFSTR) + .replace("nan", f"({_INFSTR}-{_INFSTR})") + ) + elif self._avoid_backslashes and isinstance(value, str): + self._write_str_avoiding_backslashes(value) else: self.write(repr(value)) @@ -1208,10 +1282,13 @@ def visit_IfExp(self, node): self.traverse(node.orelse) def visit_Set(self, node): - if not node.elts: - raise ValueError("Set node should have at least one item") - with self.delimit("{", "}"): - self.interleave(lambda: self.write(", "), self.traverse, node.elts) + if node.elts: + with self.delimit("{", "}"): + self.interleave(lambda: self.write(", "), self.traverse, node.elts) + else: + # `{}` would be interpreted as a dictionary literal, and + # `set` might be shadowed. Thus: + self.write('{*()}') def visit_Dict(self, node): def write_key_value_pair(k, v): @@ -1413,6 +1490,13 @@ def visit_Slice(self, node): self.write(":") self.traverse(node.step) + def visit_Match(self, node): + self.fill("match ") + self.traverse(node.subject) + with self.block(): + for case in node.cases: + self.traverse(case) + def visit_arg(self, node): self.write(node.arg) if node.annotation: @@ -1497,6 +1581,94 @@ def visit_withitem(self, node): self.write(" as ") self.traverse(node.optional_vars) + def visit_match_case(self, node): + self.fill("case ") + self.traverse(node.pattern) + if node.guard: + self.write(" if ") + self.traverse(node.guard) + with self.block(): + self.traverse(node.body) + + def visit_MatchValue(self, node): + self.traverse(node.value) + + def visit_MatchSingleton(self, node): + self._write_constant(node.value) + + def visit_MatchSequence(self, node): + with self.delimit("[", "]"): + self.interleave( + lambda: self.write(", "), self.traverse, node.patterns + ) + + def visit_MatchStar(self, node): + name = node.name + if name is None: + name = "_" + self.write(f"*{name}") + + def visit_MatchMapping(self, node): + def write_key_pattern_pair(pair): + k, p = pair + self.traverse(k) + self.write(": ") + self.traverse(p) + + with self.delimit("{", "}"): + keys = node.keys + self.interleave( + lambda: self.write(", "), + write_key_pattern_pair, + zip(keys, node.patterns, strict=True), + ) + rest = node.rest + if rest is not None: + if keys: + self.write(", ") + self.write(f"**{rest}") + + def visit_MatchClass(self, node): + self.set_precedence(_Precedence.ATOM, node.cls) + self.traverse(node.cls) + with self.delimit("(", ")"): + patterns = node.patterns + self.interleave( + lambda: self.write(", "), self.traverse, patterns + ) + attrs = node.kwd_attrs + if attrs: + def write_attr_pattern(pair): + attr, pattern = pair + self.write(f"{attr}=") + self.traverse(pattern) + + if patterns: + self.write(", ") + self.interleave( + lambda: self.write(", "), + write_attr_pattern, + zip(attrs, node.kwd_patterns, strict=True), + ) + + def visit_MatchAs(self, node): + name = node.name + pattern = node.pattern + if name is None: + self.write("_") + elif pattern is None: + self.write(node.name) + else: + with self.require_parens(_Precedence.TEST, node): + self.set_precedence(_Precedence.BOR, node.pattern) + self.traverse(node.pattern) + self.write(f" as {node.name}") + + def visit_MatchOr(self, node): + with self.require_parens(_Precedence.BOR, node): + self.set_precedence(_Precedence.BOR.next(), *node.patterns) + self.interleave(lambda: self.write(" | "), self.traverse, node.patterns) + def unparse(ast_obj): unparser = _Unparser() return unparser.visit(ast_obj) diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index eb84bfb189ccf3..200b14c2a3f21e 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -20,10 +20,6 @@ from .threads import * from .transports import * -# Exposed for _asynciomodule.c to implement now deprecated -# Task.all_tasks() method. This function will be removed in 3.9. -from .tasks import _all_tasks_compat # NoQA - __all__ = (base_events.__all__ + coroutines.__all__ + events.__all__ + diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index b2d446a51fedb5..f789635e0f893a 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -350,7 +350,7 @@ async def start_serving(self): self._start_serving() # Skip one loop iteration so that all 'loop.add_reader' # go through. - await tasks.sleep(0, loop=self._loop) + await tasks.sleep(0) async def serve_forever(self): if self._serving_forever_fut is not None: @@ -541,8 +541,7 @@ async def shutdown_asyncgens(self): results = await tasks.gather( *[ag.aclose() for ag in closing_agens], - return_exceptions=True, - loop=self) + return_exceptions=True) for result, agen in zip(results, closing_agens): if isinstance(result, Exception): @@ -1457,7 +1456,7 @@ async def create_server( fs = [self._create_server_getaddrinfo(host, port, family=family, flags=flags) for host in hosts] - infos = await tasks.gather(*fs, loop=self) + infos = await tasks.gather(*fs) infos = set(itertools.chain.from_iterable(infos)) completed = False @@ -1515,7 +1514,7 @@ async def create_server( server._start_serving() # Skip one loop iteration so that all 'loop.add_reader' # go through. - await tasks.sleep(0, loop=self) + await tasks.sleep(0) if self._debug: logger.info("%r is serving", server) @@ -1525,14 +1524,6 @@ async def connect_accepted_socket( self, protocol_factory, sock, *, ssl=None, ssl_handshake_timeout=None): - """Handle an accepted connection. - - This is used by servers that accept connections outside of - asyncio but that use asyncio to handle connections. - - This method is a coroutine. When completed, the coroutine - returns a (transport, protocol) pair. - """ if sock.type != socket.SOCK_STREAM: raise ValueError(f'A Stream Socket was expected, got {sock!r}') diff --git a/Lib/asyncio/base_futures.py b/Lib/asyncio/base_futures.py index 22f298069c505e..2c01ac98e10fca 100644 --- a/Lib/asyncio/base_futures.py +++ b/Lib/asyncio/base_futures.py @@ -1,6 +1,7 @@ __all__ = () import reprlib +from _thread import get_ident from . import format_helpers @@ -41,6 +42,16 @@ def format_cb(callback): return f'cb=[{cb}]' +# bpo-42183: _repr_running is needed for repr protection +# when a Future or Task result contains itself directly or indirectly. +# The logic is borrowed from @reprlib.recursive_repr decorator. +# Unfortunately, the direct decorator usage is impossible because of +# AttributeError: '_asyncio.Task' object has no attribute '__module__' error. +# +# After fixing this thing we can return to the decorator based approach. +_repr_running = set() + + def _future_repr_info(future): # (Future) -> str """helper function for Future.__repr__""" @@ -49,9 +60,17 @@ def _future_repr_info(future): if future._exception is not None: info.append(f'exception={future._exception!r}') else: - # use reprlib to limit the length of the output, especially - # for very long strings - result = reprlib.repr(future._result) + key = id(future), get_ident() + if key in _repr_running: + result = '...' + else: + _repr_running.add(key) + try: + # use reprlib to limit the length of the output, especially + # for very long strings + result = reprlib.repr(future._result) + finally: + _repr_running.discard(key) info.append(f'result={result}') if future._callbacks: info.append(_format_callbacks(future._callbacks)) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 0dce87b8ecc586..b966ad26bf467b 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -418,6 +418,20 @@ async def create_unix_server( """ raise NotImplementedError + async def connect_accepted_socket( + self, protocol_factory, sock, + *, ssl=None, + ssl_handshake_timeout=None): + """Handle an accepted connection. + + This is used by servers that accept connections outside of + asyncio, but use asyncio to handle connections. + + This method is a coroutine. When completed, the coroutine + returns a (transport, protocol) pair. + """ + raise NotImplementedError + async def create_datagram_endpoint(self, protocol_factory, local_addr=None, remote_addr=None, *, family=0, proto=0, flags=0, @@ -745,9 +759,16 @@ def get_event_loop(): the result of `get_event_loop_policy().get_event_loop()` call. """ # NOTE: this function is implemented in C (see _asynciomodule.c) + return _py__get_event_loop() + + +def _get_event_loop(stacklevel=3): current_loop = _get_running_loop() if current_loop is not None: return current_loop + import warnings + warnings.warn('There is no current event loop', + DeprecationWarning, stacklevel=stacklevel) return get_event_loop_policy().get_event_loop() @@ -777,6 +798,7 @@ def set_child_watcher(watcher): _py__set_running_loop = _set_running_loop _py_get_running_loop = get_running_loop _py_get_event_loop = get_event_loop +_py__get_event_loop = _get_event_loop try: @@ -784,7 +806,7 @@ def set_child_watcher(watcher): # functions in asyncio. Pure Python implementation is # about 4 times slower than C-accelerated. from _asyncio import (_get_running_loop, _set_running_loop, - get_running_loop, get_event_loop) + get_running_loop, get_event_loop, _get_event_loop) except ImportError: pass else: @@ -793,3 +815,4 @@ def set_child_watcher(watcher): _c__set_running_loop = _set_running_loop _c_get_running_loop = get_running_loop _c_get_event_loop = get_event_loop + _c__get_event_loop = _get_event_loop diff --git a/Lib/asyncio/exceptions.py b/Lib/asyncio/exceptions.py index e03602ef576234..f07e4486577381 100644 --- a/Lib/asyncio/exceptions.py +++ b/Lib/asyncio/exceptions.py @@ -34,8 +34,9 @@ class IncompleteReadError(EOFError): - expected: total number of expected bytes (or None if unknown) """ def __init__(self, partial, expected): + r_expected = 'undefined' if expected is None else repr(expected) super().__init__(f'{len(partial)} bytes read on a total of ' - f'{expected!r} expected bytes') + f'{r_expected} expected bytes') self.partial = partial self.expected = expected diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index bed4da52fd4d98..10f8f0554e4b60 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -76,7 +76,7 @@ def __init__(self, *, loop=None): the default event loop. """ if loop is None: - self._loop = events.get_event_loop() + self._loop = events._get_event_loop() else: self._loop = loop self._callbacks = [] @@ -115,7 +115,7 @@ def _log_traceback(self): @_log_traceback.setter def _log_traceback(self, val): - if bool(val): + if val: raise ValueError('_log_traceback can only be set to False') self.__log_traceback = False @@ -408,7 +408,7 @@ def wrap_future(future, *, loop=None): assert isinstance(future, concurrent.futures.Future), \ f'concurrent.futures.Future is expected, got {future!r}' if loop is None: - loop = events.get_event_loop() + loop = events._get_event_loop() new_future = loop.create_future() _chain_future(future, new_future) return new_future diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index f1ce7324785ba9..a7453fb1c77287 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -3,10 +3,9 @@ __all__ = ('Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore') import collections -import warnings -from . import events from . import exceptions +from . import mixins class _ContextManagerMixin: @@ -20,7 +19,7 @@ async def __aexit__(self, exc_type, exc, tb): self.release() -class Lock(_ContextManagerMixin): +class Lock(_ContextManagerMixin, mixins._LoopBoundMixin): """Primitive lock objects. A primitive lock is a synchronization primitive that is not owned @@ -74,16 +73,10 @@ class Lock(_ContextManagerMixin): """ - def __init__(self, *, loop=None): + def __init__(self, *, loop=mixins._marker): + super().__init__(loop=loop) self._waiters = None self._locked = False - if loop is None: - self._loop = events.get_event_loop() - else: - self._loop = loop - warnings.warn("The loop argument is deprecated since Python 3.8, " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, stacklevel=2) def __repr__(self): res = super().__repr__() @@ -109,7 +102,7 @@ async def acquire(self): if self._waiters is None: self._waiters = collections.deque() - fut = self._loop.create_future() + fut = self._get_loop().create_future() self._waiters.append(fut) # Finally block should be called before the CancelledError @@ -161,7 +154,7 @@ def _wake_up_first(self): fut.set_result(True) -class Event: +class Event(mixins._LoopBoundMixin): """Asynchronous equivalent to threading.Event. Class implementing event objects. An event manages a flag that can be set @@ -170,16 +163,10 @@ class Event: false. """ - def __init__(self, *, loop=None): + def __init__(self, *, loop=mixins._marker): + super().__init__(loop=loop) self._waiters = collections.deque() self._value = False - if loop is None: - self._loop = events.get_event_loop() - else: - self._loop = loop - warnings.warn("The loop argument is deprecated since Python 3.8, " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, stacklevel=2) def __repr__(self): res = super().__repr__() @@ -220,7 +207,7 @@ async def wait(self): if self._value: return True - fut = self._loop.create_future() + fut = self._get_loop().create_future() self._waiters.append(fut) try: await fut @@ -229,7 +216,7 @@ async def wait(self): self._waiters.remove(fut) -class Condition(_ContextManagerMixin): +class Condition(_ContextManagerMixin, mixins._LoopBoundMixin): """Asynchronous equivalent to threading.Condition. This class implements condition variable objects. A condition variable @@ -239,18 +226,11 @@ class Condition(_ContextManagerMixin): A new Lock object is created and used as the underlying lock. """ - def __init__(self, lock=None, *, loop=None): - if loop is None: - self._loop = events.get_event_loop() - else: - self._loop = loop - warnings.warn("The loop argument is deprecated since Python 3.8, " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, stacklevel=2) - + def __init__(self, lock=None, *, loop=mixins._marker): + super().__init__(loop=loop) if lock is None: - lock = Lock(loop=loop) - elif lock._loop is not self._loop: + lock = Lock() + elif lock._loop is not self._get_loop(): raise ValueError("loop argument must agree with lock") self._lock = lock @@ -284,7 +264,7 @@ async def wait(self): self.release() try: - fut = self._loop.create_future() + fut = self._get_loop().create_future() self._waiters.append(fut) try: await fut @@ -351,7 +331,7 @@ def notify_all(self): self.notify(len(self._waiters)) -class Semaphore(_ContextManagerMixin): +class Semaphore(_ContextManagerMixin, mixins._LoopBoundMixin): """A Semaphore implementation. A semaphore manages an internal counter which is decremented by each @@ -366,18 +346,12 @@ class Semaphore(_ContextManagerMixin): ValueError is raised. """ - def __init__(self, value=1, *, loop=None): + def __init__(self, value=1, *, loop=mixins._marker): + super().__init__(loop=loop) if value < 0: raise ValueError("Semaphore initial value must be >= 0") self._value = value self._waiters = collections.deque() - if loop is None: - self._loop = events.get_event_loop() - else: - self._loop = loop - warnings.warn("The loop argument is deprecated since Python 3.8, " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, stacklevel=2) def __repr__(self): res = super().__repr__() @@ -407,7 +381,7 @@ async def acquire(self): True. """ while self._value <= 0: - fut = self._loop.create_future() + fut = self._get_loop().create_future() self._waiters.append(fut) try: await fut @@ -436,12 +410,7 @@ class BoundedSemaphore(Semaphore): above the initial value. """ - def __init__(self, value=1, *, loop=None): - if loop: - warnings.warn("The loop argument is deprecated since Python 3.8, " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, stacklevel=2) - + def __init__(self, value=1, *, loop=mixins._marker): self._bound_value = value super().__init__(value, loop=loop) diff --git a/Lib/asyncio/mixins.py b/Lib/asyncio/mixins.py new file mode 100644 index 00000000000000..650df05ccc93ea --- /dev/null +++ b/Lib/asyncio/mixins.py @@ -0,0 +1,31 @@ +"""Event loop mixins.""" + +import threading +from . import events + +_global_lock = threading.Lock() + +# Used as a sentinel for loop parameter +_marker = object() + + +class _LoopBoundMixin: + _loop = None + + def __init__(self, *, loop=_marker): + if loop is not _marker: + raise TypeError( + f'As of 3.10, the *loop* parameter was removed from ' + f'{type(self).__name__}() since it is no longer necessary' + ) + + def _get_loop(self): + loop = events._get_running_loop() + + if self._loop is None: + with _global_lock: + if self._loop is None: + self._loop = loop + if loop is not self._loop: + raise RuntimeError(f'{self!r} is bound to a different event loop') + return loop diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index cd3f7c6a567891..a87ec8b2158767 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -2,10 +2,9 @@ import collections import heapq -import warnings -from . import events from . import locks +from . import mixins class QueueEmpty(Exception): @@ -18,7 +17,7 @@ class QueueFull(Exception): pass -class Queue: +class Queue(mixins._LoopBoundMixin): """A queue, useful for coordinating producer and consumer coroutines. If maxsize is less than or equal to zero, the queue size is infinite. If it @@ -30,14 +29,8 @@ class Queue: interrupted between calling qsize() and doing an operation on the Queue. """ - def __init__(self, maxsize=0, *, loop=None): - if loop is None: - self._loop = events.get_event_loop() - else: - self._loop = loop - warnings.warn("The loop argument is deprecated since Python 3.8, " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, stacklevel=2) + def __init__(self, maxsize=0, *, loop=mixins._marker): + super().__init__(loop=loop) self._maxsize = maxsize # Futures. @@ -45,7 +38,7 @@ def __init__(self, maxsize=0, *, loop=None): # Futures. self._putters = collections.deque() self._unfinished_tasks = 0 - self._finished = locks.Event(loop=loop) + self._finished = locks.Event() self._finished.set() self._init(maxsize) @@ -122,7 +115,7 @@ async def put(self, item): slot is available before adding item. """ while self.full(): - putter = self._loop.create_future() + putter = self._get_loop().create_future() self._putters.append(putter) try: await putter @@ -160,7 +153,7 @@ async def get(self): If queue is empty, wait until an item is available. """ while self.empty(): - getter = self._loop.create_future() + getter = self._get_loop().create_future() self._getters.append(getter) try: await getter diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 268635d68fb0c0..9a5e9a48479ef7 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -60,8 +60,7 @@ def _cancel_all_tasks(loop): for task in to_cancel: task.cancel() - loop.run_until_complete( - tasks.gather(*to_cancel, loop=loop, return_exceptions=True)) + loop.run_until_complete(tasks.gather(*to_cancel, return_exceptions=True)) for task in to_cancel: if task.cancelled(): diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 3c80bb88925905..080d8a62cde1e2 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -23,7 +23,7 @@ async def open_connection(host=None, port=None, *, - loop=None, limit=_DEFAULT_LIMIT, **kwds): + limit=_DEFAULT_LIMIT, **kwds): """A wrapper for create_connection() returning a (reader, writer) pair. The reader returned is a StreamReader instance; the writer is a @@ -41,12 +41,7 @@ async def open_connection(host=None, port=None, *, StreamReaderProtocol classes, just copy the code -- there's really nothing special here except some convenience.) """ - if loop is None: - loop = events.get_event_loop() - else: - warnings.warn("The loop argument is deprecated since Python 3.8, " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, stacklevel=2) + loop = events.get_running_loop() reader = StreamReader(limit=limit, loop=loop) protocol = StreamReaderProtocol(reader, loop=loop) transport, _ = await loop.create_connection( @@ -56,7 +51,7 @@ async def open_connection(host=None, port=None, *, async def start_server(client_connected_cb, host=None, port=None, *, - loop=None, limit=_DEFAULT_LIMIT, **kwds): + limit=_DEFAULT_LIMIT, **kwds): """Start a socket server, call back for each client connected. The first parameter, `client_connected_cb`, takes two parameters: @@ -78,12 +73,7 @@ async def start_server(client_connected_cb, host=None, port=None, *, The return value is the same as loop.create_server(), i.e. a Server object which can be used to stop the service. """ - if loop is None: - loop = events.get_event_loop() - else: - warnings.warn("The loop argument is deprecated since Python 3.8, " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, stacklevel=2) + loop = events.get_running_loop() def factory(): reader = StreamReader(limit=limit, loop=loop) @@ -98,14 +88,10 @@ def factory(): # UNIX Domain Sockets are supported on this platform async def open_unix_connection(path=None, *, - loop=None, limit=_DEFAULT_LIMIT, **kwds): + limit=_DEFAULT_LIMIT, **kwds): """Similar to `open_connection` but works with UNIX Domain Sockets.""" - if loop is None: - loop = events.get_event_loop() - else: - warnings.warn("The loop argument is deprecated since Python 3.8, " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, stacklevel=2) + loop = events.get_running_loop() + reader = StreamReader(limit=limit, loop=loop) protocol = StreamReaderProtocol(reader, loop=loop) transport, _ = await loop.create_unix_connection( @@ -114,14 +100,9 @@ async def open_unix_connection(path=None, *, return reader, writer async def start_unix_server(client_connected_cb, path=None, *, - loop=None, limit=_DEFAULT_LIMIT, **kwds): + limit=_DEFAULT_LIMIT, **kwds): """Similar to `start_server` but works with UNIX Domain Sockets.""" - if loop is None: - loop = events.get_event_loop() - else: - warnings.warn("The loop argument is deprecated since Python 3.8, " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, stacklevel=2) + loop = events.get_running_loop() def factory(): reader = StreamReader(limit=limit, loop=loop) @@ -144,7 +125,7 @@ class FlowControlMixin(protocols.Protocol): def __init__(self, loop=None): if loop is None: - self._loop = events.get_event_loop() + self._loop = events._get_event_loop(stacklevel=4) else: self._loop = loop self._paused = False @@ -302,9 +283,13 @@ def _get_close_waiter(self, stream): def __del__(self): # Prevent reports about unhandled exceptions. # Better than self._closed._log_traceback = False hack - closed = self._closed - if closed.done() and not closed.cancelled(): - closed.exception() + try: + closed = self._closed + except AttributeError: + pass # failed constructor + else: + if closed.done() and not closed.cancelled(): + closed.exception() class StreamWriter: @@ -400,7 +385,7 @@ def __init__(self, limit=_DEFAULT_LIMIT, loop=None): self._limit = limit if loop is None: - self._loop = events.get_event_loop() + self._loop = events._get_event_loop() else: self._loop = loop self._buffer = bytearray() diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py index c9506b158302b5..cd10231f710f11 100644 --- a/Lib/asyncio/subprocess.py +++ b/Lib/asyncio/subprocess.py @@ -1,7 +1,6 @@ __all__ = 'create_subprocess_exec', 'create_subprocess_shell' import subprocess -import warnings from . import events from . import protocols @@ -193,24 +192,14 @@ async def communicate(self, input=None): stderr = self._read_stream(2) else: stderr = self._noop() - stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr, - loop=self._loop) + stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr) await self.wait() return (stdout, stderr) async def create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None, - loop=None, limit=streams._DEFAULT_LIMIT, - **kwds): - if loop is None: - loop = events.get_event_loop() - else: - warnings.warn("The loop argument is deprecated since Python 3.8 " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, - stacklevel=2 - ) - + limit=streams._DEFAULT_LIMIT, **kwds): + loop = events.get_running_loop() protocol_factory = lambda: SubprocessStreamProtocol(limit=limit, loop=loop) transport, protocol = await loop.subprocess_shell( @@ -221,16 +210,9 @@ async def create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None, async def create_subprocess_exec(program, *args, stdin=None, stdout=None, - stderr=None, loop=None, - limit=streams._DEFAULT_LIMIT, **kwds): - if loop is None: - loop = events.get_event_loop() - else: - warnings.warn("The loop argument is deprecated since Python 3.8 " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, - stacklevel=2 - ) + stderr=None, limit=streams._DEFAULT_LIMIT, + **kwds): + loop = events.get_running_loop() protocol_factory = lambda: SubprocessStreamProtocol(limit=limit, loop=loop) transport, protocol = await loop.subprocess_exec( diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 8b05434f273b52..9a9d0d6e3cc269 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -61,30 +61,6 @@ def all_tasks(loop=None): if futures._get_loop(t) is loop and not t.done()} -def _all_tasks_compat(loop=None): - # Different from "all_task()" by returning *all* Tasks, including - # the completed ones. Used to implement deprecated "Tasks.all_task()" - # method. - if loop is None: - loop = events.get_event_loop() - # Looping over a WeakSet (_all_tasks) isn't safe as it can be updated from another - # thread while we do so. Therefore we cast it to list prior to filtering. The list - # cast itself requires iteration, so we repeat it several times ignoring - # RuntimeErrors (which are not very likely to occur). See issues 34970 and 36607 for - # details. - i = 0 - while True: - try: - tasks = list(_all_tasks) - except RuntimeError: - i += 1 - if i >= 1000: - raise - else: - break - return {t for t in tasks if futures._get_loop(t) is loop} - - def _set_task_name(task, name): if name is not None: try: @@ -370,10 +346,10 @@ def create_task(coro, *, name=None): ALL_COMPLETED = concurrent.futures.ALL_COMPLETED -async def wait(fs, *, loop=None, timeout=None, return_when=ALL_COMPLETED): +async def wait(fs, *, timeout=None, return_when=ALL_COMPLETED): """Wait for the Futures and coroutines given by fs to complete. - The sequence futures must not be empty. + The fs iterable must not be empty. Coroutines will be wrapped in Tasks. @@ -393,20 +369,17 @@ async def wait(fs, *, loop=None, timeout=None, return_when=ALL_COMPLETED): if return_when not in (FIRST_COMPLETED, FIRST_EXCEPTION, ALL_COMPLETED): raise ValueError(f'Invalid return_when value: {return_when}') - if loop is None: - loop = events.get_running_loop() - else: - warnings.warn("The loop argument is deprecated since Python 3.8, " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, stacklevel=2) + loop = events.get_running_loop() - if any(coroutines.iscoroutine(f) for f in set(fs)): + fs = set(fs) + + if any(coroutines.iscoroutine(f) for f in fs): warnings.warn("The explicit passing of coroutine objects to " "asyncio.wait() is deprecated since Python 3.8, and " "scheduled for removal in Python 3.11.", DeprecationWarning, stacklevel=2) - fs = {ensure_future(f, loop=loop) for f in set(fs)} + fs = {ensure_future(f, loop=loop) for f in fs} return await _wait(fs, timeout, return_when, loop) @@ -416,7 +389,7 @@ def _release_waiter(waiter, *args): waiter.set_result(None) -async def wait_for(fut, timeout, *, loop=None): +async def wait_for(fut, timeout): """Wait for the single Future or coroutine to complete, with timeout. Coroutine will be wrapped in Task. @@ -429,12 +402,7 @@ async def wait_for(fut, timeout, *, loop=None): This function is a coroutine. """ - if loop is None: - loop = events.get_running_loop() - else: - warnings.warn("The loop argument is deprecated since Python 3.8, " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, stacklevel=2) + loop = events.get_running_loop() if timeout is None: return await fut @@ -469,7 +437,10 @@ async def wait_for(fut, timeout, *, loop=None): return fut.result() else: fut.remove_done_callback(cb) - fut.cancel() + # We must ensure that the task is not running + # after wait_for() returns. + # See https://bugs.python.org/issue32751 + await _cancel_and_wait(fut, loop=loop) raise if fut.done(): @@ -554,7 +525,7 @@ async def _cancel_and_wait(fut, loop): # This is *not* a @coroutine! It is just an iterator (yielding Futures). -def as_completed(fs, *, loop=None, timeout=None): +def as_completed(fs, *, timeout=None): """Return an iterator whose values are coroutines. When waiting for the yielded coroutines you'll get the results (or @@ -573,17 +544,12 @@ def as_completed(fs, *, loop=None, timeout=None): Note: The futures 'f' are not necessarily members of fs. """ if futures.isfuture(fs) or coroutines.iscoroutine(fs): - raise TypeError(f"expect a list of futures, not {type(fs).__name__}") + raise TypeError(f"expect an iterable of futures, not {type(fs).__name__}") from .queues import Queue # Import here to avoid circular import problem. - done = Queue(loop=loop) + done = Queue() - if loop is None: - loop = events.get_event_loop() - else: - warnings.warn("The loop argument is deprecated since Python 3.8, " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, stacklevel=2) + loop = events._get_event_loop() todo = {ensure_future(f, loop=loop) for f in set(fs)} timeout_handle = None @@ -628,19 +594,13 @@ def __sleep0(): yield -async def sleep(delay, result=None, *, loop=None): +async def sleep(delay, result=None): """Coroutine that completes after a given time (in seconds).""" if delay <= 0: await __sleep0() return result - if loop is None: - loop = events.get_running_loop() - else: - warnings.warn("The loop argument is deprecated since Python 3.8, " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, stacklevel=2) - + loop = events.get_running_loop() future = loop.create_future() h = loop.call_later(delay, futures._set_result_unless_cancelled, @@ -656,23 +616,26 @@ def ensure_future(coro_or_future, *, loop=None): If the argument is a Future, it is returned directly. """ - if coroutines.iscoroutine(coro_or_future): - if loop is None: - loop = events.get_event_loop() - task = loop.create_task(coro_or_future) - if task._source_traceback: - del task._source_traceback[-1] - return task - elif futures.isfuture(coro_or_future): + return _ensure_future(coro_or_future, loop=loop) + + +def _ensure_future(coro_or_future, *, loop=None): + if futures.isfuture(coro_or_future): if loop is not None and loop is not futures._get_loop(coro_or_future): raise ValueError('The future belongs to a different loop than ' - 'the one specified as the loop argument') + 'the one specified as the loop argument') return coro_or_future - elif inspect.isawaitable(coro_or_future): - return ensure_future(_wrap_awaitable(coro_or_future), loop=loop) - else: - raise TypeError('An asyncio.Future, a coroutine or an awaitable is ' - 'required') + + if not coroutines.iscoroutine(coro_or_future): + if inspect.isawaitable(coro_or_future): + coro_or_future = _wrap_awaitable(coro_or_future) + else: + raise TypeError('An asyncio.Future, a coroutine or an awaitable ' + 'is required') + + if loop is None: + loop = events._get_event_loop(stacklevel=4) + return loop.create_task(coro_or_future) @types.coroutine @@ -695,7 +658,8 @@ class _GatheringFuture(futures.Future): cancelled. """ - def __init__(self, children, *, loop=None): + def __init__(self, children, *, loop): + assert loop is not None super().__init__(loop=loop) self._children = children self._cancel_requested = False @@ -715,7 +679,7 @@ def cancel(self, msg=None): return ret -def gather(*coros_or_futures, loop=None, return_exceptions=False): +def gather(*coros_or_futures, return_exceptions=False): """Return a future aggregating results from the given coroutines/futures. Coroutines will be wrapped in a future and scheduled in the event @@ -746,12 +710,7 @@ def gather(*coros_or_futures, loop=None, return_exceptions=False): gather won't cancel any other awaitables. """ if not coros_or_futures: - if loop is None: - loop = events.get_event_loop() - else: - warnings.warn("The loop argument is deprecated since Python 3.8, " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, stacklevel=2) + loop = events._get_event_loop() outer = loop.create_future() outer.set_result([]) return outer @@ -815,9 +774,10 @@ def _done_callback(fut): children = [] nfuts = 0 nfinished = 0 + loop = None for arg in coros_or_futures: if arg not in arg_to_fut: - fut = ensure_future(arg, loop=loop) + fut = _ensure_future(arg, loop=loop) if loop is None: loop = futures._get_loop(fut) if fut is not arg: @@ -841,7 +801,7 @@ def _done_callback(fut): return outer -def shield(arg, *, loop=None): +def shield(arg): """Wait for a future, shielding it from cancellation. The statement @@ -867,11 +827,7 @@ def shield(arg, *, loop=None): except CancelledError: res = None """ - if loop is not None: - warnings.warn("The loop argument is deprecated since Python 3.8, " - "and scheduled for removal in Python 3.10.", - DeprecationWarning, stacklevel=2) - inner = ensure_future(arg, loop=loop) + inner = _ensure_future(arg) if inner.done(): # Shortcut. return inner diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index f34a5b4b443736..a55b3a375fa22d 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -44,6 +44,16 @@ def _sighandler_noop(signum, frame): pass +def waitstatus_to_exitcode(status): + try: + return os.waitstatus_to_exitcode(status) + except ValueError: + # The child exited, but we don't understand its status. + # This shouldn't happen, but if it does, let's just + # return that status; perhaps that helps debug it. + return status + + class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): """Unix event loop. @@ -116,7 +126,7 @@ def add_signal_handler(self, sig, callback, *args): logger.info('set_wakeup_fd(-1) failed: %s', nexc) if exc.errno == errno.EINVAL: - raise RuntimeError(f'sig {sig} cannot be caught') + raise RuntimeError(f'sig {sig:d} cannot be caught') else: raise @@ -150,7 +160,7 @@ def remove_signal_handler(self, sig): signal.signal(sig, handler) except OSError as exc: if exc.errno == errno.EINVAL: - raise RuntimeError(f'sig {sig} cannot be caught') + raise RuntimeError(f'sig {sig:d} cannot be caught') else: raise @@ -323,7 +333,7 @@ async def create_unix_server( server._start_serving() # Skip one loop iteration so that all 'loop.add_reader' # go through. - await tasks.sleep(0, loop=self) + await tasks.sleep(0) return server @@ -941,7 +951,7 @@ def _do_wait(self, pid): " will report returncode 255", pid) else: - returncode = _compute_returncode(status) + returncode = waitstatus_to_exitcode(status) os.close(pidfd) callback(pid, returncode, *args) @@ -956,20 +966,6 @@ def remove_child_handler(self, pid): return True -def _compute_returncode(status): - if os.WIFSIGNALED(status): - # The child process died because of a signal. - return -os.WTERMSIG(status) - elif os.WIFEXITED(status): - # The child process exited (e.g sys.exit()). - return os.WEXITSTATUS(status) - else: - # The child exited, but we don't understand its status. - # This shouldn't happen, but if it does, let's just - # return that status; perhaps that helps debug it. - return status - - class BaseChildWatcher(AbstractChildWatcher): def __init__(self): @@ -1080,7 +1076,7 @@ def _do_waitpid(self, expected_pid): # The child process is still alive. return - returncode = _compute_returncode(status) + returncode = waitstatus_to_exitcode(status) if self._loop.get_debug(): logger.debug('process %s exited with returncode %s', expected_pid, returncode) @@ -1173,7 +1169,7 @@ def _do_waitpid_all(self): # A child process is still alive. return - returncode = _compute_returncode(status) + returncode = waitstatus_to_exitcode(status) with self._lock: try: @@ -1230,13 +1226,15 @@ def is_active(self): def close(self): self._callbacks.clear() - if self._saved_sighandler is not None: - handler = signal.getsignal(signal.SIGCHLD) - if handler != self._sig_chld: - logger.warning("SIGCHLD handler was changed by outside code") - else: - signal.signal(signal.SIGCHLD, self._saved_sighandler) - self._saved_sighandler = None + if self._saved_sighandler is None: + return + + handler = signal.getsignal(signal.SIGCHLD) + if handler != self._sig_chld: + logger.warning("SIGCHLD handler was changed by outside code") + else: + signal.signal(signal.SIGCHLD, self._saved_sighandler) + self._saved_sighandler = None def __enter__(self): return self @@ -1263,15 +1261,17 @@ def attach_loop(self, loop): # The reason to do it here is that attach_loop() is called from # unix policy only for the main thread. # Main thread is required for subscription on SIGCHLD signal + if self._saved_sighandler is not None: + return + + self._saved_sighandler = signal.signal(signal.SIGCHLD, self._sig_chld) if self._saved_sighandler is None: - self._saved_sighandler = signal.signal(signal.SIGCHLD, self._sig_chld) - if self._saved_sighandler is None: - logger.warning("Previous SIGCHLD handler was set by non-Python code, " - "restore to default handler on watcher close.") - self._saved_sighandler = signal.SIG_DFL + logger.warning("Previous SIGCHLD handler was set by non-Python code, " + "restore to default handler on watcher close.") + self._saved_sighandler = signal.SIG_DFL - # Set SA_RESTART to limit EINTR occurrences. - signal.siginterrupt(signal.SIGCHLD, False) + # Set SA_RESTART to limit EINTR occurrences. + signal.siginterrupt(signal.SIGCHLD, False) def _do_waitpid_all(self): for pid in list(self._callbacks): @@ -1296,7 +1296,7 @@ def _do_waitpid(self, expected_pid): # The child process is still alive. return - returncode = _compute_returncode(status) + returncode = waitstatus_to_exitcode(status) debug_log = True try: loop, callback, args = self._callbacks.pop(pid) @@ -1399,7 +1399,7 @@ def _do_waitpid(self, loop, expected_pid, callback, args): "Unknown child process pid %d, will report returncode 255", pid) else: - returncode = _compute_returncode(status) + returncode = waitstatus_to_exitcode(status) if loop.get_debug(): logger.debug('process %s exited with returncode %s', expected_pid, returncode) diff --git a/Lib/asyncore.py b/Lib/asyncore.py index ce16f11a2f5cee..eeea48888616d7 100644 --- a/Lib/asyncore.py +++ b/Lib/asyncore.py @@ -113,7 +113,7 @@ def readwrite(obj, flags): if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL): obj.handle_close() except OSError as e: - if e.args[0] not in _DISCONNECTED: + if e.errno not in _DISCONNECTED: obj.handle_error() else: obj.handle_close() @@ -236,7 +236,7 @@ def __init__(self, sock=None, map=None): try: self.addr = sock.getpeername() except OSError as err: - if err.args[0] in (ENOTCONN, EINVAL): + if err.errno in (ENOTCONN, EINVAL): # To handle the case where we got an unconnected # socket. self.connected = False @@ -346,7 +346,7 @@ def accept(self): except TypeError: return None except OSError as why: - if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN): + if why.errno in (EWOULDBLOCK, ECONNABORTED, EAGAIN): return None else: raise @@ -358,9 +358,9 @@ def send(self, data): result = self.socket.send(data) return result except OSError as why: - if why.args[0] == EWOULDBLOCK: + if why.errno == EWOULDBLOCK: return 0 - elif why.args[0] in _DISCONNECTED: + elif why.errno in _DISCONNECTED: self.handle_close() return 0 else: @@ -378,7 +378,7 @@ def recv(self, buffer_size): return data except OSError as why: # winsock sometimes raises ENOTCONN - if why.args[0] in _DISCONNECTED: + if why.errno in _DISCONNECTED: self.handle_close() return b'' else: @@ -393,7 +393,7 @@ def close(self): try: self.socket.close() except OSError as why: - if why.args[0] not in (ENOTCONN, EBADF): + if why.errno not in (ENOTCONN, EBADF): raise # log and log_info may be overridden to provide more sophisticated @@ -557,7 +557,7 @@ def close_all(map=None, ignore_all=False): try: x.close() except OSError as x: - if x.args[0] == EBADF: + if x.errno == EBADF: pass elif not ignore_all: raise diff --git a/Lib/base64.py b/Lib/base64.py index 539ad16f0e86d6..e1256ad9358e7a 100755 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -344,7 +344,7 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): global _a85chars, _a85chars2 # Delay the initialization of tables to not waste memory # if the function is never called - if _a85chars is None: + if _a85chars2 is None: _a85chars = [bytes((i,)) for i in range(33, 118)] _a85chars2 = [(a + b) for a in _a85chars for b in _a85chars] @@ -452,7 +452,7 @@ def b85encode(b, pad=False): global _b85chars, _b85chars2 # Delay the initialization of tables to not waste memory # if the function is never called - if _b85chars is None: + if _b85chars2 is None: _b85chars = [bytes((i,)) for i in _b85alphabet] _b85chars2 = [(a + b) for a in _b85chars for b in _b85chars] return _85encode(b, _b85chars, _b85chars2, pad) diff --git a/Lib/bdb.py b/Lib/bdb.py index b18a0612d8c789..abb50c092e69bb 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -34,6 +34,8 @@ def __init__(self, skip=None): self.fncache = {} self.frame_returning = None + self._load_breaks() + def canonic(self, filename): """Return canonical form of filename. @@ -365,6 +367,12 @@ def set_quit(self): # Call self.get_*break*() to see the breakpoints or better # for bp in Breakpoint.bpbynumber: if bp: bp.bpprint(). + def _add_to_breaks(self, filename, lineno): + """Add breakpoint to breaks, if not already there.""" + bp_linenos = self.breaks.setdefault(filename, []) + if lineno not in bp_linenos: + bp_linenos.append(lineno) + def set_break(self, filename, lineno, temporary=False, cond=None, funcname=None): """Set a new breakpoint for filename:lineno. @@ -377,12 +385,21 @@ def set_break(self, filename, lineno, temporary=False, cond=None, line = linecache.getline(filename, lineno) if not line: return 'Line %s:%d does not exist' % (filename, lineno) - list = self.breaks.setdefault(filename, []) - if lineno not in list: - list.append(lineno) + self._add_to_breaks(filename, lineno) bp = Breakpoint(filename, lineno, temporary, cond, funcname) return None + def _load_breaks(self): + """Apply all breakpoints (set in other instances) to this one. + + Populates this instance's breaks list from the Breakpoint class's + list, which can have breakpoints set by another Bdb instance. This + is necessary for interactive sessions to keep the breakpoints + active across multiple calls to run(). + """ + for (filename, lineno) in Breakpoint.bplist.keys(): + self._add_to_breaks(filename, lineno) + def _prune_breaks(self, filename, lineno): """Prune breakpoints for filename:lineno. @@ -681,6 +698,12 @@ def __init__(self, file, line, temporary=False, cond=None, funcname=None): else: self.bplist[file, line] = [self] + @staticmethod + def clearBreakpoints(): + Breakpoint.next = 1 + Breakpoint.bplist = {} + Breakpoint.bpbynumber = [None] + def deleteMe(self): """Delete the breakpoint from the list associated to a file:line. diff --git a/Lib/binhex.py b/Lib/binhex.py index 9559f46d5a2882..ace5217d271392 100644 --- a/Lib/binhex.py +++ b/Lib/binhex.py @@ -117,12 +117,12 @@ def _flush(self, force): first = 0 while first <= len(self.hqxdata) - self.linelen: last = first + self.linelen - self.ofp.write(self.hqxdata[first:last] + b'\n') + self.ofp.write(self.hqxdata[first:last] + b'\r') self.linelen = LINELEN first = last self.hqxdata = self.hqxdata[first:] if force: - self.ofp.write(self.hqxdata + b':\n') + self.ofp.write(self.hqxdata + b':\r') def close(self): if self.data: diff --git a/Lib/bz2.py b/Lib/bz2.py index ce07ebeb142d92..a2c588e7487f3d 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -13,7 +13,6 @@ import io import os import _compression -from threading import RLock from _bz2 import BZ2Compressor, BZ2Decompressor @@ -53,9 +52,6 @@ def __init__(self, filename, mode="r", *, compresslevel=9): If mode is 'r', the input file may be the concatenation of multiple compressed streams. """ - # This lock must be recursive, so that BufferedIOBase's - # writelines() does not deadlock. - self._lock = RLock() self._fp = None self._closefp = False self._mode = _MODE_CLOSED @@ -104,24 +100,23 @@ def close(self): May be called more than once without error. Once the file is closed, any other operation on it will raise a ValueError. """ - with self._lock: - if self._mode == _MODE_CLOSED: - return + if self._mode == _MODE_CLOSED: + return + try: + if self._mode == _MODE_READ: + self._buffer.close() + elif self._mode == _MODE_WRITE: + self._fp.write(self._compressor.flush()) + self._compressor = None + finally: try: - if self._mode == _MODE_READ: - self._buffer.close() - elif self._mode == _MODE_WRITE: - self._fp.write(self._compressor.flush()) - self._compressor = None + if self._closefp: + self._fp.close() finally: - try: - if self._closefp: - self._fp.close() - finally: - self._fp = None - self._closefp = False - self._mode = _MODE_CLOSED - self._buffer = None + self._fp = None + self._closefp = False + self._mode = _MODE_CLOSED + self._buffer = None @property def closed(self): @@ -153,12 +148,11 @@ def peek(self, n=0): Always returns at least one byte of data, unless at EOF. The exact number of bytes returned is unspecified. """ - with self._lock: - self._check_can_read() - # Relies on the undocumented fact that BufferedReader.peek() - # always returns at least one byte (except at EOF), independent - # of the value of n - return self._buffer.peek(n) + self._check_can_read() + # Relies on the undocumented fact that BufferedReader.peek() + # always returns at least one byte (except at EOF), independent + # of the value of n + return self._buffer.peek(n) def read(self, size=-1): """Read up to size uncompressed bytes from the file. @@ -166,9 +160,8 @@ def read(self, size=-1): If size is negative or omitted, read until EOF is reached. Returns b'' if the file is already at EOF. """ - with self._lock: - self._check_can_read() - return self._buffer.read(size) + self._check_can_read() + return self._buffer.read(size) def read1(self, size=-1): """Read up to size uncompressed bytes, while trying to avoid @@ -177,20 +170,18 @@ def read1(self, size=-1): Returns b'' if the file is at EOF. """ - with self._lock: - self._check_can_read() - if size < 0: - size = io.DEFAULT_BUFFER_SIZE - return self._buffer.read1(size) + self._check_can_read() + if size < 0: + size = io.DEFAULT_BUFFER_SIZE + return self._buffer.read1(size) def readinto(self, b): """Read bytes into b. Returns the number of bytes read (0 for EOF). """ - with self._lock: - self._check_can_read() - return self._buffer.readinto(b) + self._check_can_read() + return self._buffer.readinto(b) def readline(self, size=-1): """Read a line of uncompressed bytes from the file. @@ -203,9 +194,12 @@ def readline(self, size=-1): if not hasattr(size, "__index__"): raise TypeError("Integer argument expected") size = size.__index__() - with self._lock: - self._check_can_read() - return self._buffer.readline(size) + self._check_can_read() + return self._buffer.readline(size) + + def __iter__(self): + self._check_can_read() + return self._buffer.__iter__() def readlines(self, size=-1): """Read a list of lines of uncompressed bytes from the file. @@ -218,9 +212,8 @@ def readlines(self, size=-1): if not hasattr(size, "__index__"): raise TypeError("Integer argument expected") size = size.__index__() - with self._lock: - self._check_can_read() - return self._buffer.readlines(size) + self._check_can_read() + return self._buffer.readlines(size) def write(self, data): """Write a byte string to the file. @@ -229,12 +222,11 @@ def write(self, data): always len(data). Note that due to buffering, the file on disk may not reflect the data written until close() is called. """ - with self._lock: - self._check_can_write() - compressed = self._compressor.compress(data) - self._fp.write(compressed) - self._pos += len(data) - return len(data) + self._check_can_write() + compressed = self._compressor.compress(data) + self._fp.write(compressed) + self._pos += len(data) + return len(data) def writelines(self, seq): """Write a sequence of byte strings to the file. @@ -244,8 +236,7 @@ def writelines(self, seq): Line separators are not added between the written byte strings. """ - with self._lock: - return _compression.BaseStream.writelines(self, seq) + return _compression.BaseStream.writelines(self, seq) def seek(self, offset, whence=io.SEEK_SET): """Change the file position. @@ -262,17 +253,15 @@ def seek(self, offset, whence=io.SEEK_SET): Note that seeking is emulated, so depending on the parameters, this operation may be extremely slow. """ - with self._lock: - self._check_can_seek() - return self._buffer.seek(offset, whence) + self._check_can_seek() + return self._buffer.seek(offset, whence) def tell(self): """Return the current file position.""" - with self._lock: - self._check_not_closed() - if self._mode == _MODE_READ: - return self._buffer.tell() - return self._pos + self._check_not_closed() + if self._mode == _MODE_READ: + return self._buffer.tell() + return self._pos def open(filename, mode="rb", compresslevel=9, @@ -311,6 +300,7 @@ def open(filename, mode="rb", compresslevel=9, binary_file = BZ2File(filename, bz_mode, compresslevel=compresslevel) if "t" in mode: + encoding = io.text_encoding(encoding) return io.TextIOWrapper(binary_file, encoding, errors, newline) else: return binary_file diff --git a/Lib/cProfile.py b/Lib/cProfile.py index 59b4699feb5062..22a7d0aade855f 100755 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -175,7 +175,12 @@ def main(): '__package__': None, '__cached__': None, } - runctx(code, globs, None, options.outfile, options.sort) + try: + runctx(code, globs, None, options.outfile, options.sort) + except BrokenPipeError as exc: + # Prevent "Exception ignored" during interpreter shutdown. + sys.stdout = None + sys.exit(exc.errno) else: parser.print_usage() return parser diff --git a/Lib/cgi.py b/Lib/cgi.py index 77ab703cc03600..6cb8cf28bd6645 100755 --- a/Lib/cgi.py +++ b/Lib/cgi.py @@ -41,6 +41,7 @@ import html import locale import tempfile +import warnings __all__ = ["MiniFieldStorage", "FieldStorage", "parse", "parse_multipart", "parse_header", "test", "print_exception", "print_environ", @@ -77,9 +78,11 @@ def initlog(*allargs): """ global log, logfile, logfp + warnings.warn("cgi.log() is deprecated as of 3.10. Use logging instead", + DeprecationWarning, stacklevel=2) if logfile and not logfp: try: - logfp = open(logfile, "a") + logfp = open(logfile, "a", encoding="locale") except OSError: pass if not logfp: @@ -115,7 +118,8 @@ def closelog(): # 0 ==> unlimited input maxlen = 0 -def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0): +def parse(fp=None, environ=os.environ, keep_blank_values=0, + strict_parsing=0, separator='&'): """Parse a query in the environment or from a file (default stdin) Arguments, all optional: @@ -134,6 +138,9 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0): strict_parsing: flag indicating what to do with parsing errors. If false (the default), errors are silently ignored. If true, errors raise a ValueError exception. + + separator: str. The symbol to use for separating the query arguments. + Defaults to &. """ if fp is None: fp = sys.stdin @@ -154,7 +161,7 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0): if environ['REQUEST_METHOD'] == 'POST': ctype, pdict = parse_header(environ['CONTENT_TYPE']) if ctype == 'multipart/form-data': - return parse_multipart(fp, pdict) + return parse_multipart(fp, pdict, separator=separator) elif ctype == 'application/x-www-form-urlencoded': clength = int(environ['CONTENT_LENGTH']) if maxlen and clength > maxlen: @@ -178,10 +185,10 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0): qs = "" environ['QUERY_STRING'] = qs # XXX Shouldn't, really return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing, - encoding=encoding) + encoding=encoding, separator=separator) -def parse_multipart(fp, pdict, encoding="utf-8", errors="replace"): +def parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator='&'): """Parse multipart input. Arguments: @@ -194,7 +201,7 @@ def parse_multipart(fp, pdict, encoding="utf-8", errors="replace"): value is a list of values for that field. For non-file fields, the value is a list of strings. """ - # RFC 2026, Section 5.1 : The "multipart" boundary delimiters are always + # RFC 2046, Section 5.1 : The "multipart" boundary delimiters are always # represented as 7bit US-ASCII. boundary = pdict['boundary'].decode('ascii') ctype = "multipart/form-data; boundary={}".format(boundary) @@ -205,7 +212,7 @@ def parse_multipart(fp, pdict, encoding="utf-8", errors="replace"): except KeyError: pass fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors, - environ={'REQUEST_METHOD': 'POST'}) + environ={'REQUEST_METHOD': 'POST'}, separator=separator) return {k: fs.getlist(k) for k in fs} def _parseparam(s): @@ -315,7 +322,7 @@ class FieldStorage: def __init__(self, fp=None, headers=None, outerboundary=b'', environ=os.environ, keep_blank_values=0, strict_parsing=0, limit=None, encoding='utf-8', errors='replace', - max_num_fields=None): + max_num_fields=None, separator='&'): """Constructor. Read multipart/* until last part. Arguments, all optional: @@ -363,6 +370,7 @@ def __init__(self, fp=None, headers=None, outerboundary=b'', self.keep_blank_values = keep_blank_values self.strict_parsing = strict_parsing self.max_num_fields = max_num_fields + self.separator = separator if 'REQUEST_METHOD' in environ: method = environ['REQUEST_METHOD'].upper() self.qs_on_post = None @@ -589,7 +597,7 @@ def read_urlencoded(self): query = urllib.parse.parse_qsl( qs, self.keep_blank_values, self.strict_parsing, encoding=self.encoding, errors=self.errors, - max_num_fields=self.max_num_fields) + max_num_fields=self.max_num_fields, separator=self.separator) self.list = [MiniFieldStorage(key, value) for key, value in query] self.skip_lines() @@ -605,7 +613,7 @@ def read_multi(self, environ, keep_blank_values, strict_parsing): query = urllib.parse.parse_qsl( self.qs_on_post, self.keep_blank_values, self.strict_parsing, encoding=self.encoding, errors=self.errors, - max_num_fields=self.max_num_fields) + max_num_fields=self.max_num_fields, separator=self.separator) self.list.extend(MiniFieldStorage(key, value) for key, value in query) klass = self.FieldStorageClass or self.__class__ @@ -649,7 +657,7 @@ def read_multi(self, environ, keep_blank_values, strict_parsing): else self.limit - self.bytes_read part = klass(self.fp, headers, ib, environ, keep_blank_values, strict_parsing, limit, - self.encoding, self.errors, max_num_fields) + self.encoding, self.errors, max_num_fields, self.separator) if max_num_fields is not None: max_num_fields -= 1 diff --git a/Lib/codecs.py b/Lib/codecs.py index 3935490d88c8d3..e6ad6e3a052364 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -386,7 +386,7 @@ def writelines(self, list): def reset(self): - """ Flushes and resets the codec buffers used for keeping state. + """ Resets the codec buffers used for keeping internal state. Calling this method should ensure that the data on the output is put into a clean state, that allows appending @@ -620,7 +620,7 @@ def readlines(self, sizehint=None, keepends=True): def reset(self): - """ Resets the codec buffers used for keeping state. + """ Resets the codec buffers used for keeping internal state. Note that no stream repositioning should take place. This method is primarily intended to be able to recover diff --git a/Lib/codeop.py b/Lib/codeop.py index 4c10470aee7b7c..6b56be488eeb03 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -64,24 +64,21 @@ __all__ = ["compile_command", "Compile", "CommandCompiler"] -PyCF_DONT_IMPLY_DEDENT = 0x200 # Matches pythonrun.h +PyCF_DONT_IMPLY_DEDENT = 0x200 # Matches pythonrun.h. def _maybe_compile(compiler, source, filename, symbol): - # Check for source consisting of only blank lines and comments + # Check for source consisting of only blank lines and comments. for line in source.split("\n"): line = line.strip() if line and line[0] != '#': - break # Leave it alone + break # Leave it alone. else: if symbol != "eval": source = "pass" # Replace it with a 'pass' statement - err = err1 = err2 = None - code = code1 = code2 = None - try: - code = compiler(source, filename, symbol) - except SyntaxError: + return compiler(source, filename, symbol) + except SyntaxError: # Let other compile() errors propagate. pass # Catch syntax warnings after the first compile @@ -89,6 +86,7 @@ def _maybe_compile(compiler, source, filename, symbol): with warnings.catch_warnings(): warnings.simplefilter("error") + code1 = err1 = err2 = None try: code1 = compiler(source + "\n", filename, symbol) except SyntaxError as e: @@ -100,13 +98,22 @@ def _maybe_compile(compiler, source, filename, symbol): err2 = e try: - if code: - return code - if not code1 and repr(err1) == repr(err2): + if not code1 and _is_syntax_error(err1, err2): raise err1 + else: + return None finally: err1 = err2 = None +def _is_syntax_error(err1, err2): + rep1 = repr(err1) + rep2 = repr(err2) + if "was never closed" in rep1 and "was never closed" in rep2: + return False + if rep1 == rep2: + return True + return False + def _compile(source, filename, symbol): return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 5d75501645fc4a..bae0805d6686c5 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -51,22 +51,6 @@ pass -def __getattr__(name): - # For backwards compatibility, continue to make the collections ABCs - # through Python 3.6 available through the collections module. - # Note, no new collections ABCs were added in Python 3.7 - if name in _collections_abc.__all__: - obj = getattr(_collections_abc, name) - import warnings - warnings.warn("Using or importing the ABCs from 'collections' instead " - "of from 'collections.abc' is deprecated since Python 3.3, " - "and in 3.10 it will stop working", - DeprecationWarning, stacklevel=2) - globals()[name] = obj - return obj - raise AttributeError(f'module {__name__!r} has no attribute {name!r}') - - ################################################################################ ### OrderedDict ################################################################################ @@ -423,7 +407,7 @@ def namedtuple(typename, field_names, *, rename=False, defaults=None, module=Non namespace = { '_tuple_new': tuple_new, - '__builtins__': None, + '__builtins__': {}, '__name__': f'namedtuple_{typename}', } code = f'lambda _cls, {arg_list}: _tuple_new(_cls, ({arg_list}))' @@ -488,6 +472,7 @@ def __getnewargs__(self): '__repr__': __repr__, '_asdict': _asdict, '__getnewargs__': __getnewargs__, + '__match_args__': field_names, } for index, name in enumerate(field_names): doc = _sys.intern(f'Alias for field number {index}') @@ -596,6 +581,10 @@ def __missing__(self, key): # Needed so that self[missing_item] does not raise KeyError return 0 + def total(self): + 'Sum of the counts' + return sum(self.values()) + def most_common(self, n=None): '''List the n most common elements and their counts from the most common to the least. If n is None, then list all element counts. @@ -1001,7 +990,7 @@ def __len__(self): def __iter__(self): d = {} for mapping in reversed(self.maps): - d.update(mapping) # reuses stored hash values if possible + d.update(dict.fromkeys(mapping)) # reuses stored hash values if possible return iter(d) def __contains__(self, key): @@ -1025,12 +1014,15 @@ def copy(self): __copy__ = copy - def new_child(self, m=None): # like Django's Context.push() + def new_child(self, m=None, **kwargs): # like Django's Context.push() '''New ChainMap with a new map followed by all previous maps. If no map is provided, an empty dict is used. + Keyword arguments update the map or new empty dict. ''' if m is None: - m = {} + m = kwargs + elif kwargs: + m.update(kwargs) return self.__class__(m, *self.maps) @property diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py index 891600d16bee9e..86ca8b8a8414b3 100644 --- a/Lib/collections/abc.py +++ b/Lib/collections/abc.py @@ -1,2 +1,3 @@ from _collections_abc import * from _collections_abc import __all__ +from _collections_abc import _CallableGenericAlias diff --git a/Lib/colorsys.py b/Lib/colorsys.py index b93e3844067e4e..0f52512a67d87c 100644 --- a/Lib/colorsys.py +++ b/Lib/colorsys.py @@ -75,17 +75,18 @@ def yiq_to_rgb(y, i, q): def rgb_to_hls(r, g, b): maxc = max(r, g, b) minc = min(r, g, b) - # XXX Can optimize (maxc+minc) and (maxc-minc) - l = (minc+maxc)/2.0 + sumc = (maxc+minc) + rangec = (maxc-minc) + l = sumc/2.0 if minc == maxc: return 0.0, l, 0.0 if l <= 0.5: - s = (maxc-minc) / (maxc+minc) + s = rangec / sumc else: - s = (maxc-minc) / (2.0-maxc-minc) - rc = (maxc-r) / (maxc-minc) - gc = (maxc-g) / (maxc-minc) - bc = (maxc-b) / (maxc-minc) + s = rangec / (2.0-sumc) + rc = (maxc-r) / rangec + gc = (maxc-g) / rangec + bc = (maxc-b) / rangec if r == maxc: h = bc-gc elif g == maxc: diff --git a/Lib/compileall.py b/Lib/compileall.py index fe7f450c55e1c5..61b4c5c0af1ad6 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -84,12 +84,14 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, if workers < 0: raise ValueError('workers must be greater or equal to 0') if workers != 1: + # Check if this is a system where ProcessPoolExecutor can function. + from concurrent.futures.process import _check_system_limits try: - # Only import when needed, as low resource platforms may - # fail to import it - from concurrent.futures import ProcessPoolExecutor - except ImportError: + _check_system_limits() + except NotImplementedError: workers = 1 + else: + from concurrent.futures import ProcessPoolExecutor if maxlevels is None: maxlevels = sys.getrecursionlimit() files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels) @@ -405,7 +407,8 @@ def main(): # if flist is provided then load it if args.flist: try: - with (sys.stdin if args.flist=='-' else open(args.flist)) as f: + with (sys.stdin if args.flist=='-' else + open(args.flist, encoding="utf-8")) as f: for line in f: compile_dests.append(line.strip()) except OSError: diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index 00eb54881f2958..6095026cb278b9 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -386,7 +386,11 @@ def done(self): def __get_result(self): if self._exception: - raise self._exception + try: + raise self._exception + finally: + # Break a reference cycle with the exception in self._exception + self = None else: return self._result @@ -426,20 +430,24 @@ def result(self, timeout=None): timeout. Exception: If the call raised then that exception will be raised. """ - with self._condition: - if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]: - raise CancelledError() - elif self._state == FINISHED: - return self.__get_result() - - self._condition.wait(timeout) - - if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]: - raise CancelledError() - elif self._state == FINISHED: - return self.__get_result() - else: - raise TimeoutError() + try: + with self._condition: + if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]: + raise CancelledError() + elif self._state == FINISHED: + return self.__get_result() + + self._condition.wait(timeout) + + if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]: + raise CancelledError() + elif self._state == FINISHED: + return self.__get_result() + else: + raise TimeoutError() + finally: + # Break a reference cycle with the exception in self._exception + self = None def exception(self, timeout=None): """Return the exception raised by the call that the future represents. diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 90bc98bf2ecd17..764719859f7cea 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -532,6 +532,14 @@ def _check_system_limits(): if _system_limited: raise NotImplementedError(_system_limited) _system_limits_checked = True + try: + import multiprocessing.synchronize + except ImportError: + _system_limited = ( + "This Python build lacks multiprocessing.synchronize, usually due " + "to named semaphores being unavailable on this platform." + ) + raise NotImplementedError(_system_limited) try: nsems_max = os.sysconf("SC_SEM_NSEMS_MAX") except (AttributeError, ValueError): diff --git a/Lib/configparser.py b/Lib/configparser.py index 924cc56a3f150d..3b4cb5e6b2407f 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -690,6 +690,7 @@ def read(self, filenames, encoding=None): """ if isinstance(filenames, (str, bytes, os.PathLike)): filenames = [filenames] + encoding = io.text_encoding(encoding) read_ok = [] for filename in filenames: try: diff --git a/Lib/contextlib.py b/Lib/contextlib.py index ff92d9f913f4c2..eb5946145b47ee 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -9,7 +9,7 @@ __all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext", "AbstractContextManager", "AbstractAsyncContextManager", "AsyncExitStack", "ContextDecorator", "ExitStack", - "redirect_stdout", "redirect_stderr", "suppress"] + "redirect_stdout", "redirect_stderr", "suppress", "aclosing"] class AbstractContextManager(abc.ABC): @@ -80,6 +80,22 @@ def inner(*args, **kwds): return inner +class AsyncContextDecorator(object): + "A base class or mixin that enables async context managers to work as decorators." + + def _recreate_cm(self): + """Return a recreated instance of self. + """ + return self + + def __call__(self, func): + @wraps(func) + async def inner(*args, **kwds): + async with self._recreate_cm(): + return await func(*args, **kwds) + return inner + + class _GeneratorContextManagerBase: """Shared functionality for @contextmanager and @asynccontextmanager.""" @@ -167,9 +183,16 @@ def __exit__(self, type, value, traceback): class _AsyncGeneratorContextManager(_GeneratorContextManagerBase, - AbstractAsyncContextManager): + AbstractAsyncContextManager, + AsyncContextDecorator): """Helper for @asynccontextmanager.""" + def _recreate_cm(self): + # _AGCM instances are one-shot context managers, so the + # ACM must be recreated each time a decorated function is + # called + return self.__class__(self.func, self.args, self.kwds) + async def __aenter__(self): try: return await self.gen.__anext__() @@ -303,6 +326,32 @@ def __exit__(self, *exc_info): self.thing.close() +class aclosing(AbstractAsyncContextManager): + """Async context manager for safely finalizing an asynchronously cleaned-up + resource such as an async generator, calling its ``aclose()`` method. + + Code like this: + + async with aclosing(.fetch()) as agen: + + + is equivalent to this: + + agen = .fetch() + try: + + finally: + await agen.aclose() + + """ + def __init__(self, thing): + self.thing = thing + async def __aenter__(self): + return self.thing + async def __aexit__(self, *exc_info): + await self.thing.aclose() + + class _RedirectStream(AbstractContextManager): _stream = None @@ -655,7 +704,7 @@ def _fix_exception_context(new_exc, old_exc): return received_exc and suppressed_exc -class nullcontext(AbstractContextManager): +class nullcontext(AbstractContextManager, AbstractAsyncContextManager): """Context manager that does no additional processing. Used as a stand-in for a normal context manager, when a particular @@ -674,3 +723,9 @@ def __enter__(self): def __exit__(self, *excinfo): pass + + async def __aenter__(self): + return self.enter_result + + async def __aexit__(self, *excinfo): + pass diff --git a/Lib/ctypes/macholib/dyld.py b/Lib/ctypes/macholib/dyld.py index 9d86b058765a3e..1c3f8fd38b0665 100644 --- a/Lib/ctypes/macholib/dyld.py +++ b/Lib/ctypes/macholib/dyld.py @@ -6,6 +6,11 @@ from ctypes.macholib.framework import framework_info from ctypes.macholib.dylib import dylib_info from itertools import * +try: + from _ctypes import _dyld_shared_cache_contains_path +except ImportError: + def _dyld_shared_cache_contains_path(*args): + raise NotImplementedError __all__ = [ 'dyld_find', 'framework_find', @@ -122,8 +127,15 @@ def dyld_find(name, executable_path=None, env=None): dyld_executable_path_search(name, executable_path), dyld_default_search(name, env), ), env): + if os.path.isfile(path): return path + try: + if _dyld_shared_cache_contains_path(path): + return path + except NotImplementedError: + pass + raise ValueError("dylib %s could not be found" % (name,)) def framework_find(fn, executable_path=None, env=None): diff --git a/Lib/ctypes/test/test_bitfields.py b/Lib/ctypes/test/test_bitfields.py index 992b8c4da3a776..68b618a05f436b 100644 --- a/Lib/ctypes/test/test_bitfields.py +++ b/Lib/ctypes/test/test_bitfields.py @@ -233,6 +233,69 @@ class X(Structure): else: self.assertEqual(sizeof(X), sizeof(c_int) * 2) + @unittest.skipIf(os.name == 'nt', reason='Posix only') + def test_packed_posix(self): + test_cases = { + ( + ("a", c_uint8, 4), + ("b", c_uint8, 4), + ): 1, + ( + ("a", c_uint8, 1), + ("b", c_uint16, 1), + ("c", c_uint32, 1), + ("d", c_uint64, 1), + ): 1, + ( + ("a", c_uint8, 8), + ("b", c_uint16, 1), + ("c", c_uint32, 1), + ("d", c_uint64, 1), + ): 2, + ( + ("a", c_uint32, 9), + ("b", c_uint16, 10), + ("c", c_uint32, 25), + ("d", c_uint64, 1), + ): 6, + ( + ("a", c_uint32, 9), + ("b", c_uint16, 10), + ("c", c_uint32, 25), + ("d", c_uint64, 5), + ): 7, + ( + ("a", c_uint16), + ("b", c_uint16, 9), + ("c", c_uint16, 1), + ("d", c_uint16, 1), + ("e", c_uint16, 1), + ("f", c_uint16, 1), + ("g", c_uint16, 3), + ("h", c_uint32, 10), + ("i", c_uint32, 20), + ("j", c_uint32, 2), + ): 8, + ( + ("a", c_uint16, 9), + ("b", c_uint16, 10), + ("d", c_uint16), + ("c", c_uint8, 8), + ): 6, + ( + ("a", c_uint32, 9), + ("b", c_uint32), + ("c", c_uint32, 8), + ): 7, + } + + for fields, size in test_cases.items(): + with self.subTest(fields=fields): + class X(Structure): + _pack_ = 1 + _fields_ = list(fields) + self.assertEqual(sizeof(X), size) + def test_anon_bitfields(self): # anonymous bit-fields gave a strange error message class X(Structure): diff --git a/Lib/ctypes/test/test_find.py b/Lib/ctypes/test/test_find.py index 4a8a3820f3fe1c..1ff9d019b138a4 100644 --- a/Lib/ctypes/test/test_find.py +++ b/Lib/ctypes/test/test_find.py @@ -90,7 +90,7 @@ def test_find_on_libpath(self): srcname = os.path.join(d, 'dummy.c') libname = 'py_ctypes_test_dummy' dstname = os.path.join(d, 'lib%s.so' % libname) - with open(srcname, 'w') as f: + with open(srcname, 'wb') as f: pass self.assertTrue(os.path.exists(srcname)) # compile the file to a shared library diff --git a/Lib/ctypes/test/test_macholib.py b/Lib/ctypes/test/test_macholib.py index 6b3526951acfab..a1bac26a7df058 100644 --- a/Lib/ctypes/test/test_macholib.py +++ b/Lib/ctypes/test/test_macholib.py @@ -45,19 +45,22 @@ def find_lib(name): class MachOTest(unittest.TestCase): @unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test') def test_find(self): - - self.assertEqual(find_lib('pthread'), - '/usr/lib/libSystem.B.dylib') + # On Mac OS 11, system dylibs are only present in the shared cache, + # so symlinks like libpthread.dylib -> libSystem.B.dylib will not + # be resolved by dyld_find + self.assertIn(find_lib('pthread'), + ('/usr/lib/libSystem.B.dylib', '/usr/lib/libpthread.dylib')) result = find_lib('z') # Issue #21093: dyld default search path includes $HOME/lib and # /usr/local/lib before /usr/lib, which caused test failures if # a local copy of libz exists in one of them. Now ignore the head # of the path. - self.assertRegex(result, r".*/lib/libz\..*.*\.dylib") + self.assertRegex(result, r".*/lib/libz.*\.dylib") - self.assertEqual(find_lib('IOKit'), - '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit') + self.assertIn(find_lib('IOKit'), + ('/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit', + '/System/Library/Frameworks/IOKit.framework/IOKit')) if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_parameters.py b/Lib/ctypes/test/test_parameters.py index e4c25fd880cefb..38af7ac13d756c 100644 --- a/Lib/ctypes/test/test_parameters.py +++ b/Lib/ctypes/test/test_parameters.py @@ -201,6 +201,49 @@ def __dict__(self): with self.assertRaises(ZeroDivisionError): WorseStruct().__setstate__({}, b'foo') + def test_parameter_repr(self): + from ctypes import ( + c_bool, + c_char, + c_wchar, + c_byte, + c_ubyte, + c_short, + c_ushort, + c_int, + c_uint, + c_long, + c_ulong, + c_longlong, + c_ulonglong, + c_float, + c_double, + c_longdouble, + c_char_p, + c_wchar_p, + c_void_p, + ) + self.assertRegex(repr(c_bool.from_param(True)), r"^$") + self.assertEqual(repr(c_char.from_param(97)), "") + self.assertRegex(repr(c_wchar.from_param('a')), r"^$") + self.assertEqual(repr(c_byte.from_param(98)), "") + self.assertEqual(repr(c_ubyte.from_param(98)), "") + self.assertEqual(repr(c_short.from_param(511)), "") + self.assertEqual(repr(c_ushort.from_param(511)), "") + self.assertRegex(repr(c_int.from_param(20000)), r"^$") + self.assertRegex(repr(c_uint.from_param(20000)), r"^$") + self.assertRegex(repr(c_long.from_param(20000)), r"^$") + self.assertRegex(repr(c_ulong.from_param(20000)), r"^$") + self.assertRegex(repr(c_longlong.from_param(20000)), r"^$") + self.assertRegex(repr(c_ulonglong.from_param(20000)), r"^$") + self.assertEqual(repr(c_float.from_param(1.5)), "") + self.assertEqual(repr(c_double.from_param(1.5)), "") + self.assertEqual(repr(c_double.from_param(1e300)), "") + self.assertRegex(repr(c_longdouble.from_param(1.5)), r"^$") + self.assertRegex(repr(c_char_p.from_param(b'hihi')), r"^$") + self.assertRegex(repr(c_wchar_p.from_param('hihi')), r"^$") + self.assertRegex(repr(c_void_p.from_param(0x12)), r"^$") + ################################################################ if __name__ == '__main__': diff --git a/Lib/ctypes/test/test_unicode.py b/Lib/ctypes/test/test_unicode.py index c200af7b650661..60c75424b767fa 100644 --- a/Lib/ctypes/test/test_unicode.py +++ b/Lib/ctypes/test/test_unicode.py @@ -26,6 +26,14 @@ def test_buffers(self): self.assertEqual(buf[::2], 'a\xe4\xfc') self.assertEqual(buf[6:5:-1], "") + def test_embedded_null(self): + class TestStruct(ctypes.Structure): + _fields_ = [("unicode", ctypes.c_wchar_p)] + t = TestStruct() + # This would raise a ValueError: + t.unicode = "foo\0bar\0\0" + + func = ctypes.CDLL(_ctypes_test.__file__)._testfunc_p_p class StringTestCase(UnicodeTestCase): diff --git a/Lib/ctypes/test/test_values.py b/Lib/ctypes/test/test_values.py index 87eb9198ade0c7..7514fe84ff53c8 100644 --- a/Lib/ctypes/test/test_values.py +++ b/Lib/ctypes/test/test_values.py @@ -80,9 +80,9 @@ class struct_frozen(Structure): continue items.append((entry.name.decode("ascii"), entry.size)) - expected = [("__hello__", 141), - ("__phello__", -141), - ("__phello__.spam", 141), + expected = [("__hello__", 137), + ("__phello__", -137), + ("__phello__.spam", 137), ] self.assertEqual(items, expected, "PyImport_FrozenModules example " "in Doc/library/ctypes.rst may be out of date") diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 0c4b47564b0605..cbba320e01a54d 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -16,6 +16,7 @@ 'Field', 'FrozenInstanceError', 'InitVar', + 'KW_ONLY', 'MISSING', # Helper functions. @@ -152,6 +153,20 @@ # # See _hash_action (below) for a coded version of this table. +# __match_args__ +# +# +--- match_args= parameter +# | +# v | | | +# | no | yes | <--- class has __match_args__ in __dict__? +# +=======+=======+=======+ +# | False | | | +# +-------+-------+-------+ +# | True | add | | <- the default +# +=======+=======+=======+ +# __match_args__ is always added unless the class already defines it. It is a +# tuple of __init__ parameter names; non-init fields must be matched by keyword. + # Raised when an attempt is made to modify a frozen class. class FrozenInstanceError(AttributeError): pass @@ -170,6 +185,12 @@ class _MISSING_TYPE: pass MISSING = _MISSING_TYPE() +# A sentinel object to indicate that following fields are keyword-only by +# default. Use a class to give it a better repr. +class _KW_ONLY_TYPE: + pass +KW_ONLY = _KW_ONLY_TYPE() + # Since most per-field metadata will be unused, create an empty # read-only proxy that can be shared among all fields. _EMPTY_METADATA = types.MappingProxyType({}) @@ -218,7 +239,6 @@ def __repr__(self): def __class_getitem__(cls, type): return InitVar(type) - # Instances of Field are only ever created from within this module, # and only from the field() function, although Field instances are # exposed externally as (conceptually) read-only objects. @@ -239,11 +259,12 @@ class Field: 'init', 'compare', 'metadata', + 'kw_only', '_field_type', # Private: not to be used by user code. ) def __init__(self, default, default_factory, init, repr, hash, compare, - metadata): + metadata, kw_only): self.name = None self.type = None self.default = default @@ -255,6 +276,7 @@ def __init__(self, default, default_factory, init, repr, hash, compare, self.metadata = (_EMPTY_METADATA if metadata is None else types.MappingProxyType(metadata)) + self.kw_only = kw_only self._field_type = None def __repr__(self): @@ -268,6 +290,7 @@ def __repr__(self): f'hash={self.hash!r},' f'compare={self.compare!r},' f'metadata={self.metadata!r},' + f'kw_only={self.kw_only!r},' f'_field_type={self._field_type}' ')') @@ -321,17 +344,19 @@ def __repr__(self): # so that a type checker can be told (via overloads) that this is a # function whose type depends on its parameters. def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, - hash=None, compare=True, metadata=None): + hash=None, compare=True, metadata=None, kw_only=MISSING): """Return an object to identify dataclass fields. default is the default value of the field. default_factory is a 0-argument function called to initialize a field's value. If init - is True, the field will be a parameter to the class's __init__() - function. If repr is True, the field will be included in the - object's repr(). If hash is True, the field will be included in - the object's hash(). If compare is True, the field will be used - in comparison functions. metadata, if specified, must be a - mapping which is stored but not otherwise examined by dataclass. + is true, the field will be a parameter to the class's __init__() + function. If repr is true, the field will be included in the + object's repr(). If hash is true, the field will be included in the + object's hash(). If compare is true, the field will be used in + comparison functions. metadata, if specified, must be a mapping + which is stored but not otherwise examined by dataclass. If kw_only + is true, the field will become a keyword-only parameter to + __init__(). It is an error to specify both default and default_factory. """ @@ -339,7 +364,16 @@ def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, if default is not MISSING and default_factory is not MISSING: raise ValueError('cannot specify both default and default_factory') return Field(default, default_factory, init, repr, hash, compare, - metadata) + metadata, kw_only) + + +def _fields_in_init_order(fields): + # Returns the fields as __init__ will output them. It returns 2 tuples: + # the first for normal args, and the second for keyword args. + + return (tuple(f for f in fields if f.init and not f.kw_only), + tuple(f for f in fields if f.init and f.kw_only) + ) def _tuple_str(obj_name, fields): @@ -396,13 +430,10 @@ def _create_fn(name, args, body, *, globals=None, locals=None, local_vars = ', '.join(locals.keys()) txt = f"def __create_fn__({local_vars}):\n{txt}\n return {name}" - ns = {} exec(txt, globals, ns) - func = ns['__create_fn__'](**locals) - for arg, annotation in func.__annotations__.copy().items(): - func.__annotations__[arg] = locals[annotation] - return func + return ns['__create_fn__'](**locals) + def _field_assign(frozen, name, value, self_name): # If we're a frozen class, then assign to our fields in __init__ @@ -489,7 +520,8 @@ def _init_param(f): return f'{f.name}:_type_{f.name}{default}' -def _init_fn(fields, frozen, has_post_init, self_name, globals): +def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, + self_name, globals): # fields contains both real fields and InitVar pseudo-fields. # Make sure we don't have fields without defaults following fields @@ -497,9 +529,10 @@ def _init_fn(fields, frozen, has_post_init, self_name, globals): # function source code, but catching it here gives a better error # message, and future-proofs us in case we build up the function # using ast. + seen_default = False - for f in fields: - # Only consider fields in the __init__ call. + for f in std_fields: + # Only consider the non-kw-only fields in the __init__ call. if f.init: if not (f.default is MISSING and f.default_factory is MISSING): seen_default = True @@ -531,8 +564,15 @@ def _init_fn(fields, frozen, has_post_init, self_name, globals): if not body_lines: body_lines = ['pass'] + _init_params = [_init_param(f) for f in std_fields] + if kw_only_fields: + # Add the keyword-only args. Because the * can only be added if + # there's at least one keyword-only arg, there needs to be a test here + # (instead of just concatenting the lists together). + _init_params += ['*'] + _init_params += [_init_param(f) for f in kw_only_fields] return _create_fn('__init__', - [self_name] + [_init_param(f) for f in fields if f.init], + [self_name] + _init_params, body_lines, locals=locals, globals=globals, @@ -611,6 +651,9 @@ def _is_initvar(a_type, dataclasses): return (a_type is dataclasses.InitVar or type(a_type) is dataclasses.InitVar) +def _is_kw_only(a_type, dataclasses): + return a_type is dataclasses.KW_ONLY + def _is_type(annotation, cls, a_module, a_type, is_type_predicate): # Given a type annotation string, does it refer to a_type in @@ -653,11 +696,6 @@ def _is_type(annotation, cls, a_module, a_type, is_type_predicate): # a eval() penalty for every single field of every dataclass # that's defined. It was judged not worth it. - # Strip away the extra quotes as a result of double-stringifying when the - # 'annotations' feature became default. - if annotation.startswith(("'", '"')) and annotation.endswith(("'", '"')): - annotation = annotation[1:-1] - match = _MODULE_IDENTIFIER_RE.match(annotation) if match: ns = None @@ -676,10 +714,11 @@ def _is_type(annotation, cls, a_module, a_type, is_type_predicate): return False -def _get_field(cls, a_name, a_type): - # Return a Field object for this field name and type. ClassVars - # and InitVars are also returned, but marked as such (see - # f._field_type). +def _get_field(cls, a_name, a_type, default_kw_only): + # Return a Field object for this field name and type. ClassVars and + # InitVars are also returned, but marked as such (see f._field_type). + # default_kw_only is the value of kw_only to use if there isn't a field() + # that defines it. # If the default value isn't derived from Field, then it's only a # normal default value. Convert it to a Field(). @@ -704,7 +743,7 @@ def _get_field(cls, a_name, a_type): # In addition to checking for actual types here, also check for # string annotations. get_type_hints() won't always work for us # (see https://github.com/python/typing/issues/508 for example), - # plus it's expensive and would require an eval for every stirng + # plus it's expensive and would require an eval for every string # annotation. So, make a best effort to see if this is a ClassVar # or InitVar using regex's and checking that the thing referenced # is actually of the correct type. @@ -750,6 +789,19 @@ def _get_field(cls, a_name, a_type): # init=)? It makes no sense for # ClassVar and InitVar to specify init=. + # kw_only validation and assignment. + if f._field_type in (_FIELD, _FIELD_INITVAR): + # For real and InitVar fields, if kw_only wasn't specified use the + # default value. + if f.kw_only is MISSING: + f.kw_only = default_kw_only + else: + # Make sure kw_only isn't set for ClassVars + assert f._field_type is _FIELD_CLASSVAR + if f.kw_only is not MISSING: + raise TypeError(f'field {f.name} is a ClassVar but specifies ' + 'kw_only') + # For real fields, disallow mutable defaults for known types. if f._field_type is _FIELD and isinstance(f.default, (list, dict, set)): raise ValueError(f'mutable default {type(f.default)} for field ' @@ -821,7 +873,8 @@ def _hash_exception(cls, fields, globals): # version of this table. -def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): +def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, + match_args, kw_only, slots): # Now that dicts retain insertion order, there's no reason to use # an ordered dict. I am leveraging that ordering here, because # derived class fields overwrite base class fields, but the order @@ -851,7 +904,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): # Only process classes that have been processed by our # decorator. That is, they have a _FIELDS attribute. base_fields = getattr(b, _FIELDS, None) - if base_fields: + if base_fields is not None: has_dataclass_bases = True for f in base_fields.values(): fields[f.name] = f @@ -875,8 +928,27 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): # Now find fields in our class. While doing so, validate some # things, and set the default values (as class attributes) where # we can. - cls_fields = [_get_field(cls, name, type) - for name, type in cls_annotations.items()] + cls_fields = [] + # Get a reference to this module for the _is_kw_only() test. + KW_ONLY_seen = False + dataclasses = sys.modules[__name__] + for name, type in cls_annotations.items(): + # See if this is a marker to change the value of kw_only. + if (_is_kw_only(type, dataclasses) + or (isinstance(type, str) + and _is_type(type, cls, dataclasses, dataclasses.KW_ONLY, + _is_kw_only))): + # Switch the default to kw_only=True, and ignore this + # annotation: it's not a real field. + if KW_ONLY_seen: + raise TypeError(f'{name!r} is KW_ONLY, but KW_ONLY ' + 'has already been specified') + KW_ONLY_seen = True + kw_only = True + else: + # Otherwise it's a field of some type. + cls_fields.append(_get_field(cls, name, type, kw_only)) + for f in cls_fields: fields[f.name] = f @@ -931,15 +1003,22 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): if order and not eq: raise ValueError('eq must be true if order is true') + # Include InitVars and regular fields (so, not ClassVars). This is + # initialized here, outside of the "if init:" test, because std_init_fields + # is used with match_args, below. + all_init_fields = [f for f in fields.values() + if f._field_type in (_FIELD, _FIELD_INITVAR)] + (std_init_fields, + kw_only_init_fields) = _fields_in_init_order(all_init_fields) + if init: # Does this class have a post-init function? has_post_init = hasattr(cls, _POST_INIT_NAME) - # Include InitVars and regular fields (so, not ClassVars). - flds = [f for f in fields.values() - if f._field_type in (_FIELD, _FIELD_INITVAR)] _set_new_attribute(cls, '__init__', - _init_fn(flds, + _init_fn(all_init_fields, + std_init_fields, + kw_only_init_fields, frozen, has_post_init, # The name to use for the "self" @@ -1005,15 +1084,72 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): if not getattr(cls, '__doc__'): # Create a class doc-string. cls.__doc__ = (cls.__name__ + - str(inspect.signature(cls)).replace(' -> NoneType', '')) + str(inspect.signature(cls)).replace(' -> None', '')) + + if match_args: + # I could probably compute this once + _set_new_attribute(cls, '__match_args__', + tuple(f.name for f in std_init_fields)) + + if slots: + cls = _add_slots(cls, frozen) abc.update_abstractmethods(cls) return cls +# _dataclass_getstate and _dataclass_setstate are needed for pickling frozen +# classes with slots. These could be slighly more performant if we generated +# the code instead of iterating over fields. But that can be a project for +# another day, if performance becomes an issue. +def _dataclass_getstate(self): + return [getattr(self, f.name) for f in fields(self)] + + +def _dataclass_setstate(self, state): + for field, value in zip(fields(self), state): + # use setattr because dataclass may be frozen + object.__setattr__(self, field.name, value) + + +def _add_slots(cls, is_frozen): + # Need to create a new class, since we can't set __slots__ + # after a class has been created. + + # Make sure __slots__ isn't already set. + if '__slots__' in cls.__dict__: + raise TypeError(f'{cls.__name__} already specifies __slots__') + + # Create a new dict for our new class. + cls_dict = dict(cls.__dict__) + field_names = tuple(f.name for f in fields(cls)) + cls_dict['__slots__'] = field_names + for field_name in field_names: + # Remove our attributes, if present. They'll still be + # available in _MARKER. + cls_dict.pop(field_name, None) + + # Remove __dict__ itself. + cls_dict.pop('__dict__', None) + + # And finally create the class. + qualname = getattr(cls, '__qualname__', None) + cls = type(cls)(cls.__name__, cls.__bases__, cls_dict) + if qualname is not None: + cls.__qualname__ = qualname + + if is_frozen: + # Need this for pickling frozen classes with slots. + cls.__getstate__ = _dataclass_getstate + cls.__setstate__ = _dataclass_setstate + + return cls + + def dataclass(cls=None, /, *, init=True, repr=True, eq=True, order=False, - unsafe_hash=False, frozen=False): + unsafe_hash=False, frozen=False, match_args=True, + kw_only=False, slots=False): """Returns the same class as was passed in, with dunder methods added based on the fields defined in the class. @@ -1023,11 +1159,15 @@ def dataclass(cls=None, /, *, init=True, repr=True, eq=True, order=False, repr is true, a __repr__() method is added. If order is true, rich comparison dunder methods are added. If unsafe_hash is true, a __hash__() method function is added. If frozen is true, fields may - not be assigned to after instance creation. + not be assigned to after instance creation. If match_args is true, + the __match_args__ tuple is added. If kw_only is true, then by + default all fields are keyword-only. If slots is true, an + __slots__ attribute is added. """ def wrap(cls): - return _process_class(cls, init, repr, eq, order, unsafe_hash, frozen) + return _process_class(cls, init, repr, eq, order, unsafe_hash, + frozen, match_args, kw_only, slots) # See if we're being called as @dataclass or @dataclass(). if cls is None: @@ -1186,7 +1326,7 @@ def _astuple_inner(obj, tuple_factory): def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, - frozen=False): + frozen=False, match_args=True, slots=False): """Return a new dynamically created dataclass. The dataclass name will be 'cls_name'. 'fields' is an iterable @@ -1212,14 +1352,12 @@ class C(Base): if namespace is None: namespace = {} - else: - # Copy namespace since we're going to mutate it. - namespace = namespace.copy() # While we're looking through the field names, validate that they # are identifiers, are not keywords, and not duplicates. seen = set() - anns = {} + annotations = {} + defaults = {} for item in fields: if isinstance(item, str): name = item @@ -1228,7 +1366,7 @@ class C(Base): name, tp, = item elif len(item) == 3: name, tp, spec = item - namespace[name] = spec + defaults[name] = spec else: raise TypeError(f'Invalid field: {item!r}') @@ -1240,14 +1378,22 @@ class C(Base): raise TypeError(f'Field name duplicated: {name!r}') seen.add(name) - anns[name] = tp + annotations[name] = tp + + # Update 'ns' with the user-supplied namespace plus our calculated values. + def exec_body_callback(ns): + ns.update(namespace) + ns.update(defaults) + ns['__annotations__'] = annotations - namespace['__annotations__'] = anns # We use `types.new_class()` instead of simply `type()` to allow dynamic creation # of generic dataclassses. - cls = types.new_class(cls_name, bases, {}, lambda ns: ns.update(namespace)) + cls = types.new_class(cls_name, bases, {}, exec_body_callback) + + # Apply the normal decorator. return dataclass(cls, init=init, repr=repr, eq=eq, order=order, - unsafe_hash=unsafe_hash, frozen=frozen) + unsafe_hash=unsafe_hash, frozen=frozen, + match_args=match_args, slots=slots) def replace(obj, /, **changes): @@ -1288,7 +1434,7 @@ class C: continue if f.name not in changes: - if f._field_type is _FIELD_INITVAR: + if f._field_type is _FIELD_INITVAR and f.default is MISSING: raise ValueError(f"InitVar {f.name!r} " 'must be specified with replace()') changes[f.name] = getattr(obj, f.name) diff --git a/Lib/datetime.py b/Lib/datetime.py index b896b94b0fe0e4..6bf37ccfab7ac8 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -2327,7 +2327,7 @@ def _name_from_offset(delta): # This is again a requirement for a sane tzinfo class. # # 4. (x+k).s = x.s -# This follows from #2, and that datimetimetz+timedelta preserves tzinfo. +# This follows from #2, and that datetime.timetz+timedelta preserves tzinfo. # # 5. (x+k).n = x.n + k # Again follows from how arithmetic is defined. diff --git a/Lib/dis.py b/Lib/dis.py index e289e176c78ffd..3fee1ce27725cd 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -338,8 +338,11 @@ def _get_instructions_bytes(code, varnames=None, names=None, constants=None, argval, argrepr = _get_const_info(arg, constants) elif op in hasname: argval, argrepr = _get_name_info(arg, names) + elif op in hasjabs: + argval = arg*2 + argrepr = "to " + repr(argval) elif op in hasjrel: - argval = offset + 2 + arg + argval = offset + 2 + arg*2 argrepr = "to " + repr(argval) elif op in haslocal: argval, argrepr = _get_name_info(arg, varnames) @@ -384,7 +387,7 @@ def _disassemble_bytes(code, lasti=-1, varnames=None, names=None, constants=None, cells=None, linestarts=None, *, file=None, line_offset=0): # Omit the line number column entirely if we have no line number info - show_lineno = linestarts is not None + show_lineno = bool(linestarts) if show_lineno: maxlineno = max(linestarts.values()) + line_offset if maxlineno >= 1000: @@ -437,9 +440,9 @@ def findlabels(code): for offset, op, arg in _unpack_opargs(code): if arg is not None: if op in hasjrel: - label = offset + 2 + arg + label = offset + 2 + arg*2 elif op in hasjabs: - label = arg + label = arg*2 else: continue if label not in labels: @@ -449,32 +452,15 @@ def findlabels(code): def findlinestarts(code): """Find the offsets in a byte code which are start of lines in the source. - Generate pairs (offset, lineno) as described in Python/compile.c. - + Generate pairs (offset, lineno) """ - byte_increments = code.co_lnotab[0::2] - line_increments = code.co_lnotab[1::2] - bytecode_len = len(code.co_code) - - lastlineno = None - lineno = code.co_firstlineno - addr = 0 - for byte_incr, line_incr in zip(byte_increments, line_increments): - if byte_incr: - if lineno != lastlineno: - yield (addr, lineno) - lastlineno = lineno - addr += byte_incr - if addr >= bytecode_len: - # The rest of the lnotab byte offsets are past the end of - # the bytecode, so the lines were optimized away. - return - if line_incr >= 0x80: - # line_increments is an array of 8-bit signed integers - line_incr -= 0x100 - lineno += line_incr - if lineno != lastlineno: - yield (addr, lineno) + lastline = None + for start, end, line in code.co_lines(): + if line is not None and line != lastline: + lastline = line + yield start, line + return + class Bytecode: """The bytecode operations of a piece of code diff --git a/Lib/distutils/__init__.py b/Lib/distutils/__init__.py index d823d040a1c1e6..fdad6f65a78562 100644 --- a/Lib/distutils/__init__.py +++ b/Lib/distutils/__init__.py @@ -9,5 +9,12 @@ """ import sys +import warnings __version__ = sys.version[:sys.version.index(' ')] + +_DEPRECATION_MESSAGE = ("The distutils package is deprecated and slated for " + "removal in Python 3.12. Use setuptools or check " + "PEP 632 for potential alternatives") +warnings.warn(_DEPRECATION_MESSAGE, + DeprecationWarning, 2) diff --git a/Lib/distutils/command/__init__.py b/Lib/distutils/command/__init__.py index 481eea9fd4b5a5..fd0bfae7ade6ff 100644 --- a/Lib/distutils/command/__init__.py +++ b/Lib/distutils/command/__init__.py @@ -19,7 +19,6 @@ 'bdist', 'bdist_dumb', 'bdist_rpm', - 'bdist_wininst', 'check', 'upload', # These two are reserved for future use: diff --git a/Lib/distutils/command/bdist.py b/Lib/distutils/command/bdist.py index 014871d280edb5..d580a8090bd201 100644 --- a/Lib/distutils/command/bdist.py +++ b/Lib/distutils/command/bdist.py @@ -62,7 +62,7 @@ class bdist(Command): # Establish the preferred order (for the --help-formats option). format_commands = ['rpm', 'gztar', 'bztar', 'xztar', 'ztar', 'tar', - 'wininst', 'zip', 'msi'] + 'zip', 'msi'] # And the real information. format_command = {'rpm': ('bdist_rpm', "RPM distribution"), @@ -71,8 +71,6 @@ class bdist(Command): 'xztar': ('bdist_dumb', "xz'ed tar file"), 'ztar': ('bdist_dumb', "compressed tar file"), 'tar': ('bdist_dumb', "tar file"), - 'wininst': ('bdist_wininst', - "Windows executable installer"), 'zip': ('bdist_dumb', "ZIP file"), 'msi': ('bdist_msi', "Microsoft Installer") } diff --git a/Lib/distutils/command/bdist_msi.py b/Lib/distutils/command/bdist_msi.py index 0863a1883e7205..2ed017b4d664d7 100644 --- a/Lib/distutils/command/bdist_msi.py +++ b/Lib/distutils/command/bdist_msi.py @@ -1,7 +1,5 @@ # Copyright (C) 2005, 2006 Martin von Löwis # Licensed to PSF under a Contributor Agreement. -# The bdist_wininst command proper -# based on bdist_wininst """ Implements the bdist_msi command. """ diff --git a/Lib/distutils/command/bdist_wininst.py b/Lib/distutils/command/bdist_wininst.py deleted file mode 100644 index 0e9ddaa21419e9..00000000000000 --- a/Lib/distutils/command/bdist_wininst.py +++ /dev/null @@ -1,377 +0,0 @@ -"""distutils.command.bdist_wininst - -Implements the Distutils 'bdist_wininst' command: create a windows installer -exe-program.""" - -import os -import sys -import warnings -from distutils.core import Command -from distutils.util import get_platform -from distutils.dir_util import remove_tree -from distutils.errors import * -from distutils.sysconfig import get_python_version -from distutils import log - -class bdist_wininst(Command): - - description = "create an executable installer for MS Windows" - - user_options = [('bdist-dir=', None, - "temporary directory for creating the distribution"), - ('plat-name=', 'p', - "platform name to embed in generated filenames " - "(default: %s)" % get_platform()), - ('keep-temp', 'k', - "keep the pseudo-installation tree around after " + - "creating the distribution archive"), - ('target-version=', None, - "require a specific python version" + - " on the target system"), - ('no-target-compile', 'c', - "do not compile .py to .pyc on the target system"), - ('no-target-optimize', 'o', - "do not compile .py to .pyo (optimized) " - "on the target system"), - ('dist-dir=', 'd', - "directory to put final built distributions in"), - ('bitmap=', 'b', - "bitmap to use for the installer instead of python-powered logo"), - ('title=', 't', - "title to display on the installer background instead of default"), - ('skip-build', None, - "skip rebuilding everything (for testing/debugging)"), - ('install-script=', None, - "basename of installation script to be run after " - "installation or before deinstallation"), - ('pre-install-script=', None, - "Fully qualified filename of a script to be run before " - "any files are installed. This script need not be in the " - "distribution"), - ('user-access-control=', None, - "specify Vista's UAC handling - 'none'/default=no " - "handling, 'auto'=use UAC if target Python installed for " - "all users, 'force'=always use UAC"), - ] - - boolean_options = ['keep-temp', 'no-target-compile', 'no-target-optimize', - 'skip-build'] - - # bpo-10945: bdist_wininst requires mbcs encoding only available on Windows - _unsupported = (sys.platform != "win32") - - def __init__(self, *args, **kw): - super().__init__(*args, **kw) - warnings.warn("bdist_wininst command is deprecated since Python 3.8, " - "use bdist_wheel (wheel packages) instead", - DeprecationWarning, 2) - - def initialize_options(self): - self.bdist_dir = None - self.plat_name = None - self.keep_temp = 0 - self.no_target_compile = 0 - self.no_target_optimize = 0 - self.target_version = None - self.dist_dir = None - self.bitmap = None - self.title = None - self.skip_build = None - self.install_script = None - self.pre_install_script = None - self.user_access_control = None - - - def finalize_options(self): - self.set_undefined_options('bdist', ('skip_build', 'skip_build')) - - if self.bdist_dir is None: - if self.skip_build and self.plat_name: - # If build is skipped and plat_name is overridden, bdist will - # not see the correct 'plat_name' - so set that up manually. - bdist = self.distribution.get_command_obj('bdist') - bdist.plat_name = self.plat_name - # next the command will be initialized using that name - bdist_base = self.get_finalized_command('bdist').bdist_base - self.bdist_dir = os.path.join(bdist_base, 'wininst') - - if not self.target_version: - self.target_version = "" - - if not self.skip_build and self.distribution.has_ext_modules(): - short_version = get_python_version() - if self.target_version and self.target_version != short_version: - raise DistutilsOptionError( - "target version can only be %s, or the '--skip-build'" \ - " option must be specified" % (short_version,)) - self.target_version = short_version - - self.set_undefined_options('bdist', - ('dist_dir', 'dist_dir'), - ('plat_name', 'plat_name'), - ) - - if self.install_script: - for script in self.distribution.scripts: - if self.install_script == os.path.basename(script): - break - else: - raise DistutilsOptionError( - "install_script '%s' not found in scripts" - % self.install_script) - - def run(self): - if (sys.platform != "win32" and - (self.distribution.has_ext_modules() or - self.distribution.has_c_libraries())): - raise DistutilsPlatformError \ - ("distribution contains extensions and/or C libraries; " - "must be compiled on a Windows 32 platform") - - if not self.skip_build: - self.run_command('build') - - install = self.reinitialize_command('install', reinit_subcommands=1) - install.root = self.bdist_dir - install.skip_build = self.skip_build - install.warn_dir = 0 - install.plat_name = self.plat_name - - install_lib = self.reinitialize_command('install_lib') - # we do not want to include pyc or pyo files - install_lib.compile = 0 - install_lib.optimize = 0 - - if self.distribution.has_ext_modules(): - # If we are building an installer for a Python version other - # than the one we are currently running, then we need to ensure - # our build_lib reflects the other Python version rather than ours. - # Note that for target_version!=sys.version, we must have skipped the - # build step, so there is no issue with enforcing the build of this - # version. - target_version = self.target_version - if not target_version: - assert self.skip_build, "Should have already checked this" - target_version = '%d.%d' % sys.version_info[:2] - plat_specifier = ".%s-%s" % (self.plat_name, target_version) - build = self.get_finalized_command('build') - build.build_lib = os.path.join(build.build_base, - 'lib' + plat_specifier) - - # Use a custom scheme for the zip-file, because we have to decide - # at installation time which scheme to use. - for key in ('purelib', 'platlib', 'headers', 'scripts', 'data'): - value = key.upper() - if key == 'headers': - value = value + '/Include/$dist_name' - setattr(install, - 'install_' + key, - value) - - log.info("installing to %s", self.bdist_dir) - install.ensure_finalized() - - # avoid warning of 'install_lib' about installing - # into a directory not in sys.path - sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB')) - - install.run() - - del sys.path[0] - - # And make an archive relative to the root of the - # pseudo-installation tree. - from tempfile import mktemp - archive_basename = mktemp() - fullname = self.distribution.get_fullname() - arcname = self.make_archive(archive_basename, "zip", - root_dir=self.bdist_dir) - # create an exe containing the zip-file - self.create_exe(arcname, fullname, self.bitmap) - if self.distribution.has_ext_modules(): - pyversion = get_python_version() - else: - pyversion = 'any' - self.distribution.dist_files.append(('bdist_wininst', pyversion, - self.get_installer_filename(fullname))) - # remove the zip-file again - log.debug("removing temporary file '%s'", arcname) - os.remove(arcname) - - if not self.keep_temp: - remove_tree(self.bdist_dir, dry_run=self.dry_run) - - def get_inidata(self): - # Return data describing the installation. - lines = [] - metadata = self.distribution.metadata - - # Write the [metadata] section. - lines.append("[metadata]") - - # 'info' will be displayed in the installer's dialog box, - # describing the items to be installed. - info = (metadata.long_description or '') + '\n' - - # Escape newline characters - def escape(s): - return s.replace("\n", "\\n") - - for name in ["author", "author_email", "description", "maintainer", - "maintainer_email", "name", "url", "version"]: - data = getattr(metadata, name, "") - if data: - info = info + ("\n %s: %s" % \ - (name.capitalize(), escape(data))) - lines.append("%s=%s" % (name, escape(data))) - - # The [setup] section contains entries controlling - # the installer runtime. - lines.append("\n[Setup]") - if self.install_script: - lines.append("install_script=%s" % self.install_script) - lines.append("info=%s" % escape(info)) - lines.append("target_compile=%d" % (not self.no_target_compile)) - lines.append("target_optimize=%d" % (not self.no_target_optimize)) - if self.target_version: - lines.append("target_version=%s" % self.target_version) - if self.user_access_control: - lines.append("user_access_control=%s" % self.user_access_control) - - title = self.title or self.distribution.get_fullname() - lines.append("title=%s" % escape(title)) - import time - import distutils - build_info = "Built %s with distutils-%s" % \ - (time.ctime(time.time()), distutils.__version__) - lines.append("build_info=%s" % build_info) - return "\n".join(lines) - - def create_exe(self, arcname, fullname, bitmap=None): - import struct - - self.mkpath(self.dist_dir) - - cfgdata = self.get_inidata() - - installer_name = self.get_installer_filename(fullname) - self.announce("creating %s" % installer_name) - - if bitmap: - with open(bitmap, "rb") as f: - bitmapdata = f.read() - bitmaplen = len(bitmapdata) - else: - bitmaplen = 0 - - with open(installer_name, "wb") as file: - file.write(self.get_exe_bytes()) - if bitmap: - file.write(bitmapdata) - - # Convert cfgdata from unicode to ascii, mbcs encoded - if isinstance(cfgdata, str): - cfgdata = cfgdata.encode("mbcs") - - # Append the pre-install script - cfgdata = cfgdata + b"\0" - if self.pre_install_script: - # We need to normalize newlines, so we open in text mode and - # convert back to bytes. "latin-1" simply avoids any possible - # failures. - with open(self.pre_install_script, "r", - encoding="latin-1") as script: - script_data = script.read().encode("latin-1") - cfgdata = cfgdata + script_data + b"\n\0" - else: - # empty pre-install script - cfgdata = cfgdata + b"\0" - file.write(cfgdata) - - # The 'magic number' 0x1234567B is used to make sure that the - # binary layout of 'cfgdata' is what the wininst.exe binary - # expects. If the layout changes, increment that number, make - # the corresponding changes to the wininst.exe sources, and - # recompile them. - header = struct.pack("", value) + value = value.replace("$installed_base", "$base") + value = value.replace("$py_version_nodot_plat", "$py_version_nodot") + if key == "headers": + value += "/$dist_name" + if sys.version_info >= (3, 9) and key == "platlib": + # platlibdir is available since 3.9: bpo-1294959 + value = value.replace("/lib/", "/$platlibdir/") + INSTALL_SCHEMES[main_key][key] = value + +# The following part of INSTALL_SCHEMES has a different definition +# than the one in sysconfig, but because both depend on the site module, +# the outcomes should be the same. if HAS_USER_SITE: INSTALL_SCHEMES['nt_user'] = { 'purelib': '$usersite', @@ -64,11 +79,6 @@ 'data' : '$userbase', } -# The keys to an installation scheme; if any new types of files are to be -# installed, be sure to add an entry to every installation scheme above, -# and to SCHEME_KEYS here. -SCHEME_KEYS = ('purelib', 'platlib', 'headers', 'scripts', 'data') - class install(Command): @@ -169,8 +179,9 @@ def initialize_options(self): self.install_lib = None # set to either purelib or platlib self.install_scripts = None self.install_data = None - self.install_userbase = USER_BASE - self.install_usersite = USER_SITE + if HAS_USER_SITE: + self.install_userbase = USER_BASE + self.install_usersite = USER_SITE self.compile = None self.optimize = None @@ -343,8 +354,9 @@ def finalize_options(self): # Convert directories from Unix /-separated syntax to the local # convention. self.convert_paths('lib', 'purelib', 'platlib', - 'scripts', 'data', 'headers', - 'userbase', 'usersite') + 'scripts', 'data', 'headers') + if HAS_USER_SITE: + self.convert_paths('userbase', 'usersite') # Deprecated # Well, we're not actually fully completely finalized yet: we still diff --git a/Lib/distutils/command/wininst-10.0-amd64.exe b/Lib/distutils/command/wininst-10.0-amd64.exe deleted file mode 100644 index 6fa0dce1631585..00000000000000 Binary files a/Lib/distutils/command/wininst-10.0-amd64.exe and /dev/null differ diff --git a/Lib/distutils/command/wininst-10.0.exe b/Lib/distutils/command/wininst-10.0.exe deleted file mode 100644 index afc3bc6c148472..00000000000000 Binary files a/Lib/distutils/command/wininst-10.0.exe and /dev/null differ diff --git a/Lib/distutils/command/wininst-14.0-amd64.exe b/Lib/distutils/command/wininst-14.0-amd64.exe deleted file mode 100644 index 253c2e2eccefa7..00000000000000 Binary files a/Lib/distutils/command/wininst-14.0-amd64.exe and /dev/null differ diff --git a/Lib/distutils/command/wininst-14.0.exe b/Lib/distutils/command/wininst-14.0.exe deleted file mode 100644 index 46f5f356676c80..00000000000000 Binary files a/Lib/distutils/command/wininst-14.0.exe and /dev/null differ diff --git a/Lib/distutils/command/wininst-6.0.exe b/Lib/distutils/command/wininst-6.0.exe deleted file mode 100644 index f57c855a613e2d..00000000000000 Binary files a/Lib/distutils/command/wininst-6.0.exe and /dev/null differ diff --git a/Lib/distutils/command/wininst-7.1.exe b/Lib/distutils/command/wininst-7.1.exe deleted file mode 100644 index 1433bc1ad3775e..00000000000000 Binary files a/Lib/distutils/command/wininst-7.1.exe and /dev/null differ diff --git a/Lib/distutils/command/wininst-8.0.exe b/Lib/distutils/command/wininst-8.0.exe deleted file mode 100644 index 7403bfabf5cc10..00000000000000 Binary files a/Lib/distutils/command/wininst-8.0.exe and /dev/null differ diff --git a/Lib/distutils/command/wininst-9.0-amd64.exe b/Lib/distutils/command/wininst-9.0-amd64.exe deleted file mode 100644 index 94fbd4341b2a14..00000000000000 Binary files a/Lib/distutils/command/wininst-9.0-amd64.exe and /dev/null differ diff --git a/Lib/distutils/command/wininst-9.0.exe b/Lib/distutils/command/wininst-9.0.exe deleted file mode 100644 index 2ec261f9fdcd03..00000000000000 Binary files a/Lib/distutils/command/wininst-9.0.exe and /dev/null differ diff --git a/Lib/distutils/extension.py b/Lib/distutils/extension.py index c507da360aa3d9..e85032ece8916f 100644 --- a/Lib/distutils/extension.py +++ b/Lib/distutils/extension.py @@ -4,6 +4,7 @@ modules in setup scripts.""" import os +import re import warnings # This class is really only used by the "build_ext" command, so it might @@ -161,7 +162,7 @@ def read_setup_file(filename): line = file.readline() if line is None: # eof break - if _variable_rx.match(line): # VAR=VALUE, handled in first pass + if re.match(_variable_rx, line): # VAR=VALUE, handled in first pass continue if line[0] == line[-1] == "*": diff --git a/Lib/distutils/spawn.py b/Lib/distutils/spawn.py index 0d1bd0391e6f11..31df3f7faca552 100644 --- a/Lib/distutils/spawn.py +++ b/Lib/distutils/spawn.py @@ -59,13 +59,17 @@ def spawn(cmd, search_path=1, verbose=0, dry_run=0): if _cfg_target: _cfg_target_split = [int(x) for x in _cfg_target.split('.')] if _cfg_target: - # ensure that the deployment target of build process is not less - # than that used when the interpreter was built. This ensures - # extension modules are built with correct compatibility values + # Ensure that the deployment target of the build process is not + # less than 10.3 if the interpreter was built for 10.3 or later. + # This ensures extension modules are built with correct + # compatibility values, specifically LDSHARED which can use + # '-undefined dynamic_lookup' which only works on >= 10.3. cur_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', _cfg_target) - if _cfg_target_split > [int(x) for x in cur_target.split('.')]: + cur_target_split = [int(x) for x in cur_target.split('.')] + if _cfg_target_split[:2] >= [10, 3] and cur_target_split[:2] < [10, 3]: my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: ' - 'now "%s" but "%s" during configure' + 'now "%s" but "%s" during configure;' + 'must use 10.3 or later' % (cur_target, _cfg_target)) raise DistutilsPlatformError(my_msg) env = dict(os.environ, diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 37feae5df72c93..aa63093a3f0e6e 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -13,56 +13,71 @@ import os import re import sys +import warnings -from .errors import DistutilsPlatformError - -# These are needed in a couple of spots, so just compute them once. -PREFIX = os.path.normpath(sys.prefix) -EXEC_PREFIX = os.path.normpath(sys.exec_prefix) -BASE_PREFIX = os.path.normpath(sys.base_prefix) -BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) - -# Path to the base directory of the project. On Windows the binary may -# live in project/PCbuild/win32 or project/PCbuild/amd64. -# set for cross builds -if "_PYTHON_PROJECT_BASE" in os.environ: - project_base = os.path.abspath(os.environ["_PYTHON_PROJECT_BASE"]) -else: - if sys.executable: - project_base = os.path.dirname(os.path.abspath(sys.executable)) - else: - # sys.executable can be empty if argv[0] has been changed and Python is - # unable to retrieve the real program name - project_base = os.getcwd() +from functools import partial +from .errors import DistutilsPlatformError -# python_build: (Boolean) if true, we're either building Python or -# building an extension with an un-installed Python, so we use -# different (hard-wired) directories. -def _is_python_source_dir(d): - for fn in ("Setup", "Setup.local"): - if os.path.isfile(os.path.join(d, "Modules", fn)): - return True - return False +from sysconfig import ( + _PREFIX as PREFIX, + _BASE_PREFIX as BASE_PREFIX, + _EXEC_PREFIX as EXEC_PREFIX, + _BASE_EXEC_PREFIX as BASE_EXEC_PREFIX, + _PROJECT_BASE as project_base, + _PYTHON_BUILD as python_build, + _init_posix as sysconfig_init_posix, + parse_config_h as sysconfig_parse_config_h, + _parse_makefile as sysconfig_parse_makefile, + + _init_non_posix, + _is_python_source_dir, + _sys_home, + + _variable_rx, + _findvar1_rx, + _findvar2_rx, + + expand_makefile_vars, + is_python_build, + get_config_h_filename, + get_config_var, + get_config_vars, + get_makefile_filename, + get_python_version, +) + +# This is better than +# from sysconfig import _CONFIG_VARS as _config_vars +# because it makes sure that the global dictionary is initialized +# which might not be true in the time of import. +_config_vars = get_config_vars() + +if os.name == "nt": + from sysconfig import _fix_pcbuild + +warnings.warn( + 'The distutils.sysconfig module is deprecated, use sysconfig instead', + DeprecationWarning, + stacklevel=2 +) + + +# Following functions are the same as in sysconfig but with different API +def parse_config_h(fp, g=None): + return sysconfig_parse_config_h(fp, vars=g) -_sys_home = getattr(sys, '_home', None) -if os.name == 'nt': - def _fix_pcbuild(d): - if d and os.path.normcase(d).startswith( - os.path.normcase(os.path.join(PREFIX, "PCbuild"))): - return PREFIX - return d - project_base = _fix_pcbuild(project_base) - _sys_home = _fix_pcbuild(_sys_home) +def parse_makefile(fn, g=None): + return sysconfig_parse_makefile(fn, vars=g, keep_unresolved=False) -def _python_build(): - if _sys_home: - return _is_python_source_dir(_sys_home) - return _is_python_source_dir(project_base) +_python_build = partial(is_python_build, check_home=True) +_init_posix = partial(sysconfig_init_posix, _config_vars) +_init_nt = partial(_init_non_posix, _config_vars) -python_build = _python_build() +# Following functions are deprecated together with this module and they +# have no direct replacement # Calculate the build qualifier flags if they are defined. Adding the flags # to the include and lib directories only makes sense for an installation, not @@ -76,12 +91,76 @@ def _python_build(): # this attribute, which is fine. pass -def get_python_version(): - """Return a string containing the major and minor Python version, - leaving off the patchlevel. Sample return values could be '1.5' - or '2.2'. + +def customize_compiler(compiler): + """Do any platform-specific customization of a CCompiler instance. + + Mainly needed on Unix, so we can plug in the information that + varies across Unices and is stored in Python's Makefile. """ - return '%d.%d' % sys.version_info[:2] + if compiler.compiler_type == "unix": + if sys.platform == "darwin": + # Perform first-time customization of compiler-related + # config vars on OS X now that we know we need a compiler. + # This is primarily to support Pythons from binary + # installers. The kind and paths to build tools on + # the user system may vary significantly from the system + # that Python itself was built on. Also the user OS + # version and build tools may not support the same set + # of CPU architectures for universal builds. + if not _config_vars.get('CUSTOMIZED_OSX_COMPILER'): + import _osx_support + _osx_support.customize_compiler(_config_vars) + _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' + + (cc, cxx, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags) = \ + get_config_vars('CC', 'CXX', 'CFLAGS', + 'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS') + + if 'CC' in os.environ: + newcc = os.environ['CC'] + if (sys.platform == 'darwin' + and 'LDSHARED' not in os.environ + and ldshared.startswith(cc)): + # On OS X, if CC is overridden, use that as the default + # command for LDSHARED as well + ldshared = newcc + ldshared[len(cc):] + cc = newcc + if 'CXX' in os.environ: + cxx = os.environ['CXX'] + if 'LDSHARED' in os.environ: + ldshared = os.environ['LDSHARED'] + if 'CPP' in os.environ: + cpp = os.environ['CPP'] + else: + cpp = cc + " -E" # not always + if 'LDFLAGS' in os.environ: + ldshared = ldshared + ' ' + os.environ['LDFLAGS'] + if 'CFLAGS' in os.environ: + cflags = cflags + ' ' + os.environ['CFLAGS'] + ldshared = ldshared + ' ' + os.environ['CFLAGS'] + if 'CPPFLAGS' in os.environ: + cpp = cpp + ' ' + os.environ['CPPFLAGS'] + cflags = cflags + ' ' + os.environ['CPPFLAGS'] + ldshared = ldshared + ' ' + os.environ['CPPFLAGS'] + if 'AR' in os.environ: + ar = os.environ['AR'] + if 'ARFLAGS' in os.environ: + archiver = ar + ' ' + os.environ['ARFLAGS'] + else: + archiver = ar + ' ' + ar_flags + + cc_cmd = cc + ' ' + cflags + compiler.set_executables( + preprocessor=cpp, + compiler=cc_cmd, + compiler_so=cc_cmd + ' ' + ccshared, + compiler_cxx=cxx, + linker_so=ldshared, + linker_exe=cc, + archiver=archiver) + + compiler.shared_lib_extension = shlib_suffix def get_python_inc(plat_specific=0, prefix=None): @@ -167,389 +246,3 @@ def get_python_lib(plat_specific=0, standard_lib=0, prefix=None): raise DistutilsPlatformError( "I don't know where Python installs its library " "on platform '%s'" % os.name) - - - -def customize_compiler(compiler): - """Do any platform-specific customization of a CCompiler instance. - - Mainly needed on Unix, so we can plug in the information that - varies across Unices and is stored in Python's Makefile. - """ - if compiler.compiler_type == "unix": - if sys.platform == "darwin": - # Perform first-time customization of compiler-related - # config vars on OS X now that we know we need a compiler. - # This is primarily to support Pythons from binary - # installers. The kind and paths to build tools on - # the user system may vary significantly from the system - # that Python itself was built on. Also the user OS - # version and build tools may not support the same set - # of CPU architectures for universal builds. - global _config_vars - # Use get_config_var() to ensure _config_vars is initialized. - if not get_config_var('CUSTOMIZED_OSX_COMPILER'): - import _osx_support - _osx_support.customize_compiler(_config_vars) - _config_vars['CUSTOMIZED_OSX_COMPILER'] = 'True' - - (cc, cxx, cflags, ccshared, ldshared, shlib_suffix, ar, ar_flags) = \ - get_config_vars('CC', 'CXX', 'CFLAGS', - 'CCSHARED', 'LDSHARED', 'SHLIB_SUFFIX', 'AR', 'ARFLAGS') - - if 'CC' in os.environ: - newcc = os.environ['CC'] - if (sys.platform == 'darwin' - and 'LDSHARED' not in os.environ - and ldshared.startswith(cc)): - # On OS X, if CC is overridden, use that as the default - # command for LDSHARED as well - ldshared = newcc + ldshared[len(cc):] - cc = newcc - if 'CXX' in os.environ: - cxx = os.environ['CXX'] - if 'LDSHARED' in os.environ: - ldshared = os.environ['LDSHARED'] - if 'CPP' in os.environ: - cpp = os.environ['CPP'] - else: - cpp = cc + " -E" # not always - if 'LDFLAGS' in os.environ: - ldshared = ldshared + ' ' + os.environ['LDFLAGS'] - if 'CFLAGS' in os.environ: - cflags = cflags + ' ' + os.environ['CFLAGS'] - ldshared = ldshared + ' ' + os.environ['CFLAGS'] - if 'CPPFLAGS' in os.environ: - cpp = cpp + ' ' + os.environ['CPPFLAGS'] - cflags = cflags + ' ' + os.environ['CPPFLAGS'] - ldshared = ldshared + ' ' + os.environ['CPPFLAGS'] - if 'AR' in os.environ: - ar = os.environ['AR'] - if 'ARFLAGS' in os.environ: - archiver = ar + ' ' + os.environ['ARFLAGS'] - else: - archiver = ar + ' ' + ar_flags - - cc_cmd = cc + ' ' + cflags - compiler.set_executables( - preprocessor=cpp, - compiler=cc_cmd, - compiler_so=cc_cmd + ' ' + ccshared, - compiler_cxx=cxx, - linker_so=ldshared, - linker_exe=cc, - archiver=archiver) - - compiler.shared_lib_extension = shlib_suffix - - -def get_config_h_filename(): - """Return full pathname of installed pyconfig.h file.""" - if python_build: - if os.name == "nt": - inc_dir = os.path.join(_sys_home or project_base, "PC") - else: - inc_dir = _sys_home or project_base - else: - inc_dir = get_python_inc(plat_specific=1) - - return os.path.join(inc_dir, 'pyconfig.h') - - -def get_makefile_filename(): - """Return full pathname of installed Makefile from the Python build.""" - if python_build: - return os.path.join(_sys_home or project_base, "Makefile") - lib_dir = get_python_lib(plat_specific=0, standard_lib=1) - config_file = 'config-{}{}'.format(get_python_version(), build_flags) - if hasattr(sys.implementation, '_multiarch'): - config_file += '-%s' % sys.implementation._multiarch - return os.path.join(lib_dir, config_file, 'Makefile') - - -def parse_config_h(fp, g=None): - """Parse a config.h-style file. - - A dictionary containing name/value pairs is returned. If an - optional dictionary is passed in as the second argument, it is - used instead of a new dictionary. - """ - if g is None: - g = {} - define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n") - undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n") - # - while True: - line = fp.readline() - if not line: - break - m = define_rx.match(line) - if m: - n, v = m.group(1, 2) - try: v = int(v) - except ValueError: pass - g[n] = v - else: - m = undef_rx.match(line) - if m: - g[m.group(1)] = 0 - return g - - -# Regexes needed for parsing Makefile (and similar syntaxes, -# like old-style Setup files). -_variable_rx = re.compile(r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)") -_findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)") -_findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}") - -def parse_makefile(fn, g=None): - """Parse a Makefile-style file. - - A dictionary containing name/value pairs is returned. If an - optional dictionary is passed in as the second argument, it is - used instead of a new dictionary. - """ - from distutils.text_file import TextFile - fp = TextFile(fn, strip_comments=1, skip_blanks=1, join_lines=1, errors="surrogateescape") - - if g is None: - g = {} - done = {} - notdone = {} - - while True: - line = fp.readline() - if line is None: # eof - break - m = _variable_rx.match(line) - if m: - n, v = m.group(1, 2) - v = v.strip() - # `$$' is a literal `$' in make - tmpv = v.replace('$$', '') - - if "$" in tmpv: - notdone[n] = v - else: - try: - v = int(v) - except ValueError: - # insert literal `$' - done[n] = v.replace('$$', '$') - else: - done[n] = v - - # Variables with a 'PY_' prefix in the makefile. These need to - # be made available without that prefix through sysconfig. - # Special care is needed to ensure that variable expansion works, even - # if the expansion uses the name without a prefix. - renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS') - - # do variable interpolation here - while notdone: - for name in list(notdone): - value = notdone[name] - m = _findvar1_rx.search(value) or _findvar2_rx.search(value) - if m: - n = m.group(1) - found = True - if n in done: - item = str(done[n]) - elif n in notdone: - # get it on a subsequent round - found = False - elif n in os.environ: - # do it like make: fall back to environment - item = os.environ[n] - - elif n in renamed_variables: - if name.startswith('PY_') and name[3:] in renamed_variables: - item = "" - - elif 'PY_' + n in notdone: - found = False - - else: - item = str(done['PY_' + n]) - else: - done[n] = item = "" - if found: - after = value[m.end():] - value = value[:m.start()] + item + after - if "$" in after: - notdone[name] = value - else: - try: value = int(value) - except ValueError: - done[name] = value.strip() - else: - done[name] = value - del notdone[name] - - if name.startswith('PY_') \ - and name[3:] in renamed_variables: - - name = name[3:] - if name not in done: - done[name] = value - else: - # bogus variable reference; just drop it since we can't deal - del notdone[name] - - fp.close() - - # strip spurious spaces - for k, v in done.items(): - if isinstance(v, str): - done[k] = v.strip() - - # save the results in the global dictionary - g.update(done) - return g - - -def expand_makefile_vars(s, vars): - """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in - 'string' according to 'vars' (a dictionary mapping variable names to - values). Variables not present in 'vars' are silently expanded to the - empty string. The variable values in 'vars' should not contain further - variable expansions; if 'vars' is the output of 'parse_makefile()', - you're fine. Returns a variable-expanded version of 's'. - """ - - # This algorithm does multiple expansion, so if vars['foo'] contains - # "${bar}", it will expand ${foo} to ${bar}, and then expand - # ${bar}... and so forth. This is fine as long as 'vars' comes from - # 'parse_makefile()', which takes care of such expansions eagerly, - # according to make's variable expansion semantics. - - while True: - m = _findvar1_rx.search(s) or _findvar2_rx.search(s) - if m: - (beg, end) = m.span() - s = s[0:beg] + vars.get(m.group(1)) + s[end:] - else: - break - return s - - -_config_vars = None - -def _init_posix(): - """Initialize the module as appropriate for POSIX systems.""" - # _sysconfigdata is generated at build time, see the sysconfig module - name = os.environ.get('_PYTHON_SYSCONFIGDATA_NAME', - '_sysconfigdata_{abi}_{platform}_{multiarch}'.format( - abi=sys.abiflags, - platform=sys.platform, - multiarch=getattr(sys.implementation, '_multiarch', ''), - )) - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) - build_time_vars = _temp.build_time_vars - global _config_vars - _config_vars = {} - _config_vars.update(build_time_vars) - - -def _init_nt(): - """Initialize the module as appropriate for NT""" - g = {} - # set basic install directories - g['LIBDEST'] = get_python_lib(plat_specific=0, standard_lib=1) - g['BINLIBDEST'] = get_python_lib(plat_specific=1, standard_lib=1) - - # XXX hmmm.. a normal install puts include files here - g['INCLUDEPY'] = get_python_inc(plat_specific=0) - - g['EXT_SUFFIX'] = _imp.extension_suffixes()[0] - g['EXE'] = ".exe" - g['VERSION'] = get_python_version().replace(".", "") - g['BINDIR'] = os.path.dirname(os.path.abspath(sys.executable)) - - global _config_vars - _config_vars = g - - -def get_config_vars(*args): - """With no arguments, return a dictionary of all configuration - variables relevant for the current platform. Generally this includes - everything needed to build extensions and install both pure modules and - extensions. On Unix, this means every variable defined in Python's - installed Makefile; on Windows it's a much smaller set. - - With arguments, return a list of values that result from looking up - each argument in the configuration variable dictionary. - """ - global _config_vars - if _config_vars is None: - func = globals().get("_init_" + os.name) - if func: - func() - else: - _config_vars = {} - - # Normalized versions of prefix and exec_prefix are handy to have; - # in fact, these are the standard versions used most places in the - # Distutils. - _config_vars['prefix'] = PREFIX - _config_vars['exec_prefix'] = EXEC_PREFIX - - # For backward compatibility, see issue19555 - SO = _config_vars.get('EXT_SUFFIX') - if SO is not None: - _config_vars['SO'] = SO - - # Always convert srcdir to an absolute path - srcdir = _config_vars.get('srcdir', project_base) - if os.name == 'posix': - if python_build: - # If srcdir is a relative path (typically '.' or '..') - # then it should be interpreted relative to the directory - # containing Makefile. - base = os.path.dirname(get_makefile_filename()) - srcdir = os.path.join(base, srcdir) - else: - # srcdir is not meaningful since the installation is - # spread about the filesystem. We choose the - # directory containing the Makefile since we know it - # exists. - srcdir = os.path.dirname(get_makefile_filename()) - _config_vars['srcdir'] = os.path.abspath(os.path.normpath(srcdir)) - - # Convert srcdir into an absolute path if it appears necessary. - # Normally it is relative to the build directory. However, during - # testing, for example, we might be running a non-installed python - # from a different directory. - if python_build and os.name == "posix": - base = project_base - if (not os.path.isabs(_config_vars['srcdir']) and - base != os.getcwd()): - # srcdir is relative and we are not in the same directory - # as the executable. Assume executable is in the build - # directory and make srcdir absolute. - srcdir = os.path.join(base, _config_vars['srcdir']) - _config_vars['srcdir'] = os.path.normpath(srcdir) - - # OS X platforms require special customization to handle - # multi-architecture, multi-os-version installers - if sys.platform == 'darwin': - import _osx_support - _osx_support.customize_config_vars(_config_vars) - - if args: - vals = [] - for name in args: - vals.append(_config_vars.get(name)) - return vals - else: - return _config_vars - -def get_config_var(name): - """Return the value of a single variable using the dictionary - returned by 'get_config_vars()'. Equivalent to - get_config_vars().get(name) - """ - if name == 'SO': - import warnings - warnings.warn('SO is deprecated, use EXT_SUFFIX', DeprecationWarning, 2) - return get_config_vars().get(name) diff --git a/Lib/distutils/tests/test_bdist.py b/Lib/distutils/tests/test_bdist.py index 130d8bf155a45c..09ad076799b420 100644 --- a/Lib/distutils/tests/test_bdist.py +++ b/Lib/distutils/tests/test_bdist.py @@ -2,7 +2,6 @@ import os import unittest from test.support import run_unittest -import warnings from distutils.command.bdist import bdist from distutils.tests import support @@ -22,7 +21,7 @@ def test_formats(self): # what formats does bdist offer? formats = ['bztar', 'gztar', 'msi', 'rpm', 'tar', - 'wininst', 'xztar', 'zip', 'ztar'] + 'xztar', 'zip', 'ztar'] found = sorted(cmd.format_command) self.assertEqual(found, formats) @@ -34,15 +33,12 @@ def test_skip_build(self): cmd.ensure_finalized() dist.command_obj['bdist'] = cmd - names = ['bdist_dumb', 'bdist_wininst'] # bdist_rpm does not support --skip-build + names = ['bdist_dumb'] # bdist_rpm does not support --skip-build if os.name == 'nt': names.append('bdist_msi') for name in names: - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', 'bdist_wininst command is deprecated', - DeprecationWarning) - subcmd = cmd.get_finalized_command(name) + subcmd = cmd.get_finalized_command(name) if getattr(subcmd, '_unsupported', False): # command is not supported on this build continue diff --git a/Lib/distutils/tests/test_bdist_wininst.py b/Lib/distutils/tests/test_bdist_wininst.py deleted file mode 100644 index c338069a1dcf2d..00000000000000 --- a/Lib/distutils/tests/test_bdist_wininst.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Tests for distutils.command.bdist_wininst.""" -import sys -import platform -import unittest -from test.support import run_unittest -from test.support.warnings_helper import check_warnings - -from distutils.command.bdist_wininst import bdist_wininst -from distutils.tests import support - -@unittest.skipIf(sys.platform == 'win32' and platform.machine() == 'ARM64', - 'bdist_wininst is not supported in this install') -@unittest.skipIf(getattr(bdist_wininst, '_unsupported', False), - 'bdist_wininst is not supported in this install') -class BuildWinInstTestCase(support.TempdirManager, - support.LoggingSilencer, - unittest.TestCase): - - def test_get_exe_bytes(self): - - # issue5731: command was broken on non-windows platforms - # this test makes sure it works now for every platform - # let's create a command - pkg_pth, dist = self.create_dist() - with check_warnings(("", DeprecationWarning)): - cmd = bdist_wininst(dist) - cmd.ensure_finalized() - - # let's run the code that finds the right wininst*.exe file - # and make sure it finds it and returns its content - # no matter what platform we have - exe_file = cmd.get_exe_bytes() - self.assertGreater(len(exe_file), 10) - -def test_suite(): - return unittest.makeSuite(BuildWinInstTestCase) - -if __name__ == '__main__': - run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_build_clib.py b/Lib/distutils/tests/test_build_clib.py index abd8313770ef7a..19e012a58ce8cb 100644 --- a/Lib/distutils/tests/test_build_clib.py +++ b/Lib/distutils/tests/test_build_clib.py @@ -2,6 +2,7 @@ import unittest import os import sys +import sysconfig from test.support import run_unittest, missing_compiler_executable @@ -13,6 +14,15 @@ class BuildCLibTestCase(support.TempdirManager, support.LoggingSilencer, unittest.TestCase): + def setUp(self): + super().setUp() + self._backup_CONFIG_VARS = dict(sysconfig._CONFIG_VARS) + + def tearDown(self): + super().tearDown() + sysconfig._CONFIG_VARS.clear() + sysconfig._CONFIG_VARS.update(self._backup_CONFIG_VARS) + def test_check_library_dist(self): pkg_dir, dist = self.create_dist() cmd = build_clib(dist) diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py index f9e0d766d870e0..8e7364d2a2cb5f 100644 --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -35,6 +35,7 @@ def setUp(self): site.USER_BASE = self.mkdtemp() from distutils.command import build_ext build_ext.USER_BASE = site.USER_BASE + self.old_config_vars = dict(sysconfig._config_vars) # bpo-30132: On Windows, a .pdb file may be created in the current # working directory. Create a temporary working directory to cleanup @@ -48,6 +49,8 @@ def tearDown(self): site.USER_BASE = self.old_user_base from distutils.command import build_ext build_ext.USER_BASE = self.old_user_base + sysconfig._config_vars.clear() + sysconfig._config_vars.update(self.old_config_vars) super(BuildExtTestCase, self).tearDown() def build_ext(self, *args, **kwargs): @@ -493,12 +496,16 @@ def _try_compile_deployment_target(self, operator, target): # format the target value as defined in the Apple # Availability Macros. We can't use the macro names since # at least one value we test with will not exist yet. - if target[1] < 10: + if target[:2] < (10, 10): # for 10.1 through 10.9.x -> "10n0" target = '%02d%01d0' % target else: # for 10.10 and beyond -> "10nn00" - target = '%02d%02d00' % target + if len(target) >= 2: + target = '%02d%02d00' % target + else: + # 11 and later can have no minor version (11 instead of 11.0) + target = '%02d0000' % target deptarget_ext = Extension( 'deptarget', [deptarget_c], diff --git a/Lib/distutils/tests/test_config_cmd.py b/Lib/distutils/tests/test_config_cmd.py index 9aeab07b468361..0127ba71fc4bc4 100644 --- a/Lib/distutils/tests/test_config_cmd.py +++ b/Lib/distutils/tests/test_config_cmd.py @@ -2,6 +2,7 @@ import unittest import os import sys +import sysconfig from test.support import run_unittest, missing_compiler_executable from distutils.command.config import dump_file, config @@ -21,9 +22,12 @@ def setUp(self): self._logs = [] self.old_log = log.info log.info = self._info + self.old_config_vars = dict(sysconfig._CONFIG_VARS) def tearDown(self): log.info = self.old_log + sysconfig._CONFIG_VARS.clear() + sysconfig._CONFIG_VARS.update(self.old_config_vars) super(ConfigTestCase, self).tearDown() def test_dump_file(self): diff --git a/Lib/distutils/tests/test_install.py b/Lib/distutils/tests/test_install.py index 51c80e0421a8c1..0632024b3585ce 100644 --- a/Lib/distutils/tests/test_install.py +++ b/Lib/distutils/tests/test_install.py @@ -8,7 +8,7 @@ from test.support import captured_stdout, run_unittest from distutils import sysconfig -from distutils.command.install import install +from distutils.command.install import install, HAS_USER_SITE from distutils.command import install as install_module from distutils.command.build_ext import build_ext from distutils.command.install import INSTALL_SCHEMES @@ -29,6 +29,15 @@ class InstallTestCase(support.TempdirManager, support.LoggingSilencer, unittest.TestCase): + def setUp(self): + super().setUp() + self._backup_config_vars = dict(sysconfig._config_vars) + + def tearDown(self): + super().tearDown() + sysconfig._config_vars.clear() + sysconfig._config_vars.update(self._backup_config_vars) + def test_home_installation_scheme(self): # This ensure two things: # - that --home generates the desired set of directory names @@ -66,6 +75,7 @@ def check_path(got, expected): check_path(cmd.install_scripts, os.path.join(destination, "bin")) check_path(cmd.install_data, destination) + @unittest.skipUnless(HAS_USER_SITE, 'need user site') def test_user_site(self): # test install with --user # preparing the environment for the test @@ -93,8 +103,9 @@ def cleanup(): self.addCleanup(cleanup) - for key in ('nt_user', 'unix_user'): - self.assertIn(key, INSTALL_SCHEMES) + if HAS_USER_SITE: + for key in ('nt_user', 'unix_user'): + self.assertIn(key, INSTALL_SCHEMES) dist = Distribution({'name': 'xx'}) cmd = install(dist) diff --git a/Lib/distutils/tests/test_unixccompiler.py b/Lib/distutils/tests/test_unixccompiler.py index eefe4ba40291eb..24725ead1194ce 100644 --- a/Lib/distutils/tests/test_unixccompiler.py +++ b/Lib/distutils/tests/test_unixccompiler.py @@ -12,6 +12,7 @@ class UnixCCompilerTestCase(unittest.TestCase): def setUp(self): self._backup_platform = sys.platform self._backup_get_config_var = sysconfig.get_config_var + self._backup_config_vars = dict(sysconfig._config_vars) class CompilerWrapper(UnixCCompiler): def rpath_foo(self): return self.runtime_library_dir_option('/foo') @@ -20,6 +21,8 @@ def rpath_foo(self): def tearDown(self): sys.platform = self._backup_platform sysconfig.get_config_var = self._backup_get_config_var + sysconfig._config_vars.clear() + sysconfig._config_vars.update(self._backup_config_vars) @unittest.skipIf(sys.platform == 'win32', "can't test on Windows") def test_runtime_libdir_option(self): diff --git a/Lib/distutils/tests/test_util.py b/Lib/distutils/tests/test_util.py index bf0d4333f9aeaa..d4a01c6e91ce2b 100644 --- a/Lib/distutils/tests/test_util.py +++ b/Lib/distutils/tests/test_util.py @@ -54,7 +54,8 @@ def tearDown(self): os.uname = self.uname else: del os.uname - sysconfig._config_vars = copy(self._config_vars) + sysconfig._config_vars.clear() + sysconfig._config_vars.update(self._config_vars) super(UtilTestCase, self).tearDown() def _set_uname(self, uname): diff --git a/Lib/distutils/unixccompiler.py b/Lib/distutils/unixccompiler.py index 4d7a6de740ab3a..f0792de74a1a48 100644 --- a/Lib/distutils/unixccompiler.py +++ b/Lib/distutils/unixccompiler.py @@ -290,7 +290,7 @@ def find_library_file(self, dirs, lib, debug=0): cflags = sysconfig.get_config_var('CFLAGS') m = re.search(r'-isysroot\s*(\S+)', cflags) if m is None: - sysroot = '/' + sysroot = _osx_support._default_sysroot(sysconfig.get_config_var('CC')) else: sysroot = m.group(1) diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py index 4b002ecef1df8f..2ce5c5b64d62fa 100644 --- a/Lib/distutils/util.py +++ b/Lib/distutils/util.py @@ -9,6 +9,7 @@ import importlib.util import string import sys +import distutils from distutils.errors import DistutilsPlatformError from distutils.dep_util import newer from distutils.spawn import spawn @@ -419,8 +420,10 @@ def byte_compile (py_files, direct=1) """ % (optimize, force, prefix, base_dir, verbose)) + msg = distutils._DEPRECATION_MESSAGE cmd = [sys.executable] cmd.extend(subprocess._optim_args_from_interpreter_flags()) + cmd.append(f'-Wignore:{msg}:DeprecationWarning') cmd.append(script_name) spawn(cmd, dry_run=dry_run) execute(os.remove, (script_name,), "removing %s" % script_name, diff --git a/Lib/doctest.py b/Lib/doctest.py index baa503c83f8757..e95c333f48aad5 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -102,7 +102,7 @@ def _test(): import sys import traceback import unittest -from io import StringIO +from io import StringIO, IncrementalNewlineDecoder from collections import namedtuple TestResults = namedtuple('TestResults', 'failed attempted') @@ -212,23 +212,24 @@ def _normalize_module(module, depth=2): raise TypeError("Expected a module, string, or None") def _newline_convert(data): - # We have two cases to cover and we need to make sure we do - # them in the right order - for newline in ('\r\n', '\r'): - data = data.replace(newline, '\n') - return data + # The IO module provides a handy decoder for universal newline conversion + return IncrementalNewlineDecoder(None, True).decode(data, True) def _load_testfile(filename, package, module_relative, encoding): if module_relative: package = _normalize_module(package, 3) filename = _module_relative_path(package, filename) - if getattr(package, '__loader__', None) is not None: - if hasattr(package.__loader__, 'get_data'): - file_contents = package.__loader__.get_data(filename) - file_contents = file_contents.decode(encoding) - # get_data() opens files as 'rb', so one must do the equivalent - # conversion as universal newlines would do. - return _newline_convert(file_contents), filename + if (loader := getattr(package, '__loader__', None)) is None: + try: + loader = package.__spec__.loader + except AttributeError: + pass + if hasattr(loader, 'get_data'): + file_contents = loader.get_data(filename) + file_contents = file_contents.decode(encoding) + # get_data() opens files as 'rb', so one must do the equivalent + # conversion as universal newlines would do. + return _newline_convert(file_contents), filename with open(filename, encoding=encoding) as f: return f.read(), filename diff --git a/Lib/email/base64mime.py b/Lib/email/base64mime.py index 17f0818f6caa81..a7cc37365c6f9a 100644 --- a/Lib/email/base64mime.py +++ b/Lib/email/base64mime.py @@ -84,7 +84,7 @@ def body_encode(s, maxlinelen=76, eol=NL): in an email. """ if not s: - return s + return "" encvec = [] max_unencoded = maxlinelen * 3 // 4 diff --git a/Lib/email/headerregistry.py b/Lib/email/headerregistry.py index d8613ebf24e613..b590d69e8b7441 100644 --- a/Lib/email/headerregistry.py +++ b/Lib/email/headerregistry.py @@ -2,10 +2,6 @@ This module provides an implementation of the HeaderRegistry API. The implementation is designed to flexibly follow RFC5322 rules. - -Eventually HeaderRegistry will be a public API, but it isn't yet, -and will probably change some before that happens. - """ from types import MappingProxyType diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index cb2882e3360fcf..4c606b9f2a89b8 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -1,26 +1,81 @@ +import collections import os import os.path +import subprocess import sys -import runpy +import sysconfig import tempfile -import subprocess from importlib import resources -from . import _bundled - __all__ = ["version", "bootstrap"] +_PACKAGE_NAMES = ('setuptools', 'pip') +_SETUPTOOLS_VERSION = "56.0.0" +_PIP_VERSION = "21.1.1" +_PROJECTS = [ + ("setuptools", _SETUPTOOLS_VERSION, "py3"), + ("pip", _PIP_VERSION, "py3"), +] +# Packages bundled in ensurepip._bundled have wheel_name set. +# Packages from WHEEL_PKG_DIR have wheel_path set. +_Package = collections.namedtuple('Package', + ('version', 'wheel_name', 'wheel_path')) -_SETUPTOOLS_VERSION = "47.1.0" +# Directory of system wheel packages. Some Linux distribution packaging +# policies recommend against bundling dependencies. For example, Fedora +# installs wheel packages in the /usr/share/python-wheels/ directory and don't +# install the ensurepip._bundled package. +_WHEEL_PKG_DIR = sysconfig.get_config_var('WHEEL_PKG_DIR') -_PIP_VERSION = "20.2.3" -_PROJECTS = [ - ("setuptools", _SETUPTOOLS_VERSION, "py3"), - ("pip", _PIP_VERSION, "py2.py3"), -] +def _find_packages(path): + packages = {} + try: + filenames = os.listdir(path) + except OSError: + # Ignore: path doesn't exist or permission error + filenames = () + # Make the code deterministic if a directory contains multiple wheel files + # of the same package, but don't attempt to implement correct version + # comparison since this case should not happen. + filenames = sorted(filenames) + for filename in filenames: + # filename is like 'pip-20.2.3-py2.py3-none-any.whl' + if not filename.endswith(".whl"): + continue + for name in _PACKAGE_NAMES: + prefix = name + '-' + if filename.startswith(prefix): + break + else: + continue + + # Extract '20.2.2' from 'pip-20.2.2-py2.py3-none-any.whl' + version = filename.removeprefix(prefix).partition('-')[0] + wheel_path = os.path.join(path, filename) + packages[name] = _Package(version, None, wheel_path) + return packages + + +def _get_packages(): + global _PACKAGES, _WHEEL_PKG_DIR + if _PACKAGES is not None: + return _PACKAGES + + packages = {} + for name, version, py_tag in _PROJECTS: + wheel_name = f"{name}-{version}-{py_tag}-none-any.whl" + packages[name] = _Package(version, wheel_name, None) + if _WHEEL_PKG_DIR: + dir_packages = _find_packages(_WHEEL_PKG_DIR) + # only used the wheel package directory if all packages are found there + if all(name in dir_packages for name in _PACKAGE_NAMES): + packages = dir_packages + _PACKAGES = packages + return packages +_PACKAGES = None def _run_pip(args, additional_paths=None): @@ -35,14 +90,16 @@ def _run_pip(args, additional_paths=None): sys.argv[1:] = {args} runpy.run_module("pip", run_name="__main__", alter_sys=True) """ - return subprocess.run([sys.executable, "-c", code], check=True).returncode + return subprocess.run([sys.executable, '-W', 'ignore::DeprecationWarning', + "-c", code], check=True).returncode def version(): """ Returns a string specifying the bundled version of pip. """ - return _PIP_VERSION + return _get_packages()['pip'].version + def _disable_pip_configuration_settings(): # We deliberately ignore all pip environment variables @@ -104,16 +161,23 @@ def _bootstrap(*, root=None, upgrade=False, user=False, # Put our bundled wheels into a temporary directory and construct the # additional paths that need added to sys.path additional_paths = [] - for project, version, py_tag in _PROJECTS: - wheel_name = "{}-{}-{}-none-any.whl".format(project, version, py_tag) - whl = resources.read_binary( - _bundled, - wheel_name, - ) - with open(os.path.join(tmpdir, wheel_name), "wb") as fp: + for name, package in _get_packages().items(): + if package.wheel_name: + # Use bundled wheel package + from ensurepip import _bundled + wheel_name = package.wheel_name + whl = resources.read_binary(_bundled, wheel_name) + else: + # Use the wheel package directory + with open(package.wheel_path, "rb") as fp: + whl = fp.read() + wheel_name = os.path.basename(package.wheel_path) + + filename = os.path.join(tmpdir, wheel_name) + with open(filename, "wb") as fp: fp.write(whl) - additional_paths.append(os.path.join(tmpdir, wheel_name)) + additional_paths.append(filename) # Construct the arguments to be passed to the pip command args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir] @@ -126,7 +190,7 @@ def _bootstrap(*, root=None, upgrade=False, user=False, if verbosity: args += ["-" + "v" * verbosity] - return _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) + return _run_pip([*args, *_PACKAGE_NAMES], additional_paths) def _uninstall_helper(*, verbosity=0): """Helper to support a clean default uninstall process on Windows @@ -139,11 +203,14 @@ def _uninstall_helper(*, verbosity=0): except ImportError: return - # If the pip version doesn't match the bundled one, leave it alone - if pip.__version__ != _PIP_VERSION: - msg = ("ensurepip will only uninstall a matching version " - "({!r} installed, {!r} bundled)") - print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr) + # If the installed pip version doesn't match the available one, + # leave it alone + available_version = version() + if pip.__version__ != available_version: + print(f"ensurepip will only uninstall a matching version " + f"({pip.__version__!r} installed, " + f"{available_version!r} available)", + file=sys.stderr) return _disable_pip_configuration_settings() @@ -153,7 +220,7 @@ def _uninstall_helper(*, verbosity=0): if verbosity: args += ["-" + "v" * verbosity] - return _run_pip(args + [p[0] for p in reversed(_PROJECTS)]) + return _run_pip([*args, *reversed(_PACKAGE_NAMES)]) def _main(argv=None): diff --git a/Lib/ensurepip/_bundled/pip-20.2.3-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-21.1.1-py3-none-any.whl similarity index 51% rename from Lib/ensurepip/_bundled/pip-20.2.3-py2.py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-21.1.1-py3-none-any.whl index 7ebdc0f31d4e3e..291cc296fa7b48 100644 Binary files a/Lib/ensurepip/_bundled/pip-20.2.3-py2.py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-21.1.1-py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-47.1.0-py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-47.1.0-py3-none-any.whl deleted file mode 100644 index f87867ff98254a..00000000000000 Binary files a/Lib/ensurepip/_bundled/setuptools-47.1.0-py3-none-any.whl and /dev/null differ diff --git a/Lib/ensurepip/_bundled/setuptools-56.0.0-py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-56.0.0-py3-none-any.whl new file mode 100644 index 00000000000000..264ef10e826679 Binary files /dev/null and b/Lib/ensurepip/_bundled/setuptools-56.0.0-py3-none-any.whl differ diff --git a/Lib/enum.py b/Lib/enum.py index 40ff25b9cdad37..01f431001a9bf0 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1,44 +1,123 @@ import sys from types import MappingProxyType, DynamicClassAttribute +from builtins import property as _bltin_property, bin as _bltin_bin __all__ = [ - 'EnumMeta', + 'EnumType', 'EnumMeta', 'Enum', 'IntEnum', 'StrEnum', 'Flag', 'IntFlag', 'auto', 'unique', + 'property', + 'FlagBoundary', 'STRICT', 'CONFORM', 'EJECT', 'KEEP', + 'global_flag_repr', 'global_enum_repr', 'global_enum', ] +# Dummy value for Enum and Flag as there are explicit checks for them +# before they have been created. +# This is also why there are checks in EnumType like `if Enum is not None` +Enum = Flag = EJECT = None + def _is_descriptor(obj): - """Returns True if obj is a descriptor, False otherwise.""" + """ + Returns True if obj is a descriptor, False otherwise. + """ return ( hasattr(obj, '__get__') or hasattr(obj, '__set__') or - hasattr(obj, '__delete__')) - + hasattr(obj, '__delete__') + ) def _is_dunder(name): - """Returns True if a __dunder__ name, False otherwise.""" - return (len(name) > 4 and + """ + Returns True if a __dunder__ name, False otherwise. + """ + return ( + len(name) > 4 and name[:2] == name[-2:] == '__' and name[2] != '_' and - name[-3] != '_') - + name[-3] != '_' + ) def _is_sunder(name): - """Returns True if a _sunder_ name, False otherwise.""" - return (len(name) > 2 and + """ + Returns True if a _sunder_ name, False otherwise. + """ + return ( + len(name) > 2 and name[0] == name[-1] == '_' and name[1:2] != '_' and - name[-2:-1] != '_') + name[-2:-1] != '_' + ) + +def _is_private(cls_name, name): + # do not use `re` as `re` imports `enum` + pattern = '_%s__' % (cls_name, ) + pat_len = len(pattern) + if ( + len(name) > pat_len + and name.startswith(pattern) + and name[pat_len:pat_len+1] != ['_'] + and (name[-1] != '_' or name[-2] != '_') + ): + return True + else: + return False +def _is_single_bit(num): + """ + True if only one bit set in num (should be an int) + """ + if num == 0: + return False + num &= num - 1 + return num == 0 -def _make_class_unpicklable(cls): - """Make the given class un-picklable.""" +def _make_class_unpicklable(obj): + """ + Make the given obj un-picklable. + + obj should be either a dictionary, on an Enum + """ def _break_on_call_reduce(self, proto): raise TypeError('%r cannot be pickled' % self) - cls.__reduce_ex__ = _break_on_call_reduce - cls.__module__ = '' + if isinstance(obj, dict): + obj['__reduce_ex__'] = _break_on_call_reduce + obj['__module__'] = '' + else: + setattr(obj, '__reduce_ex__', _break_on_call_reduce) + setattr(obj, '__module__', '') + +def _iter_bits_lsb(num): + while num: + b = num & (~num + 1) + yield b + num ^= b + +def bin(num, max_bits=None): + """ + Like built-in bin(), except negative values are represented in + twos-compliment, and the leading bit always indicates sign + (0=positive, 1=negative). + + >>> bin(10) + '0b0 1010' + >>> bin(~10) # ~10 is -11 + '0b1 0101' + """ + + ceiling = 2 ** (num).bit_length() + if num >= 0: + s = _bltin_bin(num + ceiling).replace('1', '0', 1) + else: + s = _bltin_bin(~num ^ (ceiling - 1) + ceiling) + sign = s[:3] + digits = s[3:] + if max_bits is not None: + if len(digits) < max_bits: + digits = (sign[-1] * max_bits + digits)[-max_bits:] + return "%s %s" % (sign, digits) + _auto_null = object() class auto: @@ -47,13 +126,170 @@ class auto: """ value = _auto_null +class property(DynamicClassAttribute): + """ + This is a descriptor, used to define attributes that act differently + when accessed through an enum member and through an enum class. + Instance access is the same as property(), but access to an attribute + through the enum class will instead look in the class' _member_map_ for + a corresponding enum member. + """ + + def __get__(self, instance, ownerclass=None): + if instance is None: + try: + return ownerclass._member_map_[self.name] + except KeyError: + raise AttributeError( + '%s: no class attribute %r' % (ownerclass.__name__, self.name) + ) + else: + if self.fget is None: + # check for member + if self.name in ownerclass._member_map_: + import warnings + warnings.warn( + "accessing one member from another is not supported, " + " and will be disabled in 3.12", + DeprecationWarning, + stacklevel=2, + ) + return ownerclass._member_map_[self.name] + raise AttributeError( + '%s: no instance attribute %r' % (ownerclass.__name__, self.name) + ) + else: + return self.fget(instance) + + def __set__(self, instance, value): + if self.fset is None: + raise AttributeError( + "%s: cannot set instance attribute %r" % (self.clsname, self.name) + ) + else: + return self.fset(instance, value) + + def __delete__(self, instance): + if self.fdel is None: + raise AttributeError( + "%s: cannot delete instance attribute %r" % (self.clsname, self.name) + ) + else: + return self.fdel(instance) + + def __set_name__(self, ownerclass, name): + self.name = name + self.clsname = ownerclass.__name__ + + +class _proto_member: + """ + intermediate step for enum members between class execution and final creation + """ + + def __init__(self, value): + self.value = value + + def __set_name__(self, enum_class, member_name): + """ + convert each quasi-member into an instance of the new enum class + """ + # first step: remove ourself from enum_class + delattr(enum_class, member_name) + # second step: create member based on enum_class + value = self.value + if not isinstance(value, tuple): + args = (value, ) + else: + args = value + if enum_class._member_type_ is tuple: # special case for tuple enums + args = (args, ) # wrap it one more time + if not enum_class._use_args_: + enum_member = enum_class._new_member_(enum_class) + if not hasattr(enum_member, '_value_'): + enum_member._value_ = value + else: + enum_member = enum_class._new_member_(enum_class, *args) + if not hasattr(enum_member, '_value_'): + if enum_class._member_type_ is object: + enum_member._value_ = value + else: + try: + enum_member._value_ = enum_class._member_type_(*args) + except Exception as exc: + raise TypeError( + '_value_ not set in __new__, unable to create it' + ) from None + value = enum_member._value_ + enum_member._name_ = member_name + enum_member.__objclass__ = enum_class + enum_member.__init__(*args) + enum_member._sort_order_ = len(enum_class._member_names_) + # If another member with the same value was already defined, the + # new member becomes an alias to the existing one. + for name, canonical_member in enum_class._member_map_.items(): + if canonical_member._value_ == enum_member._value_: + enum_member = canonical_member + break + else: + # this could still be an alias if the value is multi-bit and the + # class is a flag class + if ( + Flag is None + or not issubclass(enum_class, Flag) + ): + # no other instances found, record this member in _member_names_ + enum_class._member_names_.append(member_name) + elif ( + Flag is not None + and issubclass(enum_class, Flag) + and _is_single_bit(value) + ): + # no other instances found, record this member in _member_names_ + enum_class._member_names_.append(member_name) + # get redirect in place before adding to _member_map_ + # but check for other instances in parent classes first + need_override = False + descriptor = None + for base in enum_class.__mro__[1:]: + descriptor = base.__dict__.get(member_name) + if descriptor is not None: + if isinstance(descriptor, (property, DynamicClassAttribute)): + break + else: + need_override = True + # keep looking for an enum.property + if descriptor and not need_override: + # previous enum.property found, no further action needed + pass + else: + redirect = property() + redirect.__set_name__(enum_class, member_name) + if descriptor and need_override: + # previous enum.property found, but some other inherited attribute + # is in the way; copy fget, fset, fdel to this one + redirect.fget = descriptor.fget + redirect.fset = descriptor.fset + redirect.fdel = descriptor.fdel + setattr(enum_class, member_name, redirect) + # now add to _member_map_ (even aliases) + enum_class._member_map_[member_name] = enum_member + try: + # This may fail if value is not hashable. We can't add the value + # to the map, and by-value lookups for this value will be + # linear. + enum_class._value2member_map_.setdefault(value, enum_member) + except TypeError: + # keep track of the value in a list so containment checks are quick + enum_class._unhashable_values_.append(value) + class _EnumDict(dict): - """Track enum member order and ensure member names are not reused. + """ + Track enum member order and ensure member names are not reused. - EnumMeta will use the names found in self._member_names as the + EnumType will use the names found in self._member_names as the enumeration member names. - """ def __init__(self): super().__init__() @@ -63,26 +299,33 @@ def __init__(self): self._auto_called = False def __setitem__(self, key, value): - """Changes anything not dundered or not a descriptor. + """ + Changes anything not dundered or not a descriptor. If an enum member name is used twice, an error is raised; duplicate values are not checked for. Single underscore (sunder) names are reserved. - """ - if _is_sunder(key): + if _is_private(self._cls_name, key): + # do nothing, name will be a normal attribute + pass + elif _is_sunder(key): if key not in ( - '_order_', '_create_pseudo_member_', + '_order_', '_generate_next_value_', '_missing_', '_ignore_', + '_iter_member_', '_iter_member_by_value_', '_iter_member_by_def_', ): - raise ValueError(f'_sunder_ names, such as "{key}", are ' - 'reserved for future Enum use') + raise ValueError( + '_sunder_ names, such as %r, are reserved for future Enum use' + % (key, ) + ) if key == '_generate_next_value_': # check if members already defined as auto() if self._auto_called: raise TypeError("_generate_next_value_ must be defined before members") - setattr(self, '_generate_next_value', value) + _gnv = value.__func__ if isinstance(value, staticmethod) else value + setattr(self, '_generate_next_value', _gnv) elif key == '_ignore_': if isinstance(value, str): value = value.replace(',',' ').split() @@ -91,13 +334,16 @@ def __setitem__(self, key, value): self._ignore = value already = set(value) & set(self._member_names) if already: - raise ValueError('_ignore_ cannot specify already set names: %r' % (already, )) + raise ValueError( + '_ignore_ cannot specify already set names: %r' + % (already, ) + ) elif _is_dunder(key): if key == '__order__': key = '_order_' elif key in self._member_names: # descriptor overwriting an enum? - raise TypeError('Attempted to reuse key: %r' % key) + raise TypeError('%r already defined as: %r' % (key, self[key])) elif key in self._ignore: pass elif not _is_descriptor(value): @@ -106,83 +352,107 @@ def __setitem__(self, key, value): raise TypeError('%r already defined as: %r' % (key, self[key])) if isinstance(value, auto): if value.value == _auto_null: - value.value = self._generate_next_value(key, 1, len(self._member_names), self._last_values[:]) + value.value = self._generate_next_value( + key, 1, len(self._member_names), self._last_values[:], + ) self._auto_called = True value = value.value self._member_names.append(key) self._last_values.append(value) super().__setitem__(key, value) + def update(self, members, **more_members): + try: + for name in members.keys(): + self[name] = members[name] + except AttributeError: + for name, value in members: + self[name] = value + for name, value in more_members.items(): + self[name] = value -# Dummy value for Enum as EnumMeta explicitly checks for it, but of course -# until EnumMeta finishes running the first time the Enum class doesn't exist. -# This is also why there are checks in EnumMeta like `if Enum is not None` -Enum = None +class EnumType(type): + """ + Metaclass for Enum + """ -class EnumMeta(type): - """Metaclass for Enum""" @classmethod - def __prepare__(metacls, cls, bases): + def __prepare__(metacls, cls, bases, **kwds): # check that previous enum members do not exist metacls._check_for_existing_members(cls, bases) # create the namespace dict enum_dict = _EnumDict() + enum_dict._cls_name = cls # inherit previous flags and _generate_next_value_ function member_type, first_enum = metacls._get_mixins_(cls, bases) if first_enum is not None: - enum_dict['_generate_next_value_'] = getattr(first_enum, '_generate_next_value_', None) + enum_dict['_generate_next_value_'] = getattr( + first_enum, '_generate_next_value_', None, + ) return enum_dict - def __new__(metacls, cls, bases, classdict): + def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **kwds): # an Enum class is final once enumeration items have been defined; it # cannot be mixed with other types (int, float, etc.) if it has an # inherited __new__ unless a new __new__ is defined (or the resulting # class will fail). # # remove any keys listed in _ignore_ + if _simple: + return super().__new__(metacls, cls, bases, classdict, **kwds) classdict.setdefault('_ignore_', []).append('_ignore_') ignore = classdict['_ignore_'] for key in ignore: classdict.pop(key, None) - member_type, first_enum = metacls._get_mixins_(cls, bases) - __new__, save_new, use_args = metacls._find_new_(classdict, member_type, - first_enum) - - # save enum items into separate mapping so they don't get baked into - # the new class - enum_members = {k: classdict[k] for k in classdict._member_names} - for name in classdict._member_names: - del classdict[name] - - # adjust the sunders - _order_ = classdict.pop('_order_', None) - + # + # grab member names + member_names = classdict._member_names + # # check for illegal enum names (any others?) - invalid_names = set(enum_members) & {'mro', ''} + invalid_names = set(member_names) & {'mro', ''} if invalid_names: raise ValueError('Invalid enum member name: {0}'.format( ','.join(invalid_names))) - - # create a default docstring if one has not been provided - if '__doc__' not in classdict: - classdict['__doc__'] = 'An enumeration.' - - # create our new Enum type - enum_class = super().__new__(metacls, cls, bases, classdict) - enum_class._member_names_ = [] # names in definition order - enum_class._member_map_ = {} # name->value map - enum_class._member_type_ = member_type - - # save DynamicClassAttribute attributes from super classes so we know - # if we can take the shortcut of storing members in the class dict - dynamic_attributes = {k for c in enum_class.mro() - for k, v in c.__dict__.items() - if isinstance(v, DynamicClassAttribute)} - - # Reverse value->name map for hashable values. - enum_class._value2member_map_ = {} - + # + # adjust the sunders + _order_ = classdict.pop('_order_', None) + # convert to normal dict + classdict = dict(classdict.items()) + # + # data type of member and the controlling Enum class + member_type, first_enum = metacls._get_mixins_(cls, bases) + __new__, save_new, use_args = metacls._find_new_( + classdict, member_type, first_enum, + ) + classdict['_new_member_'] = __new__ + classdict['_use_args_'] = use_args + # + # convert future enum members into temporary _proto_members + # and record integer values in case this will be a Flag + flag_mask = 0 + for name in member_names: + value = classdict[name] + if isinstance(value, int): + flag_mask |= value + classdict[name] = _proto_member(value) + # + # house-keeping structures + classdict['_member_names_'] = [] + classdict['_member_map_'] = {} + classdict['_value2member_map_'] = {} + classdict['_unhashable_values_'] = [] + classdict['_member_type_'] = member_type + # + # Flag structures (will be removed if final class is not a Flag + classdict['_boundary_'] = ( + boundary + or getattr(first_enum, '_boundary_', None) + ) + classdict['_flag_mask_'] = flag_mask + classdict['_all_bits_'] = 2 ** ((flag_mask).bit_length()) - 1 + classdict['_inverted_'] = None + # # If a custom type is mixed into the Enum, and it does not know how # to pickle itself, pickle.dumps will succeed but pickle.loads will # fail. Rather than have the error show up later and possibly far @@ -198,58 +468,21 @@ def __new__(metacls, cls, bases, classdict): methods = ('__getnewargs_ex__', '__getnewargs__', '__reduce_ex__', '__reduce__') if not any(m in member_type.__dict__ for m in methods): - _make_class_unpicklable(enum_class) - - # instantiate them, checking for duplicates as we go - # we instantiate first instead of checking for duplicates first in case - # a custom __new__ is doing something funky with the values -- such as - # auto-numbering ;) - for member_name in classdict._member_names: - value = enum_members[member_name] - if not isinstance(value, tuple): - args = (value, ) - else: - args = value - if member_type is tuple: # special case for tuple enums - args = (args, ) # wrap it one more time - if not use_args: - enum_member = __new__(enum_class) - if not hasattr(enum_member, '_value_'): - enum_member._value_ = value - else: - enum_member = __new__(enum_class, *args) - if not hasattr(enum_member, '_value_'): - if member_type is object: - enum_member._value_ = value - else: - enum_member._value_ = member_type(*args) - value = enum_member._value_ - enum_member._name_ = member_name - enum_member.__objclass__ = enum_class - enum_member.__init__(*args) - # If another member with the same value was already defined, the - # new member becomes an alias to the existing one. - for name, canonical_member in enum_class._member_map_.items(): - if canonical_member._value_ == enum_member._value_: - enum_member = canonical_member - break - else: - # Aliases don't appear in member names (only in __members__). - enum_class._member_names_.append(member_name) - # performance boost for any member that would not shadow - # a DynamicClassAttribute - if member_name not in dynamic_attributes: - setattr(enum_class, member_name, enum_member) - # now add to _member_map_ - enum_class._member_map_[member_name] = enum_member - try: - # This may fail if value is not hashable. We can't add the value - # to the map, and by-value lookups for this value will be - # linear. - enum_class._value2member_map_[value] = enum_member - except TypeError: - pass - + _make_class_unpicklable(classdict) + # + # create a default docstring if one has not been provided + if '__doc__' not in classdict: + classdict['__doc__'] = 'An enumeration.' + try: + exc = None + enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) + except RuntimeError as e: + # any exceptions raised by member.__new__ will get converted to a + # RuntimeError, so get that original exception back and raise it instead + exc = e.__cause__ or e + if exc is not None: + raise exc + # # double check that repr and friends are not the mixin's or various # things break (such as pickle) # however, if the method is defined in the Enum itself, don't replace @@ -262,7 +495,7 @@ def __new__(metacls, cls, bases, classdict): enum_method = getattr(first_enum, name, None) if obj_method is not None and obj_method is class_method: setattr(enum_class, name, enum_method) - + # # replace any other __new__ with our own (as long as Enum is not None, # anyway) -- again, this is to support pickle if Enum is not None: @@ -271,14 +504,78 @@ def __new__(metacls, cls, bases, classdict): if save_new: enum_class.__new_member__ = __new__ enum_class.__new__ = Enum.__new__ - + # # py3 support for definition order (helps keep py2/py3 code in sync) + # + # _order_ checking is spread out into three/four steps + # - if enum_class is a Flag: + # - remove any non-single-bit flags from _order_ + # - remove any aliases from _order_ + # - check that _order_ and _member_names_ match + # + # step 1: ensure we have a list if _order_ is not None: if isinstance(_order_, str): _order_ = _order_.replace(',', ' ').split() + # + # remove Flag structures if final class is not a Flag + if ( + Flag is None and cls != 'Flag' + or Flag is not None and not issubclass(enum_class, Flag) + ): + delattr(enum_class, '_boundary_') + delattr(enum_class, '_flag_mask_') + delattr(enum_class, '_all_bits_') + delattr(enum_class, '_inverted_') + elif Flag is not None and issubclass(enum_class, Flag): + # ensure _all_bits_ is correct and there are no missing flags + single_bit_total = 0 + multi_bit_total = 0 + for flag in enum_class._member_map_.values(): + flag_value = flag._value_ + if _is_single_bit(flag_value): + single_bit_total |= flag_value + else: + # multi-bit flags are considered aliases + multi_bit_total |= flag_value + if enum_class._boundary_ is not KEEP: + missed = list(_iter_bits_lsb(multi_bit_total & ~single_bit_total)) + if missed: + raise TypeError( + 'invalid Flag %r -- missing values: %s' + % (cls, ', '.join((str(i) for i in missed))) + ) + enum_class._flag_mask_ = single_bit_total + # + # set correct __iter__ + member_list = [m._value_ for m in enum_class] + if member_list != sorted(member_list): + enum_class._iter_member_ = enum_class._iter_member_by_def_ + if _order_: + # _order_ step 2: remove any items from _order_ that are not single-bit + _order_ = [ + o + for o in _order_ + if o not in enum_class._member_map_ or _is_single_bit(enum_class[o]._value_) + ] + # + if _order_: + # _order_ step 3: remove aliases from _order_ + _order_ = [ + o + for o in _order_ + if ( + o not in enum_class._member_map_ + or + (o in enum_class._member_map_ and o in enum_class._member_names_) + )] + # _order_ step 4: verify that _order_ and _member_names_ match if _order_ != enum_class._member_names_: - raise TypeError('member order does not match _order_') - + raise TypeError( + 'member order does not match _order_:\n%r\n%r' + % (enum_class._member_names_, _order_) + ) + # return enum_class def __bool__(self): @@ -287,8 +584,9 @@ def __bool__(self): """ return True - def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1): - """Either returns an existing member, or creates a new enum class. + def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None): + """ + Either returns an existing member, or creates a new enum class. This method is used both when an enum class is given a value to match to an enumeration member (i.e. Color(3)) and for the functional API @@ -310,15 +608,29 @@ def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, s not correct, unpickling will fail in some circumstances. `type`, if set, will be mixed in as the first base class. - """ if names is None: # simple value lookup return cls.__new__(cls, value) # otherwise, functional API: we're creating a new Enum type - return cls._create_(value, names, module=module, qualname=qualname, type=type, start=start) + return cls._create_( + value, + names, + module=module, + qualname=qualname, + type=type, + start=start, + boundary=boundary, + ) def __contains__(cls, member): if not isinstance(member, Enum): + import warnings + warnings.warn( + "in 3.12 __contains__ will no longer raise TypeError, but will return True or\n" + "False depending on whether the value is a member or the value of a member", + DeprecationWarning, + stacklevel=2, + ) raise TypeError( "unsupported operand type(s) for 'in': '%s' and '%s'" % ( type(member).__qualname__, cls.__class__.__qualname__)) @@ -328,22 +640,23 @@ def __delattr__(cls, attr): # nicer error message when someone tries to delete an attribute # (see issue19025). if attr in cls._member_map_: - raise AttributeError( - "%s: cannot delete Enum member." % cls.__name__) + raise AttributeError("%s: cannot delete Enum member %r." % (cls.__name__, attr)) super().__delattr__(attr) def __dir__(self): - return (['__class__', '__doc__', '__members__', '__module__'] + - self._member_names_) + return ( + ['__class__', '__doc__', '__members__', '__module__'] + + self._member_names_ + ) def __getattr__(cls, name): - """Return the enum member matching `name` + """ + Return the enum member matching `name` We use __getattr__ instead of descriptors or inserting into the enum class' __dict__ in order to support `name` and `value` being both properties for enum members (which live in the class' __dict__) and enum members themselves. - """ if _is_dunder(name): raise AttributeError(name) @@ -356,18 +669,21 @@ def __getitem__(cls, name): return cls._member_map_[name] def __iter__(cls): + """ + Returns members in definition order. + """ return (cls._member_map_[name] for name in cls._member_names_) def __len__(cls): return len(cls._member_names_) - @property + @_bltin_property def __members__(cls): - """Returns a mapping of member name->value. + """ + Returns a mapping of member name->value. This mapping lists all enum members, including aliases. Note that this is a read-only view of the internal mapping. - """ return MappingProxyType(cls._member_map_) @@ -375,23 +691,27 @@ def __repr__(cls): return "" % cls.__name__ def __reversed__(cls): + """ + Returns members in reverse definition order. + """ return (cls._member_map_[name] for name in reversed(cls._member_names_)) def __setattr__(cls, name, value): - """Block attempts to reassign Enum members. + """ + Block attempts to reassign Enum members. A simple assignment to the class namespace only changes one of the several possible ways to get an Enum member from the Enum class, resulting in an inconsistent Enumeration. - """ member_map = cls.__dict__.get('_member_map_', {}) if name in member_map: - raise AttributeError('Cannot reassign members.') + raise AttributeError('Cannot reassign member %r.' % (name, )) super().__setattr__(name, value) - def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, start=1): - """Convenience method to create a new Enum class. + def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, start=1, boundary=None): + """ + Convenience method to create a new Enum class. `names` can be: @@ -400,7 +720,6 @@ def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, s * An iterable of member names. Values are incremented by 1 from `start`. * An iterable of (member name, value) pairs. * A mapping of member name -> value pairs. - """ metacls = cls.__class__ bases = (cls, ) if type is None else (type, cls) @@ -425,7 +744,6 @@ def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, s else: member_name, member_value = item classdict[member_name] = member_value - enum_class = metacls.__new__(metacls, class_name, bases, classdict) # TODO: replace the frame hack if a blessed way to know the calling # module is ever developed @@ -435,15 +753,16 @@ def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, s except (AttributeError, ValueError, KeyError): pass if module is None: - _make_class_unpicklable(enum_class) + _make_class_unpicklable(classdict) else: - enum_class.__module__ = module + classdict['__module__'] = module if qualname is not None: - enum_class.__qualname__ = qualname + classdict['__qualname__'] = qualname - return enum_class + return metacls.__new__(metacls, class_name, bases, classdict, boundary=boundary) + + def _convert_(cls, name, module, filter, source=None, *, boundary=None): - def _convert_(cls, name, module, filter, source=None): """ Create a new Enum subclass that replaces a collection of global constants """ @@ -452,9 +771,9 @@ def _convert_(cls, name, module, filter, source=None): # module; # also, replace the __reduce_ex__ method so unpickling works in # previous Python versions - module_globals = vars(sys.modules[module]) + module_globals = sys.modules[module].__dict__ if source: - source = vars(source) + source = source.__dict__ else: source = module_globals # _value2member_map_ is populated in the same order every time @@ -470,9 +789,12 @@ def _convert_(cls, name, module, filter, source=None): except TypeError: # unless some values aren't comparable, in which case sort by name members.sort(key=lambda t: t[0]) - cls = cls(name, members, module=module) + body = {t[0]: t[1] for t in members} + body['__module__'] = module + tmp_cls = type(name, (object, ), body) + cls = _simple_enum(etype=cls, boundary=boundary or KEEP)(tmp_cls) cls.__reduce_ex__ = _reduce_ex_by_name - module_globals.update(cls.__members__) + global_enum(cls) module_globals[name] = cls return cls @@ -481,15 +803,18 @@ def _check_for_existing_members(class_name, bases): for chain in bases: for base in chain.__mro__: if issubclass(base, Enum) and base._member_names_: - raise TypeError("%s: cannot extend enumeration %r" % (class_name, base.__name__)) + raise TypeError( + "%s: cannot extend enumeration %r" + % (class_name, base.__name__) + ) @staticmethod def _get_mixins_(class_name, bases): - """Returns the type for creating enum members, and the first inherited + """ + Returns the type for creating enum members, and the first inherited enum class. bases: the tuple of bases that was given to __new__ - """ if not bases: return object, Enum @@ -501,12 +826,16 @@ def _find_data_type(bases): for base in chain.__mro__: if base is object: continue + elif issubclass(base, Enum): + if base._member_type_ is not object: + data_types.append(base._member_type_) + break elif '__new__' in base.__dict__: if issubclass(base, Enum): continue data_types.append(candidate or base) break - elif not issubclass(base, Enum): + else: candidate = base if len(data_types) > 1: raise TypeError('%r: too many data types: %r' % (class_name, data_types)) @@ -528,12 +857,12 @@ def _find_data_type(bases): @staticmethod def _find_new_(classdict, member_type, first_enum): - """Returns the __new__ to be used for creating the enum members. + """ + Returns the __new__ to be used for creating the enum members. classdict: the class dictionary given to __new__ member_type: the data type whose __new__ will be used by default first_enum: enumeration to check for an overriding __new__ - """ # now find the correct __new__, checking to see of one was defined # by the user; also check earlier enum classes in case a __new__ was @@ -541,7 +870,7 @@ def _find_new_(classdict, member_type, first_enum): __new__ = classdict.get('__new__', None) # should __new__ be saved as __new_member__ later? - save_new = __new__ is not None + save_new = first_enum is not None and __new__ is not None if __new__ is None: # check all possibles for __new_member__ before falling back to @@ -565,19 +894,21 @@ def _find_new_(classdict, member_type, first_enum): # if a non-object.__new__ is used then whatever value/tuple was # assigned to the enum member name will be passed to __new__ and to the # new enum member's __init__ - if __new__ is object.__new__: + if first_enum is None or __new__ in (Enum.__new__, object.__new__): use_args = False else: use_args = True return __new__, save_new, use_args +EnumMeta = EnumType -class Enum(metaclass=EnumMeta): - """Generic enumeration. +class Enum(metaclass=EnumType): + """ + Generic enumeration. Derive from this class to define new enumerations. - """ + def __new__(cls, value): # all enum instances are actually created during class construction # without calling this method; this method is called by the metaclass' @@ -604,21 +935,40 @@ def __new__(cls, value): except Exception as e: exc = e result = None - if isinstance(result, cls): - return result - else: - ve_exc = ValueError("%r is not a valid %s" % (value, cls.__qualname__)) - if result is None and exc is None: - raise ve_exc - elif exc is None: - exc = TypeError( - 'error in %s._missing_: returned %r instead of None or a valid member' - % (cls.__name__, result) - ) - exc.__context__ = ve_exc - raise exc + try: + if isinstance(result, cls): + return result + elif ( + Flag is not None and issubclass(cls, Flag) + and cls._boundary_ is EJECT and isinstance(result, int) + ): + return result + else: + ve_exc = ValueError("%r is not a valid %s" % (value, cls.__qualname__)) + if result is None and exc is None: + raise ve_exc + elif exc is None: + exc = TypeError( + 'error in %s._missing_: returned %r instead of None or a valid member' + % (cls.__name__, result) + ) + if not isinstance(exc, ValueError): + exc.__context__ = ve_exc + raise exc + finally: + # ensure all variables that could hold an exception are destroyed + exc = None + ve_exc = None def _generate_next_value_(name, start, count, last_values): + """ + Generate the next value when not given. + + name: the name of the member + start: the initial start value or None + count: the number of existing members + last_value: the last value assigned or None + """ for last_value in reversed(last_values): try: return last_value + 1 @@ -632,13 +982,15 @@ def _missing_(cls, value): return None def __repr__(self): - return "<%s.%s: %r>" % ( - self.__class__.__name__, self._name_, self._value_) + return "%s.%s" % ( self.__class__.__name__, self._name_) def __str__(self): - return "%s.%s" % (self.__class__.__name__, self._name_) + return "%s" % (self._name_, ) def __dir__(self): + """ + Returns all members and all public methods + """ added_behavior = [ m for cls in self.__class__.mro() @@ -648,17 +1000,29 @@ def __dir__(self): return (['__class__', '__doc__', '__module__'] + added_behavior) def __format__(self, format_spec): + """ + Returns format using actual value type unless __str__ has been overridden. + """ # mixed-in Enums should use the mixed-in type's __format__, otherwise # we can get strange results with the Enum name showing up instead of # the value # pure Enum branch, or branch with __str__ explicitly overridden - str_overridden = type(self).__str__ != Enum.__str__ + str_overridden = type(self).__str__ not in (Enum.__str__, Flag.__str__) if self._member_type_ is object or str_overridden: cls = str val = str(self) # mix-in branch else: + if not format_spec or format_spec in ('{}','{:}'): + import warnings + warnings.warn( + "in 3.12 format() will use the enum member, not the enum member's value;\n" + "use a format specifier, such as :d for an IntEnum member, to maintain " + "the current display", + DeprecationWarning, + stacklevel=2, + ) cls = self._member_type_ val = self._value_ return cls.__format__(val, format_spec) @@ -669,19 +1033,20 @@ def __hash__(self): def __reduce_ex__(self, proto): return self.__class__, (self._value_, ) - # DynamicClassAttribute is used to provide access to the `name` and - # `value` properties of enum members while keeping some measure of + # enum.property is used to provide access to the `name` and + # `value` attributes of enum members while keeping some measure of # protection from modification, while still allowing for an enumeration # to have members named `name` and `value`. This works because enumeration - # members are not set directly on the enum class -- __getattr__ is - # used to look them up. + # members are not set directly on the enum class; they are kept in a + # separate structure, _member_map_, which is where enum.property looks for + # them - @DynamicClassAttribute + @property def name(self): """The name of the Enum member.""" return self._name_ - @DynamicClassAttribute + @property def value(self): """The value of the Enum member.""" return self._value_ @@ -705,14 +1070,14 @@ def __new__(cls, *values): # it must be a string if not isinstance(values[0], str): raise TypeError('%r is not a string' % (values[0], )) - if len(values) > 1: + if len(values) >= 2: # check that encoding argument is a string if not isinstance(values[1], str): raise TypeError('encoding must be a string, not %r' % (values[1], )) - if len(values) > 2: - # check that errors argument is a string - if not isinstance(values[2], str): - raise TypeError('errors must be a string, not %r' % (values[2], )) + if len(values) == 3: + # check that errors argument is a string + if not isinstance(values[2], str): + raise TypeError('errors must be a string, not %r' % (values[2])) value = str(*values) member = str.__new__(cls, value) member._value_ = value @@ -720,12 +1085,35 @@ def __new__(cls, *values): __str__ = str.__str__ + def _generate_next_value_(name, start, count, last_values): + """ + Return the lower-cased version of the member name. + """ + return name.lower() + def _reduce_ex_by_name(self, proto): return self.name -class Flag(Enum): - """Support for flags""" +class FlagBoundary(StrEnum): + """ + control how out of range values are handled + "strict" -> error is raised [default for Flag] + "conform" -> extra bits are discarded + "eject" -> lose flag status [default for IntFlag] + "keep" -> keep flag status and all bits + """ + STRICT = auto() + CONFORM = auto() + EJECT = auto() + KEEP = auto() +STRICT, CONFORM, EJECT, KEEP = FlagBoundary + + +class Flag(Enum, boundary=STRICT): + """ + Support for flags + """ def _generate_next_value_(name, start, count, last_values): """ @@ -738,78 +1126,156 @@ def _generate_next_value_(name, start, count, last_values): """ if not count: return start if start is not None else 1 - for last_value in reversed(last_values): - try: - high_bit = _high_bit(last_value) - break - except Exception: - raise TypeError('Invalid Flag value: %r' % last_value) from None + last_value = max(last_values) + try: + high_bit = _high_bit(last_value) + except Exception: + raise TypeError('Invalid Flag value: %r' % last_value) from None return 2 ** (high_bit+1) @classmethod - def _missing_(cls, value): - original_value = value - if value < 0: - value = ~value - possible_member = cls._create_pseudo_member_(value) - if original_value < 0: - possible_member = ~possible_member - return possible_member + def _iter_member_by_value_(cls, value): + """ + Extract all members from the value in definition (i.e. increasing value) order. + """ + for val in _iter_bits_lsb(value & cls._flag_mask_): + yield cls._value2member_map_.get(val) + + _iter_member_ = _iter_member_by_value_ + + @classmethod + def _iter_member_by_def_(cls, value): + """ + Extract all members from the value in definition order. + """ + yield from sorted( + cls._iter_member_by_value_(value), + key=lambda m: m._sort_order_, + ) @classmethod - def _create_pseudo_member_(cls, value): + def _missing_(cls, value): """ - Create a composite member iff value contains only members. + Create a composite member containing all canonical members present in `value`. + + If non-member values are present, result depends on `_boundary_` setting. """ - pseudo_member = cls._value2member_map_.get(value, None) - if pseudo_member is None: - # verify all bits are accounted for - _, extra_flags = _decompose(cls, value) - if extra_flags: - raise ValueError("%r is not a valid %s" % (value, cls.__qualname__)) + if not isinstance(value, int): + raise ValueError( + "%r is not a valid %s" % (value, cls.__qualname__) + ) + # check boundaries + # - value must be in range (e.g. -16 <-> +15, i.e. ~15 <-> 15) + # - value must not include any skipped flags (e.g. if bit 2 is not + # defined, then 0d10 is invalid) + flag_mask = cls._flag_mask_ + all_bits = cls._all_bits_ + neg_value = None + if ( + not ~all_bits <= value <= all_bits + or value & (all_bits ^ flag_mask) + ): + if cls._boundary_ is STRICT: + max_bits = max(value.bit_length(), flag_mask.bit_length()) + raise ValueError( + "%s: invalid value: %r\n given %s\n allowed %s" % ( + cls.__name__, value, bin(value, max_bits), bin(flag_mask, max_bits), + )) + elif cls._boundary_ is CONFORM: + value = value & flag_mask + elif cls._boundary_ is EJECT: + return value + elif cls._boundary_ is KEEP: + if value < 0: + value = ( + max(all_bits+1, 2**(value.bit_length())) + + value + ) + else: + raise ValueError( + 'unknown flag boundary: %r' % (cls._boundary_, ) + ) + if value < 0: + neg_value = value + value = all_bits + 1 + value + # get members and unknown + unknown = value & ~flag_mask + member_value = value & flag_mask + if unknown and cls._boundary_ is not KEEP: + raise ValueError( + '%s(%r) --> unknown values %r [%s]' + % (cls.__name__, value, unknown, bin(unknown)) + ) + # normal Flag? + __new__ = getattr(cls, '__new_member__', None) + if cls._member_type_ is object and not __new__: # construct a singleton enum pseudo-member pseudo_member = object.__new__(cls) - pseudo_member._name_ = None + else: + pseudo_member = (__new__ or cls._member_type_.__new__)(cls, value) + if not hasattr(pseudo_member, '_value_'): pseudo_member._value_ = value - # use setdefault in case another thread already created a composite - # with this value + if member_value: + pseudo_member._name_ = '|'.join([ + m._name_ for m in cls._iter_member_(member_value) + ]) + if unknown: + pseudo_member._name_ += '|0x%x' % unknown + else: + pseudo_member._name_ = None + # use setdefault in case another thread already created a composite + # with this value, but only if all members are known + # note: zero is a special case -- add it + if not unknown: pseudo_member = cls._value2member_map_.setdefault(value, pseudo_member) + if neg_value is not None: + cls._value2member_map_[neg_value] = pseudo_member return pseudo_member def __contains__(self, other): + """ + Returns True if self has at least the same flags set as other. + """ if not isinstance(other, self.__class__): raise TypeError( "unsupported operand type(s) for 'in': '%s' and '%s'" % ( type(other).__qualname__, self.__class__.__qualname__)) + if other._value_ == 0 or self._value_ == 0: + return False return other._value_ & self._value_ == other._value_ def __iter__(self): - members, extra_flags = _decompose(self.__class__, self.value) - return (m for m in members if m._value_ != 0) + """ + Returns flags in definition order. + """ + yield from self._iter_member_(self._value_) + + def __len__(self): + return self._value_.bit_count() def __repr__(self): - cls = self.__class__ - if self._name_ is not None: - return '<%s.%s: %r>' % (cls.__name__, self._name_, self._value_) - members, uncovered = _decompose(cls, self._value_) - return '<%s.%s: %r>' % ( - cls.__name__, - '|'.join([str(m._name_ or m._value_) for m in members]), - self._value_, - ) + cls_name = self.__class__.__name__ + if self._name_ is None: + return "0x%x" % (self._value_, ) + if _is_single_bit(self._value_): + return '%s.%s' % (cls_name, self._name_) + if self._boundary_ is not FlagBoundary.KEEP: + return '%s.' % cls_name + ('|%s.' % cls_name).join(self.name.split('|')) + else: + name = [] + for n in self._name_.split('|'): + if n.startswith('0'): + name.append(n) + else: + name.append('%s.%s' % (cls_name, n)) + return '|'.join(name) def __str__(self): cls = self.__class__ - if self._name_ is not None: - return '%s.%s' % (cls.__name__, self._name_) - members, uncovered = _decompose(cls, self._value_) - if len(members) == 1 and members[0]._name_ is None: - return '%s.%r' % (cls.__name__, members[0]._value_) + if self._name_ is None: + return '%s(%x)' % (cls.__name__, self._value_) else: - return '%s.%s' % ( - cls.__name__, - '|'.join([str(m._name_ or m._value_) for m in members]), - ) + return self._name_ def __bool__(self): return bool(self._value_) @@ -830,85 +1296,67 @@ def __xor__(self, other): return self.__class__(self._value_ ^ other._value_) def __invert__(self): - members, uncovered = _decompose(self.__class__, self._value_) - inverted = self.__class__(0) - for m in self.__class__: - if m not in members and not (m._value_ & self._value_): - inverted = inverted | m - return self.__class__(inverted) - - -class IntFlag(int, Flag): - """Support for integer-based Flags""" + if self._inverted_ is None: + if self._boundary_ is KEEP: + # use all bits + self._inverted_ = self.__class__(~self._value_) + else: + # calculate flags not in this member + self._inverted_ = self.__class__(self._flag_mask_ ^ self._value_) + self._inverted_._inverted_ = self + return self._inverted_ - @classmethod - def _missing_(cls, value): - if not isinstance(value, int): - raise ValueError("%r is not a valid %s" % (value, cls.__qualname__)) - new_member = cls._create_pseudo_member_(value) - return new_member - @classmethod - def _create_pseudo_member_(cls, value): - pseudo_member = cls._value2member_map_.get(value, None) - if pseudo_member is None: - need_to_create = [value] - # get unaccounted for bits - _, extra_flags = _decompose(cls, value) - # timer = 10 - while extra_flags: - # timer -= 1 - bit = _high_bit(extra_flags) - flag_value = 2 ** bit - if (flag_value not in cls._value2member_map_ and - flag_value not in need_to_create - ): - need_to_create.append(flag_value) - if extra_flags == -flag_value: - extra_flags = 0 - else: - extra_flags ^= flag_value - for value in reversed(need_to_create): - # construct singleton pseudo-members - pseudo_member = int.__new__(cls, value) - pseudo_member._name_ = None - pseudo_member._value_ = value - # use setdefault in case another thread already created a composite - # with this value - pseudo_member = cls._value2member_map_.setdefault(value, pseudo_member) - return pseudo_member +class IntFlag(int, Flag, boundary=EJECT): + """ + Support for integer-based Flags + """ def __or__(self, other): - if not isinstance(other, (self.__class__, int)): + if isinstance(other, self.__class__): + other = other._value_ + elif isinstance(other, int): + other = other + else: return NotImplemented - result = self.__class__(self._value_ | self.__class__(other)._value_) - return result + value = self._value_ + return self.__class__(value | other) def __and__(self, other): - if not isinstance(other, (self.__class__, int)): + if isinstance(other, self.__class__): + other = other._value_ + elif isinstance(other, int): + other = other + else: return NotImplemented - return self.__class__(self._value_ & self.__class__(other)._value_) + value = self._value_ + return self.__class__(value & other) def __xor__(self, other): - if not isinstance(other, (self.__class__, int)): + if isinstance(other, self.__class__): + other = other._value_ + elif isinstance(other, int): + other = other + else: return NotImplemented - return self.__class__(self._value_ ^ self.__class__(other)._value_) + value = self._value_ + return self.__class__(value ^ other) __ror__ = __or__ __rand__ = __and__ __rxor__ = __xor__ - - def __invert__(self): - result = self.__class__(~self._value_) - return result - + __invert__ = Flag.__invert__ def _high_bit(value): - """returns index of highest bit, or -1 if value is zero or negative""" + """ + returns index of highest bit, or -1 if value is zero or negative + """ return value.bit_length() - 1 def unique(enumeration): - """Class decorator for enumerations ensuring unique member values.""" + """ + Class decorator for enumerations ensuring unique member values. + """ duplicates = [] for name, member in enumeration.__members__.items(): if name != member.name: @@ -920,29 +1368,348 @@ def unique(enumeration): (enumeration, alias_details)) return enumeration -def _decompose(flag, value): - """Extract all members from the value.""" - # _decompose is only called if the value is not named - not_covered = value - negative = value < 0 - members = [] - for member in flag: - member_value = member.value - if member_value and member_value & value == member_value: - members.append(member) - not_covered &= ~member_value - if not negative: - tmp = not_covered - while tmp: - flag_value = 2 ** _high_bit(tmp) - if flag_value in flag._value2member_map_: - members.append(flag._value2member_map_[flag_value]) - not_covered &= ~flag_value - tmp &= ~flag_value - if not members and value in flag._value2member_map_: - members.append(flag._value2member_map_[value]) - members.sort(key=lambda m: m._value_, reverse=True) - if len(members) > 1 and members[0].value == value: - # we have the breakdown, don't need the value member itself - members.pop(0) - return members, not_covered +def _power_of_two(value): + if value < 1: + return False + return value == 2 ** _high_bit(value) + +def global_enum_repr(self): + return '%s.%s' % (self.__class__.__module__, self._name_) + +def global_flag_repr(self): + module = self.__class__.__module__ + cls_name = self.__class__.__name__ + if self._name_ is None: + return "%x" % (module, cls_name, self._value_) + if _is_single_bit(self): + return '%s.%s' % (module, self._name_) + if self._boundary_ is not FlagBoundary.KEEP: + return module + module.join(self.name.split('|')) + else: + name = [] + for n in self._name_.split('|'): + if n.startswith('0'): + name.append(n) + else: + name.append('%s.%s' % (module, n)) + return '|'.join(name) + + +def global_enum(cls): + """ + decorator that makes the repr() of an enum member reference its module + instead of its class; also exports all members to the enum's module's + global namespace + """ + if issubclass(cls, Flag): + cls.__repr__ = global_flag_repr + else: + cls.__repr__ = global_enum_repr + sys.modules[cls.__module__].__dict__.update(cls.__members__) + return cls + +def _simple_enum(etype=Enum, *, boundary=None, use_args=None): + """ + Class decorator that converts a normal class into an :class:`Enum`. No + safety checks are done, and some advanced behavior (such as + :func:`__init_subclass__`) is not available. Enum creation can be faster + using :func:`simple_enum`. + + >>> from enum import Enum, _simple_enum + >>> @_simple_enum(Enum) + ... class Color: + ... RED = auto() + ... GREEN = auto() + ... BLUE = auto() + >>> Color + + """ + def convert_class(cls): + nonlocal use_args + cls_name = cls.__name__ + if use_args is None: + use_args = etype._use_args_ + __new__ = cls.__dict__.get('__new__') + if __new__ is not None: + new_member = __new__.__func__ + else: + new_member = etype._member_type_.__new__ + attrs = {} + body = {} + if __new__ is not None: + body['__new_member__'] = new_member + body['_new_member_'] = new_member + body['_use_args_'] = use_args + body['_generate_next_value_'] = gnv = etype._generate_next_value_ + body['_member_names_'] = member_names = [] + body['_member_map_'] = member_map = {} + body['_value2member_map_'] = value2member_map = {} + body['_unhashable_values_'] = [] + body['_member_type_'] = member_type = etype._member_type_ + if issubclass(etype, Flag): + body['_boundary_'] = boundary or etype._boundary_ + body['_flag_mask_'] = None + body['_all_bits_'] = None + body['_inverted_'] = None + for name, obj in cls.__dict__.items(): + if name in ('__dict__', '__weakref__'): + continue + if _is_dunder(name) or _is_private(cls_name, name) or _is_sunder(name) or _is_descriptor(obj): + body[name] = obj + else: + attrs[name] = obj + if cls.__dict__.get('__doc__') is None: + body['__doc__'] = 'An enumeration.' + # + # double check that repr and friends are not the mixin's or various + # things break (such as pickle) + # however, if the method is defined in the Enum itself, don't replace + # it + enum_class = type(cls_name, (etype, ), body, boundary=boundary, _simple=True) + for name in ('__repr__', '__str__', '__format__', '__reduce_ex__'): + if name in body: + continue + class_method = getattr(enum_class, name) + obj_method = getattr(member_type, name, None) + enum_method = getattr(etype, name, None) + if obj_method is not None and obj_method is class_method: + setattr(enum_class, name, enum_method) + gnv_last_values = [] + if issubclass(enum_class, Flag): + # Flag / IntFlag + single_bits = multi_bits = 0 + for name, value in attrs.items(): + if isinstance(value, auto) and auto.value is _auto_null: + value = gnv(name, 1, len(member_names), gnv_last_values) + if value in value2member_map: + # an alias to an existing member + redirect = property() + redirect.__set_name__(enum_class, name) + setattr(enum_class, name, redirect) + member_map[name] = value2member_map[value] + else: + # create the member + if use_args: + if not isinstance(value, tuple): + value = (value, ) + member = new_member(enum_class, *value) + value = value[0] + else: + member = new_member(enum_class) + if __new__ is None: + member._value_ = value + member._name_ = name + member.__objclass__ = enum_class + member.__init__(value) + redirect = property() + redirect.__set_name__(enum_class, name) + setattr(enum_class, name, redirect) + member_map[name] = member + member._sort_order_ = len(member_names) + value2member_map[value] = member + if _is_single_bit(value): + # not a multi-bit alias, record in _member_names_ and _flag_mask_ + member_names.append(name) + single_bits |= value + else: + multi_bits |= value + gnv_last_values.append(value) + enum_class._flag_mask_ = single_bits + enum_class._all_bits_ = 2 ** ((single_bits|multi_bits).bit_length()) - 1 + # set correct __iter__ + member_list = [m._value_ for m in enum_class] + if member_list != sorted(member_list): + enum_class._iter_member_ = enum_class._iter_member_by_def_ + else: + # Enum / IntEnum / StrEnum + for name, value in attrs.items(): + if isinstance(value, auto): + if value.value is _auto_null: + value.value = gnv(name, 1, len(member_names), gnv_last_values) + value = value.value + if value in value2member_map: + # an alias to an existing member + redirect = property() + redirect.__set_name__(enum_class, name) + setattr(enum_class, name, redirect) + member_map[name] = value2member_map[value] + else: + # create the member + if use_args: + if not isinstance(value, tuple): + value = (value, ) + member = new_member(enum_class, *value) + value = value[0] + else: + member = new_member(enum_class) + if __new__ is None: + member._value_ = value + member._name_ = name + member.__objclass__ = enum_class + member.__init__(value) + member._sort_order_ = len(member_names) + redirect = property() + redirect.__set_name__(enum_class, name) + setattr(enum_class, name, redirect) + member_map[name] = member + value2member_map[value] = member + member_names.append(name) + gnv_last_values.append(value) + if '__new__' in body: + enum_class.__new_member__ = enum_class.__new__ + enum_class.__new__ = Enum.__new__ + return enum_class + return convert_class + +def _test_simple_enum(checked_enum, simple_enum): + """ + A function that can be used to test an enum created with :func:`_simple_enum` + against the version created by subclassing :class:`Enum`:: + + >>> from enum import Enum, _simple_enum, _test_simple_enum + >>> @_simple_enum(Enum) + ... class Color: + ... RED = auto() + ... GREEN = auto() + ... BLUE = auto() + >>> class CheckedColor(Enum): + ... RED = auto() + ... GREEN = auto() + ... BLUE = auto() + >>> _test_simple_enum(CheckedColor, Color) + + If differences are found, a :exc:`TypeError` is raised. + """ + failed = [] + if checked_enum.__dict__ != simple_enum.__dict__: + checked_dict = checked_enum.__dict__ + checked_keys = list(checked_dict.keys()) + simple_dict = simple_enum.__dict__ + simple_keys = list(simple_dict.keys()) + member_names = set( + list(checked_enum._member_map_.keys()) + + list(simple_enum._member_map_.keys()) + ) + for key in set(checked_keys + simple_keys): + if key in ('__module__', '_member_map_', '_value2member_map_'): + # keys known to be different + continue + elif key in member_names: + # members are checked below + continue + elif key not in simple_keys: + failed.append("missing key: %r" % (key, )) + elif key not in checked_keys: + failed.append("extra key: %r" % (key, )) + else: + checked_value = checked_dict[key] + simple_value = simple_dict[key] + if callable(checked_value): + continue + if key == '__doc__': + # remove all spaces/tabs + compressed_checked_value = checked_value.replace(' ','').replace('\t','') + compressed_simple_value = simple_value.replace(' ','').replace('\t','') + if compressed_checked_value != compressed_simple_value: + failed.append("%r:\n %s\n %s" % ( + key, + "checked -> %r" % (checked_value, ), + "simple -> %r" % (simple_value, ), + )) + elif checked_value != simple_value: + failed.append("%r:\n %s\n %s" % ( + key, + "checked -> %r" % (checked_value, ), + "simple -> %r" % (simple_value, ), + )) + failed.sort() + for name in member_names: + failed_member = [] + if name not in simple_keys: + failed.append('missing member from simple enum: %r' % name) + elif name not in checked_keys: + failed.append('extra member in simple enum: %r' % name) + else: + checked_member_dict = checked_enum[name].__dict__ + checked_member_keys = list(checked_member_dict.keys()) + simple_member_dict = simple_enum[name].__dict__ + simple_member_keys = list(simple_member_dict.keys()) + for key in set(checked_member_keys + simple_member_keys): + if key in ('__module__', '__objclass__', '_inverted_'): + # keys known to be different or absent + continue + elif key not in simple_member_keys: + failed_member.append("missing key %r not in the simple enum member %r" % (key, name)) + elif key not in checked_member_keys: + failed_member.append("extra key %r in simple enum member %r" % (key, name)) + else: + checked_value = checked_member_dict[key] + simple_value = simple_member_dict[key] + if checked_value != simple_value: + failed_member.append("%r:\n %s\n %s" % ( + key, + "checked member -> %r" % (checked_value, ), + "simple member -> %r" % (simple_value, ), + )) + if failed_member: + failed.append('%r member mismatch:\n %s' % ( + name, '\n '.join(failed_member), + )) + for method in ( + '__str__', '__repr__', '__reduce_ex__', '__format__', + '__getnewargs_ex__', '__getnewargs__', '__reduce_ex__', '__reduce__' + ): + if method in simple_keys and method in checked_keys: + # cannot compare functions, and it exists in both, so we're good + continue + elif method not in simple_keys and method not in checked_keys: + # method is inherited -- check it out + checked_method = getattr(checked_enum, method, None) + simple_method = getattr(simple_enum, method, None) + if hasattr(checked_method, '__func__'): + checked_method = checked_method.__func__ + simple_method = simple_method.__func__ + if checked_method != simple_method: + failed.append("%r: %-30s %s" % ( + method, + "checked -> %r" % (checked_method, ), + "simple -> %r" % (simple_method, ), + )) + else: + # if the method existed in only one of the enums, it will have been caught + # in the first checks above + pass + if failed: + raise TypeError('enum mismatch:\n %s' % '\n '.join(failed)) + +def _old_convert_(etype, name, module, filter, source=None, *, boundary=None): + """ + Create a new Enum subclass that replaces a collection of global constants + """ + # convert all constants from source (or module) that pass filter() to + # a new Enum called name, and export the enum and its members back to + # module; + # also, replace the __reduce_ex__ method so unpickling works in + # previous Python versions + module_globals = sys.modules[module].__dict__ + if source: + source = source.__dict__ + else: + source = module_globals + # _value2member_map_ is populated in the same order every time + # for a consistent reverse mapping of number to name when there + # are multiple names for the same number. + members = [ + (name, value) + for name, value in source.items() + if filter(name)] + try: + # sort by value + members.sort(key=lambda t: (t[1], t[0])) + except TypeError: + # unless some values aren't comparable, in which case sort by name + members.sort(key=lambda t: t[0]) + cls = etype(name, members, module=module, boundary=boundary or KEEP) + cls.__reduce_ex__ = _reduce_ex_by_name + cls.__repr__ = global_enum_repr + return cls diff --git a/Lib/filecmp.py b/Lib/filecmp.py index 7a4da6beb50500..7c47eb022cc8c0 100644 --- a/Lib/filecmp.py +++ b/Lib/filecmp.py @@ -115,7 +115,9 @@ class dircmp: same_files: list of identical files. diff_files: list of filenames which differ. funny_files: list of files which could not be compared. - subdirs: a dictionary of dircmp objects, keyed by names in common_dirs. + subdirs: a dictionary of dircmp instances (or MyDirCmp instances if this + object is of type MyDirCmp, a subclass of dircmp), keyed by names + in common_dirs. """ def __init__(self, a, b, ignore=None, hide=None): # Initialize @@ -185,14 +187,15 @@ def phase3(self): # Find out differences between common files self.same_files, self.diff_files, self.funny_files = xx def phase4(self): # Find out differences between common subdirectories - # A new dircmp object is created for each common subdirectory, + # A new dircmp (or MyDirCmp if dircmp was subclassed) object is created + # for each common subdirectory, # these are stored in a dictionary indexed by filename. # The hide and ignore properties are inherited from the parent self.subdirs = {} for x in self.common_dirs: a_x = os.path.join(self.left, x) b_x = os.path.join(self.right, x) - self.subdirs[x] = dircmp(a_x, b_x, self.ignore, self.hide) + self.subdirs[x] = self.__class__(a_x, b_x, self.ignore, self.hide) def phase4_closure(self): # Recursively call phase4() on subdirectories self.phase4() diff --git a/Lib/fileinput.py b/Lib/fileinput.py index 0c31f93ed8f2e7..35347185da0484 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -3,7 +3,7 @@ Typical use is: import fileinput - for line in fileinput.input(): + for line in fileinput.input(encoding="utf-8"): process(line) This iterates over the lines of all files listed in sys.argv[1:], @@ -63,15 +63,9 @@ deleted when the output file is closed. In-place filtering is disabled when standard input is read. XXX The current implementation does not work for MS-DOS 8+3 filesystems. - -XXX Possible additions: - -- optional getopt argument processing -- isatty() -- read(), read(size), even readlines() - """ +import io import sys, os from types import GenericAlias @@ -81,7 +75,8 @@ _state = None -def input(files=None, inplace=False, backup="", *, mode="r", openhook=None): +def input(files=None, inplace=False, backup="", *, mode="r", openhook=None, + encoding=None, errors=None): """Return an instance of the FileInput class, which can be iterated. The parameters are passed to the constructor of the FileInput class. @@ -91,7 +86,8 @@ def input(files=None, inplace=False, backup="", *, mode="r", openhook=None): global _state if _state and _state._file: raise RuntimeError("input() already active") - _state = FileInput(files, inplace, backup, mode=mode, openhook=openhook) + _state = FileInput(files, inplace, backup, mode=mode, openhook=openhook, + encoding=encoding, errors=errors) return _state def close(): @@ -186,7 +182,7 @@ class FileInput: """ def __init__(self, files=None, inplace=False, backup="", *, - mode="r", openhook=None): + mode="r", openhook=None, encoding=None, errors=None): if isinstance(files, str): files = (files,) elif isinstance(files, os.PathLike): @@ -209,6 +205,17 @@ def __init__(self, files=None, inplace=False, backup="", *, self._file = None self._isstdin = False self._backupfilename = None + self._encoding = encoding + self._errors = errors + + # We can not use io.text_encoding() here because old openhook doesn't + # take encoding parameter. + if (sys.flags.warn_default_encoding and + "b" not in mode and encoding is None and openhook is None): + import warnings + warnings.warn("'encoding' argument not specified.", + EncodingWarning, 2) + # restrict mode argument to reading modes if mode not in ('r', 'rU', 'U', 'rb'): raise ValueError("FileInput opening mode must be one of " @@ -324,6 +331,13 @@ def _readline(self): self._file = None self._isstdin = False self._backupfilename = 0 + + # EncodingWarning is emitted in __init__() already + if "b" not in self._mode: + encoding = self._encoding or "locale" + else: + encoding = None + if self._filename == '-': self._filename = '' if 'b' in self._mode: @@ -341,18 +355,18 @@ def _readline(self): pass # The next few lines may raise OSError os.rename(self._filename, self._backupfilename) - self._file = open(self._backupfilename, self._mode) + self._file = open(self._backupfilename, self._mode, encoding=encoding) try: perm = os.fstat(self._file.fileno()).st_mode except OSError: - self._output = open(self._filename, self._write_mode) + self._output = open(self._filename, self._write_mode, encoding=encoding) else: mode = os.O_CREAT | os.O_WRONLY | os.O_TRUNC if hasattr(os, 'O_BINARY'): mode |= os.O_BINARY fd = os.open(self._filename, mode, perm) - self._output = os.fdopen(fd, self._write_mode) + self._output = os.fdopen(fd, self._write_mode, encoding=encoding) try: os.chmod(self._filename, perm) except OSError: @@ -362,9 +376,15 @@ def _readline(self): else: # This may raise OSError if self._openhook: - self._file = self._openhook(self._filename, self._mode) + # Custom hooks made previous to Python 3.10 didn't have + # encoding argument + if self._encoding is None: + self._file = self._openhook(self._filename, self._mode) + else: + self._file = self._openhook( + self._filename, self._mode, encoding=self._encoding, errors=self._errors) else: - self._file = open(self._filename, self._mode) + self._file = open(self._filename, self._mode, encoding=encoding, errors=self._errors) self._readline = self._file.readline # hide FileInput._readline return self._readline() @@ -395,16 +415,23 @@ def isstdin(self): __class_getitem__ = classmethod(GenericAlias) -def hook_compressed(filename, mode): +def hook_compressed(filename, mode, *, encoding=None, errors=None): + if encoding is None: # EncodingWarning is emitted in FileInput() already. + encoding = "locale" ext = os.path.splitext(filename)[1] if ext == '.gz': import gzip - return gzip.open(filename, mode) + stream = gzip.open(filename, mode) elif ext == '.bz2': import bz2 - return bz2.BZ2File(filename, mode) + stream = bz2.BZ2File(filename, mode) else: - return open(filename, mode) + return open(filename, mode, encoding=encoding, errors=errors) + + # gzip and bz2 are binary mode by default. + if "b" not in mode: + stream = io.TextIOWrapper(stream, encoding=encoding, errors=errors) + return stream def hook_encoded(encoding, errors=None): diff --git a/Lib/fnmatch.py b/Lib/fnmatch.py index 0eb1802bdb53c5..7c52c23067d40f 100644 --- a/Lib/fnmatch.py +++ b/Lib/fnmatch.py @@ -52,7 +52,7 @@ def _compile_pattern(pat): return re.compile(res).match def filter(names, pat): - """Return the subset of the list NAMES that match PAT.""" + """Construct a list from those elements of the iterable NAMES that match PAT.""" result = [] pat = os.path.normcase(pat) match = _compile_pattern(pat) diff --git a/Lib/formatter.py b/Lib/formatter.py deleted file mode 100644 index e2394de8c29195..00000000000000 --- a/Lib/formatter.py +++ /dev/null @@ -1,452 +0,0 @@ -"""Generic output formatting. - -Formatter objects transform an abstract flow of formatting events into -specific output events on writer objects. Formatters manage several stack -structures to allow various properties of a writer object to be changed and -restored; writers need not be able to handle relative changes nor any sort -of ``change back'' operation. Specific writer properties which may be -controlled via formatter objects are horizontal alignment, font, and left -margin indentations. A mechanism is provided which supports providing -arbitrary, non-exclusive style settings to a writer as well. Additional -interfaces facilitate formatting events which are not reversible, such as -paragraph separation. - -Writer objects encapsulate device interfaces. Abstract devices, such as -file formats, are supported as well as physical devices. The provided -implementations all work with abstract devices. The interface makes -available mechanisms for setting the properties which formatter objects -manage and inserting data into the output. -""" - -import sys -import warnings -warnings.warn('the formatter module is deprecated', DeprecationWarning, - stacklevel=2) - - -AS_IS = None - - -class NullFormatter: - """A formatter which does nothing. - - If the writer parameter is omitted, a NullWriter instance is created. - No methods of the writer are called by NullFormatter instances. - - Implementations should inherit from this class if implementing a writer - interface but don't need to inherit any implementation. - - """ - - def __init__(self, writer=None): - if writer is None: - writer = NullWriter() - self.writer = writer - def end_paragraph(self, blankline): pass - def add_line_break(self): pass - def add_hor_rule(self, *args, **kw): pass - def add_label_data(self, format, counter, blankline=None): pass - def add_flowing_data(self, data): pass - def add_literal_data(self, data): pass - def flush_softspace(self): pass - def push_alignment(self, align): pass - def pop_alignment(self): pass - def push_font(self, x): pass - def pop_font(self): pass - def push_margin(self, margin): pass - def pop_margin(self): pass - def set_spacing(self, spacing): pass - def push_style(self, *styles): pass - def pop_style(self, n=1): pass - def assert_line_data(self, flag=1): pass - - -class AbstractFormatter: - """The standard formatter. - - This implementation has demonstrated wide applicability to many writers, - and may be used directly in most circumstances. It has been used to - implement a full-featured World Wide Web browser. - - """ - - # Space handling policy: blank spaces at the boundary between elements - # are handled by the outermost context. "Literal" data is not checked - # to determine context, so spaces in literal data are handled directly - # in all circumstances. - - def __init__(self, writer): - self.writer = writer # Output device - self.align = None # Current alignment - self.align_stack = [] # Alignment stack - self.font_stack = [] # Font state - self.margin_stack = [] # Margin state - self.spacing = None # Vertical spacing state - self.style_stack = [] # Other state, e.g. color - self.nospace = 1 # Should leading space be suppressed - self.softspace = 0 # Should a space be inserted - self.para_end = 1 # Just ended a paragraph - self.parskip = 0 # Skipped space between paragraphs? - self.hard_break = 1 # Have a hard break - self.have_label = 0 - - def end_paragraph(self, blankline): - if not self.hard_break: - self.writer.send_line_break() - self.have_label = 0 - if self.parskip < blankline and not self.have_label: - self.writer.send_paragraph(blankline - self.parskip) - self.parskip = blankline - self.have_label = 0 - self.hard_break = self.nospace = self.para_end = 1 - self.softspace = 0 - - def add_line_break(self): - if not (self.hard_break or self.para_end): - self.writer.send_line_break() - self.have_label = self.parskip = 0 - self.hard_break = self.nospace = 1 - self.softspace = 0 - - def add_hor_rule(self, *args, **kw): - if not self.hard_break: - self.writer.send_line_break() - self.writer.send_hor_rule(*args, **kw) - self.hard_break = self.nospace = 1 - self.have_label = self.para_end = self.softspace = self.parskip = 0 - - def add_label_data(self, format, counter, blankline = None): - if self.have_label or not self.hard_break: - self.writer.send_line_break() - if not self.para_end: - self.writer.send_paragraph((blankline and 1) or 0) - if isinstance(format, str): - self.writer.send_label_data(self.format_counter(format, counter)) - else: - self.writer.send_label_data(format) - self.nospace = self.have_label = self.hard_break = self.para_end = 1 - self.softspace = self.parskip = 0 - - def format_counter(self, format, counter): - label = '' - for c in format: - if c == '1': - label = label + ('%d' % counter) - elif c in 'aA': - if counter > 0: - label = label + self.format_letter(c, counter) - elif c in 'iI': - if counter > 0: - label = label + self.format_roman(c, counter) - else: - label = label + c - return label - - def format_letter(self, case, counter): - label = '' - while counter > 0: - counter, x = divmod(counter-1, 26) - # This makes a strong assumption that lowercase letters - # and uppercase letters form two contiguous blocks, with - # letters in order! - s = chr(ord(case) + x) - label = s + label - return label - - def format_roman(self, case, counter): - ones = ['i', 'x', 'c', 'm'] - fives = ['v', 'l', 'd'] - label, index = '', 0 - # This will die of IndexError when counter is too big - while counter > 0: - counter, x = divmod(counter, 10) - if x == 9: - label = ones[index] + ones[index+1] + label - elif x == 4: - label = ones[index] + fives[index] + label - else: - if x >= 5: - s = fives[index] - x = x-5 - else: - s = '' - s = s + ones[index]*x - label = s + label - index = index + 1 - if case == 'I': - return label.upper() - return label - - def add_flowing_data(self, data): - if not data: return - prespace = data[:1].isspace() - postspace = data[-1:].isspace() - data = " ".join(data.split()) - if self.nospace and not data: - return - elif prespace or self.softspace: - if not data: - if not self.nospace: - self.softspace = 1 - self.parskip = 0 - return - if not self.nospace: - data = ' ' + data - self.hard_break = self.nospace = self.para_end = \ - self.parskip = self.have_label = 0 - self.softspace = postspace - self.writer.send_flowing_data(data) - - def add_literal_data(self, data): - if not data: return - if self.softspace: - self.writer.send_flowing_data(" ") - self.hard_break = data[-1:] == '\n' - self.nospace = self.para_end = self.softspace = \ - self.parskip = self.have_label = 0 - self.writer.send_literal_data(data) - - def flush_softspace(self): - if self.softspace: - self.hard_break = self.para_end = self.parskip = \ - self.have_label = self.softspace = 0 - self.nospace = 1 - self.writer.send_flowing_data(' ') - - def push_alignment(self, align): - if align and align != self.align: - self.writer.new_alignment(align) - self.align = align - self.align_stack.append(align) - else: - self.align_stack.append(self.align) - - def pop_alignment(self): - if self.align_stack: - del self.align_stack[-1] - if self.align_stack: - self.align = align = self.align_stack[-1] - self.writer.new_alignment(align) - else: - self.align = None - self.writer.new_alignment(None) - - def push_font(self, font): - size, i, b, tt = font - if self.softspace: - self.hard_break = self.para_end = self.softspace = 0 - self.nospace = 1 - self.writer.send_flowing_data(' ') - if self.font_stack: - csize, ci, cb, ctt = self.font_stack[-1] - if size is AS_IS: size = csize - if i is AS_IS: i = ci - if b is AS_IS: b = cb - if tt is AS_IS: tt = ctt - font = (size, i, b, tt) - self.font_stack.append(font) - self.writer.new_font(font) - - def pop_font(self): - if self.font_stack: - del self.font_stack[-1] - if self.font_stack: - font = self.font_stack[-1] - else: - font = None - self.writer.new_font(font) - - def push_margin(self, margin): - self.margin_stack.append(margin) - fstack = [m for m in self.margin_stack if m] - if not margin and fstack: - margin = fstack[-1] - self.writer.new_margin(margin, len(fstack)) - - def pop_margin(self): - if self.margin_stack: - del self.margin_stack[-1] - fstack = [m for m in self.margin_stack if m] - if fstack: - margin = fstack[-1] - else: - margin = None - self.writer.new_margin(margin, len(fstack)) - - def set_spacing(self, spacing): - self.spacing = spacing - self.writer.new_spacing(spacing) - - def push_style(self, *styles): - if self.softspace: - self.hard_break = self.para_end = self.softspace = 0 - self.nospace = 1 - self.writer.send_flowing_data(' ') - for style in styles: - self.style_stack.append(style) - self.writer.new_styles(tuple(self.style_stack)) - - def pop_style(self, n=1): - del self.style_stack[-n:] - self.writer.new_styles(tuple(self.style_stack)) - - def assert_line_data(self, flag=1): - self.nospace = self.hard_break = not flag - self.para_end = self.parskip = self.have_label = 0 - - -class NullWriter: - """Minimal writer interface to use in testing & inheritance. - - A writer which only provides the interface definition; no actions are - taken on any methods. This should be the base class for all writers - which do not need to inherit any implementation methods. - - """ - def __init__(self): pass - def flush(self): pass - def new_alignment(self, align): pass - def new_font(self, font): pass - def new_margin(self, margin, level): pass - def new_spacing(self, spacing): pass - def new_styles(self, styles): pass - def send_paragraph(self, blankline): pass - def send_line_break(self): pass - def send_hor_rule(self, *args, **kw): pass - def send_label_data(self, data): pass - def send_flowing_data(self, data): pass - def send_literal_data(self, data): pass - - -class AbstractWriter(NullWriter): - """A writer which can be used in debugging formatters, but not much else. - - Each method simply announces itself by printing its name and - arguments on standard output. - - """ - - def new_alignment(self, align): - print("new_alignment(%r)" % (align,)) - - def new_font(self, font): - print("new_font(%r)" % (font,)) - - def new_margin(self, margin, level): - print("new_margin(%r, %d)" % (margin, level)) - - def new_spacing(self, spacing): - print("new_spacing(%r)" % (spacing,)) - - def new_styles(self, styles): - print("new_styles(%r)" % (styles,)) - - def send_paragraph(self, blankline): - print("send_paragraph(%r)" % (blankline,)) - - def send_line_break(self): - print("send_line_break()") - - def send_hor_rule(self, *args, **kw): - print("send_hor_rule()") - - def send_label_data(self, data): - print("send_label_data(%r)" % (data,)) - - def send_flowing_data(self, data): - print("send_flowing_data(%r)" % (data,)) - - def send_literal_data(self, data): - print("send_literal_data(%r)" % (data,)) - - -class DumbWriter(NullWriter): - """Simple writer class which writes output on the file object passed in - as the file parameter or, if file is omitted, on standard output. The - output is simply word-wrapped to the number of columns specified by - the maxcol parameter. This class is suitable for reflowing a sequence - of paragraphs. - - """ - - def __init__(self, file=None, maxcol=72): - self.file = file or sys.stdout - self.maxcol = maxcol - NullWriter.__init__(self) - self.reset() - - def reset(self): - self.col = 0 - self.atbreak = 0 - - def send_paragraph(self, blankline): - self.file.write('\n'*blankline) - self.col = 0 - self.atbreak = 0 - - def send_line_break(self): - self.file.write('\n') - self.col = 0 - self.atbreak = 0 - - def send_hor_rule(self, *args, **kw): - self.file.write('\n') - self.file.write('-'*self.maxcol) - self.file.write('\n') - self.col = 0 - self.atbreak = 0 - - def send_literal_data(self, data): - self.file.write(data) - i = data.rfind('\n') - if i >= 0: - self.col = 0 - data = data[i+1:] - data = data.expandtabs() - self.col = self.col + len(data) - self.atbreak = 0 - - def send_flowing_data(self, data): - if not data: return - atbreak = self.atbreak or data[0].isspace() - col = self.col - maxcol = self.maxcol - write = self.file.write - for word in data.split(): - if atbreak: - if col + len(word) >= maxcol: - write('\n') - col = 0 - else: - write(' ') - col = col + 1 - write(word) - col = col + len(word) - atbreak = 1 - self.col = col - self.atbreak = data[-1].isspace() - - -def test(file = None): - w = DumbWriter() - f = AbstractFormatter(w) - if file is not None: - fp = open(file) - elif sys.argv[1:]: - fp = open(sys.argv[1]) - else: - fp = sys.stdin - try: - for line in fp: - if line == '\n': - f.end_paragraph(1) - else: - f.add_flowing_data(line) - finally: - if fp is not sys.stdin: - fp.close() - f.end_paragraph(0) - - -if __name__ == '__main__': - test() diff --git a/Lib/fractions.py b/Lib/fractions.py index de3e23b759227c..96047beb4546a5 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -380,32 +380,139 @@ def reverse(b, a): return forward, reverse + # Rational arithmetic algorithms: Knuth, TAOCP, Volume 2, 4.5.1. + # + # Assume input fractions a and b are normalized. + # + # 1) Consider addition/subtraction. + # + # Let g = gcd(da, db). Then + # + # na nb na*db ± nb*da + # a ± b == -- ± -- == ------------- == + # da db da*db + # + # na*(db//g) ± nb*(da//g) t + # == ----------------------- == - + # (da*db)//g d + # + # Now, if g > 1, we're working with smaller integers. + # + # Note, that t, (da//g) and (db//g) are pairwise coprime. + # + # Indeed, (da//g) and (db//g) share no common factors (they were + # removed) and da is coprime with na (since input fractions are + # normalized), hence (da//g) and na are coprime. By symmetry, + # (db//g) and nb are coprime too. Then, + # + # gcd(t, da//g) == gcd(na*(db//g), da//g) == 1 + # gcd(t, db//g) == gcd(nb*(da//g), db//g) == 1 + # + # Above allows us optimize reduction of the result to lowest + # terms. Indeed, + # + # g2 = gcd(t, d) == gcd(t, (da//g)*(db//g)*g) == gcd(t, g) + # + # t//g2 t//g2 + # a ± b == ----------------------- == ---------------- + # (da//g)*(db//g)*(g//g2) (da//g)*(db//g2) + # + # is a normalized fraction. This is useful because the unnormalized + # denominator d could be much larger than g. + # + # We should special-case g == 1 (and g2 == 1), since 60.8% of + # randomly-chosen integers are coprime: + # https://en.wikipedia.org/wiki/Coprime_integers#Probability_of_coprimality + # Note, that g2 == 1 always for fractions, obtained from floats: here + # g is a power of 2 and the unnormalized numerator t is an odd integer. + # + # 2) Consider multiplication + # + # Let g1 = gcd(na, db) and g2 = gcd(nb, da), then + # + # na*nb na*nb (na//g1)*(nb//g2) + # a*b == ----- == ----- == ----------------- + # da*db db*da (db//g1)*(da//g2) + # + # Note, that after divisions we're multiplying smaller integers. + # + # Also, the resulting fraction is normalized, because each of + # two factors in the numerator is coprime to each of the two factors + # in the denominator. + # + # Indeed, pick (na//g1). It's coprime with (da//g2), because input + # fractions are normalized. It's also coprime with (db//g1), because + # common factors are removed by g1 == gcd(na, db). + # + # As for addition/subtraction, we should special-case g1 == 1 + # and g2 == 1 for same reason. That happens also for multiplying + # rationals, obtained from floats. + def _add(a, b): """a + b""" - da, db = a.denominator, b.denominator - return Fraction(a.numerator * db + b.numerator * da, - da * db) + na, da = a.numerator, a.denominator + nb, db = b.numerator, b.denominator + g = math.gcd(da, db) + if g == 1: + return Fraction(na * db + da * nb, da * db, _normalize=False) + s = da // g + t = na * (db // g) + nb * s + g2 = math.gcd(t, g) + if g2 == 1: + return Fraction(t, s * db, _normalize=False) + return Fraction(t // g2, s * (db // g2), _normalize=False) __add__, __radd__ = _operator_fallbacks(_add, operator.add) def _sub(a, b): """a - b""" - da, db = a.denominator, b.denominator - return Fraction(a.numerator * db - b.numerator * da, - da * db) + na, da = a.numerator, a.denominator + nb, db = b.numerator, b.denominator + g = math.gcd(da, db) + if g == 1: + return Fraction(na * db - da * nb, da * db, _normalize=False) + s = da // g + t = na * (db // g) - nb * s + g2 = math.gcd(t, g) + if g2 == 1: + return Fraction(t, s * db, _normalize=False) + return Fraction(t // g2, s * (db // g2), _normalize=False) __sub__, __rsub__ = _operator_fallbacks(_sub, operator.sub) def _mul(a, b): """a * b""" - return Fraction(a.numerator * b.numerator, a.denominator * b.denominator) + na, da = a.numerator, a.denominator + nb, db = b.numerator, b.denominator + g1 = math.gcd(na, db) + if g1 > 1: + na //= g1 + db //= g1 + g2 = math.gcd(nb, da) + if g2 > 1: + nb //= g2 + da //= g2 + return Fraction(na * nb, db * da, _normalize=False) __mul__, __rmul__ = _operator_fallbacks(_mul, operator.mul) def _div(a, b): """a / b""" - return Fraction(a.numerator * b.denominator, - a.denominator * b.numerator) + # Same as _mul(), with inversed b. + na, da = a.numerator, a.denominator + nb, db = b.numerator, b.denominator + g1 = math.gcd(na, nb) + if g1 > 1: + na //= g1 + nb //= g1 + g2 = math.gcd(db, da) + if g2 > 1: + da //= g2 + db //= g2 + n, d = na * db, nb * da + if d < 0: + n, d = -n, -d + return Fraction(n, d, _normalize=False) __truediv__, __rtruediv__ = _operator_fallbacks(_div, operator.truediv) diff --git a/Lib/ftplib.py b/Lib/ftplib.py index 1f760ed1ce0bf0..7c5a50715f6dc6 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -102,7 +102,9 @@ class FTP: sock = None file = None welcome = None - passiveserver = 1 + passiveserver = True + # Disables https://bugs.python.org/issue43285 security if set to True. + trust_server_pasv_ipv4_address = False def __init__(self, host='', user='', passwd='', acct='', timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None, *, @@ -320,8 +322,13 @@ def makeport(self): return sock def makepasv(self): + """Internal: Does the PASV or EPSV handshake -> (address, port)""" if self.af == socket.AF_INET: - host, port = parse227(self.sendcmd('PASV')) + untrusted_host, port = parse227(self.sendcmd('PASV')) + if self.trust_server_pasv_ipv4_address: + host = untrusted_host + else: + host = self.sock.getpeername()[0] else: host, port = parse229(self.sendcmd('EPSV'), self.sock.getpeername()) return host, port diff --git a/Lib/glob.py b/Lib/glob.py index a491363f3f9395..a6cff873508266 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -34,6 +34,7 @@ def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False): zero or more directories and subdirectories. """ sys.audit("glob.glob", pathname, recursive) + sys.audit("glob.glob/2", pathname, recursive, root_dir, dir_fd) if root_dir is not None: root_dir = os.fspath(root_dir) else: diff --git a/Lib/gzip.py b/Lib/gzip.py index e422773b3edfb7..9a4e0f9c00c580 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -62,6 +62,7 @@ def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST, raise TypeError("filename must be a str or bytes object, or a file") if "t" in mode: + encoding = io.text_encoding(encoding) return io.TextIOWrapper(binary_file, encoding, errors, newline) else: return binary_file @@ -397,6 +398,10 @@ def readline(self, size=-1): self._check_not_closed() return self._buffer.readline(size) + def __iter__(self): + self._check_not_closed() + return self._buffer.__iter__() + class _GzipReader(_compression.DecompressReader): def __init__(self, fp): @@ -583,8 +588,7 @@ def main(): g = sys.stdout.buffer else: if arg[-3:] != ".gz": - print("filename doesn't end in .gz:", repr(arg)) - continue + sys.exit(f"filename doesn't end in .gz: {arg!r}") f = open(arg, "rb") g = builtins.open(arg[:-3], "wb") else: @@ -596,7 +600,7 @@ def main(): f = builtins.open(arg, "rb") g = open(arg + ".gz", "wb") while True: - chunk = f.read(1024) + chunk = f.read(io.DEFAULT_BUFFER_SIZE) if not chunk: break g.write(chunk) diff --git a/Lib/hashlib.py b/Lib/hashlib.py index 58c340d56e3ba5..21a73f3bf6cb62 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -173,6 +173,7 @@ def __hash_new(name, data=b'', **kwargs): algorithms_available = algorithms_available.union( _hashlib.openssl_md_meth_names) except ImportError: + _hashlib = None new = __py_new __get_hash = __get_builtin_constructor @@ -180,6 +181,7 @@ def __hash_new(name, data=b'', **kwargs): # OpenSSL's PKCS5_PBKDF2_HMAC requires OpenSSL 1.0+ with HMAC and SHA from _hashlib import pbkdf2_hmac except ImportError: + from warnings import warn as _warn _trans_5C = bytes((x ^ 0x5C) for x in range(256)) _trans_36 = bytes((x ^ 0x36) for x in range(256)) @@ -190,6 +192,11 @@ def pbkdf2_hmac(hash_name, password, salt, iterations, dklen=None): as OpenSSL's PKCS5_PBKDF2_HMAC for short passwords and much faster for long passwords. """ + _warn( + "Python implementation of pbkdf2_hmac() is deprecated.", + category=DeprecationWarning, + stacklevel=2 + ) if not isinstance(hash_name, str): raise TypeError(hash_name) diff --git a/Lib/hmac.py b/Lib/hmac.py index 180bc378b52d62..8b4f920db954ca 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -8,11 +8,12 @@ import _hashlib as _hashopenssl except ImportError: _hashopenssl = None - _openssl_md_meths = None + _functype = None from _operator import _compare_digest as compare_digest else: - _openssl_md_meths = frozenset(_hashopenssl.openssl_md_meth_names) compare_digest = _hashopenssl.compare_digest + _functype = type(_hashopenssl.openssl_sha256) # builtin type + import hashlib as _hashlib trans_5C = bytes((x ^ 0x5C) for x in range(256)) @@ -23,7 +24,6 @@ digest_size = None - class HMAC: """RFC 2104 HMAC class. Also complies with RFC 4231. @@ -32,7 +32,7 @@ class HMAC: blocksize = 64 # 512-bit HMAC; can be changed in subclasses. __slots__ = ( - "_digest_cons", "_inner", "_outer", "block_size", "digest_size" + "_hmac", "_inner", "_outer", "block_size", "digest_size" ) def __init__(self, key, msg=None, digestmod=''): @@ -55,15 +55,30 @@ def __init__(self, key, msg=None, digestmod=''): if not digestmod: raise TypeError("Missing required parameter 'digestmod'.") + if _hashopenssl and isinstance(digestmod, (str, _functype)): + try: + self._init_hmac(key, msg, digestmod) + except _hashopenssl.UnsupportedDigestmodError: + self._init_old(key, msg, digestmod) + else: + self._init_old(key, msg, digestmod) + + def _init_hmac(self, key, msg, digestmod): + self._hmac = _hashopenssl.hmac_new(key, msg, digestmod=digestmod) + self.digest_size = self._hmac.digest_size + self.block_size = self._hmac.block_size + + def _init_old(self, key, msg, digestmod): if callable(digestmod): - self._digest_cons = digestmod + digest_cons = digestmod elif isinstance(digestmod, str): - self._digest_cons = lambda d=b'': _hashlib.new(digestmod, d) + digest_cons = lambda d=b'': _hashlib.new(digestmod, d) else: - self._digest_cons = lambda d=b'': digestmod.new(d) + digest_cons = lambda d=b'': digestmod.new(d) - self._outer = self._digest_cons() - self._inner = self._digest_cons() + self._hmac = None + self._outer = digest_cons() + self._inner = digest_cons() self.digest_size = self._inner.digest_size if hasattr(self._inner, 'block_size'): @@ -79,13 +94,13 @@ def __init__(self, key, msg=None, digestmod=''): RuntimeWarning, 2) blocksize = self.blocksize + if len(key) > blocksize: + key = digest_cons(key).digest() + # self.blocksize is the default blocksize. self.block_size is # effective block size as well as the public API attribute. self.block_size = blocksize - if len(key) > blocksize: - key = self._digest_cons(key).digest() - key = key.ljust(blocksize, b'\0') self._outer.update(key.translate(trans_5C)) self._inner.update(key.translate(trans_36)) @@ -94,23 +109,15 @@ def __init__(self, key, msg=None, digestmod=''): @property def name(self): - return "hmac-" + self._inner.name - - @property - def digest_cons(self): - return self._digest_cons - - @property - def inner(self): - return self._inner - - @property - def outer(self): - return self._outer + if self._hmac: + return self._hmac.name + else: + return f"hmac-{self._inner.name}" def update(self, msg): """Feed data from msg into this hashing object.""" - self._inner.update(msg) + inst = self._hmac or self._inner + inst.update(msg) def copy(self): """Return a separate copy of this hashing object. @@ -119,10 +126,14 @@ def copy(self): """ # Call __new__ directly to avoid the expensive __init__. other = self.__class__.__new__(self.__class__) - other._digest_cons = self._digest_cons other.digest_size = self.digest_size - other._inner = self._inner.copy() - other._outer = self._outer.copy() + if self._hmac: + other._hmac = self._hmac.copy() + other._inner = other._outer = None + else: + other._hmac = None + other._inner = self._inner.copy() + other._outer = self._outer.copy() return other def _current(self): @@ -130,9 +141,12 @@ def _current(self): To be used only internally with digest() and hexdigest(). """ - h = self._outer.copy() - h.update(self._inner.digest()) - return h + if self._hmac: + return self._hmac + else: + h = self._outer.copy() + h.update(self._inner.digest()) + return h def digest(self): """Return the hash value of this hashing object. @@ -179,9 +193,11 @@ def digest(key, msg, digest): A hashlib constructor returning a new hash object. *OR* A module supporting PEP 247. """ - if (_hashopenssl is not None and - isinstance(digest, str) and digest in _openssl_md_meths): - return _hashopenssl.hmac_digest(key, msg, digest) + if _hashopenssl is not None and isinstance(digest, (str, _functype)): + try: + return _hashopenssl.hmac_digest(key, msg, digest) + except _hashopenssl.UnsupportedDigestmodError: + pass if callable(digest): digest_cons = digest diff --git a/Lib/html/parser.py b/Lib/html/parser.py index 60830779816a03..9e49effca1fcc5 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -46,7 +46,7 @@ |"[^"]*" # LIT-enclosed value |(?!['"])[^>\s]* # bare value ) - (?:\s*,)* # possibly followed by a comma + \s* # possibly followed by a space )?(?:\s|/(?!>))* )* )? diff --git a/Lib/http/__init__.py b/Lib/http/__init__.py index 37be765349ea00..8b980e24a5603f 100644 --- a/Lib/http/__init__.py +++ b/Lib/http/__init__.py @@ -1,8 +1,10 @@ -from enum import IntEnum +from enum import IntEnum, _simple_enum __all__ = ['HTTPStatus'] -class HTTPStatus(IntEnum): + +@_simple_enum(IntEnum) +class HTTPStatus: """HTTP status codes and reason phrases Status codes from the following RFCs are all observed: diff --git a/Lib/http/client.py b/Lib/http/client.py index 15abcfeada5915..4b1f692844474f 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -74,6 +74,7 @@ import io import re import socket +import sys import collections.abc from urllib.parse import urlsplit @@ -349,9 +350,6 @@ def begin(self): # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked" self.length = None length = self.headers.get("content-length") - - # are we using the chunked-style of transfer encoding? - tr_enc = self.headers.get("transfer-encoding") if length and not self.chunked: try: self.length = int(length) @@ -861,7 +859,7 @@ def set_tunnel(self, host, port=None, headers=None): the endpoint passed to `set_tunnel`. This done by sending an HTTP CONNECT request to the proxy server when the connection is established. - This method must be called before the HTML connection has been + This method must be called before the HTTP connection has been established. The headers argument should be a mapping of extra HTTP headers to send @@ -901,23 +899,24 @@ def set_debuglevel(self, level): self.debuglevel = level def _tunnel(self): - connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self._tunnel_host, - self._tunnel_port) - connect_bytes = connect_str.encode("ascii") - self.send(connect_bytes) + connect = b"CONNECT %s:%d HTTP/1.0\r\n" % ( + self._tunnel_host.encode("ascii"), self._tunnel_port) + headers = [connect] for header, value in self._tunnel_headers.items(): - header_str = "%s: %s\r\n" % (header, value) - header_bytes = header_str.encode("latin-1") - self.send(header_bytes) - self.send(b'\r\n') + headers.append(f"{header}: {value}\r\n".encode("latin-1")) + headers.append(b"\r\n") + # Making a single send() call instead of one per line encourages + # the host OS to use a more optimal packet size instead of + # potentially emitting a series of small packets. + self.send(b"".join(headers)) + del headers response = self.response_class(self.sock, method=self._method) (version, code, message) = response._read_status() if code != http.HTTPStatus.OK: self.close() - raise OSError("Tunnel connection failed: %d %s" % (code, - message.strip())) + raise OSError(f"Tunnel connection failed: {code} {message.strip()}") while True: line = response.fp.readline(_MAXLINE + 1) if len(line) > _MAXLINE: @@ -933,6 +932,7 @@ def _tunnel(self): def connect(self): """Connect to the host and port specified in __init__.""" + sys.audit("http.client.connect", self, self.host, self.port) self.sock = self._create_connection( (self.host,self.port), self.timeout, self.source_address) self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) @@ -980,8 +980,10 @@ def send(self, data): break if encode: datablock = datablock.encode("iso-8859-1") + sys.audit("http.client.send", self, datablock) self.sock.sendall(datablock) return + sys.audit("http.client.send", self, data) try: self.sock.sendall(data) except TypeError: @@ -1407,6 +1409,9 @@ def __init__(self, host, port=None, key_file=None, cert_file=None, self.cert_file = cert_file if context is None: context = ssl._create_default_https_context() + # send ALPN extension to indicate HTTP/1.1 protocol + if self._http_vsn == 11: + context.set_alpn_protocols(['http/1.1']) # enable PHA for TLS 1.3 connections if available if context.post_handshake_auth is not None: context.post_handshake_auth = True diff --git a/Lib/http/server.py b/Lib/http/server.py index fa204fbc15e3d7..94f730ed3445be 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -414,7 +414,7 @@ def handle_one_request(self): method = getattr(self, mname) method() self.wfile.flush() #actually send the response if not already done. - except socket.timeout as e: + except TimeoutError as e: #a read or a write timed out. Discard this connection self.log_error("Request timed out: %r", e) self.close_connection = True @@ -1092,8 +1092,7 @@ def run_cgi(self): env['PATH_INFO'] = uqrest env['PATH_TRANSLATED'] = self.translate_path(uqrest) env['SCRIPT_NAME'] = scriptname - if query: - env['QUERY_STRING'] = query + env['QUERY_STRING'] = query env['REMOTE_ADDR'] = self.client_address[0] authorization = self.headers.get("authorization") if authorization: @@ -1123,12 +1122,7 @@ def run_cgi(self): referer = self.headers.get('referer') if referer: env['HTTP_REFERER'] = referer - accept = [] - for line in self.headers.getallmatchingheaders('accept'): - if line[:1] in "\t\n\r ": - accept.append(line.strip()) - else: - accept = accept + line[7:].split(',') + accept = self.headers.get_all('accept', ()) env['HTTP_ACCEPT'] = ','.join(accept) ua = self.headers.get('user-agent') if ua: diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index 3ece623b3aa187..ed1142653d9534 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -1,7 +1,48 @@ -What's New in IDLE 3.10.0 (since 3.9.0) +What's New in IDLE 3.10.0 +(since 3.9.0) Released on 2021-10-04? -====================================== +========================= + + +bpo-37903: Add mouse actions to the shell sidebar. Left click and +optional drag selects one or more lines of text, as with the +editor line number sidebar. Right click after selecting text lines +displays a context menu with 'copy with prompts'. This zips together +prompts from the sidebar with lines from the selected text. This option +also appears on the context menu for the text. + +bpo-37892: Change Shell input indents from tabs to spaces. Shell input +now 'looks right'. Making this feasible motivated the shell sidebar. + +bpo-37903: Move the Shell input prompt to a side bar. + +bpo-43655: Make window managers on macOS and X Window recognize +IDLE dialog windows as dialogs. + +bpo-43283: Document why printing to IDLE's Shell is often slower than +printing to a system terminal and that it can be made faster by +pre-formatting a single string before printing. + +bpo-23544: Disable Debug=>Stack Viewer when user code is running or +Debugger is active, to prevent hang or crash. Patch by Zackery Spytz. + +bpo-43008: Make IDLE invoke :func:`sys.excepthook` in normal, +2-process mode. User hooks were previously ignored. +Patch by Ken Hilton. + +bpo-33065: Fix problem debugging user classes with __repr__ method. + +bpo-32631: Finish zzdummy example extension module: make menu entries +work; add docstrings and tests with 100% coverage. + +bpo-42508: Keep IDLE running on macOS. Remove obsolete workaround +that prevented running files with shortcuts when using new universal2 +installers built on macOS 11. + +bpo-42426: Fix reporting offset of the RE error in searchengine. +bpo-42416: Display docstrings in IDLE calltips in more cases, +by using inspect.getdoc. bpo-33987: Mostly finish using ttk widgets, mainly for editor, settings, and searches. Some patches by Mark Roseman. diff --git a/Lib/idlelib/autocomplete.py b/Lib/idlelib/autocomplete.py index e1e9e17311eda1..bb7ee035c4fefb 100644 --- a/Lib/idlelib/autocomplete.py +++ b/Lib/idlelib/autocomplete.py @@ -31,10 +31,11 @@ class AutoComplete: - def __init__(self, editwin=None): + def __init__(self, editwin=None, tags=None): self.editwin = editwin if editwin is not None: # not in subprocess or no-gui test self.text = editwin.text + self.tags = tags self.autocompletewindow = None # id of delayed call, and the index of the text insert when # the delayed call was issued. If _delayed_completion_id is @@ -48,7 +49,7 @@ def reload(cls): "extensions", "AutoComplete", "popupwait", type="int", default=0) def _make_autocomplete_window(self): # Makes mocking easier. - return autocomplete_w.AutoCompleteWindow(self.text) + return autocomplete_w.AutoCompleteWindow(self.text, tags=self.tags) def _remove_autocomplete_window(self, event=None): if self.autocompletewindow: diff --git a/Lib/idlelib/autocomplete_w.py b/Lib/idlelib/autocomplete_w.py index fe7a6be83d586b..d3d1e6982bfb2e 100644 --- a/Lib/idlelib/autocomplete_w.py +++ b/Lib/idlelib/autocomplete_w.py @@ -26,9 +26,11 @@ class AutoCompleteWindow: - def __init__(self, widget): + def __init__(self, widget, tags): # The widget (Text) on which we place the AutoCompleteWindow self.widget = widget + # Tags to mark inserted text with + self.tags = tags # The widgets we create self.autocompletewindow = self.listbox = self.scrollbar = None # The default foreground and background of a selection. Saved because @@ -69,7 +71,8 @@ def _change_start(self, newstart): "%s+%dc" % (self.startindex, len(self.start))) if i < len(newstart): self.widget.insert("%s+%dc" % (self.startindex, i), - newstart[i:]) + newstart[i:], + self.tags) self.start = newstart def _binary_search(self, s): diff --git a/Lib/idlelib/calltip.py b/Lib/idlelib/calltip.py index b02f87207d8db1..40bc5a0ad798fe 100644 --- a/Lib/idlelib/calltip.py +++ b/Lib/idlelib/calltip.py @@ -55,18 +55,50 @@ def refresh_calltip_event(self, event): self.open_calltip(False) def open_calltip(self, evalfuncs): - self.remove_calltip_window() + """Maybe close an existing calltip and maybe open a new calltip. + Called from (force_open|try_open|refresh)_calltip_event functions. + """ hp = HyperParser(self.editwin, "insert") sur_paren = hp.get_surrounding_brackets('(') + + # If not inside parentheses, no calltip. if not sur_paren: + self.remove_calltip_window() return + + # If a calltip is shown for the current parentheses, do + # nothing. + if self.active_calltip: + opener_line, opener_col = map(int, sur_paren[0].split('.')) + if ( + (opener_line, opener_col) == + (self.active_calltip.parenline, self.active_calltip.parencol) + ): + return + hp.set_index(sur_paren[0]) - expression = hp.get_expression() + try: + expression = hp.get_expression() + except ValueError: + expression = None if not expression: + # No expression before the opening parenthesis, e.g. + # because it's in a string or the opener for a tuple: + # Do nothing. return + + # At this point, the current index is after an opening + # parenthesis, in a section of code, preceded by a valid + # expression. If there is a calltip shown, it's not for the + # same index and should be closed. + self.remove_calltip_window() + + # Simple, fast heuristic: If the preceding expression includes + # an opening parenthesis, it likely includes a function call. if not evalfuncs and (expression.find('(') != -1): return + argspec = self.fetch_tip(expression) if not argspec: return @@ -133,6 +165,7 @@ def get_argspec(ob): ob_call = ob.__call__ except BaseException: # Buggy user object could raise anything. return '' # No popup for non-callables. + # For Get_argspecTest.test_buggy_getattr_class, CallA() & CallB(). fob = ob_call if isinstance(ob_call, types.MethodType) else ob # Initialize argspec and wrap it to get lines. @@ -153,10 +186,7 @@ def get_argspec(ob): if len(argspec) > _MAX_COLS else [argspec] if argspec else []) # Augment lines from docstring, if any, and join to get argspec. - if isinstance(ob_call, types.MethodType): - doc = ob_call.__doc__ - else: - doc = getattr(ob, "__doc__", "") + doc = inspect.getdoc(ob) if doc: for line in doc.split('\n', _MAX_LINES)[:_MAX_LINES]: line = line.strip() diff --git a/Lib/idlelib/codecontext.py b/Lib/idlelib/codecontext.py index 989b30e5994650..f2f44f5f8d4e61 100644 --- a/Lib/idlelib/codecontext.py +++ b/Lib/idlelib/codecontext.py @@ -7,11 +7,14 @@ enclosing block. The number of hint lines is determined by the maxlines variable in the codecontext section of config-extensions.def. Lines which do not open blocks are not shown in the context hints pane. + +For EditorWindows, <> is bound to CodeContext(self). +toggle_code_context_event. """ import re from sys import maxsize as INFINITY -import tkinter +from tkinter import Frame, Text, TclError from tkinter.constants import NSEW, SUNKEN from idlelib.config import idleConf @@ -83,7 +86,7 @@ def __del__(self): if self.t1 is not None: try: self.text.after_cancel(self.t1) - except tkinter.TclError: # pragma: no cover + except TclError: # pragma: no cover pass self.t1 = None @@ -111,7 +114,7 @@ def toggle_code_context_event(self, event=None): padx += widget.tk.getint(info['padx']) padx += widget.tk.getint(widget.cget('padx')) border += widget.tk.getint(widget.cget('border')) - context = self.context = tkinter.Text( + context = self.context = Text( self.editwin.text_frame, height=1, width=1, # Don't request more than we get. @@ -127,7 +130,7 @@ def toggle_code_context_event(self, event=None): line_number_colors = idleConf.GetHighlight(idleConf.CurrentTheme(), 'linenumber') - self.cell00 = tkinter.Frame(self.editwin.text_frame, + self.cell00 = Frame(self.editwin.text_frame, bg=line_number_colors['background']) self.cell00.grid(row=0, column=0, sticky=NSEW) menu_status = 'Hide' @@ -139,7 +142,7 @@ def toggle_code_context_event(self, event=None): self.text.after_cancel(self.t1) self._reset() menu_status = 'Show' - self.editwin.update_menu_label(menu='options', index='* Code Context', + self.editwin.update_menu_label(menu='options', index='*ode*ontext', label=f'{menu_status} Code Context') return "break" @@ -221,7 +224,7 @@ def jumptoline(self, event=None): """ try: self.context.index("sel.first") - except tkinter.TclError: + except TclError: lines = len(self.info) if lines == 1: # No context lines are showing. newtop = 1 diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index db1266fed3b691..3c527409731afa 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -8,15 +8,17 @@ DEBUG = False + def any(name, alternates): "Return a named group pattern matching list of alternates." return "(?P<%s>" % name + "|".join(alternates) + ")" + def make_pat(): kw = r"\b" + any("KEYWORD", keyword.kwlist) + r"\b" builtinlist = [str(name) for name in dir(builtins) - if not name.startswith('_') and \ - name not in keyword.kwlist] + if not name.startswith('_') and + name not in keyword.kwlist] builtin = r"([^.'\"\\#]\b|^)" + any("BUILTIN", builtinlist) + r"\b" comment = any("COMMENT", [r"#[^\n]*"]) stringprefix = r"(?i:r|u|f|fr|rf|b|br|rb)?" @@ -25,12 +27,14 @@ def make_pat(): sq3string = stringprefix + r"'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?" dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?' string = any("STRING", [sq3string, dq3string, sqstring, dqstring]) - return kw + "|" + builtin + "|" + comment + "|" + string +\ - "|" + any("SYNC", [r"\n"]) + return (kw + "|" + builtin + "|" + comment + "|" + string + + "|" + any("SYNC", [r"\n"])) + prog = re.compile(make_pat(), re.S) idprog = re.compile(r"\s+(\w+)", re.S) + def color_config(text): """Set color options of Text widget. @@ -49,7 +53,7 @@ def color_config(text): selectforeground=select_colors['foreground'], selectbackground=select_colors['background'], inactiveselectbackground=select_colors['background'], # new in 8.5 - ) + ) class ColorDelegator(Delegator): @@ -120,14 +124,16 @@ def LoadTagDefs(self): "BUILTIN": idleConf.GetHighlight(theme, "builtin"), "STRING": idleConf.GetHighlight(theme, "string"), "DEFINITION": idleConf.GetHighlight(theme, "definition"), - "SYNC": {'background':None,'foreground':None}, - "TODO": {'background':None,'foreground':None}, + "SYNC": {'background': None, 'foreground': None}, + "TODO": {'background': None, 'foreground': None}, "ERROR": idleConf.GetHighlight(theme, "error"), - # The following is used by ReplaceDialog: + # "hit" is used by ReplaceDialog to mark matches. It shouldn't be changed by Colorizer, but + # that currently isn't technically possible. This should be moved elsewhere in the future + # when fixing the "hit" tag's visibility, or when the replace dialog is replaced with a + # non-modal alternative. "hit": idleConf.GetHighlight(theme, "hit"), } - - if DEBUG: print('tagdefs',self.tagdefs) + if DEBUG: print('tagdefs', self.tagdefs) def insert(self, index, chars, tags=None): "Insert chars into widget at index and mark for colorizing." @@ -184,8 +190,8 @@ def toggle_colorize_event(self, event=None): if self.allow_colorizing and not self.colorizing: self.after_id = self.after(1, self.recolorize) if DEBUG: - print("auto colorizing turned",\ - self.allow_colorizing and "on" or "off") + print("auto colorizing turned", + "on" if self.allow_colorizing else "off") return "break" def recolorize(self): @@ -232,10 +238,7 @@ def recolorize_main(self): head, tail = item self.tag_remove("SYNC", head, tail) item = self.tag_prevrange("SYNC", head) - if item: - head = item[1] - else: - head = "1.0" + head = item[1] if item else "1.0" chars = "" next = head @@ -307,7 +310,7 @@ def _color_delegator(parent): # htest # "elif False: print(0)\n" "else: float(None)\n" "if iF + If + IF: 'keyword matching must respect case'\n" - "if'': x or'' # valid string-keyword no-space combinations\n" + "if'': x or'' # valid keyword-string no-space combinations\n" "async def f(): await g()\n" "# All valid prefixes for unicode and byte strings should be colored.\n" "'x', '''x''', \"x\", \"\"\"x\"\"\"\n" diff --git a/Lib/idlelib/config_key.py b/Lib/idlelib/config_key.py index 7510aa9f3d8786..9ca3a156f4b97f 100644 --- a/Lib/idlelib/config_key.py +++ b/Lib/idlelib/config_key.py @@ -4,6 +4,7 @@ from tkinter import Toplevel, Listbox, StringVar, TclError from tkinter.ttk import Frame, Button, Checkbutton, Entry, Label, Scrollbar from tkinter import messagebox +from tkinter.simpledialog import _setup_dialog import string import sys @@ -63,6 +64,7 @@ def __init__(self, parent, title, action, current_key_sequences, self.resizable(height=False, width=False) self.title(title) self.transient(parent) + _setup_dialog(self) self.grab_set() self.protocol("WM_DELETE_WINDOW", self.cancel) self.parent = parent diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index a84e1c5668f99f..c52a04b503adb4 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -18,8 +18,8 @@ HORIZONTAL, VERTICAL, ANCHOR, ACTIVE, END) from tkinter.ttk import (Frame, LabelFrame, Button, Checkbutton, Entry, Label, OptionMenu, Notebook, Radiobutton, Scrollbar, Style) -import tkinter.colorchooser as tkColorChooser -import tkinter.font as tkFont +from tkinter import colorchooser +import tkinter.font as tkfont from tkinter import messagebox from idlelib.config import idleConf, ConfigChanges @@ -609,7 +609,7 @@ def load_font_cfg(self): font_bold = configured_font[2]=='bold' # Set sorted no-duplicate editor font selection list and font_name. - fonts = sorted(set(tkFont.families(self))) + fonts = sorted(set(tkfont.families(self))) for font in fonts: self.fontlist.insert(END, font) self.font_name.set(font_name) @@ -663,7 +663,7 @@ def set_samples(self, event=None): Updates font_sample and highlight page highlight_sample. """ font_name = self.font_name.get() - font_weight = tkFont.BOLD if self.font_bold.get() else tkFont.NORMAL + font_weight = tkfont.BOLD if self.font_bold.get() else tkfont.NORMAL new_font = (font_name, self.font_size.get(), font_weight) self.font_sample['font'] = new_font self.highlight_sample['font'] = new_font @@ -1100,7 +1100,7 @@ def get_color(self): target = self.highlight_target.get() prev_color = self.style.lookup(self.frame_color_set['style'], 'background') - rgbTuplet, color_string = tkColorChooser.askcolor( + rgbTuplet, color_string = colorchooser.askcolor( parent=self, title='Pick new color for : '+target, initialcolor=prev_color) if color_string and (color_string != prev_color): @@ -2316,7 +2316,15 @@ def detach(self): Shell Preferences: Auto-Squeeze Min. Lines is the minimum number of lines of output to automatically "squeeze". -''' +''', + 'Extensions': ''' +ZzDummy: This extension is provided as an example for how to create and +use an extension. Enable indicates whether the extension is active or +not; likewise enable_editor and enable_shell indicate which windows it +will be active on. For this extension, z-text is the text that will be +inserted at or removed from the beginning of the lines of selected text, +or the current line if no selection. +''', } diff --git a/Lib/idlelib/debugger_r.py b/Lib/idlelib/debugger_r.py index 9dcfc56414c050..26204438858d8a 100644 --- a/Lib/idlelib/debugger_r.py +++ b/Lib/idlelib/debugger_r.py @@ -19,7 +19,7 @@ barrier, in particular frame and traceback objects. """ - +import reprlib import types from idlelib import debugger @@ -170,7 +170,7 @@ def dict_keys_list(self, did): def dict_item(self, did, key): dict = dicttable[did] value = dict[key] - value = repr(value) ### can't pickle module 'builtins' + value = reprlib.repr(value) ### can't pickle module 'builtins' return value #----------end class IdbAdapter---------- @@ -390,4 +390,4 @@ def restart_subprocess_debugger(rpcclt): if __name__ == "__main__": from unittest import main - main('idlelib.idle_test.test_debugger', verbosity=2, exit=False) + main('idlelib.idle_test.test_debugger_r', verbosity=2, exit=False) diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index a178eaf93c013a..fcc8a3f08ccfe3 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -12,8 +12,8 @@ from tkinter import * from tkinter.font import Font from tkinter.ttk import Scrollbar -import tkinter.simpledialog as tkSimpleDialog -import tkinter.messagebox as tkMessageBox +from tkinter import simpledialog +from tkinter import messagebox from idlelib.config import idleConf from idlelib import configdialog @@ -46,7 +46,7 @@ def _sphinx_version(): return release -class EditorWindow(object): +class EditorWindow: from idlelib.percolator import Percolator from idlelib.colorizer import ColorDelegator, color_config from idlelib.undo import UndoDelegator @@ -60,7 +60,6 @@ class EditorWindow(object): from idlelib.sidebar import LineNumbers from idlelib.format import FormatParagraph, FormatRegion, Indents, Rstrip from idlelib.parenmatch import ParenMatch - from idlelib.squeezer import Squeezer from idlelib.zoomheight import ZoomHeight filesystemencoding = sys.getfilesystemencoding() # for file names @@ -68,6 +67,7 @@ class EditorWindow(object): allow_code_context = True allow_line_numbers = True + user_input_insert_tags = None def __init__(self, flist=None, filename=None, key=None, root=None): # Delay import: runscript imports pyshell imports EditorWindow. @@ -295,9 +295,9 @@ def __init__(self, flist=None, filename=None, key=None, root=None): window.register_callback(self.postwindowsmenu) # Some abstractions so IDLE extensions are cross-IDE - self.askyesno = tkMessageBox.askyesno - self.askinteger = tkSimpleDialog.askinteger - self.showerror = tkMessageBox.showerror + self.askinteger = simpledialog.askinteger + self.askyesno = messagebox.askyesno + self.showerror = messagebox.showerror # Add pseudoevents for former extension fixed keys. # (This probably needs to be done once in the process.) @@ -311,7 +311,7 @@ def __init__(self, flist=None, filename=None, key=None, root=None): # Former extension bindings depends on frame.text being packed # (called from self.ResetColorizer()). - autocomplete = self.AutoComplete(self) + autocomplete = self.AutoComplete(self, self.user_input_insert_tags) text.bind("<>", autocomplete.autocomplete_event) text.bind("<>", autocomplete.try_open_completions_event) @@ -339,7 +339,7 @@ def __init__(self, flist=None, filename=None, key=None, root=None): text.bind("<>", self.code_context.toggle_code_context_event) else: - self.update_menu_state('options', '*Code Context', 'disabled') + self.update_menu_state('options', '*ode*ontext', 'disabled') if self.allow_line_numbers: self.line_numbers = self.LineNumbers(self) if idleConf.GetOption('main', 'EditorWindow', @@ -347,7 +347,7 @@ def __init__(self, flist=None, filename=None, key=None, root=None): self.toggle_line_numbers_event() text.bind("<>", self.toggle_line_numbers_event) else: - self.update_menu_state('options', '*Line Numbers', 'disabled') + self.update_menu_state('options', '*ine*umbers', 'disabled') def handle_winconfig(self, event=None): self.set_width() @@ -450,7 +450,9 @@ def createmenubar(self): self.menudict = menudict = {} for name, label in self.menu_specs: underline, label = prepstr(label) - menudict[name] = menu = Menu(mbar, name=name, tearoff=0) + postcommand = getattr(self, f'{name}_menu_postcommand', None) + menudict[name] = menu = Menu(mbar, name=name, tearoff=0, + postcommand=postcommand) mbar.add_cascade(label=label, menu=menu, underline=underline) if macosx.isCarbonTk(): # Insert the application menu @@ -596,7 +598,7 @@ def python_docs(self, event=None): try: os.startfile(self.help_url) except OSError as why: - tkMessageBox.showerror(title='Document Start Failure', + messagebox.showerror(title='Document Start Failure', message=str(why), parent=self.text) else: webbrowser.open(self.help_url) @@ -782,9 +784,7 @@ def _addcolorizer(self): self.color = self.ColorDelegator() # can add more colorizers here... if self.color: - self.per.removefilter(self.undo) - self.per.insertfilter(self.color) - self.per.insertfilter(self.undo) + self.per.insertfilterafter(filter=self.color, after=self.undo) def _rmcolorizer(self): if not self.color: @@ -927,7 +927,7 @@ def display_extra_help(helpfile=helpfile): try: os.startfile(helpfile) except OSError as why: - tkMessageBox.showerror(title='Document Start Failure', + messagebox.showerror(title='Document Start Failure', message=str(why), parent=self.text) else: webbrowser.open(helpfile) @@ -963,7 +963,7 @@ def update_recent_files_list(self, new_file=None): except OSError as err: if not getattr(self.root, "recentfiles_message", False): self.root.recentfiles_message = True - tkMessageBox.showwarning(title='IDLE Warning', + messagebox.showwarning(title='IDLE Warning', message="Cannot save Recent Files list to disk.\n" f" {err}\n" "Select OK to continue.", @@ -1301,8 +1301,6 @@ def smart_backspace_event(self, event): # Debug prompt is multilined.... ncharsdeleted = 0 while 1: - if chars == self.prompt_last_line: # '' unless PyShell - break chars = chars[:-1] ncharsdeleted = ncharsdeleted + 1 have = len(chars.expandtabs(tabwidth)) @@ -1311,7 +1309,8 @@ def smart_backspace_event(self, event): text.undo_block_start() text.delete("insert-%dc" % ncharsdeleted, "insert") if have < want: - text.insert("insert", ' ' * (want - have)) + text.insert("insert", ' ' * (want - have), + self.user_input_insert_tags) text.undo_block_stop() return "break" @@ -1344,7 +1343,7 @@ def smart_indent_event(self, event): effective = len(prefix.expandtabs(self.tabwidth)) n = self.indentwidth pad = ' ' * (n - effective % n) - text.insert("insert", pad) + text.insert("insert", pad, self.user_input_insert_tags) text.see("insert") return "break" finally: @@ -1375,13 +1374,14 @@ def newline_and_indent_event(self, event): if i == n: # The cursor is in or at leading indentation in a continuation # line; just inject an empty line at the start. - text.insert("insert linestart", '\n') + text.insert("insert linestart", '\n', + self.user_input_insert_tags) return "break" indent = line[:i] # Strip whitespace before insert point unless it's in the prompt. i = 0 - while line and line[-1] in " \t" and line != self.prompt_last_line: + while line and line[-1] in " \t": line = line[:-1] i += 1 if i: @@ -1392,7 +1392,7 @@ def newline_and_indent_event(self, event): text.delete("insert") # Insert new line. - text.insert("insert", '\n') + text.insert("insert", '\n', self.user_input_insert_tags) # Adjust indentation for continuations and block open/close. # First need to find the last statement. @@ -1428,7 +1428,7 @@ def newline_and_indent_event(self, event): elif c == pyparse.C_STRING_NEXT_LINES: # Inside a string which started before this line; # just mimic the current indent. - text.insert("insert", indent) + text.insert("insert", indent, self.user_input_insert_tags) elif c == pyparse.C_BRACKET: # Line up with the first (if any) element of the # last open bracket structure; else indent one @@ -1442,7 +1442,8 @@ def newline_and_indent_event(self, event): # beyond leftmost =; else to beyond first chunk of # non-whitespace on initial line. if y.get_num_lines_in_stmt() > 1: - text.insert("insert", indent) + text.insert("insert", indent, + self.user_input_insert_tags) else: self.reindent_to(y.compute_backslash_indent()) else: @@ -1453,7 +1454,7 @@ def newline_and_indent_event(self, event): # indentation of initial line of closest preceding # interesting statement. indent = y.get_base_indent_string() - text.insert("insert", indent) + text.insert("insert", indent, self.user_input_insert_tags) if y.is_block_opener(): self.smart_indent_event(event) elif indent and y.is_block_closer(): @@ -1500,7 +1501,8 @@ def reindent_to(self, column): if text.compare("insert linestart", "!=", "insert"): text.delete("insert linestart", "insert") if column: - text.insert("insert", self._make_blanks(column)) + text.insert("insert", self._make_blanks(column), + self.user_input_insert_tags) text.undo_block_stop() # Guess indentwidth from text content. @@ -1527,7 +1529,7 @@ def toggle_line_numbers_event(self, event=None): else: self.line_numbers.show_sidebar() menu_label = "Hide" - self.update_menu_label(menu='options', index='*Line Numbers', + self.update_menu_label(menu='options', index='*ine*umbers', label=f'{menu_label} Line Numbers') # "line.col" -> line, as an int @@ -1546,7 +1548,7 @@ def get_line_indent(line, tabwidth): return m.end(), len(m.group().expandtabs(tabwidth)) -class IndentSearcher(object): +class IndentSearcher: # .run() chews over the Text widget, looking for a block opener # and the stmt following it. Returns a pair, diff --git a/Lib/idlelib/extend.txt b/Lib/idlelib/extend.txt index c9cb2e8297eb35..b482f76c4fb0f7 100644 --- a/Lib/idlelib/extend.txt +++ b/Lib/idlelib/extend.txt @@ -28,8 +28,8 @@ variables: (There are a few more, but they are rarely useful.) The extension class must not directly bind Window Manager (e.g. X) events. -Rather, it must define one or more virtual events, e.g. <>, and -corresponding methods, e.g. zoom_height_event(). The virtual events will be +Rather, it must define one or more virtual events, e.g. <>, and +corresponding methods, e.g. z_in_event(). The virtual events will be bound to the corresponding methods, and Window Manager events can then be bound to the virtual events. (This indirection is done so that the key bindings can easily be changed, and so that other sources of virtual events can exist, such @@ -54,21 +54,21 @@ Extensions are not required to define menu entries for all the events they implement. (They are also not required to create keybindings, but in that case there must be empty bindings in cofig-extensions.def) -Here is a complete example: +Here is a partial example from zzdummy.py: -class ZoomHeight: +class ZzDummy: menudefs = [ - ('edit', [ - None, # Separator - ('_Zoom Height', '<>'), - ]) + ('format', [ + ('Z in', '<>'), + ('Z out', '<>'), + ] ) ] def __init__(self, editwin): self.editwin = editwin - def zoom_height_event(self, event): + def z_in_event(self, event=None): "...Do what you want here..." The final piece of the puzzle is the file "config-extensions.def", which is diff --git a/Lib/idlelib/filelist.py b/Lib/idlelib/filelist.py index 0d200854ef0007..254f5caf6b81b0 100644 --- a/Lib/idlelib/filelist.py +++ b/Lib/idlelib/filelist.py @@ -1,7 +1,7 @@ "idlelib.filelist" import os -from tkinter import messagebox as tkMessageBox +from tkinter import messagebox class FileList: @@ -20,7 +20,7 @@ def open(self, filename, action=None): filename = self.canonize(filename) if os.path.isdir(filename): # This can happen when bad filename is passed on command line: - tkMessageBox.showerror( + messagebox.showerror( "File Error", "%r is a directory." % (filename,), master=self.root) @@ -88,7 +88,7 @@ def filename_changed_edit(self, edit): if newkey in self.dict: conflict = self.dict[newkey] self.inversedict[conflict] = None - tkMessageBox.showerror( + messagebox.showerror( "Name Conflict", "You now have multiple edit windows open for %r" % (filename,), master=self.root) diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index 170999e1280173..e80384b7775222 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -5,7 +5,7 @@ - IDLE — Python 3.10.0a1 documentation + IDLE — Python 3.10.0a6 documentation @@ -18,7 +18,7 @@ @@ -32,7 +32,6 @@ -