diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index b2ce9e75df6..2284a5fb36b 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -1,14 +1,14 @@ variables: coverage: false -trigger: ['main', '3.9', '3.8', '3.7'] +trigger: ['main', '3.10', '3.9', '3.8', '3.7'] jobs: - job: Prebuild displayName: Pre-build checks pool: - vmImage: ubuntu-18.04 + vmImage: ubuntu-20.04 steps: - template: ./prebuild-checks.yml @@ -20,7 +20,7 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true')) pool: - vmImage: ubuntu-18.04 + vmImage: ubuntu-20.04 steps: - template: ./docs-steps.yml @@ -40,7 +40,7 @@ jobs: testRunPlatform: macos pool: - vmImage: macos-10.14 + vmImage: macos-10.15 steps: - template: ./macos-steps.yml @@ -52,12 +52,12 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) pool: - vmImage: ubuntu-18.04 + vmImage: ubuntu-20.04 variables: testRunTitle: '$(build.sourceBranchName)-linux' testRunPlatform: linux - openssl_version: 1.1.1k + openssl_version: 1.1.1l steps: - template: ./posix-steps.yml @@ -78,12 +78,12 @@ jobs: ) pool: - vmImage: ubuntu-18.04 + vmImage: ubuntu-20.04 variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.1k + openssl_version: 1.1.1l steps: - template: ./posix-steps.yml diff --git a/.azure-pipelines/find-tools.yml b/.azure-pipelines/find-tools.yml new file mode 100644 index 00000000000..9ad0f5622bb --- /dev/null +++ b/.azure-pipelines/find-tools.yml @@ -0,0 +1,26 @@ +# Locate a set of the tools used for builds + +steps: + - template: windows-release/find-sdk.yml + parameters: + toolname: 'signtool.exe' + + - powershell: | + $vcvarsall = (& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" ` + -prerelease ` + -latest ` + -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` + -find VC\Auxiliary\Build\vcvarsall.bat) + Write-Host "Found vcvarsall at $vcvarsall" + Write-Host "##vso[task.setVariable variable=vcvarsall]$vcvarsall" + displayName: 'Find vcvarsall.bat' + + - powershell: | + $msbuild = (& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" ` + -prerelease ` + -latest ` + -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` + -find MSBuild\Current\Bin\msbuild.exe) + Write-Host "Found MSBuild at $msbuild" + Write-Host "##vso[task.setVariable variable=msbuild]$msbuild" + displayName: 'Find MSBuild' diff --git a/.azure-pipelines/libffi-build.yml b/.azure-pipelines/libffi-build.yml new file mode 100644 index 00000000000..dd26ff215a8 --- /dev/null +++ b/.azure-pipelines/libffi-build.yml @@ -0,0 +1,86 @@ +name: $(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr) + +variables: + IntDir: '$(Build.BinariesDirectory)' + OutDir: '$(Build.ArtifactStagingDirectory)' + + # MUST BE SET AT QUEUE TIME + # SigningCertificate: 'Python Software Foundation' + # SourcesRepo: 'https://github.com/python/cpython-source-deps' + # SourceTag: 'libffi-3.4.2' + +jobs: +- job: Build_LibFFI + displayName: LibFFI + pool: + vmImage: windows-latest + + workspace: + clean: all + + steps: + - checkout: none + + - template: ./find-tools.yml + + - powershell: | + mkdir -Force "$(IntDir)\script" + iwr "https://github.com/python/cpython/raw/main/PCbuild/prepare_libffi.bat" ` + -outfile "$(IntDir)\script\prepare_libffi.bat" + displayName: 'Download build script' + + - powershell: | + git clone $(SourcesRepo) -b $(SourceTag) --depth 1 -c core.autocrlf=false -c core.eol=lf . + displayName: 'Check out LibFFI sources' + + - script: 'prepare_libffi.bat --install-cygwin' + workingDirectory: '$(IntDir)\script' + displayName: 'Install Cygwin and build' + env: + VCVARSALL: '$(vcvarsall)' + LIBFFI_SOURCE: '$(Build.SourcesDirectory)' + LIBFFI_OUT: '$(OutDir)' + + - powershell: | + if ((gci *\*.dll).Count -lt 4) { + Write-Error "Did not generate enough DLL files" + } + if ((gci *\Include\ffi.h).Count -lt 4) { + Write-Error "Did not generate enough include files" + } + failOnStderr: true + workingDirectory: '$(OutDir)' + displayName: 'Verify files were created' + + - publish: '$(OutDir)' + artifact: 'unsigned' + displayName: 'Publish unsigned build' + +- job: Sign_LibFFI + displayName: Sign LibFFI + dependsOn: Build_LibFFI + pool: + name: 'Windows Release' + + workspace: + clean: all + + steps: + - checkout: none + - download: current + artifact: unsigned + + - template: ./find-tools.yml + + - powershell: | + signtool sign /q /a ` + /n "Python Software Foundation" ` + /fd sha256 ` + /tr http://timestamp.digicert.com/ /td sha256 ` + /d "LibFFI for Python" ` + (gci "$(Pipeline.Workspace)\unsigned\*.dll" -r) + displayName: 'Sign files' + + - publish: '$(Pipeline.Workspace)\unsigned' + artifact: 'libffi' + displayName: 'Publish libffi' diff --git a/.azure-pipelines/openssl-build.yml b/.azure-pipelines/openssl-build.yml new file mode 100644 index 00000000000..8aab7ea0b94 --- /dev/null +++ b/.azure-pipelines/openssl-build.yml @@ -0,0 +1,110 @@ +name: $(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr) + +variables: + IntDir: '$(Build.BinariesDirectory)' + OutDir: '$(Build.ArtifactStagingDirectory)' + + # MUST BE SET AT QUEUE TIME + # SigningCertificate: 'Python Software Foundation' + # SourcesRepo: 'https://github.com/python/cpython-source-deps' + # SourceTag: 'openssl-1.1.1k' + +jobs: +- job: Build_SSL + displayName: OpenSSL + pool: + name: 'Windows Release' + #vmImage: windows-latest + + strategy: + matrix: + win32: + Platform: 'win32' + VCPlatform: 'amd64_x86' + OpenSSLPlatform: 'VC-WIN32 no-asm' + amd64: + Platform: 'amd64' + VCPlatform: 'amd64' + OpenSSLPlatform: 'VC-WIN64A-masm' + arm32: + Platform: 'arm32' + VCPlatform: 'amd64_arm' + OpenSSLPlatform: 'VC-WIN32-ARM' + arm64: + Platform: 'arm64' + VCPlatform: 'amd64_arm64' + OpenSSLPlatform: 'VC-WIN64-ARM' + + workspace: + clean: all + + steps: + - checkout: none + + - template: ./find-tools.yml + + - powershell: | + git clone $(SourcesRepo) -b $(SourceTag) --depth 1 . + displayName: 'Check out OpenSSL sources' + + - powershell: | + $f = gi ms\uplink.c + $c1 = gc $f + $c2 = $c1 -replace '\(\(h = GetModuleHandle\(NULL\)\) == NULL\)', '((h = GetModuleHandleA("_ssl.pyd")) == NULL) if ((h = GetModuleHandleA("_ssl_d.pyd")) == NULL) if ((h = GetModuleHandle(NULL)) == NULL /*patched*/)' + if ($c2 -ne $c1) { + $c2 | Out-File $f -Encoding ASCII + } else { + Write-Host '##warning Failed to patch uplink.c' + } + displayName: 'Apply uplink.c patch' + + - script: | + call "$(vcvarsall)" $(VCPlatform) + perl "$(Build.SourcesDirectory)\Configure" $(OpenSSLPlatform) + nmake + workingDirectory: '$(IntDir)' + displayName: 'Build OpenSSL' + + - script: | + call "$(vcvarsall)" $(VCPlatform) + signtool sign /q /a /n "$(SigningCertificate)" /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d "OpenSSL for Python" *.dll + workingDirectory: '$(IntDir)' + displayName: 'Sign OpenSSL Build' + condition: and(succeeded(), variables['SigningCertificate']) + + - task: CopyFiles@2 + displayName: 'Copy built libraries for upload' + inputs: + SourceFolder: '$(IntDir)' + Contents: | + lib*.dll + lib*.pdb + lib*.lib + include\openssl\*.h + TargetFolder: '$(OutDir)' + + - task: CopyFiles@2 + displayName: 'Copy header files for upload' + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + include\openssl\* + TargetFolder: '$(OutDir)' + + - task: CopyFiles@2 + displayName: 'Copy applink files for upload' + inputs: + SourceFolder: '$(Build.SourcesDirectory)\ms' + Contents: applink.c + TargetFolder: '$(OutDir)\include' + + - task: CopyFiles@2 + displayName: 'Copy LICENSE for upload' + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: LICENSE + TargetFolder: '$(OutDir)' + + - publish: '$(OutDir)' + artifact: '$(Platform)' + displayName: 'Publishing $(Platform)' diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index 9360d37baec..1a3bf75ed4c 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -1,14 +1,14 @@ variables: coverage: false -pr: ['main', '3.9', '3.8', '3.7'] +pr: ['main', '3.10', '3.9', '3.8', '3.7'] jobs: - job: Prebuild displayName: Pre-build checks pool: - vmImage: ubuntu-18.04 + vmImage: ubuntu-20.04 steps: - template: ./prebuild-checks.yml @@ -20,7 +20,7 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true')) pool: - vmImage: ubuntu-18.04 + vmImage: ubuntu-20.04 steps: - template: ./docs-steps.yml @@ -38,7 +38,7 @@ jobs: testRunPlatform: macos pool: - vmImage: macos-10.14 + vmImage: macos-10.15 steps: - template: ./macos-steps.yml @@ -52,12 +52,12 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) pool: - vmImage: ubuntu-18.04 + vmImage: ubuntu-20.04 variables: testRunTitle: '$(system.pullRequest.TargetBranch)-linux' testRunPlatform: linux - openssl_version: 1.1.1k + openssl_version: 1.1.1l steps: - template: ./posix-steps.yml @@ -78,12 +78,12 @@ jobs: ) pool: - vmImage: ubuntu-18.04 + vmImage: ubuntu-20.04 variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.1k + openssl_version: 1.1.1l steps: - template: ./posix-steps.yml diff --git a/.azure-pipelines/tcltk-build.yml b/.azure-pipelines/tcltk-build.yml new file mode 100644 index 00000000000..27968e886cc --- /dev/null +++ b/.azure-pipelines/tcltk-build.yml @@ -0,0 +1,65 @@ +name: tcl$(TkSourceTag)_$(Date:yyyyMMdd)$(Rev:.rr) + +variables: + IntDir: '$(Build.BinariesDirectory)\obj' + ExternalsDir: '$(Build.BinariesDirectory)\externals' + OutDir: '$(Build.ArtifactStagingDirectory)' + Configuration: 'Release' + + # MUST BE SET AT QUEUE TIME + # SigningCertificate: 'Python Software Foundation' + # SourcesRepo: 'https://github.com/python/cpython-source-deps' + # TclSourceTag: 'tcl-core-8.6.12.0' + # TkSourceTag: 'tk-8.6.12.0' + # TixSourceTag: 'tix-8.4.3.6' + +jobs: +- job: Build_TclTk + displayName: 'Tcl/Tk' + pool: + name: 'Windows Release' + #vmImage: windows-latest + + workspace: + clean: all + + steps: + - template: ./find-tools.yml + + - powershell: | + git clone $(SourcesRepo) -b $(TclSourceTag) --depth 1 "$(ExternalsDir)\$(TclSourceTag)" + displayName: 'Check out Tcl sources' + + - powershell: | + git clone $(SourcesRepo) -b $(TkSourceTag) --depth 1 "$(ExternalsDir)\$(TkSourceTag)" + displayName: 'Check out Tk sources' + + - powershell: | + git clone $(SourcesRepo) -b $(TixSourceTag) --depth 1 "$(ExternalsDir)\$(TixSourceTag)" + displayName: 'Check out Tix sources' + + # This msbuild.rsp file will be used by the build to forcibly override these variables + - powershell: | + del -Force -EA 0 msbuild.rsp + "/p:IntDir=$(IntDir)\" >> msbuild.rsp + "/p:ExternalsDir=$(ExternalsDir)\" >> msbuild.rsp + "/p:tclDir=$(ExternalsDir)\$(TclSourceTag)\" >> msbuild.rsp + "/p:tkDir=$(ExternalsDir)\$(TkSourceTag)\" >> msbuild.rsp + "/p:tixDir=$(ExternalsDir)\$(TixSourceTag)\" >> msbuild.rsp + displayName: 'Generate msbuild.rsp' + + - powershell: | + & "$(msbuild)" PCbuild\tcl.vcxproj "@msbuild.rsp" /p:Platform=Win32 /p:tcltkDir="$(OutDir)\win32" + & "$(msbuild)" PCbuild\tk.vcxproj "@msbuild.rsp" /p:Platform=Win32 /p:tcltkDir="$(OutDir)\win32" + & "$(msbuild)" PCbuild\tix.vcxproj "@msbuild.rsp" /p:Platform=Win32 /p:tcltkDir="$(OutDir)\win32" + displayName: 'Build for win32' + + - powershell: | + & "$(msbuild)" PCbuild\tcl.vcxproj "@msbuild.rsp" /p:Platform=x64 /p:tcltkDir="$(OutDir)\amd64" + & "$(msbuild)" PCbuild\tk.vcxproj "@msbuild.rsp" /p:Platform=x64 /p:tcltkDir="$(OutDir)\amd64" + & "$(msbuild)" PCbuild\tix.vcxproj "@msbuild.rsp" /p:Platform=x64 /p:tcltkDir="$(OutDir)\amd64" + displayName: 'Build for amd64' + + - publish: '$(OutDir)' + artifact: 'tcltk' + displayName: 'Publishing tcltk' diff --git a/.gitattributes b/.gitattributes index 68566e89924..3363ea8e4e7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -46,8 +46,9 @@ Modules/clinic/*.h linguist-generated=true Objects/clinic/*.h linguist-generated=true PC/clinic/*.h linguist-generated=true Python/clinic/*.h linguist-generated=true -Python/importlib.h linguist-generated=true -Python/importlib_external.h linguist-generated=true +Python/deepfreeze/*.c linguist-generated=true +Python/frozen_modules/*.h linguist-generated=true +Python/frozen_modules/MANIFEST linguist-generated=true Include/internal/pycore_ast.h linguist-generated=true Python/Python-ast.c linguist-generated=true Include/opcode.h linguist-generated=true diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4081b36f926..48671aa7a67 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,6 +4,7 @@ name: Tests # it prevents to mark a job as mandatory. A PR cannot be merged if a job is # mandatory but not scheduled because of "paths-ignore". on: + workflow_dispatch: push: branches: - 'main' @@ -63,6 +64,18 @@ jobs: - uses: actions/setup-python@v2 - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Add ccache to PATH + run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1 + - name: Check Autoconf version 2.69 and aclocal 1.16.3 + run: | + grep "Generated by GNU Autoconf 2.69" configure + grep "aclocal 1.16.3" aclocal.m4 + grep -q "runstatedir" configure + grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4 + - name: Regenerate autoconf files + run: docker run --rm -v $(pwd):/src quay.io/tiran/cpython_autoconf:269 - name: Build CPython run: | # Build Python with the libpython dynamic library @@ -71,11 +84,13 @@ jobs: make regen-stdlib-module-names - name: Check for changes run: | + git add -u changes=$(git status --porcelain) # Check for changes in regenerated files - if ! test -z "$changes" - then - echo "Generated files not up to date. Perhaps you forgot to run make regen-all or build.bat --regen ;)" + if test -n "$changes"; then + echo "Generated files not up to date." + echo "Perhaps you forgot to run make regen-all or build.bat --regen. ;)" + echo "configure files must be regenerated with a specific, unpatched version of autoconf." echo "$changes" exit 1 fi @@ -89,6 +104,8 @@ jobs: runs-on: windows-latest needs: check_source if: needs.check_source.outputs.run_tests == 'true' + env: + IncludeUwp: 'true' steps: - uses: actions/checkout@v2 - name: Build CPython @@ -103,6 +120,8 @@ jobs: runs-on: windows-latest needs: check_source if: needs.check_source.outputs.run_tests == 'true' + env: + IncludeUwp: 'true' steps: - uses: actions/checkout@v2 - name: Register MSVC problem matcher @@ -123,8 +142,12 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v2 + - name: Prepare homebrew environment variables + run: | + echo "LDFLAGS=-L$(brew --prefix tcl-tk)/lib" >> $GITHUB_ENV + echo "PKG_CONFIG_PATH=$(brew --prefix openssl@1.1)/lib/pkgconfig:$(brew --prefix tcl-tk)/lib/pkgconfig" >> $GITHUB_ENV - name: Configure CPython - run: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-dev + run: ./configure --with-pydebug --prefix=/opt/python-dev - name: Build CPython run: make -j4 - name: Display build info @@ -138,7 +161,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1k + OPENSSL_VER: 1.1.1l PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v2 @@ -153,7 +176,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v2.1.6 + uses: actions/cache@v2.1.7 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -165,13 +188,28 @@ jobs: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1 - - name: Configure CPython - run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR - - name: Build CPython + - name: Setup directory envs for out-of-tree builds + run: | + echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV + echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + - name: Create directories for read-only out-of-tree builds + run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + - name: Bind mount sources read-only + run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + - name: Configure CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR + - name: Build CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make -j4 - name: Display build info + working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make pythoninfo + - name: Remount sources writable for tests + # some tests write to srcdir, lack of pyc files slows down testing + run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw - name: Tests + working-directory: ${{ env.CPYTHON_BUILDDIR }} run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" build_ubuntu_ssltests: @@ -182,7 +220,7 @@ jobs: strategy: fail-fast: false matrix: - openssl_ver: [1.1.1k, 3.0.0-beta1] + openssl_ver: [1.1.1l, 3.0.0] env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl @@ -201,7 +239,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v2.1.6 + uses: actions/cache@v2.1.7 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -229,7 +267,7 @@ jobs: needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1k + OPENSSL_VER: 1.1.1l PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: @@ -245,7 +283,7 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v2.1.6 + uses: actions/cache@v2.1.7 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -264,4 +302,4 @@ jobs: - name: Display build info run: make pythoninfo - name: Tests - run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu -x test_ctypes test_crypt test_decimal test_faulthandler test_interpreters test___all__ test_idle test_tix test_tk test_ttk_guionly test_ttk_textonly" + run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu -x test_ctypes test_crypt test_decimal test_faulthandler test_interpreters test___all__ test_idle test_tix test_tk test_ttk_guionly test_ttk_textonly test_multiprocessing_fork test_multiprocessing_forkserver test_multiprocessing_spawn" diff --git a/.github/workflows/build_msi.yml b/.github/workflows/build_msi.yml index db5eaa17573..476a0b1bc07 100644 --- a/.github/workflows/build_msi.yml +++ b/.github/workflows/build_msi.yml @@ -1,6 +1,7 @@ name: TestsMSI on: + workflow_dispatch: push: branches: - 'main' diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index ca4476c5090..ed44409d118 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -1,6 +1,7 @@ name: Docs on: + workflow_dispatch: #push: # branches: # - 'main' @@ -37,8 +38,15 @@ jobs: run: make -j4 - name: 'Install build dependencies' run: make -C Doc/ PYTHON=../python venv - - name: 'Build documentation' - run: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W --keep-going -j4" doctest html + # Run "check doctest html" as 3 steps to get a more readable output + # in the web UI + - name: 'Check documentation' + run: make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W --keep-going -j4" check + # Use "xvfb-run" since some doctest tests open GUI windows + - name: 'Run documentation doctest' + run: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W --keep-going -j4" doctest + - name: 'Build HTML documentation' + run: make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W --keep-going -j4" html - name: 'Upload' uses: actions/upload-artifact@v2.2.4 with: diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index 0119843e47e..fc4aaba552a 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -3,6 +3,7 @@ apt-get update apt-get -yq install \ build-essential \ + pkg-config \ ccache \ gdb \ lcov \ diff --git a/.gitignore b/.gitignore index a96be679622..39de8410200 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ *.so.* *.dylib *.dll +*.wasm *.orig *.pyc *.pyd @@ -59,16 +60,27 @@ Lib/distutils/command/*.pdb Lib/lib2to3/*.pickle Lib/test/data/* !Lib/test/data/README +/_bootstrap_python /Makefile /Makefile.pre +Mac/Makefile +Mac/PythonLauncher/Info.plist +Mac/PythonLauncher/Makefile +Mac/PythonLauncher/Python Launcher +Mac/PythonLauncher/Python Launcher.app/* +Mac/Resources/app/Info.plist +Mac/Resources/framework/Info.plist +Mac/pythonw +/*.framework/ Misc/python.pc Misc/python-embed.pc Misc/python-config.sh Modules/Setup.config Modules/Setup.local +Modules/Setup.stdlib Modules/config.c Modules/ld_so_aix -Programs/_freeze_importlib +Programs/_freeze_module Programs/_testembed PC/python_nt*.h PC/pythonnt_rc*.h @@ -103,9 +115,12 @@ Tools/unicode/data/ /config.log /config.status /config.status.lineno +# hendrikmuhs/ccache-action@v1 +/.ccache /platform /profile-clean-stamp /profile-run-stamp +/Python/deepfreeze/*.c /pybuilddir.txt /pyconfig.h /python-config @@ -120,6 +135,13 @@ Tools/unicode/data/ Tools/msi/obj Tools/ssl/amd64 Tools/ssl/win32 +Tools/freeze/test/outdir + +# The frozen modules are always generated by the build so we don't +# keep them in the repo. Also see Tools/scripts/freeze_modules.py. +Python/frozen_modules/*.h +# The manifest can be generated at any time with "make regen-frozen". +Python/frozen_modules/MANIFEST # Two-trick pony for OSX and other case insensitive file systems: # Ignore ./python binary on Unix but still look into ./Python/ directory. diff --git a/Doc/Makefile b/Doc/Makefile index 24528a1c4f3..19ddafc5f2f 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -117,6 +117,10 @@ suspicious: "or in build/$(BUILDER)/suspicious.csv. If all issues are false" \ "positives, append that file to tools/susp-ignored.csv."; \ false; } + @echo "⚠ make suspicious is deprecated and will be removed soon." + @echo "⚠ Use:" + @echo "⚠ make check" + @echo "⚠ instead." coverage: BUILDER = coverage coverage: build diff --git a/Doc/bugs.rst b/Doc/bugs.rst index 82819792a1a..b3d057797c2 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -80,6 +80,10 @@ taken on the bug. Article which goes into some detail about how to create a useful bug report. This describes what kind of information is useful and why it is useful. + `Bug Writing Guidelines `_ + Information about writing a good bug report. Some of this is specific to the + Mozilla project, but describes general good practices. + .. _contributing-to-python: Getting started contributing to Python yourself diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst index 04050f7dabe..53a42e7f28e 100644 --- a/Doc/c-api/apiabiversion.rst +++ b/Doc/c-api/apiabiversion.rst @@ -58,5 +58,14 @@ See :ref:`stable` for a discussion of API and ABI stability across versions. Thus ``3.4.1a2`` is hexversion ``0x030401a2`` and ``3.10.0`` is hexversion ``0x030a00f0``. + This version is also available via the symbol :data:`Py_Version`. + +.. c:var:: const unsigned long Py_Version + + The Python runtime version number encoded in a single constant integer, with + the same format as the c:macro:`PY_VERSION_HEX` macro. + This contains the Python version used at run time. + + .. versionadded:: 3.11 All the given macros are defined in :source:`Include/patchlevel.h`. diff --git a/Doc/c-api/call.rst b/Doc/c-api/call.rst index 31dc9c8031f..739b5e97d15 100644 --- a/Doc/c-api/call.rst +++ b/Doc/c-api/call.rst @@ -185,7 +185,7 @@ Object Calling API Various functions are available for calling a Python object. Each converts its arguments to a convention supported by the called object – either *tp_call* or vectorcall. -In order to do as litle conversion as possible, pick one that best fits +In order to do as little conversion as possible, pick one that best fits the format of data you have available. The following table summarizes the available functions; diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst index 908e92653dd..2c4cfc48f9a 100644 --- a/Doc/c-api/capsule.rst +++ b/Doc/c-api/capsule.rst @@ -103,13 +103,14 @@ Refer to :ref:`using-capsules` for more information on using these objects. Import a pointer to a C object from a capsule attribute in a module. The *name* parameter should specify the full name to the attribute, as in ``module.attribute``. The *name* stored in the capsule must match this - string exactly. If *no_block* is true, import the module without blocking - (using :c:func:`PyImport_ImportModuleNoBlock`). If *no_block* is false, - import the module conventionally (using :c:func:`PyImport_ImportModule`). + string exactly. Return the capsule's internal *pointer* on success. On failure, set an exception and return ``NULL``. + .. versionchanged:: 3.3 + *no_block* has no effect anymore. + .. c:function:: int PyCapsule_IsValid(PyObject *capsule, const char *name) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 525a773b7c7..27feab92ded 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -100,7 +100,7 @@ For convenience, some of these functions will always return a This is the most common way to set the error indicator. The first argument specifies the exception type; it is normally one of the standard exceptions, e.g. :c:data:`PyExc_RuntimeError`. You need not increment its reference count. - The second argument is an error message; it is decoded from ``'utf-8``'. + The second argument is an error message; it is decoded from ``'utf-8'``. .. c:function:: void PyErr_SetObject(PyObject *type, PyObject *value) @@ -482,7 +482,6 @@ Querying the error indicator to an exception that was *already caught*, not to an exception that was freshly raised. This function steals the references of the arguments. To clear the exception state, pass ``NULL`` for all three arguments. - For general rules about the three arguments, see :c:func:`PyErr_Restore`. .. note:: @@ -493,6 +492,12 @@ Querying the error indicator .. versionadded:: 3.3 + .. versionchanged:: 3.11 + The ``type`` and ``traceback`` arguments are no longer used and + can be NULL. The interpreter now derives them from the exception + instance (the ``value`` argument). The function still steals + references of all three arguments. + Signal Handling =============== diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index 4bd2fb3837a..8c90d1e8991 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -33,6 +33,14 @@ Constructors for container types must conform to two rules: #. Once all the fields which may contain references to other containers are initialized, it must call :c:func:`PyObject_GC_Track`. +Similarly, the deallocator for the object must conform to a similar pair of +rules: + +#. Before fields which refer to other containers are invalidated, + :c:func:`PyObject_GC_UnTrack` must be called. + +#. The object's memory must be deallocated using :c:func:`PyObject_GC_Del`. + .. warning:: If a type adds the Py_TPFLAGS_HAVE_GC, then it *must* implement at least a :c:member:`~PyTypeObject.tp_traverse` handler or explicitly use one @@ -100,14 +108,6 @@ Constructors for container types must conform to two rules: .. versionadded:: 3.9 -Similarly, the deallocator for the object must conform to a similar pair of -rules: - -#. Before fields which refer to other containers are invalidated, - :c:func:`PyObject_GC_UnTrack` must be called. - -#. The object's memory must be deallocated using :c:func:`PyObject_GC_Del`. - .. c:function:: void PyObject_GC_Del(void *op) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 60564241bfd..322b9e4d251 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -110,7 +110,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Suppress error messages when calculating the module search path in :c:func:`Py_GetPath`. - Private flag used by ``_freeze_importlib`` and ``frozenmain`` programs. + Private flag used by ``_freeze_module`` and ``frozenmain`` programs. .. c:var:: int Py_HashRandomizationFlag @@ -486,7 +486,7 @@ Process-wide parameters (set by :c:func:`Py_SetProgramName` above) and some environment variables. The returned string consists of a series of directory names separated by a platform dependent delimiter character. The delimiter character is ``':'`` - on Unix and Mac OS X, ``';'`` on Windows. The returned string points into + on Unix and macOS, ``';'`` on Windows. The returned string points into static storage; the caller should not modify its value. The list :data:`sys.path` is initialized with this value on interpreter startup; it can be (and usually is) modified later to change the search path for loading @@ -518,7 +518,7 @@ Process-wide parameters default search path but uses the one provided instead. This is useful if Python is embedded by an application that has full knowledge of the location of all modules. The path components should be separated by the platform - dependent delimiter character, which is ``':'`` on Unix and Mac OS X, ``';'`` + dependent delimiter character, which is ``':'`` on Unix and macOS, ``';'`` on Windows. This also causes :data:`sys.executable` to be set to the program @@ -553,6 +553,8 @@ Process-wide parameters period. The returned string points into static storage; the caller should not modify its value. The value is available to Python code as :data:`sys.version`. + See also the :data:`Py_Version` constant. + .. c:function:: const char* Py_GetPlatform() @@ -561,7 +563,7 @@ Process-wide parameters Return the platform identifier for the current platform. On Unix, this is formed from the "official" name of the operating system, converted to lower case, followed by the major revision number; e.g., for Solaris 2.x, which is - also known as SunOS 5.x, the value is ``'sunos5'``. On Mac OS X, it is + also known as SunOS 5.x, the value is ``'sunos5'``. On macOS, it is ``'darwin'``. On Windows, it is ``'win'``. The returned string points into static storage; the caller should not modify its value. The value is available to Python code as ``sys.platform``. @@ -1173,6 +1175,26 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. versionadded:: 3.9 +.. c:function:: void PyThreadState_EnterTracing(PyThreadState *tstate) + + Suspend tracing and profiling in the Python thread state *tstate*. + + Resume them using the :c:func:`PyThreadState_LeaveTracing` function. + + .. versionadded:: 3.11 + + +.. c:function:: void PyThreadState_LeaveTracing(PyThreadState *tstate) + + Resume tracing and profiling in the Python thread state *tstate* suspended + by the :c:func:`PyThreadState_EnterTracing` function. + + See also :c:func:`PyEval_SetTrace` and :c:func:`PyEval_SetProfile` + functions. + + .. versionadded:: 3.11 + + .. c:function:: PyInterpreterState* PyInterpreterState_Get(void) Get the current interpreter. @@ -1623,6 +1645,8 @@ Python-level trace functions in previous versions. profile function is called for all monitored events except :const:`PyTrace_LINE` :const:`PyTrace_OPCODE` and :const:`PyTrace_EXCEPTION`. + See also the :func:`sys.setprofile` function. + The caller must hold the :term:`GIL`. @@ -1635,6 +1659,8 @@ Python-level trace functions in previous versions. will not receive :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION` or :const:`PyTrace_C_RETURN` as a value for the *what* parameter. + See also the :func:`sys.settrace` function. + The caller must hold the :term:`GIL`. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 2e52679ebc5..7c8fb7b1dee 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -22,7 +22,7 @@ There are two kinds of configuration: * The :ref:`Isolated Configuration ` can be used to embed Python into an application. It isolates Python from the system. For example, environments variables are ignored, the LC_CTYPE locale is left unchanged and - no signal handler is registred. + no signal handler is registered. The :c:func:`Py_RunMain` function can be used to write a customized Python program. @@ -479,6 +479,9 @@ PyConfig Fields which are already initialized are left unchanged. + Fields for :ref:`path configuration ` are no longer + calculated or modified when calling this function, as of Python 3.11. + The :c:func:`PyConfig_Read` function only parses :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments are parsed. Since Python arguments are @@ -493,6 +496,12 @@ PyConfig parsed, and arguments are only parsed if :c:member:`PyConfig.parse_argv` equals ``1``. + .. versionchanged:: 3.11 + :c:func:`PyConfig_Read` no longer calculates all paths, and so fields + listed under :ref:`Python Path Configuration ` may + no longer be updated until :c:func:`Py_InitializeFromConfig` is + called. + .. c:function:: void PyConfig_Clear(PyConfig *config) Release configuration memory. @@ -596,13 +605,16 @@ PyConfig .. versionadded:: 3.10 - .. c:member:: int no_debug_ranges + .. c:member:: int code_debug_ranges - If equals to ``1``, disables the inclusion of the end line and column + If equals to ``0``, disables the inclusion of the end line and column mappings in code objects. Also disables traceback printing carets to specific error locations. - Default: ``0``. + Set to ``0`` by the :envvar:`PYTHONNODEBUGRANGES` environment variable + and by the :option:`-X no_debug_ranges <-X>` command line option. + + Default: ``1``. .. versionadded:: 3.11 @@ -706,7 +718,7 @@ PyConfig * Otherwise, use the :term:`locale encoding`: ``nl_langinfo(CODESET)`` result. - At Python statup, the encoding name is normalized to the Python codec + At Python startup, the encoding name is normalized to the Python codec name. For example, ``"ANSI_X3.4-1968"`` is replaced with ``"ascii"``. See also the :c:member:`~PyConfig.filesystem_errors` member. @@ -845,12 +857,19 @@ PyConfig Default: value of the ``PLATLIBDIR`` macro which is set by the :option:`configure --with-platlibdir option <--with-platlibdir>` - (default: ``"lib"``). + (default: ``"lib"``, or ``"DLLs"`` on Windows). Part of the :ref:`Python Path Configuration ` input. .. versionadded:: 3.9 + .. versionchanged:: 3.11 + This macro is now used on Windows to locate the standard + library extension modules, typically under ``DLLs``. However, + for compatibility, note that this value is ignored for any + non-standard layouts, including in-tree builds and virtual + environments. + .. c:member:: wchar_t* pythonpath_env Module search paths (:data:`sys.path`) as a string separated by ``DELIM`` @@ -867,9 +886,9 @@ PyConfig Module search paths: :data:`sys.path`. - If :c:member:`~PyConfig.module_search_paths_set` is equal to 0, the - function calculating the :ref:`Python Path Configuration ` - overrides the :c:member:`~PyConfig.module_search_paths` and sets + If :c:member:`~PyConfig.module_search_paths_set` is equal to 0, + :c:func:`Py_InitializeFromConfig` will replace + :c:member:`~PyConfig.module_search_paths` and sets :c:member:`~PyConfig.module_search_paths_set` to ``1``. Default: empty list (``module_search_paths``) and ``0`` @@ -941,16 +960,16 @@ PyConfig .. c:member:: int pathconfig_warnings - On Unix, if non-zero, calculating the :ref:`Python Path Configuration - ` can log warnings into ``stderr``. If equals to 0, - suppress these warnings. - - It has no effect on Windows. + If non-zero, calculation of path configuration is allowed to log + warnings into ``stderr``. If equals to 0, suppress these warnings. Default: ``1`` in Python mode, ``0`` in isolated mode. Part of the :ref:`Python Path Configuration ` input. + .. versionchanged:: 3.11 + Now also applies on Windows. + .. c:member:: wchar_t* prefix The site-specific directory prefix where the platform independent Python @@ -1302,10 +1321,9 @@ variables, command line arguments (:c:member:`PyConfig.argv` is not parsed) and user site directory. The C standard streams (ex: ``stdout``) and the LC_CTYPE locale are left unchanged. Signal handlers are not installed. -Configuration files are still used with this configuration. Set the -:ref:`Python Path Configuration ` ("output fields") to ignore these -configuration files and avoid the function computing the default path -configuration. +Configuration files are still used with this configuration to determine +paths that are unspecified. Ensure :c:member:`PyConfig.home` is specified +to avoid computing the default path configuration. .. _init-python-config: diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 2d85d30702d..3e7890cb766 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -105,6 +105,93 @@ defined closer to where they are useful (e.g. :c:macro:`Py_RETURN_NONE`). Others of a more general utility are defined here. This is not necessarily a complete listing. +.. c:macro:: Py_ABS(x) + + Return the absolute value of ``x``. + + .. versionadded:: 3.3 + +.. c:macro:: Py_ALWAYS_INLINE + + Ask the compiler to always inline a static inline function. The compiler can + ignore it and decides to not inline the function. + + It can be used to inline performance critical static inline functions when + building Python in debug mode with function inlining disabled. For example, + MSC disables function inlining when building in debug mode. + + Marking blindly a static inline function with Py_ALWAYS_INLINE can result in + worse performances (due to increased code size for example). The compiler is + usually smarter than the developer for the cost/benefit analysis. + + If Python is :ref:`built in debug mode ` (if the ``Py_DEBUG`` + macro is defined), the :c:macro:`Py_ALWAYS_INLINE` macro does nothing. + + It must be specified before the function return type. Usage:: + + static inline Py_ALWAYS_INLINE int random(void) { return 4; } + + .. versionadded:: 3.11 + +.. c:macro:: Py_CHARMASK(c) + + Argument must be a character or an integer in the range [-128, 127] or [0, + 255]. This macro returns ``c`` cast to an ``unsigned char``. + +.. c:macro:: Py_DEPRECATED(version) + + Use this for deprecated declarations. The macro must be placed before the + symbol name. + + Example:: + + Py_DEPRECATED(3.8) PyAPI_FUNC(int) Py_OldFunction(void); + + .. versionchanged:: 3.8 + MSVC support was added. + +.. c:macro:: Py_GETENV(s) + + Like ``getenv(s)``, but returns ``NULL`` if :option:`-E` was passed on the + command line (i.e. if ``Py_IgnoreEnvironmentFlag`` is set). + +.. c:macro:: Py_MAX(x, y) + + Return the maximum value between ``x`` and ``y``. + + .. versionadded:: 3.3 + +.. c:macro:: Py_MEMBER_SIZE(type, member) + + Return the size of a structure (``type``) ``member`` in bytes. + + .. versionadded:: 3.6 + +.. c:macro:: Py_MIN(x, y) + + Return the minimum value between ``x`` and ``y``. + + .. versionadded:: 3.3 + +.. c:macro:: Py_NO_INLINE + + Disable inlining on a function. For example, it reduces the C stack + consumption: useful on LTO+PGO builds which heavily inline code (see + :issue:`33720`). + + Usage:: + + Py_NO_INLINE static int random(void) { return 4; } + + .. versionadded:: 3.11 + +.. c:macro:: Py_STRINGIFY(x) + + Convert ``x`` to a C string. E.g. ``Py_STRINGIFY(123)`` returns + ``"123"``. + + .. versionadded:: 3.4 + .. c:macro:: Py_UNREACHABLE() Use this when you have a code path that cannot be reached by design. @@ -127,47 +214,6 @@ complete listing. .. versionadded:: 3.7 -.. c:macro:: Py_ABS(x) - - Return the absolute value of ``x``. - - .. versionadded:: 3.3 - -.. c:macro:: Py_MIN(x, y) - - Return the minimum value between ``x`` and ``y``. - - .. versionadded:: 3.3 - -.. c:macro:: Py_MAX(x, y) - - Return the maximum value between ``x`` and ``y``. - - .. versionadded:: 3.3 - -.. c:macro:: Py_STRINGIFY(x) - - Convert ``x`` to a C string. E.g. ``Py_STRINGIFY(123)`` returns - ``"123"``. - - .. versionadded:: 3.4 - -.. c:macro:: Py_MEMBER_SIZE(type, member) - - Return the size of a structure (``type``) ``member`` in bytes. - - .. versionadded:: 3.6 - -.. c:macro:: Py_CHARMASK(c) - - Argument must be a character or an integer in the range [-128, 127] or [0, - 255]. This macro returns ``c`` cast to an ``unsigned char``. - -.. c:macro:: Py_GETENV(s) - - Like ``getenv(s)``, but returns ``NULL`` if :option:`-E` was passed on the - command line (i.e. if ``Py_IgnoreEnvironmentFlag`` is set). - .. c:macro:: Py_UNUSED(arg) Use this for unused arguments in a function definition to silence compiler @@ -175,18 +221,6 @@ complete listing. .. versionadded:: 3.4 -.. c:macro:: Py_DEPRECATED(version) - - Use this for deprecated declarations. The macro must be placed before the - symbol name. - - Example:: - - Py_DEPRECATED(3.8) PyAPI_FUNC(int) Py_OldFunction(void); - - .. versionchanged:: 3.8 - MSVC support was added. - .. c:macro:: PyDoc_STRVAR(name, str) Creates a variable with name ``name`` that can be used in docstrings. @@ -221,6 +255,7 @@ complete listing. {NULL, NULL} }; + .. _api-objects: Objects, Types and Reference Counts diff --git a/Doc/c-api/iter.rst b/Doc/c-api/iter.rst index 63290e0a7f0..3e388bb917a 100644 --- a/Doc/c-api/iter.rst +++ b/Doc/c-api/iter.rst @@ -9,10 +9,10 @@ There are two functions specifically for working with iterators. .. c:function:: int PyIter_Check(PyObject *o) - Return non-zero if the object *o* supports the iterator protocol, and ``0`` - otherwise. This function always succeeds. + Return non-zero if the object *o* can be safely passed to + :c:func:`PyIter_Next`, and ``0`` otherwise. This function always succeeds. -.. c:function:: int PyAiter_Check(PyObject *o) +.. c:function:: int PyAIter_Check(PyObject *o) Returns non-zero if the object 'obj' provides :class:`AsyncIterator` protocols, and ``0`` otherwise. This function always succeeds. @@ -21,10 +21,11 @@ There are two functions specifically for working with iterators. .. c:function:: PyObject* PyIter_Next(PyObject *o) - Return the next value from the iteration *o*. The object must be an iterator - (it is up to the caller to check this). If there are no remaining values, - returns ``NULL`` with no exception set. If an error occurs while retrieving - the item, returns ``NULL`` and passes along the exception. + Return the next value from the iterator *o*. The object must be an iterator + according to :c:func:`PyIter_Check` (it is up to the caller to check this). + If there are no remaining values, returns ``NULL`` with no exception set. + If an error occurs while retrieving the item, returns ``NULL`` and passes + along the exception. To write a loop which iterates over an iterator, the C code should look something like this:: diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 70cff69dc16..17e37077994 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -358,7 +358,7 @@ Object Protocol iterated. -.. c:function:: PyObject* PyObject_GetAiter(PyObject *o) +.. c:function:: PyObject* PyObject_GetAIter(PyObject *o) This is the equivalent to the Python expression ``aiter(o)``. Takes an :class:`AsyncIterable` object and returns an :class:`AsyncIterator` for it. diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 05c54ccc8da..49f2a614e35 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -99,7 +99,10 @@ the definition of all other Python objects. Return a :term:`borrowed reference`. - The :c:func:`Py_SET_TYPE` function must be used to set an object type. + Use the :c:func:`Py_SET_TYPE` function to set an object type. + + .. versionchanged:: 3.11 + :c:func:`Py_TYPE()` is changed to an inline static function. .. c:function:: int Py_IS_TYPE(PyObject *o, PyTypeObject *type) @@ -121,9 +124,10 @@ the definition of all other Python objects. Get the reference count of the Python object *o*. + Use the :c:func:`Py_SET_REFCNT()` function to set an object reference count. + .. versionchanged:: 3.10 :c:func:`Py_REFCNT()` is changed to the inline static function. - Use :c:func:`Py_SET_REFCNT()` to set an object reference count. .. c:function:: void Py_SET_REFCNT(PyObject *o, Py_ssize_t refcnt) @@ -137,7 +141,10 @@ the definition of all other Python objects. Get the size of the Python object *o*. - The :c:func:`Py_SET_SIZE` function must be used to set an object size. + Use the :c:func:`Py_SET_SIZE` function to set an object size. + + .. versionchanged:: 3.11 + :c:func:`Py_SIZE()` is changed to an inline static function. .. c:function:: void Py_SET_SIZE(PyVarObject *o, Py_ssize_t size) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 630c7db437c..f96886985e9 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -112,6 +112,13 @@ Type Objects .. versionadded:: 3.11 +.. c:function:: PyObject* PyType_GetQualName(PyTypeObject *type) + + Return the type's qualified name. Equivalent to getting the + type's ``__qualname__`` attribute. + + .. versionadded:: 3.11 + .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the @@ -180,7 +187,7 @@ The following functions and structs are used to create The *module* argument can be used to record the module in which the new class is defined. It must be a module object or ``NULL``. If not ``NULL``, the module is associated with the new type and can later be - retreived with :c:func:`PyType_GetModule`. + retrieved with :c:func:`PyType_GetModule`. The associated module is not inherited by subclasses; it must be specified for each class individually. diff --git a/Doc/c-api/typehints.rst b/Doc/c-api/typehints.rst index dfda96a4724..8b0b9b651e8 100644 --- a/Doc/c-api/typehints.rst +++ b/Doc/c-api/typehints.rst @@ -31,7 +31,7 @@ two types exist -- :ref:`GenericAlias ` and static PyMethodDef my_obj_methods[] = { // Other methods. ... - {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, "See PEP 585"} + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, "See PEP 585"} ... } diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index b17fb22b694..cd8723efef6 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -189,138 +189,138 @@ sub-slots .. table:: :widths: 26,17,12 - +---------------------------------------------------------+-----------------------------------+--------------+ - | Slot | :ref:`Type ` | special | - | | | methods | - +=========================================================+===================================+==============+ - | :c:member:`~PyAsyncMethods.am_await` | :c:type:`unaryfunc` | __await__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyAsyncMethods.am_aiter` | :c:type:`unaryfunc` | __aiter__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyAsyncMethods.am_anext` | :c:type:`unaryfunc` | __anext__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyAsyncMethods.am_send` | :c:type:`sendfunc` | | - +---------------------------------------------------------+-----------------------------------+--------------+ - | | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_add` | :c:type:`binaryfunc` | __add__ | - | | | __radd__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_inplace_add` | :c:type:`binaryfunc` | __iadd__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_subtract` | :c:type:`binaryfunc` | __sub__ | - | | | __rsub__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_inplace_subtract` | :c:type:`binaryfunc` | __sub__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_multiply` | :c:type:`binaryfunc` | __mul__ | - | | | __rmul__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_inplace_multiply` | :c:type:`binaryfunc` | __mul__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_remainder` | :c:type:`binaryfunc` | __mod__ | - | | | __rmod__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_inplace_remainder` | :c:type:`binaryfunc` | __mod__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_divmod` | :c:type:`binaryfunc` | __divmod__ | - | | | __rdivmod__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_power` | :c:type:`ternaryfunc` | __pow__ | - | | | __rpow__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_inplace_power` | :c:type:`ternaryfunc` | __pow__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_negative` | :c:type:`unaryfunc` | __neg__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_positive` | :c:type:`unaryfunc` | __pos__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_absolute` | :c:type:`unaryfunc` | __abs__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_bool` | :c:type:`inquiry` | __bool__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_invert` | :c:type:`unaryfunc` | __invert__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_lshift` | :c:type:`binaryfunc` | __lshift__ | - | | | __rlshift__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_inplace_lshift` | :c:type:`binaryfunc` | __lshift__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_rshift` | :c:type:`binaryfunc` | __rshift__ | - | | | __rrshift__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_inplace_rshift` | :c:type:`binaryfunc` | __rshift__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_and` | :c:type:`binaryfunc` | __and__ | - | | | __rand__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_inplace_and` | :c:type:`binaryfunc` | __and__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_xor` | :c:type:`binaryfunc` | __xor__ | - | | | __rxor__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_inplace_xor` | :c:type:`binaryfunc` | __xor__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_or` | :c:type:`binaryfunc` | __or__ | - | | | __ror__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_inplace_or` | :c:type:`binaryfunc` | __or__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_int` | :c:type:`unaryfunc` | __int__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_reserved` | void * | | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_float` | :c:type:`unaryfunc` | __float__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_floor_divide` | :c:type:`binaryfunc` | __floordiv__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_inplace_floor_divide` | :c:type:`binaryfunc` | __floordiv__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_true_divide` | :c:type:`binaryfunc` | __truediv__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_inplace_true_divide` | :c:type:`binaryfunc` | __truediv__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_index` | :c:type:`unaryfunc` | __index__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_matrix_multiply` | :c:type:`binaryfunc` | __matmul__ | - | | | __rmatmul__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyNumberMethods.nb_inplace_matrix_multiply` | :c:type:`binaryfunc` | __matmul__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyMappingMethods.mp_length` | :c:type:`lenfunc` | __len__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyMappingMethods.mp_subscript` | :c:type:`binaryfunc` | __getitem__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyMappingMethods.mp_ass_subscript` | :c:type:`objobjargproc` | __setitem__, | - | | | __delitem__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PySequenceMethods.sq_length` | :c:type:`lenfunc` | __len__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PySequenceMethods.sq_concat` | :c:type:`binaryfunc` | __add__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PySequenceMethods.sq_repeat` | :c:type:`ssizeargfunc` | __mul__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PySequenceMethods.sq_item` | :c:type:`ssizeargfunc` | __getitem__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PySequenceMethods.sq_ass_item` | :c:type:`ssizeobjargproc` | __setitem__ | - | | | __delitem__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PySequenceMethods.sq_contains` | :c:type:`objobjproc` | __contains__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PySequenceMethods.sq_inplace_concat` | :c:type:`binaryfunc` | __iadd__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PySequenceMethods.sq_inplace_repeat` | :c:type:`ssizeargfunc` | __imul__ | - +---------------------------------------------------------+-----------------------------------+--------------+ - | | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyBufferProcs.bf_getbuffer` | :c:func:`getbufferproc` | | - +---------------------------------------------------------+-----------------------------------+--------------+ - | :c:member:`~PyBufferProcs.bf_releasebuffer` | :c:func:`releasebufferproc` | | - +---------------------------------------------------------+-----------------------------------+--------------+ + +---------------------------------------------------------+-----------------------------------+---------------+ + | Slot | :ref:`Type ` | special | + | | | methods | + +=========================================================+===================================+===============+ + | :c:member:`~PyAsyncMethods.am_await` | :c:type:`unaryfunc` | __await__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyAsyncMethods.am_aiter` | :c:type:`unaryfunc` | __aiter__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyAsyncMethods.am_anext` | :c:type:`unaryfunc` | __anext__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyAsyncMethods.am_send` | :c:type:`sendfunc` | | + +---------------------------------------------------------+-----------------------------------+---------------+ + | | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_add` | :c:type:`binaryfunc` | __add__ | + | | | __radd__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_inplace_add` | :c:type:`binaryfunc` | __iadd__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_subtract` | :c:type:`binaryfunc` | __sub__ | + | | | __rsub__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_inplace_subtract` | :c:type:`binaryfunc` | __isub__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_multiply` | :c:type:`binaryfunc` | __mul__ | + | | | __rmul__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_inplace_multiply` | :c:type:`binaryfunc` | __imul__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_remainder` | :c:type:`binaryfunc` | __mod__ | + | | | __rmod__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_inplace_remainder` | :c:type:`binaryfunc` | __imod__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_divmod` | :c:type:`binaryfunc` | __divmod__ | + | | | __rdivmod__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_power` | :c:type:`ternaryfunc` | __pow__ | + | | | __rpow__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_inplace_power` | :c:type:`ternaryfunc` | __ipow__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_negative` | :c:type:`unaryfunc` | __neg__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_positive` | :c:type:`unaryfunc` | __pos__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_absolute` | :c:type:`unaryfunc` | __abs__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_bool` | :c:type:`inquiry` | __bool__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_invert` | :c:type:`unaryfunc` | __invert__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_lshift` | :c:type:`binaryfunc` | __lshift__ | + | | | __rlshift__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_inplace_lshift` | :c:type:`binaryfunc` | __ilshift__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_rshift` | :c:type:`binaryfunc` | __rshift__ | + | | | __rrshift__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_inplace_rshift` | :c:type:`binaryfunc` | __irshift__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_and` | :c:type:`binaryfunc` | __and__ | + | | | __rand__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_inplace_and` | :c:type:`binaryfunc` | __iand__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_xor` | :c:type:`binaryfunc` | __xor__ | + | | | __rxor__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_inplace_xor` | :c:type:`binaryfunc` | __ixor__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_or` | :c:type:`binaryfunc` | __or__ | + | | | __ror__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_inplace_or` | :c:type:`binaryfunc` | __ior__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_int` | :c:type:`unaryfunc` | __int__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_reserved` | void * | | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_float` | :c:type:`unaryfunc` | __float__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_floor_divide` | :c:type:`binaryfunc` | __floordiv__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_inplace_floor_divide` | :c:type:`binaryfunc` | __ifloordiv__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_true_divide` | :c:type:`binaryfunc` | __truediv__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_inplace_true_divide` | :c:type:`binaryfunc` | __itruediv__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_index` | :c:type:`unaryfunc` | __index__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_matrix_multiply` | :c:type:`binaryfunc` | __matmul__ | + | | | __rmatmul__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyNumberMethods.nb_inplace_matrix_multiply` | :c:type:`binaryfunc` | __imatmul__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyMappingMethods.mp_length` | :c:type:`lenfunc` | __len__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyMappingMethods.mp_subscript` | :c:type:`binaryfunc` | __getitem__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyMappingMethods.mp_ass_subscript` | :c:type:`objobjargproc` | __setitem__, | + | | | __delitem__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PySequenceMethods.sq_length` | :c:type:`lenfunc` | __len__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PySequenceMethods.sq_concat` | :c:type:`binaryfunc` | __add__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PySequenceMethods.sq_repeat` | :c:type:`ssizeargfunc` | __mul__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PySequenceMethods.sq_item` | :c:type:`ssizeargfunc` | __getitem__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PySequenceMethods.sq_ass_item` | :c:type:`ssizeobjargproc` | __setitem__ | + | | | __delitem__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PySequenceMethods.sq_contains` | :c:type:`objobjproc` | __contains__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PySequenceMethods.sq_inplace_concat` | :c:type:`binaryfunc` | __iadd__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PySequenceMethods.sq_inplace_repeat` | :c:type:`ssizeargfunc` | __imul__ | + +---------------------------------------------------------+-----------------------------------+---------------+ + | | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyBufferProcs.bf_getbuffer` | :c:func:`getbufferproc` | | + +---------------------------------------------------------+-----------------------------------+---------------+ + | :c:member:`~PyBufferProcs.bf_releasebuffer` | :c:func:`releasebufferproc` | | + +---------------------------------------------------------+-----------------------------------+---------------+ .. _slot-typedefs-table: @@ -668,6 +668,18 @@ and :c:type:`PyType_Type` effectively act as defaults.) :c:func:`PyObject_GC_Del` if the instance was allocated using :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar`. + If the type supports garbage collection (has the :const:`Py_TPFLAGS_HAVE_GC` + flag bit set), the destructor should call :c:func:`PyObject_GC_UnTrack` + before clearing any member fields. + + .. code-block:: c + + static void foo_dealloc(foo_object *self) { + PyObject_GC_UnTrack(self); + Py_CLEAR(self->ref); + Py_TYPE(self)->tp_free((PyObject *)self); + } + Finally, if the type is heap allocated (:const:`Py_TPFLAGS_HEAPTYPE`), the deallocator should decrement the reference count for its type object after calling the type deallocator. In order to avoid dangling pointers, the @@ -1509,9 +1521,9 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. c:member:: getiterfunc PyTypeObject.tp_iter - An optional pointer to a function that returns an iterator for the object. Its - presence normally signals that the instances of this type are iterable (although - sequences may be iterable without this function). + An optional pointer to a function that returns an :term:`iterator` for the + object. Its presence normally signals that the instances of this type are + :term:`iterable` (although sequences may be iterable without this function). This function has the same signature as :c:func:`PyObject_GetIter`:: @@ -1524,8 +1536,8 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. c:member:: iternextfunc PyTypeObject.tp_iternext - An optional pointer to a function that returns the next item in an iterator. - The signature is:: + An optional pointer to a function that returns the next item in an + :term:`iterator`. The signature is:: PyObject *tp_iternext(PyObject *self); @@ -2417,8 +2429,8 @@ Async Object Structures PyObject *am_await(PyObject *self); - The returned object must be an iterator, i.e. :c:func:`PyIter_Check` must - return ``1`` for it. + The returned object must be an :term:`iterator`, i.e. :c:func:`PyIter_Check` + must return ``1`` for it. This slot may be set to ``NULL`` if an object is not an :term:`awaitable`. diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 9b5c00599f1..1694cad6f43 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -1073,8 +1073,8 @@ PyInterpreterState_New:PyInterpreterState*::: PyIter_Check:int::: PyIter_Check:PyObject*:o:0: -PyAiter_Check:int::: -PyAiter_Check:PyObject*:o:0: +PyAIter_Check:int::: +PyAIter_Check:PyObject*:o:0: PyIter_Next:PyObject*::+1: PyIter_Next:PyObject*:o:0: @@ -1700,8 +1700,8 @@ PyObject_GetItem:PyObject*:key:0: PyObject_GetIter:PyObject*::+1: PyObject_GetIter:PyObject*:o:0: -PyObject_GetAiter:PyObject*::+1: -PyObject_GetAiter:PyObject*:o:0: +PyObject_GetAIter:PyObject*::+1: +PyObject_GetAIter:PyObject*:o:0: PyObject_HasAttr:int::: PyObject_HasAttr:PyObject*:o:0: @@ -2310,6 +2310,9 @@ PyType_GetFlags:PyTypeObject*:type:0: PyType_GetName:PyObject*::+1: PyType_GetName:PyTypeObject*:type:0: +PyType_GetQualName:PyObject*::+1: +PyType_GetQualName:PyTypeObject*:type:0: + PyType_GetSlot:void*::: PyType_GetSlot:PyTypeObject*:type:0: PyType_GetSlot:int:slot:: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 4938459d317..02e54e5d7f1 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,5 +1,5 @@ role,name,added,ifdef_note -function,PyAiter_Check,3.10, +function,PyAIter_Check,3.10, function,PyArg_Parse,3.2, function,PyArg_ParseTuple,3.2, function,PyArg_ParseTupleAndKeywords,3.2, @@ -189,6 +189,7 @@ var,PyExc_ArithmeticError,3.2, var,PyExc_AssertionError,3.2, var,PyExc_AttributeError,3.2, var,PyExc_BaseException,3.2, +var,PyExc_BaseExceptionGroup,3.11, var,PyExc_BlockingIOError,3.7, var,PyExc_BrokenPipeError,3.7, var,PyExc_BufferError,3.2, @@ -491,7 +492,7 @@ function,PyObject_GenericGetAttr,3.2, function,PyObject_GenericGetDict,3.10, function,PyObject_GenericSetAttr,3.2, function,PyObject_GenericSetDict,3.7, -function,PyObject_GetAiter,3.10, +function,PyObject_GetAIter,3.10, function,PyObject_GetAttr,3.2, function,PyObject_GetAttrString,3.2, function,PyObject_GetItem,3.2, @@ -569,6 +570,7 @@ function,PyStructSequence_GetItem,3.2, function,PyStructSequence_New,3.2, function,PyStructSequence_NewType,3.2, function,PyStructSequence_SetItem,3.2, +var,PyStructSequence_UnnamedField,3.11, var,PySuper_Type,3.2, function,PySys_AddWarnOption,3.2, function,PySys_AddWarnOptionUnicode,3.2, @@ -644,6 +646,7 @@ function,PyType_GetFlags,3.2, function,PyType_GetModule,3.10, function,PyType_GetModuleState,3.10, function,PyType_GetName,3.11, +function,PyType_GetQualName,3.11, function,PyType_GetSlot,3.4, function,PyType_IsSubtype,3.2, function,PyType_Modified,3.2, @@ -827,6 +830,7 @@ type,Py_UCS4,3.2, macro,Py_UNBLOCK_THREADS,3.2, var,Py_UTF8Mode,3.8, function,Py_VaBuildValue,3.2, +var,Py_Version,3.11, function,Py_XNewRef,3.10, type,Py_intptr_t,3.2, type,Py_ssize_t,3.2, diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index 56a0a2eaef7..136cf4e77b1 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -31,7 +31,7 @@ installing other Python projects, refer to the Key terms ========= -* the `Python Packaging Index `__ is a public +* the `Python Package Index `__ is a public repository of open source licensed packages made available for use by other Python users * the `Python Packaging Authority @@ -101,7 +101,7 @@ by invoking the ``pip`` module at the command line:: .. note:: - For POSIX users (including Mac OS X and Linux users), these instructions + For POSIX users (including macOS and Linux users), these instructions assume the use of a :term:`virtual environment`. For Windows users, these instructions assume that the option to @@ -127,14 +127,14 @@ involved in creating and publishing a project: * `Project structure`_ * `Building and packaging the project`_ -* `Uploading the project to the Python Packaging Index`_ +* `Uploading the project to the Python Package Index`_ * `The .pypirc file`_ .. _Project structure: \ https://packaging.python.org/tutorials/packaging-projects/#packaging-python-projects .. _Building and packaging the project: \ https://packaging.python.org/tutorials/packaging-projects/#creating-the-package-files -.. _Uploading the project to the Python Packaging Index: \ +.. _Uploading the project to the Python Package Index: \ https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives .. _The .pypirc file: \ https://packaging.python.org/specifications/pypirc/ @@ -150,7 +150,7 @@ These are quick answers or links for some common tasks. This isn't an easy topic, but here are a few tips: -* check the Python Packaging Index to see if the name is already in use +* check the Python Package Index to see if the name is already in use * check popular hosting sites like GitHub, Bitbucket, etc to see if there is already a project with that name * check what comes up in a web search for the name you're considering diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst index e4437f4106b..18a4aac4f70 100644 --- a/Doc/distutils/apiref.rst +++ b/Doc/distutils/apiref.rst @@ -373,7 +373,7 @@ This module provides the following functions. compiler object under Unix---if you supply a value for *compiler*, *plat* is ignored. - .. % Is the posix/nt only thing still true? Mac OS X seems to work, and + .. % Is the posix/nt only thing still true? macOS seems to work, and .. % returns a UnixCCompiler instance. How to document this... hmm. @@ -1119,11 +1119,11 @@ other utility module. For non-POSIX platforms, currently just returns ``sys.platform``. - For Mac OS X systems the OS version reflects the minimal version on which + For macOS systems the OS version reflects the minimal version on which binaries will run (that is, the value of ``MACOSX_DEPLOYMENT_TARGET`` during the build of Python), not the OS version of the current system. - For universal binary builds on Mac OS X the architecture value reflects + For universal binary builds on macOS the architecture value reflects the universal binary status instead of the architecture of the current processor. For 32-bit universal binaries the architecture is ``fat``, for 64-bit universal binaries the architecture is ``fat64``, and @@ -1132,7 +1132,7 @@ other utility module. a 3-way universal build (ppc, i386, x86_64) and ``intel`` is used for a universal build with the i386 and x86_64 architectures - Examples of returned values on Mac OS X: + Examples of returned values on macOS: * ``macosx-10.3-ppc`` @@ -1852,22 +1852,6 @@ Subclasses of :class:`Command` must define the following methods. .. % todo -:mod:`distutils.command.bdist_msi` --- Build a Microsoft Installer binary package -================================================================================= - -.. module:: distutils.command.bdist_msi - :synopsis: Build a binary distribution as a Windows MSI file - -.. class:: bdist_msi - -.. deprecated:: 3.9 - Use bdist_wheel (wheel packages) instead. - - Builds a `Windows Installer`_ (.msi) binary package. - - .. _Windows Installer: https://msdn.microsoft.com/en-us/library/cc185688(VS.85).aspx - - :mod:`distutils.command.bdist_rpm` --- Build a binary distribution as a Redhat RPM and SRPM =========================================================================================== diff --git a/Doc/distutils/builtdist.rst b/Doc/distutils/builtdist.rst index c4409aca284..c1d9ea533da 100644 --- a/Doc/distutils/builtdist.rst +++ b/Doc/distutils/builtdist.rst @@ -138,11 +138,6 @@ generated by each, are: +--------------------------+-------------------------------------+ | :command:`bdist_rpm` | rpm, srpm | +--------------------------+-------------------------------------+ -| :command:`bdist_msi` | msi | -+--------------------------+-------------------------------------+ - -.. note:: - bdist_msi is deprecated since Python 3.9. The following sections give details on the individual :command:`bdist_\*` commands. diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 561d1c616cc..2e3362b834e 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -127,13 +127,11 @@ Intermezzo: Errors and Exceptions An important convention throughout the Python interpreter is the following: when a function fails, it should set an exception condition and return an error value -(usually a ``NULL`` pointer). Exceptions are stored in a static global variable -inside the interpreter; if this variable is ``NULL`` no exception has occurred. A -second global variable stores the "associated value" of the exception (the -second argument to :keyword:`raise`). A third variable contains the stack -traceback in case the error originated in Python code. These three variables -are the C equivalents of the result in Python of :meth:`sys.exc_info` (see the -section on module :mod:`sys` in the Python Library Reference). It is important +(usually ``-1`` or a ``NULL`` pointer). Exception information is stored in +three members of the interpreter's thread state. These are ``NULL`` if +there is no exception. Otherwise they are the C equivalents of the members +of the Python tuple returned by :meth:`sys.exc_info`. These are the +exception type, exception instance, and a traceback object. It is important to know about them to understand how errors are passed around. The Python API defines a number of functions to set various types of exceptions. diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 6e17897ed2c..f75bee9e6f2 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -73,7 +73,19 @@ function:: newdatatype_dealloc(newdatatypeobject *obj) { free(obj->obj_UnderlyingDatatypePtr); - Py_TYPE(obj)->tp_free(obj); + Py_TYPE(obj)->tp_free((PyObject *)obj); + } + +If your type supports garbage collection, the destructor should call +:c:func:`PyObject_GC_UnTrack` before clearing any member fields:: + + static void + newdatatype_dealloc(newdatatypeobject *obj) + { + PyObject_GC_UnTrack(obj); + Py_CLEAR(obj->other_obj); + ... + Py_TYPE(obj)->tp_free((PyObject *)obj); } .. index:: @@ -381,7 +393,7 @@ analogous to the :ref:`rich comparison methods `, like :c:func:`PyObject_RichCompareBool`. This function is called with two Python objects and the operator as arguments, -where the operator is one of ``Py_EQ``, ``Py_NE``, ``Py_LE``, ``Py_GT``, +where the operator is one of ``Py_EQ``, ``Py_NE``, ``Py_LE``, ``Py_GE``, ``Py_LT`` or ``Py_GT``. It should compare the two objects with respect to the specified operator and return ``Py_True`` or ``Py_False`` if the comparison is successful, ``Py_NotImplemented`` to indicate that comparison is not diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 720b1e496eb..0437b59d55d 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -714,7 +714,7 @@ Why don't generators support the with statement? For technical reasons, a generator used directly as a context manager would not work correctly. When, as is most common, a generator is used as an iterator run to completion, no closing is needed. When it is, wrap -it as "contextlib.closing(generator)" in the 'with' statment. +it as "contextlib.closing(generator)" in the 'with' statement. Why are colons required for the if/while/def/class statements? diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 3379e41d9de..fd32b097335 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -290,9 +290,6 @@ complete example using the GNU readline library (you may want to ignore #define PY_SSIZE_T_CLEAN #include - #include - #include - #include int main (int argc, char* argv[]) { diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index a322fa23166..86c56d957cd 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -14,17 +14,8 @@ Graphic User Interface FAQ General GUI Questions ===================== -What platform-independent GUI toolkits exist for Python? -======================================================== - -Depending on what platform(s) you are aiming at, there are several. Some -of them haven't been ported to Python 3 yet. At least `Tkinter`_ and `Qt`_ -are known to be Python 3-compatible. - -.. XXX check links - -Tkinter -------- +What GUI toolkits exist for Python? +=================================== Standard builds of Python include an object-oriented interface to the Tcl/Tk widget set, called :ref:`tkinter `. This is probably the easiest to @@ -32,85 +23,14 @@ install (since it comes included with most `binary distributions `_ of Python) and use. For more info about Tk, including pointers to the source, see the `Tcl/Tk home page `_. Tcl/Tk is fully portable to the -Mac OS X, Windows, and Unix platforms. - -wxWidgets ---------- - -wxWidgets (https://www.wxwidgets.org) is a free, portable GUI class -library written in C++ that provides a native look and feel on a -number of platforms, with Windows, Mac OS X, GTK, X11, all listed as -current stable targets. Language bindings are available for a number -of languages including Python, Perl, Ruby, etc. - -`wxPython `_ is the Python binding for -wxwidgets. While it often lags slightly behind the official wxWidgets -releases, it also offers a number of features via pure Python -extensions that are not available in other language bindings. There -is an active wxPython user and developer community. - -Both wxWidgets and wxPython are free, open source, software with -permissive licences that allow their use in commercial products as -well as in freeware or shareware. - - -Qt ---- - -There are bindings available for the Qt toolkit (using either `PyQt -`_ or `PySide -`_) and for KDE (`PyKDE4 `__). -PyQt is currently more mature than PySide, but you must buy a PyQt license from -`Riverbank Computing `_ -if you want to write proprietary applications. PySide is free for all applications. - -Qt 4.5 upwards is licensed under the LGPL license; also, commercial licenses -are available from `The Qt Company `_. - -Gtk+ ----- - -The `GObject introspection bindings `_ -for Python allow you to write GTK+ 3 applications. There is also a -`Python GTK+ 3 Tutorial `_. - -The older PyGtk bindings for the `Gtk+ 2 toolkit `_ have -been implemented by James Henstridge; see . - -Kivy ----- - -`Kivy `_ is a cross-platform GUI library supporting both -desktop operating systems (Windows, macOS, Linux) and mobile devices (Android, -iOS). It is written in Python and Cython, and can use a range of windowing -backends. - -Kivy is free and open source software distributed under the MIT license. - -FLTK ----- - -Python bindings for `the FLTK toolkit `_, a simple yet -powerful and mature cross-platform windowing system, are available from `the -PyFLTK project `_. - -OpenGL ------- - -For OpenGL bindings, see `PyOpenGL `_. - - -What platform-specific GUI toolkits exist for Python? -======================================================== - -By installing the `PyObjc Objective-C bridge -`_, Python programs can use Mac OS X's -Cocoa libraries. - -:ref:`Pythonwin ` by Mark Hammond includes an interface to the -Microsoft Foundation Classes and a Python programming environment -that's written mostly in Python using the MFC classes. +macOS, Windows, and Unix platforms. +Depending on what platform(s) you are aiming at, there are also several +alternatives. A `list of cross-platform +`_ and +`platform-specific +`_ GUI +frameworks can be found on the python wiki. Tkinter questions ================= diff --git a/Doc/faq/installed.rst b/Doc/faq/installed.rst index 42296533e26..16c9a74daff 100644 --- a/Doc/faq/installed.rst +++ b/Doc/faq/installed.rst @@ -29,7 +29,7 @@ there are several possible ways it could have gotten there. * Some Windows machines also have Python installed. At this writing we're aware of computers from Hewlett-Packard and Compaq that include Python. Apparently some of HP/Compaq's administrative tools are written in Python. -* Many Unix-compatible operating systems, such as Mac OS X and some Linux +* Many Unix-compatible operating systems, such as macOS and some Linux distributions, have Python installed by default; it's included in the base installation. diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index e92973a2e8f..b9e541c150d 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -240,9 +240,6 @@ Be sure to use the :mod:`threading` module and not the :mod:`_thread` module. The :mod:`threading` module builds convenient abstractions on top of the low-level primitives provided by the :mod:`_thread` module. -Aahz has a set of slides from his threading tutorial that are helpful; see -http://www.pythoncraft.com/OSCON2001/. - None of my threads seem to run: why? ------------------------------------ @@ -614,9 +611,9 @@ use ``p.read(n)``. How do I access the serial (RS232) port? ---------------------------------------- -For Win32, POSIX (Linux, BSD, etc.), Jython: +For Win32, OSX, Linux, BSD, Jython, IronPython: - http://pyserial.sourceforge.net + https://pypi.org/project/pyserial/ For Unix, see a Usenet post by Mitch Chapman: diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index af4b489fc8a..154563a859e 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -836,6 +836,27 @@ ago? ``-190 % 12 == 2`` is useful; ``-190 % 12 == -10`` is a bug waiting to bite. +How do I get int literal attribute instead of SyntaxError? +---------------------------------------------------------- + +Trying to lookup an ``int`` literal attribute in the normal manner gives +a syntax error because the period is seen as a decimal point:: + + >>> 1.__class__ + File "", line 1 + 1.__class__ + ^ + SyntaxError: invalid decimal literal + +The solution is to separate the literal from the period +with either a space or parentheses. + + >>> 1 .__class__ + + >>> (1).__class__ + + + How do I convert a string to a number? -------------------------------------- @@ -1826,6 +1847,55 @@ For example, here is the implementation of return True return False + +How can a subclass control what data is stored in an immutable instance? +------------------------------------------------------------------------ + +When subclassing an immutable type, override the :meth:`__new__` method +instead of the :meth:`__init__` method. The latter only runs *after* an +instance is created, which is too late to alter data in an immutable +instance. + +All of these immutable classes have a different signature than their +parent class: + +.. testcode:: + + from datetime import date + + class FirstOfMonthDate(date): + "Always choose the first day of the month" + def __new__(cls, year, month, day): + return super().__new__(cls, year, month, 1) + + class NamedInt(int): + "Allow text names for some numbers" + xlat = {'zero': 0, 'one': 1, 'ten': 10} + def __new__(cls, value): + value = cls.xlat.get(value, value) + return super().__new__(cls, value) + + class TitleStr(str): + "Convert str to name suitable for a URL path" + def __new__(cls, s): + s = s.lower().replace(' ', '-') + s = ''.join([c for c in s if c.isalnum() or c == '-']) + return super().__new__(cls, s) + +The classes can be used like this: + +.. doctest:: + + >>> FirstOfMonthDate(2012, 2, 14) + FirstOfMonthDate(2012, 2, 1) + >>> NamedInt('ten') + 10 + >>> NamedInt(20) + 20 + >>> TitleStr('Blog: Why Python Rocks') + 'blog-why-python-rocks' + + How do I cache method calls? ---------------------------- @@ -2022,7 +2092,7 @@ Jim Roskind suggests performing steps in the following order in each module: * ``import`` statements * active code (including globals that are initialized from imported values). -van Rossum doesn't like this approach much because the imports appear in a +Van Rossum doesn't like this approach much because the imports appear in a strange place, but it does work. Matthias Urlichs recommends restructuring your code so that the recursive import diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index 0153a4f316e..6b95819c8ee 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -212,7 +212,7 @@ Embedding the Python interpreter in a Windows app can be summarized as follows: .. code-block:: c - #include "python.h" + #include ... Py_Initialize(); // Initialize Python. initmyAppc(); // Initialize (import) the helper class. diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 1f14946fa13..e71f6c0406a 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -510,12 +510,13 @@ Glossary :func:`functools.singledispatch` decorator, and :pep:`443`. generic type - A :term:`type` that can be parameterized; typically a container like - :class:`list`. Used for :term:`type hints ` and + A :term:`type` that can be parameterized; typically a + :ref:`container class` such as :class:`list` or + :class:`dict`. Used for :term:`type hints ` and :term:`annotations `. - See :pep:`483` for more details, and :mod:`typing` or - :ref:`generic alias type ` for its uses. + For more details, see :ref:`generic alias types`, + :pep:`483`, :pep:`484`, :pep:`585`, and the :mod:`typing` module. GIL See :term:`global interpreter lock`. @@ -658,6 +659,11 @@ Glossary More information can be found in :ref:`typeiter`. + .. impl-detail:: + + CPython does not consistently apply the requirement that an iterator + define :meth:`__iter__`. + key function A key function or collation function is a callable that returns a value used for sorting or ordering. For example, :func:`locale.strxfrm` is diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 3e61103e99c..2bc2f2d4c83 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -156,7 +156,7 @@ Manually Un-Stringizing Stringized Annotations require annotating with string values that specifically *can't* be evaluated. For example: - * :pep:`604` union types using `|`, before support for this + * :pep:`604` union types using ``|``, before support for this was added to Python 3.10. * Definitions that aren't needed at runtime, only imported when :const:`typing.TYPE_CHECKING` is true. diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 3a3653a5ee3..a3c4330296b 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -567,9 +567,6 @@ expression. Currently the following are explicitly supported: * Simple symbolic constants like ``sys.maxsize``, which must start with the name of the module -In case you're curious, this is implemented in ``from_builtin()`` -in ``Lib/inspect.py``. - (In the future, this may need to get even more elaborate, to allow full expressions like ``CONSTANT - 1``.) diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 575caeb720f..6ce062d0fa8 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -1264,6 +1264,9 @@ Using the non-data descriptor protocol, a pure Python version of def __get__(self, obj, objtype=None): return self.f + def __call__(self, *args, **kwds): + return self.f(*args, **kwds) + .. testcode:: :hide: @@ -1272,6 +1275,8 @@ Using the non-data descriptor protocol, a pure Python version of def f(x): return x * 10 + wrapped_ord = StaticMethod(ord) + .. doctest:: :hide: @@ -1279,6 +1284,8 @@ Using the non-data descriptor protocol, a pure Python version of 30 >>> E_sim().f(3) 30 + >>> wrapped_ord('A') + 65 Class methods @@ -1344,7 +1351,7 @@ Using the non-data descriptor protocol, a pure Python version of if cls is None: cls = type(obj) if hasattr(type(self.f), '__get__'): - return self.f.__get__(cls) + return self.f.__get__(cls, cls) return MethodType(self.f, cls) .. testcode:: diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 4621b66270f..91bef218b92 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -636,7 +636,7 @@ an incorrect member:: Before :class:`StrEnum`, ``Directions.NORTH`` would have been the :class:`tuple` ``('north',)``. -.. versionadded:: 3.10 +.. versionadded:: 3.11 IntFlag @@ -659,7 +659,7 @@ used. details. .. versionadded:: 3.6 -.. versionchanged:: 3.10 +.. versionchanged:: 3.11 Sample :class:`IntFlag` class:: @@ -696,7 +696,7 @@ It is also possible to name the combinations:: Named combinations are considered aliases. Aliases do not show up during iteration, but can be returned from by-value lookups. -.. versionchanged:: 3.10 +.. versionchanged:: 3.11 Another important difference between :class:`IntFlag` and :class:`Enum` is that if no flags are set (the value is 0), its boolean evaluation is :data:`False`:: @@ -728,7 +728,7 @@ be combined with them (but may lose :class:`IntFlag` membership:: >>> list(RW) [Perm.R, Perm.W] -.. versionadded:: 3.10 +.. versionadded:: 3.11 Flag @@ -789,7 +789,7 @@ value:: >>> list(purple) [Color.RED, Color.BLUE] -.. versionadded:: 3.10 +.. versionadded:: 3.11 .. note:: @@ -936,9 +936,10 @@ and raise an error if the two do not match:: _Private__names """"""""""""""" -Private names are not converted to enum members, but remain normal attributes. +:ref:`Private names ` are not converted to enum members, +but remain normal attributes. -.. versionchanged:: 3.10 +.. versionchanged:: 3.11 ``Enum`` member type @@ -961,7 +962,6 @@ it will raise a :exc:`DeprecationWarning`:: .. versionchanged:: 3.5 -.. versionchanged:: 3.10 Creating members that are mixed with other data types @@ -997,11 +997,12 @@ Plain :class:`Enum` classes always evaluate as :data:`True`. """"""""""""""""""""""""""""" If you give your enum subclass extra methods, like the `Planet`_ -class below, those methods will show up in a :func:`dir` of the member, -but not of the class:: +class below, those methods will show up in a :func:`dir` of the member and the +class. Attributes defined in an :func:`__init__` method will only show up in a +:func:`dir` of the member:: >>> dir(Planet) - ['EARTH', 'JUPITER', 'MARS', 'MERCURY', 'NEPTUNE', 'SATURN', 'URANUS', 'VENUS', '__class__', '__doc__', '__members__', '__module__'] + ['EARTH', 'JUPITER', 'MARS', 'MERCURY', 'NEPTUNE', 'SATURN', 'URANUS', 'VENUS', '__class__', '__doc__', '__init__', '__members__', '__module__', 'surface_gravity'] >>> dir(Planet.EARTH) ['__class__', '__doc__', '__module__', 'mass', 'name', 'radius', 'surface_gravity', 'value'] diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 74e861480d2..c7f8bc8f17f 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -65,11 +65,10 @@ output must only depend on its input. Some languages are very strict about purity and don't even have assignment statements such as ``a=3`` or ``c = a + b``, but it's difficult to avoid all -side effects. Printing to the screen or writing to a disk file are side -effects, for example. For example, in Python a call to the :func:`print` or -:func:`time.sleep` function both return no useful value; they're only called for -their side effects of sending some text to the screen or pausing execution for a -second. +side effects, such as printing to the screen or writing to a disk file. Another +example is a call to the :func:`print` or :func:`time.sleep` function, neither +of which returns a useful value. Both are called only for their side effects +of sending some text to the screen or pausing execution for a second. Python programs written in functional style usually won't go to the extreme of avoiding all I/O or all assignments; instead, they'll provide a diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 5a8d92ad512..f0d944940fc 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -541,6 +541,17 @@ alternative there, as well as adapting the above script to use your alternative serialization. +Running a logging socket listener in production +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To run a logging listener in production, you may need to use a process-management tool +such as `Supervisor `_. `Here +`_ is a Gist which +provides the bare-bones files to run the above functionality using Supervisor: you +will need to change the `/path/to/` parts in the Gist to reflect the actual paths you +want to use. + + .. _context-info: Adding contextual information to your logging output @@ -982,6 +993,17 @@ to this (remembering to first import :mod:`concurrent.futures`):: for i in range(10): executor.submit(worker_process, queue, worker_configurer) +Deploying Web applications using Gunicorn and uWSGI +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When deploying Web applications using `Gunicorn `_ or `uWSGI +`_ (or similar), multiple worker +processes are created to handle client requests. In such environments, avoid creating +file-based handlers directly in your web application. Instead, use a +:class:`SocketHandler` to log from the web application to a listener in a separate +process. This can be set up using a process management tool such as Supervisor - see +`Running a logging socket listener in production`_ for more details. + Using file rotation ------------------- diff --git a/Doc/includes/sqlite3/complete_statement.py b/Doc/includes/sqlite3/complete_statement.py index cd38d7305bb..a5c94796991 100644 --- a/Doc/includes/sqlite3/complete_statement.py +++ b/Doc/includes/sqlite3/complete_statement.py @@ -24,7 +24,10 @@ while True: if buffer.lstrip().upper().startswith("SELECT"): print(cur.fetchall()) except sqlite3.Error as e: - print("An error occurred:", e.args[0]) + err_msg = str(e) + err_code = e.sqlite_errorcode + err_name = e.sqlite_errorname + print(f"{err_name} ({err_code}): {err_msg}") buffer = "" con.close() diff --git a/Doc/includes/sqlite3/text_factory.py b/Doc/includes/sqlite3/text_factory.py index a857a155cdd..c0d87cd5591 100644 --- a/Doc/includes/sqlite3/text_factory.py +++ b/Doc/includes/sqlite3/text_factory.py @@ -3,9 +3,9 @@ import sqlite3 con = sqlite3.connect(":memory:") cur = con.cursor() -AUSTRIA = "\xd6sterreich" +AUSTRIA = "Österreich" -# by default, rows are returned as Unicode +# by default, rows are returned as str cur.execute("select ?", (AUSTRIA,)) row = cur.fetchone() assert row[0] == AUSTRIA diff --git a/Doc/install/index.rst b/Doc/install/index.rst index dbe9c1e1c5d..7f7be117009 100644 --- a/Doc/install/index.rst +++ b/Doc/install/index.rst @@ -206,7 +206,7 @@ directory. If you don't choose an installation directory---i.e., if you just run ``setup.py install``\ ---then the :command:`install` command installs to the standard location for third-party Python modules. This location varies by platform and -by how you built/installed Python itself. On Unix (and Mac OS X, which is also +by how you built/installed Python itself. On Unix (and macOS, which is also Unix-based), it also depends on whether the module distribution being installed is pure Python or contains extensions ("non-pure"): @@ -236,7 +236,7 @@ Notes: :file:`{prefix}` and :file:`{exec-prefix}` stand for the directories that Python is installed to, and where it finds its libraries at run-time. They are always -the same under Windows, and very often the same under Unix and Mac OS X. You +the same under Windows, and very often the same under Unix and macOS. You can find out what your Python installation uses for :file:`{prefix}` and :file:`{exec-prefix}` by running Python in interactive mode and typing a few simple commands. Under Unix, just type ``python`` at the shell prompt. Under @@ -312,7 +312,7 @@ install into it. It is enabled with a simple option:: Files will be installed into subdirectories of :data:`site.USER_BASE` (written as :file:`{userbase}` hereafter). This scheme installs pure Python modules and extension modules in the same location (also known as :data:`site.USER_SITE`). -Here are the values for UNIX, including Mac OS X: +Here are the values for UNIX, including macOS: =============== =========================================================== Type of file Installation directory @@ -735,7 +735,7 @@ Location and names of config files ---------------------------------- The names and locations of the configuration files vary slightly across -platforms. On Unix and Mac OS X, the three configuration files (in the order +platforms. On Unix and macOS, the three configuration files (in the order they are processed) are: +--------------+----------------------------------------------------------+-------+ diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index 31e9b0bde07..4bacc7ba0c2 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -44,7 +44,7 @@ Key terms ``venv``. It allows virtual environments to be used on versions of Python prior to 3.4, which either don't provide ``venv`` at all, or aren't able to automatically install ``pip`` into created environments. -* The `Python Packaging Index `__ is a public +* The `Python Package Index `__ is a public repository of open source licensed packages made available for use by other Python users. * the `Python Packaging Authority @@ -78,13 +78,13 @@ The standard packaging tools are all designed to be used from the command line. The following command will install the latest version of a module and its -dependencies from the Python Packaging Index:: +dependencies from the Python Package Index:: python -m pip install SomePackage .. note:: - For POSIX users (including Mac OS X and Linux users), the examples in + For POSIX users (including macOS and Linux users), the examples in this guide assume the use of a :term:`virtual environment`. For Windows users, the examples in this guide assume that the option to @@ -163,7 +163,7 @@ rather than attempting to install them with ``pip``. ... work with multiple versions of Python installed in parallel? ---------------------------------------------------------------- -On Linux, Mac OS X, and other POSIX systems, use the versioned Python commands +On Linux, macOS, and other POSIX systems, use the versioned Python commands in combination with the ``-m`` switch to run the appropriate copy of ``pip``:: @@ -225,8 +225,8 @@ users being expected to compile extension modules from source as part of the installation process. With the introduction of support for the binary ``wheel`` format, and the -ability to publish wheels for at least Windows and Mac OS X through the -Python Packaging Index, this problem is expected to diminish over time, +ability to publish wheels for at least Windows and macOS through the +Python Package Index, this problem is expected to diminish over time, as users are more regularly able to install pre-built extensions rather than needing to build them themselves. diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst index 1d7bd262872..fce02e28009 100644 --- a/Doc/library/2to3.rst +++ b/Doc/library/2to3.rst @@ -11,6 +11,11 @@ contains a rich set of fixers that will handle almost all code. 2to3 supporting library :mod:`lib2to3` is, however, a flexible and generic library, so it is possible to write your own fixers for 2to3. +.. deprecated-removed:: 3.11 3.13 + The ``lib2to3`` module was marked pending for deprecation in Python 3.9 + (raising :exc:`PendingDeprecationWarning` on import) and fully deprecated + in Python 3.11 (raising :exc:`DeprecationWarning`). The ``2to3`` tool is + part of that. It will be removed in Python 3.13. .. _2to3-using: @@ -333,7 +338,8 @@ and off individually. They are described here in more detail. .. 2to3fixer:: nonzero - Renames :meth:`__nonzero__` to :meth:`~object.__bool__`. + Renames definitions of methods called :meth:`__nonzero__` + to :meth:`~object.__bool__`. .. 2to3fixer:: numliterals @@ -464,12 +470,15 @@ and off individually. They are described here in more detail. -------------- -.. deprecated:: 3.10 - Python 3.9 will switch to a PEG parser (see :pep:`617`), and Python 3.10 may - include new language syntax that is not parsable by lib2to3's LL(1) parser. - The ``lib2to3`` module may be removed from the standard library in a future - Python version. Consider third-party alternatives such as `LibCST`_ or - `parso`_. +.. deprecated-removed:: 3.11 3.13 + Python 3.9 switched to a PEG parser (see :pep:`617`) while lib2to3 is + using a less flexible LL(1) parser. Python 3.10 includes new language + syntax that is not parsable by lib2to3's LL(1) parser (see :pep:`634`). + The ``lib2to3`` module was marked pending for deprecation in Python 3.9 + (raising :exc:`PendingDeprecationWarning` on import) and fully deprecated + in Python 3.11 (raising :exc:`DeprecationWarning`). + It will be removed from the standard library in Python 3.13. + Consider third-party alternatives such as `LibCST`_ or `parso`_. .. note:: diff --git a/Doc/library/__main__.rst b/Doc/library/__main__.rst index a64faf1bbe3..c5f36a7fcf0 100644 --- a/Doc/library/__main__.rst +++ b/Doc/library/__main__.rst @@ -1,25 +1,368 @@ - -:mod:`__main__` --- Top-level script environment -================================================ +:mod:`__main__` --- Top-level code environment +============================================== .. module:: __main__ - :synopsis: The environment where the top-level script is run. + :synopsis: The environment where top-level code is run. Covers command-line + interfaces, import-time behavior, and ``__name__ == '__main__'``. -------------- -``'__main__'`` is the name of the scope in which top-level code executes. -A module's __name__ is set equal to ``'__main__'`` when read from -standard input, a script, or from an interactive prompt. +In Python, the special name ``__main__`` is used for two important constructs: -A module can discover whether or not it is running in the main scope by -checking its own ``__name__``, which allows a common idiom for conditionally -executing code in a module when it is run as a script or with ``python --m`` but not when it is imported:: +1. the name of the top-level environment of the program, which can be + checked using the ``__name__ == '__main__'`` expression; and +2. the ``__main__.py`` file in Python packages. - if __name__ == "__main__": - # execute only if run as a script - main() +Both of these mechanisms are related to Python modules; how users interact with +them and how they interact with each other. They are explained in detail +below. If you're new to Python modules, see the tutorial section +:ref:`tut-modules` for an introduction. -For a package, the same effect can be achieved by including a -``__main__.py`` module, the contents of which will be executed when the -module is run with ``-m``. + +.. _name_equals_main: + +``__name__ == '__main__'`` +--------------------------- + +When a Python module or package is imported, ``__name__`` is set to the +module's name. Usually, this is the name of the Python file itself without the +``.py`` extension:: + + >>> import configparser + >>> configparser.__name__ + 'configparser' + +If the file is part of a package, ``__name__`` will also include the parent +package's path:: + + >>> from concurrent.futures import process + >>> process.__name__ + 'concurrent.futures.process' + +However, if the module is executed in the top-level code environment, +its ``__name__`` is set to the string ``'__main__'``. + +What is the "top-level code environment"? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``__main__`` is the name of the environment where top-level code is run. +"Top-level code" is the first user-specified Python module that starts running. +It's "top-level" because it imports all other modules that the program needs. +Sometimes "top-level code" is called an *entry point* to the application. + +The top-level code environment can be: + +* the scope of an interactive prompt:: + + >>> __name__ + '__main__' + +* the Python module passed to the Python interpreter as a file argument: + + .. code-block:: shell-session + + $ python3 helloworld.py + Hello, world! + +* the Python module or package passed to the Python interpreter with the + :option:`-m` argument: + + .. code-block:: shell-session + + $ python3 -m tarfile + usage: tarfile.py [-h] [-v] (...) + +* Python code read by the Python interpreter from standard input: + + .. code-block:: shell-session + + $ echo "import this" | python3 + The Zen of Python, by Tim Peters + + Beautiful is better than ugly. + Explicit is better than implicit. + ... + +* Python code passed to the Python interpreter with the :option:`-c` argument: + + .. code-block:: shell-session + + $ python3 -c "import this" + The Zen of Python, by Tim Peters + + Beautiful is better than ugly. + Explicit is better than implicit. + ... + +In each of these situations, the top-level module's ``__name__`` is set to +``'__main__'``. + +As a result, a module can discover whether or not it is running in the +top-level environment by checking its own ``__name__``, which allows a common +idiom for conditionally executing code when the module is not initialized from +an import statement:: + + if __name__ == '__main__': + # Execute when the module is not initialized from an import statement. + ... + +.. seealso:: + + For a more detailed look at how ``__name__`` is set in all situations, see + the tutorial section :ref:`tut-modules`. + + +Idiomatic Usage +^^^^^^^^^^^^^^^ + +Some modules contain code that is intended for script use only, like parsing +command-line arguments or fetching data from standard input. If a module +like this was imported from a different module, for example to unit test +it, the script code would unintentionally execute as well. + +This is where using the ``if __name__ == '__main__'`` code block comes in +handy. Code within this block won't run unless the module is executed in the +top-level environment. + +Putting as few statements as possible in the block below ``if __name___ == +'__main__'`` can improve code clarity and correctness. Most often, a function +named ``main`` encapsulates the program's primary behavior:: + + # echo.py + + import shlex + import sys + + def echo(phrase: str) -> None: + """A dummy wrapper around print.""" + # for demonstration purposes, you can imagine that there is some + # valuable and reusable logic inside this function + print(phrase) + + def main() -> int: + """Echo the input arguments to standard output""" + phrase = shlex.join(sys.argv) + echo(phrase) + return 0 + + if __name__ == '__main__': + sys.exit(main()) # next section explains the use of sys.exit + +Note that if the module didn't encapsulate code inside the ``main`` function +but instead put it directly within the ``if __name__ == '__main__'`` block, +the ``phrase`` variable would be global to the entire module. This is +error-prone as other functions within the module could be unintentionally using +the global variable instead of a local name. A ``main`` function solves this +problem. + +Using a ``main`` function has the added benefit of the ``echo`` function itself +being isolated and importable elsewhere. When ``echo.py`` is imported, the +``echo`` and ``main`` functions will be defined, but neither of them will be +called, because ``__name__ != '__main__'``. + + +Packaging Considerations +^^^^^^^^^^^^^^^^^^^^^^^^ + +``main`` functions are often used to create command-line tools by specifying +them as entry points for console scripts. When this is done, +`pip `_ inserts the function call into a template script, +where the return value of ``main`` is passed into :func:`sys.exit`. +For example:: + + sys.exit(main()) + +Since the call to ``main`` is wrapped in :func:`sys.exit`, the expectation is +that your function will return some value acceptable as an input to +:func:`sys.exit`; typically, an integer or ``None`` (which is implicitly +returned if your function does not have a return statement). + +By proactively following this convention ourselves, our module will have the +same behavior when run directly (i.e. ``python3 echo.py``) as it will have if +we later package it as a console script entry-point in a pip-installable +package. + +In particular, be careful about returning strings from your ``main`` function. +:func:`sys.exit` will interpret a string argument as a failure message, so +your program will have an exit code of ``1``, indicating failure, and the +string will be written to :data:`sys.stderr`. The ``echo.py`` example from +earlier exemplifies using the ``sys.exit(main())`` convention. + +.. seealso:: + + `Python Packaging User Guide `_ + contains a collection of tutorials and references on how to distribute and + install Python packages with modern tools. + + +``__main__.py`` in Python Packages +---------------------------------- + +If you are not familiar with Python packages, see section :ref:`tut-packages` +of the tutorial. Most commonly, the ``__main__.py`` file is used to provide +a command-line interface for a package. Consider the following hypothetical +package, "bandclass": + +.. code-block:: text + + bandclass + ├── __init__.py + ├── __main__.py + └── student.py + +``__main__.py`` will be executed when the package itself is invoked +directly from the command line using the :option:`-m` flag. For example: + +.. code-block:: shell-session + + $ python3 -m bandclass + +This command will cause ``__main__.py`` to run. How you utilize this mechanism +will depend on the nature of the package you are writing, but in this +hypothetical case, it might make sense to allow the teacher to search for +students:: + + # bandclass/__main__.py + + import sys + from .student import search_students + + student_name = sys.argv[2] if len(sys.argv) >= 2 else '' + print(f'Found student: {search_students(student_name)}') + +Note that ``from .student import search_students`` is an example of a relative +import. This import style can be used when referencing modules within a +package. For more details, see :ref:`intra-package-references` in the +:ref:`tut-modules` section of the tutorial. + +Idiomatic Usage +^^^^^^^^^^^^^^^ + +The contents of ``__main__.py`` typically isn't fenced with +``if __name__ == '__main__'`` blocks. Instead, those files are kept short, +functions to execute from other modules. Those other modules can then be +easily unit-tested and are properly reusable. + +If used, an ``if __name__ == '__main__'`` block will still work as expected +for a ``__main__.py`` file within a package, because its ``__name__`` +attribute will include the package's path if imported:: + + >>> import asyncio.__main__ + >>> asyncio.__main__.__name__ + 'asyncio.__main__' + +This won't work for ``__main__.py`` files in the root directory of a .zip file +though. Hence, for consistency, minimal ``__main__.py`` like the :mod:`venv` +one mentioned above are preferred. + +.. seealso:: + + See :mod:`venv` for an example of a package with a minimal ``__main__.py`` + in the standard library. It doesn't contain a ``if __name__ == '__main__'`` + block. You can invoke it with ``python3 -m venv [directory]``. + + See :mod:`runpy` for more details on the :option:`-m` flag to the + interpreter executable. + + See :mod:`zipapp` for how to run applications packaged as *.zip* files. In + this case Python looks for a ``__main__.py`` file in the root directory of + the archive. + + + +``import __main__`` +------------------- + +Regardless of which module a Python program was started with, other modules +running within that same program can import the top-level environment's scope +(:term:`namespace`) by importing the ``__main__`` module. This doesn't import +a ``__main__.py`` file but rather whichever module that received the special +name ``'__main__'``. + +Here is an example module that consumes the ``__main__`` namespace:: + + # namely.py + + import __main__ + + def did_user_define_their_name(): + return 'my_name' in dir(__main__) + + def print_user_name(): + if not did_user_define_their_name(): + raise ValueError('Define the variable `my_name`!') + + if '__file__' in dir(__main__): + print(__main__.my_name, "found in file", __main__.__file__) + else: + print(__main__.my_name) + +Example usage of this module could be as follows:: + + # start.py + + import sys + + from namely import print_user_name + + # my_name = "Dinsdale" + + def main(): + try: + print_user_name() + except ValueError as ve: + return str(ve) + + if __name__ == "__main__": + sys.exit(main()) + +Now, if we started our program, the result would look like this: + +.. code-block:: shell-session + + $ python3 start.py + Define the variable `my_name`! + +The exit code of the program would be 1, indicating an error. Uncommenting the +line with ``my_name = "Dinsdale"`` fixes the program and now it exits with +status code 0, indicating success: + +.. code-block:: shell-session + + $ python3 start.py + Dinsdale found in file /path/to/start.py + +Note that importing ``__main__`` doesn't cause any issues with unintentionally +running top-level code meant for script use which is put in the +``if __name__ == "__main__"`` block of the ``start`` module. Why does this work? + +Python inserts an empty ``__main__`` module in :attr:`sys.modules` at +interpreter startup, and populates it by running top-level code. In our example +this is the ``start`` module which runs line by line and imports ``namely``. +In turn, ``namely`` imports ``__main__`` (which is really ``start``). That's an +import cycle! Fortunately, since the partially populated ``__main__`` +module is present in :attr:`sys.modules`, Python passes that to ``namely``. +See :ref:`Special considerations for __main__ ` in the +import system's reference for details on how this works. + +The Python REPL is another example of a "top-level environment", so anything +defined in the REPL becomes part of the ``__main__`` scope:: + + >>> import namely + >>> namely.did_user_define_their_name() + False + >>> namely.print_user_name() + Traceback (most recent call last): + ... + ValueError: Define the variable `my_name`! + >>> my_name = 'Jabberwocky' + >>> namely.did_user_define_their_name() + True + >>> namely.print_user_name() + Jabberwocky + +Note that in this case the ``__main__`` scope doesn't contain a ``__file__`` +attribute as it's interactive. + +The ``__main__`` scope is used in the implementation of :mod:`pdb` and +:mod:`rlcompleter`. diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index ac8a2fd6195..2edf84e5f04 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1109,7 +1109,7 @@ Anything with more interesting error-handling or resource management should be done downstream after the arguments are parsed. For example, JSON or YAML conversions have complex error cases that require -better reporting than can be given by the ``type`` keyword. An +better reporting than can be given by the ``type`` keyword. A :exc:`~json.JSONDecodeError` would not be well formatted and a :exc:`FileNotFound` exception would not be handled at all. diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 19e7bcc414b..e29b5e88d71 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1266,7 +1266,7 @@ Pattern matching the pattern matches the subject. ``body`` contains a list of nodes to execute if the pattern matches and - the result of evaluating the guard expression is truthy. + the result of evaluating the guard expression is true. .. doctest:: @@ -1917,6 +1917,19 @@ and classes for traversing abstract syntax trees: ``await`` as variable names. The lowest supported version is ``(3, 4)``; the highest is ``sys.version_info[0:2]``. + If source contains a null character ('\0'), :exc:`ValueError` is raised. + + .. warning:: + Note that successfully parsing source code into an AST object doesn't + guarantee that the source code provided is valid Python code that can + be executed as the compilation step can raise further :exc:`SyntaxError` + exceptions. For instance, the source ``return 42`` generates a valid + AST node for a return statement, but it cannot be compiled alone (it needs + to be inside a function node). + + In particular, :func:`ast.parse` won't do any scoping checks, which the + compilation step does. + .. warning:: It is possible to crash the Python interpreter with a sufficiently large/complex string due to stack depth limitations diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index 02a00033152..77f1128de50 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -57,7 +57,7 @@ When the debug mode is enabled: * The execution time of the I/O selector is logged if it takes too long to perform an I/O operation. -* Callbacks taking longer than 100ms are logged. The +* Callbacks taking longer than 100 milliseconds are logged. The :attr:`loop.slow_callback_duration` attribute can be used to set the minimum execution duration in seconds that is considered "slow". diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 98ec65dde88..497128ee17f 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -216,6 +216,10 @@ Scheduling callbacks A thread-safe variant of :meth:`call_soon`. Must be used to schedule callbacks *from another thread*. + Raises :exc:`RuntimeError` if called on a loop that's been closed. + This can happen on a secondary thread when the main application is + shutting down. + See the :ref:`concurrency and multithreading ` section of the documentation. @@ -489,24 +493,9 @@ Opening network connections .. coroutinemethod:: loop.create_datagram_endpoint(protocol_factory, \ local_addr=None, remote_addr=None, *, \ family=0, proto=0, flags=0, \ - reuse_address=None, reuse_port=None, \ + reuse_port=None, \ allow_broadcast=None, sock=None) - .. note:: - The parameter *reuse_address* is no longer supported, as using - :py:data:`~sockets.SO_REUSEADDR` poses a significant security concern for - UDP. Explicitly passing ``reuse_address=True`` will raise an exception. - - When multiple processes with differing UIDs assign sockets to an - identical UDP socket address with ``SO_REUSEADDR``, incoming packets can - become randomly distributed among the sockets. - - For supported platforms, *reuse_port* can be used as a replacement for - similar functionality. With *reuse_port*, - :py:data:`~sockets.SO_REUSEPORT` is used instead, which specifically - prevents processes with differing UIDs from assigning sockets to the same - socket address. - Create a datagram connection. The socket family can be either :py:data:`~socket.AF_INET`, @@ -553,16 +542,31 @@ Opening network connections :ref:`UDP echo server protocol ` examples. .. versionchanged:: 3.4.4 - The *family*, *proto*, *flags*, *reuse_address*, *reuse_port, + The *family*, *proto*, *flags*, *reuse_address*, *reuse_port*, *allow_broadcast*, and *sock* parameters were added. .. versionchanged:: 3.8.1 - The *reuse_address* parameter is no longer supported due to security - concerns. + The *reuse_address* parameter is no longer supported, as using + :py:data:`~sockets.SO_REUSEADDR` poses a significant security concern for + UDP. Explicitly passing ``reuse_address=True`` will raise an exception. + + When multiple processes with differing UIDs assign sockets to an + identical UDP socket address with ``SO_REUSEADDR``, incoming packets can + become randomly distributed among the sockets. + + For supported platforms, *reuse_port* can be used as a replacement for + similar functionality. With *reuse_port*, + :py:data:`~sockets.SO_REUSEPORT` is used instead, which specifically + prevents processes with differing UIDs from assigning sockets to the same + socket address. .. versionchanged:: 3.8 Added support for Windows. + .. versionchanged:: 3.11 + The *reuse_address* parameter, disabled since Python 3.9.0, 3.8.1, + 3.7.6 and 3.6.10, has been entirely removed. + .. coroutinemethod:: loop.create_unix_connection(protocol_factory, \ path=None, *, ssl=None, sock=None, \ server_hostname=None, ssl_handshake_timeout=None) @@ -627,6 +631,11 @@ Creating network servers assumed and a list of multiple sockets will be returned (most likely one for IPv4 and another one for IPv6). + * The *port* parameter can be set to specify which port the server should + listen on. If ``0`` or ``None`` (the default), a random unused port will + be selected (note that if *host* resolves to multiple network interfaces, + a different random port will be selected for each interface). + * *family* can be set to either :data:`socket.AF_INET` or :data:`~socket.AF_INET6` to force the socket to use IPv4 or IPv6. If not set, the *family* will be determined from host name @@ -1234,9 +1243,10 @@ async/await code consider using the high-level .. note:: - The default asyncio event loop on **Windows** does not support - subprocesses. See :ref:`Subprocess Support on Windows - ` for details. + On Windows, the default event loop :class:`ProactorEventLoop` supports + subprocesses, whereas :class:`SelectorEventLoop` does not. See + :ref:`Subprocess Support on Windows ` for + details. .. coroutinemethod:: loop.subprocess_exec(protocol_factory, *args, \ stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index ef496a23f5c..3496387c178 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -54,6 +54,9 @@ Future Functions See also the :func:`create_task` function which is the preferred way for creating new Tasks. + Save a reference to the result of this function, to avoid + a task disappearing mid execution. + .. versionchanged:: 3.5.1 The function accepts any :term:`awaitable` object. diff --git a/Doc/library/asyncio-platforms.rst b/Doc/library/asyncio-platforms.rst index 390ee1969d0..50ad8a2ab70 100644 --- a/Doc/library/asyncio-platforms.rst +++ b/Doc/library/asyncio-platforms.rst @@ -63,7 +63,7 @@ All event loops on Windows do not support the following methods: methods are not supported. The resolution of the monotonic clock on Windows is usually around 15.6 -msec. The best resolution is 0.5 msec. The resolution depends on the +milliseconds. The best resolution is 0.5 milliseconds. The resolution depends on the hardware (availability of `HPET `_) and on the Windows configuration. diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index b3e229c24f0..ecf178adcd3 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -352,6 +352,7 @@ TCP echo client using the :func:`asyncio.open_connection` function:: print(f'Send: {message!r}') writer.write(message.encode()) + await writer.drain() data = await reader.read(100) print(f'Received: {data.decode()!r}') @@ -395,8 +396,8 @@ TCP echo server using the :func:`asyncio.start_server` function:: server = await asyncio.start_server( handle_echo, '127.0.0.1', 8888) - addr = server.sockets[0].getsockname() - print(f'Serving on {addr}') + addrs = ', '.join(str(sock.getsockname()) for sock in server.sockets) + print(f'Serving on {addrs}') async with server: await server.serve_forever() diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 07354e028b1..bfc983e304b 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -271,6 +271,11 @@ Creating Tasks task = asyncio.ensure_future(coro()) ... + .. important:: + + Save a reference to the result of this function, to avoid + a task disappearing mid execution. + .. versionadded:: 3.7 .. versionchanged:: 3.8 @@ -294,12 +299,6 @@ Sleeping tasks to run. This can be used by long-running functions to avoid blocking the event loop for the full duration of the function call. - .. deprecated-removed:: 3.8 3.10 - The ``loop`` parameter. This function has been implicitly getting the - current running loop since 3.7. See - :ref:`What's New in 3.10's Removed section ` - for more information. - .. _asyncio_example_sleep: Example of coroutine displaying the current date every second diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index f91547bd584..4ff038c8d29 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -78,6 +78,8 @@ The modern interface provides: these non-alphabet characters in the input result in a :exc:`binascii.Error`. + For more information about the strict base64 check, see :func:`binascii.a2b_base64` + .. function:: standard_b64encode(s) @@ -152,7 +154,7 @@ The modern interface provides: This version does not allow the digit 0 (zero) to the letter O (oh) and digit 1 (one) to either the letter I (eye) or letter L (el) mappings, all these characters are included in the Extended Hex Alphabet and are not - interchangable. + interchangeable. .. versionadded:: 3.10 diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index fd5df69e852..62d7efe34ab 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -8,14 +8,13 @@ .. index:: module: uu module: base64 - module: binhex -------------- The :mod:`binascii` module contains a number of methods to convert between binary and various ASCII-encoded binary representations. Normally, you will not -use these functions directly but use wrapper modules like :mod:`uu`, -:mod:`base64`, or :mod:`binhex` instead. The :mod:`binascii` module contains +use these functions directly but use wrapper modules like :mod:`uu` or +:mod:`base64` instead. The :mod:`binascii` module contains low-level functions written in C for greater speed that are used by the higher-level modules. @@ -98,45 +97,6 @@ The :mod:`binascii` module defines the following functions: stream. -.. function:: a2b_hqx(string) - - Convert binhex4 formatted ASCII data to binary, without doing RLE-decompression. - The string should contain a complete number of binary bytes, or (in case of the - last portion of the binhex4 data) have the remaining bits zero. - - .. deprecated:: 3.9 - - -.. function:: rledecode_hqx(data) - - Perform RLE-decompression on the data, as per the binhex4 standard. The - algorithm uses ``0x90`` after a byte as a repeat indicator, followed by a count. - A count of ``0`` specifies a byte value of ``0x90``. The routine returns the - decompressed data, unless data input data ends in an orphaned repeat indicator, - in which case the :exc:`Incomplete` exception is raised. - - .. versionchanged:: 3.2 - Accept only bytestring or bytearray objects as input. - - .. deprecated:: 3.9 - - -.. function:: rlecode_hqx(data) - - Perform binhex4 style RLE-compression on *data* and return the result. - - .. deprecated:: 3.9 - - -.. function:: b2a_hqx(data) - - Perform hexbin4 binary-to-ASCII translation and return the resulting string. The - argument should already be RLE-coded, and have a length divisible by 3 (except - possibly the last fragment). - - .. deprecated:: 3.9 - - .. function:: crc_hqx(data, value) Compute a 16-bit CRC value of *data*, starting with *value* as the @@ -222,9 +182,6 @@ The :mod:`binascii` module defines the following functions: Support for RFC compliant base64-style encoding in base 16, 32, 64, and 85. - Module :mod:`binhex` - Support for the binhex format used on the Macintosh. - Module :mod:`uu` Support for UU encoding used on Unix. diff --git a/Doc/library/binhex.rst b/Doc/library/binhex.rst deleted file mode 100644 index 7de6a663762..00000000000 --- a/Doc/library/binhex.rst +++ /dev/null @@ -1,59 +0,0 @@ -:mod:`binhex` --- Encode and decode binhex4 files -================================================= - -.. module:: binhex - :synopsis: Encode and decode files in binhex4 format. - -**Source code:** :source:`Lib/binhex.py` - -.. deprecated:: 3.9 - --------------- - -This module encodes and decodes files in binhex4 format, a format allowing -representation of Macintosh files in ASCII. Only the data fork is handled. - -The :mod:`binhex` module defines the following functions: - - -.. function:: binhex(input, output) - - Convert a binary file with filename *input* to binhex file *output*. The - *output* parameter can either be a filename or a file-like object (any object - supporting a :meth:`write` and :meth:`close` method). - - -.. function:: hexbin(input, output) - - Decode a binhex file *input*. *input* may be a filename or a file-like object - supporting :meth:`read` and :meth:`close` methods. The resulting file is written - to a file named *output*, unless the argument is ``None`` in which case the - output filename is read from the binhex file. - -The following exception is also defined: - - -.. exception:: Error - - Exception raised when something can't be encoded using the binhex format (for - example, a filename is too long to fit in the filename field), or when input is - not properly encoded binhex data. - - -.. seealso:: - - Module :mod:`binascii` - Support module containing ASCII-to-binary and binary-to-ASCII conversions. - - -.. _binhex-notes: - -Notes ------ - -There is an alternative, more powerful interface to the coder and decoder, see -the source for details. - -If you code or decode textfiles on non-Macintosh platforms they will still use -the old Macintosh newline convention (carriage-return as end of line). - diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index c3c04db853e..6050ff5607a 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -279,16 +279,13 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is This subclass of :class:`TextCalendar` can be passed a locale name in the constructor and will return month and weekday names in the specified locale. - If this locale includes an encoding all strings containing month and weekday - names will be returned as unicode. .. class:: LocaleHTMLCalendar(firstweekday=0, locale=None) This subclass of :class:`HTMLCalendar` can be passed a locale name in the constructor and will return month and weekday names in the specified - locale. If this locale includes an encoding all strings containing month and - weekday names will be returned as unicode. + locale. .. note:: diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst index 1494fa7d524..c151f04dfb8 100644 --- a/Doc/library/cgi.rst +++ b/Doc/library/cgi.rst @@ -89,7 +89,7 @@ To get at submitted form data, use the :class:`FieldStorage` class. If the form contains non-ASCII characters, use the *encoding* keyword parameter set to the value of the encoding defined for the document. It is usually contained in the META tag in the HEAD section of the HTML document or by the -:mailheader:`Content-Type` header). This reads the form contents from the +:mailheader:`Content-Type` header. This reads the form contents from the standard input or the environment (depending on the value of various environment variables set according to the CGI standard). Since it may consume standard input, it should be instantiated only once. diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 0dcd88f9fd5..949288b7c6d 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -204,6 +204,9 @@ wider range of codecs when working with binary files: *buffering* has the same meaning as for the built-in :func:`open` function. It defaults to -1 which means that the default buffer size will be used. + .. versionchanged:: 3.11 + The ``'U'`` mode has been removed. + .. function:: EncodedFile(file, data_encoding, file_encoding=None, errors='strict') @@ -923,7 +926,7 @@ it's a device to determine the storage layout of the encoded bytes, and vanishes once the byte sequence has been decoded into a string; as a ``ZERO WIDTH NO-BREAK SPACE`` it's a normal character that will be decoded like any other. -There's another encoding that is able to encoding the full range of Unicode +There's another encoding that is able to encode the full range of Unicode characters: UTF-8. UTF-8 is an 8-bit encoding, which means there are no issues with byte order in UTF-8. Each byte in a UTF-8 byte sequence consists of two parts: marker bits (the most significant bits) and payload bits. The marker bits diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 924d0b58d95..132b0ce7192 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -14,7 +14,7 @@ .. testsetup:: * - from collections import * + from collections.abc import * import itertools __name__ = '' @@ -24,6 +24,89 @@ This module provides :term:`abstract base classes ` that can be used to test whether a class provides a particular interface; for example, whether it is hashable or whether it is a mapping. +An :func:`issubclass` or :func:`isinstance` test for an interface works in one +of three ways. + +1) A newly written class can inherit directly from one of the +abstract base classes. The class must supply the required abstract +methods. The remaining mixin methods come from inheritance and can be +overridden if desired. Other methods may be added as needed: + +.. testcode:: + + class C(Sequence): # Direct inheritance + def __init__(self): ... # Extra method not required by the ABC + def __getitem__(self, index): ... # Required abstract method + def __len__(self): ... # Required abstract method + def count(self, value): ... # Optionally override a mixin method + +.. doctest:: + + >>> issubclass(C, Sequence) + True + >>> isinstance(C(), Sequence) + True + +2) Existing classes and built-in classes can be registered as "virtual +subclasses" of the ABCs. Those classes should define the full API +including all of the abstract methods and all of the mixin methods. +This lets users rely on :func:`issubclass` or :func:`isinstance` tests +to determine whether the full interface is supported. The exception to +this rule is for methods that are automatically inferred from the rest +of the API: + +.. testcode:: + + class D: # No inheritance + def __init__(self): ... # Extra method not required by the ABC + def __getitem__(self, index): ... # Abstract method + def __len__(self): ... # Abstract method + def count(self, value): ... # Mixin method + def index(self, value): ... # Mixin method + + Sequence.register(D) # Register instead of inherit + +.. doctest:: + + >>> issubclass(D, Sequence) + True + >>> isinstance(D(), Sequence) + True + +In this example, class :class:`D` does not need to define +``__contains__``, ``__iter__``, and ``__reversed__`` because the +:ref:`in-operator `, the :term:`iteration ` +logic, and the :func:`reversed` function automatically fall back to +using ``__getitem__`` and ``__len__``. + +3) Some simple interfaces are directly recognizable by the presence of +the required methods (unless those methods have been set to +:const:`None`): + +.. testcode:: + + class E: + def __iter__(self): ... + def __next__(next): ... + +.. doctest:: + + >>> issubclass(E, Iterable) + True + >>> isinstance(E(), Iterable) + True + +Complex interfaces do not support this last technique because an +interface is more than just the presence of method names. Interfaces +specify semantics and relationships between methods that cannot be +inferred solely from the presence of specific method names. For +example, knowing that a class supplies ``__getitem__``, ``__len__``, and +``__iter__`` is insufficient for distinguishing a :class:`Sequence` from +a :class:`Mapping`. + +.. versionadded:: 3.9 + These abstract classes now support ``[]``. See :ref:`types-genericalias` + and :pep:`585`. .. _collections-abstract-base-classes: @@ -34,67 +117,86 @@ The collections module offers the following :term:`ABCs `: .. tabularcolumns:: |l|L|L|L| -========================== ====================== ======================= ==================================================== -ABC Inherits from Abstract Methods Mixin Methods -========================== ====================== ======================= ==================================================== -:class:`Container` ``__contains__`` -:class:`Hashable` ``__hash__`` -:class:`Iterable` ``__iter__`` -:class:`Iterator` :class:`Iterable` ``__next__`` ``__iter__`` -:class:`Reversible` :class:`Iterable` ``__reversed__`` -:class:`Generator` :class:`Iterator` ``send``, ``throw`` ``close``, ``__iter__``, ``__next__`` -:class:`Sized` ``__len__`` -:class:`Callable` ``__call__`` -:class:`Collection` :class:`Sized`, ``__contains__``, - :class:`Iterable`, ``__iter__``, - :class:`Container` ``__len__`` +============================== ====================== ======================= ==================================================== +ABC Inherits from Abstract Methods Mixin Methods +============================== ====================== ======================= ==================================================== +:class:`Container` [1]_ ``__contains__`` +:class:`Hashable` [1]_ ``__hash__`` +:class:`Iterable` [1]_ [2]_ ``__iter__`` +:class:`Iterator` [1]_ :class:`Iterable` ``__next__`` ``__iter__`` +:class:`Reversible` [1]_ :class:`Iterable` ``__reversed__`` +:class:`Generator` [1]_ :class:`Iterator` ``send``, ``throw`` ``close``, ``__iter__``, ``__next__`` +:class:`Sized` [1]_ ``__len__`` +:class:`Callable` [1]_ ``__call__`` +:class:`Collection` [1]_ :class:`Sized`, ``__contains__``, + :class:`Iterable`, ``__iter__``, + :class:`Container` ``__len__`` -:class:`Sequence` :class:`Reversible`, ``__getitem__``, ``__contains__``, ``__iter__``, ``__reversed__``, - :class:`Collection` ``__len__`` ``index``, and ``count`` +:class:`Sequence` :class:`Reversible`, ``__getitem__``, ``__contains__``, ``__iter__``, ``__reversed__``, + :class:`Collection` ``__len__`` ``index``, and ``count`` -:class:`MutableSequence` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods and - ``__setitem__``, ``append``, ``reverse``, ``extend``, ``pop``, - ``__delitem__``, ``remove``, and ``__iadd__`` - ``__len__``, - ``insert`` +:class:`MutableSequence` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods and + ``__setitem__``, ``append``, ``reverse``, ``extend``, ``pop``, + ``__delitem__``, ``remove``, and ``__iadd__`` + ``__len__``, + ``insert`` -:class:`ByteString` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods - ``__len__`` +:class:`ByteString` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods + ``__len__`` -:class:`Set` :class:`Collection` ``__contains__``, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``, - ``__iter__``, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``, - ``__len__`` ``__sub__``, ``__xor__``, and ``isdisjoint`` +:class:`Set` :class:`Collection` ``__contains__``, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``, + ``__iter__``, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``, + ``__len__`` ``__sub__``, ``__xor__``, and ``isdisjoint`` -:class:`MutableSet` :class:`Set` ``__contains__``, Inherited :class:`Set` methods and - ``__iter__``, ``clear``, ``pop``, ``remove``, ``__ior__``, - ``__len__``, ``__iand__``, ``__ixor__``, and ``__isub__`` - ``add``, - ``discard`` +:class:`MutableSet` :class:`Set` ``__contains__``, Inherited :class:`Set` methods and + ``__iter__``, ``clear``, ``pop``, ``remove``, ``__ior__``, + ``__len__``, ``__iand__``, ``__ixor__``, and ``__isub__`` + ``add``, + ``discard`` -:class:`Mapping` :class:`Collection` ``__getitem__``, ``__contains__``, ``keys``, ``items``, ``values``, - ``__iter__``, ``get``, ``__eq__``, and ``__ne__`` - ``__len__`` +:class:`Mapping` :class:`Collection` ``__getitem__``, ``__contains__``, ``keys``, ``items``, ``values``, + ``__iter__``, ``get``, ``__eq__``, and ``__ne__`` + ``__len__`` -:class:`MutableMapping` :class:`Mapping` ``__getitem__``, Inherited :class:`Mapping` methods and - ``__setitem__``, ``pop``, ``popitem``, ``clear``, ``update``, - ``__delitem__``, and ``setdefault`` - ``__iter__``, - ``__len__`` +:class:`MutableMapping` :class:`Mapping` ``__getitem__``, Inherited :class:`Mapping` methods and + ``__setitem__``, ``pop``, ``popitem``, ``clear``, ``update``, + ``__delitem__``, and ``setdefault`` + ``__iter__``, + ``__len__`` -:class:`MappingView` :class:`Sized` ``__len__`` -:class:`ItemsView` :class:`MappingView`, ``__contains__``, - :class:`Set` ``__iter__`` -:class:`KeysView` :class:`MappingView`, ``__contains__``, - :class:`Set` ``__iter__`` -:class:`ValuesView` :class:`MappingView`, ``__contains__``, ``__iter__`` - :class:`Collection` -:class:`Awaitable` ``__await__`` -:class:`Coroutine` :class:`Awaitable` ``send``, ``throw`` ``close`` -:class:`AsyncIterable` ``__aiter__`` -:class:`AsyncIterator` :class:`AsyncIterable` ``__anext__`` ``__aiter__`` -:class:`AsyncGenerator` :class:`AsyncIterator` ``asend``, ``athrow`` ``aclose``, ``__aiter__``, ``__anext__`` -========================== ====================== ======================= ==================================================== +:class:`MappingView` :class:`Sized` ``__len__`` +:class:`ItemsView` :class:`MappingView`, ``__contains__``, + :class:`Set` ``__iter__`` +:class:`KeysView` :class:`MappingView`, ``__contains__``, + :class:`Set` ``__iter__`` +:class:`ValuesView` :class:`MappingView`, ``__contains__``, ``__iter__`` + :class:`Collection` +:class:`Awaitable` [1]_ ``__await__`` +:class:`Coroutine` [1]_ :class:`Awaitable` ``send``, ``throw`` ``close`` +:class:`AsyncIterable` [1]_ ``__aiter__`` +:class:`AsyncIterator` [1]_ :class:`AsyncIterable` ``__anext__`` ``__aiter__`` +:class:`AsyncGenerator` [1]_ :class:`AsyncIterator` ``asend``, ``athrow`` ``aclose``, ``__aiter__``, ``__anext__`` +============================== ====================== ======================= ==================================================== + + +.. rubric:: Footnotes + +.. [1] These ABCs override :meth:`object.__subclasshook__` to support + testing an interface by verifying the required methods are present + and have not been set to :const:`None`. This only works for simple + interfaces. More complex interfaces require registration or direct + subclassing. + +.. [2] Checking ``isinstance(obj, Iterable)`` detects classes that are + registered as :class:`Iterable` or that have an :meth:`__iter__` + method, but it does not detect classes that iterate with the + :meth:`__getitem__` method. The only reliable way to determine + whether an object is :term:`iterable` is to call ``iter(obj)``. + + +Collections Abstract Base Classes -- Detailed Descriptions +---------------------------------------------------------- .. class:: Container @@ -244,8 +346,10 @@ ABC Inherits from Abstract Methods Mixin .. versionadded:: 3.6 +Examples and Recipes +-------------------- -These ABCs allow us to ask classes or instances if they provide +ABCs allow us to ask classes or instances if they provide particular functionality, for example:: size = None diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index b1779a5b238..8bf3cb6cb12 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -343,7 +343,7 @@ All of those tests treat missing elements as having zero counts so that ``Counter(a=1) == Counter(a=1, b=0)`` returns true. .. versionadded:: 3.10 - Rich comparison operations we were added + Rich comparison operations were added. .. versionchanged:: 3.10 In equality tests, missing elements are treated as having zero counts. @@ -1175,41 +1175,98 @@ variants of :func:`functools.lru_cache`: .. testcode:: - class LRU: + from time import time - def __init__(self, func, maxsize=128): + class TimeBoundedLRU: + "LRU Cache that invalidates and refreshes old entries." + + def __init__(self, func, maxsize=128, maxage=30): + self.cache = OrderedDict() # { args : (timestamp, result)} self.func = func self.maxsize = maxsize - self.cache = OrderedDict() + self.maxage = maxage def __call__(self, *args): if args in self.cache: - value = self.cache[args] self.cache.move_to_end(args) - return value - value = self.func(*args) - if len(self.cache) >= self.maxsize: - self.cache.popitem(False) - self.cache[args] = value - return value + timestamp, result = self.cache[args] + if time() - timestamp <= self.maxage: + return result + result = self.func(*args) + self.cache[args] = time(), result + if len(self.cache) > self.maxsize: + self.cache.popitem(0) + return result + + +.. testcode:: + + class MultiHitLRUCache: + """ LRU cache that defers caching a result until + it has been requested multiple times. + + To avoid flushing the LRU cache with one-time requests, + we don't cache until a request has been made more than once. + + """ + + def __init__(self, func, maxsize=128, maxrequests=4096, cache_after=1): + self.requests = OrderedDict() # { uncached_key : request_count } + self.cache = OrderedDict() # { cached_key : function_result } + self.func = func + self.maxrequests = maxrequests # max number of uncached requests + self.maxsize = maxsize # max number of stored return values + self.cache_after = cache_after + + def __call__(self, *args): + if args in self.cache: + self.cache.move_to_end(args) + return self.cache[args] + result = self.func(*args) + self.requests[args] = self.requests.get(args, 0) + 1 + if self.requests[args] <= self.cache_after: + self.requests.move_to_end(args) + if len(self.requests) > self.maxrequests: + self.requests.popitem(0) + else: + self.requests.pop(args, None) + self.cache[args] = result + if len(self.cache) > self.maxsize: + self.cache.popitem(0) + return result .. doctest:: :hide: >>> def square(x): - ... return x ** 2 + ... return x * x ... - >>> s = LRU(square, maxsize=5) - >>> actual = [(s(x), s(x)) for x in range(20)] - >>> expected = [(x**2, x**2) for x in range(20)] - >>> actual == expected + >>> f = MultiHitLRUCache(square, maxsize=4, maxrequests=6) + >>> list(map(f, range(10))) # First requests, don't cache + [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] + >>> f(4) # Cache the second request + 16 + >>> f(6) # Cache the second request + 36 + >>> f(2) # The first request aged out, so don't cache + 4 + >>> f(6) # Cache hit + 36 + >>> f(4) # Cache hit and move to front + 16 + >>> list(f.cache.values()) + [36, 16] + >>> set(f.requests).isdisjoint(f.cache) True - >>> actual = list(s.cache.items()) - >>> expected = [((x,), x**2) for x in range(15, 20)] - >>> actual == expected + >>> list(map(f, [9, 8, 7])) # Cache these second requests + [81, 64, 49] + >>> list(map(f, [7, 9])) # Cache hits + [49, 81] + >>> list(f.cache.values()) + [16, 64, 49, 81] + >>> set(f.requests).isdisjoint(f.cache) True - :class:`UserDict` objects ------------------------- diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 897efc2f544..b4213b45115 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -231,7 +231,7 @@ that :class:`ProcessPoolExecutor` will not work in the interactive interpreter. Calling :class:`Executor` or :class:`Future` methods from a callable submitted to a :class:`ProcessPoolExecutor` will result in deadlock. -.. class:: ProcessPoolExecutor(max_workers=None, mp_context=None, initializer=None, initargs=()) +.. class:: ProcessPoolExecutor(max_workers=None, mp_context=None, initializer=None, initargs=(), max_tasks_per_child=None) An :class:`Executor` subclass that executes calls asynchronously using a pool of at most *max_workers* processes. If *max_workers* is ``None`` or not @@ -252,6 +252,11 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. pending jobs will raise a :exc:`~concurrent.futures.process.BrokenProcessPool`, as well as any attempt to submit more jobs to the pool. + *max_tasks_per_child* is an optional argument that specifies the maximum + number of tasks a single process can execute before it will exit and be + replaced with a fresh worker process. The default *max_tasks_per_child* is + ``None`` which means worker processes will live as long as the pool. + .. versionchanged:: 3.3 When one of the worker processes terminates abruptly, a :exc:`BrokenProcessPool` error is now raised. Previously, behaviour @@ -264,6 +269,10 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. Added the *initializer* and *initargs* arguments. + .. versionchanged:: 3.11 + The *max_tasks_per_child* argument was added to allow users to + control the lifetime of workers in the pool. + .. _processpoolexecutor-example: diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index 2bb425930bb..d31452edb97 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -261,7 +261,8 @@ A configuration file consists of sections, each led by a ``[section]`` header, followed by key/value entries separated by a specific string (``=`` or ``:`` by default [1]_). By default, section names are case sensitive but keys are not [1]_. Leading and trailing whitespace is removed from keys and values. -Values can be omitted, in which case the key/value delimiter may also be left +Values can be omitted if the parser is configured to allow it [1]_, +in which case the key/value delimiter may also be left out. Values can also span multiple lines, as long as they are indented deeper than the first line of the value. Depending on the parser's mode, blank lines may be treated as parts of multiline values or ignored. @@ -1200,28 +1201,6 @@ ConfigParser Objects names is stripped before :meth:`optionxform` is called. - .. method:: readfp(fp, filename=None) - - .. deprecated:: 3.2 - Use :meth:`read_file` instead. - - .. versionchanged:: 3.2 - :meth:`readfp` now iterates on *fp* instead of calling ``fp.readline()``. - - For existing code calling :meth:`readfp` with arguments which don't - support iteration, the following generator may be used as a wrapper - around the file-like object:: - - def readline_generator(fp): - line = fp.readline() - while line: - yield line - line = fp.readline() - - Instead of ``parser.readfp(fp)`` use - ``parser.read_file(readline_generator(fp))``. - - .. data:: MAX_INTERPOLATION_DEPTH The maximum depth for recursive interpolation for :meth:`get` when the *raw* @@ -1359,6 +1338,9 @@ Exceptions The ``filename`` attribute and :meth:`__init__` argument were renamed to ``source`` for consistency. + .. versionchanged:: 3.11 + The deprecated ``filename`` attribute was removed. + .. rubric:: Footnotes diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index bc38a63a52d..bb93088b342 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -130,7 +130,9 @@ Functions and classes provided: either as decorators or with :keyword:`async with` statements:: import time + from contextlib import asynccontextmanager + @asynccontextmanager async def timeit(): now = time.monotonic() try: @@ -353,6 +355,23 @@ Functions and classes provided: .. versionadded:: 3.5 +.. function:: chdir(path) + + Non parallel-safe context manager to change the current working directory. + As this changes a global state, the working directory, it is not suitable + for use in most threaded or async contexts. It is also not suitable for most + non-linear code execution, like generators, where the program execution is + temporarily relinquished -- unless explicitely desired, you should not yield + when this context manager is active. + + This is a simple wrapper around :func:`~os.chdir`, it changes the current + working directory upon entering and restores the old one on exit. + + This context manager is :ref:`reentrant `. + + .. versionadded:: 3.11 + + .. class:: ContextDecorator() A base class that enables a context manager to also be used as a decorator. @@ -900,8 +919,8 @@ but may also be used *inside* a :keyword:`!with` statement that is already using the same context manager. :class:`threading.RLock` is an example of a reentrant context manager, as are -:func:`suppress` and :func:`redirect_stdout`. Here's a very simple example of -reentrant use:: +:func:`suppress`, :func:`redirect_stdout`, and :func:`chdir`. Here's a very +simple example of reentrant use:: >>> from contextlib import redirect_stdout >>> from io import StringIO diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 14ac47f4c9e..be1dd0c9eb5 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -94,7 +94,7 @@ Context Variables # var.get() would raise a LookupError. -.. class:: contextvars.Token +.. class:: Token *Token* objects are returned by the :meth:`ContextVar.set` method. They can be passed to the :meth:`ContextVar.reset` method to revert diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 01ebf198d7c..a8bc2fa55ea 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -60,7 +60,7 @@ The :func:`deepcopy` function avoids these problems by: components copied. This module does not copy types like module, method, stack trace, stack frame, -file, socket, window, array, or any similar types. It does "copy" functions and +file, socket, window, or any similar types. It does "copy" functions and classes (shallow and deeply), by returning the original object unchanged; this is compatible with the way these are treated by the :mod:`pickle` module. diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 899ce0225ce..3a7817cfdfa 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -383,6 +383,8 @@ Dialects support the following attributes: :const:`False`. On reading, the *escapechar* removes any special meaning from the following character. It defaults to :const:`None`, which disables escaping. + .. versionchanged:: 3.11 + An empty *escapechar* is not allowed. .. attribute:: Dialect.lineterminator @@ -402,6 +404,8 @@ Dialects support the following attributes: as the *delimiter* or *quotechar*, or which contain new-line characters. It defaults to ``'"'``. + .. versionchanged:: 3.11 + An empty *quotechar* is not allowed. .. attribute:: Dialect.quoting diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index fd6422cc8c0..6e147fc66eb 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -20,7 +20,7 @@ ctypes tutorial Note: The code samples in this tutorial use :mod:`doctest` to make sure that they actually work. Since some code samples behave differently under Linux, -Windows, or Mac OS X, they contain doctest directives in comments. +Windows, or macOS, they contain doctest directives in comments. Note: Some code samples reference the ctypes :class:`c_int` type. On platforms where ``sizeof(long) == sizeof(int)`` it is an alias to :class:`c_long`. @@ -80,7 +80,7 @@ the library by creating an instance of CDLL by calling the constructor:: >>> -.. XXX Add section for Mac OS X. +.. XXX Add section for macOS. .. _ctypes-accessing-functions-from-loaded-dlls: @@ -330,10 +330,9 @@ property:: 10 b'Hi\x00lo\x00\x00\x00\x00\x00' >>> -The :func:`create_string_buffer` function replaces the :func:`c_buffer` function -(which is still available as an alias), as well as the :func:`c_string` function -from earlier ctypes releases. To create a mutable memory block containing -unicode characters of the C type :c:type:`wchar_t` use the +The :func:`create_string_buffer` function replaces the old :func:`c_buffer` +function (which is still available as an alias). To create a mutable memory +block containing unicode characters of the C type :c:type:`wchar_t`, use the :func:`create_unicode_buffer` function. @@ -1088,7 +1087,9 @@ size, we show only how this table can be read with :mod:`ctypes`:: >>> class struct_frozen(Structure): ... _fields_ = [("name", c_char_p), ... ("code", POINTER(c_ubyte)), - ... ("size", c_int)] + ... ("size", c_int), + ... ("get_code", POINTER(c_ubyte)), # Function pointer + ... ] ... >>> @@ -1096,7 +1097,7 @@ We have defined the :c:type:`struct _frozen` data type, so we can get the pointe to the table:: >>> FrozenTable = POINTER(struct_frozen) - >>> table = FrozenTable.in_dll(pythonapi, "PyImport_FrozenModules") + >>> table = FrozenTable.in_dll(pythonapi, "_PyImport_FrozenBootstrap") >>> Since ``table`` is a ``pointer`` to the array of ``struct_frozen`` records, we @@ -1112,9 +1113,7 @@ hit the ``NULL`` entry:: ... _frozen_importlib 31764 _frozen_importlib_external 41499 - __hello__ 161 - __phello__ -161 - __phello__.spam 161 + zipimport 12345 >>> The fact that standard Python has a frozen module and a frozen package @@ -1288,7 +1287,7 @@ Here are some examples:: 'libbz2.so.1.0' >>> -On OS X, :func:`find_library` tries several predefined naming schemes and paths +On macOS, :func:`find_library` tries several predefined naming schemes and paths to locate the library, and returns a full pathname if successful:: >>> from ctypes.util import find_library @@ -1320,7 +1319,7 @@ There are several ways to load shared libraries into the Python process. One way is to instantiate one of the following classes: -.. class:: CDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=0) +.. class:: CDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) Instances of this class represent loaded shared libraries. Functions in these libraries use the standard C calling convention, and are assumed to return @@ -1342,7 +1341,7 @@ way is to instantiate one of the following classes: -- A tool to find DLL dependents. -.. class:: OleDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=0) +.. class:: OleDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) Windows only: Instances of this class represent loaded shared libraries, functions in these libraries use the ``stdcall`` calling convention, and are @@ -1355,7 +1354,7 @@ way is to instantiate one of the following classes: :exc:`WindowsError` used to be raised. -.. class:: WinDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=0) +.. class:: WinDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) Windows only: Instances of this class represent loaded shared libraries, functions in these libraries use the ``stdcall`` calling convention, and are diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index efbece437af..37e822c0e2b 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -368,7 +368,7 @@ The module :mod:`curses` defines the following functions: Set the maximum time in milliseconds that can elapse between press and release events in order for them to be recognized as a click, and return the previous - interval value. The default value is 200 msec, or one fifth of a second. + interval value. The default value is 200 milliseconds, or one fifth of a second. .. function:: mousemask(mousemask) diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 4ee75f1cc5f..6bc493c9577 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -205,7 +205,7 @@ Module contents follows a field with a default value. This is true whether this occurs in a single class, or as a result of class inheritance. -.. function:: field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, hash=None, compare=True, metadata=None, kw_only=MISSING): +.. function:: field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, hash=None, compare=True, metadata=None, kw_only=MISSING) For common and simple use cases, no other functionality is required. There are, however, some dataclass features that @@ -221,10 +221,9 @@ Module contents c.mylist += [1, 2, 3] As shown above, the :const:`MISSING` value is a sentinel object used to - detect if the ``default`` and ``default_factory`` parameters are - provided. This sentinel is used because ``None`` is a valid value - for ``default``. No code should directly use the :const:`MISSING` - value. + detect if some parameters are provided by the user. This sentinel is + used because ``None`` is a valid value for some parameters with + a distinct meaning. No code should directly use the :const:`MISSING` value. The parameters to :func:`field` are: @@ -325,7 +324,10 @@ Module contents Converts the dataclass ``instance`` to a dict (by using the factory function ``dict_factory``). Each dataclass is converted to a dict of its fields, as ``name: value`` pairs. dataclasses, dicts, - lists, and tuples are recursed into. For example:: + lists, and tuples are recursed into. Other objects are copied with + :func:`copy.deepcopy`. + + Example of using :func:`asdict` on nested dataclasses:: @dataclass class Point: @@ -342,21 +344,32 @@ Module contents c = C([Point(0, 0), Point(10, 4)]) assert asdict(c) == {'mylist': [{'x': 0, 'y': 0}, {'x': 10, 'y': 4}]} - Raises :exc:`TypeError` if ``instance`` is not a dataclass instance. + To create a shallow copy, the following workaround may be used:: + + dict((field.name, getattr(instance, field.name)) for field in fields(instance)) + + :func:`asdict` raises :exc:`TypeError` if ``instance`` is not a dataclass + instance. .. function:: astuple(instance, *, tuple_factory=tuple) Converts the dataclass ``instance`` to a tuple (by using the factory function ``tuple_factory``). Each dataclass is converted to a tuple of its field values. dataclasses, dicts, lists, and - tuples are recursed into. + tuples are recursed into. Other objects are copied with + :func:`copy.deepcopy`. Continuing from the previous example:: assert astuple(p) == (10, 20) assert astuple(c) == ([(0, 0), (10, 4)],) - Raises :exc:`TypeError` if ``instance`` is not a dataclass instance. + To create a shallow copy, the following workaround may be used:: + + tuple(getattr(instance, field.name) for field in dataclasses.fields(instance)) + + :func:`astuple` raises :exc:`TypeError` if ``instance`` is not a dataclass + instance. .. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False) @@ -640,74 +653,80 @@ re-ordered :meth:`__init__` parameter list. Default factory functions ------------------------- - If a :func:`field` specifies a ``default_factory``, it is called with - zero arguments when a default value for the field is needed. For - example, to create a new instance of a list, use:: +If a :func:`field` specifies a ``default_factory``, it is called with +zero arguments when a default value for the field is needed. For +example, to create a new instance of a list, use:: - mylist: list = field(default_factory=list) + mylist: list = field(default_factory=list) - If a field is excluded from :meth:`__init__` (using ``init=False``) - and the field also specifies ``default_factory``, then the default - factory function will always be called from the generated - :meth:`__init__` function. This happens because there is no other - way to give the field an initial value. +If a field is excluded from :meth:`__init__` (using ``init=False``) +and the field also specifies ``default_factory``, then the default +factory function will always be called from the generated +:meth:`__init__` function. This happens because there is no other +way to give the field an initial value. Mutable default values ---------------------- - Python stores default member variable values in class attributes. - Consider this example, not using dataclasses:: +Python stores default member variable values in class attributes. +Consider this example, not using dataclasses:: - class C: - x = [] - def add(self, element): - self.x.append(element) + class C: + x = [] + def add(self, element): + self.x.append(element) - o1 = C() - o2 = C() - o1.add(1) - o2.add(2) - assert o1.x == [1, 2] - assert o1.x is o2.x + o1 = C() + o2 = C() + o1.add(1) + o2.add(2) + assert o1.x == [1, 2] + assert o1.x is o2.x - Note that the two instances of class ``C`` share the same class - variable ``x``, as expected. +Note that the two instances of class ``C`` share the same class +variable ``x``, as expected. - Using dataclasses, *if* this code was valid:: +Using dataclasses, *if* this code was valid:: - @dataclass - class D: - x: List = [] - def add(self, element): - self.x += element + @dataclass + class D: + x: List = [] + def add(self, element): + self.x += element - it would generate code similar to:: +it would generate code similar to:: - class D: - x = [] - def __init__(self, x=x): - self.x = x - def add(self, element): - self.x += element + class D: + x = [] + def __init__(self, x=x): + self.x = x + def add(self, element): + self.x += element - assert D().x is D().x + assert D().x is D().x - This has the same issue as the original example using class ``C``. - That is, two instances of class ``D`` that do not specify a value - for ``x`` when creating a class instance will share the same copy - of ``x``. Because dataclasses just use normal Python class - creation they also share this behavior. There is no general way - for Data Classes to detect this condition. Instead, the - :func:`dataclass` decorator will raise a :exc:`TypeError` if it - detects a default parameter of type ``list``, ``dict``, or ``set``. - This is a partial solution, but it does protect against many common - errors. +This has the same issue as the original example using class ``C``. +That is, two instances of class ``D`` that do not specify a value +for ``x`` when creating a class instance will share the same copy +of ``x``. Because dataclasses just use normal Python class +creation they also share this behavior. There is no general way +for Data Classes to detect this condition. Instead, the +:func:`dataclass` decorator will raise a :exc:`TypeError` if it +detects an unhashable default parameter. The assumption is that if +a value is unhashable, it is mutable. This is a partial solution, +but it does protect against many common errors. - Using default factory functions is a way to create new instances of - mutable types as default values for fields:: +Using default factory functions is a way to create new instances of +mutable types as default values for fields:: - @dataclass - class D: - x: list = field(default_factory=list) + @dataclass + class D: + x: list = field(default_factory=list) - assert D().x is not D().x + assert D().x is not D().x + +.. versionchanged:: 3.11 + Instead of looking for and disallowing objects of type ``list``, + ``dict``, or ``set``, unhashable objects are now not allowed as + default values. Unhashability is used to approximate + mutability. diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index dae0dd7aa55..217cdf222b8 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -27,6 +27,9 @@ on efficient attribute extraction for output formatting and manipulation. Module :mod:`time` Time access and conversions. + Module :mod:`zoneinfo` + Concrete time zones representing the IANA time zone database. + Package `dateutil `_ Third-party library with expanded time zone and parsing support. @@ -2174,14 +2177,13 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. seealso:: - `dateutil.tz `_ + :mod:`zoneinfo` The :mod:`datetime` module has a basic :class:`timezone` class (for handling arbitrary fixed offsets from UTC) and its :attr:`timezone.utc` attribute (a UTC timezone instance). - *dateutil.tz* library brings the *IANA timezone database* - (also known as the Olson database) to Python, and its usage is - recommended. + ``zoneinfo`` brings the *IANA timezone database* (also known as the Olson + database) to Python, and its usage is recommended. `IANA timezone database `_ The Time Zone Database (often called tz, tzdata or zoneinfo) contains code @@ -2357,8 +2359,8 @@ requires, and these work on all platforms with a standard C implementation. | | decimal number. | | \(9) | +-----------+--------------------------------+------------------------+-------+ | ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(5) | -| | number, zero-padded on the | 999999 | | -| | left. | | | +| | number, zero-padded to 6 | 999999 | | +| | digits. | | | +-----------+--------------------------------+------------------------+-------+ | ``%z`` | UTC offset in the form | (empty), +0000, | \(6) | | | ``±HHMM[SS[.ffffff]]`` (empty | -0400, +1030, | | @@ -2431,7 +2433,8 @@ incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. The full set of format codes supported varies across platforms, because Python calls the platform C library's :func:`strftime` function, and platform variations are common. To see the full set of format codes supported on your -platform, consult the :manpage:`strftime(3)` documentation. +platform, consult the :manpage:`strftime(3)` documentation. There are also +differences between platforms in handling of unsupported format specifiers. .. versionadded:: 3.6 ``%G``, ``%u`` and ``%V`` were added. diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 57ae547b833..2be499337a2 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -33,6 +33,8 @@ the Oracle Berkeley DB. file's format can't be guessed; or a string containing the required module name, such as ``'dbm.ndbm'`` or ``'dbm.gnu'``. +.. versionchanged:: 3.11 + Accepts :term:`path-like object` for filename. .. function:: open(file, flag='r', mode=0o666) @@ -77,6 +79,9 @@ available, as well as :meth:`get` and :meth:`setdefault`. Deleting a key from a read-only database raises database module specific error instead of :exc:`KeyError`. +.. versionchanged:: 3.11 + Accepts :term:`path-like object` for file. + Key and values are always stored as bytes. This means that when strings are used they are implicitly converted to the default encoding before being stored. @@ -202,6 +207,9 @@ supported. In addition to the dictionary-like methods, ``gdbm`` objects have the following methods: + .. versionchanged:: 3.11 + Accepts :term:`path-like object` for filename. + .. method:: gdbm.firstkey() It's possible to loop over every key in the database using this method and the @@ -216,7 +224,7 @@ supported. contains them all:: k = db.firstkey() - while k != None: + while k is not None: print(k) k = db.nextkey(k) @@ -298,6 +306,9 @@ to locate the appropriate header file to simplify building this module. In addition to the dictionary-like methods, ``ndbm`` objects provide the following method: + .. versionchanged:: 3.11 + Accepts :term:`path-like object` for filename. + .. method:: ndbm.close() Close the ``ndbm`` database. @@ -379,6 +390,9 @@ The module defines the following: flags ``'r'`` and ``'w'`` no longer creates a database if it does not exist. + .. versionchanged:: 3.11 + Accepts :term:`path-like object` for filename. + In addition to the methods provided by the :class:`collections.abc.MutableMapping` class, :class:`dumbdbm` objects provide the following methods: diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 21747069b3a..9665e8dbf6c 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -293,12 +293,13 @@ details of bytecode instructions as :class:`Instruction` instances: .. data:: argval - resolved arg value (if known), otherwise same as arg + resolved arg value (if any), otherwise ``None`` .. data:: argrepr - human readable description of operation argument + human readable description of operation argument (if any), + otherwise an empty string. .. data:: offset @@ -405,156 +406,29 @@ result back on the stack. .. versionadded:: 3.5 -**Binary operations** +**Binary and in-place operations** Binary operations remove the top of the stack (TOS) and the second top-most stack item (TOS1) from the stack. They perform the operation, and put the result back on the stack. -.. opcode:: BINARY_POWER - - Implements ``TOS = TOS1 ** TOS``. - - -.. opcode:: BINARY_MULTIPLY - - Implements ``TOS = TOS1 * TOS``. - - -.. opcode:: BINARY_MATRIX_MULTIPLY - - Implements ``TOS = TOS1 @ TOS``. - - .. versionadded:: 3.5 - - -.. opcode:: BINARY_FLOOR_DIVIDE - - Implements ``TOS = TOS1 // TOS``. - - -.. opcode:: BINARY_TRUE_DIVIDE - - Implements ``TOS = TOS1 / TOS``. - - -.. opcode:: BINARY_MODULO - - Implements ``TOS = TOS1 % TOS``. - - -.. opcode:: BINARY_ADD - - Implements ``TOS = TOS1 + TOS``. - - -.. opcode:: BINARY_SUBTRACT - - Implements ``TOS = TOS1 - TOS``. - - -.. opcode:: BINARY_SUBSCR - - Implements ``TOS = TOS1[TOS]``. - - -.. opcode:: BINARY_LSHIFT - - Implements ``TOS = TOS1 << TOS``. - - -.. opcode:: BINARY_RSHIFT - - Implements ``TOS = TOS1 >> TOS``. - - -.. opcode:: BINARY_AND - - Implements ``TOS = TOS1 & TOS``. - - -.. opcode:: BINARY_XOR - - Implements ``TOS = TOS1 ^ TOS``. - - -.. opcode:: BINARY_OR - - Implements ``TOS = TOS1 | TOS``. - - -**In-place operations** - In-place operations are like binary operations, in that they remove TOS and TOS1, and push the result back on the stack, but the operation is done in-place when TOS1 supports it, and the resulting TOS may be (but does not have to be) the original TOS1. -.. opcode:: INPLACE_POWER - Implements in-place ``TOS = TOS1 ** TOS``. +.. opcode:: BINARY_OP (op) + + Implements the binary and in-place operators (depending on the value of + *op*). + + .. versionadded:: 3.11 -.. opcode:: INPLACE_MULTIPLY +.. opcode:: BINARY_SUBSCR - Implements in-place ``TOS = TOS1 * TOS``. - - -.. opcode:: INPLACE_MATRIX_MULTIPLY - - Implements in-place ``TOS = TOS1 @ TOS``. - - .. versionadded:: 3.5 - - -.. opcode:: INPLACE_FLOOR_DIVIDE - - Implements in-place ``TOS = TOS1 // TOS``. - - -.. opcode:: INPLACE_TRUE_DIVIDE - - Implements in-place ``TOS = TOS1 / TOS``. - - -.. opcode:: INPLACE_MODULO - - Implements in-place ``TOS = TOS1 % TOS``. - - -.. opcode:: INPLACE_ADD - - Implements in-place ``TOS = TOS1 + TOS``. - - -.. opcode:: INPLACE_SUBTRACT - - Implements in-place ``TOS = TOS1 - TOS``. - - -.. opcode:: INPLACE_LSHIFT - - Implements in-place ``TOS = TOS1 << TOS``. - - -.. opcode:: INPLACE_RSHIFT - - Implements in-place ``TOS = TOS1 >> TOS``. - - -.. opcode:: INPLACE_AND - - Implements in-place ``TOS = TOS1 & TOS``. - - -.. opcode:: INPLACE_XOR - - Implements in-place ``TOS = TOS1 ^ TOS``. - - -.. opcode:: INPLACE_OR - - Implements in-place ``TOS = TOS1 | TOS``. + Implements ``TOS = TOS1[TOS]``. .. opcode:: STORE_SUBSCR @@ -710,12 +584,14 @@ iterations of the loop. .. opcode:: WITH_EXCEPT_START - Calls the function in position 7 on the stack with the top three + Calls the function in position 8 on the stack with the top three items on the stack as arguments. Used to implement the call ``context_manager.__exit__(*exc_info())`` when an exception has occurred in a :keyword:`with` statement. .. versionadded:: 3.9 + .. versionchanged:: 3.11 + The ``__exit__`` function is in position 8 of the stack rather than 7. .. opcode:: POP_EXCEPT_AND_RERAISE @@ -724,7 +600,7 @@ iterations of the loop. of the stack and sets the ``f_lasti`` attribute of the frame with that value. Then pops the next exception from the stack uses it to restore the current exception. Finally it re-raises the originally popped exception. - Used in excpetion handler cleanup. + Used in exception handler cleanup. .. versionadded:: 3.11 @@ -754,15 +630,6 @@ iterations of the loop. .. versionadded:: 3.11 -.. opcode:: COPY_DICT_WITHOUT_KEYS - - TOS is a tuple of mapping keys, and TOS1 is the match subject. Replace TOS - with a :class:`dict` formed from the items of TOS1, but without any of the - keys in TOS. - - .. versionadded:: 3.10 - - .. opcode:: GET_LEN Push ``len(TOS)`` onto the stack. @@ -794,11 +661,14 @@ iterations of the loop. TOS is a tuple of mapping keys, and TOS1 is the match subject. If TOS1 contains all of the keys in TOS, push a :class:`tuple` containing the - corresponding values, followed by ``True``. Otherwise, push ``None``, - followed by ``False``. + corresponding values. Otherwise, push ``None``. .. versionadded:: 3.10 + .. versionchanged:: 3.11 + Previously, this instruction also pushed a boolean value indicating + success (``True``) or failure (``False``). + All of the following opcodes use their arguments. @@ -1005,10 +875,13 @@ All of the following opcodes use their arguments. .. opcode:: JUMP_IF_NOT_EXC_MATCH (target) Tests whether the second value on the stack is an exception matching TOS, - and jumps if it is not. Pops two values from the stack. + and jumps if it is not. Pops one value from the stack. .. versionadded:: 3.9 + .. versionchanged:: 3.11 + This opcode no longer pops the active exception. + .. opcode:: JUMP_IF_TRUE_OR_POP (target) @@ -1120,6 +993,15 @@ All of the following opcodes use their arguments. ``i`` is no longer offset by the length of ``co_varnames``. +.. opcode:: COPY_FREE_VARS (n) + + Copies the ``n`` free variables from the closure into the frame. + Removes the need for special code on the caller's side when calling + closures. + + .. versionadded:: 3.11 + + .. opcode:: RAISE_VARARGS (argc) Raises an exception using one of the 3 forms of the ``raise`` statement, @@ -1276,17 +1158,20 @@ All of the following opcodes use their arguments. against, and TOS2 is the match subject. *count* is the number of positional sub-patterns. - Pop TOS. If TOS2 is an instance of TOS1 and has the positional and keyword - attributes required by *count* and TOS, set TOS to ``True`` and TOS1 to a - tuple of extracted attributes. Otherwise, set TOS to ``False``. + Pop TOS, TOS1, and TOS2. If TOS2 is an instance of TOS1 and has the + positional and keyword attributes required by *count* and TOS, push a tuple + of extracted attributes. Otherwise, push ``None``. .. versionadded:: 3.10 + .. versionchanged:: 3.11 + Previously, this instruction also pushed a boolean value indicating + success (``True``) or failure (``False``). + .. opcode:: GEN_START (kind) - Pops TOS. If TOS was not ``None``, raises an exception. The ``kind`` - operand corresponds to the type of generator or coroutine and determines - the error message. The legal kinds are 0 for generator, 1 for coroutine, + Pops TOS. The ``kind`` operand corresponds to the type of generator or + coroutine. The legal kinds are 0 for generator, 1 for coroutine, and 2 for async generator. .. versionadded:: 3.10 @@ -1300,6 +1185,14 @@ All of the following opcodes use their arguments. .. versionadded:: 3.10 +.. opcode:: COPY (i) + + Push the *i*-th item to the top of the stack. The item is not removed from its + original location. + + .. versionadded:: 3.11 + + .. opcode:: HAVE_ARGUMENT This is not really an opcode. It identifies the dividing line between diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index a77322f83ac..0bbb640bea2 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -485,25 +485,24 @@ Some details you should read once, but won't need to remember: .. index:: single: ^ (caret); marker -* For some :exc:`SyntaxError`\ s, Python displays the character position of the - syntax error, using a ``^`` marker:: +* For some exceptions, Python displays the position of the error using ``^`` + markers and tildes:: - >>> 1 1 + >>> 1 + None File "", line 1 - 1 1 - ^ - SyntaxError: invalid syntax + 1 + None + ~~^~~~~~ + TypeError: unsupported operand type(s) for +: 'int' and 'NoneType' Since the lines showing the position of the error come before the exception type and detail, they are not checked by doctest. For example, the following test would pass, even though it puts the ``^`` marker in the wrong location:: - >>> 1 1 - Traceback (most recent call last): + >>> 1 + None File "", line 1 - 1 1 - ^ - SyntaxError: invalid syntax + 1 + None + ^~~~~~~~ + TypeError: unsupported operand type(s) for +: 'int' and 'NoneType' .. _option-flags-and-directives: diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index f28505a6d2c..572048a255b 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -162,7 +162,8 @@ Data Types .. method:: EnumType.__dir__(cls) Returns ``['__class__', '__doc__', '__members__', '__module__']`` and the - names of the members in *cls*:: + names of the members in ``cls``. User-defined methods and methods from + mixin classes will also be included:: >>> dir(Color) ['BLUE', 'GREEN', 'RED', '__class__', '__doc__', '__members__', '__module__'] @@ -260,7 +261,7 @@ Data Types .. method:: Enum.__dir__(self) Returns ``['__class__', '__doc__', '__module__', 'name', 'value']`` and - any public methods defined on *self.__class__*:: + any public methods defined on ``self.__class__`` or a mixin class:: >>> from datetime import date >>> class Weekday(Enum): @@ -397,6 +398,7 @@ Data Types Using :class:`auto` with :class:`StrEnum` results in values of the member name, lower-cased. +.. versionadded:: 3.11 .. class:: Flag @@ -651,8 +653,8 @@ Data Types --------------- -Utilites and Decorators ------------------------ +Utilities and Decorators +------------------------ .. class:: auto diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 669979f7c5d..12d7d8abb26 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -34,6 +34,10 @@ class or one of its subclasses, and not from :exc:`BaseException`. More information on defining exceptions is available in the Python Tutorial under :ref:`tut-userexceptions`. + +Exception context +----------------- + When raising (or re-raising) an exception in an :keyword:`except` or :keyword:`finally` clause :attr:`__context__` is automatically set to the last exception caught; if the @@ -67,6 +71,25 @@ exceptions so that the final line of the traceback always shows the last exception that was raised. +Inheriting from built-in exceptions +----------------------------------- + +User code can create subclasses that inherit from an exception type. +It's recommended to only subclass one exception type at a time to avoid +any possible conflicts between how the bases handle the ``args`` +attribute, as well as due to possible memory layout incompatibilities. + +.. impl-detail:: + + Most built-in exceptions are implemented in C for efficiency, see: + :source:`Objects/exceptions.c`. Some have custom memory layouts + which makes it impossible to create a subclass that inherits from + multiple exception types. The memory layout of a type is an implementation + detail and might change between Python versions, leading to new + conflicts in the future. Therefore, it's recommended to avoid + subclassing multiple exception types altogether. + + Base classes ------------ @@ -104,6 +127,14 @@ The following exceptions are used mostly as base classes for other exceptions. tb = sys.exc_info()[2] raise OtherException(...).with_traceback(tb) + .. attribute:: __note__ + + A mutable field which is :const:`None` by default and can be set to a string. + If it is not :const:`None`, it is included in the traceback. This field can + be used to enrich exceptions after they have been caught. + + .. versionadded:: 3.11 + .. exception:: Exception @@ -748,6 +779,8 @@ The following exceptions are used as warning categories; see the (:pep:`565`). Enabling the :ref:`Python Development Mode ` shows this warning. + The deprecation policy is described in :pep:`387`. + .. exception:: PendingDeprecationWarning @@ -762,6 +795,8 @@ The following exceptions are used as warning categories; see the Ignored by the default warning filters. Enabling the :ref:`Python Development Mode ` shows this warning. + The deprecation policy is described in :pep:`387`. + .. exception:: SyntaxWarning diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index 9d8021150c4..846c8d7ba96 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -44,6 +44,11 @@ descriptor. ``F_SETPIPE_SZ`` constants, which allow to check and modify a pipe's size respectively. +.. versionchanged:: 3.11 + On FreeBSD, the fcntl module exposes the ``F_DUP2FD`` and ``F_DUP2FD_CLOEXEC`` + constants, which allow to duplicate a file descriptor, the latter setting + ``FD_CLOEXEC`` flag in addition. + The module defines the following functions: diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index 3880ed3d2bf..2a2c1b3c2b9 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -50,8 +50,8 @@ You can control how files are opened by providing an opening hook via the *openhook* parameter to :func:`fileinput.input` or :class:`FileInput()`. The hook must be a function that takes two arguments, *filename* and *mode*, and returns an accordingly opened file-like object. If *encoding* and/or *errors* -are specified, they will be passed to the hook as aditional keyword arguments. -This module provides a :func:`hook_encoded` to support compressed files. +are specified, they will be passed to the hook as additional keyword arguments. +This module provides a :func:`hook_compressed` to support compressed files. The following function is the primary interface of this module: @@ -146,14 +146,13 @@ available for subclassing as well: Class :class:`FileInput` is the implementation; its methods :meth:`filename`, :meth:`fileno`, :meth:`lineno`, :meth:`filelineno`, :meth:`isfirstline`, :meth:`isstdin`, :meth:`nextfile` and :meth:`close` correspond to the - functions of the same name in the module. In addition it has a - :meth:`~io.TextIOBase.readline` method which returns the next input line, - and a :meth:`__getitem__` method which implements the sequence behavior. - The sequence must be accessed in strictly sequential order; random access - and :meth:`~io.TextIOBase.readline` cannot be mixed. + functions of the same name in the module. In addition it is :term:`iterable` + and has a :meth:`~io.TextIOBase.readline` method which returns the next + input line. The sequence must be accessed in strictly sequential order; + random access and :meth:`~io.TextIOBase.readline` cannot be mixed. With *mode* you can specify which file mode will be passed to :func:`open`. It - must be one of ``'r'``, ``'rU'``, ``'U'`` and ``'rb'``. + must be one of ``'r'`` and ``'rb'``. The *openhook*, when given, must be a function that takes two arguments, *filename* and *mode*, and returns an accordingly opened file-like object. You @@ -171,18 +170,16 @@ available for subclassing as well: .. versionchanged:: 3.2 Can be used as a context manager. - .. deprecated:: 3.4 - The ``'rU'`` and ``'U'`` modes. - - .. deprecated:: 3.8 - Support for :meth:`__getitem__` method is deprecated. - .. versionchanged:: 3.8 The keyword parameter *mode* and *openhook* are now keyword-only. .. versionchanged:: 3.10 The keyword-only parameter *encoding* and *errors* are added. + .. versionchanged:: 3.11 + The ``'rU'`` and ``'U'`` modes and the :meth:`__getitem__` method have + been removed. + **Optional in-place filtering:** if the keyword argument ``inplace=True`` is passed to :func:`fileinput.input` or to the :class:`FileInput` constructor, the diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index d04de8f8e95..c893f2d5389 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -94,6 +94,10 @@ another rational number, or from a string. Underscores are now permitted when creating a :class:`Fraction` instance from a string, following :PEP:`515` rules. + .. versionchanged:: 3.11 + :class:`Fraction` implements ``__int__`` now to satisfy + ``typing.SupportsInt`` instance checks. + .. attribute:: numerator Numerator of the Fraction in lowest term. diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 652e30c6088..ebcd6c801d7 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -66,9 +66,6 @@ are always available. They are listed here in alphabetical order. Return an :term:`asynchronous iterator` for an :term:`asynchronous iterable`. Equivalent to calling ``x.__aiter__()``. - ``aiter(x)`` itself has an ``__aiter__()`` method that returns ``x``, - so ``aiter(aiter(x))`` is the same as ``aiter(x)``. - Note: Unlike :func:`iter`, :func:`aiter` has no 2-argument variant. .. versionadded:: 3.10 @@ -557,7 +554,7 @@ are always available. They are listed here in alphabetical order. a suite of Python statements which is then executed (unless a syntax error occurs). [#]_ If it is a code object, it is simply executed. In all cases, the code that's executed is expected to be valid as file input (see the - section "File input" in the Reference Manual). Be aware that the + section :ref:`file-input` in the Reference Manual). Be aware that the :keyword:`nonlocal`, :keyword:`yield`, and :keyword:`return` statements may not be used outside of function definitions even within the context of code passed to the @@ -736,9 +733,9 @@ are always available. They are listed here in alphabetical order. .. function:: globals() - Return a dictionary representing the current global symbol table. This is always - the dictionary of the current module (inside a function or method, this is the - module where it is defined, not the module from which it is called). + Return the dictionary implementing the current module namespace. For code within + functions, this is set when the function is defined and remains the same + regardless of where the function is called. .. function:: hasattr(object, name) @@ -916,9 +913,9 @@ are always available. They are listed here in alphabetical order. Return ``True`` if *class* is a subclass (direct, indirect, or :term:`virtual `) of *classinfo*. A class is considered a subclass of itself. *classinfo* may be a tuple of class - objects or a :ref:`types-union`, in which case every entry in *classinfo* - will be checked. In any other - case, a :exc:`TypeError` exception is raised. + objects or a :ref:`types-union`, in which case return ``True`` if *class* is a + subclass of any entry in *classinfo*. In any other case, a :exc:`TypeError` + exception is raised. .. versionchanged:: 3.10 *classinfo* can be a :ref:`types-union`. @@ -929,8 +926,8 @@ are always available. They are listed here in alphabetical order. Return an :term:`iterator` object. The first argument is interpreted very differently depending on the presence of the second argument. Without a second argument, *object* must be a collection object which supports the - iteration protocol (the :meth:`__iter__` method), or it must support the - sequence protocol (the :meth:`__getitem__` method with integer arguments + :term:`iterable` protocol (the :meth:`__iter__` method), or it must support + the sequence protocol (the :meth:`__getitem__` method with integer arguments starting at ``0``). If it does not support either of those protocols, :exc:`TypeError` is raised. If the second argument, *sentinel*, is given, then *object* must be a callable object. The iterator created in this case @@ -1060,7 +1057,7 @@ are always available. They are listed here in alphabetical order. .. function:: next(iterator[, default]) - Retrieve the next item from the *iterator* by calling its + Retrieve the next item from the :term:`iterator` by calling its :meth:`~iterator.__next__` method. If *default* is given, it is returned if the iterator is exhausted, otherwise :exc:`StopIteration` is raised. @@ -1156,12 +1153,6 @@ are always available. They are listed here in alphabetical order. first decoded using a platform-dependent encoding or using the specified *encoding* if given. - There is an additional mode character permitted, ``'U'``, which no longer - has any effect, and is considered deprecated. It previously enabled - :term:`universal newlines` in text mode, which became the default behavior - in Python 3.0. Refer to the documentation of the - :ref:`newline ` parameter for further details. - .. note:: Python doesn't depend on the underlying operating system's notion of text @@ -1304,8 +1295,7 @@ are always available. They are listed here in alphabetical order. The ``mode`` and ``flags`` arguments may have been modified or inferred from the original call. - .. versionchanged:: - 3.3 + .. versionchanged:: 3.3 * The *opener* parameter was added. * The ``'x'`` mode was added. @@ -1313,30 +1303,26 @@ are always available. They are listed here in alphabetical order. * :exc:`FileExistsError` is now raised if the file opened in exclusive creation mode (``'x'``) already exists. - .. versionchanged:: - 3.4 + .. versionchanged:: 3.4 * The file is now non-inheritable. - .. deprecated-removed:: 3.4 3.10 - - The ``'U'`` mode. - - .. versionchanged:: - 3.5 + .. versionchanged:: 3.5 * If the system call is interrupted and the signal handler does not raise an exception, the function now retries the system call instead of raising an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). * The ``'namereplace'`` error handler was added. - .. versionchanged:: - 3.6 + .. versionchanged:: 3.6 * Support added to accept objects implementing :class:`os.PathLike`. * On Windows, opening a console buffer may return a subclass of :class:`io.RawIOBase` other than :class:`io.FileIO`. + .. versionchanged:: 3.11 + The ``'U'`` mode has been removed. + .. function:: ord(c) Given a string representing one Unicode character, return an integer @@ -1356,8 +1342,11 @@ are always available. They are listed here in alphabetical order. coercion rules for binary arithmetic operators apply. For :class:`int` operands, the result has the same type as the operands (after coercion) unless the second argument is negative; in that case, all arguments are - converted to float and a float result is delivered. For example, ``10**2`` - returns ``100``, but ``10**-2`` returns ``0.01``. + converted to float and a float result is delivered. For example, ``pow(10, 2)`` + returns ``100``, but ``pow(10, -2)`` returns ``0.01``. For a negative base of + type :class:`int` or :class:`float` and a non-integral exponent, a complex + result is delivered. For example, ``pow(-9, 0.5)`` returns a value close + to ``3j``. For :class:`int` operands *base* and *exp*, if *mod* is present, *mod* must also be of integer type and *mod* must be nonzero. If *mod* is present and @@ -1603,6 +1592,15 @@ are always available. They are listed here in alphabetical order. compare equal --- this is helpful for sorting in multiple passes (for example, sort by department, then by salary grade). + The sort algorithm uses only ``<`` comparisons between items. While + defining an :meth:`~object.__lt__` method will suffice for sorting, + :PEP:`8` recommends that all six :ref:`rich comparisons + ` be implemented. This will help avoid bugs when using + the same data with other ordering tools such as :func:`max` that rely + on a different underlying method. Implementing all six comparisons + also helps avoid confusion for mixed type comparisons which can call + reflected the :meth:`~object.__gt__` method. + For sorting examples and a brief sorting tutorial, see :ref:`sortinghowto`. .. decorator:: staticmethod diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index e40fe7bb85f..c78818bfab1 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -160,10 +160,16 @@ The :mod:`functools` module defines the following functions: grow without bound. If *typed* is set to true, function arguments of different types will be - cached separately. For example, ``f(3)`` and ``f(3.0)`` will always be - treated as distinct calls with distinct results. If *typed* is false, - the implementation will usually but not always regard them as equivalent - calls and only cache a single result. + cached separately. If *typed* is false, the implementation will usually + regard them as equivalent calls and only cache a single result. (Some + types such as *str* and *int* may be cached separately even when *typed* + is false.) + + Note, type specificity applies only to the function's immediate arguments + rather than their contents. The scalar arguments, ``Decimal(42)`` and + ``Fraction(42)`` are be treated as distinct calls with distinct results. + In contrast, the tuple arguments ``('answer', Decimal(42))`` and + ``('answer', Fraction(42))`` are treated as equivalent. The wrapped function is instrumented with a :func:`cache_parameters` function that returns a new :class:`dict` showing the values for *maxsize* @@ -402,8 +408,8 @@ The :mod:`functools` module defines the following functions: dispatch>` :term:`generic function`. To define a generic function, decorate it with the ``@singledispatch`` - decorator. Note that the dispatch happens on the type of the first argument, - create your function accordingly:: + decorator. When defining a function using ``@singledispatch``, note that the + dispatch happens on the type of the first argument:: >>> from functools import singledispatch >>> @singledispatch @@ -413,9 +419,9 @@ The :mod:`functools` module defines the following functions: ... print(arg) To add overloaded implementations to the function, use the :func:`register` - attribute of the generic function. It is a decorator. For functions - annotated with types, the decorator will infer the type of the first - argument automatically:: + attribute of the generic function, which can be used as a decorator. For + functions annotated with types, the decorator will infer the type of the + first argument automatically:: >>> @fun.register ... def _(arg: int, verbose=False): @@ -441,17 +447,17 @@ The :mod:`functools` module defines the following functions: ... - To enable registering lambdas and pre-existing functions, the - :func:`register` attribute can be used in a functional form:: + To enable registering :term:`lambdas` and pre-existing functions, + the :func:`register` attribute can also be used in a functional form:: >>> def nothing(arg, verbose=False): ... print("Nothing.") ... >>> fun.register(type(None), nothing) - The :func:`register` attribute returns the undecorated function which - enables decorator stacking, pickling, as well as creating unit tests for - each variant independently:: + The :func:`register` attribute returns the undecorated function. This + enables decorator stacking, :mod:`pickling`, and the creation + of unit tests for each variant independently:: >>> @fun.register(float) ... @fun.register(Decimal) @@ -486,11 +492,12 @@ The :mod:`functools` module defines the following functions: Where there is no registered implementation for a specific type, its method resolution order is used to find a more generic implementation. The original function decorated with ``@singledispatch`` is registered - for the base ``object`` type, which means it is used if no better + for the base :class:`object` type, which means it is used if no better implementation is found. - If an implementation registered to :term:`abstract base class`, virtual - subclasses will be dispatched to that implementation:: + If an implementation is registered to an :term:`abstract base class`, + virtual subclasses of the base class will be dispatched to that + implementation:: >>> from collections.abc import Mapping >>> @fun.register @@ -503,7 +510,7 @@ The :mod:`functools` module defines the following functions: >>> fun({"a": "b"}) a => b - To check which implementation will the generic function choose for + To check which implementation the generic function will choose for a given type, use the ``dispatch()`` attribute:: >>> fun.dispatch(float) @@ -526,7 +533,7 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.4 .. versionchanged:: 3.7 - The :func:`register` attribute supports using type annotations. + The :func:`register` attribute now supports using type annotations. .. class:: singledispatchmethod(func) @@ -535,8 +542,9 @@ The :mod:`functools` module defines the following functions: dispatch>` :term:`generic function`. To define a generic method, decorate it with the ``@singledispatchmethod`` - decorator. Note that the dispatch happens on the type of the first non-self - or non-cls argument, create your function accordingly:: + decorator. When defining a function using ``@singledispatchmethod``, note + that the dispatch happens on the type of the first non-*self* or non-*cls* + argument:: class Negator: @singledispatchmethod @@ -552,9 +560,10 @@ The :mod:`functools` module defines the following functions: return not arg ``@singledispatchmethod`` supports nesting with other decorators such as - ``@classmethod``. Note that to allow for ``dispatcher.register``, - ``singledispatchmethod`` must be the *outer most* decorator. Here is the - ``Negator`` class with the ``neg`` methods being class bound:: + :func:`@classmethod`. Note that to allow for + ``dispatcher.register``, ``singledispatchmethod`` must be the *outer most* + decorator. Here is the ``Negator`` class with the ``neg`` methods bound to + the class, rather than an instance of the class:: class Negator: @singledispatchmethod @@ -572,8 +581,9 @@ The :mod:`functools` module defines the following functions: def _(cls, arg: bool): return not arg - The same pattern can be used for other similar decorators: ``staticmethod``, - ``abstractmethod``, and others. + The same pattern can be used for other similar decorators: + :func:`@staticmethod`, + :func:`@abstractmethod`, and others. .. versionadded:: 3.8 diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 33c40676f74..8cea2649ee6 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -174,19 +174,30 @@ The module defines the following items: Compress the *data*, returning a :class:`bytes` object containing the compressed data. *compresslevel* and *mtime* have the same meaning as in - the :class:`GzipFile` constructor above. + the :class:`GzipFile` constructor above. When *mtime* is set to ``0``, this + function is equivalent to :func:`zlib.compress` with *wbits* set to ``31``. + The zlib function is faster. .. versionadded:: 3.2 .. versionchanged:: 3.8 Added the *mtime* parameter for reproducible output. + .. versionchanged:: 3.11 + Speed is improved by compressing all data at once instead of in a + streamed fashion. Calls with *mtime* set to ``0`` are delegated to + :func:`zlib.compress` for better speed. .. function:: decompress(data) Decompress the *data*, returning a :class:`bytes` object containing the - uncompressed data. + uncompressed data. This function is capable of decompressing multi-member + gzip data (multiple gzip blocks concatenated together). When the data is + certain to contain only one member the :func:`zlib.decompress` function with + *wbits* set to 31 is faster. .. versionadded:: 3.2 - + .. versionchanged:: 3.11 + Speed is improved by decompressing members at once in memory instead of in + a streamed fashion. .. _gzip-usage-examples: diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 37addee6cda..0c3bd7b5ac2 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -376,10 +376,10 @@ Constructor functions also accept the following tree hashing parameters: * *depth*: maximal depth of tree (1 to 255, 255 if unlimited, 1 in sequential mode). -* *leaf_size*: maximal byte length of leaf (0 to 2**32-1, 0 if unlimited or in +* *leaf_size*: maximal byte length of leaf (0 to ``2**32-1``, 0 if unlimited or in sequential mode). -* *node_offset*: node offset (0 to 2**64-1 for BLAKE2b, 0 to 2**48-1 for +* *node_offset*: node offset (0 to ``2**64-1`` for BLAKE2b, 0 to ``2**48-1`` for BLAKE2s, 0 for the first, leftmost, leaf, or in sequential mode). * *node_depth*: node depth (0 to 255, 0 for leaves, or in sequential mode). @@ -500,7 +500,7 @@ Keyed hashing Keyed hashing can be used for authentication as a faster and simpler replacement for `Hash-based message authentication code -`_ (HMAC). +`_ (HMAC). BLAKE2 can be securely used in prefix-MAC mode thanks to the indifferentiability property inherited from BLAKE. diff --git a/Doc/library/html.entities.rst b/Doc/library/html.entities.rst index 067e1b1e5ad..7d836fe7380 100644 --- a/Doc/library/html.entities.rst +++ b/Doc/library/html.entities.rst @@ -44,4 +44,4 @@ This module defines four dictionaries, :data:`html5`, .. rubric:: Footnotes -.. [#] See https://www.w3.org/TR/html5/syntax.html#named-character-references +.. [#] See https://html.spec.whatwg.org/multipage/syntax.html#named-character-references diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index 6234e65629e..51a8c53152b 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -122,7 +122,7 @@ The following classes are provided: :mod:`http.cookiejar` and :mod:`http.cookies` modules do not depend on each other. - https://curl.haxx.se/rfc/cookie_spec.html + https://curl.se/rfc/cookie_spec.html The specification of the original Netscape cookie protocol. Though this is still the dominant protocol, the 'Netscape cookie protocol' implemented by all the major browsers (and :mod:`http.cookiejar`) only bears a passing resemblance to @@ -494,7 +494,8 @@ and ``".168.1.2"``, 192.168.1.2 is blocked, but 193.168.1.2 is not. .. method:: DefaultCookiePolicy.is_blocked(domain) - Return whether *domain* is on the blocklist for setting or receiving cookies. + Return ``True`` if *domain* is on the blocklist for setting or receiving + cookies. .. method:: DefaultCookiePolicy.allowed_domains() @@ -509,7 +510,7 @@ and ``".168.1.2"``, 192.168.1.2 is blocked, but 193.168.1.2 is not. .. method:: DefaultCookiePolicy.is_not_allowed(domain) - Return whether *domain* is not on the allowlist for setting or receiving + Return ``True`` if *domain* is not on the allowlist for setting or receiving cookies. :class:`DefaultCookiePolicy` instances have the following attributes, which are @@ -766,4 +767,3 @@ returned:: cj = CookieJar(policy) opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj)) r = opener.open("http://example.com/") - diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 91c540513cc..d740973af91 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -518,7 +518,7 @@ and not restarting the Shell thereafter. This is especially useful after adding imports at the top of a file. This also increases possible attribute completions. -Completion boxes intially exclude names beginning with '_' or, for +Completion boxes initially exclude names beginning with '_' or, for modules, not included in '__all__'. The hidden names can be accessed by typing '_' after '.', either before or after the box is opened. diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index c43457a3850..99bcfeb2d1b 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -11,7 +11,7 @@ .. versionchanged:: 3.10 ``importlib.metadata`` is no longer provisional. -**Source code:** :source:`Lib/importlib/metadata.py` +**Source code:** :source:`Lib/importlib/metadata/__init__.py` ``importlib.metadata`` is a library that provides for access to installed package metadata. Built in part on Python's import system, this library @@ -255,7 +255,7 @@ function:: Package distributions --------------------- -A convience method to resolve the distribution or +A convenience method to resolve the distribution or distributions (in the case of a namespace package) for top-level Python packages or modules:: diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index b5ee7a6b965..347e08e4a28 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -145,6 +145,10 @@ Functions .. versionadded:: 3.3 + .. versionchanged:: 3.10 + Namespace packages created/installed in a different :data:`sys.path` + location after the same namespace was already imported are noticed. + .. function:: reload(module) Reload a previously imported *module*. The argument must be a module object, @@ -383,11 +387,11 @@ ABC hierarchy:: See :pep:`302` for the exact definition for a loader. Loaders that wish to support resource reading should implement a - ``get_resource_reader(fullname)`` method as specified by + :meth:`get_resource_reader` method as specified by :class:`importlib.abc.ResourceReader`. .. versionchanged:: 3.7 - Introduced the optional ``get_resource_reader()`` method. + Introduced the optional :meth:`get_resource_reader` method. .. method:: create_module(spec) @@ -405,17 +409,17 @@ ABC hierarchy:: An abstract method that executes the module in its own namespace when a module is imported or reloaded. The module should already - be initialized when ``exec_module()`` is called. When this method exists, - :meth:`~importlib.abc.Loader.create_module` must be defined. + be initialized when :meth:`exec_module` is called. When this method exists, + :meth:`create_module` must be defined. .. versionadded:: 3.4 .. versionchanged:: 3.6 - :meth:`~importlib.abc.Loader.create_module` must also be defined. + :meth:`create_module` must also be defined. .. method:: load_module(fullname) - A legacy method for loading a module. If the module cannot be + A legacy method for loading a module. If the module cannot be loaded, :exc:`ImportError` is raised, otherwise the loaded module is returned. @@ -423,62 +427,64 @@ ABC hierarchy:: module should be used and reloaded. Otherwise the loader should create a new module and insert it into :data:`sys.modules` before any loading begins, to prevent recursion - from the import. If the loader inserted a module and the load fails, it + from the import. If the loader inserted a module and the load fails, it must be removed by the loader from :data:`sys.modules`; modules already in :data:`sys.modules` before the loader began execution should be left alone (see :func:`importlib.util.module_for_loader`). - The loader should set several attributes on the module. - (Note that some of these attributes can change when a module is + The loader should set several attributes on the module + (note that some of these attributes can change when a module is reloaded): - :attr:`__name__` - The name of the module. + The module's fully-qualified name. + It is ``'__main__'`` for an executed module. - :attr:`__file__` - The path to where the module data is stored (not set for built-in - modules). + The location the :term:`loader` used to load the module. + For example, for modules loaded from a .py file this is the filename. + It is not set on all modules (e.g. built-in modules). - :attr:`__cached__` - The path to where a compiled version of the module is/should be - stored (not set when the attribute would be inappropriate). + The filename of a compiled version of the module's code. + It is not set on all modules (e.g. built-in modules). - :attr:`__path__` - A list of strings specifying the search path within a - package. This attribute is not set on modules. + The list of locations where the package's submodules will be found. + Most of the time this is a single directory. + The import system passes this attribute to ``__import__()`` and to finders + in the same way as :attr:`sys.path` but just for the package. + It is not set on non-package modules so it can be used + as an indicator that the module is a package. - :attr:`__package__` - The fully-qualified name of the package under which the module was - loaded as a submodule (or the empty string for top-level modules). - For packages, it is the same as :attr:`__name__`. The - :func:`importlib.util.module_for_loader` decorator can handle the - details for :attr:`__package__`. + The fully-qualified name of the package the module is in (or the + empty string for a top-level module). + If the module is a package then this is the same as :attr:`__name__`. - :attr:`__loader__` - The loader used to load the module. The - :func:`importlib.util.module_for_loader` decorator can handle the - details for :attr:`__package__`. + The :term:`loader` used to load the module. When :meth:`exec_module` is available then backwards-compatible functionality is provided. .. versionchanged:: 3.4 Raise :exc:`ImportError` when called instead of - :exc:`NotImplementedError`. Functionality provided when + :exc:`NotImplementedError`. Functionality provided when :meth:`exec_module` is available. .. deprecated:: 3.4 The recommended API for loading a module is :meth:`exec_module` - (and :meth:`create_module`). Loaders should implement - it instead of load_module(). The import machinery takes care of - all the other responsibilities of load_module() when exec_module() - is implemented. + (and :meth:`create_module`). Loaders should implement it instead of + :meth:`load_module`. The import machinery takes care of all the + other responsibilities of :meth:`load_module` when + :meth:`exec_module` is implemented. .. method:: module_repr(module) - A legacy method which when implemented calculates and returns the - given module's repr, as a string. The module type's default repr() will - use the result of this method as appropriate. + A legacy method which when implemented calculates and returns the given + module's representation, as a string. The module type's default + :meth:`__repr__` will use the result of this method as appropriate. .. versionadded:: 3.3 @@ -815,20 +821,66 @@ ABC hierarchy:: .. versionadded:: 3.9 + .. attribute:: name + + Abstract. The base name of this object without any parent references. + + .. abstractmethod:: iterdir() + + Yield Traversable objects in self. + + .. abstractmethod:: is_dir() + + Return True if self is a directory. + + .. abstractmethod:: is_file() + + Return True if self is a file. + + .. abstractmethod:: joinpath(child) + + Return Traversable child in self. + + .. abstractmethod:: __truediv__(child) + + Return Traversable child in self. + + .. abstractmethod:: open(mode='r', *args, **kwargs) + + *mode* may be 'r' or 'rb' to open as text or binary. Return a handle + suitable for reading (same as :attr:`pathlib.Path.open`). + + When opening as text, accepts encoding parameters such as those + accepted by :attr:`io.TextIOWrapper`. + + .. method:: read_bytes() + + Read contents of self as bytes. + + .. method:: read_text(encoding=None) + + Read contents of self as text. + .. class:: TraversableResources An abstract base class for resource readers capable of serving - the ``files`` interface. Subclasses ResourceReader and provides - concrete implementations of the ResourceReader's abstract - methods. Therefore, any loader supplying TraversableReader - also supplies ResourceReader. + the :meth:`importlib.resources.files` interface. Subclasses + :class:`importlib.abc.ResourceReader` and provides + concrete implementations of the :class:`importlib.abc.ResourceReader`'s + abstract methods. Therefore, any loader supplying + :class:`importlib.abc.TraversableReader` also supplies ResourceReader. Loaders that wish to support resource reading are expected to implement this interface. .. versionadded:: 3.9 + .. abstractmethod:: files() + + Returns a :class:`importlib.abc.Traversable` object for the loaded + package. + :mod:`importlib.resources` -- Resources --------------------------------------- @@ -865,7 +917,9 @@ not** have to exist as physical files and directories on the file system. on `using importlib.resources `_ and `migrating from pkg_resources to importlib.resources - `_. + `_ + and + `migrating legacy usage `_. Loaders that wish to support resource reading should implement a ``get_resource_reader(fullname)`` method as specified by @@ -927,6 +981,8 @@ The following functions are available. sub-resources (i.e. it cannot be a directory). This function returns a ``typing.BinaryIO`` instance, a binary I/O stream open for reading. + .. deprecated:: 3.11 + .. function:: open_text(package, resource, encoding='utf-8', errors='strict') @@ -942,6 +998,8 @@ The following functions are available. This function returns a ``typing.TextIO`` instance, a text I/O stream open for reading. + .. deprecated:: 3.11 + .. function:: read_binary(package, resource) @@ -954,6 +1012,8 @@ The following functions are available. sub-resources (i.e. it cannot be a directory). This function returns the contents of the resource as :class:`bytes`. + .. deprecated:: 3.11 + .. function:: read_text(package, resource, encoding='utf-8', errors='strict') @@ -967,6 +1027,8 @@ The following functions are available. have the same meaning as with built-in :func:`open`. This function returns the contents of the resource as :class:`str`. + .. deprecated:: 3.11 + .. function:: path(package, resource) @@ -982,6 +1044,8 @@ The following functions are available. within *package*; it may not contain path separators and it may not have sub-resources (i.e. it cannot be a directory). + .. deprecated:: 3.11 + .. function:: is_resource(package, name) @@ -990,6 +1054,8 @@ The following functions are available. *package* is either a name or a module object which conforms to the ``Package`` requirements. + .. deprecated:: 3.11 + .. function:: contents(package) @@ -1000,6 +1066,8 @@ The following functions are available. *package* is either a name or a module object which conforms to the ``Package`` requirements. + .. deprecated:: 3.11 + :mod:`importlib.machinery` -- Importers and path hooks ------------------------------------------------------ @@ -1353,72 +1421,101 @@ find and load modules. .. versionadded:: 3.4 +.. class:: NamespaceLoader(name, path, path_finder): + + A concrete implementation of :class:`importlib.abc.InspectLoader` for + namespace packages. This is an alias for a private class and is only made + public for introspecting the ``__loader__`` attribute on namespace + packages:: + + >>> from importlib.machinery import NamespaceLoader + >>> import my_namespace + >>> isinstance(my_namespace.__loader__, NamespaceLoader) + True + >>> import importlib.abc + >>> isinstance(my_namespace.__loader__, importlib.abc.Loader) + True + + .. versionadded:: 3.11 + + .. class:: ModuleSpec(name, loader, *, origin=None, loader_state=None, is_package=None) A specification for a module's import-system-related state. This is - typically exposed as the module's ``__spec__`` attribute. In the + typically exposed as the module's :attr:`__spec__` attribute. In the descriptions below, the names in parentheses give the corresponding - attribute available directly on the module object. - E.g. ``module.__spec__.origin == module.__file__``. Note however that + attribute available directly on the module object, + e.g. ``module.__spec__.origin == module.__file__``. Note, however, that while the *values* are usually equivalent, they can differ since there is - no synchronization between the two objects. Thus it is possible to update - the module's ``__path__`` at runtime, and this will not be automatically - reflected in ``__spec__.submodule_search_locations``. + no synchronization between the two objects. For example, it is possible to update + the module's :attr:`__file__` at runtime and this will not be automatically + reflected in the module's :attr:`__spec__.origin`, and vice versa. .. versionadded:: 3.4 .. attribute:: name - (``__name__``) + (:attr:`__name__`) - A string for the fully-qualified name of the module. + The module's fully-qualified name. + The :term:`finder` should always set this attribute to a non-empty string. .. attribute:: loader - (``__loader__``) + (:attr:`__loader__`) - The :term:`Loader ` that should be used when loading - the module. :term:`Finders ` should always set this. + The :term:`loader` used to load the module. + The :term:`finder` should always set this attribute. .. attribute:: origin - (``__file__``) + (:attr:`__file__`) - Name of the place from which the module is loaded, e.g. "builtin" for - built-in modules and the filename for modules loaded from source. - Normally "origin" should be set, but it may be ``None`` (the default) - which indicates it is unspecified (e.g. for namespace packages). + The location the :term:`loader` should use to load the module. + For example, for modules loaded from a .py file this is the filename. + The :term:`finder` should always set this attribute to a meaningful value + for the :term:`loader` to use. In the uncommon case that there is not one + (like for namespace packages), it should be set to ``None``. .. attribute:: submodule_search_locations - (``__path__``) + (:attr:`__path__`) - List of strings for where to find submodules, if a package (``None`` - otherwise). + The list of locations where the package's submodules will be found. + Most of the time this is a single directory. + The :term:`finder` should set this attribute to a list, even an empty one, to indicate + to the import system that the module is a package. It should be set to ``None`` for + non-package modules. It is set automatically later to a special object for + namespace packages. .. attribute:: loader_state - Container of extra module-specific data for use during loading (or - ``None``). + The :term:`finder` may set this attribute to an object containing additional, + module-specific data to use when loading the module. Otherwise it should be + set to ``None``. .. attribute:: cached - (``__cached__``) + (:attr:`__cached__`) - String for where the compiled module should be stored (or ``None``). + The filename of a compiled version of the module's code. + The :term:`finder` should always set this attribute but it may be ``None`` + for modules that do not need compiled code stored. .. attribute:: parent - (``__package__``) + (:attr:`__package__`) - (Read-only) The fully-qualified name of the package under which the module - should be loaded as a submodule (or the empty string for top-level modules). - For packages, it is the same as :attr:`__name__`. + (Read-only) The fully-qualified name of the package the module is in (or the + empty string for a top-level module). + If the module is a package then this is the same as :attr:`name`. .. attribute:: has_location - Boolean indicating whether or not the module's "origin" - attribute refers to a loadable location. + ``True`` if the spec's :attr:`origin` refers to a loadable location, + ``False`` otherwise. This value impacts how :attr:`origin` is interpreted + and how the module's :attr:`__file__` is populated. + :mod:`importlib.util` -- Utility code for importers --------------------------------------------------- @@ -1510,8 +1607,9 @@ an :term:`importer`. :exc:`ImportError` is raised if **name** is a relative module name but **package** is a false value (e.g. ``None`` or the empty string). - :exc:`ImportError` is also raised a relative name would escape its containing - package (e.g. requesting ``..bacon`` from within the ``spam`` package). + :exc:`ImportError` is also raised if a relative name would escape its + containing package (e.g. requesting ``..bacon`` from within the ``spam`` + package). .. versionadded:: 3.3 diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index ed95c72c9a6..711f510d7dc 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -275,6 +275,24 @@ attributes: listed in the metaclass' custom :meth:`__dir__`. +.. function:: getmembers_static(object[, predicate]) + + Return all the members of an object in a list of ``(name, value)`` + pairs sorted by name without triggering dynamic lookup via the descriptor + protocol, __getattr__ or __getattribute__. Optionally, only return members + that satisfy a given predicate. + + .. note:: + + :func:`getmembers_static` may not be able to retrieve all members + that getmembers can fetch (like dynamically created attributes) + and may find members that getmembers can't (like descriptors + that raise AttributeError). It can also return descriptor objects + instead of instance members in some cases. + + .. versionadded:: 3.11 + + .. function:: getmodulename(path) Return the name of the module named by the file *path*, without including the @@ -935,26 +953,6 @@ Classes and functions times. -.. function:: getargspec(func) - - Get the names and default values of a Python function's parameters. A - :term:`named tuple` ``ArgSpec(args, varargs, keywords, defaults)`` is - returned. *args* is a list of the parameter names. *varargs* and *keywords* - are the names of the ``*`` and ``**`` parameters or ``None``. *defaults* is a - tuple of default argument values or ``None`` if there are no default - arguments; if this tuple has *n* elements, they correspond to the last - *n* elements listed in *args*. - - .. deprecated:: 3.0 - Use :func:`getfullargspec` for an updated API that is usually a drop-in - replacement, but also correctly handles function annotations and - keyword-only parameters. - - Alternatively, use :func:`signature` and - :ref:`Signature Object `, which provide a - more structured introspection API for callables. - - .. function:: getfullargspec(func) Get the names and default values of a Python function's parameters. A @@ -1015,33 +1013,6 @@ Classes and functions This function was inadvertently marked as deprecated in Python 3.5. -.. function:: formatargspec(args[, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations[, formatarg, formatvarargs, formatvarkw, formatvalue, formatreturns, formatannotations]]) - - Format a pretty argument spec from the values returned by - :func:`getfullargspec`. - - The first seven arguments are (``args``, ``varargs``, ``varkw``, - ``defaults``, ``kwonlyargs``, ``kwonlydefaults``, ``annotations``). - - The other six arguments are functions that are called to turn argument names, - ``*`` argument name, ``**`` argument name, default values, return annotation - and individual annotations into strings, respectively. - - For example: - - >>> from inspect import formatargspec, getfullargspec - >>> def f(a: int, b: float): - ... pass - ... - >>> formatargspec(*getfullargspec(f)) - '(a: int, b: float)' - - .. deprecated:: 3.5 - Use :func:`signature` and - :ref:`Signature Object `, which provide a - better introspecting API for callables. - - .. function:: formatargvalues(args[, varargs, varkw, locals, formatarg, formatvarargs, formatvarkw, formatvalue]) Format a pretty argument spec from the four values returned by diff --git a/Doc/library/intro.rst b/Doc/library/intro.rst index 8567e4d1d08..5bb33b9c10c 100644 --- a/Doc/library/intro.rst +++ b/Doc/library/intro.rst @@ -58,5 +58,5 @@ Notes on availability operating system. * If not separately noted, all functions that claim "Availability: Unix" are - supported on Mac OS X, which builds on a Unix core. + supported on macOS, which builds on a Unix core. diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 1c2263b128a..9c2dff55703 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -41,7 +41,7 @@ IP addresses, networks and interfaces: Return an :class:`IPv4Address` or :class:`IPv6Address` object depending on the IP address passed as argument. Either IPv4 or IPv6 addresses may be - supplied; integers less than 2**32 will be considered to be IPv4 by default. + supplied; integers less than ``2**32`` will be considered to be IPv4 by default. A :exc:`ValueError` is raised if *address* does not represent a valid IPv4 or IPv6 address. @@ -56,7 +56,7 @@ IP addresses, networks and interfaces: Return an :class:`IPv4Network` or :class:`IPv6Network` object depending on the IP address passed as argument. *address* is a string or integer representing the IP network. Either IPv4 or IPv6 networks may be supplied; - integers less than 2**32 will be considered to be IPv4 by default. *strict* + integers less than ``2**32`` will be considered to be IPv4 by default. *strict* is passed to :class:`IPv4Network` or :class:`IPv6Network` constructor. A :exc:`ValueError` is raised if *address* does not represent a valid IPv4 or IPv6 address, or if the network has host bits set. @@ -70,7 +70,7 @@ IP addresses, networks and interfaces: Return an :class:`IPv4Interface` or :class:`IPv6Interface` object depending on the IP address passed as argument. *address* is a string or integer representing the IP address. Either IPv4 or IPv6 addresses may be supplied; - integers less than 2**32 will be considered to be IPv4 by default. A + integers less than ``2**32`` will be considered to be IPv4 by default. A :exc:`ValueError` is raised if *address* does not represent a valid IPv4 or IPv6 address. @@ -132,6 +132,11 @@ write code that handles both IP versions correctly. Address objects are The above change was also included in Python 3.9 starting with version 3.9.5. + .. versionchanged:: 3.8.12 + + The above change was also included in Python 3.8 starting with + version 3.8.12. + .. attribute:: version The appropriate version number: ``4`` for IPv4, ``6`` for IPv6. @@ -677,7 +682,7 @@ dictionaries. Note that currently expanded netmasks are not supported. That means ``2001:db00::0/24`` is a valid argument while ``2001:db00::0/ffff:ff00::`` - not. + is not. 2. An integer that fits into 128 bits. This is equivalent to a single-address network, with the network address being *address* and diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index fd77f99a88f..61d8b869711 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -492,6 +492,8 @@ loops that truncate the stream. next(b, None) return zip(a, b) + .. versionadded:: 3.10 + .. function:: permutations(iterable, r=None) @@ -812,11 +814,27 @@ which incur interpreter overhead. return starmap(func, repeat(args, times)) def grouper(iterable, n, fillvalue=None): - "Collect data into fixed-length chunks or blocks" - # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx" + "Collect data into non-overlapping fixed-length chunks or blocks" + # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx args = [iter(iterable)] * n return zip_longest(*args, fillvalue=fillvalue) + def triplewise(iterable): + "Return overlapping triplets from an iterable" + # triplewise('ABCDEFG') -> ABC BCD CDE DEF EFG + for (a, _), (b, c) in pairwise(pairwise(iterable)): + yield a, b, c + + def sliding_window(iterable, n): + # sliding_window('ABCDEFG', 4) -> ABCD BCDE CDEF DEFG + it = iter(iterable) + window = collections.deque(islice(it, n), maxlen=n) + if len(window) == n: + yield tuple(window) + for x in it: + window.append(x) + yield tuple(window) + def roundrobin(*iterables): "roundrobin('ABC', 'D', 'EF') --> A D E B F C" # Recipe credited to George Sakkis @@ -837,6 +855,35 @@ which incur interpreter overhead. t1, t2 = tee(iterable) return filterfalse(pred, t1), filter(pred, t2) + def before_and_after(predicate, it): + """ Variant of takewhile() that allows complete + access to the remainder of the iterator. + + >>> it = iter('ABCdEfGhI') + >>> all_upper, remainder = before_and_after(str.isupper, it) + >>> ''.join(all_upper) + 'ABC' + >>> ''.join(remainder) # takewhile() would lose the 'd' + 'dEfGhI' + + Note that the first iterator must be fully + consumed before the second iterator can + generate valid results. + """ + it = iter(it) + transition = [] + def true_iterator(): + for elem in it: + if predicate(elem): + yield elem + else: + transition.append(elem) + return + def remainder_iterator(): + yield from transition + yield from it + return true_iterator(), remainder_iterator() + def powerset(iterable): "powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)" s = list(iterable) @@ -948,4 +995,3 @@ which incur interpreter overhead. c, n = c*(n-r)//n, n-1 result.append(pool[-1-n]) return tuple(result) - diff --git a/Doc/library/json.rst b/Doc/library/json.rst index c8184da80fe..1810e04cc83 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -11,9 +11,9 @@ -------------- -`JSON (JavaScript Object Notation) `_, specified by +`JSON (JavaScript Object Notation) `_, specified by :rfc:`7159` (which obsoletes :rfc:`4627`) and by -`ECMA-404 `_, +`ECMA-404 `_, is a lightweight data interchange format inspired by `JavaScript `_ object literal syntax (although it is not a strict subset of JavaScript [#rfc-errata]_ ). @@ -159,7 +159,7 @@ Basic Usage If *check_circular* is false (default: ``True``), then the circular reference check for container types will be skipped and a circular reference - will result in an :exc:`OverflowError` (or worse). + will result in an :exc:`RecursionError` (or worse). If *allow_nan* is false (default: ``True``), then it will be a :exc:`ValueError` to serialize out of range :class:`float` values (``nan``, @@ -432,7 +432,7 @@ Encoders and Decoders If *check_circular* is true (the default), then lists, dicts, and custom encoded objects will be checked for circular references during encoding to - prevent an infinite recursion (which would cause an :exc:`OverflowError`). + prevent an infinite recursion (which would cause an :exc:`RecursionError`). Otherwise, no such check takes place. If *allow_nan* is true (the default), then ``NaN``, ``Infinity``, and @@ -544,7 +544,7 @@ Standard Compliance and Interoperability ---------------------------------------- The JSON format is specified by :rfc:`7159` and by -`ECMA-404 `_. +`ECMA-404 `_. This section details this module's level of compliance with the RFC. For simplicity, :class:`JSONEncoder` and :class:`JSONDecoder` subclasses, and parameters other than those explicitly mentioned, are not considered. diff --git a/Doc/library/keyword.rst b/Doc/library/keyword.rst index 5cae79f5dc9..c3b4699cb05 100644 --- a/Doc/library/keyword.rst +++ b/Doc/library/keyword.rst @@ -9,7 +9,7 @@ -------------- This module allows a Python program to determine if a string is a -:ref:`keyword `. +:ref:`keyword ` or :ref:`soft keyword `. .. function:: iskeyword(s) @@ -26,14 +26,14 @@ This module allows a Python program to determine if a string is a .. function:: issoftkeyword(s) - Return ``True`` if *s* is a Python soft :ref:`keyword `. + Return ``True`` if *s* is a Python :ref:`soft keyword `. .. versionadded:: 3.9 .. data:: softkwlist - Sequence containing all the soft :ref:`keywords ` defined for the + Sequence containing all the :ref:`soft keywords ` defined for the interpreter. If any soft keywords are defined to only be active when particular :mod:`__future__` statements are in effect, these will be included as well. diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index d3478a94345..5a3e686802e 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -168,7 +168,7 @@ in :mod:`logging` itself) and defining handlers which are declared either in :func:`listen` socket and sending a configuration which runs whatever code the attacker wants to have executed in the victim's process. This is especially easy to do if the default port is used, but not hard even if a - different port is used). To avoid the risk of this happening, use the + different port is used. To avoid the risk of this happening, use the ``verify`` argument to :func:`listen` to prevent unrecognised configurations from being applied. diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 9b35d4daef5..ea6494f219a 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -80,6 +80,15 @@ is the module's name in the Python package namespace. If this evaluates to false, logging messages are not passed to the handlers of ancestor loggers. + Spelling it out with an example: If the propagate attribute of the logger named + ``A.B.C`` evaluates to true, any event logged to ``A.B.C`` via a method call such as + ``logging.getLogger('A.B.C').error(...)`` will [subject to passing that logger's + level and filter settings] be passed in turn to any handlers attached to loggers + named ``A.B``, ``A`` and the root logger, after first being passed to any handlers + attached to ``A.B.C``. If any logger in the chain ``A.B.C``, ``A.B``, ``A`` has its + ``propagate`` attribute set to false, then that is the last logger whose handlers + are offered the event to handle, and propagation stops at that point. + The constructor sets this attribute to ``True``. .. note:: If you attach a handler to a logger *and* one or more of its @@ -203,7 +212,7 @@ is the module's name in the Python package namespace. attributes can then be used as you like. For example, they could be incorporated into logged messages. For example:: - FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s' + FORMAT = '%(asctime)s %(clientip)-15s %(user)-8s %(message)s' logging.basicConfig(format=FORMAT) d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} logger = logging.getLogger('tcpserver') @@ -1000,7 +1009,7 @@ functions. be used as you like. For example, they could be incorporated into logged messages. For example:: - FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s' + FORMAT = '%(asctime)s %(clientip)-15s %(user)-8s %(message)s' logging.basicConfig(format=FORMAT) d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} logging.warning('Protocol problem: %s', 'connection reset', extra=d) @@ -1364,7 +1373,7 @@ with the :mod:`warnings` module. The proposal which described this feature for inclusion in the Python standard library. - `Original Python logging package `_ + `Original Python logging package `_ This is the original source for the :mod:`logging` package. The version of the package available from this site is suitable for use with Python 1.5.2, 2.1.x and 2.2.x, which do not include the :mod:`logging` package in the standard diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 71186788a65..1ad60459e8d 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -356,6 +356,13 @@ Power and logarithmic functions or ``pow(math.e, x)``. +.. function:: exp2(x) + + Return *2* raised to the power *x*. + + .. versionadded:: 3.11 + + .. function:: expm1(x) Return *e* raised to the power *x*, minus 1. Here *e* is the base of natural diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index d9825b47c71..c1ebd80abd9 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -102,7 +102,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length To ensure validity of the created memory mapping the file specified by the descriptor *fileno* is internally automatically synchronized - with physical backing store on Mac OS X and OpenVMS. + with physical backing store on macOS and OpenVMS. This example shows a simple way of using :class:`~mmap.mmap`:: @@ -256,6 +256,14 @@ To map anonymous memory, -1 should be passed as the fileno along with the length with :const:`ACCESS_READ` or :const:`ACCESS_COPY`, resizing the map will raise a :exc:`TypeError` exception. + **On Windows**: Resizing the map will raise an :exc:`OSError` if there are other + maps against the same named file. Resizing an anonymous map (ie against the + pagefile) will silently create a new map with the original data copied over + up to the length of the new size. + + .. versionchanged:: 3.11 + Correctly fails if attempting to resize when another map is held + Allows resize against an anonymous map on Windows .. method:: rfind(sub[, start[, end]]) diff --git a/Doc/library/msilib.rst b/Doc/library/msilib.rst index 21a2a205c39..22638852e31 100644 --- a/Doc/library/msilib.rst +++ b/Doc/library/msilib.rst @@ -20,10 +20,9 @@ exposes an API to create CAB files. Support for reading ``.cab`` files is currently not implemented; read support for the ``.msi`` database is possible. This package aims to provide complete access to all tables in an ``.msi`` file, -therefore, it is a fairly low-level API. Two primary applications of this -package are the :mod:`distutils` command ``bdist_msi``, and the creation of -Python installer package itself (although that currently uses a different -version of ``msilib``). +therefore, it is a fairly low-level API. One primary application of this +package is the creation of Python installer package itself (although that currently +uses a different version of ``msilib``). The package contents can be roughly split into four parts: low-level CAB routines, low-level MSI routines, higher-level MSI routines, and standard table @@ -439,9 +438,7 @@ GUI classes ----------- :mod:`msilib` provides several classes that wrap the GUI tables in an MSI -database. However, no standard user interface is provided; use -:mod:`~distutils.command.bdist_msi` to create MSI files with a user-interface -for installing Python packages. +database. However, no standard user interface is provided. .. class:: Control(dlg, name) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index c9b2a3716ff..7a1a285255f 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -412,7 +412,7 @@ For example:: multiple_results = [pool.apply_async(os.getpid, ()) for i in range(4)] print([res.get(timeout=1) for res in multiple_results]) - # make a single worker sleep for 10 secs + # make a single worker sleep for 10 seconds res = pool.apply_async(time.sleep, (10,)) try: print(res.get(timeout=1)) @@ -783,7 +783,7 @@ For an example of the usage of queues for interprocess communication see multithreading/multiprocessing semantics, this number is not reliable. Note that this may raise :exc:`NotImplementedError` on Unix platforms like - Mac OS X where ``sem_getvalue()`` is not implemented. + macOS where ``sem_getvalue()`` is not implemented. .. method:: empty() @@ -951,7 +951,8 @@ Miscellaneous use. The number of usable CPUs can be obtained with ``len(os.sched_getaffinity(0))`` - May raise :exc:`NotImplementedError`. + When the number of CPUs cannot be determined a :exc:`NotImplementedError` + is raised. .. seealso:: :func:`os.cpu_count` @@ -1029,7 +1030,13 @@ Miscellaneous The return value can be ``'fork'``, ``'spawn'``, ``'forkserver'`` or ``None``. ``'fork'`` is the default on Unix, while ``'spawn'`` is - the default on Windows. + the default on Windows and macOS. + +.. versionchanged:: 3.8 + + On macOS, the *spawn* start method is now the default. The *fork* start + method should be considered unsafe as it can lead to crashes of the + subprocess. See :issue:`33725`. .. versionadded:: 3.4 @@ -1234,7 +1241,7 @@ object -- see :ref:`multiprocessing-managers`. first argument is named *block*, as is consistent with :meth:`Lock.acquire`. .. note:: - On Mac OS X, this is indistinguishable from :class:`Semaphore` because + On macOS, this is indistinguishable from :class:`Semaphore` because ``sem_getvalue()`` is not implemented on that platform. .. class:: Condition([lock]) @@ -1373,7 +1380,7 @@ object -- see :ref:`multiprocessing-managers`. .. note:: - On Mac OS X, ``sem_timedwait`` is unsupported, so calling ``acquire()`` with + On macOS, ``sem_timedwait`` is unsupported, so calling ``acquire()`` with a timeout will emulate that function's behavior using a sleeping loop. .. note:: @@ -2629,12 +2636,13 @@ handler type) for messages from different processes to get mixed up. inherited. .. currentmodule:: multiprocessing -.. function:: log_to_stderr() +.. function:: log_to_stderr(level=None) This function performs a call to :func:`get_logger` but in addition to returning the logger created by get_logger, it adds a handler which sends output to :data:`sys.stderr` using format ``'[%(levelname)s/%(processName)s] %(message)s'``. + You can modify ``levelname`` of the logger by passing a ``level`` argument. Below is an example session with logging turned on:: diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst index cba576a29e2..2ba42b7e579 100644 --- a/Doc/library/multiprocessing.shared_memory.rst +++ b/Doc/library/multiprocessing.shared_memory.rst @@ -342,3 +342,30 @@ behind it: >>> c.shm.close() >>> c.shm.unlink() +The following examples demonstrates that ``ShareableList`` +(and underlying ``SharedMemory``) objects +can be pickled and unpickled if needed. +Note, that it will still be the same shared object. +This happens, because the deserialized object has +the same unique name and is just attached to an existing +object with the same name (if the object is still alive): + + >>> import pickle + >>> from multiprocessing import shared_memory + >>> sl = shared_memory.ShareableList(range(10)) + >>> list(sl) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + >>> deserialized_sl = pickle.loads(pickle.dumps(sl)) + >>> list(deserialized_sl) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + >>> sl[0] = -1 + >>> deserialized_sl[1] = -2 + >>> list(sl) + [-1, -2, 2, 3, 4, 5, 6, 7, 8, 9] + >>> list(deserialized_sl) + [-1, -2, 2, 3, 4, 5, 6, 7, 8, 9] + + >>> sl.shm.close() + >>> sl.shm.unlink() diff --git a/Doc/library/netdata.rst b/Doc/library/netdata.rst index e76280f2fe3..16f43a69d68 100644 --- a/Doc/library/netdata.rst +++ b/Doc/library/netdata.rst @@ -17,7 +17,6 @@ on the internet. mailbox.rst mimetypes.rst base64.rst - binhex.rst binascii.rst quopri.rst uu.rst diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index 4bf7de67c1d..88265d9b9e9 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -41,6 +41,10 @@ the Unix :program:`ftp` program and other FTP clients. .. versionchanged:: 3.10 :class:`netrc` try UTF-8 encoding before using locale specific encoding. + The entry in the netrc file no longer needs to contain all tokens. The missing + tokens' value default to an empty string. All the tokens and their values now + can contain arbitrary characters, like whitespace and non-ASCII characters. + If the login name is anonymous, it won't trigger the security check. .. exception:: NetrcParseError @@ -85,10 +89,3 @@ Instances of :class:`~netrc.netrc` have public instance variables: .. attribute:: netrc.macros Dictionary mapping macro names to string lists. - -.. note:: - - Passwords are limited to a subset of the ASCII character set. All ASCII - punctuation is allowed in passwords, however, note that whitespace and - non-printable characters are not allowed in passwords. This is a limitation - of the way the .netrc file is parsed and may be removed in the future. diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 0cdba68f377..35e9b49ea8b 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -250,6 +250,17 @@ Operations which work with sequences (some of them with mappings too) include: .. versionadded:: 3.4 + +The following operation works with callables: + +.. function:: call(obj, /, *args, **kwargs) + __call__(obj, /, *args, **kwargs) + + Return ``obj(*args, **kwargs)``. + + .. versionadded:: 3.11 + + The :mod:`operator` module also defines tools for generalized attribute and item lookups. These are useful for making fast field extractors as arguments for :func:`map`, :func:`sorted`, :meth:`itertools.groupby`, or other functions that diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 768771ee3e3..8092397be65 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -170,9 +170,10 @@ process and user. .. data:: environ - A :term:`mapping` object representing the string environment. For example, - ``environ['HOME']`` is the pathname of your home directory (on some platforms), - and is equivalent to ``getenv("HOME")`` in C. + A :term:`mapping` object where keys and values are strings that represent + the process environment. For example, ``environ['HOME']`` is the pathname + of your home directory (on some platforms), and is equivalent to + ``getenv("HOME")`` in C. This mapping is captured the first time the :mod:`os` module is imported, typically during Python startup as part of processing :file:`site.py`. Changes @@ -209,10 +210,10 @@ process and user. .. data:: environb - Bytes version of :data:`environ`: a :term:`mapping` object representing the - environment as byte strings. :data:`environ` and :data:`environb` are - synchronized (modify :data:`environb` updates :data:`environ`, and vice - versa). + Bytes version of :data:`environ`: a :term:`mapping` object where both keys + and values are :class:`bytes` objects representing the process environment. + :data:`environ` and :data:`environb` are synchronized (modifying + :data:`environb` updates :data:`environ`, and vice versa). :data:`environb` is only available if :data:`supports_bytes_environ` is ``True``. @@ -2789,7 +2790,7 @@ features: String that uniquely identifies the type of the filesystem that contains the file. - On Mac OS systems, the following attributes may also be available: + On macOS systems, the following attributes may also be available: .. attribute:: st_rsize diff --git a/Doc/library/othergui.rst b/Doc/library/othergui.rst deleted file mode 100644 index db11933b325..00000000000 --- a/Doc/library/othergui.rst +++ /dev/null @@ -1,57 +0,0 @@ -.. _other-gui-packages: - -Other Graphical User Interface Packages -======================================= - -Major cross-platform (Windows, macOS, Unix-like) GUI toolkits are -available for Python: - -.. seealso:: - - `PyGObject `_ - PyGObject provides introspection bindings for C libraries using - `GObject `_. One of - these libraries is the `GTK+ 3 `_ widget set. - GTK+ comes with many more widgets than Tkinter provides. An online - `Python GTK+ 3 Tutorial `_ - is available. - - `PyGTK `_ - PyGTK provides bindings for an older version - of the library, GTK+ 2. It provides an object oriented interface that - is slightly higher level than the C one. There are also bindings to - `GNOME `_. An online `tutorial - `_ is available. - - `PyQt `_ - PyQt is a :program:`sip`\ -wrapped binding to the Qt toolkit. Qt is an - extensive C++ GUI application development framework that is - available for Unix, Windows and macOS. :program:`sip` is a tool - for generating bindings for C++ libraries as Python classes, and - is specifically designed for Python. - - `PySide2 `_ - Also known as the Qt for Python project, PySide2 is a newer binding to the - Qt toolkit. It is provided by The Qt Company and aims to provide a - complete port of PySide to Qt 5. Compared to PyQt, its licensing scheme is - friendlier to non-open source applications. - - `wxPython `_ - wxPython is a cross-platform GUI toolkit for Python that is built around - the popular `wxWidgets `_ (formerly wxWindows) - C++ toolkit. It provides a native look and feel for applications on - Windows, macOS, and Unix systems by using each platform's native - widgets where ever possible, (GTK+ on Unix-like systems). In addition to - an extensive set of widgets, wxPython provides classes for online - documentation and context sensitive help, printing, HTML viewing, - low-level device context drawing, drag and drop, system clipboard access, - an XML-based resource format and more, including an ever growing library - of user-contributed modules. - -PyGTK, PyQt, PySide2, and wxPython, all have a modern look and feel and more -widgets than Tkinter. In addition, there are many other GUI toolkits for -Python, both cross-platform, and platform-specific. See the `GUI Programming -`_ page in the Python Wiki for a -much more complete list, and also for links to documents where the -different GUI toolkits are compared. - diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index bb9fc45df23..a0eece6c4d8 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -225,13 +225,13 @@ Windows Platform .. versionadded:: 3.8 -Mac OS Platform ---------------- +macOS Platform +-------------- .. function:: mac_ver(release='', versioninfo=('','',''), machine='') - Get Mac OS version information and return it as tuple ``(release, versioninfo, + Get macOS version information and return it as tuple ``(release, versioninfo, machine)`` with *versioninfo* being a tuple ``(version, dev_stage, non_release_version)``. diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index ce6d4a85bf5..5ded9661f08 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -133,7 +133,7 @@ The following classes are available: encoded data, which contains UID (see PList manual). It has one attribute, :attr:`data`, which can be used to retrieve the int value - of the UID. :attr:`data` must be in the range `0 <= data < 2**64`. + of the UID. :attr:`data` must be in the range ``0 <= data < 2**64``. .. versionadded:: 3.8 diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 774d46d0e96..cf324a57e79 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -622,7 +622,7 @@ procedure can be used to obtain a better constant for a given platform (see The method executes the number of Python calls given by the argument, directly and again under the profiler, measuring the time for both. It then computes the hidden overhead per profiler event, and returns that as a float. For example, -on a 1.8Ghz Intel Core i5 running Mac OS X, and using Python's time.process_time() as +on a 1.8Ghz Intel Core i5 running macOS, and using Python's time.process_time() as the timer, the magical number is about 4.04e-6. The object of this exercise is to get a fairly consistent result. If your diff --git a/Doc/library/random.rst b/Doc/library/random.rst index e444f957324..36f232dc319 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -135,15 +135,10 @@ Functions for integers values. Formerly it used a style like ``int(random()*n)`` which could produce slightly uneven distributions. - .. deprecated:: 3.10 - The automatic conversion of non-integer types to equivalent integers is - deprecated. Currently ``randrange(10.0)`` is losslessly converted to - ``randrange(10)``. In the future, this will raise a :exc:`TypeError`. - - .. deprecated:: 3.10 - The exception raised for non-integral values such as ``randrange(10.5)`` - or ``randrange('10')`` will be changed from :exc:`ValueError` to - :exc:`TypeError`. + .. versionchanged:: 3.11 + Automatic conversion of non-integer types is no longer supported. + Calls such as ``randrange(10.0)`` and ``randrange(Fraction(10, 1))`` + now raise a :exc:`TypeError`. .. function:: randint(a, b) @@ -508,7 +503,7 @@ between the effects of a drug versus a placebo:: Simulation of arrival times and service deliveries for a multiserver queue:: - from heapq import heappush, heappop + from heapq import heapify, heapreplace from random import expovariate, gauss from statistics import mean, quantiles @@ -520,14 +515,15 @@ Simulation of arrival times and service deliveries for a multiserver queue:: waits = [] arrival_time = 0.0 servers = [0.0] * num_servers # time when each server becomes available - for i in range(100_000): + heapify(servers) + for i in range(1_000_000): arrival_time += expovariate(1.0 / average_arrival_interval) - next_server_available = heappop(servers) + next_server_available = servers[0] wait = max(0.0, next_server_available - arrival_time) waits.append(wait) - service_duration = gauss(average_service_time, stdev_service_time) + service_duration = max(0.0, gauss(average_service_time, stdev_service_time)) service_completed = arrival_time + wait + service_duration - heappush(servers, service_completed) + heapreplace(servers, service_completed) print(f'Mean wait: {mean(waits):.1f} Max wait: {max(waits):.1f}') print('Quartiles:', [round(q, 1) for q in quantiles(waits)]) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 950012a9b0f..b12ce4b9744 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -824,10 +824,20 @@ form. .. function:: findall(pattern, string, flags=0) Return all non-overlapping matches of *pattern* in *string*, as a list of - strings. The *string* is scanned left-to-right, and matches are returned in - the order found. If one or more groups are present in the pattern, return a - list of groups; this will be a list of tuples if the pattern has more than - one group. Empty matches are included in the result. + strings or tuples. The *string* is scanned left-to-right, and matches + are returned in the order found. Empty matches are included in the result. + + The result depends on the number of capturing groups in the pattern. + If there are no groups, return a list of strings matching the whole + pattern. If there is exactly one group, return a list of strings + matching that group. If multiple groups are present, return a list + of tuples of strings matching the groups. Non-capturing groups do not + affect the form of the result. + + >>> re.findall(r'\bf[a-z]*', 'which foot or hand fell fastest') + ['foot', 'fell', 'fastest'] + >>> re.findall(r'(\w+)=(\d+)', 'set width=20 and height=10') + [('width', '20'), ('height', '10')] .. versionchanged:: 3.7 Non-empty matches can now start just after a previous empty match. @@ -1562,7 +1572,7 @@ find all of the adverbs in some text, they might use :func:`findall` in the following manner:: >>> text = "He was carefully disguised but captured quickly by police." - >>> re.findall(r"\w+ly", text) + >>> re.findall(r"\w+ly\b", text) ['carefully', 'quickly'] @@ -1576,7 +1586,7 @@ a writer wanted to find all of the adverbs *and their positions* in some text, they would use :func:`finditer` in the following manner:: >>> text = "He was carefully disguised but captured quickly by police." - >>> for m in re.finditer(r"\w+ly", text): + >>> for m in re.finditer(r"\w+ly\b", text): ... print('%02d-%02d: %s' % (m.start(), m.end(), m.group(0))) 07-16: carefully 40-47: quickly diff --git a/Doc/library/reprlib.rst b/Doc/library/reprlib.rst index 1a0c1b0dcd1..4b37c5ba60f 100644 --- a/Doc/library/reprlib.rst +++ b/Doc/library/reprlib.rst @@ -76,6 +76,14 @@ size limits for the representations of different object types, and methods which format specific object types. +.. attribute:: Repr.fillvalue + + This string is displayed for recursive references. It defaults to + ``...``. + + .. versionadded:: 3.11 + + .. attribute:: Repr.maxlevel Depth limit on the creation of recursive representations. The default is ``6``. diff --git a/Doc/library/select.rst b/Doc/library/select.rst index a354187c266..46b5ff8b6d5 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -534,7 +534,7 @@ https://www.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 | :const:`KQ_FILTER_PROC` | Watch for events on a process id | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_NETDEV` | Watch for events on a network device | - | | [not available on Mac OS X] | + | | [not available on macOS] | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_SIGNAL` | Returns whenever the watched signal is | | | delivered to the process | @@ -626,7 +626,7 @@ https://www.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 | :const:`KQ_NOTE_TRACKERR` | unable to attach to a child | +----------------------------+--------------------------------------------+ - :const:`KQ_FILTER_NETDEV` filter flags (not available on Mac OS X): + :const:`KQ_FILTER_NETDEV` filter flags (not available on macOS): +----------------------------+--------------------------------------------+ | Constant | Meaning | diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 684f239ef06..a50fc6f0bf7 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -45,6 +45,9 @@ lots of shared sub-objects. The keys are ordinary strings. :data:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle protocol. + .. versionchanged:: 3.11 + Accepts :term:`path-like object` for filename. + .. note:: Do not rely on the shelf being closed automatically; always call diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 11c67074921..22d6dba9e1a 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -804,6 +804,10 @@ Querying the size of the output terminal .. versionadded:: 3.3 + .. versionchanged:: 3.11 + The ``fallback`` values are also used if :func:`os.get_terminal_size` + returns zeroes. + .. _`fcopyfile`: http://www.manpagez.com/man/3/copyfile/ diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 84a569d03eb..63821866a01 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -68,10 +68,34 @@ Module contents signal (SIG*), handler (:const:`SIG_DFL`, :const:`SIG_IGN`) and sigmask (:const:`SIG_BLOCK`, :const:`SIG_UNBLOCK`, :const:`SIG_SETMASK`) related constants listed below were turned into - :class:`enums `. + :class:`enums ` (:class:`Signals`, :class:`Handlers` and :class:`Sigmasks` respectively). :func:`getsignal`, :func:`pthread_sigmask`, :func:`sigpending` and :func:`sigwait` functions return human-readable - :class:`enums `. + :class:`enums ` as :class:`Signals` objects. + + +The signal module defines three enums: + +.. class:: Signals + + :class:`enum.IntEnum` collection of SIG* constants and the CTRL_* constants. + + .. versionadded:: 3.5 + +.. class:: Handlers + + :class:`enum.IntEnum` collection the constants :const:`SIG_DFL` and :const:`SIG_IGN`. + + .. versionadded:: 3.5 + +.. class:: Sigmasks + + :class:`enum.IntEnum` collection the constants :const:`SIG_BLOCK`, :const:`SIG_UNBLOCK` and :const:`SIG_SETMASK`. + + Availability: Unix. See the man page :manpage:`sigprocmask(3)` and + :manpage:`pthread_sigmask(3)` for further information. + + .. versionadded:: 3.5 The variables defined in the :mod:`signal` module are: @@ -618,8 +642,8 @@ The :mod:`signal` module defines the following functions: .. _signal-example: -Example -------- +Examples +-------- Here is a minimal example program. It uses the :func:`alarm` function to limit the time spent waiting to open a file; this is useful if the file is for a @@ -631,7 +655,8 @@ be sent, and the handler raises an exception. :: import signal, os def handler(signum, frame): - print('Signal handler called with signal', signum) + signame = signal.Signals(signum).name + print(f'Signal handler called with signal {signame} ({signum})') raise OSError("Couldn't open device!") # Set the signal handler and a 5-second alarm diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 2e3646f6a74..e2ad3c48f97 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -32,7 +32,7 @@ It starts by constructing up to four directories from a head and a tail part. For the head part, it uses ``sys.prefix`` and ``sys.exec_prefix``; empty heads are skipped. For the tail part, it uses the empty string and then :file:`lib/site-packages` (on Windows) or -:file:`lib/python{X.Y}/site-packages` (on Unix and Macintosh). For each +:file:`lib/python{X.Y}/site-packages` (on Unix and macOS). For each of the distinct head-tail combinations, it sees if it refers to an existing directory, and if so, adds it to ``sys.path`` and also inspects the newly added path for configuration files. @@ -176,8 +176,8 @@ Module contents Path to the user site-packages for the running Python. Can be ``None`` if :func:`getusersitepackages` hasn't been called yet. Default value is - :file:`~/.local/lib/python{X.Y}/site-packages` for UNIX and non-framework Mac - OS X builds, :file:`~/Library/Python/{X.Y}/lib/python/site-packages` for Mac + :file:`~/.local/lib/python{X.Y}/site-packages` for UNIX and non-framework + macOS builds, :file:`~/Library/Python/{X.Y}/lib/python/site-packages` for macOS framework builds, and :file:`{%APPDATA%}\\Python\\Python{XY}\\site-packages` on Windows. This directory is a site directory, which means that :file:`.pth` files in it will be processed. @@ -187,8 +187,8 @@ Module contents Path to the base directory for the user site-packages. Can be ``None`` if :func:`getuserbase` hasn't been called yet. Default value is - :file:`~/.local` for UNIX and Mac OS X non-framework builds, - :file:`~/Library/Python/{X.Y}` for Mac framework builds, and + :file:`~/.local` for UNIX and macOS non-framework builds, + :file:`~/Library/Python/{X.Y}` for macOS framework builds, and :file:`{%APPDATA%}\\Python` for Windows. This value is used by Distutils to compute the installation directories for scripts, data files, Python modules, etc. for the :ref:`user installation scheme `. diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 95c7d7e0699..d6edc057f5e 100755 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -197,11 +197,15 @@ created. Socket addresses are represented as follows: - *addr* - Optional bytes-like object specifying the hardware physical address, whose interpretation depends on the device. + .. availability:: Linux >= 2.2. + - :const:`AF_QIPCRTR` is a Linux-only socket based interface for communicating with services running on co-processors in Qualcomm platforms. The address family is represented as a ``(node, port)`` tuple where the *node* and *port* are non-negative integers. + .. availability:: Linux >= 4.7. + .. versionadded:: 3.8 - :const:`IPPROTO_UDPLITE` is a variant of UDP which allows you to specify @@ -516,7 +520,7 @@ Constants .. data:: AF_LINK - .. availability:: BSD, OSX. + .. availability:: BSD, macOS. .. versionadded:: 3.4 @@ -558,7 +562,7 @@ Creating sockets The following functions all create :ref:`socket objects `. -.. function:: socket(family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None) +.. class:: socket(family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None) Create a new socket using the given address family, socket type and protocol number. The address family should be :const:`AF_INET` (the default), @@ -808,8 +812,9 @@ The :mod:`socket` module also offers various network-related services: it is interpreted as the local host. To find the fully qualified name, the hostname returned by :func:`gethostbyaddr` is checked, followed by aliases for the host, if available. The first name which includes a period is selected. In - case no fully qualified domain name is available, the hostname as returned by - :func:`gethostname` is returned. + case no fully qualified domain name is available and *name* was provided, + it is returned unchanged. If *name* was empty or equal to ``'0.0.0.0'``, + the hostname from :func:`gethostname` is returned. .. function:: gethostbyname(hostname) @@ -826,8 +831,8 @@ The :mod:`socket` module also offers various network-related services: .. function:: gethostbyname_ex(hostname) Translate a host name to IPv4 address format, extended interface. Return a - triple ``(hostname, aliaslist, ipaddrlist)`` where *hostname* is the primary - host name responding to the given *ip_address*, *aliaslist* is a (possibly + triple ``(hostname, aliaslist, ipaddrlist)`` where *hostname* is the host's + primary host name, *aliaslist* is a (possibly empty) list of alternative host names for the same address, and *ipaddrlist* is a list of IPv4 addresses for the same interface on the same host (often but not always a single address). :func:`gethostbyname_ex` does not support IPv6 name diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 6399bed7ed5..fb38182370e 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -21,16 +21,17 @@ The sqlite3 module was written by Gerhard Häring. It provides a SQL interface compliant with the DB-API 2.0 specification described by :pep:`249`, and requires SQLite 3.7.15 or newer. -To use the module, you must first create a :class:`Connection` object that +To use the module, start by creating a :class:`Connection` object that represents the database. Here the data will be stored in the :file:`example.db` file:: import sqlite3 con = sqlite3.connect('example.db') -You can also supply the special name ``:memory:`` to create a database in RAM. +The special path name ``:memory:`` can be provided to create a temporary +database in RAM. -Once you have a :class:`Connection`, you can create a :class:`Cursor` object +Once a :class:`Connection` has been established, create a :class:`Cursor` object and call its :meth:`~Cursor.execute` method to perform SQL commands:: cur = con.cursor() @@ -49,16 +50,17 @@ and call its :meth:`~Cursor.execute` method to perform SQL commands:: # Just be sure any changes have been committed or they will be lost. con.close() -The data you've saved is persistent and is available in subsequent sessions:: +The saved data is persistent: it can be reloaded in a subsequent session even +after restarting the Python interpreter:: import sqlite3 con = sqlite3.connect('example.db') cur = con.cursor() -To retrieve data after executing a SELECT statement, you can either treat the -cursor as an :term:`iterator`, call the cursor's :meth:`~Cursor.fetchone` method to -retrieve a single matching row, or call :meth:`~Cursor.fetchall` to get a list of the -matching rows. +To retrieve data after executing a SELECT statement, either treat the cursor as +an :term:`iterator`, call the cursor's :meth:`~Cursor.fetchone` method to +retrieve a single matching row, or call :meth:`~Cursor.fetchall` to get a list +of the matching rows. This example uses the iterator form:: @@ -73,27 +75,27 @@ This example uses the iterator form:: .. _sqlite3-placeholders: -Usually your SQL operations will need to use values from Python variables. You -shouldn't assemble your query using Python's string operations because doing so -is insecure; it makes your program vulnerable to an SQL injection attack -(see the `xkcd webcomic `_ for a humorous example of -what can go wrong):: +SQL operations usually need to use values from Python variables. However, +beware of using Python's string operations to assemble queries, as they +are vulnerable to SQL injection attacks (see the `xkcd webcomic +`_ for a humorous example of what can go wrong):: # Never do this -- insecure! symbol = 'RHAT' cur.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) -Instead, use the DB-API's parameter substitution. Put a placeholder wherever -you want to use a value, and then provide a tuple of values as the second -argument to the cursor's :meth:`~Cursor.execute` method. An SQL statement may +Instead, use the DB-API's parameter substitution. To insert a variable into a +query string, use a placeholder in the string, and substitute the actual values +into the query by providing them as a :class:`tuple` of values to the second +argument of the cursor's :meth:`~Cursor.execute` method. An SQL statement may use one of two kinds of placeholders: question marks (qmark style) or named placeholders (named style). For the qmark style, ``parameters`` must be a :term:`sequence `. For the named style, it can be either a :term:`sequence ` or :class:`dict` instance. The length of the :term:`sequence ` must match the number of placeholders, or a :exc:`ProgrammingError` is raised. If a :class:`dict` is given, it must contain -keys for all named parameters. Any extra items are ignored. Here's an example -of both styles: +keys for all named parameters. Any extra items are ignored. Here's an example of +both styles: .. literalinclude:: ../includes/sqlite3/execute_1.py @@ -117,6 +119,24 @@ Module functions and constants ------------------------------ +.. data:: apilevel + + String constant stating the supported DB-API level. Required by the DB-API. + Hard-coded to ``"2.0"``. + +.. data:: paramstyle + + String constant stating the type of parameter marker formatting expected by + the :mod:`sqlite3` module. Required by the DB-API. Hard-coded to + ``"qmark"``. + + .. note:: + + The :mod:`sqlite3` module supports both ``qmark`` and ``numeric`` DB-API + parameter styles, because that is what the underlying SQLite library + supports. However, the DB-API does not allow multiple values for + the ``paramstyle`` attribute. + .. data:: version The version number of this module, as a string. This is not the version of @@ -139,6 +159,44 @@ Module functions and constants The version number of the run-time SQLite library, as a tuple of integers. +.. data:: threadsafety + + Integer constant required by the DB-API 2.0, stating the level of thread + safety the :mod:`sqlite3` module supports. This attribute is set based on + the default `threading mode `_ the + underlying SQLite library is compiled with. The SQLite threading modes are: + + 1. **Single-thread**: In this mode, all mutexes are disabled and SQLite is + unsafe to use in more than a single thread at once. + 2. **Multi-thread**: In this mode, SQLite can be safely used by multiple + threads provided that no single database connection is used + simultaneously in two or more threads. + 3. **Serialized**: In serialized mode, SQLite can be safely used by + multiple threads with no restriction. + + The mappings from SQLite threading modes to DB-API 2.0 threadsafety levels + are as follows: + + +------------------+-----------------+----------------------+-------------------------------+ + | SQLite threading | `threadsafety`_ | `SQLITE_THREADSAFE`_ | DB-API 2.0 meaning | + | mode | | | | + +==================+=================+======================+===============================+ + | single-thread | 0 | 0 | Threads may not share the | + | | | | module | + +------------------+-----------------+----------------------+-------------------------------+ + | multi-thread | 1 | 2 | Threads may share the module, | + | | | | but not connections | + +------------------+-----------------+----------------------+-------------------------------+ + | serialized | 3 | 1 | Threads may share the module, | + | | | | connections and cursors | + +------------------+-----------------+----------------------+-------------------------------+ + + .. _threadsafety: https://www.python.org/dev/peps/pep-0249/#threadsafety + .. _SQLITE_THREADSAFE: https://sqlite.org/compile.html#threadsafe + + .. versionchanged:: 3.11 + Set *threadsafety* dynamically instead of hard-coding it to ``1``. + .. data:: PARSE_DECLTYPES This constant is meant to be used with the *detect_types* parameter of the @@ -271,9 +329,27 @@ Module functions and constants By default you will not get any tracebacks in user-defined functions, aggregates, converters, authorizer callbacks etc. If you want to debug them, - you can call this function with *flag* set to ``True``. Afterwards, you will - get tracebacks from callbacks on ``sys.stderr``. Use :const:`False` to - disable the feature again. + you can call this function with *flag* set to :const:`True`. Afterwards, you + will get tracebacks from callbacks on :data:`sys.stderr`. Use :const:`False` + to disable the feature again. + + Register an :func:`unraisable hook handler ` for an + improved debug experience:: + + >>> import sqlite3 + >>> sqlite3.enable_callback_tracebacks(True) + >>> cx = sqlite3.connect(":memory:") + >>> cx.set_trace_callback(lambda stmt: 5/0) + >>> cx.execute("select 1") + Exception ignored in: at 0x10b4e3ee0> + Traceback (most recent call last): + File "", line 1, in + ZeroDivisionError: division by zero + >>> import sys + >>> sys.unraisablehook = lambda unraisable: print(unraisable) + >>> cx.execute("select 1") + UnraisableHookArgs(exc_type=, exc_value=ZeroDivisionError('division by zero'), exc_traceback=, err_msg=None, object= at 0x10b4e3ee0>) + .. _sqlite3-connection-objects: @@ -460,14 +536,22 @@ Connection Objects Registers *trace_callback* to be called for each SQL statement that is actually executed by the SQLite backend. - The only argument passed to the callback is the statement (as string) that - is being executed. The return value of the callback is ignored. Note that - the backend does not only run statements passed to the :meth:`Cursor.execute` - methods. Other sources include the transaction management of the Python - module and the execution of triggers defined in the current database. + The only argument passed to the callback is the statement (as + :class:`str`) that is being executed. The return value of the callback is + ignored. Note that the backend does not only run statements passed to the + :meth:`Cursor.execute` methods. Other sources include the + :ref:`transaction management ` of the + sqlite3 module and the execution of triggers defined in the current + database. Passing :const:`None` as *trace_callback* will disable the trace callback. + .. note:: + Exceptions raised in the trace callback are not propagated. As a + development and debugging aid, use + :meth:`~sqlite3.enable_callback_tracebacks` to enable printing + tracebacks from exceptions raised in the trace callback. + .. versionadded:: 3.3 @@ -529,8 +613,8 @@ Connection Objects Using this attribute you can control what objects are returned for the ``TEXT`` data type. By default, this attribute is set to :class:`str` and the - :mod:`sqlite3` module will return Unicode objects for ``TEXT``. If you want to - return bytestrings instead, you can set it to :class:`bytes`. + :mod:`sqlite3` module will return :class:`str` objects for ``TEXT``. + If you want to return :class:`bytes` instead, you can set it to :class:`bytes`. You can also set it to any other callable that accepts a single bytestring parameter and returns the resulting object. @@ -616,6 +700,40 @@ Connection Objects .. versionadded:: 3.7 + .. method:: getlimit(category, /) + + Get a connection run-time limit. *category* is the limit category to be + queried. + + Example, query the maximum length of an SQL statement:: + + import sqlite3 + con = sqlite3.connect(":memory:") + lim = con.getlimit(sqlite3.SQLITE_LIMIT_SQL_LENGTH) + print(f"SQLITE_LIMIT_SQL_LENGTH={lim}") + + .. versionadded:: 3.11 + + + .. method:: setlimit(category, limit, /) + + Set a connection run-time limit. *category* is the limit category to be + set. *limit* is the new limit. If the new limit is a negative number, the + limit is unchanged. + + Attempts to increase a limit above its hard upper bound are silently + truncated to the hard upper bound. Regardless of whether or not the limit + was changed, the prior value of the limit is returned. + + Example, limit the number of attached databases to 1:: + + import sqlite3 + con = sqlite3.connect(":memory:") + con.setlimit(sqlite3.SQLITE_LIMIT_ATTACHED, 1) + + .. versionadded:: 3.11 + + .. _sqlite3-cursor-objects: Cursor Objects @@ -702,6 +820,14 @@ Cursor Objects The cursor will be unusable from this point forward; a :exc:`ProgrammingError` exception will be raised if any operation is attempted with the cursor. + .. method:: setinputsizes(sizes) + + Required by the DB-API. Is a no-op in :mod:`sqlite3`. + + .. method:: setoutputsize(size [, column]) + + Required by the DB-API. Is a no-op in :mod:`sqlite3`. + .. attribute:: rowcount Although the :class:`Cursor` class of the :mod:`sqlite3` module implements this @@ -836,6 +962,20 @@ Exceptions The base class of the other exceptions in this module. It is a subclass of :exc:`Exception`. + .. attribute:: sqlite_errorcode + + The numeric error code from the + `SQLite API `_ + + .. versionadded:: 3.11 + + .. attribute:: sqlite_errorname + + The symbolic name of the numeric error code + from the `SQLite API `_ + + .. versionadded:: 3.11 + .. exception:: DatabaseError Exception raised for errors that are related to the database. @@ -1027,6 +1167,12 @@ If a timestamp stored in SQLite has a fractional part longer than 6 numbers, its value will be truncated to microsecond precision by the timestamp converter. +.. note:: + + The default "timestamp" converter ignores UTC offsets in the database and + always returns a naive :class:`datetime.datetime` object. To preserve UTC + offsets in timestamps, either leave converters disabled, or register an + offset-aware converter with :func:`register_converter`. .. _sqlite3-controlling-transactions: @@ -1110,7 +1256,7 @@ committed: .. rubric:: Footnotes .. [#f1] The sqlite3 module is not built with loadable extension support by - default, because some platforms (notably Mac OS X) have SQLite + default, because some platforms (notably macOS) have SQLite libraries which are compiled without this feature. To get loadable extension support, you must pass the :option:`--enable-loadable-sqlite-extensions` option to configure. diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 21077e0f4b4..eb33d7e1778 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -18,7 +18,7 @@ This module provides access to Transport Layer Security (often known as "Secure Sockets Layer") encryption and peer authentication facilities for network sockets, both client-side and server-side. This module uses the OpenSSL -library. It is available on all modern Unix systems, Windows, Mac OS X, and +library. It is available on all modern Unix systems, Windows, macOS, and probably additional platforms, as long as OpenSSL is installed on that platform. .. note:: @@ -1576,7 +1576,7 @@ to speed up repeated connections from the same clients. Load a set of default "certification authority" (CA) certificates from default locations. On Windows it loads CA certs from the ``CA`` and - ``ROOT`` system stores. On other systems it calls + ``ROOT`` system stores. On all systems it calls :meth:`SSLContext.set_default_verify_paths`. In the future the method may load CA certificates from other locations, too. diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index f48a0a9faa6..98219eaee97 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -372,11 +372,11 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: .. data:: UF_COMPRESSED - The file is stored compressed (Mac OS X 10.6+). + The file is stored compressed (macOS 10.6+). .. data:: UF_HIDDEN - The file should not be displayed in a GUI (Mac OS X 10.5+). + The file should not be displayed in a GUI (macOS 10.5+). .. data:: SF_ARCHIVED @@ -398,7 +398,7 @@ The following flags can be used in the *flags* argument of :func:`os.chflags`: The file is a snapshot file. -See the \*BSD or Mac OS systems man page :manpage:`chflags(2)` for more information. +See the \*BSD or macOS systems man page :manpage:`chflags(2)` for more information. On Windows, the following file attribute constants are available for use when testing bits in the ``st_file_attributes`` member returned by :func:`os.stat`. diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index bb03a2ce6ee..8638abfb697 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -643,7 +643,7 @@ However, for reading convenience, most of the examples show sorted sequences. .. versionadded:: 3.10 -.. function:: linear_regression(x, y, /) +.. function:: linear_regression(x, y, /, *, proportional=False) Return the slope and intercept of `simple linear regression `_ @@ -677,8 +677,18 @@ However, for reading convenience, most of the examples show sorted sequences. >>> round(slope * 2019 + intercept) 16 + If *proportional* is true, the independent variable *x* and the + dependent variable *y* are assumed to be directly proportional. + The data is fit to a line passing through the origin. + Since the *intercept* will always be 0.0, the underlying linear + function simplifies to: + + *y = slope \* x + noise* + .. versionadded:: 3.10 + .. versionchanged:: 3.11 + Added support for *proportional*. Exceptions ---------- diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index d6eff144cce..3c0ba94c73c 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -178,13 +178,14 @@ operators are only defined where they make sense; for example, they raise a single: __ge__() (instance method) Non-identical instances of a class normally compare as non-equal unless the -class defines the :meth:`__eq__` method. +class defines the :meth:`~object.__eq__` method. Instances of a class cannot be ordered with respect to other instances of the same class, or other types of object, unless the class defines enough of the -methods :meth:`__lt__`, :meth:`__le__`, :meth:`__gt__`, and :meth:`__ge__` (in -general, :meth:`__lt__` and :meth:`__eq__` are sufficient, if you want the -conventional meanings of the comparison operators). +methods :meth:`~object.__lt__`, :meth:`~object.__le__`, :meth:`~object.__gt__`, and +:meth:`~object.__ge__` (in general, :meth:`~object.__lt__` and +:meth:`~object.__eq__` are sufficient, if you want the conventional meanings of the +comparison operators). The behavior of the :keyword:`is` and :keyword:`is not` operators cannot be customized; also they can be applied to any two objects and never raise an @@ -352,7 +353,7 @@ Notes: The numeric literals accepted include the digits ``0`` to ``9`` or any Unicode equivalent (code points with the ``Nd`` property). - See https://www.unicode.org/Public/13.0.0/ucd/extracted/DerivedNumericType.txt + See https://www.unicode.org/Public/14.0.0/ucd/extracted/DerivedNumericType.txt for a complete list of code points with the ``Nd`` property. @@ -499,7 +500,7 @@ class`. In addition, it provides a few more methods: .. versionadded:: 3.10 -.. method:: int.to_bytes(length, byteorder, *, signed=False) +.. method:: int.to_bytes(length=1, byteorder='big', *, signed=False) Return an array of bytes representing an integer. @@ -513,25 +514,45 @@ class`. In addition, it provides a few more methods: >>> x.to_bytes((x.bit_length() + 7) // 8, byteorder='little') b'\xe8\x03' - The integer is represented using *length* bytes. An :exc:`OverflowError` - is raised if the integer is not representable with the given number of - bytes. + The integer is represented using *length* bytes, and defaults to 1. An + :exc:`OverflowError` is raised if the integer is not representable with + the given number of bytes. The *byteorder* argument determines the byte order used to represent the - integer. If *byteorder* is ``"big"``, the most significant byte is at the - beginning of the byte array. If *byteorder* is ``"little"``, the most - significant byte is at the end of the byte array. To request the native - byte order of the host system, use :data:`sys.byteorder` as the byte order - value. + integer, and defaults to ``"big"``. If *byteorder* is + ``"big"``, the most significant byte is at the beginning of the byte + array. If *byteorder* is ``"little"``, the most significant byte is at + the end of the byte array. The *signed* argument determines whether two's complement is used to represent the integer. If *signed* is ``False`` and a negative integer is given, an :exc:`OverflowError` is raised. The default value for *signed* is ``False``. - .. versionadded:: 3.2 + The default values can be used to conveniently turn an integer into a + single byte object. However, when using the default arguments, don't try + to convert a value greater than 255 or you'll get an :exc:`OverflowError`:: -.. classmethod:: int.from_bytes(bytes, byteorder, *, signed=False) + >>> (65).to_bytes() + b'A' + + Equivalent to:: + + def to_bytes(n, length=1, byteorder='big', signed=False): + if byteorder == 'little': + order = range(length) + elif byteorder == 'big': + order = reversed(range(length)) + else: + raise ValueError("byteorder must be either 'little' or 'big'") + + return bytes((n >> i*8) & 0xff for i in order) + + .. versionadded:: 3.2 + .. versionchanged:: 3.11 + Added default argument values for ``length`` and ``byteorder``. + +.. classmethod:: int.from_bytes(bytes, byteorder='big', *, signed=False) Return the integer represented by the given array of bytes. @@ -550,16 +571,34 @@ class`. In addition, it provides a few more methods: iterable producing bytes. The *byteorder* argument determines the byte order used to represent the - integer. If *byteorder* is ``"big"``, the most significant byte is at the - beginning of the byte array. If *byteorder* is ``"little"``, the most - significant byte is at the end of the byte array. To request the native - byte order of the host system, use :data:`sys.byteorder` as the byte order - value. + integer, and defaults to ``"big"``. If *byteorder* is + ``"big"``, the most significant byte is at the beginning of the byte + array. If *byteorder* is ``"little"``, the most significant byte is at + the end of the byte array. To request the native byte order of the host + system, use :data:`sys.byteorder` as the byte order value. The *signed* argument indicates whether two's complement is used to represent the integer. + Equivalent to:: + + def from_bytes(bytes, byteorder='big', signed=False): + if byteorder == 'little': + little_ordered = list(bytes) + elif byteorder == 'big': + little_ordered = list(reversed(bytes)) + else: + raise ValueError("byteorder must be either 'little' or 'big'") + + n = sum(b << i*8 for i, b in enumerate(little_ordered)) + if signed and little_ordered and (little_ordered[-1] & 0x80): + n -= 1 << 8*len(little_ordered) + + return n + .. versionadded:: 3.2 + .. versionchanged:: 3.11 + Added default argument value for ``byteorder``. .. method:: int.as_integer_ratio() @@ -660,7 +699,7 @@ Hashing of numeric types ------------------------ For numbers ``x`` and ``y``, possibly of different types, it's a requirement -that ``hash(x) == hash(y)`` whenever ``x == y`` (see the :meth:`__hash__` +that ``hash(x) == hash(y)`` whenever ``x == y`` (see the :meth:`~object.__hash__` method documentation for more details). For ease of implementation and efficiency across a variety of numeric types (including :class:`int`, :class:`float`, :class:`decimal.Decimal` and :class:`fractions.Fraction`) @@ -772,21 +811,21 @@ using two distinct methods; these are used to allow user-defined classes to support iteration. Sequences, described below in more detail, always support the iteration methods. -One method needs to be defined for container objects to provide iteration +One method needs to be defined for container objects to provide :term:`iterable` support: .. XXX duplicated in reference/datamodel! .. method:: container.__iter__() - Return an iterator object. The object is required to support the iterator - protocol described below. If a container supports different types of - iteration, additional methods can be provided to specifically request + Return an :term:`iterator` object. The object is required to support the + iterator protocol described below. If a container supports different types + of iteration, additional methods can be provided to specifically request iterators for those iteration types. (An example of an object supporting multiple forms of iteration would be a tree structure which supports both breadth-first and depth-first traversal.) This method corresponds to the - :c:member:`~PyTypeObject.tp_iter` slot of the type structure for Python objects in the Python/C - API. + :c:member:`~PyTypeObject.tp_iter` slot of the type structure for Python + objects in the Python/C API. The iterator objects themselves are required to support the following two methods, which together form the :dfn:`iterator protocol`: @@ -794,18 +833,19 @@ methods, which together form the :dfn:`iterator protocol`: .. method:: iterator.__iter__() - Return the iterator object itself. This is required to allow both containers - and iterators to be used with the :keyword:`for` and :keyword:`in` statements. - This method corresponds to the :c:member:`~PyTypeObject.tp_iter` slot of the type structure for - Python objects in the Python/C API. + Return the :term:`iterator` object itself. This is required to allow both + containers and iterators to be used with the :keyword:`for` and + :keyword:`in` statements. This method corresponds to the + :c:member:`~PyTypeObject.tp_iter` slot of the type structure for Python + objects in the Python/C API. .. method:: iterator.__next__() - Return the next item from the container. If there are no further items, raise - the :exc:`StopIteration` exception. This method corresponds to the - :c:member:`~PyTypeObject.tp_iternext` slot of the type structure for Python objects in the - Python/C API. + Return the next item from the :term:`iterator`. If there are no further + items, raise the :exc:`StopIteration` exception. This method corresponds to + the :c:member:`~PyTypeObject.tp_iternext` slot of the type structure for + Python objects in the Python/C API. Python defines several iterator objects to support iteration over general and specific sequence types, dictionaries, and other more specialized forms. The @@ -1295,7 +1335,7 @@ loops. range(start, stop[, step]) The arguments to the range constructor must be integers (either built-in - :class:`int` or any object that implements the ``__index__`` special + :class:`int` or any object that implements the :meth:`~object.__index__` special method). If the *step* argument is omitted, it defaults to ``1``. If the *start* argument is omitted, it defaults to ``0``. If *step* is zero, :exc:`ValueError` is raised. @@ -3778,7 +3818,7 @@ copying. Previous versions compared the raw memory disregarding the item format and the logical array structure. - .. method:: tobytes(order=None) + .. method:: tobytes(order='C') Return the data in the buffer as a bytestring. This is equivalent to calling the :class:`bytes` constructor on the memoryview. :: @@ -4754,9 +4794,9 @@ their implementation of the context management protocol. See the Python's :term:`generator`\s and the :class:`contextlib.contextmanager` decorator provide a convenient way to implement these protocols. If a generator function is decorated with the :class:`contextlib.contextmanager` decorator, it will return a -context manager implementing the necessary :meth:`__enter__` and -:meth:`__exit__` methods, rather than the iterator produced by an undecorated -generator function. +context manager implementing the necessary :meth:`~contextmanager.__enter__` and +:meth:`~contextmanager.__exit__` methods, rather than the iterator produced by an +undecorated generator function. Note that there is no specific slot for any of these methods in the type structure for Python objects in the Python/C API. Extension types wanting to diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index af5463364c4..21a96a4bd5f 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -1149,6 +1149,8 @@ calls these functions. code was zero then return, otherwise raise :exc:`CalledProcessError`. The :exc:`CalledProcessError` object will have the return code in the :attr:`~CalledProcessError.returncode` attribute. + If :func:`check_call` was unable to start the process it will propagate the exception + that was raised. Code needing to capture stdout or stderr should use :func:`run` instead:: diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index ec12e02fb37..7d1b21f05ed 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -396,9 +396,14 @@ always available. ``(type, value, traceback)``. Their meaning is: *type* gets the type of the exception being handled (a subclass of :exc:`BaseException`); *value* gets the exception instance (an instance of the exception type); *traceback* gets - a :ref:`traceback object ` which encapsulates the call - stack at the point where the exception originally occurred. + a :ref:`traceback object ` which typically encapsulates + the call stack at the point where the exception last occurred. + .. versionchanged:: 3.11 + The ``type`` and ``traceback`` fields are now derived from the ``value`` + (the exception instance), so when an exception is modified while it is + being handled, the changes are reflected in the results of subsequent + calls to :func:`exc_info`. .. data:: exec_prefix @@ -1073,7 +1078,11 @@ always available. This is a dictionary that maps module names to modules which have already been loaded. This can be manipulated to force reloading of modules and other tricks. However, replacing the dictionary will not necessarily work as expected and - deleting essential items from the dictionary may cause Python to fail. + deleting essential items from the dictionary may cause Python to fail. If + you want to iterate over this global dictionary always use + ``sys.modules.copy()`` or ``tuple(sys.modules)`` to avoid exceptions as its + size may change during iteration as a side effect of code or activity in + other threads. .. data:: orig_argv @@ -1212,13 +1221,10 @@ always available. .. data:: prefix A string giving the site-specific directory prefix where the platform - independent Python files are installed; by default, this is the string + independent Python files are installed; on Unix, the default is ``'/usr/local'``. This can be set at build time with the ``--prefix`` - argument to the :program:`configure` script. The main collection of Python - library modules is installed in the directory :file:`{prefix}/lib/python{X.Y}` - while the platform independent header files (all except :file:`pyconfig.h`) are - stored in :file:`{prefix}/include/python{X.Y}`, where *X.Y* is the version - number of Python, for example ``3.2``. + argument to the :program:`configure` script. See + :ref:`installation_paths` for derived paths. .. note:: If a :ref:`virtual environment ` is in effect, this value will be changed in ``site.py`` to point to the virtual @@ -1725,13 +1731,13 @@ always available. .. code-block:: shell-session - $ ./python -Xa=b -Xc + $ ./python -Xpycache_prefix=some_path -Xdev Python 3.2a3+ (py3k, Oct 16 2010, 20:14:50) [GCC 4.4.3] on linux2 Type "help", "copyright", "credits" or "license" for more information. >>> import sys >>> sys._xoptions - {'a': 'b', 'c': True} + {'pycache_prefix': 'some_path', 'dev': True} .. impl-detail:: diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 2de55d86c38..713be1e02ce 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -60,6 +60,7 @@ Example of usage:: >>> sysconfig.get_config_vars('AR', 'CXX') ['ar', 'g++'] +.. _installation_paths: Installation paths ------------------ @@ -72,9 +73,9 @@ Every new component that is installed using :mod:`distutils` or a Distutils-based system will follow the same scheme to copy its file in the right places. -Python currently supports seven schemes: +Python currently supports six schemes: -- *posix_prefix*: scheme for POSIX platforms like Linux or Mac OS X. This is +- *posix_prefix*: scheme for POSIX platforms like Linux or macOS. This is the default scheme used when Python or a component is installed. - *posix_home*: scheme for POSIX platforms used when a *home* option is used upon installation. This scheme is used when a component is installed through @@ -84,6 +85,7 @@ Python currently supports seven schemes: located under the user home directory. - *nt*: scheme for NT platforms like Windows. - *nt_user*: scheme for NT platforms, when the *user* option is used. +- *osx_framework_user*: scheme for macOS, when the *user* option is used. Each scheme is itself composed of a series of paths and each path has a unique identifier. Python currently uses eight paths: @@ -94,8 +96,10 @@ identifier. Python currently uses eight paths: platform-specific. - *platlib*: directory for site-specific, platform-specific files. - *purelib*: directory for site-specific, non-platform-specific files. -- *include*: directory for non-platform-specific header files. -- *platinclude*: directory for platform-specific header files. +- *include*: directory for non-platform-specific header files for + the Python C-API. +- *platinclude*: directory for platform-specific header files for + the Python C-API. - *scripts*: directory for script files. - *data*: directory for data files. @@ -225,7 +229,7 @@ Other functions - win-amd64 (64bit Windows on AMD64, aka x86_64, Intel64, and EM64T) - win32 (all others - specifically, sys.platform is returned) - Mac OS X can return: + macOS can return: - macosx-10.6-ppc - macosx-10.4-ppc64 diff --git a/Doc/library/termios.rst b/Doc/library/termios.rst index d75a87c55a4..3b0cb60f874 100644 --- a/Doc/library/termios.rst +++ b/Doc/library/termios.rst @@ -74,6 +74,26 @@ The module defines the following functions: output, :const:`TCIOFF` to suspend input, or :const:`TCION` to restart input. +.. function:: tcgetwinsize(fd) + + Return a tuple ``(ws_row, ws_col)`` containing the tty window size for file + descriptor *fd*. Requires :const:`termios.TIOCGWINSZ` or + :const:`termios.TIOCGSIZE`. + + .. versionadded:: 3.11 + + +.. function:: tcsetwinsize(fd, winsize) + + Set the tty window size for file descriptor *fd* from *winsize*, which is + a two-item tuple ``(ws_row, ws_col)`` like the one returned by + :func:`tcgetwinsize`. Requires at least one of the pairs + (:const:`termios.TIOCGWINSZ`, :const:`termios.TIOCSWINSZ`); + (:const:`termios.TIOCGSIZE`, :const:`termios.TIOCSSIZE`) to be defined. + + .. versionadded:: 3.11 + + .. seealso:: Module :mod:`tty` diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 64d767e98fb..699db14596f 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -607,6 +607,15 @@ The :mod:`test.support` module defines the following functions: target of the "as" clause, if there is one. +.. function:: flush_std_streams() + + Call the ``flush()`` method on :data:`sys.stdout` and then on + :data:`sys.stderr`. It can be used to make sure that the logs order is + consistent before writing into stderr. + + .. versionadded:: 3.11 + + .. function:: print_warning(msg) Print a warning into :data:`sys.__stderr__`. Format the message as: @@ -684,8 +693,8 @@ The :mod:`test.support` module defines the following functions: .. decorator:: requires_mac_version(*min_version) - Decorator for the minimum version when running test on Mac OS X. If the - MAC OS X version is less than the minimum, raise :exc:`unittest.SkipTest`. + Decorator for the minimum version when running test on macOS. If the + macOS version is less than the minimum, raise :exc:`unittest.SkipTest`. .. decorator:: requires_IEEE_754 @@ -1249,7 +1258,7 @@ The :mod:`test.support.threading_helper` module provides support for threading t Context manager catching :class:`threading.Thread` exception using :func:`threading.excepthook`. - Attributes set when an exception is catched: + Attributes set when an exception is caught: * ``exc_type`` * ``exc_value`` @@ -1458,7 +1467,7 @@ The :mod:`test.support.os_helper` module provides support for os tests. .. function:: unlink(filename) Call :func:`os.unlink` on *filename*. On Windows platforms, this is - wrapped with a wait loop that checks for the existence fo the file. + wrapped with a wait loop that checks for the existence of the file. :mod:`test.support.import_helper` --- Utilities for import tests diff --git a/Doc/library/time.rst b/Doc/library/time.rst index cfd67e87501..622f66719ce 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -351,11 +351,31 @@ Functions Suspend execution of the calling thread for the given number of seconds. The argument may be a floating point number to indicate a more precise sleep - time. The actual suspension time may be less than that requested because any - caught signal will terminate the :func:`sleep` following execution of that - signal's catching routine. Also, the suspension time may be longer than - requested by an arbitrary amount because of the scheduling of other activity - in the system. + time. + + If the sleep is interrupted by a signal and no exception is raised by the + signal handler, the sleep is restarted with a recomputed timeout. + + The suspension time may be longer than requested by an arbitrary amount, + because of the scheduling of other activity in the system. + + On Windows, if *secs* is zero, the thread relinquishes the remainder of its + time slice to any other thread that is ready to run. If there are no other + threads ready to run, the function returns immediately, and the thread + continues execution. On Windows 8.1 and newer the implementation uses + a `high-resolution timer + `_ + which provides resolution of 100 nanoseconds. If *secs* is zero, ``Sleep(0)`` is used. + + Unix implementation: + + * Use ``clock_nanosleep()`` if available (resolution: 1 nanosecond); + * Or use ``nanosleep()`` if available (resolution: 1 nanosecond); + * Or use ``select()`` (resolution: 1 microsecond). + + .. versionchanged:: 3.11 + On Unix, the ``clock_nanosleep()`` and ``nanosleep()`` functions are now + used if available. On Windows, a waitable timer is now used. .. versionchanged:: 3.5 The function now sleeps at least *secs* even if the sleep is interrupted diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index d4e8b749db4..7f1c41d4639 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -233,7 +233,7 @@ Where the following options are understood: .. cmdoption:: -u, --unit=U - specify a time unit for timer output; can select nsec, usec, msec, or sec + specify a time unit for timer output; can select ``nsec``, ``usec``, ``msec``, or ``sec`` .. versionadded:: 3.5 diff --git a/Doc/library/tk.rst b/Doc/library/tk.rst index 691c9df5e8f..0cb8fda4e32 100644 --- a/Doc/library/tk.rst +++ b/Doc/library/tk.rst @@ -26,8 +26,8 @@ bundled with Python. Although its standard documentation is weak, good material is available, which includes: references, tutorials, a book and others. :mod:`tkinter` is also famous for having an outdated look and feel, which has been vastly improved in Tk 8.5. Nevertheless, there are many other -GUI libraries that you could be interested in. For more information about -alternatives, see the :ref:`other-gui-packages` section. +GUI libraries that you could be interested in. The Python wiki lists several +alternative `GUI frameworks and tools `_. .. toctree:: @@ -41,7 +41,6 @@ alternatives, see the :ref:`other-gui-packages` section. tkinter.ttk.rst tkinter.tix.rst idle.rst - othergui.rst .. Other sections I have in mind are Tkinter internals diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 39766486f64..a48bc13c8b4 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -11,9 +11,8 @@ -------------- The :mod:`tkinter` package ("Tk interface") is the standard Python interface to -the Tk GUI toolkit. Both Tk and :mod:`tkinter` are available on most Unix -platforms, as well as on Windows systems. (Tk itself is not part of Python; it -is maintained at ActiveState.) +the Tcl/Tk GUI toolkit. Both Tk and :mod:`tkinter` are available on most Unix +platforms, including macOS, as well as on Windows systems. Running ``python -m tkinter`` from the command line should open a window demonstrating a simple Tk interface, letting you know that :mod:`tkinter` is @@ -30,53 +29,50 @@ make the experience more pythonic. This documentation will concentrate on these additions and changes, and refer to the official Tcl/Tk documentation for details that are unchanged. +.. note:: + + Tcl/Tk 8.5 (2007) introduced a modern set of themed user interface components + along with a new API to use them. Both old and new APIs are still available. + Most documentation you will find online still uses the old API and + can be woefully outdated. + .. seealso:: - Tkinter documentation: + * `TkDocs `_ + Extensive tutorial on creating user interfaces with Tkinter. Explains key concepts, + and illustrates recommended approaches using the modern API. - `Python Tkinter Resources `_ - The Python Tkinter Topic Guide provides a great deal of information on using Tk - from Python and links to other sources of information on Tk. + * `Tkinter 8.5 reference: a GUI for Python `_ + Reference documentation for Tkinter 8.5 detailing available classes, methods, and options. - `TKDocs `_ - Extensive tutorial plus friendlier widget pages for some of the widgets. + Tcl/Tk Resources: - `Tkinter 8.5 reference: a GUI for Python `_ - On-line reference material. + * `Tk commands `_ + Comprehensive reference to each of the underlying Tcl/Tk commands used by Tkinter. - `Programming Python `_ - Book by Mark Lutz, has excellent coverage of Tkinter. + * `Tcl/Tk Home Page `_ + Additional documentation, and links to Tcl/Tk core development. - `Modern Tkinter for Busy Python Developers `_ - Book by Mark Roseman about building attractive and modern graphical user interfaces with Python and Tkinter. + Books: - `Python and Tkinter Programming `_ - Book by John Grayson (ISBN 1-884777-81-3). + * `Modern Tkinter for Busy Python Developers `_ + By Mark Roseman. (ISBN 978-1999149567) - Tcl/Tk documentation: + * `Python and Tkinter Programming `_ + By Alan Moore. (ISBN 978-1788835886) - `Tk commands `_ - Most commands are available as :mod:`tkinter` or :mod:`tkinter.ttk` classes. - Change '8.6' to match the version of your Tcl/Tk installation. + * `Programming Python `_ + By Mark Lutz; has excellent coverage of Tkinter. (ISBN 978-0596158101) - `Tcl/Tk recent man pages `_ - Recent Tcl/Tk manuals on www.tcl.tk. - - `ActiveState Tcl Home Page `_ - The Tk/Tcl development is largely taking place at ActiveState. - - `Tcl and the Tk Toolkit `_ - Book by John Ousterhout, the inventor of Tcl. - - `Practical Programming in Tcl and Tk `_ - Brent Welch's encyclopedic book. + * `Tcl and the Tk Toolkit (2nd edition) `_ + By John Ousterhout, inventor of Tcl/Tk, and Ken Jones; does not cover Tkinter. (ISBN 978-0321336330) Architecture ------------ Tcl/Tk is not a single library but rather consists of a few distinct -modules, each with a separate functionality and its own official +modules, each with separate functionality and its own official documentation. Python's binary releases also ship an add-on module together with it. @@ -106,33 +102,26 @@ Ttk Ttk is distributed as part of Tk, starting with Tk version 8.5. Python bindings are provided in a separate module, :mod:`tkinter.ttk`. -Tix - `Tix `_ is an older - third-party Tcl package, an add-on for Tk that adds several new widgets. - Python bindings are found in the :mod:`tkinter.tix` module. - It's deprecated in favor of Ttk. +Internally, Tk and Ttk use facilities of the underlying operating system, +i.e., Xlib on Unix/X11, Cocoa on macOS, GDI on Windows. + +When your Python application uses a class in Tkinter, e.g., to create a widget, +the :mod:`tkinter` module first assembles a Tcl/Tk command string. It passes that +Tcl command string to an internal :mod:`_tkinter` binary module, which then +calls the Tcl interpreter to evaluate it. The Tcl interpreter will then call into the +Tk and/or Ttk packages, which will in turn make calls to Xlib, Cocoa, or GDI. Tkinter Modules --------------- -Most of the time, :mod:`tkinter` is all you really need, but a number of -additional modules are available as well. The Tk interface is located in a -binary module named :mod:`_tkinter`. This module contains the low-level -interface to Tk, and should never be used directly by application programmers. -It is usually a shared library (or DLL), but might in some cases be statically -linked with the Python interpreter. +Support for Tkinter is spread across several modules. Most applications will need the +main :mod:`tkinter` module, as well as the :mod:`tkinter.ttk` module, which provides +the modern themed widget set and API:: -In addition to the Tk interface module, :mod:`tkinter` includes a number of -Python modules, :mod:`tkinter.constants` being one of the most important. -Importing :mod:`tkinter` will automatically import :mod:`tkinter.constants`, -so, usually, to use Tkinter all you need is a simple import statement:: - - import tkinter - -Or, more often:: from tkinter import * + from tkinter import ttk .. class:: Tk(screenName=None, baseName=None, className='Tk', useTk=1) @@ -155,7 +144,10 @@ Or, more often:: subsystem initialized) by calling its :meth:`loadtk` method. -Other modules that provide Tk support include: +The modules that provide Tk support include: + +:mod:`tkinter` + Main Tkinter module. :mod:`tkinter.colorchooser` Dialog to let the user choose a color. @@ -178,9 +170,35 @@ Other modules that provide Tk support include: :mod:`tkinter.simpledialog` Basic dialogs and convenience functions. +:mod:`tkinter.ttk` + Themed widget set introduced in Tk 8.5, providing modern alternatives + for many of the classic widgets in the main :mod:`tkinter` module. + +Additional modules: + +:mod:`_tkinter` + A binary module that contains the low-level interface to Tcl/Tk. + It is automatically imported by the main :mod:`tkinter` module, + and should never be used directly by application programmers. + It is usually a shared library (or DLL), but might in some cases be + statically linked with the Python interpreter. + +:mod:`idlelib` + Python's Integrated Development and Learning Environment (IDLE). Based + on :mod:`tkinter`. + +:mod:`tkinter.constants` + Symbolic constants that can be used in place of strings when passing + various parameters to Tkinter calls. Automatically imported by the + main :mod:`tkinter` module. + :mod:`tkinter.dnd` - Drag-and-drop support for :mod:`tkinter`. This is experimental and should - become deprecated when it is replaced with the Tk DND. + (experimental) Drag-and-drop support for :mod:`tkinter`. This will + become deprecated when it is replaced with the Tk DND. + +:mod:`tkinter.tix` + (deprecated) An older third-party Tcl/Tk package that adds several new + widgets. Better alternatives for most can be found in :mod:`tkinter.ttk`. :mod:`turtle` Turtle graphics in a Tk window. @@ -189,243 +207,241 @@ Other modules that provide Tk support include: Tkinter Life Preserver ---------------------- -.. sectionauthor:: Matt Conway - - This section is not designed to be an exhaustive tutorial on either Tk or -Tkinter. Rather, it is intended as a stop gap, providing some introductory -orientation on the system. +Tkinter. For that, refer to one of the external resources noted earlier. +Instead, this section provides a very quick orientation to what a Tkinter +application looks like, identifies foundational Tk concepts, and +explains how the Tkinter wrapper is structured. -Credits: - -* Tk was written by John Ousterhout while at Berkeley. - -* Tkinter was written by Steen Lumholt and Guido van Rossum. - -* This Life Preserver was written by Matt Conway at the University of Virginia. - -* The HTML rendering, and some liberal editing, was produced from a FrameMaker - version by Ken Manheimer. - -* Fredrik Lundh elaborated and revised the class interface descriptions, to get - them current with Tk 4.2. - -* Mike Clarkson converted the documentation to LaTeX, and compiled the User - Interface chapter of the reference manual. +The remainder of this section will help you to identify the classes, +methods, and options you'll need in your Tkinter application, and where to +find more detailed documentation on them, including in the official Tcl/Tk +reference manual. -How To Use This Section -^^^^^^^^^^^^^^^^^^^^^^^ +A Hello World Program +^^^^^^^^^^^^^^^^^^^^^ -This section is designed in two parts: the first half (roughly) covers -background material, while the second half can be taken to the keyboard as a -handy reference. - -When trying to answer questions of the form "how do I do blah", it is often best -to find out how to do "blah" in straight Tk, and then convert this back into the -corresponding :mod:`tkinter` call. Python programmers can often guess at the -correct Python command by looking at the Tk documentation. This means that in -order to use Tkinter, you will have to know a little bit about Tk. This document -can't fulfill that role, so the best we can do is point you to the best -documentation that exists. Here are some hints: - -* The authors strongly suggest getting a copy of the Tk man pages. - Specifically, the man pages in the ``manN`` directory are most useful. - The ``man3`` man pages describe the C interface to the Tk library and thus - are not especially helpful for script writers. - -* Addison-Wesley publishes a book called Tcl and the Tk Toolkit by John - Ousterhout (ISBN 0-201-63337-X) which is a good introduction to Tcl and Tk for - the novice. The book is not exhaustive, and for many details it defers to the - man pages. - -* :file:`tkinter/__init__.py` is a last resort for most, but can be a good - place to go when nothing else makes sense. - - -A Simple Hello World Program -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +We'll start by walking through a "Hello World" application in Tkinter. This +isn't the smallest one we could write, but has enough to illustrate some +key concepts you'll need to know. :: - import tkinter as tk - - class Application(tk.Frame): - def __init__(self, master=None): - super().__init__(master) - self.master = master - self.pack() - self.create_widgets() - - def create_widgets(self): - self.hi_there = tk.Button(self) - self.hi_there["text"] = "Hello World\n(click me)" - self.hi_there["command"] = self.say_hi - self.hi_there.pack(side="top") - - self.quit = tk.Button(self, text="QUIT", fg="red", - command=self.master.destroy) - self.quit.pack(side="bottom") - - def say_hi(self): - print("hi there, everyone!") - - root = tk.Tk() - app = Application(master=root) - app.mainloop() + from tkinter import * + from tkinter import ttk + root = Tk() + frm = ttk.Frame(root, padding=10) + frm.grid() + ttk.Label(frm, text="Hello World!").grid(column=0, row=0) + ttk.Button(frm, text="Quit", command=root.destroy).grid(column=1, row=0) + root.mainloop() -A (Very) Quick Look at Tcl/Tk ------------------------------ +After the imports, the next line creates an instance of the :class:`Tk` class, +which initializes Tk and creates its associated Tcl interpreter. It also +creates a toplevel window, known as the root window, which serves as the main +window of the application. -The class hierarchy looks complicated, but in actual practice, application -programmers almost always refer to the classes at the very bottom of the -hierarchy. +The following line creates a frame widget, which in this case will contain +a label and a button we'll create next. The frame is fit inside the root +window. -Notes: +The next line creates a label widget holding a static text string. The +:meth:`grid` method is used to specify the relative layout (position) of the +label within its containing frame widget, similar to how tables in HTML work. -* These classes are provided for the purposes of organizing certain functions - under one namespace. They aren't meant to be instantiated independently. +A button widget is then created, and placed to the right of the label. When +pressed, it will call the :meth:`destroy` method of the root window. -* The :class:`Tk` class is meant to be instantiated only once in an application. - Application programmers need not instantiate one explicitly, the system creates - one whenever any of the other classes are instantiated. - -* The :class:`Widget` class is not meant to be instantiated, it is meant only - for subclassing to make "real" widgets (in C++, this is called an 'abstract - class'). - -To make use of this reference material, there will be times when you will need -to know how to read short passages of Tk and how to identify the various parts -of a Tk command. (See section :ref:`tkinter-basic-mapping` for the -:mod:`tkinter` equivalents of what's below.) - -Tk scripts are Tcl programs. Like all Tcl programs, Tk scripts are just lists -of tokens separated by spaces. A Tk widget is just its *class*, the *options* -that help configure it, and the *actions* that make it do useful things. - -To make a widget in Tk, the command is always of the form:: - - classCommand newPathname options - -*classCommand* - denotes which kind of widget to make (a button, a label, a menu...) - -.. index:: single: . (dot); in Tkinter - -*newPathname* - is the new name for this widget. All names in Tk must be unique. To help - enforce this, widgets in Tk are named with *pathnames*, just like files in a - file system. The top level widget, the *root*, is called ``.`` (period) and - children are delimited by more periods. For example, - ``.myApp.controlPanel.okButton`` might be the name of a widget. - -*options* - configure the widget's appearance and in some cases, its behavior. The options - come in the form of a list of flags and values. Flags are preceded by a '-', - like Unix shell command flags, and values are put in quotes if they are more - than one word. - -For example:: - - button .fred -fg red -text "hi there" - ^ ^ \______________________/ - | | | - class new options - command widget (-opt val -opt val ...) - -Once created, the pathname to the widget becomes a new command. This new -*widget command* is the programmer's handle for getting the new widget to -perform some *action*. In C, you'd express this as someAction(fred, -someOptions), in C++, you would express this as fred.someAction(someOptions), -and in Tk, you say:: - - .fred someAction someOptions - -Note that the object name, ``.fred``, starts with a dot. - -As you'd expect, the legal values for *someAction* will depend on the widget's -class: ``.fred disable`` works if fred is a button (fred gets greyed out), but -does not work if fred is a label (disabling of labels is not supported in Tk). - -The legal values of *someOptions* is action dependent. Some actions, like -``disable``, require no arguments, others, like a text-entry box's ``delete`` -command, would need arguments to specify what range of text to delete. +Finally, the :meth:`mainloop` method puts everything on the display, and +responds to user input until the program terminates. -.. _tkinter-basic-mapping: -Mapping Basic Tk into Tkinter ------------------------------ +Important Tk Concepts +^^^^^^^^^^^^^^^^^^^^^ -Class commands in Tk correspond to class constructors in Tkinter. :: +Even this simple program illustrates the following key Tk concepts: - button .fred =====> fred = Button() +widgets + A Tkinter user interface is made up of individual *widgets*. Each widget is + represented as a Python object, instantiated from classes like + :class:`ttk.Frame`, :class:`ttk.Label`, and :class:`ttk.Button`. -The master of an object is implicit in the new name given to it at creation -time. In Tkinter, masters are specified explicitly. :: +widget hierarchy + Widgets are arranged in a *hierarchy*. The label and button were contained + within a frame, which in turn was contained within the root window. When + creating each *child* widget, its *parent* widget is passed as the first + argument to the widget constructor. - button .panel.fred =====> fred = Button(panel) +configuration options + Widgets have *configuration options*, which modify their appearance and + behavior, such as the text to display in a label or button. Different + classes of widgets will have different sets of options. -The configuration options in Tk are given in lists of hyphened tags followed by -values. In Tkinter, options are specified as keyword-arguments in the instance -constructor, and keyword-args for configure calls or as instance indices, in -dictionary style, for established instances. See section -:ref:`tkinter-setting-options` on setting options. :: +geometry management + Widgets aren't automatically added to the user interface when they are + created. A *geometry manager* like ``grid`` controls where in the + user interface they are placed. - button .fred -fg red =====> fred = Button(panel, fg="red") - .fred configure -fg red =====> fred["fg"] = red - OR ==> fred.config(fg="red") - -In Tk, to perform an action on a widget, use the widget name as a command, and -follow it with an action name, possibly with arguments (options). In Tkinter, -you call methods on the class instance to invoke actions on the widget. The -actions (methods) that a given widget can perform are listed in -:file:`tkinter/__init__.py`. :: - - .fred invoke =====> fred.invoke() - -To give a widget to the packer (geometry manager), you call pack with optional -arguments. In Tkinter, the Pack class holds all this functionality, and the -various forms of the pack command are implemented as methods. All widgets in -:mod:`tkinter` are subclassed from the Packer, and so inherit all the packing -methods. See the :mod:`tkinter.tix` module documentation for additional -information on the Form geometry manager. :: - - pack .fred -side left =====> fred.pack(side="left") +event loop + Tkinter reacts to user input, changes from your program, and even refreshes + the display only when actively running an *event loop*. If your program + isn't running the event loop, your user interface won't update. -How Tk and Tkinter are Related ------------------------------- +Understanding How Tkinter Wraps Tcl/Tk +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -From the top down: +When your application uses Tkinter's classes and methods, internally Tkinter +is assembling strings representing Tcl/Tk commands, and executing those +commands in the Tcl interpreter attached to your applicaton's :class:`Tk` +instance. -Your App Here (Python) - A Python application makes a :mod:`tkinter` call. +Whether it's trying to navigate reference documentation, trying to find +the right method or option, adapting some existing code, or debugging your +Tkinter application, there are times that it will be useful to understand +what those underlying Tcl/Tk commands look like. -tkinter (Python Package) - This call (say, for example, creating a button widget), is implemented in - the :mod:`tkinter` package, which is written in Python. This Python - function will parse the commands and the arguments and convert them into a - form that makes them look as if they had come from a Tk script instead of - a Python script. +To illustrate, here is the Tcl/Tk equivalent of the main part of the Tkinter +script above. -_tkinter (C) - These commands and their arguments will be passed to a C function in the - :mod:`_tkinter` - note the underscore - extension module. +:: -Tk Widgets (C and Tcl) - This C function is able to make calls into other C modules, including the C - functions that make up the Tk library. Tk is implemented in C and some Tcl. - The Tcl part of the Tk widgets is used to bind certain default behaviors to - widgets, and is executed once at the point where the Python :mod:`tkinter` - package is imported. (The user never sees this stage). + ttk::frame .frm -padding 10 + grid .frm + grid [ttk::label .frm.lbl -text "Hello World!"] -column 0 -row 0 + grid [ttk::button .frm.btn -text "Quit" -command "destroy ."] -column 1 -row 0 -Tk (C) - The Tk part of the Tk Widgets implement the final mapping to ... -Xlib (C) - the Xlib library to draw graphics on the screen. +Tcl's syntax is similar to many shell languages, where the first word is the +command to be executed, with arguments to that command following it, separated +by spaces. Without getting into too many details, notice the following: + +* The commands used to create widgets (like ``ttk::frame``) correspond to + widget classes in Tkinter. + +* Tcl widget options (like ``-text``) correspond to keyword arguments in + Tkinter. + +* Widgets are referred to by a *pathname* in Tcl (like ``.frm.btn``), + whereas Tkinter doesn't use names but object references. + +* A widget's place in the widget hierarchy is encoded in its (hierarchical) + pathname, which uses a ``.`` (dot) as a path separator. The pathname for + the root window is just ``.`` (dot). In Tkinter, the hierarchy is defined + not by pathname but by specifying the parent widget when creating each + child widget. + +* Operations which are implemented as separate *commands* in Tcl (like + ``grid`` or ``destroy``) are represented as *methods* on Tkinter widget + objects. As you'll see shortly, at other times Tcl uses what appear to be + method calls on widget objects, which more closely mirror what would is + used in Tkinter. + + +How do I...? What option does...? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you're not sure how to do something in Tkinter, and you can't immediately +find it in the tutorial or reference documentation you're using, there are a +few strategies that can be helpful. + +First, remember that the details of how individual widgets work may vary +across different versions of both Tkinter and Tcl/Tk. If you're searching +documentation, make sure it corresponds to the Python and Tcl/Tk versions +installed on your system. + +When searching for how to use an API, it helps to know the exact name of the +class, option, or method that you're using. Introspection, either in an +interactive Python shell or with :func:`print`, can help you identify what +you need. + +To find out what configuration options are available on any widget, call its +:meth:`configure` method, which returns a dictionary containing a variety of +information about each object, including its default and current values. Use +:meth:`keys` to get just the names of each option. + +:: + + btn = ttk.Button(frm, ...) + print(btn.configure().keys()) + +As most widgets have many configuration options in common, it can be useful +to find out which are specific to a particular widget class. Comparing the +list of options to that of a simpler widget, like a frame, is one way to +do that. + +:: + + print(set(btn.configure().keys()) - set(frm.configure().keys())) + +Similarly, you can find the available methods for a widget object using the +standard :func:`dir` function. If you try it, you'll see there are over 200 +common widget methods, so again identifying those specific to a widget class +is helpful. + +:: + + print(dir(btn)) + print(set(dir(btn)) - set(dir(frm))) + + +Navigating the Tcl/Tk Reference Manual +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As noted, the official `Tk commands `_ +reference manual (man pages) is often the most accurate description of what +specific operations on widgets do. Even when you know the name of the option +or method that you need, you may still have a few places to look. + +While all operations in Tkinter are implemented as method calls on widget +objects, you've seen that many Tcl/Tk operations appear as commands that +take a widget pathname as its first parameter, followed by optional +parameters, e.g. + +:: + + destroy . + grid .frm.btn -column 0 -row 0 + +Others, however, look more like methods called on a widget object (in fact, +when you create a widget in Tcl/Tk, it creates a Tcl command with the name +of the widget pathname, with the first parameter to that command being the +name of a method to call). + +:: + + .frm.btn invoke + .frm.lbl configure -text "Goodbye" + + +In the official Tcl/Tk reference documentation, you'll find most operations +that look like method calls on the man page for a specific widget (e.g., +you'll find the :meth:`invoke` method on the +`ttk::button `_ +man page), while functions that take a widget as a parameter often have +their own man page (e.g., +`grid `_). + +You'll find many common options and methods in the +`options `_ or +`ttk::widget `_ man +pages, while others are found in the man page for a specific widget class. + +You'll also find that many Tkinter methods have compound names, e.g., +:func:`winfo_x`, :func:`winfo_height`, :func:`winfo_viewable`. You'd find +documentation for all of these in the +`winfo `_ man page. + +.. note:: + Somewhat confusingly, there are also methods on all Tkinter widgets + that don't actually operate on the widget, but operate at a global + scope, independent of any widget. Examples are methods for accessing + the clipboard or the system bell. (They happen to be implemented as + methods in the base :class:`Widget` class that all Tkinter widgets + inherit from). Threading model diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index 6967d750965..2db4c0f9143 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -759,7 +759,7 @@ ones inherited from :class:`ttk.Widget`. Platform-specific notes ^^^^^^^^^^^^^^^^^^^^^^^ -* On MacOS X, toplevel windows automatically include a built-in size grip +* On macOS, toplevel windows automatically include a built-in size grip by default. Adding a :class:`Sizegrip` is harmless, since the built-in grip will just mask the widget. diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 83d5c8c6fcb..df4a38c9555 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -353,11 +353,12 @@ capture data for later printing in a lightweight fashion. .. versionchanged:: 3.6 Long sequences of repeated frames are now abbreviated. - .. method:: format_frame(frame) + .. method:: format_frame_summary(frame_summary) Returns a string for printing one of the frames involved in the stack. - This method gets called for each frame object to be printed in the - :class:`StackSummary`. + This method is called for each :class:`FrameSummary` object to be + printed by :meth:`StackSummary.format`. If it returns ``None``, the + frame is omitted from the output. .. versionadded:: 3.11 @@ -367,7 +368,7 @@ capture data for later printing in a lightweight fashion. .. versionadded:: 3.5 -:class:`FrameSummary` objects represent a single frame in a traceback. +A :class:`FrameSummary` object represents a single frame in a traceback. .. class:: FrameSummary(filename, lineno, name, lookup_line=True, locals=None, line=None) diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst index 20f668c7282..68432aeaecb 100644 --- a/Doc/library/tracemalloc.rst +++ b/Doc/library/tracemalloc.rst @@ -740,12 +740,12 @@ Traceback .. method:: format(limit=None, most_recent_first=False) - Format the traceback as a list of lines with newlines. Use the - :mod:`linecache` module to retrieve lines from the source code. - If *limit* is set, format the *limit* most recent frames if *limit* - is positive. Otherwise, format the ``abs(limit)`` oldest frames. - If *most_recent_first* is ``True``, the order of the formatted frames - is reversed, returning the most recent frame first instead of last. + Format the traceback as a list of lines. Use the :mod:`linecache` module to + retrieve lines from the source code. If *limit* is set, format the *limit* + most recent frames if *limit* is positive. Otherwise, format the + ``abs(limit)`` oldest frames. If *most_recent_first* is ``True``, the order + of the formatted frames is reversed, returning the most recent frame first + instead of last. Similar to the :func:`traceback.format_tb` function, except that :meth:`.format` does not include newlines. diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 88fe47a8d4a..5cd42f1fc29 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -243,7 +243,7 @@ Standard names are defined for the following types: .. note:: A future version of Python may stop setting this attribute by default. - To guard against this potential change, preferrably read from the + To guard against this potential change, preferably read from the :attr:`__spec__` attribute instead or use ``getattr(module, "__loader__", None)`` if you explicitly need to use this attribute. @@ -268,7 +268,7 @@ Standard names are defined for the following types: .. note:: A future version of Python may stop setting this attribute by default. - To guard against this potential change, preferrably read from the + To guard against this potential change, preferably read from the :attr:`__spec__` attribute instead or use ``getattr(module, "__package__", None)`` if you explicitly need to use this attribute. @@ -312,7 +312,7 @@ Standard names are defined for the following types: This type can now be subclassed. -.. data:: UnionType +.. class:: UnionType The type of :ref:`union type expressions`. diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index e8d7c9ee01b..735d477db43 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -17,13 +17,10 @@ -------------- -This module provides runtime support for type hints as specified by -:pep:`484`, :pep:`526`, :pep:`544`, :pep:`586`, :pep:`589`, :pep:`591`, -:pep:`612` and :pep:`613`. -The most fundamental support consists of the types :data:`Any`, :data:`Union`, -:data:`Tuple`, :data:`Callable`, :class:`TypeVar`, and -:class:`Generic`. For full specification please see :pep:`484`. For -a simplified introduction to type hints see :pep:`483`. +This module provides runtime support for type hints. The most fundamental +support consists of the types :data:`Any`, :data:`Union`, :data:`Callable`, +:class:`TypeVar`, and :class:`Generic`. For a full specification, please see +:pep:`484`. For a simplified introduction to type hints, see :pep:`483`. The function below takes and returns a string and is annotated as follows:: @@ -35,6 +32,43 @@ In the function ``greeting``, the argument ``name`` is expected to be of type :class:`str` and the return type :class:`str`. Subtypes are accepted as arguments. +.. _relevant-peps: + +Relevant PEPs +============= + +Since the initial introduction of type hints in :pep:`484` and :pep:`483`, a +number of PEPs have modified and enhanced Python's framework for type +annotations. These include: + +* :pep:`526`: Syntax for Variable Annotations + *Introducing* syntax for annotating variables outside of function + definitions, and :data:`ClassVar` +* :pep:`544`: Protocols: Structural subtyping (static duck typing) + *Introducing* :class:`Protocol` and the + :func:`@runtime_checkable` decorator +* :pep:`585`: Type Hinting Generics In Standard Collections + *Introducing* :class:`types.GenericAlias` and the ability to use standard + library classes as :ref:`generic types` +* :pep:`586`: Literal Types + *Introducing* :data:`Literal` +* :pep:`589`: TypedDict: Type Hints for Dictionaries with a Fixed Set of Keys + *Introducing* :class:`TypedDict` +* :pep:`591`: Adding a final qualifier to typing + *Introducing* :data:`Final` and the :func:`@final` decorator +* :pep:`593`: Flexible function and variable annotations + *Introducing* :data:`Annotated` +* :pep:`604`: Allow writing union types as ``X | Y`` + *Introducing* :data:`types.UnionType` and the ability to use + the binary-or operator ``|`` to signify a + :ref:`union of types` +* :pep:`612`: Parameter Specification Variables + *Introducing* :class:`ParamSpec` and :data:`Concatenate` +* :pep:`613`: Explicit Type Aliases + *Introducing* :data:`TypeAlias` +* :pep:`647`: User-Defined Type Guards + *Introducing* :data:`TypeGuard` + .. _type-aliases: Type aliases @@ -222,6 +256,7 @@ called :class:`TypeVar`. def first(l: Sequence[T]) -> T: # Generic function return l[0] +.. _user-defined-generics: User-defined generic types ========================== @@ -256,8 +291,8 @@ A user-defined class can be defined as a generic class. single type parameter ``T`` . This also makes ``T`` valid as a type within the class body. -The :class:`Generic` base class defines :meth:`__class_getitem__` so that -``LoggedVar[t]`` is valid as a type:: +The :class:`Generic` base class defines :meth:`~object.__class_getitem__` so +that ``LoggedVar[t]`` is valid as a type:: from collections.abc import Iterable @@ -321,11 +356,11 @@ not generic but implicitly inherits from ``Iterable[Any]``:: User defined generic type aliases are also supported. Examples:: from collections.abc import Iterable - from typing import TypeVar, Union + from typing import TypeVar S = TypeVar('S') - Response = Union[Iterable[S], int] + Response = Iterable[S] | int - # Return type here is same as Union[Iterable[str], int] + # Return type here is same as Iterable[str] | int def response(query: str) -> Response[str]: ... @@ -588,9 +623,9 @@ These can be used as types in annotations using ``[]``, each having a unique syn .. data:: Union - Union type; ``Union[X, Y]`` means either X or Y. + Union type; ``Union[X, Y]`` is equivalent to ``X | Y`` and means either X or Y. - To define a union, use e.g. ``Union[int, str]``. Details: + To define a union, use e.g. ``Union[int, str]`` or the shorthand ``int | str``. Details: * The arguments must be types and there must be at least one. @@ -604,18 +639,16 @@ These can be used as types in annotations using ``[]``, each having a unique syn * Redundant arguments are skipped, e.g.:: - Union[int, str, int] == Union[int, str] + Union[int, str, int] == Union[int, str] == int | str * When comparing unions, the argument order is ignored, e.g.:: Union[int, str] == Union[str, int] - * You cannot subclass or instantiate a union. + * You cannot subclass or instantiate a ``Union``. * You cannot write ``Union[X][Y]``. - * You can use ``Optional[X]`` as a shorthand for ``Union[X, None]``. - .. versionchanged:: 3.7 Don't remove explicit subclasses from unions at runtime. @@ -627,7 +660,7 @@ These can be used as types in annotations using ``[]``, each having a unique syn Optional type. - ``Optional[X]`` is equivalent to ``Union[X, None]``. + ``Optional[X]`` is equivalent to ``X | None`` (or ``Union[X, None]``). Note that this is not the same concept as an optional argument, which is one that has a default. An optional argument with a @@ -644,6 +677,10 @@ These can be used as types in annotations using ``[]``, each having a unique syn def foo(arg: Optional[int] = None) -> None: ... + .. versionchanged:: 3.10 + Optional can now be written as ``X | None``. See + :ref:`union type expressions`. + .. data:: Callable Callable type; ``Callable[[int], str]`` is a function of (int) -> str. @@ -770,7 +807,7 @@ These can be used as types in annotations using ``[]``, each having a unique syn :ref:`type variables `, and unions of any of these types. For example:: - def new_non_team_user(user_class: Type[Union[BasicUser, ProUser]]): ... + def new_non_team_user(user_class: Type[BasicUser | ProUser]): ... ``Type[Any]`` is equivalent to ``Type`` which in turn is equivalent to ``type``, which is the root of Python's metaclass hierarchy. @@ -951,7 +988,7 @@ These can be used as types in annotations using ``[]``, each having a unique syn conditional code flow and applying the narrowing to a block of code. The conditional expression here is sometimes referred to as a "type guard":: - def is_str(val: Union[str, float]): + def is_str(val: str | float): # "isinstance" type guard if isinstance(val, str): # Type of ``val`` is narrowed to ``str`` @@ -973,16 +1010,16 @@ These can be used as types in annotations using ``[]``, each having a unique syn For example:: - def is_str_list(val: List[object]) -> TypeGuard[List[str]]: + def is_str_list(val: list[object]) -> TypeGuard[list[str]]: '''Determines whether all objects in the list are strings''' return all(isinstance(x, str) for x in val) - def func1(val: List[object]): + def func1(val: list[object]): if is_str_list(val): - # Type of ``val`` is narrowed to ``List[str]``. + # Type of ``val`` is narrowed to ``list[str]``. print(" ".join(val)) else: - # Type of ``val`` remains as ``List[object]``. + # Type of ``val`` remains as ``list[object]``. print("Not a list of strings!") If ``is_str_list`` is a class or instance method, then the type in @@ -997,8 +1034,8 @@ These can be used as types in annotations using ``[]``, each having a unique syn ``TypeB`` need not be a narrower form of ``TypeA`` -- it can even be a wider form. The main reason is to allow for things like - narrowing ``List[object]`` to ``List[str]`` even though the latter - is not a subtype of the former, since ``List`` is invariant. + narrowing ``list[object]`` to ``list[str]`` even though the latter + is not a subtype of the former, since ``list`` is invariant. The responsibility of writing type-safe type guards is left to the user. ``TypeGuard`` also works with type variables. For more information, see @@ -1505,8 +1542,8 @@ Other concrete types :func:`open`. .. deprecated-removed:: 3.8 3.12 - These types are also in the ``typing.io`` namespace, which was - never supported by type checkers and will be removed. + The ``typing.io`` namespace is deprecated and will be removed. + These types should be directly imported from ``typing`` instead. .. class:: Pattern Match @@ -1519,8 +1556,8 @@ Other concrete types ``Match[bytes]``. .. deprecated-removed:: 3.8 3.12 - These types are also in the ``typing.re`` namespace, which was - never supported by type checkers and will be removed. + The ``typing.re`` namespace is deprecated and will be removed. + These types should be directly imported from ``typing`` instead. .. deprecated:: 3.9 Classes ``Pattern`` and ``Match`` from :mod:`re` now support ``[]``. @@ -2013,6 +2050,13 @@ Introspection helpers 'name': Annotated[str, 'some marker'] } + .. note:: + + :func:`get_type_hints` does not work with imported + :ref:`type aliases ` that include forward references. + Enabling postponed evaluation of annotations (:pep:`563`) may remove + the need for most forward references. + .. versionchanged:: 3.9 Added ``include_extras`` parameter as part of :pep:`593`. @@ -2024,7 +2068,7 @@ Introspection helpers For a typing object of the form ``X[Y, Z, ...]`` these functions return ``X`` and ``(Y, Z, ...)``. If ``X`` is a generic alias for a builtin or :mod:`collections` class, it gets normalized to the original class. - If ``X`` is a :class:`Union` or :class:`Literal` contained in another + If ``X`` is a union or :class:`Literal` contained in another generic type, the order of ``(Y, Z, ...)`` may be different from the order of the original arguments ``[Y, Z, ...]`` due to type caching. For unsupported objects return ``None`` and ``()`` correspondingly. @@ -2049,15 +2093,15 @@ Introspection helpers year: int is_typeddict(Film) # => True - is_typeddict(Union[list, str]) # => False + is_typeddict(list | str) # => False .. versionadded:: 3.10 .. class:: ForwardRef A class used for internal typing representation of string forward references. - For example, ``List["SomeClass"]`` is implicitly transformed into - ``List[ForwardRef("SomeClass")]``. This class should not be instantiated by + For example, ``list["SomeClass"]`` is implicitly transformed into + ``list[ForwardRef("SomeClass")]``. This class should not be instantiated by a user, but may be used by introspection tools. .. note:: diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index dade3f265b5..6276f6382a0 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -17,8 +17,8 @@ This module provides access to the Unicode Character Database (UCD) which defines character properties for all Unicode characters. The data contained in -this database is compiled from the `UCD version 13.0.0 -`_. +this database is compiled from the `UCD version 14.0.0 +`_. The module uses the same names and symbols as defined by Unicode Standard Annex #44, `"Unicode Character Database" @@ -175,6 +175,6 @@ Examples: .. rubric:: Footnotes -.. [#] https://www.unicode.org/Public/13.0.0/ucd/NameAliases.txt +.. [#] https://www.unicode.org/Public/14.0.0/ucd/NameAliases.txt -.. [#] https://www.unicode.org/Public/13.0.0/ucd/NamedSequences.txt +.. [#] https://www.unicode.org/Public/14.0.0/ucd/NamedSequences.txt diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 2f82fded994..0856c3fbded 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -2020,7 +2020,7 @@ The full list of supported magic methods is: * Context manager: ``__enter__``, ``__exit__``, ``__aenter__`` and ``__aexit__`` * Unary numeric methods: ``__neg__``, ``__pos__`` and ``__invert__`` * The numeric methods (including right hand and in-place variants): - ``__add__``, ``__sub__``, ``__mul__``, ``__matmul__``, ``__div__``, ``__truediv__``, + ``__add__``, ``__sub__``, ``__mul__``, ``__matmul__``, ``__truediv__``, ``__floordiv__``, ``__mod__``, ``__divmod__``, ``__lshift__``, ``__rshift__``, ``__and__``, ``__xor__``, ``__or__``, and ``__pow__`` * Numeric conversion methods: ``__complex__``, ``__int__``, ``__float__`` diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 99c2f6e0294..bf7a1e87928 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -151,6 +151,10 @@ The above examples show the most commonly used :mod:`unittest` features which are sufficient to meet many everyday testing needs. The remainder of the documentation explores the full feature set from first principles. +.. versionchanged:: 3.11 + The behavior of returning a value from a test method (other than the default + ``None`` value), is now deprecated. + .. _unittest-command-line-interface: @@ -223,7 +227,7 @@ Command-line options Only run test methods and classes that match the pattern or substring. This option may be used multiple times, in which case all test cases that - match of the given patterns are included. + match any of the given patterns are included. Patterns that contain a wildcard character (``*``) are matched against the test name using :meth:`fnmatch.fnmatchcase`; otherwise simple case-sensitive @@ -1233,9 +1237,6 @@ Test cases :meth:`.assertRegex`. .. versionadded:: 3.2 :meth:`.assertNotRegex`. - .. versionadded:: 3.5 - The name ``assertNotRegexpMatches`` is a deprecated alias - for :meth:`.assertNotRegex`. .. method:: assertCountEqual(first, second, msg=None) @@ -1601,40 +1602,6 @@ Test cases :mod:`unittest`-based test framework. -.. _deprecated-aliases: - -Deprecated aliases -################## - -For historical reasons, some of the :class:`TestCase` methods had one or more -aliases that are now deprecated. The following table lists the correct names -along with their deprecated aliases: - - ============================== ====================== ======================= - Method Name Deprecated alias Deprecated alias - ============================== ====================== ======================= - :meth:`.assertEqual` failUnlessEqual assertEquals - :meth:`.assertNotEqual` failIfEqual assertNotEquals - :meth:`.assertTrue` failUnless assert\_ - :meth:`.assertFalse` failIf - :meth:`.assertRaises` failUnlessRaises - :meth:`.assertAlmostEqual` failUnlessAlmostEqual assertAlmostEquals - :meth:`.assertNotAlmostEqual` failIfAlmostEqual assertNotAlmostEquals - :meth:`.assertRegex` assertRegexpMatches - :meth:`.assertNotRegex` assertNotRegexpMatches - :meth:`.assertRaisesRegex` assertRaisesRegexp - ============================== ====================== ======================= - - .. deprecated:: 3.1 - The fail* aliases listed in the second column have been deprecated. - .. deprecated:: 3.2 - The assert* aliases listed in the third column have been deprecated. - .. deprecated:: 3.2 - ``assertRegexpMatches`` and ``assertRaisesRegexp`` have been renamed to - :meth:`.assertRegex` and :meth:`.assertRaisesRegex`. - .. deprecated:: 3.5 - The ``assertNotRegexpMatches`` name is deprecated in favor of :meth:`.assertNotRegex`. - .. _testsuite-objects: Grouping tests @@ -1760,7 +1727,7 @@ Loading and running tests case is created for that method instead. - .. method:: loadTestsFromModule(module, pattern=None) + .. method:: loadTestsFromModule(module, *, pattern=None) Return a suite of all test cases contained in the given module. This method searches *module* for classes derived from :class:`TestCase` and @@ -1784,10 +1751,11 @@ Loading and running tests Support for ``load_tests`` added. .. versionchanged:: 3.5 - The undocumented and unofficial *use_load_tests* default argument is - deprecated and ignored, although it is still accepted for backward - compatibility. The method also now accepts a keyword-only argument - *pattern* which is passed to ``load_tests`` as the third argument. + Support for a keyword-only argument *pattern* has been added. + + .. versionchanged:: 3.11 + The undocumented and unofficial *use_load_tests* parameter has been + removed. .. method:: loadTestsFromName(name, module=None) @@ -2140,8 +2108,6 @@ Loading and running tests :class:`TextTestRunner`. .. versionadded:: 3.2 - This class was previously named ``_TextTestResult``. The old name still - exists as an alias but is deprecated. .. data:: defaultTestLoader @@ -2164,10 +2130,7 @@ Loading and running tests By default this runner shows :exc:`DeprecationWarning`, :exc:`PendingDeprecationWarning`, :exc:`ResourceWarning` and :exc:`ImportWarning` even if they are :ref:`ignored by default - `. Deprecation warnings caused by :ref:`deprecated unittest - methods ` are also special-cased and, when the warning - filters are ``'default'`` or ``'always'``, they will appear only once - per-module, in order to avoid too many warning messages. This behavior can + `. This behavior can be overridden using Python's :option:`!-Wd` or :option:`!-Wa` options (see :ref:`Warning control `) and leaving *warnings* to ``None``. diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index a060cc9ba7f..1478b34bc95 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -48,17 +48,29 @@ or on combining URL components into a URL string. result, except for a leading slash in the *path* component, which is retained if present. For example: + .. doctest:: + :options: +NORMALIZE_WHITESPACE + >>> from urllib.parse import urlparse - >>> o = urlparse('http://www.cwi.nl:80/%7Eguido/Python.html') - >>> o # doctest: +NORMALIZE_WHITESPACE - ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') + >>> urlparse("scheme://netloc/path;parameters?query#fragment") + ParseResult(scheme='scheme', netloc='netloc', path='/path;parameters', params='', + query='query', fragment='fragment') + >>> o = urlparse("http://docs.python.org:80/3/library/urllib.parse.html?" + ... "highlight=params#url-parsing") + >>> o + ParseResult(scheme='http', netloc='docs.python.org:80', + path='/3/library/urllib.parse.html', params='', + query='highlight=params', fragment='url-parsing') >>> o.scheme 'http' + >>> o.netloc + 'docs.python.org:80' + >>> o.hostname + 'docs.python.org' >>> o.port 80 - >>> o.geturl() - 'http://www.cwi.nl:80/%7Eguido/Python.html' + >>> o._replace(fragment="").geturl() + 'http://docs.python.org:80/3/library/urllib.parse.html?highlight=params' Following the syntax specifications in :rfc:`1808`, urlparse recognizes a netloc only if it is properly introduced by '//'. Otherwise the @@ -92,31 +104,30 @@ or on combining URL components into a URL string. The return value is a :term:`named tuple`, which means that its items can be accessed by index or as named attributes, which are: - +------------------+-------+--------------------------+----------------------+ - | Attribute | Index | Value | Value if not present | - +==================+=======+==========================+======================+ - | :attr:`scheme` | 0 | URL scheme specifier | *scheme* parameter | - +------------------+-------+--------------------------+----------------------+ - | :attr:`netloc` | 1 | Network location part | empty string | - +------------------+-------+--------------------------+----------------------+ - | :attr:`path` | 2 | Hierarchical path | empty string | - +------------------+-------+--------------------------+----------------------+ - | :attr:`params` | 3 | Parameters for last path | empty string | - | | | element | | - +------------------+-------+--------------------------+----------------------+ - | :attr:`query` | 4 | Query component | empty string | - +------------------+-------+--------------------------+----------------------+ - | :attr:`fragment` | 5 | Fragment identifier | empty string | - +------------------+-------+--------------------------+----------------------+ - | :attr:`username` | | User name | :const:`None` | - +------------------+-------+--------------------------+----------------------+ - | :attr:`password` | | Password | :const:`None` | - +------------------+-------+--------------------------+----------------------+ - | :attr:`hostname` | | Host name (lower case) | :const:`None` | - +------------------+-------+--------------------------+----------------------+ - | :attr:`port` | | Port number as integer, | :const:`None` | - | | | if present | | - +------------------+-------+--------------------------+----------------------+ + +------------------+-------+-------------------------+------------------------+ + | Attribute | Index | Value | Value if not present | + +==================+=======+=========================+========================+ + | :attr:`scheme` | 0 | URL scheme specifier | *scheme* parameter | + +------------------+-------+-------------------------+------------------------+ + | :attr:`netloc` | 1 | Network location part | empty string | + +------------------+-------+-------------------------+------------------------+ + | :attr:`path` | 2 | Hierarchical path | empty string | + +------------------+-------+-------------------------+------------------------+ + | :attr:`params` | 3 | No longer used | always an empty string | + +------------------+-------+-------------------------+------------------------+ + | :attr:`query` | 4 | Query component | empty string | + +------------------+-------+-------------------------+------------------------+ + | :attr:`fragment` | 5 | Fragment identifier | empty string | + +------------------+-------+-------------------------+------------------------+ + | :attr:`username` | | User name | :const:`None` | + +------------------+-------+-------------------------+------------------------+ + | :attr:`password` | | Password | :const:`None` | + +------------------+-------+-------------------------+------------------------+ + | :attr:`hostname` | | Host name (lower case) | :const:`None` | + +------------------+-------+-------------------------+------------------------+ + | :attr:`port` | | Port number as integer, | :const:`None` | + | | | if present | | + +------------------+-------+-------------------------+------------------------+ Reading the :attr:`port` attribute will raise a :exc:`ValueError` if an invalid port is specified in the URL. See section diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 130c7d694e8..88e93ba6b00 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -169,8 +169,8 @@ The :mod:`urllib.request` module defines the following functions: This helper function returns a dictionary of scheme to proxy server URL mappings. It scans the environment for variables named ``_proxy``, in a case insensitive approach, for all operating systems first, and when it - cannot find it, looks for proxy information from Mac OSX System - Configuration for Mac OS X and Windows Systems Registry for Windows. + cannot find it, looks for proxy information from System + Configuration for macOS and Windows Systems Registry for Windows. If both lowercase and uppercase environment variables exist (and disagree), lowercase is preferred. @@ -303,8 +303,8 @@ The following classes are provided: the list of proxies from the environment variables ``_proxy``. If no proxy environment variables are set, then in a Windows environment proxy settings are obtained from the registry's - Internet Settings section, and in a Mac OS X environment proxy information - is retrieved from the OS X System Configuration Framework. + Internet Settings section, and in a macOS environment proxy information + is retrieved from the System Configuration Framework. To disable autodetected proxy pass an empty dictionary. @@ -655,7 +655,7 @@ OpenerDirector Objects optional *timeout* parameter specifies a timeout in seconds for blocking operations like the connection attempt (if not specified, the global default timeout setting will be used). The timeout feature actually works only for - HTTP, HTTPS and FTP connections). + HTTP, HTTPS and FTP connections. .. method:: OpenerDirector.error(proto, *args) @@ -876,7 +876,17 @@ HTTPRedirectHandler Objects .. method:: HTTPRedirectHandler.http_error_307(req, fp, code, msg, hdrs) The same as :meth:`http_error_301`, but called for the 'temporary redirect' - response. + response. It does not allow changing the request method from ``POST`` + to ``GET``. + + +.. method:: HTTPRedirectHandler.http_error_308(req, fp, code, msg, hdrs) + + The same as :meth:`http_error_301`, but called for the 'permanent redirect' + response. It does not allow changing the request method from ``POST`` + to ``GET``. + + .. versionadded:: 3.11 .. _http-cookie-processor: diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 5eb50948015..092781b5ff1 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -426,7 +426,7 @@ subclass which installs setuptools and pip into a created virtual environment:: 'more target ' 'directories.') parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', - help='A directory in which to create the + help='A directory in which to create the ' 'virtual environment.') parser.add_argument('--no-setuptools', default=False, action='store_true', dest='nodist', diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 5a8df4c27dc..1102c634eda 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -88,6 +88,10 @@ support weak references but can add support through subclassing:: Extension types can easily be made to support weak references; see :ref:`weakref-support`. +When ``__slots__`` are defined for a given type, weak reference support is +disabled unless a ``'__weakref__'`` string is also present in the sequence of +strings in the ``__slots__`` declaration. +See :ref:`__slots__ documentation ` for details. .. class:: ref(object[, callback]) diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index bd0919164d8..19b3c463bb8 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -143,9 +143,9 @@ for the controller classes, all defined in this module. +------------------------+-----------------------------------------+-------+ | ``'windows-default'`` | :class:`WindowsDefault` | \(2) | +------------------------+-----------------------------------------+-------+ -| ``'macosx'`` | :class:`MacOSX('default')` | \(3) | +| ``'macosx'`` | :class:`MacOSXOSAScript('default')` | \(3) | +------------------------+-----------------------------------------+-------+ -| ``'safari'`` | :class:`MacOSX('safari')` | \(3) | +| ``'safari'`` | :class:`MacOSXOSAScript('safari')` | \(3) | +------------------------+-----------------------------------------+-------+ | ``'google-chrome'`` | :class:`Chrome('google-chrome')` | | +------------------------+-----------------------------------------+-------+ @@ -169,11 +169,14 @@ Notes: Only on Windows platforms. (3) - Only on Mac OS X platform. + Only on macOS platform. .. versionadded:: 3.3 Support for Chrome/Chromium has been added. +.. deprecated-removed:: 3.11 3.13 + :class:`MacOSX` is deprecated, use :class:`MacOSXOSAScript` instead. + Here are some simple examples:: url = 'https://docs.python.org/' diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index e92a689de0b..e924448b86d 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -151,8 +151,7 @@ also provides these miscellaneous utilities: .. class:: FileWrapper(filelike, blksize=8192) A wrapper to convert a file-like object to an :term:`iterator`. The resulting objects - support both :meth:`__getitem__` and :meth:`__iter__` iteration styles, for - compatibility with Python 2.1 and Jython. As the object is iterated over, the + are :term:`iterable`\ s. As the object is iterated over, the optional *blksize* parameter will be repeatedly passed to the *filelike* object's :meth:`read` method to obtain bytestrings to yield. When :meth:`read` returns an empty bytestring, iteration is ended and is not resumable. @@ -173,8 +172,8 @@ also provides these miscellaneous utilities: for chunk in wrapper: print(chunk) - .. deprecated:: 3.8 - Support for :meth:`sequence protocol <__getitem__>` is deprecated. + .. versionchanged:: 3.11 + Support for :meth:`__getitem__` method has been removed. :mod:`wsgiref.headers` -- WSGI response header tools diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index d3a5f872204..20984b98b17 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -156,6 +156,9 @@ module documentation. This section lists the differences between the API and The :meth:`writexml` method now preserves the attribute order specified by the user. + .. versionchanged:: 3.9 + The *standalone* parameter was added. + .. method:: Node.toxml(encoding=None, standalone=None) Return a string or byte string containing the XML represented by @@ -174,6 +177,9 @@ module documentation. This section lists the differences between the API and The :meth:`toxml` method now preserves the attribute order specified by the user. + .. versionchanged:: 3.9 + The *standalone* parameter was added. + .. method:: Node.toprettyxml(indent="\\t", newl="\\n", encoding=None, \ standalone=None) @@ -190,6 +196,8 @@ module documentation. This section lists the differences between the API and The :meth:`toprettyxml` method now preserves the attribute order specified by the user. + .. versionchanged:: 3.9 + The *standalone* parameter was added. .. _dom-example: diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index 660c75c1a1b..d1df465a598 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -114,8 +114,8 @@ DOMEventStream Objects .. class:: DOMEventStream(stream, parser, bufsize) - .. deprecated:: 3.8 - Support for :meth:`sequence protocol <__getitem__>` is deprecated. + .. versionchanged:: 3.11 + Support for :meth:`__getitem__` method has been removed. .. method:: getEvent() @@ -144,4 +144,3 @@ DOMEventStream Objects print(node.toxml()) .. method:: DOMEventStream.reset() - diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index 1981cab7cd4..e3b35162961 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -60,22 +60,26 @@ circumvent firewalls. The following table gives an overview of the known attacks and whether the various modules are vulnerable to them. -========================= ============== =============== ============== ============== ============== -kind sax etree minidom pulldom xmlrpc -========================= ============== =============== ============== ============== ============== -billion laughs **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** -quadratic blowup **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** **Vulnerable** -external entity expansion Safe (4) Safe (1) Safe (2) Safe (4) Safe (3) -`DTD`_ retrieval Safe (4) Safe Safe Safe (4) Safe -decompression bomb Safe Safe Safe Safe **Vulnerable** -========================= ============== =============== ============== ============== ============== +========================= ================== ================== ================== ================== ================== +kind sax etree minidom pulldom xmlrpc +========================= ================== ================== ================== ================== ================== +billion laughs **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) +quadratic blowup **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) +external entity expansion Safe (5) Safe (2) Safe (3) Safe (5) Safe (4) +`DTD`_ retrieval Safe (5) Safe Safe Safe (5) Safe +decompression bomb Safe Safe Safe Safe **Vulnerable** +========================= ================== ================== ================== ================== ================== -1. :mod:`xml.etree.ElementTree` doesn't expand external entities and raises a +1. Expat 2.4.1 and newer is not vulnerable to the "billion laughs" and + "quadratic blowup" vulnerabilities. Items still listed as vulnerable due to + potential reliance on system-provided libraries. Check + :data:`pyexpat.EXPAT_VERSION`. +2. :mod:`xml.etree.ElementTree` doesn't expand external entities and raises a :exc:`ParserError` when an entity occurs. -2. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns +3. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns the unexpanded entity verbatim. -3. :mod:`xmlrpclib` doesn't expand external entities and omits them. -4. Since Python 3.7.1, external general entities are no longer processed by +4. :mod:`xmlrpclib` doesn't expand external entities and omits them. +5. Since Python 3.7.1, external general entities are no longer processed by default. diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index ec60ea24db6..793c90f3c4e 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -47,7 +47,7 @@ The available exception and functions in this module are: platforms, use ``adler32(data) & 0xffffffff``. -.. function:: compress(data, /, level=-1) +.. function:: compress(data, /, level=-1, wbits=MAX_WBITS) Compresses the bytes in *data*, returning a bytes object containing compressed data. *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; @@ -55,26 +55,8 @@ The available exception and functions in this module are: is slowest and produces the most. ``0`` (Z_NO_COMPRESSION) is no compression. The default value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression (currently equivalent to level 6). - Raises the :exc:`error` exception if any error occurs. - .. versionchanged:: 3.6 - *level* can now be used as a keyword parameter. - - -.. function:: compressobj(level=-1, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY[, zdict]) - - Returns a compression object, to be used for compressing data streams that won't - fit into memory at once. - - *level* is the compression level -- an integer from ``0`` to ``9`` or ``-1``. - A value of ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, - while a value of ``9`` (Z_BEST_COMPRESSION) is slowest and produces the most. - ``0`` (Z_NO_COMPRESSION) is no compression. The default value is ``-1`` (Z_DEFAULT_COMPRESSION). - Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression - (currently equivalent to level 6). - - *method* is the compression algorithm. Currently, the only supported value is - :const:`DEFLATED`. + .. _compress-wbits: The *wbits* argument controls the size of the history buffer (or the "window size") used when compressing data, and whether a header and @@ -94,6 +76,34 @@ The available exception and functions in this module are: window size logarithm, while including a basic :program:`gzip` header and trailing checksum in the output. + Raises the :exc:`error` exception if any error occurs. + + .. versionchanged:: 3.6 + *level* can now be used as a keyword parameter. + + .. versionchanged:: 3.11 + The *wbits* parameter is now available to set window bits and + compression type. + +.. function:: compressobj(level=-1, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY[, zdict]) + + Returns a compression object, to be used for compressing data streams that won't + fit into memory at once. + + *level* is the compression level -- an integer from ``0`` to ``9`` or ``-1``. + A value of ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, + while a value of ``9`` (Z_BEST_COMPRESSION) is slowest and produces the most. + ``0`` (Z_NO_COMPRESSION) is no compression. The default value is ``-1`` (Z_DEFAULT_COMPRESSION). + Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression + (currently equivalent to level 6). + + *method* is the compression algorithm. Currently, the only supported value is + :const:`DEFLATED`. + + The *wbits* parameter controls the size of the history buffer (or the + "window size"), and what header and trailer format will be used. It has + the same meaning as `described for compress() <#compress-wbits>`__. + The *memLevel* argument controls the amount of memory used for the internal compression state. Valid values range from ``1`` to ``9``. Higher values use more memory, but are faster and produce smaller output. diff --git a/Doc/license.rst b/Doc/license.rst index c36ab3212a1..cd03411d6c9 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -655,7 +655,7 @@ OpenSSL The modules :mod:`hashlib`, :mod:`posix`, :mod:`ssl`, :mod:`crypt` use the OpenSSL library for added performance if made available by the -operating system. Additionally, the Windows and Mac OS X installers for +operating system. Additionally, the Windows and macOS installers for Python may include a copy of the OpenSSL libraries, so we include a copy of the OpenSSL license here:: diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 8afcadbfbcf..cf8ad1787b2 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -418,8 +418,8 @@ usage patterns to be encapsulated for convenient reuse. The execution of the :keyword:`with` statement with one "item" proceeds as follows: -#. The context expression (the expression given in the :token:`with_item`) is - evaluated to obtain a context manager. +#. The context expression (the expression given in the + :token:`~python-grammar:with_item`) is evaluated to obtain a context manager. #. The context manager's :meth:`__enter__` is loaded for later use. @@ -585,8 +585,8 @@ Here's an overview of the logical flow of a match statement: #. If the pattern succeeds, the corresponding guard (if present) is evaluated. In this case all name bindings are guaranteed to have happened. - * If the guard evaluates as truthy or missing, the ``block`` inside ``case_block`` is - executed. + * If the guard evaluates as true or is missing, the ``block`` inside + ``case_block`` is executed. * Otherwise, the next ``case_block`` is attempted as described above. @@ -637,10 +637,10 @@ The logical flow of a ``case`` block with a ``guard`` follows: #. If the pattern succeeded, evaluate the ``guard``. - * If the ``guard`` condition evaluates to "truthy", the case block is + * If the ``guard`` condition evaluates as true, the case block is selected. - * If the ``guard`` condition evaluates to "falsy", the case block is not + * If the ``guard`` condition evaluates as false, the case block is not selected. * If the ``guard`` raises an exception during evaluation, the exception @@ -797,7 +797,8 @@ Syntax: capture_pattern: !'_' NAME A single underscore ``_`` is not a capture pattern (this is what ``!'_'`` -expresses). It is instead treated as a :token:`wildcard_pattern`. +expresses). It is instead treated as a +:token:`~python-grammar:wildcard_pattern`. In a given pattern, a given name can only be bound once. E.g. ``case x, x: ...`` is invalid while ``case [x] | x: ...`` is allowed. @@ -1017,7 +1018,7 @@ A class pattern represents a class and its positional and keyword arguments The same keyword should not be repeated in class patterns. -The following is the logical flow for matching a mapping pattern against a +The following is the logical flow for matching a class pattern against a subject value: #. If ``name_or_attr`` is not an instance of the builtin :class:`type` , raise @@ -1055,7 +1056,7 @@ subject value: patterns using the :data:`~object.__match_args__` attribute on the class ``name_or_attr`` before matching: - I. The equivalent of ``getattr(cls, "__match_args__", ()))`` is called. + I. The equivalent of ``getattr(cls, "__match_args__", ())`` is called. * If this raises an exception, the exception bubbles up. @@ -1182,9 +1183,9 @@ is roughly equivalent to :: except that the original function is not temporarily bound to the name ``func``. .. versionchanged:: 3.9 - Functions may be decorated with any valid :token:`assignment_expression`. - Previously, the grammar was much more restrictive; see :pep:`614` for - details. + Functions may be decorated with any valid + :token:`~python-grammar:assignment_expression`. Previously, the grammar was + much more restrictive; see :pep:`614` for details. .. index:: triple: default; parameter; value @@ -1360,9 +1361,9 @@ The evaluation rules for the decorator expressions are the same as for function decorators. The result is then bound to the class name. .. versionchanged:: 3.9 - Classes may be decorated with any valid :token:`assignment_expression`. - Previously, the grammar was much more restrictive; see :pep:`614` for - details. + Classes may be decorated with any valid + :token:`~python-grammar:assignment_expression`. Previously, the grammar was + much more restrictive; see :pep:`614` for details. **Programmer's note:** Variables defined in the class definition are class attributes; they are shared by instances. Instance attributes can be set in a diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 2c76e5656b0..aa0d91ad1c9 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -188,7 +188,7 @@ Ellipsis representation in computers. The string representations of the numeric classes, computed by - :meth:`__repr__` and :meth:`__str__`, have the following + :meth:`~object.__repr__` and :meth:`~object.__str__`, have the following properties: * They are valid numeric literals which, when passed to their @@ -648,13 +648,13 @@ Callable types A function or method which uses the :keyword:`yield` statement (see section :ref:`yield`) is called a :dfn:`generator function`. Such a function, when - called, always returns an iterator object which can be used to execute the - body of the function: calling the iterator's :meth:`iterator.__next__` - method will cause the function to execute until it provides a value - using the :keyword:`!yield` statement. When the function executes a - :keyword:`return` statement or falls off the end, a :exc:`StopIteration` - exception is raised and the iterator will have reached the end of the set of - values to be returned. + called, always returns an :term:`iterator` object which can be used to + execute the body of the function: calling the iterator's + :meth:`iterator.__next__` method will cause the function to execute until + it provides a value using the :keyword:`!yield` statement. When the + function executes a :keyword:`return` statement or falls off the end, a + :exc:`StopIteration` exception is raised and the iterator will have + reached the end of the set of values to be returned. Coroutine functions .. index:: @@ -674,10 +674,11 @@ Callable types A function or method which is defined using :keyword:`async def` and which uses the :keyword:`yield` statement is called a :dfn:`asynchronous generator function`. Such a function, when called, - returns an asynchronous iterator object which can be used in an + returns an :term:`asynchronous iterator` object which can be used in an :keyword:`async for` statement to execute the body of the function. - Calling the asynchronous iterator's :meth:`aiterator.__anext__` method + Calling the asynchronous iterator's + :meth:`aiterator.__anext__ ` method will return an :term:`awaitable` which when awaited will execute until it provides a value using the :keyword:`yield` expression. When the function executes an empty :keyword:`return` @@ -715,13 +716,13 @@ Callable types Classes Classes are callable. These objects normally act as factories for new instances of themselves, but variations are possible for class types that - override :meth:`__new__`. The arguments of the call are passed to - :meth:`__new__` and, in the typical case, to :meth:`__init__` to + override :meth:`~object.__new__`. The arguments of the call are passed to + :meth:`__new__` and, in the typical case, to :meth:`~object.__init__` to initialize the new instance. Class Instances Instances of arbitrary classes can be made callable by defining a - :meth:`__call__` method in their class. + :meth:`~object.__call__` method in their class. Modules @@ -880,14 +881,14 @@ Class instances section :ref:`descriptors` for another way in which attributes of a class retrieved via its instances may differ from the objects actually stored in the class's :attr:`~object.__dict__`. If no class attribute is found, and the - object's class has a :meth:`__getattr__` method, that is called to satisfy + object's class has a :meth:`~object.__getattr__` method, that is called to satisfy the lookup. .. index:: triple: class instance; attribute; assignment Attribute assignments and deletions update the instance's dictionary, never a - class's dictionary. If the class has a :meth:`__setattr__` or - :meth:`__delattr__` method, this is called instead of updating the instance + class's dictionary. If the class has a :meth:`~object.__setattr__` or + :meth:`~object.__delattr__` method, this is called instead of updating the instance dictionary directly. .. index:: @@ -1176,7 +1177,8 @@ Internal types Slice objects .. index:: builtin: slice - Slice objects are used to represent slices for :meth:`__getitem__` + Slice objects are used to represent slices for + :meth:`~object.__getitem__` methods. They are also created by the built-in :func:`slice` function. .. index:: @@ -1229,7 +1231,8 @@ A class can implement certain operations that are invoked by special syntax (such as arithmetic operations or subscripting and slicing) by defining methods with special names. This is Python's approach to :dfn:`operator overloading`, allowing classes to define their own behavior with respect to language -operators. For instance, if a class defines a method named :meth:`__getitem__`, +operators. For instance, if a class defines a method named +:meth:`~object.__getitem__`, and ``x`` is an instance of this class, then ``x[i]`` is roughly equivalent to ``type(x).__getitem__(x, i)``. Except where mentioned, attempts to execute an operation raise an exception when no appropriate method is defined (typically @@ -1237,9 +1240,9 @@ operation raise an exception when no appropriate method is defined (typically Setting a special method to ``None`` indicates that the corresponding operation is not available. For example, if a class sets -:meth:`__iter__` to ``None``, the class is not iterable, so calling +:meth:`~object.__iter__` to ``None``, the class is not iterable, so calling :func:`iter` on its instances will raise a :exc:`TypeError` (without -falling back to :meth:`__getitem__`). [#]_ +falling back to :meth:`~object.__getitem__`). [#]_ When implementing a class that emulates any built-in type, it is important that the emulation only be implemented to the degree that it makes sense for the @@ -1271,7 +1274,7 @@ Basic customization as necessary before returning it. If :meth:`__new__` is invoked during object construction and it returns an - instance or subclass of *cls*, then the new instance’s :meth:`__init__` method + instance of *cls*, then the new instance’s :meth:`__init__` method will be invoked like ``__init__(self[, ...])``, where *self* is the new instance and the remaining arguments are the same as were passed to the object constructor. @@ -1558,7 +1561,7 @@ Basic customization This is intended to provide protection against a denial-of-service caused by carefully-chosen inputs that exploit the worst case performance of a - dict insertion, O(n^2) complexity. See + dict insertion, O(n\ :sup:`2`) complexity. See http://www.ocert.org/advisories/ocert-2011-003.html for details. Changing hash values affects the iteration order of sets. @@ -1774,28 +1777,6 @@ class' :attr:`~object.__dict__`. Called to delete the attribute on an instance *instance* of the owner class. -.. method:: object.__set_name__(self, owner, name) - - Called at the time the owning class *owner* is created. The - descriptor has been assigned to *name*. - - .. note:: - - :meth:`__set_name__` is only called implicitly as part of the - :class:`type` constructor, so it will need to be called explicitly with - the appropriate parameters when a descriptor is added to a class after - initial creation:: - - class A: - pass - descr = custom_descriptor() - A.attr = descr - descr.__set_name__(A, 'attr') - - See :ref:`class-object-creation` for more details. - - .. versionadded:: 3.6 - The attribute :attr:`__objclass__` is interpreted by the :mod:`inspect` module as specifying the class where this object was defined (setting this appropriately can assist in runtime introspection of dynamic class attributes). @@ -1811,7 +1792,8 @@ Invoking Descriptors In general, a descriptor is an object attribute with "binding behavior", one whose attribute access has been overridden by methods in the descriptor -protocol: :meth:`__get__`, :meth:`__set__`, and :meth:`__delete__`. If any of +protocol: :meth:`~object.__get__`, :meth:`~object.__set__`, and +:meth:`~object.__delete__`. If any of those methods are defined for an object, it is said to be a descriptor. The default behavior for attribute access is to get, set, or delete the @@ -1840,14 +1822,47 @@ Class Binding ``A.__dict__['x'].__get__(None, A)``. Super Binding - If ``a`` is an instance of :class:`super`, then the binding ``super(B, obj).m()`` - searches ``obj.__class__.__mro__`` for the base class ``A`` - immediately preceding ``B`` and then invokes the descriptor with the call: - ``A.__dict__['m'].__get__(obj, obj.__class__)``. + A dotted lookup such as ``super(A, a).x`` searches + ``a.__class__.__mro__`` for a base class ``B`` following ``A`` and then + returns ``B.__dict__['x'].__get__(a, A)``. If not a descriptor, ``x`` is + returned unchanged. + +.. testcode:: + :hide: + + class Desc: + def __get__(*args): + return args + + class B: + + x = Desc() + + class A(B): + + x = 999 + + def m(self): + 'Demonstrate these two descriptor invocations are equivalent' + result1 = super(A, self).x + result2 = B.__dict__['x'].__get__(self, A) + return result1 == result2 + +.. doctest:: + :hide: + + >>> a = A() + >>> a.__class__.__mro__.index(B) > a.__class__.__mro__.index(A) + True + >>> super(A, a).x == B.__dict__['x'].__get__(a, A) + True + >>> a.m() + True For instance bindings, the precedence of descriptor invocation depends on which descriptor methods are defined. A descriptor can define any combination -of :meth:`__get__`, :meth:`__set__` and :meth:`__delete__`. If it does not +of :meth:`~object.__get__`, :meth:`~object.__set__` and +:meth:`~object.__delete__`. If it does not define :meth:`__get__`, then accessing the attribute will return the descriptor object itself unless there is a value in the object's instance dictionary. If the descriptor defines :meth:`__set__` and/or :meth:`__delete__`, it is a data @@ -1858,7 +1873,8 @@ descriptors have just the :meth:`__get__` method. Data descriptors with instance dictionary. In contrast, non-data descriptors can be overridden by instances. -Python methods (including :func:`staticmethod` and :func:`classmethod`) are +Python methods (including those decorated with +:func:`@staticmethod ` and :func:`@classmethod `) are implemented as non-data descriptors. Accordingly, instances can redefine and override methods. This allows individual instances to acquire behaviors that differ from other instances of the same class. @@ -1873,46 +1889,50 @@ __slots__ ^^^^^^^^^ *__slots__* allow us to explicitly declare data members (like -properties) and deny the creation of *__dict__* and *__weakref__* +properties) and deny the creation of :attr:`~object.__dict__` and *__weakref__* (unless explicitly declared in *__slots__* or available in a parent.) -The space saved over using *__dict__* can be significant. +The space saved over using :attr:`~object.__dict__` can be significant. Attribute lookup speed can be significantly improved as well. .. data:: object.__slots__ This class variable can be assigned a string, iterable, or sequence of strings with variable names used by instances. *__slots__* reserves space - for the declared variables and prevents the automatic creation of *__dict__* + for the declared variables and prevents the automatic creation of + :attr:`~object.__dict__` and *__weakref__* for each instance. Notes on using *__slots__* """""""""""""""""""""""""" -* When inheriting from a class without *__slots__*, the *__dict__* and +* When inheriting from a class without *__slots__*, the + :attr:`~object.__dict__` and *__weakref__* attribute of the instances will always be accessible. -* Without a *__dict__* variable, instances cannot be assigned new variables not +* Without a :attr:`~object.__dict__` variable, instances cannot be assigned new + variables not listed in the *__slots__* definition. Attempts to assign to an unlisted variable name raises :exc:`AttributeError`. If dynamic assignment of new variables is desired, then add ``'__dict__'`` to the sequence of strings in the *__slots__* declaration. * Without a *__weakref__* variable for each instance, classes defining - *__slots__* do not support weak references to its instances. If weak reference + *__slots__* do not support :mod:`weak references ` to its instances. + If weak reference support is needed, then add ``'__weakref__'`` to the sequence of strings in the *__slots__* declaration. -* *__slots__* are implemented at the class level by creating descriptors - (:ref:`descriptors`) for each variable name. As a result, class attributes +* *__slots__* are implemented at the class level by creating :ref:`descriptors ` + for each variable name. As a result, class attributes cannot be used to set default values for instance variables defined by *__slots__*; otherwise, the class attribute would overwrite the descriptor assignment. * The action of a *__slots__* declaration is not limited to the class where it is defined. *__slots__* declared in parents are available in - child classes. However, child subclasses will get a *__dict__* and + child classes. However, child subclasses will get a :attr:`~object.__dict__` and *__weakref__* unless they also define *__slots__* (which should only contain names of any *additional* slots). @@ -1928,14 +1948,17 @@ Notes on using *__slots__* used; however, in the future, special meaning may be assigned to the values corresponding to each key. -* *__class__* assignment works only if both classes have the same *__slots__*. +* :attr:`~instance.__class__` assignment works only if both classes have the + same *__slots__*. -* Multiple inheritance with multiple slotted parent classes can be used, +* :ref:`Multiple inheritance ` with multiple slotted parent + classes can be used, but only one parent is allowed to have attributes created by slots (the other bases must have empty slot layouts) - violations raise :exc:`TypeError`. -* If an iterator is used for *__slots__* then a descriptor is created for each +* If an :term:`iterator` is used for *__slots__* then a :term:`descriptor` is + created for each of the iterator's values. However, the *__slots__* attribute will be an empty iterator. @@ -1944,7 +1967,7 @@ Notes on using *__slots__* Customizing class creation -------------------------- -Whenever a class inherits from another class, *__init_subclass__* is +Whenever a class inherits from another class, :meth:`~object.__init_subclass__` is called on that class. This way, it is possible to write classes which change the behavior of subclasses. This is closely related to class decorators, but where class decorators only affect the specific class they're @@ -1984,6 +2007,33 @@ class defining the method. .. versionadded:: 3.6 +When a class is created, :meth:`type.__new__` scans the class variables +and makes callbacks to those with a :meth:`~object.__set_name__` hook. + +.. method:: object.__set_name__(self, owner, name) + + Automatically called at the time the owning class *owner* is + created. The object has been assigned to *name* in that class:: + + class A: + x = C() # Automatically calls: x.__set_name__(A, 'x') + + If the class variable is assigned after the class is created, + :meth:`__set_name__` will not be called automatically. + If needed, :meth:`__set_name__` can be called directly:: + + class A: + pass + + c = C() + A.x = c # The hook is not called + c.__set_name__(A, 'x') # Manually invoke the hook + + See :ref:`class-object-creation` for more details. + + .. versionadded:: 3.6 + + .. _metaclasses: Metaclasses @@ -2070,7 +2120,8 @@ Once the appropriate metaclass has been identified, then the class namespace is prepared. If the metaclass has a ``__prepare__`` attribute, it is called as ``namespace = metaclass.__prepare__(name, bases, **kwds)`` (where the additional keyword arguments, if any, come from the class definition). The -``__prepare__`` method should be implemented as a :func:`classmethod`. The +``__prepare__`` method should be implemented as a +:func:`classmethod `. The namespace returned by ``__prepare__`` is passed in to ``__new__``, but when the final class object is created the namespace is copied into a new ``dict``. @@ -2133,15 +2184,15 @@ current call is identified based on the first argument passed to the method. Failing to do so will result in a :exc:`RuntimeError` in Python 3.8. When using the default metaclass :class:`type`, or any metaclass that ultimately -calls ``type.__new__``, the following additional customisation steps are +calls ``type.__new__``, the following additional customization steps are invoked after creating the class object: -* first, ``type.__new__`` collects all of the descriptors in the class - namespace that define a :meth:`~object.__set_name__` method; -* second, all of these ``__set_name__`` methods are called with the class - being defined and the assigned name of that particular descriptor; -* finally, the :meth:`~object.__init_subclass__` hook is called on the - immediate parent of the new class in its method resolution order. +1) The ``type.__new__`` method collects all of the attributes in the class + namespace that define a :meth:`~object.__set_name__` method; +2) Those ``__set_name__`` methods are called with the class + being defined and the assigned name of that particular attribute; +3) The :meth:`~object.__init_subclass__` hook is called on the + immediate parent of the new class in its method resolution order. After the class object is created, it is passed to the class decorators included in the class definition (if any) and the resulting object is bound @@ -2210,22 +2261,142 @@ case the instance is itself a class. Emulating generic types ----------------------- -One can implement the generic class syntax as specified by :pep:`484` -(for example ``List[int]``) by defining a special method: +When using :term:`type annotations`, it is often useful to +*parameterize* a :term:`generic type` using Python's square-brackets notation. +For example, the annotation ``list[int]`` might be used to signify a +:class:`list` in which all the elements are of type :class:`int`. + +.. seealso:: + + :pep:`484` - Type Hints + Introducing Python's framework for type annotations + + :ref:`Generic Alias Types` + Documentation for objects representing parameterized generic classes + + :ref:`Generics`, :ref:`user-defined generics` and :class:`typing.Generic` + Documentation on how to implement generic classes that can be + parameterized at runtime and understood by static type-checkers. + +A class can *generally* only be parameterized if it defines the special +class method ``__class_getitem__()``. .. classmethod:: object.__class_getitem__(cls, key) Return an object representing the specialization of a generic class by type arguments found in *key*. -This method is looked up on the class object itself, and when defined in -the class body, this method is implicitly a class method. Note, this -mechanism is primarily reserved for use with static type hints, other usage -is discouraged. + When defined on a class, ``__class_getitem__()`` is automatically a class + method. As such, there is no need for it to be decorated with + :func:`@classmethod` when it is defined. + + +The purpose of *__class_getitem__* +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The purpose of :meth:`~object.__class_getitem__` is to allow runtime +parameterization of standard-library generic classes in order to more easily +apply :term:`type hints` to these classes. + +To implement custom generic classes that can be parameterized at runtime and +understood by static type-checkers, users should either inherit from a standard +library class that already implements :meth:`~object.__class_getitem__`, or +inherit from :class:`typing.Generic`, which has its own implementation of +``__class_getitem__()``. + +Custom implementations of :meth:`~object.__class_getitem__` on classes defined +outside of the standard library may not be understood by third-party +type-checkers such as mypy. Using ``__class_getitem__()`` on any class for +purposes other than type hinting is discouraged. + + +.. _classgetitem-versus-getitem: + + +*__class_getitem__* versus *__getitem__* +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Usually, the :ref:`subscription` of an object using square +brackets will call the :meth:`~object.__getitem__` instance method defined on +the object's class. However, if the object being subscribed is itself a class, +the class method :meth:`~object.__class_getitem__` may be called instead. +``__class_getitem__()`` should return a :ref:`GenericAlias` +object if it is properly defined. + +Presented with the :term:`expression` ``obj[x]``, the Python interpreter +follows something like the following process to decide whether +:meth:`~object.__getitem__` or :meth:`~object.__class_getitem__` should be +called:: + + from inspect import isclass + + def subscribe(obj, x): + """Return the result of the expression 'obj[x]'""" + + class_of_obj = type(obj) + + # If the class of obj defines __getitem__, + # call class_of_obj.__getitem__(obj, x) + if hasattr(class_of_obj, '__getitem__'): + return class_of_obj.__getitem__(obj, x) + + # Else, if obj is a class and defines __class_getitem__, + # call obj.__class_getitem__(x) + elif isclass(obj) and hasattr(obj, '__class_getitem__'): + return obj.__class_getitem__(x) + + # Else, raise an exception + else: + raise TypeError( + f"'{class_of_obj.__name__}' object is not subscriptable" + ) + +In Python, all classes are themselves instances of other classes. The class of +a class is known as that class's :term:`metaclass`, and most classes have the +:class:`type` class as their metaclass. :class:`type` does not define +:meth:`~object.__getitem__`, meaning that expressions such as ``list[int]``, +``dict[str, float]`` and ``tuple[str, bytes]`` all result in +:meth:`~object.__class_getitem__` being called:: + + >>> # list has class "type" as its metaclass, like most classes: + >>> type(list) + + >>> type(dict) == type(list) == type(tuple) == type(str) == type(bytes) + True + >>> # "list[int]" calls "list.__class_getitem__(int)" + >>> list[int] + list[int] + >>> # list.__class_getitem__ returns a GenericAlias object: + >>> type(list[int]) + + +However, if a class has a custom metaclass that defines +:meth:`~object.__getitem__`, subscribing the class may result in different +behaviour. An example of this can be found in the :mod:`enum` module:: + + >>> from enum import Enum + >>> class Menu(Enum): + ... """A breakfast menu""" + ... SPAM = 'spam' + ... BACON = 'bacon' + ... + >>> # Enum classes have a custom metaclass: + >>> type(Menu) + + >>> # EnumMeta defines __getitem__, + >>> # so __class_getitem__ is not called, + >>> # and the result is not a GenericAlias object: + >>> Menu['SPAM'] + + >>> type(Menu['SPAM']) + + .. seealso:: - - :pep:`560` - Core support for typing module and generic types + :pep:`560` - Core Support for typing module and generic types + Introducing :meth:`~object.__class_getitem__`, and outlining when a + :ref:`subscription` results in ``__class_getitem__()`` + being called instead of :meth:`~object.__getitem__` .. _callable-types: @@ -2248,31 +2419,36 @@ Emulating container types ------------------------- The following methods can be defined to implement container objects. Containers -usually are sequences (such as lists or tuples) or mappings (like dictionaries), +usually are :term:`sequences ` (such as :class:`lists ` or +:class:`tuples `) or :term:`mappings ` (like +:class:`dictionaries `), but can represent other containers as well. The first set of methods is used either to emulate a sequence or to emulate a mapping; the difference is that for a sequence, the allowable keys should be the integers *k* for which ``0 <= k < -N`` where *N* is the length of the sequence, or slice objects, which define a +N`` where *N* is the length of the sequence, or :class:`slice` objects, which define a range of items. It is also recommended that mappings provide the methods :meth:`keys`, :meth:`values`, :meth:`items`, :meth:`get`, :meth:`clear`, :meth:`setdefault`, :meth:`pop`, :meth:`popitem`, :meth:`!copy`, and -:meth:`update` behaving similar to those for Python's standard dictionary +:meth:`update` behaving similar to those for Python's standard :class:`dictionary ` objects. The :mod:`collections.abc` module provides a :class:`~collections.abc.MutableMapping` -abstract base class to help create those methods from a base set of -:meth:`__getitem__`, :meth:`__setitem__`, :meth:`__delitem__`, and :meth:`keys`. +:term:`abstract base class` to help create those methods from a base set of +:meth:`~object.__getitem__`, :meth:`~object.__setitem__`, :meth:`~object.__delitem__`, and :meth:`keys`. Mutable sequences should provide methods :meth:`append`, :meth:`count`, :meth:`index`, :meth:`extend`, :meth:`insert`, :meth:`pop`, :meth:`remove`, -:meth:`reverse` and :meth:`sort`, like Python standard list objects. Finally, +:meth:`reverse` and :meth:`sort`, like Python standard :class:`list` +objects. Finally, sequence types should implement addition (meaning concatenation) and -multiplication (meaning repetition) by defining the methods :meth:`__add__`, -:meth:`__radd__`, :meth:`__iadd__`, :meth:`__mul__`, :meth:`__rmul__` and -:meth:`__imul__` described below; they should not define other numerical +multiplication (meaning repetition) by defining the methods +:meth:`~object.__add__`, :meth:`~object.__radd__`, :meth:`~object.__iadd__`, +:meth:`~object.__mul__`, :meth:`~object.__rmul__` and :meth:`~object.__imul__` +described below; they should not define other numerical operators. It is recommended that both mappings and sequences implement the -:meth:`__contains__` method to allow efficient use of the ``in`` operator; for +:meth:`~object.__contains__` method to allow efficient use of the ``in`` +operator; for mappings, ``in`` should search the mapping's keys; for sequences, it should search through the values. It is further recommended that both mappings and -sequences implement the :meth:`__iter__` method to allow efficient iteration +sequences implement the :meth:`~object.__iter__` method to allow efficient iteration through the container; for mappings, :meth:`__iter__` should iterate through the object's keys; for sequences, it should iterate through the values. @@ -2325,19 +2501,27 @@ through the object's keys; for sequences, it should iterate through the values. .. method:: object.__getitem__(self, key) - Called to implement evaluation of ``self[key]``. For sequence types, the - accepted keys should be integers and slice objects. Note that the special - interpretation of negative indexes (if the class wishes to emulate a sequence - type) is up to the :meth:`__getitem__` method. If *key* is of an inappropriate - type, :exc:`TypeError` may be raised; if of a value outside the set of indexes - for the sequence (after any special interpretation of negative values), - :exc:`IndexError` should be raised. For mapping types, if *key* is missing (not - in the container), :exc:`KeyError` should be raised. + Called to implement evaluation of ``self[key]``. For :term:`sequence` types, + the accepted keys should be integers and slice objects. Note that the + special interpretation of negative indexes (if the class wishes to emulate a + :term:`sequence` type) is up to the :meth:`__getitem__` method. If *key* is + of an inappropriate type, :exc:`TypeError` may be raised; if of a value + outside the set of indexes for the sequence (after any special + interpretation of negative values), :exc:`IndexError` should be raised. For + :term:`mapping` types, if *key* is missing (not in the container), + :exc:`KeyError` should be raised. .. note:: - :keyword:`for` loops expect that an :exc:`IndexError` will be raised for illegal - indexes to allow proper detection of the end of the sequence. + :keyword:`for` loops expect that an :exc:`IndexError` will be raised for + illegal indexes to allow proper detection of the end of the sequence. + + .. note:: + + When :ref:`subscripting` a *class*, the special + class method :meth:`~object.__class_getitem__` may be called instead of + ``__getitem__()``. See :ref:`classgetitem-versus-getitem` for more + details. .. method:: object.__setitem__(self, key, value) @@ -2366,12 +2550,10 @@ through the object's keys; for sequences, it should iterate through the values. .. method:: object.__iter__(self) - This method is called when an iterator is required for a container. This method - should return a new iterator object that can iterate over all the objects in the - container. For mappings, it should iterate over the keys of the container. - - Iterator objects also need to implement this method; they are required to return - themselves. For more information on iterator objects, see :ref:`typeiter`. + This method is called when an :term:`iterator` is required for a container. + This method should return a new iterator object that can iterate over all the + objects in the container. For mappings, it should iterate over the keys of + the container. .. method:: object.__reversed__(self) @@ -2572,8 +2754,8 @@ left undefined. return the value of the object truncated to an :class:`~numbers.Integral` (typically an :class:`int`). - If :meth:`__int__` is not defined then the built-in function :func:`int` - falls back to :meth:`__trunc__`. + The built-in function :func:`int` falls back to :meth:`__trunc__` if neither + :meth:`__int__` nor :meth:`__index__` is defined. .. _context-managers: @@ -2679,7 +2861,8 @@ exception:: TypeError: object of type 'C' has no len() The rationale behind this behaviour lies with a number of special methods such -as :meth:`__hash__` and :meth:`__repr__` that are implemented by all objects, +as :meth:`~object.__hash__` and :meth:`~object.__repr__` that are implemented +by all objects, including type objects. If the implicit lookup of these methods used the conventional lookup process, they would fail when invoked on the type object itself:: @@ -2702,7 +2885,7 @@ the instance when looking up special methods:: In addition to bypassing any instance attributes in the interest of correctness, implicit special method lookup generally also bypasses the -:meth:`__getattribute__` method even of the object's metaclass:: +:meth:`~object.__getattribute__` method even of the object's metaclass:: >>> class Meta(type): ... def __getattribute__(*args): @@ -2726,7 +2909,7 @@ correctness, implicit special method lookup generally also bypasses the >>> len(c) # Implicit lookup 10 -Bypassing the :meth:`__getattribute__` machinery in this fashion +Bypassing the :meth:`~object.__getattribute__` machinery in this fashion provides significant scope for speed optimisations within the interpreter, at the cost of some flexibility in the handling of special methods (the special method *must* be set on the class @@ -2743,7 +2926,7 @@ Coroutines Awaitable Objects ----------------- -An :term:`awaitable` object generally implements an :meth:`__await__` method. +An :term:`awaitable` object generally implements an :meth:`~object.__await__` method. :term:`Coroutine objects ` returned from :keyword:`async def` functions are awaitable. @@ -2751,7 +2934,7 @@ are awaitable. The :term:`generator iterator` objects returned from generators decorated with :func:`types.coroutine` - are also awaitable, but they do not implement :meth:`__await__`. + are also awaitable, but they do not implement :meth:`~object.__await__`. .. method:: object.__await__(self) @@ -2770,7 +2953,7 @@ Coroutine Objects ----------------- :term:`Coroutine objects ` are :term:`awaitable` objects. -A coroutine's execution can be controlled by calling :meth:`__await__` and +A coroutine's execution can be controlled by calling :meth:`~object.__await__` and iterating over the result. When the coroutine has finished executing and returns, the iterator raises :exc:`StopIteration`, and the exception's :attr:`~StopIteration.value` attribute holds the return value. If the @@ -2789,7 +2972,7 @@ generators, coroutines do not directly support iteration. Starts or resumes execution of the coroutine. If *value* is ``None``, this is equivalent to advancing the iterator returned by - :meth:`__await__`. If *value* is not ``None``, this method delegates + :meth:`~object.__await__`. If *value* is not ``None``, this method delegates to the :meth:`~generator.send` method of the iterator that caused the coroutine to suspend. The result (return value, :exc:`StopIteration`, or other exception) is the same as when @@ -2802,7 +2985,7 @@ generators, coroutines do not directly support iteration. the coroutine to suspend, if it has such a method. Otherwise, the exception is raised at the suspension point. The result (return value, :exc:`StopIteration`, or other exception) is the same as - when iterating over the :meth:`__await__` return value, described + when iterating over the :meth:`~object.__await__` return value, described above. If the exception is not caught in the coroutine, it propagates back to the caller. @@ -2856,11 +3039,11 @@ An example of an asynchronous iterable object:: .. versionadded:: 3.5 .. versionchanged:: 3.7 - Prior to Python 3.7, ``__aiter__`` could return an *awaitable* + Prior to Python 3.7, :meth:`~object.__aiter__` could return an *awaitable* that would resolve to an :term:`asynchronous iterator `. - Starting with Python 3.7, ``__aiter__`` must return an + Starting with Python 3.7, :meth:`~object.__aiter__` must return an asynchronous iterator object. Returning anything else will result in a :exc:`TypeError` error. @@ -2903,8 +3086,9 @@ An example of an asynchronous context manager class:: controlled conditions. It generally isn't a good idea though, since it can lead to some very strange behaviour if it is handled incorrectly. -.. [#] The :meth:`__hash__`, :meth:`__iter__`, :meth:`__reversed__`, and - :meth:`__contains__` methods have special handling for this; others +.. [#] The :meth:`~object.__hash__`, :meth:`~object.__iter__`, + :meth:`~object.__reversed__`, and :meth:`~object.__contains__` methods have + special handling for this; others will still raise a :exc:`TypeError`, but may do so by relying on the behavior that ``None`` is not callable. @@ -2915,5 +3099,6 @@ An example of an asynchronous context manager class:: *blocking* such fallback. .. [#] For operands of the same type, it is assumed that if the non-reflected - method -- such as :meth:`__add__` -- fails then the overall operation is not + method -- such as :meth:`~object.__add__` -- fails then the overall + operation is not supported, which is why the reflected method is not called. diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index 55ac01b6a84..e652915bd27 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -56,15 +56,25 @@ Binding of names .. index:: single: from; import statement -The following constructs bind names: formal parameters to functions, -:keyword:`import` statements, class and function definitions (these bind the -class or function name in the defining block), and targets that are identifiers -if occurring in an assignment, :keyword:`for` loop header, or after -:keyword:`!as` in a :keyword:`with` statement or :keyword:`except` clause. -The :keyword:`!import` statement -of the form ``from ... import *`` binds all names defined in the imported -module, except those beginning with an underscore. This form may only be used -at the module level. +The following constructs bind names: + +* formal parameters to functions, +* class definitions, +* function definitions, +* assignment expressions, +* :ref:`targets ` that are identifiers if occurring in + an assignment: + + + :keyword:`for` loop header, + + after :keyword:`!as` in a :keyword:`with` statement, :keyword:`except` + clause or in the as-pattern in structural pattern matching, + + in a capture pattern in structural pattern matching + +* :keyword:`import` statements. + +The :keyword:`!import` statement of the form ``from ... import *`` binds all +names defined in the imported module, except those beginning with an underscore. +This form may only be used at the module level. A target occurring in a :keyword:`del` statement is also considered bound for this purpose (though the actual semantics are to unbind the name). @@ -119,14 +129,14 @@ is subtle. Python lacks declarations and allows name binding operations to occur anywhere within a code block. The local variables of a code block can be determined by scanning the entire text of the block for name binding operations. -If the :keyword:`global` statement occurs within a block, all uses of the name -specified in the statement refer to the binding of that name in the top-level +If the :keyword:`global` statement occurs within a block, all uses of the names +specified in the statement refer to the bindings of those names in the top-level namespace. Names are resolved in the top-level namespace by searching the global namespace, i.e. the namespace of the module containing the code block, and the builtins namespace, the namespace of the module :mod:`builtins`. The -global namespace is searched first. If the name is not found there, the +global namespace is searched first. If the names are not found there, the builtins namespace is searched. The :keyword:`!global` statement must precede -all uses of the name. +all uses of the listed names. The :keyword:`global` statement has the same scope as a name binding operation in the same block. If the nearest enclosing scope for a free variable contains diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index b1b32dfba26..991f2d717a0 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -450,21 +450,20 @@ functions are described separately in section :ref:`asynchronous-generator-functions`. When a generator function is called, it returns an iterator known as a -generator. That generator then controls the execution of the generator function. -The execution starts when one of the generator's methods is called. At that -time, the execution proceeds to the first yield expression, where it is -suspended again, returning the value of :token:`expression_list` to the generator's -caller. By suspended, we mean that all local state is retained, including the -current bindings of local variables, the instruction pointer, the internal -evaluation stack, and the state of any exception handling. When the execution -is resumed by calling one of the -generator's methods, the function can proceed exactly as if the yield expression -were just another external call. The value of the yield expression after -resuming depends on the method which resumed the execution. If -:meth:`~generator.__next__` is used (typically via either a :keyword:`for` or -the :func:`next` builtin) then the result is :const:`None`. Otherwise, if -:meth:`~generator.send` is used, then the result will be the value passed in to -that method. +generator. That generator then controls the execution of the generator +function. The execution starts when one of the generator's methods is called. +At that time, the execution proceeds to the first yield expression, where it is +suspended again, returning the value of :token:`~python-grammar:expression_list` +to the generator's caller. By suspended, we mean that all local state is +retained, including the current bindings of local variables, the instruction +pointer, the internal evaluation stack, and the state of any exception handling. +When the execution is resumed by calling one of the generator's methods, the +function can proceed exactly as if the yield expression were just another +external call. The value of the yield expression after resuming depends on the +method which resumed the execution. If :meth:`~generator.__next__` is used +(typically via either a :keyword:`for` or the :func:`next` builtin) then the +result is :const:`None`. Otherwise, if :meth:`~generator.send` is used, then +the result will be the value passed in to that method. .. index:: single: coroutine @@ -514,8 +513,8 @@ on the right hand side of an assignment statement. usable as simple coroutines. :pep:`380` - Syntax for Delegating to a Subgenerator - The proposal to introduce the :token:`yield_from` syntax, making delegation - to subgenerators easy. + The proposal to introduce the :token:`~python-grammar:yield_from` syntax, + making delegation to subgenerators easy. :pep:`525` - Asynchronous Generators The proposal that expanded on :pep:`492` by adding generator capabilities to @@ -543,9 +542,9 @@ is already executing raises a :exc:`ValueError` exception. :meth:`~generator.__next__` method, the current yield expression always evaluates to :const:`None`. The execution then continues to the next yield expression, where the generator is suspended again, and the value of the - :token:`expression_list` is returned to :meth:`__next__`'s caller. If the - generator exits without yielding another value, a :exc:`StopIteration` - exception is raised. + :token:`~python-grammar:expression_list` is returned to :meth:`__next__`'s + caller. If the generator exits without yielding another value, a + :exc:`StopIteration` exception is raised. This method is normally called implicitly, e.g. by a :keyword:`for` loop, or by the built-in :func:`next` function. @@ -634,21 +633,20 @@ An asynchronous generator object is typically used in an :keyword:`async for` statement in a coroutine function analogously to how a generator object would be used in a :keyword:`for` statement. -Calling one of the asynchronous generator's methods returns an -:term:`awaitable` object, and the execution starts when this object -is awaited on. At that time, the execution proceeds to the first yield -expression, where it is suspended again, returning the value of -:token:`expression_list` to the awaiting coroutine. As with a generator, -suspension means that all local state is retained, including the -current bindings of local variables, the instruction pointer, the internal -evaluation stack, and the state of any exception handling. When the execution -is resumed by awaiting on the next object returned by the asynchronous -generator's methods, the function can proceed exactly as if the yield -expression were just another external call. The value of the yield expression -after resuming depends on the method which resumed the execution. If +Calling one of the asynchronous generator's methods returns an :term:`awaitable` +object, and the execution starts when this object is awaited on. At that time, +the execution proceeds to the first yield expression, where it is suspended +again, returning the value of :token:`~python-grammar:expression_list` to the +awaiting coroutine. As with a generator, suspension means that all local state +is retained, including the current bindings of local variables, the instruction +pointer, the internal evaluation stack, and the state of any exception handling. +When the execution is resumed by awaiting on the next object returned by the +asynchronous generator's methods, the function can proceed exactly as if the +yield expression were just another external call. The value of the yield +expression after resuming depends on the method which resumed the execution. If :meth:`~agen.__anext__` is used then the result is :const:`None`. Otherwise, if -:meth:`~agen.asend` is used, then the result will be the value passed in to -that method. +:meth:`~agen.asend` is used, then the result will be the value passed in to that +method. If an asynchronous generator happens to exit early by :keyword:`break`, the caller task being cancelled, or other exceptions, the generator's async cleanup code @@ -700,10 +698,10 @@ which are used to control the execution of a generator function. Returns an awaitable which when run starts to execute the asynchronous generator or resumes it at the last executed yield expression. When an asynchronous generator function is resumed with an :meth:`~agen.__anext__` - method, the current yield expression always evaluates to :const:`None` in - the returned awaitable, which when run will continue to the next yield - expression. The value of the :token:`expression_list` of the yield - expression is the value of the :exc:`StopIteration` exception raised by + method, the current yield expression always evaluates to :const:`None` in the + returned awaitable, which when run will continue to the next yield + expression. The value of the :token:`~python-grammar:expression_list` of the + yield expression is the value of the :exc:`StopIteration` exception raised by the completing coroutine. If the asynchronous generator exits without yielding another value, the awaitable instead raises a :exc:`StopAsyncIteration` exception, signalling that the asynchronous @@ -965,8 +963,8 @@ formal parameters. If there are N positional arguments, they are placed in the first N slots. Next, for each keyword argument, the identifier is used to determine the corresponding slot (if the identifier is the same as the first formal parameter name, the first slot is used, and so on). If the slot is -already filled, a :exc:`TypeError` exception is raised. Otherwise, the value of -the argument is placed in the slot, filling it (even if the expression is +already filled, a :exc:`TypeError` exception is raised. Otherwise, the +argument is placed in the slot, filling it (even if the expression is ``None``, it fills the slot). When all arguments have been processed, the slots that are still unfilled are filled with the corresponding default value from the function definition. (Default values are calculated, once, when the function is @@ -1026,7 +1024,7 @@ keyword arguments (and any ``**expression`` arguments -- see below). So:: 1 2 It is unusual for both keyword arguments and the ``*expression`` syntax to be -used in the same call, so in practice this confusion does not arise. +used in the same call, so in practice this confusion does not often arise. .. index:: single: **; in function calls @@ -1256,7 +1254,7 @@ integer; the result is that of mathematical division with the 'floor' function applied to the result. Division by zero raises the :exc:`ZeroDivisionError` exception. -This operation can be customized using the special :meth:`__div__` and +This operation can be customized using the special :meth:`__truediv__` and :meth:`__floordiv__` methods. .. index:: @@ -1424,6 +1422,8 @@ Note that ``a op1 b op2 c`` doesn't imply any kind of comparison between *a* and *c*, so that, e.g., ``x < y > z`` is perfectly legal (though perhaps not pretty). +.. _expressions-value-comparisons: + Value comparisons ----------------- @@ -1704,8 +1704,9 @@ Assignment expressions assignment_expression: [`identifier` ":="] `expression` An assignment expression (sometimes also called a "named expression" or -"walrus") assigns an :token:`expression` to an :token:`identifier`, while also -returning the value of the :token:`expression`. +"walrus") assigns an :token:`~python-grammar:expression` to an +:token:`~python-grammar:identifier`, while also returning the value of the +:token:`~python-grammar:expression`. One common use case is when handling matched regular expressions: diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 81a124f745a..69e2a947274 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -605,7 +605,7 @@ the module. ``__file__`` is optional (if set, value must be a string). It indicates the pathname of the file from which the module was loaded (if - loaded from a file), or the pathname of the shared libray file + loaded from a file), or the pathname of the shared library file for extension modules loaded dynamically from a shared library. It might be missing for certain types of modules, such as C modules that are statically linked into the interpreter, and the @@ -975,6 +975,8 @@ should expose ``XXX.YYY.ZZZ`` as a usable expression, but .moduleY is not a valid expression. +.. _import-dunder-main: + Special considerations for __main__ =================================== diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 4ad8f8be1e7..b69d0fbdd41 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -316,7 +316,7 @@ The Unicode category codes mentioned above stand for: * *Nd* - decimal numbers * *Pc* - connector punctuations * *Other_ID_Start* - explicit list of characters in `PropList.txt - `_ to support backwards + `_ to support backwards compatibility * *Other_ID_Continue* - likewise @@ -324,8 +324,8 @@ All identifiers are converted into the normal form NFKC while parsing; compariso of identifiers is based on NFKC. A non-normative HTML file listing all valid identifier characters for Unicode -4.1 can be found at -https://www.unicode.org/Public/13.0.0/ucd/DerivedCoreProperties.txt +14.0.0 can be found at +https://www.unicode.org/Public/14.0.0/ucd/DerivedCoreProperties.txt .. _keywords: @@ -385,10 +385,20 @@ classes are identified by the patterns of leading and trailing underscore characters: ``_*`` - Not imported by ``from module import *``. The special identifier ``_`` is used - in the interactive interpreter to store the result of the last evaluation; it is - stored in the :mod:`builtins` module. When not in interactive mode, ``_`` - has no special meaning and is not defined. See section :ref:`import`. + Not imported by ``from module import *``. + +``_`` + In a ``case`` pattern within a :keyword:`match` statement, ``_`` is a + :ref:`soft keyword ` that denotes a + :ref:`wildcard `. + + Separately, the interactive interpreter makes the result of the last evaluation + available in the variable ``_``. + (It is stored in the :mod:`builtins` module, alongside built-in + functions like ``print``.) + + Elsewhere, ``_`` is a regular identifier. It is often used to name + "special" items, but it is not special to Python itself. .. note:: @@ -396,6 +406,8 @@ characters: refer to the documentation for the :mod:`gettext` module for more information on this convention. + It is also commonly used for unused variables. + ``__*__`` System-defined names, informally known as "dunder" names. These names are defined by the interpreter and its implementation (including the standard library). @@ -457,10 +469,10 @@ String literals are described by the following lexical definitions: bytesescapeseq: "\" One syntactic restriction not indicated by these productions is that whitespace -is not allowed between the :token:`stringprefix` or :token:`bytesprefix` and the -rest of the literal. The source character set is defined by the encoding -declaration; it is UTF-8 if no encoding declaration is given in the source file; -see section :ref:`encodings`. +is not allowed between the :token:`~python-grammar:stringprefix` or +:token:`~python-grammar:bytesprefix` and the rest of the literal. The source +character set is defined by the encoding declaration; it is UTF-8 if no encoding +declaration is given in the source file; see section :ref:`encodings`. .. index:: triple-quoted string, Unicode Consortium, raw string single: """; string literal diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index bb1209dfc33..3d02074960f 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -655,6 +655,12 @@ and information about handling exceptions is in section :ref:`try`. The ``__suppress_context__`` attribute to suppress automatic display of the exception context. +.. versionchanged:: 3.11 + If the traceback of the active exception is modified in an :keyword:`except` + clause, a subsequent ``raise`` statement re-raises the exception with the + modified traceback. Previously, the exception was re-raised with the + traceback it had when it was caught. + .. _break: The :keyword:`!break` statement diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 2b70af3a4fc..785da2c3217 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -3,7 +3,7 @@ # Sphinx version is pinned so that new versions that introduce new warnings # won't suddenly cause build failures. Updating the version is fine as long # as no warnings are raised by doing so. -sphinx==3.2.1 +sphinx==4.2.0 blurb diff --git a/Doc/tools/extensions/suspicious.py b/Doc/tools/extensions/suspicious.py index 9e814fb94d2..c3de4d79c83 100644 --- a/Doc/tools/extensions/suspicious.py +++ b/Doc/tools/extensions/suspicious.py @@ -118,7 +118,7 @@ class CheckSuspiciousMarkupBuilder(Builder): self.logger.warning( 'Found %s/%s unused rules: %s' % ( len(unused_rules), len(self.rules), - ''.join(repr(rule) for rule in unused_rules), + '\n'.join(repr(rule) for rule in unused_rules), ) ) return diff --git a/Doc/tools/rstlint.py b/Doc/tools/rstlint.py index 91aed80b1e9..33cbaadfce9 100755 --- a/Doc/tools/rstlint.py +++ b/Doc/tools/rstlint.py @@ -43,10 +43,10 @@ directives = [ ] roles = [ - ":class:", - ":func:", - ":meth:", - ":mod:", + "(?>> '2' + 2 Traceback (most recent call last): File "", line 1, in - TypeError: Can't convert 'int' object to str implicitly + TypeError: can only concatenate str (not "int") to str The last line of the error message indicates what happened. Exceptions come in different types, and the type is printed as part of the message: the types in diff --git a/Doc/tutorial/floatingpoint.rst b/Doc/tutorial/floatingpoint.rst index b98de6e56a0..7212b40be83 100644 --- a/Doc/tutorial/floatingpoint.rst +++ b/Doc/tutorial/floatingpoint.rst @@ -133,7 +133,7 @@ with inexact values become comparable to one another:: Binary floating-point arithmetic holds many surprises like this. The problem with "0.1" is explained in precise detail below, in the "Representation Error" -section. See `The Perils of Floating Point `_ +section. See `The Perils of Floating Point `_ for a more complete account of other common surprises. As that says near the end, "there are no easy answers." Still, don't be unduly diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 4613cf76c53..33678f5a64b 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -11,6 +11,13 @@ with a prompt are output from the interpreter. Note that a secondary prompt on a line by itself in an example means you must type a blank line; this is used to end a multi-line command. +.. only:: html + + You can toggle the display of prompts and output by clicking on ``>>>`` + in the upper-right corner of an example box. If you hide the prompts + and output for an example, then you can easily copy and paste the input + lines into your interpreter. + .. index:: single: # (hash); comment Many of the examples in this manual, even those entered at the interactive @@ -269,14 +276,6 @@ to obtain individual characters, *slicing* allows you to obtain substring:: >>> word[2:5] # characters from position 2 (included) to 5 (excluded) 'tho' -Note how the start is always included, and the end always excluded. This -makes sure that ``s[:i] + s[i:]`` is always equal to ``s``:: - - >>> word[:2] + word[2:] - 'Python' - >>> word[:4] + word[4:] - 'Python' - Slice indices have useful defaults; an omitted first index defaults to zero, an omitted second index defaults to the size of the string being sliced. :: @@ -287,6 +286,14 @@ omitted second index defaults to the size of the string being sliced. :: >>> word[-2:] # characters from the second-last (included) to the end 'on' +Note how the start is always included, and the end always excluded. This +makes sure that ``s[:i] + s[i:]`` is always equal to ``s``:: + + >>> word[:2] + word[2:] + 'Python' + >>> word[:4] + word[4:] + 'Python' + One way to remember how slices work is to think of the indices as pointing *between* characters, with the left edge of the first character numbered 0. Then the right edge of the last character of a string of *n* characters has diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index af595e5ca04..f1d4957e37e 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -191,7 +191,8 @@ named :file:`spam.py` in a list of directories given by the variable file is specified). * :envvar:`PYTHONPATH` (a list of directory names, with the same syntax as the shell variable :envvar:`PATH`). -* The installation-dependent default. +* The installation-dependent default (by convention including a + ``site-packages`` directory, handled by the :mod:`site` module). .. note:: On file systems which support symlinks, the directory containing the input @@ -533,6 +534,8 @@ importing module needs to use submodules with the same name from different packages. +.. _intra-package-references: + Intra-package References ------------------------ diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index 221c11ca74b..58ad31972f6 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -88,6 +88,11 @@ For example: '~/envs/tutorial-env/lib/python3.5/site-packages'] >>> +To deactivate a virtual environment, type:: + + deactivate + +into the terminal. Managing Packages with pip ========================== diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 9e391604a1a..d341ea8bb43 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -322,7 +322,7 @@ Miscellaneous options Hash randomization is intended to provide protection against a denial-of-service caused by carefully-chosen inputs that exploit the worst - case performance of a dict construction, O(n^2) complexity. See + case performance of a dict construction, O(n\ :sup:`2`) complexity. See http://www.ocert.org/advisories/ocert-2011-003.html for details. :envvar:`PYTHONHASHSEED` allows you to set a fixed value for the hash @@ -477,9 +477,16 @@ Miscellaneous options * ``-X no_debug_ranges`` disables the inclusion of the tables mapping extra location information (end line, start column offset and end column offset) to every instruction in code objects. This is useful when smaller code - objects and pyc files are desired as well as supressing the extra visual + objects and pyc files are desired as well as suppressing the extra visual location indicators when the interpreter displays tracebacks. See also :envvar:`PYTHONNODEBUGRANGES`. + * ``-X frozen_modules`` determines whether or not frozen modules are + ignored by the import machinery. A value of "on" means they get + imported and "off" means they are ignored. The default is "on" + if this is an installed Python (the normal case). If it's under + development (running from the source tree) then the default is "off". + Note that the "importlib_bootstrap" and "importlib_bootstrap_external" + frozen modules are always used, even if this flag is set to "off". It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -518,6 +525,9 @@ Miscellaneous options .. versionadded:: 3.11 The ``-X no_debug_ranges`` option. + .. versionadded:: 3.11 + The ``-X frozen_modules`` option. + Options you shouldn't use ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -647,7 +657,7 @@ conflict. .. envvar:: PYTHONCASEOK If this is set, Python ignores case in :keyword:`import` statements. This - only works on Windows and OS X. + only works on Windows and macOS. .. envvar:: PYTHONDONTWRITEBYTECODE @@ -730,7 +740,7 @@ conflict. If this environment variable is set, ``sys.argv[0]`` will be set to its value instead of the value got through the C runtime. Only works on - Mac OS X. + macOS. .. envvar:: PYTHONWARNINGS @@ -950,7 +960,7 @@ conflict. If this variable is set, it disables the inclusion of the tables mapping extra location information (end line, start column offset and end column offset) to every instruction in code objects. This is useful when smaller - code objects and pyc files are desired as well as supressing the extra visual + code objects and pyc files are desired as well as suppressing the extra visual location indicators when the interpreter displays tracebacks. .. versionadded:: 3.11 @@ -975,3 +985,12 @@ Debug-mode variables shutting down the interpreter. Need Python configured with the :option:`--with-trace-refs` build option. + +.. envvar:: PYTHONDUMPREFSFILE=FILENAME + + If set, Python will dump objects and reference counts still alive + after shutting down the interpreter into a file called *FILENAME*. + + Need Python configured with the :option:`--with-trace-refs` build option. + + .. versionadded:: 3.11 diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index a545d5a9372..7802a7c810b 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -53,7 +53,11 @@ General Options Set the Python executable suffix to *SUFFIX*. The default suffix is ``.exe`` on Windows and macOS (``python.exe`` - executable), and an empty string on other platforms (``python`` executable). + executable), ``.wasm`` on Emscripten (``python.wasm`` executable), and + an empty string on other platforms (``python`` executable). + + .. versionchanged:: 3.11 + The default suffix on Emscripten platform is ``.wasm``. .. cmdoption:: --with-tzpath= @@ -76,7 +80,7 @@ General Options .. versionadded:: 3.9 -.. cmdoption:: --with-dbmliborder=db1:db2:... +.. cmdoption:: --with-dbmliborder= Override order to check db backends for the :mod:`dbm` module @@ -116,6 +120,17 @@ General Options .. versionadded:: 3.10 +.. cmdoption:: --with-pkg-config=[check|yes|no] + + Whether configure should use :program:`pkg-config` to detect build + dependencies. + + * ``check`` (default): :program:`pkg-config` is optional + * ``yes``: :program:`pkg-config` is mandatory + * ``no``: configure does not use :program:`pkg-config` even when present + + .. versionadded:: 3.11 + Install Options --------------- @@ -416,15 +431,19 @@ Libraries options Security Options ---------------- -.. cmdoption:: --with-hash-algorithm=[fnv|siphash24] +.. cmdoption:: --with-hash-algorithm=[fnv|siphash13|siphash24] Select hash algorithm for use in ``Python/pyhash.c``: - * ``siphash24`` (default). - * ``fnv``; + * ``siphash13`` (default); + * ``siphash24``; + * ``fnv``. .. versionadded:: 3.4 + .. versionadded:: 3.11 + ``siphash13`` is added and it is the new default. + .. cmdoption:: --with-builtin-hashlib-hashes=md5,sha1,sha256,sha512,sha3,blake2 Built-in hash modules: @@ -494,6 +513,56 @@ See ``Mac/README.rst``. :option:`--enable-framework` is set (default: ``Python``). +Cross Compiling Options +----------------------- + +Cross compiling, also known as cross building, can be used to build Python +for another CPU architecture or platform. Cross compiling requires a Python +interpreter and the :program:`_freeze_module` binary from another build. The +version of the build Python and :program:`_freeze_module` command must be +the same as the cross compiled host Python. + +.. cmdoption:: --build=BUILD + + configure for building on BUILD, usually guessed by :program:`config.guess`. + +.. cmdoption:: --host=HOST + + cross-compile to build programs to run on HOST (target platform) + +.. cmdoption:: --with-freeze-module=Programs/_freeze_module + + path to ``_freeze_module`` binary for cross compiling. + + .. versionadded:: 3.11 + +.. cmdoption:: --with-build-python=python3.xx + + path to build ``python`` binary for cross compiling + + .. versionadded:: 3.11 + +.. cmdoption:: CONFIG_SITE=file + + An environment variable that points to a file with configure overrides. + + Example *config.site* file:: + + # config.site-aarch64 + ac_cv_buggy_getaddrinfo=no + ac_cv_file__dev_ptmx=yes + ac_cv_file__dev_ptc=no + + +Cross compiling example:: + + CONFIG_SITE=config.site-aarch64 ../configure \ + --build=x86_64-pc-linux-gnu \ + --host=aarch64-unknown-linux-gnu \ + --with-freeze-module=../x86_64/Programs/_freeze_module \ + --with-build-python=../x86_64/python + + Python Build System =================== @@ -553,7 +622,7 @@ Built-in modules have no ``__file__`` attribute:: File "", line 1, in AttributeError: module 'sys' has no attribute '__file__' -Other C extensions are built as dynamic libraires, like the ``_asyncio`` module. +Other C extensions are built as dynamic libraries, like the ``_asyncio`` module. They are built with the ``Py_BUILD_CORE_MODULE`` macro defined. Example on Linux x86-64:: diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index ead71e1b079..2f132a96bef 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -1,14 +1,14 @@ .. _using-on-mac: -*************************** -Using Python on a Macintosh -*************************** +********************* +Using Python on a Mac +********************* :Author: Bob Savage -Python on a Macintosh running Mac OS X is in principle very similar to Python on +Python on a Mac running macOS is in principle very similar to Python on any other Unix platform, but there are a number of additional features such as the IDE and the Package Manager that are worth pointing out. @@ -17,7 +17,7 @@ the IDE and the Package Manager that are worth pointing out. Getting and Installing MacPython ================================ -Mac OS X 10.8 comes with Python 2.7 pre-installed by Apple. If you wish, you +macOS since version 10.8 comes with Python 2.7 pre-installed by Apple. If you wish, you are invited to install the most recent version of Python 3 from the Python website (https://www.python.org). A current "universal binary" build of Python, which runs natively on the Mac's new Intel and legacy PPC CPU's, is available @@ -54,12 +54,12 @@ section on running Python scripts from the Unix shell. How to run a Python script -------------------------- -Your best way to get started with Python on Mac OS X is through the IDLE +Your best way to get started with Python on macOS is through the IDLE integrated development environment, see section :ref:`ide` and use the Help menu when the IDE is running. If you want to run Python scripts from the Terminal window command line or from -the Finder you first need an editor to create your script. Mac OS X comes with a +the Finder you first need an editor to create your script. macOS comes with a number of standard Unix command line editors, :program:`vim` and :program:`emacs` among them. If you want a more Mac-like editor, :program:`BBEdit` or :program:`TextWrangler` from Bare Bones Software (see @@ -87,7 +87,7 @@ To run your script from the Finder you have two options: Running scripts with a GUI -------------------------- -With older versions of Python, there is one Mac OS X quirk that you need to be +With older versions of Python, there is one macOS quirk that you need to be aware of: programs that talk to the Aqua window manager (in other words, anything that has a GUI) need to be run in a special way. Use :program:`pythonw` instead of :program:`python` to start such scripts. @@ -98,7 +98,7 @@ With Python 3.9, you can use either :program:`python` or :program:`pythonw`. Configuration ------------- -Python on OS X honors all standard Unix environment variables such as +Python on macOS honors all standard Unix environment variables such as :envvar:`PYTHONPATH`, but setting these variables for programs started from the Finder is non-standard as the Finder does not read your :file:`.profile` or :file:`.cshrc` at startup. You need to create a file @@ -148,10 +148,10 @@ X by Apple, and the latest version can be downloaded and installed from https://www.activestate.com; it can also be built from source. *wxPython* is another popular cross-platform GUI toolkit that runs natively on -Mac OS X. Packages and documentation are available from https://www.wxpython.org. +macOS. Packages and documentation are available from https://www.wxpython.org. -*PyQt* is another popular cross-platform GUI toolkit that runs natively on Mac -OS X. More information can be found at +*PyQt* is another popular cross-platform GUI toolkit that runs natively on +macOS. More information can be found at https://riverbankcomputing.com/software/pyqt/intro. diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index 1d1fa8bd85d..0a1834453a0 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -162,6 +162,7 @@ Custom OpenSSL $ pushd openssl-VERSION $ ./config \ --prefix=/usr/local/custom-openssl \ + --libdir=lib \ --openssldir=/etc/ssl $ make -j1 depend $ make -j8 diff --git a/Doc/whatsnew/2.0.rst b/Doc/whatsnew/2.0.rst index c0a66920497..0e1cf1fd0ce 100644 --- a/Doc/whatsnew/2.0.rst +++ b/Doc/whatsnew/2.0.rst @@ -791,7 +791,7 @@ Previously the Python virtual machine used 16-bit numbers in its bytecode, limiting the size of source files. In particular, this affected the maximum size of literal lists and dictionaries in Python source; occasionally people who are generating Python code would run into this limit. A patch by Charles G. -Waldman raises the limit from ``2^16`` to ``2^{32}``. +Waldman raises the limit from ``2**16`` to ``2**32``. Three new convenience functions intended for adding constants to a module's dictionary at module initialization time were added: :func:`PyModule_AddObject`, diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index d19c8e01ad8..abb65222ddd 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -953,12 +953,12 @@ Several performance enhancements have been added: considered and traversed by the collector. (Contributed by Antoine Pitrou; :issue:`4688`.) -* Long integers are now stored internally either in base 2**15 or in base - 2**30, the base being determined at build time. Previously, they - were always stored in base 2**15. Using base 2**30 gives +* Long integers are now stored internally either in base ``2**15`` or in base + ``2**30``, the base being determined at build time. Previously, they + were always stored in base ``2**15``. Using base ``2**30`` gives significant performance improvements on 64-bit machines, but benchmark results on 32-bit machines have been mixed. Therefore, - the default is to use base 2**30 on 64-bit machines and base 2**15 + the default is to use base ``2**30`` on 64-bit machines and base ``2**15`` on 32-bit machines; on Unix, there's a new configure option :option:`!--enable-big-digits` that can be used to override this default. diff --git a/Doc/whatsnew/3.1.rst b/Doc/whatsnew/3.1.rst index 919fbeeb2ad..f1e6d0c4f3d 100644 --- a/Doc/whatsnew/3.1.rst +++ b/Doc/whatsnew/3.1.rst @@ -474,12 +474,12 @@ Build and C API Changes Changes to Python's build process and to the C API include: -* Integers are now stored internally either in base 2**15 or in base - 2**30, the base being determined at build time. Previously, they - were always stored in base 2**15. Using base 2**30 gives +* Integers are now stored internally either in base ``2**15`` or in base + ``2**30``, the base being determined at build time. Previously, they + were always stored in base ``2**15``. Using base ``2**30`` gives significant performance improvements on 64-bit machines, but benchmark results on 32-bit machines have been mixed. Therefore, - the default is to use base 2**30 on 64-bit machines and base 2**15 + the default is to use base ``2**30`` on 64-bit machines and base ``2**15`` on 32-bit machines; on Unix, there's a new configure option ``--enable-big-digits`` that can be used to override this default. diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index ae6ad9c52cc..11856396509 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -50,13 +50,6 @@ This article explains the new features in Python 3.10, compared to 3.9. For full details, see the :ref:`changelog `. -.. note:: - - Prerelease users should be aware that this document is currently in draft - form. It will be updated substantially as Python 3.10 moves towards release, - so it's worth checking back even after reading earlier versions. - - Summary -- Release highlights ============================= @@ -883,6 +876,13 @@ Other Language Changes collisions when creating dictionaries and sets containing multiple NaNs. (Contributed by Raymond Hettinger in :issue:`43475`.) +* A :exc:`SyntaxError` (instead of a :exc:`NameError`) will be raised when deleting + the :const:`__debug__` constant. (Contributed by Dong-hee Na in :issue:`45000`.) + +* :exc:`SyntaxError` exceptions now have ``end_lineno`` and + ``end_offset`` attributes. They will be ``None`` if not determined. + (Contributed by Pablo Galindo in :issue:`43914`.) + New Modules =========== @@ -930,6 +930,12 @@ bdb Add :meth:`~bdb.Breakpoint.clearBreakpoints` to reset all set breakpoints. (Contributed by Irit Katriel in :issue:`24160`.) +bisect +------ + +Added the possibility of providing a *key* function to the APIs in the :mod:`bisect` +module. (Contributed by Raymond Hettinger in :issue:`4356`.) + codecs ------ @@ -981,9 +987,68 @@ they are provided by the underlying curses library. dataclasses ----------- -Add ``slots`` parameter in :func:`dataclasses.dataclass` decorator. +__slots__ +~~~~~~~~~ + +Added ``slots`` parameter in :func:`dataclasses.dataclass` decorator. (Contributed by Yurii Karabas in :issue:`42269`) +Keyword-only fields +~~~~~~~~~~~~~~~~~~~ + +dataclasses now supports fields that are keyword-only in the +generated __init__ method. There are a number of ways of specifying +keyword-only fields. + +You can say that every field is keyword-only: + +.. code-block:: python + + from dataclasses import dataclass + + @dataclass(kw_only=True) + class Birthday: + name: str + birthday: datetime.date + +Both ``name`` and ``birthday`` are keyword-only parameters to the +generated __init__ method. + +You can specify keyword-only on a per-field basis: + +.. code-block:: python + + from dataclasses import dataclass + + @dataclass + class Birthday: + name: str + birthday: datetime.date = field(kw_only=True) + +Here only ``birthday`` is keyword-only. If you set ``kw_only`` on +individual fields, be aware that there are rules about re-ordering +fields due to keyword-only fields needing to follow non-keyword-only +fields. See the full dataclasses documentation for details. + +You can also specify that all fields following a KW_ONLY marker are +keyword-only. This will probably be the most common usage: + +.. code-block:: python + + from dataclasses import dataclass, KW_ONLY + + @dataclass + class Point: + x: float + y: float + _: KW_ONLY + z: float = 0.0 + t: float = 0.0 + +Here, ``z`` and ``t`` are keyword-only parameters, while ``x`` and +``y`` are not. +(Contributed by Eric V. Smith in :issue:`43532`) + .. _distutils-deprecated: distutils @@ -1108,7 +1173,7 @@ Highlight the new :ref:`soft keywords ` :keyword:`match`, :keyword:`case `, and :keyword:`_ ` in pattern-matching statements. However, this highlighting is not perfect and will be incorrect in some rare cases, including some ``_``-s in -``case`` patterns. (Contributed by Tal Einat in bpo-44010.) +``case`` patterns. (Contributed by Tal Einat in :issue:`44010`.) importlib.metadata ------------------ @@ -1117,7 +1182,7 @@ Feature parity with ``importlib_metadata`` 4.6 (`history `_). :ref:`importlib.metadata entry points ` -now provides a nicer experience +now provide a nicer experience for selecting entry points by group and name through a new :class:`importlib.metadata.EntryPoints` class. See the Compatibility Note in the docs for more info on the deprecation and usage. @@ -1401,6 +1466,13 @@ subclasses with the :func:`runtime_checkable` decorator if they want runtime protocols. (Contributed by Yurii Karabas in :issue:`38908`) +Importing from the ``typing.io`` and ``typing.re`` submodules will now emit +:exc:`DeprecationWarning`. These submodules have been deprecated since +Python 3.8 and will be removed in a future version of Python. Anything +belonging to those submodules should be imported directly from +:mod:`typing` instead. +(Contributed by Sebastian Rittau in :issue:`38291`) + unittest -------- @@ -1504,7 +1576,7 @@ Deprecated * Currently Python accepts numeric literals immediately followed by keywords, for example ``0in x``, ``1or x``, ``0if 1else 2``. It allows confusing - and ambigious expressions like ``[0x1for x in y]`` (which can be + and ambiguous expressions like ``[0x1for x in y]`` (which can be interpreted as ``[0x1 for x in y]`` or ``[0x1f or x in y]``). Starting in this release, a deprecation warning is raised if the numeric literal is immediately followed by one of keywords :keyword:`and`, :keyword:`else`, @@ -1696,6 +1768,12 @@ Deprecated requires a :ref:`debug build of Python `. (Contributed by Victor Stinner in :issue:`44584`.) +* Importing from the ``typing.io`` and ``typing.re`` submodules will now emit + :exc:`DeprecationWarning`. These submodules will be removed in a future version + of Python. Anything belonging to these submodules should be imported directly + from :mod:`typing` instead. + (Contributed by Sebastian Rittau in :issue:`38291`) + .. _whatsnew310-removed: Removed @@ -1773,7 +1851,7 @@ Changes in the Python syntax * Deprecation warning is now emitted when compiling previously valid syntax if the numeric literal is immediately followed by a keyword (like in ``0in x``). - If future releases it will be changed to syntax warning, and finally to a + In future releases it will be changed to syntax warning, and finally to a syntax error. To get rid of the warning and make the code compatible with future releases just add a space between the numeric literal and the following keyword. @@ -1835,7 +1913,7 @@ Changes in the Python API if the *globals* dictionary has no ``"__builtins__"`` key, rather than using ``{"None": None}`` as builtins: same behavior as :func:`eval` and :func:`exec` functions. Defining a function with ``def function(...): ...`` - in Python is not affected, globals cannot be overriden with this syntax: it + in Python is not affected, globals cannot be overridden with this syntax: it also inherits the current builtins. (Contributed by Victor Stinner in :issue:`42990`.) @@ -1874,6 +1952,13 @@ Changes in the C API source_buf = PyBytes_AsString(source_bytes_object); code = Py_CompileString(source_buf, filename, Py_file_input); + * For ``FrameObject`` objects, the ``f_lasti`` member now represents a wordcode + offset instead of a simple offset into the bytecode string. This means that this + number needs to be multiplied by 2 to be used with APIs that expect a byte offset + instead (like :c:func:`PyCode_Addr2Line` for example). Notice as well that the + ``f_lasti`` member of ``FrameObject`` objects is not considered stable: please + use :c:func:`PyFrame_GetLineNumber` instead. + CPython bytecode changes ======================== @@ -2218,3 +2303,6 @@ Removed These functions were undocumented, excluded from the limited C API, and were only used internally by the compiler. (Contributed by Victor Stinner in :issue:`43244`.) + +* The ``PyThreadState.use_tracing`` member has been removed to optimize Python. + (Contributed by Mark Shannon in :issue:`43760`.) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 88b6f8fa731..8d1f4eba36e 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -66,7 +66,6 @@ Summary -- Release highlights .. PEP-sized items next. - New Features ============ @@ -147,6 +146,12 @@ The :option:`-X` ``no_debug_ranges`` option and the environment variable See :pep:`657` for more details. (Contributed by Pablo Galindo, Batuhan Taskaya and Ammar Askar in :issue:`43950`.) +Exceptions can be enriched with a string ``__note__`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``__note__`` field was added to :exc:`BaseException`. It is ``None`` +by default but can be set to a string which is added to the exception's +traceback. (Contributed by Irit Katriel in :issue:`45607`.) Other Language Changes ====================== @@ -169,6 +174,25 @@ Other Language Changes (Contributed by Serhiy Storchaka in :issue:`12022`.) +Other CPython Implementation Changes +==================================== + +* Special methods :meth:`complex.__complex__` and :meth:`bytes.__bytes__` are implemented to + support :class:`typing.SupportsComplex` and :class:`typing.SupportsBytes` protocols. + (Contributed by Mark Dickinson and Dong-hee Na in :issue:`24234`.) + +* ``siphash13`` is added as a new internal hashing algorithms. It's has similar security + properties as ``siphash24`` but it is slightly faster for long inputs. ``str``, ``bytes``, + and some other types now use it as default algorithm for ``hash()``. :pep:`552` + hash-based pyc files now use ``siphash13``, too. + (Contributed by Inada Naoki in :issue:`29410`.) + +* When an active exception is re-raised by a :keyword:`raise` statement with no parameters, + the traceback attached to this exception is now always ``sys.exc_info()[1].__traceback__``. + This means that changes made to the traceback in the current :keyword:`except` clause are + reflected in the re-raised exception. + (Contributed by Irit Katriel in :issue:`45711`.) + New Modules =========== @@ -181,12 +205,25 @@ Improved Modules fractions --------- -Support :PEP:`515`-style initialization of :class:`~fractions.Fraction` from -string. (Contributed by Sergey B Kirpichev in :issue:`44258`.) +* Support :PEP:`515`-style initialization of :class:`~fractions.Fraction` from + string. (Contributed by Sergey B Kirpichev in :issue:`44258`.) + +* :class:`~fractions.Fraction` now implements an ``__int__`` method, so + that an ``isinstance(some_fraction, typing.SupportsInt)`` check passes. + (Contributed by Mark Dickinson in :issue:`44547`.) + + +inspect +------- +* Add :func:`inspect.getmembers_static`: return all members without + triggering dynamic lookup via the descriptor protocol. (Contributed by + Weipeng Hong in :issue:`30533`.) math ---- +* Add :func:`math.exp2`: return 2 raised to the power of x. + (Contributed by Gideon Mitchell in :issue:`45917`.) * Add :func:`math.cbrt`: return the cube root of x. (Contributed by Ajith Ramachandran in :issue:`44357`.) @@ -198,11 +235,19 @@ math Dickinson in :issue:`44339`.) +operator +-------- + +* A new function ``operator.call`` has been added, such that + ``operator.call(obj, *args, **kwargs) == obj(*args, **kwargs)``. + (Contributed by Antony Lee in :issue:`44019`.) + + os -- -* On Windows, :func:`os.urandom` uses ``BCryptGenRandom()`` instead of ``CryptGenRandom()`` - which is deprecated. +* On Windows, :func:`os.urandom` now uses ``BCryptGenRandom()``, + instead of ``CryptGenRandom()`` which is deprecated. (Contributed by Dong-hee Na in :issue:`44611`.) @@ -218,11 +263,78 @@ sqlite3 now raise :exc:`UnicodeEncodeError` instead of :exc:`sqlite3.ProgrammingError`. (Contributed by Erlend E. Aasland in :issue:`44688`.) +* :mod:`sqlite3` exceptions now include the SQLite extended error code as + :attr:`~sqlite3.Error.sqlite_errorcode` and the SQLite error name as + :attr:`~sqlite3.Error.sqlite_errorname`. + (Contributed by Aviv Palivoda, Daniel Shahaf, and Erlend E. Aasland in + :issue:`16379` and :issue:`24139`.) -Removed -======= -* :class:`smtpd.MailmanProxy` is now removed as it is unusable without - an external module, ``mailman``. (Contributed by Dong-hee Na in :issue:`35800`.) +* Add :meth:`~sqlite3.Connection.setlimit` and + :meth:`~sqlite3.Connection.getlimit` to :class:`sqlite3.Connection` for + setting and getting SQLite limits by connection basis. + (Contributed by Erlend E. Aasland in :issue:`45243`.) + +* :mod:`sqlite3` now sets :attr:`sqlite3.threadsafety` based on the default + threading mode the underlying SQLite library has been compiled with. + (Contributed by Erlend E. Aasland in :issue:`45613`.) + +* :mod:`sqlite3` C callbacks now use unraisable exceptions if callback + tracebacks are enabled. Users can now register an + :func:`unraisable hook handler ` to improve their debug + experience. + (Contributed by Erlend E. Aasland in :issue:`45828`.) + + +sys +--- + +* :func:`sys.exc_info` now derives the ``type`` and ``traceback`` fields + from the ``value`` (the exception instance), so when an exception is + modified while it is being handled, the changes are reflected in + the results of subsequent calls to :func:`exc_info`. + (Contributed by Irit Katriel in :issue:`45711`.) + + +threading +--------- + +* On Unix, if the ``sem_clockwait()`` function is available in the C library + (glibc 2.30 and newer), the :meth:`threading.Lock.acquire` method now uses + the monotonic clock (:data:`time.CLOCK_MONOTONIC`) for the timeout, rather + than using the system clock (:data:`time.CLOCK_REALTIME`), to not be affected + by system clock changes. + (Contributed by Victor Stinner in :issue:`41710`.) + + +time +---- + +* On Unix, :func:`time.sleep` now uses the ``clock_nanosleep()`` or + ``nanosleep()`` function, if available, which has a resolution of 1 nanosecond + (10\ :sup:`-9` seconds), rather than using ``select()`` which has a resolution + of 1 microsecond (10\ :sup:`-6` seconds). + (Contributed by Benjamin Szőke and Victor Stinner in :issue:`21302`.) + +* On Windows 8.1 and newer, :func:`time.sleep` now uses a waitable timer based + on `high-resolution timers + `_ + which has a resolution of 100 nanoseconds (10\ :sup:`-7` seconds). Previously, + it had a resolution of 1 millisecond (10\ :sup:`-3` seconds). + (Contributed by Benjamin Szőke, Dong-hee Na, Eryk Sun and Victor Stinner in :issue:`21302` and :issue:`45429`.) + + +unicodedata +----------- + +* The Unicode database has been updated to version 14.0.0. (:issue:`45190`). + + +fcntl +----- + +* On FreeBSD, the :attr:`F_DUP2FD` and :attr:`F_DUP2FD_CLOEXEC` flags respectively + are supported, the former equals to ``dup2`` usage while the latter set + the ``FD_CLOEXEC`` flag in addition. Optimizations @@ -244,29 +356,147 @@ Optimizations (Contributed by Ken Jin and Mark Shannon in :issue:`26110`, based on ideas implemented in PyPy.) -* :file:`.pdbrc` is now read with ``'utf-8'`` encoding. +* Pure ASCII strings are now normalized in constant time by :func:`unicodedata.normalize`. + (Contributed by Dong-hee Na in :issue:`44987`.) + +* :mod:`math` functions :func:`~math.comb` and :func:`~math.perm` are now up + to 10 times or more faster for large arguments (the speed up is larger for + larger *k*). + (Contributed by Serhiy Storchaka in :issue:`37295`.) CPython bytecode changes ======================== +* Replaced all numeric ``BINARY_*`` and ``INPLACE_*`` instructions with a single + :opcode:`BINARY_OP` implementation. + * Added a new :opcode:`CALL_METHOD_KW` opcode. Calls a method in a similar fashion as :opcode:`CALL_METHOD`, but also supports keyword arguments. Works in tandem with :opcode:`LOAD_METHOD`. +* Removed ``COPY_DICT_WITHOUT_KEYS``. -Build Changes -============= +* :opcode:`MATCH_CLASS` and :opcode:`MATCH_KEYS` no longer push an additional + boolean value indicating whether the match succeeded or failed. Instead, they + indicate failure with :const:`None` (where a tuple of extracted values would + otherwise be). + +* Added :opcode:`COPY`, which pushes the *i*-th item to the top of the stack. + The item is not removed from its original location. + +* :opcode:`JUMP_IF_NOT_EXC_MATCH` no longer pops the active exception. Deprecated ========== +* The :mod:`lib2to3` package and ``2to3`` tool are now deprecated and may not + be able to parse Python 3.10 or newer. See the :pep:`617` (New PEG parser for + CPython). (Contributed by Victor Stinner in :issue:`40360`.) +* :class:`webbrowser.MacOSX` is deprecated and will be removed in Python 3.13. + It is untested and undocumented and also not used by webbrowser itself. + (Contributed by Dong-hee Na in :issue:`42255`.) + +* The behavior of returning a value from a :class:`~unittest.TestCase` and + :class:`~unittest.IsolatedAsyncioTestCase` test methods (other than the + default ``None`` value), is now deprecated. + +* Deprecated the following :mod:`unittest` functions, scheduled for removal in + Python 3.13: + + * :func:`unittest.findTestCases` + * :func:`unittest.makeSuite` + * :func:`unittest.getTestCaseNames` + + Use :class:`~unittest.TestLoader` method instead: + + * :meth:`unittest.TestLoader.loadTestsFromModule` + * :meth:`unittest.TestLoader.loadTestsFromTestCase` + * :meth:`unittest.TestLoader.getTestCaseNames` + + (Contributed by Erlend E. Aasland in :issue:`5846`.) + +* The :meth:`turtle.RawTurtle.settiltangle` is deprecated since Python 3.1, + it now emits a deprecation warning and will be removed in Python 3.13. Use + :meth:`turtle.RawTurtle.tiltangle` instead (it was earlier incorrectly marked + as deprecated, its docstring is now corrected). + (Contributed by Hugo van Kemenade in :issue:`45837`.) Removed ======= +* :class:`smtpd.MailmanProxy` is now removed as it is unusable without + an external module, ``mailman``. (Contributed by Dong-hee Na in :issue:`35800`.) + +* The ``binhex`` module, deprecated in Python 3.9, is now removed. + The following :mod:`binascii` functions, deprecated in Python 3.9, are now + also removed: + + * ``a2b_hqx()``, ``b2a_hqx()``; + * ``rlecode_hqx()``, ``rledecode_hqx()``. + + The :func:`binascii.crc_hqx` function remains available. + + (Contributed by Victor Stinner in :issue:`45085`.) + +* The distutils ``bdist_msi`` command, deprecated in Python 3.9, is now removed. + Use ``bdist_wheel`` (wheel packages) instead. + (Contributed by Hugo van Kemenade in :issue:`45124`.) + +* Due to significant security concerns, the *reuse_address* parameter of + :meth:`asyncio.loop.create_datagram_endpoint`, disabled in Python 3.9, is + now entirely removed. This is because of the behavior of the socket option + ``SO_REUSEADDR`` in UDP. + (Contributed by Hugo van Kemenade in :issue:`45129`.) + +* Removed :meth:`__getitem__` methods of + :class:`xml.dom.pulldom.DOMEventStream`, :class:`wsgiref.util.FileWrapper` + and :class:`fileinput.FileInput`, deprecated since Python 3.9. + (Contributed by Hugo van Kemenade in :issue:`45132`.) + +* Removed many old deprecated :mod:`unittest` features: + + - :class:`~unittest.TestCase` method aliases ``failUnlessEqual``, + ``failIfEqual``, ``failUnless``, ``failIf``, ``failUnlessRaises``, + ``failUnlessAlmostEqual``, ``failIfAlmostEqual`` (deprecated in Python 3.1), + ``assertEquals``, ``assertNotEquals``, ``assert_``, ``assertAlmostEquals``, + ``assertNotAlmostEquals``, ``assertRegexpMatches``, ``assertRaisesRegexp`` + (deprecated in Python 3.2), and ``assertNotRegexpMatches`` (deprecated in + Python 3.5). + + - Undocumented and broken :class:`~unittest.TestCase` method + ``assertDictContainsSubset`` (deprecated in Python 3.2). + + - Undocumented :meth:` + TestLoader.loadTestsFromModule` parameter *use_load_tests* (deprecated + and ignored since Python 3.2). + + - An alias of the :class:`~unittest.TextTestResult` class: + ``_TextTestResult`` (deprecated in Python 3.2). + + (Contributed by Serhiy Storchaka in :issue:`45162`.) + +* The following deprecated functions and methods are removed in the :mod:`gettext` + module: :func:`~gettext.lgettext`, :func:`~gettext.ldgettext`, + :func:`~gettext.lngettext` and :func:`~gettext.ldngettext`. + + Function :func:`~gettext.bind_textdomain_codeset`, methods + :meth:`~gettext.NullTranslations.output_charset` and + :meth:`~gettext.NullTranslations.set_output_charset`, and the *codeset* + parameter of functions :func:`~gettext.translation` and + :func:`~gettext.install` are also removed, since they are only used for + the ``l*gettext()`` functions. + (Contributed by Dong-hee Na and Serhiy Storchaka in :issue:`44235`.) + +* Removed from the :mod:`configparser` module: + the :class:`SafeConfigParser` class, + the :attr:`filename` property of the :class:`ParsingError` class, + the :meth:`readfp` method of the :class:`ConfigParser` class, + deprecated since Python 3.2. + (Contributed by Hugo van Kemenade in :issue:`45173`.) + * The :func:`@asyncio.coroutine ` :term:`decorator` enabling legacy generator-based coroutines to be compatible with async/await code. The function has been deprecated since Python 3.8 and the removal was @@ -277,6 +507,26 @@ Removed generator-based coroutine objects in the debug mode. (Contributed by Illia Volochii in :issue:`43216`.) +* Removed the deprecated ``split()`` method of :class:`_tkinter.TkappType`. + (Contributed by Erlend E. Aasland in :issue:`38371`.) + +* Removed from the :mod:`inspect` module: + + * the ``getargspec`` function, deprecated since Python 3.0; + use :func:`inspect.signature` or :func:`inspect.getfullargspec` instead. + + * the ``formatargspec`` function, deprecated since Python 3.5; + use the :func:`inspect.signature` function and :class:`Signature` object + directly. + + * the undocumented ``Signature.from_callable`` and ``Signature.from_function`` + functions, deprecated since Python 3.5; use the + :meth:`Signature.from_callable() ` method + instead. + + (Contributed by Hugo van Kemenade in :issue:`45320`.) + + Porting to Python 3.11 ====================== @@ -292,18 +542,139 @@ Changes in the Python API Python 3.8. (Contributed by Illia Volochii in :issue:`43234`.) +* :func:`open`, :func:`io.open`, :func:`codecs.open` and + :class:`fileinput.FileInput` no longer accept ``'U'`` ("universal newline") + in the file mode. This flag was deprecated since Python 3.3. In Python 3, the + "universal newline" is used by default when a file is open in text mode. The + :ref:`newline parameter ` of :func:`open` controls + how universal newlines works. + (Contributed by Victor Stinner in :issue:`37330`.) + +* The :mod:`pdb` module now reads the :file:`.pdbrc` configuration file with + the ``'utf-8'`` encoding. + (Contributed by Srinivas Reddy Thatiparthy (శ్రీనివాస్ రెడ్డి తాటిపర్తి) in :issue:`41137`.) + +* When sorting using tuples as keys, the order of the result may differ + from earlier releases if the tuple elements don't define a total + ordering (see :ref:`expressions-value-comparisons` for + information on total ordering). It's generally true that the result + of sorting simply isn't well-defined in the absence of a total ordering + on list elements. + + +Build Changes +============= + +* CPython can now be built with the ThinLTO option via ``--with-lto=thin``. + (Contributed by Dong-hee Na and Brett Holman in :issue:`44340`.) + +* libpython is no longer linked against libcrypt. + (Contributed by Mike Gilbert in :issue:`45433`.) + +* Building Python now requires a C99 ```` header file providing + the following functions: ``copysign()``, ``hypot()``, ``isfinite()``, + ``isinf()``, ``isnan()``, ``round()``. + (Contributed by Victor Stinner in :issue:`45440`.) + +* Freelists for object structs can now be disabled. A new :program:`configure` + option :option:`!--without-freelists` can be used to disable all freelists + except empty tuple singleton. + (Contributed by Christian Heimes in :issue:`45522`) + +* ``Modules/Setup`` and ``Modules/makesetup`` have been improved and tied up. + Extension modules can now be built through ``makesetup``. All except some + test modules can be linked statically into main binary or library. + (Contributed by Brett Cannon and Christian Heimes in :issue:`45548`, + :issue:`45570`, :issue:`45571`, and :issue:`43974`.) + +* Build dependencies, compiler flags, and linker flags for most stdlib + extension modules are now detected by :program:`configure`. libffi, libnsl, + libsqlite3, zlib, bzip2, liblzma, libcrypt, and uuid flags are detected by + ``pkg-config`` (when available). + (Contributed by Christian Heimes and Erlend Egeberg Aasland in + :issue:`bpo-45847`, :issue:`45747`, and :issue:`45763`.) + +* CPython now has experimental support for cross compiling to WebAssembly + platform ``wasm32-emscripten``. The effort is inspired by previous work + like Pyodide. + (Contributed by Christian Heimes and Ethan Smith in :issue:`40280`.) C API Changes ============= -* Add a new :c:func:`PyType_GetName` function to get type's short name. - (Contributed by Hai Shi in :issue:`42035`.) New Features ------------ +* Add a new :c:func:`PyType_GetName` function to get type's short name. + (Contributed by Hai Shi in :issue:`42035`.) + +* Add a new :c:func:`PyType_GetQualName` function to get type's qualified name. + (Contributed by Hai Shi in :issue:`42035`.) + +* Add new :c:func:`PyThreadState_EnterTracing` and + :c:func:`PyThreadState_LeaveTracing` functions to the limited C API to + suspend and resume tracing and profiling. + (Contributed by Victor Stinner in :issue:`43760`.) + +* :c:func:`PyErr_SetExcInfo()` no longer uses the ``type`` and ``traceback`` + arguments, the interpreter now derives those values from the exception + instance (the ``value`` argument). The function still steals references + of all three arguments. + (Contributed by Irit Katriel in :issue:`45711`.) + +* :c:func:`PyErr_GetExcInfo()` now derives the ``type`` and ``traceback`` + fields of the result from the exception instance (the ``value`` field). + (Contributed by Irit Katriel in :issue:`45711`.) + +* Added the :c:data:`Py_Version` constant which bears the same value as + :c:macro:`PY_VERSION_HEX`. + (Contributed by Gabriele N. Tornetta in :issue:`43931`.) + + Porting to Python 3.11 ---------------------- +* The old trashcan macros (``Py_TRASHCAN_SAFE_BEGIN``/``Py_TRASHCAN_SAFE_END``) + are now deprecated. They should be replaced by the new macros + ``Py_TRASHCAN_BEGIN`` and ``Py_TRASHCAN_END``. + + A tp_dealloc function that has the old macros, such as:: + + static void + mytype_dealloc(mytype *p) + { + PyObject_GC_UnTrack(p); + Py_TRASHCAN_SAFE_BEGIN(p); + ... + Py_TRASHCAN_SAFE_END + } + + should migrate to the new macros as follows:: + + static void + mytype_dealloc(mytype *p) + { + PyObject_GC_UnTrack(p); + Py_TRASHCAN_BEGIN(p, mytype_dealloc) + ... + Py_TRASHCAN_END + } + + Note that ``Py_TRASHCAN_BEGIN`` has a second argument which + should be the deallocation function it is in. + + To support older Python versions in the same codebase, you + can define the following macros and use them throughout + the code (credit: these were copied from the ``mypy`` codebase):: + + #if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 8 + # define CPy_TRASHCAN_BEGIN(op, dealloc) Py_TRASHCAN_BEGIN(op, dealloc) + # define CPy_TRASHCAN_END(op) Py_TRASHCAN_END + #else + # define CPy_TRASHCAN_BEGIN(op, dealloc) Py_TRASHCAN_SAFE_BEGIN(op) + # define CPy_TRASHCAN_END(op) Py_TRASHCAN_SAFE_END(op) + #endif + * The :c:func:`PyType_Ready` function now raises an error if a type is defined with the :const:`Py_TPFLAGS_HAVE_GC` flag set but has no traverse function (:c:member:`PyTypeObject.tp_traverse`). @@ -314,16 +685,57 @@ Porting to Python 3.11 :ref:`static types `. (Contributed by Erlend E. Aasland in :issue:`43908`) +* Since :c:func:`Py_TYPE()` is changed to a inline static function, + ``Py_TYPE(obj) = new_type`` must be replaced with + ``Py_SET_TYPE(obj, new_type)``: see the :c:func:`Py_SET_TYPE()` function + (available since Python 3.9). For backward compatibility, this macro can be + used:: + + #if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_TYPE) + static inline void _Py_SET_TYPE(PyObject *ob, PyTypeObject *type) + { ob->ob_type = type; } + #define Py_SET_TYPE(ob, type) _Py_SET_TYPE((PyObject*)(ob), type) + #endif + + (Contributed by Victor Stinner in :issue:`39573`.) + +* Since :c:func:`Py_SIZE()` is changed to a inline static function, + ``Py_SIZE(obj) = new_size`` must be replaced with + ``Py_SET_SIZE(obj, new_size)``: see the :c:func:`Py_SET_SIZE()` function + (available since Python 3.9). For backward compatibility, this macro can be + used:: + + #if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_SIZE) + static inline void _Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) + { ob->ob_size = size; } + #define Py_SET_SIZE(ob, size) _Py_SET_SIZE((PyVarObject*)(ob), size) + #endif + + (Contributed by Victor Stinner in :issue:`39573`.) + +* ```` no longer includes the header files ````, + ````, ```` and ```` when the ``Py_LIMITED_API`` + macro is set to ``0x030b0000`` (Python 3.11) or higher. C extensions should + explicitly include the header files after ``#include ``. + (Contributed by Victor Stinner in :issue:`45434`.) + +* The non-limited API files ``cellobject.h``, ``classobject.h``, ``context.h``, + ``funcobject.h``, ``genobject.h`` and ``longintrepr.h`` have been moved to + the ``Include/cpython`` directory. Moreover, the ``eval.h`` header file was + removed. These files must not be included directly, as they are already + included in ``Python.h``: :ref:`Include Files `. If they have + been included directly, consider including ``Python.h`` instead. + (Contributed by Victor Stinner in :issue:`35134`.) + +* The :c:func:`PyUnicode_CHECK_INTERNED` macro has been excluded from the + limited C API. It was never usable there, because it used internal structures + which are not available in the limited C API. + (Contributed by Victor Stinner in :issue:`46007`.) + + Deprecated ---------- -Removed -------- - -* :c:func:`PyFrame_BlockSetup` and :c:func:`PyFrame_BlockPop` have been - removed. - (Contributed by Mark Shannon in :issue:`40222`.) - * Deprecate the following functions to configure the Python initialization: * :c:func:`PySys_AddWarnOptionUnicode` @@ -340,14 +752,50 @@ Removed ` instead (:pep:`587`). (Contributed by Victor Stinner in :issue:`44113`.) -* The following deprecated functions and methods are removed in the :mod:`gettext` - module: :func:`~gettext.lgettext`, :func:`~gettext.ldgettext`, - :func:`~gettext.lngettext` and :func:`~gettext.ldngettext`. +Removed +------- - Function :func:`~gettext.bind_textdomain_codeset`, methods - :meth:`~gettext.NullTranslations.output_charset` and - :meth:`~gettext.NullTranslations.set_output_charset`, and the *codeset* - parameter of functions :func:`~gettext.translation` and - :func:`~gettext.install` are also removed, since they are only used for - the ``l*gettext()`` functions. - (Contributed by Dong-hee Na and Serhiy Storchaka in :issue:`44235`.) +* :c:func:`PyFrame_BlockSetup` and :c:func:`PyFrame_BlockPop` have been + removed. + (Contributed by Mark Shannon in :issue:`40222`.) + +* Remove the following math macros using the ``errno`` variable: + + * ``Py_ADJUST_ERANGE1()`` + * ``Py_ADJUST_ERANGE2()`` + * ``Py_OVERFLOWED()`` + * ``Py_SET_ERANGE_IF_OVERFLOW()`` + * ``Py_SET_ERRNO_ON_MATH_ERROR()`` + + (Contributed by Victor Stinner in :issue:`45412`.) + +* Remove ``Py_UNICODE_COPY()`` and ``Py_UNICODE_FILL()`` macros, deprecated + since Python 3.3. Use ``PyUnicode_CopyCharacters()`` or ``memcpy()`` + (``wchar_t*`` string), and ``PyUnicode_Fill()`` functions instead. + (Contributed by Victor Stinner in :issue:`41123`.) + +* Remove the ``pystrhex.h`` header file. It only contains private functions. + C extensions should only include the main ```` header file. + (Contributed by Victor Stinner in :issue:`45434`.) + +* Remove the ``Py_FORCE_DOUBLE()`` macro. It was used by the + ``Py_IS_INFINITY()`` macro. + (Contributed by Victor Stinner in :issue:`45440`.) + +* The following items are no longer available when :c:macro:`Py_LIMITED_API` + is defined: + + * :c:func:`PyMarshal_WriteLongToFile` + * :c:func:`PyMarshal_WriteObjectToFile` + * :c:func:`PyMarshal_ReadObjectFromString` + * :c:func:`PyMarshal_WriteObjectToString` + * the ``Py_MARSHAL_VERSION`` macro + + These are not part of the :ref:`limited API `. + + (Contributed by Victor Stinner in :issue:`45474`.) + +* Exclude :c:func:`PyWeakref_GET_OBJECT` from the limited C API. It never + worked since the :c:type:`PyWeakReference` structure is opaque in the + limited C API. + (Contributed by Victor Stinner in :issue:`35134`.) diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 840cb061129..09ec58eca5b 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -1815,8 +1815,7 @@ names. =============================== ============================== Likewise, the ``TestCase.fail*`` methods deprecated in Python 3.1 are expected - to be removed in Python 3.3. Also see the :ref:`deprecated-aliases` section in - the :mod:`unittest` documentation. + to be removed in Python 3.3. (Contributed by Ezio Melotti; :issue:`9424`.) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index f1a903624f4..7c293a50189 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -2248,3 +2248,16 @@ separator key, with ``&`` as the default. This change also affects functions internally. For more details, please see their respective documentation. (Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) + +Notable changes in Python 3.8.12 +================================ + +Starting with Python 3.8.12 the :mod:`ipaddress` module no longer accepts +any leading zeros in IPv4 address strings. Leading zeros are ambiguous and +interpreted as octal notation by some libraries. For example the legacy +function :func:`socket.inet_aton` treats leading zeros as octal notation. +glibc implementation of modern :func:`~socket.inet_pton` does not accept +any leading zeros. + +(Originally contributed by Christian Heimes in :issue:`36384`, and backported +to 3.8 by Achraf Merzouki.) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index f1725e7df02..0d514084d6c 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -671,7 +671,7 @@ time On AIX, :func:`~time.thread_time` is now implemented with ``thread_cputime()`` which has nanosecond resolution, rather than -``clock_gettime(CLOCK_THREAD_CPUTIME_ID)`` which has a resolution of 10 ms. +``clock_gettime(CLOCK_THREAD_CPUTIME_ID)`` which has a resolution of 10 milliseconds. (Contributed by Batuhan Taskaya in :issue:`40192`) sys @@ -1378,10 +1378,6 @@ Porting to Python 3.9 becomes an alias to the :c:func:`PyObject_NewVar` macro. They no longer access directly the :c:member:`PyTypeObject.tp_basicsize` member. - * :c:func:`PyType_HasFeature` now always calls :c:func:`PyType_GetFlags`. - Previously, it accessed directly the :c:member:`PyTypeObject.tp_flags` - member when the limited C API was not used. - * :c:func:`PyObject_GET_WEAKREFS_LISTPTR` macro was converted to a function: the macro accessed directly the :c:member:`PyTypeObject.tp_weaklistoffset` member. diff --git a/Grammar/python.gram b/Grammar/python.gram index b107e474630..2c696a6a085 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -264,11 +264,11 @@ function_def[stmt_ty]: function_def_raw[stmt_ty]: | invalid_def_raw - | 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { + | 'def' n=NAME &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { _PyAST_FunctionDef(n->v.Name.id, (params) ? params : CHECK(arguments_ty, _PyPegen_empty_arguments(p)), b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA) } - | ASYNC 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { + | ASYNC 'def' n=NAME &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { CHECK_VERSION( stmt_ty, 5, @@ -618,6 +618,7 @@ expressions[expr_ty]: expression[expr_ty] (memo): | invalid_expression + | invalid_legacy_expression | a=disjunction 'if' b=disjunction 'else' c=expression { _PyAST_IfExp(b, a, c, EXTRA) } | disjunction | lambdef @@ -642,11 +643,11 @@ star_named_expression[expr_ty]: | '*' a=bitwise_or { _PyAST_Starred(a, Load, EXTRA) } | named_expression -assigment_expression[expr_ty]: +assignment_expression[expr_ty]: | a=NAME ':=' ~ b=expression { _PyAST_NamedExpr(CHECK(expr_ty, _PyPegen_set_expr_context(p, a, Store)), b, EXTRA) } named_expression[expr_ty]: - | assigment_expression + | assignment_expression | invalid_named_expression | expression !':=' @@ -903,7 +904,7 @@ setcomp[expr_ty]: | invalid_comprehension genexp[expr_ty]: - | '(' a=( assigment_expression | expression !':=') b=for_if_clauses ')' { _PyAST_GeneratorExp(a, b, EXTRA) } + | '(' a=( assignment_expression | expression !':=') b=for_if_clauses ')' { _PyAST_GeneratorExp(a, b, EXTRA) } | invalid_comprehension dictcomp[expr_ty]: @@ -918,7 +919,7 @@ arguments[expr_ty] (memo): | invalid_arguments args[expr_ty]: - | a[asdl_expr_seq*]=','.(starred_expression | ( assigment_expression | expression !':=') !'=')+ b=[',' k=kwargs {k}] { + | a[asdl_expr_seq*]=','.(starred_expression | ( assignment_expression | expression !':=') !'=')+ b=[',' k=kwargs {k}] { _PyPegen_collect_call_seqs(p, a, b, EXTRA) } | a=kwargs { _PyAST_Call(_PyPegen_dummy_name(p), CHECK_NULL_ALLOWED(asdl_expr_seq*, _PyPegen_seq_extract_starred_exprs(p, a)), @@ -1057,11 +1058,13 @@ invalid_arguments: RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, PyPegen_last_item(b, comprehension_ty)->target, "Generator expression must be parenthesized") } | a=NAME b='=' expression for_if_clauses { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Maybe you meant '==' or ':=' instead of '='?")} - | a=args for_if_clauses { _PyPegen_nonparen_genexp_in_call(p, a) } + | a=args b=for_if_clauses { _PyPegen_nonparen_genexp_in_call(p, a, b) } | args ',' a=expression b=for_if_clauses { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, asdl_seq_GET(b, b->size-1)->target, "Generator expression must be parenthesized") } | a=args ',' args { _PyPegen_arguments_parsing_error(p, a) } invalid_kwarg: + | a[Token*]=('True'|'False'|'None') b='=' { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to %s", PyBytes_AS_STRING(a->bytes)) } | a=NAME b='=' expression for_if_clauses { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Maybe you meant '==' or ':=' instead of '='?")} | !(NAME '=') a=expression b='=' { @@ -1078,10 +1081,10 @@ invalid_legacy_expression: "Missing parentheses in call to '%U'. Did you mean %U(...)?", a->v.Name.id, a->v.Name.id) : NULL} invalid_expression: - | invalid_legacy_expression # !(NAME STRING) is not matched so we don't show this error with some invalid string prefixes like: kf"dsfsdf" # Soft keywords need to also be ignored because they can be parsed as NAME NAME | !(NAME STRING | SOFT_KEYWORD) a=disjunction b=expression_without_invalid { + _PyPegen_check_legacy_stmt(p, a) ? NULL : p->tokens[p->mark-1]->level == 0 ? NULL : RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Perhaps you forgot a comma?") } | a=disjunction 'if' b=disjunction !('else'|':') { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "expected 'else' after 'if' expression") } @@ -1138,12 +1141,16 @@ invalid_dict_comprehension: invalid_parameters: | param_no_default* invalid_parameters_helper a=param_no_default { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "non-default argument follows default argument") } + | param_no_default* a='(' param_no_default+ ','? b=')' { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "Function parameters cannot be parenthesized") } invalid_parameters_helper: # This is only there to avoid type errors | a=slash_with_default { _PyPegen_singleton_seq(p, a) } | param_with_default+ invalid_lambda_parameters: | lambda_param_no_default* invalid_lambda_parameters_helper a=lambda_param_no_default { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "non-default argument follows default argument") } + | lambda_param_no_default* a='(' ','.lambda_param+ ','? b=')' { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "Lambda expression parameters cannot be parenthesized") } invalid_lambda_parameters_helper: | a=lambda_slash_with_default { _PyPegen_singleton_seq(p, a) } | lambda_param_with_default+ @@ -1169,7 +1176,7 @@ invalid_group: | '(' a='**' expression ')' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use double starred expression here") } invalid_import_from_targets: - | import_from_as_names ',' { + | import_from_as_names ',' NEWLINE { RAISE_SYNTAX_ERROR("trailing comma not allowed without surrounding parentheses") } invalid_with_stmt: diff --git a/Include/Python.h b/Include/Python.h index a83befa3117..7260ae5cd0b 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -1,88 +1,54 @@ +// Entry point of the Python C API. +// C extensions should only #include , and not include directly +// the other Python header files included by . + #ifndef Py_PYTHON_H #define Py_PYTHON_H -/* Since this is a "meta-include" file, no #ifdef __cplusplus / extern "C" { */ -/* Include nearly all Python header files */ +// Since this is a "meta-include" file, no #ifdef __cplusplus / extern "C" { +// Include Python header files #include "patchlevel.h" #include "pyconfig.h" #include "pymacconfig.h" -#include - -#ifndef UCHAR_MAX -#error "Something's broken. UCHAR_MAX should be defined in limits.h." -#endif - -#if UCHAR_MAX != 255 -#error "Python's source code assumes C's unsigned char is an 8-bit type." -#endif - #if defined(__sgi) && !defined(_SGI_MP_SOURCE) -#define _SGI_MP_SOURCE +# define _SGI_MP_SOURCE #endif -#include -#ifndef NULL -# error "Python.h requires that stdio.h define NULL." +// stdlib.h, stdio.h, errno.h and string.h headers are not used by Python +// headers, but kept for backward compatibility. They are excluded from the +// limited C API of Python 3.11. +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# include +# include // FILE* +# include // errno +# include // memcpy() #endif - -#include -#ifdef HAVE_ERRNO_H -#include -#endif -#include #ifndef MS_WINDOWS -#include +# include #endif - -/* For size_t? */ #ifdef HAVE_STDDEF_H -#include +# include // size_t #endif -/* CAUTION: Build setups should ensure that NDEBUG is defined on the - * compiler command line when building Python in release mode; else - * assert() calls won't be removed. - */ -#include +#include // assert() +#include // wchar_t #include "pyport.h" #include "pymacro.h" - -/* A convenient way for code to know if sanitizers are enabled. */ -#if defined(__has_feature) -# if __has_feature(memory_sanitizer) -# if !defined(_Py_MEMORY_SANITIZER) -# define _Py_MEMORY_SANITIZER -# endif -# endif -# if __has_feature(address_sanitizer) -# if !defined(_Py_ADDRESS_SANITIZER) -# define _Py_ADDRESS_SANITIZER -# endif -# endif -#elif defined(__GNUC__) -# if defined(__SANITIZE_ADDRESS__) -# define _Py_ADDRESS_SANITIZER -# endif -#endif - #include "pymath.h" #include "pymem.h" - #include "object.h" #include "objimpl.h" #include "typeslots.h" #include "pyhash.h" - #include "cpython/pydebug.h" - #include "bytearrayobject.h" #include "bytesobject.h" #include "unicodeobject.h" #include "longobject.h" -#include "longintrepr.h" +#include "cpython/longintrepr.h" #include "boolobject.h" #include "floatobject.h" #include "complexobject.h" @@ -96,34 +62,30 @@ #include "setobject.h" #include "methodobject.h" #include "moduleobject.h" -#include "funcobject.h" -#include "classobject.h" +#include "cpython/funcobject.h" +#include "cpython/classobject.h" #include "fileobject.h" #include "pycapsule.h" #include "code.h" #include "pyframe.h" #include "traceback.h" #include "sliceobject.h" -#include "cellobject.h" +#include "cpython/cellobject.h" #include "iterobject.h" -#include "genobject.h" +#include "cpython/initconfig.h" +#include "pystate.h" +#include "cpython/genobject.h" #include "descrobject.h" #include "genericaliasobject.h" #include "warnings.h" #include "weakrefobject.h" #include "structseq.h" -#include "namespaceobject.h" #include "cpython/picklebufobject.h" #include "cpython/pytime.h" - #include "codecs.h" #include "pyerrors.h" - -#include "cpython/initconfig.h" #include "pythread.h" -#include "pystate.h" -#include "context.h" - +#include "cpython/context.h" #include "modsupport.h" #include "compile.h" #include "pythonrun.h" @@ -133,12 +95,8 @@ #include "osmodule.h" #include "intrcheck.h" #include "import.h" - #include "abstract.h" #include "bltinmodule.h" - -#include "eval.h" - #include "cpython/pyctype.h" #include "pystrtod.h" #include "pystrcmp.h" diff --git a/Include/abstract.h b/Include/abstract.h index 929861e4a50..9e06fbbb749 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -318,7 +318,7 @@ PyAPI_FUNC(int) PyObject_DelItem(PyObject *o, PyObject *key); /* Takes an arbitrary object which must support the (character, single segment) buffer interface and returns a pointer to a read-only memory location - useable as character based input for subsequent processing. + usable as character based input for subsequent processing. Return 0 on success. buffer and buffer_len are only set in case no error occurs. Otherwise, -1 is returned and an exception set. */ @@ -374,7 +374,7 @@ PyAPI_FUNC(PyObject *) PyObject_GetIter(PyObject *); /* Takes an AsyncIterable object and returns an AsyncIterator for it. This is typically a new iterator but if the argument is an AsyncIterator, this returns itself. */ -PyAPI_FUNC(PyObject *) PyObject_GetAiter(PyObject *); +PyAPI_FUNC(PyObject *) PyObject_GetAIter(PyObject *); /* Returns non-zero if the object 'obj' provides iterator protocols, and 0 otherwise. @@ -384,7 +384,7 @@ PyAPI_FUNC(int) PyIter_Check(PyObject *); /* Returns non-zero if the object 'obj' provides AsyncIterator protocols, and 0 otherwise. This function always succeeds. */ -PyAPI_FUNC(int) PyAiter_Check(PyObject *); +PyAPI_FUNC(int) PyAIter_Check(PyObject *); /* Takes an iterator object and calls its tp_iternext slot, returning the next value. @@ -863,7 +863,7 @@ PyAPI_FUNC(int) PyObject_IsSubclass(PyObject *object, PyObject *typeorclass); #ifndef Py_LIMITED_API # define Py_CPYTHON_ABSTRACTOBJECT_H -# include "cpython/abstract.h" +# include "cpython/abstract.h" # undef Py_CPYTHON_ABSTRACTOBJECT_H #endif diff --git a/Include/bytearrayobject.h b/Include/bytearrayobject.h index 9e95433f0f2..ae2bde1c303 100644 --- a/Include/bytearrayobject.h +++ b/Include/bytearrayobject.h @@ -6,8 +6,6 @@ extern "C" { #endif -#include - /* Type PyByteArrayObject represents a mutable array of bytes. * The Python API is that of a sequence; * the bytes are mapped to ints in [0, 256). @@ -36,7 +34,7 @@ PyAPI_FUNC(int) PyByteArray_Resize(PyObject *, Py_ssize_t); #ifndef Py_LIMITED_API # define Py_CPYTHON_BYTEARRAYOBJECT_H -# include "cpython/bytearrayobject.h" +# include "cpython/bytearrayobject.h" # undef Py_CPYTHON_BYTEARRAYOBJECT_H #endif diff --git a/Include/bytesobject.h b/Include/bytesobject.h index 39c241a2dcf..4c4dc40d705 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -7,7 +7,7 @@ extern "C" { #endif -#include +#include // va_list /* Type PyBytesObject represents a byte string. An extra zero byte is @@ -59,7 +59,7 @@ PyAPI_FUNC(int) PyBytes_AsStringAndSize( #ifndef Py_LIMITED_API # define Py_CPYTHON_BYTESOBJECT_H -# include "cpython/bytesobject.h" +# include "cpython/bytesobject.h" # undef Py_CPYTHON_BYTESOBJECT_H #endif diff --git a/Include/ceval.h b/Include/ceval.h index 0f687666e2b..1b57f6ea20f 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -1,3 +1,5 @@ +/* Interface to random parts in ceval.c */ + #ifndef Py_CEVAL_H #define Py_CEVAL_H #ifdef __cplusplus @@ -5,7 +7,15 @@ extern "C" { #endif -/* Interface to random parts in ceval.c */ +PyAPI_FUNC(PyObject *) PyEval_EvalCode(PyObject *, PyObject *, PyObject *); + +PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyObject *co, + PyObject *globals, + PyObject *locals, + PyObject *const *args, int argc, + PyObject *const *kwds, int kwdc, + PyObject *const *defs, int defc, + PyObject *kwdefs, PyObject *closure); /* PyEval_CallObjectWithKeywords(), PyEval_CallObject(), PyEval_CallFunction * and PyEval_CallMethod are deprecated. Since they are officially part of the @@ -148,7 +158,7 @@ PyAPI_FUNC(void) PyEval_ReleaseThread(PyThreadState *tstate); #ifndef Py_LIMITED_API # define Py_CPYTHON_CEVAL_H -# include "cpython/ceval.h" +# include "cpython/ceval.h" # undef Py_CPYTHON_CEVAL_H #endif diff --git a/Include/code.h b/Include/code.h index b9e23eb8165..2dea3c26105 100644 --- a/Include/code.h +++ b/Include/code.h @@ -10,7 +10,7 @@ typedef struct PyCodeObject PyCodeObject; #ifndef Py_LIMITED_API # define Py_CPYTHON_CODE_H -# include "cpython/code.h" +# include "cpython/code.h" # undef Py_CPYTHON_CODE_H #endif diff --git a/Include/cpython/abstract.h b/Include/cpython/abstract.h index db850219645..55a742c31fa 100644 --- a/Include/cpython/abstract.h +++ b/Include/cpython/abstract.h @@ -58,71 +58,13 @@ PyVectorcall_NARGS(size_t n) return n & ~PY_VECTORCALL_ARGUMENTS_OFFSET; } -static inline vectorcallfunc -PyVectorcall_Function(PyObject *callable) -{ - PyTypeObject *tp; - Py_ssize_t offset; - vectorcallfunc ptr; +PyAPI_FUNC(vectorcallfunc) PyVectorcall_Function(PyObject *callable); - assert(callable != NULL); - tp = Py_TYPE(callable); - if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) { - return NULL; - } - assert(PyCallable_Check(callable)); - offset = tp->tp_vectorcall_offset; - assert(offset > 0); - memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); - return ptr; -} - -/* Call the callable object 'callable' with the "vectorcall" calling - convention. - - args is a C array for positional arguments. - - nargsf is the number of positional arguments plus optionally the flag - PY_VECTORCALL_ARGUMENTS_OFFSET which means that the caller is allowed to - modify args[-1]. - - kwnames is a tuple of keyword names. The values of the keyword arguments - are stored in "args" after the positional arguments (note that the number - of keyword arguments does not change nargsf). kwnames can also be NULL if - there are no keyword arguments. - - keywords must only contain strings and all keys must be unique. - - Return the result on success. Raise an exception and return NULL on - error. */ -static inline PyObject * -_PyObject_VectorcallTstate(PyThreadState *tstate, PyObject *callable, - PyObject *const *args, size_t nargsf, - PyObject *kwnames) -{ - vectorcallfunc func; - PyObject *res; - - assert(kwnames == NULL || PyTuple_Check(kwnames)); - assert(args != NULL || PyVectorcall_NARGS(nargsf) == 0); - - func = PyVectorcall_Function(callable); - if (func == NULL) { - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - return _PyObject_MakeTpCall(tstate, callable, args, nargs, kwnames); - } - res = func(callable, args, nargsf, kwnames); - return _Py_CheckFunctionResult(tstate, callable, res, NULL); -} - -static inline PyObject * -PyObject_Vectorcall(PyObject *callable, PyObject *const *args, - size_t nargsf, PyObject *kwnames) -{ - PyThreadState *tstate = PyThreadState_Get(); - return _PyObject_VectorcallTstate(tstate, callable, - args, nargsf, kwnames); -} +PyAPI_FUNC(PyObject *) PyObject_Vectorcall( + PyObject *callable, + PyObject *const *args, + size_t nargsf, + PyObject *kwnames); // Backwards compatibility aliases for API that was provisional in Python 3.8 #define _PyObject_Vectorcall PyObject_Vectorcall @@ -145,44 +87,13 @@ PyAPI_FUNC(PyObject *) PyObject_VectorcallDict( "tuple" and keyword arguments "dict". "dict" may also be NULL */ PyAPI_FUNC(PyObject *) PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *dict); -static inline PyObject * -_PyObject_FastCallTstate(PyThreadState *tstate, PyObject *func, PyObject *const *args, Py_ssize_t nargs) -{ - return _PyObject_VectorcallTstate(tstate, func, args, (size_t)nargs, NULL); -} +// Same as PyObject_Vectorcall(), except without keyword arguments +PyAPI_FUNC(PyObject *) _PyObject_FastCall( + PyObject *func, + PyObject *const *args, + Py_ssize_t nargs); -/* Same as PyObject_Vectorcall except without keyword arguments */ -static inline PyObject * -_PyObject_FastCall(PyObject *func, PyObject *const *args, Py_ssize_t nargs) -{ - PyThreadState *tstate = PyThreadState_Get(); - return _PyObject_FastCallTstate(tstate, func, args, nargs); -} - -/* Call a callable without any arguments - Private static inline function variant of public function - PyObject_CallNoArgs(). */ -static inline PyObject * -_PyObject_CallNoArg(PyObject *func) { - PyThreadState *tstate = PyThreadState_Get(); - return _PyObject_VectorcallTstate(tstate, func, NULL, 0, NULL); -} - -static inline PyObject * -PyObject_CallOneArg(PyObject *func, PyObject *arg) -{ - PyObject *_args[2]; - PyObject **args; - PyThreadState *tstate; - size_t nargsf; - - assert(arg != NULL); - args = _args + 1; // For PY_VECTORCALL_ARGUMENTS_OFFSET - args[0] = arg; - tstate = PyThreadState_Get(); - nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; - return _PyObject_VectorcallTstate(tstate, func, args, nargsf, NULL); -} +PyAPI_FUNC(PyObject *) PyObject_CallOneArg(PyObject *func, PyObject *arg); PyAPI_FUNC(PyObject *) PyObject_VectorcallMethod( PyObject *name, PyObject *const *args, diff --git a/Include/cellobject.h b/Include/cpython/cellobject.h similarity index 79% rename from Include/cellobject.h rename to Include/cpython/cellobject.h index 81bc784d36f..e07f9d1de79 100644 --- a/Include/cellobject.h +++ b/Include/cpython/cellobject.h @@ -1,4 +1,5 @@ /* Cell object interface */ + #ifndef Py_LIMITED_API #ifndef Py_CELLOBJECT_H #define Py_CELLOBJECT_H @@ -8,7 +9,8 @@ extern "C" { typedef struct { PyObject_HEAD - PyObject *ob_ref; /* Content of the cell or NULL when empty */ + /* Content of the cell or NULL when empty */ + PyObject *ob_ref; } PyCellObject; PyAPI_DATA(PyTypeObject) PyCell_Type; @@ -20,7 +22,7 @@ PyAPI_FUNC(PyObject *) PyCell_Get(PyObject *); PyAPI_FUNC(int) PyCell_Set(PyObject *, PyObject *); #define PyCell_GET(op) (((PyCellObject *)(op))->ob_ref) -#define PyCell_SET(op, v) ((void)(((PyCellObject *)(op))->ob_ref = v)) +#define PyCell_SET(op, v) _Py_RVALUE(((PyCellObject *)(op))->ob_ref = (v)) #ifdef __cplusplus } diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h index 44b78f6d223..caf64401307 100644 --- a/Include/cpython/ceval.h +++ b/Include/cpython/ceval.h @@ -2,6 +2,8 @@ # error "this header file must not be included directly" #endif +PyAPI_FUNC(PyObject *) _PyEval_CallTracing(PyObject *func, PyObject *args); + PyAPI_FUNC(void) PyEval_SetProfile(Py_tracefunc, PyObject *); PyAPI_DATA(int) _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); PyAPI_FUNC(void) PyEval_SetTrace(Py_tracefunc, PyObject *); diff --git a/Include/classobject.h b/Include/cpython/classobject.h similarity index 96% rename from Include/classobject.h rename to Include/cpython/classobject.h index 1952f673b7d..80df8842eb4 100644 --- a/Include/classobject.h +++ b/Include/cpython/classobject.h @@ -53,5 +53,5 @@ PyAPI_FUNC(PyObject *) PyInstanceMethod_Function(PyObject *); #ifdef __cplusplus } #endif -#endif /* !Py_CLASSOBJECT_H */ -#endif /* Py_LIMITED_API */ +#endif // !Py_CLASSOBJECT_H +#endif // !Py_LIMITED_API diff --git a/Include/context.h b/Include/cpython/context.h similarity index 99% rename from Include/context.h rename to Include/cpython/context.h index 4e5007089dd..4db079f7633 100644 --- a/Include/context.h +++ b/Include/cpython/context.h @@ -1,12 +1,10 @@ +#ifndef Py_LIMITED_API #ifndef Py_CONTEXT_H #define Py_CONTEXT_H #ifdef __cplusplus extern "C" { #endif -#ifndef Py_LIMITED_API - - PyAPI_DATA(PyTypeObject) PyContext_Type; typedef struct _pycontextobject PyContext; @@ -73,9 +71,8 @@ PyAPI_FUNC(int) PyContextVar_Reset(PyObject *var, PyObject *token); PyAPI_FUNC(PyObject *) _PyContext_NewHamtForTests(void); -#endif /* !Py_LIMITED_API */ - #ifdef __cplusplus } #endif #endif /* !Py_CONTEXT_H */ +#endif /* !Py_LIMITED_API */ diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index de3c1160b48..e97969be4de 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -3,6 +3,7 @@ #endif typedef struct _dictkeysobject PyDictKeysObject; +typedef struct _dictvalues PyDictValues; /* The ma_values pointer is NULL for a combined table * or points to an array of PyObject* for a split table @@ -22,9 +23,9 @@ typedef struct { /* If ma_values is NULL, the table is "combined": keys and values are stored in ma_keys. - If ma_values is not NULL, the table is splitted: + If ma_values is not NULL, the table is split: keys are stored in ma_keys and values are stored in ma_values */ - PyObject **ma_values; + PyDictValues *ma_values; } PyDictObject; PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key, @@ -85,4 +86,6 @@ PyAPI_FUNC(PyObject *) _PyDictView_Intersect(PyObject* self, PyObject *other); /* Gets a version number unique to the current state of the keys of dict, if possible. * Returns the version number, or zero if it was not possible to get a version number. */ -uint32_t _PyDictKeys_GetVersionForCurrentState(PyDictObject *dict); +uint32_t _PyDictKeys_GetVersionForCurrentState(PyDictKeysObject *dictkeys); + +Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); diff --git a/Include/cpython/fileutils.h b/Include/cpython/fileutils.h index ccf37e9468d..7ab2290184a 100644 --- a/Include/cpython/fileutils.h +++ b/Include/cpython/fileutils.h @@ -135,12 +135,6 @@ PyAPI_FUNC(wchar_t*) _Py_wrealpath( size_t resolved_path_len); #endif -#ifndef MS_WINDOWS -PyAPI_FUNC(int) _Py_isabs(const wchar_t *path); -#endif - -PyAPI_FUNC(int) _Py_abspath(const wchar_t *path, wchar_t **abspath_p); - PyAPI_FUNC(wchar_t*) _Py_wgetcwd( wchar_t *buf, /* Number of characters of 'buf' buffer diff --git a/Include/cpython/floatobject.h b/Include/cpython/floatobject.h new file mode 100644 index 00000000000..fffd4686902 --- /dev/null +++ b/Include/cpython/floatobject.h @@ -0,0 +1,12 @@ +#ifndef Py_CPYTHON_FLOATOBJECT_H +# error "this header file must not be included directly" +#endif + +typedef struct { + PyObject_HEAD + double ob_fval; +} PyFloatObject; + +// Macro version of PyFloat_AsDouble() trading safety for speed. +// It doesn't check if op is a double object. +#define PyFloat_AS_DOUBLE(op) (((PyFloatObject *)(op))->ob_fval) diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index e4cfac518bb..67f98a7642b 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -12,7 +12,9 @@ struct _frame { int f_lineno; /* Current line number. Only valid if non-zero */ char f_trace_lines; /* Emit per-line trace events? */ char f_trace_opcodes; /* Emit per-opcode trace events? */ - char f_own_locals_memory; /* This frame owns the memory for the locals */ + char f_owns_frame; /* This frame owns the frame */ + /* The frame data, if this frame object owns the frame */ + PyObject *_f_frame_data[1]; }; /* Standard object interface */ @@ -26,7 +28,7 @@ PyAPI_FUNC(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *, /* only internal use */ PyFrameObject* -_PyFrame_New_NoTrack(struct _interpreter_frame *, int); +_PyFrame_New_NoTrack(PyCodeObject *code); /* The rest of the interface is specific for frame objects */ diff --git a/Include/funcobject.h b/Include/cpython/funcobject.h similarity index 96% rename from Include/funcobject.h rename to Include/cpython/funcobject.h index 6bc03f57d4c..9f0560fb725 100644 --- a/Include/funcobject.h +++ b/Include/cpython/funcobject.h @@ -1,5 +1,5 @@ - /* Function object interface */ + #ifndef Py_LIMITED_API #ifndef Py_FUNCOBJECT_H #define Py_FUNCOBJECT_H @@ -76,7 +76,6 @@ PyAPI_FUNC(int) PyFunction_SetClosure(PyObject *, PyObject *); PyAPI_FUNC(PyObject *) PyFunction_GetAnnotations(PyObject *); PyAPI_FUNC(int) PyFunction_SetAnnotations(PyObject *, PyObject *); -#ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyFunction_Vectorcall( PyObject *func, PyObject *const *stack, @@ -84,7 +83,6 @@ PyAPI_FUNC(PyObject *) _PyFunction_Vectorcall( PyObject *kwnames); uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); -#endif /* Macros for direct access to these values. Type checks are *not* done, so use with care. */ @@ -103,9 +101,6 @@ uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); #define PyFunction_GET_ANNOTATIONS(func) \ (((PyFunctionObject *)func) -> func_annotations) -#define PyFunction_AS_FRAME_CONSTRUCTOR(func) \ - ((PyFrameConstructor *)&((PyFunctionObject *)(func))->func_globals) - /* The classmethod and staticmethod types lives here, too */ PyAPI_DATA(PyTypeObject) PyClassMethod_Type; PyAPI_DATA(PyTypeObject) PyStaticMethod_Type; diff --git a/Include/genobject.h b/Include/cpython/genobject.h similarity index 77% rename from Include/genobject.h rename to Include/cpython/genobject.h index 55a8b34afd6..ad2818e8816 100644 --- a/Include/genobject.h +++ b/Include/cpython/genobject.h @@ -1,4 +1,3 @@ - /* Generator object interface */ #ifndef Py_LIMITED_API @@ -8,15 +7,13 @@ extern "C" { #endif -#include "pystate.h" /* _PyErr_StackItem */ -#include "abstract.h" /* PySendResult */ +/* --- Generators --------------------------------------------------------- */ /* _PyGenObject_HEAD defines the initial segment of generator and coroutine objects. */ #define _PyGenObject_HEAD(prefix) \ PyObject_HEAD \ /* Note: gi_frame can be NULL if the generator is "finished" */ \ - struct _interpreter_frame *prefix##_xframe; \ /* The code object backing the generator */ \ PyCodeObject *prefix##_code; \ /* List of weak reference. */ \ @@ -25,7 +22,14 @@ extern "C" { PyObject *prefix##_name; \ /* Qualified name of the generator. */ \ PyObject *prefix##_qualname; \ - _PyErr_StackItem prefix##_exc_state; + _PyErr_StackItem prefix##_exc_state; \ + PyObject *prefix##_origin_or_finalizer; \ + char prefix##_hooks_inited; \ + char prefix##_closed; \ + char prefix##_running_async; \ + /* The frame */ \ + char prefix##_frame_valid; \ + PyObject *prefix##_iframe[1]; typedef struct { /* The gi_ prefix is intended to remind of generator-iterator. */ @@ -45,10 +49,11 @@ PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); PyObject *_PyGen_yf(PyGenObject *); PyAPI_FUNC(void) _PyGen_Finalize(PyObject *self); -#ifndef Py_LIMITED_API + +/* --- PyCoroObject ------------------------------------------------------- */ + typedef struct { _PyGenObject_HEAD(cr) - PyObject *cr_origin; } PyCoroObject; PyAPI_DATA(PyTypeObject) PyCoro_Type; @@ -59,22 +64,11 @@ PyObject *_PyCoro_GetAwaitableIter(PyObject *o); PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, PyObject *name, PyObject *qualname); -/* Asynchronous Generators */ + +/* --- Asynchronous Generators -------------------------------------------- */ typedef struct { _PyGenObject_HEAD(ag) - PyObject *ag_finalizer; - - /* Flag is set to 1 when hooks set up by sys.set_asyncgen_hooks - were called on the generator, to avoid calling them more - than once. */ - int ag_hooks_inited; - - /* Flag is set to 1 when aclose() is called for the first time, or - when a StopAsyncIteration exception is raised. */ - int ag_closed; - - int ag_running_async; } PyAsyncGenObject; PyAPI_DATA(PyTypeObject) PyAsyncGen_Type; @@ -89,7 +83,6 @@ PyAPI_FUNC(PyObject *) PyAsyncGen_New(PyFrameObject *, PyObject *_PyAsyncGenValueWrapperNew(PyObject *); -#endif #undef _PyGenObject_HEAD diff --git a/Include/cpython/import.h b/Include/cpython/import.h index bad68f0e098..5ec637e7ab3 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -32,6 +32,7 @@ struct _frozen { const char *name; /* ASCII encoded string */ const unsigned char *code; int size; + PyObject *(*get_code)(void); }; /* Embedding apps may change this pointer to point to their favorite diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 5f03b8c57f8..2ba1224d9b0 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -1,6 +1,9 @@ #ifndef Py_PYCORECONFIG_H #define Py_PYCORECONFIG_H #ifndef Py_LIMITED_API +#ifdef __cplusplus +extern "C" { +#endif /* --- PyStatus ----------------------------------------------- */ @@ -140,9 +143,10 @@ typedef struct PyConfig { int faulthandler; int tracemalloc; int import_time; - int no_debug_ranges; + int code_debug_ranges; int show_ref_count; int dump_refs; + wchar_t *dump_refs_file; int malloc_stats; wchar_t *filesystem_encoding; wchar_t *filesystem_errors; @@ -171,6 +175,7 @@ typedef struct PyConfig { int legacy_windows_stdio; #endif wchar_t *check_hash_pycs_mode; + int use_frozen_modules; /* --- Path configuration inputs ------------ */ int pathconfig_warnings; @@ -182,6 +187,7 @@ typedef struct PyConfig { /* --- Path configuration outputs ----------- */ int module_search_paths_set; PyWideStringList module_search_paths; + wchar_t *stdlib_dir; wchar_t *executable; wchar_t *base_executable; wchar_t *prefix; @@ -207,6 +213,9 @@ typedef struct PyConfig { // If non-zero, disallow threads, subprocesses, and fork. // Default: 0. int _isolated_interpreter; + + // If non-zero, we believe we're running from a source tree. + int _is_python_build; } PyConfig; PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config); @@ -240,5 +249,8 @@ PyAPI_FUNC(PyStatus) PyConfig_SetWideStringList(PyConfig *config, See also PyConfig.orig_argv. */ PyAPI_FUNC(void) Py_GetArgcArgv(int *argc, wchar_t ***argv); +#ifdef __cplusplus +} +#endif #endif /* !Py_LIMITED_API */ #endif /* !Py_PYCORECONFIG_H */ diff --git a/Include/cpython/listobject.h b/Include/cpython/listobject.h index e3239152c49..51687d866ce 100644 --- a/Include/cpython/listobject.h +++ b/Include/cpython/listobject.h @@ -30,5 +30,5 @@ PyAPI_FUNC(void) _PyList_DebugMallocStats(FILE *out); #define _PyList_CAST(op) (assert(PyList_Check(op)), (PyListObject *)(op)) #define PyList_GET_ITEM(op, i) (_PyList_CAST(op)->ob_item[i]) -#define PyList_SET_ITEM(op, i, v) ((void)(_PyList_CAST(op)->ob_item[i] = (v))) +#define PyList_SET_ITEM(op, i, v) _Py_RVALUE(_PyList_CAST(op)->ob_item[i] = (v)) #define PyList_GET_SIZE(op) Py_SIZE(_PyList_CAST(op)) diff --git a/Include/longintrepr.h b/Include/cpython/longintrepr.h similarity index 100% rename from Include/longintrepr.h rename to Include/cpython/longintrepr.h diff --git a/Include/cpython/longobject.h b/Include/cpython/longobject.h new file mode 100644 index 00000000000..1a73799d658 --- /dev/null +++ b/Include/cpython/longobject.h @@ -0,0 +1,95 @@ +#ifndef Py_CPYTHON_LONGOBJECT_H +# error "this header file must not be included directly" +#endif + +PyAPI_FUNC(int) _PyLong_AsInt(PyObject *); + +PyAPI_FUNC(int) _PyLong_UnsignedShort_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UnsignedInt_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UnsignedLong_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UnsignedLongLong_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *); + +/* _PyLong_Frexp returns a double x and an exponent e such that the + true value is approximately equal to x * 2**e. e is >= 0. x is + 0.0 if and only if the input is 0 (in which case, e and x are both + zeroes); otherwise, 0.5 <= abs(x) < 1.0. On overflow, which is + possible if the number of bits doesn't fit into a Py_ssize_t, sets + OverflowError and returns -1.0 for x, 0 for e. */ +PyAPI_FUNC(double) _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e); + +PyAPI_FUNC(PyObject *) PyLong_FromUnicodeObject(PyObject *u, int base); +PyAPI_FUNC(PyObject *) _PyLong_FromBytes(const char *, Py_ssize_t, int); + +/* _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. + v must not be NULL, and must be a normalized long. + There are no error cases. +*/ +PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); + +/* _PyLong_NumBits. Return the number of bits needed to represent the + absolute value of a long. For example, this returns 1 for 1 and -1, 2 + for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. + v must not be NULL, and must be a normalized long. + (size_t)-1 is returned and OverflowError set if the true result doesn't + fit in a size_t. +*/ +PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v); + +/* _PyLong_DivmodNear. Given integers a and b, compute the nearest + integer q to the exact quotient a / b, rounding to the nearest even integer + in the case of a tie. Return (q, r), where r = a - q*b. The remainder r + will satisfy abs(r) <= abs(b)/2, with equality possible only if q is + even. +*/ +PyAPI_FUNC(PyObject *) _PyLong_DivmodNear(PyObject *, PyObject *); + +/* _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in + base 256, and return a Python int with the same numeric value. + If n is 0, the integer is 0. Else: + If little_endian is 1/true, bytes[n-1] is the MSB and bytes[0] the LSB; + else (little_endian is 0/false) bytes[0] is the MSB and bytes[n-1] the + LSB. + If is_signed is 0/false, view the bytes as a non-negative integer. + If is_signed is 1/true, view the bytes as a 2's-complement integer, + non-negative if bit 0x80 of the MSB is clear, negative if set. + Error returns: + + Return NULL with the appropriate exception set if there's not + enough memory to create the Python int. +*/ +PyAPI_FUNC(PyObject *) _PyLong_FromByteArray( + const unsigned char* bytes, size_t n, + int little_endian, int is_signed); + +/* _PyLong_AsByteArray: Convert the least-significant 8*n bits of long + v to a base-256 integer, stored in array bytes. Normally return 0, + return -1 on error. + If little_endian is 1/true, store the MSB at bytes[n-1] and the LSB at + bytes[0]; else (little_endian is 0/false) store the MSB at bytes[0] and + the LSB at bytes[n-1]. + If is_signed is 0/false, it's an error if v < 0; else (v >= 0) n bytes + are filled and there's nothing special about bit 0x80 of the MSB. + If is_signed is 1/true, bytes is filled with the 2's-complement + representation of v's value. Bit 0x80 of the MSB is the sign bit. + Error returns (-1): + + is_signed is 0 and v < 0. TypeError is set in this case, and bytes + isn't altered. + + n isn't big enough to hold the full mathematical value of v. For + example, if is_signed is 0 and there are more digits in the v than + fit in n; or if is_signed is 1, v < 0, and n is just 1 bit shy of + being large enough to hold a sign bit. OverflowError is set in this + case, but bytes holds the least-significant n bytes of the true value. +*/ +PyAPI_FUNC(int) _PyLong_AsByteArray(PyLongObject* v, + unsigned char* bytes, size_t n, + int little_endian, int is_signed); + +/* _PyLong_Format: Convert the long to a string object with given base, + appending a base prefix of 0[box] if base is 2, 8 or 16. */ +PyAPI_FUNC(PyObject *) _PyLong_Format(PyObject *obj, int base); + +/* For use by the gcd function in mathmodule.c */ +PyAPI_FUNC(PyObject *) _PyLong_GCD(PyObject *, PyObject *); + +PyAPI_FUNC(PyObject *) _PyLong_Rshift(PyObject *, size_t); +PyAPI_FUNC(PyObject *) _PyLong_Lshift(PyObject *, size_t); diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 75cd0f90022..0c3957aff4b 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -289,6 +289,7 @@ typedef struct _heaptypeobject { PyObject *ht_name, *ht_slots, *ht_qualname; struct _dictkeysobject *ht_cached_keys; PyObject *ht_module; + char *_ht_tpname; // Storage for "tp_name"; see PyType_FromModuleAndSpec /* here are optional user slots, followed by the members. */ } PyHeapTypeObject; @@ -534,7 +535,16 @@ PyAPI_FUNC(int) _PyTrash_cond(PyObject *op, destructor dealloc); Py_TRASHCAN_BEGIN_CONDITION(op, \ _PyTrash_cond(_PyObject_CAST(op), (destructor)dealloc)) -/* For backwards compatibility, these macros enable the trashcan - * unconditionally */ -#define Py_TRASHCAN_SAFE_BEGIN(op) Py_TRASHCAN_BEGIN_CONDITION(op, 1) -#define Py_TRASHCAN_SAFE_END(op) Py_TRASHCAN_END +/* The following two macros, Py_TRASHCAN_SAFE_BEGIN and + * Py_TRASHCAN_SAFE_END, are deprecated since version 3.11 and + * will be removed in the future. + * Use Py_TRASHCAN_BEGIN and Py_TRASHCAN_END instead. + */ +Py_DEPRECATED(3.11) typedef int UsingDeprecatedTrashcanMacro; +#define Py_TRASHCAN_SAFE_BEGIN(op) \ + do { \ + UsingDeprecatedTrashcanMacro cond=1; \ + Py_TRASHCAN_BEGIN_CONDITION(op, cond); +#define Py_TRASHCAN_SAFE_END(op) \ + Py_TRASHCAN_END; \ + } while(0); diff --git a/Include/cpython/objimpl.h b/Include/cpython/objimpl.h index d83700e2a46..4a905c25cc8 100644 --- a/Include/cpython/objimpl.h +++ b/Include/cpython/objimpl.h @@ -90,9 +90,6 @@ PyAPI_FUNC(int) PyObject_IS_GC(PyObject *obj); # define _PyGC_FINALIZED(o) PyObject_GC_IsFinalized(o) #endif -PyAPI_FUNC(PyObject *) _PyObject_GC_Malloc(size_t size); -PyAPI_FUNC(PyObject *) _PyObject_GC_Calloc(size_t size); - /* Test if a type supports weak references */ #define PyType_SUPPORTS_WEAKREFS(t) ((t)->tp_weaklistoffset > 0) diff --git a/Include/cpython/pydebug.h b/Include/cpython/pydebug.h index 78bcb118be4..cab799f0b38 100644 --- a/Include/cpython/pydebug.h +++ b/Include/cpython/pydebug.h @@ -29,7 +29,7 @@ PyAPI_DATA(int) Py_LegacyWindowsStdioFlag; /* this is a wrapper around getenv() that pays attention to Py_IgnoreEnvironmentFlag. It should be used for getting variables like PYTHONPATH and PYTHONHOME from the environment */ -#define Py_GETENV(s) (Py_IgnoreEnvironmentFlag ? NULL : getenv(s)) +PyAPI_DATA(char*) Py_GETENV(const char *name); #ifdef __cplusplus } diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index a3ec5afdb7c..5281fde1f1a 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -6,7 +6,7 @@ /* PyException_HEAD defines the initial segment of every exception class. */ #define PyException_HEAD PyObject_HEAD PyObject *dict;\ - PyObject *args; PyObject *traceback;\ + PyObject *args; PyObject *note; PyObject *traceback;\ PyObject *context; PyObject *cause;\ char suppress_context; @@ -14,6 +14,12 @@ typedef struct { PyException_HEAD } PyBaseExceptionObject; +typedef struct { + PyException_HEAD + PyObject *msg; + PyObject *excs; +} PyBaseExceptionGroupObject; + typedef struct { PyException_HEAD PyObject *msg; @@ -143,6 +149,11 @@ PyAPI_FUNC(PyObject *) PyErr_ProgramTextObject( PyObject *filename, int lineno); +PyAPI_FUNC(PyObject *) _PyErr_ProgramDecodedTextObject( + PyObject *filename, + int lineno, + const char* encoding); + PyAPI_FUNC(PyObject *) _PyUnicodeTranslateError_Create( PyObject *object, Py_ssize_t start, diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index ab4bf8bf848..aa518281c80 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -2,8 +2,6 @@ # error "this header file must not be included directly" #endif -#include "cpython/initconfig.h" - PyAPI_FUNC(int) _PyInterpreterState_RequiresIDRef(PyInterpreterState *); PyAPI_FUNC(void) _PyInterpreterState_RequireIDRef(PyInterpreterState *, int); @@ -46,6 +44,8 @@ typedef struct _cframe { * accessed outside of their lifetime. */ int use_tracing; + /* Pointer to the currently executing frame (it can be NULL) */ + struct _interpreter_frame *current_frame; struct _cframe *previous; } CFrame; @@ -77,11 +77,15 @@ struct _ts { struct _ts *next; PyInterpreterState *interp; - /* Borrowed reference to the current frame (it can be NULL) */ - struct _interpreter_frame *frame; - int recursion_depth; + /* Has been initialized to a safe state. + + In order to be effective, this must be set to 0 during or right + after allocation. */ + int _initialized; + + int recursion_remaining; + int recursion_limit; int recursion_headroom; /* Allow 50 more calls to handle any errors. */ - int stackcheck_counter; /* 'tracing' keeps track of the execution depth when tracing/profiling. This is to prevent the actual trace/profile code from being recorded in @@ -185,6 +189,13 @@ PyAPI_FUNC(PyThreadState *) _PyThreadState_UncheckedGet(void); PyAPI_FUNC(PyObject *) _PyThreadState_GetDict(PyThreadState *tstate); +// Disable tracing and profiling. +PyAPI_FUNC(void) PyThreadState_EnterTracing(PyThreadState *tstate); + +// Reset tracing and profiling: enable them if a trace function or a profile +// function is set, otherwise disable them. +PyAPI_FUNC(void) PyThreadState_LeaveTracing(PyThreadState *tstate); + /* PyGILState */ /* Helper/diagnostic function - return 1 if the current thread @@ -263,7 +274,7 @@ PyAPI_FUNC(int) _PyInterpreterState_GetConfigCopy( PyAPI_FUNC(int) _PyInterpreterState_SetConfig( const struct PyConfig *config); -// Get the configuration of the currrent interpreter. +// Get the configuration of the current interpreter. // The caller must hold the GIL. PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); diff --git a/Include/cpython/pytime.h b/Include/cpython/pytime.h index 56607d199ed..23d4f16a8fd 100644 --- a/Include/cpython/pytime.h +++ b/Include/cpython/pytime.h @@ -1,3 +1,46 @@ +// The _PyTime_t API is written to use timestamp and timeout values stored in +// various formats and to read clocks. +// +// The _PyTime_t type is an integer to support directly common arithmetic +// operations like t1 + t2. +// +// The _PyTime_t API supports a resolution of 1 nanosecond. The _PyTime_t type +// is signed to support negative timestamps. The supported range is around +// [-292.3 years; +292.3 years]. Using the Unix epoch (January 1st, 1970), the +// supported date range is around [1677-09-21; 2262-04-11]. +// +// Formats: +// +// * seconds +// * seconds as a floating pointer number (C double) +// * milliseconds (10^-3 seconds) +// * microseconds (10^-6 seconds) +// * 100 nanoseconds (10^-7 seconds) +// * nanoseconds (10^-9 seconds) +// * timeval structure, 1 microsecond resolution (10^-6 seconds) +// * timespec structure, 1 nanosecond resolution (10^-9 seconds) +// +// Integer overflows are detected and raise OverflowError. Conversion to a +// resolution worse than 1 nanosecond is rounded correctly with the requested +// rounding mode. There are 4 rounding modes: floor (towards -inf), ceiling +// (towards +inf), half even and up (away from zero). +// +// Some functions clamp the result in the range [_PyTime_MIN; _PyTime_MAX], so +// the caller doesn't have to handle errors and doesn't need to hold the GIL. +// For example, _PyTime_Add(t1, t2) computes t1+t2 and clamp the result on +// overflow. +// +// Clocks: +// +// * System clock +// * Monotonic clock +// * Performance counter +// +// Operations like (t * k / q) with integers are implemented in a way to reduce +// the risk of integer overflow. Such operation is used to convert a clock +// value expressed in ticks with a frequency to _PyTime_t, like +// QueryPerformanceCounter() with QueryPerformanceFrequency(). + #ifndef Py_LIMITED_API #ifndef Py_PYTIME_H #define Py_PYTIME_H @@ -14,8 +57,11 @@ extern "C" { store a duration, and so indirectly a date (related to another date, like UNIX epoch). */ typedef int64_t _PyTime_t; +// _PyTime_MIN nanoseconds is around -292.3 years #define _PyTime_MIN INT64_MIN +// _PyTime_MAX nanoseconds is around +292.3 years #define _PyTime_MAX INT64_MAX +#define _SIZEOF_PYTIME_T 8 typedef enum { /* Round towards minus infinity (-inf). @@ -88,13 +134,13 @@ PyAPI_FUNC(_PyTime_t) _PyTime_FromNanoseconds(_PyTime_t ns); PyAPI_FUNC(int) _PyTime_FromNanosecondsObject(_PyTime_t *t, PyObject *obj); -/* Convert a number of seconds (Python float or int) to a timetamp. +/* Convert a number of seconds (Python float or int) to a timestamp. Raise an exception and return -1 on error, return 0 on success. */ PyAPI_FUNC(int) _PyTime_FromSecondsObject(_PyTime_t *t, PyObject *obj, _PyTime_round_t round); -/* Convert a number of milliseconds (Python float or int, 10^-3) to a timetamp. +/* Convert a number of milliseconds (Python float or int, 10^-3) to a timestamp. Raise an exception and return -1 on error, return 0 on success. */ PyAPI_FUNC(int) _PyTime_FromMillisecondsObject(_PyTime_t *t, PyObject *obj, @@ -111,13 +157,24 @@ PyAPI_FUNC(_PyTime_t) _PyTime_AsMilliseconds(_PyTime_t t, PyAPI_FUNC(_PyTime_t) _PyTime_AsMicroseconds(_PyTime_t t, _PyTime_round_t round); +/* Convert timestamp to a number of nanoseconds (10^-9 seconds). */ +PyAPI_FUNC(_PyTime_t) _PyTime_AsNanoseconds(_PyTime_t t); + +#ifdef MS_WINDOWS +// Convert timestamp to a number of 100 nanoseconds (10^-7 seconds). +PyAPI_FUNC(_PyTime_t) _PyTime_As100Nanoseconds(_PyTime_t t, + _PyTime_round_t round); +#endif + /* Convert timestamp to a number of nanoseconds (10^-9 seconds) as a Python int object. */ PyAPI_FUNC(PyObject *) _PyTime_AsNanosecondsObject(_PyTime_t t); +#ifndef MS_WINDOWS /* Create a timestamp from a timeval structure. Raise an exception and return -1 on overflow, return 0 on success. */ PyAPI_FUNC(int) _PyTime_FromTimeval(_PyTime_t *tp, struct timeval *tv); +#endif /* Convert a timestamp to a timeval structure (microsecond resolution). tv_usec is always positive. @@ -127,8 +184,9 @@ PyAPI_FUNC(int) _PyTime_AsTimeval(_PyTime_t t, struct timeval *tv, _PyTime_round_t round); -/* Similar to _PyTime_AsTimeval(), but don't raise an exception on error. */ -PyAPI_FUNC(int) _PyTime_AsTimeval_noraise(_PyTime_t t, +/* Similar to _PyTime_AsTimeval() but don't raise an exception on overflow. + On overflow, clamp tv_sec to _PyTime_t min/max. */ +PyAPI_FUNC(void) _PyTime_AsTimeval_clamp(_PyTime_t t, struct timeval *tv, _PyTime_round_t round); @@ -153,9 +211,18 @@ PyAPI_FUNC(int) _PyTime_FromTimespec(_PyTime_t *tp, struct timespec *ts); tv_nsec is always positive. Raise an exception and return -1 on error, return 0 on success. */ PyAPI_FUNC(int) _PyTime_AsTimespec(_PyTime_t t, struct timespec *ts); + +/* Similar to _PyTime_AsTimespec() but don't raise an exception on overflow. + On overflow, clamp tv_sec to _PyTime_t min/max. */ +PyAPI_FUNC(void) _PyTime_AsTimespec_clamp(_PyTime_t t, struct timespec *ts); #endif + +// Compute t1 + t2. Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. +PyAPI_FUNC(_PyTime_t) _PyTime_Add(_PyTime_t t1, _PyTime_t t2); + /* Compute ticks * mul / div. + Clamp to [_PyTime_MIN; _PyTime_MAX] on overflow. The caller must ensure that ((div - 1) * mul) cannot overflow. */ PyAPI_FUNC(_PyTime_t) _PyTime_MulDiv(_PyTime_t ticks, _PyTime_t mul, @@ -172,8 +239,8 @@ typedef struct { /* Get the current time from the system clock. If the internal clock fails, silently ignore the error and return 0. - On integer overflow, silently ignore the overflow and truncated the clock to - _PyTime_MIN or _PyTime_MAX. + On integer overflow, silently ignore the overflow and clamp the clock to + [_PyTime_MIN; _PyTime_MAX]. Use _PyTime_GetSystemClockWithInfo() to check for failure. */ PyAPI_FUNC(_PyTime_t) _PyTime_GetSystemClock(void); @@ -192,8 +259,8 @@ PyAPI_FUNC(int) _PyTime_GetSystemClockWithInfo( results of consecutive calls is valid. If the internal clock fails, silently ignore the error and return 0. - On integer overflow, silently ignore the overflow and truncated the clock to - _PyTime_MIN or _PyTime_MAX. + On integer overflow, silently ignore the overflow and clamp the clock to + [_PyTime_MIN; _PyTime_MAX]. Use _PyTime_GetMonotonicClockWithInfo() to check for failure. */ PyAPI_FUNC(_PyTime_t) _PyTime_GetMonotonicClock(void); @@ -223,8 +290,8 @@ PyAPI_FUNC(int) _PyTime_gmtime(time_t t, struct tm *tm); measure a short duration. If the internal clock fails, silently ignore the error and return 0. - On integer overflow, silently ignore the overflow and truncated the clock to - _PyTime_MIN or _PyTime_MAX. + On integer overflow, silently ignore the overflow and clamp the clock to + [_PyTime_MIN; _PyTime_MAX]. Use _PyTime_GetPerfCounterWithInfo() to check for failure. */ PyAPI_FUNC(_PyTime_t) _PyTime_GetPerfCounter(void); @@ -239,6 +306,15 @@ PyAPI_FUNC(int) _PyTime_GetPerfCounterWithInfo( _PyTime_t *t, _Py_clock_info_t *info); + +// Create a deadline. +// Pseudo code: _PyTime_GetMonotonicClock() + timeout. +PyAPI_FUNC(_PyTime_t) _PyDeadline_Init(_PyTime_t timeout); + +// Get remaining time from a deadline. +// Pseudo code: deadline - _PyTime_GetMonotonicClock(). +PyAPI_FUNC(_PyTime_t) _PyDeadline_Get(_PyTime_t deadline); + #ifdef __cplusplus } #endif diff --git a/Include/cpython/tupleobject.h b/Include/cpython/tupleobject.h index 7cada8848c4..fc37c4e6de1 100644 --- a/Include/cpython/tupleobject.h +++ b/Include/cpython/tupleobject.h @@ -23,6 +23,6 @@ PyAPI_FUNC(void) _PyTuple_MaybeUntrack(PyObject *); #define PyTuple_GET_ITEM(op, i) (_PyTuple_CAST(op)->ob_item[i]) /* Macro, *only* to be used to fill in brand new tuples */ -#define PyTuple_SET_ITEM(op, i, v) ((void)(_PyTuple_CAST(op)->ob_item[i] = v)) +#define PyTuple_SET_ITEM(op, i, v) _Py_RVALUE(_PyTuple_CAST(op)->ob_item[i] = (v)) PyAPI_FUNC(void) _PyTuple_DebugMallocStats(FILE *out); diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 95b76cecd6f..be5647c7d26 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -50,19 +50,6 @@ Py_UNICODE_ISDIGIT(ch) || \ Py_UNICODE_ISNUMERIC(ch)) -Py_DEPRECATED(3.3) static inline void -Py_UNICODE_COPY(Py_UNICODE *target, const Py_UNICODE *source, Py_ssize_t length) { - memcpy(target, source, (size_t)(length) * sizeof(Py_UNICODE)); -} - -Py_DEPRECATED(3.3) static inline void -Py_UNICODE_FILL(Py_UNICODE *target, Py_UNICODE value, Py_ssize_t length) { - Py_ssize_t i; - for (i = 0; i < length; i++) { - target[i] = value; - } -} - /* macros to work with surrogates */ #define Py_UNICODE_IS_SURROGATE(ch) (0xD800 <= (ch) && (ch) <= 0xDFFF) #define Py_UNICODE_IS_HIGH_SURROGATE(ch) (0xD800 <= (ch) && (ch) <= 0xDBFF) @@ -292,6 +279,10 @@ PyAPI_FUNC(int) _PyUnicode_CheckConsistency( #define SSTATE_INTERNED_MORTAL 1 #define SSTATE_INTERNED_IMMORTAL 2 +/* Use only if you know it's a string */ +#define PyUnicode_CHECK_INTERNED(op) \ + (((PyASCIIObject *)(op))->state.interned) + /* Return true if the string contains only ASCII characters, or 0 if not. The string may be compact (PyUnicode_IS_COMPACT_ASCII) or not, but must be ready. */ @@ -416,7 +407,7 @@ enum PyUnicode_Kind { /* Fast check to determine whether an object is ready. Equivalent to - PyUnicode_IS_COMPACT(op) || ((PyUnicodeObject*)(op))->data.any) */ + PyUnicode_IS_COMPACT(op) || ((PyUnicodeObject*)(op))->data.any */ #define PyUnicode_IS_READY(op) (((PyASCIIObject*)op)->state.ready) @@ -790,17 +781,35 @@ PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF16( /* --- Unicode-Escape Codecs ---------------------------------------------- */ -/* Helper for PyUnicode_DecodeUnicodeEscape that detects invalid escape - chars. */ -PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscape( +/* Variant of PyUnicode_DecodeUnicodeEscape that supports partial decoding. */ +PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeStateful( const char *string, /* Unicode-Escape encoded string */ Py_ssize_t length, /* size of string */ const char *errors, /* error handling */ + Py_ssize_t *consumed /* bytes consumed */ +); +/* Helper for PyUnicode_DecodeUnicodeEscape that detects invalid escape + chars. */ +PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeInternal( + const char *string, /* Unicode-Escape encoded string */ + Py_ssize_t length, /* size of string */ + const char *errors, /* error handling */ + Py_ssize_t *consumed, /* bytes consumed */ const char **first_invalid_escape /* on return, points to first invalid escaped char in string. */ ); +/* --- Raw-Unicode-Escape Codecs ---------------------------------------------- */ + +/* Variant of PyUnicode_DecodeRawUnicodeEscape that supports partial decoding. */ +PyAPI_FUNC(PyObject*) _PyUnicode_DecodeRawUnicodeEscapeStateful( + const char *string, /* Unicode-Escape encoded string */ + Py_ssize_t length, /* size of string */ + const char *errors, /* error handling */ + Py_ssize_t *consumed /* bytes consumed */ +); + /* --- Latin-1 Codecs ----------------------------------------------------- */ PyAPI_FUNC(PyObject*) _PyUnicode_AsLatin1String( @@ -1011,6 +1020,9 @@ PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*); and where the hash values are equal (i.e. a very probable match) */ PyAPI_FUNC(int) _PyUnicode_EQ(PyObject *, PyObject *); +/* Equality check. Returns -1 on failure. */ +PyAPI_FUNC(int) _PyUnicode_Equal(PyObject *, PyObject *); + PyAPI_FUNC(int) _PyUnicode_WideCharString_Converter(PyObject *, void *); PyAPI_FUNC(int) _PyUnicode_WideCharString_Opt_Converter(PyObject *, void *); diff --git a/Include/cpython/warnings.h b/Include/cpython/warnings.h new file mode 100644 index 00000000000..2ef8e3ce943 --- /dev/null +++ b/Include/cpython/warnings.h @@ -0,0 +1,20 @@ +#ifndef Py_CPYTHON_WARNINGS_H +# error "this header file must not be included directly" +#endif + +PyAPI_FUNC(int) PyErr_WarnExplicitObject( + PyObject *category, + PyObject *message, + PyObject *filename, + int lineno, + PyObject *module, + PyObject *registry); + +PyAPI_FUNC(int) PyErr_WarnExplicitFormat( + PyObject *category, + const char *filename, int lineno, + const char *module, PyObject *registry, + const char *format, ...); + +// DEPRECATED: Use PyErr_WarnEx() instead. +#define PyErr_Warn(category, msg) PyErr_WarnEx(category, msg, 1) diff --git a/Include/cpython/weakrefobject.h b/Include/cpython/weakrefobject.h new file mode 100644 index 00000000000..9efcc412df9 --- /dev/null +++ b/Include/cpython/weakrefobject.h @@ -0,0 +1,47 @@ +#ifndef Py_CPYTHON_WEAKREFOBJECT_H +# error "this header file must not be included directly" +#endif + +/* PyWeakReference is the base struct for the Python ReferenceType, ProxyType, + * and CallableProxyType. + */ +struct _PyWeakReference { + PyObject_HEAD + + /* The object to which this is a weak reference, or Py_None if none. + * Note that this is a stealth reference: wr_object's refcount is + * not incremented to reflect this pointer. + */ + PyObject *wr_object; + + /* A callable to invoke when wr_object dies, or NULL if none. */ + PyObject *wr_callback; + + /* A cache for wr_object's hash code. As usual for hashes, this is -1 + * if the hash code isn't known yet. + */ + Py_hash_t hash; + + /* If wr_object is weakly referenced, wr_object has a doubly-linked NULL- + * terminated list of weak references to it. These are the list pointers. + * If wr_object goes away, wr_object is set to Py_None, and these pointers + * have no meaning then. + */ + PyWeakReference *wr_prev; + PyWeakReference *wr_next; +}; + +PyAPI_FUNC(Py_ssize_t) _PyWeakref_GetWeakrefCount(PyWeakReference *head); + +PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); + +/* Explanation for the Py_REFCNT() check: when a weakref's target is part + of a long chain of deallocations which triggers the trashcan mechanism, + clearing the weakrefs can be delayed long after the target's refcount + has dropped to zero. In the meantime, code accessing the weakref will + be able to "see" the target object even though it is supposed to be + unreachable. See issue #16602. */ +#define PyWeakref_GET_OBJECT(ref) \ + (Py_REFCNT(((PyWeakReference *)(ref))->wr_object) > 0 \ + ? ((PyWeakReference *)(ref))->wr_object \ + : Py_None) diff --git a/Include/dictobject.h b/Include/dictobject.h index da5a36ba07f..a6233d8ae25 100644 --- a/Include/dictobject.h +++ b/Include/dictobject.h @@ -87,7 +87,7 @@ PyAPI_DATA(PyTypeObject) PyDictRevIterValue_Type; #ifndef Py_LIMITED_API # define Py_CPYTHON_DICTOBJECT_H -# include "cpython/dictobject.h" +# include "cpython/dictobject.h" # undef Py_CPYTHON_DICTOBJECT_H #endif diff --git a/Include/eval.h b/Include/eval.h deleted file mode 100644 index eda28df8f65..00000000000 --- a/Include/eval.h +++ /dev/null @@ -1,27 +0,0 @@ - -/* Interface to execute compiled code */ - -#ifndef Py_EVAL_H -#define Py_EVAL_H -#ifdef __cplusplus -extern "C" { -#endif - -PyAPI_FUNC(PyObject *) PyEval_EvalCode(PyObject *, PyObject *, PyObject *); - -PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyObject *co, - PyObject *globals, - PyObject *locals, - PyObject *const *args, int argc, - PyObject *const *kwds, int kwdc, - PyObject *const *defs, int defc, - PyObject *kwdefs, PyObject *closure); - -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyEval_CallTracing(PyObject *func, PyObject *args); -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_EVAL_H */ diff --git a/Include/fileobject.h b/Include/fileobject.h index 6ec2994aa85..4c983e7b5da 100644 --- a/Include/fileobject.h +++ b/Include/fileobject.h @@ -39,7 +39,7 @@ PyAPI_DATA(int) Py_UTF8Mode; #ifndef Py_LIMITED_API # define Py_CPYTHON_FILEOBJECT_H -# include "cpython/fileobject.h" +# include "cpython/fileobject.h" # undef Py_CPYTHON_FILEOBJECT_H #endif diff --git a/Include/fileutils.h b/Include/fileutils.h index 16f3b635dee..ba5acc84fcb 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -16,7 +16,7 @@ PyAPI_FUNC(char*) Py_EncodeLocale( #ifndef Py_LIMITED_API # define Py_CPYTHON_FILEUTILS_H -# include "cpython/fileutils.h" +# include "cpython/fileutils.h" # undef Py_CPYTHON_FILEUTILS_H #endif diff --git a/Include/floatobject.h b/Include/floatobject.h index e994aa8f29d..3b6ca478eae 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -11,107 +11,45 @@ PyFloatObject represents a (double precision) floating point number. extern "C" { #endif -#ifndef Py_LIMITED_API -typedef struct { - PyObject_HEAD - double ob_fval; -} PyFloatObject; -#endif - PyAPI_DATA(PyTypeObject) PyFloat_Type; #define PyFloat_Check(op) PyObject_TypeCheck(op, &PyFloat_Type) #define PyFloat_CheckExact(op) Py_IS_TYPE(op, &PyFloat_Type) #ifdef Py_NAN -#define Py_RETURN_NAN return PyFloat_FromDouble(Py_NAN) +# define Py_RETURN_NAN return PyFloat_FromDouble(Py_NAN) #endif -#define Py_RETURN_INF(sign) do \ - if (copysign(1., sign) == 1.) { \ - return PyFloat_FromDouble(Py_HUGE_VAL); \ - } else { \ - return PyFloat_FromDouble(-Py_HUGE_VAL); \ +#define Py_RETURN_INF(sign) \ + do { \ + if (copysign(1., sign) == 1.) { \ + return PyFloat_FromDouble(Py_HUGE_VAL); \ + } \ + else { \ + return PyFloat_FromDouble(-Py_HUGE_VAL); \ + } \ } while(0) PyAPI_FUNC(double) PyFloat_GetMax(void); PyAPI_FUNC(double) PyFloat_GetMin(void); -PyAPI_FUNC(PyObject *) PyFloat_GetInfo(void); +PyAPI_FUNC(PyObject*) PyFloat_GetInfo(void); /* Return Python float from string PyObject. */ -PyAPI_FUNC(PyObject *) PyFloat_FromString(PyObject*); +PyAPI_FUNC(PyObject*) PyFloat_FromString(PyObject*); /* Return Python float from C double. */ -PyAPI_FUNC(PyObject *) PyFloat_FromDouble(double); +PyAPI_FUNC(PyObject*) PyFloat_FromDouble(double); /* Extract C double from Python float. The macro version trades safety for speed. */ -PyAPI_FUNC(double) PyFloat_AsDouble(PyObject *); +PyAPI_FUNC(double) PyFloat_AsDouble(PyObject*); + #ifndef Py_LIMITED_API -#define PyFloat_AS_DOUBLE(op) (((PyFloatObject *)(op))->ob_fval) +# define Py_CPYTHON_FLOATOBJECT_H +# include "cpython/floatobject.h" +# undef Py_CPYTHON_FLOATOBJECT_H #endif -#ifndef Py_LIMITED_API -/* _PyFloat_{Pack,Unpack}{4,8} - * - * The struct and pickle (at least) modules need an efficient platform- - * independent way to store floating-point values as byte strings. - * The Pack routines produce a string from a C double, and the Unpack - * routines produce a C double from such a string. The suffix (4 or 8) - * specifies the number of bytes in the string. - * - * On platforms that appear to use (see _PyFloat_Init()) IEEE-754 formats - * these functions work by copying bits. On other platforms, the formats the - * 4- byte format is identical to the IEEE-754 single precision format, and - * the 8-byte format to the IEEE-754 double precision format, although the - * packing of INFs and NaNs (if such things exist on the platform) isn't - * handled correctly, and attempting to unpack a string containing an IEEE - * INF or NaN will raise an exception. - * - * On non-IEEE platforms with more precision, or larger dynamic range, than - * 754 supports, not all values can be packed; on non-IEEE platforms with less - * precision, or smaller dynamic range, not all values can be unpacked. What - * happens in such cases is partly accidental (alas). - */ - -/* The pack routines write 2, 4 or 8 bytes, starting at p. le is a bool - * argument, true if you want the string in little-endian format (exponent - * last, at p+1, p+3 or p+7), false if you want big-endian format (exponent - * first, at p). - * Return value: 0 if all is OK, -1 if error (and an exception is - * set, most likely OverflowError). - * There are two problems on non-IEEE platforms: - * 1): What this does is undefined if x is a NaN or infinity. - * 2): -0.0 and +0.0 produce the same string. - */ -PyAPI_FUNC(int) _PyFloat_Pack2(double x, unsigned char *p, int le); -PyAPI_FUNC(int) _PyFloat_Pack4(double x, unsigned char *p, int le); -PyAPI_FUNC(int) _PyFloat_Pack8(double x, unsigned char *p, int le); - -/* The unpack routines read 2, 4 or 8 bytes, starting at p. le is a bool - * argument, true if the string is in little-endian format (exponent - * last, at p+1, p+3 or p+7), false if big-endian (exponent first, at p). - * Return value: The unpacked double. On error, this is -1.0 and - * PyErr_Occurred() is true (and an exception is set, most likely - * OverflowError). Note that on a non-IEEE platform this will refuse - * to unpack a string that represents a NaN or infinity. - */ -PyAPI_FUNC(double) _PyFloat_Unpack2(const unsigned char *p, int le); -PyAPI_FUNC(double) _PyFloat_Unpack4(const unsigned char *p, int le); -PyAPI_FUNC(double) _PyFloat_Unpack8(const unsigned char *p, int le); - -PyAPI_FUNC(void) _PyFloat_DebugMallocStats(FILE* out); - -/* Format the object based on the format_spec, as defined in PEP 3101 - (Advanced String Formatting). */ -PyAPI_FUNC(int) _PyFloat_FormatAdvancedWriter( - _PyUnicodeWriter *writer, - PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); -#endif /* Py_LIMITED_API */ - #ifdef __cplusplus } #endif diff --git a/Include/frameobject.h b/Include/frameobject.h index c118af1201a..adb628f6314 100644 --- a/Include/frameobject.h +++ b/Include/frameobject.h @@ -10,7 +10,7 @@ extern "C" { #ifndef Py_LIMITED_API # define Py_CPYTHON_FRAMEOBJECT_H -# include "cpython/frameobject.h" +# include "cpython/frameobject.h" # undef Py_CPYTHON_FRAMEOBJECT_H #endif diff --git a/Include/import.h b/Include/import.h index aeef3efd0bc..a87677bb10c 100644 --- a/Include/import.h +++ b/Include/import.h @@ -88,7 +88,7 @@ PyAPI_FUNC(int) PyImport_AppendInittab( #ifndef Py_LIMITED_API # define Py_CPYTHON_IMPORT_H -# include "cpython/import.h" +# include "cpython/import.h" # undef Py_CPYTHON_IMPORT_H #endif diff --git a/Include/internal/pycore_abstract.h b/Include/internal/pycore_abstract.h index b791bf24321..b1afb2dc7be 100644 --- a/Include/internal/pycore_abstract.h +++ b/Include/internal/pycore_abstract.h @@ -16,6 +16,9 @@ _PyIndex_Check(PyObject *obj) return (tp_as_number != NULL && tp_as_number->nb_index != NULL); } +PyObject *_PyNumber_PowerNoMod(PyObject *lhs, PyObject *rhs); +PyObject *_PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_asdl.h b/Include/internal/pycore_asdl.h index c0b07c31810..5b01c7a6659 100644 --- a/Include/internal/pycore_asdl.h +++ b/Include/internal/pycore_asdl.h @@ -78,9 +78,9 @@ asdl_ ## NAME ## _seq *_Py_asdl_ ## NAME ## _seq_new(Py_ssize_t size, PyArena *a return seq; \ } -#define asdl_seq_GET_UNTYPED(S, I) (S)->elements[(I)] -#define asdl_seq_GET(S, I) (S)->typed_elements[(I)] -#define asdl_seq_LEN(S) ((S) == NULL ? 0 : (S)->size) +#define asdl_seq_GET_UNTYPED(S, I) _Py_RVALUE((S)->elements[(I)]) +#define asdl_seq_GET(S, I) _Py_RVALUE((S)->typed_elements[(I)]) +#define asdl_seq_LEN(S) _Py_RVALUE(((S) == NULL ? 0 : (S)->size)) #ifdef Py_DEBUG # define asdl_seq_SET(S, I, V) \ @@ -91,7 +91,7 @@ asdl_ ## NAME ## _seq *_Py_asdl_ ## NAME ## _seq_new(Py_ssize_t size, PyArena *a (S)->typed_elements[_asdl_i] = (V); \ } while (0) #else -# define asdl_seq_SET(S, I, V) (S)->typed_elements[I] = (V) +# define asdl_seq_SET(S, I, V) _Py_RVALUE((S)->typed_elements[I] = (V)) #endif #ifdef Py_DEBUG @@ -103,7 +103,7 @@ asdl_ ## NAME ## _seq *_Py_asdl_ ## NAME ## _seq_new(Py_ssize_t size, PyArena *a (S)->elements[_asdl_i] = (V); \ } while (0) #else -# define asdl_seq_SET_UNTYPED(S, I, V) (S)->elements[I] = (V) +# define asdl_seq_SET_UNTYPED(S, I, V) _Py_RVALUE((S)->elements[I] = (V)) #endif #ifdef __cplusplus diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h new file mode 100644 index 00000000000..b00ed9784ef --- /dev/null +++ b/Include/internal/pycore_bytesobject.h @@ -0,0 +1,30 @@ +#ifndef Py_INTERNAL_BYTESOBJECT_H +#define Py_INTERNAL_BYTESOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +/* runtime lifecycle */ + +extern PyStatus _PyBytes_InitGlobalObjects(PyInterpreterState *); +extern PyStatus _PyBytes_InitTypes(PyInterpreterState *); +extern void _PyBytes_Fini(PyInterpreterState *); + + +/* other API */ + +struct _Py_bytes_state { + PyObject *empty_string; + PyBytesObject *characters[256]; +}; + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_BYTESOBJECT_H */ diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index f7d856a54bd..f2cfd2fd53e 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -8,6 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_pystate.h" // _PyThreadState_GET() + PyAPI_FUNC(PyObject *) _PyObject_Call_Prepend( PyThreadState *tstate, PyObject *callable, @@ -28,11 +30,88 @@ PyAPI_FUNC(PyObject *) _PyObject_Call( PyObject *args, PyObject *kwargs); + +// Static inline variant of public PyVectorcall_Function(). +static inline vectorcallfunc +_PyVectorcall_FunctionInline(PyObject *callable) +{ + assert(callable != NULL); + + PyTypeObject *tp = Py_TYPE(callable); + if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) { + return NULL; + } + assert(PyCallable_Check(callable)); + + Py_ssize_t offset = tp->tp_vectorcall_offset; + assert(offset > 0); + + vectorcallfunc ptr; + memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); + return ptr; +} + + +/* Call the callable object 'callable' with the "vectorcall" calling + convention. + + args is a C array for positional arguments. + + nargsf is the number of positional arguments plus optionally the flag + PY_VECTORCALL_ARGUMENTS_OFFSET which means that the caller is allowed to + modify args[-1]. + + kwnames is a tuple of keyword names. The values of the keyword arguments + are stored in "args" after the positional arguments (note that the number + of keyword arguments does not change nargsf). kwnames can also be NULL if + there are no keyword arguments. + + keywords must only contain strings and all keys must be unique. + + Return the result on success. Raise an exception and return NULL on + error. */ static inline PyObject * -_PyObject_CallNoArgTstate(PyThreadState *tstate, PyObject *func) { +_PyObject_VectorcallTstate(PyThreadState *tstate, PyObject *callable, + PyObject *const *args, size_t nargsf, + PyObject *kwnames) +{ + vectorcallfunc func; + PyObject *res; + + assert(kwnames == NULL || PyTuple_Check(kwnames)); + assert(args != NULL || PyVectorcall_NARGS(nargsf) == 0); + + func = _PyVectorcall_FunctionInline(callable); + if (func == NULL) { + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + return _PyObject_MakeTpCall(tstate, callable, args, nargs, kwnames); + } + res = func(callable, args, nargsf, kwnames); + return _Py_CheckFunctionResult(tstate, callable, res, NULL); +} + + +static inline PyObject * +_PyObject_CallNoArgsTstate(PyThreadState *tstate, PyObject *func) { return _PyObject_VectorcallTstate(tstate, func, NULL, 0, NULL); } + +// Private static inline function variant of public PyObject_CallNoArgs() +static inline PyObject * +_PyObject_CallNoArgs(PyObject *func) { + PyThreadState *tstate = _PyThreadState_GET(); + return _PyObject_VectorcallTstate(tstate, func, NULL, 0, NULL); +} + + +static inline PyObject * +_PyObject_FastCallTstate(PyThreadState *tstate, PyObject *func, PyObject *const *args, Py_ssize_t nargs) +{ + return _PyObject_VectorcallTstate(tstate, func, args, (size_t)nargs, NULL); +} + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 66ddc991a9b..20508d4a687 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -12,11 +12,12 @@ extern "C" { struct pyruntimestate; struct _ceval_runtime_state; -#include "pycore_interp.h" /* PyInterpreterState.eval_frame */ +#include "pycore_interp.h" // PyInterpreterState.eval_frame +#include "pycore_pystate.h" // _PyThreadState_GET() extern void _Py_FinishPendingCalls(PyThreadState *tstate); extern void _PyEval_InitRuntimeState(struct _ceval_runtime_state *); -extern int _PyEval_InitState(struct _ceval_state *ceval); +extern void _PyEval_InitState(struct _ceval_state *, PyThread_type_lock); extern void _PyEval_FiniState(struct _ceval_state *ceval); PyAPI_FUNC(void) _PyEval_SignalReceived(PyInterpreterState *interp); PyAPI_FUNC(int) _PyEval_AddPendingCall( @@ -43,12 +44,15 @@ extern PyObject *_PyEval_BuiltinsFromGlobals( static inline PyObject* _PyEval_EvalFrame(PyThreadState *tstate, struct _interpreter_frame *frame, int throwflag) { + if (tstate->interp->eval_frame == NULL) { + return _PyEval_EvalFrameDefault(tstate, frame, throwflag); + } return tstate->interp->eval_frame(tstate, frame, throwflag); } extern PyObject * _PyEval_Vector(PyThreadState *tstate, - PyFrameConstructor *desc, PyObject *locals, + PyFunctionObject *func, PyObject *locals, PyObject* const* args, size_t argcount, PyObject *kwnames); @@ -71,12 +75,12 @@ extern void _PyEval_DeactivateOpCache(void); /* With USE_STACKCHECK macro defined, trigger stack checks in _Py_CheckRecursiveCall() on every 64th call to Py_EnterRecursiveCall. */ static inline int _Py_MakeRecCheck(PyThreadState *tstate) { - return (++tstate->recursion_depth > tstate->interp->ceval.recursion_limit - || ++tstate->stackcheck_counter > 64); + return (tstate->recursion_remaining-- <= 0 + || (tstate->recursion_remaining & 63) == 0); } #else static inline int _Py_MakeRecCheck(PyThreadState *tstate) { - return (++tstate->recursion_depth > tstate->interp->ceval.recursion_limit); + return tstate->recursion_remaining-- <= 0; } #endif @@ -90,18 +94,18 @@ static inline int _Py_EnterRecursiveCall(PyThreadState *tstate, } static inline int _Py_EnterRecursiveCall_inline(const char *where) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); return _Py_EnterRecursiveCall(tstate, where); } #define Py_EnterRecursiveCall(where) _Py_EnterRecursiveCall_inline(where) static inline void _Py_LeaveRecursiveCall(PyThreadState *tstate) { - tstate->recursion_depth--; + tstate->recursion_remaining++; } static inline void _Py_LeaveRecursiveCall_inline(void) { - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = _PyThreadState_GET(); _Py_LeaveRecursiveCall(tstate); } @@ -109,7 +113,7 @@ static inline void _Py_LeaveRecursiveCall_inline(void) { struct _interpreter_frame *_PyEval_GetFrame(void); -PyObject *_Py_MakeCoro(PyFrameConstructor *, struct _interpreter_frame *); +PyObject *_Py_MakeCoro(PyFunctionObject *func); #ifdef __cplusplus } diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 282089c0837..496d52f580f 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -17,6 +17,7 @@ typedef struct { uint8_t original_oparg; uint8_t counter; uint16_t index; + uint32_t version; } _PyAdaptiveEntry; @@ -30,6 +31,18 @@ typedef struct { uint32_t builtin_keys_version; } _PyLoadGlobalCache; +typedef struct { + /* Borrowed ref in LOAD_METHOD */ + PyObject *obj; +} _PyObjectCache; + +typedef struct { + uint32_t func_version; + uint16_t defaults_start; + uint16_t defaults_len; +} _PyCallCache; + + /* Add specialized versions of entries to this union. * * Do not break the invariant: sizeof(SpecializedCacheEntry) == 8 @@ -45,6 +58,8 @@ typedef union { _PyAdaptiveEntry adaptive; _PyAttrCache attr; _PyLoadGlobalCache load_global; + _PyObjectCache obj; + _PyCallCache call; } SpecializedCacheEntry; #define INSTRUCTIONS_PER_ENTRY (sizeof(SpecializedCacheEntry)/sizeof(_Py_CODEUNIT)) @@ -123,23 +138,24 @@ _GetSpecializedCacheEntryForInstruction(const _Py_CODEUNIT *first_instr, int nex #define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY) #define QUICKENING_WARMUP_COLDEST 1 -static inline void -PyCodeObject_IncrementWarmup(PyCodeObject * co) -{ - co->co_warmup++; -} - -/* Used by the interpreter to determine when a code object should be quickened */ -static inline int -PyCodeObject_IsWarmedUp(PyCodeObject * co) -{ - return (co->co_warmup == 0); -} - int _Py_Quicken(PyCodeObject *code); -extern Py_ssize_t _Py_QuickenedCount; +/* Returns 1 if quickening occurs. + * -1 if an error occurs + * 0 otherwise */ +static inline int +_Py_IncrementCountAndMaybeQuicken(PyCodeObject *code) +{ + if (code->co_warmup != 0) { + code->co_warmup++; + if (code->co_warmup == 0) { + return _Py_Quicken(code) ? -1 : 1; + } + } + return 0; +} +extern Py_ssize_t _Py_QuickenedCount; /* "Locals plus" for a code object is the set of locals + cell vars + * free vars. This relates to variable names as well as offsets into @@ -240,53 +256,6 @@ PyAPI_FUNC(PyObject *) _PyCode_GetVarnames(PyCodeObject *); PyAPI_FUNC(PyObject *) _PyCode_GetCellvars(PyCodeObject *); PyAPI_FUNC(PyObject *) _PyCode_GetFreevars(PyCodeObject *); - -/* Cache hits and misses */ - -static inline uint8_t -saturating_increment(uint8_t c) -{ - return c<<1; -} - -static inline uint8_t -saturating_decrement(uint8_t c) -{ - return (c>>1) + 128; -} - -static inline uint8_t -saturating_zero(void) -{ - return 255; -} - -/* Starting value for saturating counter. - * Technically this should be 1, but that is likely to - * cause a bit of thrashing when we optimize then get an immediate miss. - * We want to give the counter a change to stabilize, so we start at 3. - */ -static inline uint8_t -saturating_start(void) -{ - return saturating_zero()<<3; -} - -static inline void -record_cache_hit(_PyAdaptiveEntry *entry) { - entry->counter = saturating_increment(entry->counter); -} - -static inline void -record_cache_miss(_PyAdaptiveEntry *entry) { - entry->counter = saturating_decrement(entry->counter); -} - -static inline int -too_many_cache_misses(_PyAdaptiveEntry *entry) { - return entry->counter == saturating_zero(); -} - #define ADAPTIVE_CACHE_BACKOFF 64 static inline void @@ -299,7 +268,13 @@ cache_backoff(_PyAdaptiveEntry *entry) { int _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache); int _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache); int _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache); -int _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr); +int _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache); +int _Py_Specialize_BinarySubscr(PyObject *sub, PyObject *container, _Py_CODEUNIT *instr, SpecializedCacheEntry *cache); +int _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr); +int _Py_Specialize_CallFunction(PyObject *callable, _Py_CODEUNIT *instr, int nargs, SpecializedCacheEntry *cache, PyObject *builtins); +void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, + SpecializedCacheEntry *cache); +void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, SpecializedCacheEntry *cache); #define PRINT_SPECIALIZATION_STATS 0 #define PRINT_SPECIALIZATION_STATS_DETAILED 0 diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index a482dd42122..31ca0a43fae 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -7,6 +7,32 @@ #include "pycore_hamt.h" /* PyHamtObject */ + +/* runtime lifecycle */ + +PyStatus _PyContext_InitTypes(PyInterpreterState *); +void _PyContext_Fini(PyInterpreterState *); + + +/* other API */ + +#ifndef WITH_FREELISTS +// without freelists +# define PyContext_MAXFREELIST 0 +#endif + +#ifndef PyContext_MAXFREELIST +# define PyContext_MAXFREELIST 255 +#endif + +struct _Py_context_state { +#if PyContext_MAXFREELIST > 0 + // List of free PyContext objects + PyContext *freelist; + int numfree; +#endif +}; + struct _pycontextobject { PyObject_HEAD PyContext *ctx_prev; @@ -36,7 +62,4 @@ struct _pycontexttokenobject { }; -int _PyContext_Init(void); -void _PyContext_Fini(PyInterpreterState *interp); - #endif /* !Py_INTERNAL_CONTEXT_H */ diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 2becc30beb4..faa8bb49bb7 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -10,6 +10,32 @@ extern "C" { #endif +/* runtime lifecycle */ + +extern void _PyDict_Fini(PyInterpreterState *interp); + + +/* other API */ + +#ifndef WITH_FREELISTS +// without freelists +# define PyDict_MAXFREELIST 0 +#endif + +#ifndef PyDict_MAXFREELIST +# define PyDict_MAXFREELIST 80 +#endif + +struct _Py_dict_state { +#if PyDict_MAXFREELIST > 0 + /* Dictionary reuse scheme to save calls to malloc and free */ + PyDictObject *free_list[PyDict_MAXFREELIST]; + int numfree; + PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST]; + int keys_numfree; +#endif +}; + typedef struct { /* Cached hash code of me_key. */ Py_hash_t me_hash; @@ -22,6 +48,8 @@ typedef struct { */ Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); +/* Consumes references to key and value */ +int _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); #define DKIX_EMPTY (-1) #define DKIX_DUMMY (-2) /* Used internally */ @@ -71,6 +99,14 @@ struct _dictkeysobject { see the DK_ENTRIES() macro */ }; +/* This must be no more than 16, for the order vector to fit in 64 bits */ +#define SHARED_KEYS_MAX_SIZE 16 + +struct _dictvalues { + uint64_t mv_order; + PyObject *values[1]; +}; + #define DK_LOG_SIZE(dk) ((dk)->dk_log2_size) #if SIZEOF_VOID_P > 4 #define DK_SIZE(dk) (((int64_t)1)< /* struct lconv */ +// This is used after getting NULL back from Py_DecodeLocale(). +#define DECODE_LOCALE_ERR(NAME, LEN) \ + ((LEN) == (size_t)-2) \ + ? _PyStatus_ERR("cannot decode " NAME) \ + : _PyStatus_NO_MEMORY() + PyAPI_DATA(int) _Py_HasFileSystemDefaultEncodeErrors; PyAPI_FUNC(int) _Py_DecodeUTF8Ex( @@ -33,6 +39,9 @@ PyAPI_FUNC(wchar_t*) _Py_DecodeUTF8_surrogateescape( Py_ssize_t arglen, size_t *wlen); +extern int +_Py_wstat(const wchar_t *, struct stat *); + PyAPI_FUNC(int) _Py_GetForceASCII(void); /* Reset "force ASCII" mode (if it was initialized). @@ -65,6 +74,39 @@ extern int _Py_EncodeNonUnicodeWchar_InPlace( Py_ssize_t size); #endif +extern int _Py_isabs(const wchar_t *path); +extern int _Py_abspath(const wchar_t *path, wchar_t **abspath_p); +extern wchar_t * _Py_join_relfile(const wchar_t *dirname, + const wchar_t *relfile); +extern int _Py_add_relfile(wchar_t *dirname, + const wchar_t *relfile, + size_t bufsize); +extern size_t _Py_find_basename(const wchar_t *filename); +PyAPI_FUNC(wchar_t *) _Py_normpath(wchar_t *path, Py_ssize_t size); + + +// Macros to protect CRT calls against instant termination when passed an +// invalid parameter (bpo-23524). IPH stands for Invalid Parameter Handler. +// Usage: +// +// _Py_BEGIN_SUPPRESS_IPH +// ... +// _Py_END_SUPPRESS_IPH +#if defined _MSC_VER && _MSC_VER >= 1900 + +# include // _set_thread_local_invalid_parameter_handler() + + extern _invalid_parameter_handler _Py_silent_invalid_parameter_handler; +# define _Py_BEGIN_SUPPRESS_IPH \ + { _invalid_parameter_handler _Py_old_handler = \ + _set_thread_local_invalid_parameter_handler(_Py_silent_invalid_parameter_handler); +# define _Py_END_SUPPRESS_IPH \ + _set_thread_local_invalid_parameter_handler(_Py_old_handler); } +#else +# define _Py_BEGIN_SUPPRESS_IPH +# define _Py_END_SUPPRESS_IPH +#endif /* _MSC_VER >= 1900 */ + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h new file mode 100644 index 00000000000..be6045587de --- /dev/null +++ b/Include/internal/pycore_floatobject.h @@ -0,0 +1,104 @@ +#ifndef Py_INTERNAL_FLOATOBJECT_H +#define Py_INTERNAL_FLOATOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +/* runtime lifecycle */ + +extern void _PyFloat_InitState(PyInterpreterState *); +extern PyStatus _PyFloat_InitTypes(PyInterpreterState *); +extern void _PyFloat_Fini(PyInterpreterState *); + + +/* other API */ + +#ifndef WITH_FREELISTS +// without freelists +# define PyFloat_MAXFREELIST 0 +#endif + +#ifndef PyFloat_MAXFREELIST +# define PyFloat_MAXFREELIST 100 +#endif + +struct _Py_float_state { +#if PyFloat_MAXFREELIST > 0 + /* Special free list + free_list is a singly-linked list of available PyFloatObjects, + linked via abuse of their ob_type members. */ + int numfree; + PyFloatObject *free_list; +#endif +}; + +/* _PyFloat_{Pack,Unpack}{4,8} + * + * The struct and pickle (at least) modules need an efficient platform- + * independent way to store floating-point values as byte strings. + * The Pack routines produce a string from a C double, and the Unpack + * routines produce a C double from such a string. The suffix (4 or 8) + * specifies the number of bytes in the string. + * + * On platforms that appear to use (see _PyFloat_Init()) IEEE-754 formats + * these functions work by copying bits. On other platforms, the formats the + * 4- byte format is identical to the IEEE-754 single precision format, and + * the 8-byte format to the IEEE-754 double precision format, although the + * packing of INFs and NaNs (if such things exist on the platform) isn't + * handled correctly, and attempting to unpack a string containing an IEEE + * INF or NaN will raise an exception. + * + * On non-IEEE platforms with more precision, or larger dynamic range, than + * 754 supports, not all values can be packed; on non-IEEE platforms with less + * precision, or smaller dynamic range, not all values can be unpacked. What + * happens in such cases is partly accidental (alas). + */ + +/* The pack routines write 2, 4 or 8 bytes, starting at p. le is a bool + * argument, true if you want the string in little-endian format (exponent + * last, at p+1, p+3 or p+7), false if you want big-endian format (exponent + * first, at p). + * Return value: 0 if all is OK, -1 if error (and an exception is + * set, most likely OverflowError). + * There are two problems on non-IEEE platforms: + * 1): What this does is undefined if x is a NaN or infinity. + * 2): -0.0 and +0.0 produce the same string. + */ +PyAPI_FUNC(int) _PyFloat_Pack2(double x, unsigned char *p, int le); +PyAPI_FUNC(int) _PyFloat_Pack4(double x, unsigned char *p, int le); +PyAPI_FUNC(int) _PyFloat_Pack8(double x, unsigned char *p, int le); + +/* The unpack routines read 2, 4 or 8 bytes, starting at p. le is a bool + * argument, true if the string is in little-endian format (exponent + * last, at p+1, p+3 or p+7), false if big-endian (exponent first, at p). + * Return value: The unpacked double. On error, this is -1.0 and + * PyErr_Occurred() is true (and an exception is set, most likely + * OverflowError). Note that on a non-IEEE platform this will refuse + * to unpack a string that represents a NaN or infinity. + */ +PyAPI_FUNC(double) _PyFloat_Unpack2(const unsigned char *p, int le); +PyAPI_FUNC(double) _PyFloat_Unpack4(const unsigned char *p, int le); +PyAPI_FUNC(double) _PyFloat_Unpack8(const unsigned char *p, int le); + + +PyAPI_FUNC(void) _PyFloat_DebugMallocStats(FILE* out); + + +/* Format the object based on the format_spec, as defined in PEP 3101 + (Advanced String Formatting). */ +PyAPI_FUNC(int) _PyFloat_FormatAdvancedWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_FLOATOBJECT_H */ diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index cd465d73bc6..a55877b55fb 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -4,6 +4,14 @@ extern "C" { #endif + +/* runtime lifecycle */ + +extern void _PyFrame_Fini(PyInterpreterState *interp); + + +/* other API */ + /* These values are chosen so that the inline functions below all * compare f_state to zero. */ @@ -19,20 +27,25 @@ enum _framestate { typedef signed char PyFrameState; +/* + frame->f_lasti refers to the index of the last instruction, + unless it's -1 in which case next_instr should be first_instr. +*/ + typedef struct _interpreter_frame { - PyObject *f_globals; - PyObject *f_builtins; - PyObject *f_locals; - PyCodeObject *f_code; - PyFrameObject *frame_obj; - /* Borrowed reference to a generator, or NULL */ - PyObject *generator; + PyFunctionObject *f_func; /* Strong reference */ + PyObject *f_globals; /* Borrowed reference */ + PyObject *f_builtins; /* Borrowed reference */ + PyObject *f_locals; /* Strong reference, may be NULL */ + PyCodeObject *f_code; /* Strong reference */ + PyFrameObject *frame_obj; /* Strong reference, may be NULL */ + PyObject *generator; /* Borrowed reference, may be NULL */ struct _interpreter_frame *previous; int f_lasti; /* Last instruction if called */ - int stackdepth; /* Depth of value stack */ - int nlocalsplus; - PyFrameState f_state; /* What state the frame is in */ - PyObject *stack[1]; + int stacktop; /* Offset of TOS from localsplus */ + PyFrameState f_state; /* What state the frame is in */ + int depth; /* Depth of the frame in a ceval loop */ + PyObject *localsplus[1]; } InterpreterFrame; static inline int _PyFrame_IsRunnable(InterpreterFrame *f) { @@ -47,26 +60,47 @@ static inline int _PyFrameHasCompleted(InterpreterFrame *f) { return f->f_state > FRAME_EXECUTING; } +static inline PyObject **_PyFrame_Stackbase(InterpreterFrame *f) { + return f->localsplus + f->f_code->co_nlocalsplus; +} + +static inline PyObject *_PyFrame_StackPeek(InterpreterFrame *f) { + assert(f->stacktop > f->f_code->co_nlocalsplus); + return f->localsplus[f->stacktop-1]; +} + +static inline PyObject *_PyFrame_StackPop(InterpreterFrame *f) { + assert(f->stacktop > f->f_code->co_nlocalsplus); + f->stacktop--; + return f->localsplus[f->stacktop]; +} + +static inline void _PyFrame_StackPush(InterpreterFrame *f, PyObject *value) { + f->localsplus[f->stacktop] = value; + f->stacktop++; +} + #define FRAME_SPECIALS_SIZE ((sizeof(InterpreterFrame)-1)/sizeof(PyObject *)) -InterpreterFrame * -_PyInterpreterFrame_HeapAlloc(PyFrameConstructor *con, PyObject *locals); +void _PyFrame_Copy(InterpreterFrame *src, InterpreterFrame *dest); static inline void _PyFrame_InitializeSpecials( - InterpreterFrame *frame, PyFrameConstructor *con, + InterpreterFrame *frame, PyFunctionObject *func, PyObject *locals, int nlocalsplus) { - frame->f_code = (PyCodeObject *)Py_NewRef(con->fc_code); - frame->f_builtins = Py_NewRef(con->fc_builtins); - frame->f_globals = Py_NewRef(con->fc_globals); + Py_INCREF(func); + frame->f_func = func; + frame->f_code = (PyCodeObject *)Py_NewRef(func->func_code); + frame->f_builtins = func->func_builtins; + frame->f_globals = func->func_globals; frame->f_locals = Py_XNewRef(locals); - frame->nlocalsplus = nlocalsplus; - frame->stackdepth = 0; + frame->stacktop = nlocalsplus; frame->frame_obj = NULL; frame->generator = NULL; frame->f_lasti = -1; frame->f_state = FRAME_CREATED; + frame->depth = 0; } /* Gets the pointer to the locals array @@ -75,7 +109,19 @@ _PyFrame_InitializeSpecials( static inline PyObject** _PyFrame_GetLocalsArray(InterpreterFrame *frame) { - return ((PyObject **)frame) - frame->nlocalsplus; + return frame->localsplus; +} + +static inline PyObject** +_PyFrame_GetStackPointer(InterpreterFrame *frame) +{ + return frame->localsplus+frame->stacktop; +} + +static inline void +_PyFrame_SetStackPointer(InterpreterFrame *frame, PyObject **stack_pointer) +{ + frame->stacktop = (int)(stack_pointer - frame->localsplus); } /* For use by _PyFrame_GetFrameObject @@ -98,15 +144,15 @@ _PyFrame_GetFrameObject(InterpreterFrame *frame) /* Clears all references in the frame. * If take is non-zero, then the InterpreterFrame frame - * may be transfered to the frame object it references + * may be transferred to the frame object it references * instead of being cleared. Either way * the caller no longer owns the references * in the frame. * take should be set to 1 for heap allocated * frames like the ones in generators and coroutines. */ -int -_PyFrame_Clear(InterpreterFrame * frame, int take); +void +_PyFrame_Clear(InterpreterFrame * frame); int _PyFrame_Traverse(InterpreterFrame *frame, visitproc visit, void *arg); @@ -118,7 +164,22 @@ void _PyFrame_LocalsToFast(InterpreterFrame *frame, int clear); InterpreterFrame *_PyThreadState_PushFrame( - PyThreadState *tstate, PyFrameConstructor *con, PyObject *locals); + PyThreadState *tstate, PyFunctionObject *func, PyObject *locals); + +extern InterpreterFrame * +_PyThreadState_BumpFramePointerSlow(PyThreadState *tstate, size_t size); + +static inline InterpreterFrame * +_PyThreadState_BumpFramePointer(PyThreadState *tstate, size_t size) +{ + PyObject **base = tstate->datastack_top; + PyObject **top = base + size; + if (top < tstate->datastack_limit) { + tstate->datastack_top = top; + return (InterpreterFrame *)base; + } + return _PyThreadState_BumpFramePointerSlow(tstate, size); +} void _PyThreadState_PopFrame(PyThreadState *tstate, InterpreterFrame *frame); diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h new file mode 100644 index 00000000000..dc4422df3eb --- /dev/null +++ b/Include/internal/pycore_function.h @@ -0,0 +1,11 @@ +#ifndef Py_INTERNAL_FUNCTION_H +#define Py_INTERNAL_FUNCTION_H + + +#include "Python.h" + +PyFunctionObject * +_PyFunction_FromConstructor(PyFrameConstructor *constr); + + +#endif /* !Py_INTERNAL_FUNCTION_H */ diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 9db4a4716fa..a23dca80549 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -43,7 +43,7 @@ typedef struct { // Lowest bit of _gc_next is used for flags only in GC. // But it is always 0 for normal code. #define _PyGCHead_NEXT(g) ((PyGC_Head*)(g)->_gc_next) -#define _PyGCHead_SET_NEXT(g, p) ((g)->_gc_next = (uintptr_t)(p)) +#define _PyGCHead_SET_NEXT(g, p) _Py_RVALUE((g)->_gc_next = (uintptr_t)(p)) // Lowest two bits of _gc_prev is used for _PyGC_PREV_MASK_* flags. #define _PyGCHead_PREV(g) ((PyGC_Head*)((g)->_gc_prev & _PyGC_PREV_MASK)) @@ -56,7 +56,7 @@ typedef struct { #define _PyGCHead_FINALIZED(g) \ (((g)->_gc_prev & _PyGC_PREV_MASK_FINALIZED) != 0) #define _PyGCHead_SET_FINALIZED(g) \ - ((g)->_gc_prev |= _PyGC_PREV_MASK_FINALIZED) + _Py_RVALUE((g)->_gc_prev |= _PyGC_PREV_MASK_FINALIZED) #define _PyGC_FINALIZED(o) \ _PyGCHead_FINALIZED(_Py_AS_GC(o)) @@ -167,7 +167,6 @@ extern Py_ssize_t _PyGC_CollectNoFail(PyThreadState *tstate); // Functions to clear types free lists -extern void _PyFrame_ClearFreeList(PyInterpreterState *interp); extern void _PyTuple_ClearFreeList(PyInterpreterState *interp); extern void _PyFloat_ClearFreeList(PyInterpreterState *interp); extern void _PyList_ClearFreeList(PyInterpreterState *interp); diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h new file mode 100644 index 00000000000..74a676df4ad --- /dev/null +++ b/Include/internal/pycore_genobject.h @@ -0,0 +1,46 @@ +#ifndef Py_INTERNAL_GENOBJECT_H +#define Py_INTERNAL_GENOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +/* runtime lifecycle */ + +extern void _PyAsyncGen_Fini(PyInterpreterState *); + + +/* other API */ + +#ifndef WITH_FREELISTS +// without freelists +# define _PyAsyncGen_MAXFREELIST 0 +#endif + +#ifndef _PyAsyncGen_MAXFREELIST +# define _PyAsyncGen_MAXFREELIST 80 +#endif + +struct _Py_async_gen_state { +#if _PyAsyncGen_MAXFREELIST > 0 + /* Freelists boost performance 6-10%; they also reduce memory + fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend + are short-living objects that are instantiated for every + __anext__() call. */ + struct _PyAsyncGenWrappedValue* value_freelist[_PyAsyncGen_MAXFREELIST]; + int value_numfree; + + struct PyAsyncGenASend* asend_freelist[_PyAsyncGen_MAXFREELIST]; + int asend_numfree; +#endif +}; + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_GENOBJECT_H */ diff --git a/Include/internal/pycore_hamt.h b/Include/internal/pycore_hamt.h index aaf65590955..cf9c19e022d 100644 --- a/Include/internal/pycore_hamt.h +++ b/Include/internal/pycore_hamt.h @@ -8,6 +8,14 @@ #define _Py_HAMT_MAX_TREE_DEPTH 7 +/* runtime lifecycle */ + +PyStatus _PyHamt_InitTypes(PyInterpreterState *); +void _PyHamt_Fini(PyInterpreterState *); + + +/* other API */ + #define PyHamt_Check(o) Py_IS_TYPE(o, &_PyHamt_Type) @@ -110,7 +118,4 @@ PyObject * _PyHamt_NewIterValues(PyHamtObject *o); /* Return a Items iterator over "o". */ PyObject * _PyHamt_NewIterItems(PyHamtObject *o); -int _PyHamt_Init(void); -void _PyHamt_Fini(void); - #endif /* !Py_INTERNAL_HAMT_H */ diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index e21ed0a7a06..aee1f66a3ea 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -10,6 +10,16 @@ extern PyStatus _PyImport_ReInitLock(void); #endif extern PyObject* _PyImport_BootstrapImp(PyThreadState *tstate); +struct _module_alias { + const char *name; /* ASCII encoded string */ + const char *orig; /* ASCII encoded string */ +}; + +PyAPI_DATA(const struct _frozen *) _PyImport_FrozenBootstrap; +PyAPI_DATA(const struct _frozen *) _PyImport_FrozenStdlib; +PyAPI_DATA(const struct _frozen *) _PyImport_FrozenTest; +extern const struct _module_alias * _PyImport_FrozenAliases; + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 4b009e816b4..a2f4cd182e8 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -155,6 +155,7 @@ extern PyStatus _PyConfig_Copy( extern PyStatus _PyConfig_InitPathConfig( PyConfig *config, int compute_path_config); +extern PyStatus _PyConfig_InitImportConfig(PyConfig *config); extern PyStatus _PyConfig_Read(PyConfig *config, int compute_path_config); extern PyStatus _PyConfig_Write(const PyConfig *config, struct pyruntimestate *runtime); @@ -165,6 +166,10 @@ extern PyStatus _PyConfig_SetPyArgv( PyAPI_FUNC(PyObject*) _PyConfig_AsDict(const PyConfig *config); PyAPI_FUNC(int) _PyConfig_FromDict(PyConfig *config, PyObject *dict); +extern void _Py_DumpPathConfig(PyThreadState *tstate); + +PyAPI_FUNC(PyObject*) _Py_Get_Getpath_CodeObject(void); + /* --- Function used for testing ---------------------------------- */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index bfd082b5882..e4d7b1b8752 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -10,8 +10,18 @@ extern "C" { #include "pycore_atomic.h" // _Py_atomic_address #include "pycore_ast_state.h" // struct ast_state +#include "pycore_bytesobject.h" // struct _Py_bytes_state +#include "pycore_context.h" // struct _Py_context_state +#include "pycore_dict.h" // struct _Py_dict_state +#include "pycore_exceptions.h" // struct _Py_exc_state +#include "pycore_floatobject.h" // struct _Py_float_state +#include "pycore_genobject.h" // struct _Py_async_gen_state #include "pycore_gil.h" // struct _gil_runtime_state #include "pycore_gc.h" // struct _gc_runtime_state +#include "pycore_list.h" // struct _Py_list_state +#include "pycore_tuple.h" // struct _Py_tuple_state +#include "pycore_typeobject.h" // struct type_cache +#include "pycore_unicodeobject.h" // struct _Py_unicode_state #include "pycore_warnings.h" // struct _warnings_runtime_state struct _pending_calls { @@ -44,132 +54,6 @@ struct _ceval_state { #endif }; -/* fs_codec.encoding is initialized to NULL. - Later, it is set to a non-NULL string by _PyUnicode_InitEncodings(). */ -struct _Py_unicode_fs_codec { - char *encoding; // Filesystem encoding (encoded to UTF-8) - int utf8; // encoding=="utf-8"? - char *errors; // Filesystem errors (encoded to UTF-8) - _Py_error_handler error_handler; -}; - -struct _Py_bytes_state { - PyObject *empty_string; - PyBytesObject *characters[256]; -}; - -struct _Py_unicode_ids { - Py_ssize_t size; - PyObject **array; -}; - -struct _Py_unicode_state { - // The empty Unicode object is a singleton to improve performance. - PyObject *empty_string; - /* Single character Unicode strings in the Latin-1 range are being - shared as well. */ - PyObject *latin1[256]; - struct _Py_unicode_fs_codec fs_codec; - - /* This dictionary holds all interned unicode strings. Note that references - to strings in this dictionary are *not* counted in the string's ob_refcnt. - When the interned string reaches a refcnt of 0 the string deallocation - function will delete the reference from this dictionary. - - Another way to look at this is that to say that the actual reference - count of a string is: s->ob_refcnt + (s->state ? 2 : 0) - */ - PyObject *interned; - - // Unicode identifiers (_Py_Identifier): see _PyUnicode_FromId() - struct _Py_unicode_ids ids; -}; - -struct _Py_float_state { - /* Special free list - free_list is a singly-linked list of available PyFloatObjects, - linked via abuse of their ob_type members. */ - int numfree; - PyFloatObject *free_list; -}; - -/* Speed optimization to avoid frequent malloc/free of small tuples */ -#ifndef PyTuple_MAXSAVESIZE - // Largest tuple to save on free list -# define PyTuple_MAXSAVESIZE 20 -#endif -#ifndef PyTuple_MAXFREELIST - // Maximum number of tuples of each size to save -# define PyTuple_MAXFREELIST 2000 -#endif - -struct _Py_tuple_state { -#if PyTuple_MAXSAVESIZE > 0 - /* Entries 1 up to PyTuple_MAXSAVESIZE are free lists, - entry 0 is the empty tuple () of which at most one instance - will be allocated. */ - PyTupleObject *free_list[PyTuple_MAXSAVESIZE]; - int numfree[PyTuple_MAXSAVESIZE]; -#endif -}; - -/* Empty list reuse scheme to save calls to malloc and free */ -#ifndef PyList_MAXFREELIST -# define PyList_MAXFREELIST 80 -#endif - -struct _Py_list_state { - PyListObject *free_list[PyList_MAXFREELIST]; - int numfree; -}; - -#ifndef PyDict_MAXFREELIST -# define PyDict_MAXFREELIST 80 -#endif - -struct _Py_dict_state { - /* Dictionary reuse scheme to save calls to malloc and free */ - PyDictObject *free_list[PyDict_MAXFREELIST]; - int numfree; - PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST]; - int keys_numfree; -}; - -struct _Py_frame_state { - PyFrameObject *free_list; - /* number of frames currently in free_list */ - int numfree; -}; - -#ifndef _PyAsyncGen_MAXFREELIST -# define _PyAsyncGen_MAXFREELIST 80 -#endif - -struct _Py_async_gen_state { - /* Freelists boost performance 6-10%; they also reduce memory - fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend - are short-living objects that are instantiated for every - __anext__() call. */ - struct _PyAsyncGenWrappedValue* value_freelist[_PyAsyncGen_MAXFREELIST]; - int value_numfree; - - struct PyAsyncGenASend* asend_freelist[_PyAsyncGen_MAXFREELIST]; - int asend_numfree; -}; - -struct _Py_context_state { - // List of free PyContext objects - PyContext *freelist; - int numfree; -}; - -struct _Py_exc_state { - // The dict mapping from errno codes to OSError subclasses - PyObject *errnomap; - PyBaseExceptionObject *memerrors_freelist; - int memerrors_numfree; -}; - // atexit state typedef struct { @@ -185,42 +69,24 @@ struct atexit_state { }; -// Type attribute lookup cache: speed up attribute and method lookups, -// see _PyType_Lookup(). -struct type_cache_entry { - unsigned int version; // initialized from type->tp_version_tag - PyObject *name; // reference to exactly a str or None - PyObject *value; // borrowed reference or NULL -}; - -#define MCACHE_SIZE_EXP 12 -#define MCACHE_STATS 0 - -struct type_cache { - struct type_cache_entry hashtable[1 << MCACHE_SIZE_EXP]; -#if MCACHE_STATS - size_t hits; - size_t misses; - size_t collisions; -#endif -}; - - /* interpreter state */ -#define _PY_NSMALLPOSINTS 257 -#define _PY_NSMALLNEGINTS 5 - -// _PyLong_GetZero() and _PyLong_GetOne() must always be available -#if _PY_NSMALLPOSINTS < 2 -# error "_PY_NSMALLPOSINTS must be greater than 1" -#endif - // The PyInterpreterState typedef is in Include/pystate.h. struct _is { struct _is *next; - struct _ts *tstate_head; + + struct pythreads { + uint64_t next_unique_id; + struct _ts *head; + /* Used in Modules/_threadmodule.c. */ + long count; + /* Support for runtime thread stack size tuning. + A value of 0 means using the platform's default stack size + or the size specified by the THREAD_STACK_SIZE macro. */ + /* Used in Python/thread.c. */ + size_t stacksize; + } threads; /* Reference to the _PyRuntime global variable. This field exists to not have to pass runtime in addition to tstate to a function. @@ -232,6 +98,11 @@ struct _is { int requires_idref; PyThread_type_lock id_mutex; + /* Has been initialized to a safe state. + + In order to be effective, this must be set to 0 during or right + after allocation. */ + int _initialized; int finalizing; struct _ceval_state ceval; @@ -246,14 +117,9 @@ struct _is { PyObject *builtins; // importlib module PyObject *importlib; - - /* Used in Modules/_threadmodule.c. */ - long num_threads; - /* Support for runtime thread stack size tuning. - A value of 0 means using the platform's default stack size - or the size specified by the THREAD_STACK_SIZE macro. */ - /* Used in Python/thread.c. */ - size_t pythread_stacksize; + // override for config->use_frozen_modules (for tests) + // (-1: "off", 1: "on", 0: no override) + int override_frozen_modules; PyObject *codec_search_path; PyObject *codec_search_cache; @@ -281,19 +147,11 @@ struct _is { PyObject *after_forkers_child; #endif - uint64_t tstate_next_unique_id; - struct _warnings_runtime_state warnings; struct atexit_state atexit; PyObject *audit_hooks; - /* Small integers are preallocated in this array so that they - can be shared. - The integers that are preallocated are those in the range - -_PY_NSMALLNEGINTS (inclusive) to _PY_NSMALLPOSINTS (not inclusive). - */ - PyLongObject* small_ints[_PY_NSMALLNEGINTS + _PY_NSMALLPOSINTS]; struct _Py_bytes_state bytes; struct _Py_unicode_state unicode; struct _Py_float_state float_state; @@ -304,7 +162,6 @@ struct _is { struct _Py_tuple_state tuple; struct _Py_list_state list; struct _Py_dict_state dict_state; - struct _Py_frame_state frame; struct _Py_async_gen_state async_gen; struct _Py_context_state context; struct _Py_exc_state exc_state; diff --git a/Include/cpython/interpreteridobject.h b/Include/internal/pycore_interpreteridobject.h similarity index 51% rename from Include/cpython/interpreteridobject.h rename to Include/internal/pycore_interpreteridobject.h index 5076584209b..804831e76de 100644 --- a/Include/cpython/interpreteridobject.h +++ b/Include/internal/pycore_interpreteridobject.h @@ -1,11 +1,22 @@ -#ifndef Py_CPYTHON_INTERPRETERIDOBJECT_H -# error "this header file must not be included directly" +/* Interpreter ID Object */ + +#ifndef Py_INTERNAL_INTERPRETERIDOBJECT_H +#define Py_INTERNAL_INTERPRETERIDOBJECT_H +#ifdef __cplusplus +extern "C" { #endif -/* Interpreter ID Object */ +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif PyAPI_DATA(PyTypeObject) _PyInterpreterID_Type; PyAPI_FUNC(PyObject *) _PyInterpreterID_New(int64_t); PyAPI_FUNC(PyObject *) _PyInterpreterState_GetIDObject(PyInterpreterState *); PyAPI_FUNC(PyInterpreterState *) _PyInterpreterID_LookUp(PyObject *); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_INTERPRETERIDOBJECT_H diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index f18fb052c49..0717a1f9563 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -11,6 +11,30 @@ extern "C" { #include "listobject.h" // _PyList_CAST() +/* runtime lifecycle */ + +extern void _PyList_Fini(PyInterpreterState *); + + +/* other API */ + +#ifndef WITH_FREELISTS +// without freelists +# define PyList_MAXFREELIST 0 +#endif + +/* Empty list reuse scheme to save calls to malloc and free */ +#ifndef PyList_MAXFREELIST +# define PyList_MAXFREELIST 80 +#endif + +struct _Py_list_state { +#if PyList_MAXFREELIST > 0 + PyListObject *free_list[PyList_MAXFREELIST]; + int numfree; +#endif +}; + #define _PyList_ITEMS(op) (_PyList_CAST(op)->ob_item) diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 2bea3a55ec8..a5639ceb692 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -8,31 +8,58 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_interp.h" // PyInterpreterState.small_ints +#include "pycore_long_state.h" // _PyLong_SMALL_INTS #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_runtime.h" // _PyRuntime -// Don't call this function but _PyLong_GetZero() and _PyLong_GetOne() -static inline PyObject* __PyLong_GetSmallInt_internal(int value) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(-_PY_NSMALLNEGINTS <= value && value < _PY_NSMALLPOSINTS); - size_t index = _PY_NSMALLNEGINTS + value; - PyObject *obj = (PyObject*)interp->small_ints[index]; - // _PyLong_GetZero(), _PyLong_GetOne() and get_small_int() must not be - // called before _PyLong_Init() nor after _PyLong_Fini(). - assert(obj != NULL); - return obj; -} + +/* runtime lifecycle */ + +extern void _PyLong_InitGlobalObjects(PyInterpreterState *); +extern PyStatus _PyLong_InitTypes(PyInterpreterState *); + + +/* other API */ // Return a borrowed reference to the zero singleton. // The function cannot return NULL. static inline PyObject* _PyLong_GetZero(void) -{ return __PyLong_GetSmallInt_internal(0); } +{ return (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS]; } // Return a borrowed reference to the one singleton. // The function cannot return NULL. static inline PyObject* _PyLong_GetOne(void) -{ return __PyLong_GetSmallInt_internal(1); } +{ return (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS+1]; } + +PyObject *_PyLong_Add(PyLongObject *left, PyLongObject *right); +PyObject *_PyLong_Multiply(PyLongObject *left, PyLongObject *right); +PyObject *_PyLong_Subtract(PyLongObject *left, PyLongObject *right); + +/* Used by Python/mystrtoul.c, _PyBytes_FromHex(), + _PyBytes_DecodeEscape(), etc. */ +PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; + +/* Format the object based on the format_spec, as defined in PEP 3101 + (Advanced String Formatting). */ +PyAPI_FUNC(int) _PyLong_FormatAdvancedWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + PyObject *format_spec, + Py_ssize_t start, + Py_ssize_t end); + +PyAPI_FUNC(int) _PyLong_FormatWriter( + _PyUnicodeWriter *writer, + PyObject *obj, + int base, + int alternate); + +PyAPI_FUNC(char*) _PyLong_FormatBytesWriter( + _PyBytesWriter *writer, + char *str, + PyObject *obj, + int base, + int alternate); #ifdef __cplusplus } diff --git a/Include/internal/pycore_long_state.h b/Include/internal/pycore_long_state.h new file mode 100644 index 00000000000..5fe8e623a9e --- /dev/null +++ b/Include/internal/pycore_long_state.h @@ -0,0 +1,33 @@ +#ifndef Py_INTERNAL_LONG_STATE_H +#define Py_INTERNAL_LONG_STATE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#define _PY_NSMALLPOSINTS 257 +#define _PY_NSMALLNEGINTS 5 + +// _PyLong_GetZero() and _PyLong_GetOne() must always be available +#if _PY_NSMALLPOSINTS < 2 +# error "_PY_NSMALLPOSINTS must be greater than 1" +#endif + +struct _Py_long_state { + /* Small integers are preallocated in this array so that they + * can be shared. + * The integers that are preallocated are those in the range + *-_PY_NSMALLNEGINTS (inclusive) to _PY_NSMALLPOSINTS (not inclusive). + */ + PyLongObject small_ints[_PY_NSMALLNEGINTS + _PY_NSMALLPOSINTS]; +}; + +#define _PyLong_SMALL_INTS _PyRuntime.int_state.small_ints + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_LONG_STATE_H */ diff --git a/Include/internal/pycore_namespace.h b/Include/internal/pycore_namespace.h new file mode 100644 index 00000000000..cb76f040693 --- /dev/null +++ b/Include/internal/pycore_namespace.h @@ -0,0 +1,20 @@ +// Simple namespace object interface + +#ifndef Py_INTERNAL_NAMESPACE_H +#define Py_INTERNAL_NAMESPACE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +PyAPI_DATA(PyTypeObject) _PyNamespace_Type; + +PyAPI_FUNC(PyObject *) _PyNamespace_New(PyObject *kwds); + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_NAMESPACE_H diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 744b41ae5d9..9041a4dc8a3 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -168,7 +168,16 @@ _PyObject_IS_GC(PyObject *obj) // Fast inlined version of PyType_IS_GC() #define _PyType_IS_GC(t) _PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) -// Usage: assert(_Py_CheckSlotResult(obj, "__getitem__", result != NULL))); +static inline size_t +_PyType_PreHeaderSize(PyTypeObject *tp) +{ + return _PyType_IS_GC(tp) * sizeof(PyGC_Head) + + _PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT) * 2 * sizeof(PyObject *); +} + +void _PyObject_GC_Link(PyObject *op); + +// Usage: assert(_Py_CheckSlotResult(obj, "__getitem__", result != NULL)); extern int _Py_CheckSlotResult( PyObject *obj, const char *slot_name, @@ -181,6 +190,28 @@ extern int _Py_CheckSlotResult( extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); extern int _PyObject_InitializeDict(PyObject *obj); +extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, + PyObject *name, PyObject *value); +PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values, + PyObject *name); + +static inline PyDictValues **_PyObject_ValuesPointer(PyObject *obj) +{ + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + return ((PyDictValues **)obj)-4; +} + +static inline PyObject **_PyObject_ManagedDictPointer(PyObject *obj) +{ + assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + return ((PyObject **)obj)-3; +} + +PyObject ** _PyObject_DictPointer(PyObject *); +int _PyObject_VisitInstanceAttributes(PyObject *self, visitproc visit, void *arg); +void _PyObject_ClearInstanceAttributes(PyObject *self); +void _PyObject_FreeInstanceAttributes(PyObject *self); +int _PyObject_IsInstanceDictEmpty(PyObject *); #ifdef __cplusplus } diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h index 15447f54490..b8deaa0c3eb 100644 --- a/Include/internal/pycore_pathconfig.h +++ b/Include/internal/pycore_pathconfig.h @@ -8,64 +8,15 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -typedef struct _PyPathConfig { - /* Full path to the Python program */ - wchar_t *program_full_path; - wchar_t *prefix; - wchar_t *exec_prefix; - /* Set by Py_SetPath(), or computed by _PyConfig_InitPathConfig() */ - wchar_t *module_search_path; - /* Python program name */ - wchar_t *program_name; - /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */ - wchar_t *home; -#ifdef MS_WINDOWS - /* isolated and site_import are used to set Py_IsolatedFlag and - Py_NoSiteFlag flags on Windows in read_pth_file(). These fields - are ignored when their value are equal to -1 (unset). */ - int isolated; - int site_import; - /* Set when a venv is detected */ - wchar_t *base_executable; -#endif -} _PyPathConfig; +PyAPI_FUNC(void) _PyPathConfig_ClearGlobal(void); +extern PyStatus _PyPathConfig_ReadGlobal(PyConfig *config); +extern PyStatus _PyPathConfig_UpdateGlobal(const PyConfig *config); +extern const wchar_t * _PyPathConfig_GetGlobalModuleSearchPath(void); -#ifdef MS_WINDOWS -# define _PyPathConfig_INIT \ - {.module_search_path = NULL, \ - .isolated = -1, \ - .site_import = -1} -#else -# define _PyPathConfig_INIT \ - {.module_search_path = NULL} -#endif -/* Note: _PyPathConfig_INIT sets other fields to 0/NULL */ - -PyAPI_DATA(_PyPathConfig) _Py_path_config; -#ifdef MS_WINDOWS -PyAPI_DATA(wchar_t*) _Py_dll_path; -#endif - -extern void _PyPathConfig_ClearGlobal(void); - -extern PyStatus _PyPathConfig_Calculate( - _PyPathConfig *pathconfig, - const PyConfig *config); extern int _PyPathConfig_ComputeSysPath0( const PyWideStringList *argv, PyObject **path0); -extern PyStatus _Py_FindEnvConfigValue( - FILE *env_file, - const wchar_t *key, - wchar_t **value_p); -#ifdef MS_WINDOWS -extern wchar_t* _Py_GetDLLPath(void); -#endif - -extern PyStatus _PyConfig_WritePathConfig(const PyConfig *config); -extern void _Py_DumpPathConfig(PyThreadState *tstate); -extern PyObject* _PyPathConfig_AsDict(void); #ifdef __cplusplus } diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index a5e97fe23fb..3134afeb864 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -8,6 +8,14 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif + +/* runtime lifecycle */ + +extern PyStatus _PyErr_InitTypes(PyInterpreterState *); + + +/* other API */ + static inline PyObject* _PyErr_Occurred(PyThreadState *tstate) { assert(tstate != NULL); @@ -28,6 +36,8 @@ static inline void _PyErr_ClearExcState(_PyErr_StackItem *exc_state) Py_XDECREF(tb); } +PyAPI_FUNC(PyObject*) _PyErr_StackItemToExcInfoTuple( + _PyErr_StackItem *err_info); PyAPI_FUNC(void) _PyErr_Fetch( PyThreadState *tstate, @@ -90,6 +100,12 @@ extern PyObject* _Py_Offer_Suggestions(PyObject* exception); PyAPI_FUNC(Py_ssize_t) _Py_UTF8_Edit_Cost(PyObject *str_a, PyObject *str_b, Py_ssize_t max_cost); +PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( + const char *func, + const char *message); + +#define _Py_FatalRefcountError(message) _Py_FatalRefcountErrorFunc(__func__, message) + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 524be9d4cbb..766e889f237 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -49,13 +49,6 @@ PyAPI_FUNC(int) _Py_IsLocaleCoercionTarget(const char *ctype_loc); /* Various one-time initializers */ -extern PyStatus _PyUnicode_Init(PyInterpreterState *interp); -extern PyStatus _PyUnicode_InitTypes(void); -extern PyStatus _PyBytes_Init(PyInterpreterState *interp); -extern int _PyStructSequence_Init(void); -extern int _PyLong_Init(PyInterpreterState *interp); -extern int _PyLong_InitTypes(void); -extern PyStatus _PyTuple_Init(PyInterpreterState *interp); extern PyStatus _PyFaulthandler_Init(int enable); extern int _PyTraceMalloc_Init(int enable); extern PyObject * _PyBuiltin_Init(PyInterpreterState *interp); @@ -65,15 +58,9 @@ extern PyStatus _PySys_Create( extern PyStatus _PySys_ReadPreinitWarnOptions(PyWideStringList *options); extern PyStatus _PySys_ReadPreinitXOptions(PyConfig *config); extern int _PySys_UpdateConfig(PyThreadState *tstate); -extern PyStatus _PyExc_Init(PyInterpreterState *interp); -extern PyStatus _PyErr_InitTypes(void); extern PyStatus _PyBuiltins_AddExceptions(PyObject * bltinmod); -extern void _PyFloat_Init(void); -extern int _PyFloat_InitTypes(void); extern PyStatus _Py_HashRandomization_Init(const PyConfig *); -extern PyStatus _PyTypes_Init(void); -extern PyStatus _PyTypes_InitSlotDefs(void); extern PyStatus _PyImportZip_Init(PyThreadState *tstate); extern PyStatus _PyGC_Init(PyInterpreterState *interp); extern PyStatus _PyAtExit_Init(PyInterpreterState *interp); @@ -81,27 +68,13 @@ extern PyStatus _PyAtExit_Init(PyInterpreterState *interp); /* Various internal finalizers */ -extern void _PyFrame_Fini(PyInterpreterState *interp); -extern void _PyDict_Fini(PyInterpreterState *interp); -extern void _PyTuple_Fini(PyInterpreterState *interp); -extern void _PyList_Fini(PyInterpreterState *interp); -extern void _PyBytes_Fini(PyInterpreterState *interp); -extern void _PyFloat_Fini(PyInterpreterState *interp); -extern void _PySlice_Fini(PyInterpreterState *interp); -extern void _PyAsyncGen_Fini(PyInterpreterState *interp); - extern int _PySignal_Init(int install_signal_handlers); extern void _PySignal_Fini(void); -extern void _PyExc_Fini(PyInterpreterState *interp); extern void _PyImport_Fini(void); extern void _PyImport_Fini2(void); extern void _PyGC_Fini(PyInterpreterState *interp); -extern void _PyType_Fini(PyInterpreterState *interp); extern void _Py_HashRandomization_Fini(void); -extern void _PyUnicode_Fini(PyInterpreterState *interp); -extern void _PyUnicode_ClearInterned(PyInterpreterState *interp); -extern void _PyLong_Fini(PyInterpreterState *interp); extern void _PyFaulthandler_Fini(void); extern void _PyHash_Fini(void); extern void _PyTraceMalloc_Fini(void); @@ -122,6 +95,7 @@ PyAPI_FUNC(PyStatus) _Py_PreInitializeFromConfig( const PyConfig *config, const struct _PyArgv *args); +PyAPI_FUNC(wchar_t *) _Py_GetStdlibDir(void); PyAPI_FUNC(int) _Py_HandleSystemExit(int *exitcode_p); diff --git a/Include/internal/pycore_pymath.h b/Include/internal/pycore_pymath.h new file mode 100644 index 00000000000..395b71452e4 --- /dev/null +++ b/Include/internal/pycore_pymath.h @@ -0,0 +1,184 @@ +#ifndef Py_INTERNAL_PYMATH_H +#define Py_INTERNAL_PYMATH_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// Extra declarations +#if !defined(_MSC_VER) && !defined(__STDC__) +extern double fmod (double, double); +extern double frexp (double, int *); +extern double ldexp (double, int); +extern double modf (double, double *); +extern double pow(double, double); +#endif // !defined(_MSC_VER) && !defined(__STDC__) + + +/* _Py_ADJUST_ERANGE1(x) + * _Py_ADJUST_ERANGE2(x, y) + * Set errno to 0 before calling a libm function, and invoke one of these + * macros after, passing the function result(s) (_Py_ADJUST_ERANGE2 is useful + * for functions returning complex results). This makes two kinds of + * adjustments to errno: (A) If it looks like the platform libm set + * errno=ERANGE due to underflow, clear errno. (B) If it looks like the + * platform libm overflowed but didn't set errno, force errno to ERANGE. In + * effect, we're trying to force a useful implementation of C89 errno + * behavior. + * Caution: + * This isn't reliable. C99 no longer requires libm to set errno under + * any exceptional condition, but does require +- HUGE_VAL return + * values on overflow. A 754 box *probably* maps HUGE_VAL to a + * double infinity, and we're cool if that's so, unless the input + * was an infinity and an infinity is the expected result. A C89 + * system sets errno to ERANGE, so we check for that too. We're + * out of luck if a C99 754 box doesn't map HUGE_VAL to +Inf, or + * if the returned result is a NaN, or if a C89 box returns HUGE_VAL + * in non-overflow cases. + */ +static inline void _Py_ADJUST_ERANGE1(double x) +{ + if (errno == 0) { + if (x == Py_HUGE_VAL || x == -Py_HUGE_VAL) { + errno = ERANGE; + } + } + else if (errno == ERANGE && x == 0.0) { + errno = 0; + } +} + +static inline void _Py_ADJUST_ERANGE2(double x, double y) +{ + if (x == Py_HUGE_VAL || x == -Py_HUGE_VAL || + y == Py_HUGE_VAL || y == -Py_HUGE_VAL) + { + if (errno == 0) { + errno = ERANGE; + } + } + else if (errno == ERANGE) { + errno = 0; + } +} + +// Return whether integral type *type* is signed or not. +#define _Py_IntegralTypeSigned(type) \ + ((type)(-1) < 0) + +// Return the maximum value of integral type *type*. +#define _Py_IntegralTypeMax(type) \ + ((_Py_IntegralTypeSigned(type)) ? (((((type)1 << (sizeof(type)*CHAR_BIT - 2)) - 1) << 1) + 1) : ~(type)0) + +// Return the minimum value of integral type *type*. +#define _Py_IntegralTypeMin(type) \ + ((_Py_IntegralTypeSigned(type)) ? -_Py_IntegralTypeMax(type) - 1 : 0) + +// Check whether *v* is in the range of integral type *type*. This is most +// useful if *v* is floating-point, since demoting a floating-point *v* to an +// integral type that cannot represent *v*'s integral part is undefined +// behavior. +#define _Py_InIntegralTypeRange(type, v) \ + (_Py_IntegralTypeMin(type) <= v && v <= _Py_IntegralTypeMax(type)) + + +//--- Implementation of the HAVE_PY_SET_53BIT_PRECISION macro ------------- +//--- defined in pyport.h ------------------------------------------------- +// +// Give appropriate definitions for the following three macros: +// +// _Py_SET_53BIT_PRECISION_HEADER : any variable declarations needed to +// use the two macros below. +// _Py_SET_53BIT_PRECISION_START : store original FPU settings, and +// set FPU to 53-bit precision/round-half-to-even +// _Py_SET_53BIT_PRECISION_END : restore original FPU settings + +// Get and set x87 control word for gcc/x86 +#ifdef HAVE_GCC_ASM_FOR_X87 + +// Functions defined in Python/pymath.c +extern unsigned short _Py_get_387controlword(void); +extern void _Py_set_387controlword(unsigned short); + +#define _Py_SET_53BIT_PRECISION_HEADER \ + unsigned short old_387controlword, new_387controlword +#define _Py_SET_53BIT_PRECISION_START \ + do { \ + old_387controlword = _Py_get_387controlword(); \ + new_387controlword = (old_387controlword & ~0x0f00) | 0x0200; \ + if (new_387controlword != old_387controlword) { \ + _Py_set_387controlword(new_387controlword); \ + } \ + } while (0) +#define _Py_SET_53BIT_PRECISION_END \ + do { \ + if (new_387controlword != old_387controlword) { \ + _Py_set_387controlword(old_387controlword); \ + } \ + } while (0) +#endif + +// Get and set x87 control word for VisualStudio/x86. +// x87 is not supported in 64-bit or ARM. +#if defined(_MSC_VER) && !defined(_WIN64) && !defined(_M_ARM) + +#include // __control87_2() + +#define _Py_SET_53BIT_PRECISION_HEADER \ + unsigned int old_387controlword, new_387controlword, out_387controlword + // We use the __control87_2 function to set only the x87 control word. + // The SSE control word is unaffected. +#define _Py_SET_53BIT_PRECISION_START \ + do { \ + __control87_2(0, 0, &old_387controlword, NULL); \ + new_387controlword = \ + (old_387controlword & ~(_MCW_PC | _MCW_RC)) | (_PC_53 | _RC_NEAR); \ + if (new_387controlword != old_387controlword) { \ + __control87_2(new_387controlword, _MCW_PC | _MCW_RC, \ + &out_387controlword, NULL); \ + } \ + } while (0) +#define _Py_SET_53BIT_PRECISION_END \ + do { \ + if (new_387controlword != old_387controlword) { \ + __control87_2(old_387controlword, _MCW_PC | _MCW_RC, \ + &out_387controlword, NULL); \ + } \ + } while (0) +#endif + +#ifdef HAVE_GCC_ASM_FOR_MC68881 +#define _Py_SET_53BIT_PRECISION_HEADER \ + unsigned int old_fpcr, new_fpcr +#define _Py_SET_53BIT_PRECISION_START \ + do { \ + __asm__ ("fmove.l %%fpcr,%0" : "=g" (old_fpcr)); \ + /* Set double precision / round to nearest. */ \ + new_fpcr = (old_fpcr & ~0xf0) | 0x80; \ + if (new_fpcr != old_fpcr) { \ + __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (new_fpcr));\ + } \ + } while (0) +#define _Py_SET_53BIT_PRECISION_END \ + do { \ + if (new_fpcr != old_fpcr) { \ + __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (old_fpcr)); \ + } \ + } while (0) +#endif + +// Default definitions are empty +#ifndef _Py_SET_53BIT_PRECISION_HEADER +# define _Py_SET_53BIT_PRECISION_HEADER +# define _Py_SET_53BIT_PRECISION_START +# define _Py_SET_53BIT_PRECISION_END +#endif + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_PYMATH_H */ diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 30db3f2b670..d70deee710e 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -103,4 +103,4 @@ void _PyObject_VirtualFree(void *, size_t size); #ifdef __cplusplus } #endif -#endif /* !Py_INTERNAL_PYMEM_H */ +#endif // !Py_INTERNAL_PYMEM_H diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 4b894f3eff4..06f58fb5100 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -21,12 +21,27 @@ _Py_IsMainThread(void) } +static inline PyInterpreterState * +_PyInterpreterState_Main(void) +{ + return _PyRuntime.interpreters.main; +} + static inline int _Py_IsMainInterpreter(PyInterpreterState *interp) { - /* Use directly _PyRuntime rather than tstate->interp->runtime, since - this function is used in performance critical code path (ceval) */ - return (interp == _PyRuntime.interpreters.main); + return (interp == _PyInterpreterState_Main()); +} + + +static inline const PyConfig * +_Py_GetMainConfig(void) +{ + PyInterpreterState *interp = _PyInterpreterState_Main(); + if (interp == NULL) { + return NULL; + } + return _PyInterpreterState_GetConfig(interp); } @@ -34,7 +49,7 @@ _Py_IsMainInterpreter(PyInterpreterState *interp) static inline int _Py_ThreadCanHandleSignals(PyInterpreterState *interp) { - return (_Py_IsMainThread() && interp == _PyRuntime.interpreters.main); + return (_Py_IsMainThread() && _Py_IsMainInterpreter(interp)); } @@ -71,7 +86,7 @@ _PyRuntimeState_GetThreadState(_PyRuntimeState *runtime) The caller must hold the GIL. - See also PyThreadState_Get() and PyThreadState_GET(). */ + See also PyThreadState_Get() and _PyThreadState_UncheckedGet(). */ static inline PyThreadState* _PyThreadState_GET(void) { @@ -82,10 +97,6 @@ _PyThreadState_GET(void) #endif } -/* Redefine PyThreadState_GET() as an alias to _PyThreadState_GET() */ -#undef PyThreadState_GET -#define PyThreadState_GET() _PyThreadState_GET() - PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalError_TstateNULL(const char *func); static inline void @@ -118,14 +129,33 @@ static inline PyInterpreterState* _PyInterpreterState_GET(void) { } -/* Other */ +// PyThreadState functions +PyAPI_FUNC(void) _PyThreadState_SetCurrent(PyThreadState *tstate); +// We keep this around exclusively for stable ABI compatibility. PyAPI_FUNC(void) _PyThreadState_Init( PyThreadState *tstate); PyAPI_FUNC(void) _PyThreadState_DeleteExcept( _PyRuntimeState *runtime, PyThreadState *tstate); +static inline void +_PyThreadState_PauseTracing(PyThreadState *tstate) +{ + tstate->cframe->use_tracing = 0; +} + +static inline void +_PyThreadState_ResumeTracing(PyThreadState *tstate) +{ + int use_tracing = (tstate->c_tracefunc != NULL + || tstate->c_profilefunc != NULL); + tstate->cframe->use_tracing = (use_tracing ? 255 : 0); +} + + +/* Other */ + PyAPI_FUNC(PyThreadState *) _PyThreadState_Swap( struct _gilstate_runtime_state *gilstate, PyThreadState *newts); diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index bcd710c4496..bd88510d1f0 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -10,6 +10,8 @@ extern "C" { #include "pycore_atomic.h" /* _Py_atomic_address */ #include "pycore_gil.h" // struct _gil_runtime_state +#include "pycore_long_state.h" // struct _Py_long_state +#include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids /* ceval state */ @@ -49,16 +51,15 @@ typedef struct _Py_AuditHookEntry { void *userData; } _Py_AuditHookEntry; -struct _Py_unicode_runtime_ids { - PyThread_type_lock lock; - // next_index value must be preserved when Py_Initialize()/Py_Finalize() - // is called multiple times: see _PyUnicode_FromId() implementation. - Py_ssize_t next_index; -}; - /* Full Python runtime state */ typedef struct pyruntimestate { + /* Has been initialized to a safe state. + + In order to be effective, this must be set to 0 during or right + after allocation. */ + int _initialized; + /* Is running Py_PreInitialize()? */ int preinitializing; @@ -100,6 +101,8 @@ typedef struct pyruntimestate { unsigned long main_thread; + struct _Py_long_state int_state; + #define NEXITFUNCS 32 void (*exitfuncs[NEXITFUNCS])(void); int nexitfuncs; @@ -121,9 +124,18 @@ typedef struct pyruntimestate { } _PyRuntimeState; #define _PyRuntimeState_INIT \ - {.preinitialized = 0, .core_initialized = 0, .initialized = 0} + { \ + ._initialized = 0, \ + } /* Note: _PyRuntimeState_INIT sets other fields to 0/NULL */ +static inline void +_PyRuntimeState_reset(_PyRuntimeState *runtime) +{ + /* Make it match _PyRuntimeState_INIT. */ + memset(runtime, 0, sizeof(*runtime)); +} + PyAPI_DATA(_PyRuntimeState) _PyRuntime; diff --git a/Include/internal/pycore_sliceobject.h b/Include/internal/pycore_sliceobject.h new file mode 100644 index 00000000000..e81834c041e --- /dev/null +++ b/Include/internal/pycore_sliceobject.h @@ -0,0 +1,20 @@ +#ifndef Py_INTERNAL_SLICEOBJECT_H +#define Py_INTERNAL_SLICEOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +/* runtime lifecycle */ + +extern void _PySlice_Fini(PyInterpreterState *); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_SLICEOBJECT_H */ diff --git a/Include/internal/pycore_strhex.h b/Include/internal/pycore_strhex.h new file mode 100644 index 00000000000..1633671da0f --- /dev/null +++ b/Include/internal/pycore_strhex.h @@ -0,0 +1,36 @@ +#ifndef Py_INTERNAL_STRHEX_H +#define Py_INTERNAL_STRHEX_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +// Returns a str() containing the hex representation of argbuf. +PyAPI_FUNC(PyObject*) _Py_strhex(const + char* argbuf, + const Py_ssize_t arglen); + +// Returns a bytes() containing the ASCII hex representation of argbuf. +PyAPI_FUNC(PyObject*) _Py_strhex_bytes( + const char* argbuf, + const Py_ssize_t arglen); + +// These variants include support for a separator between every N bytes: +PyAPI_FUNC(PyObject*) _Py_strhex_with_sep( + const char* argbuf, + const Py_ssize_t arglen, + const PyObject* sep, + const int bytes_per_group); +PyAPI_FUNC(PyObject*) _Py_strhex_bytes_with_sep( + const char* argbuf, + const Py_ssize_t arglen, + const PyObject* sep, + const int bytes_per_group); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_STRHEX_H */ diff --git a/Include/internal/pycore_structseq.h b/Include/internal/pycore_structseq.h index 84c8d477e0d..3a61cb9a126 100644 --- a/Include/internal/pycore_structseq.h +++ b/Include/internal/pycore_structseq.h @@ -9,6 +9,13 @@ extern "C" { #endif +/* runtime lifecycle */ + +extern PyStatus _PyStructSequence_InitState(PyInterpreterState *); + + +/* other API */ + PyAPI_FUNC(int) _PyStructSequence_InitType( PyTypeObject *type, PyStructSequence_Desc *desc, diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 4c1b7d3519c..28935f4ed55 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -13,6 +13,13 @@ struct _mod; // Type defined in pycore_ast.h typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock, AnnotationBlock } _Py_block_ty; +typedef enum _comprehension_type { + NoComprehension = 0, + ListComprehension = 1, + DictComprehension = 2, + SetComprehension = 3, + GeneratorExpression = 4 } _Py_comprehension_ty; + struct _symtable_entry; struct symtable { @@ -42,14 +49,14 @@ typedef struct _symtable_entry { PyObject *ste_varnames; /* list of function parameters */ PyObject *ste_children; /* list of child blocks */ PyObject *ste_directives;/* locations of global and nonlocal statements */ - _Py_block_ty ste_type; /* module, class, or function */ + _Py_block_ty ste_type; /* module, class or function */ int ste_nested; /* true if block is nested */ unsigned ste_free : 1; /* true if block has free variables */ unsigned ste_child_free : 1; /* true if a child block has free vars, including free refs to globals */ unsigned ste_generator : 1; /* true if namespace is a generator */ unsigned ste_coroutine : 1; /* true if namespace is a coroutine */ - unsigned ste_comprehension : 1; /* true if namespace is a list comprehension */ + _Py_comprehension_ty ste_comprehension; /* Kind of comprehension (if any) */ unsigned ste_varargs : 1; /* true if block has varargs */ unsigned ste_varkeywords : 1; /* true if block has varkeywords */ unsigned ste_returns_value : 1; /* true if namespace uses return with diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index 4d282308769..84dbe27044f 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -51,7 +51,7 @@ PyAPI_FUNC(void) _Py_DumpTraceback( _PyGILState_GetInterpreterStateUnsafe() in last resort. It is better to pass NULL to interp and current_tstate, the function tries - different options to retrieve these informations. + different options to retrieve this information. This function is signal safe. */ @@ -87,6 +87,17 @@ PyAPI_FUNC(PyObject*) _PyTraceBack_FromFrame( PyObject *tb_next, PyFrameObject *frame); +#define EXCEPTION_TB_HEADER "Traceback (most recent call last):\n" +#define EXCEPTION_GROUP_TB_HEADER "Exception Group Traceback (most recent call last):\n" + +/* Write the traceback tb to file f. Prefix each line with + indent spaces followed by the margin (if it is not NULL). */ +PyAPI_FUNC(int) _PyTraceBack_Print_Indented( + PyObject *tb, int indent, const char* margin, + const char *header_margin, const char *header, PyObject *f); +PyAPI_FUNC(int) _Py_WriteIndentedMargin(int, const char*, PyObject *); +PyAPI_FUNC(int) _Py_WriteIndent(int, PyObject *); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index d1d0d2a92e4..624c21caec1 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -10,9 +10,47 @@ extern "C" { #include "tupleobject.h" /* _PyTuple_CAST() */ + +/* runtime lifecycle */ + +extern PyStatus _PyTuple_InitGlobalObjects(PyInterpreterState *); +extern PyStatus _PyTuple_InitTypes(PyInterpreterState *); +extern void _PyTuple_Fini(PyInterpreterState *); + + +/* other API */ + +#ifndef WITH_FREELISTS +// without freelists +// for tuples only store empty tuple singleton +# define PyTuple_MAXSAVESIZE 1 +# define PyTuple_MAXFREELIST 1 +#endif + +/* Speed optimization to avoid frequent malloc/free of small tuples */ +#ifndef PyTuple_MAXSAVESIZE + // Largest tuple to save on free list +# define PyTuple_MAXSAVESIZE 20 +#endif +#ifndef PyTuple_MAXFREELIST + // Maximum number of tuples of each size to save +# define PyTuple_MAXFREELIST 2000 +#endif + +struct _Py_tuple_state { +#if PyTuple_MAXSAVESIZE > 0 + /* Entries 1 up to PyTuple_MAXSAVESIZE are free lists, + entry 0 is the empty tuple () of which at most one instance + will be allocated. */ + PyTupleObject *free_list[PyTuple_MAXSAVESIZE]; + int numfree[PyTuple_MAXSAVESIZE]; +#endif +}; + #define _PyTuple_ITEMS(op) (_PyTuple_CAST(op)->ob_item) extern PyObject *_PyTuple_FromArray(PyObject *const *, Py_ssize_t); +extern PyObject *_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); #ifdef __cplusplus } diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h new file mode 100644 index 00000000000..7fd8a1f3509 --- /dev/null +++ b/Include/internal/pycore_typeobject.h @@ -0,0 +1,47 @@ +#ifndef Py_INTERNAL_TYPEOBJECT_H +#define Py_INTERNAL_TYPEOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +/* runtime lifecycle */ + +extern PyStatus _PyTypes_InitState(PyInterpreterState *); +extern PyStatus _PyTypes_InitTypes(PyInterpreterState *); +extern void _PyTypes_Fini(PyInterpreterState *); + + +/* other API */ + +// Type attribute lookup cache: speed up attribute and method lookups, +// see _PyType_Lookup(). +struct type_cache_entry { + unsigned int version; // initialized from type->tp_version_tag + PyObject *name; // reference to exactly a str or None + PyObject *value; // borrowed reference or NULL +}; + +#define MCACHE_SIZE_EXP 12 +#define MCACHE_STATS 0 + +struct type_cache { + struct type_cache_entry hashtable[1 << MCACHE_SIZE_EXP]; +#if MCACHE_STATS + size_t hits; + size_t misses; + size_t collisions; +#endif +}; + +extern PyStatus _PyTypes_InitSlotDefs(void); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_TYPEOBJECT_H */ diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h new file mode 100644 index 00000000000..c50c42011a9 --- /dev/null +++ b/Include/internal/pycore_unicodeobject.h @@ -0,0 +1,71 @@ +#ifndef Py_INTERNAL_UNICODEOBJECT_H +#define Py_INTERNAL_UNICODEOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +/* runtime lifecycle */ + +extern void _PyUnicode_InitState(PyInterpreterState *); +extern PyStatus _PyUnicode_InitGlobalObjects(PyInterpreterState *); +extern PyStatus _PyUnicode_InitTypes(PyInterpreterState *); +extern void _PyUnicode_Fini(PyInterpreterState *); + + +/* other API */ + +struct _Py_unicode_runtime_ids { + PyThread_type_lock lock; + // next_index value must be preserved when Py_Initialize()/Py_Finalize() + // is called multiple times: see _PyUnicode_FromId() implementation. + Py_ssize_t next_index; +}; + +/* fs_codec.encoding is initialized to NULL. + Later, it is set to a non-NULL string by _PyUnicode_InitEncodings(). */ +struct _Py_unicode_fs_codec { + char *encoding; // Filesystem encoding (encoded to UTF-8) + int utf8; // encoding=="utf-8"? + char *errors; // Filesystem errors (encoded to UTF-8) + _Py_error_handler error_handler; +}; + +struct _Py_unicode_ids { + Py_ssize_t size; + PyObject **array; +}; + +struct _Py_unicode_state { + // The empty Unicode object is a singleton to improve performance. + PyObject *empty_string; + /* Single character Unicode strings in the Latin-1 range are being + shared as well. */ + PyObject *latin1[256]; + struct _Py_unicode_fs_codec fs_codec; + + /* This dictionary holds all interned unicode strings. Note that references + to strings in this dictionary are *not* counted in the string's ob_refcnt. + When the interned string reaches a refcnt of 0 the string deallocation + function will delete the reference from this dictionary. + + Another way to look at this is that to say that the actual reference + count of a string is: s->ob_refcnt + (s->state ? 2 : 0) + */ + PyObject *interned; + + // Unicode identifiers (_Py_Identifier): see _PyUnicode_FromId() + struct _Py_unicode_ids ids; +}; + +extern void _PyUnicode_ClearInterned(PyInterpreterState *); + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_UNICODEOBJECT_H */ diff --git a/Include/internal/pycore_warnings.h b/Include/internal/pycore_warnings.h index f728ec3077b..efb4f1cd7ea 100644 --- a/Include/internal/pycore_warnings.h +++ b/Include/internal/pycore_warnings.h @@ -19,6 +19,10 @@ struct _warnings_runtime_state { extern int _PyWarnings_InitState(PyInterpreterState *interp); +PyAPI_FUNC(PyObject*) _PyWarnings_Init(void); + +extern void _PyErr_WarnUnawaitedCoroutine(PyObject *coro); + #ifdef __cplusplus } #endif diff --git a/Include/interpreteridobject.h b/Include/interpreteridobject.h deleted file mode 100644 index e744fcdc9ff..00000000000 --- a/Include/interpreteridobject.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef Py_INTERPRETERIDOBJECT_H -#define Py_INTERPRETERIDOBJECT_H - -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_INTERPRETERIDOBJECT_H -# include "cpython/interpreteridobject.h" -# undef Py_CPYTHON_INTERPRETERIDOBJECT_H -#endif - -#ifdef __cplusplus -} -#endif -#endif /* !Py_INTERPRETERIDOBJECT_H */ diff --git a/Include/listobject.h b/Include/listobject.h index 2a8a25525d1..eff42c188f1 100644 --- a/Include/listobject.h +++ b/Include/listobject.h @@ -42,7 +42,7 @@ PyAPI_FUNC(PyObject *) PyList_AsTuple(PyObject *); #ifndef Py_LIMITED_API # define Py_CPYTHON_LISTOBJECT_H -# include "cpython/listobject.h" +# include "cpython/listobject.h" # undef Py_CPYTHON_LISTOBJECT_H #endif diff --git a/Include/longobject.h b/Include/longobject.h index e2301d7abfc..7fe8f58cb3e 100644 --- a/Include/longobject.h +++ b/Include/longobject.h @@ -26,9 +26,6 @@ PyAPI_FUNC(Py_ssize_t) PyLong_AsSsize_t(PyObject *); PyAPI_FUNC(size_t) PyLong_AsSize_t(PyObject *); PyAPI_FUNC(unsigned long) PyLong_AsUnsignedLong(PyObject *); PyAPI_FUNC(unsigned long) PyLong_AsUnsignedLongMask(PyObject *); -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) _PyLong_AsInt(PyObject *); -#endif PyAPI_FUNC(PyObject *) PyLong_GetInfo(void); /* It may be useful in the future. I've added it in the PyInt -> PyLong @@ -65,30 +62,6 @@ PyAPI_FUNC(PyObject *) PyLong_GetInfo(void); # error "void* different in size from int, long and long long" #endif /* SIZEOF_VOID_P */ -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) _PyLong_UnsignedShort_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyLong_UnsignedInt_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyLong_UnsignedLong_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyLong_UnsignedLongLong_Converter(PyObject *, void *); -PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *); -#endif - -/* Used by Python/mystrtoul.c, _PyBytes_FromHex(), - _PyBytes_DecodeEscape(), etc. */ -#ifndef Py_LIMITED_API -PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; -#endif - -/* _PyLong_Frexp returns a double x and an exponent e such that the - true value is approximately equal to x * 2**e. e is >= 0. x is - 0.0 if and only if the input is 0 (in which case, e and x are both - zeroes); otherwise, 0.5 <= abs(x) < 1.0. On overflow, which is - possible if the number of bits doesn't fit into a Py_ssize_t, sets - OverflowError and returns -1.0 for x, 0 for e. */ -#ifndef Py_LIMITED_API -PyAPI_FUNC(double) _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e); -#endif - PyAPI_FUNC(double) PyLong_AsDouble(PyObject *); PyAPI_FUNC(PyObject *) PyLong_FromVoidPtr(void *); PyAPI_FUNC(void *) PyLong_AsVoidPtr(PyObject *); @@ -101,102 +74,6 @@ PyAPI_FUNC(unsigned long long) PyLong_AsUnsignedLongLongMask(PyObject *); PyAPI_FUNC(long long) PyLong_AsLongLongAndOverflow(PyObject *, int *); PyAPI_FUNC(PyObject *) PyLong_FromString(const char *, char **, int); -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) PyLong_FromUnicodeObject(PyObject *u, int base); -PyAPI_FUNC(PyObject *) _PyLong_FromBytes(const char *, Py_ssize_t, int); -#endif - -#ifndef Py_LIMITED_API -/* _PyLong_Sign. Return 0 if v is 0, -1 if v < 0, +1 if v > 0. - v must not be NULL, and must be a normalized long. - There are no error cases. -*/ -PyAPI_FUNC(int) _PyLong_Sign(PyObject *v); - - -/* _PyLong_NumBits. Return the number of bits needed to represent the - absolute value of a long. For example, this returns 1 for 1 and -1, 2 - for 2 and -2, and 2 for 3 and -3. It returns 0 for 0. - v must not be NULL, and must be a normalized long. - (size_t)-1 is returned and OverflowError set if the true result doesn't - fit in a size_t. -*/ -PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v); - -/* _PyLong_DivmodNear. Given integers a and b, compute the nearest - integer q to the exact quotient a / b, rounding to the nearest even integer - in the case of a tie. Return (q, r), where r = a - q*b. The remainder r - will satisfy abs(r) <= abs(b)/2, with equality possible only if q is - even. -*/ -PyAPI_FUNC(PyObject *) _PyLong_DivmodNear(PyObject *, PyObject *); - -/* _PyLong_FromByteArray: View the n unsigned bytes as a binary integer in - base 256, and return a Python int with the same numeric value. - If n is 0, the integer is 0. Else: - If little_endian is 1/true, bytes[n-1] is the MSB and bytes[0] the LSB; - else (little_endian is 0/false) bytes[0] is the MSB and bytes[n-1] the - LSB. - If is_signed is 0/false, view the bytes as a non-negative integer. - If is_signed is 1/true, view the bytes as a 2's-complement integer, - non-negative if bit 0x80 of the MSB is clear, negative if set. - Error returns: - + Return NULL with the appropriate exception set if there's not - enough memory to create the Python int. -*/ -PyAPI_FUNC(PyObject *) _PyLong_FromByteArray( - const unsigned char* bytes, size_t n, - int little_endian, int is_signed); - -/* _PyLong_AsByteArray: Convert the least-significant 8*n bits of long - v to a base-256 integer, stored in array bytes. Normally return 0, - return -1 on error. - If little_endian is 1/true, store the MSB at bytes[n-1] and the LSB at - bytes[0]; else (little_endian is 0/false) store the MSB at bytes[0] and - the LSB at bytes[n-1]. - If is_signed is 0/false, it's an error if v < 0; else (v >= 0) n bytes - are filled and there's nothing special about bit 0x80 of the MSB. - If is_signed is 1/true, bytes is filled with the 2's-complement - representation of v's value. Bit 0x80 of the MSB is the sign bit. - Error returns (-1): - + is_signed is 0 and v < 0. TypeError is set in this case, and bytes - isn't altered. - + n isn't big enough to hold the full mathematical value of v. For - example, if is_signed is 0 and there are more digits in the v than - fit in n; or if is_signed is 1, v < 0, and n is just 1 bit shy of - being large enough to hold a sign bit. OverflowError is set in this - case, but bytes holds the least-significant n bytes of the true value. -*/ -PyAPI_FUNC(int) _PyLong_AsByteArray(PyLongObject* v, - unsigned char* bytes, size_t n, - int little_endian, int is_signed); - -/* _PyLong_Format: Convert the long to a string object with given base, - appending a base prefix of 0[box] if base is 2, 8 or 16. */ -PyAPI_FUNC(PyObject *) _PyLong_Format(PyObject *obj, int base); - -PyAPI_FUNC(int) _PyLong_FormatWriter( - _PyUnicodeWriter *writer, - PyObject *obj, - int base, - int alternate); - -PyAPI_FUNC(char*) _PyLong_FormatBytesWriter( - _PyBytesWriter *writer, - char *str, - PyObject *obj, - int base, - int alternate); - -/* Format the object based on the format_spec, as defined in PEP 3101 - (Advanced String Formatting). */ -PyAPI_FUNC(int) _PyLong_FormatAdvancedWriter( - _PyUnicodeWriter *writer, - PyObject *obj, - PyObject *format_spec, - Py_ssize_t start, - Py_ssize_t end); -#endif /* Py_LIMITED_API */ /* These aren't really part of the int object, but they're handy. The functions are in Python/mystrtoul.c. @@ -205,13 +82,9 @@ PyAPI_FUNC(unsigned long) PyOS_strtoul(const char *, char **, int); PyAPI_FUNC(long) PyOS_strtol(const char *, char **, int); #ifndef Py_LIMITED_API -/* For use by the gcd function in mathmodule.c */ -PyAPI_FUNC(PyObject *) _PyLong_GCD(PyObject *, PyObject *); -#endif /* !Py_LIMITED_API */ - -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyLong_Rshift(PyObject *, size_t); -PyAPI_FUNC(PyObject *) _PyLong_Lshift(PyObject *, size_t); +# define Py_CPYTHON_LONGOBJECT_H +# include "cpython/longobject.h" +# undef Py_CPYTHON_LONGOBJECT_H #endif #ifdef __cplusplus diff --git a/Include/marshal.h b/Include/marshal.h index 09d9337e57b..f8b0de80cfc 100644 --- a/Include/marshal.h +++ b/Include/marshal.h @@ -3,26 +3,29 @@ #ifndef Py_MARSHAL_H #define Py_MARSHAL_H +#ifndef Py_LIMITED_API + #ifdef __cplusplus extern "C" { #endif -#define Py_MARSHAL_VERSION 4 - -PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int); -PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int); +PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, + Py_ssize_t); PyAPI_FUNC(PyObject *) PyMarshal_WriteObjectToString(PyObject *, int); -#ifndef Py_LIMITED_API +#define Py_MARSHAL_VERSION 4 + PyAPI_FUNC(long) PyMarshal_ReadLongFromFile(FILE *); PyAPI_FUNC(int) PyMarshal_ReadShortFromFile(FILE *); PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromFile(FILE *); PyAPI_FUNC(PyObject *) PyMarshal_ReadLastObjectFromFile(FILE *); -#endif -PyAPI_FUNC(PyObject *) PyMarshal_ReadObjectFromString(const char *, - Py_ssize_t); + +PyAPI_FUNC(void) PyMarshal_WriteLongToFile(long, FILE *, int); +PyAPI_FUNC(void) PyMarshal_WriteObjectToFile(PyObject *, FILE *, int); #ifdef __cplusplus } #endif + +#endif /* Py_LIMITED_API */ #endif /* !Py_MARSHAL_H */ diff --git a/Include/methodobject.h b/Include/methodobject.h index 9ffe8e1a3dd..1be5873a305 100644 --- a/Include/methodobject.h +++ b/Include/methodobject.h @@ -103,11 +103,9 @@ PyAPI_FUNC(PyObject *) PyCMethod_New(PyMethodDef *, PyObject *, #ifndef Py_LIMITED_API - -#define Py_CPYTHON_METHODOBJECT_H -#include "cpython/methodobject.h" -#undef Py_CPYTHON_METHODOBJECT_H - +# define Py_CPYTHON_METHODOBJECT_H +# include "cpython/methodobject.h" +# undef Py_CPYTHON_METHODOBJECT_H #endif #ifdef __cplusplus diff --git a/Include/modsupport.h b/Include/modsupport.h index 7d37b499422..baf47f0038d 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -7,7 +7,7 @@ extern "C" { /* Module support interface */ -#include +#include // va_list /* If PY_SSIZE_T_CLEAN is defined, each functions treats #-specifier to mean Py_ssize_t */ diff --git a/Include/namespaceobject.h b/Include/namespaceobject.h deleted file mode 100644 index 0c8d95c0f13..00000000000 --- a/Include/namespaceobject.h +++ /dev/null @@ -1,19 +0,0 @@ - -/* simple namespace object interface */ - -#ifndef NAMESPACEOBJECT_H -#define NAMESPACEOBJECT_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -PyAPI_DATA(PyTypeObject) _PyNamespace_Type; - -PyAPI_FUNC(PyObject *) _PyNamespace_New(PyObject *kwds); -#endif /* !Py_LIMITED_API */ - -#ifdef __cplusplus -} -#endif -#endif /* !NAMESPACEOBJECT_H */ diff --git a/Include/object.h b/Include/object.h index 23ebad84ab4..e5544e8b588 100644 --- a/Include/object.h +++ b/Include/object.h @@ -134,10 +134,16 @@ static inline Py_ssize_t _Py_REFCNT(const PyObject *ob) { // bpo-39573: The Py_SET_TYPE() function must be used to set an object type. -#define Py_TYPE(ob) (_PyObject_CAST(ob)->ob_type) +static inline PyTypeObject* _Py_TYPE(const PyObject *ob) { + return ob->ob_type; +} +#define Py_TYPE(ob) _Py_TYPE(_PyObject_CAST_CONST(ob)) // bpo-39573: The Py_SET_SIZE() function must be used to set an object size. -#define Py_SIZE(ob) (_PyVarObject_CAST(ob)->ob_size) +static inline Py_ssize_t _Py_SIZE(const PyVarObject *ob) { + return ob->ob_size; +} +#define Py_SIZE(ob) _Py_SIZE(_PyVarObject_CAST_CONST(ob)) static inline int _Py_IS_TYPE(const PyObject *ob, const PyTypeObject *type) { @@ -241,6 +247,7 @@ PyAPI_FUNC(void *) PyType_GetModuleState(struct _typeobject *); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030B0000 PyAPI_FUNC(PyObject *) PyType_GetName(PyTypeObject *); +PyAPI_FUNC(PyObject *) PyType_GetQualName(PyTypeObject *); #endif /* Generic type check */ @@ -326,6 +333,13 @@ given type object has a specified feature. */ #ifndef Py_LIMITED_API + +/* Placement of dict (and values) pointers are managed by the VM, not by the type. + * The VM will automatically set tp_dictoffset. Should not be used for variable sized + * classes, such as classes that extend tuple. + */ +#define Py_TPFLAGS_MANAGED_DICT (1 << 4) + /* Set if instances of the type object are treated as sequences for pattern matching */ #define Py_TPFLAGS_SEQUENCE (1 << 5) /* Set if instances of the type object are treated as mappings for pattern matching */ @@ -593,7 +607,7 @@ static inline PyObject* _Py_XNewRef(PyObject *obj) } // Py_NewRef() and Py_XNewRef() are exported as functions for the stable ABI. -// Names overriden with macros by static inline functions for best +// Names overridden with macros by static inline functions for best // performances. #define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj)) #define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj)) @@ -716,7 +730,7 @@ times. #ifndef Py_LIMITED_API # define Py_CPYTHON_OBJECT_H -# include "cpython/object.h" +# include "cpython/object.h" # undef Py_CPYTHON_OBJECT_H #endif diff --git a/Include/objimpl.h b/Include/objimpl.h index 450befad679..9b98c112ac2 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -205,7 +205,7 @@ PyAPI_FUNC(int) PyObject_GC_IsFinalized(PyObject *); #ifndef Py_LIMITED_API # define Py_CPYTHON_OBJIMPL_H -# include "cpython/objimpl.h" +# include "cpython/objimpl.h" # undef Py_CPYTHON_OBJIMPL_H #endif diff --git a/Include/opcode.h b/Include/opcode.h index 50b78328959..f22f7e94f61 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -6,153 +6,163 @@ extern "C" { #endif - /* Instruction opcodes for compiled code */ -#define POP_TOP 1 -#define ROT_TWO 2 -#define ROT_THREE 3 -#define DUP_TOP 4 -#define DUP_TOP_TWO 5 -#define ROT_FOUR 6 -#define NOP 9 -#define UNARY_POSITIVE 10 -#define UNARY_NEGATIVE 11 -#define UNARY_NOT 12 -#define UNARY_INVERT 15 -#define BINARY_MATRIX_MULTIPLY 16 -#define INPLACE_MATRIX_MULTIPLY 17 -#define BINARY_POWER 19 -#define BINARY_MULTIPLY 20 -#define BINARY_MODULO 22 -#define BINARY_ADD 23 -#define BINARY_SUBTRACT 24 -#define BINARY_SUBSCR 25 -#define BINARY_FLOOR_DIVIDE 26 -#define BINARY_TRUE_DIVIDE 27 -#define INPLACE_FLOOR_DIVIDE 28 -#define INPLACE_TRUE_DIVIDE 29 -#define GET_LEN 30 -#define MATCH_MAPPING 31 -#define MATCH_SEQUENCE 32 -#define MATCH_KEYS 33 -#define COPY_DICT_WITHOUT_KEYS 34 -#define PUSH_EXC_INFO 35 -#define POP_EXCEPT_AND_RERAISE 37 -#define WITH_EXCEPT_START 49 -#define GET_AITER 50 -#define GET_ANEXT 51 -#define BEFORE_ASYNC_WITH 52 -#define BEFORE_WITH 53 -#define END_ASYNC_FOR 54 -#define INPLACE_ADD 55 -#define INPLACE_SUBTRACT 56 -#define INPLACE_MULTIPLY 57 -#define INPLACE_MODULO 59 -#define STORE_SUBSCR 60 -#define DELETE_SUBSCR 61 -#define BINARY_LSHIFT 62 -#define BINARY_RSHIFT 63 -#define BINARY_AND 64 -#define BINARY_XOR 65 -#define BINARY_OR 66 -#define INPLACE_POWER 67 -#define GET_ITER 68 -#define GET_YIELD_FROM_ITER 69 -#define PRINT_EXPR 70 -#define LOAD_BUILD_CLASS 71 -#define YIELD_FROM 72 -#define GET_AWAITABLE 73 -#define LOAD_ASSERTION_ERROR 74 -#define INPLACE_LSHIFT 75 -#define INPLACE_RSHIFT 76 -#define INPLACE_AND 77 -#define INPLACE_XOR 78 -#define INPLACE_OR 79 -#define LIST_TO_TUPLE 82 -#define RETURN_VALUE 83 -#define IMPORT_STAR 84 -#define SETUP_ANNOTATIONS 85 -#define YIELD_VALUE 86 -#define POP_EXCEPT 89 -#define HAVE_ARGUMENT 90 -#define STORE_NAME 90 -#define DELETE_NAME 91 -#define UNPACK_SEQUENCE 92 -#define FOR_ITER 93 -#define UNPACK_EX 94 -#define STORE_ATTR 95 -#define DELETE_ATTR 96 -#define STORE_GLOBAL 97 -#define DELETE_GLOBAL 98 -#define ROT_N 99 -#define LOAD_CONST 100 -#define LOAD_NAME 101 -#define BUILD_TUPLE 102 -#define BUILD_LIST 103 -#define BUILD_SET 104 -#define BUILD_MAP 105 -#define LOAD_ATTR 106 -#define COMPARE_OP 107 -#define IMPORT_NAME 108 -#define IMPORT_FROM 109 -#define JUMP_FORWARD 110 -#define JUMP_IF_FALSE_OR_POP 111 -#define JUMP_IF_TRUE_OR_POP 112 -#define JUMP_ABSOLUTE 113 -#define POP_JUMP_IF_FALSE 114 -#define POP_JUMP_IF_TRUE 115 -#define LOAD_GLOBAL 116 -#define IS_OP 117 -#define CONTAINS_OP 118 -#define RERAISE 119 -#define JUMP_IF_NOT_EXC_MATCH 121 -#define LOAD_FAST 124 -#define STORE_FAST 125 -#define DELETE_FAST 126 -#define GEN_START 129 -#define RAISE_VARARGS 130 -#define CALL_FUNCTION 131 -#define MAKE_FUNCTION 132 -#define BUILD_SLICE 133 -#define MAKE_CELL 135 -#define LOAD_CLOSURE 136 -#define LOAD_DEREF 137 -#define STORE_DEREF 138 -#define DELETE_DEREF 139 -#define CALL_FUNCTION_KW 141 -#define CALL_FUNCTION_EX 142 -#define EXTENDED_ARG 144 -#define LIST_APPEND 145 -#define SET_ADD 146 -#define MAP_ADD 147 -#define LOAD_CLASSDEREF 148 -#define MATCH_CLASS 152 -#define FORMAT_VALUE 155 -#define BUILD_CONST_KEY_MAP 156 -#define BUILD_STRING 157 -#define LOAD_METHOD 160 -#define CALL_METHOD 161 -#define LIST_EXTEND 162 -#define SET_UPDATE 163 -#define DICT_MERGE 164 -#define DICT_UPDATE 165 -#define CALL_METHOD_KW 166 -#define BINARY_SUBSCR_ADAPTIVE 7 -#define BINARY_SUBSCR_LIST_INT 8 -#define BINARY_SUBSCR_TUPLE_INT 13 -#define BINARY_SUBSCR_DICT 14 -#define JUMP_ABSOLUTE_QUICK 18 -#define LOAD_ATTR_ADAPTIVE 21 -#define LOAD_ATTR_SPLIT_KEYS 36 -#define LOAD_ATTR_WITH_HINT 38 -#define LOAD_ATTR_SLOT 39 -#define LOAD_ATTR_MODULE 40 -#define LOAD_GLOBAL_ADAPTIVE 41 -#define LOAD_GLOBAL_MODULE 42 -#define LOAD_GLOBAL_BUILTIN 43 -#define STORE_ATTR_ADAPTIVE 44 -#define STORE_ATTR_SPLIT_KEYS 45 -#define STORE_ATTR_SLOT 46 -#define STORE_ATTR_WITH_HINT 47 +/* Instruction opcodes for compiled code */ +#define POP_TOP 1 +#define ROT_TWO 2 +#define ROT_THREE 3 +#define DUP_TOP 4 +#define DUP_TOP_TWO 5 +#define ROT_FOUR 6 +#define NOP 9 +#define UNARY_POSITIVE 10 +#define UNARY_NEGATIVE 11 +#define UNARY_NOT 12 +#define UNARY_INVERT 15 +#define BINARY_SUBSCR 25 +#define GET_LEN 30 +#define MATCH_MAPPING 31 +#define MATCH_SEQUENCE 32 +#define MATCH_KEYS 33 +#define PUSH_EXC_INFO 35 +#define POP_EXCEPT_AND_RERAISE 37 +#define WITH_EXCEPT_START 49 +#define GET_AITER 50 +#define GET_ANEXT 51 +#define BEFORE_ASYNC_WITH 52 +#define BEFORE_WITH 53 +#define END_ASYNC_FOR 54 +#define STORE_SUBSCR 60 +#define DELETE_SUBSCR 61 +#define GET_ITER 68 +#define GET_YIELD_FROM_ITER 69 +#define PRINT_EXPR 70 +#define LOAD_BUILD_CLASS 71 +#define YIELD_FROM 72 +#define GET_AWAITABLE 73 +#define LOAD_ASSERTION_ERROR 74 +#define LIST_TO_TUPLE 82 +#define RETURN_VALUE 83 +#define IMPORT_STAR 84 +#define SETUP_ANNOTATIONS 85 +#define YIELD_VALUE 86 +#define POP_EXCEPT 89 +#define HAVE_ARGUMENT 90 +#define STORE_NAME 90 +#define DELETE_NAME 91 +#define UNPACK_SEQUENCE 92 +#define FOR_ITER 93 +#define UNPACK_EX 94 +#define STORE_ATTR 95 +#define DELETE_ATTR 96 +#define STORE_GLOBAL 97 +#define DELETE_GLOBAL 98 +#define ROT_N 99 +#define LOAD_CONST 100 +#define LOAD_NAME 101 +#define BUILD_TUPLE 102 +#define BUILD_LIST 103 +#define BUILD_SET 104 +#define BUILD_MAP 105 +#define LOAD_ATTR 106 +#define COMPARE_OP 107 +#define IMPORT_NAME 108 +#define IMPORT_FROM 109 +#define JUMP_FORWARD 110 +#define JUMP_IF_FALSE_OR_POP 111 +#define JUMP_IF_TRUE_OR_POP 112 +#define JUMP_ABSOLUTE 113 +#define POP_JUMP_IF_FALSE 114 +#define POP_JUMP_IF_TRUE 115 +#define LOAD_GLOBAL 116 +#define IS_OP 117 +#define CONTAINS_OP 118 +#define RERAISE 119 +#define COPY 120 +#define JUMP_IF_NOT_EXC_MATCH 121 +#define BINARY_OP 122 +#define LOAD_FAST 124 +#define STORE_FAST 125 +#define DELETE_FAST 126 +#define GEN_START 129 +#define RAISE_VARARGS 130 +#define CALL_FUNCTION 131 +#define MAKE_FUNCTION 132 +#define BUILD_SLICE 133 +#define MAKE_CELL 135 +#define LOAD_CLOSURE 136 +#define LOAD_DEREF 137 +#define STORE_DEREF 138 +#define DELETE_DEREF 139 +#define CALL_FUNCTION_KW 141 +#define CALL_FUNCTION_EX 142 +#define EXTENDED_ARG 144 +#define LIST_APPEND 145 +#define SET_ADD 146 +#define MAP_ADD 147 +#define LOAD_CLASSDEREF 148 +#define COPY_FREE_VARS 149 +#define MATCH_CLASS 152 +#define FORMAT_VALUE 155 +#define BUILD_CONST_KEY_MAP 156 +#define BUILD_STRING 157 +#define LOAD_METHOD 160 +#define CALL_METHOD 161 +#define LIST_EXTEND 162 +#define SET_UPDATE 163 +#define DICT_MERGE 164 +#define DICT_UPDATE 165 +#define CALL_METHOD_KW 166 +#define BINARY_OP_ADAPTIVE 7 +#define BINARY_OP_ADD_INT 8 +#define BINARY_OP_ADD_FLOAT 13 +#define BINARY_OP_ADD_UNICODE 14 +#define BINARY_OP_INPLACE_ADD_UNICODE 16 +#define BINARY_OP_MULTIPLY_INT 17 +#define BINARY_OP_MULTIPLY_FLOAT 18 +#define BINARY_OP_SUBTRACT_INT 19 +#define BINARY_OP_SUBTRACT_FLOAT 20 +#define COMPARE_OP_ADAPTIVE 21 +#define COMPARE_OP_FLOAT_JUMP 22 +#define COMPARE_OP_INT_JUMP 23 +#define COMPARE_OP_STR_JUMP 24 +#define BINARY_SUBSCR_ADAPTIVE 26 +#define BINARY_SUBSCR_GETITEM 27 +#define BINARY_SUBSCR_LIST_INT 28 +#define BINARY_SUBSCR_TUPLE_INT 29 +#define BINARY_SUBSCR_DICT 34 +#define STORE_SUBSCR_ADAPTIVE 36 +#define STORE_SUBSCR_LIST_INT 38 +#define STORE_SUBSCR_DICT 39 +#define CALL_FUNCTION_ADAPTIVE 40 +#define CALL_FUNCTION_BUILTIN_O 41 +#define CALL_FUNCTION_BUILTIN_FAST 42 +#define CALL_FUNCTION_LEN 43 +#define CALL_FUNCTION_ISINSTANCE 44 +#define CALL_FUNCTION_PY_SIMPLE 45 +#define JUMP_ABSOLUTE_QUICK 46 +#define LOAD_ATTR_ADAPTIVE 47 +#define LOAD_ATTR_INSTANCE_VALUE 48 +#define LOAD_ATTR_WITH_HINT 55 +#define LOAD_ATTR_SLOT 56 +#define LOAD_ATTR_MODULE 57 +#define LOAD_GLOBAL_ADAPTIVE 58 +#define LOAD_GLOBAL_MODULE 59 +#define LOAD_GLOBAL_BUILTIN 62 +#define LOAD_METHOD_ADAPTIVE 63 +#define LOAD_METHOD_CACHED 64 +#define LOAD_METHOD_CLASS 65 +#define LOAD_METHOD_MODULE 66 +#define LOAD_METHOD_NO_DICT 67 +#define STORE_ATTR_ADAPTIVE 75 +#define STORE_ATTR_INSTANCE_VALUE 76 +#define STORE_ATTR_SLOT 77 +#define STORE_ATTR_WITH_HINT 78 +#define LOAD_FAST__LOAD_FAST 79 +#define STORE_FAST__LOAD_FAST 80 +#define LOAD_FAST__LOAD_CONST 81 +#define LOAD_CONST__LOAD_FAST 87 +#define STORE_FAST__STORE_FAST 88 +#define DO_TRACING 255 #ifdef NEED_OPCODE_JUMP_TABLES static uint32_t _PyOpcode_RelativeJump[8] = { 0U, @@ -176,6 +186,37 @@ static uint32_t _PyOpcode_Jump[8] = { }; #endif /* OPCODE_TABLES */ +#define HAS_CONST(op) (false\ + || ((op) == 100) \ + ) + +#define NB_ADD 0 +#define NB_AND 1 +#define NB_FLOOR_DIVIDE 2 +#define NB_LSHIFT 3 +#define NB_MATRIX_MULTIPLY 4 +#define NB_MULTIPLY 5 +#define NB_REMAINDER 6 +#define NB_OR 7 +#define NB_POWER 8 +#define NB_RSHIFT 9 +#define NB_SUBTRACT 10 +#define NB_TRUE_DIVIDE 11 +#define NB_XOR 12 +#define NB_INPLACE_ADD 13 +#define NB_INPLACE_AND 14 +#define NB_INPLACE_FLOOR_DIVIDE 15 +#define NB_INPLACE_LSHIFT 16 +#define NB_INPLACE_MATRIX_MULTIPLY 17 +#define NB_INPLACE_MULTIPLY 18 +#define NB_INPLACE_REMAINDER 19 +#define NB_INPLACE_OR 20 +#define NB_INPLACE_POWER 21 +#define NB_INPLACE_RSHIFT 22 +#define NB_INPLACE_SUBTRACT 23 +#define NB_INPLACE_TRUE_DIVIDE 24 +#define NB_INPLACE_XOR 25 + #define HAS_ARG(op) ((op) >= HAVE_ARGUMENT) /* Reserve some bytecodes for internal use in the compiler. diff --git a/Include/patchlevel.h b/Include/patchlevel.h index f37c4d48e37..70205dbac07 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 11 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 0 +#define PY_RELEASE_SERIAL 3 /* Version as a string */ -#define PY_VERSION "3.11.0a0" +#define PY_VERSION "3.11.0a3+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Include/pyerrors.h b/Include/pyerrors.h index f5d1c711577..77d791427d4 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -60,11 +60,14 @@ PyAPI_FUNC(const char *) PyExceptionClass_Name(PyObject *); #define PyExceptionInstance_Class(x) ((PyObject*)Py_TYPE(x)) +#define _PyBaseExceptionGroup_Check(x) \ + PyObject_TypeCheck(x, (PyTypeObject *)PyExc_BaseExceptionGroup) /* Predefined exceptions */ PyAPI_DATA(PyObject *) PyExc_BaseException; PyAPI_DATA(PyObject *) PyExc_Exception; +PyAPI_DATA(PyObject *) PyExc_BaseExceptionGroup; #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000 PyAPI_DATA(PyObject *) PyExc_StopAsyncIteration; #endif @@ -314,7 +317,7 @@ PyAPI_FUNC(int) PyOS_vsnprintf(char *str, size_t size, const char *format, va_l #ifndef Py_LIMITED_API # define Py_CPYTHON_ERRORS_H -# include "cpython/pyerrors.h" +# include "cpython/pyerrors.h" # undef Py_CPYTHON_ERRORS_H #endif diff --git a/Include/pyhash.h b/Include/pyhash.h index a314ea907b7..182d223fab1 100644 --- a/Include/pyhash.h +++ b/Include/pyhash.h @@ -114,11 +114,10 @@ PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); /* hash algorithm selection * - * The values for Py_HASH_SIPHASH24 and Py_HASH_FNV are hard-coded in the + * The values for Py_HASH_* are hard-coded in the * configure script. * - * - FNV is available on all platforms and architectures. - * - SIPHASH24 only works on platforms that don't require aligned memory for integers. + * - FNV and SIPHASH* are available on all platforms and architectures. * - With EXTERNAL embedders can provide an alternative implementation with:: * * PyHash_FuncDef PyHash_Func = {...}; @@ -128,10 +127,11 @@ PyAPI_FUNC(PyHash_FuncDef*) PyHash_GetFuncDef(void); #define Py_HASH_EXTERNAL 0 #define Py_HASH_SIPHASH24 1 #define Py_HASH_FNV 2 +#define Py_HASH_SIPHASH13 3 #ifndef Py_HASH_ALGORITHM # ifndef HAVE_ALIGNED_REQUIRED -# define Py_HASH_ALGORITHM Py_HASH_SIPHASH24 +# define Py_HASH_ALGORITHM Py_HASH_SIPHASH13 # else # define Py_HASH_ALGORITHM Py_HASH_FNV # endif /* uint64_t && uint32_t && aligned */ diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 9b2dd0868eb..e4c3b09c963 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -62,9 +62,13 @@ typedef void (*PyOS_sighandler_t)(int); PyAPI_FUNC(PyOS_sighandler_t) PyOS_getsig(int); PyAPI_FUNC(PyOS_sighandler_t) PyOS_setsig(int, PyOS_sighandler_t); +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030B0000 +PyAPI_DATA(const unsigned long) Py_Version; +#endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_PYLIFECYCLE_H -# include "cpython/pylifecycle.h" +# include "cpython/pylifecycle.h" # undef Py_CPYTHON_PYLIFECYCLE_H #endif diff --git a/Include/pymacro.h b/Include/pymacro.h index 202b936d964..2728496976d 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -129,4 +129,8 @@ Py_FatalError("Unreachable C code path reached") #endif +// Prevent using an expression as a l-value. +// For example, "int x; _Py_RVALUE(x) = 1;" fails with a compiler error. +#define _Py_RVALUE(EXPR) ((void)0, (EXPR)) + #endif /* Py_PYMACRO_H */ diff --git a/Include/pymath.h b/Include/pymath.h index f869724334a..57310fc097e 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -1,42 +1,9 @@ +// Symbols and macros to supply platform-independent interfaces to mathematical +// functions and constants. + #ifndef Py_PYMATH_H #define Py_PYMATH_H -#include "pyconfig.h" /* include for defines */ - -/************************************************************************** -Symbols and macros to supply platform-independent interfaces to mathematical -functions and constants -**************************************************************************/ - -/* Python provides implementations for copysign, round and hypot in - * Python/pymath.c just in case your math library doesn't provide the - * functions. - * - *Note: PC/pyconfig.h defines copysign as _copysign - */ -#ifndef HAVE_COPYSIGN -extern double copysign(double, double); -#endif - -#ifndef HAVE_ROUND -extern double round(double); -#endif - -#ifndef HAVE_HYPOT -extern double hypot(double, double); -#endif - -/* extra declarations */ -#ifndef _MSC_VER -#ifndef __STDC__ -extern double fmod (double, double); -extern double frexp (double, int *); -extern double ldexp (double, int); -extern double modf (double, double *); -extern double pow(double, double); -#endif /* __STDC__ */ -#endif /* _MSC_VER */ - /* High precision definition of pi and e (Euler) * The values are taken from libc6's math.h. */ @@ -60,84 +27,17 @@ extern double pow(double, double); #define Py_MATH_TAU 6.2831853071795864769252867665590057683943L #endif - -/* On x86, Py_FORCE_DOUBLE forces a floating-point number out of an x87 FPU - register and into a 64-bit memory location, rounding from extended - precision to double precision in the process. On other platforms it does - nothing. */ - -/* we take double rounding as evidence of x87 usage */ -#ifndef Py_LIMITED_API -#ifndef Py_FORCE_DOUBLE -# ifdef X87_DOUBLE_ROUNDING -PyAPI_FUNC(double) _Py_force_double(double); -# define Py_FORCE_DOUBLE(X) (_Py_force_double(X)) -# else -# define Py_FORCE_DOUBLE(X) (X) -# endif -#endif -#endif - -#ifndef Py_LIMITED_API -#ifdef HAVE_GCC_ASM_FOR_X87 -PyAPI_FUNC(unsigned short) _Py_get_387controlword(void); -PyAPI_FUNC(void) _Py_set_387controlword(unsigned short); -#endif -#endif - -/* Py_IS_NAN(X) - * Return 1 if float or double arg is a NaN, else 0. - * Caution: - * X is evaluated more than once. - * This may not work on all platforms. Each platform has *some* - * way to spell this, though -- override in pyconfig.h if you have - * a platform where it doesn't work. - * Note: PC/pyconfig.h defines Py_IS_NAN as _isnan - */ -#ifndef Py_IS_NAN -#if defined HAVE_DECL_ISNAN && HAVE_DECL_ISNAN == 1 +// Py_IS_NAN(X) +// Return 1 if float or double arg is a NaN, else 0. #define Py_IS_NAN(X) isnan(X) -#else -#define Py_IS_NAN(X) ((X) != (X)) -#endif -#endif -/* Py_IS_INFINITY(X) - * Return 1 if float or double arg is an infinity, else 0. - * Caution: - * X is evaluated more than once. - * This implementation may set the underflow flag if |X| is very small; - * it really can't be implemented correctly (& easily) before C99. - * Override in pyconfig.h if you have a better spelling on your platform. - * Py_FORCE_DOUBLE is used to avoid getting false negatives from a - * non-infinite value v sitting in an 80-bit x87 register such that - * v becomes infinite when spilled from the register to 64-bit memory. - * Note: PC/pyconfig.h defines Py_IS_INFINITY as _isinf - */ -#ifndef Py_IS_INFINITY -# if defined HAVE_DECL_ISINF && HAVE_DECL_ISINF == 1 -# define Py_IS_INFINITY(X) isinf(X) -# else -# define Py_IS_INFINITY(X) ((X) && \ - (Py_FORCE_DOUBLE(X)*0.5 == Py_FORCE_DOUBLE(X))) -# endif -#endif +// Py_IS_INFINITY(X) +// Return 1 if float or double arg is an infinity, else 0. +#define Py_IS_INFINITY(X) isinf(X) -/* Py_IS_FINITE(X) - * Return 1 if float or double arg is neither infinite nor NAN, else 0. - * Some compilers (e.g. VisualStudio) have intrinsics for this, so a special - * macro for this particular test is useful - * Note: PC/pyconfig.h defines Py_IS_FINITE as _finite - */ -#ifndef Py_IS_FINITE -#if defined HAVE_DECL_ISFINITE && HAVE_DECL_ISFINITE == 1 +// Py_IS_FINITE(X) +// Return 1 if float or double arg is neither infinite nor NAN, else 0. #define Py_IS_FINITE(X) isfinite(X) -#elif defined HAVE_FINITE -#define Py_IS_FINITE(X) finite(X) -#else -#define Py_IS_FINITE(X) (!Py_IS_INFINITY(X) && !Py_IS_NAN(X)) -#endif -#endif /* HUGE_VAL is supposed to expand to a positive double infinity. Python * uses Py_HUGE_VAL instead because some platforms are broken in this @@ -147,7 +47,7 @@ PyAPI_FUNC(void) _Py_set_387controlword(unsigned short); * config to #define Py_HUGE_VAL to something that works on your platform. */ #ifndef Py_HUGE_VAL -#define Py_HUGE_VAL HUGE_VAL +# define Py_HUGE_VAL HUGE_VAL #endif /* Py_NAN @@ -156,75 +56,24 @@ PyAPI_FUNC(void) _Py_set_387controlword(unsigned short); * doesn't support NaNs. */ #if !defined(Py_NAN) && !defined(Py_NO_NAN) -#if !defined(__INTEL_COMPILER) - #define Py_NAN (Py_HUGE_VAL * 0.) -#else /* __INTEL_COMPILER */ - #if defined(ICC_NAN_STRICT) +# if !defined(__INTEL_COMPILER) +# define Py_NAN (Py_HUGE_VAL * 0.) +# else /* __INTEL_COMPILER */ +# if defined(ICC_NAN_STRICT) #pragma float_control(push) #pragma float_control(precise, on) #pragma float_control(except, on) - #if defined(_MSC_VER) - __declspec(noinline) - #else /* Linux */ - __attribute__((noinline)) - #endif /* _MSC_VER */ - static double __icc_nan() + Py_NO_INLINE static double __icc_nan() { return sqrt(-1.0); } #pragma float_control (pop) - #define Py_NAN __icc_nan() - #else /* ICC_NAN_RELAXED as default for Intel Compiler */ +# define Py_NAN __icc_nan() +# else /* ICC_NAN_RELAXED as default for Intel Compiler */ static const union { unsigned char buf[8]; double __icc_nan; } __nan_store = {0,0,0,0,0,0,0xf8,0x7f}; - #define Py_NAN (__nan_store.__icc_nan) - #endif /* ICC_NAN_STRICT */ -#endif /* __INTEL_COMPILER */ +# define Py_NAN (__nan_store.__icc_nan) +# endif /* ICC_NAN_STRICT */ +# endif /* __INTEL_COMPILER */ #endif -/* Py_OVERFLOWED(X) - * Return 1 iff a libm function overflowed. Set errno to 0 before calling - * a libm function, and invoke this macro after, passing the function - * result. - * Caution: - * This isn't reliable. C99 no longer requires libm to set errno under - * any exceptional condition, but does require +- HUGE_VAL return - * values on overflow. A 754 box *probably* maps HUGE_VAL to a - * double infinity, and we're cool if that's so, unless the input - * was an infinity and an infinity is the expected result. A C89 - * system sets errno to ERANGE, so we check for that too. We're - * out of luck if a C99 754 box doesn't map HUGE_VAL to +Inf, or - * if the returned result is a NaN, or if a C89 box returns HUGE_VAL - * in non-overflow cases. - * X is evaluated more than once. - * Some platforms have better way to spell this, so expect some #ifdef'ery. - * - * OpenBSD uses 'isinf()' because a compiler bug on that platform causes - * the longer macro version to be mis-compiled. This isn't optimal, and - * should be removed once a newer compiler is available on that platform. - * The system that had the failure was running OpenBSD 3.2 on Intel, with - * gcc 2.95.3. - * - * According to Tim's checkin, the FreeBSD systems use isinf() to work - * around a FPE bug on that platform. - */ -#if defined(__FreeBSD__) || defined(__OpenBSD__) -#define Py_OVERFLOWED(X) isinf(X) -#else -#define Py_OVERFLOWED(X) ((X) != 0.0 && (errno == ERANGE || \ - (X) == Py_HUGE_VAL || \ - (X) == -Py_HUGE_VAL)) -#endif - -/* Return whether integral type *type* is signed or not. */ -#define _Py_IntegralTypeSigned(type) ((type)(-1) < 0) -/* Return the maximum value of integral type *type*. */ -#define _Py_IntegralTypeMax(type) ((_Py_IntegralTypeSigned(type)) ? (((((type)1 << (sizeof(type)*CHAR_BIT - 2)) - 1) << 1) + 1) : ~(type)0) -/* Return the minimum value of integral type *type*. */ -#define _Py_IntegralTypeMin(type) ((_Py_IntegralTypeSigned(type)) ? -_Py_IntegralTypeMax(type) - 1 : 0) -/* Check whether *v* is in the range of integral type *type*. This is most - * useful if *v* is floating-point, since demoting a floating-point *v* to an - * integral type that cannot represent *v*'s integral part is undefined - * behavior. */ -#define _Py_InIntegralTypeRange(type, v) (_Py_IntegralTypeMin(type) <= v && v <= _Py_IntegralTypeMax(type)) - #endif /* Py_PYMATH_H */ diff --git a/Include/pymem.h b/Include/pymem.h index 66cdb0d2973..c15ad10dfcf 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -93,7 +93,7 @@ PyAPI_FUNC(void) PyMem_Free(void *ptr); #ifndef Py_LIMITED_API # define Py_CPYTHON_PYMEM_H -# include "cpython/pymem.h" +# include "cpython/pymem.h" # undef Py_CPYTHON_PYMEM_H #endif diff --git a/Include/pyport.h b/Include/pyport.h index b2b53dd2f77..81b1bde841e 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -5,6 +5,14 @@ #include +#include +#ifndef UCHAR_MAX +# error "limits.h must define UCHAR_MAX" +#endif +#if UCHAR_MAX != 255 +# error "Python's source code assumes C's unsigned char is an 8-bit type" +#endif + /* Defines to build Python and its standard library: * @@ -79,13 +87,17 @@ Used in: Py_SAFE_DOWNCAST /* If PYLONG_BITS_IN_DIGIT is not defined then we'll use 30-bit digits if all the necessary integer types are available, and we're on a 64-bit platform - (as determined by SIZEOF_VOID_P); otherwise we use 15-bit digits. */ + (as determined by SIZEOF_VOID_P); otherwise we use 15-bit digits. + + From pyodide: WASM has 32 bit pointers but has native 64 bit arithmetic + so it is more efficient to use 30 bit digits. + */ #ifndef PYLONG_BITS_IN_DIGIT -#if SIZEOF_VOID_P >= 8 -#define PYLONG_BITS_IN_DIGIT 30 +#if SIZEOF_VOID_P >= 8 || defined(__wasm__) +# define PYLONG_BITS_IN_DIGIT 30 #else -#define PYLONG_BITS_IN_DIGIT 15 +# define PYLONG_BITS_IN_DIGIT 15 #endif #endif @@ -193,11 +205,10 @@ typedef Py_ssize_t Py_ssize_clean_t; # define Py_LOCAL_INLINE(type) static inline type #endif -/* Py_MEMCPY is kept for backwards compatibility, - * see https://bugs.python.org/issue28126 */ -#define Py_MEMCPY memcpy - -#include +// bpo-28126: Py_MEMCPY is kept for backwards compatibility, +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 +# define Py_MEMCPY memcpy +#endif #ifdef HAVE_IEEEFP_H #include /* needed for 'finite' declaration on some platforms */ @@ -209,17 +220,10 @@ typedef Py_ssize_t Py_ssize_clean_t; * WRAPPER FOR and/or * ********************************************/ -#ifdef TIME_WITH_SYS_TIME -#include -#include -#else /* !TIME_WITH_SYS_TIME */ #ifdef HAVE_SYS_TIME_H #include -#else /* !HAVE_SYS_TIME_H */ +#endif #include -#endif /* !HAVE_SYS_TIME_H */ -#endif /* !TIME_WITH_SYS_TIME */ - /****************************** * WRAPPER FOR * @@ -316,69 +320,6 @@ extern "C" { #define Py_SAFE_DOWNCAST(VALUE, WIDE, NARROW) (NARROW)(VALUE) #endif -/* Py_SET_ERRNO_ON_MATH_ERROR(x) - * If a libm function did not set errno, but it looks like the result - * overflowed or not-a-number, set errno to ERANGE or EDOM. Set errno - * to 0 before calling a libm function, and invoke this macro after, - * passing the function result. - * Caution: - * This isn't reliable. See Py_OVERFLOWED comments. - * X is evaluated more than once. - */ -#if defined(__FreeBSD__) || defined(__OpenBSD__) || (defined(__hpux) && defined(__ia64)) -#define _Py_SET_EDOM_FOR_NAN(X) if (isnan(X)) errno = EDOM; -#else -#define _Py_SET_EDOM_FOR_NAN(X) ; -#endif -#define Py_SET_ERRNO_ON_MATH_ERROR(X) \ - do { \ - if (errno == 0) { \ - if ((X) == Py_HUGE_VAL || (X) == -Py_HUGE_VAL) \ - errno = ERANGE; \ - else _Py_SET_EDOM_FOR_NAN(X) \ - } \ - } while(0) - -/* Py_SET_ERANGE_IF_OVERFLOW(x) - * An alias of Py_SET_ERRNO_ON_MATH_ERROR for backward-compatibility. - */ -#define Py_SET_ERANGE_IF_OVERFLOW(X) Py_SET_ERRNO_ON_MATH_ERROR(X) - -/* Py_ADJUST_ERANGE1(x) - * Py_ADJUST_ERANGE2(x, y) - * Set errno to 0 before calling a libm function, and invoke one of these - * macros after, passing the function result(s) (Py_ADJUST_ERANGE2 is useful - * for functions returning complex results). This makes two kinds of - * adjustments to errno: (A) If it looks like the platform libm set - * errno=ERANGE due to underflow, clear errno. (B) If it looks like the - * platform libm overflowed but didn't set errno, force errno to ERANGE. In - * effect, we're trying to force a useful implementation of C89 errno - * behavior. - * Caution: - * This isn't reliable. See Py_OVERFLOWED comments. - * X and Y may be evaluated more than once. - */ -#define Py_ADJUST_ERANGE1(X) \ - do { \ - if (errno == 0) { \ - if ((X) == Py_HUGE_VAL || (X) == -Py_HUGE_VAL) \ - errno = ERANGE; \ - } \ - else if (errno == ERANGE && (X) == 0.0) \ - errno = 0; \ - } while(0) - -#define Py_ADJUST_ERANGE2(X, Y) \ - do { \ - if ((X) == Py_HUGE_VAL || (X) == -Py_HUGE_VAL || \ - (Y) == Py_HUGE_VAL || (Y) == -Py_HUGE_VAL) { \ - if (errno == 0) \ - errno = ERANGE; \ - } \ - else if (errno == ERANGE) \ - errno = 0; \ - } while(0) - /* The functions _Py_dg_strtod and _Py_dg_dtoa in Python/dtoa.c (which are * required to support the short float repr introduced in Python 3.1) require * that the floating-point unit that's being used for arithmetic operations @@ -390,85 +331,24 @@ extern "C" { * * #define HAVE_PY_SET_53BIT_PRECISION 1 * - * and also give appropriate definitions for the following three macros: - * - * _PY_SET_53BIT_PRECISION_START : store original FPU settings, and - * set FPU to 53-bit precision/round-half-to-even - * _PY_SET_53BIT_PRECISION_END : restore original FPU settings - * _PY_SET_53BIT_PRECISION_HEADER : any variable declarations needed to - * use the two macros above. - * * The macros are designed to be used within a single C function: see * Python/pystrtod.c for an example of their use. */ -/* get and set x87 control word for gcc/x86 */ +// HAVE_PY_SET_53BIT_PRECISION macro must be kept in sync with pycore_pymath.h #ifdef HAVE_GCC_ASM_FOR_X87 -#define HAVE_PY_SET_53BIT_PRECISION 1 -/* _Py_get/set_387controlword functions are defined in Python/pymath.c */ -#define _Py_SET_53BIT_PRECISION_HEADER \ - unsigned short old_387controlword, new_387controlword -#define _Py_SET_53BIT_PRECISION_START \ - do { \ - old_387controlword = _Py_get_387controlword(); \ - new_387controlword = (old_387controlword & ~0x0f00) | 0x0200; \ - if (new_387controlword != old_387controlword) \ - _Py_set_387controlword(new_387controlword); \ - } while (0) -#define _Py_SET_53BIT_PRECISION_END \ - if (new_387controlword != old_387controlword) \ - _Py_set_387controlword(old_387controlword) + // Get and set x87 control word for gcc/x86 +# define HAVE_PY_SET_53BIT_PRECISION 1 #endif - -/* get and set x87 control word for VisualStudio/x86 */ -#if defined(_MSC_VER) && !defined(_WIN64) && !defined(_M_ARM) /* x87 not supported in 64-bit or ARM */ -#define HAVE_PY_SET_53BIT_PRECISION 1 -#define _Py_SET_53BIT_PRECISION_HEADER \ - unsigned int old_387controlword, new_387controlword, out_387controlword -/* We use the __control87_2 function to set only the x87 control word. - The SSE control word is unaffected. */ -#define _Py_SET_53BIT_PRECISION_START \ - do { \ - __control87_2(0, 0, &old_387controlword, NULL); \ - new_387controlword = \ - (old_387controlword & ~(_MCW_PC | _MCW_RC)) | (_PC_53 | _RC_NEAR); \ - if (new_387controlword != old_387controlword) \ - __control87_2(new_387controlword, _MCW_PC | _MCW_RC, \ - &out_387controlword, NULL); \ - } while (0) -#define _Py_SET_53BIT_PRECISION_END \ - do { \ - if (new_387controlword != old_387controlword) \ - __control87_2(old_387controlword, _MCW_PC | _MCW_RC, \ - &out_387controlword, NULL); \ - } while (0) +#if defined(_MSC_VER) && !defined(_WIN64) && !defined(_M_ARM) + // Get and set x87 control word for VisualStudio/x86. + // x87 not supported in 64-bit or ARM. +# define HAVE_PY_SET_53BIT_PRECISION 1 #endif - #ifdef HAVE_GCC_ASM_FOR_MC68881 -#define HAVE_PY_SET_53BIT_PRECISION 1 -#define _Py_SET_53BIT_PRECISION_HEADER \ - unsigned int old_fpcr, new_fpcr -#define _Py_SET_53BIT_PRECISION_START \ - do { \ - __asm__ ("fmove.l %%fpcr,%0" : "=g" (old_fpcr)); \ - /* Set double precision / round to nearest. */ \ - new_fpcr = (old_fpcr & ~0xf0) | 0x80; \ - if (new_fpcr != old_fpcr) \ - __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (new_fpcr)); \ - } while (0) -#define _Py_SET_53BIT_PRECISION_END \ - do { \ - if (new_fpcr != old_fpcr) \ - __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (old_fpcr)); \ - } while (0) +# define HAVE_PY_SET_53BIT_PRECISION 1 #endif -/* default definitions are empty */ -#ifndef HAVE_PY_SET_53BIT_PRECISION -#define _Py_SET_53BIT_PRECISION_HEADER -#define _Py_SET_53BIT_PRECISION_START -#define _Py_SET_53BIT_PRECISION_END -#endif /* If we can't guarantee 53-bit precision, don't use the code in Python/dtoa.c, but fall back to standard code. This @@ -485,14 +365,14 @@ extern "C" { #if !defined(DOUBLE_IS_LITTLE_ENDIAN_IEEE754) && \ !defined(DOUBLE_IS_BIG_ENDIAN_IEEE754) && \ !defined(DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754) -#define PY_NO_SHORT_FLOAT_REPR +# define PY_NO_SHORT_FLOAT_REPR #endif /* double rounding is symptomatic of use of extended precision on x86. If we're seeing double rounding, and we don't have any mechanism available for changing the FPU rounding precision, then don't use Python/dtoa.c. */ #if defined(X87_DOUBLE_ROUNDING) && !defined(HAVE_PY_SET_53BIT_PRECISION) -#define PY_NO_SHORT_FLOAT_REPR +# define PY_NO_SHORT_FLOAT_REPR #endif @@ -557,19 +437,51 @@ extern "C" { #define _Py_HOT_FUNCTION #endif -/* _Py_NO_INLINE - * Disable inlining on a function. For example, it helps to reduce the C stack - * consumption. - * - * Usage: - * int _Py_NO_INLINE x(void) { return 3; } - */ -#if defined(_MSC_VER) -# define _Py_NO_INLINE __declspec(noinline) -#elif defined(__GNUC__) || defined(__clang__) -# define _Py_NO_INLINE __attribute__ ((noinline)) +// Ask the compiler to always inline a static inline function. The compiler can +// ignore it and decides to not inline the function. +// +// It can be used to inline performance critical static inline functions when +// building Python in debug mode with function inlining disabled. For example, +// MSC disables function inlining when building in debug mode. +// +// Marking blindly a static inline function with Py_ALWAYS_INLINE can result in +// worse performances (due to increased code size for example). The compiler is +// usually smarter than the developer for the cost/benefit analysis. +// +// If Python is built in debug mode (if the Py_DEBUG macro is defined), the +// Py_ALWAYS_INLINE macro does nothing. +// +// It must be specified before the function return type. Usage: +// +// static inline Py_ALWAYS_INLINE int random(void) { return 4; } +#if defined(Py_DEBUG) + // If Python is built in debug mode, usually compiler optimizations are + // disabled. In this case, Py_ALWAYS_INLINE can increase a lot the stack + // memory usage. For example, forcing inlining using gcc -O0 increases the + // stack usage from 6 KB to 15 KB per Python function call. +# define Py_ALWAYS_INLINE +#elif defined(__GNUC__) || defined(__clang__) || defined(__INTEL_COMPILER) +# define Py_ALWAYS_INLINE __attribute__((always_inline)) +#elif defined(_MSC_VER) +# define Py_ALWAYS_INLINE __forceinline #else -# define _Py_NO_INLINE +# define Py_ALWAYS_INLINE +#endif + +// Py_NO_INLINE +// Disable inlining on a function. For example, it reduces the C stack +// consumption: useful on LTO+PGO builds which heavily inline code (see +// bpo-33720). +// +// Usage: +// +// Py_NO_INLINE static int random(void) { return 4; } +#if defined(__GNUC__) || defined(__clang__) || defined(__INTEL_COMPILER) +# define Py_NO_INLINE __attribute__ ((noinline)) +#elif defined(_MSC_VER) +# define Py_NO_INLINE __declspec(noinline) +#else +# define Py_NO_INLINE #endif /************************************************************************** @@ -796,26 +708,6 @@ extern char * _getpty(int *, int, mode_t, int); # define PY_LITTLE_ENDIAN 1 #endif -#ifdef Py_BUILD_CORE -/* - * Macros to protect CRT calls against instant termination when passed an - * invalid parameter (issue23524). - */ -#if defined _MSC_VER && _MSC_VER >= 1900 - -extern _invalid_parameter_handler _Py_silent_invalid_parameter_handler; -#define _Py_BEGIN_SUPPRESS_IPH { _invalid_parameter_handler _Py_old_handler = \ - _set_thread_local_invalid_parameter_handler(_Py_silent_invalid_parameter_handler); -#define _Py_END_SUPPRESS_IPH _set_thread_local_invalid_parameter_handler(_Py_old_handler); } - -#else - -#define _Py_BEGIN_SUPPRESS_IPH -#define _Py_END_SUPPRESS_IPH - -#endif /* _MSC_VER >= 1900 */ -#endif /* Py_BUILD_CORE */ - #ifdef __ANDROID__ /* The Android langinfo.h header is not used. */ # undef HAVE_LANGINFO_H @@ -882,4 +774,22 @@ extern _invalid_parameter_handler _Py_silent_invalid_parameter_handler; #endif +/* A convenient way for code to know if sanitizers are enabled. */ +#if defined(__has_feature) +# if __has_feature(memory_sanitizer) +# if !defined(_Py_MEMORY_SANITIZER) +# define _Py_MEMORY_SANITIZER +# endif +# endif +# if __has_feature(address_sanitizer) +# if !defined(_Py_ADDRESS_SANITIZER) +# define _Py_ADDRESS_SANITIZER +# endif +# endif +#elif defined(__GNUC__) +# if defined(__SANITIZE_ADDRESS__) +# define _Py_ADDRESS_SANITIZER +# endif +#endif + #endif /* Py_PYPORT_H */ diff --git a/Include/pystate.h b/Include/pystate.h index bae440778b2..b6ee0ede81d 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -66,18 +66,10 @@ PyAPI_FUNC(void) PyThreadState_Delete(PyThreadState *); The caller must hold the GIL. - See also PyThreadState_GET() and _PyThreadState_GET(). */ + See also _PyThreadState_UncheckedGet() and _PyThreadState_GET(). */ PyAPI_FUNC(PyThreadState *) PyThreadState_Get(void); -/* Get the current Python thread state. - - Macro using PyThreadState_Get() or _PyThreadState_GET() depending if - pycore_pystate.h is included or not (this header redefines the macro). - - If PyThreadState_Get() is used, issue a fatal error if the current thread - state is NULL. - - See also PyThreadState_Get() and _PyThreadState_GET(). */ +// Alias to PyThreadState_Get() #define PyThreadState_GET() PyThreadState_Get() PyAPI_FUNC(PyThreadState *) PyThreadState_Swap(PyThreadState *); @@ -140,7 +132,7 @@ PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); #ifndef Py_LIMITED_API # define Py_CPYTHON_PYSTATE_H -# include "cpython/pystate.h" +# include "cpython/pystate.h" # undef Py_CPYTHON_PYSTATE_H #endif diff --git a/Include/pystrhex.h b/Include/pystrhex.h deleted file mode 100644 index a4f36305bac..00000000000 --- a/Include/pystrhex.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef Py_STRHEX_H -#define Py_STRHEX_H - -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -/* Returns a str() containing the hex representation of argbuf. */ -PyAPI_FUNC(PyObject*) _Py_strhex(const char* argbuf, const Py_ssize_t arglen); -/* Returns a bytes() containing the ASCII hex representation of argbuf. */ -PyAPI_FUNC(PyObject*) _Py_strhex_bytes(const char* argbuf, const Py_ssize_t arglen); -/* These variants include support for a separator between every N bytes: */ -PyAPI_FUNC(PyObject*) _Py_strhex_with_sep(const char* argbuf, const Py_ssize_t arglen, const PyObject* sep, const int bytes_per_group); -PyAPI_FUNC(PyObject*) _Py_strhex_bytes_with_sep(const char* argbuf, const Py_ssize_t arglen, const PyObject* sep, const int bytes_per_group); -#endif /* !Py_LIMITED_API */ - -#ifdef __cplusplus -} -#endif - -#endif /* !Py_STRHEX_H */ diff --git a/Include/pythonrun.h b/Include/pythonrun.h index b0a2fc3002d..02715775581 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -34,7 +34,7 @@ PyAPI_FUNC(int) PyOS_CheckStack(void); #ifndef Py_LIMITED_API # define Py_CPYTHON_PYTHONRUN_H -# include "cpython/pythonrun.h" +# include "cpython/pythonrun.h" # undef Py_CPYTHON_PYTHONRUN_H #endif diff --git a/Include/pythread.h b/Include/pythread.h index bb9d8641221..1a6092c4ad0 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -61,9 +61,11 @@ PyAPI_FUNC(int) _PyThread_at_fork_reinit(PyThread_type_lock *lock); convert microseconds to nanoseconds. */ # define PY_TIMEOUT_MAX (LLONG_MAX / 1000) #elif defined (NT_THREADS) - /* In the NT API, the timeout is a DWORD and is expressed in milliseconds */ -# if 0xFFFFFFFFLL * 1000 < LLONG_MAX -# define PY_TIMEOUT_MAX (0xFFFFFFFFLL * 1000) + // WaitForSingleObject() accepts timeout in milliseconds in the range + // [0; 0xFFFFFFFE] (DWORD type). INFINITE value (0xFFFFFFFF) means no + // timeout. 0xFFFFFFFE milliseconds is around 49.7 days. +# if 0xFFFFFFFELL * 1000 < LLONG_MAX +# define PY_TIMEOUT_MAX (0xFFFFFFFELL * 1000) # else # define PY_TIMEOUT_MAX LLONG_MAX # endif diff --git a/Include/structseq.h b/Include/structseq.h index 8f51c89163a..e89265a67c3 100644 --- a/Include/structseq.h +++ b/Include/structseq.h @@ -19,7 +19,7 @@ typedef struct PyStructSequence_Desc { int n_in_sequence; } PyStructSequence_Desc; -extern const char * const PyStructSequence_UnnamedField; +PyAPI_DATA(const char * const) PyStructSequence_UnnamedField; #ifndef Py_LIMITED_API PyAPI_FUNC(void) PyStructSequence_InitType(PyTypeObject *type, diff --git a/Include/sysmodule.h b/Include/sysmodule.h index 8c8f7c42594..3463c622309 100644 --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -31,7 +31,7 @@ PyAPI_FUNC(PyObject *) PySys_GetXOptions(void); #ifndef Py_LIMITED_API # define Py_CPYTHON_SYSMODULE_H -# include "cpython/sysmodule.h" +# include "cpython/sysmodule.h" # undef Py_CPYTHON_SYSMODULE_H #endif diff --git a/Include/traceback.h b/Include/traceback.h index 781e5a6eec4..2dfa2ada4f2 100644 --- a/Include/traceback.h +++ b/Include/traceback.h @@ -16,7 +16,7 @@ PyAPI_DATA(PyTypeObject) PyTraceBack_Type; #ifndef Py_LIMITED_API # define Py_CPYTHON_TRACEBACK_H -# include "cpython/traceback.h" +# include "cpython/traceback.h" # undef Py_CPYTHON_TRACEBACK_H #endif diff --git a/Include/tupleobject.h b/Include/tupleobject.h index e796a320192..dc68e3fc5c6 100644 --- a/Include/tupleobject.h +++ b/Include/tupleobject.h @@ -36,7 +36,7 @@ PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...); #ifndef Py_LIMITED_API # define Py_CPYTHON_TUPLEOBJECT_H -# include "cpython/tupleobject.h" +# include "cpython/tupleobject.h" # undef Py_CPYTHON_TUPLEOBJECT_H #endif diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index b0ac086a6be..6426c5d06b4 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -1,7 +1,7 @@ #ifndef Py_UNICODEOBJECT_H #define Py_UNICODEOBJECT_H -#include +#include // va_list /* @@ -269,10 +269,6 @@ PyAPI_FUNC(PyObject *) PyUnicode_InternFromString( // and will be removed in Python 3.12. Use PyUnicode_InternInPlace() instead. Py_DEPRECATED(3.10) PyAPI_FUNC(void) PyUnicode_InternImmortal(PyObject **); -/* Use only if you know it's a string */ -#define PyUnicode_CHECK_INTERNED(op) \ - (((PyASCIIObject *)(op))->state.interned) - /* --- wchar_t support for platforms which support it --------------------- */ #ifdef HAVE_WCHAR_H @@ -1043,7 +1039,7 @@ PyAPI_FUNC(int) PyUnicode_IsIdentifier(PyObject *s); #ifndef Py_LIMITED_API # define Py_CPYTHON_UNICODEOBJECT_H -# include "cpython/unicodeobject.h" +# include "cpython/unicodeobject.h" # undef Py_CPYTHON_UNICODEOBJECT_H #endif diff --git a/Include/warnings.h b/Include/warnings.h index a675bb5dfcb..18ac1543a3c 100644 --- a/Include/warnings.h +++ b/Include/warnings.h @@ -4,14 +4,11 @@ extern "C" { #endif -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject*) _PyWarnings_Init(void); -#endif - PyAPI_FUNC(int) PyErr_WarnEx( PyObject *category, const char *message, /* UTF-8 encoded string */ Py_ssize_t stack_level); + PyAPI_FUNC(int) PyErr_WarnFormat( PyObject *category, Py_ssize_t stack_level, @@ -26,15 +23,7 @@ PyAPI_FUNC(int) PyErr_ResourceWarning( const char *format, /* ASCII-encoded string */ ...); #endif -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) PyErr_WarnExplicitObject( - PyObject *category, - PyObject *message, - PyObject *filename, - int lineno, - PyObject *module, - PyObject *registry); -#endif + PyAPI_FUNC(int) PyErr_WarnExplicit( PyObject *category, const char *message, /* UTF-8 encoded string */ @@ -44,20 +33,9 @@ PyAPI_FUNC(int) PyErr_WarnExplicit( PyObject *registry); #ifndef Py_LIMITED_API -PyAPI_FUNC(int) -PyErr_WarnExplicitFormat(PyObject *category, - const char *filename, int lineno, - const char *module, PyObject *registry, - const char *format, ...); -#endif - -/* DEPRECATED: Use PyErr_WarnEx() instead. */ -#ifndef Py_LIMITED_API -#define PyErr_Warn(category, msg) PyErr_WarnEx(category, msg, 1) -#endif - -#ifndef Py_LIMITED_API -void _PyErr_WarnUnawaitedCoroutine(PyObject *coro); +# define Py_CPYTHON_WARNINGS_H +# include "cpython/warnings.h" +# undef Py_CPYTHON_WARNINGS_H #endif #ifdef __cplusplus diff --git a/Include/weakrefobject.h b/Include/weakrefobject.h index ac4b4821c8a..f071e9c759a 100644 --- a/Include/weakrefobject.h +++ b/Include/weakrefobject.h @@ -6,40 +6,8 @@ extern "C" { #endif - typedef struct _PyWeakReference PyWeakReference; -/* PyWeakReference is the base struct for the Python ReferenceType, ProxyType, - * and CallableProxyType. - */ -#ifndef Py_LIMITED_API -struct _PyWeakReference { - PyObject_HEAD - - /* The object to which this is a weak reference, or Py_None if none. - * Note that this is a stealth reference: wr_object's refcount is - * not incremented to reflect this pointer. - */ - PyObject *wr_object; - - /* A callable to invoke when wr_object dies, or NULL if none. */ - PyObject *wr_callback; - - /* A cache for wr_object's hash code. As usual for hashes, this is -1 - * if the hash code isn't known yet. - */ - Py_hash_t hash; - - /* If wr_object is weakly referenced, wr_object has a doubly-linked NULL- - * terminated list of weak references to it. These are the list pointers. - * If wr_object goes away, wr_object is set to Py_None, and these pointers - * have no meaning then. - */ - PyWeakReference *wr_prev; - PyWeakReference *wr_next; -}; -#endif - PyAPI_DATA(PyTypeObject) _PyWeakref_RefType; PyAPI_DATA(PyTypeObject) _PyWeakref_ProxyType; PyAPI_DATA(PyTypeObject) _PyWeakref_CallableProxyType; @@ -56,30 +24,18 @@ PyAPI_DATA(PyTypeObject) _PyWeakref_CallableProxyType; PyAPI_FUNC(PyObject *) PyWeakref_NewRef(PyObject *ob, - PyObject *callback); + PyObject *callback); PyAPI_FUNC(PyObject *) PyWeakref_NewProxy(PyObject *ob, - PyObject *callback); + PyObject *callback); PyAPI_FUNC(PyObject *) PyWeakref_GetObject(PyObject *ref); + #ifndef Py_LIMITED_API -PyAPI_FUNC(Py_ssize_t) _PyWeakref_GetWeakrefCount(PyWeakReference *head); - -PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); +# define Py_CPYTHON_WEAKREFOBJECT_H +# include "cpython/weakrefobject.h" +# undef Py_CPYTHON_WEAKREFOBJECT_H #endif -/* Explanation for the Py_REFCNT() check: when a weakref's target is part - of a long chain of deallocations which triggers the trashcan mechanism, - clearing the weakrefs can be delayed long after the target's refcount - has dropped to zero. In the meantime, code accessing the weakref will - be able to "see" the target object even though it is supposed to be - unreachable. See issue #16602. */ - -#define PyWeakref_GET_OBJECT(ref) \ - (Py_REFCNT(((PyWeakReference *)(ref))->wr_object) > 0 \ - ? ((PyWeakReference *)(ref))->wr_object \ - : Py_None) - - #ifdef __cplusplus } #endif diff --git a/Lib/__hello__.py b/Lib/__hello__.py new file mode 100644 index 00000000000..d37bd2766ac --- /dev/null +++ b/Lib/__hello__.py @@ -0,0 +1,7 @@ +initialized = True + +def main(): + print("Hello world!") + +if __name__ == '__main__': + main() diff --git a/Lib/__phello__.foo.py b/Lib/__phello__.foo.py deleted file mode 100644 index 8e8623ee1da..00000000000 --- a/Lib/__phello__.foo.py +++ /dev/null @@ -1 +0,0 @@ -# This file exists as a helper for the test.test_frozen module. diff --git a/Lib/__phello__/__init__.py b/Lib/__phello__/__init__.py new file mode 100644 index 00000000000..d37bd2766ac --- /dev/null +++ b/Lib/__phello__/__init__.py @@ -0,0 +1,7 @@ +initialized = True + +def main(): + print("Hello world!") + +if __name__ == '__main__': + main() diff --git a/Lib/sqlite3/test/__init__.py b/Lib/__phello__/ham/__init__.py similarity index 100% rename from Lib/sqlite3/test/__init__.py rename to Lib/__phello__/ham/__init__.py diff --git a/Lib/__phello__/ham/eggs.py b/Lib/__phello__/ham/eggs.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Lib/__phello__/spam.py b/Lib/__phello__/spam.py new file mode 100644 index 00000000000..d37bd2766ac --- /dev/null +++ b/Lib/__phello__/spam.py @@ -0,0 +1,7 @@ +initialized = True + +def main(): + print("Hello world!") + +if __name__ == '__main__': + main() diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index 326b36d6116..aa66c8b9f41 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -481,7 +481,7 @@ def customize_compiler(_config_vars): This customization is performed when the first extension module build is requested - in distutils.sysconfig.customize_compiler). + in distutils.sysconfig.customize_compiler. """ # Find a compiler to use for extension module builds diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 56e9a0cb33c..d7119742b9d 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -101,7 +101,6 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, 'b' binary mode 't' text mode (default) '+' open a disk file for updating (reading and writing) - 'U' universal newline mode (deprecated) ========= =============================================================== The default mode is 'rt' (open for reading text). For binary random @@ -117,10 +116,6 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, returned as strings, the bytes having been first decoded using a platform-dependent encoding or using the specified encoding if given. - 'U' mode is deprecated and will raise an exception in future versions - of Python. It has no effect in Python 3. Use newline to control - universal newlines mode. - buffering is an optional integer used to set the buffering policy. Pass 0 to switch buffering off (only allowed in binary mode), 1 to select line buffering (only usable in text mode), and an integer > 1 to indicate @@ -206,7 +201,7 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, if errors is not None and not isinstance(errors, str): raise TypeError("invalid errors: %r" % errors) modes = set(mode) - if modes - set("axrwb+tU") or len(mode) > len(modes): + if modes - set("axrwb+t") or len(mode) > len(modes): raise ValueError("invalid mode: %r" % mode) creating = "x" in modes reading = "r" in modes @@ -215,13 +210,6 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, updating = "+" in modes text = "t" in modes binary = "b" in modes - if "U" in modes: - if creating or writing or appending or updating: - raise ValueError("mode U cannot be combined with 'x', 'w', 'a', or '+'") - import warnings - warnings.warn("'U' mode is deprecated", - DeprecationWarning, 2) - reading = True if text and binary: raise ValueError("can't have text and binary mode at once") if creating + reading + writing + appending > 1: diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py index b267780f0ce..2a27684324d 100644 --- a/Lib/_weakrefset.py +++ b/Lib/_weakrefset.py @@ -51,10 +51,14 @@ class WeakSet: self.update(data) def _commit_removals(self): - l = self._pending_removals + pop = self._pending_removals.pop discard = self.data.discard - while l: - discard(l.pop()) + while True: + try: + item = pop() + except IndexError: + return + discard(item) def __iter__(self): with _IterationGuard(self): diff --git a/Lib/argparse.py b/Lib/argparse.py index 33c5d705bc6..b44fa4f0f65 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -526,12 +526,13 @@ class HelpFormatter(object): parts = [action_header] # if there was help for the action, add lines of help text - if action.help: + if action.help and action.help.strip(): help_text = self._expand_help(action) - help_lines = self._split_lines(help_text, help_width) - parts.append('%*s%s\n' % (indent_first, '', help_lines[0])) - for line in help_lines[1:]: - parts.append('%*s%s\n' % (help_position, '', line)) + if help_text: + help_lines = self._split_lines(help_text, help_width) + parts.append('%*s%s\n' % (indent_first, '', help_lines[0])) + for line in help_lines[1:]: + parts.append('%*s%s\n' % (help_position, '', line)) # or add a newline if the description doesn't end with one elif not action_header.endswith('\n'): @@ -876,7 +877,7 @@ class BooleanOptionalAction(Action): _option_strings.append(option_string) if help is not None and default is not None: - help += f" (default: {default})" + help += " (default: %(default)s)" super().__init__( option_strings=_option_strings, diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 125d32da889..cbf6d5db0a0 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -66,10 +66,6 @@ _HAS_IPv6 = hasattr(socket, 'AF_INET6') # Maximum timeout passed to select to avoid OS limitations MAXIMUM_SELECT_TIMEOUT = 24 * 3600 -# Used for deprecation and removal of `loop.create_datagram_endpoint()`'s -# *reuse_address* parameter -_unset = object() - def _format_handle(handle): cb = handle._callback @@ -710,6 +706,8 @@ class BaseEventLoop(events.AbstractEventLoop): Any positional arguments after the callback will be passed to the callback when it is called. """ + if delay is None: + raise TypeError('delay must not be None') timer = self.call_at(self.time() + delay, callback, *args, context=context) if timer._source_traceback: @@ -721,6 +719,8 @@ class BaseEventLoop(events.AbstractEventLoop): Absolute time corresponds to the event loop's time() method. """ + if when is None: + raise TypeError("when cannot be None") self._check_closed() if self._debug: self._check_thread() @@ -1235,7 +1235,7 @@ class BaseEventLoop(events.AbstractEventLoop): async def create_datagram_endpoint(self, protocol_factory, local_addr=None, remote_addr=None, *, family=0, proto=0, flags=0, - reuse_address=_unset, reuse_port=None, + reuse_port=None, allow_broadcast=None, sock=None): """Create datagram connection.""" if sock is not None: @@ -1248,7 +1248,7 @@ class BaseEventLoop(events.AbstractEventLoop): # show the problematic kwargs in exception msg opts = dict(local_addr=local_addr, remote_addr=remote_addr, family=family, proto=proto, flags=flags, - reuse_address=reuse_address, reuse_port=reuse_port, + reuse_port=reuse_port, allow_broadcast=allow_broadcast) problems = ', '.join(f'{k}={v}' for k, v in opts.items() if v) raise ValueError( @@ -1311,19 +1311,6 @@ class BaseEventLoop(events.AbstractEventLoop): exceptions = [] - # bpo-37228 - if reuse_address is not _unset: - if reuse_address: - raise ValueError("Passing `reuse_address=True` is no " - "longer supported, as the usage of " - "SO_REUSEPORT in UDP poses a significant " - "security concern.") - else: - warnings.warn("The *reuse_address* parameter has been " - "deprecated as of 3.5.10 and is scheduled " - "for removal in 3.11.", DeprecationWarning, - stacklevel=2) - for ((family, proto), (local_address, remote_address)) in addr_pairs_info: sock = None @@ -1439,7 +1426,7 @@ class BaseEventLoop(events.AbstractEventLoop): 'host/port and sock can not be specified at the same time') if reuse_address is None: - reuse_address = os.name == 'posix' and sys.platform != 'cygwin' + reuse_address = os.name == "posix" and sys.platform != "cygwin" sockets = [] if host == '': hosts = [None] diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index b966ad26bf4..d91fe8db2b0 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -101,7 +101,6 @@ class TimerHandle(Handle): __slots__ = ['_scheduled', '_when'] def __init__(self, when, callback, args, loop, context=None): - assert when is not None super().__init__(callback, args, loop, context) if self._source_traceback: del self._source_traceback[-1] @@ -479,7 +478,7 @@ class AbstractEventLoop: # The reason to accept file-like object instead of just file descriptor # is: we need to own pipe and close it at transport finishing # Can got complicated errors if pass f.fileno(), - # close fd in pipe transport then close f and vise versa. + # close fd in pipe transport then close f and vice versa. raise NotImplementedError async def connect_write_pipe(self, protocol_factory, pipe): @@ -492,7 +491,7 @@ class AbstractEventLoop: # The reason to accept file-like object instead of just file descriptor # is: we need to own pipe and close it at transport finishing # Can got complicated errors if pass f.fileno(), - # close fd in pipe transport then close f and vise versa. + # close fd in pipe transport then close f and vice versa. raise NotImplementedError async def subprocess_shell(self, protocol_factory, cmd, *, @@ -661,7 +660,8 @@ class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): def set_event_loop(self, loop): """Set the event loop.""" self._local._set_called = True - assert loop is None or isinstance(loop, AbstractEventLoop) + if loop is not None and not isinstance(loop, AbstractEventLoop): + raise TypeError(f"loop must be an instance of AbstractEventLoop or None, not '{type(loop).__name__}'") self._local._loop = loop def new_event_loop(self): @@ -745,7 +745,8 @@ def set_event_loop_policy(policy): If policy is None, the default policy is restored.""" global _event_loop_policy - assert policy is None or isinstance(policy, AbstractEventLoopPolicy) + if policy is not None and not isinstance(policy, AbstractEventLoopPolicy): + raise TypeError(f"policy must be an instance of AbstractEventLoopPolicy or None, not '{type(policy).__name__}'") _event_loop_policy = policy diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index a7453fb1c77..4fef64e3921 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -230,8 +230,6 @@ class Condition(_ContextManagerMixin, mixins._LoopBoundMixin): super().__init__(loop=loop) if lock is None: lock = Lock() - elif lock._loop is not self._get_loop(): - raise ValueError("loop argument must agree with lock") self._lock = lock # Export the lock's locked(), acquire() and release() methods. diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 9a9d0d6e3cc..53eef84427b 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -415,11 +415,9 @@ async def wait_for(fut, timeout): await _cancel_and_wait(fut, loop=loop) try: - fut.result() + return fut.result() except exceptions.CancelledError as exc: raise exceptions.TimeoutError() from exc - else: - raise exceptions.TimeoutError() waiter = loop.create_future() timeout_handle = loop.call_later(timeout, _release_waiter, waiter) @@ -455,11 +453,9 @@ async def wait_for(fut, timeout): # exception, we should re-raise it # See https://bugs.python.org/issue40607 try: - fut.result() + return fut.result() except exceptions.CancelledError as exc: raise exceptions.TimeoutError() from exc - else: - raise exceptions.TimeoutError() finally: timeout_handle.cancel() diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index e4f445e9502..c88b818de62 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -1379,7 +1379,7 @@ class ThreadedChildWatcher(AbstractChildWatcher): def remove_child_handler(self, pid): # asyncio never calls remove_child_handler() !!! # The method is no-op but is implemented because - # abstract base classe requires it + # abstract base classes require it. return True def attach_loop(self, loop): diff --git a/Lib/base64.py b/Lib/base64.py index e1256ad9358..7e9c2a2ca47 100755 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -76,15 +76,16 @@ def b64decode(s, altchars=None, validate=False): normal base-64 alphabet nor the alternative alphabet are discarded prior to the padding check. If validate is True, these non-alphabet characters in the input result in a binascii.Error. + For more information about the strict base64 check, see: + + https://docs.python.org/3.11/library/binascii.html#binascii.a2b_base64 """ s = _bytes_from_decode_data(s) if altchars is not None: altchars = _bytes_from_decode_data(altchars) assert len(altchars) == 2, repr(altchars) s = s.translate(bytes.maketrans(altchars, b'+/')) - if validate and not re.fullmatch(b'[A-Za-z0-9+/]*={0,2}', s): - raise binascii.Error('Non-base64 digit found') - return binascii.a2b_base64(s) + return binascii.a2b_base64(s, strict_mode=validate) def standard_b64encode(s): @@ -181,7 +182,7 @@ def _b32encode(alphabet, s): from_bytes = int.from_bytes b32tab2 = _b32tab2[alphabet] for i in range(0, len(s), 5): - c = from_bytes(s[i: i + 5], 'big') + c = from_bytes(s[i: i + 5]) # big endian encoded += (b32tab2[c >> 30] + # bits 1 - 10 b32tab2[(c >> 20) & 0x3ff] + # bits 11 - 20 b32tab2[(c >> 10) & 0x3ff] + # bits 21 - 30 @@ -233,13 +234,13 @@ def _b32decode(alphabet, s, casefold=False, map01=None): acc = (acc << 5) + b32rev[c] except KeyError: raise binascii.Error('Non-base32 digit found') from None - decoded += acc.to_bytes(5, 'big') + decoded += acc.to_bytes(5) # big endian # Process the last, partial quanta if l % 8 or padchars not in {0, 1, 3, 4, 6}: raise binascii.Error('Incorrect padding') if padchars and decoded: acc <<= 5 * padchars - last = acc.to_bytes(5, 'big') + last = acc.to_bytes(5) # big endian leftover = (43 - 5 * padchars) // 8 # 1: 4, 3: 3, 4: 2, 6: 1 decoded[-5:] = last[:leftover] return bytes(decoded) @@ -566,15 +567,17 @@ def decodebytes(s): def main(): """Small main program""" import sys, getopt + usage = """usage: %s [-h|-d|-e|-u|-t] [file|-] + -h: print this help message and exit + -d, -u: decode + -e: encode (default) + -t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0] try: - opts, args = getopt.getopt(sys.argv[1:], 'deut') + opts, args = getopt.getopt(sys.argv[1:], 'hdeut') except getopt.error as msg: sys.stdout = sys.stderr print(msg) - print("""usage: %s [-d|-e|-u|-t] [file|-] - -d, -u: decode - -e: encode (default) - -t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0]) + print(usage) sys.exit(2) func = encode for o, a in opts: @@ -582,6 +585,7 @@ def main(): if o == '-d': func = decode if o == '-u': func = decode if o == '-t': test(); return + if o == '-h': print(usage); return if args and args[0] != '-': with open(args[0], 'rb') as f: func(f, sys.stdout.buffer) diff --git a/Lib/binhex.py b/Lib/binhex.py deleted file mode 100644 index ace5217d271..00000000000 --- a/Lib/binhex.py +++ /dev/null @@ -1,502 +0,0 @@ -"""Macintosh binhex compression/decompression. - -easy interface: -binhex(inputfilename, outputfilename) -hexbin(inputfilename, outputfilename) -""" - -# -# Jack Jansen, CWI, August 1995. -# -# The module is supposed to be as compatible as possible. Especially the -# easy interface should work "as expected" on any platform. -# XXXX Note: currently, textfiles appear in mac-form on all platforms. -# We seem to lack a simple character-translate in python. -# (we should probably use ISO-Latin-1 on all but the mac platform). -# XXXX The simple routines are too simple: they expect to hold the complete -# files in-core. Should be fixed. -# XXXX It would be nice to handle AppleDouble format on unix -# (for servers serving macs). -# XXXX I don't understand what happens when you get 0x90 times the same byte on -# input. The resulting code (xx 90 90) would appear to be interpreted as an -# escaped *value* of 0x90. All coders I've seen appear to ignore this nicety... -# -import binascii -import contextlib -import io -import os -import struct -import warnings - -warnings.warn('the binhex module is deprecated', DeprecationWarning, - stacklevel=2) - - -__all__ = ["binhex","hexbin","Error"] - -class Error(Exception): - pass - -# States (what have we written) -_DID_HEADER = 0 -_DID_DATA = 1 - -# Various constants -REASONABLY_LARGE = 32768 # Minimal amount we pass the rle-coder -LINELEN = 64 -RUNCHAR = b"\x90" - -# -# This code is no longer byte-order dependent - - -class FInfo: - def __init__(self): - self.Type = '????' - self.Creator = '????' - self.Flags = 0 - -def getfileinfo(name): - finfo = FInfo() - with io.open(name, 'rb') as fp: - # Quick check for textfile - data = fp.read(512) - if 0 not in data: - finfo.Type = 'TEXT' - fp.seek(0, 2) - dsize = fp.tell() - dir, file = os.path.split(name) - file = file.replace(':', '-', 1) - return file, finfo, dsize, 0 - -class openrsrc: - def __init__(self, *args): - pass - - def read(self, *args): - return b'' - - def write(self, *args): - pass - - def close(self): - pass - - -# DeprecationWarning is already emitted on "import binhex". There is no need -# to repeat the warning at each call to deprecated binascii functions. -@contextlib.contextmanager -def _ignore_deprecation_warning(): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - yield - - -class _Hqxcoderengine: - """Write data to the coder in 3-byte chunks""" - - def __init__(self, ofp): - self.ofp = ofp - self.data = b'' - self.hqxdata = b'' - self.linelen = LINELEN - 1 - - def write(self, data): - self.data = self.data + data - datalen = len(self.data) - todo = (datalen // 3) * 3 - data = self.data[:todo] - self.data = self.data[todo:] - if not data: - return - with _ignore_deprecation_warning(): - self.hqxdata = self.hqxdata + binascii.b2a_hqx(data) - self._flush(0) - - def _flush(self, force): - first = 0 - while first <= len(self.hqxdata) - self.linelen: - last = first + self.linelen - self.ofp.write(self.hqxdata[first:last] + b'\r') - self.linelen = LINELEN - first = last - self.hqxdata = self.hqxdata[first:] - if force: - self.ofp.write(self.hqxdata + b':\r') - - def close(self): - if self.data: - with _ignore_deprecation_warning(): - self.hqxdata = self.hqxdata + binascii.b2a_hqx(self.data) - self._flush(1) - self.ofp.close() - del self.ofp - -class _Rlecoderengine: - """Write data to the RLE-coder in suitably large chunks""" - - def __init__(self, ofp): - self.ofp = ofp - self.data = b'' - - def write(self, data): - self.data = self.data + data - if len(self.data) < REASONABLY_LARGE: - return - with _ignore_deprecation_warning(): - rledata = binascii.rlecode_hqx(self.data) - self.ofp.write(rledata) - self.data = b'' - - def close(self): - if self.data: - with _ignore_deprecation_warning(): - rledata = binascii.rlecode_hqx(self.data) - self.ofp.write(rledata) - self.ofp.close() - del self.ofp - -class BinHex: - def __init__(self, name_finfo_dlen_rlen, ofp): - name, finfo, dlen, rlen = name_finfo_dlen_rlen - close_on_error = False - if isinstance(ofp, str): - ofname = ofp - ofp = io.open(ofname, 'wb') - close_on_error = True - try: - ofp.write(b'(This file must be converted with BinHex 4.0)\r\r:') - hqxer = _Hqxcoderengine(ofp) - self.ofp = _Rlecoderengine(hqxer) - self.crc = 0 - if finfo is None: - finfo = FInfo() - self.dlen = dlen - self.rlen = rlen - self._writeinfo(name, finfo) - self.state = _DID_HEADER - except: - if close_on_error: - ofp.close() - raise - - def _writeinfo(self, name, finfo): - nl = len(name) - if nl > 63: - raise Error('Filename too long') - d = bytes([nl]) + name.encode("latin-1") + b'\0' - tp, cr = finfo.Type, finfo.Creator - if isinstance(tp, str): - tp = tp.encode("latin-1") - if isinstance(cr, str): - cr = cr.encode("latin-1") - d2 = tp + cr - - # Force all structs to be packed with big-endian - d3 = struct.pack('>h', finfo.Flags) - d4 = struct.pack('>ii', self.dlen, self.rlen) - info = d + d2 + d3 + d4 - self._write(info) - self._writecrc() - - def _write(self, data): - self.crc = binascii.crc_hqx(data, self.crc) - self.ofp.write(data) - - def _writecrc(self): - # XXXX Should this be here?? - # self.crc = binascii.crc_hqx('\0\0', self.crc) - if self.crc < 0: - fmt = '>h' - else: - fmt = '>H' - self.ofp.write(struct.pack(fmt, self.crc)) - self.crc = 0 - - def write(self, data): - if self.state != _DID_HEADER: - raise Error('Writing data at the wrong time') - self.dlen = self.dlen - len(data) - self._write(data) - - def close_data(self): - if self.dlen != 0: - raise Error('Incorrect data size, diff=%r' % (self.rlen,)) - self._writecrc() - self.state = _DID_DATA - - def write_rsrc(self, data): - if self.state < _DID_DATA: - self.close_data() - if self.state != _DID_DATA: - raise Error('Writing resource data at the wrong time') - self.rlen = self.rlen - len(data) - self._write(data) - - def close(self): - if self.state is None: - return - try: - if self.state < _DID_DATA: - self.close_data() - if self.state != _DID_DATA: - raise Error('Close at the wrong time') - if self.rlen != 0: - raise Error("Incorrect resource-datasize, diff=%r" % (self.rlen,)) - self._writecrc() - finally: - self.state = None - ofp = self.ofp - del self.ofp - ofp.close() - -def binhex(inp, out): - """binhex(infilename, outfilename): create binhex-encoded copy of a file""" - finfo = getfileinfo(inp) - ofp = BinHex(finfo, out) - - with io.open(inp, 'rb') as ifp: - # XXXX Do textfile translation on non-mac systems - while True: - d = ifp.read(128000) - if not d: break - ofp.write(d) - ofp.close_data() - - ifp = openrsrc(inp, 'rb') - while True: - d = ifp.read(128000) - if not d: break - ofp.write_rsrc(d) - ofp.close() - ifp.close() - -class _Hqxdecoderengine: - """Read data via the decoder in 4-byte chunks""" - - def __init__(self, ifp): - self.ifp = ifp - self.eof = 0 - - def read(self, totalwtd): - """Read at least wtd bytes (or until EOF)""" - decdata = b'' - wtd = totalwtd - # - # The loop here is convoluted, since we don't really now how - # much to decode: there may be newlines in the incoming data. - while wtd > 0: - if self.eof: return decdata - wtd = ((wtd + 2) // 3) * 4 - data = self.ifp.read(wtd) - # - # Next problem: there may not be a complete number of - # bytes in what we pass to a2b. Solve by yet another - # loop. - # - while True: - try: - with _ignore_deprecation_warning(): - decdatacur, self.eof = binascii.a2b_hqx(data) - break - except binascii.Incomplete: - pass - newdata = self.ifp.read(1) - if not newdata: - raise Error('Premature EOF on binhex file') - data = data + newdata - decdata = decdata + decdatacur - wtd = totalwtd - len(decdata) - if not decdata and not self.eof: - raise Error('Premature EOF on binhex file') - return decdata - - def close(self): - self.ifp.close() - -class _Rledecoderengine: - """Read data via the RLE-coder""" - - def __init__(self, ifp): - self.ifp = ifp - self.pre_buffer = b'' - self.post_buffer = b'' - self.eof = 0 - - def read(self, wtd): - if wtd > len(self.post_buffer): - self._fill(wtd - len(self.post_buffer)) - rv = self.post_buffer[:wtd] - self.post_buffer = self.post_buffer[wtd:] - return rv - - def _fill(self, wtd): - self.pre_buffer = self.pre_buffer + self.ifp.read(wtd + 4) - if self.ifp.eof: - with _ignore_deprecation_warning(): - self.post_buffer = self.post_buffer + \ - binascii.rledecode_hqx(self.pre_buffer) - self.pre_buffer = b'' - return - - # - # Obfuscated code ahead. We have to take care that we don't - # end up with an orphaned RUNCHAR later on. So, we keep a couple - # of bytes in the buffer, depending on what the end of - # the buffer looks like: - # '\220\0\220' - Keep 3 bytes: repeated \220 (escaped as \220\0) - # '?\220' - Keep 2 bytes: repeated something-else - # '\220\0' - Escaped \220: Keep 2 bytes. - # '?\220?' - Complete repeat sequence: decode all - # otherwise: keep 1 byte. - # - mark = len(self.pre_buffer) - if self.pre_buffer[-3:] == RUNCHAR + b'\0' + RUNCHAR: - mark = mark - 3 - elif self.pre_buffer[-1:] == RUNCHAR: - mark = mark - 2 - elif self.pre_buffer[-2:] == RUNCHAR + b'\0': - mark = mark - 2 - elif self.pre_buffer[-2:-1] == RUNCHAR: - pass # Decode all - else: - mark = mark - 1 - - with _ignore_deprecation_warning(): - self.post_buffer = self.post_buffer + \ - binascii.rledecode_hqx(self.pre_buffer[:mark]) - self.pre_buffer = self.pre_buffer[mark:] - - def close(self): - self.ifp.close() - -class HexBin: - def __init__(self, ifp): - if isinstance(ifp, str): - ifp = io.open(ifp, 'rb') - # - # Find initial colon. - # - while True: - ch = ifp.read(1) - if not ch: - raise Error("No binhex data found") - # Cater for \r\n terminated lines (which show up as \n\r, hence - # all lines start with \r) - if ch == b'\r': - continue - if ch == b':': - break - - hqxifp = _Hqxdecoderengine(ifp) - self.ifp = _Rledecoderengine(hqxifp) - self.crc = 0 - self._readheader() - - def _read(self, len): - data = self.ifp.read(len) - self.crc = binascii.crc_hqx(data, self.crc) - return data - - def _checkcrc(self): - filecrc = struct.unpack('>h', self.ifp.read(2))[0] & 0xffff - #self.crc = binascii.crc_hqx('\0\0', self.crc) - # XXXX Is this needed?? - self.crc = self.crc & 0xffff - if filecrc != self.crc: - raise Error('CRC error, computed %x, read %x' - % (self.crc, filecrc)) - self.crc = 0 - - def _readheader(self): - len = self._read(1) - fname = self._read(ord(len)) - rest = self._read(1 + 4 + 4 + 2 + 4 + 4) - self._checkcrc() - - type = rest[1:5] - creator = rest[5:9] - flags = struct.unpack('>h', rest[9:11])[0] - self.dlen = struct.unpack('>l', rest[11:15])[0] - self.rlen = struct.unpack('>l', rest[15:19])[0] - - self.FName = fname - self.FInfo = FInfo() - self.FInfo.Creator = creator - self.FInfo.Type = type - self.FInfo.Flags = flags - - self.state = _DID_HEADER - - def read(self, *n): - if self.state != _DID_HEADER: - raise Error('Read data at wrong time') - if n: - n = n[0] - n = min(n, self.dlen) - else: - n = self.dlen - rv = b'' - while len(rv) < n: - rv = rv + self._read(n-len(rv)) - self.dlen = self.dlen - n - return rv - - def close_data(self): - if self.state != _DID_HEADER: - raise Error('close_data at wrong time') - if self.dlen: - dummy = self._read(self.dlen) - self._checkcrc() - self.state = _DID_DATA - - def read_rsrc(self, *n): - if self.state == _DID_HEADER: - self.close_data() - if self.state != _DID_DATA: - raise Error('Read resource data at wrong time') - if n: - n = n[0] - n = min(n, self.rlen) - else: - n = self.rlen - self.rlen = self.rlen - n - return self._read(n) - - def close(self): - if self.state is None: - return - try: - if self.rlen: - dummy = self.read_rsrc(self.rlen) - self._checkcrc() - finally: - self.state = None - self.ifp.close() - -def hexbin(inp, out): - """hexbin(infilename, outfilename) - Decode binhexed file""" - ifp = HexBin(inp) - finfo = ifp.FInfo - if not out: - out = ifp.FName - - with io.open(out, 'wb') as ofp: - # XXXX Do translation on non-mac systems - while True: - d = ifp.read(128000) - if not d: break - ofp.write(d) - ifp.close_data() - - d = ifp.read_rsrc(128000) - if d: - ofp = openrsrc(out, 'wb') - ofp.write(d) - while True: - d = ifp.read_rsrc(128000) - if not d: break - ofp.write(d) - ofp.close() - - ifp.close() diff --git a/Lib/bz2.py b/Lib/bz2.py index 7f1d20632ef..fabe4f73c8d 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -197,10 +197,6 @@ class BZ2File(_compression.BaseStream): self._check_can_read() return self._buffer.readline(size) - def __iter__(self): - self._check_can_read() - return self._buffer.__iter__() - def readlines(self, size=-1): """Read a list of lines of uncompressed bytes from the file. diff --git a/Lib/calendar.py b/Lib/calendar.py index 7311a017372..663bb946b0d 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -558,9 +558,7 @@ class different_locale: class LocaleTextCalendar(TextCalendar): """ This class can be passed a locale name in the constructor and will return - month and weekday names in the specified locale. If this locale includes - an encoding all strings containing month and weekday names will be returned - as unicode. + month and weekday names in the specified locale. """ def __init__(self, firstweekday=0, locale=None): @@ -581,9 +579,7 @@ class LocaleTextCalendar(TextCalendar): class LocaleHTMLCalendar(HTMLCalendar): """ This class can be passed a locale name in the constructor and will return - month and weekday names in the specified locale. If this locale includes - an encoding all strings containing month and weekday names will be returned - as unicode. + month and weekday names in the specified locale. """ def __init__(self, firstweekday=0, locale=None): HTMLCalendar.__init__(self, firstweekday) diff --git a/Lib/cgitb.py b/Lib/cgitb.py index 17ddda37688..ec156843099 100644 --- a/Lib/cgitb.py +++ b/Lib/cgitb.py @@ -31,6 +31,7 @@ import tempfile import time import tokenize import traceback +from html import escape as html_escape def reset(): """Return a string that resets the CGI and browser to a known state.""" @@ -105,10 +106,16 @@ def html(einfo, context=5): etype = etype.__name__ pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable date = time.ctime(time.time()) - head = '' + pydoc.html.heading( - '%s' % - strong(pydoc.html.escape(str(etype))), - '#ffffff', '#6622aa', pyver + '
' + date) + ''' + head = f''' + + + + + +
 
+ 
+{html_escape(str(etype))}		 +{pyver}
{date}

A problem occurred in a Python script. Here is the sequence of function calls leading up to the error, in the order they occurred.

''' diff --git a/Lib/compileall.py b/Lib/compileall.py index 454465bd8fc..330a90786ef 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -4,7 +4,7 @@ When called as a script with arguments, this compiles the directories given as arguments recursively; the -l option prevents it from recursing into directories. -Without arguments, if compiles all modules on sys.path, without +Without arguments, it compiles all modules on sys.path, without recursing into subdirectories. (Even though it should do so for packages -- for now, you'll have to deal with packages separately.) @@ -221,8 +221,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if not force: try: mtime = int(os.stat(fullname).st_mtime) - expect = struct.pack('<4sll', importlib.util.MAGIC_NUMBER, - 0, mtime) + expect = struct.pack('<4sLL', importlib.util.MAGIC_NUMBER, + 0, mtime & 0xFFFF_FFFF) for cfile in opt_cfiles.values(): with open(cfile, 'rb') as chandle: actual = chandle.read(12) @@ -367,9 +367,9 @@ def main(): 'environment variable is set, and ' '"timestamp" otherwise.')) parser.add_argument('-o', action='append', type=int, dest='opt_levels', - help=('Optimization levels to run compilation with.' - 'Default is -1 which uses optimization level of' - 'Python interpreter itself (specified by -O).')) + help=('Optimization levels to run compilation with. ' + 'Default is -1 which uses the optimization level ' + 'of the Python interpreter itself (see -O).')) parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', help='Ignore symlinks pointing outsite of the DIR') parser.add_argument('--hardlink-dupes', action='store_true', diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 9904db78c5b..695f7733305 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -141,10 +141,11 @@ class _WorkItem(object): self.kwargs = kwargs class _ResultItem(object): - def __init__(self, work_id, exception=None, result=None): + def __init__(self, work_id, exception=None, result=None, exit_pid=None): self.work_id = work_id self.exception = exception self.result = result + self.exit_pid = exit_pid class _CallItem(object): def __init__(self, work_id, fn, args, kwargs): @@ -201,17 +202,19 @@ def _process_chunk(fn, chunk): return [fn(*args) for args in chunk] -def _sendback_result(result_queue, work_id, result=None, exception=None): +def _sendback_result(result_queue, work_id, result=None, exception=None, + exit_pid=None): """Safely send back the given result or exception""" try: result_queue.put(_ResultItem(work_id, result=result, - exception=exception)) + exception=exception, exit_pid=exit_pid)) except BaseException as e: exc = _ExceptionWithTraceback(e, e.__traceback__) - result_queue.put(_ResultItem(work_id, exception=exc)) + result_queue.put(_ResultItem(work_id, exception=exc, + exit_pid=exit_pid)) -def _process_worker(call_queue, result_queue, initializer, initargs): +def _process_worker(call_queue, result_queue, initializer, initargs, max_tasks=None): """Evaluates calls from call_queue and places the results in result_queue. This worker is run in a separate process. @@ -232,25 +235,38 @@ def _process_worker(call_queue, result_queue, initializer, initargs): # The parent will notice that the process stopped and # mark the pool broken return + num_tasks = 0 + exit_pid = None while True: call_item = call_queue.get(block=True) if call_item is None: # Wake up queue management thread result_queue.put(os.getpid()) return + + if max_tasks is not None: + num_tasks += 1 + if num_tasks >= max_tasks: + exit_pid = os.getpid() + try: r = call_item.fn(*call_item.args, **call_item.kwargs) except BaseException as e: exc = _ExceptionWithTraceback(e, e.__traceback__) - _sendback_result(result_queue, call_item.work_id, exception=exc) + _sendback_result(result_queue, call_item.work_id, exception=exc, + exit_pid=exit_pid) else: - _sendback_result(result_queue, call_item.work_id, result=r) + _sendback_result(result_queue, call_item.work_id, result=r, + exit_pid=exit_pid) del r # Liberate the resource as soon as possible, to avoid holding onto # open files or shared memory that is not needed anymore del call_item + if exit_pid is not None: + return + class _ExecutorManagerThread(threading.Thread): """Manages the communication between this process and the worker processes. @@ -301,6 +317,10 @@ class _ExecutorManagerThread(threading.Thread): # A queue.Queue of work ids e.g. Queue([5, 6, ...]). self.work_ids_queue = executor._work_ids + # Maximum number of tasks a worker process can execute before + # exiting safely + self.max_tasks_per_child = executor._max_tasks_per_child + # A dict mapping work ids to _WorkItems e.g. # {5: <_WorkItem...>, 6: <_WorkItem...>, ...} self.pending_work_items = executor._pending_work_items @@ -320,15 +340,23 @@ class _ExecutorManagerThread(threading.Thread): return if result_item is not None: self.process_result_item(result_item) + + process_exited = result_item.exit_pid is not None + if process_exited: + p = self.processes.pop(result_item.exit_pid) + p.join() + # Delete reference to result_item to avoid keeping references # while waiting on new results. del result_item - # attempt to increment idle process count - executor = self.executor_reference() - if executor is not None: - executor._idle_worker_semaphore.release() - del executor + if executor := self.executor_reference(): + if process_exited: + with self.shutdown_lock: + executor._adjust_process_count() + else: + executor._idle_worker_semaphore.release() + del executor if self.is_shutting_down(): self.flag_executor_shutting_down() @@ -372,7 +400,7 @@ class _ExecutorManagerThread(threading.Thread): assert not self.thread_wakeup._closed wakeup_reader = self.thread_wakeup._reader readers = [result_reader, wakeup_reader] - worker_sentinels = [p.sentinel for p in self.processes.values()] + worker_sentinels = [p.sentinel for p in list(self.processes.values())] ready = mp.connection.wait(readers + worker_sentinels) cause = None @@ -578,7 +606,7 @@ class BrokenProcessPool(_base.BrokenExecutor): class ProcessPoolExecutor(_base.Executor): def __init__(self, max_workers=None, mp_context=None, - initializer=None, initargs=()): + initializer=None, initargs=(), *, max_tasks_per_child=None): """Initializes a new ProcessPoolExecutor instance. Args: @@ -589,6 +617,11 @@ class ProcessPoolExecutor(_base.Executor): object should provide SimpleQueue, Queue and Process. initializer: A callable used to initialize worker processes. initargs: A tuple of arguments to pass to the initializer. + max_tasks_per_child: The maximum number of tasks a worker process can + complete before it will exit and be replaced with a fresh + worker process, to enable unused resources to be freed. The + default value is None, which means worker process will live + as long as the executor will live. """ _check_system_limits() @@ -616,6 +649,13 @@ class ProcessPoolExecutor(_base.Executor): self._initializer = initializer self._initargs = initargs + if max_tasks_per_child is not None: + if not isinstance(max_tasks_per_child, int): + raise TypeError("max_tasks_per_child must be an integer") + elif max_tasks_per_child <= 0: + raise ValueError("max_tasks_per_child must be >= 1") + self._max_tasks_per_child = max_tasks_per_child + # Management thread self._executor_manager_thread = None @@ -678,7 +718,8 @@ class ProcessPoolExecutor(_base.Executor): args=(self._call_queue, self._result_queue, self._initializer, - self._initargs)) + self._initargs, + self._max_tasks_per_child)) p.start() self._processes[p.pid] = p diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py index b7a2cac7f57..51c942f51ab 100644 --- a/Lib/concurrent/futures/thread.py +++ b/Lib/concurrent/futures/thread.py @@ -36,6 +36,12 @@ def _python_exit(): # See bpo-39812 for context. threading._register_atexit(_python_exit) +# At fork, reinitialize the `_global_shutdown_lock` lock in the child process +if hasattr(os, 'register_at_fork'): + os.register_at_fork(before=_global_shutdown_lock.acquire, + after_in_child=_global_shutdown_lock._at_fork_reinit, + after_in_parent=_global_shutdown_lock.release) + class _WorkItem(object): def __init__(self, future, fn, args, kwargs): diff --git a/Lib/configparser.py b/Lib/configparser.py index 042a5c74b69..c10309acf1a 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -146,13 +146,12 @@ import itertools import os import re import sys -import warnings __all__ = ["NoSectionError", "DuplicateOptionError", "DuplicateSectionError", "NoOptionError", "InterpolationError", "InterpolationDepthError", "InterpolationMissingOptionError", "InterpolationSyntaxError", "ParsingError", "MissingSectionHeaderError", - "ConfigParser", "SafeConfigParser", "RawConfigParser", + "ConfigParser", "RawConfigParser", "Interpolation", "BasicInterpolation", "ExtendedInterpolation", "LegacyInterpolation", "SectionProxy", "ConverterMapping", "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH"] @@ -312,26 +311,6 @@ class ParsingError(Error): self.errors = [] self.args = (source, ) - @property - def filename(self): - """Deprecated, use `source'.""" - warnings.warn( - "The 'filename' attribute will be removed in future versions. " - "Use 'source' instead.", - DeprecationWarning, stacklevel=2 - ) - return self.source - - @filename.setter - def filename(self, value): - """Deprecated, user `source'.""" - warnings.warn( - "The 'filename' attribute will be removed in future versions. " - "Use 'source' instead.", - DeprecationWarning, stacklevel=2 - ) - self.source = value - def append(self, lineno, line): self.errors.append((lineno, line)) self.message += '\n\t[line %2d]: %s' % (lineno, line) @@ -754,15 +733,6 @@ class RawConfigParser(MutableMapping): elements_added.add((section, key)) self.set(section, key, value) - def readfp(self, fp, filename=None): - """Deprecated, use read_file instead.""" - warnings.warn( - "This method will be removed in future versions. " - "Use 'parser.read_file()' instead.", - DeprecationWarning, stacklevel=2 - ) - self.read_file(fp, source=filename) - def get(self, section, option, *, raw=False, vars=None, fallback=_UNSET): """Get an option value for a given section. @@ -1225,19 +1195,6 @@ class ConfigParser(RawConfigParser): self._interpolation = hold_interpolation -class SafeConfigParser(ConfigParser): - """ConfigParser alias for backwards compatibility purposes.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - warnings.warn( - "The SafeConfigParser class has been renamed to ConfigParser " - "in Python 3.2. This alias will be removed in future versions." - " Use ConfigParser directly instead.", - DeprecationWarning, stacklevel=2 - ) - - class SectionProxy(MutableMapping): """A proxy for a single section from a parser.""" diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 8343d7e5196..ee722585057 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -1,5 +1,6 @@ """Utilities for with-statement contexts. See PEP 343.""" import abc +import os import sys import _collections_abc from collections import deque @@ -9,7 +10,8 @@ from types import MethodType, GenericAlias __all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext", "AbstractContextManager", "AbstractAsyncContextManager", "AsyncExitStack", "ContextDecorator", "ExitStack", - "redirect_stdout", "redirect_stderr", "suppress", "aclosing"] + "redirect_stdout", "redirect_stderr", "suppress", "aclosing", + "chdir"] class AbstractContextManager(abc.ABC): @@ -191,6 +193,14 @@ class _AsyncGeneratorContextManager( ): """Helper for @asynccontextmanager decorator.""" + def __call__(self, func): + @wraps(func) + async def inner(*args, **kwds): + async with self.__class__(self.func, self.args, self.kwds): + return await func(*args, **kwds) + + return inner + async def __aenter__(self): # do not keep args and kwds alive unnecessarily # they are only needed for recreation, which is not possible anymore @@ -545,10 +555,10 @@ class ExitStack(_BaseExitStack, AbstractContextManager): # Context may not be correct, so find the end of the chain while 1: exc_context = new_exc.__context__ - if exc_context is old_exc: + if exc_context is None or exc_context is old_exc: # Context is already set correctly (see issue 20317) return - if exc_context is None or exc_context is frame_exc: + if exc_context is frame_exc: break new_exc = exc_context # Change the end of the chain to point to the exception @@ -685,10 +695,10 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager): # Context may not be correct, so find the end of the chain while 1: exc_context = new_exc.__context__ - if exc_context is old_exc: + if exc_context is None or exc_context is old_exc: # Context is already set correctly (see issue 20317) return - if exc_context is None or exc_context is frame_exc: + if exc_context is frame_exc: break new_exc = exc_context # Change the end of the chain to point to the exception @@ -754,3 +764,18 @@ class nullcontext(AbstractContextManager, AbstractAsyncContextManager): async def __aexit__(self, *excinfo): pass + + +class chdir(AbstractContextManager): + """Non thread-safe context manager to change the current working directory.""" + + def __init__(self, path): + self.path = path + self._old_cwd = [] + + def __enter__(self): + self._old_cwd.append(os.getcwd()) + os.chdir(self.path) + + def __exit__(self, *excinfo): + os.chdir(self._old_cwd.pop()) diff --git a/Lib/copy.py b/Lib/copy.py index dd41c54dffe..69bac980be2 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -39,8 +39,8 @@ Python's deep copy operation avoids these problems by: set of components copied This version does not copy types like module, class, function, method, -nor stack trace, stack frame, nor file, socket, window, nor array, nor -any similar types. +nor stack trace, stack frame, nor file, socket, window, nor any +similar types. Classes can use the same interfaces to control copying that they use to control pickling: they can define methods called __getinitargs__(), diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 4afa4ebd422..b08629e8df4 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -65,12 +65,8 @@ def create_string_buffer(init, size=None): return buf raise TypeError(init) -def c_buffer(init, size=None): -## "deprecated, use create_string_buffer instead" -## import warnings -## warnings.warn("c_buffer is deprecated, use create_string_buffer instead", -## DeprecationWarning, stacklevel=2) - return create_string_buffer(init, size) +# Alias to create_string_buffer() for backward compatibility +c_buffer = create_string_buffer _c_functype_cache = {} def CFUNCTYPE(restype, *argtypes, **kw): diff --git a/Lib/ctypes/_aix.py b/Lib/ctypes/_aix.py index 190cac6507e..fc3e95cbcc8 100644 --- a/Lib/ctypes/_aix.py +++ b/Lib/ctypes/_aix.py @@ -163,7 +163,7 @@ def get_legacy(members): return member else: # 32-bit legacy names - both shr.o and shr4.o exist. - # shr.o is the preffered name so we look for shr.o first + # shr.o is the preferred name so we look for shr.o first # i.e., shr4.o is returned only when shr.o does not exist for name in ['shr.o', 'shr4.o']: member = get_one_match(re.escape(name), members) @@ -282,7 +282,7 @@ def find_shared(paths, name): if path.exists(archive): members = get_shared(get_ld_headers(archive)) member = get_member(re.escape(name), members) - if member != None: + if member is not None: return (base, member) else: return (None, None) @@ -307,7 +307,7 @@ def find_library(name): libpaths = get_libpaths() (base, member) = find_shared(libpaths, name) - if base != None: + if base is not None: return f"{base}({member})" # To get here, a member in an archive has not been found diff --git a/Lib/ctypes/test/test_functions.py b/Lib/ctypes/test/test_functions.py index d3c6536f276..f9e92e1cc6b 100644 --- a/Lib/ctypes/test/test_functions.py +++ b/Lib/ctypes/test/test_functions.py @@ -35,34 +35,24 @@ class FunctionTestCase(unittest.TestCase): # wasn't checked, and it even crashed Python. # Found by Greg Chapman. - try: + with self.assertRaises(TypeError): class X(object, Array): _length_ = 5 _type_ = "i" - except TypeError: - pass - from _ctypes import _Pointer - try: + with self.assertRaises(TypeError): class X(object, _Pointer): pass - except TypeError: - pass from _ctypes import _SimpleCData - try: + with self.assertRaises(TypeError): class X(object, _SimpleCData): _type_ = "i" - except TypeError: - pass - try: + with self.assertRaises(TypeError): class X(object, Structure): _fields_ = [] - except TypeError: - pass - @need_symbol('c_wchar') def test_wchar_parm(self): @@ -209,15 +199,6 @@ class FunctionTestCase(unittest.TestCase): result = f(byref(c_int(99))) self.assertNotEqual(result.contents, 99) - def test_errors(self): - f = dll._testfunc_p_p - f.restype = c_int - - class X(Structure): - _fields_ = [("y", c_int)] - - self.assertRaises(TypeError, f, X()) #cannot convert parameter - ################################################################ def test_shorts(self): f = dll._testfunc_callback_i_if diff --git a/Lib/ctypes/test/test_strings.py b/Lib/ctypes/test/test_strings.py index 5434efda10c..12e208828a7 100644 --- a/Lib/ctypes/test/test_strings.py +++ b/Lib/ctypes/test/test_strings.py @@ -85,74 +85,6 @@ class WStringArrayTestCase(unittest.TestCase): w = c_wchar(u) self.assertEqual(w.value, u) -class StringTestCase(unittest.TestCase): - @unittest.skip('test disabled') - def test_basic_strings(self): - cs = c_string("abcdef") - - # Cannot call len on a c_string any longer - self.assertRaises(TypeError, len, cs) - self.assertEqual(sizeof(cs), 7) - - # The value property is the string up to the first terminating NUL. - self.assertEqual(cs.value, "abcdef") - self.assertEqual(c_string("abc\000def").value, "abc") - - # The raw property is the total buffer contents: - self.assertEqual(cs.raw, "abcdef\000") - self.assertEqual(c_string("abc\000def").raw, "abc\000def\000") - - # We can change the value: - cs.value = "ab" - self.assertEqual(cs.value, "ab") - self.assertEqual(cs.raw, "ab\000\000\000\000\000") - - cs.raw = "XY" - self.assertEqual(cs.value, "XY") - self.assertEqual(cs.raw, "XY\000\000\000\000\000") - - self.assertRaises(TypeError, c_string, "123") - - @unittest.skip('test disabled') - def test_sized_strings(self): - - # New in releases later than 0.4.0: - self.assertRaises(TypeError, c_string, None) - - # New in releases later than 0.4.0: - # c_string(number) returns an empty string of size number - self.assertEqual(len(c_string(32).raw), 32) - self.assertRaises(ValueError, c_string, -1) - self.assertRaises(ValueError, c_string, 0) - - # These tests fail, because it is no longer initialized -## self.assertEqual(c_string(2).value, "") -## self.assertEqual(c_string(2).raw, "\000\000") - self.assertEqual(c_string(2).raw[-1], "\000") - self.assertEqual(len(c_string(2).raw), 2) - - @unittest.skip('test disabled') - def test_initialized_strings(self): - - self.assertEqual(c_string("ab", 4).raw[:2], "ab") - self.assertEqual(c_string("ab", 4).raw[:2:], "ab") - self.assertEqual(c_string("ab", 4).raw[:2:-1], "ba") - self.assertEqual(c_string("ab", 4).raw[:2:2], "a") - self.assertEqual(c_string("ab", 4).raw[-1], "\000") - self.assertEqual(c_string("ab", 2).raw, "a\000") - - @unittest.skip('test disabled') - def test_toolong(self): - cs = c_string("abcdef") - # Much too long string: - self.assertRaises(ValueError, setattr, cs, "value", "123456789012345") - - # One char too long values: - self.assertRaises(ValueError, setattr, cs, "value", "1234567") - - @unittest.skip('test disabled') - def test_perf(self): - check_perf() @need_symbol('c_wchar') class WStringTestCase(unittest.TestCase): @@ -208,25 +140,6 @@ def run_test(rep, msg, func, arg): stop = clock() print("%20s: %.2f us" % (msg, ((stop-start)*1e6/5/rep))) -def check_perf(): - # Construct 5 objects - - REP = 200000 - - run_test(REP, "c_string(None)", c_string, None) - run_test(REP, "c_string('abc')", c_string, 'abc') - -# Python 2.3 -OO, win2k, P4 700 MHz: -# -# c_string(None): 1.75 us -# c_string('abc'): 2.74 us - -# Python 2.2 -OO, win2k, P4 700 MHz: -# -# c_string(None): 2.95 us -# c_string('abc'): 3.67 us - if __name__ == '__main__': -## check_perf() unittest.main() diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py index 245cd94c5cd..97ad2b8ed8a 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/ctypes/test/test_structures.py @@ -443,7 +443,7 @@ class StructureTestCase(unittest.TestCase): s = Test(1, 2, 3) # Test the StructUnionType_paramfunc() code path which copies the - # structure: if the stucture is larger than sizeof(void*). + # structure: if the structure is larger than sizeof(void*). self.assertGreater(sizeof(s), sizeof(c_void_p)) dll = CDLL(_ctypes_test.__file__) @@ -451,7 +451,7 @@ class StructureTestCase(unittest.TestCase): func.argtypes = (Test,) func.restype = None func(s) - # bpo-37140: Passing the structure by refrence must not call + # bpo-37140: Passing the structure by reference must not call # its finalizer! self.assertEqual(finalizer_calls, []) self.assertEqual(s.first, 1) diff --git a/Lib/ctypes/test/test_values.py b/Lib/ctypes/test/test_values.py index 96a5f7cc1c5..3e8b13768b4 100644 --- a/Lib/ctypes/test/test_values.py +++ b/Lib/ctypes/test/test_values.py @@ -2,9 +2,12 @@ A testcase which accesses *values* in a dll. """ +import _imp +import importlib.util import unittest import sys from ctypes import * +from test.support import import_helper import _ctypes_test @@ -50,45 +53,42 @@ class PythonValuesTestCase(unittest.TestCase): class struct_frozen(Structure): _fields_ = [("name", c_char_p), ("code", POINTER(c_ubyte)), - ("size", c_int)] + ("size", c_int), + ("get_code", POINTER(c_ubyte)), # Function ptr + ] FrozenTable = POINTER(struct_frozen) - ft = FrozenTable.in_dll(pythonapi, "PyImport_FrozenModules") - # ft is a pointer to the struct_frozen entries: - items = [] - # _frozen_importlib changes size whenever importlib._bootstrap - # changes, so it gets a special case. We should make sure it's - # found, but don't worry about its size too much. The same - # applies to _frozen_importlib_external. - bootstrap_seen = [] - bootstrap_expected = [ - b'_frozen_importlib', - b'_frozen_importlib_external', - b'zipimport', - ] - for entry in ft: - # This is dangerous. We *can* iterate over a pointer, but - # the loop will not terminate (maybe with an access - # violation;-) because the pointer instance has no size. - if entry.name is None: - break + modules = [] + for group in ["Bootstrap", "Stdlib", "Test"]: + ft = FrozenTable.in_dll(pythonapi, f"_PyImport_Frozen{group}") + # ft is a pointer to the struct_frozen entries: + for entry in ft: + # This is dangerous. We *can* iterate over a pointer, but + # the loop will not terminate (maybe with an access + # violation;-) because the pointer instance has no size. + if entry.name is None: + break + modname = entry.name.decode("ascii") + modules.append(modname) + with self.subTest(modname): + # Do a sanity check on entry.size and entry.code. + self.assertGreater(abs(entry.size), 10) + self.assertTrue([entry.code[i] for i in range(abs(entry.size))]) + # Check the module's package-ness. + with import_helper.frozen_modules(): + spec = importlib.util.find_spec(modname) + if entry.size < 0: + # It's a package. + self.assertIsNotNone(spec.submodule_search_locations) + else: + self.assertIsNone(spec.submodule_search_locations) - if entry.name in bootstrap_expected: - bootstrap_seen.append(entry.name) - self.assertTrue(entry.size, - "{!r} was reported as having no size".format(entry.name)) - continue - items.append((entry.name.decode("ascii"), entry.size)) - - expected = [("__hello__", 164), - ("__phello__", -164), - ("__phello__.spam", 164), - ] - self.assertEqual(items, expected, "PyImport_FrozenModules example " - "in Doc/library/ctypes.rst may be out of date") - - self.assertEqual(sorted(bootstrap_seen), bootstrap_expected, - "frozen bootstrap modules did not match PyImport_FrozenModules") + with import_helper.frozen_modules(): + expected = _imp._frozen_module_names() + self.maxDiff = None + self.assertEqual(modules, expected, + "_PyImport_FrozenBootstrap example " + "in Doc/library/ctypes.rst may be out of date") from ctypes import _pointer_type_cache del _pointer_type_cache[struct_frozen] diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 95ff39287be..b327462080f 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -229,7 +229,7 @@ class InitVar: self.type = type def __repr__(self): - if isinstance(self.type, type): + if isinstance(self.type, type) and not isinstance(self.type, GenericAlias): type_name = self.type.__name__ else: # typing objects, e.g. List[int] @@ -447,7 +447,7 @@ def _field_assign(frozen, name, value, self_name): return f'{self_name}.{name}={value}' -def _field_init(f, frozen, globals, self_name): +def _field_init(f, frozen, globals, self_name, slots): # Return the text of the line in the body of __init__ that will # initialize this field. @@ -487,9 +487,15 @@ def _field_init(f, frozen, globals, self_name): globals[default_name] = f.default value = f.name else: - # This field does not need initialization. Signify that - # to the caller by returning None. - return None + # If the class has slots, then initialize this field. + if slots and f.default is not MISSING: + globals[default_name] = f.default + value = default_name + else: + # This field does not need initialization: reading from it will + # just use the class attribute that contains the default. + # Signify that to the caller by returning None. + return None # Only test this now, so that we can create variables for the # default. However, return None to signify that we're not going @@ -521,7 +527,7 @@ def _init_param(f): def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, - self_name, globals): + self_name, globals, slots): # fields contains both real fields and InitVar pseudo-fields. # Make sure we don't have fields without defaults following fields @@ -548,7 +554,7 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, body_lines = [] for f in fields: - line = _field_init(f, frozen, locals, self_name) + line = _field_init(f, frozen, locals, self_name, slots) # line is None means that this field doesn't require # initialization (it's a pseudo-field). Just skip it. if line: @@ -802,8 +808,10 @@ def _get_field(cls, a_name, a_type, default_kw_only): raise TypeError(f'field {f.name} is a ClassVar but specifies ' 'kw_only') - # For real fields, disallow mutable defaults for known types. - if f._field_type is _FIELD and isinstance(f.default, (list, dict, set)): + # For real fields, disallow mutable defaults. Use unhashable as a proxy + # indicator for mutability. Read the __hash__ attribute from the class, + # not the instance. + if f._field_type is _FIELD and f.default.__class__.__hash__ is None: raise ValueError(f'mutable default {type(f.default)} for field ' f'{f.name} is not allowed: use default_factory') @@ -1027,6 +1035,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, '__dataclass_self__' if 'self' in fields else 'self', globals, + slots, )) # Get the fields as a list, and include only real fields. This is @@ -1204,7 +1213,7 @@ def _is_dataclass_instance(obj): def is_dataclass(obj): """Returns True if obj is a dataclass or an instance of a dataclass.""" - cls = obj if isinstance(obj, type) else type(obj) + cls = obj if isinstance(obj, type) and not isinstance(obj, GenericAlias) else type(obj) return hasattr(cls, _FIELDS) @@ -1326,7 +1335,7 @@ def _astuple_inner(obj, tuple_factory): def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, - frozen=False, match_args=True, slots=False): + frozen=False, match_args=True, kw_only=False, slots=False): """Return a new dynamically created dataclass. The dataclass name will be 'cls_name'. 'fields' is an iterable @@ -1387,13 +1396,13 @@ def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, ns['__annotations__'] = annotations # We use `types.new_class()` instead of simply `type()` to allow dynamic creation - # of generic dataclassses. + # of generic dataclasses. cls = types.new_class(cls_name, bases, {}, exec_body_callback) # Apply the normal decorator. return dataclass(cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen, - match_args=match_args, slots=slots) + match_args=match_args, kw_only=kw_only, slots=slots) def replace(obj, /, **changes): diff --git a/Lib/dbm/__init__.py b/Lib/dbm/__init__.py index f65da521af4..8055d3769f9 100644 --- a/Lib/dbm/__init__.py +++ b/Lib/dbm/__init__.py @@ -109,17 +109,18 @@ def whichdb(filename): """ # Check for ndbm first -- this has a .pag and a .dir file + filename = os.fsencode(filename) try: - f = io.open(filename + ".pag", "rb") + f = io.open(filename + b".pag", "rb") f.close() - f = io.open(filename + ".dir", "rb") + f = io.open(filename + b".dir", "rb") f.close() return "dbm.ndbm" except OSError: # some dbm emulations based on Berkeley DB generate a .db file # some do not, but they should be caught by the bsd checks try: - f = io.open(filename + ".db", "rb") + f = io.open(filename + b".db", "rb") f.close() # guarantee we can actually open the file using dbm # kind of overkill, but since we are dealing with emulations @@ -134,12 +135,12 @@ def whichdb(filename): # Check for dumbdbm next -- this has a .dir and a .dat file try: # First check for presence of files - os.stat(filename + ".dat") - size = os.stat(filename + ".dir").st_size + os.stat(filename + b".dat") + size = os.stat(filename + b".dir").st_size # dumbdbm files with no keys are empty if size == 0: return "dbm.dumb" - f = io.open(filename + ".dir", "rb") + f = io.open(filename + b".dir", "rb") try: if f.read(1) in (b"'", b'"'): return "dbm.dumb" diff --git a/Lib/dbm/dumb.py b/Lib/dbm/dumb.py index 864ad371ec9..754624ccc8f 100644 --- a/Lib/dbm/dumb.py +++ b/Lib/dbm/dumb.py @@ -46,6 +46,7 @@ class _Database(collections.abc.MutableMapping): _io = _io # for _commit() def __init__(self, filebasename, mode, flag='c'): + filebasename = self._os.fsencode(filebasename) self._mode = mode self._readonly = (flag == 'r') @@ -54,14 +55,14 @@ class _Database(collections.abc.MutableMapping): # where key is the string key, pos is the offset into the dat # file of the associated value's first byte, and siz is the number # of bytes in the associated value. - self._dirfile = filebasename + '.dir' + self._dirfile = filebasename + b'.dir' # The data file is a binary file pointed into by the directory # file, and holds the values associated with keys. Each value # begins at a _BLOCKSIZE-aligned byte offset, and is a raw # binary 8-bit string value. - self._datfile = filebasename + '.dat' - self._bakfile = filebasename + '.bak' + self._datfile = filebasename + b'.dat' + self._bakfile = filebasename + b'.bak' # The index is an in-memory dict, mirroring the directory file. self._index = None # maps keys to (pos, siz) pairs diff --git a/Lib/difflib.py b/Lib/difflib.py index 0dda80d3875..afd8a0c7c5b 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -62,7 +62,7 @@ class SequenceMatcher: notion, pairing up elements that appear uniquely in each sequence. That, and the method here, appear to yield more intuitive difference reports than does diff. This method appears to be the least vulnerable - to synching up on blocks of "junk lines", though (like blank lines in + to syncing up on blocks of "junk lines", though (like blank lines in ordinary text files, or maybe "

" lines in HTML files). That may be because this is the only method of the 3 that has a *concept* of "junk" . @@ -115,38 +115,6 @@ class SequenceMatcher: case. SequenceMatcher is quadratic time for the worst case and has expected-case behavior dependent in a complicated way on how many elements the sequences have in common; best case time is linear. - - Methods: - - __init__(isjunk=None, a='', b='') - Construct a SequenceMatcher. - - set_seqs(a, b) - Set the two sequences to be compared. - - set_seq1(a) - Set the first sequence to be compared. - - set_seq2(b) - Set the second sequence to be compared. - - find_longest_match(alo=0, ahi=None, blo=0, bhi=None) - Find longest matching block in a[alo:ahi] and b[blo:bhi]. - - get_matching_blocks() - Return list of triples describing matching subsequences. - - get_opcodes() - Return list of 5-tuples describing how to turn a into b. - - ratio() - Return a measure of the sequences' similarity (float in [0,1]). - - quick_ratio() - Return an upper bound on .ratio() relatively quickly. - - real_quick_ratio() - Return an upper bound on ratio() very quickly. """ def __init__(self, isjunk=None, a='', b='', autojunk=True): @@ -837,14 +805,6 @@ class Differ: + 4. Complicated is better than complex. ? ++++ ^ ^ + 5. Flat is better than nested. - - Methods: - - __init__(linejunk=None, charjunk=None) - Construct a text differencer, with optional filters. - - compare(a, b) - Compare two sequences of lines; generate the resulting delta. """ def __init__(self, linejunk=None, charjunk=None): diff --git a/Lib/dis.py b/Lib/dis.py index 3cacd9a44a9..ac0c6e7f04c 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -7,6 +7,7 @@ import io from opcode import * from opcode import __all__ as _opcodes_all +from opcode import _nb_ops __all__ = ["code_info", "dis", "disassemble", "distb", "disco", "findlinestarts", "findlabels", "show_code", @@ -26,6 +27,8 @@ FORMAT_VALUE_CONVERTERS = ( MAKE_FUNCTION = opmap['MAKE_FUNCTION'] MAKE_FUNCTION_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure') +LOAD_CONST = opmap['LOAD_CONST'] +BINARY_OP = opmap['BINARY_OP'] def _try_compile(source, name): """Attempts to compile the given source, first as an expression and @@ -125,6 +128,13 @@ def pretty_flags(flags): names.append(hex(flags)) return ", ".join(names) +class _Unknown: + def __repr__(self): + return "" + +# Sentinel to represent values that cannot be calculated +UNKNOWN = _Unknown() + def _get_code_object(x): """Helper to handle methods, compiled or raw code objects, and strings.""" # Extract functions from methods. @@ -310,33 +320,46 @@ def get_instructions(x, *, first_line=None): co.co_names, co.co_consts, linestarts, line_offset, co_positions=co.co_positions()) -def _get_const_info(const_index, const_list): +def _get_const_value(op, arg, co_consts): + """Helper to get the value of the const in a hasconst op. + + Returns the dereferenced constant if this is possible. + Otherwise (if it is a LOAD_CONST and co_consts is not + provided) returns the dis.UNKNOWN sentinel. + """ + assert op in hasconst + + argval = UNKNOWN + if op == LOAD_CONST: + if co_consts is not None: + argval = co_consts[arg] + return argval + +def _get_const_info(op, arg, co_consts): """Helper to get optional details about const references - Returns the dereferenced constant and its repr if the constant - list is defined. - Otherwise returns the constant index and its repr(). + Returns the dereferenced constant and its repr if the value + can be calculated. + Otherwise returns the sentinel value dis.UNKNOWN for the value + and an empty string for its repr. """ - argval = const_index - if const_list is not None: - argval = const_list[const_index] - return argval, repr(argval) + argval = _get_const_value(op, arg, co_consts) + argrepr = repr(argval) if argval is not UNKNOWN else '' + return argval, argrepr def _get_name_info(name_index, get_name, **extrainfo): """Helper to get optional details about named references Returns the dereferenced name as both value and repr if the name list is defined. - Otherwise returns the name index and its repr(). + Otherwise returns the sentinel value dis.UNKNOWN for the value + and an empty string for its repr. """ - argval = name_index if get_name is not None: argval = get_name(name_index, **extrainfo) - argrepr = argval + return argval, argval else: - argrepr = repr(argval) - return argval, argrepr - + return UNKNOWN, '' def parse_varint(iterator): b = next(iterator) @@ -364,14 +387,14 @@ def parse_exception_table(code): return entries def _get_instructions_bytes(code, varname_from_oparg=None, - names=None, constants=None, + names=None, co_consts=None, linestarts=None, line_offset=0, exception_entries=(), co_positions=None): """Iterate over the instructions in a bytecode string. Generates a sequence of Instruction namedtuples giving the details of each opcode. Additional information about the code's runtime environment - (e.g. variable names, constants) can be specified using optional + (e.g. variable names, co_consts) can be specified using optional arguments. """ @@ -401,7 +424,7 @@ def _get_instructions_bytes(code, varname_from_oparg=None, # raw name index for LOAD_GLOBAL, LOAD_CONST, etc. argval = arg if op in hasconst: - argval, argrepr = _get_const_info(arg, constants) + argval, argrepr = _get_const_info(op, arg, co_consts) elif op in hasname: argval, argrepr = _get_name_info(arg, get_name) elif op in hasjabs: @@ -425,6 +448,8 @@ def _get_instructions_bytes(code, varname_from_oparg=None, elif op == MAKE_FUNCTION: argrepr = ', '.join(s for i, s in enumerate(MAKE_FUNCTION_FLAGS) if arg & (1<= 2: + from_op = opargs[i-1] + level_op = opargs[i-2] + if (from_op[0] in hasconst and level_op[0] in hasconst): + level = _get_const_value(level_op[0], level_op[1], consts) + fromlist = _get_const_value(from_op[0], from_op[1], consts) + yield (names[oparg], level, fromlist) + +def _find_store_names(co): + """Find names of variables which are written in the code + + Generate sequence of strings + """ + STORE_OPS = { + opmap['STORE_NAME'], + opmap['STORE_GLOBAL'] + } + + names = co.co_names + for _, op, arg in _unpack_opargs(co.co_code): + if op in STORE_OPS: + yield names[arg] + class Bytecode: """The bytecode operations of a piece of code @@ -564,7 +628,8 @@ class Bytecode: co.co_names, co.co_consts, self._linestarts, line_offset=self._line_offset, - exception_entries=self.exception_entries) + exception_entries=self.exception_entries, + co_positions=co.co_positions()) def __repr__(self): return "{}({!r})".format(self.__class__.__name__, @@ -591,7 +656,7 @@ class Bytecode: with io.StringIO() as output: _disassemble_bytes(co.co_code, varname_from_oparg=co._varname_from_oparg, - names=co.co_names, constants=co.co_consts, + names=co.co_names, co_consts=co.co_consts, linestarts=self._linestarts, line_offset=self._line_offset, file=output, diff --git a/Lib/distutils/ccompiler.py b/Lib/distutils/ccompiler.py index b5ef143e72c..4c47f2ed245 100644 --- a/Lib/distutils/ccompiler.py +++ b/Lib/distutils/ccompiler.py @@ -392,7 +392,7 @@ class CCompiler: return output_dir, macros, include_dirs def _prep_compile(self, sources, output_dir, depends=None): - """Decide which souce files must be recompiled. + """Decide which source files must be recompiled. Determine the list of object files corresponding to 'sources', and figure out which ones really need to be recompiled. diff --git a/Lib/distutils/command/bdist.py b/Lib/distutils/command/bdist.py index d580a8090bd..60309e1ff2f 100644 --- a/Lib/distutils/command/bdist.py +++ b/Lib/distutils/command/bdist.py @@ -61,8 +61,7 @@ class bdist(Command): 'nt': 'zip'} # Establish the preferred order (for the --help-formats option). - format_commands = ['rpm', 'gztar', 'bztar', 'xztar', 'ztar', 'tar', - 'zip', 'msi'] + format_commands = ['rpm', 'gztar', 'bztar', 'xztar', 'ztar', 'tar', 'zip'] # And the real information. format_command = {'rpm': ('bdist_rpm', "RPM distribution"), @@ -72,10 +71,8 @@ class bdist(Command): 'ztar': ('bdist_dumb', "compressed tar file"), 'tar': ('bdist_dumb', "tar file"), 'zip': ('bdist_dumb', "ZIP file"), - 'msi': ('bdist_msi', "Microsoft Installer") } - def initialize_options(self): self.bdist_base = None self.plat_name = None diff --git a/Lib/distutils/command/bdist_msi.py b/Lib/distutils/command/bdist_msi.py deleted file mode 100644 index 2ed017b4d66..00000000000 --- a/Lib/distutils/command/bdist_msi.py +++ /dev/null @@ -1,747 +0,0 @@ -# Copyright (C) 2005, 2006 Martin von Löwis -# Licensed to PSF under a Contributor Agreement. -""" -Implements the bdist_msi command. -""" - -import os -import sys -import warnings -from distutils.core import Command -from distutils.dir_util import remove_tree -from distutils.sysconfig import get_python_version -from distutils.version import StrictVersion -from distutils.errors import DistutilsOptionError -from distutils.util import get_platform -from distutils import log -import msilib -from msilib import schema, sequence, text -from msilib import Directory, Feature, Dialog, add_data - -class PyDialog(Dialog): - """Dialog class with a fixed layout: controls at the top, then a ruler, - then a list of buttons: back, next, cancel. Optionally a bitmap at the - left.""" - def __init__(self, *args, **kw): - """Dialog(database, name, x, y, w, h, attributes, title, first, - default, cancel, bitmap=true)""" - Dialog.__init__(self, *args) - ruler = self.h - 36 - bmwidth = 152*ruler/328 - #if kw.get("bitmap", True): - # self.bitmap("Bitmap", 0, 0, bmwidth, ruler, "PythonWin") - self.line("BottomLine", 0, ruler, self.w, 0) - - def title(self, title): - "Set the title text of the dialog at the top." - # name, x, y, w, h, flags=Visible|Enabled|Transparent|NoPrefix, - # text, in VerdanaBold10 - self.text("Title", 15, 10, 320, 60, 0x30003, - r"{\VerdanaBold10}%s" % title) - - def back(self, title, next, name = "Back", active = 1): - """Add a back button with a given title, the tab-next button, - its name in the Control table, possibly initially disabled. - - Return the button, so that events can be associated""" - if active: - flags = 3 # Visible|Enabled - else: - flags = 1 # Visible - return self.pushbutton(name, 180, self.h-27 , 56, 17, flags, title, next) - - def cancel(self, title, next, name = "Cancel", active = 1): - """Add a cancel button with a given title, the tab-next button, - its name in the Control table, possibly initially disabled. - - Return the button, so that events can be associated""" - if active: - flags = 3 # Visible|Enabled - else: - flags = 1 # Visible - return self.pushbutton(name, 304, self.h-27, 56, 17, flags, title, next) - - def next(self, title, next, name = "Next", active = 1): - """Add a Next button with a given title, the tab-next button, - its name in the Control table, possibly initially disabled. - - Return the button, so that events can be associated""" - if active: - flags = 3 # Visible|Enabled - else: - flags = 1 # Visible - return self.pushbutton(name, 236, self.h-27, 56, 17, flags, title, next) - - def xbutton(self, name, title, next, xpos): - """Add a button with a given title, the tab-next button, - its name in the Control table, giving its x position; the - y-position is aligned with the other buttons. - - Return the button, so that events can be associated""" - return self.pushbutton(name, int(self.w*xpos - 28), self.h-27, 56, 17, 3, title, next) - -class bdist_msi(Command): - - description = "create a Microsoft Installer (.msi) binary distribution" - - user_options = [('bdist-dir=', None, - "temporary directory for creating the distribution"), - ('plat-name=', 'p', - "platform name to embed in generated filenames " - "(default: %s)" % get_platform()), - ('keep-temp', 'k', - "keep the pseudo-installation tree around after " + - "creating the distribution archive"), - ('target-version=', None, - "require a specific python version" + - " on the target system"), - ('no-target-compile', 'c', - "do not compile .py to .pyc on the target system"), - ('no-target-optimize', 'o', - "do not compile .py to .pyo (optimized) " - "on the target system"), - ('dist-dir=', 'd', - "directory to put final built distributions in"), - ('skip-build', None, - "skip rebuilding everything (for testing/debugging)"), - ('install-script=', None, - "basename of installation script to be run after " - "installation or before deinstallation"), - ('pre-install-script=', None, - "Fully qualified filename of a script to be run before " - "any files are installed. This script need not be in the " - "distribution"), - ] - - boolean_options = ['keep-temp', 'no-target-compile', 'no-target-optimize', - 'skip-build'] - - all_versions = ['2.0', '2.1', '2.2', '2.3', '2.4', - '2.5', '2.6', '2.7', '2.8', '2.9', - '3.0', '3.1', '3.2', '3.3', '3.4', - '3.5', '3.6', '3.7', '3.8', '3.9'] - other_version = 'X' - - def __init__(self, *args, **kw): - super().__init__(*args, **kw) - warnings.warn("bdist_msi command is deprecated since Python 3.9, " - "use bdist_wheel (wheel packages) instead", - DeprecationWarning, 2) - - def initialize_options(self): - self.bdist_dir = None - self.plat_name = None - self.keep_temp = 0 - self.no_target_compile = 0 - self.no_target_optimize = 0 - self.target_version = None - self.dist_dir = None - self.skip_build = None - self.install_script = None - self.pre_install_script = None - self.versions = None - - def finalize_options(self): - self.set_undefined_options('bdist', ('skip_build', 'skip_build')) - - if self.bdist_dir is None: - bdist_base = self.get_finalized_command('bdist').bdist_base - self.bdist_dir = os.path.join(bdist_base, 'msi') - - short_version = get_python_version() - if (not self.target_version) and self.distribution.has_ext_modules(): - self.target_version = short_version - - if self.target_version: - self.versions = [self.target_version] - if not self.skip_build and self.distribution.has_ext_modules()\ - and self.target_version != short_version: - raise DistutilsOptionError( - "target version can only be %s, or the '--skip-build'" - " option must be specified" % (short_version,)) - else: - self.versions = list(self.all_versions) - - self.set_undefined_options('bdist', - ('dist_dir', 'dist_dir'), - ('plat_name', 'plat_name'), - ) - - if self.pre_install_script: - raise DistutilsOptionError( - "the pre-install-script feature is not yet implemented") - - if self.install_script: - for script in self.distribution.scripts: - if self.install_script == os.path.basename(script): - break - else: - raise DistutilsOptionError( - "install_script '%s' not found in scripts" - % self.install_script) - self.install_script_key = None - - def run(self): - if not self.skip_build: - self.run_command('build') - - install = self.reinitialize_command('install', reinit_subcommands=1) - install.prefix = self.bdist_dir - install.skip_build = self.skip_build - install.warn_dir = 0 - - install_lib = self.reinitialize_command('install_lib') - # we do not want to include pyc or pyo files - install_lib.compile = 0 - install_lib.optimize = 0 - - if self.distribution.has_ext_modules(): - # If we are building an installer for a Python version other - # than the one we are currently running, then we need to ensure - # our build_lib reflects the other Python version rather than ours. - # Note that for target_version!=sys.version, we must have skipped the - # build step, so there is no issue with enforcing the build of this - # version. - target_version = self.target_version - if not target_version: - assert self.skip_build, "Should have already checked this" - target_version = '%d.%d' % sys.version_info[:2] - plat_specifier = ".%s-%s" % (self.plat_name, target_version) - build = self.get_finalized_command('build') - build.build_lib = os.path.join(build.build_base, - 'lib' + plat_specifier) - - log.info("installing to %s", self.bdist_dir) - install.ensure_finalized() - - # avoid warning of 'install_lib' about installing - # into a directory not in sys.path - sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB')) - - install.run() - - del sys.path[0] - - self.mkpath(self.dist_dir) - fullname = self.distribution.get_fullname() - installer_name = self.get_installer_filename(fullname) - installer_name = os.path.abspath(installer_name) - if os.path.exists(installer_name): os.unlink(installer_name) - - metadata = self.distribution.metadata - author = metadata.author - if not author: - author = metadata.maintainer - if not author: - author = "UNKNOWN" - version = metadata.get_version() - # ProductVersion must be strictly numeric - # XXX need to deal with prerelease versions - sversion = "%d.%d.%d" % StrictVersion(version).version - # Prefix ProductName with Python x.y, so that - # it sorts together with the other Python packages - # in Add-Remove-Programs (APR) - fullname = self.distribution.get_fullname() - if self.target_version: - product_name = "Python %s %s" % (self.target_version, fullname) - else: - product_name = "Python %s" % (fullname) - self.db = msilib.init_database(installer_name, schema, - product_name, msilib.gen_uuid(), - sversion, author) - msilib.add_tables(self.db, sequence) - props = [('DistVersion', version)] - email = metadata.author_email or metadata.maintainer_email - if email: - props.append(("ARPCONTACT", email)) - if metadata.url: - props.append(("ARPURLINFOABOUT", metadata.url)) - if props: - add_data(self.db, 'Property', props) - - self.add_find_python() - self.add_files() - self.add_scripts() - self.add_ui() - self.db.Commit() - - if hasattr(self.distribution, 'dist_files'): - tup = 'bdist_msi', self.target_version or 'any', fullname - self.distribution.dist_files.append(tup) - - if not self.keep_temp: - remove_tree(self.bdist_dir, dry_run=self.dry_run) - - def add_files(self): - db = self.db - cab = msilib.CAB("distfiles") - rootdir = os.path.abspath(self.bdist_dir) - - root = Directory(db, cab, None, rootdir, "TARGETDIR", "SourceDir") - f = Feature(db, "Python", "Python", "Everything", - 0, 1, directory="TARGETDIR") - - items = [(f, root, '')] - for version in self.versions + [self.other_version]: - target = "TARGETDIR" + version - name = default = "Python" + version - desc = "Everything" - if version is self.other_version: - title = "Python from another location" - level = 2 - else: - title = "Python %s from registry" % version - level = 1 - f = Feature(db, name, title, desc, 1, level, directory=target) - dir = Directory(db, cab, root, rootdir, target, default) - items.append((f, dir, version)) - db.Commit() - - seen = {} - for feature, dir, version in items: - todo = [dir] - while todo: - dir = todo.pop() - for file in os.listdir(dir.absolute): - afile = os.path.join(dir.absolute, file) - if os.path.isdir(afile): - short = "%s|%s" % (dir.make_short(file), file) - default = file + version - newdir = Directory(db, cab, dir, file, default, short) - todo.append(newdir) - else: - if not dir.component: - dir.start_component(dir.logical, feature, 0) - if afile not in seen: - key = seen[afile] = dir.add_file(file) - if file==self.install_script: - if self.install_script_key: - raise DistutilsOptionError( - "Multiple files with name %s" % file) - self.install_script_key = '[#%s]' % key - else: - key = seen[afile] - add_data(self.db, "DuplicateFile", - [(key + version, dir.component, key, None, dir.logical)]) - db.Commit() - cab.commit(db) - - def add_find_python(self): - """Adds code to the installer to compute the location of Python. - - Properties PYTHON.MACHINE.X.Y and PYTHON.USER.X.Y will be set from the - registry for each version of Python. - - Properties TARGETDIRX.Y will be set from PYTHON.USER.X.Y if defined, - else from PYTHON.MACHINE.X.Y. - - Properties PYTHONX.Y will be set to TARGETDIRX.Y\\python.exe""" - - start = 402 - for ver in self.versions: - install_path = r"SOFTWARE\Python\PythonCore\%s\InstallPath" % ver - machine_reg = "python.machine." + ver - user_reg = "python.user." + ver - machine_prop = "PYTHON.MACHINE." + ver - user_prop = "PYTHON.USER." + ver - machine_action = "PythonFromMachine" + ver - user_action = "PythonFromUser" + ver - exe_action = "PythonExe" + ver - target_dir_prop = "TARGETDIR" + ver - exe_prop = "PYTHON" + ver - if msilib.Win64: - # type: msidbLocatorTypeRawValue + msidbLocatorType64bit - Type = 2+16 - else: - Type = 2 - add_data(self.db, "RegLocator", - [(machine_reg, 2, install_path, None, Type), - (user_reg, 1, install_path, None, Type)]) - add_data(self.db, "AppSearch", - [(machine_prop, machine_reg), - (user_prop, user_reg)]) - add_data(self.db, "CustomAction", - [(machine_action, 51+256, target_dir_prop, "[" + machine_prop + "]"), - (user_action, 51+256, target_dir_prop, "[" + user_prop + "]"), - (exe_action, 51+256, exe_prop, "[" + target_dir_prop + "]\\python.exe"), - ]) - add_data(self.db, "InstallExecuteSequence", - [(machine_action, machine_prop, start), - (user_action, user_prop, start + 1), - (exe_action, None, start + 2), - ]) - add_data(self.db, "InstallUISequence", - [(machine_action, machine_prop, start), - (user_action, user_prop, start + 1), - (exe_action, None, start + 2), - ]) - add_data(self.db, "Condition", - [("Python" + ver, 0, "NOT TARGETDIR" + ver)]) - start += 4 - assert start < 500 - - def add_scripts(self): - if self.install_script: - start = 6800 - for ver in self.versions + [self.other_version]: - install_action = "install_script." + ver - exe_prop = "PYTHON" + ver - add_data(self.db, "CustomAction", - [(install_action, 50, exe_prop, self.install_script_key)]) - add_data(self.db, "InstallExecuteSequence", - [(install_action, "&Python%s=3" % ver, start)]) - start += 1 - # XXX pre-install scripts are currently refused in finalize_options() - # but if this feature is completed, it will also need to add - # entries for each version as the above code does - if self.pre_install_script: - scriptfn = os.path.join(self.bdist_dir, "preinstall.bat") - with open(scriptfn, "w") as f: - # The batch file will be executed with [PYTHON], so that %1 - # is the path to the Python interpreter; %0 will be the path - # of the batch file. - # rem =""" - # %1 %0 - # exit - # """ - # - f.write('rem ="""\n%1 %0\nexit\n"""\n') - with open(self.pre_install_script) as fin: - f.write(fin.read()) - add_data(self.db, "Binary", - [("PreInstall", msilib.Binary(scriptfn)) - ]) - add_data(self.db, "CustomAction", - [("PreInstall", 2, "PreInstall", None) - ]) - add_data(self.db, "InstallExecuteSequence", - [("PreInstall", "NOT Installed", 450)]) - - - def add_ui(self): - db = self.db - x = y = 50 - w = 370 - h = 300 - title = "[ProductName] Setup" - - # see "Dialog Style Bits" - modal = 3 # visible | modal - modeless = 1 # visible - track_disk_space = 32 - - # UI customization properties - add_data(db, "Property", - # See "DefaultUIFont Property" - [("DefaultUIFont", "DlgFont8"), - # See "ErrorDialog Style Bit" - ("ErrorDialog", "ErrorDlg"), - ("Progress1", "Install"), # modified in maintenance type dlg - ("Progress2", "installs"), - ("MaintenanceForm_Action", "Repair"), - # possible values: ALL, JUSTME - ("WhichUsers", "ALL") - ]) - - # Fonts, see "TextStyle Table" - add_data(db, "TextStyle", - [("DlgFont8", "Tahoma", 9, None, 0), - ("DlgFontBold8", "Tahoma", 8, None, 1), #bold - ("VerdanaBold10", "Verdana", 10, None, 1), - ("VerdanaRed9", "Verdana", 9, 255, 0), - ]) - - # UI Sequences, see "InstallUISequence Table", "Using a Sequence Table" - # Numbers indicate sequence; see sequence.py for how these action integrate - add_data(db, "InstallUISequence", - [("PrepareDlg", "Not Privileged or Windows9x or Installed", 140), - ("WhichUsersDlg", "Privileged and not Windows9x and not Installed", 141), - # In the user interface, assume all-users installation if privileged. - ("SelectFeaturesDlg", "Not Installed", 1230), - # XXX no support for resume installations yet - #("ResumeDlg", "Installed AND (RESUME OR Preselected)", 1240), - ("MaintenanceTypeDlg", "Installed AND NOT RESUME AND NOT Preselected", 1250), - ("ProgressDlg", None, 1280)]) - - add_data(db, 'ActionText', text.ActionText) - add_data(db, 'UIText', text.UIText) - ##################################################################### - # Standard dialogs: FatalError, UserExit, ExitDialog - fatal=PyDialog(db, "FatalError", x, y, w, h, modal, title, - "Finish", "Finish", "Finish") - fatal.title("[ProductName] Installer ended prematurely") - fatal.back("< Back", "Finish", active = 0) - fatal.cancel("Cancel", "Back", active = 0) - fatal.text("Description1", 15, 70, 320, 80, 0x30003, - "[ProductName] setup ended prematurely because of an error. Your system has not been modified. To install this program at a later time, please run the installation again.") - fatal.text("Description2", 15, 155, 320, 20, 0x30003, - "Click the Finish button to exit the Installer.") - c=fatal.next("Finish", "Cancel", name="Finish") - c.event("EndDialog", "Exit") - - user_exit=PyDialog(db, "UserExit", x, y, w, h, modal, title, - "Finish", "Finish", "Finish") - user_exit.title("[ProductName] Installer was interrupted") - user_exit.back("< Back", "Finish", active = 0) - user_exit.cancel("Cancel", "Back", active = 0) - user_exit.text("Description1", 15, 70, 320, 80, 0x30003, - "[ProductName] setup was interrupted. Your system has not been modified. " - "To install this program at a later time, please run the installation again.") - user_exit.text("Description2", 15, 155, 320, 20, 0x30003, - "Click the Finish button to exit the Installer.") - c = user_exit.next("Finish", "Cancel", name="Finish") - c.event("EndDialog", "Exit") - - exit_dialog = PyDialog(db, "ExitDialog", x, y, w, h, modal, title, - "Finish", "Finish", "Finish") - exit_dialog.title("Completing the [ProductName] Installer") - exit_dialog.back("< Back", "Finish", active = 0) - exit_dialog.cancel("Cancel", "Back", active = 0) - exit_dialog.text("Description", 15, 235, 320, 20, 0x30003, - "Click the Finish button to exit the Installer.") - c = exit_dialog.next("Finish", "Cancel", name="Finish") - c.event("EndDialog", "Return") - - ##################################################################### - # Required dialog: FilesInUse, ErrorDlg - inuse = PyDialog(db, "FilesInUse", - x, y, w, h, - 19, # KeepModeless|Modal|Visible - title, - "Retry", "Retry", "Retry", bitmap=False) - inuse.text("Title", 15, 6, 200, 15, 0x30003, - r"{\DlgFontBold8}Files in Use") - inuse.text("Description", 20, 23, 280, 20, 0x30003, - "Some files that need to be updated are currently in use.") - inuse.text("Text", 20, 55, 330, 50, 3, - "The following applications are using files that need to be updated by this setup. Close these applications and then click Retry to continue the installation or Cancel to exit it.") - inuse.control("List", "ListBox", 20, 107, 330, 130, 7, "FileInUseProcess", - None, None, None) - c=inuse.back("Exit", "Ignore", name="Exit") - c.event("EndDialog", "Exit") - c=inuse.next("Ignore", "Retry", name="Ignore") - c.event("EndDialog", "Ignore") - c=inuse.cancel("Retry", "Exit", name="Retry") - c.event("EndDialog","Retry") - - # See "Error Dialog". See "ICE20" for the required names of the controls. - error = Dialog(db, "ErrorDlg", - 50, 10, 330, 101, - 65543, # Error|Minimize|Modal|Visible - title, - "ErrorText", None, None) - error.text("ErrorText", 50,9,280,48,3, "") - #error.control("ErrorIcon", "Icon", 15, 9, 24, 24, 5242881, None, "py.ico", None, None) - error.pushbutton("N",120,72,81,21,3,"No",None).event("EndDialog","ErrorNo") - error.pushbutton("Y",240,72,81,21,3,"Yes",None).event("EndDialog","ErrorYes") - error.pushbutton("A",0,72,81,21,3,"Abort",None).event("EndDialog","ErrorAbort") - error.pushbutton("C",42,72,81,21,3,"Cancel",None).event("EndDialog","ErrorCancel") - error.pushbutton("I",81,72,81,21,3,"Ignore",None).event("EndDialog","ErrorIgnore") - error.pushbutton("O",159,72,81,21,3,"Ok",None).event("EndDialog","ErrorOk") - error.pushbutton("R",198,72,81,21,3,"Retry",None).event("EndDialog","ErrorRetry") - - ##################################################################### - # Global "Query Cancel" dialog - cancel = Dialog(db, "CancelDlg", 50, 10, 260, 85, 3, title, - "No", "No", "No") - cancel.text("Text", 48, 15, 194, 30, 3, - "Are you sure you want to cancel [ProductName] installation?") - #cancel.control("Icon", "Icon", 15, 15, 24, 24, 5242881, None, - # "py.ico", None, None) - c=cancel.pushbutton("Yes", 72, 57, 56, 17, 3, "Yes", "No") - c.event("EndDialog", "Exit") - - c=cancel.pushbutton("No", 132, 57, 56, 17, 3, "No", "Yes") - c.event("EndDialog", "Return") - - ##################################################################### - # Global "Wait for costing" dialog - costing = Dialog(db, "WaitForCostingDlg", 50, 10, 260, 85, modal, title, - "Return", "Return", "Return") - costing.text("Text", 48, 15, 194, 30, 3, - "Please wait while the installer finishes determining your disk space requirements.") - c = costing.pushbutton("Return", 102, 57, 56, 17, 3, "Return", None) - c.event("EndDialog", "Exit") - - ##################################################################### - # Preparation dialog: no user input except cancellation - prep = PyDialog(db, "PrepareDlg", x, y, w, h, modeless, title, - "Cancel", "Cancel", "Cancel") - prep.text("Description", 15, 70, 320, 40, 0x30003, - "Please wait while the Installer prepares to guide you through the installation.") - prep.title("Welcome to the [ProductName] Installer") - c=prep.text("ActionText", 15, 110, 320, 20, 0x30003, "Pondering...") - c.mapping("ActionText", "Text") - c=prep.text("ActionData", 15, 135, 320, 30, 0x30003, None) - c.mapping("ActionData", "Text") - prep.back("Back", None, active=0) - prep.next("Next", None, active=0) - c=prep.cancel("Cancel", None) - c.event("SpawnDialog", "CancelDlg") - - ##################################################################### - # Feature (Python directory) selection - seldlg = PyDialog(db, "SelectFeaturesDlg", x, y, w, h, modal, title, - "Next", "Next", "Cancel") - seldlg.title("Select Python Installations") - - seldlg.text("Hint", 15, 30, 300, 20, 3, - "Select the Python locations where %s should be installed." - % self.distribution.get_fullname()) - - seldlg.back("< Back", None, active=0) - c = seldlg.next("Next >", "Cancel") - order = 1 - c.event("[TARGETDIR]", "[SourceDir]", ordering=order) - for version in self.versions + [self.other_version]: - order += 1 - c.event("[TARGETDIR]", "[TARGETDIR%s]" % version, - "FEATURE_SELECTED AND &Python%s=3" % version, - ordering=order) - c.event("SpawnWaitDialog", "WaitForCostingDlg", ordering=order + 1) - c.event("EndDialog", "Return", ordering=order + 2) - c = seldlg.cancel("Cancel", "Features") - c.event("SpawnDialog", "CancelDlg") - - c = seldlg.control("Features", "SelectionTree", 15, 60, 300, 120, 3, - "FEATURE", None, "PathEdit", None) - c.event("[FEATURE_SELECTED]", "1") - ver = self.other_version - install_other_cond = "FEATURE_SELECTED AND &Python%s=3" % ver - dont_install_other_cond = "FEATURE_SELECTED AND &Python%s<>3" % ver - - c = seldlg.text("Other", 15, 200, 300, 15, 3, - "Provide an alternate Python location") - c.condition("Enable", install_other_cond) - c.condition("Show", install_other_cond) - c.condition("Disable", dont_install_other_cond) - c.condition("Hide", dont_install_other_cond) - - c = seldlg.control("PathEdit", "PathEdit", 15, 215, 300, 16, 1, - "TARGETDIR" + ver, None, "Next", None) - c.condition("Enable", install_other_cond) - c.condition("Show", install_other_cond) - c.condition("Disable", dont_install_other_cond) - c.condition("Hide", dont_install_other_cond) - - ##################################################################### - # Disk cost - cost = PyDialog(db, "DiskCostDlg", x, y, w, h, modal, title, - "OK", "OK", "OK", bitmap=False) - cost.text("Title", 15, 6, 200, 15, 0x30003, - r"{\DlgFontBold8}Disk Space Requirements") - cost.text("Description", 20, 20, 280, 20, 0x30003, - "The disk space required for the installation of the selected features.") - cost.text("Text", 20, 53, 330, 60, 3, - "The highlighted volumes (if any) do not have enough disk space " - "available for the currently selected features. You can either " - "remove some files from the highlighted volumes, or choose to " - "install less features onto local drive(s), or select different " - "destination drive(s).") - cost.control("VolumeList", "VolumeCostList", 20, 100, 330, 150, 393223, - None, "{120}{70}{70}{70}{70}", None, None) - cost.xbutton("OK", "Ok", None, 0.5).event("EndDialog", "Return") - - ##################################################################### - # WhichUsers Dialog. Only available on NT, and for privileged users. - # This must be run before FindRelatedProducts, because that will - # take into account whether the previous installation was per-user - # or per-machine. We currently don't support going back to this - # dialog after "Next" was selected; to support this, we would need to - # find how to reset the ALLUSERS property, and how to re-run - # FindRelatedProducts. - # On Windows9x, the ALLUSERS property is ignored on the command line - # and in the Property table, but installer fails according to the documentation - # if a dialog attempts to set ALLUSERS. - whichusers = PyDialog(db, "WhichUsersDlg", x, y, w, h, modal, title, - "AdminInstall", "Next", "Cancel") - whichusers.title("Select whether to install [ProductName] for all users of this computer.") - # A radio group with two options: allusers, justme - g = whichusers.radiogroup("AdminInstall", 15, 60, 260, 50, 3, - "WhichUsers", "", "Next") - g.add("ALL", 0, 5, 150, 20, "Install for all users") - g.add("JUSTME", 0, 25, 150, 20, "Install just for me") - - whichusers.back("Back", None, active=0) - - c = whichusers.next("Next >", "Cancel") - c.event("[ALLUSERS]", "1", 'WhichUsers="ALL"', 1) - c.event("EndDialog", "Return", ordering = 2) - - c = whichusers.cancel("Cancel", "AdminInstall") - c.event("SpawnDialog", "CancelDlg") - - ##################################################################### - # Installation Progress dialog (modeless) - progress = PyDialog(db, "ProgressDlg", x, y, w, h, modeless, title, - "Cancel", "Cancel", "Cancel", bitmap=False) - progress.text("Title", 20, 15, 200, 15, 0x30003, - r"{\DlgFontBold8}[Progress1] [ProductName]") - progress.text("Text", 35, 65, 300, 30, 3, - "Please wait while the Installer [Progress2] [ProductName]. " - "This may take several minutes.") - progress.text("StatusLabel", 35, 100, 35, 20, 3, "Status:") - - c=progress.text("ActionText", 70, 100, w-70, 20, 3, "Pondering...") - c.mapping("ActionText", "Text") - - #c=progress.text("ActionData", 35, 140, 300, 20, 3, None) - #c.mapping("ActionData", "Text") - - c=progress.control("ProgressBar", "ProgressBar", 35, 120, 300, 10, 65537, - None, "Progress done", None, None) - c.mapping("SetProgress", "Progress") - - progress.back("< Back", "Next", active=False) - progress.next("Next >", "Cancel", active=False) - progress.cancel("Cancel", "Back").event("SpawnDialog", "CancelDlg") - - ################################################################### - # Maintenance type: repair/uninstall - maint = PyDialog(db, "MaintenanceTypeDlg", x, y, w, h, modal, title, - "Next", "Next", "Cancel") - maint.title("Welcome to the [ProductName] Setup Wizard") - maint.text("BodyText", 15, 63, 330, 42, 3, - "Select whether you want to repair or remove [ProductName].") - g=maint.radiogroup("RepairRadioGroup", 15, 108, 330, 60, 3, - "MaintenanceForm_Action", "", "Next") - #g.add("Change", 0, 0, 200, 17, "&Change [ProductName]") - g.add("Repair", 0, 18, 200, 17, "&Repair [ProductName]") - g.add("Remove", 0, 36, 200, 17, "Re&move [ProductName]") - - maint.back("< Back", None, active=False) - c=maint.next("Finish", "Cancel") - # Change installation: Change progress dialog to "Change", then ask - # for feature selection - #c.event("[Progress1]", "Change", 'MaintenanceForm_Action="Change"', 1) - #c.event("[Progress2]", "changes", 'MaintenanceForm_Action="Change"', 2) - - # Reinstall: Change progress dialog to "Repair", then invoke reinstall - # Also set list of reinstalled features to "ALL" - c.event("[REINSTALL]", "ALL", 'MaintenanceForm_Action="Repair"', 5) - c.event("[Progress1]", "Repairing", 'MaintenanceForm_Action="Repair"', 6) - c.event("[Progress2]", "repairs", 'MaintenanceForm_Action="Repair"', 7) - c.event("Reinstall", "ALL", 'MaintenanceForm_Action="Repair"', 8) - - # Uninstall: Change progress to "Remove", then invoke uninstall - # Also set list of removed features to "ALL" - c.event("[REMOVE]", "ALL", 'MaintenanceForm_Action="Remove"', 11) - c.event("[Progress1]", "Removing", 'MaintenanceForm_Action="Remove"', 12) - c.event("[Progress2]", "removes", 'MaintenanceForm_Action="Remove"', 13) - c.event("Remove", "ALL", 'MaintenanceForm_Action="Remove"', 14) - - # Close dialog when maintenance action scheduled - c.event("EndDialog", "Return", 'MaintenanceForm_Action<>"Change"', 20) - #c.event("NewDialog", "SelectFeaturesDlg", 'MaintenanceForm_Action="Change"', 21) - - maint.cancel("Cancel", "RepairRadioGroup").event("SpawnDialog", "CancelDlg") - - def get_installer_filename(self, fullname): - # Factored out to allow overriding in subclasses - if self.target_version: - base_name = "%s.%s-py%s.msi" % (fullname, self.plat_name, - self.target_version) - else: - base_name = "%s.%s.msi" % (fullname, self.plat_name) - installer_name = os.path.join(self.dist_dir, base_name) - return installer_name diff --git a/Lib/distutils/command/check.py b/Lib/distutils/command/check.py index ada25006467..73a30f3afd8 100644 --- a/Lib/distutils/command/check.py +++ b/Lib/distutils/command/check.py @@ -83,7 +83,7 @@ class check(Command): name, version, URL Recommended fields: - (author and author_email) or (maintainer and maintainer_email)) + (author and author_email) or (maintainer and maintainer_email) Warns if any are missing. """ diff --git a/Lib/distutils/command/install.py b/Lib/distutils/command/install.py index 26696cfb9dc..01d5331a630 100644 --- a/Lib/distutils/command/install.py +++ b/Lib/distutils/command/install.py @@ -31,7 +31,7 @@ SCHEME_KEYS = ('purelib', 'platlib', 'headers', 'scripts', 'data') # while making the sysconfig module the single point of truth. # This makes it easier for OS distributions where they need to # alter locations for packages installations in a single place. -# Note that this module is depracated (PEP 632); all consumers +# Note that this module is deprecated (PEP 632); all consumers # of this information should switch to using sysconfig directly. INSTALL_SCHEMES = {"unix_prefix": {}, "unix_home": {}, "nt": {}} @@ -43,7 +43,7 @@ for key in SCHEME_KEYS: sys_key = key sys_scheme = sysconfig._INSTALL_SCHEMES[sys_scheme_name] if key == "headers" and key not in sys_scheme: - # On POSIX-y platofrms, Python will: + # On POSIX-y platforms, Python will: # - Build from .h files in 'headers' (only there when # building CPython) # - Install .h files to 'include' diff --git a/Lib/distutils/msvc9compiler.py b/Lib/distutils/msvc9compiler.py index 6934e964abd..a7976fbe3ed 100644 --- a/Lib/distutils/msvc9compiler.py +++ b/Lib/distutils/msvc9compiler.py @@ -673,7 +673,7 @@ class MSVCCompiler(CCompiler) : # If a manifest should be embedded, return a tuple of # (manifest_filename, resource_id). Returns None if no manifest # should be embedded. See http://bugs.python.org/issue7833 for why - # we want to avoid any manifest for extension modules if we can) + # we want to avoid any manifest for extension modules if we can. for arg in ld_args: if arg.startswith("/MANIFESTFILE:"): temp_manifest = arg.split(":", 1)[1] diff --git a/Lib/distutils/tests/test_archive_util.py b/Lib/distutils/tests/test_archive_util.py index edcec2513e0..8aec84078ed 100644 --- a/Lib/distutils/tests/test_archive_util.py +++ b/Lib/distutils/tests/test_archive_util.py @@ -390,7 +390,7 @@ class ArchiveUtilTestCase(support.TempdirManager, archive.close() def test_suite(): - return unittest.makeSuite(ArchiveUtilTestCase) + return unittest.TestLoader().loadTestsFromTestCase(ArchiveUtilTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_bdist.py b/Lib/distutils/tests/test_bdist.py index 55fa3930dd9..241fc9ad75f 100644 --- a/Lib/distutils/tests/test_bdist.py +++ b/Lib/distutils/tests/test_bdist.py @@ -18,13 +18,12 @@ class BuildTestCase(support.TempdirManager, # we can set the format dist = self.create_dist()[1] cmd = bdist(dist) - cmd.formats = ['msi'] + cmd.formats = ['tar'] cmd.ensure_finalized() - self.assertEqual(cmd.formats, ['msi']) + self.assertEqual(cmd.formats, ['tar']) # what formats does bdist offer? - formats = ['bztar', 'gztar', 'msi', 'rpm', 'tar', - 'xztar', 'zip', 'ztar'] + formats = ['bztar', 'gztar', 'rpm', 'tar', 'xztar', 'zip', 'ztar'] found = sorted(cmd.format_command) self.assertEqual(found, formats) @@ -36,11 +35,7 @@ class BuildTestCase(support.TempdirManager, cmd.ensure_finalized() dist.command_obj['bdist'] = cmd - names = ['bdist_dumb'] # bdist_rpm does not support --skip-build - if os.name == 'nt': - names.append('bdist_msi') - - for name in names: + for name in ['bdist_dumb']: # bdist_rpm does not support --skip-build subcmd = cmd.get_finalized_command(name) if getattr(subcmd, '_unsupported', False): # command is not supported on this build @@ -50,7 +45,8 @@ class BuildTestCase(support.TempdirManager, def test_suite(): - return unittest.makeSuite(BuildTestCase) + return unittest.TestLoader().loadTestsFromTestCase(BuildTestCase) + if __name__ == '__main__': run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_bdist_dumb.py b/Lib/distutils/tests/test_bdist_dumb.py index 01a233bce37..bb860c8ac70 100644 --- a/Lib/distutils/tests/test_bdist_dumb.py +++ b/Lib/distutils/tests/test_bdist_dumb.py @@ -91,7 +91,7 @@ class BuildDumbTestCase(support.TempdirManager, self.assertEqual(contents, sorted(wanted)) def test_suite(): - return unittest.makeSuite(BuildDumbTestCase) + return unittest.TestLoader().loadTestsFromTestCase(BuildDumbTestCase) if __name__ == '__main__': run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_bdist_msi.py b/Lib/distutils/tests/test_bdist_msi.py deleted file mode 100644 index a61266a14f9..00000000000 --- a/Lib/distutils/tests/test_bdist_msi.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Tests for distutils.command.bdist_msi.""" -import sys -import unittest -from test.support import run_unittest -from test.support.warnings_helper import check_warnings -from distutils.tests import support - - -@unittest.skipUnless(sys.platform == 'win32', 'these tests require Windows') -class BDistMSITestCase(support.TempdirManager, - support.LoggingSilencer, - unittest.TestCase): - - def test_minimal(self): - # minimal test XXX need more tests - from distutils.command.bdist_msi import bdist_msi - project_dir, dist = self.create_dist() - with check_warnings(("", DeprecationWarning)): - cmd = bdist_msi(dist) - cmd.ensure_finalized() - - -def test_suite(): - return unittest.makeSuite(BDistMSITestCase) - -if __name__ == '__main__': - run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_bdist_rpm.py b/Lib/distutils/tests/test_bdist_rpm.py index 6453a02b88f..f1eb9ba4493 100644 --- a/Lib/distutils/tests/test_bdist_rpm.py +++ b/Lib/distutils/tests/test_bdist_rpm.py @@ -44,7 +44,7 @@ class BuildRpmTestCase(support.TempdirManager, # spurious sdtout/stderr output under Mac OS X @unittest.skipUnless(sys.platform.startswith('linux'), 'spurious sdtout/stderr output under Mac OS X') - @requires_zlib + @requires_zlib() @unittest.skipIf(find_executable('rpm') is None, 'the rpm command is not found') @unittest.skipIf(find_executable('rpmbuild') is None, @@ -87,7 +87,7 @@ class BuildRpmTestCase(support.TempdirManager, # spurious sdtout/stderr output under Mac OS X @unittest.skipUnless(sys.platform.startswith('linux'), 'spurious sdtout/stderr output under Mac OS X') - @requires_zlib + @requires_zlib() # http://bugs.python.org/issue1533164 @unittest.skipIf(find_executable('rpm') is None, 'the rpm command is not found') @@ -129,7 +129,7 @@ class BuildRpmTestCase(support.TempdirManager, os.remove(os.path.join(pkg_dir, 'dist', 'foo-0.1-1.noarch.rpm')) def test_suite(): - return unittest.makeSuite(BuildRpmTestCase) + return unittest.TestLoader().loadTestsFromTestCase(BuildRpmTestCase) if __name__ == '__main__': run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_build.py b/Lib/distutils/tests/test_build.py index b020a5ba356..83a9e4f4dd2 100644 --- a/Lib/distutils/tests/test_build.py +++ b/Lib/distutils/tests/test_build.py @@ -50,7 +50,7 @@ class BuildTestCase(support.TempdirManager, self.assertEqual(cmd.executable, os.path.normpath(sys.executable)) def test_suite(): - return unittest.makeSuite(BuildTestCase) + return unittest.TestLoader().loadTestsFromTestCase(BuildTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_build_clib.py b/Lib/distutils/tests/test_build_clib.py index 19e012a58ce..601a1b10fa1 100644 --- a/Lib/distutils/tests/test_build_clib.py +++ b/Lib/distutils/tests/test_build_clib.py @@ -138,7 +138,7 @@ class BuildCLibTestCase(support.TempdirManager, self.assertIn('libfoo.a', os.listdir(build_temp)) def test_suite(): - return unittest.makeSuite(BuildCLibTestCase) + return unittest.TestLoader().loadTestsFromTestCase(BuildCLibTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py index 8e7364d2a2c..3ee567d0455 100644 --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -545,8 +545,8 @@ class ParallelBuildExtTestCase(BuildExtTestCase): def test_suite(): suite = unittest.TestSuite() - suite.addTest(unittest.makeSuite(BuildExtTestCase)) - suite.addTest(unittest.makeSuite(ParallelBuildExtTestCase)) + suite.addTest(unittest.TestLoader().loadTestsFromTestCase(BuildExtTestCase)) + suite.addTest(unittest.TestLoader().loadTestsFromTestCase(ParallelBuildExtTestCase)) return suite if __name__ == '__main__': diff --git a/Lib/distutils/tests/test_build_py.py b/Lib/distutils/tests/test_build_py.py index 0712e92c6ab..a590a485a2b 100644 --- a/Lib/distutils/tests/test_build_py.py +++ b/Lib/distutils/tests/test_build_py.py @@ -173,7 +173,7 @@ class BuildPyTestCase(support.TempdirManager, def test_suite(): - return unittest.makeSuite(BuildPyTestCase) + return unittest.TestLoader().loadTestsFromTestCase(BuildPyTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_build_scripts.py b/Lib/distutils/tests/test_build_scripts.py index 954fc763987..f299e51ef79 100644 --- a/Lib/distutils/tests/test_build_scripts.py +++ b/Lib/distutils/tests/test_build_scripts.py @@ -106,7 +106,7 @@ class BuildScriptsTestCase(support.TempdirManager, self.assertIn(name, built) def test_suite(): - return unittest.makeSuite(BuildScriptsTestCase) + return unittest.TestLoader().loadTestsFromTestCase(BuildScriptsTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_check.py b/Lib/distutils/tests/test_check.py index e534aca1d47..91bcdceb43b 100644 --- a/Lib/distutils/tests/test_check.py +++ b/Lib/distutils/tests/test_check.py @@ -157,7 +157,7 @@ class CheckTestCase(support.LoggingSilencer, 'restructuredtext': 1}) def test_suite(): - return unittest.makeSuite(CheckTestCase) + return unittest.TestLoader().loadTestsFromTestCase(CheckTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_clean.py b/Lib/distutils/tests/test_clean.py index c605afd8601..92367499cef 100644 --- a/Lib/distutils/tests/test_clean.py +++ b/Lib/distutils/tests/test_clean.py @@ -43,7 +43,7 @@ class cleanTestCase(support.TempdirManager, cmd.run() def test_suite(): - return unittest.makeSuite(cleanTestCase) + return unittest.TestLoader().loadTestsFromTestCase(cleanTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_cmd.py b/Lib/distutils/tests/test_cmd.py index cf5197c30fc..2319214a9e3 100644 --- a/Lib/distutils/tests/test_cmd.py +++ b/Lib/distutils/tests/test_cmd.py @@ -120,7 +120,7 @@ class CommandTestCase(unittest.TestCase): debug.DEBUG = False def test_suite(): - return unittest.makeSuite(CommandTestCase) + return unittest.TestLoader().loadTestsFromTestCase(CommandTestCase) if __name__ == '__main__': run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_config.py b/Lib/distutils/tests/test_config.py index 344084afb77..8ab70efb161 100644 --- a/Lib/distutils/tests/test_config.py +++ b/Lib/distutils/tests/test_config.py @@ -135,7 +135,7 @@ class PyPIRCCommandTestCase(BasePyPIRCCommandTestCase): def test_suite(): - return unittest.makeSuite(PyPIRCCommandTestCase) + return unittest.TestLoader().loadTestsFromTestCase(PyPIRCCommandTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_config_cmd.py b/Lib/distutils/tests/test_config_cmd.py index 0127ba71fc4..072f9ebe714 100644 --- a/Lib/distutils/tests/test_config_cmd.py +++ b/Lib/distutils/tests/test_config_cmd.py @@ -94,7 +94,7 @@ class ConfigTestCase(support.LoggingSilencer, self.assertFalse(os.path.exists(f)) def test_suite(): - return unittest.makeSuite(ConfigTestCase) + return unittest.TestLoader().loadTestsFromTestCase(ConfigTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_core.py b/Lib/distutils/tests/test_core.py index 4e6694a3d1c..700a22da045 100644 --- a/Lib/distutils/tests/test_core.py +++ b/Lib/distutils/tests/test_core.py @@ -134,7 +134,7 @@ class CoreTestCase(support.EnvironGuard, unittest.TestCase): self.assertEqual(stdout.readlines()[0], wanted) def test_suite(): - return unittest.makeSuite(CoreTestCase) + return unittest.TestLoader().loadTestsFromTestCase(CoreTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_cygwinccompiler.py b/Lib/distutils/tests/test_cygwinccompiler.py index 9dc869de4c8..0912ffd15c8 100644 --- a/Lib/distutils/tests/test_cygwinccompiler.py +++ b/Lib/distutils/tests/test_cygwinccompiler.py @@ -148,7 +148,7 @@ class CygwinCCompilerTestCase(support.TempdirManager, self.assertRaises(ValueError, get_msvcr) def test_suite(): - return unittest.makeSuite(CygwinCCompilerTestCase) + return unittest.TestLoader().loadTestsFromTestCase(CygwinCCompilerTestCase) if __name__ == '__main__': run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_dep_util.py b/Lib/distutils/tests/test_dep_util.py index c6fae39cfb1..0d52740a9ed 100644 --- a/Lib/distutils/tests/test_dep_util.py +++ b/Lib/distutils/tests/test_dep_util.py @@ -74,7 +74,7 @@ class DepUtilTestCase(support.TempdirManager, unittest.TestCase): def test_suite(): - return unittest.makeSuite(DepUtilTestCase) + return unittest.TestLoader().loadTestsFromTestCase(DepUtilTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_dir_util.py b/Lib/distutils/tests/test_dir_util.py index d436cf83191..1b1f3bbb02e 100644 --- a/Lib/distutils/tests/test_dir_util.py +++ b/Lib/distutils/tests/test_dir_util.py @@ -133,7 +133,7 @@ class DirUtilTestCase(support.TempdirManager, unittest.TestCase): def test_suite(): - return unittest.makeSuite(DirUtilTestCase) + return unittest.TestLoader().loadTestsFromTestCase(DirUtilTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_dist.py b/Lib/distutils/tests/test_dist.py index f8a9e86b16f..2ef70d987f3 100644 --- a/Lib/distutils/tests/test_dist.py +++ b/Lib/distutils/tests/test_dist.py @@ -521,8 +521,8 @@ class MetadataTestCase(support.TempdirManager, support.EnvironGuard, def test_suite(): suite = unittest.TestSuite() - suite.addTest(unittest.makeSuite(DistributionTestCase)) - suite.addTest(unittest.makeSuite(MetadataTestCase)) + suite.addTest(unittest.TestLoader().loadTestsFromTestCase(DistributionTestCase)) + suite.addTest(unittest.TestLoader().loadTestsFromTestCase(MetadataTestCase)) return suite if __name__ == "__main__": diff --git a/Lib/distutils/tests/test_extension.py b/Lib/distutils/tests/test_extension.py index 81fad02dbec..2b08930eafb 100644 --- a/Lib/distutils/tests/test_extension.py +++ b/Lib/distutils/tests/test_extension.py @@ -64,7 +64,7 @@ class ExtensionTestCase(unittest.TestCase): "Unknown Extension options: 'chic'") def test_suite(): - return unittest.makeSuite(ExtensionTestCase) + return unittest.TestLoader().loadTestsFromTestCase(ExtensionTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_file_util.py b/Lib/distutils/tests/test_file_util.py index c7783b858d5..a614219a10b 100644 --- a/Lib/distutils/tests/test_file_util.py +++ b/Lib/distutils/tests/test_file_util.py @@ -118,7 +118,7 @@ class FileUtilTestCase(support.TempdirManager, unittest.TestCase): def test_suite(): - return unittest.makeSuite(FileUtilTestCase) + return unittest.TestLoader().loadTestsFromTestCase(FileUtilTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_filelist.py b/Lib/distutils/tests/test_filelist.py index cee97d439e6..98c97e49f80 100644 --- a/Lib/distutils/tests/test_filelist.py +++ b/Lib/distutils/tests/test_filelist.py @@ -331,8 +331,8 @@ class FindAllTestCase(unittest.TestCase): def test_suite(): return unittest.TestSuite([ - unittest.makeSuite(FileListTestCase), - unittest.makeSuite(FindAllTestCase), + unittest.TestLoader().loadTestsFromTestCase(FileListTestCase), + unittest.TestLoader().loadTestsFromTestCase(FindAllTestCase), ]) diff --git a/Lib/distutils/tests/test_install.py b/Lib/distutils/tests/test_install.py index 0632024b358..b2a3887f0bb 100644 --- a/Lib/distutils/tests/test_install.py +++ b/Lib/distutils/tests/test_install.py @@ -254,7 +254,7 @@ class InstallTestCase(support.TempdirManager, def test_suite(): - return unittest.makeSuite(InstallTestCase) + return unittest.TestLoader().loadTestsFromTestCase(InstallTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_install_data.py b/Lib/distutils/tests/test_install_data.py index 32ab296a320..6191d2fa6ee 100644 --- a/Lib/distutils/tests/test_install_data.py +++ b/Lib/distutils/tests/test_install_data.py @@ -69,7 +69,7 @@ class InstallDataTestCase(support.TempdirManager, self.assertTrue(os.path.exists(os.path.join(inst, rone))) def test_suite(): - return unittest.makeSuite(InstallDataTestCase) + return unittest.TestLoader().loadTestsFromTestCase(InstallDataTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_install_headers.py b/Lib/distutils/tests/test_install_headers.py index 2217b321e6e..1aa4d09cdef 100644 --- a/Lib/distutils/tests/test_install_headers.py +++ b/Lib/distutils/tests/test_install_headers.py @@ -33,7 +33,7 @@ class InstallHeadersTestCase(support.TempdirManager, self.assertEqual(len(cmd.get_outputs()), 2) def test_suite(): - return unittest.makeSuite(InstallHeadersTestCase) + return unittest.TestLoader().loadTestsFromTestCase(InstallHeadersTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_install_lib.py b/Lib/distutils/tests/test_install_lib.py index fda6315bbc7..652653f2b2c 100644 --- a/Lib/distutils/tests/test_install_lib.py +++ b/Lib/distutils/tests/test_install_lib.py @@ -109,7 +109,7 @@ class InstallLibTestCase(support.TempdirManager, def test_suite(): - return unittest.makeSuite(InstallLibTestCase) + return unittest.TestLoader().loadTestsFromTestCase(InstallLibTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_install_scripts.py b/Lib/distutils/tests/test_install_scripts.py index 1f7b1038cb2..648db3b11da 100644 --- a/Lib/distutils/tests/test_install_scripts.py +++ b/Lib/distutils/tests/test_install_scripts.py @@ -76,7 +76,7 @@ class InstallScriptsTestCase(support.TempdirManager, def test_suite(): - return unittest.makeSuite(InstallScriptsTestCase) + return unittest.TestLoader().loadTestsFromTestCase(InstallScriptsTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_log.py b/Lib/distutils/tests/test_log.py index 75cf900617b..ec2ae028de8 100644 --- a/Lib/distutils/tests/test_log.py +++ b/Lib/distutils/tests/test_log.py @@ -40,7 +40,7 @@ class TestLog(unittest.TestCase): 'Fαtal\t\\xc8rr\\u014dr') def test_suite(): - return unittest.makeSuite(TestLog) + return unittest.TestLoader().loadTestsFromTestCase(TestLog) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_msvc9compiler.py b/Lib/distutils/tests/test_msvc9compiler.py index 77a07ef39dd..6235405e312 100644 --- a/Lib/distutils/tests/test_msvc9compiler.py +++ b/Lib/distutils/tests/test_msvc9compiler.py @@ -178,7 +178,7 @@ class msvc9compilerTestCase(support.TempdirManager, def test_suite(): - return unittest.makeSuite(msvc9compilerTestCase) + return unittest.TestLoader().loadTestsFromTestCase(msvc9compilerTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_msvccompiler.py b/Lib/distutils/tests/test_msvccompiler.py index b518d6a78b3..dd67c3eb6d5 100644 --- a/Lib/distutils/tests/test_msvccompiler.py +++ b/Lib/distutils/tests/test_msvccompiler.py @@ -75,7 +75,7 @@ class msvccompilerTestCase(support.TempdirManager, raise unittest.SkipTest("VS 2015 is not installed") def test_suite(): - return unittest.makeSuite(msvccompilerTestCase) + return unittest.TestLoader().loadTestsFromTestCase(msvccompilerTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_register.py b/Lib/distutils/tests/test_register.py index bba48633c9c..7805c6d3c9f 100644 --- a/Lib/distutils/tests/test_register.py +++ b/Lib/distutils/tests/test_register.py @@ -318,7 +318,7 @@ class RegisterTestCase(BasePyPIRCCommandTestCase): def test_suite(): - return unittest.makeSuite(RegisterTestCase) + return unittest.TestLoader().loadTestsFromTestCase(RegisterTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_sdist.py b/Lib/distutils/tests/test_sdist.py index 752e9db5ba5..46b3a13e470 100644 --- a/Lib/distutils/tests/test_sdist.py +++ b/Lib/distutils/tests/test_sdist.py @@ -487,7 +487,7 @@ class SDistTestCase(BasePyPIRCCommandTestCase): archive.close() def test_suite(): - return unittest.makeSuite(SDistTestCase) + return unittest.TestLoader().loadTestsFromTestCase(SDistTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_spawn.py b/Lib/distutils/tests/test_spawn.py index 4ec767b120e..631d6455d45 100644 --- a/Lib/distutils/tests/test_spawn.py +++ b/Lib/distutils/tests/test_spawn.py @@ -131,7 +131,7 @@ class SpawnTestCase(support.TempdirManager, def test_suite(): - return unittest.makeSuite(SpawnTestCase) + return unittest.TestLoader().loadTestsFromTestCase(SpawnTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_sysconfig.py b/Lib/distutils/tests/test_sysconfig.py index 59676b0e0b0..3697206229d 100644 --- a/Lib/distutils/tests/test_sysconfig.py +++ b/Lib/distutils/tests/test_sysconfig.py @@ -269,7 +269,7 @@ class SysconfigTestCase(support.EnvironGuard, unittest.TestCase): def test_suite(): suite = unittest.TestSuite() - suite.addTest(unittest.makeSuite(SysconfigTestCase)) + suite.addTest(unittest.TestLoader().loadTestsFromTestCase(SysconfigTestCase)) return suite diff --git a/Lib/distutils/tests/test_text_file.py b/Lib/distutils/tests/test_text_file.py index 7e76240a9a3..ebac3d52f90 100644 --- a/Lib/distutils/tests/test_text_file.py +++ b/Lib/distutils/tests/test_text_file.py @@ -101,7 +101,7 @@ class TextFileTestCase(support.TempdirManager, unittest.TestCase): in_file.close() def test_suite(): - return unittest.makeSuite(TextFileTestCase) + return unittest.TestLoader().loadTestsFromTestCase(TextFileTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_unixccompiler.py b/Lib/distutils/tests/test_unixccompiler.py index 24725ead119..a3484d4f94c 100644 --- a/Lib/distutils/tests/test_unixccompiler.py +++ b/Lib/distutils/tests/test_unixccompiler.py @@ -139,7 +139,7 @@ class UnixCCompilerTestCase(unittest.TestCase): def test_suite(): - return unittest.makeSuite(UnixCCompilerTestCase) + return unittest.TestLoader().loadTestsFromTestCase(UnixCCompilerTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_upload.py b/Lib/distutils/tests/test_upload.py index 74f0bc0a671..d6797414883 100644 --- a/Lib/distutils/tests/test_upload.py +++ b/Lib/distutils/tests/test_upload.py @@ -217,7 +217,7 @@ class uploadTestCase(BasePyPIRCCommandTestCase): def test_suite(): - return unittest.makeSuite(uploadTestCase) + return unittest.TestLoader().loadTestsFromTestCase(uploadTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_util.py b/Lib/distutils/tests/test_util.py index d4a01c6e91c..812d44ec9bb 100644 --- a/Lib/distutils/tests/test_util.py +++ b/Lib/distutils/tests/test_util.py @@ -304,7 +304,7 @@ class UtilTestCase(support.EnvironGuard, unittest.TestCase): def test_suite(): - return unittest.makeSuite(UtilTestCase) + return unittest.TestLoader().loadTestsFromTestCase(UtilTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/tests/test_version.py b/Lib/distutils/tests/test_version.py index 8671cd2fc5c..1563e0227b6 100644 --- a/Lib/distutils/tests/test_version.py +++ b/Lib/distutils/tests/test_version.py @@ -81,7 +81,7 @@ class VersionTestCase(unittest.TestCase): (v1, v2, res)) def test_suite(): - return unittest.makeSuite(VersionTestCase) + return unittest.TestLoader().loadTestsFromTestCase(VersionTestCase) if __name__ == "__main__": run_unittest(test_suite()) diff --git a/Lib/distutils/unixccompiler.py b/Lib/distutils/unixccompiler.py index f0792de74a1..d00c48981eb 100644 --- a/Lib/distutils/unixccompiler.py +++ b/Lib/distutils/unixccompiler.py @@ -215,7 +215,8 @@ class UnixCCompiler(CCompiler): return "-L" + dir def _is_gcc(self, compiler_name): - return "gcc" in compiler_name or "g++" in compiler_name + # clang uses same syntax for rpath as gcc + return any(name in compiler_name for name in ("gcc", "g++", "clang")) def runtime_library_dir_option(self, dir): # XXX Hackish, at the very least. See Python bug #445902: diff --git a/Lib/doctest.py b/Lib/doctest.py index ba898f65403..b27cbdfed46 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1034,10 +1034,8 @@ class DocTestFinder: if inspect.isclass(obj) and self._recurse: for valname, val in obj.__dict__.items(): # Special handling for staticmethod/classmethod. - if isinstance(val, staticmethod): - val = getattr(obj, valname) - if isinstance(val, classmethod): - val = getattr(obj, valname).__func__ + if isinstance(val, (staticmethod, classmethod)): + val = val.__func__ # Recurse to methods, properties, and nested classes. if ((inspect.isroutine(val) or inspect.isclass(val) or diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py index 4d27f87974b..ba5ad5a36d0 100644 --- a/Lib/email/_parseaddr.py +++ b/Lib/email/_parseaddr.py @@ -67,6 +67,8 @@ def _parsedate_tz(data): if not data: return None data = data.split() + if not data: # This happens for whitespace-only input. + return None # The FWS after the comma after the day-of-week is optional, so search and # adjust for this. if data[0].endswith(',') or data[0].lower() in _daynames: @@ -126,6 +128,8 @@ def _parsedate_tz(data): tss = 0 elif len(tm) == 3: [thh, tmm, tss] = tm + else: + return None else: return None try: diff --git a/Lib/email/contentmanager.py b/Lib/email/contentmanager.py index 3cf62dc8621..fcf278dbccb 100644 --- a/Lib/email/contentmanager.py +++ b/Lib/email/contentmanager.py @@ -144,7 +144,7 @@ def _encode_text(string, charset, cte, policy): linesep = policy.linesep.encode('ascii') def embedded_body(lines): return linesep.join(lines) + linesep def normal_body(lines): return b'\n'.join(lines) + b'\n' - if cte==None: + if cte is None: # Use heuristics to decide on the "best" encoding. if max((len(x) for x in lines), default=0) <= policy.max_line_length: try: diff --git a/Lib/email/errors.py b/Lib/email/errors.py index 1d258c34fc9..3ad00565549 100644 --- a/Lib/email/errors.py +++ b/Lib/email/errors.py @@ -110,4 +110,4 @@ class NonASCIILocalPartDefect(HeaderDefect): # parsing messages decoded from binary. class InvalidDateDefect(HeaderDefect): - """Header has unparseable or invalid date""" + """Header has unparsable or invalid date""" diff --git a/Lib/encodings/raw_unicode_escape.py b/Lib/encodings/raw_unicode_escape.py index 2b919b40d37..46c8e070dd1 100644 --- a/Lib/encodings/raw_unicode_escape.py +++ b/Lib/encodings/raw_unicode_escape.py @@ -21,15 +21,16 @@ class IncrementalEncoder(codecs.IncrementalEncoder): def encode(self, input, final=False): return codecs.raw_unicode_escape_encode(input, self.errors)[0] -class IncrementalDecoder(codecs.IncrementalDecoder): - def decode(self, input, final=False): - return codecs.raw_unicode_escape_decode(input, self.errors)[0] +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def _buffer_decode(self, input, errors, final): + return codecs.raw_unicode_escape_decode(input, errors, final) class StreamWriter(Codec,codecs.StreamWriter): pass class StreamReader(Codec,codecs.StreamReader): - pass + def decode(self, input, errors='strict'): + return codecs.raw_unicode_escape_decode(input, errors, False) ### encodings module API diff --git a/Lib/encodings/unicode_escape.py b/Lib/encodings/unicode_escape.py index 817f93265a4..9b1ce99b339 100644 --- a/Lib/encodings/unicode_escape.py +++ b/Lib/encodings/unicode_escape.py @@ -21,15 +21,16 @@ class IncrementalEncoder(codecs.IncrementalEncoder): def encode(self, input, final=False): return codecs.unicode_escape_encode(input, self.errors)[0] -class IncrementalDecoder(codecs.IncrementalDecoder): - def decode(self, input, final=False): - return codecs.unicode_escape_decode(input, self.errors)[0] +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def _buffer_decode(self, input, errors, final): + return codecs.unicode_escape_decode(input, errors, final) class StreamWriter(Codec,codecs.StreamWriter): pass class StreamReader(Codec,codecs.StreamReader): - pass + def decode(self, input, errors='strict'): + return codecs.unicode_escape_decode(input, errors, False) ### encodings module API diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 651f876e24b..94f1b6604cb 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,8 +10,8 @@ from importlib import resources __all__ = ["version", "bootstrap"] _PACKAGE_NAMES = ('setuptools', 'pip') -_SETUPTOOLS_VERSION = "57.4.0" -_PIP_VERSION = "21.2.3" +_SETUPTOOLS_VERSION = "58.1.0" +_PIP_VERSION = "21.2.4" _PROJECTS = [ ("setuptools", _SETUPTOOLS_VERSION, "py3"), ("pip", _PIP_VERSION, "py3"), @@ -41,7 +41,7 @@ def _find_packages(path): # comparison since this case should not happen. filenames = sorted(filenames) for filename in filenames: - # filename is like 'pip-20.2.3-py2.py3-none-any.whl' + # filename is like 'pip-21.2.4-py3-none-any.whl' if not filename.endswith(".whl"): continue for name in _PACKAGE_NAMES: @@ -51,7 +51,7 @@ def _find_packages(path): else: continue - # Extract '20.2.2' from 'pip-20.2.2-py2.py3-none-any.whl' + # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl' version = filename.removeprefix(prefix).partition('-')[0] wheel_path = os.path.join(path, filename) packages[name] = _Package(version, None, wheel_path) diff --git a/Lib/ensurepip/_bundled/pip-21.2.3-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-21.2.3-py3-none-any.whl deleted file mode 100644 index d417df63d9e..00000000000 Binary files a/Lib/ensurepip/_bundled/pip-21.2.3-py3-none-any.whl and /dev/null differ diff --git a/Lib/ensurepip/_bundled/pip-21.2.4-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-21.2.4-py3-none-any.whl new file mode 100644 index 00000000000..46d3012c59b Binary files /dev/null and b/Lib/ensurepip/_bundled/pip-21.2.4-py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-57.4.0-py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-58.1.0-py3-none-any.whl similarity index 80% rename from Lib/ensurepip/_bundled/setuptools-57.4.0-py3-none-any.whl rename to Lib/ensurepip/_bundled/setuptools-58.1.0-py3-none-any.whl index af8f8ba2100..18c8c22958f 100644 Binary files a/Lib/ensurepip/_bundled/setuptools-57.4.0-py3-none-any.whl and b/Lib/ensurepip/_bundled/setuptools-58.1.0-py3-none-any.whl differ diff --git a/Lib/enum.py b/Lib/enum.py index 84e3cc17bbc..8efc38c3d78 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -235,11 +235,18 @@ class _proto_member: enum_member._sort_order_ = len(enum_class._member_names_) # If another member with the same value was already defined, the # new member becomes an alias to the existing one. - for name, canonical_member in enum_class._member_map_.items(): - if canonical_member._value_ == enum_member._value_: - enum_member = canonical_member - break - else: + try: + try: + # try to do a fast lookup to avoid the quadratic loop + enum_member = enum_class._value2member_map_[value] + except TypeError: + for name, canonical_member in enum_class._member_map_.items(): + if canonical_member._value_ == value: + enum_member = canonical_member + break + else: + raise KeyError + except KeyError: # this could still be an alias if the value is multi-bit and the # class is a flag class if ( @@ -301,7 +308,7 @@ class _EnumDict(dict): """ def __init__(self): super().__init__() - self._member_names = [] + self._member_names = {} # use a dict to keep insertion order self._last_values = [] self._ignore = [] self._auto_called = False @@ -365,7 +372,7 @@ class _EnumDict(dict): ) self._auto_called = True value = value.value - self._member_names.append(key) + self._member_names[key] = None self._last_values.append(value) super().__setitem__(key, value) @@ -628,10 +635,60 @@ class EnumType(type): super().__delattr__(attr) def __dir__(self): - return ( - ['__class__', '__doc__', '__members__', '__module__'] - + self._member_names_ - ) + # Start off with the desired result for dir(Enum) + cls_dir = {'__class__', '__doc__', '__members__', '__module__'} + add_to_dir = cls_dir.add + mro = self.__mro__ + this_module = globals().values() + is_from_this_module = lambda cls: any(cls is thing for thing in this_module) + first_enum_base = next(cls for cls in mro if is_from_this_module(cls)) + enum_dict = Enum.__dict__ + sentinel = object() + # special-case __new__ + ignored = {'__new__', *filter(_is_sunder, enum_dict)} + add_to_ignored = ignored.add + + # We want these added to __dir__ + # if and only if they have been user-overridden + enum_dunders = set(filter(_is_dunder, enum_dict)) + + # special-case __new__ + if self.__new__ is not first_enum_base.__new__: + add_to_dir('__new__') + + for cls in mro: + # Ignore any classes defined in this module + if cls is object or is_from_this_module(cls): + continue + + cls_lookup = cls.__dict__ + + # If not an instance of EnumType, + # ensure all attributes excluded from that class's `dir()` are ignored here. + if not isinstance(cls, EnumType): + cls_lookup = set(cls_lookup).intersection(dir(cls)) + + for attr_name in cls_lookup: + # Already seen it? Carry on + if attr_name in cls_dir or attr_name in ignored: + continue + # Sunders defined in Enum.__dict__ are already in `ignored`, + # But sunders defined in a subclass won't be (we want all sunders excluded). + elif _is_sunder(attr_name): + add_to_ignored(attr_name) + # Not an "enum dunder"? Add it to dir() output. + elif attr_name not in enum_dunders: + add_to_dir(attr_name) + # Is an "enum dunder", and is defined by a class from enum.py? Ignore it. + elif getattr(self, attr_name) is getattr(first_enum_base, attr_name, sentinel): + add_to_ignored(attr_name) + # Is an "enum dunder", and is either user-defined or defined by a mixin class? + # Add it to dir() output. + else: + add_to_dir(attr_name) + + # sort the output before returning it, so that the result is deterministic. + return sorted(cls_dir) def __getattr__(cls, name): """ @@ -978,13 +1035,10 @@ class Enum(metaclass=EnumType): """ Returns all members and all public methods """ - added_behavior = [ - m - for cls in self.__class__.mro() - for m in cls.__dict__ - if m[0] != '_' and m not in self._member_map_ - ] + [m for m in self.__dict__ if m[0] != '_'] - return (['__class__', '__doc__', '__module__'] + added_behavior) + cls = type(self) + to_exclude = {'__members__', '__init__', '__new__', *cls._member_names_} + filtered_self_dict = (name for name in self.__dict__ if not name.startswith('_')) + return sorted({'name', 'value', *dir(cls), *filtered_self_dict} - to_exclude) def __format__(self, format_spec): """ @@ -1390,17 +1444,28 @@ def _power_of_two(value): return value == 2 ** _high_bit(value) def global_enum_repr(self): - return '%s.%s' % (self.__class__.__module__, self._name_) + """ + use module.enum_name instead of class.enum_name + + the module is the last module in case of a multi-module name + """ + module = self.__class__.__module__.split('.')[-1] + return '%s.%s' % (module, self._name_) def global_flag_repr(self): - module = self.__class__.__module__ + """ + use module.flag_name instead of class.flag_name + + the module is the last module in case of a multi-module name + """ + module = self.__class__.__module__.split('.')[-1] cls_name = self.__class__.__name__ if self._name_ is None: - return "%x" % (module, cls_name, self._value_) + return "%s.%s(0x%x)" % (module, cls_name, self._value_) if _is_single_bit(self): return '%s.%s' % (module, self._name_) if self._boundary_ is not FlagBoundary.KEEP: - return module + module.join(self.name.split('|')) + return '|'.join(['%s.%s' % (module, name) for name in self.name.split('|')]) else: name = [] for n in self._name_.split('|'): diff --git a/Lib/fileinput.py b/Lib/fileinput.py index 35347185da0..9f41c18510d 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -217,15 +217,10 @@ class FileInput: EncodingWarning, 2) # restrict mode argument to reading modes - if mode not in ('r', 'rU', 'U', 'rb'): - raise ValueError("FileInput opening mode must be one of " - "'r', 'rU', 'U' and 'rb'") - if 'U' in mode: - import warnings - warnings.warn("'U' mode is deprecated", - DeprecationWarning, 2) + if mode not in ('r', 'rb'): + raise ValueError("FileInput opening mode must be 'r' or 'rb'") self._mode = mode - self._write_mode = mode.replace('r', 'w') if 'U' not in mode else 'w' + self._write_mode = mode.replace('r', 'w') if openhook: if inplace: raise ValueError("FileInput cannot use an opening hook in inplace mode") @@ -262,21 +257,6 @@ class FileInput: self.nextfile() # repeat with next file - def __getitem__(self, i): - import warnings - warnings.warn( - "Support for indexing FileInput objects is deprecated. " - "Use iterator protocol instead.", - DeprecationWarning, - stacklevel=2 - ) - if i != self.lineno(): - raise RuntimeError("accessing lines out of order") - try: - return self.__next__() - except StopIteration: - raise IndexError("end of input reached") - def nextfile(self): savestdout = self._savestdout self._savestdout = None diff --git a/Lib/fractions.py b/Lib/fractions.py index 180cd94c287..f9ac882ec00 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -594,8 +594,15 @@ class Fraction(numbers.Rational): """abs(a)""" return Fraction(abs(a._numerator), a._denominator, _normalize=False) + def __int__(a, _index=operator.index): + """int(a)""" + if a._numerator < 0: + return _index(-(-a._numerator // a._denominator)) + else: + return _index(a._numerator // a._denominator) + def __trunc__(a): - """trunc(a)""" + """math.trunc(a)""" if a._numerator < 0: return -(-a._numerator // a._denominator) else: diff --git a/Lib/functools.py b/Lib/functools.py index 77ec852805c..ccac6f89996 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -837,6 +837,14 @@ def singledispatch(func): dispatch_cache[cls] = impl return impl + def _is_union_type(cls): + from typing import get_origin, Union + return get_origin(cls) in {Union, types.UnionType} + + def _is_valid_union_type(cls): + from typing import get_args + return _is_union_type(cls) and all(isinstance(arg, type) for arg in get_args(cls)) + def register(cls, func=None): """generic_func.register(cls, func) -> func @@ -845,7 +853,7 @@ def singledispatch(func): """ nonlocal cache_token if func is None: - if isinstance(cls, type): + if isinstance(cls, type) or _is_valid_union_type(cls): return lambda f: register(cls, f) ann = getattr(cls, '__annotations__', {}) if not ann: @@ -859,12 +867,25 @@ def singledispatch(func): # only import typing if annotation parsing is necessary from typing import get_type_hints argname, cls = next(iter(get_type_hints(func).items())) - if not isinstance(cls, type): - raise TypeError( - f"Invalid annotation for {argname!r}. " - f"{cls!r} is not a class." - ) - registry[cls] = func + if not isinstance(cls, type) and not _is_valid_union_type(cls): + if _is_union_type(cls): + raise TypeError( + f"Invalid annotation for {argname!r}. " + f"{cls!r} not all arguments are classes." + ) + else: + raise TypeError( + f"Invalid annotation for {argname!r}. " + f"{cls!r} is not a class." + ) + + if _is_union_type(cls): + from typing import get_args + + for arg in get_args(cls): + registry[arg] = func + else: + registry[cls] = func if cache_token is None and hasattr(cls, '__abstractmethods__'): cache_token = get_cache_token() dispatch_cache.clear() diff --git a/Lib/graphlib.py b/Lib/graphlib.py index d0e7a4814c5..636545648e1 100644 --- a/Lib/graphlib.py +++ b/Lib/graphlib.py @@ -1,3 +1,5 @@ +from types import GenericAlias + __all__ = ["TopologicalSorter", "CycleError"] _NODE_OUT = -1 @@ -17,7 +19,7 @@ class _NodeInfo: self.npredecessors = 0 # List of successor nodes. The list can contain duplicated elements as - # long as they're all reflected in the successor's npredecessors attribute). + # long as they're all reflected in the successor's npredecessors attribute. self.successors = [] @@ -244,3 +246,5 @@ class TopologicalSorter: node_group = self.get_ready() yield from node_group self.done(*node_group) + + __class_getitem__ = classmethod(GenericAlias) diff --git a/Lib/gzip.py b/Lib/gzip.py index 3d837b74480..6773ea3eef0 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -398,9 +398,58 @@ class GzipFile(_compression.BaseStream): self._check_not_closed() return self._buffer.readline(size) - def __iter__(self): - self._check_not_closed() - return self._buffer.__iter__() + +def _read_exact(fp, n): + '''Read exactly *n* bytes from `fp` + + This method is required because fp may be unbuffered, + i.e. return short reads. + ''' + data = fp.read(n) + while len(data) < n: + b = fp.read(n - len(data)) + if not b: + raise EOFError("Compressed file ended before the " + "end-of-stream marker was reached") + data += b + return data + + +def _read_gzip_header(fp): + '''Read a gzip header from `fp` and progress to the end of the header. + + Returns last mtime if header was present or None otherwise. + ''' + magic = fp.read(2) + if magic == b'': + return None + + if magic != b'\037\213': + raise BadGzipFile('Not a gzipped file (%r)' % magic) + + (method, flag, last_mtime) = struct.unpack(" bytes: + """ + Write a simple gzip header with no extra fields. + :param compresslevel: Compresslevel used to determine the xfl bytes. + :param mtime: The mtime (must support conversion to a 32-bit integer). + :return: A bytes object representing the gzip header. + """ + if mtime is None: + mtime = time.time() + if compresslevel == _COMPRESS_LEVEL_BEST: + xfl = 2 + elif compresslevel == _COMPRESS_LEVEL_FAST: + xfl = 4 + else: + xfl = 0 + # Pack ID1 and ID2 magic bytes, method (8=deflate), header flags (no extra + # fields added to header), mtime, xfl and os (255 for unknown OS). + return struct.pack("", "/>"): - lineno, offset = self.getpos() - if "\n" in self.__starttag_text: - lineno = lineno + self.__starttag_text.count("\n") - offset = len(self.__starttag_text) \ - - self.__starttag_text.rfind("\n") - else: - offset = offset + len(self.__starttag_text) self.handle_data(rawdata[i:endpos]) return endpos if end.endswith('/>'): @@ -405,7 +398,7 @@ class HTMLParser(_markupbase.ParserBase): tagname = namematch.group(1).lower() # consume and ignore other stuff between the name and the > # Note: this is not 100% correct, since we might have things like - # , but looking for > after tha name should cover + # , but looking for > after the name should cover # most of the cases and is much simpler gtpos = rawdata.find('>', namematch.end()) self.handle_endtag(tagname) diff --git a/Lib/http/client.py b/Lib/http/client.py index 08cf2ed9b37..f54172fd0de 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -70,6 +70,7 @@ Req-sent-unread-response _CS_REQ_SENT import email.parser import email.message +import errno import http import io import re @@ -592,8 +593,8 @@ class HTTPResponse(io.BufferedIOBase): amt -= chunk_left self.chunk_left = 0 return b''.join(value) - except IncompleteRead: - raise IncompleteRead(b''.join(value)) + except IncompleteRead as exc: + raise IncompleteRead(b''.join(value)) from exc def _readinto_chunked(self, b): assert self.chunked != _UNKNOWN @@ -939,7 +940,12 @@ class HTTPConnection: sys.audit("http.client.connect", self, self.host, self.port) self.sock = self._create_connection( (self.host,self.port), self.timeout, self.source_address) - self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + # Might fail in OSs that don't implement TCP_NODELAY + try: + self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + except OSError as e: + if e.errno != errno.ENOPROTOOPT: + raise if self._tunnel_host: self._tunnel() diff --git a/Lib/idlelib/ChangeLog b/Lib/idlelib/ChangeLog index d7d7e1efdb1..c8960cfa535 100644 --- a/Lib/idlelib/ChangeLog +++ b/Lib/idlelib/ChangeLog @@ -1175,7 +1175,7 @@ Wed Mar 10 05:18:02 1999 Guido van Rossum classes in selected module methods of selected class - Sinlge clicking in a directory, module or class item updates the next + Single clicking in a directory, module or class item updates the next column with info about the selected item. Double clicking in a module, class or method item opens the file (and selects the clicked item if it is a class or method). diff --git a/Lib/idlelib/autocomplete.py b/Lib/idlelib/autocomplete.py index bb7ee035c4f..032d3122531 100644 --- a/Lib/idlelib/autocomplete.py +++ b/Lib/idlelib/autocomplete.py @@ -9,6 +9,12 @@ import os import string import sys +# Modified keyword list is used in fetch_completions. +completion_kwds = [s for s in keyword.kwlist + if s not in {'True', 'False', 'None'}] # In builtins. +completion_kwds.extend(('match', 'case')) # Context keywords. +completion_kwds.sort() + # Two types of completions; defined here for autocomplete_w import below. ATTRS, FILES = 0, 1 from idlelib import autocomplete_w @@ -177,9 +183,7 @@ class AutoComplete: namespace = {**__main__.__builtins__.__dict__, **__main__.__dict__} bigl = eval("dir()", namespace) - kwds = (s for s in keyword.kwlist - if s not in {'True', 'False', 'None'}) - bigl.extend(kwds) + bigl.extend(completion_kwds) bigl.sort() if "__all__" in bigl: smalll = sorted(eval("__all__", namespace)) diff --git a/Lib/idlelib/autocomplete_w.py b/Lib/idlelib/autocomplete_w.py index 13ff60ae449..0f835a9cc1d 100644 --- a/Lib/idlelib/autocomplete_w.py +++ b/Lib/idlelib/autocomplete_w.py @@ -206,7 +206,7 @@ class AutoCompleteWindow: scrollbar.config(command=listbox.yview) scrollbar.pack(side=RIGHT, fill=Y) listbox.pack(side=LEFT, fill=BOTH, expand=True) - acw.update_idletasks() # Need for tk8.6.8 on macOS: #40128. + #acw.update_idletasks() # Need for tk8.6.8 on macOS: #40128. acw.lift() # work around bug in Tk 8.5.18+ (issue #24570) # Initialize the listbox selection diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index 2b1c2afff08..2468afa7148 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -24,7 +24,7 @@ - + @@ -59,7 +59,7 @@ modules |

  • - next |
  • -

    Completion boxes intially exclude names beginning with ‘_’ or, for +

    Completion boxes initially exclude names beginning with ‘_’ or, for modules, not included in ‘__all__’. The hidden names can be accessed by typing ‘_’ after ‘.’, either before or after the box is opened.

    @@ -937,8 +937,8 @@ also used for testing.

    tkinter.tix — Extension widgets for Tk

    Next topic

    -

    Other Graphical User Interface Packages

    +

    Development Tools

    This Page