mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-12-23 09:19:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'main' into tracebacklimit-doc

Browse source
This commit is contained in:
Sebastian Rittau 2025-12-22 21:03:08 +01:00 • committed by GitHub
commit e22d906539
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
593 changed files with 76489 additions and 12866 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.gitattributes vendored
View file

@ -68,6 +68,7 @@ PCbuild/readme.txt dos
**/clinic/*.cpp.h generated
**/clinic/*.h.h generated
*_db.h generated
Doc/_static/tachyon-example-*.html generated
Doc/c-api/lifecycle.dot.svg generated
Doc/data/stable_abi.dat generated
Doc/library/token-list.inc generated
@ -88,6 +89,7 @@ Lib/test/certdata/*.pem generated
Lib/test/certdata/*.0 generated
Lib/test/levenshtein_examples.json generated
Lib/test/test_stable_abi_ctypes.py generated
Lib/test/test_zoneinfo/data/*.json generated
Lib/token.py generated
Misc/sbom.spdx.json generated
Objects/typeslots.inc generated

14
.github/CODEOWNERS vendored
View file

@ -126,6 +126,9 @@ Doc/howto/clinic.rst @erlend-aasland @AA-Turner
# C Analyser
Tools/c-analyzer/ @ericsnowcurrently
# C API Documentation Checks
Tools/check-c-api-docs/ @ZeroIntensity
# Fuzzing
Modules/_xxtestfuzz/ @ammaraskar
@ -286,10 +289,10 @@ Tools/jit/ @brandtbucher @savannahostrowski @diegorusso
InternalDocs/jit.md @brandtbucher @savannahostrowski @diegorusso @AA-Turner
# Micro-op / μop / Tier 2 Optimiser
Python/optimizer.c @markshannon
Python/optimizer.c @markshannon @Fidget-Spinner
Python/optimizer_analysis.c @markshannon @tomasr8 @Fidget-Spinner
Python/optimizer_bytecodes.c @markshannon @tomasr8 @Fidget-Spinner
Python/optimizer_symbols.c @markshannon @tomasr8
Python/optimizer_symbols.c @markshannon @tomasr8 @Fidget-Spinner
# Parser, Lexer, and Grammar
Grammar/python.gram @pablogsal @lysnikolaou
@ -319,7 +322,7 @@ Tools/build/generate_global_objects.py @ericsnowcurrently
# Remote Debugging
Python/remote_debug.h @pablogsal
Python/remote_debugging.c @pablogsal
Modules/_remote_debugging_module.c @pablogsal @ambv @1st1
Modules/_remote_debugging/ @pablogsal
# Sub-Interpreters
**/*crossinterp* @ericsnowcurrently
@ -534,6 +537,11 @@ Lib/pydoc.py @AA-Turner
Lib/pydoc_data/ @AA-Turner
Lib/test/test_pydoc/ @AA-Turner
# Profiling (Sampling)
Doc/library/profiling*.rst @pablogsal
Lib/profiling/ @pablogsal
Lib/test/test_profiling/ @pablogsal
# PyREPL
Lib/_pyrepl/ @pablogsal @lysnikolaou @ambv
Lib/test/test_pyrepl/ @pablogsal @lysnikolaou @ambv

3
.github/ISSUE_TEMPLATE/config.yml vendored
View file

@ -5,3 +5,6 @@ contact_links:
- name: "Proposing new features"
about: "Submit major feature proposal (e.g. syntax changes) to an ideas forum first."
url: "https://discuss.python.org/c/ideas/6"
- name: "Python Install Manager issues"
about: "Report issues with the Python Install Manager (for Windows)"
url: "https://github.com/python/pymanager/issues"

69
.github/workflows/build.yml vendored
View file

@ -142,6 +142,9 @@ jobs:
- name: Check for unsupported C global variables
if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME
run: make check-c-globals
- name: Check for undocumented C APIs
run: make check-c-api-docs
build-windows:
name: >-
@ -188,7 +191,7 @@ jobs:
macOS
${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }}
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
if: needs.build-context.outputs.run-macos == 'true'
strategy:
fail-fast: false
matrix:
@ -214,7 +217,7 @@ jobs:
${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }}
${{ fromJSON(matrix.bolt) && '(bolt)' || '' }}
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
if: needs.build-context.outputs.run-ubuntu == 'true'
strategy:
fail-fast: false
matrix:
@ -245,7 +248,7 @@ jobs:
runs-on: ${{ matrix.os }}
timeout-minutes: 60
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
if: needs.build-context.outputs.run-ubuntu == 'true'
strategy:
fail-fast: false
matrix:
@ -301,7 +304,7 @@ jobs:
runs-on: ${{ matrix.os }}
timeout-minutes: 60
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
if: needs.build-context.outputs.run-ubuntu == 'true'
strategy:
fail-fast: false
matrix:
@ -365,7 +368,7 @@ jobs:
build-android:
name: Android (${{ matrix.arch }})
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
if: needs.build-context.outputs.run-android == 'true'
timeout-minutes: 60
strategy:
fail-fast: false
@ -387,9 +390,9 @@ jobs:
build-ios:
name: iOS
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
if: needs.build-context.outputs.run-ios == 'true'
timeout-minutes: 60
runs-on: macos-15
runs-on: macos-14
steps:
- uses: actions/checkout@v4
with:
@ -402,15 +405,15 @@ jobs:
# https://github.com/actions/runner-images/issues/12751.
- name: Select Xcode version
run: |
sudo xcode-select --switch /Applications/Xcode_16.4.app
sudo xcode-select --switch /Applications/Xcode_15.4.app
- name: Build and test
run: python3 Apple ci iOS --fast-ci --simulator 'iPhone 16e,OS=18.5'
run: python3 Apple ci iOS --fast-ci --simulator 'iPhone SE (3rd generation),OS=17.5'
build-wasi:
name: 'WASI'
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
if: needs.build-context.outputs.run-wasi == 'true'
uses: ./.github/workflows/reusable-wasi.yml
test-hypothesis:
@ -418,7 +421,7 @@ jobs:
runs-on: ubuntu-24.04
timeout-minutes: 60
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
if: needs.build-context.outputs.run-ubuntu == 'true'
env:
OPENSSL_VER: 3.0.18
PYTHONSTRICTEXTENSIONBUILD: 1
@ -525,7 +528,7 @@ jobs:
runs-on: ${{ matrix.os }}
timeout-minutes: 60
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
if: needs.build-context.outputs.run-ubuntu == 'true'
strategy:
fail-fast: false
matrix:
@ -578,7 +581,7 @@ jobs:
# ${{ '' } is a hack to nest jobs under the same sidebar category.
name: Sanitizers${{ '' }} # zizmor: ignore[obfuscation]
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
if: needs.build-context.outputs.run-ubuntu == 'true'
strategy:
fail-fast: false
matrix:
@ -603,7 +606,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 60
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
if: needs.build-context.outputs.run-ubuntu == 'true'
steps:
- uses: actions/checkout@v4
with:
@ -709,25 +712,24 @@ jobs:
test-hypothesis,
cifuzz,
allowed-skips: >-
${{
!fromJSON(needs.build-context.outputs.run-docs)
&& '
check-docs,
'
|| ''
}}
${{ !fromJSON(needs.build-context.outputs.run-docs) && 'check-docs,' || '' }}
${{
needs.build-context.outputs.run-tests != 'true'
&& '
check-autoconf-regen,
check-generated-files,
build-macos,
'
|| ''
}}
${{ !fromJSON(needs.build-context.outputs.run-windows-tests) && 'build-windows,' || '' }}
${{ !fromJSON(needs.build-context.outputs.run-ci-fuzz) && 'cifuzz,' || '' }}
${{ !fromJSON(needs.build-context.outputs.run-macos) && 'build-macos,' || '' }}
${{
!fromJSON(needs.build-context.outputs.run-ubuntu)
&& '
build-ubuntu,
build-ubuntu-ssltests-awslc,
build-ubuntu-ssltests-openssl,
build-android,
build-ios,
build-wasi,
test-hypothesis,
build-asan,
build-san,
@ -735,18 +737,7 @@ jobs:
'
|| ''
}}
${{
!fromJSON(needs.build-context.outputs.run-windows-tests)
&& '
build-windows,
'
|| ''
}}
${{
!fromJSON(needs.build-context.outputs.run-ci-fuzz)
&& '
cifuzz,
'
|| ''
}}
${{ !fromJSON(needs.build-context.outputs.run-android) && 'build-android,' || '' }}
${{ !fromJSON(needs.build-context.outputs.run-ios) && 'build-ios,' || '' }}
${{ !fromJSON(needs.build-context.outputs.run-wasi) && 'build-wasi,' || '' }}
jobs: ${{ toJSON(needs) }}

2
.github/workflows/mypy.yml vendored
View file

@ -26,6 +26,7 @@ on:
- "Tools/build/update_file.py"
- "Tools/build/verify_ensurepip_wheels.py"
- "Tools/cases_generator/**"
- "Tools/check-c-api-docs/**"
- "Tools/clinic/**"
- "Tools/jit/**"
- "Tools/peg_generator/**"
@ -58,6 +59,7 @@ jobs:
"Lib/tomllib",
"Tools/build",
"Tools/cases_generator",
"Tools/check-c-api-docs",
"Tools/clinic",
"Tools/jit",
"Tools/peg_generator",

44
.github/workflows/reusable-context.yml vendored
View file

@ -17,21 +17,36 @@ on: # yamllint disable-line rule:truthy
# || 'falsy-branch'
# }}
#
run-docs:
description: Whether to build the docs
value: ${{ jobs.compute-changes.outputs.run-docs }} # bool
run-tests:
description: Whether to run the regular tests
value: ${{ jobs.compute-changes.outputs.run-tests }} # bool
run-windows-tests:
description: Whether to run the Windows tests
value: ${{ jobs.compute-changes.outputs.run-windows-tests }} # bool
run-windows-msi:
description: Whether to run the MSI installer smoke tests
value: ${{ jobs.compute-changes.outputs.run-windows-msi }} # bool
run-android:
description: Whether to run the Android tests
value: ${{ jobs.compute-changes.outputs.run-android }} # bool
run-ci-fuzz:
description: Whether to run the CIFuzz job
value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }} # bool
run-docs:
description: Whether to build the docs
value: ${{ jobs.compute-changes.outputs.run-docs }} # bool
run-ios:
description: Whether to run the iOS tests
value: ${{ jobs.compute-changes.outputs.run-ios }} # bool
run-macos:
description: Whether to run the macOS tests
value: ${{ jobs.compute-changes.outputs.run-macos }} # bool
run-tests:
description: Whether to run the regular tests
value: ${{ jobs.compute-changes.outputs.run-tests }} # bool
run-ubuntu:
description: Whether to run the Ubuntu tests
value: ${{ jobs.compute-changes.outputs.run-ubuntu }} # bool
run-wasi:
description: Whether to run the WASI tests
value: ${{ jobs.compute-changes.outputs.run-wasi }} # bool
run-windows-msi:
description: Whether to run the MSI installer smoke tests
value: ${{ jobs.compute-changes.outputs.run-windows-msi }} # bool
run-windows-tests:
description: Whether to run the Windows tests
value: ${{ jobs.compute-changes.outputs.run-windows-tests }} # bool
jobs:
compute-changes:
@ -39,9 +54,14 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 10
outputs:
run-android: ${{ steps.changes.outputs.run-android }}
run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }}
run-docs: ${{ steps.changes.outputs.run-docs }}
run-ios: ${{ steps.changes.outputs.run-ios }}
run-macos: ${{ steps.changes.outputs.run-macos }}
run-tests: ${{ steps.changes.outputs.run-tests }}
run-ubuntu: ${{ steps.changes.outputs.run-ubuntu }}
run-wasi: ${{ steps.changes.outputs.run-wasi }}
run-windows-msi: ${{ steps.changes.outputs.run-windows-msi }}
run-windows-tests: ${{ steps.changes.outputs.run-windows-tests }}
steps:

1
.gitignore vendored
View file

@ -45,6 +45,7 @@ gmon.out
.pytest_cache/
.ruff_cache/
.DS_Store
.pixi/
*.exe

39
Android/android.py
View file

@ -29,6 +29,7 @@ in_source_tree = (
ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists()
)
ENV_SCRIPT = ANDROID_DIR / "android-env.sh"
TESTBED_DIR = ANDROID_DIR / "testbed"
CROSS_BUILD_DIR = PYTHON_DIR / "cross-build"
@ -129,12 +130,11 @@ def android_env(host):
sysconfig_filename = next(sysconfig_files).name
host = re.fullmatch(r"_sysconfigdata__android_(.+).py", sysconfig_filename)[1]
env_script = ANDROID_DIR / "android-env.sh"
env_output = subprocess.run(
f"set -eu; "
f"HOST={host}; "
f"PREFIX={prefix}; "
f". {env_script}; "
f". {ENV_SCRIPT}; "
f"export",
check=True, shell=True, capture_output=True, encoding='utf-8',
).stdout
@ -151,7 +151,7 @@ def android_env(host):
env[key] = value
if not env:
raise ValueError(f"Found no variables in {env_script.name} output:\n"
raise ValueError(f"Found no variables in {ENV_SCRIPT.name} output:\n"
+ env_output)
return env
@ -281,15 +281,30 @@ def clean_all(context):
def setup_ci():
# https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/
if "GITHUB_ACTIONS" in os.environ and platform.system() == "Linux":
run(
["sudo", "tee", "/etc/udev/rules.d/99-kvm4all.rules"],
input='KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\n',
text=True,
)
run(["sudo", "udevadm", "control", "--reload-rules"])
run(["sudo", "udevadm", "trigger", "--name-match=kvm"])
if "GITHUB_ACTIONS" in os.environ:
# Enable emulator hardware acceleration
# (https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/).
if platform.system() == "Linux":
run(
["sudo", "tee", "/etc/udev/rules.d/99-kvm4all.rules"],
input='KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\n',
text=True,
)
run(["sudo", "udevadm", "control", "--reload-rules"])
run(["sudo", "udevadm", "trigger", "--name-match=kvm"])
# Free up disk space by deleting unused versions of the NDK
# (https://github.com/freakboy3742/pyspamsum/pull/108).
for line in ENV_SCRIPT.read_text().splitlines():
if match := re.fullmatch(r"ndk_version=(.+)", line):
ndk_version = match[1]
break
else:
raise ValueError(f"Failed to find NDK version in {ENV_SCRIPT.name}")
for item in (android_home / "ndk").iterdir():
if item.name[0].isdigit() and item.name != ndk_version:
delete_glob(item)
def setup_sdk():

4
Android/testbed/app/build.gradle.kts
View file

@ -79,7 +79,7 @@ android {
val androidEnvFile = file("../../android-env.sh").absoluteFile
namespace = "org.python.testbed"
compileSdk = 34
compileSdk = 35
defaultConfig {
applicationId = "org.python.testbed"
@ -92,7 +92,7 @@ android {
}
throw GradleException("Failed to find API level in $androidEnvFile")
}
targetSdk = 34
targetSdk = 35
versionCode = 1
versionName = "1.0"

3260
Doc/_static/tachyon-example-flamegraph.html generated Normal file
View file

File diff suppressed because one or more lines are too long

3804
Doc/_static/tachyon-example-heatmap.html generated Normal file
View file

File diff suppressed because one or more lines are too long

8
Doc/c-api/extension-modules.rst
View file

@ -29,7 +29,7 @@ and must be named after the module name plus an extension listed in
Extension export hook
.....................
.. versionadded:: next
.. versionadded:: 3.15
Support for the :samp:`PyModExport_{<name>}` export hook was added in Python
3.15. The older way of defining modules is still available: consult either
@ -191,7 +191,7 @@ the :c:data:`Py_mod_multiple_interpreters` slot.
``PyInit`` function
...................
.. deprecated:: next
.. deprecated:: 3.15
This functionality is :term:`soft deprecated`.
It will not get new features, but there are no plans to remove it.
@ -272,7 +272,7 @@ For example, a module called ``spam`` would be defined like this::
Legacy single-phase initialization
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.. deprecated:: next
.. deprecated:: 3.15
Single-phase initialization is :term:`soft deprecated`.
It is a legacy mechanism to initialize extension
@ -282,7 +282,7 @@ Legacy single-phase initialization
However, there are no plans to remove support for it.
In single-phase initialization, the old-style
:ref:`initializaton function <extension-pyinit>` (``PyInit_modulename``)
:ref:`initialization function <extension-pyinit>` (``PyInit_modulename``)
should create, populate and return a module object.
This is typically done using :c:func:`PyModule_Create` and functions like
:c:func:`PyModule_AddObjectRef`.

4
Doc/c-api/gcsupport.rst
View file

@ -232,6 +232,10 @@ The :c:member:`~PyTypeObject.tp_traverse` handler must have the following type:
object argument. If *visit* returns a non-zero value that value should be
returned immediately.
The traversal function must not have any side effects. Implementations
may not modify the reference counts of any Python objects nor create or
destroy any Python objects.
To simplify writing :c:member:`~PyTypeObject.tp_traverse` handlers, a :c:func:`Py_VISIT` macro is
provided. In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse` implementation
must name its arguments exactly *visit* and *arg*:

19
Doc/c-api/import.rst
View file

@ -129,8 +129,7 @@ Importing Modules
of :class:`~importlib.machinery.SourceFileLoader` otherwise.
The module's :attr:`~module.__file__` attribute will be set to the code
object's :attr:`~codeobject.co_filename`. If applicable,
:attr:`~module.__cached__` will also be set.
object's :attr:`~codeobject.co_filename`.
This function will reload the module if it was already imported. See
:c:func:`PyImport_ReloadModule` for the intended way to reload a module.
@ -142,10 +141,13 @@ Importing Modules
:c:func:`PyImport_ExecCodeModuleWithPathnames`.
.. versionchanged:: 3.12
The setting of :attr:`~module.__cached__` and :attr:`~module.__loader__`
The setting of ``__cached__`` and :attr:`~module.__loader__`
is deprecated. See :class:`~importlib.machinery.ModuleSpec` for
alternatives.
.. versionchanged:: 3.15
``__cached__`` is no longer set.
.. c:function:: PyObject* PyImport_ExecCodeModuleEx(const char *name, PyObject *co, const char *pathname)
@ -157,16 +159,19 @@ Importing Modules
.. c:function:: PyObject* PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, PyObject *cpathname)
Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`~module.__cached__`
attribute of the module object is set to *cpathname* if it is
non-``NULL``. Of the three functions, this is the preferred one to use.
Like :c:func:`PyImport_ExecCodeModuleEx`, but the path to any compiled file
via *cpathname* is used appropriately when non-``NULL``. Of the three
functions, this is the preferred one to use.
.. versionadded:: 3.3
.. versionchanged:: 3.12
Setting :attr:`~module.__cached__` is deprecated. See
Setting ``__cached__`` is deprecated. See
:class:`~importlib.machinery.ModuleSpec` for alternatives.
.. versionchanged:: 3.15
``__cached__`` no longer set.
.. c:function:: PyObject* PyImport_ExecCodeModuleWithPathnames(const char *name, PyObject *co, const char *pathname, const char *cpathname)

40
Doc/c-api/intro.rst
View file

@ -107,6 +107,46 @@ header files properly declare the entry points to be ``extern "C"``. As a result
there is no need to do anything special to use the API from C++.
.. _capi-system-includes:
System includes
---------------
:file:`Python.h` includes several standard header files.
C extensions should include the standard headers that they use,
and should not rely on these implicit includes.
The implicit includes are:
* ``<assert.h>``
* ``<intrin.h>`` (on Windows)
* ``<inttypes.h>``
* ``<limits.h>``
* ``<math.h>``
* ``<stdarg.h>``
* ``<wchar.h>``
* ``<sys/types.h>`` (if present)
The following are included for backwards compatibility, unless using
:ref:`Limited API <limited-c-api>` 3.13 or newer:
* ``<ctype.h>``
* ``<unistd.h>`` (on POSIX)
The following are included for backwards compatibility, unless using
:ref:`Limited API <limited-c-api>` 3.11 or newer:
* ``<errno.h>``
* ``<stdio.h>``
* ``<stdlib.h>``
* ``<string.h>``
.. note::
Since Python may define some pre-processor definitions which affect the standard
headers on some systems, you *must* include :file:`Python.h` before any standard
headers are included.
Useful macros
=============

32
Doc/c-api/module.rst
View file

@ -142,7 +142,7 @@ Modules created using the C API are typically defined using an
array of :dfn:`slots`.
The slots provide a "description" of how a module should be created.
.. versionchanged:: next
.. versionchanged:: 3.15
Previously, a :c:type:`PyModuleDef` struct was necessary to define modules.
The older way of defining modules is still available: consult either the
@ -190,7 +190,7 @@ Metadata slots
However, it is still recommended to include this slot for introspection
and debugging purposes.
.. versionadded:: next
.. versionadded:: 3.15
Use :c:member:`PyModuleDef.m_name` instead to support previous versions.
@ -201,7 +201,7 @@ Metadata slots
Usually it is set to a variable created with :c:macro:`PyDoc_STRVAR`.
.. versionadded:: next
.. versionadded:: 3.15
Use :c:member:`PyModuleDef.m_doc` instead to support previous versions.
@ -332,7 +332,7 @@ Creation and initialization slots
.. versionadded:: 3.5
.. versionchanged:: next
.. versionchanged:: 3.15
The *slots* argument may be a ``ModuleSpec``-like object, rather than
a true :py:class:`~importlib.machinery.ModuleSpec` instance.
@ -365,7 +365,7 @@ Creation and initialization slots
.. versionadded:: 3.5
.. versionchanged:: next
.. versionchanged:: 3.15
Repeated ``Py_mod_exec`` slots are disallowed, except in
:c:type:`PyModuleDef.m_slots`.
@ -384,7 +384,7 @@ Creation and initialization slots
The table must be statically allocated (or otherwise guaranteed to outlive
the module object).
.. versionadded:: next
.. versionadded:: 3.15
Use :c:member:`PyModuleDef.m_methods` instead to support previous versions.
@ -434,7 +434,7 @@ To retrieve the state from a given module, use the following functions:
On error, set *\*result* to -1, and return -1 with an exception set.
.. versionadded:: next
.. versionadded:: 3.15
@ -459,7 +459,7 @@ defining the module state.
Use :c:func:`PyModule_GetStateSize` to retrieve the size of a given module.
.. versionadded:: next
.. versionadded:: 3.15
Use :c:member:`PyModuleDef.m_size` instead to support previous versions.
@ -482,7 +482,7 @@ defining the module state.
(:c:data:`Py_mod_state_size`) is greater than 0 and the module state
(as returned by :c:func:`PyModule_GetState`) is ``NULL``.
.. versionadded:: next
.. versionadded:: 3.15
Use :c:member:`PyModuleDef.m_size` instead to support previous versions.
@ -510,7 +510,7 @@ defining the module state.
the cyclic garbage collector is not involved and
the :c:macro:`Py_mod_state_free` function is called directly.
.. versionadded:: next
.. versionadded:: 3.15
Use :c:member:`PyModuleDef.m_clear` instead to support previous versions.
@ -532,7 +532,7 @@ defining the module state.
(:c:data:`Py_mod_state_size`) is greater than 0 and the module state
(as returned by :c:func:`PyModule_GetState`) is ``NULL``.
.. versionadded:: next
.. versionadded:: 3.15
Use :c:member:`PyModuleDef.m_free` instead to support previous versions.
@ -588,12 +588,12 @@ A module's token -- and the *your_token* value to use in the above code -- is:
behave as if it was created from that :c:type:`PyModuleDef`.
In particular, the module state must have matching layout and semantics.
Modules created from :c:type:`PyModuleDef` allways use the address of
Modules created from :c:type:`PyModuleDef` always use the address of
the :c:type:`PyModuleDef` as the token.
This means that :c:macro:`!Py_mod_token` cannot be used in
:c:member:`PyModuleDef.m_slots`.
.. versionadded:: next
.. versionadded:: 3.15
.. c:function:: int PyModule_GetToken(PyObject *module, void** result)
@ -601,7 +601,7 @@ A module's token -- and the *your_token* value to use in the above code -- is:
On error, set *\*result* to NULL, and return -1 with an exception set.
.. versionadded:: next
.. versionadded:: 3.15
See also :c:func:`PyType_GetModuleByToken`.
@ -641,7 +641,7 @@ rather than from an extension's :ref:`export hook <extension-export-hook>`.
:c:func:`!PyModule_FromSlotsAndSpec` call.
In particular, it may be heap-allocated.
.. versionadded:: next
.. versionadded:: 3.15
.. c:function:: int PyModule_Exec(PyObject *module)
@ -654,7 +654,7 @@ rather than from an extension's :ref:`export hook <extension-export-hook>`.
:ref:`legacy single-phase initialization <single-phase-initialization>`,
this function does nothing and returns 0.
.. versionadded:: next
.. versionadded:: 3.15

2
Doc/c-api/object.rst
View file

@ -111,7 +111,7 @@ Object Protocol
object type name: str
object repr : 'abcdef'
.. versionadded:: next
.. versionadded:: 3.15
.. c:function:: int PyObject_HasAttrWithError(PyObject *o, PyObject *attr_name)

2
Doc/c-api/set.rst
View file

@ -147,7 +147,7 @@ subtypes but not for instances of :class:`frozenset` or its subtypes.
Return ``1`` if found and removed, ``0`` if not found (no action taken), and ``-1`` if an
error is encountered. Does not raise :exc:`KeyError` for missing keys. Raise a
:exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~frozenset.discard`
:exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~set.discard`
method, this function does not automatically convert unhashable sets into
temporary frozensets. Raise :exc:`SystemError` if *set* is not an
instance of :class:`set` or its subtype.

7
Doc/c-api/tuple.rst
View file

@ -148,8 +148,11 @@ Tuple Objects
Struct Sequence Objects
-----------------------
Struct sequence objects are the C equivalent of :func:`~collections.namedtuple`
objects, i.e. a sequence whose items can also be accessed through attributes.
A struct sequence object is a :term:`named tuple`, that is, a sequence
whose items can also be accessed through attributes.
It is similar to :func:`collections.namedtuple`, but provides a slightly
different interface.
To create a struct sequence, you first have to create a specific struct sequence
type.

2
Doc/c-api/type.rst
View file

@ -317,7 +317,7 @@ Type Objects
and other places where a method's defining class cannot be passed using the
:c:type:`PyCMethod` calling convention.
.. versionadded:: next
.. versionadded:: 3.15
.. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def)

5
Doc/c-api/typeobj.rst
View file

@ -1569,6 +1569,11 @@ and :c:data:`PyType_Type` effectively act as defaults.)
but the instance has no strong reference to the elements inside it, as they
are allowed to be removed even if the instance is still alive).
.. warning::
The traversal function must not have any side effects. It must not
modify the reference counts of any Python objects nor create or destroy
any Python objects.
Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to
:c:func:`!local_traverse` to have these specific names; don't name them just
anything.

7
Doc/conf.py
View file

@ -436,7 +436,12 @@ latex_appendices = ['glossary', 'about', 'license', 'copyright']
epub_author = 'Python Documentation Authors'
epub_publisher = 'Python Software Foundation'
epub_exclude_files = ('index.xhtml', 'download.xhtml')
epub_exclude_files = (
'index.xhtml',
'download.xhtml',
'_static/tachyon-example-flamegraph.html',
'_static/tachyon-example-heatmap.html',
)
# index pages are not valid xhtml
# https://github.com/sphinx-doc/sphinx/issues/12359

7
Doc/deprecations/c-api-pending-removal-in-3.20.rst
View file

@ -1,6 +1,13 @@
Pending removal in Python 3.20
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
* :c:func:`!_PyObject_CallMethodId`, :c:func:`!_PyObject_GetAttrId` and
:c:func:`!_PyUnicode_FromId` are deprecated since 3.15 and will be removed in
3.20. Instead, use :c:func:`PyUnicode_InternFromString()` and cache the result in
the module state, then call :c:func:`PyObject_CallMethod` or
:c:func:`PyObject_GetAttr`.
(Contributed by Victor Stinner in :gh:`141049`.)
* The ``cval`` field in :c:type:`PyComplexObject` (:gh:`128813`).
Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex`
to convert a Python complex number to/from the C :c:type:`Py_complex`

4
Doc/deprecations/pending-removal-in-3.15.rst
View file

@ -3,9 +3,9 @@ Pending removal in Python 3.15
* The import system:
* Setting :attr:`~module.__cached__` on a module while
* Setting ``__cached__`` on a module while
failing to set :attr:`__spec__.cached <importlib.machinery.ModuleSpec.cached>`
is deprecated. In Python 3.15, :attr:`!__cached__` will cease to be set or
is deprecated. In Python 3.15, ``__cached__`` will cease to be set or
take into consideration by the import system or standard library. (:gh:`97879`)
* Setting :attr:`~module.__package__` on a module while

12
Doc/deprecations/pending-removal-in-3.20.rst
View file

@ -1,14 +1,16 @@
Pending removal in Python 3.20
------------------------------
* The ``__version__`` attribute has been deprecated in these standard library
modules and will be removed in Python 3.20.
Use :py:data:`sys.version_info` instead.
* The ``__version__``, ``version`` and ``VERSION`` attributes have been
deprecated in these standard library modules and will be removed in
Python 3.20. Use :py:data:`sys.version_info` instead.
- :mod:`argparse`
- :mod:`csv`
- :mod:`ctypes`
- :mod:`!ctypes.macholib`
- :mod:`decimal` (use :data:`decimal.SPEC_VERSION` instead)
- :mod:`http.server`
- :mod:`imaplib`
- :mod:`ipaddress`
- :mod:`json`
@ -21,6 +23,10 @@ Pending removal in Python 3.20
- :mod:`tabnanny`
- :mod:`tkinter.font`
- :mod:`tkinter.ttk`
- :mod:`wsgiref.simple_server`
- :mod:`xml.etree.ElementTree`
- :mod:`!xml.sax.expatreader`
- :mod:`xml.sax.handler`
- :mod:`zlib`
(Contributed by Hugo van Kemenade and Stan Ulbrych in :gh:`76007`.)

421
Doc/extending/extending.rst
View file

@ -3,154 +3,20 @@
.. _extending-intro:
******************************
Extending Python with C or C++
******************************
********************************
Using the C API: Assorted topics
********************************
It is quite easy to add new built-in modules to Python, if you know how to
program in C. Such :dfn:`extension modules` can do two things that can't be
done directly in Python: they can implement new built-in object types, and they
can call C library functions and system calls.
To support extensions, the Python API (Application Programmers Interface)
defines a set of functions, macros and variables that provide access to most
aspects of the Python run-time system. The Python API is incorporated in a C
source file by including the header ``"Python.h"``.
The compilation of an extension module depends on its intended use as well as on
your system setup; details are given in later chapters.
.. note::
The C extension interface is specific to CPython, and extension modules do
not work on other Python implementations. In many cases, it is possible to
avoid writing C extensions and preserve portability to other implementations.
For example, if your use case is calling C library functions or system calls,
you should consider using the :mod:`ctypes` module or the `cffi
<https://cffi.readthedocs.io/>`_ library rather than writing
custom C code.
These modules let you write Python code to interface with C code and are more
portable between implementations of Python than writing and compiling a C
extension module.
.. _extending-simpleexample:
A Simple Example
================
Let's create an extension module called ``spam`` (the favorite food of Monty
Python fans...) and let's say we want to create a Python interface to the C
library function :c:func:`system` [#]_. This function takes a null-terminated
character string as argument and returns an integer. We want this function to
be callable from Python as follows:
.. code-block:: pycon
>>> import spam
>>> status = spam.system("ls -l")
Begin by creating a file :file:`spammodule.c`. (Historically, if a module is
called ``spam``, the C file containing its implementation is called
:file:`spammodule.c`; if the module name is very long, like ``spammify``, the
module name can be just :file:`spammify.c`.)
The first two lines of our file can be::
#define PY_SSIZE_T_CLEAN
#include <Python.h>
which pulls in the Python API (you can add a comment describing the purpose of
the module and a copyright notice if you like).
.. note::
Since Python may define some pre-processor definitions which affect the standard
headers on some systems, you *must* include :file:`Python.h` before any standard
headers are included.
``#define PY_SSIZE_T_CLEAN`` was used to indicate that ``Py_ssize_t`` should be
used in some APIs instead of ``int``.
It is not necessary since Python 3.13, but we keep it here for backward compatibility.
See :ref:`arg-parsing-string-and-buffers` for a description of this macro.
All user-visible symbols defined by :file:`Python.h` have a prefix of ``Py`` or
``PY``, except those defined in standard header files.
.. tip::
For backward compatibility, :file:`Python.h` includes several standard header files.
C extensions should include the standard headers that they use,
and should not rely on these implicit includes.
If using the limited C API version 3.13 or newer, the implicit includes are:
* ``<assert.h>``
* ``<intrin.h>`` (on Windows)
* ``<inttypes.h>``
* ``<limits.h>``
* ``<math.h>``
* ``<stdarg.h>``
* ``<wchar.h>``
* ``<sys/types.h>`` (if present)
If :c:macro:`Py_LIMITED_API` is not defined, or is set to version 3.12 or older,
the headers below are also included:
* ``<ctype.h>``
* ``<unistd.h>`` (on POSIX)
If :c:macro:`Py_LIMITED_API` is not defined, or is set to version 3.10 or older,
the headers below are also included:
* ``<errno.h>``
* ``<stdio.h>``
* ``<stdlib.h>``
* ``<string.h>``
The next thing we add to our module file is the C function that will be called
when the Python expression ``spam.system(string)`` is evaluated (we'll see
shortly how it ends up being called)::
static PyObject *
spam_system(PyObject *self, PyObject *args)
{
const char *command;
int sts;
if (!PyArg_ParseTuple(args, "s", &command))
return NULL;
sts = system(command);
return PyLong_FromLong(sts);
}
There is a straightforward translation from the argument list in Python (for
example, the single expression ``"ls -l"``) to the arguments passed to the C
function. The C function always has two arguments, conventionally named *self*
and *args*.
The *self* argument points to the module object for module-level functions;
for a method it would point to the object instance.
The *args* argument will be a pointer to a Python tuple object containing the
arguments. Each item of the tuple corresponds to an argument in the call's
argument list. The arguments are Python objects --- in order to do anything
with them in our C function we have to convert them to C values. The function
:c:func:`PyArg_ParseTuple` in the Python API checks the argument types and
converts them to C values. It uses a template string to determine the required
types of the arguments as well as the types of the C variables into which to
store the converted values. More about this later.
:c:func:`PyArg_ParseTuple` returns true (nonzero) if all arguments have the right
type and its components have been stored in the variables whose addresses are
passed. It returns false (zero) if an invalid argument list was passed. In the
latter case it also raises an appropriate exception so the calling function can
return ``NULL`` immediately (as we saw in the example).
The :ref:`tutorial <first-extension-module>` walked you through
creating a C API extension module, but left many areas unexplained.
This document looks at several concepts that you'll need to learn
in order to write more complex extensions.
.. _extending-errors:
Intermezzo: Errors and Exceptions
=================================
Errors and Exceptions
=====================
An important convention throughout the Python interpreter is the following: when
a function fails, it should set an exception condition and return an error value
@ -321,194 +187,14 @@ call to :c:func:`PyErr_SetString` as shown below::
}
.. _backtoexample:
Back to the Example
===================
Going back to our example function, you should now be able to understand this
statement::
if (!PyArg_ParseTuple(args, "s", &command))
return NULL;
It returns ``NULL`` (the error indicator for functions returning object pointers)
if an error is detected in the argument list, relying on the exception set by
:c:func:`PyArg_ParseTuple`. Otherwise the string value of the argument has been
copied to the local variable :c:data:`!command`. This is a pointer assignment and
you are not supposed to modify the string to which it points (so in Standard C,
the variable :c:data:`!command` should properly be declared as ``const char
*command``).
The next statement is a call to the Unix function :c:func:`system`, passing it
the string we just got from :c:func:`PyArg_ParseTuple`::
sts = system(command);
Our :func:`!spam.system` function must return the value of :c:data:`!sts` as a
Python object. This is done using the function :c:func:`PyLong_FromLong`. ::
return PyLong_FromLong(sts);
In this case, it will return an integer object. (Yes, even integers are objects
on the heap in Python!)
If you have a C function that returns no useful argument (a function returning
:c:expr:`void`), the corresponding Python function must return ``None``. You
need this idiom to do so (which is implemented by the :c:macro:`Py_RETURN_NONE`
macro)::
Py_INCREF(Py_None);
return Py_None;
:c:data:`Py_None` is the C name for the special Python object ``None``. It is a
genuine Python object rather than a ``NULL`` pointer, which means "error" in most
contexts, as we have seen.
.. _methodtable:
The Module's Method Table and Initialization Function
=====================================================
I promised to show how :c:func:`!spam_system` is called from Python programs.
First, we need to list its name and address in a "method table"::
static PyMethodDef spam_methods[] = {
...
{"system", spam_system, METH_VARARGS,
"Execute a shell command."},
...
{NULL, NULL, 0, NULL} /* Sentinel */
};
Note the third entry (``METH_VARARGS``). This is a flag telling the interpreter
the calling convention to be used for the C function. It should normally always
be ``METH_VARARGS`` or ``METH_VARARGS | METH_KEYWORDS``; a value of ``0`` means
that an obsolete variant of :c:func:`PyArg_ParseTuple` is used.
When using only ``METH_VARARGS``, the function should expect the Python-level
parameters to be passed in as a tuple acceptable for parsing via
:c:func:`PyArg_ParseTuple`; more information on this function is provided below.
The :c:macro:`METH_KEYWORDS` bit may be set in the third field if keyword
arguments should be passed to the function. In this case, the C function should
accept a third ``PyObject *`` parameter which will be a dictionary of keywords.
Use :c:func:`PyArg_ParseTupleAndKeywords` to parse the arguments to such a
function.
The method table must be referenced in the module definition structure::
static struct PyModuleDef spam_module = {
...
.m_methods = spam_methods,
...
};
This structure, in turn, must be passed to the interpreter in the module's
initialization function. The initialization function must be named
:c:func:`!PyInit_name`, where *name* is the name of the module, and should be the
only non-\ ``static`` item defined in the module file::
PyMODINIT_FUNC
PyInit_spam(void)
{
return PyModuleDef_Init(&spam_module);
}
Note that :c:macro:`PyMODINIT_FUNC` declares the function as ``PyObject *`` return type,
declares any special linkage declarations required by the platform, and for C++
declares the function as ``extern "C"``.
:c:func:`!PyInit_spam` is called when each interpreter imports its module
:mod:`!spam` for the first time. (See below for comments about embedding Python.)
A pointer to the module definition must be returned via :c:func:`PyModuleDef_Init`,
so that the import machinery can create the module and store it in ``sys.modules``.
When embedding Python, the :c:func:`!PyInit_spam` function is not called
automatically unless there's an entry in the :c:data:`PyImport_Inittab` table.
To add the module to the initialization table, use :c:func:`PyImport_AppendInittab`,
optionally followed by an import of the module::
#define PY_SSIZE_T_CLEAN
#include <Python.h>
int
main(int argc, char *argv[])
{
PyStatus status;
PyConfig config;
PyConfig_InitPythonConfig(&config);
/* Add a built-in module, before Py_Initialize */
if (PyImport_AppendInittab("spam", PyInit_spam) == -1) {
fprintf(stderr, "Error: could not extend in-built modules table\n");
exit(1);
}
/* Pass argv[0] to the Python interpreter */
status = PyConfig_SetBytesString(&config, &config.program_name, argv[0]);
if (PyStatus_Exception(status)) {
goto exception;
}
/* Initialize the Python interpreter. Required.
If this step fails, it will be a fatal error. */
status = Py_InitializeFromConfig(&config);
if (PyStatus_Exception(status)) {
goto exception;
}
PyConfig_Clear(&config);
/* Optionally import the module; alternatively,
import can be deferred until the embedded script
imports it. */
PyObject *pmodule = PyImport_ImportModule("spam");
if (!pmodule) {
PyErr_Print();
fprintf(stderr, "Error: could not import module 'spam'\n");
}
// ... use Python C API here ...
return 0;
exception:
PyConfig_Clear(&config);
Py_ExitStatusException(status);
}
.. note::
If you declare a global variable or a local static one, the module may
experience unintended side-effects on re-initialisation, for example when
removing entries from ``sys.modules`` or importing compiled modules into
multiple interpreters within a process
(or following a :c:func:`fork` without an intervening :c:func:`exec`).
If module state is not yet fully :ref:`isolated <isolating-extensions-howto>`,
authors should consider marking the module as having no support for subinterpreters
(via :c:macro:`Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED`).
A more substantial example module is included in the Python source distribution
as :file:`Modules/xxlimited.c`. This file may be used as a template or simply
read as an example.
.. _compilation:
Compilation and Linkage
=======================
Embedding an extension
======================
There are two more things to do before you can use your new extension: compiling
and linking it with the Python system. If you use dynamic loading, the details
may depend on the style of dynamic loading your system uses; see the chapters
about building extension modules (chapter :ref:`building`) and additional
information that pertains only to building on Windows (chapter
:ref:`building-on-windows`) for more information about this.
If you can't use dynamic loading, or if you want to make your module a permanent
If you want to make your module a permanent
part of the Python interpreter, you will have to change the configuration setup
and rebuild the interpreter. Luckily, this is very simple on Unix: just place
and rebuild the interpreter. On Unix, place
your file (:file:`spammodule.c` for example) in the :file:`Modules/` directory
of an unpacked source distribution, add a line to the file
:file:`Modules/Setup.local` describing your file:
@ -536,7 +222,7 @@ on the line in the configuration file as well, for instance:
Calling Python Functions from C
===============================
So far we have concentrated on making C functions callable from Python. The
The tutorial concentrated on making C functions callable from Python. The
reverse is also useful: calling Python functions from C. This is especially the
case for libraries that support so-called "callback" functions. If a C
interface makes use of callbacks, the equivalent Python often needs to provide a
@ -581,7 +267,7 @@ be part of a module definition::
}
This function must be registered with the interpreter using the
:c:macro:`METH_VARARGS` flag; this is described in section :ref:`methodtable`. The
:c:macro:`METH_VARARGS` flag in :c:type:`PyMethodDef.ml_flags`. The
:c:func:`PyArg_ParseTuple` function and its arguments are documented in section
:ref:`parsetuple`.
@ -676,14 +362,21 @@ the above example, we use :c:func:`Py_BuildValue` to construct the dictionary. :
Py_DECREF(result);
.. index:: single: PyArg_ParseTuple (C function)
.. _parsetuple:
Extracting Parameters in Extension Functions
============================================
.. index:: single: PyArg_ParseTuple (C function)
The :ref:`tutorial <first-extension-module>` uses a ":c:data:`METH_O`"
function, which is limited to a single Python argument.
If you want more, you can use :c:data:`METH_VARARGS` instead.
With this flag, the C function will receive a :py:class:`tuple` of arguments
instead of a single object.
The :c:func:`PyArg_ParseTuple` function is declared as follows::
For unpacking the tuple, CPython provides the :c:func:`PyArg_ParseTuple`
function, declared as follows::
int PyArg_ParseTuple(PyObject *arg, const char *format, ...);
@ -693,6 +386,19 @@ whose syntax is explained in :ref:`arg-parsing` in the Python/C API Reference
Manual. The remaining arguments must be addresses of variables whose type is
determined by the format string.
For example, to receive a single Python :py:class:`str` object and turn it
into a C buffer, you would use ``"s"`` as the format string::
const char *command;
if (!PyArg_ParseTuple(args, "s", &command)) {
return NULL;
}
If an error is detected in the argument list, :c:func:`!PyArg_ParseTuple`
returns ``NULL`` (the error indicator for functions returning object pointers);
your function may return ``NULL``, relying on the exception set by
:c:func:`PyArg_ParseTuple`.
Note that while :c:func:`PyArg_ParseTuple` checks that the Python arguments have
the required types, it cannot check the validity of the addresses of C variables
passed to the call: if you make mistakes there, your code will probably crash or
@ -703,7 +409,6 @@ Note that any Python object references which are provided to the caller are
Some example calls::
#define PY_SSIZE_T_CLEAN
#include <Python.h>
::
@ -773,6 +478,17 @@ Some example calls::
Keyword Parameters for Extension Functions
==========================================
If you also want your function to accept
:term:`keyword arguments <keyword argument>`, use the :c:data:`METH_KEYWORDS`
flag in combination with :c:data:`METH_VARARGS`.
(:c:data:`!METH_KEYWORDS` can also be used with other flags; see its
documentation for the allowed combinations.)
In this case, the C function should accept a third ``PyObject *`` parameter
which will be a dictionary of keywords.
Use :c:func:`PyArg_ParseTupleAndKeywords` to parse the arguments to such a
function.
.. index:: single: PyArg_ParseTupleAndKeywords (C function)
The :c:func:`PyArg_ParseTupleAndKeywords` function is declared as follows::
@ -833,19 +549,6 @@ Philbrick (philbrick@hks.com)::
{NULL, NULL, 0, NULL} /* sentinel */
};
static struct PyModuleDef keywdarg_module = {
.m_base = PyModuleDef_HEAD_INIT,
.m_name = "keywdarg",
.m_size = 0,
.m_methods = keywdarg_methods,
};
PyMODINIT_FUNC
PyInit_keywdarg(void)
{
return PyModuleDef_Init(&keywdarg_module);
}
.. _buildvalue:
@ -986,11 +689,11 @@ needed. Ownership of a reference can be transferred. There are three ways to
dispose of an owned reference: pass it on, store it, or call :c:func:`Py_DECREF`.
Forgetting to dispose of an owned reference creates a memory leak.
It is also possible to :dfn:`borrow` [#]_ a reference to an object. The
It is also possible to :dfn:`borrow` [#borrow]_ a reference to an object. The
borrower of a reference should not call :c:func:`Py_DECREF`. The borrower must
not hold on to the object longer than the owner from which it was borrowed.
Using a borrowed reference after the owner has disposed of it risks using freed
memory and should be avoided completely [#]_.
memory and should be avoided completely [#dont-check-refcount]_.
The advantage of borrowing over owning a reference is that you don't need to
take care of disposing of the reference on all possible paths through the code
@ -1169,7 +872,7 @@ checking.
The C function calling mechanism guarantees that the argument list passed to C
functions (``args`` in the examples) is never ``NULL`` --- in fact it guarantees
that it is always a tuple [#]_.
that it is always a tuple [#old-calling-convention]_.
It is a severe error to ever let a ``NULL`` pointer "escape" to the Python user.
@ -1226,8 +929,8 @@ the module whose functions one wishes to call might not have been loaded yet!
Portability therefore requires not to make any assumptions about symbol
visibility. This means that all symbols in extension modules should be declared
``static``, except for the module's initialization function, in order to
avoid name clashes with other extension modules (as discussed in section
:ref:`methodtable`). And it means that symbols that *should* be accessible from
avoid name clashes with other extension modules. And it means that symbols
that *should* be accessible from
other extension modules must be exported in a different way.
Python provides a special mechanism to pass C-level information (pointers) from
@ -1269,8 +972,9 @@ file corresponding to the module provides a macro that takes care of importing
the module and retrieving its C API pointers; client modules only have to call
this macro before accessing the C API.
The exporting module is a modification of the :mod:`!spam` module from section
:ref:`extending-simpleexample`. The function :func:`!spam.system` does not call
The exporting module is a modification of the :mod:`!spam` module from the
:ref:`tutorial <first-extension-module>`.
The function :func:`!spam.system` does not call
the C library function :c:func:`system` directly, but a function
:c:func:`!PySpam_System`, which would of course do something more complicated in
reality (such as adding "spam" to every command). This function
@ -1412,15 +1116,14 @@ code distribution).
.. rubric:: Footnotes
.. [#] An interface for this function already exists in the standard module :mod:`os`
--- it was chosen as a simple and straightforward example.
.. [#borrow] The metaphor of "borrowing" a reference is not completely correct:
the owner still has a copy of the reference.
.. [#] The metaphor of "borrowing" a reference is not completely correct: the owner
still has a copy of the reference.
.. [#] Checking that the reference count is at least 1 **does not work** --- the
.. [#dont-check-refcount] Checking that the reference count is at least 1
**does not work** --- the
reference count itself could be in freed memory and may thus be reused for
another object!
.. [#] These guarantees don't hold when you use the "old" style calling convention ---
.. [#old-calling-convention] These guarantees don't hold when you use the
"old" style calling convention ---
this is still found in much existing code.

667
Doc/extending/first-extension-module.rst Normal file
View file

@ -0,0 +1,667 @@
.. highlight:: c
.. _extending-simpleexample:
.. _first-extension-module:
*********************************
Your first C API extension module
*********************************
This tutorial will take you through creating a simple
Python extension module written in C or C++.
We will use the low-level Python C API directly.
For easier ways to create extension modules, see
the :ref:`recommended third party tools <c-api-tools>`.
The tutorial assumes basic knowledge about Python: you should be able to
define functions in Python code before starting to write them in C.
See :ref:`tutorial-index` for an introduction to Python itself.
The tutorial should be approachable for anyone who can write a basic C library.
While we will mention several concepts that a C beginner would not be expected
to know, like ``static`` functions or linkage declarations, understanding these
is not necessary for success.
We will focus on giving you a "feel" of what Python's C API is like.
It will not teach you important concepts, like error handling
and reference counting, which are covered in later chapters.
We will assume that you use a Unix-like system (including macOS and
Linux), or Windows.
On other systems, you might need to adjust some details -- for example,
a system command name.
You need to have a suitable C compiler and Python development headers installed.
On Linux, headers are often in a package like ``python3-dev``
or ``python3-devel``.
You need to be able to install Python packages.
This tutorial uses `pip <https://pip.pypa.io/>`__ (``pip install``), but you
can substitute any tool that can build and install ``pyproject.toml``-based
projects, like `uv <https://docs.astral.sh/uv/>`_ (``uv pip install``).
Preferably, have a :ref:`virtual environment <venv-def>` activated.
.. note::
This tutorial uses APIs that were added in CPython 3.15.
To create an extension that's compatible with earlier versions of CPython,
please follow an earlier version of this documentation.
This tutorial uses C syntax added in C11 and C++20.
If your extension needs to be compatible with earlier standards,
please follow tutorials in documentation for Python 3.14 or below.
What we'll do
=============
Let's create an extension module called ``spam`` [#why-spam]_,
which will include a Python interface to the C
standard library function :c:func:`system`.
This function is defined in ``stdlib.h``.
It takes a C string as argument, runs the argument as a system
command, and returns a result value as an integer.
A manual page for :c:func:`system` might summarize it this way::
#include <stdlib.h>
int system(const char *command);
Note that like many functions in the C standard library,
this function is already exposed in Python.
In production, use :py:func:`os.system` or :py:func:`subprocess.run`
rather than the module you'll write here.
We want this function to be callable from Python as follows:
.. code-block:: pycon
>>> import spam
>>> status = spam.system("whoami")
User Name
>>> status
0
.. note::
The system command ``whoami`` prints out your username.
It's useful in tutorials like this one because it has the same name on
both Unix and Windows.
Start with the headers
======================
Begin by creating a directory for this tutorial, and switching to it
on the command line.
Then, create a file named :file:`spammodule.c` in your directory.
[#why-spammodule]_
In this file, we'll include two headers: :file:`Python.h` to pull in
all declarations of the Python C API, and :file:`stdlib.h` for the
:c:func:`system` function. [#stdlib-h]_
Add the following lines to :file:`spammodule.c`:
.. literalinclude:: ../includes/capi-extension/spammodule-01.c
:start-at: <Python.h>
:end-at: <stdlib.h>
Be sure to put :file:`stdlib.h`, and any other standard library includes,
*after* :file:`Python.h`.
On some systems, Python may define some pre-processor definitions
that affect the standard headers.
Running your build tool
=======================
With only the includes in place, your extension won't do anything.
Still, it's a good time to compile it and try to import it.
This will ensure that your build tool works, so that you can make
and test incremental changes as you follow the rest of the text.
CPython itself does not come with a tool to build extension modules;
it is recommended to use a third-party project for this.
In this tutorial, we'll use `meson-python`_.
(If you want to use another one, see :ref:`first-extension-other-tools`.)
.. at the time of writing, meson-python has the least overhead for a
simple extension using PyModExport.
Change this if another tool makes things easier.
``meson-python`` requires defining a "project" using two extra files.
First, add ``pyproject.toml`` with these contents:
.. code-block:: toml
[build-system]
build-backend = 'mesonpy'
requires = ['meson-python']
[project]
# Placeholder project information
# (change this before distributing the module)
name = 'sampleproject'
version = '0'
Then, create ``meson.build`` containing the following:
.. code-block:: meson
project('sampleproject', 'c')
py = import('python').find_installation(pure: false)
py.extension_module(
'spam', # name of the importable Python module
'spammodule.c', # the C source file
install: true,
)
.. note::
See `meson-python documentation <meson-python>`_ for details on
configuration.
Now, build install the *project in the current directory* (``.``) via ``pip``:
.. code-block:: sh
python -m pip install .
.. tip::
If you don't have ``pip`` installed, run ``python -m ensurepip``,
preferably in a :ref:`virtual environment <venv-def>`.
(Or, if you prefer another tool that can build and install
``pyproject.toml``-based projects, use that.)
.. _meson-python: https://mesonbuild.com/meson-python/
.. _virtual environment: https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments
Note that you will need to run this command again every time you change your
extension.
Unlike Python, C has an explicit compilation step.
When your extension is compiled and installed, start Python and try to
import it.
This should fail with the following exception:
.. code-block:: pycon
>>> import spam
Traceback (most recent call last):
...
ImportError: dynamic module does not define module export function (PyModExport_spam or PyInit_spam)
Module export hook
==================
The exception you got when you tried to import the module told you that Python
is looking for a "module export function", also known as a
:ref:`module export hook <extension-export-hook>`.
Let's define one.
First, add a prototype below the ``#include`` lines:
.. literalinclude:: ../includes/capi-extension/spammodule-01.c
:start-after: /// Export hook prototype
:end-before: ///
.. tip::
The prototype is not strictly necessary, but some modern compilers emit
warnings without it.
It's generally better to add the prototype than to disable the warning.
The :c:macro:`PyMODEXPORT_FUNC` macro declares the function's
return type, and adds any special linkage declarations needed
to make the function visible and usable when CPython loads it.
After the prototype, add the function itself.
For now, make it return ``NULL``:
.. code-block:: c
PyMODEXPORT_FUNC
PyModExport_spam(void)
{
return NULL;
}
Compile and load the module again.
You should get a different error this time.
.. code-block:: pycon
>>> import spam
Traceback (most recent call last):
...
SystemError: module export hook for module 'spam' failed without setting an exception
Simply returning ``NULL`` is *not* correct behavior for an export hook,
and CPython complains about it.
That's good -- it means that CPython found the function!
Let's now make it do something useful.
The slot table
==============
Rather than ``NULL``, the export hook should return the information needed to
create a module.
Let's start with the basics: the name and docstring.
The information should be defined in a ``static`` array of
:c:type:`PyModuleDef_Slot` entries, which are essentially key-value pairs.
Define this array just before your export hook:
.. code-block:: c
static PyModuleDef_Slot spam_slots[] = {
{Py_mod_name, "spam"},
{Py_mod_doc, "A wonderful module with an example function"},
{0, NULL}
};
For both :c:data:`Py_mod_name` and :c:data:`Py_mod_doc`, the values are C
strings -- that is, NUL-terminated, UTF-8 encoded byte arrays.
Note the zero-filled sentinel entry at the end.
If you forget it, you'll trigger undefined behavior.
The array is defined as ``static`` -- that is, not visible outside this ``.c`` file.
This will be a common theme.
CPython only needs to access the export hook; all global variables
and all other functions should generally be ``static``, so that they don't
clash with other extensions.
Return this array from your export hook instead of ``NULL``:
.. code-block:: c
:emphasize-lines: 4
PyMODEXPORT_FUNC
PyModExport_spam(void)
{
return spam_slots;
}
Now, recompile and try it out:
.. code-block:: pycon
>>> import spam
>>> print(spam)
<module 'spam' from '/home/encukou/dev/cpython/spam.so'>
You have an extension module!
Try ``help(spam)`` to see the docstring.
The next step will be adding a function.
.. _backtoexample:
Exposing a function
===================
To expose the :c:func:`system` C function directly to Python,
we'll need to write a layer of glue code to convert arguments from Python
objects to C values, and the C return value back to Python.
One of the simplest ways to write glue code is a ":c:data:`METH_O`" function,
which takes two Python objects and returns one.
All Python objects -- regardless of the Python type -- are represented in C
as pointers to the :c:type:`PyObject` structure.
Add such a function above the slots array::
static PyObject *
spam_system(PyObject *self, PyObject *arg)
{
Py_RETURN_NONE;
}
For now, we ignore the arguments, and use the :c:macro:`Py_RETURN_NONE`
macro, which expands to a ``return`` statement that properly returns
a Python :py:data:`None` object.
Recompile your extension to make sure you don't have syntax errors.
We haven't yet added ``spam_system`` to the module, so you might get a
warning that ``spam_system`` is unused.
.. _methodtable:
Method definitions
------------------
To expose the C function to Python, you will need to provide several pieces of
information in a structure called
:c:type:`PyMethodDef` [#why-pymethoddef]_:
* ``ml_name``: the name of the Python function;
* ``ml_doc``: a docstring;
* ``ml_meth``: the C function to be called; and
* ``ml_flags``: a set of flags describing details like how Python arguments are
passed to the C function.
We'll use :c:data:`METH_O` here -- the flag that matches our
``spam_system`` function's signature.
Because modules typically create several functions, these definitions
need to be collected in an array, with a zero-filled sentinel at the end.
Add this array just below the ``spam_system`` function:
.. literalinclude:: ../includes/capi-extension/spammodule-01.c
:start-after: /// Module method table
:end-before: ///
As with module slots, a zero-filled sentinel marks the end of the array.
Next, we'll add the method to the module.
Add a :c:data:`Py_mod_methods` slot to your :c:type:`PyMethodDef` array:
.. literalinclude:: ../includes/capi-extension/spammodule-01.c
:start-after: /// Module slot table
:end-before: ///
:emphasize-lines: 5
Recompile your extension again, and test it.
Be sure to restart the Python interpreter, so that ``import spam`` picks
up the new version of the module.
You should now be able to call the function:
.. code-block:: pycon
>>> import spam
>>> print(spam.system)
<built-in function system>
>>> print(spam.system('whoami'))
None
Note that our ``spam.system`` does not yet run the ``whoami`` command;
it only returns ``None``.
Check that the function accepts exactly one argument, as specified by
the :c:data:`METH_O` flag:
.. code-block:: pycon
>>> print(spam.system('too', 'many', 'arguments'))
Traceback (most recent call last):
...
TypeError: spam.system() takes exactly one argument (3 given)
Returning an integer
====================
Now, let's take a look at the return value.
Instead of ``None``, we'll want ``spam.system`` to return a number -- that is,
a Python :py:type:`int` object.
Eventually this will be the exit code of a system command,
but let's start with a fixed value, say, ``3``.
The Python C API provides a function to create a Python :py:type:`int` object
from a C ``int`` value: :c:func:`PyLong_FromLong`. [#why-pylongfromlong]_
To call it, replace the ``Py_RETURN_NONE`` with the following 3 lines:
.. this could be a one-liner, but we want to show the data types here
.. code-block:: c
:emphasize-lines: 4-6
static PyObject *
spam_system(PyObject *self, PyObject *arg)
{
int status = 3;
PyObject *result = PyLong_FromLong(status);
return result;
}
Recompile, restart the Python interpreter again,
and check that the function now returns 3:
.. code-block:: pycon
>>> import spam
>>> spam.system('whoami')
3
Accepting a string
==================
Finally, let's handle the function argument.
Our C function, :c:func:`!spam_system`, takes two arguments.
The first one, ``PyObject *self``, will be set to the ``spam`` module
object.
This isn't useful in our case, so we'll ignore it.
The other one, ``PyObject *arg``, will be set to the object that the user
passed from Python.
We expect that it should be a Python string.
In order to use the information in it, we will need
to convert it to a C value -- in this case, a C string (``const char *``).
There's a slight type mismatch here: Python's :py:class:`str` objects store
Unicode text, but C strings are arrays of bytes.
So, we'll need to *encode* the data, and we'll use the UTF-8 encoding for it.
(UTF-8 might not always be correct for system commands, but it's what
:py:meth:`str.encode` uses by default,
and the C API has special support for it.)
The function to encode a Python string into a UTF-8 buffer is named
:c:func:`PyUnicode_AsUTF8` [#why-pyunicodeasutf8]_.
Call it like this:
.. code-block:: c
:emphasize-lines: 4
static PyObject *
spam_system(PyObject *self, PyObject *arg)
{
const char *command = PyUnicode_AsUTF8(arg);
int status = 3;
PyObject *result = PyLong_FromLong(status);
return result;
}
If :c:func:`PyUnicode_AsUTF8` is successful, *command* will point to the
resulting array of bytes.
This buffer is managed by the *arg* object, which means we don't need to free
it, but we must follow some rules:
* We should only use the buffer inside the ``spam_system`` function.
When ``spam_system`` returns, *arg* and the buffer it manages might be
garbage-collected.
* We must not modify it. This is why we use ``const``.
If :c:func:`PyUnicode_AsUTF8` was *not* successful, it returns a ``NULL``
pointer.
When calling *any* Python C API, we always need to handle such error cases.
The way to do this in general is left for later chapters of this documentation.
For now, be assured that we are already handling errors from
:c:func:`PyLong_FromLong` correctly.
For the :c:func:`PyUnicode_AsUTF8` call, the correct way to handle errors is
returning ``NULL`` from ``spam_system``.
Add an ``if`` block for this:
.. code-block:: c
:emphasize-lines: 5-7
static PyObject *
spam_system(PyObject *self, PyObject *arg)
{
const char *command = PyUnicode_AsUTF8(arg);
if (command == NULL) {
return NULL;
}
int status = 3;
PyObject *result = PyLong_FromLong(status);
return result;
}
That's it for the setup.
Now, all that is left is calling the C library function :c:func:`system` with
the ``char *`` buffer, and using its result instead of the ``3``:
.. code-block:: c
:emphasize-lines: 8
static PyObject *
spam_system(PyObject *self, PyObject *arg)
{
const char *command = PyUnicode_AsUTF8(arg);
if (command == NULL) {
return NULL;
}
int status = system(command);
PyObject *result = PyLong_FromLong(status);
return result;
}
Compile your module, restart Python, and test.
This time, you should see your username -- the output of the ``whoami``
system command:
.. code-block:: pycon
>>> import spam
>>> result = spam.system('whoami')
User Name
>>> result
0
You might also want to test error cases:
.. code-block:: pycon
>>> import spam
>>> result = spam.system('nonexistent-command')
sh: line 1: nonexistent-command: command not found
>>> result
32512
>>> spam.system(3)
Traceback (most recent call last):
...
TypeError: bad argument type for built-in operation
The result
==========
Congratulations!
You have written a complete Python C API extension module,
and completed this tutorial!
Here is the entire source file, for your convenience:
.. _extending-spammodule-source:
.. literalinclude:: ../includes/capi-extension/spammodule-01.c
:start-at: ///
.. _first-extension-other-tools:
Appendix: Other build tools
===========================
You should be able to follow this tutorial -- except the
*Running your build tool* section itself -- with a build tool other
than ``meson-python``.
The Python Packaging User Guide has a `list of recommended tools <https://packaging.python.org/en/latest/guides/tool-recommendations/#build-backends-for-extension-modules>`_;
be sure to choose one for the C language.
Workaround for missing PyInit function
--------------------------------------
If your build tool output complains about missing ``PyInit_spam``,
add the following function to your module for now:
.. code-block:: c
// A workaround
void *PyInit_spam(void) { return NULL; }
This is a shim for an old-style :ref:`initialization function <extension-export-hook>`,
which was required in extension modules for CPython 3.14 and below.
Current CPython does not need it, but some build tools may still assume that
all extension modules need to define it.
If you use this workaround, you will get the exception
``SystemError: initialization of spam failed without raising an exception``
instead of
``ImportError: dynamic module does not define module export function``.
Compiling directly
------------------
Using a third-party build tool is heavily recommended,
as it will take care of various details of your platform and Python
installation, of naming the resulting extension, and, later, of distributing
your work.
If you are building an extension for as *specific* system, or for yourself
only, you might instead want to run your compiler directly.
The way to do this is system-specific; be prepared for issues you will need
to solve yourself.
Linux
^^^^^
On Linux, the Python development package may include a ``python3-config``
command that prints out the required compiler flags.
If you use it, check that it corresponds to the CPython interpreter you'll use
to load the module.
Then, start with the following command:
.. code-block:: sh
gcc --shared $(python3-config --cflags --ldflags) spammodule.c -o spam.so
This should generate a ``spam.so`` file that you need to put in a directory
on :py:attr:`sys.path`.
.. rubric:: Footnotes
.. [#why-spam] ``spam`` is the favorite food of Monty Python fans...
.. [#why-spammodule] The source file name is entirely up to you,
though some tools can be picky about the ``.c`` extension.
This tutorial uses the traditional ``*module.c`` suffix.
Some people would just use :file:`spam.c` to implement a module
named ``spam``,
projects where Python isn't the primary language might use ``py_spam.c``,
and so on.
.. [#stdlib-h] Including :file:`stdlib.h` is technically not necessary,
since :file:`Python.h` includes it and
:ref:`several other standard headers <capi-system-includes>` for its own use
or for backwards compatibility.
However, it is good practice to explicitly include what you need.
.. [#why-pymethoddef] The :c:type:`!PyMethodDef` structure is also used
to create methods of classes, so there's no separate
":c:type:`!PyFunctionDef`".
.. [#why-pylongfromlong] The name :c:func:`PyLong_FromLong`
might not seem obvious.
``PyLong`` refers to a the Python :py:class:`int`, which was originally
called ``long``; the ``FromLong`` refers to the C ``long`` (or ``long int``)
type.
.. [#why-pyunicodeasutf8] Here, ``PyUnicode`` refers to the original name of
the Python :py:class:`str` class: ``unicode``.

86
Doc/extending/index.rst
View file

@ -5,15 +5,17 @@
##################################################
This document describes how to write modules in C or C++ to extend the Python
interpreter with new modules. Those modules can not only define new functions
but also new object types and their methods. The document also describes how
interpreter with new modules. Those modules can do what Python code does --
define functions, object types and methods -- but also interact with native
libraries or achieve better performance by avoiding the overhead of an
interpreter. The document also describes how
to embed the Python interpreter in another application, for use as an extension
language. Finally, it shows how to compile and link extension modules so that
they can be loaded dynamically (at run time) into the interpreter, if the
underlying operating system supports this feature.
This document assumes basic knowledge about Python. For an informal
introduction to the language, see :ref:`tutorial-index`. :ref:`reference-index`
This document assumes basic knowledge about C and Python. For an informal
introduction to Python, see :ref:`tutorial-index`. :ref:`reference-index`
gives a more formal definition of the language. :ref:`library-index` documents
the existing object types, functions and modules (both built-in and written in
Python) that give the language its wide application range.
@ -21,37 +23,75 @@ Python) that give the language its wide application range.
For a detailed description of the whole Python/C API, see the separate
:ref:`c-api-index`.
To support extensions, Python's C API (Application Programmers Interface)
defines a set of functions, macros and variables that provide access to most
aspects of the Python run-time system. The Python API is incorporated in a C
source file by including the header ``"Python.h"``.
.. note::
The C extension interface is specific to CPython, and extension modules do
not work on other Python implementations. In many cases, it is possible to
avoid writing C extensions and preserve portability to other implementations.
For example, if your use case is calling C library functions or system calls,
you should consider using the :mod:`ctypes` module or the `cffi
<https://cffi.readthedocs.io/>`_ library rather than writing
custom C code.
These modules let you write Python code to interface with C code and are more
portable between implementations of Python than writing and compiling a C
extension module.
.. toctree::
:hidden:
first-extension-module.rst
extending.rst
newtypes_tutorial.rst
newtypes.rst
building.rst
windows.rst
embedding.rst
Recommended third party tools
=============================
This guide only covers the basic tools for creating extensions provided
This document only covers the basic tools for creating extensions provided
as part of this version of CPython. Some :ref:`third party tools
<c-api-tools>` offer both simpler and more sophisticated approaches to creating
C and C++ extensions for Python.
While this document is aimed at extension authors, it should also be helpful to
the authors of such tools.
For example, the tutorial module can serve as a simple test case for a build
tool or sample expected output of a code generator.
Creating extensions without third party tools
=============================================
C API Tutorial
==============
This tutorial describes how to write a simple module in C or C++,
using the Python C API -- that is, using the basic tools provided
as part of this version of CPython.
#. :ref:`first-extension-module`
Guides for intermediate topics
==============================
This section of the guide covers creating C and C++ extensions without
assistance from third party tools. It is intended primarily for creators
of those tools, rather than being a recommended way to create your own
C extensions.
.. seealso::
:pep:`489` -- Multi-phase extension module initialization
.. toctree::
:maxdepth: 2
:numbered:
extending.rst
newtypes_tutorial.rst
newtypes.rst
building.rst
windows.rst
* :ref:`extending-intro`
* :ref:`defining-new-types`
* :ref:`new-types-topics`
* :ref:`building`
* :ref:`building-on-windows`
Embedding the CPython runtime in a larger application
=====================================================
@ -61,8 +101,4 @@ interpreter as the main application, it is desirable to instead embed
the CPython runtime inside a larger application. This section covers
some of the details involved in doing that successfully.
.. toctree::
:maxdepth: 2
:numbered:
embedding.rst
* :ref:`embedding`

174
Doc/glossary.rst
View file

@ -134,6 +134,14 @@ Glossary
iterator's :meth:`~object.__anext__` method until it raises a
:exc:`StopAsyncIteration` exception. Introduced by :pep:`492`.
atomic operation
An operation that appears to execute as a single, indivisible step: no
other thread can observe it half-done, and its effects become visible all
at once. Python does not guarantee that high-level statements are atomic
(for example, ``x += 1`` performs multiple bytecode operations and is not
atomic). Atomicity is only guaranteed where explicitly documented. See
also :term:`race condition` and :term:`data race`.
attached thread state
A :term:`thread state` that is active for the current OS thread.
@ -289,6 +297,22 @@ Glossary
advanced mathematical feature. If you're not aware of a need for them,
it's almost certain you can safely ignore them.
concurrency
The ability of a computer program to perform multiple tasks at the same
time. Python provides libraries for writing programs that make use of
different forms of concurrency. :mod:`asyncio` is a library for dealing
with asynchronous tasks and coroutines. :mod:`threading` provides
access to operating system threads and :mod:`multiprocessing` to
operating system processes. Multi-core processors can execute threads and
processes on different CPU cores at the same time (see
:term:`parallelism`).
concurrent modification
When multiple threads modify shared data at the same time. Concurrent
modification without proper synchronization can cause
:term:`race conditions <race condition>`, and might also trigger a
:term:`data race <data race>`, data corruption, or both.
context
This term has different meanings depending on where and how it is used.
Some common meanings:
@ -363,6 +387,28 @@ Glossary
the :term:`cyclic garbage collector <garbage collection>` is to identify these groups and break the reference
cycles so that the memory can be reclaimed.
data race
A situation where multiple threads access the same memory location
concurrently, at least one of the accesses is a write, and the threads
do not use any synchronization to control their access. Data races
lead to :term:`non-deterministic` behavior and can cause data corruption.
Proper use of :term:`locks <lock>` and other :term:`synchronization primitives
<synchronization primitive>` prevents data races. Note that data races
can only happen in native code, but that :term:`native code` might be
exposed in a Python API. See also :term:`race condition` and
:term:`thread-safe`.
deadlock
A situation in which two or more tasks (threads, processes, or coroutines)
wait indefinitely for each other to release resources or complete actions,
preventing any from making progress. For example, if thread A holds lock
1 and waits for lock 2, while thread B holds lock 2 and waits for lock 1,
both threads will wait indefinitely. In Python this often arises from
acquiring multiple locks in conflicting orders or from circular
join/await dependencies. Deadlocks can be avoided by always acquiring
multiple :term:`locks <lock>` in a consistent order. See also
:term:`lock` and :term:`reentrant`.
decorator
A function returning another function, usually applied as a function
transformation using the ``@wrapper`` syntax. Common examples for
@ -662,6 +708,14 @@ Glossary
requires the GIL to be held in order to use it. This refers to having an
:term:`attached thread state`.
global state
Data that is accessible throughout a program, such as module-level
variables, class variables, or C static variables in :term:`extension modules
<extension module>`. In multi-threaded programs, global state shared
between threads typically requires synchronization to avoid
:term:`race conditions <race condition>` and
:term:`data races <data race>`.
hash-based pyc
A bytecode cache file that uses the hash rather than the last-modified
time of the corresponding source file to determine its validity. See
@ -706,7 +760,9 @@ Glossary
tuples. Such an object cannot be altered. A new object has to
be created if a different value has to be stored. They play an important
role in places where a constant hash value is needed, for example as a key
in a dictionary.
in a dictionary. Immutable objects are inherently :term:`thread-safe`
because their state cannot be modified after creation, eliminating concerns
about improperly synchronized :term:`concurrent modification`.
import path
A list of locations (or :term:`path entries <path entry>`) that are
@ -796,8 +852,9 @@ Glossary
CPython does not consistently apply the requirement that an iterator
define :meth:`~iterator.__iter__`.
And also please note that the free-threading CPython does not guarantee
the thread-safety of iterator operations.
And also please note that :term:`free-threaded <free threading>`
CPython does not guarantee :term:`thread-safe` behavior of iterator
operations.
key function
@ -813,7 +870,7 @@ Glossary
:func:`itertools.groupby`.
There are several ways to create a key function. For example. the
:meth:`str.lower` method can serve as a key function for case insensitive
:meth:`str.casefold` method can serve as a key function for case insensitive
sorts. Alternatively, a key function can be built from a
:keyword:`lambda` expression such as ``lambda r: (r[0], r[2])``. Also,
:func:`operator.attrgetter`, :func:`operator.itemgetter`, and
@ -835,10 +892,11 @@ Glossary
:keyword:`if` statements.
In a multi-threaded environment, the LBYL approach can risk introducing a
race condition between "the looking" and "the leaping". For example, the
code, ``if key in mapping: return mapping[key]`` can fail if another
:term:`race condition` between "the looking" and "the leaping". For example,
the code, ``if key in mapping: return mapping[key]`` can fail if another
thread removes *key* from *mapping* after the test, but before the lookup.
This issue can be solved with locks or by using the EAFP approach.
This issue can be solved with :term:`locks <lock>` or by using the
:term:`EAFP` approach. See also :term:`thread-safe`.
lexical analyzer
@ -857,6 +915,19 @@ Glossary
clause is optional. If omitted, all elements in ``range(256)`` are
processed.
lock
A :term:`synchronization primitive` that allows only one thread at a
time to access a shared resource. A thread must acquire a lock before
accessing the protected resource and release it afterward. If a thread
attempts to acquire a lock that is already held by another thread, it
will block until the lock becomes available. Python's :mod:`threading`
module provides :class:`~threading.Lock` (a basic lock) and
:class:`~threading.RLock` (a :term:`reentrant` lock). Locks are used
to prevent :term:`race conditions <race condition>` and ensure
:term:`thread-safe` access to shared data. Alternative design patterns
to locks exist such as queues, producer/consumer patterns, and
thread-local state. See also :term:`deadlock`, and :term:`reentrant`.
loader
An object that loads a module.
It must define the :meth:`!exec_module` and :meth:`!create_module` methods
@ -942,8 +1013,11 @@ Glossary
See :term:`method resolution order`.
mutable
Mutable objects can change their value but keep their :func:`id`. See
also :term:`immutable`.
An :term:`object` with state that is allowed to change during the course
of the program. In multi-threaded programs, mutable objects that are
shared between threads require careful synchronization to avoid
:term:`race conditions <race condition>`. See also :term:`immutable`,
:term:`thread-safe`, and :term:`concurrent modification`.
named tuple
The term "named tuple" applies to any type or class that inherits from
@ -995,6 +1069,13 @@ Glossary
See also :term:`module`.
native code
Code that is compiled to machine instructions and runs directly on the
processor, as opposed to code that is interpreted or runs in a virtual
machine. In the context of Python, native code typically refers to
C, C++, Rust or Fortran code in :term:`extension modules <extension module>`
that can be called from Python. See also :term:`extension module`.
nested scope
The ability to refer to a variable in an enclosing definition. For
instance, a function defined inside another function can refer to
@ -1011,6 +1092,15 @@ Glossary
properties, :meth:`~object.__getattribute__`, class methods, and static
methods.
non-deterministic
Behavior where the outcome of a program can vary between executions with
the same inputs. In multi-threaded programs, non-deterministic behavior
often results from :term:`race conditions <race condition>` where the
relative timing or interleaving of threads affects the result.
Proper synchronization using :term:`locks <lock>` and other
:term:`synchronization primitives <synchronization primitive>` helps
ensure deterministic behavior.
object
Any data with state (attributes or value) and defined behavior
(methods). Also the ultimate base class of any :term:`new-style
@ -1041,6 +1131,16 @@ Glossary
See also :term:`regular package` and :term:`namespace package`.
parallelism
Executing multiple operations at the same time (e.g. on multiple CPU
cores). In Python builds with the
:term:`global interpreter lock (GIL) <global interpreter lock>`, only one
thread runs Python bytecode at a time, so taking advantage of multiple
CPU cores typically involves multiple processes
(e.g. :mod:`multiprocessing`) or native extensions that release the GIL.
In :term:`free-threaded <free threading>` Python, multiple Python threads
can run Python code simultaneously on different cores.
parameter
A named entity in a :term:`function` (or method) definition that
specifies an :term:`argument` (or in some cases, arguments) that the
@ -1215,6 +1315,18 @@ Glossary
>>> email.mime.text.__name__
'email.mime.text'
race condition
A condition of a program where the its behavior
depends on the relative timing or ordering of events, particularly in
multi-threaded programs. Race conditions can lead to
:term:`non-deterministic` behavior and bugs that are difficult to
reproduce. A :term:`data race` is a specific type of race condition
involving unsynchronized access to shared memory. The :term:`LBYL`
coding style is particularly susceptible to race conditions in
multi-threaded code. Using :term:`locks <lock>` and other
:term:`synchronization primitives <synchronization primitive>`
helps prevent race conditions.
reference count
The number of references to an object. When the reference count of an
object drops to zero, it is deallocated. Some objects are
@ -1236,6 +1348,25 @@ Glossary
See also :term:`namespace package`.
reentrant
A property of a function or :term:`lock` that allows it to be called or
acquired multiple times by the same thread without causing errors or a
:term:`deadlock`.
For functions, reentrancy means the function can be safely called again
before a previous invocation has completed, which is important when
functions may be called recursively or from signal handlers. Thread-unsafe
functions may be :term:`non-deterministic` if they're called reentrantly in a
multithreaded program.
For locks, Python's :class:`threading.RLock` (reentrant lock) is
reentrant, meaning a thread that already holds the lock can acquire it
again without blocking. In contrast, :class:`threading.Lock` is not
reentrant - attempting to acquire it twice from the same thread will cause
a deadlock.
See also :term:`lock` and :term:`deadlock`.
REPL
An acronym for the "read–eval–print loop", another name for the
:term:`interactive` interpreter shell.
@ -1340,6 +1471,18 @@ Glossary
See also :term:`borrowed reference`.
synchronization primitive
A basic building block for coordinating (synchronizing) the execution of
multiple threads to ensure :term:`thread-safe` access to shared resources.
Python's :mod:`threading` module provides several synchronization primitives
including :class:`~threading.Lock`, :class:`~threading.RLock`,
:class:`~threading.Semaphore`, :class:`~threading.Condition`,
:class:`~threading.Event`, and :class:`~threading.Barrier`. Additionally,
the :mod:`queue` module provides multi-producer, multi-consumer queues
that are especially useful in multithreaded programs. These
primitives help prevent :term:`race conditions <race condition>` and
coordinate thread execution. See also :term:`lock`.
t-string
t-strings
String literals prefixed with ``t`` or ``T`` are commonly called
@ -1392,6 +1535,19 @@ Glossary
See :ref:`Thread State and the Global Interpreter Lock <threads>` for more
information.
thread-safe
A module, function, or class that behaves correctly when used by multiple
threads concurrently. Thread-safe code uses appropriate
:term:`synchronization primitives <synchronization primitive>` like
:term:`locks <lock>` to protect shared mutable state, or is designed
to avoid shared mutable state entirely. In the
:term:`free-threaded <free threading>` build, built-in types like
:class:`dict`, :class:`list`, and :class:`set` use internal locking
to make many operations thread-safe, although thread safety is not
necessarily guaranteed. Code that is not thread-safe may experience
:term:`race conditions <race condition>` and :term:`data races <data race>`
when used in multi-threaded programs.
token
A small unit of source code, generated by the

2
Doc/howto/gdb_helpers.rst
View file

@ -136,7 +136,7 @@ enabled::
at Objects/unicodeobject.c:551
#7 0x0000000000440d94 in PyUnicodeUCS2_FromString (u=0x5c2b8d "__lltrace__") at Objects/unicodeobject.c:569
#8 0x0000000000584abd in PyDict_GetItemString (v=
{'Yuck': <type at remote 0xad4730>, '__builtins__': <module at remote 0x7ffff7fd5ee8>, '__file__': 'Lib/test/crashers/nasty_eq_vs_dict.py', '__package__': None, 'y': <Yuck(i=0) at remote 0xaacd80>, 'dict': {0: 0, 1: 1, 2: 2, 3: 3}, '__cached__': None, '__name__': '__main__', 'z': <Yuck(i=0) at remote 0xaace60>, '__doc__': None}, key=
{'Yuck': <type at remote 0xad4730>, '__builtins__': <module at remote 0x7ffff7fd5ee8>, '__file__': 'Lib/test/crashers/nasty_eq_vs_dict.py', '__package__': None, 'y': <Yuck(i=0) at remote 0xaacd80>, 'dict': {0: 0, 1: 1, 2: 2, 3: 3}, '__name__': '__main__', 'z': <Yuck(i=0) at remote 0xaace60>, '__doc__': None}, key=
0x5c2b8d "__lltrace__") at Objects/dictobject.c:2171
Notice how the dictionary argument to ``PyDict_GetItemString`` is displayed

55
Doc/includes/capi-extension/spammodule-01.c Normal file
View file

@ -0,0 +1,55 @@
/* This file needs to be kept in sync with the tutorial
* at Doc/extending/first-extension-module.rst
*/
/// Includes
#include <Python.h>
#include <stdlib.h> // for system()
/// Implementation of spam.system
static PyObject *
spam_system(PyObject *self, PyObject *arg)
{
const char *command = PyUnicode_AsUTF8(arg);
if (command == NULL) {
return NULL;
}
int status = system(command);
PyObject *result = PyLong_FromLong(status);
return result;
}
/// Module method table
static PyMethodDef spam_methods[] = {
{
.ml_name="system",
.ml_meth=spam_system,
.ml_flags=METH_O,
.ml_doc="Execute a shell command.",
},
{NULL, NULL, 0, NULL} /* Sentinel */
};
/// Module slot table
static PyModuleDef_Slot spam_slots[] = {
{Py_mod_name, "spam"},
{Py_mod_doc, "A wonderful module with an example function"},
{Py_mod_methods, spam_methods},
{0, NULL}
};
/// Export hook prototype
PyMODEXPORT_FUNC PyModExport_spam(void);
/// Module export hook
PyMODEXPORT_FUNC
PyModExport_spam(void)
{
return spam_slots;
}

31
Doc/library/argparse.rst
View file

@ -645,6 +645,27 @@ are set.
.. versionadded:: 3.14
To highlight inline code in your description or epilog text, you can use
backticks::
>>> parser = argparse.ArgumentParser(
... formatter_class=argparse.RawDescriptionHelpFormatter,
... epilog='''Examples:
... `python -m myapp --verbose`
... `python -m myapp --config settings.json`
... ''')
When colors are enabled, the text inside backticks will be displayed in a
distinct color to help examples stand out. When colors are disabled, backticks
are preserved as-is, which is readable in plain text.
.. note::
Backtick markup only applies to description and epilog text. It does not
apply to individual argument ``help`` strings.
.. versionadded:: 3.15
The add_argument() method
-------------------------
@ -1349,7 +1370,7 @@ behavior::
>>> parser.parse_args('--foo XXX'.split())
Namespace(bar='XXX')
.. versionchanged:: next
.. versionchanged:: 3.15
Single-dash long option now takes precedence over short options.
@ -1452,7 +1473,7 @@ this API may be passed as the ``action`` parameter to
.. versionadded:: 3.9
.. versionchanged:: next
.. versionchanged:: 3.15
Added support for single-dash options.
Added support for alternate prefix_chars_.
@ -1679,7 +1700,7 @@ The Namespace object
Other utilities
---------------
Sub-commands
Subcommands
^^^^^^^^^^^^
.. method:: ArgumentParser.add_subparsers(*, [title], [description], [prog], \
@ -1708,7 +1729,7 @@ Sub-commands
* *description* - description for the sub-parser group in help output, by
default ``None``
* *prog* - usage information that will be displayed with sub-command help,
* *prog* - usage information that will be displayed with subcommand help,
by default the name of the program and any positional arguments before the
subparser argument
@ -1718,7 +1739,7 @@ Sub-commands
* action_ - the basic type of action to be taken when this argument is
encountered at the command line
* dest_ - name of the attribute under which sub-command name will be
* dest_ - name of the attribute under which subcommand name will be
stored; by default ``None`` and no value is stored
* required_ - Whether or not a subcommand must be provided, by default

19
Doc/library/ast.rst
View file

@ -139,12 +139,13 @@ Node classes
The :meth:`~object.__repr__` output of :class:`~ast.AST` nodes includes
the values of the node fields.
.. deprecated:: 3.8
.. deprecated-removed:: 3.8 3.14
Old classes :class:`!ast.Num`, :class:`!ast.Str`, :class:`!ast.Bytes`,
:class:`!ast.NameConstant` and :class:`!ast.Ellipsis` are still available,
but they will be removed in future Python releases. In the meantime,
instantiating them will return an instance of a different class.
Previous versions of Python provided the AST classes :class:`!ast.Num`,
:class:`!ast.Str`, :class:`!ast.Bytes`, :class:`!ast.NameConstant` and
:class:`!ast.Ellipsis`, which were deprecated in Python 3.8. These classes
were removed in Python 3.14, and their functionality has been replaced with
:class:`ast.Constant`.
.. deprecated:: 3.9
@ -2419,12 +2420,12 @@ and classes for traversing abstract syntax trees:
during traversal. For this a special visitor exists
(:class:`NodeTransformer`) that allows modifications.
.. deprecated:: 3.8
.. deprecated-removed:: 3.8 3.14
Methods :meth:`!visit_Num`, :meth:`!visit_Str`, :meth:`!visit_Bytes`,
:meth:`!visit_NameConstant` and :meth:`!visit_Ellipsis` are deprecated
now and will not be called in future Python versions. Add the
:meth:`visit_Constant` method to handle all constant nodes.
:meth:`!visit_NameConstant` and :meth:`!visit_Ellipsis` will not be called
in Python 3.14+. Add the :meth:`visit_Constant` method instead to handle
all constant nodes.
.. class:: NodeTransformer()

2
Doc/library/asyncio-queue.rst
View file

@ -107,7 +107,7 @@ Queue
The queue can no longer grow.
Future calls to :meth:`~Queue.put` raise :exc:`QueueShutDown`.
Currently blocked callers of :meth:`~Queue.put` will be unblocked
and will raise :exc:`QueueShutDown` in the formerly blocked thread.
and will raise :exc:`QueueShutDown` in the formerly awaiting task.
If *immediate* is false (the default), the queue can be wound
down normally with :meth:`~Queue.get` calls to extract tasks

5
Doc/library/calendar.rst
View file

@ -158,6 +158,11 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is
:class:`TextCalendar` instances have the following methods:
.. method:: prweek(theweek, width)
Print a week's calendar as returned by :meth:`formatweek` and without a
final newline.
.. method:: formatday(theday, weekday, width)

7
Doc/library/cmdline.rst
View file

@ -12,7 +12,7 @@ The following modules have a command-line interface.
* :ref:`calendar <calendar-cli>`
* :mod:`code`
* :ref:`compileall <compileall-cli>`
* :mod:`cProfile`: see :ref:`profile <profile-cli>`
* ``cProfile``: see :ref:`profiling.tracing <profiling-tracing-cli>`
* :ref:`dis <dis-cli>`
* :ref:`doctest <doctest-cli>`
* :mod:`!encodings.rot_13`
@ -31,8 +31,9 @@ The following modules have a command-line interface.
* :ref:`pickletools <pickletools-cli>`
* :ref:`platform <platform-cli>`
* :mod:`poplib`
* :ref:`profile <profile-cli>`
* :mod:`pstats`
* :ref:`profiling.sampling <profiling-sampling>`
* :ref:`profiling.tracing <profiling-tracing-cli>`
* :ref:`pstats <pstats-cli>`
* :ref:`py_compile <py_compile-cli>`
* :mod:`pyclbr`
* :mod:`pydoc`

5
Doc/library/ctypes.rst
View file

@ -1388,6 +1388,9 @@ On Linux, :func:`~ctypes.util.find_library` tries to run external programs
(``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the library file.
It returns the filename of the library file.
Note that if the output of these programs does not correspond to the dynamic
linker used by Python, the result of this function may be misleading.
.. versionchanged:: 3.6
On Linux, the value of the environment variable ``LD_LIBRARY_PATH`` is used
when searching for libraries, if a library cannot be found by any other means.
@ -2132,6 +2135,8 @@ Utility functions
The exact functionality is system dependent.
See :ref:`ctypes-finding-shared-libraries` for complete documentation.
.. function:: find_msvcrt()
:module: ctypes.util

43
Doc/library/datetime.rst
View file

@ -2651,9 +2651,42 @@ Broadly speaking, ``d.strftime(fmt)`` acts like the :mod:`time` module's
``time.strftime(fmt, d.timetuple())`` although not all objects support a
:meth:`~date.timetuple` method.
For the :meth:`.datetime.strptime` class method, the default value is
``1900-01-01T00:00:00.000``: any components not specified in the format string
will be pulled from the default value. [#]_
For the :meth:`.datetime.strptime` and :meth:`.date.strptime` class methods,
the default value is ``1900-01-01T00:00:00.000``: any components not specified
in the format string will be pulled from the default value.
.. note::
When used to parse partial dates lacking a year, :meth:`.datetime.strptime`
and :meth:`.date.strptime` will raise when encountering February 29 because
the default year of 1900 is *not* a leap year. Always add a default leap
year to partial date strings before parsing.
.. testsetup::
# doctest seems to turn the warning into an error which makes it
# show up and require matching and prevents the actual interesting
# exception from being raised.
# Manually apply the catch_warnings context manager
import warnings
catch_warnings = warnings.catch_warnings()
catch_warnings.__enter__()
warnings.simplefilter("ignore")
.. testcleanup::
catch_warnings.__exit__()
.. doctest::
>>> from datetime import datetime
>>> value = "2/29"
>>> datetime.strptime(value, "%m/%d")
Traceback (most recent call last):
...
ValueError: day 29 must be in range 1..28 for month 2 in year 1900
>>> datetime.strptime(f"1904 {value}", "%Y %m/%d")
datetime.datetime(1904, 2, 29, 0, 0)
Using ``datetime.strptime(date_string, format)`` is equivalent to::
@ -2790,7 +2823,7 @@ Notes:
include a year in the format. If the value you need to parse lacks a year,
append an explicit dummy leap year. Otherwise your code will raise an
exception when it encounters leap day because the default year used by the
parser is not a leap year. Users run into this bug every four years...
parser (1900) is not a leap year. Users run into that bug every leap year.
.. doctest::
@ -2817,5 +2850,3 @@ Notes:
.. [#] See R. H. van Gent's `guide to the mathematics of the ISO 8601 calendar
<https://web.archive.org/web/20220531051136/https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm>`_
for a good explanation.
.. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since 1900 is not a leap year.

5
Doc/library/debug.rst
View file

@ -1,5 +1,5 @@
***********************
Debugging and Profiling
Debugging and profiling
***********************
These libraries help you with Python development: the debugger enables you to
@ -15,7 +15,8 @@ intrusive debugging or patching.
bdb.rst
faulthandler.rst
pdb.rst
profile.rst
profiling.rst
pstats.rst
timeit.rst
trace.rst
tracemalloc.rst

30
Doc/library/decimal.rst
View file

@ -34,10 +34,12 @@ The :mod:`decimal` module provides support for fast correctly rounded
decimal floating-point arithmetic. It offers several advantages over the
:class:`float` datatype:
* Decimal "is based on a floating-point model which was designed with people
in mind, and necessarily has a paramount guiding principle -- computers must
provide an arithmetic that works in the same way as the arithmetic that
people learn at school." -- excerpt from the decimal arithmetic specification.
* Decimal "is based on a `floating-point model
<https://speleotrove.com/decimal/damodel.html#refnumber>`__ which was designed
with people in mind, and necessarily has a paramount guiding principle --
computers must provide an arithmetic that works in the same way as the
arithmetic that people learn at school." -- excerpt from the decimal
arithmetic specification.
* Decimal numbers can be represented exactly. In contrast, numbers like
``1.1`` and ``2.2`` do not have exact representations in binary
@ -238,6 +240,26 @@ floating-point flying circus:
>>> c % a
Decimal('0.77')
Decimals can be formatted (with :func:`format` built-in or :ref:`f-strings`) in
fixed-point or scientific notation, using the same formatting syntax (see
:ref:`formatspec`) as builtin :class:`float` type:
.. doctest::
>>> format(Decimal('2.675'), "f")
'2.675'
>>> format(Decimal('2.675'), ".2f")
'2.68'
>>> f"{Decimal('2.675'):.2f}"
'2.68'
>>> format(Decimal('2.675'), ".2e")
'2.68e+0'
>>> with localcontext() as ctx:
... ctx.rounding = ROUND_DOWN
... print(format(Decimal('2.675'), ".2f"))
...
2.67
And some mathematical functions are also available to Decimal:
>>> getcontext().prec = 28

5
Doc/library/enum.rst
View file

@ -947,12 +947,13 @@ Utilities and Decorators
the member's name. Care must be taken if mixing *auto()* with manually
specified values.
*auto* instances are only resolved when at the top level of an assignment:
*auto* instances are only resolved when at the top level of an assignment, either by
itself or as part of a tuple:
* ``FIRST = auto()`` will work (auto() is replaced with ``1``);
* ``SECOND = auto(), -2`` will work (auto is replaced with ``2``, so ``2, -2`` is
used to create the ``SECOND`` enum member;
* ``THREE = [auto(), -3]`` will *not* work (``<auto instance>, -3`` is used to
* ``THREE = [auto(), -3]`` will *not* work (``[<auto instance>, -3]`` is used to
create the ``THREE`` enum member)
.. versionchanged:: 3.11.1

10
Doc/library/exceptions.rst
View file

@ -742,8 +742,8 @@ depending on the system error code.
.. attribute:: characters_written
An integer containing the number of characters written to the stream
before it blocked. This attribute is available when using the
An integer containing the number of **bytes** written to the stream
before it blocked. This attribute is available when using the
buffered I/O classes from the :mod:`io` module.
.. exception:: ChildProcessError
@ -978,6 +978,12 @@ their subgroups based on the types of the contained exceptions.
raises a :exc:`TypeError` if any contained exception is not an
:exc:`Exception` subclass.
.. impl-detail::
The ``excs`` parameter may be any sequence, but lists and tuples are
specifically processed more efficiently here. For optimal performance,
pass a tuple as ``excs``.
.. attribute:: message
The ``msg`` argument to the constructor. This is a read-only attribute.

12
Doc/library/functions.rst
View file

@ -340,8 +340,8 @@ are always available. They are listed here in alphabetical order.
It is needed to unambiguous :ref:`filter <warning-filter>` syntax warnings
by module name.
This function raises :exc:`SyntaxError` if the compiled source is invalid,
and :exc:`ValueError` if the source contains null bytes.
This function raises :exc:`SyntaxError` or :exc:`ValueError` if the compiled
source is invalid.
If you want to parse Python code into its AST representation, see
:func:`ast.parse`.
@ -526,7 +526,7 @@ are always available. They are listed here in alphabetical order.
>>> dir() # show the names in the module namespace # doctest: +SKIP
['__builtins__', '__name__', 'struct']
>>> dir(struct) # show the names in the struct module # doctest: +SKIP
['Struct', '__all__', '__builtins__', '__cached__', '__doc__', '__file__',
['Struct', '__all__', '__builtins__', '__doc__', '__file__',
'__initializing__', '__loader__', '__name__', '__package__',
'_clearcache', 'calcsize', 'error', 'pack', 'pack_into',
'unpack', 'unpack_from']
@ -606,16 +606,16 @@ are always available. They are listed here in alphabetical order.
This function executes arbitrary code. Calling it with
user-supplied input may lead to security vulnerabilities.
The *expression* argument is parsed and evaluated as a Python expression
The *source* argument is parsed and evaluated as a Python expression
(technically speaking, a condition list) using the *globals* and *locals*
mappings as global and local namespace. If the *globals* dictionary is
present and does not contain a value for the key ``__builtins__``, a
reference to the dictionary of the built-in module :mod:`builtins` is
inserted under that key before *expression* is parsed. That way you can
inserted under that key before *source* is parsed. That way you can
control what builtins are available to the executed code by inserting your
own ``__builtins__`` dictionary into *globals* before passing it to
:func:`eval`. If the *locals* mapping is omitted it defaults to the
*globals* dictionary. If both mappings are omitted, the expression is
*globals* dictionary. If both mappings are omitted, the source is
executed with the *globals* and *locals* in the environment where
:func:`eval` is called. Note, *eval()* will only have access to the
:term:`nested scopes <nested scope>` (non-locals) in the enclosing

4
Doc/library/gc.rst
View file

@ -118,7 +118,7 @@ The :mod:`gc` module provides the following functions:
.. versionadded:: 3.4
.. versionchanged:: next
.. versionchanged:: 3.15
Add ``duration`` and ``candidates``.
@ -340,7 +340,7 @@ values but should not rebind them):
.. versionadded:: 3.3
.. versionchanged:: next
.. versionchanged:: 3.15
Add "duration" and "candidates".

15
Doc/library/http.client.rst
View file

@ -319,6 +319,12 @@ HTTPConnection Objects
:class:`str` or bytes-like object that is not also a file as the
body representation.
.. note::
Note that you must have read the whole response or call :meth:`close`
if :meth:`getresponse` raised an non-:exc:`ConnectionError` exception
before you can send a new request to the server.
.. versionchanged:: 3.2
*body* can now be an iterable.
@ -334,16 +340,15 @@ HTTPConnection Objects
Should be called after a request is sent to get the response from the server.
Returns an :class:`HTTPResponse` instance.
.. note::
Note that you must have read the whole response before you can send a new
request to the server.
.. versionchanged:: 3.5
If a :exc:`ConnectionError` or subclass is raised, the
:class:`HTTPConnection` object will be ready to reconnect when
a new request is sent.
Note that this does not apply to :exc:`OSError`\s raised by the underlying
socket. Instead the caller is responsible to call :meth:`close` on the
existing connection.
.. method:: HTTPConnection.set_debuglevel(level)

115
Doc/library/importlib.rst
View file

@ -210,6 +210,12 @@ Functions
:exc:`ModuleNotFoundError` is raised when the module being reloaded lacks
a :class:`~importlib.machinery.ModuleSpec`.
.. versionchanged:: 3.15
If *module* is a lazy module that has not yet been materialized (i.e.,
loaded via :class:`importlib.util.LazyLoader` and not yet accessed),
calling :func:`reload` is a no-op and returns the module unchanged.
This prevents the reload from unintentionally triggering the lazy load.
.. warning::
This function is not thread-safe. Calling it from multiple threads can result
in unexpected behavior. It's recommended to use the :class:`threading.Lock`
@ -320,6 +326,9 @@ ABC hierarchy::
.. versionchanged:: 3.7
Introduced the optional :meth:`get_resource_reader` method.
.. versionchanged:: 3.15
Removed the ``load_module()`` method.
.. method:: create_module(spec)
A method that returns the module object to use when
@ -344,47 +353,6 @@ ABC hierarchy::
.. versionchanged:: 3.6
:meth:`create_module` must also be defined.
.. method:: load_module(fullname)
A legacy method for loading a module. If the module cannot be
loaded, :exc:`ImportError` is raised, otherwise the loaded module is
returned.
If the requested module already exists in :data:`sys.modules`, that
module should be used and reloaded.
Otherwise the loader should create a new module and insert it into
:data:`sys.modules` before any loading begins, to prevent recursion
from the import. If the loader inserted a module and the load fails, it
must be removed by the loader from :data:`sys.modules`; modules already
in :data:`sys.modules` before the loader began execution should be left
alone.
The loader should set several attributes on the module
(note that some of these attributes can change when a module is
reloaded):
- :attr:`module.__name__`
- :attr:`module.__file__`
- :attr:`module.__cached__` *(deprecated)*
- :attr:`module.__path__`
- :attr:`module.__package__` *(deprecated)*
- :attr:`module.__loader__` *(deprecated)*
When :meth:`exec_module` is available then backwards-compatible
functionality is provided.
.. versionchanged:: 3.4
Raise :exc:`ImportError` when called instead of
:exc:`NotImplementedError`. Functionality provided when
:meth:`exec_module` is available.
.. deprecated-removed:: 3.4 3.15
The recommended API for loading a module is :meth:`exec_module`
(and :meth:`create_module`). Loaders should implement it instead of
:meth:`load_module`. The import machinery takes care of all the
other responsibilities of :meth:`load_module` when
:meth:`exec_module` is implemented.
.. class:: ResourceLoader
@ -490,13 +458,6 @@ ABC hierarchy::
.. versionadded:: 3.4
.. method:: load_module(fullname)
Implementation of :meth:`Loader.load_module`.
.. deprecated-removed:: 3.4 3.15
use :meth:`exec_module` instead.
.. class:: ExecutionLoader
@ -530,6 +491,9 @@ ABC hierarchy::
.. versionadded:: 3.3
.. versionchanged:: 3.15
Removed the ``load_module()`` method.
.. attribute:: name
The name of the module the loader can handle.
@ -538,13 +502,6 @@ ABC hierarchy::
Path to the file of the module.
.. method:: load_module(fullname)
Calls super's ``load_module()``.
.. deprecated-removed:: 3.4 3.15
Use :meth:`Loader.exec_module` instead.
.. method:: get_filename(fullname)
:abstractmethod:
@ -576,6 +533,9 @@ ABC hierarchy::
optimization to speed up loading by removing the parsing step of Python's
compiler, and so no bytecode-specific API is exposed.
.. versionchanged:: 3.15
Removed the ``load_module()`` method.
.. method:: path_stats(path)
Optional abstract method which returns a :class:`dict` containing
@ -629,13 +589,6 @@ ABC hierarchy::
.. versionadded:: 3.4
.. method:: load_module(fullname)
Concrete implementation of :meth:`Loader.load_module`.
.. deprecated-removed:: 3.4 3.15
Use :meth:`exec_module` instead.
.. method:: get_source(fullname)
Concrete implementation of :meth:`InspectLoader.get_source`.
@ -1059,6 +1012,9 @@ find and load modules.
.. versionadded:: 3.3
.. versionchanged:: 3.15
Removed the ``load_module()`` method.
.. attribute:: name
The name of the module that this loader will handle.
@ -1079,15 +1035,6 @@ find and load modules.
Concrete implementation of :meth:`importlib.abc.SourceLoader.set_data`.
.. method:: load_module(name=None)
Concrete implementation of :meth:`importlib.abc.Loader.load_module` where
specifying the name of the module to load is optional.
.. deprecated-removed:: 3.6 3.15
Use :meth:`importlib.abc.Loader.exec_module` instead.
.. class:: SourcelessFileLoader(fullname, path)
@ -1101,6 +1048,9 @@ find and load modules.
.. versionadded:: 3.3
.. versionchanged:: 3.15
Removed the ``load_module()`` method.
.. attribute:: name
The name of the module the loader will handle.
@ -1122,15 +1072,6 @@ find and load modules.
Returns ``None`` as bytecode files have no source when this loader is
used.
.. method:: load_module(name=None)
Concrete implementation of :meth:`importlib.abc.Loader.load_module` where
specifying the name of the module to load is optional.
.. deprecated-removed:: 3.6 3.15
Use :meth:`importlib.abc.Loader.exec_module` instead.
.. class:: ExtensionFileLoader(fullname, path)
@ -1262,8 +1203,7 @@ find and load modules.
.. attribute:: cached
The filename of a compiled version of the module's code
(see :attr:`module.__cached__`).
The filename of a compiled version of the module's code.
The :term:`finder` should always set this attribute but it may be ``None``
for modules that do not need compiled code stored.
@ -1365,7 +1305,7 @@ an :term:`importer`.
.. versionadded:: 3.4
.. function:: cache_from_source(path, debug_override=None, *, optimization=None)
.. function:: cache_from_source(path, *, optimization=None)
Return the :pep:`3147`/:pep:`488` path to the byte-compiled file associated
with the source *path*. For example, if *path* is ``/foo/bar/baz.py`` the return
@ -1384,12 +1324,6 @@ an :term:`importer`.
``/foo/bar/__pycache__/baz.cpython-32.opt-2.pyc``. The string representation
of *optimization* can only be alphanumeric, else :exc:`ValueError` is raised.
The *debug_override* parameter is deprecated and can be used to override
the system's value for ``__debug__``. A ``True`` value is the equivalent of
setting *optimization* to the empty string. A ``False`` value is the same as
setting *optimization* to ``1``. If both *debug_override* an *optimization*
are not ``None`` then :exc:`TypeError` is raised.
.. versionadded:: 3.4
.. versionchanged:: 3.5
@ -1399,6 +1333,9 @@ an :term:`importer`.
.. versionchanged:: 3.6
Accepts a :term:`path-like object`.
.. versionchanged:: 3.15
The *debug_override* parameter was removed.
.. function:: source_from_cache(path)

2
Doc/library/inspect.rst
View file

@ -640,7 +640,7 @@ Retrieving source code
Added parameters *inherit_class_doc* and *fallback_to_class_doc*.
Documentation strings on :class:`~functools.cached_property`
objects are now inherited if not overriden.
objects are now inherited if not overridden.
.. function:: getcomments(object)

25
Doc/library/locale.rst
View file

@ -48,8 +48,19 @@ The :mod:`locale` module defines the following exception and functions:
If *locale* is omitted or ``None``, the current setting for *category* is
returned.
Example::
>>> import locale
>>> loc = locale.setlocale(locale.LC_ALL) # get current locale
# use German locale; name and availability varies with platform
>>> locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8')
>>> locale.strcoll('f\xe4n', 'foo') # compare a string containing an umlaut
>>> locale.setlocale(locale.LC_ALL, '') # use user's preferred locale
>>> locale.setlocale(locale.LC_ALL, 'C') # use default (C) locale
>>> locale.setlocale(locale.LC_ALL, loc) # restore saved locale
:func:`setlocale` is not thread-safe on most systems. Applications typically
start with a call of ::
start with a call of::
import locale
locale.setlocale(locale.LC_ALL, '')
@ -580,18 +591,6 @@ The :mod:`locale` module defines the following exception and functions:
:func:`localeconv`.
Example::
>>> import locale
>>> loc = locale.getlocale() # get current locale
# use German locale; name might vary with platform
>>> locale.setlocale(locale.LC_ALL, 'de_DE')
>>> locale.strcoll('f\xe4n', 'foo') # compare a string containing an umlaut
>>> locale.setlocale(locale.LC_ALL, '') # use user's preferred locale
>>> locale.setlocale(locale.LC_ALL, 'C') # use default (C) locale
>>> locale.setlocale(locale.LC_ALL, loc) # restore saved locale
Background, details, hints, tips and caveats
--------------------------------------------

11
Doc/library/mmap.rst
View file

@ -328,6 +328,17 @@ To map anonymous memory, -1 should be passed as the fileno along with the length
.. versionadded:: 3.13
.. method:: set_name(name, /)
Annotate the memory mapping with the given *name* for easier identification
in ``/proc/<pid>/maps`` if the kernel supports the feature and :option:`-X dev <-X>` is passed
to Python or if Python is built in :ref:`debug mode <debug-build>`.
The length of *name* must not exceed 67 bytes including the ``'\0'`` terminator.
.. availability:: Linux >= 5.17 (kernel built with ``CONFIG_ANON_VMA_NAME`` option)
.. versionadded:: next
.. method:: size()
Return the length of the file, which can be larger than the size of the

86
Doc/library/multiprocessing.rst
View file

@ -521,6 +521,21 @@ Reference
The :mod:`multiprocessing` package mostly replicates the API of the
:mod:`threading` module.
.. _global-start-method:
Global start method
^^^^^^^^^^^^^^^^^^^
Python supports several ways to create and initialize a process.
The global start method sets the default mechanism for creating a process.
Several multiprocessing functions and methods that may also instantiate
certain objects will implicitly set the global start method to the system's default,
if it hasn’t been set already. The global start method can only be set once.
If you need to change the start method from the system default, you must
proactively set the global start method before calling functions or methods,
or creating these objects.
:class:`Process` and exceptions
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@ -910,6 +925,9 @@ For an example of the usage of queues for interprocess communication see
locks/semaphores. When a process first puts an item on the queue a feeder
thread is started which transfers objects from a buffer into the pipe.
Instantiating this class may set the global start method. See
:ref:`global-start-method` for more details.
The usual :exc:`queue.Empty` and :exc:`queue.Full` exceptions from the
standard library's :mod:`queue` module are raised to signal timeouts.
@ -1025,6 +1043,9 @@ For an example of the usage of queues for interprocess communication see
It is a simplified :class:`Queue` type, very close to a locked :class:`Pipe`.
Instantiating this class may set the global start method. See
:ref:`global-start-method` for more details.
.. method:: close()
Close the queue: release internal resources.
@ -1055,6 +1076,9 @@ For an example of the usage of queues for interprocess communication see
:class:`JoinableQueue`, a :class:`Queue` subclass, is a queue which
additionally has :meth:`task_done` and :meth:`join` methods.
Instantiating this class may set the global start method. See
:ref:`global-start-method` for more details.
.. method:: task_done()
Indicate that a formerly enqueued task is complete. Used by queue
@ -1167,8 +1191,8 @@ Miscellaneous
:mod:`multiprocessing` module.
If *method* is ``None`` then the default context is returned. Note that if
the global start method has not been set, this will set it to the
default method.
the global start method has not been set, this will set it to the system default
See :ref:`global-start-method` for more details.
Otherwise *method* should be ``'fork'``, ``'spawn'``,
``'forkserver'``. :exc:`ValueError` is raised if the specified
start method is not available. See :ref:`multiprocessing-start-methods`.
@ -1179,10 +1203,9 @@ Miscellaneous
Return the name of start method used for starting processes.
If the global start method has not been set and *allow_none* is
``False``, then the start method is set to the default and the name
is returned. If the start method has not been set and *allow_none* is
``True`` then ``None`` is returned.
If the global start method is not set and *allow_none* is ``False``, the global start
method is set to the default, and its name is returned. See
:ref:`global-start-method` for more details.
The return value can be ``'fork'``, ``'spawn'``, ``'forkserver'``
or ``None``. See :ref:`multiprocessing-start-methods`.
@ -1409,6 +1432,9 @@ object -- see :ref:`multiprocessing-managers`.
A barrier object: a clone of :class:`threading.Barrier`.
Instantiating this class may set the global start method. See
:ref:`global-start-method` for more details.
.. versionadded:: 3.3
.. class:: BoundedSemaphore([value])
@ -1416,6 +1442,9 @@ object -- see :ref:`multiprocessing-managers`.
A bounded semaphore object: a close analog of
:class:`threading.BoundedSemaphore`.
Instantiating this class may set the global start method. See
:ref:`global-start-method` for more details.
A solitary difference from its close analog exists: its ``acquire`` method's
first argument is named *block*, as is consistent with :meth:`Lock.acquire`.
@ -1436,6 +1465,9 @@ object -- see :ref:`multiprocessing-managers`.
If *lock* is specified then it should be a :class:`Lock` or :class:`RLock`
object from :mod:`multiprocessing`.
Instantiating this class may set the global start method. See
:ref:`global-start-method` for more details.
.. versionchanged:: 3.3
The :meth:`~threading.Condition.wait_for` method was added.
@ -1443,6 +1475,8 @@ object -- see :ref:`multiprocessing-managers`.
A clone of :class:`threading.Event`.
Instantiating this class may set the global start method. See
:ref:`global-start-method` for more details.
.. class:: Lock()
@ -1458,6 +1492,9 @@ object -- see :ref:`multiprocessing-managers`.
instance of ``multiprocessing.synchronize.Lock`` initialized with a
default context.
Instantiating this class may set the global start method. See
:ref:`global-start-method` for more details.
:class:`Lock` supports the :term:`context manager` protocol and thus may be
used in :keyword:`with` statements.
@ -1515,6 +1552,9 @@ object -- see :ref:`multiprocessing-managers`.
instance of ``multiprocessing.synchronize.RLock`` initialized with a
default context.
Instantiating this class may set the global start method. See
:ref:`global-start-method` for more details.
:class:`RLock` supports the :term:`context manager` protocol and thus may be
used in :keyword:`with` statements.
@ -1574,6 +1614,9 @@ object -- see :ref:`multiprocessing-managers`.
A semaphore object: a close analog of :class:`threading.Semaphore`.
Instantiating this class may set the global start method. See
:ref:`global-start-method` for more details.
A solitary difference from its close analog exists: its ``acquire`` method's
first argument is named *block*, as is consistent with :meth:`Lock.acquire`.
@ -1718,7 +1761,7 @@ processes.
attributes which allow one to use it to store and retrieve strings -- see
documentation for :mod:`ctypes`.
.. function:: Array(typecode_or_type, size_or_initializer, *, lock=True)
.. function:: Array(typecode_or_type, size_or_initializer, *, lock=True, ctx=None)
The same as :func:`RawArray` except that depending on the value of *lock* a
process-safe synchronization wrapper may be returned instead of a raw ctypes
@ -1732,9 +1775,13 @@ processes.
automatically protected by a lock, so it will not necessarily be
"process-safe".
Note that *lock* is a keyword-only argument.
*ctx* is a context object, or ``None`` (use the current context). If ``None``,
calling this may set the global start method. See
:ref:`global-start-method` for more details.
.. function:: Value(typecode_or_type, *args, lock=True)
Note that *lock* and *ctx* are keyword-only parameters.
.. function:: Value(typecode_or_type, *args, lock=True, ctx=None)
The same as :func:`RawValue` except that depending on the value of *lock* a
process-safe synchronization wrapper may be returned instead of a raw ctypes
@ -1747,19 +1794,27 @@ processes.
automatically protected by a lock, so it will not necessarily be
"process-safe".
Note that *lock* is a keyword-only argument.
*ctx* is a context object, or ``None`` (use the current context). If ``None``,
calling this may set the global start method. See
:ref:`global-start-method` for more details.
Note that *lock* and *ctx* are keyword-only parameters.
.. function:: copy(obj)
Return a ctypes object allocated from shared memory which is a copy of the
ctypes object *obj*.
.. function:: synchronized(obj[, lock])
.. function:: synchronized(obj, lock=None, ctx=None)
Return a process-safe wrapper object for a ctypes object which uses *lock* to
synchronize access. If *lock* is ``None`` (the default) then a
:class:`multiprocessing.RLock` object is created automatically.
*ctx* is a context object, or ``None`` (use the current context). If ``None``,
calling this may set the global start method. See
:ref:`global-start-method` for more details.
A synchronized wrapper will have two methods in addition to those of the
object it wraps: :meth:`get_obj` returns the wrapped object and
:meth:`get_lock` returns the lock object used for synchronization.
@ -1877,8 +1932,9 @@ their parent process exits. The manager classes are defined in the
*serializer* must be ``'pickle'`` (use :mod:`pickle` serialization) or
``'xmlrpclib'`` (use :mod:`xmlrpc.client` serialization).
*ctx* is a context object, or ``None`` (use the current context). See the
:func:`get_context` function.
*ctx* is a context object, or ``None`` (use the current context). If ``None``,
calling this may set the global start method. See
:ref:`global-start-method` for more details.
*shutdown_timeout* is a timeout in seconds used to wait until the process
used by the manager completes in the :meth:`shutdown` method. If the
@ -2371,7 +2427,9 @@ with the :class:`Pool` class.
the worker processes. Usually a pool is created using the
function :func:`multiprocessing.Pool` or the :meth:`Pool` method
of a context object. In both cases *context* is set
appropriately.
appropriately. If ``None``, calling this function will have the side effect
of setting the current global start method if it has not been set already.
See the :func:`get_context` function.
Note that the methods of the pool object should only be called by
the process which created the pool.

35
Doc/library/os.path.rst
View file

@ -57,8 +57,9 @@ the :mod:`glob` module.)
.. function:: abspath(path)
Return a normalized absolutized version of the pathname *path*. On most
platforms, this is equivalent to calling the function :func:`normpath` as
follows: ``normpath(join(os.getcwd(), path))``.
platforms, this is equivalent to calling ``normpath(join(os.getcwd(), path))``.
.. seealso:: :func:`os.path.join` and :func:`os.path.normpath`.
.. versionchanged:: 3.6
Accepts a :term:`path-like object`.
@ -243,6 +244,8 @@ the :mod:`glob` module.)
begins with a slash, on Windows that it begins with two (back)slashes, or a
drive letter, colon, and (back)slash together.
.. seealso:: :func:`abspath`
.. versionchanged:: 3.6
Accepts a :term:`path-like object`.
@ -357,14 +360,28 @@ the :mod:`glob` module.)
concatenation of *path* and all members of *\*paths*, with exactly one
directory separator following each non-empty part, except the last. That is,
the result will only end in a separator if the last part is either empty or
ends in a separator. If a segment is an absolute path (which on Windows
requires both a drive and a root), then all previous segments are ignored and
joining continues from the absolute path segment.
ends in a separator.
If a segment is an absolute path (which on Windows requires both a drive and
a root), then all previous segments are ignored and joining continues from the
absolute path segment. On Linux, for example::
>>> os.path.join('/home/foo', 'bar')
'/home/foo/bar'
>>> os.path.join('/home/foo', '/home/bar')
'/home/bar'
On Windows, the drive is not reset when a rooted path segment (e.g.,
``r'\foo'``) is encountered. If a segment is on a different drive or is an
absolute path, all previous segments are ignored and the drive is reset. Note
that since there is a current directory for each drive,
absolute path, all previous segments are ignored and the drive is reset. For
example::
>>> os.path.join('c:\\', 'foo')
'c:\\foo'
>>> os.path.join('c:\\foo', 'd:\\bar')
'd:\\bar'
Note that since there is a current directory for each drive,
``os.path.join("c:", "foo")`` represents a path relative to the current
directory on drive :file:`C:` (:file:`c:foo`), not :file:`c:\\foo`.
@ -527,8 +544,8 @@ the :mod:`glob` module.)
*path* is empty, both *head* and *tail* are empty. Trailing slashes are
stripped from *head* unless it is the root (one or more slashes only). In
all cases, ``join(head, tail)`` returns a path to the same location as *path*
(but the strings may differ). Also see the functions :func:`dirname` and
:func:`basename`.
(but the strings may differ). Also see the functions :func:`join`,
:func:`dirname` and :func:`basename`.
.. versionchanged:: 3.6
Accepts a :term:`path-like object`.

3
Doc/library/pdb.rst
View file

@ -520,7 +520,8 @@ can be overridden by the local file.
To remove all commands from a breakpoint, type ``commands`` and follow it
immediately with ``end``; that is, give no commands.
With no *bpnumber* argument, ``commands`` refers to the last breakpoint set.
With no *bpnumber* argument, ``commands`` refers to the most recently set
breakpoint that still exists.
You can use breakpoint commands to start your program up again. Simply use
the :pdbcmd:`continue` command, or :pdbcmd:`step`,

778
Doc/library/profile.rst
View file

@ -1,458 +1,64 @@
.. _profile:
********************
The Python Profilers
********************
****************************************
:mod:`!profile` --- Pure Python profiler
****************************************
**Source code:** :source:`Lib/profile.py`, :source:`Lib/pstats.py`, and :source:`Lib/profile/sample.py`
.. module:: profile
:synopsis: Pure Python profiler (deprecated).
:deprecated:
**Source code:** :source:`Lib/profile.py`
--------------
.. _profiler-introduction:
.. deprecated-removed:: 3.15 3.17
Introduction to the profilers
=============================
The :mod:`profile` module is deprecated and will be removed in Python 3.17.
Use :mod:`profiling.tracing` instead.
.. index::
single: statistical profiling
single: profiling, statistical
single: deterministic profiling
single: profiling, deterministic
The :mod:`profile` module provides a pure Python implementation of a
deterministic profiler. While useful for understanding profiler internals or
extending profiler behavior through subclassing, its pure Python implementation
introduces significant overhead compared to the C-based :mod:`profiling.tracing`
module.
Python provides both :dfn:`statistical profiling` and :dfn:`deterministic profiling` of
Python programs. A :dfn:`profile` is a set of statistics that describes how
often and for how long various parts of the program executed. These statistics
can be formatted into reports via the :mod:`pstats` module.
For most profiling tasks, use:
The Python standard library provides three different profiling implementations:
- :mod:`profiling.sampling` for production debugging with zero overhead
- :mod:`profiling.tracing` for development and testing
**Statistical Profiler:**
1. :mod:`!profiling.sampling` provides statistical profiling of running Python processes
using periodic stack sampling. It can attach to any running Python process without
requiring code modification or restart, making it ideal for production debugging.
Migration
=========
**Deterministic Profilers:**
Migrating from :mod:`profile` to :mod:`profiling.tracing` is straightforward.
The APIs are compatible::
2. :mod:`cProfile` is recommended for development and testing; it's a C extension with
reasonable overhead that makes it suitable for profiling long-running
programs. Based on :mod:`lsprof`, contributed by Brett Rosen and Ted
Czotter.
# Old (deprecated)
import profile
profile.run('my_function()')
3. :mod:`profile`, a pure Python module whose interface is imitated by
:mod:`cProfile`, but which adds significant overhead to profiled programs.
If you're trying to extend the profiler in some way, the task might be easier
with this module. Originally designed and written by Jim Roskind.
# New (recommended)
import profiling.tracing
profiling.tracing.run('my_function()')
For most code, replacing ``import profile`` with ``import profiling.tracing``
(and using ``profiling.tracing`` instead of ``profile`` throughout) provides
a straightforward migration path.
.. note::
The profiler modules are designed to provide an execution profile for a given
program, not for benchmarking purposes (for that, there is :mod:`timeit` for
reasonably accurate results). This particularly applies to benchmarking
Python code against C code: the profilers introduce overhead for Python code,
but not for C-level functions, and so the C code would seem faster than any
Python one.
The ``cProfile`` module remains available as a backward-compatible alias
to :mod:`profiling.tracing`. Existing code using ``import cProfile`` will
continue to work without modification.
**Profiler Comparison:**
+-------------------+--------------------------+----------------------+----------------------+
| Feature | Statistical | Deterministic | Deterministic |
| | (``profiling.sampling``) | (``cProfile``) | (``profile``) |
+===================+==========================+======================+======================+
| **Target** | Running process | Code you run | Code you run |
+-------------------+--------------------------+----------------------+----------------------+
| **Overhead** | Virtually none | Moderate | High |
+-------------------+--------------------------+----------------------+----------------------+
| **Accuracy** | Statistical approx. | Exact call counts | Exact call counts |
+-------------------+--------------------------+----------------------+----------------------+
| **Setup** | Attach to any PID | Instrument code | Instrument code |
+-------------------+--------------------------+----------------------+----------------------+
| **Use Case** | Production debugging | Development/testing | Profiler extension |
+-------------------+--------------------------+----------------------+----------------------+
| **Implementation**| C extension | C extension | Pure Python |
+-------------------+--------------------------+----------------------+----------------------+
:mod:`!profile` and :mod:`!profiling.tracing` module reference
==============================================================
.. note::
The statistical profiler (:mod:`!profiling.sampling`) is recommended for most production
use cases due to its extremely low overhead and ability to profile running processes
without modification. It can attach to any Python process and collect performance
data with minimal impact on execution speed, making it ideal for debugging
performance issues in live applications.
.. _statistical-profiling:
What Is Statistical Profiling?
==============================
:dfn:`Statistical profiling` works by periodically interrupting a running
program to capture its current call stack. Rather than monitoring every
function entry and exit like deterministic profilers, it takes snapshots at
regular intervals to build a statistical picture of where the program spends
its time.
The sampling profiler uses process memory reading (via system calls like
``process_vm_readv`` on Linux, ``vm_read`` on macOS, and ``ReadProcessMemory`` on
Windows) to attach to a running Python process and extract stack trace
information without requiring any code modification or restart of the target
process. This approach provides several key advantages over traditional
profiling methods.
The fundamental principle is that if a function appears frequently in the
collected stack samples, it is likely consuming significant CPU time. By
analyzing thousands of samples, the profiler can accurately estimate the
relative time spent in different parts of the program. The statistical nature
means that while individual measurements may vary, the aggregate results
converge to represent the true performance characteristics of the application.
Since statistical profiling operates externally to the target process, it
introduces virtually no overhead to the running program. The profiler process
runs separately and reads the target process memory without interrupting its
execution. This makes it suitable for profiling production systems where
performance impact must be minimized.
The accuracy of statistical profiling improves with the number of samples
collected. Short-lived functions may be missed or underrepresented, while
long-running functions will be captured proportionally to their execution time.
This characteristic makes statistical profiling particularly effective for
identifying the most significant performance bottlenecks rather than providing
exhaustive coverage of all function calls.
Statistical profiling excels at answering questions like "which functions
consume the most CPU time?" and "where should I focus optimization efforts?"
rather than "exactly how many times was this function called?" The trade-off
between precision and practicality makes it an invaluable tool for performance
analysis in real-world applications.
.. _profile-instant:
Instant User's Manual
=====================
This section is provided for users that "don't want to read the manual." It
provides a very brief overview, and allows a user to rapidly perform profiling
on an existing application.
**Statistical Profiling (Recommended for Production):**
To profile an existing running process::
python -m profiling.sampling 1234
To profile with custom settings::
python -m profiling.sampling -i 50 -d 30 1234
**Deterministic Profiling (Development/Testing):**
To profile a function that takes a single argument, you can do::
import cProfile
import re
cProfile.run('re.compile("foo|bar")')
(Use :mod:`profile` instead of :mod:`cProfile` if the latter is not available on
your system.)
The above action would run :func:`re.compile` and print profile results like
the following::
214 function calls (207 primitive calls) in 0.002 seconds
Ordered by: cumulative time
ncalls tottime percall cumtime percall filename:lineno(function)
1 0.000 0.000 0.002 0.002 {built-in method builtins.exec}
1 0.000 0.000 0.001 0.001 <string>:1(<module>)
1 0.000 0.000 0.001 0.001 __init__.py:250(compile)
1 0.000 0.000 0.001 0.001 __init__.py:289(_compile)
1 0.000 0.000 0.000 0.000 _compiler.py:759(compile)
1 0.000 0.000 0.000 0.000 _parser.py:937(parse)
1 0.000 0.000 0.000 0.000 _compiler.py:598(_code)
1 0.000 0.000 0.000 0.000 _parser.py:435(_parse_sub)
The first line indicates that 214 calls were monitored. Of those calls, 207
were :dfn:`primitive`, meaning that the call was not induced via recursion. The
next line: ``Ordered by: cumulative time`` indicates the output is sorted
by the ``cumtime`` values. The column headings include:
ncalls
for the number of calls.
tottime
for the total time spent in the given function (and excluding time made in
calls to sub-functions)
percall
is the quotient of ``tottime`` divided by ``ncalls``
cumtime
is the cumulative time spent in this and all subfunctions (from invocation
till exit). This figure is accurate *even* for recursive functions.
percall
is the quotient of ``cumtime`` divided by primitive calls
filename:lineno(function)
provides the respective data of each function
When there are two numbers in the first column (for example ``3/1``), it means
that the function recursed. The second value is the number of primitive calls
and the former is the total number of calls. Note that when the function does
not recurse, these two values are the same, and only the single figure is
printed.
Instead of printing the output at the end of the profile run, you can save the
results to a file by specifying a filename to the :func:`run` function::
import cProfile
import re
cProfile.run('re.compile("foo|bar")', 'restats')
The :class:`pstats.Stats` class reads profile results from a file and formats
them in various ways.
.. _sampling-profiler-cli:
Statistical Profiler Command Line Interface
===========================================
.. program:: profiling.sampling
The :mod:`!profiling.sampling` module can be invoked as a script to profile running processes::
python -m profiling.sampling [options] PID
**Basic Usage Examples:**
Profile process 1234 for 10 seconds with default settings::
python -m profiling.sampling 1234
Profile with custom interval and duration, save to file::
python -m profiling.sampling -i 50 -d 30 -o profile.stats 1234
Generate collapsed stacks to use with tools like `flamegraph.pl
<https://github.com/brendangregg/FlameGraph>`_::
python -m profiling.sampling --collapsed 1234
Profile all threads, sort by total time::
python -m profiling.sampling -a --sort-tottime 1234
Profile with real-time sampling statistics::
python -m profiling.sampling --realtime-stats 1234
**Command Line Options:**
.. option:: PID
Process ID of the Python process to profile (required)
.. option:: -i, --interval INTERVAL
Sampling interval in microseconds (default: 100)
.. option:: -d, --duration DURATION
Sampling duration in seconds (default: 10)
.. option:: -a, --all-threads
Sample all threads in the process instead of just the main thread
.. option:: --native
Include artificial ``<native>`` frames to denote calls to non-Python code.
.. option:: --no-gc
Don't include artificial ``<GC>`` frames to denote active garbage collection.
.. option:: --realtime-stats
Print real-time sampling statistics during profiling
.. option:: --pstats
Generate pstats output (default)
.. option:: --collapsed
Generate collapsed stack traces for flamegraphs
.. option:: -o, --outfile OUTFILE
Save output to a file
**Sorting Options (pstats format only):**
.. option:: --sort-nsamples
Sort by number of direct samples
.. option:: --sort-tottime
Sort by total time
.. option:: --sort-cumtime
Sort by cumulative time (default)
.. option:: --sort-sample-pct
Sort by sample percentage
.. option:: --sort-cumul-pct
Sort by cumulative sample percentage
.. option:: --sort-nsamples-cumul
Sort by cumulative samples
.. option:: --sort-name
Sort by function name
.. option:: -l, --limit LIMIT
Limit the number of rows in the output (default: 15)
.. option:: --no-summary
Disable the summary section in the output
**Understanding Statistical Profile Output:**
The statistical profiler produces output similar to deterministic profilers but with different column meanings::
Profile Stats:
nsamples sample% tottime (ms) cumul% cumtime (ms) filename:lineno(function)
45/67 12.5 23.450 18.6 56.780 mymodule.py:42(process_data)
23/23 6.4 15.230 6.4 15.230 <built-in>:0(len)
**Column Meanings:**
- **nsamples**: ``direct/cumulative`` - Times function was directly executing / on call stack
- **sample%**: Percentage of total samples where function was directly executing
- **tottime**: Estimated time spent directly in this function
- **cumul%**: Percentage of samples where function was anywhere on call stack
- **cumtime**: Estimated cumulative time including called functions
- **filename:lineno(function)**: Location and name of the function
.. _profile-cli:
Deterministic Profiler Command Line Interface
=============================================
.. program:: cProfile
The files :mod:`cProfile` and :mod:`profile` can also be invoked as a script to
profile another script. For example::
python -m cProfile [-o output_file] [-s sort_order] (-m module | myscript.py)
.. option:: -o <output_file>
Writes the profile results to a file instead of to stdout.
.. option:: -s <sort_order>
Specifies one of the :func:`~pstats.Stats.sort_stats` sort values
to sort the output by.
This only applies when :option:`-o <cProfile -o>` is not supplied.
.. option:: -m <module>
Specifies that a module is being profiled instead of a script.
.. versionadded:: 3.7
Added the ``-m`` option to :mod:`cProfile`.
.. versionadded:: 3.8
Added the ``-m`` option to :mod:`profile`.
The :mod:`pstats` module's :class:`~pstats.Stats` class has a variety of methods
for manipulating and printing the data saved into a profile results file::
import pstats
from pstats import SortKey
p = pstats.Stats('restats')
p.strip_dirs().sort_stats(-1).print_stats()
The :meth:`~pstats.Stats.strip_dirs` method removed the extraneous path from all
the module names. The :meth:`~pstats.Stats.sort_stats` method sorted all the
entries according to the standard module/line/name string that is printed. The
:meth:`~pstats.Stats.print_stats` method printed out all the statistics. You
might try the following sort calls::
p.sort_stats(SortKey.NAME)
p.print_stats()
The first call will actually sort the list by function name, and the second call
will print out the statistics. The following are some interesting calls to
experiment with::
p.sort_stats(SortKey.CUMULATIVE).print_stats(10)
This sorts the profile by cumulative time in a function, and then only prints
the ten most significant lines. If you want to understand what algorithms are
taking time, the above line is what you would use.
If you were looking to see what functions were looping a lot, and taking a lot
of time, you would do::
p.sort_stats(SortKey.TIME).print_stats(10)
to sort according to time spent within each function, and then print the
statistics for the top ten functions.
You might also try::
p.sort_stats(SortKey.FILENAME).print_stats('__init__')
This will sort all the statistics by file name, and then print out statistics
for only the class init methods (since they are spelled with ``__init__`` in
them). As one final example, you could try::
p.sort_stats(SortKey.TIME, SortKey.CUMULATIVE).print_stats(.5, 'init')
This line sorts statistics with a primary key of time, and a secondary key of
cumulative time, and then prints out some of the statistics. To be specific, the
list is first culled down to 50% (re: ``.5``) of its original size, then only
lines containing ``init`` are maintained, and that sub-sub-list is printed.
If you wondered what functions called the above functions, you could now (``p``
is still sorted according to the last criteria) do::
p.print_callers(.5, 'init')
and you would get a list of callers for each of the listed functions.
If you want more functionality, you're going to have to read the manual, or
guess what the following functions do::
p.print_callees()
p.add('restats')
Invoked as a script, the :mod:`pstats` module is a statistics browser for
reading and examining profile dumps. It has a simple line-oriented interface
(implemented using :mod:`cmd`) and interactive help.
:mod:`profile` and :mod:`cProfile` Module Reference
=======================================================
.. module:: cProfile
.. module:: profile
:synopsis: Python source profiler.
Both the :mod:`profile` and :mod:`cProfile` modules provide the following
functions:
Both the :mod:`profile` and :mod:`profiling.tracing` modules provide the
following functions:
.. function:: run(command, filename=None, sort=-1)
@ -480,7 +86,7 @@ functions:
.. class:: Profile(timer=None, timeunit=0.0, subcalls=True, builtins=True)
This class is normally only used if more precise control over profiling is
needed than what the :func:`cProfile.run` function provides.
needed than what the :func:`profiling.tracing.run` function provides.
A custom timer can be supplied for measuring how long code takes to run via
the *timer* argument. This must be a function that returns a single number
@ -492,9 +98,12 @@ functions:
Directly using the :class:`Profile` class allows formatting profile results
without writing the profile data to a file::
import cProfile, pstats, io
import profiling.tracing
import pstats
import io
from pstats import SortKey
pr = cProfile.Profile()
pr = profiling.tracing.Profile()
pr.enable()
# ... do something ...
pr.disable()
@ -505,11 +114,12 @@ functions:
print(s.getvalue())
The :class:`Profile` class can also be used as a context manager (supported
only in :mod:`cProfile` module. see :ref:`typecontextmanager`)::
only in :mod:`profiling.tracing`, not in the deprecated :mod:`profile`
module; see :ref:`typecontextmanager`)::
import cProfile
import profiling.tracing
with cProfile.Profile() as pr:
with profiling.tracing.Profile() as pr:
# ... do something ...
pr.print_stats()
@ -519,11 +129,11 @@ functions:
.. method:: enable()
Start collecting profiling data. Only in :mod:`cProfile`.
Start collecting profiling data. Only in :mod:`profiling.tracing`.
.. method:: disable()
Stop collecting profiling data. Only in :mod:`cProfile`.
Stop collecting profiling data. Only in :mod:`profiling.tracing`.
.. method:: create_stats()
@ -537,7 +147,7 @@ functions:
The *sort* parameter specifies the sorting order of the displayed
statistics. It accepts a single key or a tuple of keys to enable
multi-level sorting, as in :func:`Stats.sort_stats <pstats.Stats.sort_stats>`.
multi-level sorting, as in :meth:`pstats.Stats.sort_stats`.
.. versionadded:: 3.13
:meth:`~Profile.print_stats` now accepts a tuple of keys.
@ -564,237 +174,43 @@ returns. If the interpreter is terminated (e.g. via a :func:`sys.exit` call
during the called command/function execution) no profiling results will be
printed.
.. _profile-stats:
The :class:`Stats` Class
========================
Differences from :mod:`!profiling.tracing`
==========================================
Analysis of the profiler data is done using the :class:`~pstats.Stats` class.
The :mod:`profile` module differs from :mod:`profiling.tracing` in several
ways:
.. module:: pstats
:synopsis: Statistics object for use with the profiler.
**Higher overhead.** The pure Python implementation is significantly slower
than the C implementation, making it unsuitable for profiling long-running
programs or performance-sensitive code.
.. class:: Stats(*filenames or profile, stream=sys.stdout)
**Calibration support.** The :mod:`profile` module supports calibration to
compensate for profiling overhead. This is not needed in :mod:`profiling.tracing`
because the C implementation has negligible overhead.
This class constructor creates an instance of a "statistics object" from a
*filename* (or list of filenames) or from a :class:`Profile` instance. Output
will be printed to the stream specified by *stream*.
**Custom timers.** Both modules support custom timers, but :mod:`profile`
accepts timer functions that return tuples (like :func:`os.times`), while
:mod:`profiling.tracing` requires a function returning a single number.
The file selected by the above constructor must have been created by the
corresponding version of :mod:`profile` or :mod:`cProfile`. To be specific,
there is *no* file compatibility guaranteed with future versions of this
profiler, and there is no compatibility with files produced by other
profilers, or the same profiler run on a different operating system. If
several files are provided, all the statistics for identical functions will
be coalesced, so that an overall view of several processes can be considered
in a single report. If additional files need to be combined with data in an
existing :class:`~pstats.Stats` object, the :meth:`~pstats.Stats.add` method
can be used.
**Subclassing.** The pure Python implementation is easier to subclass and
extend for custom profiling behavior.
Instead of reading the profile data from a file, a :class:`cProfile.Profile`
or :class:`profile.Profile` object can be used as the profile data source.
:class:`Stats` objects have the following methods:
.. method:: strip_dirs()
This method for the :class:`Stats` class removes all leading path
information from file names. It is very useful in reducing the size of
the printout to fit within (close to) 80 columns. This method modifies
the object, and the stripped information is lost. After performing a
strip operation, the object is considered to have its entries in a
"random" order, as it was just after object initialization and loading.
If :meth:`~pstats.Stats.strip_dirs` causes two function names to be
indistinguishable (they are on the same line of the same filename, and
have the same function name), then the statistics for these two entries
are accumulated into a single entry.
.. method:: add(*filenames)
This method of the :class:`Stats` class accumulates additional profiling
information into the current profiling object. Its arguments should refer
to filenames created by the corresponding version of :func:`profile.run`
or :func:`cProfile.run`. Statistics for identically named (re: file, line,
name) functions are automatically accumulated into single function
statistics.
.. method:: dump_stats(filename)
Save the data loaded into the :class:`Stats` object to a file named
*filename*. The file is created if it does not exist, and is overwritten
if it already exists. This is equivalent to the method of the same name
on the :class:`profile.Profile` and :class:`cProfile.Profile` classes.
.. method:: sort_stats(*keys)
This method modifies the :class:`Stats` object by sorting it according to
the supplied criteria. The argument can be either a string or a SortKey
enum identifying the basis of a sort (example: ``'time'``, ``'name'``,
``SortKey.TIME`` or ``SortKey.NAME``). The SortKey enums argument have
advantage over the string argument in that it is more robust and less
error prone.
When more than one key is provided, then additional keys are used as
secondary criteria when there is equality in all keys selected before
them. For example, ``sort_stats(SortKey.NAME, SortKey.FILE)`` will sort
all the entries according to their function name, and resolve all ties
(identical function names) by sorting by file name.
For the string argument, abbreviations can be used for any key names, as
long as the abbreviation is unambiguous.
The following are the valid string and SortKey:
+------------------+---------------------+----------------------+
| Valid String Arg | Valid enum Arg | Meaning |
+==================+=====================+======================+
| ``'calls'`` | SortKey.CALLS | call count |
+------------------+---------------------+----------------------+
| ``'cumulative'`` | SortKey.CUMULATIVE | cumulative time |
+------------------+---------------------+----------------------+
| ``'cumtime'`` | N/A | cumulative time |
+------------------+---------------------+----------------------+
| ``'file'`` | N/A | file name |
+------------------+---------------------+----------------------+
| ``'filename'`` | SortKey.FILENAME | file name |
+------------------+---------------------+----------------------+
| ``'module'`` | N/A | file name |
+------------------+---------------------+----------------------+
| ``'ncalls'`` | N/A | call count |
+------------------+---------------------+----------------------+
| ``'pcalls'`` | SortKey.PCALLS | primitive call count |
+------------------+---------------------+----------------------+
| ``'line'`` | SortKey.LINE | line number |
+------------------+---------------------+----------------------+
| ``'name'`` | SortKey.NAME | function name |
+------------------+---------------------+----------------------+
| ``'nfl'`` | SortKey.NFL | name/file/line |
+------------------+---------------------+----------------------+
| ``'stdname'`` | SortKey.STDNAME | standard name |
+------------------+---------------------+----------------------+
| ``'time'`` | SortKey.TIME | internal time |
+------------------+---------------------+----------------------+
| ``'tottime'`` | N/A | internal time |
+------------------+---------------------+----------------------+
Note that all sorts on statistics are in descending order (placing most
time consuming items first), where as name, file, and line number searches
are in ascending order (alphabetical). The subtle distinction between
``SortKey.NFL`` and ``SortKey.STDNAME`` is that the standard name is a
sort of the name as printed, which means that the embedded line numbers
get compared in an odd way. For example, lines 3, 20, and 40 would (if
the file names were the same) appear in the string order 20, 3 and 40.
In contrast, ``SortKey.NFL`` does a numeric compare of the line numbers.
In fact, ``sort_stats(SortKey.NFL)`` is the same as
``sort_stats(SortKey.NAME, SortKey.FILENAME, SortKey.LINE)``.
For backward-compatibility reasons, the numeric arguments ``-1``, ``0``,
``1``, and ``2`` are permitted. They are interpreted as ``'stdname'``,
``'calls'``, ``'time'``, and ``'cumulative'`` respectively. If this old
style format (numeric) is used, only one sort key (the numeric key) will
be used, and additional arguments will be silently ignored.
.. For compatibility with the old profiler.
.. versionadded:: 3.7
Added the SortKey enum.
.. method:: reverse_order()
This method for the :class:`Stats` class reverses the ordering of the
basic list within the object. Note that by default ascending vs
descending order is properly selected based on the sort key of choice.
.. This method is provided primarily for compatibility with the old
profiler.
.. method:: print_stats(*restrictions)
This method for the :class:`Stats` class prints out a report as described
in the :func:`profile.run` definition.
The order of the printing is based on the last
:meth:`~pstats.Stats.sort_stats` operation done on the object (subject to
caveats in :meth:`~pstats.Stats.add` and
:meth:`~pstats.Stats.strip_dirs`).
The arguments provided (if any) can be used to limit the list down to the
significant entries. Initially, the list is taken to be the complete set
of profiled functions. Each restriction is either an integer (to select a
count of lines), or a decimal fraction between 0.0 and 1.0 inclusive (to
select a percentage of lines), or a string that will be interpreted as a
regular expression (to pattern match the standard name that is printed).
If several restrictions are provided, then they are applied sequentially.
For example::
print_stats(.1, 'foo:')
would first limit the printing to first 10% of list, and then only print
functions that were part of filename :file:`.\*foo:`. In contrast, the
command::
print_stats('foo:', .1)
would limit the list to all functions having file names :file:`.\*foo:`,
and then proceed to only print the first 10% of them.
.. method:: print_callers(*restrictions)
This method for the :class:`Stats` class prints a list of all functions
that called each function in the profiled database. The ordering is
identical to that provided by :meth:`~pstats.Stats.print_stats`, and the
definition of the restricting argument is also identical. Each caller is
reported on its own line. The format differs slightly depending on the
profiler that produced the stats:
* With :mod:`profile`, a number is shown in parentheses after each caller
to show how many times this specific call was made. For convenience, a
second non-parenthesized number repeats the cumulative time spent in the
function at the right.
* With :mod:`cProfile`, each caller is preceded by three numbers: the
number of times this specific call was made, and the total and
cumulative times spent in the current function while it was invoked by
this specific caller.
.. method:: print_callees(*restrictions)
This method for the :class:`Stats` class prints a list of all function
that were called by the indicated function. Aside from this reversal of
direction of calls (re: called vs was called by), the arguments and
ordering are identical to the :meth:`~pstats.Stats.print_callers` method.
.. method:: get_stats_profile()
This method returns an instance of StatsProfile, which contains a mapping
of function names to instances of FunctionProfile. Each FunctionProfile
instance holds information related to the function's profile such as how
long the function took to run, how many times it was called, etc...
.. versionadded:: 3.9
Added the following dataclasses: StatsProfile, FunctionProfile.
Added the following function: get_stats_profile.
.. _deterministic-profiling:
What Is Deterministic Profiling?
What is deterministic profiling?
================================
:dfn:`Deterministic profiling` is meant to reflect the fact that all *function
call*, *function return*, and *exception* events are monitored, and precise
timings are made for the intervals between these events (during which time the
user's code is executing). In contrast, :dfn:`statistical profiling` (which is
provided by the :mod:`!profiling.sampling` module) periodically samples the effective instruction pointer, and
deduces where time is being spent. The latter technique traditionally involves
less overhead (as the code does not need to be instrumented), but provides only
relative indications of where time is being spent.
provided by the :mod:`profiling.sampling` module) periodically samples the
effective instruction pointer, and deduces where time is being spent. The
latter technique traditionally involves less overhead (as the code does not
need to be instrumented), but provides only relative indications of where time
is being spent.
In Python, since there is an interpreter active during execution, the presence
of instrumented code is not required in order to do deterministic profiling.
@ -838,15 +254,16 @@ this error. The error that accumulates in this fashion is typically less than
the accuracy of the clock (less than one clock tick), but it *can* accumulate
and become very significant.
The problem is more important with :mod:`profile` than with the lower-overhead
:mod:`cProfile`. For this reason, :mod:`profile` provides a means of
calibrating itself for a given platform so that this error can be
probabilistically (on the average) removed. After the profiler is calibrated, it
will be more accurate (in a least square sense), but it will sometimes produce
negative numbers (when call counts are exceptionally low, and the gods of
probability work against you :-). ) Do *not* be alarmed by negative numbers in
the profile. They should *only* appear if you have calibrated your profiler,
and the results are actually better than without calibration.
The problem is more important with the deprecated :mod:`profile` module than
with the lower-overhead :mod:`profiling.tracing`. For this reason,
:mod:`profile` provides a means of calibrating itself for a given platform so
that this error can be probabilistically (on the average) removed. After the
profiler is calibrated, it will be more accurate (in a least square sense), but
it will sometimes produce negative numbers (when call counts are exceptionally
low, and the gods of probability work against you :-). ) Do *not* be alarmed
by negative numbers in the profile. They should *only* appear if you have
calibrated your profiler, and the results are actually better than without
calibration.
.. _profile-calibration:
@ -892,6 +309,7 @@ When you have a consistent answer, there are three ways you can use it::
If you have a choice, you are better off choosing a smaller constant, and then
your results will "less often" show up as negative in profile statistics.
.. _profile-timers:
Using a custom timer
@ -904,7 +322,7 @@ to the :class:`Profile` class constructor::
pr = profile.Profile(your_time_func)
The resulting profiler will then call ``your_time_func``. Depending on whether
you are using :class:`profile.Profile` or :class:`cProfile.Profile`,
you are using :class:`profile.Profile` or :class:`profiling.tracing.Profile`,
``your_time_func``'s return value will be interpreted differently:
:class:`profile.Profile`
@ -923,20 +341,32 @@ you are using :class:`profile.Profile` or :class:`cProfile.Profile`,
replacement dispatch method that best handles your timer call, along with the
appropriate calibration constant.
:class:`cProfile.Profile`
:class:`profiling.tracing.Profile`
``your_time_func`` should return a single number. If it returns integers,
you can also invoke the class constructor with a second argument specifying
the real duration of one unit of time. For example, if
``your_integer_time_func`` returns times measured in thousands of seconds,
you would construct the :class:`Profile` instance as follows::
pr = cProfile.Profile(your_integer_time_func, 0.001)
pr = profiling.tracing.Profile(your_integer_time_func, 0.001)
As the :class:`cProfile.Profile` class cannot be calibrated, custom timer
functions should be used with care and should be as fast as possible. For
the best results with a custom timer, it might be necessary to hard-code it
in the C source of the internal :mod:`!_lsprof` module.
As the :class:`profiling.tracing.Profile` class cannot be calibrated, custom
timer functions should be used with care and should be as fast as possible.
For the best results with a custom timer, it might be necessary to hard-code
it in the C source of the internal :mod:`!_lsprof` module.
Python 3.3 adds several new functions in :mod:`time` that can be used to make
precise measurements of process or wall-clock time. For example, see
:func:`time.perf_counter`.
.. seealso::
:mod:`profiling`
Overview of Python profiling tools.
:mod:`profiling.tracing`
Recommended replacement for this module.
:mod:`pstats`
Statistical analysis and formatting for profile data.

270
Doc/library/profiling.rst Normal file
View file

@ -0,0 +1,270 @@
.. highlight:: shell-session
.. _profiling-module:
***************************************
:mod:`profiling` --- Python profilers
***************************************
.. module:: profiling
:synopsis: Python profiling tools for performance analysis.
.. versionadded:: 3.15
**Source code:** :source:`Lib/profiling/`
--------------
.. index::
single: statistical profiling
single: profiling, statistical
single: deterministic profiling
single: profiling, deterministic
Introduction to profiling
=========================
A :dfn:`profile` is a set of statistics that describes how often and for how
long various parts of a program execute. These statistics help identify
performance bottlenecks and guide optimization efforts. Python provides two
fundamentally different approaches to collecting this information: statistical
sampling and deterministic tracing.
The :mod:`profiling` package organizes Python's built-in profiling tools under
a single namespace. It contains two submodules, each implementing a different
profiling methodology:
:mod:`profiling.sampling`
A statistical profiler that periodically samples the call stack. Run scripts
directly or attach to running processes by PID. Provides multiple output
formats (flame graphs, heatmaps, Firefox Profiler), GIL analysis, GC tracking,
and multiple profiling modes (wall-clock, CPU, GIL) with virtually no overhead.
:mod:`profiling.tracing`
A deterministic profiler that traces every function call, return, and
exception event. Provides exact call counts and precise timing information,
capturing every invocation including very fast functions.
.. note::
The profiler modules are designed to provide an execution profile for a
given program, not for benchmarking purposes. For benchmarking, use the
:mod:`timeit` module, which provides reasonably accurate timing
measurements. This distinction is particularly important when comparing
Python code against C code: deterministic profilers introduce overhead for
Python code but not for C-level functions, which can skew comparisons.
.. _choosing-a-profiler:
Choosing a profiler
===================
For most performance analysis, use the statistical profiler
(:mod:`profiling.sampling`). It has minimal overhead, works for both development
and production, and provides rich visualization options including flamegraphs,
heatmaps, GIL analysis, and more.
Use the deterministic profiler (:mod:`profiling.tracing`) when you need **exact
call counts** and cannot afford to miss any function calls. Since it instruments
every function call and return, it will capture even very fast functions that
complete between sampling intervals. The tradeoff is higher overhead.
The following table summarizes the key differences:
+--------------------+------------------------------+------------------------------+
| Feature | Statistical sampling | Deterministic |
| | (:mod:`profiling.sampling`) | (:mod:`profiling.tracing`) |
+====================+==============================+==============================+
| **Overhead** | Virtually none | Moderate |
+--------------------+------------------------------+------------------------------+
| **Accuracy** | Statistical estimate | Exact call counts |
+--------------------+------------------------------+------------------------------+
| **Output formats** | pstats, flamegraph, heatmap, | pstats |
| | gecko, collapsed | |
+--------------------+------------------------------+------------------------------+
| **Profiling modes**| Wall-clock, CPU, GIL | Wall-clock |
+--------------------+------------------------------+------------------------------+
| **Special frames** | GC, native (C extensions) | N/A |
+--------------------+------------------------------+------------------------------+
| **Attach to PID** | Yes | No |
+--------------------+------------------------------+------------------------------+
When to use statistical sampling
--------------------------------
The statistical profiler (:mod:`profiling.sampling`) is recommended for most
performance analysis tasks. Use it the same way you would use
:mod:`profiling.tracing`::
python -m profiling.sampling run script.py
One of the main strengths of the sampling profiler is its variety of output
formats. Beyond traditional pstats tables, it can generate interactive
flamegraphs that visualize call hierarchies, line-level source heatmaps that
show exactly where time is spent in your code, and Firefox Profiler output for
timeline-based analysis.
The profiler also provides insight into Python interpreter behavior that
deterministic profiling cannot capture. Use ``--mode gil`` to identify GIL
contention in multi-threaded code, ``--mode cpu`` to measure actual CPU time
excluding I/O waits, or inspect ``<GC>`` frames to understand garbage collection
overhead. The ``--native`` option reveals time spent in C extensions, helping
distinguish Python overhead from library performance.
For multi-threaded applications, the ``-a`` option samples all threads
simultaneously, showing how work is distributed. And for production debugging,
the ``attach`` command connects to any running Python process by PID without
requiring a restart or code changes.
When to use deterministic tracing
---------------------------------
The deterministic profiler (:mod:`profiling.tracing`) instruments every function
call and return. This approach has higher overhead than sampling, but guarantees
complete coverage of program execution.
The primary reason to choose deterministic tracing is when you need exact call
counts. Statistical profiling estimates frequency based on sampling, which may
undercount short-lived functions that complete between samples. If you need to
verify that an optimization actually reduced the number of function calls, or
if you want to trace the complete call graph to understand caller-callee
relationships, deterministic tracing is the right choice.
Deterministic tracing also excels at capturing functions that execute in
microseconds. Such functions may not appear frequently enough in statistical
samples, but deterministic tracing records every invocation regardless of
duration.
Quick start
===========
This section provides the minimal steps needed to start profiling. For complete
documentation, see the dedicated pages for each profiler.
Statistical profiling
---------------------
To profile a script, use the :mod:`profiling.sampling` module with the ``run``
command::
python -m profiling.sampling run script.py
python -m profiling.sampling run -m mypackage.module
This runs the script under the profiler and prints a summary of where time was
spent. For an interactive flamegraph::
python -m profiling.sampling run --flamegraph script.py
To profile an already-running process, use the ``attach`` command with the
process ID::
python -m profiling.sampling attach 1234
For custom settings, specify the sampling interval (in microseconds) and
duration (in seconds)::
python -m profiling.sampling run -i 50 -d 30 script.py
Deterministic profiling
-----------------------
To profile a script from the command line::
python -m profiling.tracing script.py
To profile a piece of code programmatically:
.. code-block:: python
import profiling.tracing
profiling.tracing.run('my_function()')
This executes the given code under the profiler and prints a summary showing
exact function call counts and timing.
.. _profile-output:
Understanding profile output
============================
Both profilers collect function-level statistics, though they present them in
different formats. The sampling profiler offers multiple visualizations
(flamegraphs, heatmaps, Firefox Profiler, pstats tables), while the
deterministic profiler produces pstats-compatible output. Regardless of format,
the underlying concepts are the same.
Key profiling concepts:
**Direct time** (also called *self time* or *tottime*)
Time spent executing code in the function itself, excluding time spent in
functions it called. High direct time indicates the function contains
expensive operations.
**Cumulative time** (also called *total time* or *cumtime*)
Time spent in the function and all functions it called. This measures the
total cost of calling a function, including its entire call subtree.
**Call count** (also called *ncalls* or *samples*)
How many times the function was called (deterministic) or sampled
(statistical). In deterministic profiling, this is exact. In statistical
profiling, it represents the number of times the function appeared in a
stack sample.
**Primitive calls**
Calls that are not induced by recursion. When a function recurses, the total
call count includes recursive invocations, but primitive calls counts only
the initial entry. Displayed as ``total/primitive`` (for example, ``3/1``
means three total calls, one primitive).
**Caller/Callee relationships**
Which functions called a given function (callers) and which functions it
called (callees). Flamegraphs visualize this as nested rectangles; pstats
can display it via the :meth:`~pstats.Stats.print_callers` and
:meth:`~pstats.Stats.print_callees` methods.
Legacy compatibility
====================
For backward compatibility, the ``cProfile`` module remains available as an
alias to :mod:`profiling.tracing`. Existing code using ``import cProfile`` will
continue to work without modification in all future Python versions.
.. deprecated:: 3.15
The pure Python :mod:`profile` module is deprecated and will be removed in
Python 3.17. Use :mod:`profiling.tracing` (or its alias ``cProfile``)
instead. See :mod:`profile` for migration guidance.
.. seealso::
:mod:`profiling.sampling`
Statistical sampling profiler with flamegraphs, heatmaps, and GIL analysis.
Recommended for most users.
:mod:`profiling.tracing`
Deterministic tracing profiler for exact call counts.
:mod:`pstats`
Statistics analysis and formatting for profile data.
:mod:`timeit`
Module for measuring execution time of small code snippets.
.. rubric:: Submodules
.. toctree::
:maxdepth: 1
profiling.tracing.rst
profiling.sampling.rst

1393
Doc/library/profiling.sampling.rst Normal file
View file

File diff suppressed because it is too large Load diff

331
Doc/library/profiling.tracing.rst Normal file
View file

@ -0,0 +1,331 @@
.. _profiling-tracing:
****************************************************
:mod:`profiling.tracing` --- Deterministic profiler
****************************************************
.. module:: profiling.tracing
:synopsis: Deterministic tracing profiler for Python programs.
.. module:: cProfile
:synopsis: Alias for profiling.tracing (backward compatibility).
:noindex:
.. versionadded:: 3.15
**Source code:** :source:`Lib/profiling/tracing/`
--------------
The :mod:`profiling.tracing` module provides deterministic profiling of Python
programs. It monitors every function call, function return, and exception event,
recording precise timing for each. This approach provides exact call counts and
complete visibility into program execution, making it ideal for development and
testing scenarios.
.. note::
This module is also available as ``cProfile`` for backward compatibility.
The ``cProfile`` name will continue to work in all future Python versions.
Use whichever import style suits your codebase::
# Preferred (new style)
import profiling.tracing
profiling.tracing.run('my_function()')
# Also works (backward compatible)
import cProfile
cProfile.run('my_function()')
What is deterministic profiling?
================================
:dfn:`Deterministic profiling` captures every function call, function return,
and exception event during program execution. The profiler measures the precise
time intervals between these events, providing exact statistics about how the
program behaves.
In contrast to :ref:`statistical profiling <profiling-sampling>`, which samples
the call stack periodically to estimate where time is spent, deterministic
profiling records every event. This means you get exact call counts rather than
statistical approximations. The trade-off is that instrumenting every event
introduces overhead that can slow down program execution.
Python's interpreted nature makes deterministic profiling practical. The
interpreter already dispatches events for function calls and returns, so the
profiler can hook into this mechanism without requiring code modification. The
overhead tends to be moderate relative to the inherent cost of interpretation,
making deterministic profiling suitable for most development workflows.
Deterministic profiling helps answer questions like:
- How many times was this function called?
- What is the complete call graph of my program?
- Which functions are called by a particular function?
- Are there unexpected function calls happening?
Call count statistics can identify bugs (surprising counts) and inline
expansion opportunities (high call counts). Internal time statistics reveal
"hot loops" that warrant optimization. Cumulative time statistics help identify
algorithmic inefficiencies. The handling of cumulative times in this profiler
allows direct comparison of recursive and iterative implementations.
.. _profiling-tracing-cli:
Command-line interface
======================
.. program:: profiling.tracing
The :mod:`profiling.tracing` module can be invoked as a script to profile
another script or module:
.. code-block:: shell-session
python -m profiling.tracing [-o output_file] [-s sort_order] (-m module | script.py)
This runs the specified script or module under the profiler and prints the
results to standard output (or saves them to a file).
.. option:: -o <output_file>
Write the profile results to a file instead of standard output. The output
file can be read by the :mod:`pstats` module for later analysis.
.. option:: -s <sort_order>
Sort the output by the specified key. This accepts any of the sort keys
recognized by :meth:`pstats.Stats.sort_stats`, such as ``cumulative``,
``time``, ``calls``, or ``name``. This option only applies when
:option:`-o <profiling.tracing -o>` is not specified.
.. option:: -m <module>
Profile a module instead of a script. The module is located using the
standard import mechanism.
.. versionadded:: 3.7
The ``-m`` option for ``cProfile``.
.. versionadded:: 3.8
The ``-m`` option for :mod:`profile`.
Programmatic usage examples
===========================
For more control over profiling, use the module's functions and classes
directly.
Basic profiling
---------------
The simplest approach uses the :func:`!run` function::
import profiling.tracing
profiling.tracing.run('my_function()')
This profiles the given code string and prints a summary to standard output.
To save results for later analysis::
profiling.tracing.run('my_function()', 'output.prof')
Using the :class:`!Profile` class
---------------------------------
The :class:`!Profile` class provides fine-grained control::
import profiling.tracing
import pstats
from io import StringIO
pr = profiling.tracing.Profile()
pr.enable()
# ... code to profile ...
pr.disable()
# Print results
s = StringIO()
ps = pstats.Stats(pr, stream=s).sort_stats(pstats.SortKey.CUMULATIVE)
ps.print_stats()
print(s.getvalue())
The :class:`!Profile` class also works as a context manager::
import profiling.tracing
with profiling.tracing.Profile() as pr:
# ... code to profile ...
pr.print_stats()
Module reference
================
.. currentmodule:: profiling.tracing
.. function:: run(command, filename=None, sort=-1)
Profile execution of a command and print or save the results.
This function executes the *command* string using :func:`exec` in the
``__main__`` module's namespace::
exec(command, __main__.__dict__, __main__.__dict__)
If *filename* is not provided, the function creates a :class:`pstats.Stats`
instance and prints a summary to standard output. If *filename* is
provided, the raw profile data is saved to that file for later analysis
with :mod:`pstats`.
The *sort* argument specifies the sort order for printed output, accepting
any value recognized by :meth:`pstats.Stats.sort_stats`.
.. function:: runctx(command, globals, locals, filename=None, sort=-1)
Profile execution of a command with explicit namespaces.
Like :func:`run`, but executes the command with the specified *globals*
and *locals* mappings instead of using the ``__main__`` module's namespace::
exec(command, globals, locals)
.. class:: Profile(timer=None, timeunit=0.0, subcalls=True, builtins=True)
A profiler object that collects execution statistics.
The optional *timer* argument specifies a custom timing function. If not
provided, the profiler uses a platform-appropriate default timer. When
supplying a custom timer, it must return a single number representing the
current time. If the timer returns integers, use *timeunit* to specify the
duration of one time unit (for example, ``0.001`` for milliseconds).
The *subcalls* argument controls whether the profiler tracks call
relationships between functions. The *builtins* argument controls whether
built-in functions are profiled.
.. versionchanged:: 3.8
Added context manager support.
.. method:: enable()
Start collecting profiling data.
.. method:: disable()
Stop collecting profiling data.
.. method:: create_stats()
Stop collecting data and record the results internally as the current
profile.
.. method:: print_stats(sort=-1)
Create a :class:`pstats.Stats` object from the current profile and print
the results to standard output.
The *sort* argument specifies the sorting order. It accepts a single
key or a tuple of keys for multi-level sorting, using the same values
as :meth:`pstats.Stats.sort_stats`.
.. versionadded:: 3.13
Support for a tuple of sort keys.
.. method:: dump_stats(filename)
Write the current profile data to *filename*. The file can be read by
:class:`pstats.Stats` for later analysis.
.. method:: run(cmd)
Profile the command string via :func:`exec`.
.. method:: runctx(cmd, globals, locals)
Profile the command string via :func:`exec` with the specified
namespaces.
.. method:: runcall(func, /, *args, **kwargs)
Profile a function call. Returns whatever *func* returns::
result = pr.runcall(my_function, arg1, arg2, keyword=value)
.. note::
Profiling requires that the profiled code returns normally. If the
interpreter terminates (for example, via :func:`sys.exit`) during
profiling, no results will be available.
Using a custom timer
====================
The :class:`Profile` class accepts a custom timing function, allowing you to
measure different aspects of execution such as wall-clock time or CPU time.
Pass the timing function to the constructor::
pr = profiling.tracing.Profile(my_timer_function)
The timer function must return a single number representing the current time.
If it returns integers, also specify *timeunit* to indicate the duration of
one unit::
# Timer returns time in milliseconds
pr = profiling.tracing.Profile(my_ms_timer, 0.001)
For best performance, the timer function should be as fast as possible. The
profiler calls it frequently, so timer overhead directly affects profiling
overhead.
The :mod:`time` module provides several functions suitable for use as custom
timers:
- :func:`time.perf_counter` for high-resolution wall-clock time
- :func:`time.process_time` for CPU time (excluding sleep)
- :func:`time.monotonic` for monotonic clock time
Limitations
===========
Deterministic profiling has inherent limitations related to timing accuracy.
The underlying timer typically has a resolution of about one millisecond.
Measurements cannot be more accurate than this resolution. With enough
measurements, timing errors tend to average out, but individual measurements
may be imprecise.
There is also latency between when an event occurs and when the profiler
captures the timestamp. Similarly, there is latency after reading the
timestamp before user code resumes. Functions called frequently accumulate
this latency, which can make them appear slower than they actually are. This
error is typically less than one clock tick per call but can become
significant for functions called many times.
The :mod:`profiling.tracing` module (and its ``cProfile`` alias) is
implemented as a C extension with low overhead, so these timing issues are
less pronounced than with the deprecated pure Python :mod:`profile` module.
.. seealso::
:mod:`profiling`
Overview of Python profiling tools and guidance on choosing a profiler.
:mod:`profiling.sampling`
Statistical sampling profiler for production use.
:mod:`pstats`
Statistics analysis and formatting for profile data.
:mod:`profile`
Deprecated pure Python profiler (includes calibration documentation).

362
Doc/library/pstats.rst Normal file
View file

@ -0,0 +1,362 @@
.. _pstats-module:
********************************************
:mod:`pstats` --- Statistics for profilers
********************************************
.. module:: pstats
:synopsis: Statistics object for analyzing profiler output.
**Source code:** :source:`Lib/pstats.py`
--------------
The :mod:`pstats` module provides tools for reading, manipulating, and
displaying profiling statistics generated by Python's profilers. It reads
output from both :mod:`profiling.tracing` (deterministic profiler) and
:mod:`profiling.sampling` (statistical profiler).
Reading and displaying profile data
===================================
The :class:`Stats` class is the primary interface for working with profile
data. It can read statistics from files or directly from a
:class:`~profiling.tracing.Profile` object.
Load statistics from a file and print a basic report::
import pstats
p = pstats.Stats('profile_output.prof')
p.print_stats()
The :class:`Stats` object provides methods for sorting and filtering the
data before printing. For example, to see the ten functions with the highest
cumulative time::
from pstats import SortKey
p = pstats.Stats('profile_output.prof')
p.sort_stats(SortKey.CUMULATIVE).print_stats(10)
Working with statistics
-----------------------
The :class:`Stats` class supports method chaining, making it convenient to
perform multiple operations::
p = pstats.Stats('restats')
p.strip_dirs().sort_stats(-1).print_stats()
The :meth:`~Stats.strip_dirs` method removes directory paths from filenames,
making the output more compact. The :meth:`~Stats.sort_stats` method accepts
various keys to control the sort order.
Different sort keys highlight different aspects of performance::
from pstats import SortKey
# Functions that consume the most cumulative time
p.sort_stats(SortKey.CUMULATIVE).print_stats(10)
# Functions that consume the most time in their own code
p.sort_stats(SortKey.TIME).print_stats(10)
# Functions sorted by name
p.sort_stats(SortKey.NAME).print_stats()
Filtering output
----------------
The :meth:`~Stats.print_stats` method accepts restrictions that filter
which functions are displayed. Restrictions can be integers (limiting the
count), floats between 0 and 1 (selecting a percentage), or strings (matching
function names via regular expression).
Print only the top 10%::
p.print_stats(.1)
Print only functions whose names contain "init"::
p.print_stats('init')
Combine restrictions (they apply sequentially)::
# Top 10%, then only those containing "init"
p.print_stats(.1, 'init')
# Functions in files matching "foo:", limited to top 50%
p.sort_stats(SortKey.FILENAME).print_stats('foo:', .5)
Analyzing call relationships
----------------------------
The :meth:`~Stats.print_callers` method shows which functions called each
displayed function::
p.print_callers()
The :meth:`~Stats.print_callees` method shows the opposite relationship,
listing which functions each displayed function called::
p.print_callees()
Both methods accept the same restriction arguments as :meth:`~Stats.print_stats`.
Combining multiple profiles
---------------------------
Statistics from multiple profiling runs can be combined into a single
:class:`Stats` object::
# Load multiple files at once
p = pstats.Stats('run1.prof', 'run2.prof', 'run3.prof')
# Or add files incrementally
p = pstats.Stats('run1.prof')
p.add('run2.prof')
p.add('run3.prof')
When files are combined, statistics for identical functions (same file, line,
and name) are accumulated, giving an aggregate view across all profiling runs.
The :class:`!Stats` class
=========================
.. class:: Stats(*filenames_or_profile, stream=sys.stdout)
Create a statistics object from profile data.
The arguments can be filenames (strings or path-like objects) or
:class:`~profiling.tracing.Profile` objects. If multiple sources are
provided, their statistics are combined.
The *stream* argument specifies where output from :meth:`print_stats` and
related methods is written. It defaults to :data:`sys.stdout`.
The profile data format is specific to the Python version that created it.
There is no compatibility guarantee between Python versions or between
different profilers.
.. method:: strip_dirs()
Remove leading path information from all filenames.
This method modifies the object in place and returns it for method
chaining. After stripping, the statistics are considered to be in
random order.
If stripping causes two functions to become indistinguishable (same
filename, line number, and function name), their statistics are
combined into a single entry.
.. method:: add(*filenames)
Add profiling data from additional files.
The files must have been created by the same profiler type. Statistics
for identical functions are accumulated.
.. method:: dump_stats(filename)
Save the current statistics to a file.
The file is created if it does not exist and overwritten if it does.
The saved data can be loaded by creating a new :class:`Stats` object.
.. method:: sort_stats(*keys)
Sort the statistics according to the specified criteria.
Each key can be a string or a :class:`SortKey` enum member. When
multiple keys are provided, later keys break ties in earlier keys.
Using :class:`SortKey` enum members is preferred over strings as it
provides better error checking::
from pstats import SortKey
p.sort_stats(SortKey.CUMULATIVE)
Valid sort keys:
+------------------+------------------------+----------------------+
| String | Enum | Meaning |
+==================+========================+======================+
| ``'calls'`` | ``SortKey.CALLS`` | call count |
+------------------+------------------------+----------------------+
| ``'cumulative'`` | ``SortKey.CUMULATIVE`` | cumulative time |
+------------------+------------------------+----------------------+
| ``'cumtime'`` | N/A | cumulative time |
+------------------+------------------------+----------------------+
| ``'file'`` | N/A | file name |
+------------------+------------------------+----------------------+
| ``'filename'`` | ``SortKey.FILENAME`` | file name |
+------------------+------------------------+----------------------+
| ``'module'`` | N/A | file name |
+------------------+------------------------+----------------------+
| ``'ncalls'`` | N/A | call count |
+------------------+------------------------+----------------------+
| ``'pcalls'`` | ``SortKey.PCALLS`` | primitive call count |
+------------------+------------------------+----------------------+
| ``'line'`` | ``SortKey.LINE`` | line number |
+------------------+------------------------+----------------------+
| ``'name'`` | ``SortKey.NAME`` | function name |
+------------------+------------------------+----------------------+
| ``'nfl'`` | ``SortKey.NFL`` | name/file/line |
+------------------+------------------------+----------------------+
| ``'stdname'`` | ``SortKey.STDNAME`` | standard name |
+------------------+------------------------+----------------------+
| ``'time'`` | ``SortKey.TIME`` | internal time |
+------------------+------------------------+----------------------+
| ``'tottime'`` | N/A | internal time |
+------------------+------------------------+----------------------+
All sorts on statistics are in descending order (most time consuming
first), while name, file, and line number sorts are ascending
(alphabetical).
The difference between ``SortKey.NFL`` and ``SortKey.STDNAME`` is that
NFL sorts line numbers numerically while STDNAME sorts them as strings.
``sort_stats(SortKey.NFL)`` is equivalent to
``sort_stats(SortKey.NAME, SortKey.FILENAME, SortKey.LINE)``.
For backward compatibility, the numeric arguments ``-1``, ``0``, ``1``,
and ``2`` are also accepted, meaning ``'stdname'``, ``'calls'``,
``'time'``, and ``'cumulative'`` respectively.
.. versionadded:: 3.7
The :class:`SortKey` enum.
.. method:: reverse_order()
Reverse the current sort order.
By default, the sort direction is chosen appropriately for the sort key
(descending for time-based keys, ascending for name-based keys). This
method inverts that choice.
.. method:: print_stats(*restrictions)
Print a report of the profiling statistics.
The output includes a header line summarizing the data, followed by a
table of function statistics sorted according to the last
:meth:`sort_stats` call.
Restrictions filter the output. Each restriction is either:
- An integer: limits output to that many entries
- A float between 0.0 and 1.0: selects that fraction of entries
- A string: matches function names via regular expression
Restrictions are applied sequentially. For example::
print_stats(.1, 'foo:')
First limits to the top 10%, then filters to functions matching 'foo:'.
.. method:: print_callers(*restrictions)
Print the callers of each function in the statistics.
For each function in the filtered results, shows which functions called
it and how often.
With :mod:`profiling.tracing` (or ``cProfile``), each caller line
shows three numbers: the number of calls from that caller, and the
total and cumulative times for those specific calls.
Accepts the same restriction arguments as :meth:`print_stats`.
.. method:: print_callees(*restrictions)
Print the functions called by each function in the statistics.
This is the inverse of :meth:`print_callers`, showing which functions
each listed function called.
Accepts the same restriction arguments as :meth:`print_stats`.
.. method:: get_stats_profile()
Return a ``StatsProfile`` object containing the statistics.
The returned object provides programmatic access to the profile data,
with function names mapped to ``FunctionProfile`` objects
containing timing and call count information.
.. versionadded:: 3.9
.. class:: SortKey
An enumeration of valid sort keys for :meth:`Stats.sort_stats`.
.. attribute:: CALLS
Sort by call count.
.. attribute:: CUMULATIVE
Sort by cumulative time.
.. attribute:: FILENAME
Sort by file name.
.. attribute:: LINE
Sort by line number.
.. attribute:: NAME
Sort by function name.
.. attribute:: NFL
Sort by name, then file, then line number (numeric line sort).
.. attribute:: PCALLS
Sort by primitive (non-recursive) call count.
.. attribute:: STDNAME
Sort by standard name (string-based line sort).
.. attribute:: TIME
Sort by internal time (time in function excluding subcalls).
.. _pstats-cli:
Command-line interface
======================
The :mod:`pstats` module can be invoked as a script to interactively browse
profile data::
python -m pstats profile_output.prof
This opens a line-oriented interface (built on :mod:`cmd`) for examining the
statistics. Type ``help`` at the prompt for available commands.
.. seealso::
:mod:`profiling`
Overview of Python profiling tools.
:mod:`profiling.tracing`
Deterministic tracing profiler.
:mod:`profiling.sampling`
Statistical sampling profiler.

2
Doc/library/random.rst
View file

@ -78,7 +78,7 @@ Bookkeeping functions
instead of the system time (see the :func:`os.urandom` function for details
on availability).
If *a* is an int, it is used directly.
If *a* is an int, its absolute value is used directly.
With version 2 (the default), a :class:`str`, :class:`bytes`, or :class:`bytearray`
object gets converted to an :class:`int` and all of its bits are used.

9
Doc/library/readline.rst
View file

@ -246,6 +246,15 @@ Startup hooks
if Python was compiled for a version of the library that supports it.
.. function:: get_pre_input_hook()
Get the current pre-input hook function, or ``None`` if no pre-input hook
function has been set. This function only exists if Python was compiled
for a version of the library that supports it.
.. versionadded:: 3.15
.. _readline-completion:
Completion

28
Doc/library/runpy.rst
View file

@ -50,10 +50,10 @@ The :mod:`runpy` module provides two functions:
overridden by :func:`run_module`.
The special global variables ``__name__``, ``__spec__``, ``__file__``,
``__cached__``, ``__loader__`` and ``__package__`` are set in the globals
dictionary before the module code is executed. (Note that this is a
minimal set of variables - other variables may be set implicitly as an
interpreter implementation detail.)
``__loader__`` and ``__package__`` are set in the globals dictionary before
the module code is executed. (Note that this is a minimal set of variables -
other variables may be set implicitly as an interpreter implementation
detail.)
``__name__`` is set to *run_name* if this optional argument is not
:const:`None`, to ``mod_name + '.__main__'`` if the named module is a
@ -63,7 +63,7 @@ The :mod:`runpy` module provides two functions:
module (that is, ``__spec__.name`` will always be *mod_name* or
``mod_name + '.__main__'``, never *run_name*).
``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` are
``__file__``, ``__loader__`` and ``__package__`` are
:ref:`set as normal <import-mod-attrs>` based on the module spec.
If the argument *alter_sys* is supplied and evaluates to :const:`True`,
@ -98,6 +98,9 @@ The :mod:`runpy` module provides two functions:
``__package__`` are deprecated. See
:class:`~importlib.machinery.ModuleSpec` for alternatives.
.. versionchanged:: 3.15
``__cached__`` is no longer set.
.. function:: run_path(path_name, init_globals=None, run_name=None)
.. index::
@ -125,23 +128,23 @@ The :mod:`runpy` module provides two functions:
overridden by :func:`run_path`.
The special global variables ``__name__``, ``__spec__``, ``__file__``,
``__cached__``, ``__loader__`` and ``__package__`` are set in the globals
dictionary before the module code is executed. (Note that this is a
minimal set of variables - other variables may be set implicitly as an
interpreter implementation detail.)
``__loader__`` and ``__package__`` are set in the globals dictionary before
the module code is executed. (Note that this is a minimal set of variables -
other variables may be set implicitly as an interpreter implementation
detail.)
``__name__`` is set to *run_name* if this optional argument is not
:const:`None` and to ``'<run_path>'`` otherwise.
If *file_path* directly references a script file (whether as source
or as precompiled byte code), then ``__file__`` will be set to
*file_path*, and ``__spec__``, ``__cached__``, ``__loader__`` and
*file_path*, and ``__spec__``, ``__loader__`` and
``__package__`` will all be set to :const:`None`.
If *file_path* is a reference to a valid :data:`sys.path` entry, then
``__spec__`` will be set appropriately for the imported :mod:`__main__`
module (that is, ``__spec__.name`` will always be ``__main__``).
``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` will be
``__file__``, ``__loader__`` and ``__package__`` will be
:ref:`set as normal <import-mod-attrs>` based on the module spec.
A number of alterations are also made to the :mod:`sys` module. Firstly,
@ -173,6 +176,9 @@ The :mod:`runpy` module provides two functions:
The setting of ``__cached__``, ``__loader__``, and
``__package__`` are deprecated.
.. versionchanged:: 3.15
``__cached__`` is no longer set.
.. seealso::
:pep:`338` -- Executing modules as scripts

6
Doc/library/smtplib.rst
View file

@ -458,9 +458,11 @@ An :class:`SMTP` instance has the following methods:
Send mail. The required arguments are an :rfc:`822` from-address string, a list
of :rfc:`822` to-address strings (a bare string will be treated as a list with 1
address), and a message string. The caller may pass a list of ESMTP options
(such as ``8bitmime``) to be used in ``MAIL FROM`` commands as *mail_options*.
(such as ``"8bitmime"``) to be used in ``MAIL FROM`` commands as *mail_options*.
ESMTP options (such as ``DSN`` commands) that should be used with all ``RCPT``
commands can be passed as *rcpt_options*. (If you need to use different ESMTP
commands can be passed as *rcpt_options*. Each option should be passed as a string
containing the full text of the option, including any potential key
(for instance, ``"NOTIFY=SUCCESS,FAILURE"``). (If you need to use different ESMTP
options to different recipients you have to use the low-level methods such as
:meth:`!mail`, :meth:`!rcpt` and :meth:`!data` to send the message.)

2
Doc/library/socket.rst
View file

@ -482,7 +482,7 @@ The AF_* and SOCK_* constants are now :class:`AddressFamily` and
.. versionchanged:: 3.14
Added support for ``TCP_QUICKACK`` on Windows platforms when available.
.. versionchanged:: next
.. versionchanged:: 3.15
``IPV6_HDRINCL`` was added.

449
Doc/library/stdtypes.rst
View file

@ -46,8 +46,10 @@ Any object can be tested for truth value, for use in an :keyword:`if` or
By default, an object is considered true unless its class defines either a
:meth:`~object.__bool__` method that returns ``False`` or a
:meth:`~object.__len__` method that
returns zero, when called with the object. [1]_ Here are most of the built-in
objects considered false:
returns zero, when called with the object. [1]_ If one of the methods raises an
exception when called, the exception is propagated and the object does
not have a truth value (for example, :data:`NotImplemented`).
Here are most of the built-in objects considered false:
.. index::
single: None (Built-in object)
@ -164,7 +166,7 @@ This table summarizes the comparison operations:
pair: object; numeric
pair: objects; comparing
Objects of different types, except different numeric types, never compare equal.
Unless stated otherwise, objects of different types never compare equal.
The ``==`` operator is always defined but for some object types (for example,
class objects) is equivalent to :keyword:`is`. The ``<``, ``<=``, ``>`` and ``>=``
operators are only defined where they make sense; for example, they raise a
@ -2155,6 +2157,21 @@ expression support in the :mod:`re` module).
that have the Unicode numeric value property, e.g. U+2155,
VULGAR FRACTION ONE FIFTH. Formally, numeric characters are those with the property
value Numeric_Type=Digit, Numeric_Type=Decimal or Numeric_Type=Numeric.
For example:
.. doctest::
>>> '0123456789'.isnumeric()
True
>>> '٠١٢٣٤٥٦٧٨٩'.isnumeric() # Arabic-indic digit zero to nine
True
>>> '⅕'.isnumeric() # Vulgar fraction one fifth
True
>>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric()
(False, True, True)
See also :meth:`isdecimal` and :meth:`isdigit`. Numeric characters are
a superset of decimal numbers.
.. method:: str.isprintable()
@ -2246,6 +2263,19 @@ expression support in the :mod:`re` module).
done using the specified *fillchar* (default is an ASCII space). The
original string is returned if *width* is less than or equal to ``len(s)``.
For example:
.. doctest::
>>> 'Python'.ljust(10)
'Python '
>>> 'Python'.ljust(10, '.')
'Python....'
>>> 'Monty Python'.ljust(10, '.')
'Monty Python'
See also :meth:`rjust`.
.. method:: str.lower()
@ -2656,6 +2686,8 @@ expression support in the :mod:`re` module).
single: : (colon); in formatted string literal
single: = (equals); for help in debugging using string literals
.. _stdtypes-fstrings:
Formatted String Literals (f-strings)
-------------------------------------
@ -2664,123 +2696,147 @@ Formatted String Literals (f-strings)
The :keyword:`await` and :keyword:`async for` can be used in expressions
within f-strings.
.. versionchanged:: 3.8
Added the debugging operator (``=``)
Added the debug specifier (``=``)
.. versionchanged:: 3.12
Many restrictions on expressions within f-strings have been removed.
Notably, nested strings, comments, and backslashes are now permitted.
An :dfn:`f-string` (formally a :dfn:`formatted string literal`) is
a string literal that is prefixed with ``f`` or ``F``.
This type of string literal allows embedding arbitrary Python expressions
within *replacement fields*, which are delimited by curly brackets (``{}``).
These expressions are evaluated at runtime, similarly to :meth:`str.format`,
and are converted into regular :class:`str` objects.
For example:
This type of string literal allows embedding the results of arbitrary Python
expressions within *replacement fields*, which are delimited by curly
brackets (``{}``).
Each replacement field must contain an expression, optionally followed by:
.. doctest::
* a *debug specifier* -- an equal sign (``=``);
* a *conversion specifier* -- ``!s``, ``!r`` or ``!a``; and/or
* a *format specifier* prefixed with a colon (``:``).
>>> who = 'nobody'
>>> nationality = 'Spanish'
>>> f'{who.title()} expects the {nationality} Inquisition!'
'Nobody expects the Spanish Inquisition!'
See the :ref:`Lexical Analysis section on f-strings <f-strings>` for details
on the syntax of these fields.
It is also possible to use a multi line f-string:
Debug specifier
^^^^^^^^^^^^^^^
.. doctest::
.. versionadded:: 3.8
>>> f'''This is a string
... on two lines'''
'This is a string\non two lines'
If a debug specifier -- an equal sign (``=``) -- appears after the replacement
field expression, the resulting f-string will contain the expression's source,
the equal sign, and the value of the expression.
This is often useful for debugging::
A single opening curly bracket, ``'{'``, marks a *replacement field* that
can contain any Python expression:
>>> number = 14.3
>>> f'{number=}'
'number=14.3'
.. doctest::
Whitespace before, inside and after the expression, as well as whitespace
after the equal sign, is significant --- it is retained in the result::
>>> nationality = 'Spanish'
>>> f'The {nationality} Inquisition!'
'The Spanish Inquisition!'
>>> f'{ number - 4 = }'
' number - 4 = 10.3'
To include a literal ``{`` or ``}``, use a double bracket:
.. doctest::
Conversion specifier
^^^^^^^^^^^^^^^^^^^^
>>> x = 42
>>> f'{{x}} is {x}'
'{x} is 42'
Functions can also be used, and :ref:`format specifiers <formatstrings>`:
.. doctest::
>>> from math import sqrt
>>> f'√2 \N{ALMOST EQUAL TO} {sqrt(2):.5f}'
'√2 ≈ 1.41421'
Any non-string expression is converted using :func:`str`, by default:
.. doctest::
By default, the value of a replacement field expression is converted to
a string using :func:`str`::
>>> from fractions import Fraction
>>> f'{Fraction(1, 3)}'
>>> one_third = Fraction(1, 3)
>>> f'{one_third}'
'1/3'
To use an explicit conversion, use the ``!`` (exclamation mark) operator,
followed by any of the valid formats, which are:
When a debug specifier but no format specifier is used, the default conversion
instead uses :func:`repr`::
========== ==============
Conversion Meaning
========== ==============
``!a`` :func:`ascii`
``!r`` :func:`repr`
``!s`` :func:`str`
========== ==============
>>> f'{one_third = }'
'one_third = Fraction(1, 3)'
For example:
The conversion can be specified explicitly using one of these specifiers:
.. doctest::
* ``!s`` for :func:`str`
* ``!r`` for :func:`repr`
* ``!a`` for :func:`ascii`
>>> from fractions import Fraction
>>> f'{Fraction(1, 3)!s}'
For example::
>>> str(one_third)
'1/3'
>>> f'{Fraction(1, 3)!r}'
>>> repr(one_third)
'Fraction(1, 3)'
>>> question = '¿Dónde está el Presidente?'
>>> print(f'{question!a}')
'\xbfD\xf3nde est\xe1 el Presidente?'
While debugging it may be helpful to see both the expression and its value,
by using the equals sign (``=``) after the expression.
This preserves spaces within the brackets, and can be used with a converter.
By default, the debugging operator uses the :func:`repr` (``!r``) conversion.
For example:
>>> f'{one_third!s} is {one_third!r}'
'1/3 is Fraction(1, 3)'
.. doctest::
>>> string = "¡kočka 😸!"
>>> ascii(string)
"'\\xa1ko\\u010dka \\U0001f638!'"
>>> f'{string = !a}'
"string = '\\xa1ko\\u010dka \\U0001f638!'"
Format specifier
^^^^^^^^^^^^^^^^
After the expression has been evaluated, and possibly converted using an
explicit conversion specifier, it is formatted using the :func:`format` function.
If the replacement field includes a *format specifier* introduced by a colon
(``:``), the specifier is passed to :func:`!format` as the second argument.
The result of :func:`!format` is then used as the final value for the
replacement field. For example::
>>> from fractions import Fraction
>>> calculation = Fraction(1, 3)
>>> f'{calculation=}'
'calculation=Fraction(1, 3)'
>>> f'{calculation = }'
'calculation = Fraction(1, 3)'
>>> f'{calculation = !s}'
'calculation = 1/3'
>>> one_third = Fraction(1, 3)
>>> f'{one_third:.6f}'
'0.333333'
>>> f'{one_third:_^+10}'
'___+1/3___'
>>> >>> f'{one_third!r:_^20}'
'___Fraction(1, 3)___'
>>> f'{one_third = :~>10}~'
'one_third = ~~~~~~~1/3~'
Once the output has been evaluated, it can be formatted using a
:ref:`format specifier <formatstrings>` following a colon (``':'``).
After the expression has been evaluated, and possibly converted to a string,
the :meth:`!__format__` method of the result is called with the format specifier,
or the empty string if no format specifier is given.
The formatted result is then used as the final value for the replacement field.
For example:
.. _stdtypes-tstrings:
.. doctest::
Template String Literals (t-strings)
------------------------------------
>>> from fractions import Fraction
>>> f'{Fraction(1, 7):.6f}'
'0.142857'
>>> f'{Fraction(1, 7):_^+10}'
'___+1/7___'
An :dfn:`t-string` (formally a :dfn:`template string literal`) is
a string literal that is prefixed with ``t`` or ``T``.
These strings follow the same syntax and evaluation rules as
:ref:`formatted string literals <stdtypes-fstrings>`,
with for the following differences:
* Rather than evaluating to a ``str`` object, template string literals evaluate
to a :class:`string.templatelib.Template` object.
* The :func:`format` protocol is not used.
Instead, the format specifier and conversions (if any) are passed to
a new :class:`~string.templatelib.Interpolation` object that is created
for each evaluated expression.
It is up to code that processes the resulting :class:`~string.templatelib.Template`
object to decide how to handle format specifiers and conversions.
* Format specifiers containing nested replacement fields are evaluated eagerly,
prior to being passed to the :class:`~string.templatelib.Interpolation` object.
For instance, an interpolation of the form ``{amount:.{precision}f}`` will
evaluate the inner expression ``{precision}`` to determine the value of the
``format_spec`` attribute.
If ``precision`` were to be ``2``, the resulting format specifier
would be ``'.2f'``.
* When the equals sign ``'='`` is provided in an interpolation expression,
the text of the expression is appended to the literal string that precedes
the relevant interpolation.
This includes the equals sign and any surrounding whitespace.
The :class:`!Interpolation` instance for the expression will be created as
normal, except that :attr:`~string.templatelib.Interpolation.conversion` will
be set to '``r``' (:func:`repr`) by default.
If an explicit conversion or format specifier are provided,
this will override the default behaviour.
.. _old-string-formatting:
@ -4800,7 +4856,7 @@ other sequence-like behavior.
There are currently two built-in set types, :class:`set` and :class:`frozenset`.
The :class:`set` type is mutable --- the contents can be changed using methods
like :meth:`add <frozenset.add>` and :meth:`remove <frozenset.add>`.
like :meth:`~set.add` and :meth:`~set.remove`.
Since it is mutable, it has no hash value and cannot be used as
either a dictionary key or as an element of another set.
The :class:`frozenset` type is immutable and :term:`hashable` ---
@ -4822,164 +4878,172 @@ The constructors for both classes work the same:
objects. If *iterable* is not specified, a new empty set is
returned.
Sets can be created by several means:
Sets can be created by several means:
* Use a comma-separated list of elements within braces: ``{'jack', 'sjoerd'}``
* Use a set comprehension: ``{c for c in 'abracadabra' if c not in 'abc'}``
* Use the type constructor: ``set()``, ``set('foobar')``, ``set(['a', 'b', 'foo'])``
* Use a comma-separated list of elements within braces: ``{'jack', 'sjoerd'}``
* Use a set comprehension: ``{c for c in 'abracadabra' if c not in 'abc'}``
* Use the type constructor: ``set()``, ``set('foobar')``, ``set(['a', 'b', 'foo'])``
Instances of :class:`set` and :class:`frozenset` provide the following
operations:
Instances of :class:`set` and :class:`frozenset` provide the following
operations:
.. describe:: len(s)
.. describe:: len(s)
Return the number of elements in set *s* (cardinality of *s*).
Return the number of elements in set *s* (cardinality of *s*).
.. describe:: x in s
.. describe:: x in s
Test *x* for membership in *s*.
Test *x* for membership in *s*.
.. describe:: x not in s
.. describe:: x not in s
Test *x* for non-membership in *s*.
Test *x* for non-membership in *s*.
.. method:: isdisjoint(other, /)
.. method:: frozenset.isdisjoint(other, /)
set.isdisjoint(other, /)
Return ``True`` if the set has no elements in common with *other*. Sets are
disjoint if and only if their intersection is the empty set.
Return ``True`` if the set has no elements in common with *other*. Sets are
disjoint if and only if their intersection is the empty set.
.. method:: issubset(other, /)
set <= other
.. method:: frozenset.issubset(other, /)
set.issubset(other, /)
.. describe:: set <= other
Test whether every element in the set is in *other*.
Test whether every element in the set is in *other*.
.. method:: set < other
.. describe:: set < other
Test whether the set is a proper subset of *other*, that is,
``set <= other and set != other``.
Test whether the set is a proper subset of *other*, that is,
``set <= other and set != other``.
.. method:: issuperset(other, /)
set >= other
.. method:: frozenset.issuperset(other, /)
set.issuperset(other, /)
.. describe:: set >= other
Test whether every element in *other* is in the set.
Test whether every element in *other* is in the set.
.. method:: set > other
.. describe:: set > other
Test whether the set is a proper superset of *other*, that is, ``set >=
other and set != other``.
Test whether the set is a proper superset of *other*, that is, ``set >=
other and set != other``.
.. method:: union(*others)
set | other | ...
.. method:: frozenset.union(*others)
set.union(*others)
.. describe:: set | other | ...
Return a new set with elements from the set and all others.
Return a new set with elements from the set and all others.
.. method:: intersection(*others)
set & other & ...
.. method:: frozenset.intersection(*others)
set.intersection(*others)
.. describe:: set & other & ...
Return a new set with elements common to the set and all others.
Return a new set with elements common to the set and all others.
.. method:: difference(*others)
set - other - ...
.. method:: frozenset.difference(*others)
set.difference(*others)
.. describe:: set - other - ...
Return a new set with elements in the set that are not in the others.
Return a new set with elements in the set that are not in the others.
.. method:: symmetric_difference(other, /)
set ^ other
.. method:: frozenset.symmetric_difference(other, /)
set.symmetric_difference(other, /)
.. describe:: set ^ other
Return a new set with elements in either the set or *other* but not both.
Return a new set with elements in either the set or *other* but not both.
.. method:: copy()
.. method:: frozenset.copy()
set.copy()
Return a shallow copy of the set.
Return a shallow copy of the set.
Note, the non-operator versions of :meth:`union`, :meth:`intersection`,
:meth:`difference`, :meth:`symmetric_difference`, :meth:`issubset`, and
:meth:`issuperset` methods will accept any iterable as an argument. In
contrast, their operator based counterparts require their arguments to be
sets. This precludes error-prone constructions like ``set('abc') & 'cbs'``
in favor of the more readable ``set('abc').intersection('cbs')``.
Note, the non-operator versions of :meth:`~frozenset.union`,
:meth:`~frozenset.intersection`, :meth:`~frozenset.difference`, :meth:`~frozenset.symmetric_difference`, :meth:`~frozenset.issubset`, and
:meth:`~frozenset.issuperset` methods will accept any iterable as an argument. In
contrast, their operator based counterparts require their arguments to be
sets. This precludes error-prone constructions like ``set('abc') & 'cbs'``
in favor of the more readable ``set('abc').intersection('cbs')``.
Both :class:`set` and :class:`frozenset` support set to set comparisons. Two
sets are equal if and only if every element of each set is contained in the
other (each is a subset of the other). A set is less than another set if and
only if the first set is a proper subset of the second set (is a subset, but
is not equal). A set is greater than another set if and only if the first set
is a proper superset of the second set (is a superset, but is not equal).
Both :class:`set` and :class:`frozenset` support set to set comparisons. Two
sets are equal if and only if every element of each set is contained in the
other (each is a subset of the other). A set is less than another set if and
only if the first set is a proper subset of the second set (is a subset, but
is not equal). A set is greater than another set if and only if the first set
is a proper superset of the second set (is a superset, but is not equal).
Instances of :class:`set` are compared to instances of :class:`frozenset`
based on their members. For example, ``set('abc') == frozenset('abc')``
returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``.
Instances of :class:`set` are compared to instances of :class:`frozenset`
based on their members. For example, ``set('abc') == frozenset('abc')``
returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``.
The subset and equality comparisons do not generalize to a total ordering
function. For example, any two nonempty disjoint sets are not equal and are not
subsets of each other, so *all* of the following return ``False``: ``a<b``,
``a==b``, or ``a>b``.
The subset and equality comparisons do not generalize to a total ordering
function. For example, any two nonempty disjoint sets are not equal and are not
subsets of each other, so *all* of the following return ``False``: ``a<b``,
``a==b``, or ``a>b``.
Since sets only define partial ordering (subset relationships), the output of
the :meth:`list.sort` method is undefined for lists of sets.
Since sets only define partial ordering (subset relationships), the output of
the :meth:`list.sort` method is undefined for lists of sets.
Set elements, like dictionary keys, must be :term:`hashable`.
Set elements, like dictionary keys, must be :term:`hashable`.
Binary operations that mix :class:`set` instances with :class:`frozenset`
return the type of the first operand. For example: ``frozenset('ab') |
set('bc')`` returns an instance of :class:`frozenset`.
Binary operations that mix :class:`set` instances with :class:`frozenset`
return the type of the first operand. For example: ``frozenset('ab') |
set('bc')`` returns an instance of :class:`frozenset`.
The following table lists operations available for :class:`set` that do not
apply to immutable instances of :class:`frozenset`:
The following table lists operations available for :class:`set` that do not
apply to immutable instances of :class:`frozenset`:
.. method:: update(*others)
set |= other | ...
.. method:: set.update(*others)
.. describe:: set |= other | ...
Update the set, adding elements from all others.
Update the set, adding elements from all others.
.. method:: intersection_update(*others)
set &= other & ...
.. method:: set.intersection_update(*others)
.. describe:: set &= other & ...
Update the set, keeping only elements found in it and all others.
Update the set, keeping only elements found in it and all others.
.. method:: difference_update(*others)
set -= other | ...
.. method:: set.difference_update(*others)
.. describe:: set -= other | ...
Update the set, removing elements found in others.
Update the set, removing elements found in others.
.. method:: symmetric_difference_update(other, /)
set ^= other
.. method:: set.symmetric_difference_update(other, /)
.. describe:: set ^= other
Update the set, keeping only elements found in either set, but not in both.
Update the set, keeping only elements found in either set, but not in both.
.. method:: add(elem, /)
.. method:: set.add(elem, /)
Add element *elem* to the set.
Add element *elem* to the set.
.. method:: remove(elem, /)
.. method:: set.remove(elem, /)
Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is
not contained in the set.
Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is
not contained in the set.
.. method:: discard(elem, /)
.. method:: set.discard(elem, /)
Remove element *elem* from the set if it is present.
Remove element *elem* from the set if it is present.
.. method:: pop()
.. method:: set.pop()
Remove and return an arbitrary element from the set. Raises
:exc:`KeyError` if the set is empty.
Remove and return an arbitrary element from the set. Raises
:exc:`KeyError` if the set is empty.
.. method:: clear()
.. method:: set.clear()
Remove all elements from the set.
Remove all elements from the set.
Note, the non-operator versions of the :meth:`update`,
:meth:`intersection_update`, :meth:`difference_update`, and
:meth:`symmetric_difference_update` methods will accept any iterable as an
argument.
Note, the non-operator versions of the :meth:`~set.update`,
:meth:`~set.intersection_update`, :meth:`~set.difference_update`, and
:meth:`~set.symmetric_difference_update` methods will accept any iterable as an
argument.
Note, the *elem* argument to the :meth:`~object.__contains__`,
:meth:`remove`, and
:meth:`discard` methods may be a set. To support searching for an equivalent
frozenset, a temporary one is created from *elem*.
Note, the *elem* argument to the :meth:`~object.__contains__`,
:meth:`~set.remove`, and
:meth:`~set.discard` methods may be a set. To support searching for an equivalent
frozenset, a temporary one is created from *elem*.
.. _typesmapping:
@ -5038,9 +5102,6 @@ can be used interchangeably to index the same dictionary entry.
being added is already present, the value from the keyword argument
replaces the value from the positional argument.
Providing keyword arguments as in the first example only works for keys that
are valid Python identifiers. Otherwise, any valid keys can be used.
Dictionaries compare equal if and only if they have the same ``(key,
value)`` pairs (regardless of ordering). Order comparisons ('<', '<=', '>=', '>') raise
:exc:`TypeError`. To illustrate dictionary creation and equality,
@ -5426,9 +5487,11 @@ before the statement body is executed and exited when the statement ends:
Returning a true value from this method will cause the :keyword:`with` statement
to suppress the exception and continue execution with the statement immediately
following the :keyword:`!with` statement. Otherwise the exception continues
propagating after this method has finished executing. Exceptions that occur
during execution of this method will replace any exception that occurred in the
body of the :keyword:`!with` statement.
propagating after this method has finished executing.
If this method raises an exception while handling an earlier exception from the
:keyword:`with` block, the new exception is raised, and the original exception
is stored in its :attr:`~BaseException.__context__` attribute.
The exception passed in should never be reraised explicitly - instead, this
method should return a false value to indicate that the method completed

3
Doc/library/string.rst
View file

@ -546,6 +546,9 @@ The available presentation types for :class:`float` and
| | :class:`float`, and shows all coefficient digits |
| | for :class:`~decimal.Decimal`. If ``p=0``, the decimal |
| | point is omitted unless the ``#`` option is used. |
| | |
| | For :class:`float`, the exponent always contains at |
| | least two digits, and is zero if the value is zero. |
+---------+----------------------------------------------------------+
| ``'E'`` | Scientific notation. Same as ``'e'`` except it uses |
| | an upper case 'E' as the separator character. |

3
Doc/library/superseded.rst
View file

@ -1,7 +1,7 @@
.. _superseded:
******************
Superseded Modules
Superseded modules
******************
The modules described in this chapter have been superseded by other modules
@ -24,3 +24,4 @@ currently no modules in this latter category.
:maxdepth: 1
getopt.rst
profile.rst

BIN
Doc/library/tachyon-flamegraph.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 60 KiB

BIN
Doc/library/tachyon-gecko-calltree.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 83 KiB

BIN
Doc/library/tachyon-gecko-flamegraph.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 39 KiB

BIN
Doc/library/tachyon-gecko-opcodes.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 49 KiB

BIN
Doc/library/tachyon-heatmap-with-opcodes.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 69 KiB

BIN
Doc/library/tachyon-heatmap.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 34 KiB

BIN
Doc/library/tachyon-live-mode-1.gif Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 6.2 MiB

BIN
Doc/library/tachyon-live-mode-2.gif Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 3.4 MiB

BIN
Doc/library/tachyon-logo.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 110 KiB

BIN
Doc/library/tachyon-pstats.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 107 KiB

4
Doc/library/typing.rst
View file

@ -2869,8 +2869,8 @@ ABCs and Protocols for working with I/O
---------------------------------------
.. class:: IO[AnyStr]
TextIO[AnyStr]
BinaryIO[AnyStr]
TextIO
BinaryIO
Generic class ``IO[AnyStr]`` and its subclasses ``TextIO(IO[str])``
and ``BinaryIO(IO[bytes])``

2
Doc/reference/compound_stmts.rst
View file

@ -388,7 +388,7 @@ type of the target ``e`` is consistently :exc:`BaseExceptionGroup`::
... except* BlockingIOError as e:
... print(repr(e))
...
ExceptionGroup('', (BlockingIOError()))
ExceptionGroup('', (BlockingIOError(),))
:keyword:`break`, :keyword:`continue` and :keyword:`return`
cannot appear in an :keyword:`!except*` clause.

42
Doc/reference/datamodel.rst
View file

@ -449,7 +449,7 @@ Sets
These represent a mutable set. They are created by the built-in :func:`set`
constructor and can be modified afterwards by several methods, such as
:meth:`add <frozenset.add>`.
:meth:`~set.add`.
Frozen sets
@ -895,7 +895,6 @@ Attribute assignment updates the module's namespace dictionary, e.g.,
single: __loader__ (module attribute)
single: __path__ (module attribute)
single: __file__ (module attribute)
single: __cached__ (module attribute)
single: __doc__ (module attribute)
single: __annotations__ (module attribute)
single: __annotate__ (module attribute)
@ -1044,43 +1043,28 @@ this approach.
instead of :attr:`!module.__path__`.
.. attribute:: module.__file__
.. attribute:: module.__cached__
:attr:`!__file__` and :attr:`!__cached__` are both optional attributes that
:attr:`!__file__` is an optional attribute that
may or may not be set. Both attributes should be a :class:`str` when they
are available.
:attr:`!__file__` indicates the pathname of the file from which the module
was loaded (if loaded from a file), or the pathname of the shared library
file for extension modules loaded dynamically from a shared library.
It might be missing for certain types of modules, such as C modules that are
statically linked into the interpreter, and the
An optional attribute, :attr:`!__file__` indicates the pathname of the file
from which the module was loaded (if loaded from a file), or the pathname of
the shared library file for extension modules loaded dynamically from a
shared library. It might be missing for certain types of modules, such as C
modules that are statically linked into the interpreter, and the
:ref:`import system <importsystem>` may opt to leave it unset if it
has no semantic meaning (for example, a module loaded from a database).
If :attr:`!__file__` is set then the :attr:`!__cached__` attribute might
also be set, which is the path to any compiled version of
the code (for example, a byte-compiled file). The file does not need to exist
to set this attribute; the path can simply point to where the
compiled file *would* exist (see :pep:`3147`).
Note that :attr:`!__cached__` may be set even if :attr:`!__file__` is not
set. However, that scenario is quite atypical. Ultimately, the
:term:`loader` is what makes use of the module spec provided by the
:term:`finder` (from which :attr:`!__file__` and :attr:`!__cached__` are
derived). So if a loader can load from a cached module but otherwise does
not load from a file, that atypical scenario may be appropriate.
It is **strongly** recommended that you use
:attr:`module.__spec__.cached <importlib.machinery.ModuleSpec.cached>`
instead of :attr:`!module.__cached__`.
.. deprecated-removed:: 3.13 3.15
Setting :attr:`!__cached__` on a module while failing to set
Setting ``__cached__`` on a module while failing to set
:attr:`!__spec__.cached` is deprecated. In Python 3.15,
:attr:`!__cached__` will cease to be set or taken into consideration by
``__cached__`` will cease to be set or taken into consideration by
the import system or standard library.
.. versionchanged:: 3.15
``__cached__`` is no longer set.
Other writable attributes on module objects
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@ -2656,7 +2640,7 @@ Notes on using *__slots__*:
of the iterator's values. However, the *__slots__* attribute will be an empty
iterator.
.. versionchanged:: next
.. versionchanged:: 3.15
Allowed defining the *__dict__* and *__weakref__* *__slots__* for any class.

2
Doc/reference/expressions.rst
View file

@ -174,7 +174,7 @@ Formally:
.. grammar-snippet::
:group: python-grammar
strings: ( `STRING` | fstring)+ | tstring+
strings: ( `STRING` | `fstring`)+ | `tstring`+
This feature is defined at the syntactical level, so it only works with literals.
To concatenate string expressions at run time, the '+' operator may be used::

30
Doc/reference/import.rst
View file

@ -359,21 +359,16 @@ of what happens during the loading portion of import::
if spec.loader is None:
# unsupported
raise ImportError
if spec.origin is None and spec.submodule_search_locations is not None:
# namespace package
sys.modules[spec.name] = module
elif not hasattr(spec.loader, 'exec_module'):
module = spec.loader.load_module(spec.name)
else:
sys.modules[spec.name] = module
sys.modules[spec.name] = module
try:
spec.loader.exec_module(module)
except BaseException:
try:
spec.loader.exec_module(module)
except BaseException:
try:
del sys.modules[spec.name]
except KeyError:
pass
raise
del sys.modules[spec.name]
except KeyError:
pass
raise
return sys.modules[spec.name]
Note the following details:
@ -408,7 +403,10 @@ Note the following details:
.. versionchanged:: 3.4
The import system has taken over the boilerplate responsibilities of
loaders. These were previously performed by the
:meth:`importlib.abc.Loader.load_module` method.
``importlib.abc.Loader.load_module`` method.
.. versionchanged:: 3.15
The ``load_module`` method is no longer used.
Loaders
-------
@ -443,7 +441,7 @@ import machinery will create the new module itself.
The :meth:`~importlib.abc.Loader.create_module` method of loaders.
.. versionchanged:: 3.4
The :meth:`~importlib.abc.Loader.load_module` method was replaced by
The ``importlib.abc.Loader.load_module`` method was replaced by
:meth:`~importlib.abc.Loader.exec_module` and the import
machinery assumed all the boilerplate responsibilities of loading.

363
Doc/reference/lexical_analysis.rst
View file

@ -345,7 +345,15 @@ Whitespace between tokens
Except at the beginning of a logical line or in string literals, the whitespace
characters space, tab and formfeed can be used interchangeably to separate
tokens. Whitespace is needed between two tokens only if their concatenation
tokens:
.. grammar-snippet::
:group: python-grammar
whitespace: ' ' | tab | formfeed
Whitespace is needed between two tokens only if their concatenation
could otherwise be interpreted as a different token. For example, ``ab`` is one
token, but ``a b`` is two tokens. However, ``+a`` and ``+ a`` both produce
two tokens, ``+`` and ``a``, as ``+a`` is not a valid token.
@ -1032,124 +1040,59 @@ f-strings
---------
.. versionadded:: 3.6
.. versionchanged:: 3.7
The :keyword:`await` and :keyword:`async for` can be used in expressions
within f-strings.
.. versionchanged:: 3.8
Added the debug specifier (``=``)
.. versionchanged:: 3.12
Many restrictions on expressions within f-strings have been removed.
Notably, nested strings, comments, and backslashes are now permitted.
A :dfn:`formatted string literal` or :dfn:`f-string` is a string literal
that is prefixed with '``f``' or '``F``'. These strings may contain
replacement fields, which are expressions delimited by curly braces ``{}``.
While other string literals always have a constant value, formatted strings
are really expressions evaluated at run time.
that is prefixed with '``f``' or '``F``'.
Unlike other string literals, f-strings do not have a constant value.
They may contain *replacement fields* delimited by curly braces ``{}``.
Replacement fields contain expressions which are evaluated at run time.
For example::
Escape sequences are decoded like in ordinary string literals (except when
a literal is also marked as a raw string). After decoding, the grammar
for the contents of the string is:
>>> who = 'nobody'
>>> nationality = 'Spanish'
>>> f'{who.title()} expects the {nationality} Inquisition!'
'Nobody expects the Spanish Inquisition!'
.. productionlist:: python-grammar
f_string: (`literal_char` | "{{" | "}}" | `replacement_field`)*
replacement_field: "{" `f_expression` ["="] ["!" `conversion`] [":" `format_spec`] "}"
f_expression: (`conditional_expression` | "*" `or_expr`)
: ("," `conditional_expression` | "," "*" `or_expr`)* [","]
: | `yield_expression`
conversion: "s" | "r" | "a"
format_spec: (`literal_char` | `replacement_field`)*
literal_char: <any code point except "{", "}" or NULL>
Any doubled curly braces (``{{`` or ``}}``) outside replacement fields
are replaced with the corresponding single curly brace::
The parts of the string outside curly braces are treated literally,
except that any doubled curly braces ``'{{'`` or ``'}}'`` are replaced
with the corresponding single curly brace. A single opening curly
bracket ``'{'`` marks a replacement field, which starts with a
Python expression. To display both the expression text and its value after
evaluation, (useful in debugging), an equal sign ``'='`` may be added after the
expression. A conversion field, introduced by an exclamation point ``'!'`` may
follow. A format specifier may also be appended, introduced by a colon ``':'``.
A replacement field ends with a closing curly bracket ``'}'``.
>>> print(f'{{...}}')
{...}
Other characters outside replacement fields are treated like in ordinary
string literals.
This means that escape sequences are decoded (except when a literal is
also marked as a raw string), and newlines are possible in triple-quoted
f-strings::
>>> name = 'Galahad'
>>> favorite_color = 'blue'
>>> print(f'{name}:\t{favorite_color}')
Galahad: blue
>>> print(rf"C:\Users\{name}")
C:\Users\Galahad
>>> print(f'''Three shall be the number of the counting
... and the number of the counting shall be three.''')
Three shall be the number of the counting
and the number of the counting shall be three.
Expressions in formatted string literals are treated like regular
Python expressions surrounded by parentheses, with a few exceptions.
An empty expression is not allowed, and both :keyword:`lambda` and
assignment expressions ``:=`` must be surrounded by explicit parentheses.
Python expressions.
Each expression is evaluated in the context where the formatted string literal
appears, in order from left to right. Replacement expressions can contain
newlines in both single-quoted and triple-quoted f-strings and they can contain
comments. Everything that comes after a ``#`` inside a replacement field
is a comment (even closing braces and quotes). In that case, replacement fields
must be closed in a different line.
.. code-block:: text
>>> f"abc{a # This is a comment }"
... + 3}"
'abc5'
.. versionchanged:: 3.7
Prior to Python 3.7, an :keyword:`await` expression and comprehensions
containing an :keyword:`async for` clause were illegal in the expressions
in formatted string literals due to a problem with the implementation.
.. versionchanged:: 3.12
Prior to Python 3.12, comments were not allowed inside f-string replacement
fields.
When the equal sign ``'='`` is provided, the output will have the expression
text, the ``'='`` and the evaluated value. Spaces after the opening brace
``'{'``, within the expression and after the ``'='`` are all retained in the
output. By default, the ``'='`` causes the :func:`repr` of the expression to be
provided, unless there is a format specified. When a format is specified it
defaults to the :func:`str` of the expression unless a conversion ``'!r'`` is
declared.
.. versionadded:: 3.8
The equal sign ``'='``.
If a conversion is specified, the result of evaluating the expression
is converted before formatting. Conversion ``'!s'`` calls :func:`str` on
the result, ``'!r'`` calls :func:`repr`, and ``'!a'`` calls :func:`ascii`.
The result is then formatted using the :func:`format` protocol. The
format specifier is passed to the :meth:`~object.__format__` method of the
expression or conversion result. An empty string is passed when the
format specifier is omitted. The formatted result is then included in
the final value of the whole string.
Top-level format specifiers may include nested replacement fields. These nested
fields may include their own conversion fields and :ref:`format specifiers
<formatspec>`, but may not include more deeply nested replacement fields. The
:ref:`format specifier mini-language <formatspec>` is the same as that used by
the :meth:`str.format` method.
Formatted string literals may be concatenated, but replacement fields
cannot be split across literals.
Some examples of formatted string literals::
>>> name = "Fred"
>>> f"He said his name is {name!r}."
"He said his name is 'Fred'."
>>> f"He said his name is {repr(name)}." # repr() is equivalent to !r
"He said his name is 'Fred'."
>>> width = 10
>>> precision = 4
>>> value = decimal.Decimal("12.34567")
>>> f"result: {value:{width}.{precision}}" # nested fields
'result: 12.35'
>>> today = datetime(year=2017, month=1, day=27)
>>> f"{today:%B %d, %Y}" # using date format specifier
'January 27, 2017'
>>> f"{today=:%B %d, %Y}" # using date format specifier and debugging
'today=January 27, 2017'
>>> number = 1024
>>> f"{number:#0x}" # using integer format specifier
'0x400'
>>> foo = "bar"
>>> f"{ foo = }" # preserves whitespace
" foo = 'bar'"
>>> line = "The mill's closed"
>>> f"{line = }"
'line = "The mill\'s closed"'
>>> f"{line = :20}"
"line = The mill's closed "
>>> f"{line = !r:20}"
'line = "The mill\'s closed" '
appears, in order from left to right.
An empty expression is not allowed, and both :keyword:`lambda` and
assignment expressions ``:=`` must be surrounded by explicit parentheses::
>>> f'{(half := 1/2)}, {half * 42}'
'0.5, 21.0'
Reusing the outer f-string quoting type inside a replacement field is
permitted::
@ -1158,10 +1101,6 @@ permitted::
>>> f"abc {a["x"]} def"
'abc 2 def'
.. versionchanged:: 3.12
Prior to Python 3.12, reuse of the same quoting type of the outer f-string
inside a replacement field was not possible.
Backslashes are also allowed in replacement fields and are evaluated the same
way as in any other context::
@ -1172,23 +1111,84 @@ way as in any other context::
b
c
.. versionchanged:: 3.12
Prior to Python 3.12, backslashes were not permitted inside an f-string
replacement field.
It is possible to nest f-strings::
Formatted string literals cannot be used as docstrings, even if they do not
include expressions.
>>> name = 'world'
>>> f'Repeated:{f' hello {name}' * 3}'
'Repeated: hello world hello world hello world'
::
Portable Python programs should not use more than 5 levels of nesting.
.. impl-detail::
CPython does not limit nesting of f-strings.
Replacement expressions can contain newlines in both single-quoted and
triple-quoted f-strings and they can contain comments.
Everything that comes after a ``#`` inside a replacement field
is a comment (even closing braces and quotes).
This means that replacement fields with comments must be closed in a
different line:
.. code-block:: text
>>> a = 2
>>> f"abc{a # This comment }" continues until the end of the line
... + 3}"
'abc5'
After the expression, replacement fields may optionally contain:
* a *debug specifier* -- an equal sign (``=``), optionally surrounded by
whitespace on one or both sides;
* a *conversion specifier* -- ``!s``, ``!r`` or ``!a``; and/or
* a *format specifier* prefixed with a colon (``:``).
See the :ref:`Standard Library section on f-strings <stdtypes-fstrings>`
for details on how these fields are evaluated.
As that section explains, *format specifiers* are passed as the second argument
to the :func:`format` function to format a replacement field value.
For example, they can be used to specify a field width and padding characters
using the :ref:`Format Specification Mini-Language <formatspec>`::
>>> number = 14.3
>>> f'{number:20.7f}'
' 14.3000000'
Top-level format specifiers may include nested replacement fields::
>>> field_size = 20
>>> precision = 7
>>> f'{number:{field_size}.{precision}f}'
' 14.3000000'
These nested fields may include their own conversion fields and
:ref:`format specifiers <formatspec>`::
>>> number = 3
>>> f'{number:{field_size}}'
' 3'
>>> f'{number:{field_size:05}}'
'00000000000000000003'
However, these nested fields may not include more deeply nested replacement
fields.
Formatted string literals cannot be used as :term:`docstrings <docstring>`,
even if they do not include expressions::
>>> def foo():
... f"Not a docstring"
...
>>> foo.__doc__ is None
True
>>> print(foo.__doc__)
None
See also :pep:`498` for the proposal that added formatted string literals,
and :meth:`str.format`, which uses a related format string mechanism.
.. seealso::
* :pep:`498` -- Literal String Interpolation
* :pep:`701` -- Syntactic formalization of f-strings
* :meth:`str.format`, which uses a related format string mechanism.
.. _t-strings:
@ -1201,36 +1201,99 @@ t-strings
A :dfn:`template string literal` or :dfn:`t-string` is a string literal
that is prefixed with '``t``' or '``T``'.
These strings follow the same syntax and evaluation rules as
:ref:`formatted string literals <f-strings>`, with the following differences:
These strings follow the same syntax rules as
:ref:`formatted string literals <f-strings>`.
For differences in evaluation rules, see the
:ref:`Standard Library section on t-strings <stdtypes-tstrings>`
* Rather than evaluating to a ``str`` object, template string literals evaluate
to a :class:`string.templatelib.Template` object.
* The :func:`format` protocol is not used.
Instead, the format specifier and conversions (if any) are passed to
a new :class:`~string.templatelib.Interpolation` object that is created
for each evaluated expression.
It is up to code that processes the resulting :class:`~string.templatelib.Template`
object to decide how to handle format specifiers and conversions.
Formal grammar for f-strings
----------------------------
* Format specifiers containing nested replacement fields are evaluated eagerly,
prior to being passed to the :class:`~string.templatelib.Interpolation` object.
For instance, an interpolation of the form ``{amount:.{precision}f}`` will
evaluate the inner expression ``{precision}`` to determine the value of the
``format_spec`` attribute.
If ``precision`` were to be ``2``, the resulting format specifier
would be ``'.2f'``.
F-strings are handled partly by the :term:`lexical analyzer`, which produces the
tokens :py:data:`~token.FSTRING_START`, :py:data:`~token.FSTRING_MIDDLE`
and :py:data:`~token.FSTRING_END`, and partly by the parser, which handles
expressions in the replacement field.
The exact way the work is split is a CPython implementation detail.
* When the equals sign ``'='`` is provided in an interpolation expression,
the text of the expression is appended to the literal string that precedes
the relevant interpolation.
This includes the equals sign and any surrounding whitespace.
The :class:`!Interpolation` instance for the expression will be created as
normal, except that :attr:`~string.templatelib.Interpolation.conversion` will
be set to '``r``' (:func:`repr`) by default.
If an explicit conversion or format specifier are provided,
this will override the default behaviour.
Correspondingly, the f-string grammar is a mix of
:ref:`lexical and syntactic definitions <notation-lexical-vs-syntactic>`.
Whitespace is significant in these situations:
* There may be no whitespace in :py:data:`~token.FSTRING_START` (between
the prefix and quote).
* Whitespace in :py:data:`~token.FSTRING_MIDDLE` is part of the literal
string contents.
* In ``fstring_replacement_field``, if ``f_debug_specifier`` is present,
all whitespace after the opening brace until the ``f_debug_specifier``,
as well as whitespace immediately following ``f_debug_specifier``,
is retained as part of the expression.
.. impl-detail::
The expression is not handled in the tokenization phase; it is
retrieved from the source code using locations of the ``{`` token
and the token after ``=``.
The ``FSTRING_MIDDLE`` definition uses
:ref:`negative lookaheads <lexical-lookaheads>` (``!``)
to indicate special characters (backslash, newline, ``{``, ``}``) and
sequences (``f_quote``).
.. grammar-snippet::
:group: python-grammar
fstring: `FSTRING_START` `fstring_middle`* `FSTRING_END`
FSTRING_START: `fstringprefix` ("'" | '"' | "'''" | '"""')
FSTRING_END: `f_quote`
fstringprefix: <("f" | "fr" | "rf"), case-insensitive>
f_debug_specifier: '='
f_quote: <the quote character(s) used in FSTRING_START>
fstring_middle:
| `fstring_replacement_field`
| `FSTRING_MIDDLE`
FSTRING_MIDDLE:
| (!"\" !`newline` !'{' !'}' !`f_quote`) `source_character`
| `stringescapeseq`
| "{{"
| "}}"
| <newline, in triple-quoted f-strings only>
fstring_replacement_field:
| '{' `f_expression` [`f_debug_specifier`] [`fstring_conversion`]
[`fstring_full_format_spec`] '}'
fstring_conversion:
| "!" ("s" | "r" | "a")
fstring_full_format_spec:
| ':' `fstring_format_spec`*
fstring_format_spec:
| `FSTRING_MIDDLE`
| `fstring_replacement_field`
f_expression:
| ','.(`conditional_expression` | "*" `or_expr`)+ [","]
| `yield_expression`
.. note::
In the above grammar snippet, the ``f_quote`` and ``FSTRING_MIDDLE`` rules
are context-sensitive -- they depend on the contents of ``FSTRING_START``
of the nearest enclosing ``fstring``.
Constructing a more traditional formal grammar from this template is left
as an exercise for the reader.
The grammar for t-strings is identical to the one for f-strings, with *t*
instead of *f* at the beginning of rule and token names and in the prefix.
.. grammar-snippet::
:group: python-grammar
tstring: TSTRING_START tstring_middle* TSTRING_END
<rest of the t-string grammar is omitted; see above>
.. _numbers:

1
Doc/tools/.nitignore
View file

@ -23,7 +23,6 @@ Doc/library/multiprocessing.rst
Doc/library/optparse.rst
Doc/library/os.rst
Doc/library/pickletools.rst
Doc/library/profile.rst
Doc/library/pyexpat.rst
Doc/library/select.rst
Doc/library/socket.rst

6
Doc/tools/templates/customsourcelink.html
View file

@ -8,6 +8,12 @@
rel="nofollow">{{ _('Show source') }}
</a>
</li>
{% if language != "en" %}
<li>
<a href="https://github.com/python/python-docs-{{ language }}/blob/{{ version }}/{{ pagename }}.po?plain=1"
rel="nofollow">{{ _('Show translation source') }}</a>
</li>
{% endif %}
</ul>
</div>
{%- endif %}

23
Doc/tutorial/controlflow.rst
View file

@ -1039,31 +1039,28 @@ blank, visually separating the summary from the rest of the description. The
following lines should be one or more paragraphs describing the object's calling
conventions, its side effects, etc.
The Python parser does not strip indentation from multi-line string literals in
Python, so tools that process documentation have to strip indentation if
desired. This is done using the following convention. The first non-blank line
*after* the first line of the string determines the amount of indentation for
the entire documentation string. (We can't use the first line since it is
generally adjacent to the string's opening quotes so its indentation is not
apparent in the string literal.) Whitespace "equivalent" to this indentation is
then stripped from the start of all lines of the string. Lines that are
indented less should not occur, but if they occur all their leading whitespace
should be stripped. Equivalence of whitespace should be tested after expansion
of tabs (to 8 spaces, normally).
The Python parser strips indentation from multi-line string literals when they
serve as module, class, or function docstrings.
Here is an example of a multi-line docstring::
>>> def my_function():
... """Do nothing, but document it.
...
... No, really, it doesn't do anything.
... No, really, it doesn't do anything:
...
... >>> my_function()
... >>>
... """
... pass
...
>>> print(my_function.__doc__)
Do nothing, but document it.
No, really, it doesn't do anything.
No, really, it doesn't do anything:
>>> my_function()
>>>
.. _tut-annotations:

4
Doc/using/configure.rst
View file

@ -339,12 +339,12 @@ General Options
.. code-block:: json
{
"_gdbm": "The '_gdbm' module is not available in this distribution"
"_gdbm": "The '_gdbm' module is not available in this distribution",
"tkinter": "Install the python-tk package to use tkinter",
"_tkinter": "Install the python-tk package to use tkinter",
}
.. versionadded:: next
.. versionadded:: 3.15
.. option:: --enable-pystats

47
Doc/using/windows.rst
View file

@ -290,8 +290,12 @@ Passing ``--dry-run`` will generate output and logs, but will not modify any
installs.
In addition to the above options, the ``--target`` option will extract the
runtime to the specified directory instead of doing a normal install. This is
useful for embedding runtimes into larger applications.
runtime to the specified directory instead of doing a normal install.
This is useful for embedding runtimes into larger applications.
Unlike a normal install, ``py`` will not be aware of the extracted runtime,
and no Start menu or other shortcuts will be created.
To launch the runtime, directly execute the main executable (typically
``python.exe``) in the target directory.
.. code::
@ -378,10 +382,13 @@ overridden installs may resolve settings differently.
A global configuration file may be configured by an administrator, and would be
read first. The user configuration file is stored at
:file:`%AppData%\\Python\\pymanager.json` (by default) and is read next,
:file:`%AppData%\\Python\\pymanager.json`
(note that this location is under ``Roaming``, not ``Local``) and is read next,
overwriting any settings from earlier files. An additional configuration file
may be specified as the ``PYTHON_MANAGER_CONFIG`` environment variable or the
``--config`` command line option (but not both).
These locations may be modified by administrative customization options listed
later.
The following settings are those that are considered likely to be modified in
normal use. Later sections list those that are intended for administrative
@ -420,8 +427,8 @@ customization.
* - ``automatic_install``
- ``PYTHON_MANAGER_AUTOMATIC_INSTALL``
- True to allow automatic installs when specifying a particular runtime
to launch.
- True to allow automatic installs when using ``py exec`` to launch.
Other commands will not automatically install.
By default, true.
* - ``include_unmanaged``
@ -799,6 +806,12 @@ default).
* -
- Check that the ``py`` and ``pymanager`` commands work.
* -
- Ensure your :envvar:`PATH` variable contains the entry for
``%UserProfile%\AppData\Local\Microsoft\WindowsApps``.
The operating system includes this entry once by default, after other
user paths. If removed, shortcuts will not be found.
* - ``py`` gives me a "command not found" error when I type it in my terminal.
- Did you :ref:`install the Python install manager <pymanager>`?
@ -809,6 +822,12 @@ default).
The "Python (default windowed)" and "Python install manager" commands
may also need refreshing.
* -
- Ensure your :envvar:`PATH` variable contains the entry for
``%UserProfile%\AppData\Local\Microsoft\WindowsApps``.
The operating system includes this entry once by default, after other
user paths. If removed, shortcuts will not be found.
* - ``py`` gives me a "can't open file" error when I type commands in my
terminal.
- This usually means you have the legacy launcher installed and
@ -839,7 +858,7 @@ default).
- Prerelease and experimental installs that are not managed by the Python
install manager may be chosen ahead of stable releases.
Configure your default tag or uninstall the prerelease runtime
and reinstall using ``py install``.
and reinstall it using ``py install``.
* - ``pythonw`` or ``pyw`` don't launch the same runtime as ``python`` or ``py``
- Click Start, open "Manage app execution aliases", and check that your
@ -869,6 +888,20 @@ default).
the `legacy launcher`_, or with the Python install manager when installed
from the MSI.
* - I have installed the Python install manager multiple times.
- It is possible to install from the Store or WinGet, from the MSIX on
the Python website, and from the MSI, all at once.
They are all compatible and will share configuration and runtimes.
* -
- See the earlier :ref:`pymanager-advancedinstall` section for ways to
uninstall the install manager other than the typical Installed Apps
(Add and Remove Programs) settings page.
* - My old ``py.ini`` settings no longer work.
- The new Python install manager no longer supports this configuration file
or its settings, and so it will be ignored.
See :ref:`pymanager-config` for information about configuration settings.
.. _windows-embeddable:
@ -886,7 +919,7 @@ To install an embedded distribution, we recommend using ``py install`` with the
.. code::
$> py install 3.14-embed --target=runtime
$> py install 3.14-embed --target=<directory>
When extracted, the embedded distribution is (almost) fully isolated from the
user's system, including environment variables, system registry settings, and

4
Doc/whatsnew/2.3.rst
View file

@ -66,7 +66,7 @@ Here's a simple example::
The union and intersection of sets can be computed with the :meth:`~frozenset.union` and
:meth:`~frozenset.intersection` methods; an alternative notation uses the bitwise operators
``&`` and ``|``. Mutable sets also have in-place versions of these methods,
:meth:`!union_update` and :meth:`~frozenset.intersection_update`. ::
:meth:`!union_update` and :meth:`~set.intersection_update`. ::
>>> S1 = sets.Set([1,2,3])
>>> S2 = sets.Set([4,5,6])
@ -87,7 +87,7 @@ It's also possible to take the symmetric difference of two sets. This is the
set of all elements in the union that aren't in the intersection. Another way
of putting it is that the symmetric difference contains all elements that are in
exactly one set. Again, there's an alternative notation (``^``), and an
in-place version with the ungainly name :meth:`~frozenset.symmetric_difference_update`. ::
in-place version with the ungainly name :meth:`~set.symmetric_difference_update`. ::
>>> S1 = sets.Set([1,2,3,4])
>>> S2 = sets.Set([3,4,5,6])

8
Doc/whatsnew/2.5.rst
View file

@ -1277,12 +1277,12 @@ complete list of changes, or look through the SVN logs for all the details.
with the new ':keyword:`with`' statement. See section :ref:`contextlibmod`
for more about this module.
* New module: The :mod:`cProfile` module is a C implementation of the existing
:mod:`profile` module that has much lower overhead. The module's interface is
the same as :mod:`profile`: you run ``cProfile.run('main()')`` to profile a
* New module: The :mod:`!cProfile` module is a C implementation of the existing
:mod:`!profile` module that has much lower overhead. The module's interface is
the same as :mod:`!profile`: you run ``cProfile.run('main()')`` to profile a
function, can save profile data to a file, etc. It's not yet known if the
Hotshot profiler, which is also written in C but doesn't match the
:mod:`profile` module's interface, will continue to be maintained in future
:mod:`!profile` module's interface, will continue to be maintained in future
versions of Python. (Contributed by Armin Rigo.)
Also, the :mod:`pstats` module for analyzing the data measured by the profiler

4
Doc/whatsnew/3.10.rst
View file

@ -1622,7 +1622,7 @@ Deprecated
compatibility. Specifically,
:meth:`!find_loader`/:meth:`!find_module`
(superseded by :meth:`~importlib.abc.MetaPathFinder.find_spec`),
:meth:`~importlib.abc.Loader.load_module`
``importlib.abc.Loader.load_module``
(superseded by :meth:`~importlib.abc.Loader.exec_module`),
:meth:`!module_repr` (which the import system
takes care of for you), the ``__package__`` attribute
@ -1652,7 +1652,7 @@ Deprecated
preference for :meth:`~zipimport.zipimporter.exec_module`.
(Contributed by Brett Cannon in :issue:`26131`.)
* The use of :meth:`~importlib.abc.Loader.load_module` by the import
* The use of ``importlib.abc.Loader.load_module`` by the import
system now triggers an :exc:`ImportWarning` as
:meth:`~importlib.abc.Loader.exec_module` is preferred.
(Contributed by Brett Cannon in :issue:`26131`.)

2
Doc/whatsnew/3.12.rst
View file

@ -1337,7 +1337,7 @@ Deprecated
it was :exc:`ImportWarning`).
(Contributed by Brett Cannon in :gh:`65961`.)
* Setting :attr:`~module.__package__` or :attr:`~module.__cached__` on a
* Setting :attr:`~module.__package__` or ``__cached__`` on a
module is deprecated, and will cease to be set or taken into consideration by
the import system in Python 3.14. (Contributed by Brett Cannon in :gh:`65961`.)

285
Doc/whatsnew/3.15.rst
View file

@ -66,126 +66,126 @@ Summary -- Release highlights
.. PEP-sized items next.
* :pep:`799`: :ref:`A dedicated profiling package for organizing Python
profiling tools <whatsnew315-profiling-package>`
* :pep:`799`: :ref:`Tachyon: High frequency statistical sampling profiler
profiling tools <whatsnew315-sampling-profiler>`
* :pep:`686`: :ref:`Python now uses UTF-8 as the default encoding
<whatsnew315-utf8-default>`
* :pep:`782`: :ref:`A new PyBytesWriter C API to create a Python bytes object
<whatsnew315-pep782>`
* :ref:`The JIT compiler has been significantly upgraded <whatsnew315-jit>`
* :ref:`Improved error messages <whatsnew315-improved-error-messages>`
New features
============
.. _whatsnew315-profiling-package:
:pep:`799`: A dedicated profiling package
-----------------------------------------
A new :mod:`profiling` module has been added to organize Python's built-in
profiling tools under a single, coherent namespace. This module contains:
* :mod:`profiling.tracing`: deterministic function-call tracing (relocated from
``cProfile``).
* :mod:`profiling.sampling`: a new statistical sampling profiler (named Tachyon).
The ``cProfile`` module remains as an alias for backwards compatibility.
The :mod:`profile` module is deprecated and will be removed in Python 3.17.
.. seealso:: :pep:`799` for further details.
(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`138122`.)
.. _whatsnew315-sampling-profiler:
:pep:`799`: High frequency statistical sampling profiler
--------------------------------------------------------
Tachyon: High frequency statistical sampling profiler
-----------------------------------------------------
A new statistical sampling profiler has been added to the new :mod:`!profiling` module as
:mod:`!profiling.sampling`. This profiler enables low-overhead performance analysis of
.. image:: ../library/tachyon-logo.png
:alt: Tachyon profiler logo
:align: center
:width: 200px
A new statistical sampling profiler (Tachyon) has been added as
:mod:`profiling.sampling`. This profiler enables low-overhead performance analysis of
running Python processes without requiring code modification or process restart.
Unlike deterministic profilers (:mod:`cProfile` and :mod:`profile`) that instrument
Unlike deterministic profilers (such as :mod:`profiling.tracing`) that instrument
every function call, the sampling profiler periodically captures stack traces from
running processes. This approach provides virtually zero overhead while achieving
sampling rates of **up to 1,000,000 Hz**, making it the fastest sampling profiler
available for Python (at the time of its contribution) and ideal for debugging
performance issues in production environments.
performance issues in production environments. This capability is particularly
valuable for debugging performance issues in production systems where traditional
profiling approaches would be too intrusive.
Key features include:
* **Zero-overhead profiling**: Attach to any running Python process without
affecting its performance
* **No code modification required**: Profile existing applications without restart
* **Real-time statistics**: Monitor sampling quality during data collection
* **Multiple output formats**: Generate both detailed statistics and flamegraph data
* **Thread-aware profiling**: Option to profile all threads or just the main thread
affecting its performance. Ideal for production debugging where you can't afford
to restart or slow down your application.
Profile process 1234 for 10 seconds with default settings:
* **No code modification required**: Profile existing applications without restart.
Simply point the profiler at a running process by PID and start collecting data.
.. code-block:: shell
* **Flexible target modes**:
python -m profiling.sampling 1234
* Profile running processes by PID (``attach``) - attach to already-running applications
* Run and profile scripts directly (``run``) - profile from the very start of execution
* Execute and profile modules (``run -m``) - profile packages run as ``python -m module``
Profile with custom interval and duration, save to file:
* **Multiple profiling modes**: Choose what to measure based on your performance investigation:
.. code-block:: shell
* **Wall-clock time** (``--mode wall``, default): Measures real elapsed time including I/O,
network waits, and blocking operations. Use this to understand where your program spends
calendar time, including when waiting for external resources.
* **CPU time** (``--mode cpu``): Measures only active CPU execution time, excluding I/O waits
and blocking. Use this to identify CPU-bound bottlenecks and optimize computational work.
* **GIL-holding time** (``--mode gil``): Measures time spent holding Python's Global Interpreter
Lock. Use this to identify which threads dominate GIL usage in multi-threaded applications.
* **Exception handling time** (``--mode exception``): Captures samples only from threads with
an active exception. Use this to analyze exception handling overhead.
python -m profiling.sampling -i 50 -d 30 -o profile.stats 1234
* **Thread-aware profiling**: Option to profile all threads (``-a``) or just the main thread,
essential for understanding multi-threaded application behavior.
Generate collapsed stacks for flamegraph:
* **Multiple output formats**: Choose the visualization that best fits your workflow:
.. code-block:: shell
* ``--pstats``: Detailed tabular statistics compatible with :mod:`pstats`. Shows function-level
timing with direct and cumulative samples. Best for detailed analysis and integration with
existing Python profiling tools.
* ``--collapsed``: Generates collapsed stack traces (one line per stack). This format is
specifically designed for creating flamegraphs with external tools like Brendan Gregg's
FlameGraph scripts or speedscope.
* ``--flamegraph``: Generates a self-contained interactive HTML flamegraph using D3.js.
Opens directly in your browser for immediate visual analysis. Flamegraphs show the call
hierarchy where width represents time spent, making it easy to spot bottlenecks at a glance.
* ``--gecko``: Generates Gecko Profiler format compatible with Firefox Profiler
(https://profiler.firefox.com). Upload the output to Firefox Profiler for advanced
timeline-based analysis with features like stack charts, markers, and network activity.
* ``--heatmap``: Generates an interactive HTML heatmap visualization with line-level sample
counts. Creates a directory with per-file heatmaps showing exactly where time is spent
at the source code level.
python -m profiling.sampling --collapsed 1234
* **Live interactive mode**: Real-time TUI profiler with a top-like interface (``--live``).
Monitor performance as your application runs with interactive sorting and filtering.
Profile all threads and sort by total time:
* **Async-aware profiling**: Profile async/await code with task-based stack reconstruction
(``--async-aware``). See which coroutines are consuming time, with options to show only
running tasks or all tasks including those waiting.
.. code-block:: shell
* **Opcode-level profiling**: Gather bytecode opcode information for instruction-level
profiling (``--opcodes``). Shows which bytecode instructions are executing, including
specializations from the adaptive interpreter.
python -m profiling.sampling -a --sort-tottime 1234
See :mod:`profiling.sampling` for the complete documentation, including all
available output formats, profiling modes, and configuration options.
The profiler generates statistical estimates of where time is spent:
.. code-block:: text
Real-time sampling stats: Mean: 100261.5Hz (9.97µs) Min: 86333.4Hz (11.58µs) Max: 118807.2Hz (8.42µs) Samples: 400001
Captured 498841 samples in 5.00 seconds
Sample rate: 99768.04 samples/sec
Error rate: 0.72%
Profile Stats:
nsamples sample% tottime (s) cumul% cumtime (s) filename:lineno(function)
43/418858 0.0 0.000 87.9 4.189 case.py:667(TestCase.run)
3293/418812 0.7 0.033 87.9 4.188 case.py:613(TestCase._callTestMethod)
158562/158562 33.3 1.586 33.3 1.586 test_compile.py:725(TestSpecifics.test_compiler_recursion_limit.<locals>.check_limit)
129553/129553 27.2 1.296 27.2 1.296 ast.py:46(parse)
0/128129 0.0 0.000 26.9 1.281 test_ast.py:884(AST_Tests.test_ast_recursion_limit.<locals>.check_limit)
7/67446 0.0 0.000 14.2 0.674 test_compile.py:729(TestSpecifics.test_compiler_recursion_limit)
6/60380 0.0 0.000 12.7 0.604 test_ast.py:888(AST_Tests.test_ast_recursion_limit)
3/50020 0.0 0.000 10.5 0.500 test_compile.py:727(TestSpecifics.test_compiler_recursion_limit)
1/38011 0.0 0.000 8.0 0.380 test_ast.py:886(AST_Tests.test_ast_recursion_limit)
1/25076 0.0 0.000 5.3 0.251 test_compile.py:728(TestSpecifics.test_compiler_recursion_limit)
22361/22362 4.7 0.224 4.7 0.224 test_compile.py:1368(TestSpecifics.test_big_dict_literal)
4/18008 0.0 0.000 3.8 0.180 test_ast.py:889(AST_Tests.test_ast_recursion_limit)
11/17696 0.0 0.000 3.7 0.177 subprocess.py:1038(Popen.__init__)
16968/16968 3.6 0.170 3.6 0.170 subprocess.py:1900(Popen._execute_child)
2/16941 0.0 0.000 3.6 0.169 test_compile.py:730(TestSpecifics.test_compiler_recursion_limit)
Legend:
nsamples: Direct/Cumulative samples (direct executing / on call stack)
sample%: Percentage of total samples this function was directly executing
tottime: Estimated total time spent directly in this function
cumul%: Percentage of total samples when this function was on the call stack
cumtime: Estimated cumulative time (including time in called functions)
filename:lineno(function): Function location and name
Summary of Interesting Functions:
Functions with Highest Direct/Cumulative Ratio (Hot Spots):
1.000 direct/cumulative ratio, 33.3% direct samples: test_compile.py:(TestSpecifics.test_compiler_recursion_limit.<locals>.check_limit)
1.000 direct/cumulative ratio, 27.2% direct samples: ast.py:(parse)
1.000 direct/cumulative ratio, 3.6% direct samples: subprocess.py:(Popen._execute_child)
Functions with Highest Call Frequency (Indirect Calls):
418815 indirect calls, 87.9% total stack presence: case.py:(TestCase.run)
415519 indirect calls, 87.9% total stack presence: case.py:(TestCase._callTestMethod)
159470 indirect calls, 33.5% total stack presence: test_compile.py:(TestSpecifics.test_compiler_recursion_limit)
Functions with Highest Call Magnification (Cumulative/Direct):
12267.9x call magnification, 159470 indirect calls from 13 direct: test_compile.py:(TestSpecifics.test_compiler_recursion_limit)
10581.7x call magnification, 116388 indirect calls from 11 direct: test_ast.py:(AST_Tests.test_ast_recursion_limit)
9740.9x call magnification, 418815 indirect calls from 43 direct: case.py:(TestCase.run)
The profiler automatically identifies performance bottlenecks through statistical
analysis, highlighting functions with high CPU usage and call frequency patterns.
This capability is particularly valuable for debugging performance issues in
production systems where traditional profiling approaches would be too intrusive.
.. seealso:: :pep:`799` for further details.
(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`135953`.)
(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`135953` and :gh:`138122`.)
.. _whatsnew315-improved-error-messages:
@ -424,6 +424,10 @@ argparse
default to ``True``. This enables suggestions for mistyped arguments by default.
(Contributed by Jakob Schluse in :gh:`140450`.)
* Added backtick markup support in description and epilog text to highlight
inline code when color output is enabled.
(Contributed by Savannah Ostrowski in :gh:`142390`.)
calendar
--------
@ -588,6 +592,11 @@ mmap
not be duplicated.
(Contributed by Serhiy Storchaka in :gh:`78502`.)
* Added the :meth:`mmap.mmap.set_name` method
to annotate an anonymous memory mapping
if Linux kernel supports :manpage:`PR_SET_VMA_ANON_NAME <PR_SET_VMA(2const)>` (Linux 5.17 or newer).
(Contributed by Donghee Na in :gh:`142419`.)
os
--
@ -847,6 +856,91 @@ csv
(Contributed by Maurycy Pawłowski-Wieroński in :gh:`137628`.)
.. _whatsnew315-jit:
Upgraded JIT compiler
=====================
Results from the `pyperformance <https://github.com/python/pyperformance>`__
benchmark suite report
`3-4% <https://github.com/facebookexperimental/free-threading-benchmarking/blob/main/results/bm-20251214-3.15.0a2%2B-6cddf04-JIT/bm-20251214-vultr-x86_64-python-6cddf04344a1e8ca9df5-3.15.0a2%2B-6cddf04-vs-base.svg>`__
geometric mean performance improvement for the JIT over the standard CPython
interpreter built with all optimizations enabled. The speedups for JIT
builds versus no JIT builds range from roughly 20% slowdown to over
100% speedup (ignoring the ``unpack_sequence`` microbenchmark) on
x86-64 Linux and AArch64 macOS systems.
.. attention::
These results are not yet final.
The major upgrades to the JIT are:
* LLVM 21 build-time dependency
* New tracing frontend
* Basic register allocation in the JIT
* More JIT optimizations
* Better machine code generation
.. rubric:: LLVM 21 build-time dependency
The JIT compiler now uses LLVM 21 for build-time stencil generation. As
always, LLVM is only needed when building CPython with the JIT enabled;
end users running Python do not need LLVM installed. Instructions for
installing LLVM can be found in the `JIT compiler documentation
<https://github.com/python/cpython/blob/main/Tools/jit/README.md>`__
for all supported platforms.
(Contributed by Savannah Ostrowski in :gh:`140973`.)
.. rubric:: A new tracing frontend
The JIT compiler now supports significantly more bytecode operations and
control flow than in Python 3.14, enabling speedups on a wider variety of
code. For example, simple Python object creation is now understood by the
3.15 JIT compiler. Overloaded operations and generators are also partially
supported. This was made possible by an overhauled JIT tracing frontend
that records actual execution paths through code, rather than estimating
them as the previous implementation did.
(Contributed by Ken Jin in :gh:`139109`. Support for Windows added by
Mark Shannon in :gh:`141703`.)
.. rubric:: Basic register allocation in the JIT
A basic form of register allocation has been added to the JIT compiler's
optimizer. This allows the JIT compiler to avoid certain stack operations
altogether and instead operate on registers. This allows the JIT to produce
more efficient traces by avoiding reads and writes to memory.
(Contributed by Mark Shannon in :gh:`135379`.)
.. rubric:: More JIT optimizations
More `constant-propagation <https://en.wikipedia.org/wiki/Constant_folding>`__
is now performed. This means when the JIT compiler detects that certain user
code results in constants, the code can be simplified by the JIT.
(Contributed by Ken Jin and Savannah Ostrowski in :gh:`132732`.)
The JIT avoids :term:`reference count`\ s where possible. This generally
reduces the cost of most operations in Python.
(Contributed by Ken Jin, Donghee Na, Zheao Li, Savannah Ostrowski,
Noam Cohen, Tomas Roun, PuQing in :gh:`134584`.)
.. rubric:: Better machine code generation
The JIT compiler's machine code generator now produces better machine code
for x86-64 and AArch64 macOS and Linux targets. In general, users should
experience lower memory usage for generated machine code and more efficient
machine code versus the old JIT.
(Contributed by Brandt Bucher in :gh:`136528` and :gh:`136528`.
Implementation for AArch64 contributed by Mark Shannon in :gh:`139855`.
Additional optimizations for AArch64 contributed by Mark Shannon and
Diego Russo in :gh:`140683` and :gh:`142305`.)
Removed
=======
@ -1015,14 +1109,16 @@ New deprecations
* ``__version__``
* The ``__version__`` attribute has been deprecated in these standard library
modules and will be removed in Python 3.20.
Use :py:data:`sys.version_info` instead.
* The ``__version__``, ``version`` and ``VERSION`` attributes have been
deprecated in these standard library modules and will be removed in
Python 3.20. Use :py:data:`sys.version_info` instead.
- :mod:`argparse`
- :mod:`csv`
- :mod:`ctypes`
- :mod:`!ctypes.macholib`
- :mod:`decimal` (use :data:`decimal.SPEC_VERSION` instead)
- :mod:`http.server`
- :mod:`imaplib`
- :mod:`ipaddress`
- :mod:`json`
@ -1035,6 +1131,10 @@ New deprecations
- :mod:`tabnanny`
- :mod:`tkinter.font`
- :mod:`tkinter.ttk`
- :mod:`wsgiref.simple_server`
- :mod:`xml.etree.ElementTree`
- :mod:`!xml.sax.expatreader`
- :mod:`xml.sax.handler`
- :mod:`zlib`
(Contributed by Hugo van Kemenade and Stan Ulbrych in :gh:`76007`.)
@ -1211,6 +1311,13 @@ Deprecated C APIs
use the :c:type:`PyBytesWriter` API instead.
(Contributed by Victor Stinner in :gh:`129813`.)
* :c:func:`!_PyObject_CallMethodId`, :c:func:`!_PyObject_GetAttrId` and
:c:func:`!_PyUnicode_FromId` are deprecated since 3.15 and will be removed in
3.20. Instead, use :c:func:`PyUnicode_InternFromString()` and cache the result in
the module state, then call :c:func:`PyObject_CallMethod` or
:c:func:`PyObject_GetAttr`.
(Contributed by Victor Stinner in :gh:`141049`.)
* Deprecate :c:member:`~PyComplexObject.cval` field of the
:c:type:`PyComplexObject` type.
Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex`
@ -1253,6 +1360,12 @@ Build changes
modules that are missing or packaged separately.
(Contributed by Stan Ulbrych and Petr Viktorin in :gh:`139707`.)
* Annotating anonymous mmap usage is now supported if Linux kernel supports
:manpage:`PR_SET_VMA_ANON_NAME <PR_SET_VMA(2const)>` (Linux 5.17 or newer).
Annotations are visible in ``/proc/<pid>/maps`` if the kernel supports the feature
and :option:`-X dev <-X>` is passed to the Python or Python is built in :ref:`debug mode <debug-build>`.
(Contributed by Donghee Na in :gh:`141770`)
Porting to Python 3.15
======================

2
Doc/whatsnew/3.2.rst
View file

@ -312,7 +312,7 @@ cluttering source directories, the *pyc* files are now collected in a
Aside from the filenames and target directories, the new scheme has a few
aspects that are visible to the programmer:
* Imported modules now have a :attr:`~module.__cached__` attribute which stores
* Imported modules now have a ``__cached__`` attribute which stores
the name of the actual file that was imported:
>>> import collections

8
Doc/whatsnew/3.4.rst
View file

@ -2086,10 +2086,10 @@ Deprecations in the Python API
:meth:`!importlib.abc.PathEntryFinder.find_loader` and
:meth:`!find_module` are replaced by
:meth:`importlib.abc.PathEntryFinder.find_spec`; all of the :samp:`{xxx}Loader` ABC
``load_module`` methods (:meth:`!importlib.abc.Loader.load_module`,
:meth:`!importlib.abc.InspectLoader.load_module`,
:meth:`!importlib.abc.FileLoader.load_module`,
:meth:`!importlib.abc.SourceLoader.load_module`) should no longer be
``load_module`` methods (``importlib.abc.Loader.load_module``,
``importlib.abc.InspectLoader.load_module``,
``importlib.abc.FileLoader.load_module``,
``importlib.abc.SourceLoader.load_module``) should no longer be
implemented, instead loaders should implement an
``exec_module`` method
(:meth:`importlib.abc.Loader.exec_module`,

6
Doc/whatsnew/3.6.rst
View file

@ -2006,10 +2006,10 @@ deprecated.
importlib
~~~~~~~~~
The :meth:`importlib.machinery.SourceFileLoader.load_module` and
:meth:`importlib.machinery.SourcelessFileLoader.load_module` methods
The ``importlib.machinery.SourceFileLoader.load_module`` and
``importlib.machinery.SourcelessFileLoader.load_module`` methods
are now deprecated. They were the only remaining implementations of
:meth:`importlib.abc.Loader.load_module` in :mod:`importlib` that had not
``importlib.abc.Loader.load_module`` in :mod:`importlib` that had not
been deprecated in previous versions of Python in favour of
:meth:`importlib.abc.Loader.exec_module`.

2
Doc/whatsnew/3.7.rst
View file

@ -841,7 +841,7 @@ and by Alexander Mohr and Ilya Kulakov in :issue:`29302`.)
cProfile
--------
The :mod:`cProfile` command line now accepts ``-m module_name`` as an
The :mod:`!cProfile` command line now accepts ``-m module_name`` as an
alternative to script path. (Contributed by Sanyam Khurana in :issue:`21862`.)

3
Grammar/python.gram
View file

@ -1251,8 +1251,7 @@ invalid_expression:
# !(NAME STRING) is not matched so we don't show this error with some invalid string prefixes like: kf"dsfsdf"
# Soft keywords need to also be ignored because they can be parsed as NAME NAME
| !(NAME STRING | SOFT_KEYWORD) a=disjunction b=expression_without_invalid {
_PyPegen_check_legacy_stmt(p, a) ? NULL : p->tokens[p->mark-1]->level == 0 ? NULL :
RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Perhaps you forgot a comma?") }
_PyPegen_raise_error_for_missing_comma(p, a, b) }
| a=disjunction 'if' b=disjunction !('else'|':') { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "expected 'else' after 'if' expression") }
| a=disjunction 'if' b=disjunction 'else' !expression {
RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("expected expression after 'else', but statement is given") }

2
Include/cpython/abstract.h
View file

@ -6,7 +6,7 @@
/* Like PyObject_CallMethod(), but expect a _Py_Identifier*
as the method name. */
PyAPI_FUNC(PyObject*) _PyObject_CallMethodId(
Py_DEPRECATED(3.15) PyAPI_FUNC(PyObject*) _PyObject_CallMethodId(
PyObject *obj,
_Py_Identifier *name,
const char *format, ...);

2
Include/cpython/object.h
View file

@ -300,7 +300,7 @@ PyAPI_FUNC(void) PyUnstable_Object_Dump(PyObject *);
// Alias for backward compatibility
#define _PyObject_Dump PyUnstable_Object_Dump
PyAPI_FUNC(PyObject*) _PyObject_GetAttrId(PyObject *, _Py_Identifier *);
Py_DEPRECATED(3.15) PyAPI_FUNC(PyObject*) _PyObject_GetAttrId(PyObject *, _Py_Identifier *);
PyAPI_FUNC(PyObject **) _PyObject_GetDictPtr(PyObject *);
PyAPI_FUNC(void) PyObject_CallFinalizer(PyObject *);

14
Include/cpython/pyatomic.h
View file

@ -523,6 +523,9 @@ _Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value);
static inline void
_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value);
static inline void
_Py_atomic_store_int8_release(int8_t *obj, int8_t value);
static inline void
_Py_atomic_store_int_release(int *obj, int value);
@ -591,6 +594,17 @@ static inline void _Py_atomic_fence_release(void);
// --- aliases ---------------------------------------------------------------
// Compilers don't really support "consume" semantics, so we fake it. Use
// "acquire" with TSan to support false positives. Use "relaxed" otherwise,
// because CPUs on all platforms we support respect address dependencies without
// extra barriers.
// See 2.6.7 in https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2055r0.pdf
#if defined(_Py_THREAD_SANITIZER)
# define _Py_atomic_load_ptr_consume _Py_atomic_load_ptr_acquire
#else
# define _Py_atomic_load_ptr_consume _Py_atomic_load_ptr_relaxed
#endif
#if SIZEOF_LONG == 8
# define _Py_atomic_load_ulong(p) \
_Py_atomic_load_uint64((uint64_t *)p)

4
Include/cpython/pyatomic_gcc.h
View file

@ -572,6 +572,10 @@ static inline void
_Py_atomic_store_int_release(int *obj, int value)
{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); }
static inline void
_Py_atomic_store_int8_release(int8_t *obj, int8_t value)
{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); }
static inline void
_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value)
{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); }

Some files were not shown because too many files have changed in this diff Show more