diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml
index 15a83dd0370..1576599379c 100644
--- a/.azure-pipelines/ci.yml
+++ b/.azure-pipelines/ci.yml
@@ -59,7 +59,7 @@ jobs:
variables:
testRunTitle: '$(build.sourceBranchName)-linux'
testRunPlatform: linux
- openssl_version: 1.1.0j
+ openssl_version: 1.1.1b
steps:
- template: ./posix-steps.yml
@@ -116,7 +116,7 @@ jobs:
variables:
testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
testRunPlatform: linux-coverage
- openssl_version: 1.1.0j
+ openssl_version: 1.1.1b
steps:
- template: ./posix-steps.yml
diff --git a/.gitignore b/.gitignore
index 9e60f5a2069..9445ef1e2c5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -36,6 +36,7 @@ Lib/test/data/*
Makefile
Makefile.pre
Misc/python.pc
+Misc/python-embed.pc
Misc/python-config.sh
Modules/Setup
Modules/Setup.config
@@ -70,6 +71,7 @@ PCbuild/*.VC.opendb
PCbuild/.vs/
PCbuild/amd64/
PCbuild/arm32/
+PCbuild/arm64/
PCbuild/obj/
PCbuild/win32/
.purify
diff --git a/.travis.yml b/.travis.yml
index 6d57ebb1d2f..c1efe24b646 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -32,7 +32,8 @@ matrix:
allow_failures:
- env: OPTIONAL=true
include:
- - os: linux
+ - name: "CPython tests"
+ os: linux
language: c
compiler: clang
# gcc also works, but to keep the # of concurrent builds down, we use one C
@@ -43,7 +44,8 @@ matrix:
apt:
packages:
- xvfb
- - os: linux
+ - name: "Documentation build"
+ os: linux
language: python
# Build the docs against a stable version of Python so code bugs don't hold up doc-related PRs.
python: 3.6
@@ -56,7 +58,8 @@ matrix:
- python -m pip install sphinx==1.8.2 blurb python-docs-theme
script:
- make check suspicious html SPHINXOPTS="-q -W -j4"
- - os: linux
+ - name: "Documentation tests"
+ os: linux
language: c
compiler: clang
env: TESTING=doctest
@@ -70,7 +73,8 @@ matrix:
- make -C Doc/ PYTHON=../python venv
script:
xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W -j4" doctest
- - os: osx
+ - name: "Mac OS X tests"
+ os: osx
language: c
compiler: clang
# Testing under macOS is optional until testing stability has been demonstrated.
@@ -79,7 +83,31 @@ matrix:
# Python 3 is needed for Argument Clinic and multissl
- HOMEBREW_NO_AUTO_UPDATE=1 brew install xz python3
- export PATH=$(brew --prefix)/bin:$(brew --prefix)/sbin:$PATH
- - os: linux
+ - name: "Test code coverage (Python)"
+ os: linux
+ language: c
+ compiler: gcc
+ env: OPTIONAL=true
+ addons:
+ apt:
+ packages:
+ - xvfb
+ before_script:
+ - ./configure
+ - make -j4
+ # Need a venv that can parse covered code.
+ - ./python -m venv venv
+ - ./venv/bin/python -m pip install -U coverage
+ - ./venv/bin/python -m test.pythoninfo
+ script:
+ # Skip tests that re-run the entire test suite.
+ - xvfb-run ./venv/bin/python -m coverage run --branch --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures || true
+ after_script: # Probably should be after_success once test suite updated to run under coverage.py.
+ # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files.
+ - source ./venv/bin/activate
+ - bash <(curl -s https://codecov.io/bash)
+ - name: "Test code coverage (C)"
+ os: linux
language: c
compiler: gcc
env: OPTIONAL=true
@@ -90,18 +118,10 @@ matrix:
- xvfb
before_script:
- ./configure
- - make coverage -s -j4
- # Need a venv that can parse covered code.
- - ./python -m venv venv
- - ./venv/bin/python -m pip install -U coverage
- - ./venv/bin/python -m test.pythoninfo
script:
- # Skip tests that re-run the entire test suite.
- - xvfb-run ./venv/bin/python -m coverage run --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures
+ - xvfb-run make -j4 coverage-report
after_script: # Probably should be after_success once test suite updated to run under coverage.py.
- # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files.
- - source ./venv/bin/activate
- - make coverage-lcov
+ - make pythoninfo
- bash <(curl -s https://codecov.io/bash)
diff --git a/Doc/Makefile b/Doc/Makefile
index cf1bb88b0b8..6f86728ea83 100644
--- a/Doc/Makefile
+++ b/Doc/Makefile
@@ -132,6 +132,7 @@ clean:
venv:
$(PYTHON) -m venv $(VENVDIR)
+ $(VENVDIR)/bin/python3 -m pip install -U pip setuptools
$(VENVDIR)/bin/python3 -m pip install -U Sphinx blurb python-docs-theme
@echo "The venv has been created in the $(VENVDIR) directory"
diff --git a/Doc/c-api/abstract.rst b/Doc/c-api/abstract.rst
index ad538811127..0edd1d5f624 100644
--- a/Doc/c-api/abstract.rst
+++ b/Doc/c-api/abstract.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _abstract:
diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst
index 25a867f139c..8e8a92003c5 100644
--- a/Doc/c-api/allocation.rst
+++ b/Doc/c-api/allocation.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _allocating-objects:
diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst
index 890a0383931..b8a8f2ff886 100644
--- a/Doc/c-api/apiabiversion.rst
+++ b/Doc/c-api/apiabiversion.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _apiabiversion:
diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst
index b41130ede41..ba9ca5e0d30 100644
--- a/Doc/c-api/arg.rst
+++ b/Doc/c-api/arg.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _arg-parsing:
diff --git a/Doc/c-api/bool.rst b/Doc/c-api/bool.rst
index a9fb342f7c0..ce8de6e22f4 100644
--- a/Doc/c-api/bool.rst
+++ b/Doc/c-api/bool.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _boolobjects:
diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst
index c7c1e3cc745..72d965053cf 100644
--- a/Doc/c-api/buffer.rst
+++ b/Doc/c-api/buffer.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. index::
single: buffer protocol
diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst
index 41b6e3c71be..b4a9660a916 100644
--- a/Doc/c-api/bytearray.rst
+++ b/Doc/c-api/bytearray.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _bytearrayobjects:
diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst
index 5b9ebf6b6af..9a62fb682d6 100644
--- a/Doc/c-api/bytes.rst
+++ b/Doc/c-api/bytes.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _bytesobjects:
diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst
index 8eb6695e22d..3c921bbde34 100644
--- a/Doc/c-api/capsule.rst
+++ b/Doc/c-api/capsule.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _capsules:
diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst
index 427259cc24d..8514afe928e 100644
--- a/Doc/c-api/cell.rst
+++ b/Doc/c-api/cell.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _cell-objects:
diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst
index 10d89f297c8..fd3f6919d6d 100644
--- a/Doc/c-api/code.rst
+++ b/Doc/c-api/code.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _codeobjects:
@@ -40,6 +40,7 @@ bound into a function.
:c:func:`PyCode_New` directly can bind you to a precise Python
version since the definition of the bytecode changes often.
+ .. audit-event:: code.__new__ "code filename name argcount kwonlyargcount nlocals stacksize flags"
.. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst
index fc63b57a855..675bd013e89 100644
--- a/Doc/c-api/complex.rst
+++ b/Doc/c-api/complex.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _complexobjects:
diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst
index 9558a4a583c..f83d711795c 100644
--- a/Doc/c-api/concrete.rst
+++ b/Doc/c-api/concrete.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _concrete:
diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst
index c344c8d71ae..a7cde7fb195 100644
--- a/Doc/c-api/contextvars.rst
+++ b/Doc/c-api/contextvars.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _contextvarsobjects:
diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst
index c46722d782a..ed101c791ec 100644
--- a/Doc/c-api/conversion.rst
+++ b/Doc/c-api/conversion.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _string-conversion:
diff --git a/Doc/c-api/coro.rst b/Doc/c-api/coro.rst
index 2fe50b5d8c4..50428c7eb43 100644
--- a/Doc/c-api/coro.rst
+++ b/Doc/c-api/coro.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _coro-objects:
diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst
index b7949e23500..44a04373d19 100644
--- a/Doc/c-api/datetime.rst
+++ b/Doc/c-api/datetime.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _datetimeobjects:
@@ -106,6 +106,12 @@ Macros to create objects:
.. versionadded:: 3.6
+.. c:function:: PyObject* PyTime_FromTime(int hour, int minute, int second, int usecond)
+
+ Return a :class:`datetime.time` object with the specified hour, minute, second and
+ microsecond.
+
+
.. c:function:: PyObject* PyTime_FromTimeAndFold(int hour, int minute, int second, int usecond, int fold)
Return a :class:`datetime.time` object with the specified hour, minute, second,
@@ -114,12 +120,6 @@ Macros to create objects:
.. versionadded:: 3.6
-.. c:function:: PyObject* PyTime_FromTime(int hour, int minute, int second, int usecond)
-
- Return a :class:`datetime.time` object with the specified hour, minute, second and
- microsecond.
-
-
.. c:function:: PyObject* PyDelta_FromDSU(int days, int seconds, int useconds)
Return a :class:`datetime.timedelta` object representing the given number
diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst
index c8f6fa5bcd7..1005140c7ac 100644
--- a/Doc/c-api/descriptor.rst
+++ b/Doc/c-api/descriptor.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _descriptor-objects:
diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst
index 0ced5a5fd00..e970771893e 100644
--- a/Doc/c-api/dict.rst
+++ b/Doc/c-api/dict.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _dictobjects:
diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst
index 13f0aff1cf9..d3f6daa8347 100644
--- a/Doc/c-api/exceptions.rst
+++ b/Doc/c-api/exceptions.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _exceptionhandling:
@@ -72,6 +72,9 @@ Printing and clearing
.. c:function:: void PyErr_WriteUnraisable(PyObject *obj)
+ Call :func:`sys.unraisablehook` using the current exception and *obj*
+ argument.
+
This utility function prints a warning message to ``sys.stderr`` when an
exception has been set but it is impossible for the interpreter to actually
raise the exception. It is used, for example, when an exception occurs in an
@@ -81,6 +84,8 @@ Printing and clearing
in which the unraisable exception occurred. If possible,
the repr of *obj* will be printed in the warning message.
+ An exception must be set when calling this function.
+
Raising exceptions
==================
@@ -514,13 +519,13 @@ Signal Handling
single: SIGINT
single: KeyboardInterrupt (built-in exception)
- This function simulates the effect of a :const:`SIGINT` signal arriving --- the
- next time :c:func:`PyErr_CheckSignals` is called, :exc:`KeyboardInterrupt` will
- be raised. It may be called without holding the interpreter lock.
-
- .. % XXX This was described as obsolete, but is used in
- .. % _thread.interrupt_main() (used from IDLE), so it's still needed.
+ Simulate the effect of a :const:`SIGINT` signal arriving. The next time
+ :c:func:`PyErr_CheckSignals` is called, the Python signal handler for
+ :const:`SIGINT` will be called.
+ If :const:`SIGINT` isn't handled by Python (it was set to
+ :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does
+ nothing.
.. c:function:: int PySignal_SetWakeupFd(int fd)
diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst
index 6f2ecee51fd..543dc60b9f5 100644
--- a/Doc/c-api/file.rst
+++ b/Doc/c-api/file.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _fileobjects:
@@ -60,6 +60,32 @@ the :mod:`io` APIs instead.
raised if the end of the file is reached immediately.
+.. c:function:: int PyFile_SetOpenCodeHook(Py_OpenCodeHookFunction handler)
+
+ Overrides the normal behavior of :func:`io.open_code` to pass its parameter
+ through the provided handler.
+
+ The handler is a function of type :c:type:`PyObject *(\*)(PyObject *path,
+ void *userData)`, where *path* is guaranteed to be :c:type:`PyUnicodeObject`.
+
+ The *userData* pointer is passed into the hook function. Since hook
+ functions may be called from different runtimes, this pointer should not
+ refer directly to Python state.
+
+ As this hook is intentionally used during import, avoid importing new modules
+ during its execution unless they are known to be frozen or available in
+ ``sys.modules``.
+
+ Once a hook has been set, it cannot be removed or replaced, and later calls to
+ :c:func:`PyFile_SetOpenCodeHook` will fail. On failure, the function returns
+ -1 and sets an exception if the interpreter has been initialized.
+
+ This function is safe to call before :c:func:`Py_Initialize`.
+
+ .. versionadded:: 3.8
+
+
+
.. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags)
.. index:: single: Py_PRINT_RAW
diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst
index 27a75e3e0c2..8a996422ce4 100644
--- a/Doc/c-api/float.rst
+++ b/Doc/c-api/float.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _floatobjects:
diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst
index 17279c732ae..02c4ebdbed4 100644
--- a/Doc/c-api/function.rst
+++ b/Doc/c-api/function.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _function-objects:
diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst
index b739aacea8d..b3f30b2ed9e 100644
--- a/Doc/c-api/gcsupport.rst
+++ b/Doc/c-api/gcsupport.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _supporting-cycle-detection:
diff --git a/Doc/c-api/gen.rst b/Doc/c-api/gen.rst
index 1efbae4fcba..8d54021c181 100644
--- a/Doc/c-api/gen.rst
+++ b/Doc/c-api/gen.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _gen-objects:
diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst
index 8cdc256e7c9..86cc4031610 100644
--- a/Doc/c-api/import.rst
+++ b/Doc/c-api/import.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _importing:
diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst
index 367c069a7ff..a0ac4d21d13 100644
--- a/Doc/c-api/init.rst
+++ b/Doc/c-api/init.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _initialization:
@@ -877,13 +877,6 @@ code, or when embedding the Python interpreter:
and is not released.
-.. c:function:: void PyEval_ReInitThreads()
-
- This function is called from :c:func:`PyOS_AfterFork_Child` to ensure
- that newly created child processes don't hold locks referring to threads
- which are not running in the child process.
-
-
The following functions use thread-local storage, and are not compatible
with sub-interpreters:
diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst
index 69aef0da04f..672936a458f 100644
--- a/Doc/c-api/intro.rst
+++ b/Doc/c-api/intro.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _api-intro:
@@ -156,7 +156,7 @@ complete listing.
.. c:macro:: Py_UNUSED(arg)
Use this for unused arguments in a function definition to silence compiler
- warnings, e.g. ``PyObject* func(PyObject *Py_UNUSED(ignored))``.
+ warnings. Example: ``int func(int a, int Py_UNUSED(b)) { return a; }``.
.. versionadded:: 3.4
diff --git a/Doc/c-api/iter.rst b/Doc/c-api/iter.rst
index 2ba444d1de6..546efb518a7 100644
--- a/Doc/c-api/iter.rst
+++ b/Doc/c-api/iter.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _iterator:
diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst
index 82cb4eba8ab..4d91e4a224f 100644
--- a/Doc/c-api/iterator.rst
+++ b/Doc/c-api/iterator.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _iterator-objects:
diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst
index 5b263a7b1cd..279783a243a 100644
--- a/Doc/c-api/list.rst
+++ b/Doc/c-api/list.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _listobjects:
diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst
index 8093f4b627a..6be29f9cd32 100644
--- a/Doc/c-api/long.rst
+++ b/Doc/c-api/long.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _longobjects:
diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst
index e37dec9949a..4244b47af75 100644
--- a/Doc/c-api/mapping.rst
+++ b/Doc/c-api/mapping.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _mapping:
diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst
index 17ec621610b..b086830feb1 100644
--- a/Doc/c-api/marshal.rst
+++ b/Doc/c-api/marshal.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _marshalling-utils:
diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst
index 65a207691b8..ab49e48782d 100644
--- a/Doc/c-api/memory.rst
+++ b/Doc/c-api/memory.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _memory:
diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst
index 9f6bfd751ad..77afb020405 100644
--- a/Doc/c-api/memoryview.rst
+++ b/Doc/c-api/memoryview.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _memoryview-objects:
diff --git a/Doc/c-api/method.rst b/Doc/c-api/method.rst
index 7a2a84fe110..1ad805e269a 100644
--- a/Doc/c-api/method.rst
+++ b/Doc/c-api/method.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _instancemethod-objects:
diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst
index 017b656854a..68cbda2938f 100644
--- a/Doc/c-api/module.rst
+++ b/Doc/c-api/module.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _moduleobjects:
diff --git a/Doc/c-api/none.rst b/Doc/c-api/none.rst
index 45568fe657b..26d2b7aab20 100644
--- a/Doc/c-api/none.rst
+++ b/Doc/c-api/none.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _noneobject:
diff --git a/Doc/c-api/number.rst b/Doc/c-api/number.rst
index 9fb220b192b..74932f6e744 100644
--- a/Doc/c-api/number.rst
+++ b/Doc/c-api/number.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _number:
diff --git a/Doc/c-api/objbuffer.rst b/Doc/c-api/objbuffer.rst
index 3572564b13e..6b82a642d7e 100644
--- a/Doc/c-api/objbuffer.rst
+++ b/Doc/c-api/objbuffer.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
Old Buffer Protocol
-------------------
diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst
index a64ff2e6b58..ffc35241e7a 100644
--- a/Doc/c-api/object.rst
+++ b/Doc/c-api/object.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _object:
diff --git a/Doc/c-api/objimpl.rst b/Doc/c-api/objimpl.rst
index 7023e519eda..8bd8c107c98 100644
--- a/Doc/c-api/objimpl.rst
+++ b/Doc/c-api/objimpl.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _newtypes:
diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst
index 225a1feb250..6b07c87d62e 100644
--- a/Doc/c-api/refcounting.rst
+++ b/Doc/c-api/refcounting.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _countingrefs:
diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst
index 96893652f73..080ea3222cf 100644
--- a/Doc/c-api/reflection.rst
+++ b/Doc/c-api/reflection.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _reflection:
diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst
index 6d22f35e22b..d11a2dde54d 100644
--- a/Doc/c-api/sequence.rst
+++ b/Doc/c-api/sequence.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _sequence:
diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst
index 64b6dde3b74..074fcb877e1 100644
--- a/Doc/c-api/set.rst
+++ b/Doc/c-api/set.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _setobjects:
diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst
index 8ad9a29b256..d924308890b 100644
--- a/Doc/c-api/slice.rst
+++ b/Doc/c-api/slice.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _slice-objects:
diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst
index 5b771dd4adc..9c05cb3c82d 100644
--- a/Doc/c-api/stable.rst
+++ b/Doc/c-api/stable.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _stable:
diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst
index da45da1d3c7..5e0cfd0264f 100644
--- a/Doc/c-api/structures.rst
+++ b/Doc/c-api/structures.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _common-structs:
diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst
index c8ea78363df..7d870a8d4e4 100644
--- a/Doc/c-api/sys.rst
+++ b/Doc/c-api/sys.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _os:
@@ -289,6 +289,56 @@ accessible to C code. They all work with the current interpreter thread's
.. versionadded:: 3.2
+.. c:function:: int PySys_Audit(const char *event, const char *format, ...)
+
+ .. index:: single: audit events
+
+ Raises an auditing event with any active hooks. Returns zero for success
+ and non-zero with an exception set on failure.
+
+ If any hooks have been added, *format* and other arguments will be used
+ to construct a tuple to pass. Apart from ``N``, the same format characters
+ as used in :c:func:`Py_BuildValue` are available. If the built value is not
+ a tuple, it will be added into a single-element tuple. (The ``N`` format
+ option consumes a reference, but since there is no way to know whether
+ arguments to this function will be consumed, using it may cause reference
+ leaks.)
+
+ :func:`sys.audit` performs the same function from Python code.
+
+ .. versionadded:: 3.8
+
+
+.. c:function:: int PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData)
+
+ .. index:: single: audit events
+
+ Adds to the collection of active auditing hooks. Returns zero for success
+ and non-zero on failure. If the runtime has been initialized, also sets an
+ error on failure. Hooks added through this API are called for all
+ interpreters created by the runtime.
+
+ This function is safe to call before :c:func:`Py_Initialize`. When called
+ after runtime initialization, existing audit hooks are notified and may
+ silently abort the operation by raising an error subclassed from
+ :class:`Exception` (other errors will not be silenced).
+
+ The hook function is of type :c:type:`int (*)(const char *event, PyObject
+ *args, void *userData)`, where *args* is guaranteed to be a
+ :c:type:`PyTupleObject`. The hook function is always called with the GIL
+ held by the Python interpreter that raised the event.
+
+ The *userData* pointer is passed into the hook function. Since hook
+ functions may be called from different runtimes, this pointer should not
+ refer directly to Python state.
+
+ See :pep:`578` for a detailed description of auditing. Functions in the
+ runtime and standard library that raise events include the details in each
+ function's documentation.
+
+ .. versionadded:: 3.8
+
+
.. _processcontrol:
Process Control
diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst
index 20bf9f0b080..259ec4fb485 100644
--- a/Doc/c-api/tuple.rst
+++ b/Doc/c-api/tuple.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _tupleobjects:
diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst
index 4dfd53fb9f0..8f8367ab77c 100644
--- a/Doc/c-api/type.rst
+++ b/Doc/c-api/type.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _typeobjects:
@@ -95,18 +95,6 @@ Type Objects
from a type's base class. Return ``0`` on success, or return ``-1`` and sets an
exception on error.
-.. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec)
-
- Creates and returns a heap type object from the *spec* passed to the function.
-
-.. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
-
- Creates and returns a heap type object from the *spec*. In addition to that,
- the created heap type contains all types contained by the *bases* tuple as base
- types. This allows the caller to reference other heap types as base types.
-
- .. versionadded:: 3.3
-
.. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot)
Return the function pointer stored in the given slot. If the
@@ -115,4 +103,107 @@ Type Objects
Callers will typically cast the result pointer into the appropriate
function type.
+ See :c:member:`PyType_Slot.slot` for possible values of the *slot* argument.
+
+ An exception is raised if *type* is not a heap type.
+
.. versionadded:: 3.4
+
+
+Creating Heap-Allocated Types
+.............................
+
+The following functions and structs are used to create
+:ref:`heap types `.
+
+.. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
+
+ Creates and returns a heap type object from the *spec*.
+
+ If *bases* is a tuple, the created heap type contains all types contained
+ in it as base types.
+
+ If *bases* is *NULL*, the *Py_tp_base* slot is used instead.
+ If that also is *NULL*, the new type derives from :class:`object`.
+
+ This function calls :c:func:`PyType_Ready` on the new type.
+
+ .. versionadded:: 3.3
+
+.. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec)
+
+ Equivalent to ``PyType_FromSpecWithBases(spec, NULL)``.
+
+.. c:type:: PyType_Spec
+
+ Structure defining a type's behavior.
+
+ .. c:member:: const char* PyType_Spec.name
+
+ Name of the type, used to set :c:member:`PyTypeObject.tp_name`.
+
+ .. c:member:: const char* PyType_Spec.doc
+
+ Type docstring, used to set :c:member:`PyTypeObject.tp_doc`.
+
+ .. c:member:: int PyType_Spec.basicsize
+ .. c:member:: int PyType_Spec.itemsize
+
+ Size of the instance in bytes, used to set
+ :c:member:`PyTypeObject.tp_basicsize` and
+ :c:member:`PyTypeObject.tp_itemsize`.
+
+ .. c:member:: int PyType_Spec.flags
+
+ Type flags, used to set :c:member:`PyTypeObject.tp_flags`.
+
+ If the ``Py_TPFLAGS_HEAPTYPE`` flag is not set,
+ :c:func:`PyType_FromSpecWithBases` sets it automatically.
+
+ .. c:member:: PyType_Slot *PyType_Spec.slots
+
+ Array of :c:type:`PyType_Slot` structures.
+ Terminated by the special slot value ``{0, NULL}``.
+
+.. c:type:: PyType_Slot
+
+ Structure defining optional functionality of a type, containing a slot ID
+ and a value pointer.
+
+ .. c:member:: int PyType_Slot.slot
+
+ A slot ID.
+
+ Slot IDs are named like the field names of the structures
+ :c:type:`PyTypeObject`, :c:type:`PyNumberMethods`,
+ :c:type:`PySequenceMethods`, :c:type:`PyMappingMethods` and
+ :c:type:`PyAsyncMethods` with an added ``Py_`` prefix.
+ For example, use:
+
+ * ``Py_tp_dealloc`` to set :c:member:`PyTypeObject.tp_dealloc`
+ * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add`
+ * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length`
+
+ The following fields cannot be set using *PyType_Spec* and *PyType_Slot*:
+
+ * :c:member:`~PyTypeObject.tp_dict`
+ * :c:member:`~PyTypeObject.tp_mro`
+ * :c:member:`~PyTypeObject.tp_cache`
+ * :c:member:`~PyTypeObject.tp_subclasses`
+ * :c:member:`~PyTypeObject.tp_weaklist`
+ * :c:member:`~PyTypeObject.tp_print`
+ * :c:member:`~PyTypeObject.tp_weaklistoffset`
+ * :c:member:`~PyTypeObject.tp_dictoffset`
+ * :c:member:`~PyBufferProcs.bf_getbuffer`
+ * :c:member:`~PyBufferProcs.bf_releasebuffer`
+
+ Setting :c:data:`Py_tp_bases` may be problematic on some platforms.
+ To avoid issues, use the *bases* argument of
+ :py:func:`PyType_FromSpecWithBases` instead.
+
+ .. c:member:: void *PyType_Slot.pfunc
+
+ The desired value of the slot. In most cases, this is a pointer
+ to a function.
+
+ May not be *NULL*.
diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst
index 0647a493303..e0ea9b9b5f9 100644
--- a/Doc/c-api/typeobj.rst
+++ b/Doc/c-api/typeobj.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _type-structs:
@@ -1822,17 +1822,36 @@ objects on the thread which called tp_dealloc will not violate any assumptions
of the library.
+.. _heap-types:
+
Heap Types
----------
-In addition to defining Python types statically, you can define them
-dynamically (i.e. to the heap) using :c:func:`PyType_FromSpec` and
+Traditionally, types defined in C code are *static*, that is,
+a static :c:type:`PyTypeObject` structure is defined directly in code
+and initialized using :c:func:`PyType_Ready`.
+
+This results in types that are limited relative to types defined in Python:
+
+* Static types are limited to one base, i.e. they cannot use multiple
+ inheritance.
+* Static type objects (but not necessarily their instances) are immutable.
+ It is not possible to add or modify the type object's attributes from Python.
+* Static type objects are shared across
+ :ref:`sub-interpreters `, so they should not
+ include any subinterpreter-specific state.
+
+Also, since *PyTypeObject* is not part of the :ref:`stable ABI `,
+any extension modules using static types must be compiled for a specific
+Python minor version.
+
+An alternative to static types is *heap-allocated types*, or *heap types*
+for short, which correspond closely to classes created by Python's
+``class`` statement.
+
+This is done by filling a :c:type:`PyType_Spec` structure and calling
:c:func:`PyType_FromSpecWithBases`.
-.. XXX Explain how to use PyType_FromSpec().
-
-.. XXX Document PyType_Spec.
-
.. _number-structs:
diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst
index 724c798456f..a150a9c39d8 100644
--- a/Doc/c-api/unicode.rst
+++ b/Doc/c-api/unicode.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _unicodeobjects:
diff --git a/Doc/c-api/utilities.rst b/Doc/c-api/utilities.rst
index d4484fb2712..a805b564763 100644
--- a/Doc/c-api/utilities.rst
+++ b/Doc/c-api/utilities.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _utilities:
diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst
index 317093e9561..a7ff08cfb6b 100644
--- a/Doc/c-api/veryhigh.rst
+++ b/Doc/c-api/veryhigh.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _veryhigh:
diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst
index 6cb3e33fe84..7a4f8615b9c 100644
--- a/Doc/c-api/weakref.rst
+++ b/Doc/c-api/weakref.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _weakrefobjects:
diff --git a/Doc/conf.py b/Doc/conf.py
index afe66270c10..e85ea5b2d2f 100644
--- a/Doc/conf.py
+++ b/Doc/conf.py
@@ -23,6 +23,9 @@ try:
except ImportError:
_tkinter = None
'''
+
+manpages_url = 'https://manpages.debian.org/{path}'
+
# General substitutions.
project = 'Python'
copyright = '2001-%s, Python Software Foundation' % time.strftime('%Y')
diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat
index 35527c179f3..213ddcb61fa 100644
--- a/Doc/data/refcounts.dat
+++ b/Doc/data/refcounts.dat
@@ -413,6 +413,16 @@ PyDateTime_FromDateAndTime:int:minute::
PyDateTime_FromDateAndTime:int:second::
PyDateTime_FromDateAndTime:int:usecond::
+PyDateTime_FromDateAndTimeAndFold:PyObject*::+1:
+PyDateTime_FromDateAndTimeAndFold:int:year::
+PyDateTime_FromDateAndTimeAndFold:int:month::
+PyDateTime_FromDateAndTimeAndFold:int:day::
+PyDateTime_FromDateAndTimeAndFold:int:hour::
+PyDateTime_FromDateAndTimeAndFold:int:minute::
+PyDateTime_FromDateAndTimeAndFold:int:second::
+PyDateTime_FromDateAndTimeAndFold:int:usecond::
+PyDateTime_FromDateAndTimeAndFold:int:fold::
+
PyDateTime_FromTimestamp:PyObject*::+1:
PyDateTime_FromTimestamp:PyObject*:args:0:
@@ -2210,6 +2220,13 @@ PyTime_FromTime:int:minute::
PyTime_FromTime:int:second::
PyTime_FromTime:int:usecond::
+PyTime_FromTimeAndFold:PyObject*::+1:
+PyTime_FromTimeAndFold:int:hour::
+PyTime_FromTimeAndFold:int:minute::
+PyTime_FromTimeAndFold:int:second::
+PyTime_FromTimeAndFold:int:usecond::
+PyTime_FromTimeAndFold:int:fold::
+
PyTraceMalloc_Track:int:::
PyTraceMalloc_Track:unsigned int:domain::
PyTraceMalloc_Track:uintptr_t:ptr::
diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst
index 5dd14b1f7a6..2e46c7ac863 100644
--- a/Doc/distributing/index.rst
+++ b/Doc/distributing/index.rst
@@ -113,11 +113,17 @@ recommended tools`_.
.. _currently recommended tools: https://packaging.python.org/guides/tool-recommendations/#packaging-tool-recommendations
-Reading the guide
-=================
+.. index::
+ single: Python Package Index (PyPI)
+ single: PyPI; (see Python Package Index (PyPI))
+
+.. _publishing-python-packages:
+
+Reading the Python Packaging User Guide
+=======================================
The Python Packaging User Guide covers the various key steps and elements
-involved in creating a project:
+involved in creating and publishing a project:
* `Project structure`_
* `Building and packaging the project`_
diff --git a/Doc/distutils/_setuptools_disclaimer.rst b/Doc/distutils/_setuptools_disclaimer.rst
new file mode 100644
index 00000000000..cc75858326d
--- /dev/null
+++ b/Doc/distutils/_setuptools_disclaimer.rst
@@ -0,0 +1,5 @@
+.. note::
+
+ This document is being retained solely until the ``setuptools`` documentation
+ at https://setuptools.readthedocs.io/en/latest/setuptools.html
+ independently covers all of the relevant information currently included here.
diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst
index 1facc0408d5..2601d30f63e 100644
--- a/Doc/distutils/apiref.rst
+++ b/Doc/distutils/apiref.rst
@@ -4,6 +4,16 @@
API Reference
*************
+.. seealso::
+
+ `New and changed setup.py arguments in setuptools `_
+ The ``setuptools`` project adds new capabilities to the ``setup`` function
+ and other APIs, makes the API consistent across different Python versions,
+ and is hence recommended over using ``distutils`` directly.
+
+.. _setuptools-setup-py: https://setuptools.readthedocs.io/en/latest/setuptools.html#new-and-changed-setup-keywords
+
+.. include:: ./_setuptools_disclaimer.rst
:mod:`distutils.core` --- Core Distutils functionality
======================================================
@@ -280,7 +290,7 @@ the full reference.
.. versionchanged:: 3.8
On Unix, C extensions are no longer linked to libpython except on
- Android.
+ Android and Cygwin.
.. class:: Distribution
diff --git a/Doc/distutils/builtdist.rst b/Doc/distutils/builtdist.rst
index f1f34712616..f44d0d039f4 100644
--- a/Doc/distutils/builtdist.rst
+++ b/Doc/distutils/builtdist.rst
@@ -4,6 +4,8 @@
Creating Built Distributions
****************************
+.. include:: ./_setuptools_disclaimer.rst
+
A "built distribution" is what you're probably used to thinking of either as a
"binary package" or an "installer" (depending on your background). It's not
necessarily binary, though, because it might contain only Python source code
diff --git a/Doc/distutils/commandref.rst b/Doc/distutils/commandref.rst
index 6a2ac960f1e..0f6fe2aba86 100644
--- a/Doc/distutils/commandref.rst
+++ b/Doc/distutils/commandref.rst
@@ -4,6 +4,8 @@
Command Reference
*****************
+.. include:: ./_setuptools_disclaimer.rst
+
.. % \section{Building modules: the \protect\command{build} command family}
.. % \label{build-cmds}
.. % \subsubsection{\protect\command{build}}
diff --git a/Doc/distutils/configfile.rst b/Doc/distutils/configfile.rst
index 0874d05fe70..2a5c8329e31 100644
--- a/Doc/distutils/configfile.rst
+++ b/Doc/distutils/configfile.rst
@@ -4,6 +4,8 @@
Writing the Setup Configuration File
************************************
+.. include:: ./_setuptools_disclaimer.rst
+
Often, it's not possible to write down everything needed to build a distribution
*a priori*: you may need to get some information from the user, or from the
user's system, in order to proceed. As long as that information is fairly
diff --git a/Doc/distutils/examples.rst b/Doc/distutils/examples.rst
index f81e06b5e60..4ac552c7c69 100644
--- a/Doc/distutils/examples.rst
+++ b/Doc/distutils/examples.rst
@@ -4,6 +4,8 @@
Examples
********
+.. include:: ./_setuptools_disclaimer.rst
+
This chapter provides a number of basic examples to help get started with
distutils. Additional information about using distutils can be found in the
Distutils Cookbook.
diff --git a/Doc/distutils/extending.rst b/Doc/distutils/extending.rst
index 501fd7c564c..1075e81779a 100644
--- a/Doc/distutils/extending.rst
+++ b/Doc/distutils/extending.rst
@@ -4,6 +4,8 @@
Extending Distutils
*******************
+.. include:: ./_setuptools_disclaimer.rst
+
Distutils can be extended in various ways. Most extensions take the form of new
commands or replacements for existing commands. New commands may be written to
support new types of platform-specific packaging, for example, while
diff --git a/Doc/distutils/index.rst b/Doc/distutils/index.rst
index d6f7640fcb6..1f72a255424 100644
--- a/Doc/distutils/index.rst
+++ b/Doc/distutils/index.rst
@@ -12,10 +12,7 @@
:ref:`distributing-index`
The up to date module distribution documentations
-This document describes the Python Distribution Utilities ("Distutils") from
-the module developer's point of view, describing how to use the Distutils to
-make Python modules and extensions easily available to a wider audience with
-very little overhead for build/release/install mechanics.
+.. include:: ./_setuptools_disclaimer.rst
.. note::
@@ -25,6 +22,11 @@ very little overhead for build/release/install mechanics.
recommendations section `__
in the Python Packaging User Guide for more information.
+This document describes the Python Distribution Utilities ("Distutils") from
+the module developer's point of view, describing the underlying capabilities
+that ``setuptools`` builds on to allow Python developers to make Python modules
+and extensions readily available to a wider audience.
+
.. toctree::
:maxdepth: 2
:numbered:
@@ -34,7 +36,6 @@ very little overhead for build/release/install mechanics.
configfile.rst
sourcedist.rst
builtdist.rst
- packageindex.rst
examples.rst
extending.rst
commandref.rst
diff --git a/Doc/distutils/introduction.rst b/Doc/distutils/introduction.rst
index 7721484fe73..1f8a560e138 100644
--- a/Doc/distutils/introduction.rst
+++ b/Doc/distutils/introduction.rst
@@ -4,6 +4,8 @@
An Introduction to Distutils
****************************
+.. include:: ./_setuptools_disclaimer.rst
+
This document covers using the Distutils to distribute your Python modules,
concentrating on the role of developer/distributor: if you're looking for
information on installing Python modules, you should refer to the
diff --git a/Doc/distutils/packageindex.rst b/Doc/distutils/packageindex.rst
index f74c4396e54..ccb9a598b2b 100644
--- a/Doc/distutils/packageindex.rst
+++ b/Doc/distutils/packageindex.rst
@@ -1,6 +1,4 @@
-.. index::
- single: Python Package Index (PyPI)
- single: PyPI; (see Python Package Index (PyPI))
+:orphan:
.. _package-index:
@@ -12,6 +10,7 @@ The `Python Package Index (PyPI)`_ stores metadata describing distributions
packaged with distutils and other publishing tools, as well the distribution
archives themselves.
-Detailed instructions on using PyPI at :ref:`distributing-index`.
+References to up to date PyPI documentation can be found at
+:ref:`publishing-python-packages`.
.. _Python Package Index (PyPI): https://pypi.org
diff --git a/Doc/distutils/setupscript.rst b/Doc/distutils/setupscript.rst
index 1f99f62f6af..4386a60b664 100644
--- a/Doc/distutils/setupscript.rst
+++ b/Doc/distutils/setupscript.rst
@@ -4,6 +4,8 @@
Writing the Setup Script
************************
+.. include:: ./_setuptools_disclaimer.rst
+
The setup script is the centre of all activity in building, distributing, and
installing modules using the Distutils. The main purpose of the setup script is
to describe your module distribution to the Distutils, so that the various
diff --git a/Doc/distutils/sourcedist.rst b/Doc/distutils/sourcedist.rst
index 0ac8ef41ddc..0600663d00e 100644
--- a/Doc/distutils/sourcedist.rst
+++ b/Doc/distutils/sourcedist.rst
@@ -4,6 +4,8 @@
Creating a Source Distribution
******************************
+.. include:: ./_setuptools_disclaimer.rst
+
As shown in section :ref:`distutils-simple-example`, you use the :command:`sdist` command
to create a source distribution. In the simplest case, ::
diff --git a/Doc/distutils/uploading.rst b/Doc/distutils/uploading.rst
index 4bce6997f9f..4c391cab072 100644
--- a/Doc/distutils/uploading.rst
+++ b/Doc/distutils/uploading.rst
@@ -4,4 +4,5 @@
Uploading Packages to the Package Index
***************************************
-The contents of this page have moved to the section :ref:`package-index`.
+References to up to date PyPI documentation can be found at
+:ref:`publishing-python-packages`.
diff --git a/Doc/extending/building.rst b/Doc/extending/building.rst
index 9bfad7fc318..753b5511ed9 100644
--- a/Doc/extending/building.rst
+++ b/Doc/extending/building.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _building:
@@ -20,7 +20,7 @@ The initialization function has the signature:
It returns either a fully-initialized module, or a :c:type:`PyModuleDef`
instance. See :ref:`initializing-modules` for details.
-.. highlightlang:: python
+.. highlight:: python
For modules with ASCII-only names, the function must be named
``PyInit_``, with ```` replaced by the name of the
@@ -43,7 +43,7 @@ function corresponding to the filename is found.
See the *"Multiple modules in one library"* section in :pep:`489` for details.
-.. highlightlang:: c
+.. highlight:: c
Building C and C++ Extensions with distutils
============================================
diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst
index 13d83b72f82..483bc852f6c 100644
--- a/Doc/extending/embedding.rst
+++ b/Doc/extending/embedding.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _embedding:
diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst
index 433178ab64d..e459514b2b5 100644
--- a/Doc/extending/extending.rst
+++ b/Doc/extending/extending.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _extending-intro:
diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst
index 8b9549d7e39..308c06705e8 100644
--- a/Doc/extending/newtypes.rst
+++ b/Doc/extending/newtypes.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _new-types-topics:
diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst
index b4bf9b9e6f7..59c8cc01222 100644
--- a/Doc/extending/newtypes_tutorial.rst
+++ b/Doc/extending/newtypes_tutorial.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _defining-new-types:
diff --git a/Doc/extending/windows.rst b/Doc/extending/windows.rst
index 67bdd475aeb..c7b92c6ea24 100644
--- a/Doc/extending/windows.rst
+++ b/Doc/extending/windows.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _building-on-windows:
diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst
index f14e8cc824e..9660a701427 100644
--- a/Doc/faq/programming.rst
+++ b/Doc/faq/programming.rst
@@ -792,7 +792,7 @@ Its documentation looks like this::
invoked using the three argument form.
The slash at the end of the parameter list means that all three parameters are
-positional-only. Thus, calling :func:`pow` with keyword aguments would lead to
+positional-only. Thus, calling :func:`pow` with keyword arguments would lead to
an error::
>>> pow(x=3, y=4)
diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst
index ad97cd0932a..a181086e9ce 100644
--- a/Doc/faq/windows.rst
+++ b/Doc/faq/windows.rst
@@ -1,6 +1,6 @@
:tocdepth: 2
-.. highlightlang:: none
+.. highlight:: none
.. _windows-faq:
diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index 5810a6b7997..d3ce3652551 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -15,10 +15,10 @@ Glossary
``...``
Can refer to:
- * The default Python prompt of the interactive shell when entering code for
- an indented code block, when within a pair of matching left and right
- delimiters (parentheses, square brackets, curly braces or triple quotes),
- or after specifying a decorator.
+ * The default Python prompt of the interactive shell when entering the
+ code for an indented code block, when within a pair of matching left and
+ right delimiters (parentheses, square brackets, curly braces or triple
+ quotes), or after specifying a decorator.
* The :const:`Ellipsis` built-in constant.
@@ -225,6 +225,15 @@ Glossary
statement by defining :meth:`__enter__` and :meth:`__exit__` methods.
See :pep:`343`.
+ context variable
+ A variable which can have different values depending on its context.
+ This is similar to Thread-Local Storage in which each execution
+ thread may have a different value for a variable. However, with context
+ variables, there may be several contexts in one execution thread and the
+ main usage for context variables is to keep track of variables in
+ concurrent asynchronous tasks.
+ See :mod:`contextvars`.
+
contiguous
.. index:: C-contiguous, Fortran contiguous
diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst
index 5b2457a1682..cfd9f2e4075 100644
--- a/Doc/howto/clinic.rst
+++ b/Doc/howto/clinic.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
**********************
Argument Clinic How-To
diff --git a/Doc/howto/cporting.rst b/Doc/howto/cporting.rst
index b638e32f5d0..ce7700fc599 100644
--- a/Doc/howto/cporting.rst
+++ b/Doc/howto/cporting.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
.. _cporting-howto:
diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst
index 50cde359503..909deb5fed3 100644
--- a/Doc/howto/instrumentation.rst
+++ b/Doc/howto/instrumentation.rst
@@ -332,6 +332,15 @@ Available static markers
.. versionadded:: 3.7
+.. c:function:: audit(str event, void *tuple)
+
+ Fires when :func:`sys.audit` or :c:func:`PySys_Audit` is called.
+ ``arg0`` is the event name as C string, ``arg1`` is a :c:type:`PyObject`
+ pointer to a tuple object.
+
+ .. versionadded:: 3.8
+
+
SystemTap Tapsets
-----------------
diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst
index 7b2d7b8abf8..24c3235e4ad 100644
--- a/Doc/howto/unicode.rst
+++ b/Doc/howto/unicode.rst
@@ -135,17 +135,22 @@ used than UTF-8.) UTF-8 uses the following rules:
UTF-8 has several convenient properties:
1. It can handle any Unicode code point.
-2. A Unicode string is turned into a sequence of bytes containing no embedded zero
- bytes. This avoids byte-ordering issues, and means UTF-8 strings can be
- processed by C functions such as ``strcpy()`` and sent through protocols that
- can't handle zero bytes.
+2. A Unicode string is turned into a sequence of bytes that contains embedded
+ zero bytes only where they represent the null character (U+0000). This means
+ that UTF-8 strings can be processed by C functions such as ``strcpy()`` and sent
+ through protocols that can't handle zero bytes for anything other than
+ end-of-string markers.
3. A string of ASCII text is also valid UTF-8 text.
4. UTF-8 is fairly compact; the majority of commonly used characters can be
represented with one or two bytes.
5. If bytes are corrupted or lost, it's possible to determine the start of the
next UTF-8-encoded code point and resynchronize. It's also unlikely that
random 8-bit data will look like valid UTF-8.
-
+6. UTF-8 is a byte oriented encoding. The encoding specifies that each
+ character is represented by a specific sequence of one or more bytes. This
+ avoids the byte-ordering issues that can occur with integer and word oriented
+ encodings, like UTF-16 and UTF-32, where the sequence of bytes varies depending
+ on the hardware on which the string was encoded.
References
diff --git a/Doc/includes/sqlite3/adapter_datetime.py b/Doc/includes/sqlite3/adapter_datetime.py
index be33395100c..d5221d80c35 100644
--- a/Doc/includes/sqlite3/adapter_datetime.py
+++ b/Doc/includes/sqlite3/adapter_datetime.py
@@ -13,3 +13,5 @@ cur = con.cursor()
now = datetime.datetime.now()
cur.execute("select ?", (now,))
print(cur.fetchone()[0])
+
+con.close()
diff --git a/Doc/includes/sqlite3/adapter_point_1.py b/Doc/includes/sqlite3/adapter_point_1.py
index 6b1af841564..77daf8f16d2 100644
--- a/Doc/includes/sqlite3/adapter_point_1.py
+++ b/Doc/includes/sqlite3/adapter_point_1.py
@@ -14,3 +14,5 @@ cur = con.cursor()
p = Point(4.0, -3.2)
cur.execute("select ?", (p,))
print(cur.fetchone()[0])
+
+con.close()
diff --git a/Doc/includes/sqlite3/adapter_point_2.py b/Doc/includes/sqlite3/adapter_point_2.py
index d670700f049..cb86331692b 100644
--- a/Doc/includes/sqlite3/adapter_point_2.py
+++ b/Doc/includes/sqlite3/adapter_point_2.py
@@ -15,3 +15,5 @@ cur = con.cursor()
p = Point(4.0, -3.2)
cur.execute("select ?", (p,))
print(cur.fetchone()[0])
+
+con.close()
diff --git a/Doc/includes/sqlite3/connect_db_1.py b/Doc/includes/sqlite3/connect_db_1.py
deleted file mode 100644
index 1b975232865..00000000000
--- a/Doc/includes/sqlite3/connect_db_1.py
+++ /dev/null
@@ -1,3 +0,0 @@
-import sqlite3
-
-con = sqlite3.connect("mydb")
diff --git a/Doc/includes/sqlite3/connect_db_2.py b/Doc/includes/sqlite3/connect_db_2.py
deleted file mode 100644
index f9728b36135..00000000000
--- a/Doc/includes/sqlite3/connect_db_2.py
+++ /dev/null
@@ -1,3 +0,0 @@
-import sqlite3
-
-con = sqlite3.connect(":memory:")
diff --git a/Doc/includes/sqlite3/countcursors.py b/Doc/includes/sqlite3/countcursors.py
index ef3e70a2a9c..112f47703a2 100644
--- a/Doc/includes/sqlite3/countcursors.py
+++ b/Doc/includes/sqlite3/countcursors.py
@@ -13,3 +13,5 @@ con = sqlite3.connect(":memory:", factory=CountCursorsConnection)
cur1 = con.cursor()
cur2 = con.cursor()
print(con.numcursors)
+
+con.close()
diff --git a/Doc/includes/sqlite3/ctx_manager.py b/Doc/includes/sqlite3/ctx_manager.py
index 7af4ad1ecfb..6db77d45046 100644
--- a/Doc/includes/sqlite3/ctx_manager.py
+++ b/Doc/includes/sqlite3/ctx_manager.py
@@ -14,3 +14,7 @@ try:
con.execute("insert into person(firstname) values (?)", ("Joe",))
except sqlite3.IntegrityError:
print("couldn't add Joe twice")
+
+# Connection object used as context manager only commits or rollbacks transactions,
+# so the connection object should be closed manually
+con.close()
diff --git a/Doc/includes/sqlite3/execsql_fetchonerow.py b/Doc/includes/sqlite3/execsql_fetchonerow.py
index 078873bfc97..115bcb50c7c 100644
--- a/Doc/includes/sqlite3/execsql_fetchonerow.py
+++ b/Doc/includes/sqlite3/execsql_fetchonerow.py
@@ -15,3 +15,5 @@ for (name_last, age) in cur:
cur.execute(SELECT)
for row in cur:
print('%s is %d years old.' % (row[0], row[1]))
+
+con.close()
diff --git a/Doc/includes/sqlite3/execsql_printall_1.py b/Doc/includes/sqlite3/execsql_printall_1.py
index a4ce5c52814..19306e6e3ca 100644
--- a/Doc/includes/sqlite3/execsql_printall_1.py
+++ b/Doc/includes/sqlite3/execsql_printall_1.py
@@ -11,3 +11,5 @@ cur.execute("select * from people order by age")
# Retrieve all rows as a sequence and print that sequence:
print(cur.fetchall())
+
+con.close()
diff --git a/Doc/includes/sqlite3/execute_1.py b/Doc/includes/sqlite3/execute_1.py
index f864a8984e4..3466b1265a5 100644
--- a/Doc/includes/sqlite3/execute_1.py
+++ b/Doc/includes/sqlite3/execute_1.py
@@ -14,3 +14,5 @@ cur.execute("insert into people values (?, ?)", (who, age))
cur.execute("select * from people where name_last=:who and age=:age", {"who": who, "age": age})
print(cur.fetchone())
+
+con.close()
diff --git a/Doc/includes/sqlite3/execute_3.py b/Doc/includes/sqlite3/execute_3.py
deleted file mode 100644
index 0353683fc70..00000000000
--- a/Doc/includes/sqlite3/execute_3.py
+++ /dev/null
@@ -1,12 +0,0 @@
-import sqlite3
-
-con = sqlite3.connect("mydb")
-
-cur = con.cursor()
-
-who = "Yeltsin"
-age = 72
-
-cur.execute("select name_last, age from people where name_last=:who and age=:age",
- locals())
-print(cur.fetchone())
diff --git a/Doc/includes/sqlite3/executemany_1.py b/Doc/includes/sqlite3/executemany_1.py
index efae10637c7..edf6f8b7ebe 100644
--- a/Doc/includes/sqlite3/executemany_1.py
+++ b/Doc/includes/sqlite3/executemany_1.py
@@ -22,3 +22,5 @@ cur.executemany("insert into characters(c) values (?)", theIter)
cur.execute("select c from characters")
print(cur.fetchall())
+
+con.close()
diff --git a/Doc/includes/sqlite3/executemany_2.py b/Doc/includes/sqlite3/executemany_2.py
index 527358ebc28..02a594c861e 100644
--- a/Doc/includes/sqlite3/executemany_2.py
+++ b/Doc/includes/sqlite3/executemany_2.py
@@ -13,3 +13,5 @@ cur.executemany("insert into characters(c) values (?)", char_generator())
cur.execute("select c from characters")
print(cur.fetchall())
+
+con.close()
diff --git a/Doc/includes/sqlite3/executescript.py b/Doc/includes/sqlite3/executescript.py
index 7e5358178d4..aea8943fbee 100644
--- a/Doc/includes/sqlite3/executescript.py
+++ b/Doc/includes/sqlite3/executescript.py
@@ -22,3 +22,4 @@ cur.executescript("""
1987
);
""")
+con.close()
diff --git a/Doc/includes/sqlite3/insert_more_people.py b/Doc/includes/sqlite3/insert_more_people.py
index edbc79e7e5b..10cf937243f 100644
--- a/Doc/includes/sqlite3/insert_more_people.py
+++ b/Doc/includes/sqlite3/insert_more_people.py
@@ -14,3 +14,5 @@ for person in newPeople:
# The changes will not be saved unless the transaction is committed explicitly:
con.commit()
+
+con.close()
diff --git a/Doc/includes/sqlite3/load_extension.py b/Doc/includes/sqlite3/load_extension.py
index b997c70668a..624cfe262f3 100644
--- a/Doc/includes/sqlite3/load_extension.py
+++ b/Doc/includes/sqlite3/load_extension.py
@@ -24,3 +24,5 @@ con.executescript("""
""")
for row in con.execute("select rowid, name, ingredients from recipe where name match 'pie'"):
print(row)
+
+con.close()
diff --git a/Doc/includes/sqlite3/md5func.py b/Doc/includes/sqlite3/md5func.py
index 0056b2d6ce8..16dc348bf00 100644
--- a/Doc/includes/sqlite3/md5func.py
+++ b/Doc/includes/sqlite3/md5func.py
@@ -9,3 +9,5 @@ con.create_function("md5", 1, md5sum)
cur = con.cursor()
cur.execute("select md5(?)", (b"foo",))
print(cur.fetchone()[0])
+
+con.close()
diff --git a/Doc/includes/sqlite3/mysumaggr.py b/Doc/includes/sqlite3/mysumaggr.py
index d2dfd2c0b98..11f96395b6c 100644
--- a/Doc/includes/sqlite3/mysumaggr.py
+++ b/Doc/includes/sqlite3/mysumaggr.py
@@ -18,3 +18,5 @@ cur.execute("insert into test(i) values (1)")
cur.execute("insert into test(i) values (2)")
cur.execute("select mysum(i) from test")
print(cur.fetchone()[0])
+
+con.close()
diff --git a/Doc/includes/sqlite3/parse_colnames.py b/Doc/includes/sqlite3/parse_colnames.py
index cc68c76459e..5f01dbfe1cb 100644
--- a/Doc/includes/sqlite3/parse_colnames.py
+++ b/Doc/includes/sqlite3/parse_colnames.py
@@ -6,3 +6,5 @@ cur = con.cursor()
cur.execute('select ? as "x [timestamp]"', (datetime.datetime.now(),))
dt = cur.fetchone()[0]
print(dt, type(dt))
+
+con.close()
diff --git a/Doc/includes/sqlite3/pysqlite_datetime.py b/Doc/includes/sqlite3/pysqlite_datetime.py
index 68d49358a57..5d843f906b3 100644
--- a/Doc/includes/sqlite3/pysqlite_datetime.py
+++ b/Doc/includes/sqlite3/pysqlite_datetime.py
@@ -18,3 +18,5 @@ cur.execute('select current_date as "d [date]", current_timestamp as "ts [timest
row = cur.fetchone()
print("current_date", row[0], type(row[0]))
print("current_timestamp", row[1], type(row[1]))
+
+con.close()
diff --git a/Doc/includes/sqlite3/row_factory.py b/Doc/includes/sqlite3/row_factory.py
index e436ffc6c80..9de6e7b1b90 100644
--- a/Doc/includes/sqlite3/row_factory.py
+++ b/Doc/includes/sqlite3/row_factory.py
@@ -11,3 +11,5 @@ con.row_factory = dict_factory
cur = con.cursor()
cur.execute("select 1 as a")
print(cur.fetchone()["a"])
+
+con.close()
diff --git a/Doc/includes/sqlite3/rowclass.py b/Doc/includes/sqlite3/rowclass.py
index 92b5ad60cb5..fc60287069a 100644
--- a/Doc/includes/sqlite3/rowclass.py
+++ b/Doc/includes/sqlite3/rowclass.py
@@ -10,3 +10,5 @@ for row in cur:
assert row["name"] == row["nAmE"]
assert row[1] == row["age"]
assert row[1] == row["AgE"]
+
+con.close()
diff --git a/Doc/includes/sqlite3/shortcut_methods.py b/Doc/includes/sqlite3/shortcut_methods.py
index 71600d4f60c..98a39411495 100644
--- a/Doc/includes/sqlite3/shortcut_methods.py
+++ b/Doc/includes/sqlite3/shortcut_methods.py
@@ -18,3 +18,7 @@ for row in con.execute("select firstname, lastname from person"):
print(row)
print("I just deleted", con.execute("delete from person").rowcount, "rows")
+
+# close is not a shortcut method and it's not called automatically,
+# so the connection object should be closed manually
+con.close()
diff --git a/Doc/includes/sqlite3/simple_tableprinter.py b/Doc/includes/sqlite3/simple_tableprinter.py
index 231d8726cd4..148a1707f94 100644
--- a/Doc/includes/sqlite3/simple_tableprinter.py
+++ b/Doc/includes/sqlite3/simple_tableprinter.py
@@ -24,3 +24,5 @@ for row in cur:
print(fieldValue.ljust(FIELD_MAX_WIDTH), end=' ')
print() # Finish the row with a newline.
+
+con.close()
diff --git a/Doc/includes/sqlite3/text_factory.py b/Doc/includes/sqlite3/text_factory.py
index 5f96cdb58da..a857a155cdd 100644
--- a/Doc/includes/sqlite3/text_factory.py
+++ b/Doc/includes/sqlite3/text_factory.py
@@ -25,3 +25,5 @@ con.text_factory = lambda x: x.decode("utf-8") + "foo"
cur.execute("select ?", ("bar",))
row = cur.fetchone()
assert row[0] == "barfoo"
+
+con.close()
diff --git a/Doc/install/index.rst b/Doc/install/index.rst
index f6a8cd6833a..a91606c0f38 100644
--- a/Doc/install/index.rst
+++ b/Doc/install/index.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: none
+.. highlight:: none
.. _install-index:
@@ -13,23 +13,10 @@
.. seealso::
:ref:`installing-index`
- The up to date module installation documentations
-
-.. The audience for this document includes people who don't know anything
- about Python and aren't about to learn the language just in order to
- install and maintain it for their users, i.e. system administrators.
- Thus, I have to be sure to explain the basics at some point:
- sys.path and PYTHONPATH at least. Should probably give pointers to
- other docs on "import site", PYTHONSTARTUP, PYTHONHOME, etc.
-
- Finally, it might be useful to include all the material from my "Care
- and Feeding of a Python Installation" talk in here somewhere. Yow!
-
-This document describes the Python Distribution Utilities ("Distutils") from the
-end-user's point-of-view, describing how to extend the capabilities of a
-standard Python installation by building and installing third-party Python
-modules and extensions.
+ The up to date module installation documentation. For regular Python
+ usage, you almost certainly want that document rather than this one.
+.. include:: ../distutils/_setuptools_disclaimer.rst
.. note::
@@ -46,59 +33,26 @@ modules and extensions.
Introduction
============
-Although Python's extensive standard library covers many programming needs,
-there often comes a time when you need to add some new functionality to your
-Python installation in the form of third-party modules. This might be necessary
-to support your own programming, or to support an application that you want to
-use and that happens to be written in Python.
+In Python 2.0, the ``distutils`` API was first added to the standard library.
+This provided Linux distro maintainers with a standard way of converting
+Python projects into Linux distro packages, and system administrators with a
+standard way of installing them directly onto target systems.
-In the past, there has been little support for adding third-party modules to an
-existing Python installation. With the introduction of the Python Distribution
-Utilities (Distutils for short) in Python 2.0, this changed.
+In the many years since Python 2.0 was released, tightly coupling the build
+system and package installer to the language runtime release cycle has turned
+out to be problematic, and it is now recommended that projects use the
+``pip`` package installer and the ``setuptools`` build system, rather than
+using ``distutils`` directly.
-This document is aimed primarily at the people who need to install third-party
-Python modules: end-users and system administrators who just need to get some
-Python application running, and existing Python programmers who want to add some
-new goodies to their toolbox. You don't need to know Python to read this
-document; there will be some brief forays into using Python's interactive mode
-to explore your installation, but that's it. If you're looking for information
-on how to distribute your own Python modules so that others may use them, see
-the :ref:`distutils-index` manual. :ref:`debug-setup-script` may also be of
-interest.
-
-
-.. _inst-trivial-install:
-
-Best case: trivial installation
--------------------------------
-
-In the best case, someone will have prepared a special version of the module
-distribution you want to install that is targeted specifically at your platform
-and is installed just like any other software on your platform. For example,
-the module developer might make an executable installer available for Windows
-users, an RPM package for users of RPM-based Linux systems (Red Hat, SuSE,
-Mandrake, and many others), a Debian package for users of Debian-based Linux
-systems, and so forth.
-
-In that case, you would download the installer appropriate to your platform and
-do the obvious thing with it: run it if it's an executable installer, ``rpm
---install`` it if it's an RPM, etc. You don't need to run Python or a setup
-script, you don't need to compile anything---you might not even need to read any
-instructions (although it's always a good idea to do so anyway).
-
-Of course, things will not always be that easy. You might be interested in a
-module distribution that doesn't have an easy-to-use installer for your
-platform. In that case, you'll have to start with the source distribution
-released by the module's author/maintainer. Installing from a source
-distribution is not too hard, as long as the modules are packaged in the
-standard way. The bulk of this document is about building and installing
-modules from standard source distributions.
+See :ref:`installing-index` and :ref:`distributing-index` for more details.
+This legacy documentation is being retained only until we're confident that the
+``setuptools`` documentation covers everything needed.
.. _inst-new-standard:
-The new standard: Distutils
----------------------------
+Distutils based source distributions
+------------------------------------
If you download a module source distribution, you can tell pretty quickly if it
was packaged and distributed in the standard way, i.e. using the Distutils.
diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst
index 747b9223b62..dc44aa64b8e 100644
--- a/Doc/installing/index.rst
+++ b/Doc/installing/index.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: none
+.. highlight:: none
.. _installing-index:
diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst
index acffabf24ba..a6ce945c720 100644
--- a/Doc/library/_thread.rst
+++ b/Doc/library/_thread.rst
@@ -53,8 +53,12 @@ This module defines the following constants and functions:
.. function:: interrupt_main()
- Raise a :exc:`KeyboardInterrupt` exception in the main thread. A subthread can
- use this function to interrupt the main thread.
+ Simulate the effect of a :data:`signal.SIGINT` signal arriving in the main
+ thread. A thread can use this function to interrupt the main thread.
+
+ If :data:`signal.SIGINT` isn't handled by Python (it was set to
+ :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does
+ nothing.
.. function:: exit()
@@ -85,6 +89,18 @@ This module defines the following constants and functions:
may be recycled when a thread exits and another thread is created.
+.. function:: get_native_id()
+
+ Return the native integral Thread ID of the current thread assigned by the kernel.
+ This is a non-negative integer.
+ Its value may be used to uniquely identify this particular thread system-wide
+ (until the thread terminates, after which the value may be recycled by the OS).
+
+ .. availability:: Windows, FreeBSD, Linux, macOS.
+
+ .. versionadded:: 3.8
+
+
.. function:: stack_size([size])
Return the thread stack size used when creating new threads. The optional
diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst
index cef197f3055..b77a38ccd48 100644
--- a/Doc/library/argparse.rst
+++ b/Doc/library/argparse.rst
@@ -797,6 +797,15 @@ how the command-line arguments should be handled. The supplied actions are:
>>> parser.parse_args(['--version'])
PROG 2.0
+* ``'extend'`` - This stores a list, and extends each argument value to the
+ list.
+ Example usage::
+
+ >>> parser = argparse.ArgumentParser()
+ >>> parser.add_argument("--foo", action="extend", nargs="+", type=str)
+ >>> parser.parse_args(["--foo", "f1", "--foo", "f2", "f3", "f4"])
+ Namespace(foo=['f1', 'f2', 'f3', 'f4'])
+
You may also specify an arbitrary action by passing an Action subclass or
other object that implements the same interface. The recommended way to do
this is to extend :class:`Action`, overriding the ``__call__`` method
diff --git a/Doc/library/array.rst b/Doc/library/array.rst
index 4ac7bb5391a..1f95dd61b9f 100644
--- a/Doc/library/array.rst
+++ b/Doc/library/array.rst
@@ -83,6 +83,7 @@ The module defines the following type:
to add initial items to the array. Otherwise, the iterable initializer is
passed to the :meth:`extend` method.
+ .. audit-event:: array.__new__ "typecode initializer"
.. data:: typecodes
diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst
index a1297f5fb7f..d94fa587cd3 100644
--- a/Doc/library/asyncio-task.rst
+++ b/Doc/library/asyncio-task.rst
@@ -40,7 +40,7 @@ be executed::
>>> main()
-To actually run a coroutine asyncio provides three main mechanisms:
+To actually run a coroutine, asyncio provides three main mechanisms:
* The :func:`asyncio.run` function to run the top-level
entry point "main()" function (see the above example.)
@@ -279,8 +279,8 @@ Sleeping
``sleep()`` always suspends the current task, allowing other tasks
to run.
- The *loop* argument is deprecated and scheduled for removal
- in Python 3.10.
+ .. deprecated-removed:: 3.8 3.10
+ The *loop* parameter.
.. _asyncio_example_sleep:
@@ -437,8 +437,8 @@ Timeouts
If the wait is cancelled, the future *aw* is also cancelled.
- The *loop* argument is deprecated and scheduled for removal
- in Python 3.10.
+ .. deprecated-removed:: 3.8 3.10
+ The *loop* parameter.
.. _asyncio_example_waitfor:
@@ -478,10 +478,12 @@ Waiting Primitives
set concurrently and block until the condition specified
by *return_when*.
- If any awaitable in *aws* is a coroutine, it is automatically
- scheduled as a Task. Passing coroutines objects to
- ``wait()`` directly is deprecated as it leads to
- :ref:`confusing behavior `.
+ .. deprecated:: 3.8
+
+ If any awaitable in *aws* is a coroutine, it is automatically
+ scheduled as a Task. Passing coroutines objects to
+ ``wait()`` directly is deprecated as it leads to
+ :ref:`confusing behavior `.
Returns two sets of Tasks/Futures: ``(done, pending)``.
@@ -489,8 +491,8 @@ Waiting Primitives
done, pending = await asyncio.wait(aws)
- The *loop* argument is deprecated and scheduled for removal
- in Python 3.10.
+ .. deprecated-removed:: 3.8 3.10
+ The *loop* parameter.
*timeout* (a float or int), if specified, can be used to control
the maximum number of seconds to wait before returning.
@@ -550,6 +552,8 @@ Waiting Primitives
if task in done:
# Everything will work as expected now.
+ .. deprecated:: 3.8
+
Passing coroutine objects to ``wait()`` directly is
deprecated.
@@ -868,8 +872,10 @@ Task Object
If *loop* is ``None``, the :func:`get_event_loop` function
is used to get the current loop.
- This method is **deprecated** and will be removed in
- Python 3.9. Use the :func:`asyncio.all_tasks` function instead.
+ .. deprecated-removed:: 3.7 3.9
+
+ Do not call this as a task method. Use the :func:`asyncio.all_tasks`
+ function instead.
.. classmethod:: current_task(loop=None)
@@ -878,9 +884,10 @@ Task Object
If *loop* is ``None``, the :func:`get_event_loop` function
is used to get the current loop.
- This method is **deprecated** and will be removed in
- Python 3.9. Use the :func:`asyncio.current_task` function
- instead.
+ .. deprecated-removed:: 3.7 3.9
+
+ Do not call this as a task method. Use the
+ :func:`asyncio.current_task` function instead.
.. _asyncio_generator_based_coro:
@@ -916,12 +923,13 @@ enforced.
async def main():
await old_style_coroutine()
- This decorator is **deprecated** and is scheduled for removal in
- Python 3.10.
-
This decorator should not be used for :keyword:`async def`
coroutines.
+ .. deprecated-removed:: 3.8 3.10
+
+ Use :keyword:`async def` instead.
+
.. function:: iscoroutine(obj)
Return ``True`` if *obj* is a :ref:`coroutine object `.
diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst
index 946cc67dd30..277de601cb7 100644
--- a/Doc/library/bz2.rst
+++ b/Doc/library/bz2.rst
@@ -83,7 +83,7 @@ All of the classes in this module may safely be accessed from multiple threads.
The *buffering* argument is ignored. Its use is deprecated since Python 3.0.
- If *mode* is ``'w'`` or ``'a'``, *compresslevel* can be a number between
+ If *mode* is ``'w'`` or ``'a'``, *compresslevel* can be an integer between
``1`` and ``9`` specifying the level of compression: ``1`` produces the
least compression, and ``9`` (default) produces the most compression.
@@ -148,7 +148,7 @@ Incremental (de)compression
incrementally. For one-shot compression, use the :func:`compress` function
instead.
- *compresslevel*, if given, must be a number between ``1`` and ``9``. The
+ *compresslevel*, if given, must be an integer between ``1`` and ``9``. The
default is ``9``.
.. method:: compress(data)
@@ -234,9 +234,9 @@ One-shot (de)compression
.. function:: compress(data, compresslevel=9)
- Compress *data*.
+ Compress *data*, a :term:`bytes-like object `.
- *compresslevel*, if given, must be a number between ``1`` and ``9``. The
+ *compresslevel*, if given, must be an integer between ``1`` and ``9``. The
default is ``9``.
For incremental compression, use a :class:`BZ2Compressor` instead.
@@ -244,7 +244,7 @@ One-shot (de)compression
.. function:: decompress(data)
- Decompress *data*.
+ Decompress *data*, a :term:`bytes-like object `.
If *data* is the concatenation of multiple compressed streams, decompress
all of the streams.
@@ -254,3 +254,77 @@ One-shot (de)compression
.. versionchanged:: 3.3
Support for multi-stream inputs was added.
+.. _bz2-usage-examples:
+
+Examples of usage
+-----------------
+
+Below are some examples of typical usage of the :mod:`bz2` module.
+
+Using :func:`compress` and :func:`decompress` to demonstrate round-trip compression:
+
+ >>> import bz2
+
+ >>> data = b"""\
+ ... Donec rhoncus quis sapien sit amet molestie. Fusce scelerisque vel augue
+ ... nec ullamcorper. Nam rutrum pretium placerat. Aliquam vel tristique lorem,
+ ... sit amet cursus ante. In interdum laoreet mi, sit amet ultrices purus
+ ... pulvinar a. Nam gravida euismod magna, non varius justo tincidunt feugiat.
+ ... Aliquam pharetra lacus non risus vehicula rutrum. Maecenas aliquam leo
+ ... felis. Pellentesque semper nunc sit amet nibh ullamcorper, ac elementum
+ ... dolor luctus. Curabitur lacinia mi ornare consectetur vestibulum."""
+
+ >>> c = bz2.compress(data)
+ >>> len(data) / len(c) # Data compression ratio
+ 1.513595166163142
+
+ >>> d = bz2.decompress(c)
+ >>> data == d # Check equality to original object after round-trip
+ True
+
+Using :class:`BZ2Compressor` for incremental compression:
+
+ >>> import bz2
+
+ >>> def gen_data(chunks=10, chunksize=1000):
+ ... """Yield incremental blocks of chunksize bytes."""
+ ... for _ in range(chunks):
+ ... yield b"z" * chunksize
+ ...
+ >>> comp = bz2.BZ2Compressor()
+ >>> out = b""
+ >>> for chunk in gen_data():
+ ... # Provide data to the compressor object
+ ... out = out + comp.compress(chunk)
+ ...
+ >>> # Finish the compression process. Call this once you have
+ >>> # finished providing data to the compressor.
+ >>> out = out + comp.flush()
+
+The example above uses a very "nonrandom" stream of data
+(a stream of `b"z"` chunks). Random data tends to compress poorly,
+while ordered, repetitive data usually yields a high compression ratio.
+
+Writing and reading a bzip2-compressed file in binary mode:
+
+ >>> import bz2
+
+ >>> data = b"""\
+ ... Donec rhoncus quis sapien sit amet molestie. Fusce scelerisque vel augue
+ ... nec ullamcorper. Nam rutrum pretium placerat. Aliquam vel tristique lorem,
+ ... sit amet cursus ante. In interdum laoreet mi, sit amet ultrices purus
+ ... pulvinar a. Nam gravida euismod magna, non varius justo tincidunt feugiat.
+ ... Aliquam pharetra lacus non risus vehicula rutrum. Maecenas aliquam leo
+ ... felis. Pellentesque semper nunc sit amet nibh ullamcorper, ac elementum
+ ... dolor luctus. Curabitur lacinia mi ornare consectetur vestibulum."""
+
+ >>> with bz2.open("myfile.bz2", "wb") as f:
+ ... # Write compressed data to file
+ ... unused = f.write(data)
+
+ >>> with bz2.open("myfile.bz2", "rb") as f:
+ ... # Decompress data from file
+ ... content = f.read()
+
+ >>> content == data # Check equality to original object after round-trip
+ True
diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst
index 9d81730f201..28cd96b0e12 100644
--- a/Doc/library/cmath.rst
+++ b/Doc/library/cmath.rst
@@ -6,13 +6,12 @@
--------------
-This module is always available. It provides access to mathematical functions
-for complex numbers. The functions in this module accept integers,
-floating-point numbers or complex numbers as arguments. They will also accept
-any Python object that has either a :meth:`__complex__` or a :meth:`__float__`
-method: these methods are used to convert the object to a complex or
-floating-point number, respectively, and the function is then applied to the
-result of the conversion.
+This module provides access to mathematical functions for complex numbers. The
+functions in this module accept integers, floating-point numbers or complex
+numbers as arguments. They will also accept any Python object that has either a
+:meth:`__complex__` or a :meth:`__float__` method: these methods are used to
+convert the object to a complex or floating-point number, respectively, and
+the function is then applied to the result of the conversion.
.. note::
diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst
index 8d3daa35d15..2e9314e0fab 100644
--- a/Doc/library/codecs.rst
+++ b/Doc/library/codecs.rst
@@ -1106,10 +1106,6 @@ particular, the following variants typically exist:
+-----------------+--------------------------------+--------------------------------+
| cp1258 | windows-1258 | Vietnamese |
+-----------------+--------------------------------+--------------------------------+
-| cp65001 | | Alias to ``utf_8`` encoding |
-| | | |
-| | | .. versionadded:: 3.3 |
-+-----------------+--------------------------------+--------------------------------+
| euc_jp | eucjp, ujis, u-jis | Japanese |
+-----------------+--------------------------------+--------------------------------+
| euc_jis_2004 | jisx0213, eucjis2004 | Japanese |
@@ -1234,7 +1230,7 @@ particular, the following variants typically exist:
+-----------------+--------------------------------+--------------------------------+
| utf_7 | U7, unicode-1-1-utf-7 | all languages |
+-----------------+--------------------------------+--------------------------------+
-| utf_8 | U8, UTF, utf8 | all languages |
+| utf_8 | U8, UTF, utf8, cp65001 | all languages |
+-----------------+--------------------------------+--------------------------------+
| utf_8_sig | | all languages |
+-----------------+--------------------------------+--------------------------------+
@@ -1245,6 +1241,9 @@ particular, the following variants typically exist:
The utf-32\* decoders no longer decode
byte sequences that correspond to surrogate code points.
+.. versionchanged:: 3.8
+ ``cp65001`` is now an alias to ``utf_8``.
+
Python Specific Encodings
-------------------------
diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst
index 64de970fec9..ae21db216fd 100644
--- a/Doc/library/collections.rst
+++ b/Doc/library/collections.rst
@@ -1017,15 +1017,6 @@ fields:
.. versionchanged:: 3.5
Property docstrings became writeable.
-Default values can be implemented by using :meth:`~somenamedtuple._replace` to
-customize a prototype instance:
-
- >>> Account = namedtuple('Account', 'owner balance transaction_count')
- >>> default_account = Account('', 0.0, 0)
- >>> johns_account = default_account._replace(owner='John')
- >>> janes_account = default_account._replace(owner='Jane')
-
-
.. seealso::
* See :class:`typing.NamedTuple` for a way to add type hints for named
@@ -1141,7 +1132,7 @@ original insertion position is changed and moved to the end::
def __setitem__(self, key, value):
super().__setitem__(key, value)
- super().move_to_end(key)
+ self.move_to_end(key)
An :class:`OrderedDict` would also be useful for implementing
variants of :func:`functools.lru_cache`::
diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst
index 5e08616e934..bb5000a0736 100644
--- a/Doc/library/compileall.rst
+++ b/Doc/library/compileall.rst
@@ -158,7 +158,8 @@ Public functions
The argument *workers* specifies how many workers are used to
compile files in parallel. The default is to not use multiple workers.
If the platform can't use multiple workers and *workers* argument is given,
- then sequential compilation will be used as a fallback. If *workers* is
+ then sequential compilation will be used as a fallback. If *workers*
+ is 0, the number of cores in the system is used. If *workers* is
lower than ``0``, a :exc:`ValueError` will be raised.
*invalidation_mode* should be a member of the
@@ -184,6 +185,9 @@ Public functions
.. versionchanged:: 3.7
The *invalidation_mode* parameter was added.
+ .. versionchanged:: 3.8
+ Setting *workers* to 0 now chooses the optimal number of cores.
+
.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP)
Compile the file with path *fullname*. Return a true value if the file
diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst
index baab0de8f8a..97172c588ae 100644
--- a/Doc/library/ctypes.rst
+++ b/Doc/library/ctypes.rst
@@ -1509,6 +1509,17 @@ object is available:
:c:type:`int`, which is of course not always the truth, so you have to assign
the correct :attr:`restype` attribute to use these functions.
+.. audit-event:: ctypes.dlopen name
+
+ Loading a library through any of these objects raises an
+ :ref:`auditing event ` ``ctypes.dlopen`` with string argument
+ ``name``, the name used to load the library.
+
+.. audit-event:: ctypes.dlsym "library name"
+
+ Accessing a function on a loaded library raises an auditing event
+ ``ctypes.dlsym`` with arguments ``library`` (the library object) and ``name``
+ (the symbol's name as a string or integer).
.. _ctypes-foreign-functions:
@@ -2032,6 +2043,12 @@ Data types
This method returns a ctypes type instance using the memory specified by
*address* which must be an integer.
+ .. audit-event:: ctypes.cdata address
+
+ This method, and others that indirectly call this method, raises an
+ :func:`auditing event ` ``ctypes.cdata`` with argument
+ ``address``.
+
.. method:: from_param(obj)
This method adapts *obj* to a ctypes type. It is called with the actual
@@ -2376,7 +2393,7 @@ other data types containing pointer type fields.
and so on). Later assignments to the :attr:`_fields_` class variable will
raise an AttributeError.
- It is possible to defined sub-subclasses of structure types, they inherit
+ It is possible to define sub-subclasses of structure types, they inherit
the fields of the base class plus the :attr:`_fields_` defined in the
sub-subclass, if any.
@@ -2424,7 +2441,7 @@ other data types containing pointer type fields.
td.lptdesc = POINTER(some_type)
td.u.lptdesc = POINTER(some_type)
- It is possible to defined sub-subclasses of structures, they inherit the
+ It is possible to define sub-subclasses of structures, they inherit the
fields of the base class. If the subclass definition has a separate
:attr:`_fields_` variable, the fields specified in this are appended to the
fields of the base class.
diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst
index 2a4d9ce8a35..7d1e7538a29 100644
--- a/Doc/library/curses.rst
+++ b/Doc/library/curses.rst
@@ -708,9 +708,16 @@ the following methods and attributes:
.. note::
- Writing outside the window, subwindow, or pad raises :exc:`curses.error`.
- Attempting to write to the lower right corner of a window, subwindow,
- or pad will cause an exception to be raised after the string is printed.
+ * Writing outside the window, subwindow, or pad raises :exc:`curses.error`.
+ Attempting to write to the lower right corner of a window, subwindow,
+ or pad will cause an exception to be raised after the string is printed.
+
+ * A `bug in ncurses `_, the backend
+ for this Python module, can cause SegFaults when resizing windows. This
+ is fixed in ncurses-6.1-20190511. If you are stuck with an earlier
+ ncurses, you can avoid triggering this if you do not call :func:`addstr`
+ with a *str* that has embedded newlines. Instead, call :func:`addstr`
+ separately for each line.
.. method:: window.attroff(attr)
diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst
index abdc9773548..3c45e56b5f4 100644
--- a/Doc/library/datetime.rst
+++ b/Doc/library/datetime.rst
@@ -2048,6 +2048,9 @@ For :class:`date` objects, the format codes for hours, minutes, seconds, and
microseconds should not be used, as :class:`date` objects have no such
values. If they're used anyway, ``0`` is substituted for them.
+For the :meth:`datetime.strptime` class method, the default value is ``1900-01-01T00:00:00.000``:
+any components not specified in the format string will be pulled from the default value. [#]_
+
The full set of format codes supported varies across platforms, because Python
calls the platform C library's :func:`strftime` function, and platform
variations are common. To see the full set of format codes supported on your
@@ -2282,3 +2285,4 @@ Notes:
.. rubric:: Footnotes
.. [#] If, that is, we ignore the effects of Relativity
+.. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since ``1900`` is not a leap year.
diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst
index c09ae8cbc60..2d9bae6a7ee 100644
--- a/Doc/library/email.generator.rst
+++ b/Doc/library/email.generator.rst
@@ -36,7 +36,7 @@ something that contains only ASCII characters, using the standard email RFC
Content Transfer Encoding techniques for encoding email messages for transport
over channels that are not "8 bit clean".
-To accomodate reproducible processing of SMIME-signed messages
+To accommodate reproducible processing of SMIME-signed messages
:class:`Generator` disables header folding for message parts of type
``multipart/signed`` and all subparts.
diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst
index af9dff34a80..14be492f55a 100644
--- a/Doc/library/fileinput.rst
+++ b/Doc/library/fileinput.rst
@@ -54,7 +54,7 @@ provided by this module.
The following function is the primary interface of this module:
-.. function:: input(files=None, inplace=False, backup='', bufsize=0, mode='r', openhook=None)
+.. function:: input(files=None, inplace=False, backup='', *, mode='r', openhook=None)
Create an instance of the :class:`FileInput` class. The instance will be used
as global state for the functions of this module, and is also returned to use
@@ -72,8 +72,9 @@ The following function is the primary interface of this module:
.. versionchanged:: 3.2
Can be used as a context manager.
- .. deprecated-removed:: 3.6 3.8
- The *bufsize* parameter.
+ .. versionchanged:: 3.8
+ The keyword parameters *mode* and *openhook* are now keyword-only.
+
The following functions use the global state created by :func:`fileinput.input`;
if there is no active state, :exc:`RuntimeError` is raised.
@@ -135,7 +136,7 @@ The class which implements the sequence behavior provided by the module is
available for subclassing as well:
-.. class:: FileInput(files=None, inplace=False, backup='', bufsize=0, mode='r', openhook=None)
+.. class:: FileInput(files=None, inplace=False, backup='', *, mode='r', openhook=None)
Class :class:`FileInput` is the implementation; its methods :meth:`filename`,
:meth:`fileno`, :meth:`lineno`, :meth:`filelineno`, :meth:`isfirstline`,
@@ -160,18 +161,20 @@ available for subclassing as well:
with FileInput(files=('spam.txt', 'eggs.txt')) as input:
process(input)
+
.. versionchanged:: 3.2
Can be used as a context manager.
.. deprecated:: 3.4
The ``'rU'`` and ``'U'`` modes.
- .. deprecated-removed:: 3.6 3.8
- The *bufsize* parameter.
-
.. deprecated:: 3.8
Support for :meth:`__getitem__` method is deprecated.
+ .. versionchanged:: 3.8
+ The keyword parameter *mode* and *openhook* are now keyword-only.
+
+
**Optional in-place filtering:** if the keyword argument ``inplace=True`` is
passed to :func:`fileinput.input` or to the :class:`FileInput` constructor, the
diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index 613e4f74ac4..7170a781720 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -257,6 +257,12 @@ are always available. They are listed here in alphabetical order.
can be found as the :attr:`~__future__._Feature.compiler_flag` attribute on
the :class:`~__future__._Feature` instance in the :mod:`__future__` module.
+ The optional argument *flags* also controls whether the compiled source is
+ allowed to contain top-level ``await``, ``async for`` and ``async with``.
+ When the bit ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` is set, the return code
+ object has ``CO_COROUTINE`` set in ``co_code``, and can be interactively
+ executed via ``await eval(code_object)``.
+
The argument *optimize* specifies the optimization level of the compiler; the
default value of ``-1`` selects the optimization level of the interpreter as
given by :option:`-O` options. Explicit levels are ``0`` (no optimization;
@@ -269,6 +275,12 @@ are always available. They are listed here in alphabetical order.
If you want to parse Python code into its AST representation, see
:func:`ast.parse`.
+ .. audit-event:: compile "source filename"
+
+ Raises an :func:`auditing event ` ``compile`` with arguments
+ ``source`` and ``filename``. This event may also be raised by implicit
+ compilation.
+
.. note::
When compiling a string with multi-line code in ``'single'`` or
@@ -290,6 +302,10 @@ are always available. They are listed here in alphabetical order.
Previously, :exc:`TypeError` was raised when null bytes were encountered
in *source*.
+ .. versionadded:: 3.8
+ ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable
+ support for top-level ``await``, ``async for``, and ``async with``.
+
.. class:: complex([real[, imag]])
@@ -463,6 +479,11 @@ are always available. They are listed here in alphabetical order.
See :func:`ast.literal_eval` for a function that can safely evaluate strings
with expressions containing only literals.
+ .. audit-event:: exec code_object
+
+ Raises an :func:`auditing event ` ``exec`` with the code object as
+ the argument. Code compilation events may also be raised.
+
.. index:: builtin: exec
.. function:: exec(object[, globals[, locals]])
@@ -492,6 +513,11 @@ are always available. They are listed here in alphabetical order.
builtins are available to the executed code by inserting your own
``__builtins__`` dictionary into *globals* before passing it to :func:`exec`.
+ .. audit-event:: exec code_object
+
+ Raises an :func:`auditing event ` ``exec`` with the code object as
+ the argument. Code compilation events may also be raised.
+
.. note::
The built-in functions :func:`globals` and :func:`locals` return the current
@@ -737,6 +763,16 @@ are always available. They are listed here in alphabetical order.
If the :mod:`readline` module was loaded, then :func:`input` will use it
to provide elaborate line editing and history features.
+ .. audit-event:: builtins.input prompt
+
+ Raises an :func:`auditing event ` ``builtins.input`` with
+ argument ``prompt`` before reading input
+
+ .. audit-event:: builtins.input/result result
+
+ Raises an auditing event ``builtins.input/result`` with the result after
+ successfully reading input.
+
.. class:: int([x])
int(x, base=10)
@@ -1166,6 +1202,11 @@ are always available. They are listed here in alphabetical order.
(where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`,
and :mod:`shutil`.
+ .. audit-event:: open "file mode flags"
+
+ The ``mode`` and ``flags`` arguments may have been modified or inferred from
+ the original call.
+
.. versionchanged::
3.3
diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst
index 8850a33f4ab..3349a94446d 100644
--- a/Doc/library/gzip.rst
+++ b/Doc/library/gzip.rst
@@ -59,6 +59,14 @@ The module defines the following items:
.. versionchanged:: 3.6
Accepts a :term:`path-like object`.
+.. exception:: BadGzipFile
+
+ An exception raised for invalid gzip files. It inherits :exc:`OSError`.
+ :exc:`EOFError` and :exc:`zlib.error` can also be raised for invalid gzip
+ files.
+
+ .. versionadded:: 3.8
+
.. class:: GzipFile(filename=None, mode=None, compresslevel=9, fileobj=None, mtime=None)
Constructor for the :class:`GzipFile` class, which simulates most of the
diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst
index ad449112832..c51cf19e97b 100644
--- a/Doc/library/idle.rst
+++ b/Doc/library/idle.rst
@@ -50,7 +50,7 @@ default title and context menu.
On macOS, there is one application menu. It dynamically changes according
to the window currently selected. It has an IDLE menu, and some entries
-described below are moved around to conform to Apple guidlines.
+described below are moved around to conform to Apple guidelines.
File menu (Shell and Editor)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -700,6 +700,9 @@ If ``sys`` is reset by user code, such as with ``importlib.reload(sys)``,
IDLE's changes are lost and input from the keyboard and output to the screen
will not work correctly.
+When user code raises SystemExit either directly or by calling sys.exit, IDLE
+returns to a Shell prompt instead of exiting.
+
User output in Shell
^^^^^^^^^^^^^^^^^^^^
@@ -769,7 +772,7 @@ facilitate development of tkinter programs. Enter ``import tkinter as tk;
root = tk.Tk()`` in standard Python and nothing appears. Enter the same
in IDLE and a tk window appears. In standard Python, one must also enter
``root.update()`` to see the window. IDLE does the equivalent in the
-background, about 20 times a second, which is about every 50 milleseconds.
+background, about 20 times a second, which is about every 50 milliseconds.
Next enter ``b = tk.Button(root, text='button'); b.pack()``. Again,
nothing visibly changes in standard Python until one enters ``root.update()``.
diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst
index d12f122a57b..81824ddc1e5 100644
--- a/Doc/library/inspect.rst
+++ b/Doc/library/inspect.rst
@@ -948,11 +948,6 @@ Classes and functions
APIs. This function is retained primarily for use in code that needs to
maintain compatibility with the Python 2 ``inspect`` module API.
- .. deprecated:: 3.8
- Use :func:`signature` and
- :ref:`Signature Object `, which provide a
- better introspecting API for callables.
-
.. versionchanged:: 3.4
This function is now based on :func:`signature`, but still ignores
``__wrapped__`` attributes and includes the already bound first
diff --git a/Doc/library/io.rst b/Doc/library/io.rst
index 0f1251687ae..2fb27c3aad7 100644
--- a/Doc/library/io.rst
+++ b/Doc/library/io.rst
@@ -120,6 +120,27 @@ High-level Module Interface
This is an alias for the builtin :func:`open` function.
+ .. audit-event:: open "path mode flags"
+
+ This function raises an :func:`auditing event ` ``open`` with
+ arguments ``path``, ``mode`` and ``flags``. The ``mode`` and ``flags``
+ arguments may have been modified or inferred from the original call.
+
+
+.. function:: open_code(path)
+
+ Opens the provided file with mode ``'rb'``. This function should be used
+ when the intent is to treat the contents as executable code.
+
+ ``path`` should be an absolute path.
+
+ The behavior of this function may be overridden by an earlier call to the
+ :c:func:`PyFile_SetOpenCodeHook`, however, it should always be considered
+ interchangeable with ``open(path, 'rb')``. Overriding the behavior is
+ intended for additional validation or preprocessing of the file.
+
+ .. versionadded:: 3.8
+
.. exception:: BlockingIOError
diff --git a/Doc/library/math.rst b/Doc/library/math.rst
index 7129525c788..bf660ae9def 100644
--- a/Doc/library/math.rst
+++ b/Doc/library/math.rst
@@ -10,8 +10,8 @@
--------------
-This module is always available. It provides access to the mathematical
-functions defined by the C standard.
+This module provides access to the mathematical functions defined by the C
+standard.
These functions cannot be used with complex numbers; use the functions of the
same name from the :mod:`cmath` module if you require support for complex
@@ -166,6 +166,20 @@ Number-theoretic and representation functions
Return ``True`` if *x* is a NaN (not a number), and ``False`` otherwise.
+.. function:: isqrt(n)
+
+ Return the integer square root of the nonnegative integer *n*. This is the
+ floor of the exact square root of *n*, or equivalently the greatest integer
+ *a* such that *a*\ ² |nbsp| ≤ |nbsp| *n*.
+
+ For some applications, it may be more convenient to have the least integer
+ *a* such that *n* |nbsp| ≤ |nbsp| *a*\ ², or in other words the ceiling of
+ the exact square root of *n*. For positive *n*, this can be computed using
+ ``a = 1 + isqrt(n - 1)``.
+
+ .. versionadded:: 3.8
+
+
.. function:: ldexp(x, i)
Return ``x * (2**i)``. This is essentially the inverse of function
@@ -538,3 +552,6 @@ Constants
Module :mod:`cmath`
Complex number versions of many of these functions.
+
+.. |nbsp| unicode:: 0xA0
+ :trim:
diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst
index 0f895d76b83..a82caf86e80 100644
--- a/Doc/library/mmap.rst
+++ b/Doc/library/mmap.rst
@@ -67,6 +67,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length
will be relative to the offset from the beginning of the file. *offset*
defaults to 0. *offset* must be a multiple of the :const:`ALLOCATIONGRANULARITY`.
+ .. audit-event:: mmap.__new__ "fileno length access offset"
.. class:: mmap(fileno, length, flags=MAP_SHARED, prot=PROT_WRITE|PROT_READ, access=ACCESS_DEFAULT[, offset])
:noindex:
@@ -155,6 +156,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length
mm.close()
+ .. audit-event:: mmap.__new__ "fileno length access offset"
Memory-mapped file objects support the following methods:
diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst
index a5ecfa6cc1c..cc6dd4e9d70 100644
--- a/Doc/library/multiprocessing.rst
+++ b/Doc/library/multiprocessing.rst
@@ -131,13 +131,17 @@ to start a process. These *start methods* are
handles on Windows.
On Unix using the *spawn* or *forkserver* start methods will also
-start a *semaphore tracker* process which tracks the unlinked named
-semaphores created by processes of the program. When all processes
-have exited the semaphore tracker unlinks any remaining semaphores.
+start a *resource tracker* process which tracks the unlinked named
+system resources (such as named semaphores or
+:class:`~multiprocessing.shared_memory.SharedMemory` objects) created
+by processes of the program. When all processes
+have exited the resource tracker unlinks any remaining tracked object.
Usually there should be none, but if a process was killed by a signal
-there may be some "leaked" semaphores. (Unlinking the named semaphores
-is a serious matter since the system allows only a limited number, and
-they will not be automatically unlinked until the next reboot.)
+there may be some "leaked" resources. (Neither leaked semaphores nor shared
+memory segments will be automatically unlinked until the next reboot. This is
+problematic for both objects because the system allows only a limited number of
+named semaphores, and shared memory segments occupy some space in the main
+memory.)
To select a start method you use the :func:`set_start_method` in
the ``if __name__ == '__main__'`` clause of the main module. For
@@ -940,6 +944,14 @@ Miscellaneous
An analogue of :func:`threading.current_thread`.
+.. function:: parent_process()
+
+ Return the :class:`Process` object corresponding to the parent process of
+ the :func:`current_process`. For the main process, ``parent_process`` will
+ be ``None``.
+
+ .. versionadded:: 3.8
+
.. function:: freeze_support()
Add support for when a program which uses :mod:`multiprocessing` has been
diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst
index a167e3b885f..8e7ee8bfe78 100644
--- a/Doc/library/os.path.rst
+++ b/Doc/library/os.path.rst
@@ -87,7 +87,7 @@ the :mod:`glob` module.)
.. function:: commonpath(paths)
Return the longest common sub-path of each pathname in the sequence
- *paths*. Raise ValueError if *paths* contains both absolute and relative
+ *paths*. Raise :exc:`ValueError` if *paths* contains both absolute and relative
pathnames, or if *paths* is empty. Unlike :func:`commonprefix`, this
returns a valid path.
@@ -324,9 +324,9 @@ the :mod:`glob` module.)
.. function:: normcase(path)
- Normalize the case of a pathname. On Unix and Mac OS X, this returns the
- path unchanged; on case-insensitive filesystems, it converts the path to
- lowercase. On Windows, it also converts forward slashes to backward slashes.
+ Normalize the case of a pathname. On Windows, convert all characters in the
+ pathname to lowercase, and also convert forward slashes to backward slashes.
+ On other operating systems, return the path unchanged.
Raise a :exc:`TypeError` if the type of *path* is not ``str`` or ``bytes`` (directly
or indirectly through the :class:`os.PathLike` interface).
diff --git a/Doc/library/os.rst b/Doc/library/os.rst
index e77a8fed377..6df2b49c532 100644
--- a/Doc/library/os.rst
+++ b/Doc/library/os.rst
@@ -32,6 +32,7 @@ Notes on the availability of these functions:
objects, and result in an object of the same type, if a path or file name is
returned.
+* On VxWorks, os.fork, os.execv and os.spawn*p* are not supported.
.. note::
@@ -650,7 +651,7 @@ process and user.
File Object Creation
--------------------
-This function creates new :term:`file objects `. (See also
+These functions create new :term:`file objects `. (See also
:func:`~os.open` for opening file descriptors.)
@@ -828,11 +829,14 @@ as internal buffering of data.
most *length* bytes in size. As of Python 3.3, this is equivalent to
``os.truncate(fd, length)``.
+ .. audit-event:: os.truncate "fd length"
+
.. availability:: Unix, Windows.
.. versionchanged:: 3.5
Added support for Windows
+
.. function:: get_blocking(fd)
Get the blocking mode of the file descriptor: ``False`` if the
@@ -844,6 +848,7 @@ as internal buffering of data.
.. versionadded:: 3.5
+
.. function:: isatty(fd)
Return ``True`` if the file descriptor *fd* is open and connected to a
@@ -911,6 +916,8 @@ as internal buffering of data.
This function can support :ref:`paths relative to directory descriptors
` with the *dir_fd* parameter.
+ .. audit-event:: open "path mode flags"
+
.. versionchanged:: 3.4
The new file descriptor is now non-inheritable.
@@ -2755,6 +2762,8 @@ features:
This function can support :ref:`specifying a file descriptor `.
+ .. audit-event:: os.truncate "path length"
+
.. availability:: Unix, Windows.
.. versionadded:: 3.3
@@ -3578,6 +3587,9 @@ written in Python, such as a mail server's external command delivery program.
process. On Windows, the process id will actually be the process handle, so can
be used with the :func:`waitpid` function.
+ Note on VxWorks, this function doesn't return ``-signal`` when the new process is
+ killed. Instead it raises OSError exception.
+
The "l" and "v" variants of the :func:`spawn\* ` functions differ in how
command-line arguments are passed. The "l" variants are perhaps the easiest
to work with if the number of parameters is fixed when the code is written; the
@@ -3711,6 +3723,8 @@ written in Python, such as a mail server's external command delivery program.
to using this function. See the :ref:`subprocess-replacements` section in
the :mod:`subprocess` documentation for some helpful recipes.
+ .. audit-event:: os.system command
+
.. availability:: Unix, Windows.
diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst
index 41aebc4f61c..166de8de1f0 100644
--- a/Doc/library/pathlib.rst
+++ b/Doc/library/pathlib.rst
@@ -1048,11 +1048,20 @@ call fails (for example because the path doesn't exist).
otherwise :exc:`FileExistsError` is raised.
-.. method:: Path.unlink()
+.. method:: Path.unlink(missing_ok=False)
Remove this file or symbolic link. If the path points to a directory,
use :func:`Path.rmdir` instead.
+ If *missing_ok* is false (the default), :exc:`FileNotFoundError` is
+ raised if the path does not exist.
+
+ If *missing_ok* is true, :exc:`FileNotFoundError` exceptions will be
+ ignored (same behavior as the POSIX ``rm -f`` command).
+
+ .. versionchanged:: 3.8
+ The *missing_ok* parameter was added.
+
.. method:: Path.link_to(target)
diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst
index 55005f00943..f4c41ac68d2 100644
--- a/Doc/library/pickle.rst
+++ b/Doc/library/pickle.rst
@@ -427,6 +427,7 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and
how they can be loaded, potentially reducing security risks. Refer to
:ref:`pickle-restrict` for details.
+ .. audit-event:: pickle.find_class "module name"
.. _pickle-picklable:
@@ -642,7 +643,7 @@ or both.
by other classes as long as they implement :meth:`__setitem__`.
* Optionally, a callable with a ``(obj, state)`` signature. This
- callable allows the user to programatically control the state-updating
+ callable allows the user to programmatically control the state-updating
behavior of a specific object, instead of using ``obj``'s static
:meth:`__setstate__` method. If not ``None``, this callable will have
priority over ``obj``'s :meth:`__setstate__`.
diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst
index 8bd6b63a8ee..d84fcac0ef2 100644
--- a/Doc/library/plistlib.rst
+++ b/Doc/library/plistlib.rst
@@ -36,6 +36,10 @@ or :class:`datetime.datetime` objects.
.. versionchanged:: 3.4
New API, old API deprecated. Support for binary format plists added.
+.. versionchanged:: 3.8
+ Support added for reading and writing :class:`UID` tokens in binary plists as used
+ by NSKeyedArchiver and NSKeyedUnarchiver.
+
.. seealso::
`PList manual page `_
@@ -179,6 +183,16 @@ The following classes are available:
.. deprecated:: 3.4 Use a :class:`bytes` object instead.
+.. class:: UID(data)
+
+ Wraps an :class:`int`. This is used when reading or writing NSKeyedArchiver
+ encoded data, which contains UID (see PList manual).
+
+ It has one attribute, :attr:`data` which can be used to retrieve the int value
+ of the UID. :attr:`data` must be in the range `0 <= data <= 2**64`.
+
+ .. versionadded:: 3.8
+
The following constants are available:
diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst
index 0ab766065d6..12268437d07 100644
--- a/Doc/library/pty.rst
+++ b/Doc/library/pty.rst
@@ -43,11 +43,32 @@ The :mod:`pty` module defines the following functions:
Spawn a process, and connect its controlling terminal with the current
process's standard io. This is often used to baffle programs which insist on
- reading from the controlling terminal.
+ reading from the controlling terminal. It is expected that the process
+ spawned behind the pty will eventually terminate, and when it does *spawn*
+ will return.
+
+ The functions *master_read* and *stdin_read* are passed a file descriptor
+ which they should read from, and they should always return a byte string. In
+ order to force spawn to return before the child process exits an
+ :exc:`OSError` should be thrown.
+
+ The default implementation for both functions will read and return up to 1024
+ bytes each time the function is called. The *master_read* callback is passed
+ the pseudoterminal’s master file descriptor to read output from the child
+ process, and *stdin_read* is passed file descriptor 0, to read from the
+ parent process's standard input.
+
+ Returning an empty byte string from either callback is interpreted as an
+ end-of-file (EOF) condition, and that callback will not be called after
+ that. If *stdin_read* signals EOF the controlling terminal can no longer
+ communicate with the parent process OR the child process. Unless the child
+ process will quit without any input, *spawn* will then loop forever. If
+ *master_read* signals EOF the same behavior results (on linux at least).
+
+ If both callbacks signal EOF then *spawn* will probably never return, unless
+ *select* throws an error on your platform when passed three empty lists. This
+ is a bug, documented in `issue 26228 `_.
- The functions *master_read* and *stdin_read* should be functions which read from
- a file descriptor. The defaults try to read 1024 bytes each time they are
- called.
.. versionchanged:: 3.4
:func:`spawn` now returns the status value from :func:`os.waitpid`
diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst
index 2dc872fd077..4af5a168060 100644
--- a/Doc/library/shutil.rst
+++ b/Doc/library/shutil.rst
@@ -54,7 +54,7 @@ Directory and files operations
*dst* and return *dst* in the most efficient way possible.
*src* and *dst* are path names given as strings.
- *dst* must be the complete target file name; look at :func:`shutil.copy`
+ *dst* must be the complete target file name; look at :func:`~shutil.copy`
for a copy that accepts a target directory path. If *src* and *dst*
specify the same file, :exc:`SameFileError` is raised.
@@ -218,7 +218,7 @@ Directory and files operations
already exists.
Permissions and times of directories are copied with :func:`copystat`,
- individual files are copied using :func:`shutil.copy2`.
+ individual files are copied using :func:`~shutil.copy2`.
If *symlinks* is true, symbolic links in the source tree are represented as
symbolic links in the new tree and the metadata of the original links will
@@ -246,8 +246,8 @@ Directory and files operations
If *copy_function* is given, it must be a callable that will be used to copy
each file. It will be called with the source path and the destination path
- as arguments. By default, :func:`shutil.copy2` is used, but any function
- that supports the same signature (like :func:`shutil.copy`) can be used.
+ Â as arguments. By default, :func:`~shutil.copy2` is used, but any function
+ Â that supports the same signature (like :func:`~shutil.copy`) can be used.
.. versionchanged:: 3.3
Copy metadata when *symlinks* is false.
diff --git a/Doc/library/site.rst b/Doc/library/site.rst
index dfc40d17944..9e4d402c1f5 100644
--- a/Doc/library/site.rst
+++ b/Doc/library/site.rst
@@ -8,7 +8,7 @@
--------------
-.. highlightlang:: none
+.. highlight:: none
**This module is automatically imported during initialization.** The automatic
import can be suppressed using the interpreter's :option:`-S` option.
diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst
index 379633a3b60..e23a4f5380b 100644
--- a/Doc/library/socket.rst
+++ b/Doc/library/socket.rst
@@ -526,6 +526,8 @@ The following functions all create :ref:`socket objects `.
The newly created socket is :ref:`non-inheritable `.
+ .. audit-event:: socket.__new__ "self family type protocol"
+
.. versionchanged:: 3.3
The AF_CAN family was added.
The AF_RDS family was added.
@@ -718,6 +720,8 @@ The :mod:`socket` module also offers various network-related services:
:const:`AF_INET6`), and is meant to be passed to the :meth:`socket.connect`
method.
+ .. audit-event:: socket.getaddrinfo "host port family type protocol"
+
The following example fetches address information for a hypothetical TCP
connection to ``example.org`` on port 80 (results may differ on your
system if IPv6 isn't enabled)::
@@ -753,6 +757,8 @@ The :mod:`socket` module also offers various network-related services:
interface. :func:`gethostbyname` does not support IPv6 name resolution, and
:func:`getaddrinfo` should be used instead for IPv4/v6 dual stack support.
+ .. audit-event:: socket.gethostbyname hostname
+
.. function:: gethostbyname_ex(hostname)
@@ -765,12 +771,16 @@ The :mod:`socket` module also offers various network-related services:
resolution, and :func:`getaddrinfo` should be used instead for IPv4/v6 dual
stack support.
+ .. audit-event:: socket.gethostbyname hostname
+
.. function:: gethostname()
Return a string containing the hostname of the machine where the Python
interpreter is currently executing.
+ .. audit-event:: socket.gethostname
+
Note: :func:`gethostname` doesn't always return the fully qualified domain
name; use :func:`getfqdn` for that.
@@ -785,6 +795,8 @@ The :mod:`socket` module also offers various network-related services:
domain name, use the function :func:`getfqdn`. :func:`gethostbyaddr` supports
both IPv4 and IPv6.
+ .. audit-event:: socket.gethostbyaddr ip_address
+
.. function:: getnameinfo(sockaddr, flags)
@@ -798,6 +810,8 @@ The :mod:`socket` module also offers various network-related services:
For more information about *flags* you can consult :manpage:`getnameinfo(3)`.
+ .. audit-event:: socket.getnameinfo sockaddr
+
.. function:: getprotobyname(protocolname)
Translate an Internet protocol name (for example, ``'icmp'``) to a constant
@@ -813,6 +827,8 @@ The :mod:`socket` module also offers various network-related services:
service. The optional protocol name, if given, should be ``'tcp'`` or
``'udp'``, otherwise any protocol will match.
+ .. audit-event:: socket.getservbyname "servicename protocolname"
+
.. function:: getservbyport(port[, protocolname])
@@ -820,6 +836,8 @@ The :mod:`socket` module also offers various network-related services:
service. The optional protocol name, if given, should be ``'tcp'`` or
``'udp'``, otherwise any protocol will match.
+ .. audit-event:: socket.getservbyport "port protocolname"
+
.. function:: ntohl(x)
@@ -1003,6 +1021,8 @@ The :mod:`socket` module also offers various network-related services:
Set the machine's hostname to *name*. This will raise an
:exc:`OSError` if you don't have enough rights.
+ .. audit-event:: socket.sethostname name
+
.. availability:: Unix.
.. versionadded:: 3.3
@@ -1078,6 +1098,7 @@ to sockets.
Bind the socket to *address*. The socket must not already be bound. (The format
of *address* depends on the address family --- see above.)
+ .. audit-event:: socket.bind "self address"
.. method:: socket.close()
@@ -1115,6 +1136,8 @@ to sockets.
:exc:`InterruptedError` exception if the connection is interrupted by a
signal (or the exception raised by the signal handler).
+ .. audit-event:: socket.connect "self address"
+
.. versionchanged:: 3.5
The method now waits until the connection completes instead of raising an
:exc:`InterruptedError` exception if the connection is interrupted by a
@@ -1131,6 +1154,7 @@ to sockets.
:c:data:`errno` variable. This is useful to support, for example, asynchronous
connects.
+ .. audit-event:: socket.connect "self address"
.. method:: socket.detach()
@@ -1472,6 +1496,8 @@ to sockets.
bytes sent. (The format of *address* depends on the address family --- see
above.)
+ .. audit-event:: socket.sendto "self address"
+
.. versionchanged:: 3.5
If the system call is interrupted and the signal handler does not raise
an exception, the method now retries the system call instead of raising
@@ -1511,6 +1537,8 @@ to sockets.
.. availability:: most Unix platforms, possibly others.
+ .. audit-event:: socket.sendmsg "self address"
+
.. versionadded:: 3.3
.. versionchanged:: 3.5
diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst
index 37087ac5af4..20fca54aab1 100644
--- a/Doc/library/sqlite3.rst
+++ b/Doc/library/sqlite3.rst
@@ -537,6 +537,7 @@ Connection Objects
with open('dump.sql', 'w') as f:
for line in con.iterdump():
f.write('%s\n' % line)
+ con.close()
.. method:: backup(target, *, pages=0, progress=None, name="main", sleep=0.250)
@@ -573,8 +574,11 @@ Connection Objects
print(f'Copied {total-remaining} of {total} pages...')
con = sqlite3.connect('existing_db.db')
- with sqlite3.connect('backup.db') as bck:
+ bck = sqlite3.connect('backup.db')
+ with bck:
con.backup(bck, pages=1, progress=progress)
+ bck.close()
+ con.close()
Example 2, copy an existing database into a transient copy::
diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst
index fb7df4e7188..bc841fda72f 100644
--- a/Doc/library/statistics.rst
+++ b/Doc/library/statistics.rst
@@ -511,22 +511,33 @@ However, for reading convenience, most of the examples show sorted sequences.
is not least 1.
The *dist* can be any iterable containing sample data or it can be an
- instance of a class that defines an :meth:`~inv_cdf` method.
+ instance of a class that defines an :meth:`~inv_cdf` method. For meaningful
+ results, the number of data points in *dist* should be larger than *n*.
Raises :exc:`StatisticsError` if there are not at least two data points.
For sample data, the cut points are linearly interpolated from the
two nearest data points. For example, if a cut point falls one-third
of the distance between two sample values, ``100`` and ``112``, the
- cut-point will evaluate to ``104``. Other selection methods may be
- offered in the future (for example choose ``100`` as the nearest
- value or compute ``106`` as the midpoint). This might matter if
- there are too few samples for a given number of cut points.
+ cut-point will evaluate to ``104``.
- If *method* is set to *inclusive*, *dist* is treated as population data.
- The minimum value is treated as the 0th percentile and the maximum
- value is treated as the 100th percentile. If *dist* is an instance of
- a class that defines an :meth:`~inv_cdf` method, setting *method*
- has no effect.
+ The *method* for computing quantiles can be varied depending on
+ whether the data in *dist* includes or excludes the lowest and
+ highest possible values from the population.
+
+ The default *method* is "exclusive" and is used for data sampled from
+ a population that can have more extreme values than found in the
+ samples. The portion of the population falling below the *i-th* of
+ *m* data points is computed as ``i / (m + 1)``.
+
+ Setting the *method* to "inclusive" is used for describing population
+ data or for samples that include the extreme points. The minimum
+ value in *dist* is treated as the 0th percentile and the maximum
+ value is treated as the 100th percentile. The portion of the
+ population falling below the *i-th* of *m* data points is computed as
+ ``(i - 1) / (m - 1)``.
+
+ If *dist* is an instance of a class that defines an
+ :meth:`~inv_cdf` method, setting *method* has no effect.
.. doctest::
diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst
index 53337291dd3..293a1ab6a0d 100644
--- a/Doc/library/stdtypes.rst
+++ b/Doc/library/stdtypes.rst
@@ -1114,7 +1114,7 @@ Notes:
item is removed and returned.
(3)
- ``remove`` raises :exc:`ValueError` when *x* is not found in *s*.
+ :meth:`remove` raises :exc:`ValueError` when *x* is not found in *s*.
(4)
The :meth:`reverse` method modifies the sequence in place for economy of
@@ -1124,7 +1124,9 @@ Notes:
(5)
:meth:`clear` and :meth:`!copy` are included for consistency with the
interfaces of mutable containers that don't support slicing operations
- (such as :class:`dict` and :class:`set`)
+ (such as :class:`dict` and :class:`set`). :meth:`!copy` is not part of the
+ :class:`collections.abc.MutableSequence` ABC, but most concrete
+ mutable sequence classes provide it.
.. versionadded:: 3.3
:meth:`clear` and :meth:`!copy` methods.
diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst
index 3280c95cacb..d840b461f98 100644
--- a/Doc/library/subprocess.rst
+++ b/Doc/library/subprocess.rst
@@ -55,7 +55,9 @@ compatibility with older versions, see the :ref:`call-function-trio` section.
If *capture_output* is true, stdout and stderr will be captured.
When used, the internal :class:`Popen` object is automatically created with
``stdout=PIPE`` and ``stderr=PIPE``. The *stdout* and *stderr* arguments may
- not be supplied at the same time as *capture_output*.
+ not be supplied at the same time as *capture_output*. If you wish to capture
+ and combine both streams into one, use ``stdout=PIPE`` and ``stderr=STDOUT``
+ instead of *capture_output*.
The *timeout* argument is passed to :meth:`Popen.communicate`. If the timeout
expires, the child process will be killed and waited for. The
diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index 591972e9b78..0294f74368c 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -19,6 +19,30 @@ always available.
.. versionadded:: 3.2
+.. function:: addaudithook(hook)
+
+ Adds the callable *hook* to the collection of active auditing hooks for the
+ current interpreter.
+
+ When an auditing event is raised through the :func:`sys.audit` function, each
+ hook will be called in the order it was added with the event name and the
+ tuple of arguments. Native hooks added by :c:func:`PySys_AddAuditHook` are
+ called first, followed by hooks added in the current interpreter.
+
+ Calling this function will trigger an event for all existing hooks, and if
+ any raise an exception derived from :class:`Exception`, the add will be
+ silently ignored. As a result, callers cannot assume that their hook has been
+ added unless they control all existing hooks.
+
+ .. versionadded:: 3.8
+
+ .. impl-detail::
+
+ When tracing is enabled, Python hooks are only traced if the callable has
+ a ``__cantrace__`` member that is set to a true value. Otherwise, trace
+ functions will not see the hook.
+
+
.. data:: argv
The list of command line arguments passed to a Python script. ``argv[0]`` is the
@@ -37,6 +61,30 @@ always available.
``[os.fsencode(arg) for arg in sys.argv]``.
+.. _auditing:
+
+.. function:: audit(event, *args)
+
+ .. index:: single: auditing
+
+ Raises an auditing event with any active hooks. The event name is a string
+ identifying the event and its associated schema, which is the number and
+ types of arguments. The schema for a given event is considered public and
+ stable API and should not be modified between releases.
+
+ This function will raise the first exception raised by any hook. In general,
+ these errors should not be handled and should terminate the process as
+ quickly as possible.
+
+ Hooks are added using the :func:`sys.addaudithook` or
+ :c:func:`PySys_AddAuditHook` functions.
+
+ The native equivalent of this function is :c:func:`PySys_Audit`. Using the
+ native function is preferred when possible.
+
+ .. versionadded:: 3.8
+
+
.. data:: base_exec_prefix
Set during Python startup, before ``site.py`` is run, to the same value as
@@ -114,6 +162,8 @@ always available.
This function should be used for internal and specialized purposes only.
+ .. audit-event:: sys._current_frames
+
.. function:: breakpointhook()
@@ -248,16 +298,19 @@ always available.
before the program exits. The handling of such top-level exceptions can be
customized by assigning another three-argument function to ``sys.excepthook``.
+ See also :func:`unraisablehook` which handles unraisable exceptions.
+
.. data:: __breakpointhook__
__displayhook__
__excepthook__
+ __unraisablehook__
These objects contain the original values of ``breakpointhook``,
- ``displayhook``, and ``excepthook`` at the start of the program. They are
- saved so that ``breakpointhook``, ``displayhook`` and ``excepthook`` can be
- restored in case they happen to get replaced with broken or alternative
- objects.
+ ``displayhook``, ``excepthook``, and ``unraisablehook`` at the start of the
+ program. They are saved so that ``breakpointhook``, ``displayhook`` and
+ ``excepthook``, ``unraisablehook`` can be restored in case they happen to
+ get replaced with broken or alternative objects.
.. versionadded:: 3.7
__breakpointhook__
@@ -614,6 +667,8 @@ always available.
that is deeper than the call stack, :exc:`ValueError` is raised. The default
for *depth* is zero, returning the frame at the top of the call stack.
+ .. audit-event:: sys._getframe
+
.. impl-detail::
This function should be used for internal and specialized purposes only.
@@ -1143,6 +1198,8 @@ always available.
``'return'``, ``'c_call'``, ``'c_return'``, or ``'c_exception'``. *arg* depends
on the event type.
+ .. audit-event:: sys.setprofile
+
The events have the following meaning:
``'call'``
@@ -1204,8 +1261,8 @@ always available.
Set the system's trace function, which allows you to implement a Python
source code debugger in Python. The function is thread-specific; for a
- debugger to support multiple threads, it must be registered using
- :func:`settrace` for each thread being debugged.
+ debugger to support multiple threads, it must register a trace function using
+ :func:`settrace` for each thread being debugged or use :func:`threading.settrace`.
Trace functions should have three arguments: *frame*, *event*, and
*arg*. *frame* is the current stack frame. *event* is a string: ``'call'``,
@@ -1263,6 +1320,8 @@ always available.
For more information on code and frame objects, refer to :ref:`types`.
+ .. audit-event:: sys.settrace
+
.. impl-detail::
The :func:`settrace` function is intended only for implementing debuggers,
@@ -1283,6 +1342,13 @@ always available.
first time. The *finalizer* will be called when an asynchronous generator
is about to be garbage collected.
+ .. audit-event:: sys.set_asyncgen_hooks_firstiter
+
+ .. audit-event:: sys.set_asyncgen_hooks_finalizer
+
+ Two auditing events are raised because the underlying API consists of two
+ calls, each of which must raise its own event.
+
.. versionadded:: 3.6
See :pep:`525` for more details, and for a reference example of a
*finalizer* method see the implementation of
@@ -1353,7 +1419,7 @@ always available.
This function has been added on a provisional basis (see :pep:`411`
for details.) Use it only for debugging purposes.
- .. deprecated:: 3.7
+ .. deprecated-removed:: 3.7 3.8
The coroutine wrapper functionality has been deprecated, and
will be removed in 3.8. See :issue:`32591` for details.
@@ -1487,6 +1553,28 @@ always available.
is suppressed and only the exception type and value are printed.
+.. function:: unraisablehook(unraisable, /)
+
+ Handle an unraisable exception.
+
+ Called when an exception has occurred but there is no way for Python to
+ handle it. For example, when a destructor raises an exception or during
+ garbage collection (:func:`gc.collect`).
+
+ The *unraisable* argument has the following attributes:
+
+ * *exc_type*: Exception type.
+ * *exc_value*: Exception value, can be ``None``.
+ * *exc_traceback*: Exception traceback, can be ``None``.
+ * *object*: Object causing the exception, can be ``None``.
+
+ :func:`sys.unraisablehook` can be overridden to control how unraisable
+ exceptions are handled.
+
+ See also :func:`excepthook` which handles uncaught exceptions.
+
+ .. versionadded:: 3.8
+
.. data:: version
A string containing the version number of the Python interpreter plus additional
diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst
index c58a6ad75d0..1df512f1d63 100644
--- a/Doc/library/threading.rst
+++ b/Doc/library/threading.rst
@@ -49,6 +49,18 @@ This module defines the following functions:
.. versionadded:: 3.3
+.. function:: get_native_id()
+
+ Return the native integral Thread ID of the current thread assigned by the kernel.
+ This is a non-negative integer.
+ Its value may be used to uniquely identify this particular thread system-wide
+ (until the thread terminates, after which the value may be recycled by the OS).
+
+ .. availability:: Windows, FreeBSD, Linux, macOS.
+
+ .. versionadded:: 3.8
+
+
.. function:: enumerate()
Return a list of all :class:`Thread` objects currently alive. The list
@@ -297,6 +309,26 @@ since it is impossible to detect the termination of alien threads.
another thread is created. The identifier is available even after the
thread has exited.
+ .. attribute:: native_id
+
+ The native integral thread ID of this thread.
+ This is a non-negative integer, or ``None`` if the thread has not
+ been started. See the :func:`get_native_id` function.
+ This represents the Thread ID (``TID``) as assigned to the
+ thread by the OS (kernel). Its value may be used to uniquely identify
+ this particular thread system-wide (until the thread terminates,
+ after which the value may be recycled by the OS).
+
+ .. note::
+
+ Similar to Process IDs, Thread IDs are only valid (guaranteed unique
+ system-wide) from the time the thread is created until the thread
+ has been terminated.
+
+ .. availability:: Windows, FreeBSD, Linux, macOS.
+
+ .. versionadded:: 3.8
+
.. method:: is_alive()
Return whether the thread is alive.
@@ -937,7 +969,7 @@ As an example, here is a simple way to synchronize a client and server thread::
Return the barrier to the default, empty state. Any threads waiting on it
will receive the :class:`BrokenBarrierError` exception.
- Note that using this function may can require some external
+ Note that using this function may require some external
synchronization if there are other threads whose state is unknown. If a
barrier is broken it may be better to just leave it and create a new one.
@@ -945,7 +977,7 @@ As an example, here is a simple way to synchronize a client and server thread::
Put the barrier into a broken state. This causes any active or future
calls to :meth:`wait` to fail with the :class:`BrokenBarrierError`. Use
- this for example if one of the needs to abort, to avoid deadlocking the
+ this for example if one of the threads needs to abort, to avoid deadlocking the
application.
It may be preferable to simply create the barrier with a sensible
diff --git a/Doc/library/time.rst b/Doc/library/time.rst
index 170f8dc629b..6d0ceafa522 100644
--- a/Doc/library/time.rst
+++ b/Doc/library/time.rst
@@ -136,30 +136,6 @@ Functions
Unlike the C function of the same name, :func:`asctime` does not add a
trailing newline.
-
-.. function:: clock()
-
- .. index::
- single: CPU time
- single: processor time
- single: benchmarking
-
- On Unix, return the current processor time as a floating point number expressed
- in seconds. The precision, and in fact the very definition of the meaning of
- "processor time", depends on that of the C function of the same name.
-
- On Windows, this function returns wall-clock seconds elapsed since the first
- call to this function, as a floating point number, based on the Win32 function
- :c:func:`QueryPerformanceCounter`. The resolution is typically better than one
- microsecond.
-
- .. availability:: Windows, Unix. Not available on VxWorks.
-
- .. deprecated:: 3.3
- The behaviour of this function depends on the platform: use
- :func:`perf_counter` or :func:`process_time` instead, depending on your
- requirements, to have a well defined behaviour.
-
.. function:: pthread_getcpuclockid(thread_id)
Return the *clk_id* of the thread-specific CPU-time clock for the specified *thread_id*.
diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst
index 8ca37034f79..ef7a4e40be6 100644
--- a/Doc/library/timeit.rst
+++ b/Doc/library/timeit.rst
@@ -44,8 +44,12 @@ This can be achieved from the :ref:`python-interface` with::
>>> timeit.timeit('"-".join(map(str, range(100)))', number=10000)
0.23702679807320237
+A callable can also be passed from the :ref:`python-interface`::
-Note however that :mod:`timeit` will automatically determine the number of
+ >>> timeit.timeit(lambda: "-".join(map(str, range(100))), number=10000)
+ 0.19665591977536678
+
+Note however that :func:`.timeit` will automatically determine the number of
repetitions only when the command-line interface is used. In the
:ref:`timeit-examples` section you can find more advanced examples.
diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst
index c2523ed5296..86a3db8467e 100644
--- a/Doc/library/typing.rst
+++ b/Doc/library/typing.rst
@@ -529,6 +529,12 @@ The module defines the following classes, functions and decorators:
An ABC with one abstract method ``__bytes__``.
+.. class:: SupportsIndex
+
+ An ABC with one abstract method ``__index__``.
+
+ .. versionadded:: 3.8
+
.. class:: SupportsAbs
An ABC with one abstract method ``__abs__`` that is covariant
diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst
index ed00ee6d0c2..163da9aecdb 100644
--- a/Doc/library/unittest.mock.rst
+++ b/Doc/library/unittest.mock.rst
@@ -201,9 +201,11 @@ The Mock Class
.. testsetup::
+ import asyncio
+ import inspect
import unittest
from unittest.mock import sentinel, DEFAULT, ANY
- from unittest.mock import patch, call, Mock, MagicMock, PropertyMock
+ from unittest.mock import patch, call, Mock, MagicMock, PropertyMock, AsyncMock
from unittest.mock import mock_open
:class:`Mock` is a flexible mock object intended to replace the use of stubs and
@@ -851,6 +853,217 @@ object::
>>> p.assert_called_once_with()
+.. class:: AsyncMock(spec=None, side_effect=None, return_value=DEFAULT, wraps=None, name=None, spec_set=None, unsafe=False, **kwargs)
+
+ An asynchronous version of :class:`Mock`. The :class:`AsyncMock` object will
+ behave so the object is recognized as an async function, and the result of a
+ call is an awaitable.
+
+ >>> mock = AsyncMock()
+ >>> asyncio.iscoroutinefunction(mock)
+ True
+ >>> inspect.isawaitable(mock()) # doctest: +SKIP
+ True
+
+ The result of ``mock()`` is an async function which will have the outcome
+ of ``side_effect`` or ``return_value``:
+
+ - if ``side_effect`` is a function, the async function will return the
+ result of that function,
+ - if ``side_effect`` is an exception, the async function will raise the
+ exception,
+ - if ``side_effect`` is an iterable, the async function will return the
+ next value of the iterable, however, if the sequence of result is
+ exhausted, ``StopIteration`` is raised immediately,
+ - if ``side_effect`` is not defined, the async function will return the
+ value defined by ``return_value``, hence, by default, the async function
+ returns a new :class:`AsyncMock` object.
+
+
+ Setting the *spec* of a :class:`Mock` or :class:`MagicMock` to an async function
+ will result in a coroutine object being returned after calling.
+
+ >>> async def async_func(): pass
+ ...
+ >>> mock = MagicMock(async_func)
+ >>> mock
+
+ >>> mock() # doctest: +SKIP
+
+
+ .. method:: assert_awaited()
+
+ Assert that the mock was awaited at least once.
+
+ >>> mock = AsyncMock()
+ >>> async def main():
+ ... await mock()
+ ...
+ >>> asyncio.run(main())
+ >>> mock.assert_awaited()
+ >>> mock_2 = AsyncMock()
+ >>> mock_2.assert_awaited()
+ Traceback (most recent call last):
+ ...
+ AssertionError: Expected mock to have been awaited.
+
+ .. method:: assert_awaited_once()
+
+ Assert that the mock was awaited exactly once.
+
+ >>> mock = AsyncMock()
+ >>> async def main():
+ ... await mock()
+ ...
+ >>> asyncio.run(main())
+ >>> mock.assert_awaited_once()
+ >>> asyncio.run(main())
+ >>> mock.method.assert_awaited_once()
+ Traceback (most recent call last):
+ ...
+ AssertionError: Expected mock to have been awaited once. Awaited 2 times.
+
+ .. method:: assert_awaited_with(*args, **kwargs)
+
+ Assert that the last await was with the specified arguments.
+
+ >>> mock = AsyncMock()
+ >>> async def main(*args, **kwargs):
+ ... await mock(*args, **kwargs)
+ ...
+ >>> asyncio.run(main('foo', bar='bar'))
+ >>> mock.assert_awaited_with('foo', bar='bar')
+ >>> mock.assert_awaited_with('other')
+ Traceback (most recent call last):
+ ...
+ AssertionError: expected call not found.
+ Expected: mock('other')
+ Actual: mock('foo', bar='bar')
+
+ .. method:: assert_awaited_once_with(*args, **kwargs)
+
+ Assert that the mock was awaited exactly once and with the specified
+ arguments.
+
+ >>> mock = AsyncMock()
+ >>> async def main(*args, **kwargs):
+ ... await mock(*args, **kwargs)
+ ...
+ >>> asyncio.run(main('foo', bar='bar'))
+ >>> mock.assert_awaited_once_with('foo', bar='bar')
+ >>> asyncio.run(main('foo', bar='bar'))
+ >>> mock.assert_awaited_once_with('foo', bar='bar')
+ Traceback (most recent call last):
+ ...
+ AssertionError: Expected mock to have been awaited once. Awaited 2 times.
+
+ .. method:: assert_any_await(*args, **kwargs)
+
+ Assert the mock has ever been awaited with the specified arguments.
+
+ >>> mock = AsyncMock()
+ >>> async def main(*args, **kwargs):
+ ... await mock(*args, **kwargs)
+ ...
+ >>> asyncio.run(main('foo', bar='bar'))
+ >>> asyncio.run(main('hello'))
+ >>> mock.assert_any_await('foo', bar='bar')
+ >>> mock.assert_any_await('other')
+ Traceback (most recent call last):
+ ...
+ AssertionError: mock('other') await not found
+
+ .. method:: assert_has_awaits(calls, any_order=False)
+
+ Assert the mock has been awaited with the specified calls.
+ The :attr:`await_args_list` list is checked for the awaits.
+
+ If *any_order* is False (the default) then the awaits must be
+ sequential. There can be extra calls before or after the
+ specified awaits.
+
+ If *any_order* is True then the awaits can be in any order, but
+ they must all appear in :attr:`await_args_list`.
+
+ >>> mock = AsyncMock()
+ >>> async def main(*args, **kwargs):
+ ... await mock(*args, **kwargs)
+ ...
+ >>> calls = [call("foo"), call("bar")]
+ >>> mock.assert_has_calls(calls)
+ Traceback (most recent call last):
+ ...
+ AssertionError: Calls not found.
+ Expected: [call('foo'), call('bar')]
+ >>> asyncio.run(main('foo'))
+ >>> asyncio.run(main('bar'))
+ >>> mock.assert_has_calls(calls)
+
+ .. method:: assert_not_awaited()
+
+ Assert that the mock was never awaited.
+
+ >>> mock = AsyncMock()
+ >>> mock.assert_not_awaited()
+
+ .. method:: reset_mock(*args, **kwargs)
+
+ See :func:`Mock.reset_mock`. Also sets :attr:`await_count` to 0,
+ :attr:`await_args` to None, and clears the :attr:`await_args_list`.
+
+ .. attribute:: await_count
+
+ An integer keeping track of how many times the mock object has been awaited.
+
+ >>> mock = AsyncMock()
+ >>> async def main():
+ ... await mock()
+ ...
+ >>> asyncio.run(main())
+ >>> mock.await_count
+ 1
+ >>> asyncio.run(main())
+ >>> mock.await_count
+ 2
+
+ .. attribute:: await_args
+
+ This is either ``None`` (if the mock hasn’t been awaited), or the arguments that
+ the mock was last awaited with. Functions the same as :attr:`Mock.call_args`.
+
+ >>> mock = AsyncMock()
+ >>> async def main(*args):
+ ... await mock(*args)
+ ...
+ >>> mock.await_args
+ >>> asyncio.run(main('foo'))
+ >>> mock.await_args
+ call('foo')
+ >>> asyncio.run(main('bar'))
+ >>> mock.await_args
+ call('bar')
+
+
+ .. attribute:: await_args_list
+
+ This is a list of all the awaits made to the mock object in sequence (so the
+ length of the list is the number of times it has been awaited). Before any
+ awaits have been made it is an empty list.
+
+ >>> mock = AsyncMock()
+ >>> async def main(*args):
+ ... await mock(*args)
+ ...
+ >>> mock.await_args_list
+ []
+ >>> asyncio.run(main('foo'))
+ >>> mock.await_args_list
+ [call('foo')]
+ >>> asyncio.run(main('bar'))
+ >>> mock.await_args_list
+ [call('foo'), call('bar')]
+
+
Calling
~~~~~~~
diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst
index 14fa27bb08a..1895ae74b4f 100644
--- a/Doc/library/urllib.request.rst
+++ b/Doc/library/urllib.request.rst
@@ -95,6 +95,12 @@ The :mod:`urllib.request` module defines the following functions:
parameter to ``urllib.urlopen``, can be obtained by using
:class:`ProxyHandler` objects.
+ .. audit-event:: urllib.request "fullurl data headers method"
+
+ The default opener raises an :func:`auditing event `
+ ``urllib.request`` with arguments ``fullurl``, ``data``, ``headers``,
+ ``method`` taken from the request object.
+
.. versionchanged:: 3.2
*cafile* and *capath* were added.
@@ -118,6 +124,7 @@ The :mod:`urllib.request` module defines the following functions:
:func:`ssl.create_default_context` select the system's trusted CA
certificates for you.
+
.. function:: install_opener(opener)
Install an :class:`OpenerDirector` instance as the default global opener.
diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst
index d121f320d6a..a481a3509d4 100644
--- a/Doc/library/warnings.rst
+++ b/Doc/library/warnings.rst
@@ -19,10 +19,10 @@ Python programmers issue warnings by calling the :func:`warn` function defined
in this module. (C programmers use :c:func:`PyErr_WarnEx`; see
:ref:`exceptionhandling` for details).
-Warning messages are normally written to ``sys.stderr``, but their disposition
+Warning messages are normally written to :data:`sys.stderr`, but their disposition
can be changed flexibly, from ignoring all warnings to turning them into
-exceptions. The disposition of warnings can vary based on the warning category
-(see below), the text of the warning message, and the source location where it
+exceptions. The disposition of warnings can vary based on the :ref:`warning category
+`, the text of the warning message, and the source location where it
is issued. Repetitions of a particular warning for the same source location are
typically suppressed.
@@ -31,7 +31,7 @@ determination is made whether a message should be issued or not; next, if a
message is to be issued, it is formatted and printed using a user-settable hook.
The determination whether to issue a warning message is controlled by the
-warning filter, which is a sequence of matching rules and actions. Rules can be
+:ref:`warning filter `, which is a sequence of matching rules and actions. Rules can be
added to the filter by calling :func:`filterwarnings` and reset to its default
state by calling :func:`resetwarnings`.
@@ -181,9 +181,9 @@ Describing Warning Filters
The warnings filter is initialized by :option:`-W` options passed to the Python
interpreter command line and the :envvar:`PYTHONWARNINGS` environment variable.
The interpreter saves the arguments for all supplied entries without
-interpretation in ``sys.warnoptions``; the :mod:`warnings` module parses these
+interpretation in :data:`sys.warnoptions`; the :mod:`warnings` module parses these
when it is first imported (invalid options are ignored, after printing a
-message to ``sys.stderr``).
+message to :data:`sys.stderr`).
Individual warnings filters are specified as a sequence of fields separated by
colons::
@@ -192,7 +192,7 @@ colons::
The meaning of each of these fields is as described in :ref:`warning-filter`.
When listing multiple filters on a single line (as for
-:envvar:`PYTHONWARNINGS`), the individual filters are separated by commas,and
+:envvar:`PYTHONWARNINGS`), the individual filters are separated by commas and
the filters listed later take precedence over those listed before them (as
they're applied left-to-right, and the most recently applied filters take
precedence over earlier ones).
@@ -395,12 +395,12 @@ Available Functions
.. function:: warn(message, category=None, stacklevel=1, source=None)
Issue a warning, or maybe ignore it or raise an exception. The *category*
- argument, if given, must be a warning category class (see above); it defaults to
- :exc:`UserWarning`. Alternatively *message* can be a :exc:`Warning` instance,
+ argument, if given, must be a :ref:`warning category class `; it
+ defaults to :exc:`UserWarning`. Alternatively, *message* can be a :exc:`Warning` instance,
in which case *category* will be ignored and ``message.__class__`` will be used.
- In this case the message text will be ``str(message)``. This function raises an
+ In this case, the message text will be ``str(message)``. This function raises an
exception if the particular warning issued is changed into an error by the
- warnings filter see above. The *stacklevel* argument can be used by wrapper
+ :ref:`warnings filter `. The *stacklevel* argument can be used by wrapper
functions written in Python, like this::
def deprecation(message):
@@ -444,7 +444,7 @@ Available Functions
Write a warning to a file. The default implementation calls
``formatwarning(message, category, filename, lineno, line)`` and writes the
- resulting string to *file*, which defaults to ``sys.stderr``. You may replace
+ resulting string to *file*, which defaults to :data:`sys.stderr`. You may replace
this function with any callable by assigning to ``warnings.showwarning``.
*line* is a line of source code to be included in the warning
message; if *line* is not supplied, :func:`showwarning` will
diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst
index aa1831dbc60..2138697ff06 100644
--- a/Doc/library/zipimport.rst
+++ b/Doc/library/zipimport.rst
@@ -6,6 +6,8 @@
.. moduleauthor:: Just van Rossum
+**Source code:** :source:`Lib/zipimport.py`
+
--------------
This module adds the ability to import Python modules (:file:`\*.py`,
diff --git a/Doc/license.rst b/Doc/license.rst
index bf2e4c522ce..d877f456775 100644
--- a/Doc/license.rst
+++ b/Doc/license.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: none
+.. highlight:: none
.. _history-and-license:
diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst
index ba7130d6362..49cb86b5608 100644
--- a/Doc/reference/executionmodel.rst
+++ b/Doc/reference/executionmodel.rst
@@ -243,7 +243,7 @@ re-entering the offending piece of code from the top).
When an exception is not handled at all, the interpreter terminates execution of
the program, or returns to its interactive main loop. In either case, it prints
-a stack backtrace, except when the exception is :exc:`SystemExit`.
+a stack traceback, except when the exception is :exc:`SystemExit`.
Exceptions are identified by class instances. The :keyword:`except` clause is
selected depending on the class of the instance: it must reference the class of
diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst
index cf7d05eef67..52b41929d7b 100644
--- a/Doc/reference/expressions.rst
+++ b/Doc/reference/expressions.rst
@@ -144,7 +144,7 @@ the single expression that makes up the expression list.
.. index:: pair: empty; tuple
An empty pair of parentheses yields an empty tuple object. Since tuples are
-immutable, the rules for literals apply (i.e., two occurrences of the empty
+immutable, the same rules as for literals apply (i.e., two occurrences of the empty
tuple may or may not yield the same object).
.. index::
@@ -196,7 +196,7 @@ the comprehension is executed in a separate implicitly nested scope. This ensure
that names assigned to in the target list don't "leak" into the enclosing scope.
The iterable expression in the leftmost :keyword:`!for` clause is evaluated
-directly in the enclosing scope and then passed as an argument to the implictly
+directly in the enclosing scope and then passed as an argument to the implicitly
nested scope. Subsequent :keyword:`!for` clauses and any filter condition in the
leftmost :keyword:`!for` clause cannot be evaluated in the enclosing scope as
they may depend on the values obtained from the leftmost iterable. For example:
@@ -479,8 +479,8 @@ will raise :exc:`AttributeError` or :exc:`TypeError`, while
When the underlying iterator is complete, the :attr:`~StopIteration.value`
attribute of the raised :exc:`StopIteration` instance becomes the value of
the yield expression. It can be either set explicitly when raising
-:exc:`StopIteration`, or automatically when the sub-iterator is a generator
-(by returning a value from the sub-generator).
+:exc:`StopIteration`, or automatically when the subiterator is a generator
+(by returning a value from the subgenerator).
.. versionchanged:: 3.3
Added ``yield from `` to delegate control flow to a subiterator.
@@ -499,7 +499,7 @@ on the right hand side of an assignment statement.
:pep:`380` - Syntax for Delegating to a Subgenerator
The proposal to introduce the :token:`yield_from` syntax, making delegation
- to sub-generators easy.
+ to subgenerators easy.
:pep:`525` - Asynchronous Generators
The proposal that expanded on :pep:`492` by adding generator capabilities to
@@ -608,7 +608,7 @@ Asynchronous generator functions
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
The presence of a yield expression in a function or method defined using
-:keyword:`async def` further defines the function as a
+:keyword:`async def` further defines the function as an
:term:`asynchronous generator` function.
When an asynchronous generator function is called, it returns an
@@ -673,13 +673,13 @@ which are used to control the execution of a generator function.
Returns an awaitable which when run starts to execute the asynchronous
generator or resumes it at the last executed yield expression. When an
- asynchronous generator function is resumed with a :meth:`~agen.__anext__`
+ asynchronous generator function is resumed with an :meth:`~agen.__anext__`
method, the current yield expression always evaluates to :const:`None` in
the returned awaitable, which when run will continue to the next yield
expression. The value of the :token:`expression_list` of the yield
expression is the value of the :exc:`StopIteration` exception raised by
the completing coroutine. If the asynchronous generator exits without
- yielding another value, the awaitable instead raises an
+ yielding another value, the awaitable instead raises a
:exc:`StopAsyncIteration` exception, signalling that the asynchronous
iteration has completed.
@@ -707,7 +707,7 @@ which are used to control the execution of a generator function.
where the asynchronous generator was paused, and returns the next value
yielded by the generator function as the value of the raised
:exc:`StopIteration` exception. If the asynchronous generator exits
- without yielding another value, an :exc:`StopAsyncIteration` exception is
+ without yielding another value, a :exc:`StopAsyncIteration` exception is
raised by the awaitable.
If the generator function does not catch the passed-in exception, or
raises a different exception, then when the awaitable is run that exception
@@ -1579,7 +1579,7 @@ if :keyword:`in` raised that exception).
pair: membership; test
object: sequence
-The operator :keyword:`not in` is defined to have the inverse true value of
+The operator :keyword:`not in` is defined to have the inverse truth value of
:keyword:`in`.
.. index::
@@ -1594,8 +1594,8 @@ The operator :keyword:`not in` is defined to have the inverse true value of
Identity comparisons
--------------------
-The operators :keyword:`is` and :keyword:`is not` test for object identity: ``x
-is y`` is true if and only if *x* and *y* are the same object. Object identity
+The operators :keyword:`is` and :keyword:`is not` test for an object's identity: ``x
+is y`` is true if and only if *x* and *y* are the same object. An Object's identity
is determined using the :meth:`id` function. ``x is not y`` yields the inverse
truth value. [#]_
diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst
index 1cbe421ded9..cc1b2f57a70 100644
--- a/Doc/reference/lexical_analysis.rst
+++ b/Doc/reference/lexical_analysis.rst
@@ -70,7 +70,7 @@ Comments
A comment starts with a hash character (``#``) that is not part of a string
literal, and ends at the end of the physical line. A comment signifies the end
of the logical line unless the implicit line joining rules are invoked. Comments
-are ignored by the syntax; they are not tokens.
+are ignored by the syntax.
.. _encodings:
diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py
index 66317430881..f79b25048a5 100644
--- a/Doc/tools/extensions/pyspecific.py
+++ b/Doc/tools/extensions/pyspecific.py
@@ -151,6 +151,45 @@ class Availability(Directive):
return [pnode]
+# Support for documenting audit event
+
+class AuditEvent(Directive):
+
+ has_content = True
+ required_arguments = 1
+ optional_arguments = 1
+ final_argument_whitespace = True
+
+ _label = [
+ "Raises an :ref:`auditing event ` {name} with no arguments.",
+ "Raises an :ref:`auditing event ` {name} with argument {args}.",
+ "Raises an :ref:`auditing event ` {name} with arguments {args}.",
+ ]
+
+ def run(self):
+ if len(self.arguments) >= 2 and self.arguments[1]:
+ args = [
+ "``{}``".format(a.strip())
+ for a in self.arguments[1].strip("'\"").split()
+ if a.strip()
+ ]
+ else:
+ args = []
+
+ label = translators['sphinx'].gettext(self._label[min(2, len(args))])
+ text = label.format(name="``{}``".format(self.arguments[0]),
+ args=", ".join(args))
+
+ pnode = nodes.paragraph(text, classes=["audit-hook"])
+ if self.content:
+ self.state.nested_parse(self.content, self.content_offset, pnode)
+ else:
+ n, m = self.state.inline_text(text, self.lineno)
+ pnode.extend(n + m)
+
+ return [pnode]
+
+
# Support for documenting decorators
class PyDecoratorMixin(object):
@@ -271,7 +310,7 @@ class DeprecatedRemoved(Directive):
translatable=False)
node.append(para)
env = self.state.document.settings.env
- env.note_versionchange('deprecated', version[0], node, self.lineno)
+ env.get_domain('changeset').note_changeset(node)
return [node] + messages
@@ -424,6 +463,7 @@ def setup(app):
app.add_role('source', source_role)
app.add_directive('impl-detail', ImplementationDetail)
app.add_directive('availability', Availability)
+ app.add_directive('audit-event', AuditEvent)
app.add_directive('deprecated-removed', DeprecatedRemoved)
app.add_builder(PydocTopicsBuilder)
app.add_builder(suspicious.CheckSuspiciousMarkupBuilder)
diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv
index 250c59bc603..85ff9d02aeb 100644
--- a/Doc/tools/susp-ignored.csv
+++ b/Doc/tools/susp-ignored.csv
@@ -4,7 +4,7 @@ c-api/sequence,,:i2,del o[i1:i2]
c-api/sequence,,:i2,o[i1:i2]
c-api/unicode,,:end,str[start:end]
c-api/unicode,,:start,unicode[start:start+length]
-distutils/examples,267,`,This is the description of the ``foobar`` package.
+distutils/examples,274,`,This is the description of the ``foobar`` package.
distutils/setupscript,,::,
extending/embedding,,:numargs,"if(!PyArg_ParseTuple(args, "":numargs""))"
extending/extending,,:myfunction,"PyArg_ParseTuple(args, ""D:myfunction"", &c);"
diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst
index ab92af044e7..dddfd850cd2 100644
--- a/Doc/tutorial/interpreter.rst
+++ b/Doc/tutorial/interpreter.rst
@@ -24,11 +24,11 @@ Python guru or system administrator. (E.g., :file:`/usr/local/python` is a
popular alternative location.)
On Windows machines, the Python installation is usually placed in
-:file:`C:\\Python36`, though you can change this when you're running the
+:file:`C:\\Python38`, though you can change this when you're running the
installer. To add this directory to your path, you can type the following
command into :ref:`a command prompt window `::
- set path=%path%;C:\python36
+ set path=%path%;C:\python38
Typing an end-of-file character (:kbd:`Control-D` on Unix, :kbd:`Control-Z` on
Windows) at the primary prompt causes the interpreter to exit with a zero exit
diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst
index fd47ce2ab53..87af7e8be13 100644
--- a/Doc/using/cmdline.rst
+++ b/Doc/using/cmdline.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: sh
+.. highlight:: sh
.. ATTENTION: You probably should update Misc/python.man, too, if you modify
this file.
@@ -182,13 +182,13 @@ Generic options
.. code-block:: none
- Python 3.6.0b2+
+ Python 3.8.0b2+
When given twice, print more information about the build, like:
.. code-block:: none
- Python 3.6.0b2+ (3.6:84a3c5003510+, Oct 26 2016, 02:33:55)
+ Python 3.8.0b2+ (3.8:0c076caaa8, Apr 20 2019, 21:55:00)
[GCC 6.2.0 20161005]
.. versionadded:: 3.6
diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst
index 31cf87c66f5..021f0d35a8e 100644
--- a/Doc/using/unix.rst
+++ b/Doc/using/unix.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: sh
+.. highlight:: sh
.. _using-on-unix:
diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst
index 2f3eb0ea3f0..44b646fddfc 100644
--- a/Doc/using/windows.rst
+++ b/Doc/using/windows.rst
@@ -1,4 +1,4 @@
-.. highlightlang:: none
+.. highlight:: none
.. _using-on-windows:
@@ -214,13 +214,13 @@ of available options is shown below.
For example, to silently install a default, system-wide Python installation,
you could use the following command (from an elevated command prompt)::
- python-3.6.0.exe /quiet InstallAllUsers=1 PrependPath=1 Include_test=0
+ python-3.8.0.exe /quiet InstallAllUsers=1 PrependPath=1 Include_test=0
To allow users to easily install a personal copy of Python without the test
suite, you could provide a shortcut with the following command. This will
display a simplified initial page and disallow customization::
- python-3.6.0.exe InstallAllUsers=0 Include_launcher=0 Include_test=0
+ python-3.8.0.exe InstallAllUsers=0 Include_launcher=0 Include_test=0
SimpleInstall=1 SimpleInstallDescription="Just for me, no test suite."
(Note that omitting the launcher also omits file associations, and is only
@@ -257,13 +257,13 @@ where a large number of installations are going to be performed it is very
useful to have a locally cached copy.
Execute the following command from Command Prompt to download all possible
-required files. Remember to substitute ``python-3.6.0.exe`` for the actual
+required files. Remember to substitute ``python-3.8.0.exe`` for the actual
name of your installer, and to create layouts in their own directories to
avoid collisions between files with the same name.
::
- python-3.6.0.exe /layout [optional target directory]
+ python-3.8.0.exe /layout [optional target directory]
You may also specify the ``/quiet`` option to hide the progress display.
@@ -530,7 +530,7 @@ To temporarily set environment variables, open Command Prompt and use the
.. code-block:: doscon
- C:\>set PATH=C:\Program Files\Python 3.6;%PATH%
+ C:\>set PATH=C:\Program Files\Python 3.8;%PATH%
C:\>set PYTHONPATH=%PYTHONPATH%;C:\My_python_lib
C:\>python
@@ -603,7 +603,7 @@ of your Python installation, delimited by a semicolon from other entries. An
example variable could look like this (assuming the first two entries already
existed)::
- C:\WINDOWS\system32;C:\WINDOWS;C:\Program Files\Python 3.6
+ C:\WINDOWS\system32;C:\WINDOWS;C:\Program Files\Python 3.8
.. _launcher:
diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 6bb03422c6b..b5d70b5a2ec 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -138,7 +138,9 @@ environment variable, can be set using the new ``./configure --with-trace-refs``
build option.
(Contributed by Victor Stinner in :issue:`36465`.)
-On Unix, C extensions are no longer linked to libpython. It is now possible
+On Unix, C extensions are no longer linked to libpython except on Android
+and Cygwin.
+It is now possible
for a statically linked Python to load a C extension built using a shared
library Python.
(Contributed by Victor Stinner in :issue:`21536`.)
@@ -148,6 +150,24 @@ extensions compiled in release mode and for C extensions compiled with the
stable ABI.
(Contributed by Victor Stinner in :issue:`36722`.)
+To embed Python into an application, a new ``--embed`` option must be passed to
+``python3-config --libs --embed`` to get ``-lpython3.8`` (link the application
+to libpython). To support both 3.8 and older, try ``python3-config --libs
+--embed`` first and fallback to ``python3-config --libs`` (without ``--embed``)
+if the previous command fails.
+
+Add a pkg-config ``python-3.8-embed`` module to embed Python into an
+application: ``pkg-config python-3.8-embed --libs`` includes ``-lpython3.8``.
+To support both 3.8 and older, try ``pkg-config python-X.Y-embed --libs`` first
+and fallback to ``pkg-config python-X.Y --libs`` (without ``--embed``) if the
+previous command fails (replace ``X.Y`` with the Python version).
+
+On the other hand, ``pkg-config python3.8 --libs`` no longer contains
+``-lpython3.8``. C extensions must not be linked to libpython (except on
+Android and Cygwin, whose cases are handled by the script);
+this change is backward incompatible on purpose.
+(Contributed by Victor Stinner in :issue:`36721`.)
+
f-strings now support = for quick and easy debugging
-----------------------------------------------------
@@ -236,6 +256,16 @@ asyncio
On Windows, the default event loop is now :class:`~asyncio.ProactorEventLoop`.
+builtins
+--------
+
+The :func:`compile` built-in has been improved to accept the
+``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` flag. With this new flag passed,
+:func:`compile` will allow top-level ``await``, ``async for`` and ``async with``
+constructs that are usually considered invalid syntax. Asynchronous code object
+marked with the ``CO_COROUTINE`` flag may then be returned.
+
+(Contributed by Matthias Bussonnier in :issue:`34616`)
collections
-----------
@@ -289,6 +319,15 @@ for :func:`property`, :func:`classmethod`, and :func:`staticmethod`::
self.bit_rate = round(bit_rate / 1000.0, 1)
self.duration = ceil(duration)
+io
+--
+
+In development mode (:option:`-X` ``env``) and in debug build, the
+:class:`io.IOBase` finalizer now logs the exception if the ``close()`` method
+fails. The exception is ignored silently by default in release build.
+(Contributed by Victor Stinner in :issue:`18748`.)
+
+
gc
--
@@ -303,6 +342,11 @@ gzip
Added the *mtime* parameter to :func:`gzip.compress` for reproducible output.
(Contributed by Guo Ci Teo in :issue:`34898`.)
+A :exc:`~gzip.BadGzipFile` exception is now raised instead of :exc:`OSError`
+for certain types of invalid or corrupt gzip files.
+(Contributed by Filip Gruszczyński, Michele Orrù, and Zackery Spytz in
+:issue:`6584`.)
+
idlelib and IDLE
----------------
@@ -338,6 +382,9 @@ Added new function, :func:`math.prod`, as analogous function to :func:`sum`
that returns the product of a 'start' value (default: 1) times an iterable of
numbers. (Contributed by Pablo Galindo in :issue:`35606`)
+Added new function :func:`math.isqrt` for computing integer square roots.
+(Contributed by Mark Dickinson in :issue:`36887`.)
+
os
--
@@ -388,6 +435,14 @@ to a path.
(Contributed by Joannah Nanjekye in :issue:`26978`)
+plistlib
+--------
+
+Added new :class:`plistlib.UID` and enabled support for reading and writing
+NSKeyedArchiver-encoded binary plists.
+(Contributed by Jon Janzen in :issue:`26707`.)
+
+
socket
------
@@ -464,6 +519,16 @@ and manipulating normal distributions of a random variable.
[7.672102882379219, 12.000027119750287, 4.647488369766392]
+sys
+---
+
+Add new :func:`sys.unraisablehook` function which can be overridden to control
+how "unraisable exceptions" are handled. It is called when an exception has
+occurred but there is no way for Python to handle it. For example, when a
+destructor raises an exception or during garbage collection
+(:func:`gc.collect`).
+
+
tarfile
-------
@@ -521,6 +586,10 @@ unicodedata
unittest
--------
+* XXX Added :class:`AsyncMock` to support an asynchronous version of :class:`Mock`.
+ Appropriate new assert functions for testing have been added as well.
+ (Contributed by Lisa Roach in :issue:`26467`).
+
* Added :func:`~unittest.addModuleCleanup()` and
:meth:`~unittest.TestCase.addClassCleanup()` to unittest to support
cleanups for :func:`~unittest.setUpModule()` and
@@ -766,10 +835,6 @@ Deprecated
`.
(Contributed by Serhiy Storchaka in :issue:`36492`.)
-* The function :func:`~inspect.getfullargspec` in the :mod:`inspect`
- module is deprecated in favor of the :func:`inspect.signature`
- API. (Contributed by Pablo Galindo in :issue:`36751`.)
-
API and Feature Removals
========================
@@ -782,6 +847,10 @@ The following features and APIs have been removed from Python 3.8:
* The function :func:`platform.popen` has been removed, it was deprecated since
Python 3.3: use :func:`os.popen` instead.
+* The function :func:`time.clock` has been removed, it was deprecated since Python
+ 3.3: use :func:`time.perf_counter` or :func:`time.process_time` instead, depending
+ on your requirements, to have a well defined behavior.
+
* The ``pyvenv`` script has been removed in favor of ``python3.8 -m venv``
to help eliminate confusion as to what Python interpreter the ``pyvenv``
script is tied to. (Contributed by Brett Cannon in :issue:`25427`.)
@@ -807,6 +876,10 @@ The following features and APIs have been removed from Python 3.8:
exposed to the user.
(Contributed by Aviv Palivoda in :issue:`30262`.)
+* The ``bufsize`` keyword argument of :func:`fileinput.input` and
+ :func:`fileinput.FileInput` which was ignored and deprecated since Python 3.6
+ has been removed. :issue:`36952` (Contributed by Matthias Bussonnier)
+
Porting to Python 3.8
=====================
@@ -976,16 +1049,25 @@ Changes in the Python API
replacement by a pure Python implementation. (Contributed by Pablo Galindo
in :issue:`36623`.)
+* :class:`types.CodeType` has a new parameter in the second position of the
+ constructor (*posonlyargcount*) to support positional-only arguments defined
+ in :pep:`570`.
+
Changes in the C API
--------------------
-* On Unix, C extensions are no longer linked to libpython except on
- Android. When Python is embedded, ``libpython`` must not be loaded with
+* The :c:func:`PyEval_ReInitThreads` function has been removed from the C API.
+ It should not be called explicitly: use :c:func:`PyOS_AfterFork_Child`
+ instead.
+ (Contributed by Victor Stinner in :issue:`36728`.)
+
+* On Unix, C extensions are no longer linked to libpython except on Android
+ and Cygwin. When Python is embedded, ``libpython`` must not be loaded with
``RTLD_LOCAL``, but ``RTLD_GLOBAL`` instead. Previously, using
- ``RTLD_LOCAL``, it was already not possible to load C extensions which were
- not linked to ``libpython``, like C extensions of the standard library built
- by the ``*shared*`` section of ``Modules/Setup``.
+ ``RTLD_LOCAL``, it was already not possible to load C extensions which
+ were not linked to ``libpython``, like C extensions of the standard
+ library built by the ``*shared*`` section of ``Modules/Setup``.
* Use of ``#`` variants of formats in parsing or building value (e.g.
:c:func:`PyArg_ParseTuple`, :c:func:`Py_BuildValue`, :c:func:`PyObject_CallFunction`,
diff --git a/Include/Python-ast.h b/Include/Python-ast.h
index 08d50ffcddf..2fc50e3f53a 100644
--- a/Include/Python-ast.h
+++ b/Include/Python-ast.h
@@ -467,6 +467,7 @@ struct _type_ignore {
union {
struct {
int lineno;
+ string tag;
} TypeIgnore;
} v;
@@ -702,8 +703,8 @@ alias_ty _Py_alias(identifier name, identifier asname, PyArena *arena);
#define withitem(a0, a1, a2) _Py_withitem(a0, a1, a2)
withitem_ty _Py_withitem(expr_ty context_expr, expr_ty optional_vars, PyArena
*arena);
-#define TypeIgnore(a0, a1) _Py_TypeIgnore(a0, a1)
-type_ignore_ty _Py_TypeIgnore(int lineno, PyArena *arena);
+#define TypeIgnore(a0, a1, a2) _Py_TypeIgnore(a0, a1, a2)
+type_ignore_ty _Py_TypeIgnore(int lineno, string tag, PyArena *arena);
PyObject* PyAST_mod2obj(mod_ty t);
mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);
diff --git a/Include/ceval.h b/Include/ceval.h
index 11283c0a570..8cdf353b05f 100644
--- a/Include/ceval.h
+++ b/Include/ceval.h
@@ -58,7 +58,6 @@ PyAPI_FUNC(int) PyEval_MergeCompilerFlags(PyCompilerFlags *cf);
#endif
PyAPI_FUNC(int) Py_AddPendingCall(int (*func)(void *), void *arg);
-PyAPI_FUNC(void) _PyEval_SignalReceived(void);
PyAPI_FUNC(int) Py_MakePendingCalls(void);
/* Protection against deeply nested recursive calls
@@ -192,14 +191,10 @@ PyAPI_FUNC(void) PyEval_RestoreThread(PyThreadState *);
PyAPI_FUNC(int) PyEval_ThreadsInitialized(void);
PyAPI_FUNC(void) PyEval_InitThreads(void);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(void) _PyEval_FiniThreads(void);
-#endif /* !Py_LIMITED_API */
PyAPI_FUNC(void) PyEval_AcquireLock(void) Py_DEPRECATED(3.2);
PyAPI_FUNC(void) PyEval_ReleaseLock(void) /* Py_DEPRECATED(3.2) */;
PyAPI_FUNC(void) PyEval_AcquireThread(PyThreadState *tstate);
PyAPI_FUNC(void) PyEval_ReleaseThread(PyThreadState *tstate);
-PyAPI_FUNC(void) PyEval_ReInitThreads(void);
#ifndef Py_LIMITED_API
PyAPI_FUNC(void) _PyEval_SetSwitchInterval(unsigned long microseconds);
@@ -221,7 +216,6 @@ PyAPI_FUNC(Py_ssize_t) _PyEval_RequestCodeExtraIndex(freefunc);
#ifndef Py_LIMITED_API
PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *);
PyAPI_FUNC(int) _PyEval_SliceIndexNotNone(PyObject *, Py_ssize_t *);
-PyAPI_FUNC(void) _PyEval_SignalAsyncExc(void);
#endif
/* Masks and values used by FORMAT_VALUE opcode. */
diff --git a/Include/compile.h b/Include/compile.h
index 13708678675..a833caa06b9 100644
--- a/Include/compile.h
+++ b/Include/compile.h
@@ -23,6 +23,7 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
#define PyCF_ONLY_AST 0x0400
#define PyCF_IGNORE_COOKIE 0x0800
#define PyCF_TYPE_COMMENTS 0x1000
+#define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000
#ifndef Py_LIMITED_API
typedef struct {
diff --git a/Include/cpython/coreconfig.h b/Include/cpython/coreconfig.h
index 47a6baa1118..ef5fde20e88 100644
--- a/Include/cpython/coreconfig.h
+++ b/Include/cpython/coreconfig.h
@@ -15,41 +15,16 @@ typedef struct {
} _type;
const char *_func;
const char *err_msg;
-#ifdef MS_WINDOWS
- unsigned int exitcode;
-#else
int exitcode;
-#endif
} _PyInitError;
-/* Almost all errors causing Python initialization to fail */
-#ifdef _MSC_VER
- /* Visual Studio 2015 doesn't implement C99 __func__ in C */
-# define _Py_INIT_GET_FUNC() __FUNCTION__
-#else
-# define _Py_INIT_GET_FUNC() __func__
-#endif
-
-#define _Py_INIT_OK() \
- (_PyInitError){._type = _Py_INIT_ERR_TYPE_OK,}
- /* other fields are set to 0 */
-#define _Py_INIT_ERR(ERR_MSG) \
- (_PyInitError){ \
- ._type = _Py_INIT_ERR_TYPE_ERROR, \
- ._func = _Py_INIT_GET_FUNC(), \
- .err_msg = (ERR_MSG)}
- /* other fields are set to 0 */
-#define _Py_INIT_NO_MEMORY() _Py_INIT_ERR("memory allocation failed")
-#define _Py_INIT_EXIT(EXITCODE) \
- (_PyInitError){ \
- ._type = _Py_INIT_ERR_TYPE_EXIT, \
- .exitcode = (EXITCODE)}
-#define _Py_INIT_IS_ERROR(err) \
- (err._type == _Py_INIT_ERR_TYPE_ERROR)
-#define _Py_INIT_IS_EXIT(err) \
- (err._type == _Py_INIT_ERR_TYPE_EXIT)
-#define _Py_INIT_FAILED(err) \
- (err._type != _Py_INIT_ERR_TYPE_OK)
+PyAPI_FUNC(_PyInitError) _PyInitError_Ok(void);
+PyAPI_FUNC(_PyInitError) _PyInitError_Error(const char *err_msg);
+PyAPI_FUNC(_PyInitError) _PyInitError_NoMemory(void);
+PyAPI_FUNC(_PyInitError) _PyInitError_Exit(int exitcode);
+PyAPI_FUNC(int) _PyInitError_IsError(_PyInitError err);
+PyAPI_FUNC(int) _PyInitError_IsExit(_PyInitError err);
+PyAPI_FUNC(int) _PyInitError_Failed(_PyInitError err);
/* --- _PyWstrList ------------------------------------------------ */
@@ -60,16 +35,27 @@ typedef struct {
wchar_t **items;
} _PyWstrList;
-#define _PyWstrList_INIT (_PyWstrList){.length = 0, .items = NULL}
-
/* --- _PyPreConfig ----------------------------------------------- */
#define _Py_CONFIG_VERSION 1
+typedef enum {
+ /* Py_Initialize() API: backward compatibility with Python 3.6 and 3.7 */
+ _PyConfig_INIT_COMPAT = 1,
+ _PyConfig_INIT_PYTHON = 2,
+ _PyConfig_INIT_ISOLATED = 3
+} _PyConfigInitEnum;
+
+
typedef struct {
int _config_version; /* Internal configuration version,
used for ABI compatibility */
+ int _config_init; /* _PyConfigInitEnum value */
+
+ /* Parse _Py_PreInitializeFromArgs() arguments?
+ See _PyCoreConfig.parse_argv */
+ int parse_argv;
/* If greater than 0, enable isolated mode: sys.path contains
neither the script's directory nor the user's site-packages directory.
@@ -83,6 +69,10 @@ typedef struct {
set to !Py_IgnoreEnvironmentFlag. */
int use_environment;
+ /* Set the LC_CTYPE locale to the user preferred locale? If equals to 0,
+ set coerce_c_locale and coerce_c_locale_warn to 0. */
+ int configure_locale;
+
/* Coerce the LC_CTYPE locale if it's equal to "C"? (PEP 538)
Set to 0 by PYTHONCOERCECLOCALE=0. Set to 1 by PYTHONCOERCECLOCALE=1.
@@ -90,12 +80,18 @@ typedef struct {
If it is equal to 1, LC_CTYPE locale is read to decide it it should be
coerced or not (ex: PYTHONCOERCECLOCALE=1). Internally, it is set to 2
- if the LC_CTYPE locale must be coerced. */
+ if the LC_CTYPE locale must be coerced.
+
+ Disable by default (set to 0). Set it to -1 to let Python decides if it
+ should be enabled or not. */
int coerce_c_locale;
/* Emit a warning if the LC_CTYPE locale is coerced?
- Disabled by default. Set to 1 by PYTHONCOERCECLOCALE=warn. */
+ Set to 1 by PYTHONCOERCECLOCALE=warn.
+
+ Disable by default (set to 0). Set it to -1 to let Python decides if it
+ should be enabled or not. */
int coerce_c_locale_warn;
#ifdef MS_WINDOWS
@@ -120,28 +116,18 @@ typedef struct {
Set to 0 by "-X utf8=0" and PYTHONUTF8=0.
If equals to -1, it is set to 1 if the LC_CTYPE locale is "C" or
- "POSIX", otherwise inherit Py_UTF8Mode value. */
+ "POSIX", otherwise it is set to 0. Inherit Py_UTF8Mode value value. */
int utf8_mode;
int dev_mode; /* Development mode. PYTHONDEVMODE, -X dev */
- char *allocator; /* Memory allocator: PYTHONMALLOC */
+
+ /* Memory allocator: PYTHONMALLOC env var.
+ See PyMemAllocatorName for valid values. */
+ int allocator;
} _PyPreConfig;
-#ifdef MS_WINDOWS
-# define _PyPreConfig_WINDOWS_INIT \
- .legacy_windows_fs_encoding = -1,
-#else
-# define _PyPreConfig_WINDOWS_INIT
-#endif
-
-#define _PyPreConfig_INIT \
- (_PyPreConfig){ \
- _PyPreConfig_WINDOWS_INIT \
- ._config_version = _Py_CONFIG_VERSION, \
- .isolated = -1, \
- .use_environment = -1, \
- .dev_mode = -1, \
- .allocator = NULL}
+PyAPI_FUNC(void) _PyPreConfig_InitPythonConfig(_PyPreConfig *config);
+PyAPI_FUNC(void) _PyPreConfig_InitIsolatedConfig(_PyPreConfig *config);
/* --- _PyCoreConfig ---------------------------------------------- */
@@ -149,6 +135,7 @@ typedef struct {
typedef struct {
int _config_version; /* Internal configuration version,
used for ABI compatibility */
+ int _config_init; /* _PyConfigInitEnum value */
int isolated; /* Isolated mode? see _PyPreConfig.isolated */
int use_environment; /* Use environment variables? see _PyPreConfig.use_environment */
@@ -211,32 +198,30 @@ typedef struct {
wchar_t *filesystem_errors;
wchar_t *pycache_prefix; /* PYTHONPYCACHEPREFIX, -X pycache_prefix=PATH */
- wchar_t *program_name; /* Program name, see also Py_GetProgramName() */
- _PyWstrList argv; /* Command line arguments */
- wchar_t *program; /* argv[0] or "" */
+ int parse_argv; /* Parse argv command line arguments? */
+
+ /* Command line arguments (sys.argv).
+
+ Set parse_argv to 1 to parse argv as Python command line arguments
+ and then strip Python arguments from argv.
+
+ If argv is empty, an empty string is added to ensure that sys.argv
+ always exists and is never empty. */
+ _PyWstrList argv;
+
+ /* Program name:
+
+ - If Py_SetProgramName() was called, use its value.
+ - On macOS, use PYTHONEXECUTABLE environment variable if set.
+ - If WITH_NEXT_FRAMEWORK macro is defined, use __PYVENV_LAUNCHER__
+ environment variable is set.
+ - Use argv[0] if available and non-empty.
+ - Use "python" on Windows, or "python3 on other platforms. */
+ wchar_t *program_name;
+
_PyWstrList xoptions; /* Command line -X options */
_PyWstrList warnoptions; /* Warnings options */
- /* Path configuration inputs */
- wchar_t *module_search_path_env; /* PYTHONPATH environment variable */
- wchar_t *home; /* PYTHONHOME environment variable,
- see also Py_SetPythonHome(). */
-
- /* Path configuration outputs */
- int use_module_search_paths; /* If non-zero, use module_search_paths */
- _PyWstrList module_search_paths; /* sys.path paths. Computed if
- use_module_search_paths is equal
- to zero. */
-
- wchar_t *executable; /* sys.executable */
- wchar_t *prefix; /* sys.prefix */
- wchar_t *base_prefix; /* sys.base_prefix */
- wchar_t *exec_prefix; /* sys.exec_prefix */
- wchar_t *base_exec_prefix; /* sys.base_exec_prefix */
-#ifdef MS_WINDOWS
- wchar_t *dll_path; /* Windows DLL path */
-#endif
-
/* If equal to zero, disable the import of the module site and the
site-dependent manipulations of sys.path that it entails. Also disable
these manipulations if site is explicitly imported later (call
@@ -322,6 +307,14 @@ typedef struct {
!Py_NoUserSiteDirectory. */
int user_site_directory;
+ /* If non-zero, configure C standard steams (stdio, stdout,
+ stderr):
+
+ - Set O_BINARY mode on Windows.
+ - If buffered_stdio is equal to zero, make streams unbuffered.
+ Otherwise, enable streams buffering if interactive is non-zero. */
+ int configure_c_stdio;
+
/* If equal to 0, enable unbuffered mode: force the stdout and stderr
streams to be unbuffered.
@@ -354,6 +347,25 @@ typedef struct {
int legacy_windows_stdio;
#endif
+ /* --- Path configuration inputs ------------ */
+
+ wchar_t *module_search_path_env; /* PYTHONPATH environment variable */
+ wchar_t *home; /* PYTHONHOME environment variable,
+ see also Py_SetPythonHome(). */
+
+ /* --- Path configuration outputs ----------- */
+
+ int use_module_search_paths; /* If non-zero, use module_search_paths */
+ _PyWstrList module_search_paths; /* sys.path paths. Computed if
+ use_module_search_paths is equal
+ to zero. */
+
+ wchar_t *executable; /* sys.executable */
+ wchar_t *prefix; /* sys.prefix */
+ wchar_t *base_prefix; /* sys.base_prefix */
+ wchar_t *exec_prefix; /* sys.exec_prefix */
+ wchar_t *base_exec_prefix; /* sys.base_exec_prefix */
+
/* --- Parameter only used by Py_Main() ---------- */
/* Skip the first line of the source ('run_filename' parameter), allowing use of non-Unix forms of
@@ -386,47 +398,36 @@ typedef struct {
See PEP 552 "Deterministic pycs" for more details. */
wchar_t *check_hash_pycs_mode;
- /* If greater than 0, suppress _PyPathConfig_Calculate() warnings.
+ /* If greater than 0, suppress _PyPathConfig_Calculate() warnings on Unix.
+ The parameter has no effect on Windows.
- If set to -1 (default), inherit Py_FrozenFlag value. */
- int _frozen;
+ If set to -1 (default), inherit !Py_FrozenFlag value. */
+ int pathconfig_warnings;
+
+ /* If equal to 0, stop Python initialization before the "main" phase */
+ int _init_main;
} _PyCoreConfig;
-#ifdef MS_WINDOWS
-# define _PyCoreConfig_WINDOWS_INIT \
- .legacy_windows_stdio = -1,
-#else
-# define _PyCoreConfig_WINDOWS_INIT
-#endif
-
-#define _PyCoreConfig_INIT \
- (_PyCoreConfig){ \
- _PyCoreConfig_WINDOWS_INIT \
- ._config_version = _Py_CONFIG_VERSION, \
- .isolated = -1, \
- .use_environment = -1, \
- .dev_mode = -1, \
- .install_signal_handlers = 1, \
- .use_hash_seed = -1, \
- .faulthandler = -1, \
- .tracemalloc = -1, \
- .use_module_search_paths = 0, \
- .site_import = -1, \
- .bytes_warning = -1, \
- .inspect = -1, \
- .interactive = -1, \
- .optimization_level = -1, \
- .parser_debug= -1, \
- .write_bytecode = -1, \
- .verbose = -1, \
- .quiet = -1, \
- .user_site_directory = -1, \
- .buffered_stdio = -1, \
- ._install_importlib = 1, \
- .check_hash_pycs_mode = NULL, \
- ._frozen = -1}
-/* Note: _PyCoreConfig_INIT sets other fields to 0/NULL */
+PyAPI_FUNC(_PyInitError) _PyCoreConfig_InitPythonConfig(_PyCoreConfig *config);
+PyAPI_FUNC(_PyInitError) _PyCoreConfig_InitIsolatedConfig(_PyCoreConfig *config);
+PyAPI_FUNC(void) _PyCoreConfig_Clear(_PyCoreConfig *);
+PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetString(
+ _PyCoreConfig *config,
+ wchar_t **config_str,
+ const wchar_t *str);
+PyAPI_FUNC(_PyInitError) _PyCoreConfig_DecodeLocale(
+ _PyCoreConfig *config,
+ wchar_t **config_str,
+ const char *str);
+PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config);
+PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetArgv(
+ _PyCoreConfig *config,
+ Py_ssize_t argc,
+ char * const *argv);
+PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetWideArgv(_PyCoreConfig *config,
+ Py_ssize_t argc,
+ wchar_t * const *argv);
#ifdef __cplusplus
}
diff --git a/Include/cpython/fileobject.h b/Include/cpython/fileobject.h
new file mode 100644
index 00000000000..57eac13c064
--- /dev/null
+++ b/Include/cpython/fileobject.h
@@ -0,0 +1,32 @@
+#ifndef Py_CPYTHON_FILEOBJECT_H
+# error "this header file must not be included directly"
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+PyAPI_FUNC(char *) Py_UniversalNewlineFgets(char *, int, FILE*, PyObject *);
+
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03060000
+PyAPI_DATA(const char *) Py_FileSystemDefaultEncodeErrors;
+#endif
+
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000
+PyAPI_DATA(int) Py_UTF8Mode;
+#endif
+
+/* The std printer acts as a preliminary sys.stderr until the new io
+ infrastructure is in place. */
+PyAPI_FUNC(PyObject *) PyFile_NewStdPrinter(int);
+PyAPI_DATA(PyTypeObject) PyStdPrinter_Type;
+
+typedef PyObject * (*Py_OpenCodeHookFunction)(PyObject *, void *);
+
+PyAPI_FUNC(PyObject *) PyFile_OpenCode(const char *utf8path);
+PyAPI_FUNC(PyObject *) PyFile_OpenCodeObject(PyObject *path);
+PyAPI_FUNC(int) PyFile_SetOpenCodeHook(Py_OpenCodeHookFunction hook, void *userData);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h
index 2366c774e7f..ba5666465d7 100644
--- a/Include/cpython/pylifecycle.h
+++ b/Include/cpython/pylifecycle.h
@@ -18,11 +18,11 @@ PyAPI_FUNC(_PyInitError) _Py_PreInitialize(
const _PyPreConfig *src_config);
PyAPI_FUNC(_PyInitError) _Py_PreInitializeFromArgs(
const _PyPreConfig *src_config,
- int argc,
+ Py_ssize_t argc,
char **argv);
PyAPI_FUNC(_PyInitError) _Py_PreInitializeFromWideArgs(
const _PyPreConfig *src_config,
- int argc,
+ Py_ssize_t argc,
wchar_t **argv);
PyAPI_FUNC(int) _Py_IsCoreInitialized(void);
@@ -34,12 +34,13 @@ PyAPI_FUNC(_PyInitError) _Py_InitializeFromConfig(
const _PyCoreConfig *config);
PyAPI_FUNC(_PyInitError) _Py_InitializeFromArgs(
const _PyCoreConfig *config,
- int argc,
- char **argv);
+ Py_ssize_t argc,
+ char * const *argv);
PyAPI_FUNC(_PyInitError) _Py_InitializeFromWideArgs(
const _PyCoreConfig *config,
- int argc,
- wchar_t **argv);
+ Py_ssize_t argc,
+ wchar_t * const *argv);
+PyAPI_FUNC(_PyInitError) _Py_InitializeMain(void);
PyAPI_FUNC(int) _Py_RunMain(void);
@@ -68,8 +69,8 @@ PyAPI_FUNC(int) _PyOS_URandom(void *buffer, Py_ssize_t size);
PyAPI_FUNC(int) _PyOS_URandomNonblock(void *buffer, Py_ssize_t size);
/* Legacy locale support */
-PyAPI_FUNC(void) _Py_CoerceLegacyLocale(int warn);
-PyAPI_FUNC(int) _Py_LegacyLocaleDetected(void);
+PyAPI_FUNC(int) _Py_CoerceLegacyLocale(int warn);
+PyAPI_FUNC(int) _Py_LegacyLocaleDetected(int warn);
PyAPI_FUNC(char *) _Py_SetLocaleFromEnv(int category);
#ifdef __cplusplus
diff --git a/Include/cpython/pymem.h b/Include/cpython/pymem.h
index bd66506639a..79f063b1217 100644
--- a/Include/cpython/pymem.h
+++ b/Include/cpython/pymem.h
@@ -11,12 +11,8 @@ PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize);
PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size);
PyAPI_FUNC(void) PyMem_RawFree(void *ptr);
-/* Configure the Python memory allocators. Pass NULL to use default
- allocators. */
-PyAPI_FUNC(int) _PyMem_SetupAllocators(const char *opt);
-
/* Try to get the allocators name set by _PyMem_SetupAllocators(). */
-PyAPI_FUNC(const char*) _PyMem_GetAllocatorsName(void);
+PyAPI_FUNC(const char*) _PyMem_GetCurrentAllocatorName(void);
PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize);
@@ -41,6 +37,19 @@ typedef enum {
PYMEM_DOMAIN_OBJ
} PyMemAllocatorDomain;
+typedef enum {
+ PYMEM_ALLOCATOR_NOT_SET = 0,
+ PYMEM_ALLOCATOR_DEFAULT = 1,
+ PYMEM_ALLOCATOR_DEBUG = 2,
+ PYMEM_ALLOCATOR_MALLOC = 3,
+ PYMEM_ALLOCATOR_MALLOC_DEBUG = 4,
+#ifdef WITH_PYMALLOC
+ PYMEM_ALLOCATOR_PYMALLOC = 5,
+ PYMEM_ALLOCATOR_PYMALLOC_DEBUG = 6,
+#endif
+} PyMemAllocatorName;
+
+
typedef struct {
/* user context passed as the first argument to the 4 functions */
void *ctx;
diff --git a/Include/cpython/sysmodule.h b/Include/cpython/sysmodule.h
new file mode 100644
index 00000000000..72d8ffed29f
--- /dev/null
+++ b/Include/cpython/sysmodule.h
@@ -0,0 +1,21 @@
+#ifndef Py_CPYTHON_SYSMODULE_H
+# error "this header file must not be included directly"
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+PyAPI_FUNC(PyObject *) _PySys_GetObjectId(_Py_Identifier *key);
+PyAPI_FUNC(int) _PySys_SetObjectId(_Py_Identifier *key, PyObject *);
+
+PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *);
+
+typedef int(*Py_AuditHookFunction)(const char *, PyObject *, void *);
+
+PyAPI_FUNC(int) PySys_Audit(const char*, const char *, ...);
+PyAPI_FUNC(int) PySys_AddAuditHook(Py_AuditHookFunction, void*);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/Include/cpython/traceback.h b/Include/cpython/traceback.h
new file mode 100644
index 00000000000..746097daaf9
--- /dev/null
+++ b/Include/cpython/traceback.h
@@ -0,0 +1,22 @@
+#ifndef Py_CPYTHON_TRACEBACK_H
+# error "this header file must not be included directly"
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct _traceback {
+ PyObject_HEAD
+ struct _traceback *tb_next;
+ struct _frame *tb_frame;
+ int tb_lasti;
+ int tb_lineno;
+} PyTracebackObject;
+
+PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int);
+PyAPI_FUNC(void) _PyTraceback_Add(const char *, const char *, int);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/Include/fileobject.h b/Include/fileobject.h
index 89e8dd6a285..456887ef9d0 100644
--- a/Include/fileobject.h
+++ b/Include/fileobject.h
@@ -15,32 +15,13 @@ PyAPI_FUNC(PyObject *) PyFile_GetLine(PyObject *, int);
PyAPI_FUNC(int) PyFile_WriteObject(PyObject *, PyObject *, int);
PyAPI_FUNC(int) PyFile_WriteString(const char *, PyObject *);
PyAPI_FUNC(int) PyObject_AsFileDescriptor(PyObject *);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(char *) Py_UniversalNewlineFgets(char *, int, FILE*, PyObject *);
-#endif
/* The default encoding used by the platform file system APIs
If non-NULL, this is different than the default encoding for strings
*/
PyAPI_DATA(const char *) Py_FileSystemDefaultEncoding;
-#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03060000
-PyAPI_DATA(const char *) Py_FileSystemDefaultEncodeErrors;
-#endif
PyAPI_DATA(int) Py_HasFileSystemDefaultEncoding;
-#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000
-PyAPI_DATA(int) Py_UTF8Mode;
-#endif
-
-/* Internal API
-
- The std printer acts as a preliminary sys.stderr until the new io
- infrastructure is in place. */
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject *) PyFile_NewStdPrinter(int);
-PyAPI_DATA(PyTypeObject) PyStdPrinter_Type;
-#endif /* Py_LIMITED_API */
-
/* A routine to check if a file descriptor can be select()-ed. */
#ifdef _MSC_VER
/* On Windows, any socket fd can be select()-ed, no matter how high */
@@ -49,6 +30,12 @@ PyAPI_DATA(PyTypeObject) PyStdPrinter_Type;
#define _PyIsSelectable_fd(FD) ((unsigned int)(FD) < (unsigned int)FD_SETSIZE)
#endif
+#ifndef Py_LIMITED_API
+# define Py_CPYTHON_FILEOBJECT_H
+# include "cpython/fileobject.h"
+# undef Py_CPYTHON_FILEOBJECT_H
+#endif
+
#ifdef __cplusplus
}
#endif
diff --git a/Include/frameobject.h b/Include/frameobject.h
index a95baf8867a..3bad86a66f7 100644
--- a/Include/frameobject.h
+++ b/Include/frameobject.h
@@ -1,4 +1,3 @@
-
/* Frame object interface */
#ifndef Py_LIMITED_API
diff --git a/Include/genobject.h b/Include/genobject.h
index 16b983339cc..6755963f332 100644
--- a/Include/genobject.h
+++ b/Include/genobject.h
@@ -8,6 +8,8 @@
extern "C" {
#endif
+#include "pystate.h" /* _PyErr_StackItem */
+
struct _frame; /* Avoid including frameobject.h */
/* _PyGenObject_HEAD defines the initial segment of generator
diff --git a/Include/import.h b/Include/import.h
index c664803478a..13c614933c7 100644
--- a/Include/import.h
+++ b/Include/import.h
@@ -8,8 +8,6 @@ extern "C" {
#endif
#ifndef Py_LIMITED_API
-PyAPI_FUNC(_PyInitError) _PyImportZip_Init(void);
-
PyMODINIT_FUNC PyInit__imp(void);
#endif /* !Py_LIMITED_API */
PyAPI_FUNC(long) PyImport_GetMagicNumber(void);
diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h
index 0bb19f1aa3b..37170ed438f 100644
--- a/Include/internal/pycore_ceval.h
+++ b/Include/internal/pycore_ceval.h
@@ -9,50 +9,24 @@ extern "C" {
#endif
#include "pycore_atomic.h"
+#include "pycore_pystate.h"
#include "pythread.h"
-PyAPI_FUNC(void) _Py_FinishPendingCalls(void);
-
-struct _pending_calls {
- int finishing;
- PyThread_type_lock lock;
- /* Request for running pending calls. */
- _Py_atomic_int calls_to_do;
- /* Request for looking at the `async_exc` field of the current
- thread state.
- Guarded by the GIL. */
- int async_exc;
-#define NPENDINGCALLS 32
- struct {
- int (*func)(void *);
- void *arg;
- } calls[NPENDINGCALLS];
- int first;
- int last;
-};
-
-#include "pycore_gil.h"
-
-struct _ceval_runtime_state {
- int recursion_limit;
- /* Records whether tracing is on for any thread. Counts the number
- of threads for which tstate->c_tracefunc is non-NULL, so if the
- value is 0, we know we don't have to check this thread's
- c_tracefunc. This speeds up the if statement in
- PyEval_EvalFrameEx() after fast_next_opcode. */
- int tracing_possible;
- /* This single variable consolidates all requests to break out of
- the fast path in the eval loop. */
- _Py_atomic_int eval_breaker;
- /* Request for dropping the GIL */
- _Py_atomic_int gil_drop_request;
- struct _pending_calls pending;
- /* Request for checking signals. */
- _Py_atomic_int signals_pending;
- struct _gil_runtime_state gil;
-};
-
+PyAPI_FUNC(void) _Py_FinishPendingCalls(_PyRuntimeState *runtime);
PyAPI_FUNC(void) _PyEval_Initialize(struct _ceval_runtime_state *);
+PyAPI_FUNC(void) _PyEval_FiniThreads(
+ struct _ceval_runtime_state *ceval);
+PyAPI_FUNC(void) _PyEval_SignalReceived(
+ struct _ceval_runtime_state *ceval);
+PyAPI_FUNC(int) _PyEval_AddPendingCall(
+ PyThreadState *tstate,
+ struct _ceval_runtime_state *ceval,
+ int (*func)(void *),
+ void *arg);
+PyAPI_FUNC(void) _PyEval_SignalAsyncExc(
+ struct _ceval_runtime_state *ceval);
+PyAPI_FUNC(void) _PyEval_ReInitThreads(
+ _PyRuntimeState *runtime);
#ifdef __cplusplus
}
diff --git a/Include/internal/pycore_coreconfig.h b/Include/internal/pycore_coreconfig.h
index d48904e482a..b17737a90ba 100644
--- a/Include/internal/pycore_coreconfig.h
+++ b/Include/internal/pycore_coreconfig.h
@@ -10,9 +10,41 @@ extern "C" {
#include "pycore_pystate.h" /* _PyRuntimeState */
+/* --- _PyInitError ----------------------------------------------- */
+
+/* Almost all errors causing Python initialization to fail */
+#ifdef _MSC_VER
+ /* Visual Studio 2015 doesn't implement C99 __func__ in C */
+# define _Py_INIT_GET_FUNC() __FUNCTION__
+#else
+# define _Py_INIT_GET_FUNC() __func__
+#endif
+
+#define _Py_INIT_OK() \
+ (_PyInitError){._type = _Py_INIT_ERR_TYPE_OK,}
+ /* other fields are set to 0 */
+#define _Py_INIT_ERR(ERR_MSG) \
+ (_PyInitError){ \
+ ._type = _Py_INIT_ERR_TYPE_ERROR, \
+ ._func = _Py_INIT_GET_FUNC(), \
+ .err_msg = (ERR_MSG)}
+ /* other fields are set to 0 */
+#define _Py_INIT_NO_MEMORY() _Py_INIT_ERR("memory allocation failed")
+#define _Py_INIT_EXIT(EXITCODE) \
+ (_PyInitError){ \
+ ._type = _Py_INIT_ERR_TYPE_EXIT, \
+ .exitcode = (EXITCODE)}
+#define _Py_INIT_IS_ERROR(err) \
+ (err._type == _Py_INIT_ERR_TYPE_ERROR)
+#define _Py_INIT_IS_EXIT(err) \
+ (err._type == _Py_INIT_ERR_TYPE_EXIT)
+#define _Py_INIT_FAILED(err) \
+ (err._type != _Py_INIT_ERR_TYPE_OK)
/* --- _PyWstrList ------------------------------------------------ */
+#define _PyWstrList_INIT (_PyWstrList){.length = 0, .items = NULL}
+
#ifndef NDEBUG
PyAPI_FUNC(int) _PyWstrList_CheckConsistency(const _PyWstrList *list);
#endif
@@ -29,10 +61,10 @@ PyAPI_FUNC(int) _PyWstrList_Extend(_PyWstrList *list,
/* --- _PyArgv ---------------------------------------------------- */
typedef struct {
- int argc;
+ Py_ssize_t argc;
int use_bytes_argv;
- char **bytes_argv;
- wchar_t **wchar_argv;
+ char * const *bytes_argv;
+ wchar_t * const *wchar_argv;
} _PyArgv;
PyAPI_FUNC(_PyInitError) _PyArgv_AsWstrList(const _PyArgv *args,
@@ -88,45 +120,37 @@ PyAPI_FUNC(_PyInitError) _PyPreCmdline_Read(_PyPreCmdline *cmdline,
/* --- _PyPreConfig ----------------------------------------------- */
-PyAPI_FUNC(void) _PyPreConfig_Clear(_PyPreConfig *config);
-PyAPI_FUNC(int) _PyPreConfig_Copy(_PyPreConfig *config,
+PyAPI_FUNC(void) _PyPreConfig_InitCompatConfig(_PyPreConfig *config);
+PyAPI_FUNC(void) _PyPreConfig_InitFromCoreConfig(
+ _PyPreConfig *config,
+ const _PyCoreConfig *coreconfig);
+PyAPI_FUNC(void) _PyPreConfig_InitFromPreConfig(
+ _PyPreConfig *config,
+ const _PyPreConfig *config2);
+PyAPI_FUNC(void) _PyPreConfig_Copy(_PyPreConfig *config,
const _PyPreConfig *config2);
PyAPI_FUNC(PyObject*) _PyPreConfig_AsDict(const _PyPreConfig *config);
-PyAPI_FUNC(void) _PyCoreConfig_GetCoreConfig(_PyPreConfig *config,
+PyAPI_FUNC(void) _PyPreConfig_GetCoreConfig(_PyPreConfig *config,
const _PyCoreConfig *core_config);
PyAPI_FUNC(_PyInitError) _PyPreConfig_Read(_PyPreConfig *config,
const _PyArgv *args);
-PyAPI_FUNC(_PyInitError) _PyPreConfig_Write(_PyPreConfig *config);
+PyAPI_FUNC(_PyInitError) _PyPreConfig_Write(const _PyPreConfig *config);
/* --- _PyCoreConfig ---------------------------------------------- */
-PyAPI_FUNC(void) _PyCoreConfig_Clear(_PyCoreConfig *);
+PyAPI_FUNC(void) _PyCoreConfig_InitCompatConfig(_PyCoreConfig *config);
PyAPI_FUNC(_PyInitError) _PyCoreConfig_Copy(
_PyCoreConfig *config,
const _PyCoreConfig *config2);
-PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetString(
- wchar_t **config_str,
- const wchar_t *str);
-PyAPI_FUNC(_PyInitError) _PyCoreConfig_DecodeLocale(
- wchar_t **config_str,
- const char *str);
PyAPI_FUNC(_PyInitError) _PyCoreConfig_InitPathConfig(_PyCoreConfig *config);
PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetPathConfig(
const _PyCoreConfig *config);
-PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config);
PyAPI_FUNC(void) _PyCoreConfig_Write(const _PyCoreConfig *config,
_PyRuntimeState *runtime);
PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetPyArgv(
_PyCoreConfig *config,
const _PyArgv *args);
-PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetArgv(
- _PyCoreConfig *config,
- int argc,
- char **argv);
-PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetWideArgv(_PyCoreConfig *config,
- int argc,
- wchar_t **argv);
/* --- Function used for testing ---------------------------------- */
diff --git a/Include/internal/pycore_getopt.h b/Include/internal/pycore_getopt.h
index 834b8c8a140..7f0dd13ae57 100644
--- a/Include/internal/pycore_getopt.h
+++ b/Include/internal/pycore_getopt.h
@@ -17,6 +17,6 @@ typedef struct {
int val;
} _PyOS_LongOption;
-extern int _PyOS_GetOpt(Py_ssize_t argc, wchar_t **argv, int *longindex);
+extern int _PyOS_GetOpt(Py_ssize_t argc, wchar_t * const *argv, int *longindex);
#endif /* !Py_INTERNAL_PYGETOPT_H */
diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h
index 9eb8e88df76..bee391187cc 100644
--- a/Include/internal/pycore_pathconfig.h
+++ b/Include/internal/pycore_pathconfig.h
@@ -53,6 +53,10 @@ PyAPI_FUNC(int) _Py_FindEnvConfigValue(
wchar_t *value,
size_t value_size);
+#ifdef MS_WINDOWS
+extern wchar_t* _Py_GetDLLPath(void);
+#endif
+
#ifdef __cplusplus
}
#endif
diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h
new file mode 100644
index 00000000000..23327ef7839
--- /dev/null
+++ b/Include/internal/pycore_pyerrors.h
@@ -0,0 +1,62 @@
+#ifndef Py_INTERNAL_PYERRORS_H
+#define Py_INTERNAL_PYERRORS_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+# error "this header requires Py_BUILD_CORE define"
+#endif
+
+static inline PyObject* _PyErr_Occurred(PyThreadState *tstate)
+{
+ return tstate == NULL ? NULL : tstate->curexc_type;
+}
+
+
+PyAPI_FUNC(void) _PyErr_Fetch(
+ PyThreadState *tstate,
+ PyObject **type,
+ PyObject **value,
+ PyObject **traceback);
+
+PyAPI_FUNC(int) _PyErr_ExceptionMatches(
+ PyThreadState *tstate,
+ PyObject *exc);
+
+PyAPI_FUNC(void) _PyErr_Restore(
+ PyThreadState *tstate,
+ PyObject *type,
+ PyObject *value,
+ PyObject *traceback);
+
+PyAPI_FUNC(void) _PyErr_SetObject(
+ PyThreadState *tstate,
+ PyObject *type,
+ PyObject *value);
+
+PyAPI_FUNC(void) _PyErr_Clear(PyThreadState *tstate);
+
+PyAPI_FUNC(void) _PyErr_SetNone(PyThreadState *tstate, PyObject *exception);
+
+PyAPI_FUNC(void) _PyErr_SetString(
+ PyThreadState *tstate,
+ PyObject *exception,
+ const char *string);
+
+PyAPI_FUNC(PyObject *) _PyErr_Format(
+ PyThreadState *tstate,
+ PyObject *exception,
+ const char *format,
+ ...);
+
+PyAPI_FUNC(void) _PyErr_NormalizeException(
+ PyThreadState *tstate,
+ PyObject **exc,
+ PyObject **val,
+ PyObject **tb);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_PYERRORS_H */
diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h
index adb1f5d90a5..13a31c262da 100644
--- a/Include/internal/pycore_pylifecycle.h
+++ b/Include/internal/pycore_pylifecycle.h
@@ -48,12 +48,15 @@ extern int _PySys_InitMain(
PyInterpreterState *interp);
extern _PyInitError _PyImport_Init(PyInterpreterState *interp);
extern _PyInitError _PyExc_Init(void);
+extern _PyInitError _PyErr_Init(void);
extern _PyInitError _PyBuiltins_AddExceptions(PyObject * bltinmod);
extern _PyInitError _PyImportHooks_Init(void);
extern int _PyFloat_Init(void);
extern _PyInitError _Py_HashRandomization_Init(const _PyCoreConfig *);
extern _PyInitError _PyTypes_Init(void);
+extern _PyInitError _PyImportZip_Init(PyInterpreterState *interp);
+
/* Various internal finalizers */
@@ -81,7 +84,7 @@ extern void PyLong_Fini(void);
extern void _PyFaulthandler_Fini(void);
extern void _PyHash_Fini(void);
extern int _PyTraceMalloc_Fini(void);
-extern void _PyWarnings_Fini(_PyRuntimeState *runtime);
+extern void _PyWarnings_Fini(PyInterpreterState *interp);
extern void _PyGILState_Init(
_PyRuntimeState *runtime,
@@ -98,6 +101,13 @@ PyAPI_FUNC(_PyInitError) _Py_PreInitializeFromCoreConfig(
const _PyCoreConfig *coreconfig,
const _PyArgv *args);
+
+PyAPI_FUNC(int) _Py_HandleSystemExit(int *exitcode_p);
+
+PyAPI_FUNC(PyObject*) _PyErr_WriteUnraisableDefaultHook(PyObject *unraisable);
+
+PyAPI_FUNC(void) _PyErr_Print(PyThreadState *tstate);
+
#ifdef __cplusplus
}
#endif
diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h
index 20f3b5e4006..22677d37385 100644
--- a/Include/internal/pycore_pymem.h
+++ b/Include/internal/pycore_pymem.h
@@ -1,5 +1,5 @@
-#ifndef Py_INTERNAL_MEM_H
-#define Py_INTERNAL_MEM_H
+#ifndef Py_INTERNAL_PYMEM_H
+#define Py_INTERNAL_PYMEM_H
#ifdef __cplusplus
extern "C" {
#endif
@@ -179,7 +179,16 @@ static inline int _PyMem_IsPtrFreed(void *ptr)
#endif
}
+PyAPI_FUNC(int) _PyMem_GetAllocatorName(
+ const char *name,
+ PyMemAllocatorName *allocator);
+
+/* Configure the Python memory allocators.
+ Pass PYMEM_ALLOCATOR_DEFAULT to use default allocators.
+ PYMEM_ALLOCATOR_NOT_SET does nothing. */
+PyAPI_FUNC(int) _PyMem_SetupAllocators(PyMemAllocatorName allocator);
+
#ifdef __cplusplus
}
#endif
-#endif /* !Py_INTERNAL_MEM_H */
+#endif /* !Py_INTERNAL_PYMEM_H */
diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h
index 67bcd147e28..ef1d8a061f1 100644
--- a/Include/internal/pycore_pystate.h
+++ b/Include/internal/pycore_pystate.h
@@ -9,15 +9,56 @@ extern "C" {
#endif
#include "cpython/coreconfig.h"
+#include "fileobject.h"
#include "pystate.h"
#include "pythread.h"
+#include "sysmodule.h"
-#include "pycore_ceval.h"
+#include "pycore_gil.h" /* _gil_runtime_state */
#include "pycore_pathconfig.h"
#include "pycore_pymem.h"
#include "pycore_warnings.h"
+/* ceval state */
+
+struct _pending_calls {
+ int finishing;
+ PyThread_type_lock lock;
+ /* Request for running pending calls. */
+ _Py_atomic_int calls_to_do;
+ /* Request for looking at the `async_exc` field of the current
+ thread state.
+ Guarded by the GIL. */
+ int async_exc;
+#define NPENDINGCALLS 32
+ struct {
+ int (*func)(void *);
+ void *arg;
+ } calls[NPENDINGCALLS];
+ int first;
+ int last;
+};
+
+struct _ceval_runtime_state {
+ int recursion_limit;
+ /* Records whether tracing is on for any thread. Counts the number
+ of threads for which tstate->c_tracefunc is non-NULL, so if the
+ value is 0, we know we don't have to check this thread's
+ c_tracefunc. This speeds up the if statement in
+ PyEval_EvalFrameEx() after fast_next_opcode. */
+ int tracing_possible;
+ /* This single variable consolidates all requests to break out of
+ the fast path in the eval loop. */
+ _Py_atomic_int eval_breaker;
+ /* Request for dropping the GIL */
+ _Py_atomic_int gil_drop_request;
+ struct _pending_calls pending;
+ /* Request for checking signals. */
+ _Py_atomic_int signals_pending;
+ struct _gil_runtime_state gil;
+};
+
/* interpreter state */
typedef PyObject* (*_PyFrameEvalFunction)(struct _frame *, int);
@@ -90,6 +131,10 @@ struct _is {
PyObject *pyexitmodule;
uint64_t tstate_next_unique_id;
+
+ struct _warnings_runtime_state warnings;
+
+ PyObject *audit_hooks;
};
PyAPI_FUNC(struct _is*) _PyInterpreterState_LookUpID(PY_INT64_T);
@@ -113,6 +158,13 @@ struct _xidregitem {
struct _xidregitem *next;
};
+/* runtime audit hook state */
+
+typedef struct _Py_AuditHookEntry {
+ struct _Py_AuditHookEntry *next;
+ Py_AuditHookFunction hookCFunction;
+ void *userData;
+} _Py_AuditHookEntry;
/* GIL state */
@@ -179,11 +231,15 @@ typedef struct pyruntimestate {
int nexitfuncs;
struct _gc_runtime_state gc;
- struct _warnings_runtime_state warnings;
struct _ceval_runtime_state ceval;
struct _gilstate_runtime_state gilstate;
_PyPreConfig preconfig;
+
+ Py_OpenCodeHookFunction open_code_hook;
+ void *open_code_userdata;
+ _Py_AuditHookEntry *audit_hook_head;
+
// XXX Consolidate globals found via the check-c-globals script.
} _PyRuntimeState;
@@ -202,13 +258,16 @@ PyAPI_FUNC(_PyInitError) _PyRuntime_Initialize(void);
PyAPI_FUNC(void) _PyRuntime_Finalize(void);
-#define _Py_CURRENTLY_FINALIZING(tstate) \
- (_PyRuntime.finalizing == tstate)
+#define _Py_CURRENTLY_FINALIZING(runtime, tstate) \
+ (runtime->finalizing == tstate)
/* Variable and macro for in-line access to current thread
and interpreter state */
+#define _PyRuntimeState_GetThreadState(runtime) \
+ ((PyThreadState*)_Py_atomic_load_relaxed(&(runtime)->gilstate.tstate_current))
+
/* Get the current Python thread state.
Efficient macro reading directly the 'gilstate.tstate_current' atomic
@@ -218,8 +277,7 @@ PyAPI_FUNC(void) _PyRuntime_Finalize(void);
The caller must hold the GIL.
See also PyThreadState_Get() and PyThreadState_GET(). */
-#define _PyThreadState_GET() \
- ((PyThreadState*)_Py_atomic_load_relaxed(&_PyRuntime.gilstate.tstate_current))
+#define _PyThreadState_GET() _PyRuntimeState_GetThreadState(&_PyRuntime)
/* Redefine PyThreadState_GET() as an alias to _PyThreadState_GET() */
#undef PyThreadState_GET
@@ -241,7 +299,13 @@ PyAPI_FUNC(void) _PyRuntime_Finalize(void);
PyAPI_FUNC(void) _PyThreadState_Init(
_PyRuntimeState *runtime,
PyThreadState *tstate);
-PyAPI_FUNC(void) _PyThreadState_DeleteExcept(PyThreadState *tstate);
+PyAPI_FUNC(void) _PyThreadState_DeleteExcept(
+ _PyRuntimeState *runtime,
+ PyThreadState *tstate);
+
+PyAPI_FUNC(PyThreadState *) _PyThreadState_Swap(
+ struct _gilstate_runtime_state *gilstate,
+ PyThreadState *newts);
PyAPI_FUNC(_PyInitError) _PyInterpreterState_Enable(_PyRuntimeState *runtime);
PyAPI_FUNC(void) _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime);
diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h
new file mode 100644
index 00000000000..bf4d7fe5851
--- /dev/null
+++ b/Include/internal/pycore_traceback.h
@@ -0,0 +1,96 @@
+#ifndef Py_INTERNAL_TRACEBACK_H
+#define Py_INTERNAL_TRACEBACK_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+# error "this header requires Py_BUILD_CORE define"
+#endif
+
+#include "pystate.h" /* PyInterpreterState */
+
+/* Write the Python traceback into the file 'fd'. For example:
+
+ Traceback (most recent call first):
+ File "xxx", line xxx in
+ File "xxx", line xxx in
+ ...
+ File "xxx", line xxx in
+
+ This function is written for debug purpose only, to dump the traceback in
+ the worst case: after a segmentation fault, at fatal error, etc. That's why,
+ it is very limited. Strings are truncated to 100 characters and encoded to
+ ASCII with backslashreplace. It doesn't write the source code, only the
+ function name, filename and line number of each frame. Write only the first
+ 100 frames: if the traceback is truncated, write the line " ...".
+
+ This function is signal safe. */
+
+PyAPI_FUNC(void) _Py_DumpTraceback(
+ int fd,
+ PyThreadState *tstate);
+
+/* Write the traceback of all threads into the file 'fd'. current_thread can be
+ NULL.
+
+ Return NULL on success, or an error message on error.
+
+ This function is written for debug purpose only. It calls
+ _Py_DumpTraceback() for each thread, and so has the same limitations. It
+ only write the traceback of the first 100 threads: write "..." if there are
+ more threads.
+
+ If current_tstate is NULL, the function tries to get the Python thread state
+ of the current thread. It is not an error if the function is unable to get
+ the current Python thread state.
+
+ If interp is NULL, the function tries to get the interpreter state from
+ the current Python thread state, or from
+ _PyGILState_GetInterpreterStateUnsafe() in last resort.
+
+ It is better to pass NULL to interp and current_tstate, the function tries
+ different options to retrieve these informations.
+
+ This function is signal safe. */
+
+PyAPI_FUNC(const char*) _Py_DumpTracebackThreads(
+ int fd,
+ PyInterpreterState *interp,
+ PyThreadState *current_tstate);
+
+/* Write a Unicode object into the file descriptor fd. Encode the string to
+ ASCII using the backslashreplace error handler.
+
+ Do nothing if text is not a Unicode object. The function accepts Unicode
+ string which is not ready (PyUnicode_WCHAR_KIND).
+
+ This function is signal safe. */
+PyAPI_FUNC(void) _Py_DumpASCII(int fd, PyObject *text);
+
+/* Format an integer as decimal into the file descriptor fd.
+
+ This function is signal safe. */
+PyAPI_FUNC(void) _Py_DumpDecimal(
+ int fd,
+ unsigned long value);
+
+/* Format an integer as hexadecimal into the file descriptor fd with at least
+ width digits.
+
+ The maximum width is sizeof(unsigned long)*2 digits.
+
+ This function is signal safe. */
+PyAPI_FUNC(void) _Py_DumpHexadecimal(
+ int fd,
+ unsigned long value,
+ Py_ssize_t width);
+
+PyAPI_FUNC(PyObject*) _PyTraceBack_FromFrame(
+ PyObject *tb_next,
+ struct _frame *frame);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_TRACEBACK_H */
diff --git a/Include/longobject.h b/Include/longobject.h
index b696f544b9c..a24bbea3a90 100644
--- a/Include/longobject.h
+++ b/Include/longobject.h
@@ -230,6 +230,9 @@ PyAPI_FUNC(PyObject *) _PyLong_GCD(PyObject *, PyObject *);
#ifndef Py_LIMITED_API
PyAPI_DATA(PyObject *) _PyLong_Zero;
PyAPI_DATA(PyObject *) _PyLong_One;
+
+PyAPI_FUNC(PyObject *) _PyLong_Rshift(PyObject *, size_t);
+PyAPI_FUNC(PyObject *) _PyLong_Lshift(PyObject *, size_t);
#endif
#ifdef __cplusplus
diff --git a/Include/object.h b/Include/object.h
index 13e88a6dc6f..6464f33be49 100644
--- a/Include/object.h
+++ b/Include/object.h
@@ -649,11 +649,11 @@ times.
When deallocating a container object, it's possible to trigger an unbounded
chain of deallocations, as each Py_DECREF in turn drops the refcount on "the
-next" object in the chain to 0. This can easily lead to stack faults, and
+next" object in the chain to 0. This can easily lead to stack overflows,
especially in threads (which typically have less stack space to work with).
-A container object that participates in cyclic gc can avoid this by
-bracketing the body of its tp_dealloc function with a pair of macros:
+A container object can avoid this by bracketing the body of its tp_dealloc
+function with a pair of macros:
static void
mytype_dealloc(mytype *p)
@@ -661,14 +661,14 @@ mytype_dealloc(mytype *p)
... declarations go here ...
PyObject_GC_UnTrack(p); // must untrack first
- Py_TRASHCAN_SAFE_BEGIN(p)
+ Py_TRASHCAN_BEGIN(p, mytype_dealloc)
... The body of the deallocator goes here, including all calls ...
... to Py_DECREF on contained objects. ...
- Py_TRASHCAN_SAFE_END(p)
+ Py_TRASHCAN_END // there should be no code after this
}
CAUTION: Never return from the middle of the body! If the body needs to
-"get out early", put a label immediately before the Py_TRASHCAN_SAFE_END
+"get out early", put a label immediately before the Py_TRASHCAN_END
call, and goto it. Else the call-depth counter (see below) will stay
above 0 forever, and the trashcan will never get emptied.
@@ -684,6 +684,12 @@ notices this, and calls another routine to deallocate all the objects that
may have been added to the list of deferred deallocations. In effect, a
chain of N deallocations is broken into (N-1)/(PyTrash_UNWIND_LEVEL-1) pieces,
with the call stack never exceeding a depth of PyTrash_UNWIND_LEVEL.
+
+Since the tp_dealloc of a subclass typically calls the tp_dealloc of the base
+class, we need to ensure that the trashcan is only triggered on the tp_dealloc
+of the actual class being deallocated. Otherwise we might end up with a
+partially-deallocated object. To check this, the tp_dealloc function must be
+passed as second argument to Py_TRASHCAN_BEGIN().
*/
/* The new thread-safe private API, invoked by the macros below. */
@@ -692,21 +698,38 @@ PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(void);
#define PyTrash_UNWIND_LEVEL 50
-#define Py_TRASHCAN_SAFE_BEGIN(op) \
+#define Py_TRASHCAN_BEGIN_CONDITION(op, cond) \
do { \
- PyThreadState *_tstate = PyThreadState_GET(); \
- if (_tstate->trash_delete_nesting < PyTrash_UNWIND_LEVEL) { \
- ++_tstate->trash_delete_nesting;
- /* The body of the deallocator is here. */
-#define Py_TRASHCAN_SAFE_END(op) \
+ PyThreadState *_tstate = NULL; \
+ /* If "cond" is false, then _tstate remains NULL and the deallocator \
+ * is run normally without involving the trashcan */ \
+ if (cond) { \
+ _tstate = PyThreadState_GET(); \
+ if (_tstate->trash_delete_nesting >= PyTrash_UNWIND_LEVEL) { \
+ /* Store the object (to be deallocated later) and jump past \
+ * Py_TRASHCAN_END, skipping the body of the deallocator */ \
+ _PyTrash_thread_deposit_object(_PyObject_CAST(op)); \
+ break; \
+ } \
+ ++_tstate->trash_delete_nesting; \
+ }
+ /* The body of the deallocator is here. */
+#define Py_TRASHCAN_END \
+ if (_tstate) { \
--_tstate->trash_delete_nesting; \
if (_tstate->trash_delete_later && _tstate->trash_delete_nesting <= 0) \
_PyTrash_thread_destroy_chain(); \
} \
- else \
- _PyTrash_thread_deposit_object(_PyObject_CAST(op)); \
} while (0);
+#define Py_TRASHCAN_BEGIN(op, dealloc) Py_TRASHCAN_BEGIN_CONDITION(op, \
+ Py_TYPE(op)->tp_dealloc == (destructor)(dealloc))
+
+/* For backwards compatibility, these macros enable the trashcan
+ * unconditionally */
+#define Py_TRASHCAN_SAFE_BEGIN(op) Py_TRASHCAN_BEGIN_CONDITION(op, 1)
+#define Py_TRASHCAN_SAFE_END(op) Py_TRASHCAN_END
+
#ifndef Py_LIMITED_API
# define Py_CPYTHON_OBJECT_H
diff --git a/Include/pydtrace.d b/Include/pydtrace.d
index a6a5e7ec224..5e6a626b01b 100644
--- a/Include/pydtrace.d
+++ b/Include/pydtrace.d
@@ -12,6 +12,7 @@ provider python {
probe gc__done(long);
probe import__find__load__start(const char *);
probe import__find__load__done(const char *, int);
+ probe audit(const char *, void *);
};
#pragma D attributes Evolving/Evolving/Common provider python provider
diff --git a/Include/pydtrace.h b/Include/pydtrace.h
index 7a04278166b..75f8e7f7097 100644
--- a/Include/pydtrace.h
+++ b/Include/pydtrace.h
@@ -36,6 +36,7 @@ static inline void PyDTrace_INSTANCE_DELETE_START(int arg0) {}
static inline void PyDTrace_INSTANCE_DELETE_DONE(int arg0) {}
static inline void PyDTrace_IMPORT_FIND_LOAD_START(const char *arg0) {}
static inline void PyDTrace_IMPORT_FIND_LOAD_DONE(const char *arg0, int arg1) {}
+static inline void PyDTrace_AUDIT(const char *arg0, void *arg1) {}
static inline int PyDTrace_LINE_ENABLED(void) { return 0; }
static inline int PyDTrace_FUNCTION_ENTRY_ENABLED(void) { return 0; }
@@ -48,6 +49,7 @@ static inline int PyDTrace_INSTANCE_DELETE_START_ENABLED(void) { return 0; }
static inline int PyDTrace_INSTANCE_DELETE_DONE_ENABLED(void) { return 0; }
static inline int PyDTrace_IMPORT_FIND_LOAD_START_ENABLED(void) { return 0; }
static inline int PyDTrace_IMPORT_FIND_LOAD_DONE_ENABLED(void) { return 0; }
+static inline int PyDTrace_AUDIT_ENABLED(void) { return 0; }
#endif /* !WITH_DTRACE */
diff --git a/Include/pymacro.h b/Include/pymacro.h
index 546f9c6e702..1890619099a 100644
--- a/Include/pymacro.h
+++ b/Include/pymacro.h
@@ -89,10 +89,15 @@
/* Check if pointer "p" is aligned to "a"-bytes boundary. */
#define _Py_IS_ALIGNED(p, a) (!((uintptr_t)(p) & (uintptr_t)((a) - 1)))
-#ifdef __GNUC__
-#define Py_UNUSED(name) _unused_ ## name __attribute__((unused))
+/* Use this for unused arguments in a function definition to silence compiler
+ * warnings. Example:
+ *
+ * int func(int a, int Py_UNUSED(b)) { return a; }
+ */
+#if defined(__GNUC__) || defined(__clang__)
+# define Py_UNUSED(name) _unused_ ## name __attribute__((unused))
#else
-#define Py_UNUSED(name) _unused_ ## name
+# define Py_UNUSED(name) _unused_ ## name
#endif
#define Py_UNREACHABLE() abort()
diff --git a/Include/pythread.h b/Include/pythread.h
index bc1d92cd1ff..40f12d257c1 100644
--- a/Include/pythread.h
+++ b/Include/pythread.h
@@ -26,6 +26,11 @@ PyAPI_FUNC(unsigned long) PyThread_start_new_thread(void (*)(void *), void *);
PyAPI_FUNC(void) _Py_NO_RETURN PyThread_exit_thread(void);
PyAPI_FUNC(unsigned long) PyThread_get_thread_ident(void);
+#if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__) || defined(_WIN32)
+#define PY_HAVE_THREAD_NATIVE_ID
+PyAPI_FUNC(unsigned long) PyThread_get_thread_native_id(void);
+#endif
+
PyAPI_FUNC(PyThread_type_lock) PyThread_allocate_lock(void);
PyAPI_FUNC(void) PyThread_free_lock(PyThread_type_lock);
PyAPI_FUNC(int) PyThread_acquire_lock(PyThread_type_lock, int);
diff --git a/Include/sysmodule.h b/Include/sysmodule.h
index c5547ff6742..670e5d283f7 100644
--- a/Include/sysmodule.h
+++ b/Include/sysmodule.h
@@ -9,10 +9,6 @@ extern "C" {
PyAPI_FUNC(PyObject *) PySys_GetObject(const char *);
PyAPI_FUNC(int) PySys_SetObject(const char *, PyObject *);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject *) _PySys_GetObjectId(_Py_Identifier *key);
-PyAPI_FUNC(int) _PySys_SetObjectId(_Py_Identifier *key, PyObject *);
-#endif
PyAPI_FUNC(void) PySys_SetArgv(int, wchar_t **);
PyAPI_FUNC(void) PySys_SetArgvEx(int, wchar_t **, int);
@@ -34,7 +30,9 @@ PyAPI_FUNC(void) PySys_AddXOption(const wchar_t *);
PyAPI_FUNC(PyObject *) PySys_GetXOptions(void);
#ifndef Py_LIMITED_API
-PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *);
+# define Py_CPYTHON_SYSMODULE_H
+# include "cpython/sysmodule.h"
+# undef Py_CPYTHON_SYSMODULE_H
#endif
#ifdef __cplusplus
diff --git a/Include/traceback.h b/Include/traceback.h
index b5874100f47..b451927fafa 100644
--- a/Include/traceback.h
+++ b/Include/traceback.h
@@ -1,117 +1,26 @@
-
#ifndef Py_TRACEBACK_H
#define Py_TRACEBACK_H
#ifdef __cplusplus
extern "C" {
#endif
-#include "pystate.h"
-
struct _frame;
/* Traceback interface */
-#ifndef Py_LIMITED_API
-typedef struct _traceback {
- PyObject_HEAD
- struct _traceback *tb_next;
- struct _frame *tb_frame;
- int tb_lasti;
- int tb_lineno;
-} PyTracebackObject;
-#endif
PyAPI_FUNC(int) PyTraceBack_Here(struct _frame *);
PyAPI_FUNC(int) PyTraceBack_Print(PyObject *, PyObject *);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int);
-PyAPI_FUNC(void) _PyTraceback_Add(const char *, const char *, int);
-#endif
/* Reveal traceback type so we can typecheck traceback objects */
PyAPI_DATA(PyTypeObject) PyTraceBack_Type;
#define PyTraceBack_Check(v) (Py_TYPE(v) == &PyTraceBack_Type)
-#ifndef Py_LIMITED_API
-/* Write the Python traceback into the file 'fd'. For example:
-
- Traceback (most recent call first):
- File "xxx", line xxx in
- File "xxx", line xxx in
- ...
- File "xxx", line xxx in
-
- This function is written for debug purpose only, to dump the traceback in
- the worst case: after a segmentation fault, at fatal error, etc. That's why,
- it is very limited. Strings are truncated to 100 characters and encoded to
- ASCII with backslashreplace. It doesn't write the source code, only the
- function name, filename and line number of each frame. Write only the first
- 100 frames: if the traceback is truncated, write the line " ...".
-
- This function is signal safe. */
-
-PyAPI_FUNC(void) _Py_DumpTraceback(
- int fd,
- PyThreadState *tstate);
-
-/* Write the traceback of all threads into the file 'fd'. current_thread can be
- NULL.
-
- Return NULL on success, or an error message on error.
-
- This function is written for debug purpose only. It calls
- _Py_DumpTraceback() for each thread, and so has the same limitations. It
- only write the traceback of the first 100 threads: write "..." if there are
- more threads.
-
- If current_tstate is NULL, the function tries to get the Python thread state
- of the current thread. It is not an error if the function is unable to get
- the current Python thread state.
-
- If interp is NULL, the function tries to get the interpreter state from
- the current Python thread state, or from
- _PyGILState_GetInterpreterStateUnsafe() in last resort.
-
- It is better to pass NULL to interp and current_tstate, the function tries
- different options to retrieve these informations.
-
- This function is signal safe. */
-
-PyAPI_FUNC(const char*) _Py_DumpTracebackThreads(
- int fd,
- PyInterpreterState *interp,
- PyThreadState *current_tstate);
-#endif /* !Py_LIMITED_API */
#ifndef Py_LIMITED_API
-
-/* Write a Unicode object into the file descriptor fd. Encode the string to
- ASCII using the backslashreplace error handler.
-
- Do nothing if text is not a Unicode object. The function accepts Unicode
- string which is not ready (PyUnicode_WCHAR_KIND).
-
- This function is signal safe. */
-PyAPI_FUNC(void) _Py_DumpASCII(int fd, PyObject *text);
-
-/* Format an integer as decimal into the file descriptor fd.
-
- This function is signal safe. */
-PyAPI_FUNC(void) _Py_DumpDecimal(
- int fd,
- unsigned long value);
-
-/* Format an integer as hexadecimal into the file descriptor fd with at least
- width digits.
-
- The maximum width is sizeof(unsigned long)*2 digits.
-
- This function is signal safe. */
-PyAPI_FUNC(void) _Py_DumpHexadecimal(
- int fd,
- unsigned long value,
- Py_ssize_t width);
-
-#endif /* !Py_LIMITED_API */
+# define Py_CPYTHON_TRACEBACK_H
+# include "cpython/traceback.h"
+# undef Py_CPYTHON_TRACEBACK_H
+#endif
#ifdef __cplusplus
}
diff --git a/Lib/_pyio.py b/Lib/_pyio.py
index af2ce30c278..5baca4df82f 100644
--- a/Lib/_pyio.py
+++ b/Lib/_pyio.py
@@ -33,6 +33,10 @@ DEFAULT_BUFFER_SIZE = 8 * 1024 # bytes
# Rebind for compatibility
BlockingIOError = BlockingIOError
+# Does io.IOBase finalizer log the exception if the close() method fails?
+# The exception is ignored silently by default in release build.
+_IOBASE_EMITS_UNRAISABLE = (hasattr(sys, "gettotalrefcount") or sys.flags.dev_mode)
+
def open(file, mode="r", buffering=-1, encoding=None, errors=None,
newline=None, closefd=True, opener=None):
@@ -250,6 +254,29 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
result.close()
raise
+# Define a default pure-Python implementation for open_code()
+# that does not allow hooks. Warn on first use. Defined for tests.
+def _open_code_with_warning(path):
+ """Opens the provided file with mode ``'rb'``. This function
+ should be used when the intent is to treat the contents as
+ executable code.
+
+ ``path`` should be an absolute path.
+
+ When supported by the runtime, this function can be hooked
+ in order to allow embedders more control over code files.
+ This functionality is not supported on the current runtime.
+ """
+ import warnings
+ warnings.warn("_pyio.open_code() may not be using hooks",
+ RuntimeWarning, 2)
+ return open(path, "rb")
+
+try:
+ open_code = io.open_code
+except AttributeError:
+ open_code = _open_code_with_warning
+
class DocDescriptor:
"""Helper for builtins.open.__doc__
@@ -378,15 +405,18 @@ class IOBase(metaclass=abc.ABCMeta):
def __del__(self):
"""Destructor. Calls close()."""
- # The try/except block is in case this is called at program
- # exit time, when it's possible that globals have already been
- # deleted, and then the close() call might fail. Since
- # there's nothing we can do about such failures and they annoy
- # the end users, we suppress the traceback.
- try:
+ if _IOBASE_EMITS_UNRAISABLE:
self.close()
- except:
- pass
+ else:
+ # The try/except block is in case this is called at program
+ # exit time, when it's possible that globals have already been
+ # deleted, and then the close() call might fail. Since
+ # there's nothing we can do about such failures and they annoy
+ # the end users, we suppress the traceback.
+ try:
+ self.close()
+ except:
+ pass
### Inquiries ###
diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py
index 304c66f59bd..7a84823622e 100644
--- a/Lib/_weakrefset.py
+++ b/Lib/_weakrefset.py
@@ -194,3 +194,6 @@ class WeakSet:
def isdisjoint(self, other):
return len(self.intersection(other)) == 0
+
+ def __repr__(self):
+ return repr(self.data)
diff --git a/Lib/argparse.py b/Lib/argparse.py
index 798766f6c40..ef888f063b3 100644
--- a/Lib/argparse.py
+++ b/Lib/argparse.py
@@ -1154,6 +1154,12 @@ class _SubParsersAction(Action):
vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, [])
getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings)
+class _ExtendAction(_AppendAction):
+ def __call__(self, parser, namespace, values, option_string=None):
+ items = getattr(namespace, self.dest, None)
+ items = _copy_items(items)
+ items.extend(values)
+ setattr(namespace, self.dest, items)
# ==============
# Type classes
@@ -1262,6 +1268,7 @@ class _ActionsContainer(object):
self.register('action', 'help', _HelpAction)
self.register('action', 'version', _VersionAction)
self.register('action', 'parsers', _SubParsersAction)
+ self.register('action', 'extend', _ExtendAction)
# raise an exception if the conflict handler is invalid
self._get_handler()
diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py
index 9613ac2a114..de9fa4f4f7f 100644
--- a/Lib/asyncio/base_events.py
+++ b/Lib/asyncio/base_events.py
@@ -102,7 +102,7 @@ def _set_reuseport(sock):
'SO_REUSEPORT defined but not implemented.')
-def _ipaddr_info(host, port, family, type, proto):
+def _ipaddr_info(host, port, family, type, proto, flowinfo=0, scopeid=0):
# Try to skip getaddrinfo if "host" is already an IP. Users might have
# handled name resolution in their own code and pass in resolved IPs.
if not hasattr(socket, 'inet_pton'):
@@ -151,7 +151,7 @@ def _ipaddr_info(host, port, family, type, proto):
socket.inet_pton(af, host)
# The host has already been resolved.
if _HAS_IPv6 and af == socket.AF_INET6:
- return af, type, proto, '', (host, port, 0, 0)
+ return af, type, proto, '', (host, port, flowinfo, scopeid)
else:
return af, type, proto, '', (host, port)
except OSError:
@@ -1348,7 +1348,7 @@ class BaseEventLoop(events.AbstractEventLoop):
family=0, type=socket.SOCK_STREAM,
proto=0, flags=0, loop):
host, port = address[:2]
- info = _ipaddr_info(host, port, family, type, proto)
+ info = _ipaddr_info(host, port, family, type, proto, *address[2:])
if info is not None:
# "host" is already a resolved IP.
return [info]
diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py
index c665ebe33ee..9664ea74d75 100644
--- a/Lib/asyncio/coroutines.py
+++ b/Lib/asyncio/coroutines.py
@@ -7,6 +7,7 @@ import os
import sys
import traceback
import types
+import warnings
from . import base_futures
from . import constants
@@ -107,6 +108,9 @@ def coroutine(func):
If the coroutine is not yielded from before it is destroyed,
an error message is logged.
"""
+ warnings.warn('"@coroutine" decorator is deprecated since Python 3.8, use "async def" instead',
+ DeprecationWarning,
+ stacklevel=2)
if inspect.iscoroutinefunction(func):
# In Python 3.5 that's all we need to do for coroutines
# defined with "async def".
diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py
index 639bd11bd06..d59eb8f210c 100644
--- a/Lib/asyncio/locks.py
+++ b/Lib/asyncio/locks.py
@@ -3,12 +3,13 @@
__all__ = ('Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore')
import collections
+import types
import warnings
from . import events
from . import futures
from . import exceptions
-from .coroutines import coroutine
+from .import coroutines
class _ContextManager:
@@ -55,7 +56,7 @@ class _ContextManagerMixin:
# always raises; that's how the with-statement works.
pass
- @coroutine
+ @types.coroutine
def __iter__(self):
# This is not a coroutine. It is meant to enable the idiom:
#
@@ -78,6 +79,9 @@ class _ContextManagerMixin:
yield from self.acquire()
return _ContextManager(self)
+ # The flag is needed for legacy asyncio.iscoroutine()
+ __iter__._is_coroutine = coroutines._is_coroutine
+
async def __acquire_ctx(self):
await self.acquire()
return _ContextManager(self)
diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py
index 29968214f8e..6461d307763 100644
--- a/Lib/asyncio/selector_events.py
+++ b/Lib/asyncio/selector_events.py
@@ -428,32 +428,35 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop):
if n == len(data):
# all data sent
return
- else:
- data = bytearray(memoryview(data)[n:])
fut = self.create_future()
fd = sock.fileno()
fut.add_done_callback(
functools.partial(self._sock_write_done, fd))
- self.add_writer(fd, self._sock_sendall, fut, sock, data)
+ # use a trick with a list in closure to store a mutable state
+ self.add_writer(fd, self._sock_sendall, fut, sock,
+ memoryview(data), [n])
return await fut
- def _sock_sendall(self, fut, sock, data):
+ def _sock_sendall(self, fut, sock, view, pos):
if fut.done():
# Future cancellation can be scheduled on previous loop iteration
return
+ start = pos[0]
try:
- n = sock.send(data)
+ n = sock.send(view[start:])
except (BlockingIOError, InterruptedError):
return
except Exception as exc:
fut.set_exception(exc)
return
- if n == len(data):
+ start += n
+
+ if start == len(view):
fut.set_result(None)
else:
- del data[:n]
+ pos[0] = start
async def sock_connect(self, sock, address):
"""Connect to a remote socket at address.
diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py
index d9a9f5e72d3..2f0cbfdbe85 100644
--- a/Lib/asyncio/streams.py
+++ b/Lib/asyncio/streams.py
@@ -329,6 +329,13 @@ class StreamReaderProtocol(FlowControlMixin, protocols.Protocol):
closed.exception()
+def _swallow_unhandled_exception(task):
+ # Do a trick to suppress unhandled exception
+ # if stream.write() was used without await and
+ # stream.drain() was paused and resumed with an exception
+ task.exception()
+
+
class StreamWriter:
"""Wraps a Transport.
@@ -393,7 +400,9 @@ class StreamWriter:
# fast path, the stream is not paused
# no need to wait for resume signal
return self._complete_fut
- return self._loop.create_task(self.drain())
+ ret = self._loop.create_task(self.drain())
+ ret.add_done_callback(_swallow_unhandled_exception)
+ return ret
def write_eof(self):
return self._transport.write_eof()
@@ -430,9 +439,7 @@ class StreamWriter:
# Wait for protocol.connection_lost() call
# Raise connection closing error if any,
# ConnectionResetError otherwise
- fut = self._protocol._get_close_waiter(self)
- await fut
- raise ConnectionResetError('Connection lost')
+ await sleep(0)
await self._protocol._drain_helper()
diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py
index 211b9126b01..1dc595298c5 100644
--- a/Lib/asyncio/tasks.py
+++ b/Lib/asyncio/tasks.py
@@ -23,7 +23,7 @@ from . import coroutines
from . import events
from . import exceptions
from . import futures
-from .coroutines import coroutine
+from .coroutines import _is_coroutine
# Helper to generate new task names
# This uses itertools.count() instead of a "+= 1" operation because the latter
@@ -95,7 +95,7 @@ class Task(futures._PyFuture): # Inherit Python Task implementation
None is returned when called not in the context of a Task.
"""
- warnings.warn("Task.current_task() is deprecated, "
+ warnings.warn("Task.current_task() is deprecated since Python 3.7, "
"use asyncio.current_task() instead",
DeprecationWarning,
stacklevel=2)
@@ -109,7 +109,7 @@ class Task(futures._PyFuture): # Inherit Python Task implementation
By default all tasks for the current event loop are returned.
"""
- warnings.warn("Task.all_tasks() is deprecated, "
+ warnings.warn("Task.all_tasks() is deprecated since Python 3.7, "
"use asyncio.all_tasks() instead",
DeprecationWarning,
stacklevel=2)
@@ -388,8 +388,8 @@ async def wait(fs, *, loop=None, timeout=None, return_when=ALL_COMPLETED):
if loop is None:
loop = events.get_running_loop()
else:
- warnings.warn("The loop argument is deprecated and scheduled for "
- "removal in Python 3.10.",
+ warnings.warn("The loop argument is deprecated since Python 3.8, "
+ "and scheduled for removal in Python 3.10.",
DeprecationWarning, stacklevel=2)
fs = {ensure_future(f, loop=loop) for f in set(fs)}
@@ -418,8 +418,8 @@ async def wait_for(fut, timeout, *, loop=None):
if loop is None:
loop = events.get_running_loop()
else:
- warnings.warn("The loop argument is deprecated and scheduled for "
- "removal in Python 3.10.",
+ warnings.warn("The loop argument is deprecated since Python 3.8, "
+ "and scheduled for removal in Python 3.10.",
DeprecationWarning, stacklevel=2)
if timeout is None:
@@ -600,8 +600,8 @@ async def sleep(delay, result=None, *, loop=None):
if loop is None:
loop = events.get_running_loop()
else:
- warnings.warn("The loop argument is deprecated and scheduled for "
- "removal in Python 3.10.",
+ warnings.warn("The loop argument is deprecated since Python 3.8, "
+ "and scheduled for removal in Python 3.10.",
DeprecationWarning, stacklevel=2)
future = loop.create_future()
@@ -638,7 +638,7 @@ def ensure_future(coro_or_future, *, loop=None):
'required')
-@coroutine
+@types.coroutine
def _wrap_awaitable(awaitable):
"""Helper for asyncio.ensure_future().
@@ -647,6 +647,8 @@ def _wrap_awaitable(awaitable):
"""
return (yield from awaitable.__await__())
+_wrap_awaitable._is_coroutine = _is_coroutine
+
class _GatheringFuture(futures.Future):
"""Helper for gather().
diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py
index 73d4bda7c23..1aa3b396086 100644
--- a/Lib/asyncio/unix_events.py
+++ b/Lib/asyncio/unix_events.py
@@ -757,12 +757,18 @@ class _UnixSubprocessTransport(base_subprocess.BaseSubprocessTransport):
# other end). Notably this is needed on AIX, and works
# just fine on other platforms.
stdin, stdin_w = socket.socketpair()
- self._proc = subprocess.Popen(
- args, shell=shell, stdin=stdin, stdout=stdout, stderr=stderr,
- universal_newlines=False, bufsize=bufsize, **kwargs)
- if stdin_w is not None:
- stdin.close()
- self._proc.stdin = open(stdin_w.detach(), 'wb', buffering=bufsize)
+ try:
+ self._proc = subprocess.Popen(
+ args, shell=shell, stdin=stdin, stdout=stdout, stderr=stderr,
+ universal_newlines=False, bufsize=bufsize, **kwargs)
+ if stdin_w is not None:
+ stdin.close()
+ self._proc.stdin = open(stdin_w.detach(), 'wb', buffering=bufsize)
+ stdin_w = None
+ finally:
+ if stdin_w is not None:
+ stdin.close()
+ stdin_w.close()
class AbstractChildWatcher:
diff --git a/Lib/codecs.py b/Lib/codecs.py
index 6b028adb1d2..884be0b2c02 100644
--- a/Lib/codecs.py
+++ b/Lib/codecs.py
@@ -838,7 +838,7 @@ class StreamRecoder:
def writelines(self, list):
- data = ''.join(list)
+ data = b''.join(list)
data, bytesdecoded = self.decode(data, self.errors)
return self.writer.write(data)
diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py
index 706907ad4a2..cb7f1bb1fcf 100644
--- a/Lib/collections/__init__.py
+++ b/Lib/collections/__init__.py
@@ -1038,6 +1038,13 @@ class UserDict(_collections_abc.MutableMapping):
# Now, add the methods in dicts but not in MutableMapping
def __repr__(self): return repr(self.data)
+ def __copy__(self):
+ inst = self.__class__.__new__(self.__class__)
+ inst.__dict__.update(self.__dict__)
+ # Create a copy and avoid triggering descriptors
+ inst.__dict__["data"] = self.__dict__["data"].copy()
+ return inst
+
def copy(self):
if self.__class__ is UserDict:
return UserDict(self.data.copy())
@@ -1050,6 +1057,7 @@ class UserDict(_collections_abc.MutableMapping):
self.data = data
c.update(self)
return c
+
@classmethod
def fromkeys(cls, iterable, value=None):
d = cls()
@@ -1118,6 +1126,12 @@ class UserList(_collections_abc.MutableSequence):
def __imul__(self, n):
self.data *= n
return self
+ def __copy__(self):
+ inst = self.__class__.__new__(self.__class__)
+ inst.__dict__.update(self.__dict__)
+ # Create a copy and avoid triggering descriptors
+ inst.__dict__["data"] = self.__dict__["data"][:]
+ return inst
def append(self, item): self.data.append(item)
def insert(self, i, item): self.data.insert(i, item)
def pop(self, i=-1): return self.data.pop(i)
@@ -1200,9 +1214,8 @@ class UserString(_collections_abc.Sequence):
__rmul__ = __mul__
def __mod__(self, args):
return self.__class__(self.data % args)
- def __rmod__(self, format):
- return self.__class__(format % args)
-
+ def __rmod__(self, template):
+ return self.__class__(str(template) % self)
# the following methods are defined in alphabetical order:
def capitalize(self): return self.__class__(self.data.capitalize())
def casefold(self):
diff --git a/Lib/compileall.py b/Lib/compileall.py
index aa65c6b904e..49306d9dabb 100644
--- a/Lib/compileall.py
+++ b/Lib/compileall.py
@@ -67,20 +67,20 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None,
invalidation_mode: how the up-to-dateness of the pyc will be checked
"""
ProcessPoolExecutor = None
- if workers is not None:
- if workers < 0:
- raise ValueError('workers must be greater or equal to 0')
- elif workers != 1:
- try:
- # Only import when needed, as low resource platforms may
- # fail to import it
- from concurrent.futures import ProcessPoolExecutor
- except ImportError:
- workers = 1
+ if workers < 0:
+ raise ValueError('workers must be greater or equal to 0')
+ if workers != 1:
+ try:
+ # Only import when needed, as low resource platforms may
+ # fail to import it
+ from concurrent.futures import ProcessPoolExecutor
+ except ImportError:
+ workers = 1
files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
ddir=ddir)
success = True
- if workers is not None and workers != 1 and ProcessPoolExecutor is not None:
+ if workers != 1 and ProcessPoolExecutor is not None:
+ # If workers == 0, let ProcessPoolExecutor choose
workers = workers or None
with ProcessPoolExecutor(max_workers=workers) as executor:
results = executor.map(partial(compile_file,
@@ -290,9 +290,6 @@ def main():
print("Error reading file list {}".format(args.flist))
return False
- if args.workers is not None:
- args.workers = args.workers or None
-
if args.invalidation_mode:
ivl_mode = args.invalidation_mode.replace('-', '_').upper()
invalidation_mode = py_compile.PycInvalidationMode[ivl_mode]
diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py
index 8f155f0ea82..6001e3bdb81 100644
--- a/Lib/concurrent/futures/_base.py
+++ b/Lib/concurrent/futures/_base.py
@@ -404,7 +404,10 @@ class Future(object):
if self._state not in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]:
self._done_callbacks.append(fn)
return
- fn(self)
+ try:
+ fn(self)
+ except Exception:
+ LOGGER.exception('exception calling callback for %r', self)
def result(self, timeout=None):
"""Return the result of the call that the future represents.
diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py
index 2af31a106dd..ad6b4c20b56 100644
--- a/Lib/concurrent/futures/thread.py
+++ b/Lib/concurrent/futures/thread.py
@@ -80,7 +80,14 @@ def _worker(executor_reference, work_queue, initializer, initargs):
work_item.run()
# Delete references to object. See issue16284
del work_item
+
+ # attempt to increment idle count
+ executor = executor_reference()
+ if executor is not None:
+ executor._idle_semaphore.release()
+ del executor
continue
+
executor = executor_reference()
# Exit if:
# - The interpreter is shutting down OR
@@ -133,6 +140,7 @@ class ThreadPoolExecutor(_base.Executor):
self._max_workers = max_workers
self._work_queue = queue.SimpleQueue()
+ self._idle_semaphore = threading.Semaphore(0)
self._threads = set()
self._broken = False
self._shutdown = False
@@ -178,12 +186,15 @@ class ThreadPoolExecutor(_base.Executor):
submit.__doc__ = _base.Executor.submit.__doc__
def _adjust_thread_count(self):
+ # if idle threads are available, don't spin new threads
+ if self._idle_semaphore.acquire(timeout=0):
+ return
+
# When the executor gets lost, the weakref callback will wake up
# the worker threads.
def weakref_cb(_, q=self._work_queue):
q.put(None)
- # TODO(bquinlan): Should avoid creating new threads if there are more
- # idle threads than items in the work queue.
+
num_threads = len(self._threads)
if num_threads < self._max_workers:
thread_name = '%s_%d' % (self._thread_name_prefix or self,
diff --git a/Lib/ctypes/test/test_arrays.py b/Lib/ctypes/test/test_arrays.py
index 0fc5d7ebf84..87ecbf04e7e 100644
--- a/Lib/ctypes/test/test_arrays.py
+++ b/Lib/ctypes/test/test_arrays.py
@@ -69,6 +69,17 @@ class ArrayTestCase(unittest.TestCase):
from operator import delitem
self.assertRaises(TypeError, delitem, ca, 0)
+ def test_step_overflow(self):
+ a = (c_int * 5)()
+ a[3::sys.maxsize] = (1,)
+ self.assertListEqual(a[3::sys.maxsize], [1])
+ a = (c_char * 5)()
+ a[3::sys.maxsize] = b"A"
+ self.assertEqual(a[3::sys.maxsize], b"A")
+ a = (c_wchar * 5)()
+ a[3::sys.maxsize] = u"X"
+ self.assertEqual(a[3::sys.maxsize], u"X")
+
def test_numeric_arrays(self):
alen = 5
diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py
index c3b9602461f..2d7cdf063f0 100644
--- a/Lib/distutils/command/build_ext.py
+++ b/Lib/distutils/command/build_ext.py
@@ -714,20 +714,32 @@ class build_ext(Command):
# don't extend ext.libraries, it may be shared with other
# extensions, it is a reference to the original list
return ext.libraries + [pythonlib]
- # On Android only the main executable and LD_PRELOADs are considered
- # to be RTLD_GLOBAL, all the dependencies of the main executable
- # remain RTLD_LOCAL and so the shared libraries must be linked with
- # libpython when python is built with a shared python library (issue
- # bpo-21536).
else:
+ # On Android only the main executable and LD_PRELOADs are considered
+ # to be RTLD_GLOBAL, all the dependencies of the main executable
+ # remain RTLD_LOCAL and so the shared libraries must be linked with
+ # libpython when python is built with a shared python library (issue
+ # bpo-21536).
+ # On Cygwin (and if required, other POSIX-like platforms based on
+ # Windows like MinGW) it is simply necessary that all symbols in
+ # shared libraries are resolved at link time.
from distutils.sysconfig import get_config_var
+ link_libpython = False
if get_config_var('Py_ENABLE_SHARED'):
- # Either a native build on an Android device or the
- # cross-compilation of Python.
- if (hasattr(sys, 'getandroidapilevel') or
- ('_PYTHON_HOST_PLATFORM' in os.environ and
- get_config_var('ANDROID_API_LEVEL') != 0)):
- ldversion = get_config_var('LDVERSION')
- return ext.libraries + ['python' + ldversion]
+ # A native build on an Android device or on Cygwin
+ if hasattr(sys, 'getandroidapilevel'):
+ link_libpython = True
+ elif sys.platform == 'cygwin':
+ link_libpython = True
+ elif '_PYTHON_HOST_PLATFORM' in os.environ:
+ # We are cross-compiling for one of the relevant platforms
+ if get_config_var('ANDROID_API_LEVEL') != 0:
+ link_libpython = True
+ elif get_config_var('MACHDEP') == 'cygwin':
+ link_libpython = True
+
+ if link_libpython:
+ ldversion = get_config_var('LDVERSION')
+ return ext.libraries + ['python' + ldversion]
return ext.libraries
diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py
index 922daa2560f..649f1539fa0 100644
--- a/Lib/email/_header_value_parser.py
+++ b/Lib/email/_header_value_parser.py
@@ -68,6 +68,7 @@ XXX: provide complete list of token types.
"""
import re
+import sys
import urllib # For urllib.parse.unquote
from string import hexdigits
from operator import itemgetter
@@ -2590,7 +2591,7 @@ def _refold_parse_tree(parse_tree, *, policy):
"""
# max_line_length 0/None means no limit, ie: infinitely long.
- maxlen = policy.max_line_length or float("+inf")
+ maxlen = policy.max_line_length or sys.maxsize
encoding = 'utf-8' if policy.utf8 else 'us-ascii'
lines = ['']
last_ew = None
@@ -2625,7 +2626,7 @@ def _refold_parse_tree(parse_tree, *, policy):
want_encoding = False
last_ew = None
if part.syntactic_break:
- encoded_part = part.fold(policy=policy)[:-1] # strip nl
+ encoded_part = part.fold(policy=policy)[:-len(policy.linesep)]
if policy.linesep not in encoded_part:
# It fits on a single line
if len(encoded_part) > maxlen - len(lines[-1]):
@@ -2723,16 +2724,19 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset):
lines.append(' ')
# XXX We'll get an infinite loop here if maxlen is <= 7
continue
- first_part = to_encode[:text_space]
- ew = _ew.encode(first_part, charset=encode_as)
- excess = len(ew) - remaining_space
- if excess > 0:
- # encode always chooses the shortest encoding, so this
- # is guaranteed to fit at this point.
- first_part = first_part[:-excess]
- ew = _ew.encode(first_part)
- lines[-1] += ew
- to_encode = to_encode[len(first_part):]
+
+ to_encode_word = to_encode[:text_space]
+ encoded_word = _ew.encode(to_encode_word, charset=encode_as)
+ excess = len(encoded_word) - remaining_space
+ while excess > 0:
+ # Since the chunk to encode is guaranteed to fit into less than 100 characters,
+ # shrinking it by one at a time shouldn't take long.
+ to_encode_word = to_encode_word[:-1]
+ encoded_word = _ew.encode(to_encode_word, charset=encode_as)
+ excess = len(encoded_word) - remaining_space
+ lines[-1] += encoded_word
+ to_encode = to_encode[len(to_encode_word):]
+
if to_encode:
lines.append(' ')
new_last_ew = len(lines[-1])
diff --git a/Lib/email/header.py b/Lib/email/header.py
index 7b30a039da1..4ab0032bc66 100644
--- a/Lib/email/header.py
+++ b/Lib/email/header.py
@@ -431,7 +431,7 @@ class _ValueFormatter:
if end_of_line != (' ', ''):
self._current_line.push(*end_of_line)
if len(self._current_line) > 0:
- if self._current_line.is_onlyws():
+ if self._current_line.is_onlyws() and self._lines:
self._lines[-1] += str(self._current_line)
else:
self._lines.append(str(self._current_line))
diff --git a/Lib/email/policy.py b/Lib/email/policy.py
index 5131311ac5e..611deb50bb5 100644
--- a/Lib/email/policy.py
+++ b/Lib/email/policy.py
@@ -3,6 +3,7 @@ code that adds all the email6 features.
"""
import re
+import sys
from email._policybase import Policy, Compat32, compat32, _extend_docstrings
from email.utils import _has_surrogates
from email.headerregistry import HeaderRegistry as HeaderRegistry
@@ -203,7 +204,7 @@ class EmailPolicy(Policy):
def _fold(self, name, value, refold_binary=False):
if hasattr(value, 'name'):
return value.fold(policy=self)
- maxlen = self.max_line_length if self.max_line_length else float('inf')
+ maxlen = self.max_line_length if self.max_line_length else sys.maxsize
lines = value.splitlines()
refold = (self.refold_source == 'all' or
self.refold_source == 'long' and
diff --git a/Lib/fileinput.py b/Lib/fileinput.py
index 0764aa5e4d2..d868e74cd5e 100644
--- a/Lib/fileinput.py
+++ b/Lib/fileinput.py
@@ -80,8 +80,7 @@ __all__ = ["input", "close", "nextfile", "filename", "lineno", "filelineno",
_state = None
-def input(files=None, inplace=False, backup="", bufsize=0,
- mode="r", openhook=None):
+def input(files=None, inplace=False, backup="", *, mode="r", openhook=None):
"""Return an instance of the FileInput class, which can be iterated.
The parameters are passed to the constructor of the FileInput class.
@@ -91,7 +90,7 @@ def input(files=None, inplace=False, backup="", bufsize=0,
global _state
if _state and _state._file:
raise RuntimeError("input() already active")
- _state = FileInput(files, inplace, backup, bufsize, mode, openhook)
+ _state = FileInput(files, inplace, backup, mode=mode, openhook=openhook)
return _state
def close():
@@ -173,7 +172,7 @@ def isstdin():
return _state.isstdin()
class FileInput:
- """FileInput([files[, inplace[, backup[, bufsize, [, mode[, openhook]]]]]])
+ """FileInput([files[, inplace[, backup]]], *, mode=None, openhook=None)
Class FileInput is the implementation of the module; its methods
filename(), lineno(), fileline(), isfirstline(), isstdin(), fileno(),
@@ -185,7 +184,7 @@ class FileInput:
sequential order; random access and readline() cannot be mixed.
"""
- def __init__(self, files=None, inplace=False, backup="", bufsize=0,
+ def __init__(self, files=None, inplace=False, backup="", *,
mode="r", openhook=None):
if isinstance(files, str):
files = (files,)
@@ -201,10 +200,6 @@ class FileInput:
self._files = files
self._inplace = inplace
self._backup = backup
- if bufsize:
- import warnings
- warnings.warn('bufsize is deprecated and ignored',
- DeprecationWarning, stacklevel=2)
self._savestdout = None
self._output = None
self._filename = None
diff --git a/Lib/functools.py b/Lib/functools.py
index 28d9f6f75fd..c863341eec5 100644
--- a/Lib/functools.py
+++ b/Lib/functools.py
@@ -861,9 +861,11 @@ def singledispatch(func):
# only import typing if annotation parsing is necessary
from typing import get_type_hints
argname, cls = next(iter(get_type_hints(func).items()))
- assert isinstance(cls, type), (
- f"Invalid annotation for {argname!r}. {cls!r} is not a class."
- )
+ if not isinstance(cls, type):
+ raise TypeError(
+ f"Invalid annotation for {argname!r}. "
+ f"{cls!r} is not a class."
+ )
registry[cls] = func
if cache_token is None and hasattr(cls, '__abstractmethods__'):
cache_token = get_cache_token()
diff --git a/Lib/gzip.py b/Lib/gzip.py
index 7c861874198..2968f475efa 100644
--- a/Lib/gzip.py
+++ b/Lib/gzip.py
@@ -11,7 +11,7 @@ import builtins
import io
import _compression
-__all__ = ["GzipFile", "open", "compress", "decompress"]
+__all__ = ["BadGzipFile", "GzipFile", "open", "compress", "decompress"]
FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16
@@ -112,6 +112,11 @@ class _PaddedFile:
def seekable(self):
return True # Allows fast-forwarding even in unseekable streams
+
+class BadGzipFile(OSError):
+ """Exception raised in some cases for invalid gzip files."""
+
+
class GzipFile(_compression.BaseStream):
"""The GzipFile class simulates most of the methods of a file object with
the exception of the truncate() method.
@@ -413,12 +418,12 @@ class _GzipReader(_compression.DecompressReader):
return False
if magic != b'\037\213':
- raise OSError('Not a gzipped file (%r)' % magic)
+ raise BadGzipFile('Not a gzipped file (%r)' % magic)
(method, flag,
self._last_mtime) = struct.unpack("
- IDLE — Python 3.8.0a3 documentation
+ IDLE — Python 3.8.0a4 documentation
@@ -19,7 +19,7 @@
@@ -72,7 +72,7 @@
@@ -130,7 +130,7 @@ window. They currently have the same top menu but a different
default title and context menu.
On macOS, there is one application menu. It dynamically changes according
to the window currently selected. It has an IDLE menu, and some entries
-described below are moved around to conform to Apple guidlines.
+described below are moved around to conform to Apple guidelines.
@@ -659,6 +659,8 @@ will then be attached to that window for input and output.
If sys is reset by user code, such as with importlib.reload(sys),
IDLE’s changes are lost and input from the keyboard and output to the screen
will not work correctly.
+
When user code raises SystemExit either directly or by calling sys.exit, IDLE
+returns to a Shell prompt instead of exiting.