Merge branch 'master' into feature/34632-importlib-metadata

.azure-pipelines/ci.yml

@@ -59,7 +59,7 @@ jobs:
   variables:
     testRunTitle: '$(build.sourceBranchName)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.0j
+    openssl_version: 1.1.1b
 
   steps:
   - template: ./posix-steps.yml
@@ -116,7 +116,7 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.0j
+    openssl_version: 1.1.1b
 
   steps:
   - template: ./posix-steps.yml

.gitignore

@@ -36,6 +36,7 @@ Lib/test/data/*
 Makefile
 Makefile.pre
 Misc/python.pc
+Misc/python-embed.pc
 Misc/python-config.sh
 Modules/Setup
 Modules/Setup.config
@@ -70,6 +71,7 @@ PCbuild/*.VC.opendb
 PCbuild/.vs/
 PCbuild/amd64/
 PCbuild/arm32/
+PCbuild/arm64/
 PCbuild/obj/
 PCbuild/win32/
 .purify

.travis.yml

@@ -32,7 +32,8 @@ matrix:
   allow_failures:
     - env: OPTIONAL=true
   include:
-    - os: linux
+    - name: "CPython tests"
+      os: linux
       language: c
       compiler: clang
       # gcc also works, but to keep the # of concurrent builds down, we use one C
@@ -43,7 +44,8 @@ matrix:
         apt:
           packages:
             - xvfb
-    - os: linux
+    - name: "Documentation build"
+      os: linux
       language: python
       # Build the docs against a stable version of Python so code bugs don't hold up doc-related PRs.
       python: 3.6
@@ -56,7 +58,8 @@ matrix:
         - python -m pip install sphinx==1.8.2 blurb python-docs-theme
       script:
         - make check suspicious html SPHINXOPTS="-q -W -j4"
-    - os: linux
+    - name: "Documentation tests"
+      os: linux
       language: c
       compiler: clang
       env: TESTING=doctest
@@ -70,7 +73,8 @@ matrix:
         - make -C Doc/ PYTHON=../python venv
       script:
         xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W -j4" doctest
-    - os: osx
+    - name: "Mac OS X tests"
+      os: osx
       language: c
       compiler: clang
       # Testing under macOS is optional until testing stability has been demonstrated.
@@ -79,7 +83,31 @@ matrix:
         # Python 3 is needed for Argument Clinic and multissl
         - HOMEBREW_NO_AUTO_UPDATE=1 brew install xz python3
         - export PATH=$(brew --prefix)/bin:$(brew --prefix)/sbin:$PATH
-    - os: linux
+    - name: "Test code coverage (Python)"
+      os: linux
+      language: c
+      compiler: gcc
+      env: OPTIONAL=true
+      addons:
+        apt:
+          packages:
+            - xvfb
+      before_script:
+        - ./configure
+        - make -j4
+        # Need a venv that can parse covered code.
+        - ./python -m venv venv
+        - ./venv/bin/python -m pip install -U coverage
+        - ./venv/bin/python -m test.pythoninfo
+      script:
+        # Skip tests that re-run the entire test suite.
+        - xvfb-run ./venv/bin/python -m coverage run --branch --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures || true
+      after_script:  # Probably should be after_success once test suite updated to run under coverage.py.
+        # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files.
+        - source ./venv/bin/activate
+        - bash <(curl -s https://codecov.io/bash)
+    - name: "Test code coverage (C)"
+      os: linux
       language: c
       compiler: gcc
       env: OPTIONAL=true
@@ -90,18 +118,10 @@ matrix:
             - xvfb
       before_script:
         - ./configure
-        - make coverage -s -j4
-        # Need a venv that can parse covered code.
-        - ./python -m venv venv
-        - ./venv/bin/python -m pip install -U coverage
-        - ./venv/bin/python -m test.pythoninfo
       script:
-        # Skip tests that re-run the entire test suite.
+        - xvfb-run make -j4 coverage-report
-        - xvfb-run ./venv/bin/python -m coverage run --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures
       after_script:  # Probably should be after_success once test suite updated to run under coverage.py.
-        # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files.
+        - make pythoninfo
-        - source ./venv/bin/activate
-        - make coverage-lcov
         - bash <(curl -s https://codecov.io/bash)
 
 

Doc/Makefile

@@ -132,6 +132,7 @@ clean:
 
 venv:
 	$(PYTHON) -m venv $(VENVDIR)
+	$(VENVDIR)/bin/python3 -m pip install -U pip setuptools
 	$(VENVDIR)/bin/python3 -m pip install -U Sphinx blurb python-docs-theme
 	@echo "The venv has been created in the $(VENVDIR) directory"
 

Doc/c-api/abstract.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _abstract:
 

Doc/c-api/allocation.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _allocating-objects:
 

Doc/c-api/apiabiversion.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _apiabiversion:
 

Doc/c-api/arg.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _arg-parsing:
 

Doc/c-api/bool.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _boolobjects:
 

Doc/c-api/buffer.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. index::
    single: buffer protocol

Doc/c-api/bytearray.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _bytearrayobjects:
 

Doc/c-api/bytes.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _bytesobjects:
 

Doc/c-api/capsule.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _capsules:
 

Doc/c-api/cell.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _cell-objects:
 

Doc/c-api/code.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _codeobjects:
 
@@ -40,6 +40,7 @@ bound into a function.
    :c:func:`PyCode_New` directly can bind you to a precise Python
    version since the definition of the bytecode changes often.
 
+   .. audit-event:: code.__new__ "code filename name argcount kwonlyargcount nlocals stacksize flags"
 
 .. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
 

Doc/c-api/complex.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _complexobjects:
 

Doc/c-api/concrete.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 
 .. _concrete:

Doc/c-api/contextvars.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _contextvarsobjects:
 

Doc/c-api/conversion.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _string-conversion:
 

Doc/c-api/coro.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _coro-objects:
 

Doc/c-api/datetime.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _datetimeobjects:
 
@@ -106,6 +106,12 @@ Macros to create objects:
    .. versionadded:: 3.6
 
 
+.. c:function:: PyObject* PyTime_FromTime(int hour, int minute, int second, int usecond)
+
+   Return a :class:`datetime.time` object with the specified hour, minute, second and
+   microsecond.
+
+
 .. c:function:: PyObject* PyTime_FromTimeAndFold(int hour, int minute, int second, int usecond, int fold)
 
    Return a :class:`datetime.time` object with the specified hour, minute, second,
@@ -114,12 +120,6 @@ Macros to create objects:
    .. versionadded:: 3.6
 
 
-.. c:function:: PyObject* PyTime_FromTime(int hour, int minute, int second, int usecond)
-
-   Return a :class:`datetime.time` object with the specified hour, minute, second and
-   microsecond.
-
-
 .. c:function:: PyObject* PyDelta_FromDSU(int days, int seconds, int useconds)
 
    Return a :class:`datetime.timedelta` object representing the given number

Doc/c-api/descriptor.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _descriptor-objects:
 

Doc/c-api/dict.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _dictobjects:
 

Doc/c-api/exceptions.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 
 .. _exceptionhandling:
@@ -72,6 +72,9 @@ Printing and clearing
 
 .. c:function:: void PyErr_WriteUnraisable(PyObject *obj)
 
+   Call :func:`sys.unraisablehook` using the current exception and *obj*
+   argument.
+
    This utility function prints a warning message to ``sys.stderr`` when an
    exception has been set but it is impossible for the interpreter to actually
    raise the exception.  It is used, for example, when an exception occurs in an
@@ -81,6 +84,8 @@ Printing and clearing
    in which the unraisable exception occurred. If possible,
    the repr of *obj* will be printed in the warning message.
 
+   An exception must be set when calling this function.
+
 
 Raising exceptions
 ==================
@@ -514,13 +519,13 @@ Signal Handling
       single: SIGINT
       single: KeyboardInterrupt (built-in exception)
 
-   This function simulates the effect of a :const:`SIGINT` signal arriving --- the
+   Simulate the effect of a :const:`SIGINT` signal arriving. The next time
-   next time :c:func:`PyErr_CheckSignals` is called,  :exc:`KeyboardInterrupt` will
+   :c:func:`PyErr_CheckSignals` is called,  the Python signal handler for
-   be raised.  It may be called without holding the interpreter lock.
+   :const:`SIGINT` will be called.
-
-   .. % XXX This was described as obsolete, but is used in
-   .. % _thread.interrupt_main() (used from IDLE), so it's still needed.
 
+   If :const:`SIGINT` isn't handled by Python (it was set to
+   :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does
+   nothing.
 
 .. c:function:: int PySignal_SetWakeupFd(int fd)
 

Doc/c-api/file.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _fileobjects:
 
@@ -60,6 +60,32 @@ the :mod:`io` APIs instead.
    raised if the end of the file is reached immediately.
 
 
+.. c:function:: int PyFile_SetOpenCodeHook(Py_OpenCodeHookFunction handler)
+
+   Overrides the normal behavior of :func:`io.open_code` to pass its parameter
+   through the provided handler.
+
+   The handler is a function of type :c:type:`PyObject *(\*)(PyObject *path,
+   void *userData)`, where *path* is guaranteed to be :c:type:`PyUnicodeObject`.
+
+   The *userData* pointer is passed into the hook function. Since hook
+   functions may be called from different runtimes, this pointer should not
+   refer directly to Python state.
+
+   As this hook is intentionally used during import, avoid importing new modules
+   during its execution unless they are known to be frozen or available in
+   ``sys.modules``.
+
+   Once a hook has been set, it cannot be removed or replaced, and later calls to
+   :c:func:`PyFile_SetOpenCodeHook` will fail. On failure, the function returns
+   -1 and sets an exception if the interpreter has been initialized.
+
+   This function is safe to call before :c:func:`Py_Initialize`.
+
+   .. versionadded:: 3.8
+
+
+
 .. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags)
 
    .. index:: single: Py_PRINT_RAW

Doc/c-api/float.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _floatobjects:
 

Doc/c-api/function.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _function-objects:
 

Doc/c-api/gcsupport.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _supporting-cycle-detection:
 

Doc/c-api/gen.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _gen-objects:
 

Doc/c-api/import.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _importing:
 

Doc/c-api/init.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 
 .. _initialization:
@@ -877,13 +877,6 @@ code, or when embedding the Python interpreter:
    and is not released.
 
 
-.. c:function:: void PyEval_ReInitThreads()
-
-   This function is called from :c:func:`PyOS_AfterFork_Child` to ensure
-   that newly created child processes don't hold locks referring to threads
-   which are not running in the child process.
-
-
 The following functions use thread-local storage, and are not compatible
 with sub-interpreters:
 

Doc/c-api/intro.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 
 .. _api-intro:
@@ -156,7 +156,7 @@ complete listing.
 .. c:macro:: Py_UNUSED(arg)
 
    Use this for unused arguments in a function definition to silence compiler
-   warnings, e.g. ``PyObject* func(PyObject *Py_UNUSED(ignored))``.
+   warnings. Example: ``int func(int a, int Py_UNUSED(b)) { return a; }``.
 
    .. versionadded:: 3.4
 

Doc/c-api/iter.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _iterator:
 

Doc/c-api/iterator.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _iterator-objects:
 

Doc/c-api/list.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _listobjects:
 

Doc/c-api/long.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _longobjects:
 

Doc/c-api/mapping.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _mapping:
 

Doc/c-api/marshal.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _marshalling-utils:
 

Doc/c-api/memory.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 
 .. _memory:

Doc/c-api/memoryview.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _memoryview-objects:
 

Doc/c-api/method.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _instancemethod-objects:
 

Doc/c-api/module.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _moduleobjects:
 

Doc/c-api/none.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _noneobject:
 

Doc/c-api/number.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _number:
 

Doc/c-api/objbuffer.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 Old Buffer Protocol
 -------------------

Doc/c-api/object.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _object:
 

Doc/c-api/objimpl.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _newtypes:
 

Doc/c-api/refcounting.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 
 .. _countingrefs:

Doc/c-api/reflection.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _reflection:
 

Doc/c-api/sequence.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _sequence:
 

Doc/c-api/set.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _setobjects:
 

Doc/c-api/slice.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _slice-objects:
 

Doc/c-api/stable.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _stable:
 

Doc/c-api/structures.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _common-structs:
 

Doc/c-api/sys.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _os:
 
@@ -289,6 +289,56 @@ accessible to C code.  They all work with the current interpreter thread's
    .. versionadded:: 3.2
 
 
+.. c:function:: int PySys_Audit(const char *event, const char *format, ...)
+
+   .. index:: single: audit events
+
+   Raises an auditing event with any active hooks. Returns zero for success
+   and non-zero with an exception set on failure.
+
+   If any hooks have been added, *format* and other arguments will be used
+   to construct a tuple to pass. Apart from ``N``, the same format characters
+   as used in :c:func:`Py_BuildValue` are available. If the built value is not
+   a tuple, it will be added into a single-element tuple. (The ``N`` format
+   option consumes a reference, but since there is no way to know whether
+   arguments to this function will be consumed, using it may cause reference
+   leaks.)
+
+   :func:`sys.audit` performs the same function from Python code.
+
+   .. versionadded:: 3.8
+
+
+.. c:function:: int PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData)
+
+   .. index:: single: audit events
+
+   Adds to the collection of active auditing hooks. Returns zero for success
+   and non-zero on failure. If the runtime has been initialized, also sets an
+   error on failure. Hooks added through this API are called for all
+   interpreters created by the runtime.
+
+   This function is safe to call before :c:func:`Py_Initialize`. When called
+   after runtime initialization, existing audit hooks are notified and may
+   silently abort the operation by raising an error subclassed from
+   :class:`Exception` (other errors will not be silenced).
+
+   The hook function is of type :c:type:`int (*)(const char *event, PyObject
+   *args, void *userData)`, where *args* is guaranteed to be a
+   :c:type:`PyTupleObject`. The hook function is always called with the GIL
+   held by the Python interpreter that raised the event.
+
+   The *userData* pointer is passed into the hook function. Since hook
+   functions may be called from different runtimes, this pointer should not
+   refer directly to Python state.
+
+   See :pep:`578` for a detailed description of auditing. Functions in the
+   runtime and standard library that raise events include the details in each
+   function's documentation.
+
+   .. versionadded:: 3.8
+
+
 .. _processcontrol:
 
 Process Control

Doc/c-api/tuple.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _tupleobjects:
 

Doc/c-api/type.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _typeobjects:
 
@@ -95,18 +95,6 @@ Type Objects
    from a type's base class.  Return ``0`` on success, or return ``-1`` and sets an
    exception on error.
 
-.. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec)
-
-   Creates and returns a heap type object from the *spec* passed to the function.
-
-.. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
-
-   Creates and returns a heap type object from the *spec*. In addition to that,
-   the created heap type contains all types contained by the *bases* tuple as base
-   types. This allows the caller to reference other heap types as base types.
-
-   .. versionadded:: 3.3
-
 .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot)
 
    Return the function pointer stored in the given slot. If the
@@ -115,4 +103,107 @@ Type Objects
    Callers will typically cast the result pointer into the appropriate
    function type.
 
+   See :c:member:`PyType_Slot.slot` for possible values of the *slot* argument.
+
+   An exception is raised if *type* is not a heap type.
+
    .. versionadded:: 3.4
+
+
+Creating Heap-Allocated Types
+.............................
+
+The following functions and structs are used to create
+:ref:`heap types <heap-types>`.
+
+.. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
+
+   Creates and returns a heap type object from the *spec*.
+
+   If *bases* is a tuple, the created heap type contains all types contained
+   in it as base types.
+
+   If *bases* is *NULL*, the *Py_tp_base* slot is used instead.
+   If that also is *NULL*, the new type derives from :class:`object`.
+
+   This function calls :c:func:`PyType_Ready` on the new type.
+
+   .. versionadded:: 3.3
+
+.. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec)
+
+   Equivalent to ``PyType_FromSpecWithBases(spec, NULL)``.
+
+.. c:type:: PyType_Spec
+
+   Structure defining a type's behavior.
+
+   .. c:member:: const char* PyType_Spec.name
+
+      Name of the type, used to set :c:member:`PyTypeObject.tp_name`.
+
+   .. c:member:: const char* PyType_Spec.doc
+
+      Type docstring, used to set :c:member:`PyTypeObject.tp_doc`.
+
+   .. c:member:: int PyType_Spec.basicsize
+   .. c:member:: int PyType_Spec.itemsize
+
+      Size of the instance in bytes, used to set
+      :c:member:`PyTypeObject.tp_basicsize` and
+      :c:member:`PyTypeObject.tp_itemsize`.
+
+   .. c:member:: int PyType_Spec.flags
+
+      Type flags, used to set :c:member:`PyTypeObject.tp_flags`.
+
+      If the ``Py_TPFLAGS_HEAPTYPE`` flag is not set,
+      :c:func:`PyType_FromSpecWithBases` sets it automatically.
+
+   .. c:member:: PyType_Slot *PyType_Spec.slots
+
+      Array of :c:type:`PyType_Slot` structures.
+      Terminated by the special slot value ``{0, NULL}``.
+
+.. c:type:: PyType_Slot
+
+   Structure defining optional functionality of a type, containing a slot ID
+   and a value pointer.
+
+   .. c:member:: int PyType_Slot.slot
+
+      A slot ID.
+
+      Slot IDs are named like the field names of the structures
+      :c:type:`PyTypeObject`, :c:type:`PyNumberMethods`,
+      :c:type:`PySequenceMethods`, :c:type:`PyMappingMethods` and
+      :c:type:`PyAsyncMethods` with an added ``Py_`` prefix.
+      For example, use:
+
+      * ``Py_tp_dealloc`` to set :c:member:`PyTypeObject.tp_dealloc`
+      * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add`
+      * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length`
+
+      The following fields cannot be set using *PyType_Spec* and *PyType_Slot*:
+
+      * :c:member:`~PyTypeObject.tp_dict`
+      * :c:member:`~PyTypeObject.tp_mro`
+      * :c:member:`~PyTypeObject.tp_cache`
+      * :c:member:`~PyTypeObject.tp_subclasses`
+      * :c:member:`~PyTypeObject.tp_weaklist`
+      * :c:member:`~PyTypeObject.tp_print`
+      * :c:member:`~PyTypeObject.tp_weaklistoffset`
+      * :c:member:`~PyTypeObject.tp_dictoffset`
+      * :c:member:`~PyBufferProcs.bf_getbuffer`
+      * :c:member:`~PyBufferProcs.bf_releasebuffer`
+
+      Setting :c:data:`Py_tp_bases` may be problematic on some platforms.
+      To avoid issues, use the *bases* argument of
+      :py:func:`PyType_FromSpecWithBases` instead.
+
+   .. c:member:: void *PyType_Slot.pfunc
+
+      The desired value of the slot. In most cases, this is a pointer
+      to a function.
+
+      May not be *NULL*.

Doc/c-api/typeobj.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _type-structs:
 
@@ -1822,17 +1822,36 @@ objects on the thread which called tp_dealloc will not violate any assumptions
 of the library.
 
 
+.. _heap-types:
+
 Heap Types
 ----------
 
-In addition to defining Python types statically, you can define them
+Traditionally, types defined in C code are *static*, that is,
-dynamically (i.e. to the heap) using  :c:func:`PyType_FromSpec` and
+a static :c:type:`PyTypeObject` structure is defined directly in code
+and initialized using :c:func:`PyType_Ready`.
+
+This results in types that are limited relative to types defined in Python:
+
+* Static types are limited to one base, i.e. they cannot use multiple
+  inheritance.
+* Static type objects (but not necessarily their instances) are immutable.
+  It is not possible to add or modify the type object's attributes from Python.
+* Static type objects are shared across
+  :ref:`sub-interpreters <sub-interpreter-support>`, so they should not
+  include any subinterpreter-specific state.
+
+Also, since *PyTypeObject* is not part of the :ref:`stable ABI <stable>`,
+any extension modules using static types must be compiled for a specific
+Python minor version.
+
+An alternative to static types is *heap-allocated types*, or *heap types*
+for short, which correspond closely to classes created by Python's
+``class`` statement.
+
+This is done by filling a :c:type:`PyType_Spec` structure and calling
 :c:func:`PyType_FromSpecWithBases`.
 
-.. XXX Explain how to use PyType_FromSpec().
-
-.. XXX Document PyType_Spec.
-
 
 .. _number-structs:
 

Doc/c-api/unicode.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _unicodeobjects:
 

Doc/c-api/utilities.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _utilities:
 

Doc/c-api/veryhigh.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 
 .. _veryhigh:

Doc/c-api/weakref.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _weakrefobjects:
 

Doc/conf.py

@@ -23,6 +23,9 @@ try:
 except ImportError:
     _tkinter = None
 '''
+
+manpages_url = 'https://manpages.debian.org/{path}'
+
 # General substitutions.
 project = 'Python'
 copyright = '2001-%s, Python Software Foundation' % time.strftime('%Y')

Doc/data/refcounts.dat

@@ -413,6 +413,16 @@ PyDateTime_FromDateAndTime:int:minute::
 PyDateTime_FromDateAndTime:int:second::
 PyDateTime_FromDateAndTime:int:usecond::
 
+PyDateTime_FromDateAndTimeAndFold:PyObject*::+1:
+PyDateTime_FromDateAndTimeAndFold:int:year::
+PyDateTime_FromDateAndTimeAndFold:int:month::
+PyDateTime_FromDateAndTimeAndFold:int:day::
+PyDateTime_FromDateAndTimeAndFold:int:hour::
+PyDateTime_FromDateAndTimeAndFold:int:minute::
+PyDateTime_FromDateAndTimeAndFold:int:second::
+PyDateTime_FromDateAndTimeAndFold:int:usecond::
+PyDateTime_FromDateAndTimeAndFold:int:fold::
+
 PyDateTime_FromTimestamp:PyObject*::+1:
 PyDateTime_FromTimestamp:PyObject*:args:0:
 
@@ -2210,6 +2220,13 @@ PyTime_FromTime:int:minute::
 PyTime_FromTime:int:second::
 PyTime_FromTime:int:usecond::
 
+PyTime_FromTimeAndFold:PyObject*::+1:
+PyTime_FromTimeAndFold:int:hour::
+PyTime_FromTimeAndFold:int:minute::
+PyTime_FromTimeAndFold:int:second::
+PyTime_FromTimeAndFold:int:usecond::
+PyTime_FromTimeAndFold:int:fold::
+
 PyTraceMalloc_Track:int:::
 PyTraceMalloc_Track:unsigned int:domain::
 PyTraceMalloc_Track:uintptr_t:ptr::

Doc/distributing/index.rst

@@ -113,11 +113,17 @@ recommended tools`_.
 
 .. _currently recommended tools: https://packaging.python.org/guides/tool-recommendations/#packaging-tool-recommendations
 
-Reading the guide
+.. index::
-=================
+   single: Python Package Index (PyPI)
+   single: PyPI; (see Python Package Index (PyPI))
+
+.. _publishing-python-packages:
+
+Reading the Python Packaging User Guide
+=======================================
 
 The Python Packaging User Guide covers the various key steps and elements
-involved in creating a project:
+involved in creating and publishing a project:
 
 * `Project structure`_
 * `Building and packaging the project`_

Doc/distutils/_setuptools_disclaimer.rst

@@ -0,0 +1,5 @@
+.. note::
+
+   This document is being retained solely until the ``setuptools`` documentation
+   at https://setuptools.readthedocs.io/en/latest/setuptools.html
+   independently covers all of the relevant information currently included here.

Doc/distutils/apiref.rst

@@ -4,6 +4,16 @@
 API Reference
 *************
 
+.. seealso::
+
+   `New and changed setup.py arguments in setuptools <setuptools-setup-py>`_
+      The ``setuptools`` project adds new capabilities to the ``setup`` function
+      and other APIs, makes the API consistent across different Python versions,
+      and is hence recommended over using ``distutils`` directly.
+
+.. _setuptools-setup-py: https://setuptools.readthedocs.io/en/latest/setuptools.html#new-and-changed-setup-keywords
+
+.. include:: ./_setuptools_disclaimer.rst
 
 :mod:`distutils.core` --- Core Distutils functionality
 ======================================================
@@ -280,7 +290,7 @@ the full reference.
    .. versionchanged:: 3.8
 
       On Unix, C extensions are no longer linked to libpython except on
-      Android.
+      Android and Cygwin.
 
 
 .. class:: Distribution

Doc/distutils/builtdist.rst

@@ -4,6 +4,8 @@
 Creating Built Distributions
 ****************************
 
+.. include:: ./_setuptools_disclaimer.rst
+
 A "built distribution" is what you're probably used to thinking of either as a
 "binary package" or an "installer" (depending on your background).  It's not
 necessarily binary, though, because it might contain only Python source code

Doc/distutils/commandref.rst

@@ -4,6 +4,8 @@
 Command Reference
 *****************
 
+.. include:: ./_setuptools_disclaimer.rst
+
 .. % \section{Building modules: the \protect\command{build} command family}
 .. % \label{build-cmds}
 .. % \subsubsection{\protect\command{build}}

Doc/distutils/configfile.rst

@@ -4,6 +4,8 @@
 Writing the Setup Configuration File
 ************************************
 
+.. include:: ./_setuptools_disclaimer.rst
+
 Often, it's not possible to write down everything needed to build a distribution
 *a priori*: you may need to get some information from the user, or from the
 user's system, in order to proceed.  As long as that information is fairly

Doc/distutils/examples.rst

@@ -4,6 +4,8 @@
 Examples
 ********
 
+.. include:: ./_setuptools_disclaimer.rst
+
 This chapter provides a number of basic examples to help get started with
 distutils.  Additional information about using distutils can be found in the
 Distutils Cookbook.

Doc/distutils/extending.rst

@@ -4,6 +4,8 @@
 Extending Distutils
 *******************
 
+.. include:: ./_setuptools_disclaimer.rst
+
 Distutils can be extended in various ways.  Most extensions take the form of new
 commands or replacements for existing commands.  New commands may be written to
 support new types of platform-specific packaging, for example, while

Doc/distutils/index.rst

@@ -12,10 +12,7 @@
    :ref:`distributing-index`
       The up to date module distribution documentations
 
-This document describes the Python Distribution Utilities ("Distutils") from
+.. include:: ./_setuptools_disclaimer.rst
-the module developer's point of view, describing how to use the Distutils to
-make Python modules and extensions easily available to a wider audience with
-very little overhead for build/release/install mechanics.
 
 .. note::
 
@@ -25,6 +22,11 @@ very little overhead for build/release/install mechanics.
    recommendations section <https://packaging.python.org/guides/tool-recommendations/>`__
    in the Python Packaging User Guide for more information.
 
+This document describes the Python Distribution Utilities ("Distutils") from
+the module developer's point of view, describing the underlying capabilities
+that ``setuptools`` builds on to allow Python developers to make Python modules
+and extensions readily available to a wider audience.
+
 .. toctree::
    :maxdepth: 2
    :numbered:
@@ -34,7 +36,6 @@ very little overhead for build/release/install mechanics.
    configfile.rst
    sourcedist.rst
    builtdist.rst
-   packageindex.rst
    examples.rst
    extending.rst
    commandref.rst

Doc/distutils/introduction.rst

@@ -4,6 +4,8 @@
 An Introduction to Distutils
 ****************************
 
+.. include:: ./_setuptools_disclaimer.rst
+
 This document covers using the Distutils to distribute your Python modules,
 concentrating on the role of developer/distributor: if you're looking for
 information on installing Python modules, you should refer to the

Doc/distutils/packageindex.rst

@@ -1,6 +1,4 @@
-.. index::
+:orphan:
-   single: Python Package Index (PyPI)
-   single: PyPI; (see Python Package Index (PyPI))
 
 .. _package-index:
 
@@ -12,6 +10,7 @@ The `Python Package Index (PyPI)`_ stores metadata describing distributions
 packaged with distutils and other publishing tools, as well the distribution
 archives themselves.
 
-Detailed instructions on using PyPI at :ref:`distributing-index`.
+References to up to date PyPI documentation can be found at
+:ref:`publishing-python-packages`.
 
 .. _Python Package Index (PyPI): https://pypi.org

Doc/distutils/setupscript.rst

@@ -4,6 +4,8 @@
 Writing the Setup Script
 ************************
 
+.. include:: ./_setuptools_disclaimer.rst
+
 The setup script is the centre of all activity in building, distributing, and
 installing modules using the Distutils.  The main purpose of the setup script is
 to describe your module distribution to the Distutils, so that the various

Doc/distutils/sourcedist.rst

@@ -4,6 +4,8 @@
 Creating a Source Distribution
 ******************************
 
+.. include:: ./_setuptools_disclaimer.rst
+
 As shown in section :ref:`distutils-simple-example`, you use the :command:`sdist` command
 to create a source distribution.  In the simplest case, ::
 

Doc/distutils/uploading.rst

@@ -4,4 +4,5 @@
 Uploading Packages to the Package Index
 ***************************************
 
-The contents of this page have moved to the section :ref:`package-index`.
+References to up to date PyPI documentation can be found at
+:ref:`publishing-python-packages`.

Doc/extending/building.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _building:
 
@@ -20,7 +20,7 @@ The initialization function has the signature:
 It returns either a fully-initialized module, or a :c:type:`PyModuleDef`
 instance. See :ref:`initializing-modules` for details.
 
-.. highlightlang:: python
+.. highlight:: python
 
 For modules with ASCII-only names, the function must be named
 ``PyInit_<modulename>``, with ``<modulename>`` replaced by the name of the
@@ -43,7 +43,7 @@ function corresponding to the filename is found.
 See the *"Multiple modules in one library"* section in :pep:`489` for details.
 
 
-.. highlightlang:: c
+.. highlight:: c
 
 Building C and C++ Extensions with distutils
 ============================================

Doc/extending/embedding.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 
 .. _embedding:

Doc/extending/extending.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 
 .. _extending-intro:

Doc/extending/newtypes.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _new-types-topics:
 

Doc/extending/newtypes_tutorial.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _defining-new-types:
 

Doc/extending/windows.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 
 .. _building-on-windows:

Doc/faq/programming.rst

@@ -792,7 +792,7 @@ Its documentation looks like this::
       invoked using the three argument form.
 
 The slash at the end of the parameter list means that all three parameters are
-positional-only. Thus, calling :func:`pow` with keyword aguments would lead to
+positional-only. Thus, calling :func:`pow` with keyword arguments would lead to
 an error::
 
    >>> pow(x=3, y=4)

Doc/faq/windows.rst

@@ -1,6 +1,6 @@
 :tocdepth: 2
 
-.. highlightlang:: none
+.. highlight:: none
 
 .. _windows-faq:
 

Doc/glossary.rst

@@ -15,10 +15,10 @@ Glossary
    ``...``
       Can refer to:
 
-      * The default Python prompt of the interactive shell when entering code for
+      * The default Python prompt of the interactive shell when entering the
-        an indented code block, when within a pair of matching left and right
+        code for an indented code block, when within a pair of matching left and
-        delimiters (parentheses, square brackets, curly braces or triple quotes),
+        right delimiters (parentheses, square brackets, curly braces or triple
-        or after specifying a decorator.
+        quotes), or after specifying a decorator.
 
       * The :const:`Ellipsis` built-in constant.
 
@@ -225,6 +225,15 @@ Glossary
       statement by defining :meth:`__enter__` and :meth:`__exit__` methods.
       See :pep:`343`.
 
+    context variable
+      A variable which can have different values depending on its context.
+      This is similar to Thread-Local Storage in which each execution
+      thread may have a different value for a variable. However, with context
+      variables, there may be several contexts in one execution thread and the
+      main usage for context variables is to keep track of variables in
+      concurrent asynchronous tasks.
+      See :mod:`contextvars`.
+
    contiguous
       .. index:: C-contiguous, Fortran contiguous
 

Doc/howto/clinic.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 **********************
 Argument Clinic How-To

Doc/howto/cporting.rst

@@ -1,4 +1,4 @@
-.. highlightlang:: c
+.. highlight:: c
 
 .. _cporting-howto:
 

Doc/howto/instrumentation.rst

@@ -332,6 +332,15 @@ Available static markers
    .. versionadded:: 3.7
 
 
+.. c:function:: audit(str event, void *tuple)
+
+   Fires when :func:`sys.audit` or :c:func:`PySys_Audit` is called.
+   ``arg0`` is the event name as C string, ``arg1`` is a :c:type:`PyObject`
+   pointer to a tuple object.
+
+   .. versionadded:: 3.8
+
+
 SystemTap Tapsets
 -----------------
 

Doc/howto/unicode.rst

@@ -135,17 +135,22 @@ used than UTF-8.)  UTF-8 uses the following rules:
 UTF-8 has several convenient properties:
 
 1. It can handle any Unicode code point.
-2. A Unicode string is turned into a sequence of bytes containing no embedded zero
+2. A Unicode string is turned into a sequence of bytes that contains embedded
-   bytes.  This avoids byte-ordering issues, and means UTF-8 strings can be
+   zero bytes only where they represent the null character (U+0000). This means
-   processed by C functions such as ``strcpy()`` and sent through protocols that
+   that UTF-8 strings can be processed by C functions such as ``strcpy()`` and sent
-   can't handle zero bytes.
+   through protocols that can't handle zero bytes for anything other than
+   end-of-string markers.
 3. A string of ASCII text is also valid UTF-8 text.
 4. UTF-8 is fairly compact; the majority of commonly used characters can be
    represented with one or two bytes.
 5. If bytes are corrupted or lost, it's possible to determine the start of the
    next UTF-8-encoded code point and resynchronize.  It's also unlikely that
    random 8-bit data will look like valid UTF-8.
-
+6. UTF-8 is a byte oriented encoding. The encoding specifies that each
+   character is represented by a specific sequence of one or more bytes. This
+   avoids the byte-ordering issues that can occur with integer and word oriented
+   encodings, like UTF-16 and UTF-32, where the sequence of bytes varies depending
+   on the hardware on which the string was encoded.
 
 
 References

Doc/includes/sqlite3/adapter_datetime.py

@@ -13,3 +13,5 @@ cur = con.cursor()
 now = datetime.datetime.now()
 cur.execute("select ?", (now,))
 print(cur.fetchone()[0])
+
+con.close()

Doc/includes/sqlite3/adapter_point_1.py

@@ -14,3 +14,5 @@ cur = con.cursor()
 p = Point(4.0, -3.2)
 cur.execute("select ?", (p,))
 print(cur.fetchone()[0])
+
+con.close()

Doc/includes/sqlite3/adapter_point_2.py

@@ -15,3 +15,5 @@ cur = con.cursor()
 p = Point(4.0, -3.2)
 cur.execute("select ?", (p,))
 print(cur.fetchone()[0])
+
+con.close()

Doc/includes/sqlite3/connect_db_1.py

@@ -1,3 +0,0 @@
-import sqlite3
-
-con = sqlite3.connect("mydb")

Doc/includes/sqlite3/connect_db_2.py

@@ -1,3 +0,0 @@
-import sqlite3
-
-con = sqlite3.connect(":memory:")

Doc/includes/sqlite3/countcursors.py

@@ -13,3 +13,5 @@ con = sqlite3.connect(":memory:", factory=CountCursorsConnection)
 cur1 = con.cursor()
 cur2 = con.cursor()
 print(con.numcursors)
+
+con.close()

Doc/includes/sqlite3/ctx_manager.py

@@ -14,3 +14,7 @@ try:
         con.execute("insert into person(firstname) values (?)", ("Joe",))
 except sqlite3.IntegrityError:
     print("couldn't add Joe twice")
+
+# Connection object used as context manager only commits or rollbacks transactions,
+# so the connection object should be closed manually
+con.close()

Doc/includes/sqlite3/execsql_fetchonerow.py

@@ -15,3 +15,5 @@ for (name_last, age) in cur:
 cur.execute(SELECT)
 for row in cur:
     print('%s is %d years old.' % (row[0], row[1]))
+
+con.close()

Doc/includes/sqlite3/execsql_printall_1.py

@@ -11,3 +11,5 @@ cur.execute("select * from people order by age")
 
 # Retrieve all rows as a sequence and print that sequence:
 print(cur.fetchall())
+
+con.close()