diff --git a/.gitattributes b/.gitattributes index 9ec17d12406..0a998a5369f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -26,6 +26,7 @@ Lib/test/decimaltestdata/*.decTest -text Lib/test/test_email/data/*.txt -text Lib/test/xmltestdata/* -text Lib/test/coding20731.py -text +Lib/test/test_importlib/data01/* -text # CRLF files *.bat text eol=crlf diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8e0647fdc1f..17d7ef4d7a2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,17 +5,24 @@ # https://git-scm.com/docs/gitignore#_pattern_format # asyncio -**/*asyncio* @1st1 +**/*asyncio* @1st1 @asvetlov # Core +**/*context* @1st1 **/*genobject* @1st1 +**/*hamt* @1st1 # Hashing **/*hashlib* @python/crypto-team **/*pyhash* @python/crypto-team -# Import (including importlib) -**/*import* @python/import-team +# Import (including importlib). +# Ignoring importlib.h so as to not get flagged on +# all pull requests that change the the emitted +# bytecode. +**/*import*.c @python/import-team +**/*import*.py @python/import-team + # SSL **/*ssl* @python/crypto-team @@ -50,4 +57,8 @@ Python/bootstrap_hash.c @python/crypto-team **/*functools* @ncoghlan @rhettinger **/*decimal* @rhettinger @skrah +**/*dataclasses* @ericvsmith + **/*idlelib* @terryjreedy + +**/*typing* @gvanrossum @ilevkivskyi diff --git a/.github/appveyor.yml b/.github/appveyor.yml index b052b28f8aa..129c119df61 100644 --- a/.github/appveyor.yml +++ b/.github/appveyor.yml @@ -1,4 +1,4 @@ -version: 3.7.0a0.{build} +version: 3.8build{build} clone_depth: 5 branches: only: @@ -6,7 +6,7 @@ branches: - /\d\.\d/ - buildbot-custom cache: - - externals -> PCbuild\* + - externals -> PCbuild build_script: - cmd: PCbuild\build.bat -e - cmd: PCbuild\win32\python.exe -m test.pythoninfo @@ -16,20 +16,3 @@ environment: HOST_PYTHON: C:\Python36\python.exe image: - Visual Studio 2017 - -# Only trigger AppVeyor if actual code or its configuration changes -only_commits: - files: - - .github/appveyor.yml - - .gitattributes - - Grammar/ - - Include/ - - Lib/ - - Modules/ - - Objects/ - - PC/ - - PCbuild/ - - Parser/ - - Programs/ - - Python/ - - Tools/ diff --git a/.gitignore b/.gitignore index 59206541ee4..05fb6cba087 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,10 @@ +# added for local development +.buildaix/ +Modules/python.exp +buildaix/ +installp/ +.gitignore + # Two-trick pony for OSX and other case insensitive file systems: # Ignore ./python binary on Unix but still look into ./Python/ directory. /python diff --git a/.travis.yml b/.travis.yml index c207bd72da2..d7387e5f983 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,19 @@ group: beta cache: - pip - ccache + - directories: + - $HOME/multissl + +env: + global: + - OPENSSL=1.1.0g + - OPENSSL_DIR="$HOME/multissl/openssl/${OPENSSL}" + - PATH="${OPENSSL_DIR}/bin:$PATH" + - CFLAGS="-I${OPENSSL_DIR}/include" + - LDFLAGS="-L${OPENSSL_DIR}/lib" + # Set rpath with env var instead of -Wl,-rpath linker flag + # OpenSSL ignores LDFLAGS when linking bin/openssl + - LD_RUN_PATH="${OPENSSL_DIR}/lib" branches: only: @@ -48,6 +61,10 @@ matrix: echo "Only docs were updated, stopping build process." exit fi + python3 Tools/ssl/multissltests.py --steps=library \ + --base-directory ${HOME}/multissl \ + --openssl ${OPENSSL} >/dev/null + openssl version ./configure make -s -j4 # Need a venv that can parse covered code. @@ -66,11 +83,32 @@ matrix: before_script: - | set -e - if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.rst$)|(^Doc)|(^Misc)' + if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then + files_changed=$(git diff --name-only $TRAVIS_COMMIT_RANGE) + else + # Pull requests are slightly complicated because merging the PR commit without + # rebasing causes it to retain its old commit date. Meaning in history if any + # commits have been made on master that post-date it, they will be accidentally + # included in the diff if we use the TRAVIS_COMMIT_RANGE variable. + files_changed=$(git diff --name-only HEAD $(git merge-base HEAD $TRAVIS_BRANCH)) + fi + + # Prints changed files in this commit to help debug doc-only build issues. + echo "Files changed: " + echo $files_changed + + if ! echo $files_changed | grep -qvE '(\.rst$)|(^Doc)|(^Misc)' then echo "Only docs were updated, stopping build process." exit fi + if [ "${TESTING}" != "docs" ]; then + # clang complains about unused-parameter a lot, redirect stderr + python3 Tools/ssl/multissltests.py --steps=library \ + --base-directory ${HOME}/multissl \ + --openssl ${OPENSSL} >/dev/null 2>&1 + fi + openssl version ./configure --with-pydebug make -j4 make -j4 regen-all clinic diff --git a/Doc/bugs.rst b/Doc/bugs.rst index bc1d10f379c..109e9eb202d 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -68,7 +68,7 @@ taken on the bug. .. seealso:: - `How to Report Bugs Effectively `_ + `How to Report Bugs Effectively `_ Article which goes into some detail about how to create a useful bug report. This describes what kind of information is useful and why it is useful. diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index 305e990368c..78724619ea3 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -13,6 +13,16 @@ the module initialisation function. The macro puts a pointer to a C structure into a static variable, :c:data:`PyDateTimeAPI`, that is used by the following macros. +Macro for access to the UTC singleton: + +.. c:var:: PyObject* PyDateTime_TimeZone_UTC + + Returns the time zone singleton representing UTC, the same object as + :attr:`datetime.timezone.utc`. + + .. versionadded:: 3.7 + + Type-check macros: .. c:function:: int PyDate_Check(PyObject *ob) @@ -79,27 +89,41 @@ Macros to create objects: .. c:function:: PyObject* PyDate_FromDate(int year, int month, int day) - Return a ``datetime.date`` object with the specified year, month and day. + Return a :class:`datetime.date` object with the specified year, month and day. .. c:function:: PyObject* PyDateTime_FromDateAndTime(int year, int month, int day, int hour, int minute, int second, int usecond) - Return a ``datetime.datetime`` object with the specified year, month, day, hour, + Return a :class:`datetime.datetime` object with the specified year, month, day, hour, minute, second and microsecond. .. c:function:: PyObject* PyTime_FromTime(int hour, int minute, int second, int usecond) - Return a ``datetime.time`` object with the specified hour, minute, second and + Return a :class:`datetime.time` object with the specified hour, minute, second and microsecond. .. c:function:: PyObject* PyDelta_FromDSU(int days, int seconds, int useconds) - Return a ``datetime.timedelta`` object representing the given number of days, - seconds and microseconds. Normalization is performed so that the resulting - number of microseconds and seconds lie in the ranges documented for - ``datetime.timedelta`` objects. + Return a :class:`datetime.timedelta` object representing the given number + of days, seconds and microseconds. Normalization is performed so that the + resulting number of microseconds and seconds lie in the ranges documented for + :class:`datetime.timedelta` objects. + +.. c:function:: PyObject* PyTimeZone_FromOffset(PyDateTime_DeltaType* offset) + + Return a :class:`datetime.timezone` object with an unnamed fixed offset + represented by the *offset* argument. + + .. versionadded:: 3.7 + +.. c:function:: PyObject* PyTimeZone_FromOffsetAndName(PyDateTime_DeltaType* offset, PyUnicode* name) + + Return a :class:`datetime.timezone` object with a fixed offset represented + by the *offset* argument and with tzname *name*. + + .. versionadded:: 3.7 Macros to extract fields from date objects. The argument must be an instance of @@ -199,11 +223,11 @@ Macros for the convenience of modules implementing the DB API: .. c:function:: PyObject* PyDateTime_FromTimestamp(PyObject *args) - Create and return a new ``datetime.datetime`` object given an argument tuple - suitable for passing to ``datetime.datetime.fromtimestamp()``. + Create and return a new :class:`datetime.datetime` object given an argument + tuple suitable for passing to :meth:`datetime.datetime.fromtimestamp()`. .. c:function:: PyObject* PyDate_FromTimestamp(PyObject *args) - Create and return a new ``datetime.date`` object given an argument tuple - suitable for passing to ``datetime.date.fromtimestamp()``. + Create and return a new :class:`datetime.date` object given an argument + tuple suitable for passing to :meth:`datetime.date.fromtimestamp()`. diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index dc1939db17e..bae49d5ba81 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -7,6 +7,213 @@ Initialization, Finalization, and Threads ***************************************** +.. _pre-init-safe: + +Before Python Initialization +============================ + +In an application embedding Python, the :c:func:`Py_Initialize` function must +be called before using any other Python/C API functions; with the exception of +a few functions and the :ref:`global configuration variables +`. + +The following functions can be safely called before Python is initialized: + +* Configuration functions: + + * :c:func:`PyImport_AppendInittab` + * :c:func:`PyImport_ExtendInittab` + * :c:func:`PyInitFrozenExtensions` + * :c:func:`PyMem_SetAllocator` + * :c:func:`PyMem_SetupDebugHooks` + * :c:func:`PyObject_SetArenaAllocator` + * :c:func:`Py_SetPath` + * :c:func:`Py_SetProgramName` + * :c:func:`Py_SetPythonHome` + * :c:func:`Py_SetStandardStreamEncoding` + +* Informative functions: + + * :c:func:`PyMem_GetAllocator` + * :c:func:`PyObject_GetArenaAllocator` + * :c:func:`Py_GetBuildInfo` + * :c:func:`Py_GetCompiler` + * :c:func:`Py_GetCopyright` + * :c:func:`Py_GetPlatform` + * :c:func:`Py_GetVersion` + +* Utilities: + + * :c:func:`Py_DecodeLocale` + +* Memory allocators: + + * :c:func:`PyMem_RawMalloc` + * :c:func:`PyMem_RawRealloc` + * :c:func:`PyMem_RawCalloc` + * :c:func:`PyMem_RawFree` + +.. note:: + + The following functions **should not be called** before + :c:func:`Py_Initialize`: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`, + :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, + :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome`, + :c:func:`Py_GetProgramName` and :c:func:`PyEval_InitThreads`. + + +.. _global-conf-vars: + +Global configuration variables +============================== + +Python has variables for the global configuration to control different features +and options. By default, these flags are controlled by :ref:`command line +options `. + +When a flag is set by an option, the value of the flag is the number of times +that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag` +to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. + +.. c:var:: Py_BytesWarningFlag + + Issue a warning when comparing :class:`bytes` or :class:`bytearray` with + :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater + or equal to ``2``. + + Set by the :option:`-b` option. + +.. c:var:: Py_DebugFlag + + Turn on parser debugging output (for expert only, depending on compilation + options). + + Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment + variable. + +.. c:var:: Py_DontWriteBytecodeFlag + + If set to non-zero, Python won't try to write ``.pyc`` files on the + import of source modules. + + Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` + environment variable. + +.. c:var:: Py_FrozenFlag + + Suppress error messages when calculating the module search path in + :c:func:`Py_GetPath`. + + Private flag used by ``_freeze_importlib`` and ``frozenmain`` programs. + +.. c:var:: Py_HashRandomizationFlag + + Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to + a non-empty string. + + If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment + variable to initialize the secret hash seed. + +.. c:var:: Py_IgnoreEnvironmentFlag + + Ignore all :envvar:`PYTHON*` environment variables, e.g. + :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. + + Set by the :option:`-E` and :option:`-I` options. + +.. c:var:: Py_InspectFlag + + When a script is passed as first argument or the :option:`-c` option is used, + enter interactive mode after executing the script or the command, even when + :data:`sys.stdin` does not appear to be a terminal. + + Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment + variable. + +.. c:var:: Py_InteractiveFlag + + Set by the :option:`-i` option. + +.. c:var:: Py_IsolatedFlag + + Run Python in isolated mode. In isolated mode :data:`sys.path` contains + neither the script's directory nor the user's site-packages directory. + + Set by the :option:`-I` option. + + .. versionadded:: 3.4 + +.. c:var:: Py_LegacyWindowsFSEncodingFlag + + If the flag is non-zero, use the ``mbcs`` encoding instead of the UTF-8 + encoding for the filesystem encoding. + + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment + variable is set to a non-empty string. + + See :pep:`529` for more details. + + Availability: Windows. + +.. c:var:: Py_LegacyWindowsStdioFlag + + If the flag is non-zero, use :class:`io.FileIO` instead of + :class:`WindowsConsoleIO` for :mod:`sys` standard streams. + + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment + variable is set to a non-empty string. + + See :pep:`528` for more details. + + Availability: Windows. + +.. c:var:: Py_NoSiteFlag + + Disable the import of the module :mod:`site` and the site-dependent + manipulations of :data:`sys.path` that it entails. Also disable these + manipulations if :mod:`site` is explicitly imported later (call + :func:`site.main` if you want them to be triggered). + + Set by the :option:`-S` option. + +.. c:var:: Py_NoUserSiteDirectory + + Don't add the :data:`user site-packages directory ` to + :data:`sys.path`. + + Set by the :option:`-s` and :option:`-I` options, and the + :envvar:`PYTHONNOUSERSITE` environment variable. + +.. c:var:: Py_OptimizeFlag + + Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment + variable. + +.. c:var:: Py_QuietFlag + + Don't display the copyright and version messages even in interactive mode. + + Set by the :option:`-q` option. + + .. versionadded:: 3.2 + +.. c:var:: Py_UnbufferedStdioFlag + + Force the stdout and stderr streams to be unbuffered. + + Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` + environment variable. + +.. c:var:: Py_VerboseFlag + + Print a message each time a module is initialized, showing the place + (filename or built-in module) from which it is loaded. If greater or equal + to ``2``, print a message for each file that is checked for when + searching for a module. Also provides information on module cleanup at exit. + + Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment + variable. + Initializing and finalizing the interpreter =========================================== @@ -27,9 +234,11 @@ Initializing and finalizing the interpreter single: PySys_SetArgvEx() single: Py_FinalizeEx() - Initialize the Python interpreter. In an application embedding Python, this - should be called before using any other Python/C API functions; with the - exception of :c:func:`Py_SetProgramName`, :c:func:`Py_SetPythonHome` and :c:func:`Py_SetPath`. This initializes + Initialize the Python interpreter. In an application embedding Python, + this should be called before using any other Python/C API functions; see + :ref:`Before Python Initialization ` for the few exceptions. + + This initializes the table of loaded modules (``sys.modules``), and creates the fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. It also initializes the module search path (``sys.path``). It does not set ``sys.argv``; use @@ -129,7 +338,7 @@ Process-wide parameters .. versionadded:: 3.4 -.. c:function:: void Py_SetProgramName(wchar_t *name) +.. c:function:: void Py_SetProgramName(const wchar_t *name) .. index:: single: Py_Initialize() @@ -396,7 +605,7 @@ Process-wide parameters .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. -.. c:function:: void Py_SetPythonHome(wchar_t *home) +.. c:function:: void Py_SetPythonHome(const wchar_t *home) Set the default "home" directory, that is, the location of the standard Python libraries. See :envvar:`PYTHONHOME` for the meaning of the @@ -478,15 +687,14 @@ This is so common that a pair of macros exists to simplify it:: The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the -block. These two macros are still available when Python is compiled without -thread support (they simply have an empty expansion). +block. -When thread support is enabled, the block above expands to the following code:: +The block above expands to the following code:: PyThreadState *_save; _save = PyEval_SaveThread(); - ...Do some blocking I/O operation... + ... Do some blocking I/O operation ... PyEval_RestoreThread(_save); .. index:: @@ -609,36 +817,24 @@ code, or when embedding the Python interpreter: This is a no-op when called for a second time. + .. versionchanged:: 3.7 + This function is now called by :c:func:`Py_Initialize()`, so you don't + have to call it yourself anymore. + .. versionchanged:: 3.2 This function cannot be called before :c:func:`Py_Initialize()` anymore. .. index:: module: _thread - .. note:: - - When only the main thread exists, no GIL operations are needed. This is a - common situation (most Python programs do not use threads), and the lock - operations slow the interpreter down a bit. Therefore, the lock is not - created initially. This situation is equivalent to having acquired the lock: - when there is only a single thread, all object accesses are safe. Therefore, - when this function initializes the global interpreter lock, it also acquires - it. Before the Python :mod:`_thread` module creates a new thread, knowing - that either it has the lock or the lock hasn't been created yet, it calls - :c:func:`PyEval_InitThreads`. When this call returns, it is guaranteed that - the lock has been created and that the calling thread has acquired it. - - It is **not** safe to call this function when it is unknown which thread (if - any) currently has the global interpreter lock. - - This function is not available when thread support is disabled at compile time. - .. c:function:: int PyEval_ThreadsInitialized() Returns a non-zero value if :c:func:`PyEval_InitThreads` has been called. This function can be called without holding the GIL, and therefore can be used to - avoid calls to the locking API when running single-threaded. This function is - not available when thread support is disabled at compile time. + avoid calls to the locking API when running single-threaded. + + .. versionchanged:: 3.7 + The :term:`GIL` is now initialized by :c:func:`Py_Initialize()`. .. c:function:: PyThreadState* PyEval_SaveThread() @@ -646,8 +842,7 @@ code, or when embedding the Python interpreter: Release the global interpreter lock (if it has been created and thread support is enabled) and reset the thread state to *NULL*, returning the previous thread state (which is not *NULL*). If the lock has been created, - the current thread must have acquired it. (This function is available even - when thread support is disabled at compile time.) + the current thread must have acquired it. .. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) @@ -655,8 +850,7 @@ code, or when embedding the Python interpreter: Acquire the global interpreter lock (if it has been created and thread support is enabled) and set the thread state to *tstate*, which must not be *NULL*. If the lock has been created, the current thread must not have - acquired it, otherwise deadlock ensues. (This function is available even - when thread support is disabled at compile time.) + acquired it, otherwise deadlock ensues. .. c:function:: PyThreadState* PyThreadState_Get() @@ -748,7 +942,7 @@ example usage in the Python source distribution. This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. Note that it contains an opening brace; it must be matched with a following :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this - macro. It is a no-op when thread support is disabled at compile time. + macro. .. c:macro:: Py_END_ALLOW_THREADS @@ -756,29 +950,29 @@ example usage in the Python source distribution. This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains a closing brace; it must be matched with an earlier :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of - this macro. It is a no-op when thread support is disabled at compile time. + this macro. .. c:macro:: Py_BLOCK_THREADS This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to - :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. It is a no-op when - thread support is disabled at compile time. + :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. .. c:macro:: Py_UNBLOCK_THREADS This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable - declaration. It is a no-op when thread support is disabled at compile time. + declaration. Low-level API ------------- -All of the following functions are only available when thread support is enabled -at compile time, and must be called only when the global interpreter lock has -been created. +All of the following functions must be called after :c:func:`Py_Initialize`. + +.. versionchanged:: 3.7 + :c:func:`Py_Initialize()` now initializes the :term:`GIL`. .. c:function:: PyInterpreterState* PyInterpreterState_New() @@ -859,8 +1053,7 @@ been created. If this thread already has the lock, deadlock ensues. :c:func:`PyEval_RestoreThread` is a higher-level function which is always - available (even when thread support isn't enabled or when threads have - not been initialized). + available (even when threads have not been initialized). .. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) @@ -872,8 +1065,7 @@ been created. reported. :c:func:`PyEval_SaveThread` is a higher-level function which is always - available (even when thread support isn't enabled or when threads have - not been initialized). + available (even when threads have not been initialized). .. c:function:: void PyEval_AcquireLock() @@ -1068,18 +1260,18 @@ Python-level trace functions in previous versions. registration function as *obj*, *frame* is the frame object to which the event pertains, *what* is one of the constants :const:`PyTrace_CALL`, :const:`PyTrace_EXCEPTION`, :const:`PyTrace_LINE`, :const:`PyTrace_RETURN`, - :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION`, or - :const:`PyTrace_C_RETURN`, and *arg* depends on the value of *what*: + :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION`, :const:`PyTrace_C_RETURN`, + or :const:`PyTrace_OPCODE`, and *arg* depends on the value of *what*: +------------------------------+--------------------------------------+ | Value of *what* | Meaning of *arg* | +==============================+======================================+ - | :const:`PyTrace_CALL` | Always *NULL*. | + | :const:`PyTrace_CALL` | Always :c:data:`Py_None`. | +------------------------------+--------------------------------------+ | :const:`PyTrace_EXCEPTION` | Exception information as returned by | | | :func:`sys.exc_info`. | +------------------------------+--------------------------------------+ - | :const:`PyTrace_LINE` | Always *NULL*. | + | :const:`PyTrace_LINE` | Always :c:data:`Py_None`. | +------------------------------+--------------------------------------+ | :const:`PyTrace_RETURN` | Value being returned to the caller, | | | or *NULL* if caused by an exception. | @@ -1090,7 +1282,8 @@ Python-level trace functions in previous versions. +------------------------------+--------------------------------------+ | :const:`PyTrace_C_RETURN` | Function object being called. | +------------------------------+--------------------------------------+ - + | :const:`PyTrace_OPCODE` | Always :c:data:`Py_None`. | + +------------------------------+--------------------------------------+ .. c:var:: int PyTrace_CALL @@ -1114,14 +1307,15 @@ Python-level trace functions in previous versions. .. c:var:: int PyTrace_LINE - The value passed as the *what* parameter to a trace function (but not a - profiling function) when a line-number event is being reported. + The value passed as the *what* parameter to a :c:type:`Py_tracefunc` function + (but not a profiling function) when a line-number event is being reported. + It may be disabled for a frame by setting :attr:`f_trace_lines` to *0* on that frame. .. c:var:: int PyTrace_RETURN The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a - call is returning without propagating an exception. + call is about to return. .. c:var:: int PyTrace_C_CALL @@ -1142,22 +1336,32 @@ Python-level trace functions in previous versions. function has returned. +.. c:var:: int PyTrace_OPCODE + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions (but not + profiling functions) when a new opcode is about to be executed. This event is + not emitted by default: it must be explicitly requested by setting + :attr:`f_trace_opcodes` to *1* on the frame. + + .. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) Set the profiler function to *func*. The *obj* parameter is passed to the function as its first parameter, and may be any Python object, or *NULL*. If the profile function needs to maintain state, using a different value for *obj* for each thread provides a convenient and thread-safe place to store it. The - profile function is called for all monitored events except the line-number - events. + profile function is called for all monitored events except :const:`PyTrace_LINE` + :const:`PyTrace_OPCODE` and :const:`PyTrace_EXCEPTION`. .. c:function:: void PyEval_SetTrace(Py_tracefunc func, PyObject *obj) Set the tracing function to *func*. This is similar to :c:func:`PyEval_SetProfile`, except the tracing function does receive line-number - events. - + events and per-opcode events, but does not receive any event related to C function + objects being called. Any trace function registered using :c:func:`PyEval_SetTrace` + will not receive :const:`PyTrace_C_CALL`, :const:`PyTrace_C_EXCEPTION` or + :const:`PyTrace_C_RETURN` as a value for the *what* parameter. .. _advanced-debugging: diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index f50680b3d29..4f16b578eb5 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -10,6 +10,9 @@ Integer Objects All integers are implemented as "long" integer objects of arbitrary size. +On error, most ``PyLong_As*`` APIs return ``(return type)-1`` which cannot be +distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. + .. c:type:: PyLongObject This subtype of :c:type:`PyObject` represents a Python integer object. @@ -134,6 +137,8 @@ All integers are implemented as "long" integer objects of arbitrary size. Raise :exc:`OverflowError` if the value of *obj* is out of range for a :c:type:`long`. + Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate. + .. c:function:: long PyLong_AsLongAndOverflow(PyObject *obj, int *overflow) @@ -146,6 +151,8 @@ All integers are implemented as "long" integer objects of arbitrary size. return ``-1``; otherwise, set *\*overflow* to ``0``. If any other exception occurs set *\*overflow* to ``0`` and return ``-1`` as usual. + Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate. + .. c:function:: long long PyLong_AsLongLong(PyObject *obj) @@ -159,6 +166,8 @@ All integers are implemented as "long" integer objects of arbitrary size. Raise :exc:`OverflowError` if the value of *obj* is out of range for a :c:type:`long`. + Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate. + .. c:function:: long long PyLong_AsLongLongAndOverflow(PyObject *obj, int *overflow) @@ -171,6 +180,8 @@ All integers are implemented as "long" integer objects of arbitrary size. and return ``-1``; otherwise, set *\*overflow* to ``0``. If any other exception occurs set *\*overflow* to ``0`` and return ``-1`` as usual. + Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate. + .. versionadded:: 3.2 @@ -186,6 +197,8 @@ All integers are implemented as "long" integer objects of arbitrary size. Raise :exc:`OverflowError` if the value of *pylong* is out of range for a :c:type:`Py_ssize_t`. + Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate. + .. c:function:: unsigned long PyLong_AsUnsignedLong(PyObject *pylong) @@ -199,15 +212,25 @@ All integers are implemented as "long" integer objects of arbitrary size. Raise :exc:`OverflowError` if the value of *pylong* is out of range for a :c:type:`unsigned long`. + Returns ``(unsigned long)-1`` on error. + Use :c:func:`PyErr_Occurred` to disambiguate. + .. c:function:: size_t PyLong_AsSize_t(PyObject *pylong) + .. index:: + single: SIZE_MAX + single: OverflowError (built-in exception) + Return a C :c:type:`size_t` representation of *pylong*. *pylong* must be an instance of :c:type:`PyLongObject`. Raise :exc:`OverflowError` if the value of *pylong* is out of range for a :c:type:`size_t`. + Returns ``(size_t)-1`` on error. + Use :c:func:`PyErr_Occurred` to disambiguate. + .. c:function:: unsigned long long PyLong_AsUnsignedLongLong(PyObject *pylong) @@ -220,6 +243,9 @@ All integers are implemented as "long" integer objects of arbitrary size. Raise :exc:`OverflowError` if the value of *pylong* is out of range for an :c:type:`unsigned long long`. + Returns ``(unsigned long long)-1`` on error. + Use :c:func:`PyErr_Occurred` to disambiguate. + .. versionchanged:: 3.1 A negative *pylong* now raises :exc:`OverflowError`, not :exc:`TypeError`. @@ -233,6 +259,8 @@ All integers are implemented as "long" integer objects of arbitrary size. If the value of *obj* is out of range for an :c:type:`unsigned long`, return the reduction of that value modulo ``ULONG_MAX + 1``. + Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate. + .. c:function:: unsigned long long PyLong_AsUnsignedLongLongMask(PyObject *obj) @@ -243,6 +271,8 @@ All integers are implemented as "long" integer objects of arbitrary size. If the value of *obj* is out of range for an :c:type:`unsigned long long`, return the reduction of that value modulo ``PY_ULLONG_MAX + 1``. + Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate. + .. c:function:: double PyLong_AsDouble(PyObject *pylong) @@ -252,6 +282,8 @@ All integers are implemented as "long" integer objects of arbitrary size. Raise :exc:`OverflowError` if the value of *pylong* is out of range for a :c:type:`double`. + Returns -1.0 on error. Use :c:func:`PyErr_Occurred` to disambiguate. + .. c:function:: void* PyLong_AsVoidPtr(PyObject *pylong) @@ -259,3 +291,5 @@ All integers are implemented as "long" integer objects of arbitrary size. If *pylong* cannot be converted, an :exc:`OverflowError` will be raised. This is only assured to produce a usable :c:type:`void` pointer for values created with :c:func:`PyLong_FromVoidPtr`. + + Returns NULL on error. Use :c:func:`PyErr_Occurred` to disambiguate. diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 4b1e666ef35..2af0c46d451 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -100,9 +100,10 @@ The following function sets are wrappers to the system allocator. These functions are thread-safe, the :term:`GIL ` does not need to be held. -The default raw memory block allocator uses the following functions: -:c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`; call -``malloc(1)`` (or ``calloc(1, 1)``) when requesting zero bytes. +The :ref:`default raw memory allocator ` uses +the following functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` +and :c:func:`free`; call ``malloc(1)`` (or ``calloc(1, 1)``) when requesting +zero bytes. .. versionadded:: 3.4 @@ -165,7 +166,8 @@ The following function sets, modeled after the ANSI C standard, but specifying behavior when requesting zero bytes, are available for allocating and releasing memory from the Python heap. -By default, these functions use :ref:`pymalloc memory allocator `. +The :ref:`default memory allocator ` uses the +:ref:`pymalloc memory allocator `. .. warning:: @@ -270,7 +272,8 @@ The following function sets, modeled after the ANSI C standard, but specifying behavior when requesting zero bytes, are available for allocating and releasing memory from the Python heap. -By default, these functions use :ref:`pymalloc memory allocator `. +The :ref:`default object allocator ` uses the +:ref:`pymalloc memory allocator `. .. warning:: @@ -326,6 +329,31 @@ By default, these functions use :ref:`pymalloc memory allocator `. If *p* is *NULL*, no operation is performed. +.. _default-memory-allocators: + +Default Memory Allocators +========================= + +Default memory allocators: + +=============================== ==================== ================== ===================== ==================== +Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc +=============================== ==================== ================== ===================== ==================== +Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc`` +Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug +Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` +Release build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug +=============================== ==================== ================== ===================== ==================== + +Legend: + +* Name: value for :envvar:`PYTHONMALLOC` environment variable +* ``malloc``: system allocators from the standard C library, C functions: + :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free` +* ``pymalloc``: :ref:`pymalloc memory allocator ` +* "+ debug": with debug hooks installed by :c:func:`PyMem_SetupDebugHooks` + + Customize Memory Allocators =========================== @@ -431,7 +459,8 @@ Customize Memory Allocators displayed if :mod:`tracemalloc` is tracing Python memory allocations and the memory block was traced. - These hooks are installed by default if Python is compiled in debug + These hooks are :ref:`installed by default ` if + Python is compiled in debug mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install debug hooks on a Python compiled in release mode. @@ -453,9 +482,9 @@ to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" with a fixed size of 256 KiB. It falls back to :c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. -*pymalloc* is the default allocator of the :c:data:`PYMEM_DOMAIN_MEM` (ex: -:c:func:`PyMem_Malloc`) and :c:data:`PYMEM_DOMAIN_OBJ` (ex: -:c:func:`PyObject_Malloc`) domains. +*pymalloc* is the :ref:`default allocator ` of the +:c:data:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) and +:c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains. The arena allocator uses the following functions: diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 95d9d657ce9..e4da96c493c 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -106,6 +106,16 @@ Operating System Utilities surrogate character, escape the bytes using the surrogateescape error handler instead of decoding them. + Encoding, highest priority to lowest priority: + + * ``UTF-8`` on macOS and Android; + * ``UTF-8`` if the Python UTF-8 mode is enabled; + * ``ASCII`` if the ``LC_CTYPE`` locale is ``"C"``, + ``nl_langinfo(CODESET)`` returns the ``ASCII`` encoding (or an alias), + and :c:func:`mbstowcs` and :c:func:`wcstombs` functions uses the + ``ISO-8859-1`` encoding. + * the current locale encoding. + Return a pointer to a newly allocated wide character string, use :c:func:`PyMem_RawFree` to free the memory. If size is not ``NULL``, write the number of wide characters excluding the null character into ``*size`` @@ -127,6 +137,9 @@ Operating System Utilities .. versionadded:: 3.5 + .. versionchanged:: 3.7 + The function now uses the UTF-8 encoding in the UTF-8 mode. + .. c:function:: char* Py_EncodeLocale(const wchar_t *text, size_t *error_pos) @@ -134,16 +147,31 @@ Operating System Utilities :ref:`surrogateescape error handler `: surrogate characters in the range U+DC80..U+DCFF are converted to bytes 0x80..0xFF. + Encoding, highest priority to lowest priority: + + * ``UTF-8`` on macOS and Android; + * ``UTF-8`` if the Python UTF-8 mode is enabled; + * ``ASCII`` if the ``LC_CTYPE`` locale is ``"C"``, + ``nl_langinfo(CODESET)`` returns the ``ASCII`` encoding (or an alias), + and :c:func:`mbstowcs` and :c:func:`wcstombs` functions uses the + ``ISO-8859-1`` encoding. + * the current locale encoding. + + The function uses the UTF-8 encoding in the Python UTF-8 mode. + Return a pointer to a newly allocated byte string, use :c:func:`PyMem_Free` to free the memory. Return ``NULL`` on encoding error or memory allocation error - If error_pos is not ``NULL``, ``*error_pos`` is set to the index of the - invalid character on encoding error, or set to ``(size_t)-1`` otherwise. + If error_pos is not ``NULL``, ``*error_pos`` is set to ``(size_t)-1`` on + success, or set to the index of the invalid character on encoding error. Use the :c:func:`Py_DecodeLocale` function to decode the bytes string back to a wide character string. + .. versionchanged:: 3.7 + The function now uses the UTF-8 encoding in the UTF-8 mode. + .. seealso:: The :c:func:`PyUnicode_EncodeFSDefault` and @@ -151,6 +179,9 @@ Operating System Utilities .. versionadded:: 3.5 + .. versionchanged:: 3.7 + The function now supports the UTF-8 mode. + .. _systemfunctions: diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 45aff1b7e3c..92e22b16a4e 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -760,7 +760,8 @@ system. Py_ssize_t len, \ const char *errors) - Decode a string from the current locale encoding. The supported + Decode a string from UTF-8 on Android, or from the current locale encoding + on other platforms. The supported error handlers are ``"strict"`` and ``"surrogateescape"`` (:pep:`383`). The decoder uses ``"strict"`` error handler if *errors* is ``NULL``. *str* must end with a null character but @@ -770,12 +771,20 @@ system. :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at Python startup). + This function ignores the Python UTF-8 mode. + .. seealso:: The :c:func:`Py_DecodeLocale` function. .. versionadded:: 3.3 + .. versionchanged:: 3.7 + The function now also uses the current locale encoding for the + ``surrogateescape`` error handler, except on Android. Previously, :c:func:`Py_DecodeLocale` + was used for the ``surrogateescape``, and the current locale encoding was + used for ``strict``. + .. c:function:: PyObject* PyUnicode_DecodeLocale(const char *str, const char *errors) @@ -787,7 +796,8 @@ system. .. c:function:: PyObject* PyUnicode_EncodeLocale(PyObject *unicode, const char *errors) - Encode a Unicode object to the current locale encoding. The + Encode a Unicode object to UTF-8 on Android, or to the current locale + encoding on other platforms. The supported error handlers are ``"strict"`` and ``"surrogateescape"`` (:pep:`383`). The encoder uses ``"strict"`` error handler if *errors* is ``NULL``. Return a :class:`bytes` object. *unicode* cannot @@ -797,12 +807,21 @@ system. :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at Python startup). + This function ignores the Python UTF-8 mode. + .. seealso:: The :c:func:`Py_EncodeLocale` function. .. versionadded:: 3.3 + .. versionchanged:: 3.7 + The function now also uses the current locale encoding for the + ``surrogateescape`` error handler, except on Android. Previously, + :c:func:`Py_EncodeLocale` + was used for the ``surrogateescape``, and the current locale encoding was + used for ``strict``. + File System Encoding """""""""""""""""""" diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 3897fdd8282..cefe9d44bf4 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -301,7 +301,7 @@ the same library that the Python runtime is using. set to *NULL*. -.. c:function:: PyObject* PyEval_EvalCodeEx(PyObject *co, PyObject *globals, PyObject *locals, PyObject **args, int argcount, PyObject **kws, int kwcount, PyObject **defs, int defcount, PyObject *kwdefs, PyObject *closure) +.. c:function:: PyObject* PyEval_EvalCodeEx(PyObject *co, PyObject *globals, PyObject *locals, PyObject *const *args, int argcount, PyObject *const *kws, int kwcount, PyObject *const *defs, int defcount, PyObject *kwdefs, PyObject *closure) Evaluate a precompiled code object, given a particular environment for its evaluation. This environment consists of a dictionary of global variables, diff --git a/Doc/conf.py b/Doc/conf.py index aaee983984b..19a2f7d67ff 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -90,11 +90,10 @@ html_split_index = True # Options for LaTeX output # ------------------------ +latex_engine = 'xelatex' + # Get LaTeX to handle Unicode correctly latex_elements = { - 'inputenc': r'\usepackage[utf8x]{inputenc}', - 'utf8extra': '', - 'fontenc': r'\usepackage[T1,T2A]{fontenc}', } # Additional stuff for the LaTeX preamble. diff --git a/Doc/copyright.rst b/Doc/copyright.rst index 2b5400cd9fb..540ff5ef059 100644 --- a/Doc/copyright.rst +++ b/Doc/copyright.rst @@ -4,7 +4,7 @@ Copyright Python and this documentation is: -Copyright © 2001-2017 Python Software Foundation. All rights reserved. +Copyright © 2001-2018 Python Software Foundation. All rights reserved. Copyright © 2000 BeOpen.com. All rights reserved. diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index b1cad48c3e5..6dc86fc5e54 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -177,6 +177,14 @@ PyDelta_FromDSU:int:days:: PyDelta_FromDSU:int:seconds:: PyDelta_FromDSU:int:useconds:: +PyTimeZone_FromOffset:PyObject*::+1: +PyTimeZone_FromOffset:PyDateTime_DeltaType*:offset:+1:Reference count not increased if offset is +00:00 + +PyTimeZone_FromOffsetAndName:PyObject*::+1: +PyTimeZone_FromOffsetAndName:PyDateTime_DeltaType*:offset:+1:Reference count not increased if offset is +00:00 and name == NULL +PyTimeZone_FromOffsetAndName:PyUnicode*:name:+1: + + PyDescr_NewClassMethod:PyObject*::+1: PyDescr_NewClassMethod:PyTypeObject*:type:: PyDescr_NewClassMethod:PyMethodDef*:method:: diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index 82ecd2c1ef4..aedbe712d3d 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -62,7 +62,7 @@ Key terms locally. .. _setuptools: https://setuptools.readthedocs.io/en/latest/ -.. _wheel: https://wheel.readthedocs.org +.. _wheel: https://wheel.readthedocs.io/ Open source licensing and collaboration ======================================= @@ -111,7 +111,7 @@ by invoking the ``pip`` module at the command line:: The Python Packaging User Guide includes more details on the `currently recommended tools`_. -.. _currently recommended tools: https://packaging.python.org/en/latest/current/#packaging-tool-recommendations +.. _currently recommended tools: https://packaging.python.org/guides/tool-recommendations/#packaging-tool-recommendations Reading the guide ================= @@ -124,11 +124,11 @@ involved in creating a project: * `Uploading the project to the Python Packaging Index`_ .. _Project structure: \ - https://packaging.python.org/en/latest/distributing/ + https://packaging.python.org/tutorials/distributing-packages/ .. _Building and packaging the project: \ - https://packaging.python.org/en/latest/distributing/#packaging-your-project + https://packaging.python.org/tutorials/distributing-packages/#packaging-your-project .. _Uploading the project to the Python Packaging Index: \ - https://packaging.python.org/en/latest/distributing/#uploading-your-project-to-pypi + https://packaging.python.org/tutorials/distributing-packages/#uploading-your-project-to-pypi How do I...? @@ -160,7 +160,7 @@ Python Packaging User Guide for more information and recommendations. .. seealso:: `Python Packaging User Guide: Binary Extensions - `__ + `__ .. other topics: diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst index 7cde1a0701e..9fce46ad266 100644 --- a/Doc/distutils/apiref.rst +++ b/Doc/distutils/apiref.rst @@ -285,6 +285,10 @@ the full reference. See the :func:`setup` function for a list of keyword arguments accepted by the Distribution constructor. :func:`setup` creates a Distribution instance. + .. versionchanged:: 3.7 + :class:`~distutils.core.Distribution` now warns if ``classifiers``, + ``keywords`` and ``platforms`` fields are not specified as a list or + a string. .. class:: Command diff --git a/Doc/distutils/index.rst b/Doc/distutils/index.rst index c565bcc5628..d6f7640fcb6 100644 --- a/Doc/distutils/index.rst +++ b/Doc/distutils/index.rst @@ -22,7 +22,7 @@ very little overhead for build/release/install mechanics. This guide only covers the basic tools for building and distributing extensions that are provided as part of this version of Python. Third party tools offer easier to use and more secure alternatives. Refer to the `quick - recommendations section `__ + recommendations section `__ in the Python Packaging User Guide for more information. .. toctree:: diff --git a/Doc/distutils/setupscript.rst b/Doc/distutils/setupscript.rst index 38e0202e4ac..952607a4073 100644 --- a/Doc/distutils/setupscript.rst +++ b/Doc/distutils/setupscript.rst @@ -581,17 +581,19 @@ This information includes: | | description of the | | | | | package | | | +----------------------+---------------------------+-----------------+--------+ -| ``long_description`` | longer description of the | long string | \(5) | +| ``long_description`` | longer description of the | long string | \(4) | | | package | | | +----------------------+---------------------------+-----------------+--------+ -| ``download_url`` | location where the | URL | \(4) | +| ``download_url`` | location where the | URL | | | | package may be downloaded | | | +----------------------+---------------------------+-----------------+--------+ -| ``classifiers`` | a list of classifiers | list of strings | \(4) | +| ``classifiers`` | a list of classifiers | list of strings | (6)(7) | +----------------------+---------------------------+-----------------+--------+ -| ``platforms`` | a list of platforms | list of strings | | +| ``platforms`` | a list of platforms | list of strings | (6)(8) | +----------------------+---------------------------+-----------------+--------+ -| ``license`` | license for the package | short string | \(6) | +| ``keywords`` | a list of keywords | list of strings | (6)(8) | ++----------------------+---------------------------+-----------------+--------+ +| ``license`` | license for the package | short string | \(5) | +----------------------+---------------------------+-----------------+--------+ Notes: @@ -607,22 +609,30 @@ Notes: provided, distutils lists it as the author in :file:`PKG-INFO`. (4) - These fields should not be used if your package is to be compatible with Python - versions prior to 2.2.3 or 2.3. The list is available from the `PyPI website - `_. - -(5) The ``long_description`` field is used by PyPI when you are :ref:`registering ` a package, to :ref:`build its home page `. -(6) +(5) The ``license`` field is a text indicating the license covering the package where the license is not a selection from the "License" Trove classifiers. See the ``Classifier`` field. Notice that there's a ``licence`` distribution option which is deprecated but still acts as an alias for ``license``. +(6) + This field must be a list. + +(7) + The valid classifiers are listed on + `PyPI `_. + +(8) + To preserve backward compatibility, this field also accepts a string. If + you pass a comma-separated string ``'foo, bar'``, it will be converted to + ``['foo', 'bar']``, Otherwise, it will be converted to a list of one + string. + 'short string' A single line of text, not more than 200 characters. @@ -650,7 +660,7 @@ information is sometimes used to indicate sub-releases. These are 1.0.1a2 the second alpha release of the first patch version of 1.0 -``classifiers`` are specified in a Python list:: +``classifiers`` must be specified in a list:: setup(..., classifiers=[ @@ -671,6 +681,11 @@ information is sometimes used to indicate sub-releases. These are ], ) +.. versionchanged:: 3.7 + :class:`~distutils.core.setup` now raises a :exc:`TypeError` if + ``classifiers``, ``keywords`` and ``platforms`` fields are not specified + as a list. + .. _debug-setup-script: Debugging the setup script diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 7c273533aba..e02f7837b69 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -27,7 +27,8 @@ your system setup; details are given in later chapters. avoid writing C extensions and preserve portability to other implementations. For example, if your use case is calling C library functions or system calls, you should consider using the :mod:`ctypes` module or the `cffi - `_ library rather than writing custom C code. + `_ library rather than writing + custom C code. These modules let you write Python code to interface with C code and are more portable between implementations of Python than writing and compiling a C extension module. @@ -40,7 +41,7 @@ A Simple Example Let's create an extension module called ``spam`` (the favorite food of Monty Python fans...) and let's say we want to create a Python interface to the C -library function :c:func:`system`. [#]_ This function takes a null-terminated +library function :c:func:`system` [#]_. This function takes a null-terminated character string as argument and returns an integer. We want this function to be callable from Python as follows:: @@ -917,7 +918,7 @@ It is also possible to :dfn:`borrow` [#]_ a reference to an object. The borrower of a reference should not call :c:func:`Py_DECREF`. The borrower must not hold on to the object longer than the owner from which it was borrowed. Using a borrowed reference after the owner has disposed of it risks using freed -memory and should be avoided completely. [#]_ +memory and should be avoided completely [#]_. The advantage of borrowing over owning a reference is that you don't need to take care of disposing of the reference on all possible paths through the code @@ -1088,7 +1089,7 @@ checking. The C function calling mechanism guarantees that the argument list passed to C functions (``args`` in the examples) is never *NULL* --- in fact it guarantees -that it is always a tuple. [#]_ +that it is always a tuple [#]_. It is a severe error to ever let a *NULL* pointer "escape" to the Python user. diff --git a/Doc/extending/index.rst b/Doc/extending/index.rst index 9eec8c273af..80594e357fd 100644 --- a/Doc/extending/index.rst +++ b/Doc/extending/index.rst @@ -32,7 +32,7 @@ approaches to creating C and C++ extensions for Python. .. seealso:: - `Python Packaging User Guide: Binary Extensions `_ + `Python Packaging User Guide: Binary Extensions `_ The Python Packaging User Guide not only covers several available tools that simplify the creation of binary extensions, but also discusses the various reasons why creating an extension module may be diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 0e36ba0aec0..62fbdb87a53 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -659,7 +659,7 @@ Fortunately, Python's cyclic-garbage collector will eventually figure out that the list is garbage and free it. In the second version of the :class:`Noddy` example, we allowed any kind of -object to be stored in the :attr:`first` or :attr:`last` attributes. [#]_ This +object to be stored in the :attr:`first` or :attr:`last` attributes [#]_. This means that :class:`Noddy` objects can participate in cycles:: >>> import noddy2 diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 1bd800b1a81..2e56fbc2f42 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -343,7 +343,7 @@ each Python stack frame. Also, extensions can call back into Python at almost random moments. Therefore, a complete threads implementation requires thread support for C. -Answer 2: Fortunately, there is `Stackless Python `_, +Answer 2: Fortunately, there is `Stackless Python `_, which has a completely redesigned interpreter loop that avoids the C stack. diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 3eafdf193c8..88996e48035 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -53,7 +53,7 @@ with a tool such as `SWIG `_. `SIP `__, `CXX `_ `Boost `_, or `Weave -`_ are also +`_ are also alternatives for wrapping C++ libraries. diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index d4a97fd81af..0d1cb198da8 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -272,7 +272,7 @@ The Python project's infrastructure is located all over the world. `www.python.org `_ is graciously hosted by `Rackspace `_, with CDN caching provided by `Fastly `_. `Upfront Systems -`_ hosts `bugs.python.org +`_ hosts `bugs.python.org `_. Many other Python services like `the Wiki `_ are hosted by `Oregon State University Open Source Lab `_. diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index 38e1796267f..4f9979bf55e 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -43,7 +43,7 @@ number of platforms, with Windows, Mac OS X, GTK, X11, all listed as current stable targets. Language bindings are available for a number of languages including Python, Perl, Ruby, etc. -wxPython (http://www.wxpython.org) is the Python binding for +`wxPython `_ is the Python binding for wxwidgets. While it often lags slightly behind the official wxWidgets releases, it also offers a number of features via pure Python extensions that are not available in other language bindings. There @@ -72,9 +72,9 @@ Gtk+ The `GObject introspection bindings `_ for Python allow you to write GTK+ 3 applications. There is also a -`Python GTK+ 3 Tutorial `_. +`Python GTK+ 3 Tutorial `_. -The older PyGtk bindings for the `Gtk+ 2 toolkit `_ have +The older PyGtk bindings for the `Gtk+ 2 toolkit `_ have been implemented by James Henstridge; see . Kivy diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index b5fdfa42cdb..f84feadd780 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -419,7 +419,7 @@ Python program effectively only uses one CPU, due to the insistence that Back in the days of Python 1.5, Greg Stein actually implemented a comprehensive patch set (the "free threading" patches) that removed the GIL and replaced it with fine-grained locking. Adam Olsen recently did a similar experiment -in his `python-safethread `_ +in his `python-safethread `_ project. Unfortunately, both experiments exhibited a sharp drop in single-thread performance (at least 30% slower), due to the amount of fine-grained locking necessary to compensate for the removal of the GIL. diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 1022373d387..1a2f582a31a 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -100,7 +100,7 @@ which don't. One is Thomas Heller's py2exe (Windows only) at http://www.py2exe.org/ -Another tool is Anthony Tuininga's `cx_Freeze `_. +Another tool is Anthony Tuininga's `cx_Freeze `_. Are there coding standards or a style guide for Python programs? diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index 6ac83e45d2e..d703f286222 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -170,8 +170,8 @@ offender. How do I make an executable from a Python script? ------------------------------------------------- -See http://cx-freeze.sourceforge.net/ for a distutils extension that allows you -to create console and GUI executables from Python code. +See `cx_Freeze `_ for a distutils extension +that allows you to create console and GUI executables from Python code. `py2exe `_, the most popular extension for building Python 2.x-based executables, does not yet support Python 3 but a version that does is in development. diff --git a/Doc/glossary.rst b/Doc/glossary.rst index b947520b96b..dcfe086b38b 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -126,7 +126,7 @@ Glossary BDFL Benevolent Dictator For Life, a.k.a. `Guido van Rossum - `_, Python's creator. + `_, Python's creator. binary file A :term:`file object` able to read and write @@ -372,9 +372,11 @@ Glossary may be accessed via the :attr:`__annotations__` special attribute of a function object. - Python itself does not assign any particular meaning to function - annotations. They are intended to be interpreted by third-party libraries - or tools. See :pep:`3107`, which describes some of their potential uses. + See also the :term:`variable annotation` glossary entry. + + Annotations are meant to provide a standard way for programmers to + document types of functions they design. See :pep:`484`, which + describes this functionality. __future__ A pseudo-module which programmers can use to enable new language features @@ -391,7 +393,8 @@ Glossary garbage collection The process of freeing memory when it is not used anymore. Python performs garbage collection via reference counting and a cyclic garbage - collector that is able to detect and break reference cycles. + collector that is able to detect and break reference cycles. The + garbage collector can be controlled using the :mod:`gc` module. .. index:: single: generator @@ -458,6 +461,12 @@ Glossary is believed that overcoming this performance issue would make the implementation much more complicated and therefore costlier to maintain. + + hash-based pyc + A bytecode cache file that uses the the hash rather than the last-modified + time of the corresponding source file to determine its validity. See + :ref:`pyc-invalidation`. + hashable An object is *hashable* if it has a hash value which never changes during its lifetime (it needs a :meth:`__hash__` method), and can be compared to @@ -1014,10 +1023,11 @@ Glossary attribute of a class or module object and can be accessed using :func:`typing.get_type_hints`. - Python itself does not assign any particular meaning to variable - annotations. They are intended to be interpreted by third-party libraries - or type checking tools. See :pep:`526`, :pep:`484` which describe - some of their potential uses. + See also the :term:`function annotation` glossary entry. + + Annotations are meant to provide a standard way for programmers to + document types of functions they design. See :pep:`484` and :pep:`526` + which describe this functionality. virtual environment A cooperatively isolated runtime environment that allows Python users diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst index 1d3bfb87dc0..19d65d6996b 100644 --- a/Doc/howto/curses.rst +++ b/Doc/howto/curses.rst @@ -543,7 +543,7 @@ learn more about submitting patches to Python. * `Writing Programs with NCURSES `_: a lengthy tutorial for C programmers. -* `The ncurses man page `_ +* `The ncurses man page `_ * `The ncurses FAQ `_ * `"Use curses... don't swear" `_: video of a PyCon 2013 talk on controlling terminals using curses or Urwid. diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 71ee417f2bc..4d2d052d291 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -941,7 +941,7 @@ Using file rotation ------------------- .. sectionauthor:: Doug Hellmann, Vinay Sajip (changes) -.. (see ) +.. (see ) Sometimes you want to let a log file grow to a certain size, then open a new file and log to that. You may want to keep a certain number of these files, and diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 8074b0f1099..4ee68b4747e 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -314,7 +314,7 @@ favourite beverage and carry on. If your logging needs are simple, then use the above examples to incorporate logging into your own scripts, and if you run into problems or don't understand something, please post a question on the comp.lang.python Usenet -group (available at https://groups.google.com/group/comp.lang.python) and you +group (available at https://groups.google.com/forum/#!forum/comp.lang.python) and you should receive help before too long. Still here? You can carry on reading the next few sections, which provide a diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst index 8562d237374..98c81206741 100644 --- a/Doc/howto/pyporting.rst +++ b/Doc/howto/pyporting.rst @@ -433,12 +433,12 @@ to make sure everything functions as expected in both versions of Python. .. _Futurize: http://python-future.org/automatic_conversion.html .. _importlib: https://docs.python.org/3/library/importlib.html#module-importlib .. _importlib2: https://pypi.python.org/pypi/importlib2 -.. _Modernize: https://python-modernize.readthedocs.org/en/latest/ +.. _Modernize: https://python-modernize.readthedocs.io/ .. _mypy: http://mypy-lang.org/ .. _Porting to Python 3: http://python3porting.com/ .. _Pylint: https://pypi.python.org/pypi/pylint -.. _Python 3 Q & A: https://ncoghlan-devs-python-notes.readthedocs.org/en/latest/python3/questions_and_answers.html +.. _Python 3 Q & A: https://ncoghlan-devs-python-notes.readthedocs.io/en/latest/python3/questions_and_answers.html .. _pytype: https://github.com/google/pytype .. _python-future: http://python-future.org/ @@ -449,4 +449,4 @@ to make sure everything functions as expected in both versions of Python. .. _"What's New": https://docs.python.org/3/whatsnew/index.html -.. _Why Python 3 exists: http://www.snarky.ca/why-python-3-exists +.. _Why Python 3 exists: https://snarky.ca/why-python-3-exists diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index e8466ee5423..bdf687ee455 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -289,6 +289,8 @@ Putting REs in strings keeps the Python language simpler, but has one disadvantage which is the topic of the next section. +.. _the-backslash-plague: + The Backslash Plague -------------------- @@ -327,6 +329,13 @@ backslashes are not handled in any special way in a string literal prefixed with while ``"\n"`` is a one-character string containing a newline. Regular expressions will often be written in Python code using this raw string notation. +In addition, special escape sequences that are valid in regular expressions, +but not valid as Python string literals, now result in a +:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`, +which means the sequences will be invalid if raw string notation or escaping +the backslashes isn't used. + + +-------------------+------------------+ | Regular String | Raw string | +===================+==================+ @@ -457,10 +466,16 @@ In actual programs, the most common style is to store the Two pattern methods return all of the matches for a pattern. :meth:`~re.Pattern.findall` returns a list of matching strings:: - >>> p = re.compile('\d+') + >>> p = re.compile(r'\d+') >>> p.findall('12 drummers drumming, 11 pipers piping, 10 lords a-leaping') ['12', '11', '10'] +The ``r`` prefix, making the literal a raw string literal, is needed in this +example because escape sequences in a normal "cooked" string literal that are +not recognized by Python, as opposed to regular expressions, now result in a +:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`. See +:ref:`the-backslash-plague`. + :meth:`~re.Pattern.findall` has to create the entire list before it can be returned as the result. The :meth:`~re.Pattern.finditer` method returns a sequence of :ref:`match object ` instances as an :term:`iterator`:: @@ -844,7 +859,7 @@ backreferences in a RE. For example, the following RE detects doubled words in a string. :: - >>> p = re.compile(r'(\b\w+)\s+\1') + >>> p = re.compile(r'\b(\w+)\s+\1\b') >>> p.search('Paris in the the spring').group() 'the the' @@ -943,9 +958,9 @@ number of the group. There's naturally a variant that uses the group name instead of the number. This is another Python extension: ``(?P=name)`` indicates that the contents of the group called *name* should again be matched at the current point. The regular expression for finding doubled words, -``(\b\w+)\s+\1`` can also be written as ``(?P\b\w+)\s+(?P=word)``:: +``\b(\w+)\s+\1\b`` can also be written as ``\b(?P\w+)\s+(?P=word)\b``:: - >>> p = re.compile(r'(?P\b\w+)\s+(?P=word)') + >>> p = re.compile(r'\b(?P\w+)\s+(?P=word)\b') >>> p.search('Paris in the the spring').group() 'the the' @@ -1096,11 +1111,11 @@ following calls:: The module-level function :func:`re.split` adds the RE to be used as the first argument, but is otherwise the same. :: - >>> re.split('[\W]+', 'Words, words, words.') + >>> re.split(r'[\W]+', 'Words, words, words.') ['Words', 'words', 'words', ''] - >>> re.split('([\W]+)', 'Words, words, words.') + >>> re.split(r'([\W]+)', 'Words, words, words.') ['Words', ', ', 'words', ', ', 'words', '.', ''] - >>> re.split('[\W]+', 'Words, words, words.', 1) + >>> re.split(r'[\W]+', 'Words, words, words.', 1) ['Words', 'words, words.'] @@ -1140,12 +1155,12 @@ new string value and the number of replacements that were performed:: >>> p.subn('colour', 'no colours at all') ('no colours at all', 0) -Empty matches are replaced only when they're not adjacent to a previous match. +Empty matches are replaced only when they're not adjacent to a previous empty match. :: >>> p = re.compile('x*') >>> p.sub('-', 'abxd') - '-a-b-d-' + '-a-b--d-' If *replacement* is a string, any backslash escapes in it are processed. That is, ``\n`` is converted to a single newline character, ``\r`` is converted to a diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index 9649b9c609c..093f4454af1 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -214,10 +214,10 @@ difficult reading. `A chronology `_ of the origin and development of Unicode is also available on the site. To help understand the standard, Jukka Korpela has written `an introductory -guide `_ to reading the +guide `_ to reading the Unicode character tables. -Another `good introductory article `_ +Another `good introductory article `_ was written by Joel Spolsky. If this introduction didn't make things clear to you, you should try reading this alternate article before continuing. @@ -463,7 +463,7 @@ The string in this example has the number 57 written in both Thai and Arabic numerals:: import re - p = re.compile('\d+') + p = re.compile(r'\d+') s = "Over \u0e55\u0e57 57 flavours" m = p.search(s) @@ -487,7 +487,7 @@ References Some good alternative discussions of Python's Unicode support are: * `Processing Text Files in Python 3 `_, by Nick Coghlan. -* `Pragmatic Unicode `_, a PyCon 2012 presentation by Ned Batchelder. +* `Pragmatic Unicode `_, a PyCon 2012 presentation by Ned Batchelder. The :class:`str` type is described in the Python library reference at :ref:`textseq`. diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 8d383e03ee8..c1fd5cf0680 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -403,7 +403,7 @@ fetched, particularly the headers sent by the server. It is currently an :class:`http.client.HTTPMessage` instance. Typical headers include 'Content-length', 'Content-type', and so on. See the -`Quick Reference to HTTP Headers `_ +`Quick Reference to HTTP Headers `_ for a useful listing of HTTP headers with brief explanations of their meaning and use. diff --git a/Doc/includes/email-read-alternative.py b/Doc/includes/email-read-alternative.py index 3f5ab24c0fb..5ea84e62584 100644 --- a/Doc/includes/email-read-alternative.py +++ b/Doc/includes/email-read-alternative.py @@ -21,7 +21,7 @@ print('To:', msg['to']) print('From:', msg['from']) print('Subject:', msg['subject']) -# If we want to print a priview of the message content, we can extract whatever +# If we want to print a preview of the message content, we can extract whatever # the least formatted payload is and print the first three lines. Of course, # if the message has no plain text part printing the first three lines of html # is probably useless, but this is just a conceptual example. diff --git a/Doc/includes/shoddy.c b/Doc/includes/shoddy.c index 0c6d412b3c4..0ef47653277 100644 --- a/Doc/includes/shoddy.c +++ b/Doc/includes/shoddy.c @@ -17,7 +17,7 @@ Shoddy_increment(Shoddy *self, PyObject *unused) static PyMethodDef Shoddy_methods[] = { {"increment", (PyCFunction)Shoddy_increment, METH_NOARGS, PyDoc_STR("increment state counter")}, - {NULL, NULL}, + {NULL}, }; static int diff --git a/Doc/install/index.rst b/Doc/install/index.rst index bc080b00c59..0545b8f33d0 100644 --- a/Doc/install/index.rst +++ b/Doc/install/index.rst @@ -36,7 +36,7 @@ modules and extensions. This guide only covers the basic tools for building and distributing extensions that are provided as part of this version of Python. Third party tools offer easier to use and more secure alternatives. Refer to the `quick - recommendations section `__ + recommendations section `__ in the Python Packaging User Guide for more information. diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index 09bb8251c35..f9a224be92b 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -48,7 +48,7 @@ Key terms repository of open source licensed packages made available for use by other Python users. * the `Python Packaging Authority - `__ are the group of + `__ are the group of developers and documentation authors responsible for the maintenance and evolution of the standard packaging tools and the associated metadata and file format standards. They maintain a variety of tools, documentation, diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst index faf06d91c3d..deb5e10f6ef 100644 --- a/Doc/library/2to3.rst +++ b/Doc/library/2to3.rst @@ -351,7 +351,7 @@ and off individually. They are described here in more detail. ================================== ============================================= From To ================================== ============================================= - ``operator.isCallable(obj)`` ``hasattr(obj, '__call__')`` + ``operator.isCallable(obj)`` ``callable(obj)`` ``operator.sequenceIncludes(obj)`` ``operator.contains(obj)`` ``operator.isSequenceType(obj)`` ``isinstance(obj, collections.abc.Sequence)`` ``operator.isMappingType(obj)`` ``isinstance(obj, collections.abc.Mapping)`` diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index 73d8b6b7e8a..e3d749e6017 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -90,6 +90,11 @@ language using this mechanism: | generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | | | | | *StopIteration handling inside generators* | +------------------+-------------+--------------+---------------------------------------------+ +| annotations | 3.7.0b1 | 4.0 | :pep:`563`: | +| | | | *Postponed evaluation of annotations* | ++------------------+-------------+--------------+---------------------------------------------+ + +.. XXX Adding a new entry? Remember to update simple_stmts.rst, too. .. seealso:: diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index bda46f46c43..b7f610ba8b2 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -261,5 +261,5 @@ and classes for traversing abstract syntax trees: .. seealso:: - `Green Tree Snakes `_, an external documentation resource, has good + `Green Tree Snakes `_, an external documentation resource, has good details on working with Python ASTs. diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index b2ad87b5d02..100fff561c5 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -81,12 +81,11 @@ is called. If you wait for a future, you should check early if the future was cancelled to avoid useless operations. Example:: - @coroutine - def slow_operation(fut): + async def slow_operation(fut): if fut.cancelled(): return # ... slow computation ... - yield from fut + await fut # ... The :func:`shield` function can also be used to ignore cancellation. @@ -99,7 +98,7 @@ Concurrency and multithreading An event loop runs in a thread and executes all callbacks and tasks in the same thread. While a task is running in the event loop, no other task is running in -the same thread. But when the task uses ``yield from``, the task is suspended +the same thread. But when the task uses ``await``, the task is suspended and the event loop executes the next task. To schedule a callback from a different thread, the @@ -192,8 +191,7 @@ Example with the bug:: import asyncio - @asyncio.coroutine - def test(): + async def test(): print("never scheduled") test() @@ -216,9 +214,9 @@ The fix is to call the :func:`ensure_future` function or the Detect exceptions never consumed -------------------------------- -Python usually calls :func:`sys.displayhook` on unhandled exceptions. If +Python usually calls :func:`sys.excepthook` on unhandled exceptions. If :meth:`Future.set_exception` is called, but the exception is never consumed, -:func:`sys.displayhook` is not called. Instead, :ref:`a log is emitted +:func:`sys.excepthook` is not called. Instead, :ref:`a log is emitted ` when the future is deleted by the garbage collector, with the traceback where the exception was raised. @@ -270,10 +268,9 @@ traceback where the task was created. Output in debug mode:: There are different options to fix this issue. The first option is to chain the coroutine in another coroutine and use classic try/except:: - @asyncio.coroutine - def handle_exception(): + async def handle_exception(): try: - yield from bug() + await bug() except Exception: print("exception consumed") @@ -300,7 +297,7 @@ Chain coroutines correctly -------------------------- When a coroutine function calls other coroutine functions and tasks, they -should be chained explicitly with ``yield from``. Otherwise, the execution is +should be chained explicitly with ``await``. Otherwise, the execution is not guaranteed to be sequential. Example with different bugs using :func:`asyncio.sleep` to simulate slow @@ -308,26 +305,22 @@ operations:: import asyncio - @asyncio.coroutine - def create(): - yield from asyncio.sleep(3.0) + async def create(): + await asyncio.sleep(3.0) print("(1) create file") - @asyncio.coroutine - def write(): - yield from asyncio.sleep(1.0) + async def write(): + await asyncio.sleep(1.0) print("(2) write into file") - @asyncio.coroutine - def close(): + async def close(): print("(3) close file") - @asyncio.coroutine - def test(): + async def test(): asyncio.ensure_future(create()) asyncio.ensure_future(write()) asyncio.ensure_future(close()) - yield from asyncio.sleep(2.0) + await asyncio.sleep(2.0) loop.stop() loop = asyncio.get_event_loop() @@ -359,24 +352,22 @@ The loop stopped before the ``create()`` finished, ``close()`` has been called before ``write()``, whereas coroutine functions were called in this order: ``create()``, ``write()``, ``close()``. -To fix the example, tasks must be marked with ``yield from``:: +To fix the example, tasks must be marked with ``await``:: - @asyncio.coroutine - def test(): - yield from asyncio.ensure_future(create()) - yield from asyncio.ensure_future(write()) - yield from asyncio.ensure_future(close()) - yield from asyncio.sleep(2.0) + async def test(): + await asyncio.ensure_future(create()) + await asyncio.ensure_future(write()) + await asyncio.ensure_future(close()) + await asyncio.sleep(2.0) loop.stop() Or without ``asyncio.ensure_future()``:: - @asyncio.coroutine - def test(): - yield from create() - yield from write() - yield from close() - yield from asyncio.sleep(2.0) + async def test(): + await create() + await write() + await close() + await asyncio.sleep(2.0) loop.stop() diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index e635cba659a..a19c670d0f3 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -171,7 +171,7 @@ a different clock than :func:`time.time`. Arrange for the *callback* to be called after the given *delay* seconds (either an int or float). - An instance of :class:`asyncio.Handle` is returned, which can be + An instance of :class:`asyncio.TimerHandle` is returned, which can be used to cancel the callback. *callback* will be called exactly once per call to :meth:`call_later`. @@ -193,7 +193,7 @@ a different clock than :func:`time.time`. This method's behavior is the same as :meth:`call_later`. - An instance of :class:`asyncio.Handle` is returned, which can be + An instance of :class:`asyncio.TimerHandle` is returned, which can be used to cancel the callback. :ref:`Use functools.partial to pass keywords to the callback @@ -235,9 +235,6 @@ Tasks interoperability. In this case, the result type is a subclass of :class:`Task`. - This method was added in Python 3.4.2. Use the :func:`async` function to - support also older Python versions. - .. versionadded:: 3.4.2 .. method:: AbstractEventLoop.set_task_factory(factory) @@ -264,7 +261,7 @@ Tasks Creating connections -------------------- -.. coroutinemethod:: AbstractEventLoop.create_connection(protocol_factory, host=None, port=None, \*, ssl=None, family=0, proto=0, flags=0, sock=None, local_addr=None, server_hostname=None) +.. coroutinemethod:: AbstractEventLoop.create_connection(protocol_factory, host=None, port=None, \*, ssl=None, family=0, proto=0, flags=0, sock=None, local_addr=None, server_hostname=None, ssl_handshake_timeout=None) Create a streaming transport connection to a given Internet *host* and *port*: socket family :py:data:`~socket.AF_INET` or @@ -272,9 +269,8 @@ Creating connections socket type :py:data:`~socket.SOCK_STREAM`. *protocol_factory* must be a callable returning a :ref:`protocol ` instance. - This method is a :ref:`coroutine ` which will try to - establish the connection in the background. When successful, the - coroutine returns a ``(transport, protocol)`` pair. + This method will try to establish the connection in the background. + When successful, it returns a ``(transport, protocol)`` pair. The chronological synopsis of the underlying operation is as follows: @@ -329,6 +325,14 @@ Creating connections to bind the socket to locally. The *local_host* and *local_port* are looked up using getaddrinfo(), similarly to *host* and *port*. + * *ssl_handshake_timeout* is (for an SSL connection) the time in seconds + to wait for the SSL handshake to complete before aborting the connection. + ``10.0`` seconds if ``None`` (default). + + .. versionadded:: 3.7 + + The *ssl_handshake_timeout* parameter. + .. versionchanged:: 3.5 On Windows with :class:`ProactorEventLoop`, SSL/TLS is now supported. @@ -347,9 +351,8 @@ Creating connections :py:data:`~socket.SOCK_DGRAM`. *protocol_factory* must be a callable returning a :ref:`protocol ` instance. - This method is a :ref:`coroutine ` which will try to - establish the connection in the background. When successful, the - coroutine returns a ``(transport, protocol)`` pair. + This method will try to establish the connection in the background. + When successful, the it returns a ``(transport, protocol)`` pair. Options changing how the connection is created: @@ -391,30 +394,37 @@ Creating connections :ref:`UDP echo server protocol ` examples. -.. coroutinemethod:: AbstractEventLoop.create_unix_connection(protocol_factory, path, \*, ssl=None, sock=None, server_hostname=None) +.. coroutinemethod:: AbstractEventLoop.create_unix_connection(protocol_factory, path=None, \*, ssl=None, sock=None, server_hostname=None, ssl_handshake_timeout=None) Create UNIX connection: socket family :py:data:`~socket.AF_UNIX`, socket type :py:data:`~socket.SOCK_STREAM`. The :py:data:`~socket.AF_UNIX` socket family is used to communicate between processes on the same machine efficiently. - This method is a :ref:`coroutine ` which will try to - establish the connection in the background. When successful, the - coroutine returns a ``(transport, protocol)`` pair. + This method will try to establish the connection in the background. + When successful, the it returns a ``(transport, protocol)`` pair. *path* is the name of a UNIX domain socket, and is required unless a *sock* - parameter is specified. Abstract UNIX sockets, :class:`str`, and - :class:`bytes` paths are supported. + parameter is specified. Abstract UNIX sockets, :class:`str`, + :class:`bytes`, and :class:`~pathlib.Path` paths are supported. See the :meth:`AbstractEventLoop.create_connection` method for parameters. Availability: UNIX. + .. versionadded:: 3.7 + + The *ssl_handshake_timeout* parameter. + + .. versionchanged:: 3.7 + + The *path* parameter can now be a :class:`~pathlib.Path` object. + Creating listening connections ------------------------------ -.. coroutinemethod:: AbstractEventLoop.create_server(protocol_factory, host=None, port=None, \*, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None, reuse_port=None) +.. coroutinemethod:: AbstractEventLoop.create_server(protocol_factory, host=None, port=None, \*, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None, reuse_port=None, ssl_handshake_timeout=None, start_serving=True) Create a TCP server (socket type :data:`~socket.SOCK_STREAM`) bound to *host* and *port*. @@ -458,7 +468,19 @@ Creating listening connections set this flag when being created. This option is not supported on Windows. - This method is a :ref:`coroutine `. + * *ssl_handshake_timeout* is (for an SSL server) the time in seconds to wait + for the SSL handshake to complete before aborting the connection. + ``10.0`` seconds if ``None`` (default). + + * *start_serving* set to ``True`` (the default) causes the created server + to start accepting connections immediately. When set to ``False``, + the user should await on :meth:`Server.start_serving` or + :meth:`Server.serve_forever` to make the server to start accepting + connections. + + .. versionadded:: 3.7 + + *ssl_handshake_timeout* and *start_serving* parameters. .. versionchanged:: 3.5 @@ -474,16 +496,26 @@ Creating listening connections The *host* parameter can now be a sequence of strings. -.. coroutinemethod:: AbstractEventLoop.create_unix_server(protocol_factory, path=None, \*, sock=None, backlog=100, ssl=None) +.. coroutinemethod:: AbstractEventLoop.create_unix_server(protocol_factory, path=None, \*, sock=None, backlog=100, ssl=None, ssl_handshake_timeout=None, start_serving=True) Similar to :meth:`AbstractEventLoop.create_server`, but specific to the socket family :py:data:`~socket.AF_UNIX`. - This method is a :ref:`coroutine `. + *path* is the name of a UNIX domain socket, and is required unless a *sock* + parameter is specified. Abstract UNIX sockets, :class:`str`, + :class:`bytes`, and :class:`~pathlib.Path` paths are supported. Availability: UNIX. -.. coroutinemethod:: BaseEventLoop.connect_accepted_socket(protocol_factory, sock, \*, ssl=None) + .. versionadded:: 3.7 + + The *ssl_handshake_timeout* parameter. + + .. versionchanged:: 3.7 + + The *path* parameter can now be a :class:`~pathlib.Path` object. + +.. coroutinemethod:: BaseEventLoop.connect_accepted_socket(protocol_factory, sock, \*, ssl=None, ssl_handshake_timeout=None) Handle an accepted connection. @@ -498,8 +530,81 @@ Creating listening connections * *ssl* can be set to an :class:`~ssl.SSLContext` to enable SSL over the accepted connections. - This method is a :ref:`coroutine `. When completed, the - coroutine returns a ``(transport, protocol)`` pair. + * *ssl_handshake_timeout* is (for an SSL connection) the time in seconds to + wait for the SSL handshake to complete before aborting the connection. + ``10.0`` seconds if ``None`` (default). + + When completed it returns a ``(transport, protocol)`` pair. + + .. versionadded:: 3.7 + + The *ssl_handshake_timeout* parameter. + + .. versionadded:: 3.5.3 + + +File Transferring +----------------- + +.. coroutinemethod:: AbstractEventLoop.sendfile(transport, file, \ + offset=0, count=None, \ + *, fallback=True) + + Send a *file* to *transport*, return the total number of bytes + which were sent. + + The method uses high-performance :meth:`os.sendfile` if available. + + *file* must be a regular file object opened in binary mode. + + *offset* tells from where to start reading the file. If specified, + *count* is the total number of bytes to transmit as opposed to + sending the file until EOF is reached. File position is updated on + return or also in case of error in which case :meth:`file.tell() + ` can be used to figure out the number of bytes + which were sent. + + *fallback* set to ``True`` makes asyncio to manually read and send + the file when the platform does not support the sendfile syscall + (e.g. Windows or SSL socket on Unix). + + Raise :exc:`SendfileNotAvailableError` if the system does not support + *sendfile* syscall and *fallback* is ``False``. + + .. versionadded:: 3.7 + + +TLS Upgrade +----------- + +.. coroutinemethod:: AbstractEventLoop.start_tls(transport, protocol, sslcontext, \*, server_side=False, server_hostname=None, ssl_handshake_timeout=None) + + Upgrades an existing connection to TLS. + + Returns a new transport instance, that the *protocol* must start using + immediately after the *await*. The *transport* instance passed to + the *start_tls* method should never be used again. + + Parameters: + + * *transport* and *protocol* instances that methods like + :meth:`~AbstractEventLoop.create_server` and + :meth:`~AbstractEventLoop.create_connection` return. + + * *sslcontext*: a configured instance of :class:`~ssl.SSLContext`. + + * *server_side* pass ``True`` when a server-side connection is being + upgraded (like the one created by :meth:`~AbstractEventLoop.create_server`). + + * *server_hostname*: sets or overrides the host name that the target + server's certificate will be matched against. + + * *ssl_handshake_timeout* is (for an SSL connection) the time in seconds to + wait for the SSL handshake to complete before aborting the connection. + ``10.0`` seconds if ``None`` (default). + + .. versionadded:: 3.7 + Watch file descriptors ---------------------- @@ -553,7 +658,10 @@ Low-level socket operations With :class:`SelectorEventLoop` event loop, the socket *sock* must be non-blocking. - This method is a :ref:`coroutine `. + .. versionchanged:: 3.7 + Even though the method was always documented as a coroutine + method, before Python 3.7 it returned a :class:`Future`. + Since Python 3.7, this is an ``async def`` method. .. coroutinemethod:: AbstractEventLoop.sock_recv_into(sock, buf) @@ -566,8 +674,6 @@ Low-level socket operations With :class:`SelectorEventLoop` event loop, the socket *sock* must be non-blocking. - This method is a :ref:`coroutine `. - .. versionadded:: 3.7 .. coroutinemethod:: AbstractEventLoop.sock_sendall(sock, data) @@ -584,7 +690,10 @@ Low-level socket operations With :class:`SelectorEventLoop` event loop, the socket *sock* must be non-blocking. - This method is a :ref:`coroutine `. + .. versionchanged:: 3.7 + Even though the method was always documented as a coroutine + method, before Python 3.7 it returned an :class:`Future`. + Since Python 3.7, this is an ``async def`` method. .. coroutinemethod:: AbstractEventLoop.sock_connect(sock, address) @@ -594,8 +703,6 @@ Low-level socket operations With :class:`SelectorEventLoop` event loop, the socket *sock* must be non-blocking. - This method is a :ref:`coroutine `. - .. versionchanged:: 3.5.2 ``address`` no longer needs to be resolved. ``sock_connect`` will try to check if the *address* is already resolved by calling @@ -622,12 +729,45 @@ Low-level socket operations The socket *sock* must be non-blocking. - This method is a :ref:`coroutine `. + .. versionchanged:: 3.7 + Even though the method was always documented as a coroutine + method, before Python 3.7 it returned a :class:`Future`. + Since Python 3.7, this is an ``async def`` method. .. seealso:: :meth:`AbstractEventLoop.create_server` and :func:`start_server`. +.. coroutinemethod:: AbstractEventLoop.sock_sendfile(sock, file, \ + offset=0, count=None, \ + *, fallback=True) + + Send a file using high-performance :mod:`os.sendfile` if possible + and return the total number of bytes which were sent. + + Asynchronous version of :meth:`socket.socket.sendfile`. + + *sock* must be non-blocking :class:`~socket.socket` of + :const:`socket.SOCK_STREAM` type. + + *file* must be a regular file object opened in binary mode. + + *offset* tells from where to start reading the file. If specified, + *count* is the total number of bytes to transmit as opposed to + sending the file until EOF is reached. File position is updated on + return or also in case of error in which case :meth:`file.tell() + ` can be used to figure out the number of bytes + which were sent. + + *fallback* set to ``True`` makes asyncio to manually read and send + the file when the platform does not support the sendfile syscall + (e.g. Windows or SSL socket on Unix). + + Raise :exc:`SendfileNotAvailableError` if the system does not support + *sendfile* syscall and *fallback* is ``False``. + + .. versionadded:: 3.7 + Resolve host name ----------------- @@ -642,6 +782,12 @@ Resolve host name This method is a :ref:`coroutine `, similar to :meth:`socket.getnameinfo` function but non-blocking. +.. versionchanged:: 3.7 + Both *getaddrinfo* and *getnameinfo* methods were always documented + to return a coroutine, but prior to Python 3.7 they were, in fact, + returning :class:`asyncio.Future` objects. Starting with Python 3.7 + both methods are coroutines. + Connect pipes ------------- @@ -661,8 +807,6 @@ Use :class:`ProactorEventLoop` to support pipes on Windows. With :class:`SelectorEventLoop` event loop, the *pipe* is set to non-blocking mode. - This method is a :ref:`coroutine `. - .. coroutinemethod:: AbstractEventLoop.connect_write_pipe(protocol_factory, pipe) Register write pipe in eventloop. @@ -675,8 +819,6 @@ Use :class:`ProactorEventLoop` to support pipes on Windows. With :class:`SelectorEventLoop` event loop, the *pipe* is set to non-blocking mode. - This method is a :ref:`coroutine `. - .. seealso:: The :meth:`AbstractEventLoop.subprocess_exec` and @@ -716,7 +858,7 @@ Call a function in an :class:`~concurrent.futures.Executor` (pool of threads or pool of processes). By default, an event loop uses a thread pool executor (:class:`~concurrent.futures.ThreadPoolExecutor`). -.. coroutinemethod:: AbstractEventLoop.run_in_executor(executor, func, \*args) +.. method:: AbstractEventLoop.run_in_executor(executor, func, \*args) Arrange for a *func* to be called in the specified executor. @@ -726,7 +868,7 @@ pool of processes). By default, an event loop uses a thread pool executor :ref:`Use functools.partial to pass keywords to the *func* `. - This method is a :ref:`coroutine `. + This method returns a :class:`asyncio.Future` object. .. versionchanged:: 3.5.3 :meth:`BaseEventLoop.run_in_executor` no longer configures the @@ -827,8 +969,26 @@ Server Server listening on sockets. - Object created by the :meth:`AbstractEventLoop.create_server` method and the - :func:`start_server` function. Don't instantiate the class directly. + Object created by :meth:`AbstractEventLoop.create_server`, + :meth:`AbstractEventLoop.create_unix_server`, :func:`start_server`, + and :func:`start_unix_server` functions. Don't instantiate the class + directly. + + *Server* objects are asynchronous context managers. When used in an + ``async with`` statement, it's guaranteed that the Server object is + closed and not accepting new connections when the ``async with`` + statement is completed:: + + srv = await loop.create_server(...) + + async with srv: + # some code + + # At this point, srv is closed and no longer accepts new connections. + + + .. versionchanged:: 3.7 + Server object is an asynchronous context manager since Python 3.7. .. method:: close() @@ -841,17 +1001,74 @@ Server The server is closed asynchronously, use the :meth:`wait_closed` coroutine to wait until the server is closed. + .. method:: get_loop() + + Gives the event loop associated with the server object. + + .. versionadded:: 3.7 + + .. coroutinemethod:: start_serving() + + Start accepting connections. + + This method is idempotent, so it can be called when + the server is already being serving. + + The new *start_serving* keyword-only parameter to + :meth:`AbstractEventLoop.create_server` and + :meth:`asyncio.start_server` allows to create a Server object + that is not accepting connections right away. In which case + this method, or :meth:`Server.serve_forever` can be used + to make the Server object to start accepting connections. + + .. versionadded:: 3.7 + + .. coroutinemethod:: serve_forever() + + Start accepting connections until the coroutine is cancelled. + Cancellation of ``serve_forever`` task causes the server + to be closed. + + This method can be called if the server is already accepting + connections. Only one ``serve_forever`` task can exist per + one *Server* object. + + Example:: + + async def client_connected(reader, writer): + # Communicate with the client with + # reader/writer streams. For example: + await reader.readline() + + async def main(host, port): + srv = await asyncio.start_server( + client_connected, host, port) + await loop.serve_forever() + + asyncio.run(main('127.0.0.1', 0)) + + .. versionadded:: 3.7 + + .. method:: is_serving() + + Return ``True`` if the server is accepting new connections. + + .. versionadded:: 3.7 + .. coroutinemethod:: wait_closed() Wait until the :meth:`close` method completes. - This method is a :ref:`coroutine `. - .. attribute:: sockets List of :class:`socket.socket` objects the server is listening to, or ``None`` if the server is closed. + .. versionchanged:: 3.7 + Prior to Python 3.7 ``Server.sockets`` used to return the + internal list of server's sockets directly. In 3.7 a copy + of that list is returned. + Handle ------ @@ -859,8 +1076,7 @@ Handle .. class:: Handle A callback wrapper object returned by :func:`AbstractEventLoop.call_soon`, - :func:`AbstractEventLoop.call_soon_threadsafe`, :func:`AbstractEventLoop.call_later`, - and :func:`AbstractEventLoop.call_at`. + :func:`AbstractEventLoop.call_soon_threadsafe`. .. method:: cancel() @@ -873,6 +1089,34 @@ Handle .. versionadded:: 3.7 +.. class:: TimerHandle + + A callback wrapper object returned by :func:`AbstractEventLoop.call_later`, + and :func:`AbstractEventLoop.call_at`. + + The class is inherited from :class:`Handle`. + + .. method:: when() + + Return a scheduled callback time as :class:`float` seconds. + + The time is an absolute timestamp, using the same time + reference as :meth:`AbstractEventLoop.time`. + + .. versionadded:: 3.7 + + +SendfileNotAvailableError +------------------------- + + +.. exception:: SendfileNotAvailableError + + Sendfile syscall is not available, subclass of :exc:`RuntimeError`. + + Raised if the OS does not support senfile syscall for + given socket or file type. + Event loop examples ------------------- @@ -952,10 +1196,7 @@ Wait until a file descriptor received some data using the :meth:`AbstractEventLoop.add_reader` method and then close the event loop:: import asyncio - try: - from socket import socketpair - except ImportError: - from asyncio.windows_utils import socketpair + from socket import socketpair # Create a pair of connected file descriptors rsock, wsock = socketpair() diff --git a/Doc/library/asyncio-eventloops.rst b/Doc/library/asyncio-eventloops.rst index 7970e9039df..3051fde5b93 100644 --- a/Doc/library/asyncio-eventloops.rst +++ b/Doc/library/asyncio-eventloops.rst @@ -25,6 +25,13 @@ the execution of the process. Equivalent to calling ``get_event_loop_policy().new_event_loop()``. +.. function:: get_running_loop() + + Return the running event loop in the current OS thread. If there + is no running event loop a :exc:`RuntimeError` is raised. + + .. versionadded:: 3.7 + .. _asyncio-event-loops: @@ -189,10 +196,15 @@ An event loop policy must implement the following interface: The default policy defines context as the current thread, and manages an event -loop per thread that interacts with :mod:`asyncio`. If the current thread -doesn't already have an event loop associated with it, the default policy's -:meth:`~AbstractEventLoopPolicy.get_event_loop` method creates one when -called from the main thread, but raises :exc:`RuntimeError` otherwise. +loop per thread that interacts with :mod:`asyncio`. An exception to this rule +happens when :meth:`~AbstractEventLoopPolicy.get_event_loop` is called from a +running future/coroutine, in which case it will return the current loop +running that future/coroutine. + +If the current thread doesn't already have an event loop associated with it, +the default policy's :meth:`~AbstractEventLoopPolicy.get_event_loop` method +creates one when called from the main thread, but raises :exc:`RuntimeError` +otherwise. Access to the global loop policy diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index cd84ae76b5d..004cac80d90 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -118,17 +118,31 @@ ReadTransport Interface for read-only transports. + .. method:: is_reading() + + Return ``True`` if the transport is receiving new data. + + .. versionadded:: 3.7 + .. method:: pause_reading() Pause the receiving end of the transport. No data will be passed to the protocol's :meth:`data_received` method until :meth:`resume_reading` is called. + .. versionchanged:: 3.7 + The method is idempotent, i.e. it can be called when the + transport is already paused or closed. + .. method:: resume_reading() Resume the receiving end. The protocol's :meth:`data_received` method will be called once again if some data is available for reading. + .. versionchanged:: 3.7 + The method is idempotent, i.e. it can be called when the + transport is already reading. + WriteTransport -------------- @@ -319,6 +333,16 @@ Protocol classes The base class for implementing streaming protocols (for use with e.g. TCP and SSL transports). +.. class:: BufferedProtocol + + A base class for implementing streaming protocols with manual + control of the receive buffer. + + .. versionadded:: 3.7 + **Important:** this has been been added to asyncio in Python 3.7 + *on a provisional basis*! Treat it as an experimental API that + might be changed or removed in Python 3.8. + .. class:: DatagramProtocol The base class for implementing datagram protocols (for use with @@ -414,10 +438,68 @@ and, if called, :meth:`data_received` won't be called after it. State machine: - start -> :meth:`~BaseProtocol.connection_made` - [-> :meth:`~Protocol.data_received` \*] - [-> :meth:`~Protocol.eof_received` ?] - -> :meth:`~BaseProtocol.connection_lost` -> end +.. code-block:: none + + start -> connection_made + [-> data_received]* + [-> eof_received]? + -> connection_lost -> end + + +Streaming protocols with manual receive buffer control +------------------------------------------------------ + +.. versionadded:: 3.7 + **Important:** :class:`BufferedProtocol` has been been added to + asyncio in Python 3.7 *on a provisional basis*! Consider it as an + experimental API that might be changed or removed in Python 3.8. + + +Event methods, such as :meth:`AbstractEventLoop.create_server` and +:meth:`AbstractEventLoop.create_connection`, accept factories that +return protocols that implement this interface. + +The idea of BufferedProtocol is that it allows to manually allocate +and control the receive buffer. Event loops can then use the buffer +provided by the protocol to avoid unnecessary data copies. This +can result in noticeable performance improvement for protocols that +receive big amounts of data. Sophisticated protocols can allocate +the buffer only once at creation time. + +The following callbacks are called on :class:`BufferedProtocol` +instances: + +.. method:: BufferedProtocol.get_buffer() + + Called to allocate a new receive buffer. Must return an object + that implements the :ref:`buffer protocol `. + +.. method:: BufferedProtocol.buffer_updated(nbytes) + + Called when the buffer was updated with the received data. + + *nbytes* is the total number of bytes that were written to the buffer. + +.. method:: BufferedProtocol.eof_received() + + See the documentation of the :meth:`Protocol.eof_received` method. + + +:meth:`get_buffer` can be called an arbitrary number of times during +a connection. However, :meth:`eof_received` is called at most once +and, if called, :meth:`get_buffer` and :meth:`buffer_updated` +won't be called after it. + +State machine: + +.. code-block:: none + + start -> connection_made + [-> get_buffer + [-> buffer_updated]? + ]* + [-> eof_received]? + -> connection_lost -> end Datagram protocols @@ -488,8 +570,9 @@ Coroutines can be scheduled in a protocol method using :func:`ensure_future`, but there is no guarantee made about the execution order. Protocols are not aware of coroutines created in protocol methods and so will not wait for them. -To have a reliable execution order, use :ref:`stream objects ` in a -coroutine with ``yield from``. For example, the :meth:`StreamWriter.drain` +To have a reliable execution order, +use :ref:`stream objects ` in a +coroutine with ``await``. For example, the :meth:`StreamWriter.drain` coroutine can be used to wait until the write buffer is flushed. @@ -589,7 +672,7 @@ received data and close the connection:: :meth:`Transport.close` can be called immediately after :meth:`WriteTransport.write` even if data are not sent yet on the socket: both -methods are asynchronous. ``yield from`` is not needed because these transport +methods are asynchronous. ``await`` is not needed because these transport methods are not coroutines. .. seealso:: @@ -690,10 +773,7 @@ Wait until a socket receives data using the the event loop :: import asyncio - try: - from socket import socketpair - except ImportError: - from asyncio.windows_utils import socketpair + from socket import socketpair # Create a pair of connected sockets rsock, wsock = socketpair() diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index ea787550082..65497f29d89 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -24,7 +24,7 @@ Queue A queue, useful for coordinating producer and consumer coroutines. If *maxsize* is less than or equal to zero, the queue size is infinite. If - it is an integer greater than ``0``, then ``yield from put()`` will block + it is an integer greater than ``0``, then ``await put()`` will block when the queue reaches *maxsize*, until an item is removed by :meth:`get`. Unlike the standard library :mod:`queue`, you can reliably know this Queue's diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index 491afdd610c..099b59ee582 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -201,6 +201,21 @@ StreamWriter Close the transport: see :meth:`BaseTransport.close`. + .. method:: is_closing() + + Return ``True`` if the writer is closing or is closed. + + .. versionadded:: 3.7 + + .. coroutinemethod:: wait_closed() + + Wait until the writer is closed. + + Should be called after :meth:`close` to wait until the underlying + connection (and the associated transport/protocol pair) is closed. + + .. versionadded:: 3.7 + .. coroutinemethod:: drain() Let the write buffer of the underlying transport a chance to be flushed. @@ -208,7 +223,7 @@ StreamWriter The intended use is to write:: w.write(data) - yield from w.drain() + await w.drain() When the size of the transport buffer reaches the high-water limit (the protocol is paused), block until the size of the buffer is drained down @@ -301,15 +316,14 @@ TCP echo client using the :func:`asyncio.open_connection` function:: import asyncio - @asyncio.coroutine - def tcp_echo_client(message, loop): - reader, writer = yield from asyncio.open_connection('127.0.0.1', 8888, - loop=loop) + async def tcp_echo_client(message, loop): + reader, writer = await asyncio.open_connection('127.0.0.1', 8888, + loop=loop) print('Send: %r' % message) writer.write(message.encode()) - data = yield from reader.read(100) + data = await reader.read(100) print('Received: %r' % data.decode()) print('Close the socket') @@ -335,16 +349,15 @@ TCP echo server using the :func:`asyncio.start_server` function:: import asyncio - @asyncio.coroutine - def handle_echo(reader, writer): - data = yield from reader.read(100) + async def handle_echo(reader, writer): + data = await reader.read(100) message = data.decode() addr = writer.get_extra_info('peername') print("Received %r from %r" % (message, addr)) print("Send: %r" % message) writer.write(data) - yield from writer.drain() + await writer.drain() print("Close the client socket") writer.close() @@ -387,13 +400,13 @@ Simple example querying HTTP headers of the URL passed on the command line:: connect = asyncio.open_connection(url.hostname, 443, ssl=True) else: connect = asyncio.open_connection(url.hostname, 80) - reader, writer = yield from connect + reader, writer = await connect query = ('HEAD {path} HTTP/1.0\r\n' 'Host: {hostname}\r\n' '\r\n').format(path=url.path or '/', hostname=url.hostname) writer.write(query.encode('latin-1')) while True: - line = yield from reader.readline() + line = await reader.readline() if not line: break line = line.decode('latin1').rstrip() @@ -426,24 +439,20 @@ Coroutine waiting until a socket receives data using the :func:`open_connection` function:: import asyncio - try: - from socket import socketpair - except ImportError: - from asyncio.windows_utils import socketpair + from socket import socketpair - @asyncio.coroutine - def wait_for_data(loop): + async def wait_for_data(loop): # Create a pair of connected sockets rsock, wsock = socketpair() # Register the open socket to wait for data - reader, writer = yield from asyncio.open_connection(sock=rsock, loop=loop) + reader, writer = await asyncio.open_connection(sock=rsock, loop=loop) # Simulate the reception of data from the network loop.call_soon(wsock.send, 'abc'.encode()) # Wait for data - data = yield from reader.read(100) + data = await reader.read(100) # Got data, we are done: close the socket print("Received:", data.decode()) diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 1c1d0be918d..280b7640037 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -347,21 +347,20 @@ wait for the subprocess exit. The subprocess is created by the def process_exited(self): self.exit_future.set_result(True) - @asyncio.coroutine - def get_date(loop): + async def get_date(loop): code = 'import datetime; print(datetime.datetime.now())' exit_future = asyncio.Future(loop=loop) # Create the subprocess controlled by the protocol DateProtocol, # redirect the standard output into a pipe - create = loop.subprocess_exec(lambda: DateProtocol(exit_future), - sys.executable, '-c', code, - stdin=None, stderr=None) - transport, protocol = yield from create + transport, protocol = await loop.subprocess_exec( + lambda: DateProtocol(exit_future), + sys.executable, '-c', code, + stdin=None, stderr=None) # Wait for the subprocess exit using the process_exited() method # of the protocol - yield from exit_future + await exit_future # Close the stdout pipe transport.close() @@ -398,16 +397,16 @@ function:: code = 'import datetime; print(datetime.datetime.now())' # Create the subprocess, redirect the standard output into a pipe - create = asyncio.create_subprocess_exec(sys.executable, '-c', code, - stdout=asyncio.subprocess.PIPE) - proc = yield from create + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdout=asyncio.subprocess.PIPE) # Read one line of output - data = yield from proc.stdout.readline() + data = await proc.stdout.readline() line = data.decode('ascii').rstrip() # Wait for the subprocess exit - yield from proc.wait() + await proc.wait() return line if sys.platform == "win32": diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 14e3defbf41..3e574f41e80 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -23,11 +23,9 @@ module (:class:`~threading.Lock`, :class:`~threading.Event`, :class:`~threading.BoundedSemaphore`), but it has no *timeout* parameter. The :func:`asyncio.wait_for` function can be used to cancel a task after a timeout. -Locks ------ Lock -^^^^ +---- .. class:: Lock(\*, loop=None) @@ -37,8 +35,9 @@ Lock particular coroutine when locked. A primitive lock is in one of two states, 'locked' or 'unlocked'. - It is created in the unlocked state. It has two basic methods, :meth:`acquire` - and :meth:`release`. When the state is unlocked, acquire() changes the state to + The lock is created in the unlocked state. + It has two basic methods, :meth:`acquire` and :meth:`release`. + When the state is unlocked, acquire() changes the state to locked and returns immediately. When the state is locked, acquire() blocks until a call to release() in another coroutine changes it to unlocked, then the acquire() call resets it to locked and returns. The release() method @@ -51,38 +50,12 @@ Lock resets the state to unlocked; first coroutine which is blocked in acquire() is being processed. - :meth:`acquire` is a coroutine and should be called with ``yield from``. + :meth:`acquire` is a coroutine and should be called with ``await``. - Locks also support the context management protocol. ``(yield from lock)`` - should be used as the context manager expression. + Locks support the :ref:`context management protocol `. This class is :ref:`not thread safe `. - Usage:: - - lock = Lock() - ... - yield from lock - try: - ... - finally: - lock.release() - - Context manager usage:: - - lock = Lock() - ... - with (yield from lock): - ... - - Lock objects can be tested for locking state:: - - if not lock.locked(): - yield from lock - else: - # lock is acquired - ... - .. method:: locked() Return ``True`` if the lock is acquired. @@ -110,7 +83,7 @@ Lock Event -^^^^^ +----- .. class:: Event(\*, loop=None) @@ -151,7 +124,7 @@ Event Condition -^^^^^^^^^ +--------- .. class:: Condition(lock=None, \*, loop=None) @@ -166,6 +139,9 @@ Condition object, and it is used as the underlying lock. Otherwise, a new :class:`Lock` object is created and used as the underlying lock. + Conditions support the :ref:`context management protocol + `. + This class is :ref:`not thread safe `. .. coroutinemethod:: acquire() @@ -239,11 +215,8 @@ Condition This method is a :ref:`coroutine `. -Semaphores ----------- - Semaphore -^^^^^^^^^ +--------- .. class:: Semaphore(value=1, \*, loop=None) @@ -254,12 +227,13 @@ Semaphore counter can never go below zero; when :meth:`acquire` finds that it is zero, it blocks, waiting until some other coroutine calls :meth:`release`. - Semaphores also support the context management protocol. - The optional argument gives the initial value for the internal counter; it defaults to ``1``. If the value given is less than ``0``, :exc:`ValueError` is raised. + Semaphores support the :ref:`context management protocol + `. + This class is :ref:`not thread safe `. .. coroutinemethod:: acquire() @@ -285,7 +259,7 @@ Semaphore BoundedSemaphore -^^^^^^^^^^^^^^^^ +---------------- .. class:: BoundedSemaphore(value=1, \*, loop=None) @@ -293,3 +267,39 @@ BoundedSemaphore This raises :exc:`ValueError` in :meth:`~Semaphore.release` if it would increase the value above the initial value. + + Bounded semapthores support the :ref:`context management + protocol `. + + This class is :ref:`not thread safe `. + + +.. _async-with-locks: + +Using locks, conditions and semaphores in the :keyword:`async with` statement +----------------------------------------------------------------------------- + +:class:`Lock`, :class:`Condition`, :class:`Semaphore`, and +:class:`BoundedSemaphore` objects can be used in :keyword:`async with` +statements. + +The :meth:`acquire` method will be called when the block is entered, +and :meth:`release` will be called when the block is exited. Hence, +the following snippet:: + + async with lock: + # do something... + +is equivalent to:: + + await lock.acquire() + try: + # do something... + finally: + lock.release() + +.. deprecated:: 3.7 + + Lock acquiring using ``await lock`` or ``yield from lock`` and + :keyword:`with` statement (``with await lock``, ``with (yield from + lock)``) are deprecated. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index ff35b0add9d..71dbe06c899 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -92,6 +92,24 @@ Coroutines (and tasks) can only run when the event loop is running. used in a callback-style code, wrap its result with :func:`ensure_future`. +.. function:: asyncio.run(coro, \*, debug=False) + + This function runs the passed coroutine, taking care of + managing the asyncio event loop and finalizing asynchronous + generators. + + This function cannot be called when another asyncio event loop is + running in the same thread. + + If debug is True, the event loop will be run in debug mode. + + This function always creates a new event loop and closes it at + the end. It should be used as a main entry point for asyncio + programs, and should ideally only be called once. + + .. versionadded:: 3.7 + + .. _asyncio-hello-world-coroutine: Example: Hello World coroutine @@ -104,10 +122,7 @@ Example of coroutine displaying ``"Hello World"``:: async def hello_world(): print("Hello World!") - loop = asyncio.get_event_loop() - # Blocking call which returns when the hello_world() coroutine is done - loop.run_until_complete(hello_world()) - loop.close() + asyncio.run(hello_world()) .. seealso:: @@ -127,7 +142,8 @@ using the :meth:`sleep` function:: import asyncio import datetime - async def display_date(loop): + async def display_date(): + loop = asyncio.get_running_loop() end_time = loop.time() + 5.0 while True: print(datetime.datetime.now()) @@ -135,10 +151,7 @@ using the :meth:`sleep` function:: break await asyncio.sleep(1) - loop = asyncio.get_event_loop() - # Blocking call which returns when the display_date() coroutine is done - loop.run_until_complete(display_date(loop)) - loop.close() + asyncio.run(display_date()) .. seealso:: @@ -293,6 +306,12 @@ Future If the future is already done when this method is called, raises :exc:`InvalidStateError`. + .. method:: get_loop() + + Return the event loop the future object is bound to. + + .. versionadded:: 3.7 + Example: Future with run_until_complete() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -358,10 +377,21 @@ with the result. Task ---- +.. function:: create_task(coro) + + Wrap a :ref:`coroutine ` *coro* into a task and schedule + its execution. Return the task object. + + The task is executed in :func:`get_running_loop` context, + :exc:`RuntimeError` is raised if there is no running loop in + current thread. + + .. versionadded:: 3.7 + .. class:: Task(coro, \*, loop=None) - Schedule the execution of a :ref:`coroutine `: wrap it in a - future. A task is a subclass of :class:`Future`. + A unit for concurrent running of :ref:`coroutines `, + subclass of :class:`Future`. A task is responsible for executing a coroutine object in an event loop. If the wrapped coroutine yields from a future, the task suspends the execution @@ -386,7 +416,7 @@ Task ` did not complete. It is probably a bug and a warning is logged: see :ref:`Pending task destroyed `. - Don't directly create :class:`Task` instances: use the :func:`ensure_future` + Don't directly create :class:`Task` instances: use the :func:`create_task` function or the :meth:`AbstractEventLoop.create_task` method. This class is :ref:`not thread safe `. @@ -504,6 +534,28 @@ Task functions the event loop object used by the underlying task or coroutine. If it's not provided, the default event loop is used. + +.. function:: current_task(loop=None): + + Return the current running :class:`Task` instance or ``None``, if + no task is running. + + If *loop* is ``None`` :func:`get_running_loop` is used to get + the current loop. + + .. versionadded:: 3.7 + + +.. function:: all_tasks(loop=None): + + Return a set of :class:`Task` objects created for the loop. + + If *loop* is ``None`` :func:`get_event_loop` is used for getting + current loop. + + .. versionadded:: 3.7 + + .. function:: as_completed(fs, \*, loop=None, timeout=None) Return an iterator whose values, when waited for, are :class:`Future` @@ -515,7 +567,7 @@ Task functions Example:: for f in as_completed(fs): - result = yield from f # The 'yield from' may raise + result = await f # The 'await' may raise # Use result .. note:: @@ -534,15 +586,15 @@ Task functions .. versionchanged:: 3.5.1 The function accepts any :term:`awaitable` object. + .. note:: + + :func:`create_task` (added in Python 3.7) is the preferable way + for spawning new tasks. + .. seealso:: - The :meth:`AbstractEventLoop.create_task` method. - -.. function:: async(coro_or_future, \*, loop=None) - - A deprecated alias to :func:`ensure_future`. - - .. deprecated:: 3.4.4 + The :func:`create_task` function and + :meth:`AbstractEventLoop.create_task` method. .. function:: wrap_future(future, \*, loop=None) @@ -636,11 +688,11 @@ Task functions The statement:: - res = yield from shield(something()) + res = await shield(something()) is exactly equivalent to the statement:: - res = yield from something() + res = await something() *except* that if the coroutine containing it is cancelled, the task running in ``something()`` is not cancelled. From the point of view of @@ -653,7 +705,7 @@ Task functions combine ``shield()`` with a try/except clause, as follows:: try: - res = yield from shield(something()) + res = await shield(something()) except CancelledError: res = None @@ -696,7 +748,7 @@ Task functions Usage:: - done, pending = yield from asyncio.wait(fs) + done, pending = await asyncio.wait(fs) .. note:: @@ -720,7 +772,7 @@ Task functions This function is a :ref:`coroutine `, usage:: - result = yield from asyncio.wait_for(fut, 60.0) + result = await asyncio.wait_for(fut, 60.0) .. versionchanged:: 3.4.3 If the wait is cancelled, the future *fut* is now also cancelled. diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index 5c60b604c68..c2c058e474c 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -20,6 +20,9 @@ at interpreter termination time they will be run in the order ``C``, ``B``, program is killed by a signal not handled by Python, when a Python fatal internal error is detected, or when :func:`os._exit` is called. +.. versionchanged:: 3.7 + When used with C-API subinterpreters, registered functions + are local to the interpreter they were registered in. .. function:: register(func, *args, **kwargs) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 6e249ecf2b1..74b24e10ede 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -977,10 +977,14 @@ e.g. ``'utf-8'`` is a valid alias for the ``'utf_8'`` codec. Some common encodings can bypass the codecs lookup machinery to improve performance. These optimization opportunities are only - recognized by CPython for a limited set of aliases: utf-8, utf8, - latin-1, latin1, iso-8859-1, mbcs (Windows only), ascii, utf-16, - and utf-32. Using alternative spellings for these encodings may - result in slower execution. + recognized by CPython for a limited set of (case insensitive) + aliases: utf-8, utf8, latin-1, latin1, iso-8859-1, iso8859-1, mbcs + (Windows only), ascii, us-ascii, utf-16, utf16, utf-32, utf32, and + the same using underscores instead of dashes. Using alternative + aliases for these encodings may result in slower execution. + + .. versionchanged:: 3.6 + Optimization opportunity recognized for us-ascii. Many of the character sets support the same languages. They vary in individual characters (e.g. whether the EURO SIGN is supported or not), and in the diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 60154532094..2a3fb142f72 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -87,7 +87,8 @@ ABC Inherits from Abstract Methods Mixin :class:`Set` ``__iter__`` :class:`KeysView` :class:`MappingView`, ``__contains__``, :class:`Set` ``__iter__`` -:class:`ValuesView` :class:`MappingView` ``__contains__``, ``__iter__`` +:class:`ValuesView` :class:`MappingView`, ``__contains__``, ``__iter__`` + :class:`Collection` :class:`Awaitable` ``__await__`` :class:`Coroutine` :class:`Awaitable` ``send``, ``throw`` ``close`` :class:`AsyncIterable` ``__aiter__`` diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index cda829694a3..96475b9f490 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -35,8 +35,8 @@ Python's general purpose built-in containers, :class:`dict`, :class:`list`, .. versionchanged:: 3.3 Moved :ref:`collections-abstract-base-classes` to the :mod:`collections.abc` module. - For backwards compatibility, they continue to be visible in this module - as well. + For backwards compatibility, they continue to be visible in this module through + Python 3.7. Subsequently, they will be removed entirely. :class:`ChainMap` objects @@ -509,11 +509,14 @@ or subtracting from an empty counter. .. versionadded:: 3.2 - .. method:: rotate(n) + .. method:: rotate(n=1) - Rotate the deque *n* steps to the right. If *n* is negative, rotate to - the left. Rotating one step to the right is equivalent to: - ``d.appendleft(d.pop())``. + Rotate the deque *n* steps to the right. If *n* is negative, rotate + to the left. + + When the deque is not empty, rotating one step to the right is equivalent + to ``d.appendleft(d.pop())``, and rotating one step to the left is + equivalent to ``d.append(d.popleft())``. Deque objects also provide one read-only attribute: @@ -618,6 +621,25 @@ added elements by appending to the right and popping to the left:: d.append(elem) yield s / n +A `round-robin scheduler +`_ can be implemented with +input iterators stored in a :class:`deque`. Values are yielded from the active +iterator in position zero. If that iterator is exhausted, it can be removed +with :meth:`~deque.popleft`; otherwise, it can be cycled back to the end with +the :meth:`~deque.rotate` method:: + + def roundrobin(*iterables): + "roundrobin('ABC', 'D', 'EF') --> A D E B F C" + iterators = deque(map(iter, iterables)) + while iterators: + try: + while True: + yield next(iterators[0]) + iterators.rotate(-1) + except StopIteration: + # Remove an exhausted iterator. + iterators.popleft() + The :meth:`rotate` method provides a way to implement :class:`deque` slicing and deletion. For example, a pure Python implementation of ``del d[n]`` relies on the :meth:`rotate` method to position elements to be popped:: @@ -763,7 +785,7 @@ Named tuples assign meaning to each position in a tuple and allow for more reada self-documenting code. They can be used wherever regular tuples are used, and they add the ability to access fields by name instead of position index. -.. function:: namedtuple(typename, field_names, *, rename=False, module=None) +.. function:: namedtuple(typename, field_names, *, rename=False, defaults=None, module=None) Returns a new tuple subclass named *typename*. The new subclass is used to create tuple-like objects that have fields accessible by attribute lookup as @@ -786,6 +808,13 @@ they add the ability to access fields by name instead of position index. converted to ``['abc', '_1', 'ghi', '_3']``, eliminating the keyword ``def`` and the duplicate fieldname ``abc``. + *defaults* can be ``None`` or an :term:`iterable` of default values. + Since fields with a default value must come after any fields without a + default, the *defaults* are applied to the rightmost parameters. For + example, if the fieldnames are ``['x', 'y', 'z']`` and the defaults are + ``(1, 2)``, then ``x`` will be a required argument, ``y`` will default to + ``1``, and ``z`` will default to ``2``. + If *module* is defined, the ``__module__`` attribute of the named tuple is set to that value. @@ -805,6 +834,10 @@ they add the ability to access fields by name instead of position index. .. versionchanged:: 3.7 Remove the *verbose* parameter and the :attr:`_source` attribute. + .. versionchanged:: 3.7 + Added the *defaults* parameter and the :attr:`_field_defaults` + attribute. + .. doctest:: :options: +NORMALIZE_WHITESPACE @@ -892,6 +925,18 @@ field names, the method and attribute names start with an underscore. >>> Pixel(11, 22, 128, 255, 0) Pixel(x=11, y=22, red=128, green=255, blue=0) +.. attribute:: somenamedtuple._fields_defaults + + Dictionary mapping field names to default values. + + .. doctest:: + + >>> Account = namedtuple('Account', ['type', 'balance'], defaults=[0]) + >>> Account._fields_defaults + {'balance': 0} + >>> Account('premium') + Account(type='premium', balance=0) + To retrieve a field whose name is stored in a string, use the :func:`getattr` function: diff --git a/Doc/library/colorsys.rst b/Doc/library/colorsys.rst index c33f531cc1d..1360c7cc9f1 100644 --- a/Doc/library/colorsys.rst +++ b/Doc/library/colorsys.rst @@ -21,7 +21,7 @@ spaces, the coordinates are all between 0 and 1. .. seealso:: More information about color spaces can be found at - http://www.poynton.com/ColorFAQ.html and + http://poynton.ca/ColorFAQ.html and https://www.cambridgeincolour.com/tutorials/color-spaces.htm. The :mod:`colorsys` module defines the following functions: diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index c1af02b0d82..7b3963da894 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -83,6 +83,16 @@ compile Python sources. If ``0`` is used, then the result of :func:`os.cpu_count()` will be used. +.. cmdoption:: --invalidation-mode [timestamp|checked-hash|unchecked-hash] + + Control how the generated pycs will be invalidated at runtime. The default + setting, ``timestamp``, means that ``.pyc`` files with the source timestamp + and size embedded will be generated. The ``checked-hash`` and + ``unchecked-hash`` values cause hash-based pycs to be generated. Hash-based + pycs embed a hash of the source file contents rather than a timestamp. See + :ref:`pyc-invalidation` for more information on how Python validates bytecode + cache files at runtime. + .. versionchanged:: 3.2 Added the ``-i``, ``-b`` and ``-h`` options. @@ -91,6 +101,9 @@ compile Python sources. was changed to a multilevel value. ``-b`` will always produce a byte-code file ending in ``.pyc``, never ``.pyo``. +.. versionchanged:: 3.7 + Added the ``--invalidation-mode`` parameter. + There is no command-line option to control the optimization level used by the :func:`compile` function, because the Python interpreter itself already @@ -99,7 +112,7 @@ provides the option: :program:`python -O -m compileall`. Public functions ---------------- -.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1) +.. function:: compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP) Recursively descend the directory tree named by *dir*, compiling all :file:`.py` files along the way. Return a true value if all the files compiled successfully, @@ -140,6 +153,10 @@ Public functions then sequential compilation will be used as a fallback. If *workers* is lower than ``0``, a :exc:`ValueError` will be raised. + *invalidation_mode* should be a member of the + :class:`py_compile.PycInvalidationMode` enum and controls how the generated + pycs are invalidated at runtime. + .. versionchanged:: 3.2 Added the *legacy* and *optimize* parameter. @@ -156,7 +173,10 @@ Public functions .. versionchanged:: 3.6 Accepts a :term:`path-like object`. -.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1) + .. versionchanged:: 3.7 + The *invalidation_mode* parameter was added. + +.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP) Compile the file with path *fullname*. Return a true value if the file compiled successfully, and a false value otherwise. @@ -184,6 +204,10 @@ Public functions *optimize* specifies the optimization level for the compiler. It is passed to the built-in :func:`compile` function. + *invalidation_mode* should be a member of the + :class:`py_compile.PycInvalidationMode` enum and controls how the generated + pycs are invalidated at runtime. + .. versionadded:: 3.2 .. versionchanged:: 3.5 @@ -193,7 +217,10 @@ Public functions The *legacy* parameter only writes out ``.pyc`` files, not ``.pyo`` files no matter what the value of *optimize* is. -.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, quiet=0, legacy=False, optimize=-1) + .. versionchanged:: 3.7 + The *invalidation_mode* parameter was added. + +.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, quiet=0, legacy=False, optimize=-1, invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP) Byte-compile all the :file:`.py` files found along ``sys.path``. Return a true value if all the files compiled successfully, and a false value otherwise. @@ -213,6 +240,9 @@ Public functions The *legacy* parameter only writes out ``.pyc`` files, not ``.pyo`` files no matter what the value of *optimize* is. + .. versionchanged:: 3.7 + The *invalidation_mode* parameter was added. + To force a recompile of all the :file:`.py` files in the :file:`Lib/` subdirectory and all its subdirectories:: diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index d4b698e1c17..707d24dc252 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -40,21 +40,29 @@ Executor Objects .. method:: map(func, *iterables, timeout=None, chunksize=1) - Equivalent to :func:`map(func, *iterables) ` except *func* is executed - asynchronously and several calls to *func* may be made concurrently. The - returned iterator raises a :exc:`concurrent.futures.TimeoutError` if - :meth:`~iterator.__next__` is called and the result isn't available + Similar to :func:`map(func, *iterables) ` except: + + * the *iterables* are collected immediately rather than lazily; + + * *func* is executed asynchronously and several calls to + *func* may be made concurrently. + + The returned iterator raises a :exc:`concurrent.futures.TimeoutError` + if :meth:`~iterator.__next__` is called and the result isn't available after *timeout* seconds from the original call to :meth:`Executor.map`. *timeout* can be an int or a float. If *timeout* is not specified or - ``None``, there is no limit to the wait time. If a call raises an - exception, then that exception will be raised when its value is - retrieved from the iterator. When using :class:`ProcessPoolExecutor`, this - method chops *iterables* into a number of chunks which it submits to the - pool as separate tasks. The (approximate) size of these chunks can be - specified by setting *chunksize* to a positive integer. For very long - iterables, using a large value for *chunksize* can significantly improve - performance compared to the default size of 1. With :class:`ThreadPoolExecutor`, - *chunksize* has no effect. + ``None``, there is no limit to the wait time. + + If a *func* call raises an exception, then that exception will be + raised when its value is retrieved from the iterator. + + When using :class:`ProcessPoolExecutor`, this method chops *iterables* + into a number of chunks which it submits to the pool as separate + tasks. The (approximate) size of these chunks can be specified by + setting *chunksize* to a positive integer. For very long iterables, + using a large value for *chunksize* can significantly improve + performance compared to the default size of 1. With + :class:`ThreadPoolExecutor`, *chunksize* has no effect. .. versionchanged:: 3.5 Added the *chunksize* argument. diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 19793693b7b..54d3a8e4296 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -29,6 +29,17 @@ Functions and classes provided: .. versionadded:: 3.6 +.. class:: AbstractAsyncContextManager + + An :term:`abstract base class` for classes that implement + :meth:`object.__aenter__` and :meth:`object.__aexit__`. A default + implementation for :meth:`object.__aenter__` is provided which returns + ``self`` while :meth:`object.__aexit__` is an abstract method which by default + returns ``None``. See also the definition of + :ref:`async-context-managers`. + + .. versionadded:: 3.7 + .. decorator:: contextmanager @@ -137,6 +148,28 @@ Functions and classes provided: ``page.close()`` will be called when the :keyword:`with` block is exited. +.. _simplifying-support-for-single-optional-context-managers: + +.. function:: nullcontext(enter_result=None) + + Return a context manager that returns enter_result from ``__enter__``, but + otherwise does nothing. It is intended to be used as a stand-in for an + optional context manager, for example:: + + def process_file(file_or_path): + if isinstance(file_or_path, str): + # If string, open file + cm = open(file_or_path) + else: + # Caller is responsible for closing file + cm = nullcontext(file_or_path) + + with cm as file: + # Perform processing on the file + + .. versionadded:: 3.7 + + .. function:: suppress(*exceptions) Return a context manager that suppresses any of the specified exceptions @@ -402,6 +435,44 @@ Functions and classes provided: callbacks registered, the arguments passed in will indicate that no exception occurred. +.. class:: AsyncExitStack() + + An :ref:`asynchronous context manager `, similar + to :class:`ExitStack`, that supports combining both synchronous and + asynchronous context managers, as well as having coroutines for + cleanup logic. + + The :meth:`close` method is not implemented, :meth:`aclose` must be used + instead. + + .. method:: enter_async_context(cm) + + Similar to :meth:`enter_context` but expects an asynchronous context + manager. + + .. method:: push_async_exit(exit) + + Similar to :meth:`push` but expects either an asynchronous context manager + or a coroutine. + + .. method:: push_async_callback(callback, *args, **kwds) + + Similar to :meth:`callback` but expects a coroutine. + + .. method:: aclose() + + Similar to :meth:`close` but properly handles awaitables. + + Continuing the example for :func:`asynccontextmanager`:: + + async with AsyncExitStack() as stack: + connections = [await stack.enter_async_context(get_connection()) + for i in range(5)] + # All opened connections will automatically be released at the end of + # the async with statement, even if attempts to open a connection + # later in the list raise an exception. + + .. versionadded:: 3.7 Examples and Recipes -------------------- @@ -433,24 +504,6 @@ statements to manage arbitrary resources that don't natively support the context management protocol. -Simplifying support for single optional context managers -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In the specific case of a single optional context manager, :class:`ExitStack` -instances can be used as a "do nothing" context manager, allowing a context -manager to easily be omitted without affecting the overall structure of -the source code:: - - def debug_trace(details): - if __debug__: - return TraceContext(details) - # Don't do anything special with the context in release mode - return ExitStack() - - with debug_trace(): - # Suite is traced in debug mode, but runs normally otherwise - - Catching exceptions from ``__enter__`` methods ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index dce51a16e86..c1b164ebc1f 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -436,6 +436,21 @@ Other constructors, all class methods: d``. +.. classmethod:: date.fromisoformat(date_string) + + Return a :class:`date` corresponding to a *date_string* in the format emitted + by :meth:`date.isoformat`. Specifically, this function supports strings in + the format(s) ``YYYY-MM-DD``. + + .. caution:: + + This does not support parsing arbitrary ISO 8601 strings - it is only intended + as the inverse operation of :meth:`date.isoformat`. + + .. versionadded:: 3.7 + + + Class attributes: .. attribute:: date.min @@ -819,6 +834,21 @@ Other constructors, all class methods: Added the *tzinfo* argument. +.. classmethod:: datetime.fromisoformat(date_string) + + Return a :class:`datetime` corresponding to a *date_string* in one of the + formats emitted by :meth:`date.isoformat` and :meth:`datetime.isoformat`. + Specifically, this function supports strings in the format(s) + ``YYYY-MM-DD[*HH[:MM[:SS[.mmm[mmm]]]][+HH:MM[:SS[.ffffff]]]]``, + where ``*`` can match any single character. + + .. caution:: + + This does not support parsing arbitrary ISO 8601 strings - it is only intended + as the inverse operation of :meth:`datetime.isoformat`. + + .. versionadded:: 3.7 + .. classmethod:: datetime.strptime(date_string, format) Return a :class:`.datetime` corresponding to *date_string*, parsed according to @@ -1486,6 +1516,23 @@ In boolean contexts, a :class:`.time` object is always considered to be true. error-prone and has been removed in Python 3.5. See :issue:`13936` for full details. + +Other constructor: + +.. classmethod:: time.fromisoformat(time_string) + + Return a :class:`time` corresponding to a *time_string* in one of the + formats emitted by :meth:`time.isoformat`. Specifically, this function supports + strings in the format(s) ``HH[:MM[:SS[.mmm[mmm]]]][+HH:MM[:SS[.ffffff]]]``. + + .. caution:: + + This does not support parsing arbitrary ISO 8601 strings - it is only intended + as the inverse operation of :meth:`time.isoformat`. + + .. versionadded:: 3.7 + + Instance methods: .. method:: time.replace(hour=self.hour, minute=self.minute, second=self.second, \ @@ -1587,7 +1634,6 @@ Instance methods: ``self.tzinfo.tzname(None)``, or raises an exception if the latter doesn't return ``None`` or a string object. - Example: >>> from datetime import time, tzinfo, timedelta diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 32e80b2cf6e..1abc36c04a7 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -339,9 +339,23 @@ The module defines the following: dumbdbm database is created, files with :file:`.dat` and :file:`.dir` extensions are created. - The optional *flag* argument supports only the semantics of ``'c'`` - and ``'n'`` values. Other values will default to database being always - opened for update, and will be created if it does not exist. + The optional *flag* argument can be: + + +---------+-------------------------------------------+ + | Value | Meaning | + +=========+===========================================+ + | ``'r'`` | Open existing database for reading only | + | | (default) | + +---------+-------------------------------------------+ + | ``'w'`` | Open existing database for reading and | + | | writing | + +---------+-------------------------------------------+ + | ``'c'`` | Open database for reading and writing, | + | | creating it if it doesn't exist | + +---------+-------------------------------------------+ + | ``'n'`` | Always create a new, empty database, open | + | | for reading and writing | + +---------+-------------------------------------------+ The optional *mode* argument is the Unix mode of the file, used only when the database has to be created. It defaults to octal ``0o666`` (and will be modified @@ -351,9 +365,10 @@ The module defines the following: :func:`.open` always creates a new database when the flag has the value ``'n'``. - .. deprecated-removed:: 3.6 3.8 - Creating database in ``'r'`` and ``'w'`` modes. Modifying database in - ``'r'`` mode. + .. versionchanged:: 3.8 + A database opened with flags ``'r'`` is now read-only. Opening with + flags ``'r'`` and ``'w'`` no longer creates a database if it does not + exist. In addition to the methods provided by the :class:`collections.abc.MutableMapping` class, :class:`dumbdbm` objects diff --git a/Doc/library/development.rst b/Doc/library/development.rst index d2b5fa2aa4f..ab34e1f7ce5 100644 --- a/Doc/library/development.rst +++ b/Doc/library/development.rst @@ -24,3 +24,6 @@ The list of modules described in this chapter is: unittest.mock-examples.rst 2to3.rst test.rst + +See also the Python development mode: the :option:`-X` ``dev`` option and +:envvar:`PYTHONDEVMODE` environment variable. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 01d46f88fb5..535b36efbf2 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -339,12 +339,16 @@ The Python compiler currently generates the following bytecode instructions. Duplicates the reference on top of the stack. + .. versionadded:: 3.2 + .. opcode:: DUP_TOP_TWO Duplicates the two references on top of the stack, leaving them in the same order. + .. versionadded:: 3.2 + **Unary operations** @@ -555,11 +559,14 @@ the original TOS1. the CO_ITERABLE_COROUTINE flag, or resolves ``o.__await__``. + .. versionadded:: 3.5 + .. opcode:: GET_AITER Implements ``TOS = TOS.__aiter__()``. + .. versionadded:: 3.5 .. versionchanged:: 3.7 Returning awaitable objects from ``__aiter__`` is no longer supported. @@ -570,17 +577,23 @@ the original TOS1. Implements ``PUSH(get_awaitable(TOS.__anext__()))``. See ``GET_AWAITABLE`` for details about ``get_awaitable`` + .. versionadded:: 3.5 + .. opcode:: BEFORE_ASYNC_WITH Resolves ``__aenter__`` and ``__aexit__`` from the object on top of the stack. Pushes ``__aexit__`` and result of ``__aenter__()`` to the stack. + .. versionadded:: 3.5 + .. opcode:: SETUP_ASYNC_WITH Creates a new frame object. + .. versionadded:: 3.5 + **Miscellaneous opcodes** @@ -618,6 +631,8 @@ the original TOS1. Calls ``dict.setitem(TOS1[-i], TOS, TOS1)``. Used to implement dict comprehensions. + .. versionadded:: 3.1 + For all of the :opcode:`SET_ADD`, :opcode:`LIST_APPEND` and :opcode:`MAP_ADD` instructions, while the added value or key/value pair is popped off, the container object remains on the stack so that it is available for further @@ -640,6 +655,7 @@ iterations of the loop. .. versionadded:: 3.3 + .. opcode:: SETUP_ANNOTATIONS Checks whether ``__annotations__`` is defined in ``locals()``, if not it is @@ -649,6 +665,7 @@ iterations of the loop. .. versionadded:: 3.6 + .. opcode:: IMPORT_STAR Loads all symbols not starting with ``'_'`` directly from the module TOS to @@ -694,6 +711,8 @@ iterations of the loop. store it in (a) variable(s) (:opcode:`STORE_FAST`, :opcode:`STORE_NAME`, or :opcode:`UNPACK_SEQUENCE`). + .. versionadded:: 3.2 + .. opcode:: WITH_CLEANUP_START @@ -924,23 +943,31 @@ All of the following opcodes use their arguments. If TOS is true, sets the bytecode counter to *target*. TOS is popped. + .. versionadded:: 3.1 + .. opcode:: POP_JUMP_IF_FALSE (target) If TOS is false, sets the bytecode counter to *target*. TOS is popped. + .. versionadded:: 3.1 + .. opcode:: JUMP_IF_TRUE_OR_POP (target) If TOS is true, sets the bytecode counter to *target* and leaves TOS on the stack. Otherwise (TOS is false), TOS is popped. + .. versionadded:: 3.1 + .. opcode:: JUMP_IF_FALSE_OR_POP (target) If TOS is false, sets the bytecode counter to *target* and leaves TOS on the stack. Otherwise (TOS is true), TOS is popped. + .. versionadded:: 3.1 + .. opcode:: JUMP_ABSOLUTE (target) @@ -993,13 +1020,6 @@ All of the following opcodes use their arguments. Deletes local ``co_varnames[var_num]``. -.. opcode:: STORE_ANNOTATION (namei) - - Stores TOS as ``locals()['__annotations__'][co_names[namei]] = TOS``. - - .. versionadded:: 3.6 - - .. opcode:: LOAD_CLOSURE (i) Pushes a reference to the cell contained in slot *i* of the cell and free @@ -1020,6 +1040,8 @@ All of the following opcodes use their arguments. consulting the cell. This is used for loading free variables in class bodies. + .. versionadded:: 3.4 + .. opcode:: STORE_DEREF (i) @@ -1032,6 +1054,8 @@ All of the following opcodes use their arguments. Empties the cell contained in slot *i* of the cell and free variable storage. Used by the :keyword:`del` statement. + .. versionadded:: 3.2 + .. opcode:: RAISE_VARARGS (argc) diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index 1e64e1066c7..cc8e8225a20 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -53,7 +53,7 @@ over channels that are not "8 bit clean". :data:`~email.policy.compat32` policy and ``False`` for all others). *mangle_from_* is intended for use when messages are stored in unix mbox format (see :mod:`mailbox` and `WHY THE CONTENT-LENGTH FORMAT IS BAD - `_). + `_). If *maxheaderlen* is not ``None``, refold any header lines that are longer than *maxheaderlen*, or if ``0``, do not rewrap any headers. If @@ -154,7 +154,7 @@ to be using :class:`BytesGenerator`, and not :class:`Generator`. :data:`~email.policy.compat32` policy and ``False`` for all others). *mangle_from_* is intended for use when messages are stored in unix mbox format (see :mod:`mailbox` and `WHY THE CONTENT-LENGTH FORMAT IS BAD - `_). + `_). If *maxheaderlen* is not ``None``, refold any header lines that are longer than *maxheaderlen*, or if ``0``, do not rewrap any headers. If diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 6548adf789d..fc65a3d078f 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -379,7 +379,8 @@ The rules for what is allowed are as follows: names that start and end with a single underscore are reserved by enum and cannot be used; all other attributes defined within an enumeration will become members of this enumeration, with the exception of special methods (:meth:`__str__`, -:meth:`__add__`, etc.) and descriptors (methods are also descriptors). +:meth:`__add__`, etc.), descriptors (methods are also descriptors), and +variable names listed in :attr:`_ignore_`. Note: if your enumeration defines :meth:`__new__` and/or :meth:`__init__` then whatever value(s) were given to the enum member will be passed into those @@ -654,7 +655,7 @@ value and let :class:`Flag` select an appropriate value. Like :class:`IntFlag`, if a combination of :class:`Flag` members results in no flags being set, the boolean evaluation is :data:`False`:: - >>> from enum import Flag + >>> from enum import Flag, auto >>> class Color(Flag): ... RED = auto() ... BLUE = auto() @@ -943,6 +944,25 @@ will be passed to those methods:: 9.802652743337129 +TimePeriod +^^^^^^^^^^ + +An example to show the :attr:`_ignore_` attribute in use:: + + >>> from datetime import timedelta + >>> class Period(timedelta, Enum): + ... "different lengths of time" + ... _ignore_ = 'Period i' + ... Period = vars() + ... for i in range(367): + ... Period['day_%d' % i] = i + ... + >>> list(Period)[:2] + [, ] + >>> list(Period)[-2:] + [, ] + + How are Enums different? ------------------------ @@ -994,6 +1014,9 @@ Supported ``_sunder_`` names - ``_missing_`` -- a lookup function used when a value is not found; may be overridden +- ``_ignore_`` -- a list of names, either as a :func:`list` or a :func:`str`, + that will not be transformed into members, and will be removed from the final + class - ``_order_`` -- used in Python 2/3 code to ensure member order is consistent (class attribute, removed during class creation) - ``_generate_next_value_`` -- used by the `Functional API`_ and by @@ -1001,6 +1024,7 @@ Supported ``_sunder_`` names overridden .. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` +.. versionadded:: 3.7 ``_ignore_`` To help keep Python 2 / Python 3 code in sync an :attr:`_order_` attribute can be provided. It will be checked against the actual order of the enumeration diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index a6b20a5ac95..42b99cc47e0 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -154,10 +154,7 @@ The following exceptions are the exceptions that are usually raised. .. exception:: FloatingPointError - Raised when a floating point operation fails. This exception is always defined, - but can only be raised when Python is configured with the - ``--with-fpectl`` option, or the :const:`WANT_SIGFPE_HANDLER` symbol is - defined in the :file:`pyconfig.h` file. + Not currently used. .. exception:: GeneratorExit @@ -370,17 +367,21 @@ The following exceptions are the exceptions that are usually raised. raised, and the value returned by the function is used as the :attr:`value` parameter to the constructor of the exception. - If a generator function defined in the presence of a ``from __future__ - import generator_stop`` directive raises :exc:`StopIteration`, it will be - converted into a :exc:`RuntimeError` (retaining the :exc:`StopIteration` - as the new exception's cause). + If a generator code directly or indirectly raises :exc:`StopIteration`, + it is converted into a :exc:`RuntimeError` (retaining the + :exc:`StopIteration` as the new exception's cause). .. versionchanged:: 3.3 Added ``value`` attribute and the ability for generator functions to use it to return a value. .. versionchanged:: 3.5 - Introduced the RuntimeError transformation. + Introduced the RuntimeError transformation via + ``from __future__ import generator_stop``, see :pep:`479`. + + .. versionchanged:: 3.7 + Enable :pep:`479` for all code by default: a :exc:`StopIteration` + error raised in a generator is transformed into a :exc:`RuntimeError`. .. exception:: StopAsyncIteration @@ -664,11 +665,13 @@ depending on the system error code. :pep:`3151` - Reworking the OS and IO exception hierarchy +.. _warning-categories-as-exceptions: + Warnings -------- -The following exceptions are used as warning categories; see the :mod:`warnings` -module for more information. +The following exceptions are used as warning categories; see the +:ref:`warning-categories` documentation for more details. .. exception:: Warning @@ -682,12 +685,14 @@ module for more information. .. exception:: DeprecationWarning - Base class for warnings about deprecated features. + Base class for warnings about deprecated features when those warnings are + intended for other Python developers. .. exception:: PendingDeprecationWarning - Base class for warnings about features which will be deprecated in the future. + Base class for warnings about features which will be deprecated in the + future. .. exception:: SyntaxWarning @@ -702,8 +707,8 @@ module for more information. .. exception:: FutureWarning - Base class for warnings about constructs that will change semantically in the - future. + Base class for warnings about deprecated features when those warnings are + intended for end users of applications that are written in Python. .. exception:: ImportWarning @@ -723,7 +728,8 @@ module for more information. .. exception:: ResourceWarning - Base class for warnings related to resource usage. + Base class for warnings related to resource usage. Ignored by the default + warning filters. .. versionadded:: 3.2 diff --git a/Doc/library/fpectl.rst b/Doc/library/fpectl.rst deleted file mode 100644 index 96607165ba4..00000000000 --- a/Doc/library/fpectl.rst +++ /dev/null @@ -1,121 +0,0 @@ -:mod:`fpectl` --- Floating point exception control -================================================== - -.. module:: fpectl - :platform: Unix - :synopsis: Provide control for floating point exception handling. - -.. moduleauthor:: Lee Busby -.. sectionauthor:: Lee Busby - -.. note:: - - The :mod:`fpectl` module is not built by default, and its usage is discouraged - and may be dangerous except in the hands of experts. See also the section - :ref:`fpectl-limitations` on limitations for more details. - -.. index:: single: IEEE-754 - --------------- - -Most computers carry out floating point operations in conformance with the -so-called IEEE-754 standard. On any real computer, some floating point -operations produce results that cannot be expressed as a normal floating point -value. For example, try :: - - >>> import math - >>> math.exp(1000) - inf - >>> math.exp(1000) / math.exp(1000) - nan - -(The example above will work on many platforms. DEC Alpha may be one exception.) -"Inf" is a special, non-numeric value in IEEE-754 that stands for "infinity", -and "nan" means "not a number." Note that, other than the non-numeric results, -nothing special happened when you asked Python to carry out those calculations. -That is in fact the default behaviour prescribed in the IEEE-754 standard, and -if it works for you, stop reading now. - -In some circumstances, it would be better to raise an exception and stop -processing at the point where the faulty operation was attempted. The -:mod:`fpectl` module is for use in that situation. It provides control over -floating point units from several hardware manufacturers, allowing the user to -turn on the generation of :const:`SIGFPE` whenever any of the IEEE-754 -exceptions Division by Zero, Overflow, or Invalid Operation occurs. In tandem -with a pair of wrapper macros that are inserted into the C code comprising your -python system, :const:`SIGFPE` is trapped and converted into the Python -:exc:`FloatingPointError` exception. - -The :mod:`fpectl` module defines the following functions and may raise the given -exception: - - -.. function:: turnon_sigfpe() - - Turn on the generation of :const:`SIGFPE`, and set up an appropriate signal - handler. - - -.. function:: turnoff_sigfpe() - - Reset default handling of floating point exceptions. - - -.. exception:: FloatingPointError - - After :func:`turnon_sigfpe` has been executed, a floating point operation that - raises one of the IEEE-754 exceptions Division by Zero, Overflow, or Invalid - operation will in turn raise this standard Python exception. - - -.. _fpectl-example: - -Example -------- - -The following example demonstrates how to start up and test operation of the -:mod:`fpectl` module. :: - - >>> import fpectl - >>> import fpetest - >>> fpectl.turnon_sigfpe() - >>> fpetest.test() - overflow PASS - FloatingPointError: Overflow - - div by 0 PASS - FloatingPointError: Division by zero - [ more output from test elided ] - >>> import math - >>> math.exp(1000) - Traceback (most recent call last): - File "", line 1, in - FloatingPointError: in math_1 - - -.. _fpectl-limitations: - -Limitations and other considerations ------------------------------------- - -Setting up a given processor to trap IEEE-754 floating point errors currently -requires custom code on a per-architecture basis. You may have to modify -:mod:`fpectl` to control your particular hardware. - -Conversion of an IEEE-754 exception to a Python exception requires that the -wrapper macros ``PyFPE_START_PROTECT`` and ``PyFPE_END_PROTECT`` be inserted -into your code in an appropriate fashion. Python itself has been modified to -support the :mod:`fpectl` module, but many other codes of interest to numerical -analysts have not. - -The :mod:`fpectl` module is not thread-safe. - - -.. seealso:: - - Some files in the source distribution may be interesting in learning more about - how this module operates. The include file :file:`Include/pyfpe.h` discusses the - implementation of this module at some length. :file:`Modules/fpetestmodule.c` - gives several examples of use. Many additional examples can be found in - :file:`Objects/floatobject.c`. - diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index e47225718c1..bfb813cf390 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1423,7 +1423,7 @@ are always available. They are listed here in alphabetical order. a regular function and do something with its result. This is needed in some cases where you need a reference to a function from a class body and you want to avoid the automatic transformation to instance - method. For these cases, use this idiom: + method. For these cases, use this idiom:: class C: builtin_open = staticmethod(open) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 28062c11890..a81e819103a 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -281,23 +281,34 @@ The :mod:`functools` module defines the following functions: ... print(arg) To add overloaded implementations to the function, use the :func:`register` - attribute of the generic function. It is a decorator, taking a type - parameter and decorating a function implementing the operation for that - type:: + attribute of the generic function. It is a decorator. For functions + annotated with types, the decorator will infer the type of the first + argument automatically:: - >>> @fun.register(int) - ... def _(arg, verbose=False): + >>> @fun.register + ... def _(arg: int, verbose=False): ... if verbose: ... print("Strength in numbers, eh?", end=" ") ... print(arg) ... - >>> @fun.register(list) - ... def _(arg, verbose=False): + >>> @fun.register + ... def _(arg: list, verbose=False): ... if verbose: ... print("Enumerate this:") ... for i, elem in enumerate(arg): ... print(i, elem) + For code which doesn't use type annotations, the appropriate type + argument can be passed explicitly to the decorator itself:: + + >>> @fun.register(complex) + ... def _(arg, verbose=False): + ... if verbose: + ... print("Better than complicated.", end=" ") + ... print(arg.real, arg.imag) + ... + + To enable registering lambdas and pre-existing functions, the :func:`register` attribute can be used in a functional form:: @@ -368,6 +379,9 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.4 + .. versionchanged:: 3.7 + The :func:`register` attribute supports using type annotations. + .. function:: update_wrapper(wrapper, wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst index 5eb9f04a8da..82b11919a3d 100644 --- a/Doc/library/getpass.rst +++ b/Doc/library/getpass.rst @@ -42,8 +42,10 @@ The :mod:`getpass` module provides two functions: Return the "login name" of the user. This function checks the environment variables :envvar:`LOGNAME`, - :envvar:`USER`, :envvar:`LNAME` and :envvar:`USERNAME`, in order, and returns - the value of the first one which is set to a non-empty string. If none are set, - the login name from the password database is returned on systems which support - the :mod:`pwd` module, otherwise, an exception is raised. + :envvar:`USER`, :envvar:`LNAME` and :envvar:`USERNAME`, in order, and + returns the value of the first one which is set to a non-empty string. If + none are set, the login name from the password database is returned on + systems which support the :mod:`pwd` module, otherwise, an exception is + raised. + In general, this function should be preferred over :func:`os.getlogin()`. diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 452705f4d2a..eda18adc9e5 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -482,7 +482,7 @@ Keyed hashing Keyed hashing can be used for authentication as a faster and simpler replacement for `Hash-based message authentication code -`_ (HMAC). +`_ (HMAC). BLAKE2 can be securely used in prefix-MAC mode thanks to the indifferentiability property inherited from BLAKE. @@ -562,7 +562,7 @@ on the hash function used in digital signatures. by the signer. (`NIST SP-800-106 "Randomized Hashing for Digital Signatures" - `_) + `_) In BLAKE2 the salt is processed as a one-time input to the hash function during initialization, rather than as an input to each compression function. @@ -699,7 +699,7 @@ implementation, extension code, and this documentation: You should have received a copy of the CC0 Public Domain Dedication along with this software. If not, see - http://creativecommons.org/publicdomain/zero/1.0/. + https://creativecommons.org/publicdomain/zero/1.0/. The following people have helped with development or contributed their changes to the project and the public domain according to the Creative Commons Public @@ -728,7 +728,7 @@ Domain Dedication 1.0 Universal: https://blake2.net Official BLAKE2 website. - http://csrc.nist.gov/publications/fips/fips180-2/fips180-2.pdf + https://csrc.nist.gov/csrc/media/publications/fips/180/2/archive/2002-08-01/documents/fips180-2.pdf The FIPS 180-2 publication on Secure Hash Algorithms. https://en.wikipedia.org/wiki/Cryptographic_hash_function#Cryptographic_hash_algorithms diff --git a/Doc/library/heapq.rst b/Doc/library/heapq.rst index e36ca8d7b34..0b1a3c8b00c 100644 --- a/Doc/library/heapq.rst +++ b/Doc/library/heapq.rst @@ -187,6 +187,17 @@ a tie-breaker so that two tasks with the same priority are returned in the order they were added. And since no two entry counts are the same, the tuple comparison will never attempt to directly compare two tasks. +Another solution to the problem of non-comparable tasks is to create a wrapper +class that ignores the task item and only compares the priority field:: + + from dataclasses import dataclass, field + from typing import Any + + @dataclass(order=True) + class PrioritizedItem: + priority: int + item: Any=field(compare=False) + The remaining challenges revolve around finding a pending task and making changes to its priority or removing it entirely. Finding a task can be done with a dictionary pointing to an entry in the queue. diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index adbf78a7b46..fcda86cf797 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -31,6 +31,21 @@ This module implements the HMAC algorithm as described by :rfc:`2104`. MD5 as implicit default digest for *digestmod* is deprecated. +.. function:: digest(key, msg, digest) + + Return digest of *msg* for given secret *key* and *digest*. The + function is equivalent to ``HMAC(key, msg, digest).digest()``, but + uses an optimized C or inline implementation, which is faster for messages + that fit into memory. The parameters *key*, *msg*, and *digest* have + the same meaning as in :func:`~hmac.new`. + + CPython implementation detail, the optimized C implementation is only used + when *digest* is a string and name of a digest algorithm, which is + supported by OpenSSL. + + .. versionadded:: 3.7 + + An HMAC object has the following methods: .. method:: HMAC.update(msg) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 3d350e8d0d5..c80f4606f3d 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -67,6 +67,9 @@ generically as an :term:`importer`) to participate in the import process. :pep:`489` Multi-phase extension module initialization + :pep:`552` + Deterministic pycs + :pep:`3120` Using UTF-8 as the Default Source Encoding @@ -366,6 +369,13 @@ ABC hierarchy:: An abstract base class for a :term:`loader`. See :pep:`302` for the exact definition for a loader. + For loaders that wish to support resource reading, they should + implement a ``get_resource_reader(fullname)`` method as specified + by :class:`importlib.abc.ResourceReader`. + + .. versionchanged:: 3.7 + Introduced the optional ``get_resource_reader()`` method. + .. method:: create_module(spec) A method that returns the module object to use when @@ -465,12 +475,88 @@ ABC hierarchy:: The import machinery now takes care of this automatically. +.. class:: ResourceReader + + An :term:`abstract base class` to provide the ability to read + *resources*. + + From the perspective of this ABC, a *resource* is a binary + artifact that is shipped within a package. Typically this is + something like a data file that lives next to the ``__init__.py`` + file of the package. The purpose of this class is to help abstract + out the accessing of such data files so that it does not matter if + the package and its data file(s) are stored in a e.g. zip file + versus on the file system. + + For any of methods of this class, a *resource* argument is + expected to be a :term:`path-like object` which represents + conceptually just a file name. This means that no subdirectory + paths should be included in the *resource* argument. This is + because the location of the package the reader is for, acts as the + "directory". Hence the metaphor for directories and file + names is packages and resources, respectively. This is also why + instances of this class are expected to directly correlate to + a specific package (instead of potentially representing multiple + packages or a module). + + Loaders that wish to support resource reading are expected to + provide a method called ``get_resource_loader(fullname)`` which + returns an object implementing this ABC's interface. If the module + specified by fullname is not a package, this method should return + :const:`None`. An object compatible with this ABC should only be + returned when the specified module is a package. + + .. versionadded:: 3.7 + + .. abstractmethod:: open_resource(resource) + + Returns an opened, :term:`file-like object` for binary reading + of the *resource*. + + If the resource cannot be found, :exc:`FileNotFoundError` is + raised. + + .. abstractmethod:: resource_path(resource) + + Returns the file system path to the *resource*. + + If the resource does not concretely exist on the file system, + raise :exc:`FileNotFoundError`. + + .. abstractmethod:: is_resource(name) + + Returns ``True`` if the named *name* is considered a resource. + :exc:`FileNotFoundError` is raised if *name* does not exist. + + .. abstractmethod:: contents() + + Returns an :term:`iterator` of strings over the contents of + the package. Do note that it is not required that all names + returned by the iterator be actual resources, e.g. it is + acceptable to return names for which :meth:`is_resource` would + be false. + + Allowing non-resource names to be returned is to allow for + situations where how a package and its resources are stored + are known a priori and the non-resource names would be useful. + For instance, returning subdirectory names is allowed so that + when it is known that the package and resources are stored on + the file system then those subdirectory names can be used + directly. + + The abstract method returns an iterator of no items. + + .. class:: ResourceLoader An abstract base class for a :term:`loader` which implements the optional :pep:`302` protocol for loading arbitrary resources from the storage back-end. + .. deprecated:: 3.7 + This ABC is deprecated in favour of supporting resource loading + through :class:`importlib.abc.ResourceReader`. + .. abstractmethod:: get_data(path) An abstract method to return the bytes for the data located at *path*. @@ -706,6 +792,131 @@ ABC hierarchy:: itself does not end in ``__init__``. +:mod:`importlib.resources` -- Resources +--------------------------------------- + +.. module:: importlib.resources + :synopsis: Package resource reading, opening, and access + +**Source code:** :source:`Lib/importlib/resources.py` + +-------------- + +.. versionadded:: 3.7 + +This module leverages Python's import system to provide access to *resources* +within *packages*. If you can import a package, you can access resources +within that package. Resources can be opened or read, in either binary or +text mode. + +Resources are roughly akin to files inside directories, though it's important +to keep in mind that this is just a metaphor. Resources and packages **do +not** have to exist as physical files and directories on the file system. + +Loaders can support resources by implementing the :class:`ResourceReader` +abstract base class. + +The following types are defined. + +.. data:: Package + + The ``Package`` type is defined as ``Union[str, ModuleType]``. This means + that where the function describes accepting a ``Package``, you can pass in + either a string or a module. Module objects must have a resolvable + ``__spec__.submodule_search_locations`` that is not ``None``. + +.. data:: Resource + + This type describes the resource names passed into the various functions + in this package. This is defined as ``Union[str, os.PathLike]``. + + +The following functions are available. + +.. function:: open_binary(package, resource) + + Open for binary reading the *resource* within *package*. + + *package* is either a name or a module object which conforms to the + ``Package`` requirements. *resource* is the name of the resource to open + within *package*; it may not contain path separators and it may not have + sub-resources (i.e. it cannot be a directory). This function returns a + ``typing.BinaryIO`` instance, a binary I/O stream open for reading. + + +.. function:: open_text(package, resource, encoding='utf-8', errors='strict') + + Open for text reading the *resource* within *package*. By default, the + resource is opened for reading as UTF-8. + + *package* is either a name or a module object which conforms to the + ``Package`` requirements. *resource* is the name of the resource to open + within *package*; it may not contain path separators and it may not have + sub-resources (i.e. it cannot be a directory). *encoding* and *errors* + have the same meaning as with built-in :func:`open`. + + This function returns a ``typing.TextIO`` instance, a text I/O stream open + for reading. + + +.. function:: read_binary(package, resource) + + Read and return the contents of the *resource* within *package* as + ``bytes``. + + *package* is either a name or a module object which conforms to the + ``Package`` requirements. *resource* is the name of the resource to open + within *package*; it may not contain path separators and it may not have + sub-resources (i.e. it cannot be a directory). This function returns the + contents of the resource as :class:`bytes`. + + +.. function:: read_text(package, resource, encoding='utf-8', errors='strict') + + Read and return the contents of *resource* within *package* as a ``str``. + By default, the contents are read as strict UTF-8. + + *package* is either a name or a module object which conforms to the + ``Package`` requirements. *resource* is the name of the resource to open + within *package*; it may not contain path separators and it may not have + sub-resources (i.e. it cannot be a directory). *encoding* and *errors* + have the same meaning as with built-in :func:`open`. This function + returns the contents of the resource as :class:`str`. + + +.. function:: path(package, resource) + + Return the path to the *resource* as an actual file system path. This + function returns a context manager for use in a :keyword:`with` statement. + The context manager provides a :class:`pathlib.Path` object. + + Exiting the context manager cleans up any temporary file created when the + resource needs to be extracted from e.g. a zip file. + + *package* is either a name or a module object which conforms to the + ``Package`` requirements. *resource* is the name of the resource to open + within *package*; it may not contain path separators and it may not have + sub-resources (i.e. it cannot be a directory). + + +.. function:: is_resource(package, name) + + Return ``True`` if there is a resource named *name* in the package, + otherwise ``False``. Remember that directories are *not* resources! + *package* is either a name or a module object which conforms to the + ``Package`` requirements. + + +.. function:: contents(package) + + Return an iterator over the named items within the package. The iterator + returns :class:`str` resources (e.g. files) and non-resources + (e.g. directories). The iterator does not recurse into subdirectories. + + *package* is either a name or a module object which conforms to the + ``Package`` requirements. + + :mod:`importlib.machinery` -- Importers and path hooks ------------------------------------------------------ @@ -1080,7 +1291,7 @@ find and load modules. Name of the place from which the module is loaded, e.g. "builtin" for built-in modules and the filename for modules loaded from source. Normally "origin" should be set, but it may be ``None`` (the default) - which indicates it is unspecified. + which indicates it is unspecified (e.g. for namespace packages). .. attribute:: submodule_search_locations @@ -1327,6 +1538,14 @@ an :term:`importer`. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. +.. function:: source_hash(source_bytes) + + Return the hash of *source_bytes* as bytes. A hash-based ``.pyc`` file embeds + the :func:`source_hash` of the corresponding source file's contents in its + header. + + .. versionadded:: 3.7 + .. class:: LazyLoader(loader) A class which postpones the execution of the loader of a module until the diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 6be28a2b31c..bc4316fabae 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -34,6 +34,9 @@ provided as convenient choices for the second argument to :func:`getmembers`. They also help you determine when you can expect to find the following special attributes: +.. this function name is too big to fit in the ascii-art table below +.. |coroutine-origin-link| replace:: :func:`sys.set_coroutine_origin_tracking_depth` + +-----------+-------------------+---------------------------+ | Type | Attribute | Description | +===========+===================+===========================+ @@ -215,6 +218,10 @@ attributes: +-----------+-------------------+---------------------------+ | | cr_code | code | +-----------+-------------------+---------------------------+ +| | cr_origin | where coroutine was | +| | | created, or ``None``. See | +| | | |coroutine-origin-link| | ++-----------+-------------------+---------------------------+ | builtin | __doc__ | documentation string | +-----------+-------------------+---------------------------+ | | __name__ | original name of this | @@ -234,6 +241,9 @@ attributes: The ``__name__`` attribute of generators is now set from the function name, instead of the code name, and it can now be modified. +.. versionchanged:: 3.7 + + Add ``cr_origin`` attribute to coroutines. .. function:: getmembers(object[, predicate]) @@ -592,7 +602,13 @@ function. .. attribute:: Signature.parameters An ordered mapping of parameters' names to the corresponding - :class:`Parameter` objects. + :class:`Parameter` objects. Parameters appear in strict definition + order, including keyword-only parameters. + + .. versionchanged:: 3.7 + Python only explicitly guaranteed that it preserved the declaration + order of keyword-only parameters as of version 3.7, although in practice + this order had always been preserved in Python 3. .. attribute:: Signature.return_annotation @@ -885,7 +901,7 @@ Classes and functions *defaults* is an *n*-tuple of default argument values corresponding to the last *n* positional parameters, or ``None`` if there are no such defaults defined. - *kwonlyargs* is a list of keyword-only parameter names. + *kwonlyargs* is a list of keyword-only parameter names in declaration order. *kwonlydefaults* is a dictionary mapping parameter names from *kwonlyargs* to the default values used if no argument is supplied. *annotations* is a dictionary mapping parameter names to annotations. @@ -911,6 +927,11 @@ Classes and functions single-source Python 2/3 code migrating away from the legacy :func:`getargspec` API. + .. versionchanged:: 3.7 + Python only explicitly guaranteed that it preserved the declaration + order of keyword-only parameters as of version 3.7, although in practice + this order had always been preserved in Python 3. + .. function:: getargvalues(frame) diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 8a695ad99dd..5c71d900fd2 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -205,8 +205,8 @@ ABC Inherits Stub Methods Mixin M ``writable``, and ``writelines`` :class:`RawIOBase` :class:`IOBase` ``readinto`` and Inherited :class:`IOBase` methods, ``read``, ``write`` and ``readall`` -:class:`BufferedIOBase` :class:`IOBase` ``detach``, ``read``, Inherited :class:`IOBase` methods, ``readinto`` - ``read1``, and ``write`` +:class:`BufferedIOBase` :class:`IOBase` ``detach``, ``read``, Inherited :class:`IOBase` methods, ``readinto``, + ``read1``, and ``write`` and ``readinto1`` :class:`TextIOBase` :class:`IOBase` ``detach``, ``read``, Inherited :class:`IOBase` methods, ``encoding``, ``readline``, and ``errors``, and ``newlines`` ``write`` @@ -385,14 +385,17 @@ I/O Base Classes .. method:: read(size=-1) Read up to *size* bytes from the object and return them. As a convenience, - if *size* is unspecified or -1, :meth:`readall` is called. Otherwise, - only one system call is ever made. Fewer than *size* bytes may be - returned if the operating system call returns fewer than *size* bytes. + if *size* is unspecified or -1, all bytes until EOF are returned. + Otherwise, only one system call is ever made. Fewer than *size* bytes may + be returned if the operating system call returns fewer than *size* bytes. If 0 bytes are returned, and *size* was not 0, this indicates end of file. If the object is in non-blocking mode and no bytes are available, ``None`` is returned. + The default implementation defers to :meth:`readall` and + :meth:`readinto`. + .. method:: readall() Read and return all the bytes from the stream until EOF, using multiple @@ -901,7 +904,7 @@ Text I/O locale encoding using :func:`locale.setlocale`, use the current locale encoding instead of the user preferred encoding. - :class:`TextIOWrapper` provides one attribute in addition to those of + :class:`TextIOWrapper` provides these members in addition to those of :class:`TextIOBase` and its parents: .. attribute:: line_buffering @@ -915,11 +918,19 @@ Text I/O .. versionadded:: 3.7 - .. method:: reconfigure(*, line_buffering=None, write_through=None) + .. method:: reconfigure(*[, encoding][, errors][, newline][, \ + line_buffering][, write_through]) - Reconfigure this text stream using new settings for *line_buffering* - and *write_through*. Passing ``None`` as an argument will retain - the current setting for that parameter. + Reconfigure this text stream using new settings for *encoding*, + *errors*, *newline*, *line_buffering* and *write_through*. + + Parameters not specified keep current settings, except + ``errors='strict`` is used when *encoding* is specified but + *errors* is not specified. + + It is not possible to change the encoding or newline if some data + has already been read from the stream. On the other hand, changing + encoding after write is possible. This method does an implicit stream flush before setting the new parameters. diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index fa6c340bb7b..0b3829f19fa 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -32,7 +32,7 @@ operator can be mapped across two vectors to form an efficient dot-product: ``sum(map(operator.mul, vector1, vector2))``. -**Infinite Iterators:** +**Infinite iterators:** ================== ================= ================================================= ========================================= Iterator Arguments Results Example @@ -61,7 +61,7 @@ Iterator Arguments Results :func:`zip_longest` p, q, ... (p[0], q[0]), (p[1], q[1]), ... ``zip_longest('ABCD', 'xy', fillvalue='-') --> Ax By C- D-`` ============================ ============================ ================================================= ============================================================= -**Combinatoric generators:** +**Combinatoric iterators:** ============================================== ==================== ============================================================= Iterator Arguments Results @@ -753,15 +753,16 @@ which incur interpreter overhead. def roundrobin(*iterables): "roundrobin('ABC', 'D', 'EF') --> A D E B F C" # Recipe credited to George Sakkis - pending = len(iterables) + num_active = len(iterables) nexts = cycle(iter(it).__next__ for it in iterables) - while pending: + while num_active: try: for next in nexts: yield next() except StopIteration: - pending -= 1 - nexts = cycle(islice(nexts, pending)) + # Remove the iterator we just exhausted from the cycle. + num_active -= 1 + nexts = cycle(islice(nexts, num_active)) def partition(pred, iterable): 'Use a predicate to partition entries into false entries and true entries' @@ -858,6 +859,29 @@ which incur interpreter overhead. indices = sorted(random.randrange(n) for i in range(r)) return tuple(pool[i] for i in indices) + def nth_combination(iterable, r, index): + 'Equivalent to list(combinations(iterable, r))[index]' + pool = tuple(iterable) + n = len(pool) + if r < 0 or r > n: + raise ValueError + c = 1 + k = min(r, n-r) + for i in range(1, k+1): + c = c * (n - k + i) // i + if index < 0: + index += c + if index < 0 or index >= c: + raise IndexError + result = [] + while r: + c, n, r = c*r//n, n-1, r-1 + while index >= c: + index -= c + c, n = c*(n-r)//n, n-1 + result.append(pool[-1-n]) + return tuple(result) + Note, many of the above recipes can be optimized by replacing global lookups with local variables defined as default values. For example, the *dotproduct* recipe can be written as:: diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index e8567a7b658..2fd44fe8e90 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -147,6 +147,16 @@ The :mod:`locale` module defines the following exception and functions: | ``CHAR_MAX`` | Nothing is specified in this locale. | +--------------+-----------------------------------------+ + The function sets temporarily the ``LC_CTYPE`` locale to the ``LC_NUMERIC`` + locale to decode ``decimal_point`` and ``thousands_sep`` byte strings if + they are non-ASCII or longer than 1 byte, and the ``LC_NUMERIC`` locale is + different than the ``LC_CTYPE`` locale. This temporary change affects other + threads. + + .. versionchanged:: 3.7 + The function now sets temporarily the ``LC_CTYPE`` locale to the + ``LC_NUMERIC`` locale in some cases. + .. function:: nl_langinfo(option) @@ -316,6 +326,13 @@ The :mod:`locale` module defines the following exception and functions: preferences, so this function is not thread-safe. If invoking setlocale is not necessary or desired, *do_setlocale* should be set to ``False``. + On Android or in the UTF-8 mode (:option:`-X` ``utf8`` option), always + return ``'UTF-8'``, the locale and the *do_setlocale* argument are ignored. + + .. versionchanged:: 3.7 + The function now always returns ``UTF-8`` on Android or if the UTF-8 mode + is enabled. + .. function:: normalize(localename) diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 148f131cac5..1ed129c00d4 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -91,12 +91,12 @@ is the module's name in the Python package namespace. scenario is to attach handlers only to the root logger, and to let propagation take care of the rest. - .. method:: Logger.setLevel(lvl) + .. method:: Logger.setLevel(level) - Sets the threshold for this logger to *lvl*. Logging messages which are less - severe than *lvl* will be ignored; logging messages which have severity *lvl* + Sets the threshold for this logger to *level*. Logging messages which are less + severe than *level* will be ignored; logging messages which have severity *level* or higher will be emitted by whichever handler or handlers service this logger, - unless a handler's level has been set to a higher severity level than *lvl*. + unless a handler's level has been set to a higher severity level than *level*. When a logger is created, the level is set to :const:`NOTSET` (which causes all messages to be processed when the logger is the root logger, or delegation @@ -117,7 +117,7 @@ is the module's name in the Python package namespace. See :ref:`levels` for a list of levels. .. versionchanged:: 3.2 - The *lvl* parameter now accepts a string representation of the + The *level* parameter now accepts a string representation of the level such as 'INFO' as an alternative to the integer constants such as :const:`INFO`. Note, however, that levels are internally stored as integers, and methods such as e.g. :meth:`getEffectiveLevel` and @@ -267,14 +267,14 @@ is the module's name in the Python package namespace. message. This method should only be called from an exception handler. - .. method:: Logger.addFilter(filt) + .. method:: Logger.addFilter(filter) - Adds the specified filter *filt* to this logger. + Adds the specified filter *filter* to this logger. - .. method:: Logger.removeFilter(filt) + .. method:: Logger.removeFilter(filter) - Removes the specified filter *filt* from this logger. + Removes the specified filter *filter* from this logger. .. method:: Logger.filter(record) @@ -393,33 +393,34 @@ subclasses. However, the :meth:`__init__` method in subclasses needs to call Releases the thread lock acquired with :meth:`acquire`. - .. method:: Handler.setLevel(lvl) + .. method:: Handler.setLevel(level) - Sets the threshold for this handler to *lvl*. Logging messages which are less - severe than *lvl* will be ignored. When a handler is created, the level is set - to :const:`NOTSET` (which causes all messages to be processed). + Sets the threshold for this handler to *level*. Logging messages which are + less severe than *level* will be ignored. When a handler is created, the + level is set to :const:`NOTSET` (which causes all messages to be + processed). See :ref:`levels` for a list of levels. .. versionchanged:: 3.2 - The *lvl* parameter now accepts a string representation of the + The *level* parameter now accepts a string representation of the level such as 'INFO' as an alternative to the integer constants such as :const:`INFO`. - .. method:: Handler.setFormatter(form) + .. method:: Handler.setFormatter(fmt) - Sets the :class:`Formatter` for this handler to *form*. + Sets the :class:`Formatter` for this handler to *fmt*. - .. method:: Handler.addFilter(filt) + .. method:: Handler.addFilter(filter) - Adds the specified filter *filt* to this handler. + Adds the specified filter *filter* to this handler. - .. method:: Handler.removeFilter(filt) + .. method:: Handler.removeFilter(filter) - Removes the specified filter *filt* from this handler. + Removes the specified filter *filter* from this handler. .. method:: Handler.filter(record) diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index 81244c2ed02..d901ad2cc1c 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -491,7 +491,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. `Configuring Netscape Mail on Unix: Why The Content-Length Format is Bad `_ An argument for using the original mbox format rather than a variation. - `"mbox" is a family of several mutually incompatible mailbox formats `_ + `"mbox" is a family of several mutually incompatible mailbox formats `_ A history of mbox variations. @@ -620,7 +620,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. `nmh - Message Handling System `_ Home page of :program:`nmh`, an updated version of the original :program:`mh`. - `MH & nmh: Email for Users & Programmers `_ + `MH & nmh: Email for Users & Programmers `_ A GPL-licensed book on :program:`mh` and :program:`nmh`, with some information on the mailbox format. diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 9e4946323a9..55eb41b86f5 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -461,7 +461,7 @@ Constants Tau is a circle constant equal to 2\ *π*, the ratio of a circle's circumference to its radius. To learn more about Tau, check out Vi Hart's video `Pi is (still) Wrong `_, and start celebrating - `Tau day `_ by eating twice as much pie! + `Tau day `_ by eating twice as much pie! .. versionadded:: 3.6 diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 3619cccfe63..8b3081cb80c 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1837,8 +1837,8 @@ Running the following commands creates a server for a single shared queue which remote clients can access:: >>> from multiprocessing.managers import BaseManager - >>> import queue - >>> queue = queue.Queue() + >>> from queue import Queue + >>> queue = Queue() >>> class QueueManager(BaseManager): pass >>> QueueManager.register('get_queue', callable=lambda:queue) >>> m = QueueManager(address=('', 50000), authkey=b'abracadabra') diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index 64aa3ac7c8a..3d29ac49b91 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -20,8 +20,10 @@ the Unix :program:`ftp` program and other FTP clients. A :class:`~netrc.netrc` instance or subclass instance encapsulates data from a netrc file. The initialization argument, if present, specifies the file to parse. If - no argument is given, the file :file:`.netrc` in the user's home directory will - be read. Parse errors will raise :exc:`NetrcParseError` with diagnostic + no argument is given, the file :file:`.netrc` in the user's home directory -- + as determined by :func:`os.path.expanduser` -- will be read. Otherwise, + a :exc:`FileNotFoundError` exception will be raised. + Parse errors will raise :exc:`NetrcParseError` with diagnostic information including the file name, line number, and terminating token. If no argument is specified on a POSIX system, the presence of passwords in the :file:`.netrc` file will raise a :exc:`NetrcParseError` if the file @@ -32,6 +34,10 @@ the Unix :program:`ftp` program and other FTP clients. .. versionchanged:: 3.4 Added the POSIX permission check. + .. versionchanged:: 3.7 + :func:`os.path.expanduser` is used to find the location of the + :file:`.netrc` file when *file* is not passed as argument. + .. exception:: NetrcParseError @@ -82,4 +88,3 @@ Instances of :class:`~netrc.netrc` have public instance variables: punctuation is allowed in passwords, however, note that whitespace and non-printable characters are not allowed in passwords. This is a limitation of the way the .netrc file is parsed and may be removed in the future. - diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 2ef187db2dc..337c7c29941 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -567,7 +567,7 @@ An option group is obtained using the class :class:`OptionGroup`: where - * parser is the :class:`OptionParser` instance the group will be insterted in + * parser is the :class:`OptionParser` instance the group will be inserted in to * title is the group title * description, optional, is a long description of the group diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 06d4ecedd86..36bb21c222e 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -240,8 +240,9 @@ the :mod:`glob` module.) .. function:: isfile(path) - Return ``True`` if *path* is an existing regular file. This follows symbolic - links, so both :func:`islink` and :func:`isfile` can be true for the same path. + Return ``True`` if *path* is an :func:`existing ` regular file. + This follows symbolic links, so both :func:`islink` and :func:`isfile` can + be true for the same path. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -249,8 +250,9 @@ the :mod:`glob` module.) .. function:: isdir(path) - Return ``True`` if *path* is an existing directory. This follows symbolic - links, so both :func:`islink` and :func:`isdir` can be true for the same path. + Return ``True`` if *path* is an :func:`existing ` directory. This + follows symbolic links, so both :func:`islink` and :func:`isdir` can be true + for the same path. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -258,8 +260,9 @@ the :mod:`glob` module.) .. function:: islink(path) - Return ``True`` if *path* refers to a directory entry that is a symbolic link. - Always ``False`` if symbolic links are not supported by the Python runtime. + Return ``True`` if *path* refers to an :func:`existing ` directory + entry that is a symbolic link. Always ``False`` if symbolic links are not + supported by the Python runtime. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 95c81137230..bae432d33b0 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -325,10 +325,11 @@ process and user. .. function:: getlogin() Return the name of the user logged in on the controlling terminal of the - process. For most purposes, it is more useful to use the environment - variables :envvar:`LOGNAME` or :envvar:`USERNAME` to find out who the user - is, or ``pwd.getpwuid(os.getuid())[0]`` to get the login name of the current - real user id. + process. For most purposes, it is more useful to use + :func:`getpass.getuser` since the latter checks the environment variables + :envvar:`LOGNAME` or :envvar:`USERNAME` to find out who the user is, and + falls back to ``pwd.getpwuid(os.getuid())[0]`` to get the login name of the + current real user id. Availability: Unix, Windows. @@ -735,13 +736,17 @@ as internal buffering of data. .. function:: dup2(fd, fd2, inheritable=True) - Duplicate file descriptor *fd* to *fd2*, closing the latter first if necessary. - The file descriptor *fd2* is :ref:`inheritable ` by default, - or non-inheritable if *inheritable* is ``False``. + Duplicate file descriptor *fd* to *fd2*, closing the latter first if + necessary. Return *fd2*. The new file descriptor is :ref:`inheritable + ` by default or non-inheritable if *inheritable* + is ``False``. .. versionchanged:: 3.4 Add the optional *inheritable* parameter. + .. versionchanged:: 3.7 + Return *fd2* on success. Previously, ``None`` was always returned. + .. function:: fchmod(fd, mode) @@ -1097,6 +1102,45 @@ or `the MSDN `_ on Windo .. versionadded:: 3.3 +.. function:: pwritev(fd, buffers, offset, flags=0) + + Combines the functionality of :func:`os.writev` and :func:`os.pwrite`. It + writes the contents of *buffers* to file descriptor *fd* at offset *offset*. + *buffers* must be a sequence of :term:`bytes-like objects `. + Buffers are processed in array order. Entire contents of first buffer is written + before proceeding to second, and so on. The operating system may set a limit + (sysconf() value SC_IOV_MAX) on the number of buffers that can be used. + :func:`~os.pwritev` writes the contents of each object to the file descriptor + and returns the total number of bytes written. + + The *flags* argument contains a bitwise OR of zero or more of the following + flags: + + - RWF_DSYNC + - RWF_SYNC + + Using non-zero flags requires Linux 4.7 or newer. + + Availability: Linux (version 2.6.30), FreeBSD 6.0 and newer, + OpenBSD (version 2.7 and newer). + + .. versionadded:: 3.7 + +.. data:: RWF_DSYNC (since Linux 4.7) + Provide a per-write equivalent of the O_DSYNC open(2) flag. This flag + is meaningful only for pwritev2(), and its effect applies only to the + data range written by the system call. + + .. versionadded:: 3.7 + +.. data:: RWF_SYNC (since Linux 4.7) + Provide a per-write equivalent of the O_SYNC open(2) flag. This flag is + meaningful only for pwritev2(), and its effect applies only to the data + range written by the system call. + + .. versionadded:: 3.7 + + .. function:: read(fd, n) Read at most *n* bytes from file descriptor *fd*. Return a bytestring containing the @@ -1191,6 +1235,51 @@ or `the MSDN `_ on Windo .. versionadded:: 3.3 +.. function:: preadv(fd, buffers, offset, flags=0) + + Combines the functionality of :func:`os.readv` and :func:`os.pread`. It + reads from a file descriptor *fd* into a number of mutable :term:`bytes-like + objects ` *buffers*. As :func:`os.readv`, it will transfer + data into each buffer until it is full and then move on to the next buffer in + the sequence to hold the rest of the data. Its fourth argument, *offset*, + specifies the file offset at which the input operation is to be performed. + :func:`~os.preadv` return the total number of bytes read (which can be less than + the total capacity of all the objects). + + The flags argument contains a bitwise OR of zero or more of the following + flags: + + - RWF_HIPRI + - RWF_NOWAIT + + Using non-zero flags requires Linux 4.6 or newer. + + Availability: Linux (version 2.6.30), FreeBSD 6.0 and newer, + OpenBSD (version 2.7 and newer). + + .. versionadded:: 3.7 + + +.. data:: RWF_HIPRI (since Linux 4.6) + High priority read/write. Allows block-based filesystems to use polling + of the device, which provides lower latency, but may use additional + resources. (Currently, this feature is usable only on a file descriptor + opened using the O_DIRECT flag.) + + .. versionadded:: 3.7 + + +.. data:: RWF_NOWAIT (since Linux 4.14) + Do not wait for data which is not immediately available. If this flag + is specified, the preadv2() system call will return instantly + if it would have to read data from the backing storage or wait for a lock. + If some data was successfully read, it will return the number of bytes + read. If no bytes were read, it will return -1 and set errno to EAGAIN. + Currently, this flag is meaningful only for preadv2(). + + .. versionadded:: 3.7 + + .. function:: tcgetpgrp(fd) Return the process group associated with the terminal given by *fd* (an open @@ -2385,6 +2474,14 @@ features: Time of file creation. + On Solaris and derivatives, the following attributes may also be + available: + + .. attribute:: st_fstype + + String that uniquely identifies the type of the filesystem that + contains the file. + On Mac OS systems, the following attributes may also be available: .. attribute:: st_rsize @@ -2428,6 +2525,8 @@ features: .. versionadded:: 3.5 Added the :attr:`st_file_attributes` member on Windows. + .. versionadded:: 3.7 + Added the :attr:`st_fstype` member to Solaris/derivatives. .. function:: statvfs(path) @@ -2436,7 +2535,7 @@ features: correspond to the members of the :c:type:`statvfs` structure, namely: :attr:`f_bsize`, :attr:`f_frsize`, :attr:`f_blocks`, :attr:`f_bfree`, :attr:`f_bavail`, :attr:`f_files`, :attr:`f_ffree`, :attr:`f_favail`, - :attr:`f_flag`, :attr:`f_namemax`. + :attr:`f_flag`, :attr:`f_namemax`, :attr:`f_fsid`. Two module-level constants are defined for the :attr:`f_flag` attribute's bit-flags: if :const:`ST_RDONLY` is set, the filesystem is mounted @@ -2471,6 +2570,9 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionadded:: 3.7 + Added :attr:`f_fsid`. + .. data:: supports_dir_fd diff --git a/Doc/library/ossaudiodev.rst b/Doc/library/ossaudiodev.rst index ec40c0b93ab..522bb7e0924 100644 --- a/Doc/library/ossaudiodev.rst +++ b/Doc/library/ossaudiodev.rst @@ -14,7 +14,7 @@ the standard audio interface for Linux and recent versions of FreeBSD. .. Things will get more complicated for future Linux versions, since ALSA is in the standard kernel as of 2.5.x. Presumably if you use ALSA, you'll have to make sure its OSS compatibility layer - is active to use ossaudiodev, but you're gonna need it for the vast + is active to use ossaudiodev, but you're going to need it for the vast majority of Linux audio apps anyway. Sounds like things are also complicated for other BSDs. In response @@ -447,4 +447,3 @@ The remaining methods are specific to audio mixing: microphone input:: mixer.setrecsrc (1 << ossaudiodev.SOUND_MIXER_MIC) - diff --git a/Doc/library/othergui.rst b/Doc/library/othergui.rst index d40abe16765..4548459f8e2 100644 --- a/Doc/library/othergui.rst +++ b/Doc/library/othergui.rst @@ -11,9 +11,9 @@ available for Python: `PyGObject `_ PyGObject provides introspection bindings for C libraries using `GObject `_. One of - these libraries is the `GTK+ 3 `_ widget set. + these libraries is the `GTK+ 3 `_ widget set. GTK+ comes with many more widgets than Tkinter provides. An online - `Python GTK+ 3 Tutorial `_ + `Python GTK+ 3 Tutorial `_ is available. `PyGTK `_ @@ -35,7 +35,7 @@ available for Python: Compared to PyQt, its licensing scheme is friendlier to non-open source applications. - `wxPython `_ + `wxPython `_ wxPython is a cross-platform GUI toolkit for Python that is built around the popular `wxWidgets `_ (formerly wxWindows) C++ toolkit. It provides a native look and feel for applications on diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 4f3148fb5c3..c6720cb5997 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -61,6 +61,12 @@ useful than quitting the debugger upon program's exit. :file:`pdb.py` now accepts a ``-c`` option that executes commands as if given in a :file:`.pdbrc` file, see :ref:`debugger-commands`. +.. versionadded:: 3.7 + :file:`pdb.py` now accepts a ``-m`` option that execute modules similar to the way + ``python3 -m`` does. As with a script, the debugger will pause execution just + before the first line of the module. + + The typical usage to break into the debugger from a running program is to insert :: @@ -326,16 +332,19 @@ by the local file. (com) end (Pdb) - To remove all commands from a breakpoint, type commands and follow it + To remove all commands from a breakpoint, type ``commands`` and follow it immediately with ``end``; that is, give no commands. - With no *bpnumber* argument, commands refers to the last breakpoint set. + With no *bpnumber* argument, ``commands`` refers to the last breakpoint set. You can use breakpoint commands to start your program up again. Simply use - the continue command, or step, or any other command that resumes execution. + the :pdbcmd:`continue` command, or :pdbcmd:`step`, + or any other command that resumes execution. - Specifying any command resuming execution (currently continue, step, next, - return, jump, quit and their abbreviations) terminates the command list (as if + Specifying any command resuming execution + (currently :pdbcmd:`continue`, :pdbcmd:`step`, :pdbcmd:`next`, + :pdbcmd:`return`, :pdbcmd:`jump`, :pdbcmd:`quit` and their abbreviations) + terminates the command :pdbcmd:`list` (as if that command was immediately followed by end). This is because any time you resume execution (even with a simple next or step), you may encounter another breakpoint—which could have its own command list, leading to ambiguities about diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 6e8430fa8e6..d0c4cf937c8 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -370,7 +370,7 @@ The :mod:`pickle` module exports two classes, :class:`Pickler` and Python 2 names to the new names used in Python 3. The *encoding* and *errors* tell pickle how to decode 8-bit string instances pickled by Python 2; these default to 'ASCII' and 'strict', respectively. The *encoding* can - be 'bytes' to read these ß8-bit string instances as bytes objects. + be 'bytes' to read these 8-bit string instances as bytes objects. .. method:: load() diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 7d306a2bcaf..20c086c8b52 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -38,7 +38,7 @@ or :class:`datetime.datetime` objects. .. seealso:: - `PList manual page `_ + `PList manual page `_ Apple's documentation of the file format. diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 48426a00c9a..a6dc56f43cb 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -139,6 +139,7 @@ The :mod:`pstats` module's :class:`~pstats.Stats` class has a variety of methods for manipulating and printing the data saved into a profile results file:: import pstats + from pstats import SortKey p = pstats.Stats('restats') p.strip_dirs().sort_stats(-1).print_stats() @@ -148,14 +149,14 @@ entries according to the standard module/line/name string that is printed. The :meth:`~pstats.Stats.print_stats` method printed out all the statistics. You might try the following sort calls:: - p.sort_stats('name') + p.sort_stats(SortKey.NAME) p.print_stats() The first call will actually sort the list by function name, and the second call will print out the statistics. The following are some interesting calls to experiment with:: - p.sort_stats('cumulative').print_stats(10) + p.sort_stats(SortKey.CUMULATIVE).print_stats(10) This sorts the profile by cumulative time in a function, and then only prints the ten most significant lines. If you want to understand what algorithms are @@ -164,20 +165,20 @@ taking time, the above line is what you would use. If you were looking to see what functions were looping a lot, and taking a lot of time, you would do:: - p.sort_stats('time').print_stats(10) + p.sort_stats(SortKey.TIME).print_stats(10) to sort according to time spent within each function, and then print the statistics for the top ten functions. You might also try:: - p.sort_stats('file').print_stats('__init__') + p.sort_stats(SortKey.FILENAME).print_stats('__init__') This will sort all the statistics by file name, and then print out statistics for only the class init methods (since they are spelled with ``__init__`` in them). As one final example, you could try:: - p.sort_stats('time', 'cumulative').print_stats(.5, 'init') + p.sort_stats(SortKey.TIME, SortKey.CUMULATIVE).print_stats(.5, 'init') This line sorts statistics with a primary key of time, and a secondary key of cumulative time, and then prints out some of the statistics. To be specific, the @@ -250,12 +251,13 @@ functions: without writing the profile data to a file:: import cProfile, pstats, io + from pstats import SortKey pr = cProfile.Profile() pr.enable() # ... do something ... pr.disable() s = io.StringIO() - sortby = 'cumulative' + sortby = SortKey.CUMULATIVE ps = pstats.Stats(pr, stream=s).sort_stats(sortby) ps.print_stats() print(s.getvalue()) @@ -361,60 +363,65 @@ Analysis of the profiler data is done using the :class:`~pstats.Stats` class. .. method:: sort_stats(*keys) This method modifies the :class:`Stats` object by sorting it according to - the supplied criteria. The argument is typically a string identifying the - basis of a sort (example: ``'time'`` or ``'name'``). + the supplied criteria. The argument can be either a string or a SortKey + enum identifying the basis of a sort (example: ``'time'``, ``'name'``, + ``SortKey.TIME`` or ``SortKey.NAME``). The SortKey enums argument have + advantage over the string argument in that it is more robust and less + error prone. When more than one key is provided, then additional keys are used as secondary criteria when there is equality in all keys selected before - them. For example, ``sort_stats('name', 'file')`` will sort all the - entries according to their function name, and resolve all ties (identical - function names) by sorting by file name. + them. For example, ``sort_stats(SortKey.NAME, SortKey.FILE)`` will sort + all the entries according to their function name, and resolve all ties + (identical function names) by sorting by file name. - Abbreviations can be used for any key names, as long as the abbreviation - is unambiguous. The following are the keys currently defined: + For the string argument, abbreviations can be used for any key names, as + long as the abbreviation is unambiguous. - +------------------+----------------------+ - | Valid Arg | Meaning | - +==================+======================+ - | ``'calls'`` | call count | - +------------------+----------------------+ - | ``'cumulative'`` | cumulative time | - +------------------+----------------------+ - | ``'cumtime'`` | cumulative time | - +------------------+----------------------+ - | ``'file'`` | file name | - +------------------+----------------------+ - | ``'filename'`` | file name | - +------------------+----------------------+ - | ``'module'`` | file name | - +------------------+----------------------+ - | ``'ncalls'`` | call count | - +------------------+----------------------+ - | ``'pcalls'`` | primitive call count | - +------------------+----------------------+ - | ``'line'`` | line number | - +------------------+----------------------+ - | ``'name'`` | function name | - +------------------+----------------------+ - | ``'nfl'`` | name/file/line | - +------------------+----------------------+ - | ``'stdname'`` | standard name | - +------------------+----------------------+ - | ``'time'`` | internal time | - +------------------+----------------------+ - | ``'tottime'`` | internal time | - +------------------+----------------------+ + The following are the valid string and SortKey: + + +------------------+---------------------+----------------------+ + | Valid String Arg | Valid enum Arg | Meaning | + +==================+=====================+======================+ + | ``'calls'`` | SortKey.CALLS | call count | + +------------------+---------------------+----------------------+ + | ``'cumulative'`` | SortKey.CUMULATIVE | cumulative time | + +------------------+---------------------+----------------------+ + | ``'cumtime'`` | N/A | cumulative time | + +------------------+---------------------+----------------------+ + | ``'file'`` | N/A | file name | + +------------------+---------------------+----------------------+ + | ``'filename'`` | SortKey.FILENAME | file name | + +------------------+---------------------+----------------------+ + | ``'module'`` | N/A | file name | + +------------------+---------------------+----------------------+ + | ``'ncalls'`` | N/A | call count | + +------------------+---------------------+----------------------+ + | ``'pcalls'`` | SortKey.PCALLS | primitive call count | + +------------------+---------------------+----------------------+ + | ``'line'`` | SortKey.LINE | line number | + +------------------+---------------------+----------------------+ + | ``'name'`` | SortKey.NAME | function name | + +------------------+---------------------+----------------------+ + | ``'nfl'`` | SortKey.NFL | name/file/line | + +------------------+---------------------+----------------------+ + | ``'stdname'`` | SortKey.STDNAME | standard name | + +------------------+---------------------+----------------------+ + | ``'time'`` | SortKey.TIME | internal time | + +------------------+---------------------+----------------------+ + | ``'tottime'`` | N/A | internal time | + +------------------+---------------------+----------------------+ Note that all sorts on statistics are in descending order (placing most time consuming items first), where as name, file, and line number searches are in ascending order (alphabetical). The subtle distinction between - ``'nfl'`` and ``'stdname'`` is that the standard name is a sort of the - name as printed, which means that the embedded line numbers get compared - in an odd way. For example, lines 3, 20, and 40 would (if the file names - were the same) appear in the string order 20, 3 and 40. In contrast, - ``'nfl'`` does a numeric compare of the line numbers. In fact, - ``sort_stats('nfl')`` is the same as ``sort_stats('name', 'file', - 'line')``. + ``SortKey.NFL`` and ``SortKey.STDNAME`` is that the standard name is a + sort of the name as printed, which means that the embedded line numbers + get compared in an odd way. For example, lines 3, 20, and 40 would (if + the file names were the same) appear in the string order 20, 3 and 40. + In contrast, ``SortKey.NFL`` does a numeric compare of the line numbers. + In fact, ``sort_stats(SortKey.NFL)`` is the same as + ``sort_stats(SortKey.NAME, SortKey.FILENAME, SortKey.LINE)``. For backward-compatibility reasons, the numeric arguments ``-1``, ``0``, ``1``, and ``2`` are permitted. They are interpreted as ``'stdname'``, @@ -424,6 +431,8 @@ Analysis of the profiler data is done using the :class:`~pstats.Stats` class. .. For compatibility with the old profiler. + .. versionadded:: 3.7 + Added the SortKey enum. .. method:: reverse_order() diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index 0af8fb15b36..d720e010505 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -27,7 +27,7 @@ byte-code cache files in the directory containing the source code. Exception raised when an error occurs while attempting to compile the file. -.. function:: compile(file, cfile=None, dfile=None, doraise=False, optimize=-1) +.. function:: compile(file, cfile=None, dfile=None, doraise=False, optimize=-1, invalidation_mode=PycInvalidationMode.TIMESTAMP) Compile a source file to byte-code and write out the byte-code cache file. The source code is loaded from the file named *file*. The byte-code is @@ -53,6 +53,12 @@ byte-code cache files in the directory containing the source code. :func:`compile` function. The default of ``-1`` selects the optimization level of the current interpreter. + *invalidation_mode* should be a member of the :class:`PycInvalidationMode` + enum and controls how the generated ``.pyc`` files are invalidated at + runtime. If the :envvar:`SOURCE_DATE_EPOCH` environment variable is set, + *invalidation_mode* will be forced to + :attr:`PycInvalidationMode.CHECKED_HASH`. + .. versionchanged:: 3.2 Changed default value of *cfile* to be :PEP:`3147`-compliant. Previous default was *file* + ``'c'`` (``'o'`` if optimization was enabled). @@ -65,6 +71,44 @@ byte-code cache files in the directory containing the source code. caveat that :exc:`FileExistsError` is raised if *cfile* is a symlink or non-regular file. + .. versionchanged:: 3.7 + The *invalidation_mode* parameter was added as specified in :pep:`552`. + If the :envvar:`SOURCE_DATE_EPOCH` environment variable is set, + *invalidation_mode* will be forced to + :attr:`PycInvalidationMode.CHECKED_HASH`. + + +.. class:: PycInvalidationMode + + A enumeration of possible methods the interpreter can use to determine + whether a bytecode file is up to date with a source file. The ``.pyc`` file + indicates the desired invalidation mode in its header. See + :ref:`pyc-invalidation` for more information on how Python invalidates + ``.pyc`` files at runtime. + + .. versionadded:: 3.7 + + .. attribute:: TIMESTAMP + + The ``.pyc`` file includes the timestamp and size of the source file, + which Python will compare against the metadata of the source file at + runtime to determine if the ``.pyc`` file needs to be regenerated. + + .. attribute:: CHECKED_HASH + + The ``.pyc`` file includes a hash of the source file content, which Python + will compare against the source at runtime to determine if the ``.pyc`` + file needs to be regenerated. + + .. attribute:: UNCHECKED_HASH + + Like :attr:`CHECKED_HASH`, the ``.pyc`` file includes a hash of the source + file content. However, Python will at runtime assume the ``.pyc`` file is + up to date and not validate the ``.pyc`` against the source file at all. + + This option is useful when the ``.pycs`` are kept up to date by some + system external to Python like a build system. + .. function:: main(args=None) diff --git a/Doc/library/python.rst b/Doc/library/python.rst index f307d7db6db..440dc6632b8 100644 --- a/Doc/library/python.rst +++ b/Doc/library/python.rst @@ -24,4 +24,3 @@ overview: gc.rst inspect.rst site.rst - fpectl.rst diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index bd0fc2d8f3c..1520faa9b83 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -23,8 +23,14 @@ the first retrieved (operating like a stack). With a priority queue, the entries are kept sorted (using the :mod:`heapq` module) and the lowest valued entry is retrieved first. -Internally, the module uses locks to temporarily block competing threads; -however, it is not designed to handle reentrancy within a thread. +Internally, those three types of queues use locks to temporarily block +competing threads; however, they are not designed to handle reentrancy +within a thread. + +In addition, the module implements a "simple" +:abbr:`FIFO (first-in, first-out)` queue type where +specific implementations can provide additional guarantees +in exchange for the smaller functionality. The :mod:`queue` module defines the following classes and exceptions: @@ -56,6 +62,24 @@ The :mod:`queue` module defines the following classes and exceptions: one returned by ``sorted(list(entries))[0]``). A typical pattern for entries is a tuple in the form: ``(priority_number, data)``. + If the *data* elements are not comparable, the data can be wrapped in a class + that ignores the data item and only compares the priority number:: + + from dataclasses import dataclass, field + from typing import Any + + @dataclass(order=True) + class PrioritizedItem: + priority: int + item: Any=field(compare=False) + +.. class:: SimpleQueue() + + Constructor for an unbounded :abbr:`FIFO (first-in, first-out)` queue. + Simple queues lack advanced functionality such as task tracking. + + .. versionadded:: 3.7 + .. exception:: Empty @@ -191,6 +215,60 @@ Example of how to wait for enqueued tasks to be completed:: t.join() +SimpleQueue Objects +------------------- + +:class:`SimpleQueue` objects provide the public methods described below. + +.. method:: SimpleQueue.qsize() + + Return the approximate size of the queue. Note, qsize() > 0 doesn't + guarantee that a subsequent get() will not block. + + +.. method:: SimpleQueue.empty() + + Return ``True`` if the queue is empty, ``False`` otherwise. If empty() + returns ``False`` it doesn't guarantee that a subsequent call to get() + will not block. + + +.. method:: SimpleQueue.put(item, block=True, timeout=None) + + Put *item* into the queue. The method never blocks and always succeeds + (except for potential low-level errors such as failure to allocate memory). + The optional args *block* and *timeout* are ignored and only provided + for compatibility with :meth:`Queue.put`. + + .. impl-detail:: + This method has a C implementation which is reentrant. That is, a + ``put()`` or ``get()`` call can be interrupted by another ``put()`` + call in the same thread without deadlocking or corrupting internal + state inside the queue. This makes it appropriate for use in + destructors such as ``__del__`` methods or :mod:`weakref` callbacks. + + +.. method:: SimpleQueue.put_nowait(item) + + Equivalent to ``put(item)``, provided for compatibility with + :meth:`Queue.put_nowait`. + + +.. method:: SimpleQueue.get(block=True, timeout=None) + + Remove and return an item from the queue. If optional args *block* is true and + *timeout* is ``None`` (the default), block if necessary until an item is available. + If *timeout* is a positive number, it blocks at most *timeout* seconds and + raises the :exc:`Empty` exception if no item was available within that time. + Otherwise (*block* is false), return an item if one is immediately available, + else raise the :exc:`Empty` exception (*timeout* is ignored in that case). + + +.. method:: SimpleQueue.get_nowait() + + Equivalent to ``get(False)``. + + .. seealso:: Class :class:`multiprocessing.Queue` @@ -200,4 +278,3 @@ Example of how to wait for enqueued tasks to be completed:: :class:`collections.deque` is an alternative implementation of unbounded queues with fast atomic :meth:`~collections.deque.append` and :meth:`~collections.deque.popleft` operations that do not require locking. - diff --git a/Doc/library/quopri.rst b/Doc/library/quopri.rst index a3f94a0ad06..86717c00c3c 100644 --- a/Doc/library/quopri.rst +++ b/Doc/library/quopri.rst @@ -34,9 +34,10 @@ sending a graphics file. Encode the contents of the *input* file and write the resulting quoted-printable data to the *output* file. *input* and *output* must be - :term:`binary file objects `. *quotetabs*, a flag which controls - whether to encode embedded spaces and tabs must be provideda and when true it - encodes such embedded whitespace, and when false it leaves them unencoded. + :term:`binary file objects `. *quotetabs*, a + non-optional flag which controls whether to encode embedded spaces + and tabs; when true it encodes such embedded whitespace, and when + false it leaves them unencoded. Note that spaces and tabs appearing at the end of lines are always encoded, as per :rfc:`1521`. *header* is a flag which controls if spaces are encoded as underscores as per :rfc:`1522`. diff --git a/Doc/library/re.rst b/Doc/library/re.rst index ca09a91c0d2..a15f00b7585 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -345,7 +345,7 @@ The special characters are: This example looks for a word following a hyphen: - >>> m = re.search('(?<=-)\w+', 'spam-egg') + >>> m = re.search(r'(?<=-)\w+', 'spam-egg') >>> m.group(0) 'egg' @@ -692,11 +692,11 @@ form. splits occur, and the remainder of the string is returned as the final element of the list. :: - >>> re.split('\W+', 'Words, words, words.') + >>> re.split(r'\W+', 'Words, words, words.') ['Words', 'words', 'words', ''] - >>> re.split('(\W+)', 'Words, words, words.') + >>> re.split(r'(\W+)', 'Words, words, words.') ['Words', ', ', 'words', ', ', 'words', '.', ''] - >>> re.split('\W+', 'Words, words, words.', 1) + >>> re.split(r'\W+', 'Words, words, words.', 1) ['Words', 'words, words.'] >>> re.split('[a-f]+', '0a3B9', flags=re.IGNORECASE) ['0', '3', '9'] @@ -705,43 +705,28 @@ form. the string, the result will start with an empty string. The same holds for the end of the string:: - >>> re.split('(\W+)', '...words, words...') + >>> re.split(r'(\W+)', '...words, words...') ['', '...', 'words', ', ', 'words', '...', ''] That way, separator components are always found at the same relative indices within the result list. - .. note:: + Empty matches for the pattern split the string only when not adjacent + to a previous empty match. - :func:`split` doesn't currently split a string on an empty pattern match. - For example:: - - >>> re.split('x*', 'axbc') - ['a', 'bc'] - - Even though ``'x*'`` also matches 0 'x' before 'a', between 'b' and 'c', - and after 'c', currently these matches are ignored. The correct behavior - (i.e. splitting on empty matches too and returning ``['', 'a', 'b', 'c', - '']``) will be implemented in future versions of Python, but since this - is a backward incompatible change, a :exc:`FutureWarning` will be raised - in the meanwhile. - - Patterns that can only match empty strings currently never split the - string. Since this doesn't match the expected behavior, a - :exc:`ValueError` will be raised starting from Python 3.5:: - - >>> re.split("^$", "foo\n\nbar\n", flags=re.M) - Traceback (most recent call last): - File "", line 1, in - ... - ValueError: split() requires a non-empty pattern match. + >>> re.split(r'\b', 'Words, words, words.') + ['', 'Words', ', ', 'words', ', ', 'words', '.'] + >>> re.split(r'\W*', '...words...') + ['', '', 'w', 'o', 'r', 'd', 's', '', ''] + >>> re.split(r'(\W*)', '...words...') + ['', '...', '', '', 'w', '', 'o', '', 'r', '', 'd', '', 's', '...', '', '', ''] .. versionchanged:: 3.1 Added the optional flags argument. - .. versionchanged:: 3.5 - Splitting on a pattern that could match an empty string now raises - a warning. Patterns that can only match empty strings are now rejected. + .. versionchanged:: 3.7 + Added support of splitting on a pattern that could match an empty string. + .. function:: findall(pattern, string, flags=0) @@ -749,8 +734,10 @@ form. strings. The *string* is scanned left-to-right, and matches are returned in the order found. If one or more groups are present in the pattern, return a list of groups; this will be a list of tuples if the pattern has more than - one group. Empty matches are included in the result unless they touch the - beginning of another match. + one group. Empty matches are included in the result. + + .. versionchanged:: 3.7 + Non-empty matches can now start just after a previous empty match. .. function:: finditer(pattern, string, flags=0) @@ -758,8 +745,10 @@ form. Return an :term:`iterator` yielding :ref:`match objects ` over all non-overlapping matches for the RE *pattern* in *string*. The *string* is scanned left-to-right, and matches are returned in the order found. Empty - matches are included in the result unless they touch the beginning of another - match. + matches are included in the result. + + .. versionchanged:: 3.7 + Non-empty matches can now start just after a previous empty match. .. function:: sub(pattern, repl, string, count=0, flags=0) @@ -795,8 +784,8 @@ form. The optional argument *count* is the maximum number of pattern occurrences to be replaced; *count* must be a non-negative integer. If omitted or zero, all occurrences will be replaced. Empty matches for the pattern are replaced only - when not adjacent to a previous match, so ``sub('x*', '-', 'abc')`` returns - ``'-a-b-c-'``. + when not adjacent to a previous empty match, so ``sub('x*', '-', 'abxd')`` returns + ``'-a-b--d-'``. In string-type *repl* arguments, in addition to the character escapes and backreferences described above, @@ -822,6 +811,9 @@ form. Unknown escapes in *repl* consisting of ``'\'`` and an ASCII letter now are errors. + Empty matches for the pattern are replaced when adjacent to a previous + non-empty match. + .. function:: subn(pattern, repl, string, count=0, flags=0) diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 4d4a6161057..6094a7b8714 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -69,7 +69,7 @@ Scheduler Objects Schedule a new event. The *time* argument should be a numeric type compatible with the return value of the *timefunc* function passed to the constructor. Events scheduled for the same *time* will be executed in the order of their - *priority*. + *priority*. A lower number represents a higher priority. Executing the event means executing ``action(*argument, **kwargs)``. *argument* is a sequence holding the positional arguments for *action*. diff --git a/Doc/library/secrets.rst b/Doc/library/secrets.rst index 9bf848f9114..28ce472c7e7 100644 --- a/Doc/library/secrets.rst +++ b/Doc/library/secrets.rst @@ -130,7 +130,7 @@ Other functions Return ``True`` if strings *a* and *b* are equal, otherwise ``False``, in such a way as to reduce the risk of - `timing attacks `_. + `timing attacks `_. See :func:`hmac.compare_digest` for additional details. @@ -173,7 +173,7 @@ three digits: break -Generate an `XKCD-style passphrase `_: +Generate an `XKCD-style passphrase `_: .. testcode:: diff --git a/Doc/library/select.rst b/Doc/library/select.rst index bd5442c6a27..e252e7adb92 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -264,7 +264,7 @@ object. Edge and Level Trigger Polling (epoll) Objects ---------------------------------------------- - http://linux.die.net/man/4/epoll + https://linux.die.net/man/4/epoll *eventmask* diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 2b84fa29372..1527deb167f 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -318,7 +318,8 @@ Directory and files operations Return disk usage statistics about the given path as a :term:`named tuple` with the attributes *total*, *used* and *free*, which are the amount of - total, used and free space, in bytes. + total, used and free space, in bytes. On Windows, *path* must be a + directory; on Unix, it can be a file or directory. .. versionadded:: 3.3 diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 2bc39d9f133..67eaa2c6381 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -300,7 +300,7 @@ The :mod:`signal` module defines the following functions: Availability: Unix. -.. function:: set_wakeup_fd(fd) +.. function:: set_wakeup_fd(fd, *, warn_on_full_buffer=True) Set the wakeup file descriptor to *fd*. When a signal is received, the signal number is written as a single byte into the fd. This can be used by @@ -312,16 +312,36 @@ The :mod:`signal` module defines the following functions: If not -1, *fd* must be non-blocking. It is up to the library to remove any bytes from *fd* before calling poll or select again. - Use for example ``struct.unpack('%uB' % len(data), data)`` to decode the - signal numbers list. - When threads are enabled, this function can only be called from the main thread; attempting to call it from other threads will cause a :exc:`ValueError` exception to be raised. + There are two common ways to use this function. In both approaches, + you use the fd to wake up when a signal arrives, but then they + differ in how they determine *which* signal or signals have + arrived. + + In the first approach, we read the data out of the fd's buffer, and + the byte values give you the signal numbers. This is simple, but in + rare cases it can run into a problem: generally the fd will have a + limited amount of buffer space, and if too many signals arrive too + quickly, then the buffer may become full, and some signals may be + lost. If you use this approach, then you should set + ``warn_on_full_buffer=True``, which will at least cause a warning + to be printed to stderr when signals are lost. + + In the second approach, we use the wakeup fd *only* for wakeups, + and ignore the actual byte values. In this case, all we care about + is whether the fd's buffer is empty or non-empty; a full buffer + doesn't indicate a problem at all. If you use this approach, then + you should set ``warn_on_full_buffer=False``, so that your users + are not confused by spurious warning messages. + .. versionchanged:: 3.5 On Windows, the function now also supports socket handles. + .. versionchanged:: 3.7 + Added ``warn_on_full_buffer`` parameter. .. function:: siginterrupt(signalnum, flag) diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 43daf790b77..ae408133a7c 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -97,12 +97,12 @@ not mentioned in either path configuration file. After these path manipulations, an attempt is made to import a module named :mod:`sitecustomize`, which can perform arbitrary site-specific customizations. It is typically created by a system administrator in the site-packages -directory. If this import fails with an :exc:`ImportError` exception, it is -silently ignored. If Python is started without output streams available, as +directory. If this import fails with an :exc:`ImportError` or its subclass +exception, and the exception's :attr:`name` attribute equals to ``'sitecustomize'``, +it is silently ignored. If Python is started without output streams available, as with :file:`pythonw.exe` on Windows (which is used by default to start IDLE), -attempted output from :mod:`sitecustomize` is ignored. Any exception other -than :exc:`ImportError` causes a silent and perhaps mysterious failure of the -process. +attempted output from :mod:`sitecustomize` is ignored. Any other exception +causes a silent and perhaps mysterious failure of the process. .. index:: module: usercustomize @@ -110,7 +110,9 @@ After this, an attempt is made to import a module named :mod:`usercustomize`, which can perform arbitrary user-specific customizations, if :data:`ENABLE_USER_SITE` is true. This file is intended to be created in the user site-packages directory (see below), which is part of ``sys.path`` unless -disabled by :option:`-s`. An :exc:`ImportError` will be silently ignored. +disabled by :option:`-s`. If this import fails with an :exc:`ImportError` or +its subclass exception, and the exception's :attr:`name` attribute equals to +``'usercustomize'``, it is silently ignored. Note that for some non-Unix systems, ``sys.prefix`` and ``sys.exec_prefix`` are empty, and the path manipulations are skipped; however the import of diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 42fd7ea0f0b..7f0d4ede7cc 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -461,10 +461,15 @@ The following functions all create :ref:`socket objects `. :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other ``SOCK_`` constants. The protocol number is usually zero and may be omitted or in the case where the address family is :const:`AF_CAN` the protocol should be one - of :const:`CAN_RAW`, :const:`CAN_BCM` or :const:`CAN_ISOTP`. If *fileno* is specified, the other - arguments are ignored, causing the socket with the specified file descriptor - to return. Unlike :func:`socket.fromfd`, *fileno* will return the same - socket and not a duplicate. This may help close a detached socket using + of :const:`CAN_RAW`, :const:`CAN_BCM` or :const:`CAN_ISOTP` + + If *fileno* is specified, the values for *family*, *type*, and *proto* are + auto-detected from the specified file descriptor. Auto-detection can be + overruled by calling the function with explicit *family*, *type*, or *proto* + arguments. This only affects how Python represents e.g. the return value + of :meth:`socket.getpeername` but not the actual OS resource. Unlike + :func:`socket.fromfd`, *fileno* will return the same socket and not a + duplicate. This may help close a detached socket using :meth:`socket.close()`. The newly created socket is :ref:`non-inheritable `. @@ -482,6 +487,20 @@ The following functions all create :ref:`socket objects `. .. versionchanged:: 3.7 The CAN_ISOTP protocol was added. + .. versionchanged:: 3.7 + When :const:`SOCK_NONBLOCK` or :const:`SOCK_CLOEXEC` + bit flags are applied to *type* they are cleared, and + :attr:`socket.type` will not reflect them. They are still passed + to the underlying system `socket()` call. Therefore:: + + sock = socket.socket( + socket.AF_INET, + socket.SOCK_STREAM | socket.SOCK_NONBLOCK) + + will still create a non-blocking socket on OSes that support + ``SOCK_NONBLOCK``, but ``sock.type`` will be set to + ``socket.SOCK_STREAM``. + .. function:: socketpair([family[, type[, proto]]]) Build a pair of connected socket objects using the given address family, socket @@ -564,6 +583,14 @@ Other functions The :mod:`socket` module also offers various network-related services: +.. function:: close(fd) + + Close a socket file descriptor. This is like :func:`os.close`, but for + sockets. On some platforms (most noticeable Windows) :func:`os.close` + does not work for socket file descriptors. + + .. versionadded:: 3.7 + .. function:: getaddrinfo(host, port, family=0, type=0, proto=0, flags=0) Translate the *host*/*port* argument into a sequence of 5-tuples that contain @@ -1065,6 +1092,16 @@ to sockets. to decode C structures encoded as byte strings). +.. method:: socket.getblocking() + + Return ``True`` if socket is in blocking mode, ``False`` if in + non-blocking. + + This is equivalent to checking ``socket.gettimeout() == 0``. + + .. versionadded:: 3.7 + + .. method:: socket.gettimeout() Return the timeout in seconds (float) associated with socket operations, @@ -1417,6 +1454,10 @@ to sockets. * ``sock.setblocking(False)`` is equivalent to ``sock.settimeout(0.0)`` + .. versionchanged:: 3.7 + The method no longer applies :const:`SOCK_NONBLOCK` flag on + :attr:`socket.type`. + .. method:: socket.settimeout(value) @@ -1429,6 +1470,10 @@ to sockets. For further information, please consult the :ref:`notes on socket timeouts `. + .. versionchanged:: 3.7 + The method no longer toggles :const:`SOCK_NONBLOCK` flag on + :attr:`socket.type`. + .. method:: socket.setsockopt(level, optname, value: int) .. method:: socket.setsockopt(level, optname, value: buffer) diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index e12c8c97844..56000273837 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -115,6 +115,21 @@ server classes. :class:`ForkingMixIn` and the Forking classes mentioned below are only available on POSIX platforms that support :func:`~os.fork`. + :meth:`socketserver.ForkingMixIn.server_close` waits until all child + processes complete. + + :meth:`socketserver.ThreadingMixIn.server_close` waits until all non-daemon + threads complete. Use daemonic threads by setting + :data:`ThreadingMixIn.daemon_threads` to ``True`` to not wait until threads + complete. + + .. versionchanged:: 3.7 + + :meth:`socketserver.ForkingMixIn.server_close` and + :meth:`socketserver.ThreadingMixIn.server_close` now waits until all + child processes and non-daemonic threads complete. + + .. class:: ForkingTCPServer ForkingUDPServer ThreadingTCPServer diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index c7b9af4037f..e7676a9f3a5 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -107,7 +107,7 @@ This example uses the iterator form:: The SQLite web page; the documentation describes the syntax and the available data types for the supported SQL dialect. - http://www.w3schools.com/sql/ + https://www.w3schools.com/sql/ Tutorial, reference and examples for learning SQL syntax. :pep:`249` - Database API Specification 2.0 diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 9b3bdd5489d..aa1075d4b02 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -146,9 +146,10 @@ Functions, Constants, and Exceptions .. exception:: CertificateError - Raised to signal an error with a certificate (such as mismatching - hostname). Certificate errors detected by OpenSSL, though, raise - an :exc:`SSLCertVerificationError`. + An alias for :exc:`SSLCertVerificationError`. + + .. versionchanged:: 3.7 + The exception is now an alias for :exc:`SSLCertVerificationError`. Socket creation @@ -429,6 +430,16 @@ Certificate handling Matching of IP addresses, when present in the subjectAltName field of the certificate, is now supported. + .. versionchanged:: 3.7 + The function is no longer used to TLS connections. Hostname matching + is now performed by OpenSSL. + + Allow wildcard when it is the leftmost and the only character + in that segment. Partial wildcards like ``www*.example.com`` are no + longer supported. + + .. deprecated:: 3.7 + .. function:: cert_time_to_seconds(cert_time) Return the time in seconds since the Epoch, given the ``cert_time`` @@ -846,6 +857,14 @@ Constants .. versionadded:: 3.5 +.. data:: HAS_NEVER_CHECK_COMMON_NAME + + Whether the OpenSSL library has built-in support not checking subject + common name and :attr:`SSLContext.hostname_checks_common_name` is + writeable. + + .. versionadded:: 3.7 + .. data:: HAS_ECDH Whether the OpenSSL library has built-in support for Elliptic Curve-based @@ -864,9 +883,9 @@ Constants .. data:: HAS_NPN Whether the OpenSSL library has built-in support for *Next Protocol - Negotiation* as described in the `NPN draft specification - `_. When true, - you can use the :meth:`SSLContext.set_npn_protocols` method to advertise + Negotiation* as described in the `Application Layer Protocol + Negotiation `_. + When true, you can use the :meth:`SSLContext.set_npn_protocols` method to advertise which protocols you want to support. .. versionadded:: 3.3 @@ -1071,6 +1090,12 @@ SSL sockets also have the following additional methods and attributes: The socket timeout is no more reset each time bytes are received or sent. The socket timeout is now to maximum total duration of the handshake. + .. versionchanged:: 3.7 + Hostname or IP address is matched by OpenSSL during handshake. The + function :func:`match_hostname` is no longer used. In case OpenSSL + refuses a hostname or IP address, the handshake is aborted early and + a TLS alert message is send to the peer. + .. method:: SSLSocket.getpeercert(binary_form=False) If there is no certificate for the peer on the other end of the connection, @@ -1370,7 +1395,7 @@ to speed up repeated connections from the same clients. The *capath* string, if present, is the path to a directory containing several CA certificates in PEM format, following an `OpenSSL specific layout - `_. + `_. The *cadata* object, if present, is either an ASCII string of one or more PEM-encoded certificates or a :term:`bytes-like object` of DER-encoded @@ -1497,8 +1522,8 @@ to speed up repeated connections from the same clients. Specify which protocols the socket should advertise during the SSL/TLS handshake. It should be a list of strings, like ``['http/1.1', 'spdy/2']``, ordered by preference. The selection of a protocol will happen during the - handshake, and will play out according to the `NPN draft specification - `_. After a + handshake, and will play out according to the `Application Layer Protocol Negotiation + `_. After a successful handshake, the :meth:`SSLSocket.selected_npn_protocol` method will return the agreed-upon protocol. @@ -1586,7 +1611,7 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.3 .. seealso:: - `SSL/TLS & Perfect Forward Secrecy `_ + `SSL/TLS & Perfect Forward Secrecy `_ Vincent Bernat. .. method:: SSLContext.wrap_socket(sock, server_side=False, \ @@ -1659,8 +1684,7 @@ to speed up repeated connections from the same clients. .. method:: SSLContext.session_stats() Get statistics about the SSL sessions created or managed by this context. - A dictionary is returned which maps the names of each `piece of information - `_ to their + A dictionary is returned which maps the names of each `piece of information `_ to their numeric values. For example, here is the total number of hits and misses in the session cache since the context was created:: @@ -1727,6 +1751,17 @@ to speed up repeated connections from the same clients. The protocol version chosen when constructing the context. This attribute is read-only. +.. attribute:: SSLContext.hostname_checks_common_name + + Whether :attr:`~SSLContext.check_hostname` falls back to verify the cert's + subject common name in the absence of a subject alternative name + extension (default: true). + + .. versionadded:: 3.7 + + .. note:: + Only writeable with OpenSSL 1.1.0 or higher. + .. attribute:: SSLContext.verify_flags The flags for certificate verification operations. You can set flags like @@ -2321,6 +2356,10 @@ in this case, the :func:`match_hostname` function can be used. This common check is automatically performed when :attr:`SSLContext.check_hostname` is enabled. +.. versionchanged:: 3.7 + Hostname matchings is now performed by OpenSSL. Python no longer uses + :func:`match_hostname`. + In server mode, if you want to authenticate your clients using the SSL layer (rather than using a higher-level authentication mechanism), you'll also have to specify :const:`CERT_REQUIRED` and similarly check the client certificate. @@ -2361,7 +2400,7 @@ enabled when negotiating a SSL session is possible through the :meth:`SSLContext.set_ciphers` method. Starting from Python 3.2.3, the ssl module disables certain weak ciphers by default, but you may want to further restrict the cipher choice. Be sure to read OpenSSL's documentation -about the `cipher list format `_. +about the `cipher list format `_. If you want to check which ciphers are enabled by a given cipher list, use :meth:`SSLContext.get_ciphers` or the ``openssl ciphers`` command on your system. @@ -2389,10 +2428,10 @@ successful call of :func:`~ssl.RAND_add`, :func:`~ssl.RAND_bytes` or `RFC 1422: Privacy Enhancement for Internet Electronic Mail: Part II: Certificate-Based Key Management `_ Steve Kent - `RFC 4086: Randomness Requirements for Security `_ + `RFC 4086: Randomness Requirements for Security `_ Donald E., Jeffrey I. Schiller - `RFC 5280: Internet X.509 Public Key Infrastructure Certificate and Certificate Revocation List (CRL) Profile `_ + `RFC 5280: Internet X.509 Public Key Infrastructure Certificate and Certificate Revocation List (CRL) Profile `_ D. Cooper `RFC 5246: The Transport Layer Security (TLS) Protocol Version 1.2 `_ diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 2aa778c4d06..bc3817836b9 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -257,8 +257,6 @@ However, for reading convenience, most of the examples show sorted sequences. * "Statistics for the Behavioral Sciences", Frederick J Gravetter and Larry B Wallnau (8th Edition). - * Calculating the `median `_. - * The `SSMEDIAN `_ function in the Gnome Gnumeric spreadsheet, including `this discussion diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 0bcafd33b20..ad7f578e086 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -973,9 +973,9 @@ Notes: (8) ``index`` raises :exc:`ValueError` when *x* is not found in *s*. - When supported, the additional arguments to the index method allow - efficient searching of subsections of the sequence. Passing the extra - arguments is roughly equivalent to using ``s[i:j].index(x)``, only + Not all implementations support passing the additional arguments *i* and *j*. + These arguments allow efficient searching of subsections of the sequence. Passing + the extra arguments is roughly equivalent to using ``s[i:j].index(x)``, only without copying any data and with the returned index being relative to the start of the sequence rather than the start of the slice. @@ -1599,6 +1599,20 @@ expression support in the :mod:`re` module). See :ref:`formatstrings` for a description of the various formatting options that can be specified in format strings. + .. note:: + When formatting a number (:class:`int`, :class:`float`, :class:`float` + and subclasses) with the ``n`` type (ex: ``'{:n}'.format(1234)``), the + function sets temporarily the ``LC_CTYPE`` locale to the ``LC_NUMERIC`` + locale to decode ``decimal_point`` and ``thousands_sep`` fields of + :c:func:`localeconv` if they are non-ASCII or longer than 1 byte, and the + ``LC_NUMERIC`` locale is different than the ``LC_CTYPE`` locale. This + temporary change affects other threads. + + .. versionchanged:: 3.7 + When formatting a number with the ``n`` type, the function sets + temporarily the ``LC_CTYPE`` locale to the ``LC_NUMERIC`` locale in some + cases. + .. method:: str.format_map(mapping) @@ -1639,6 +1653,15 @@ expression support in the :mod:`re` module). from the "Alphabetic" property defined in the Unicode Standard. +.. method:: str.isascii() + + Return true if the string is empty or all characters in the string are ASCII, + false otherwise. + ASCII characters have code points in the range U+0000-U+007F. + + .. versionadded:: 3.7 + + .. method:: str.isdecimal() Return true if all characters in the string are decimal @@ -2927,6 +2950,16 @@ place, and instead produce new objects. False +.. method:: bytes.isascii() + bytearray.isascii() + + Return true if the sequence is empty or all bytes in the sequence are ASCII, + false otherwise. + ASCII bytes are in the range 0-0x7F. + + .. versionadded:: 3.7 + + .. method:: bytes.isdigit() bytearray.isdigit() diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 5b254285257..ee8ea857da4 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -202,9 +202,9 @@ The grammar for a replacement field is as follows: .. productionlist:: sf replacement_field: "{" [`field_name`] ["!" `conversion`] [":" `format_spec`] "}" field_name: arg_name ("." `attribute_name` | "[" `element_index` "]")* - arg_name: [`identifier` | `integer`] + arg_name: [`identifier` | `digit`+] attribute_name: `identifier` - element_index: `integer` | `index_string` + element_index: `digit`+ | `index_string` index_string: + conversion: "r" | "s" | "a" format_spec: @@ -304,9 +304,9 @@ The general form of a *standard format specifier* is: fill: align: "<" | ">" | "=" | "^" sign: "+" | "-" | " " - width: `integer` + width: `digit`+ grouping_option: "_" | "," - precision: `integer` + precision: `digit`+ type: "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%" If a valid *align* value is specified, it can be preceded by a *fill* @@ -755,17 +755,14 @@ attributes: * *idpattern* -- This is the regular expression describing the pattern for non-braced placeholders. The default value is the regular expression - ``(?-i:[_a-zA-Z][_a-zA-Z0-9]*)``. If this is given and *braceidpattern* is + ``(?a:[_a-z][_a-z0-9]*)``. If this is given and *braceidpattern* is ``None`` this pattern will also apply to braced placeholders. .. note:: Since default *flags* is ``re.IGNORECASE``, pattern ``[a-z]`` can match - with some non-ASCII characters. That's why we use local ``-i`` flag here. - - While *flags* is kept to ``re.IGNORECASE`` for backward compatibility, - you can override it to ``0`` or ``re.IGNORECASE | re.ASCII`` when - subclassing. + with some non-ASCII characters. That's why we use the local ``a`` flag + here. .. versionchanged:: 3.7 *braceidpattern* can be used to define separate patterns used inside and diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index a2c184a0460..8673c4b18d8 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -47,12 +47,14 @@ compatibility with older versions, see the :ref:`call-function-trio` section. The arguments shown above are merely the most common ones, described below in :ref:`frequently-used-arguments` (hence the use of keyword-only notation in the abbreviated signature). The full function signature is largely the - same as that of the :class:`Popen` constructor - apart from *timeout*, - *input* and *check*, all the arguments to this function are passed through to - that interface. + same as that of the :class:`Popen` constructor - most of the arguments to + this function are passed through to that interface. (*timeout*, *input*, + *check*, and *capture_output* are not.) - This does not capture stdout or stderr by default. To do so, pass - :data:`PIPE` for the *stdout* and/or *stderr* arguments. + If *capture_output* is true, stdout and stderr will be captured. + When used, the internal :class:`Popen` object is automatically created with + ``stdout=PIPE`` and ``stderr=PIPE``. The *stdout* and *stderr* arguments may + not be used as well. The *timeout* argument is passed to :meth:`Popen.communicate`. If the timeout expires, the child process will be killed and waited for. The @@ -86,9 +88,9 @@ compatibility with older versions, see the :ref:`call-function-trio` section. ... subprocess.CalledProcessError: Command 'exit 1' returned non-zero exit status 1 - >>> subprocess.run(["ls", "-l", "/dev/null"], stdout=subprocess.PIPE) + >>> subprocess.run(["ls", "-l", "/dev/null"], capture_output=True) CompletedProcess(args=['ls', '-l', '/dev/null'], returncode=0, - stdout=b'crw-rw-rw- 1 root root 1, 3 Jan 23 16:23 /dev/null\n') + stdout=b'crw-rw-rw- 1 root root 1, 3 Jan 23 16:23 /dev/null\n', stderr=b'') .. versionadded:: 3.5 @@ -98,7 +100,8 @@ compatibility with older versions, see the :ref:`call-function-trio` section. .. versionchanged:: 3.7 - Added the *text* parameter, as a more understandable alias of *universal_newlines* + Added the *text* parameter, as a more understandable alias of *universal_newlines*. + Added the *capture_output* parameter. .. class:: CompletedProcess @@ -332,12 +335,12 @@ functions. the class uses the Windows ``CreateProcess()`` function. The arguments to :class:`Popen` are as follows. - *args* should be a sequence of program arguments or else a single string. - By default, the program to execute is the first item in *args* if *args* is - a sequence. If *args* is a string, the interpretation is - platform-dependent and described below. See the *shell* and *executable* - arguments for additional differences from the default behavior. Unless - otherwise stated, it is recommended to pass *args* as a sequence. + *args* should be a sequence of program arguments or else a single string or + :term:`path-like object`. By default, the program to execute is the first + item in *args* if *args* is a sequence. If *args* is a string, the + interpretation is platform-dependent and described below. See the *shell* + and *executable* arguments for additional differences from the default + behavior. Unless otherwise stated, it is recommended to pass *args* as a sequence. On POSIX, if *args* is a string, the string is interpreted as the name or path of the program to execute. However, this can only be done if not @@ -452,17 +455,20 @@ functions. common use of *preexec_fn* to call os.setsid() in the child. If *close_fds* is true, all file descriptors except :const:`0`, :const:`1` and - :const:`2` will be closed before the child process is executed. (POSIX only). - The default varies by platform: Always true on POSIX. On Windows it is - true when *stdin*/*stdout*/*stderr* are :const:`None`, false otherwise. + :const:`2` will be closed before the child process is executed. On Windows, if *close_fds* is true then no handles will be inherited by the - child process. Note that on Windows, you cannot set *close_fds* to true and - also redirect the standard handles by setting *stdin*, *stdout* or *stderr*. + child process unless explicitly passed in the ``handle_list`` element of + :attr:`STARTUPINFO.lpAttributeList`, or by standard handle redirection. .. versionchanged:: 3.2 The default for *close_fds* was changed from :const:`False` to what is described above. + .. versionchanged:: 3.7 + On Windows the default for *close_fds* was changed from :const:`False` to + :const:`True` when redirecting the standard handles. It's now possible to + set *close_fds* to :const:`True` when redirecting the standard handles. + *pass_fds* is an optional sequence of file descriptors to keep open between the parent and child. Providing any *pass_fds* forces *close_fds* to be :const:`True`. (POSIX only) @@ -545,6 +551,10 @@ functions. Popen destructor now emits a :exc:`ResourceWarning` warning if the child process is still running. + .. versionchanged:: 3.7 + *args*, or the first element of *args* if *args* is a sequence, can now + be a :term:`path-like object`. + Exceptions ^^^^^^^^^^ @@ -764,7 +774,7 @@ The :class:`STARTUPINFO` class and following constants are only available on Windows. .. class:: STARTUPINFO(*, dwFlags=0, hStdInput=None, hStdOutput=None, \ - hStdError=None, wShowWindow=0) + hStdError=None, wShowWindow=0, lpAttributeList=None) Partial support of the Windows `STARTUPINFO `__ @@ -814,6 +824,33 @@ on Windows. :data:`SW_HIDE` is provided for this attribute. It is used when :class:`Popen` is called with ``shell=True``. + .. attribute:: lpAttributeList + + A dictionary of additional attributes for process creation as given in + ``STARTUPINFOEX``, see + `UpdateProcThreadAttribute `__. + + Supported attributes: + + **handle_list** + Sequence of handles that will be inherited. *close_fds* must be true if + non-empty. + + The handles must be temporarily made inheritable by + :func:`os.set_handle_inheritable` when passed to the :class:`Popen` + constructor, else :class:`OSError` will be raised with Windows error + ``ERROR_INVALID_PARAMETER`` (87). + + .. warning:: + + In a multithreaded process, use caution to avoid leaking handles + that are marked inheritable when combining this feature with + concurrent calls to other process creation functions that inherit + all handles such as :func:`os.system`. This also applies to + standard handle redirection, which temporarily creates inheritable + handles. + + .. versionadded:: 3.7 Windows Constants ^^^^^^^^^^^^^^^^^ @@ -1050,6 +1087,9 @@ calls these functions. .. versionchanged:: 3.4 Support for the *input* keyword argument was added. + .. versionchanged:: 3.6 + *encoding* and *errors* were added. See :func:`run` for details. + .. _subprocess-replacements: Replacing Older Functions with the :mod:`subprocess` Module diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index faf540c4ea4..67925e426bb 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -313,6 +313,9 @@ always available. has caught :exc:`SystemExit` (such as an error flushing buffered data in the standard streams), the exit status is changed to 120. + .. versionchanged:: 3.7 + Added ``utf8_mode`` attribute for the new :option:`-X` ``utf8`` flag. + .. data:: flags @@ -334,6 +337,8 @@ always available. :const:`bytes_warning` :option:`-b` :const:`quiet` :option:`-q` :const:`hash_randomization` :option:`-R` + :const:`dev_mode` :option:`-X` ``dev`` + :const:`utf8_mode` :option:`-X` ``utf8`` ============================= ============================= .. versionchanged:: 3.2 @@ -345,6 +350,10 @@ always available. .. versionchanged:: 3.3 Removed obsolete ``division_warning`` attribute. + .. versionchanged:: 3.7 + Added ``dev_mode`` attribute for the new :option:`-X` ``dev`` flag + and ``utf8_mode`` attribute for the new :option:`-X` ``utf8`` flag. + .. data:: float_info @@ -488,6 +497,8 @@ always available. :func:`os.fsencode` and :func:`os.fsdecode` should be used to ensure that the correct encoding and errors mode are used. + * In the UTF-8 mode, the encoding is ``utf-8`` on any platform. + * On Mac OS X, the encoding is ``'utf-8'``. * On Unix, the encoding is the locale encoding. @@ -502,6 +513,10 @@ always available. Windows is no longer guaranteed to return ``'mbcs'``. See :pep:`529` and :func:`_enablelegacywindowsfsencoding` for more information. + .. versionchanged:: 3.7 + Return 'utf-8' in the UTF-8 mode. + + .. function:: getfilesystemencodeerrors() Return the name of the error mode used to convert between Unicode filenames @@ -660,6 +675,18 @@ always available. for details.) +.. function:: get_coroutine_origin_tracking_depth() + + Get the current coroutine origin tracking depth, as set by + func:`set_coroutine_origin_tracking_depth`. + + .. versionadded:: 3.7 + + .. note:: + This function has been added on a provisional basis (see :pep:`411` + for details.) Use it only for debugging purposes. + + .. function:: get_coroutine_wrapper() Returns ``None``, or a wrapper set by :func:`set_coroutine_wrapper`. @@ -671,6 +698,10 @@ always available. This function has been added on a provisional basis (see :pep:`411` for details.) Use it only for debugging purposes. + .. deprecated:: 3.7 + The coroutine wrapper functionality has been deprecated, and + will be removed in 3.8. See :issue:`32591` for details. + .. data:: hash_info @@ -1053,13 +1084,39 @@ always available. Set the system's profile function, which allows you to implement a Python source code profiler in Python. See chapter :ref:`profile` for more information on the Python profiler. The system's profile function is called similarly to the - system's trace function (see :func:`settrace`), but it isn't called for each - executed line of code (only on call and return, but the return event is reported - even when an exception has been set). The function is thread-specific, but - there is no way for the profiler to know about context switches between threads, - so it does not make sense to use this in the presence of multiple threads. Also, - its return value is not used, so it can simply return ``None``. + system's trace function (see :func:`settrace`), but it is called with different events, + for example it isn't called for each executed line of code (only on call and return, + but the return event is reported even when an exception has been set). The function is + thread-specific, but there is no way for the profiler to know about context switches between + threads, so it does not make sense to use this in the presence of multiple threads. Also, + its return value is not used, so it can simply return ``None``. Error in the profile + function will cause itself unset. + Profile functions should have three arguments: *frame*, *event*, and + *arg*. *frame* is the current stack frame. *event* is a string: ``'call'``, + ``'return'``, ``'c_call'``, ``'c_return'``, or ``'c_exception'``. *arg* depends + on the event type. + + The events have the following meaning: + + ``'call'`` + A function is called (or some other code block entered). The + profile function is called; *arg* is ``None``. + + ``'return'`` + A function (or other code block) is about to return. The profile + function is called; *arg* is the value that will be returned, or ``None`` + if the event is caused by an exception being raised. + + ``'c_call'`` + A C function is about to be called. This may be an extension function or + a built-in. *arg* is the C function object. + + ``'c_return'`` + A C function has returned. *arg* is the C function object. + + ``'c_exception'`` + A C function has raised an exception. *arg* is the C function object. .. function:: setrecursionlimit(limit) @@ -1106,8 +1163,8 @@ always available. Trace functions should have three arguments: *frame*, *event*, and *arg*. *frame* is the current stack frame. *event* is a string: ``'call'``, - ``'line'``, ``'return'``, ``'exception'``, ``'c_call'``, ``'c_return'``, or - ``'c_exception'``, ``'opcode'``. *arg* depends on the event type. + ``'line'``, ``'return'``, ``'exception'`` or ``'opcode'``. *arg* depends on + the event type. The trace function is invoked (with *event* set to ``'call'``) whenever a new local scope is entered; it should return a reference to a local trace @@ -1117,6 +1174,9 @@ always available. function for further tracing in that scope), or ``None`` to turn off tracing in that scope. + If there is any error occurred in the trace function, it will be unset, just + like ``settrace(None)`` is called. + The events have the following meaning: ``'call'`` @@ -1144,16 +1204,6 @@ always available. tuple ``(exception, value, traceback)``; the return value specifies the new local trace function. - ``'c_call'`` - A C function is about to be called. This may be an extension function or - a built-in. *arg* is the C function object. - - ``'c_return'`` - A C function has returned. *arg* is the C function object. - - ``'c_exception'`` - A C function has raised an exception. *arg* is the C function object. - ``'opcode'`` The interpreter is about to execute a new opcode (see :mod:`dis` for opcode details). The local trace function is called; *arg* is @@ -1197,6 +1247,26 @@ always available. This function has been added on a provisional basis (see :pep:`411` for details.) +.. function:: set_coroutine_origin_tracking_depth(depth) + + Allows enabling or disabling coroutine origin tracking. When + enabled, the ``cr_origin`` attribute on coroutine objects will + contain a tuple of (filename, line number, function name) tuples + describing the traceback where the coroutine object was created, + with the most recent call first. When disabled, ``cr_origin`` will + be None. + + To enable, pass a *depth* value greater than zero; this sets the + number of frames whose information will be captured. To disable, + pass set *depth* to zero. + + This setting is thread-specific. + + .. versionadded:: 3.7 + + .. note:: + This function has been added on a provisional basis (see :pep:`411` + for details.) Use it only for debugging purposes. .. function:: set_coroutine_wrapper(wrapper) @@ -1237,6 +1307,10 @@ always available. This function has been added on a provisional basis (see :pep:`411` for details.) Use it only for debugging purposes. + .. deprecated:: 3.7 + The coroutine wrapper functionality has been deprecated, and + will be removed in 3.8. See :issue:`32591` for details. + .. function:: _enablelegacywindowsfsencoding() Changes the default filesystem encoding and errors mode to 'mbcs' and diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 2450716a1d9..9cd07158e7f 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -451,7 +451,8 @@ be finalized; only the internally used file object will be closed. See the (directory, fifo, symbolic link, etc.). If given, *arcname* specifies an alternative name for the file in the archive. Directories are added recursively by default. This can be avoided by setting *recursive* to - :const:`False`. If *filter* is given, it + :const:`False`. Recursion adds entries in sorted order. + If *filter* is given, it should be a function that takes a :class:`TarInfo` object argument and returns the changed :class:`TarInfo` object. If it instead returns :const:`None` the :class:`TarInfo` object will be excluded from the @@ -460,6 +461,9 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.2 Added the *filter* parameter. + .. versionchanged:: 3.7 + Recursion adds entries in sorted order. + .. method:: TarFile.addfile(tarinfo, fileobj=None) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 95e27cab7c8..b94021b4eb8 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -684,8 +684,8 @@ Semaphores also support the :ref:`context management protocol `. .. class:: Semaphore(value=1) - This class implements semaphore objects. A semaphore manages a counter - representing the number of :meth:`release` calls minus the number of + This class implements semaphore objects. A semaphore manages an atomic + counter representing the number of :meth:`release` calls minus the number of :meth:`acquire` calls, plus an initial value. The :meth:`acquire` method blocks if necessary until it can return without making the counter negative. If not given, *value* defaults to 1. @@ -701,19 +701,19 @@ Semaphores also support the :ref:`context management protocol `. Acquire a semaphore. - When invoked without arguments: if the internal counter is larger than - zero on entry, decrement it by one and return immediately. If it is zero - on entry, block, waiting until some other thread has called - :meth:`~Semaphore.release` to make it larger than zero. This is done - with proper interlocking so that if multiple :meth:`acquire` calls are - blocked, :meth:`~Semaphore.release` will wake exactly one of them up. - The implementation may pick one at random, so the order in which - blocked threads are awakened should not be relied on. Returns - true (or blocks indefinitely). + When invoked without arguments: + + * If the internal counter is larger than zero on entry, decrement it by + one and return true immediately. + * If the internal counter is zero on entry, block until awoken by a call to + :meth:`~Semaphore.release`. Once awoken (and the counter is greater + than 0), decrement the counter by 1 and return true. Exactly one + thread will be awoken by each call to :meth:`~Semaphore.release`. The + order in which threads are awoken should not be relied on. When invoked with *blocking* set to false, do not block. If a call - without an argument would block, return false immediately; otherwise, - do the same thing as when called without arguments, and return true. + without an argument would block, return false immediately; otherwise, do + the same thing as when called without arguments, and return true. When invoked with a *timeout* other than ``None``, it will block for at most *timeout* seconds. If acquire does not complete successfully in diff --git a/Doc/library/time.rst b/Doc/library/time.rst index ccbb3f37877..3eddc3f1ad3 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -792,7 +792,7 @@ These constants are used as parameters for :func:`clock_getres` and High-resolution per-process timer from the CPU. - Availability: FreeBSD 3 or later, NetBSD 7 or later, OpenBSD. + Availability: FreeBSD, NetBSD 7 or later, OpenBSD. .. versionadded:: 3.7 @@ -812,7 +812,7 @@ These constants are used as parameters for :func:`clock_getres` and suspended, providing accurate uptime measurement, both absolute and interval. - Availability: FreeBSD 7 or later, OpenBSD 5.5 or later. + Availability: FreeBSD, OpenBSD 5.5 or later. .. versionadded:: 3.7 diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 3e1faed8c7b..6d90e43e33b 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -35,10 +35,10 @@ this should open a window demonstrating a simple Tk interface. `Tcl/Tk manual `_ Official manual for the latest tcl/tk version. - `Programming Python `_ + `Programming Python `_ Book by Mark Lutz, has excellent coverage of Tkinter. - `Modern Tkinter for Busy Python Developers `_ + `Modern Tkinter for Busy Python Developers `_ Book by Mark Rozerman about building attractive and modern graphical user interfaces with Python and Tkinter. `Python and Tkinter Programming `_ @@ -154,7 +154,7 @@ background material, while the second half can be taken to the keyboard as a handy reference. When trying to answer questions of the form "how do I do blah", it is often best -to find out how to do"blah" in straight Tk, and then convert this back into the +to find out how to do "blah" in straight Tk, and then convert this back into the corresponding :mod:`tkinter` call. Python programmers can often guess at the correct Python command by looking at the Tk documentation. This means that in order to use Tkinter, you will have to know a little bit about Tk. This document @@ -183,7 +183,7 @@ documentation that exists. Here are some hints: `ActiveState Tcl Home Page `_ The Tk/Tcl development is largely taking place at ActiveState. - `Tcl and the Tk Toolkit `_ + `Tcl and the Tk Toolkit `_ The book by John Ousterhout, the inventor of Tcl. `Practical Programming in Tcl and Tk `_ diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index 927f7f3c6c1..9c0c4cde347 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -26,7 +26,7 @@ appearance. .. seealso:: - `Tk Widget Styling Support `_ + `Tk Widget Styling Support `_ A document introducing theming support for Tk @@ -1094,14 +1094,13 @@ ttk.Treeview the tree. - .. method:: selection(selop=None, items=None) + .. method:: selection() - If *selop* is not specified, returns selected items. Otherwise, it will - act according to the following selection methods. + Returns a tuple of selected items. - .. deprecated-removed:: 3.6 3.8 - Using ``selection()`` for changing the selection state is deprecated. - Use the following selection methods instead. + .. versionchanged:: 3.8 + ``selection()`` no longer takes arguments. For changing the selection + state use the following selection methods. .. method:: selection_set(*items) diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst index 048ee64aac9..2d327c02540 100644 --- a/Doc/library/tracemalloc.rst +++ b/Doc/library/tracemalloc.rst @@ -650,8 +650,8 @@ Traceback .. class:: Traceback - Sequence of :class:`Frame` instances sorted from the most recent frame to - the oldest frame. + Sequence of :class:`Frame` instances sorted from the oldest frame to the + most recent frame. A traceback contains at least ``1`` frame. If the ``tracemalloc`` module failed to get a frame, the filename ``""`` at line number ``0`` is @@ -663,11 +663,17 @@ Traceback The :attr:`Trace.traceback` attribute is an instance of :class:`Traceback` instance. - .. method:: format(limit=None) + .. versionchanged:: 3.7 + Frames are now sorted from the oldest to the most recent, instead of most recent to oldest. - Format the traceback as a list of lines with newlines. Use the - :mod:`linecache` module to retrieve lines from the source code. If - *limit* is set, only format the *limit* most recent frames. + .. method:: format(limit=None, most_recent_first=False) + + Format the traceback as a list of lines with newlines. Use the + :mod:`linecache` module to retrieve lines from the source code. + If *limit* is set, format the *limit* most recent frames if *limit* + is positive. Otherwise, format the ``abs(limit)`` oldest frames. + If *most_recent_first* is ``True``, the order of the formatted frames + is reversed, returning the most recent frame first instead of last. Similar to the :func:`traceback.format_tb` function, except that :meth:`.format` does not include newlines. diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 89aca9c9df8..bbc1d1301d2 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -155,6 +155,14 @@ Standard names are defined for the following types: .. versionadded:: 3.7 +.. data:: ClassMethodDescriptorType + + The type of *unbound* class methods of some built-in data types such as + ``dict.__dict__['fromkeys']``. + + .. versionadded:: 3.7 + + .. class:: ModuleType(name, doc=None) The type of :term:`modules `. Constructor takes the name of the diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 9883d8bbe86..9c4777ac2fe 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -146,6 +146,8 @@ See :pep:`484` for more details. ``Derived`` is expected. This is useful when you want to prevent logic errors with minimal runtime cost. +.. versionadded:: 3.5.2 + Callable -------- @@ -494,6 +496,8 @@ The module defines the following classes, functions and decorators: ``Type[Any]`` is equivalent to ``Type`` which in turn is equivalent to ``type``, which is the root of Python's metaclass hierarchy. + .. versionadded:: 3.5.2 + .. class:: Iterable(Generic[T_co]) A generic version of :class:`collections.abc.Iterable`. @@ -674,6 +678,8 @@ The module defines the following classes, functions and decorators: A generic version of :class:`collections.defaultdict`. + .. versionadded:: 3.5.2 + .. class:: Counter(collections.Counter, Dict[T, int]) A generic version of :class:`collections.Counter`. @@ -762,13 +768,15 @@ The module defines the following classes, functions and decorators: def add_unicode_checkmark(text: Text) -> Text: return text + u' \u2713' + .. versionadded:: 3.5.2 + .. class:: io Wrapper namespace for I/O stream types. - This defines the generic type ``IO[AnyStr]`` and aliases ``TextIO`` - and ``BinaryIO`` for respectively ``IO[str]`` and ``IO[bytes]``. - These represent the types of I/O streams such as returned by + This defines the generic type ``IO[AnyStr]`` and subclasses ``TextIO`` + and ``BinaryIO``, deriving from ``IO[str]`` and ``IO[bytes]``, + respectively. These represent the types of I/O streams such as returned by :func:`open`. These types are also accessible directly as ``typing.IO``, @@ -847,6 +855,8 @@ The module defines the following classes, functions and decorators: UserId = NewType('UserId', int) first_user = UserId(1) + .. versionadded:: 3.5.2 + .. function:: cast(typ, val) Cast a value to a type. @@ -1054,3 +1064,5 @@ The module defines the following classes, functions and decorators: "forward reference", to hide the ``expensive_mod`` reference from the interpreter runtime. Type annotations for local variables are not evaluated, so the second annotation does not need to be enclosed in quotes. + + .. versionadded:: 3.5.2 diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 05f33740d75..5bf3d575520 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -1008,7 +1008,7 @@ subclass. True Sometimes this is inconvenient. For example, `one user -`_ is subclassing mock to +`_ is subclassing mock to created a `Twisted adaptor `_. Having this applied to attributes too actually causes errors. @@ -1251,7 +1251,7 @@ With a bit of tweaking you could have the comparison function raise the :exc:`AssertionError` directly and provide a more useful failure message. As of version 1.5, the Python testing library `PyHamcrest -`_ provides similar functionality, +`_ provides similar functionality, that may be useful here, in the form of its equality matcher (`hamcrest.library.integration.match_equality -`_). +`_). diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index b6eb8ccb59d..ac9dd3b6f79 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -2370,12 +2370,12 @@ Sealing mocks .. function:: seal(mock) - Seal will disable the creation of mock children by preventing to get or set - any new attribute on the sealed mock. The sealing process is performed recursively. + Seal will disable the creation of mock children by preventing getting or setting + of any new attribute on the sealed mock. The sealing process is performed recursively. If a mock instance is assigned to an attribute instead of being dynamically created - it won't be considered in the sealing chain. This allows to prevent seal from fixing - part of the mock object. + it won't be considered in the sealing chain. This allows one to prevent seal from + fixing part of the mock object. >>> mock = Mock() >>> mock.submock.attribute1 = 2 diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index e52f140029c..93ccd0fd611 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -56,7 +56,7 @@ test runner Kent Beck's original paper on testing frameworks using the pattern shared by :mod:`unittest`. - `Nose `_ and `py.test `_ + `Nose `_ and `py.test `_ Third-party unittest frameworks with a lighter-weight syntax for writing tests. For example, ``assert func(10) == 42``. @@ -219,6 +219,22 @@ Command-line options Stop the test run on the first error or failure. +.. cmdoption:: -k + + Only run test methods and classes that match the pattern or substring. + This option may be used multiple times, in which case all test cases that + match of the given patterns are included. + + Patterns that contain a wildcard character (``*``) are matched against the + test name using :meth:`fnmatch.fnmatchcase`; otherwise simple case-sensitive + substring matching is used. + + Patterns are matched against the fully qualified test method name as + imported by the test loader. + + For example, ``-k foo`` matches ``foo_tests.SomeTest.test_something``, + ``bar_tests.SomeTest.test_foo``, but not ``bar_tests.FooTest.test_something``. + .. cmdoption:: --locals Show local variables in tracebacks. @@ -229,6 +245,9 @@ Command-line options .. versionadded:: 3.5 The command-line option ``--locals``. +.. versionadded:: 3.7 + The command-line option ``-k``. + The command line can also be used for test discovery, for running all of the tests in a project or just a subset. @@ -1745,6 +1764,21 @@ Loading and running tests This affects all the :meth:`loadTestsFrom\*` methods. + .. attribute:: testNamePatterns + + List of Unix shell-style wildcard test name patterns that test methods + have to match to be included in test suites (see ``-v`` option). + + If this attribute is not ``None`` (the default), all test methods to be + included in test suites must match one of the patterns in this list. + Note that matches are always performed using :meth:`fnmatch.fnmatchcase`, + so unlike patterns passed to the ``-v`` option, simple substring patterns + will have to be converted using ``*`` wildcards. + + This affects all the :meth:`loadTestsFrom\*` methods. + + .. versionadded:: 3.7 + .. class:: TestResult diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 5a10f9571e9..413d8b6ffdf 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -62,7 +62,7 @@ The :mod:`urllib.request` module defines the following functions: * :meth:`~urllib.response.addinfourl.info` --- return the meta-information of the page, such as headers, in the form of an :func:`email.message_from_string` instance (see - `Quick Reference to HTTP Headers `_) + `Quick Reference to HTTP Headers `_) * :meth:`~urllib.response.addinfourl.getcode` -- return the HTTP status code of the response. diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index 7d31932f965..e3b90e673ca 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -69,10 +69,10 @@ structure of :file:`robots.txt` files, see http://www.robotstxt.org/orig.html. .. method:: request_rate(useragent) Returns the contents of the ``Request-rate`` parameter from - ``robots.txt`` in the form of a :func:`~collections.namedtuple` - ``(requests, seconds)``. If there is no such parameter or it doesn't - apply to the *useragent* specified or the ``robots.txt`` entry for this - parameter has invalid syntax, return ``None``. + ``robots.txt`` as a :term:`named tuple` ``RequestRate(requests, seconds)``. + If there is no such parameter or it doesn't apply to the *useragent* + specified or the ``robots.txt`` entry for this parameter has invalid + syntax, return ``None``. .. versionadded:: 3.6 diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index ea9ea7dc7d9..8ec75a79acf 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -156,10 +156,18 @@ The :mod:`uuid` module defines the following functions: Get the hardware address as a 48-bit positive integer. The first time this runs, it may launch a separate program, which could be quite slow. If all - attempts to obtain the hardware address fail, we choose a random 48-bit number - with its eighth bit set to 1 as recommended in RFC 4122. "Hardware address" - means the MAC address of a network interface, and on a machine with multiple - network interfaces the MAC address of any one of them may be returned. + attempts to obtain the hardware address fail, we choose a random 48-bit + number with the multicast bit (least significant bit of the first octet) + set to 1 as recommended in RFC 4122. "Hardware address" means the MAC + address of a network interface. On a machine with multiple network + interfaces, universally administered MAC addresses (i.e. where the second + least significant bit of the first octet is *unset*) will be preferred over + locally administered MAC addresses, but with no other ordering guarantees. + + .. versionchanged:: 3.7 + Universally administered MAC addresses are preferred over locally + administered MAC addresses, since the former are guaranteed to be + globally unique, while the latter are not. .. index:: single: getnode diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 17c80c8d982..6707be7fc82 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -458,4 +458,4 @@ subclass which installs setuptools and pip into a created virtual environment:: This script is also available for download `online -`_. +`_. diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index f67f4bc24c4..b04bd79e4bb 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -51,8 +51,17 @@ Warning Categories ------------------ There are a number of built-in exceptions that represent warning categories. -This categorization is useful to be able to filter out groups of warnings. The -following warnings category classes are currently defined: +This categorization is useful to be able to filter out groups of warnings. + +While these are technically +:ref:`built-in exceptions `, they are +documented here, because conceptually they belong to the warnings mechanism. + +User code can define additional warning categories by subclassing one of the +standard warning categories. A warning category must always be a subclass of +the :exc:`Warning` class. + +The following warnings category classes are currently defined: .. tabularcolumns:: |l|p{0.6\linewidth}| @@ -66,7 +75,9 @@ following warnings category classes are currently defined: | :exc:`UserWarning` | The default category for :func:`warn`. | +----------------------------------+-----------------------------------------------+ | :exc:`DeprecationWarning` | Base category for warnings about deprecated | -| | features (ignored by default). | +| | features when those warnings are intended for | +| | other Python developers (ignored by default, | +| | unless triggered by code in ``__main__``). | +----------------------------------+-----------------------------------------------+ | :exc:`SyntaxWarning` | Base category for warnings about dubious | | | syntactic features. | @@ -74,8 +85,10 @@ following warnings category classes are currently defined: | :exc:`RuntimeWarning` | Base category for warnings about dubious | | | runtime features. | +----------------------------------+-----------------------------------------------+ -| :exc:`FutureWarning` | Base category for warnings about constructs | -| | that will change semantically in the future. | +| :exc:`FutureWarning` | Base category for warnings about deprecated | +| | features when those warnings are intended for | +| | end users of applications that are written in | +| | Python. | +----------------------------------+-----------------------------------------------+ | :exc:`PendingDeprecationWarning` | Base category for warnings about features | | | that will be deprecated in the future | @@ -95,13 +108,12 @@ following warnings category classes are currently defined: | | resource usage. | +----------------------------------+-----------------------------------------------+ - -While these are technically built-in exceptions, they are documented here, -because conceptually they belong to the warnings mechanism. - -User code can define additional warning categories by subclassing one of the -standard warning categories. A warning category must always be a subclass of -the :exc:`Warning` class. +.. versionchanged:: 3.7 + Previously :exc:`DeprecationWarning` and :exc:`FutureWarning` were + distinguished based on whether a feature was being removed entirely or + changing its behaviour. They are now distinguished based on their + intended audience and the way they're handled by the default warnings + filters. .. _warning-filter: @@ -114,7 +126,7 @@ into errors (raising an exception). Conceptually, the warnings filter maintains an ordered list of filter specifications; any specific warning is matched against each filter -specification in the list in turn until a match is found; the match determines +specification in the list in turn until a match is found; the filter determines the disposition of the match. Each entry is a tuple of the form (*action*, *message*, *category*, *module*, *lineno*), where: @@ -123,19 +135,19 @@ the disposition of the match. Each entry is a tuple of the form (*action*, +---------------+----------------------------------------------+ | Value | Disposition | +===============+==============================================+ + | ``"default"`` | print the first occurrence of matching | + | | warnings for each location (module + | + | | line number) where the warning is issued | + +---------------+----------------------------------------------+ | ``"error"`` | turn matching warnings into exceptions | +---------------+----------------------------------------------+ | ``"ignore"`` | never print matching warnings | +---------------+----------------------------------------------+ | ``"always"`` | always print matching warnings | +---------------+----------------------------------------------+ - | ``"default"`` | print the first occurrence of matching | - | | warnings for each location where the warning | - | | is issued | - +---------------+----------------------------------------------+ | ``"module"`` | print the first occurrence of matching | | | warnings for each module where the warning | - | | is issued | + | | is issued (regardless of line number) | +---------------+----------------------------------------------+ | ``"once"`` | print only the first occurrence of matching | | | warnings, regardless of location | @@ -157,33 +169,119 @@ the disposition of the match. Each entry is a tuple of the form (*action*, Since the :exc:`Warning` class is derived from the built-in :exc:`Exception` class, to turn a warning into an error we simply raise ``category(message)``. +If a warning is reported and doesn't match any registered filter then the +"default" action is applied (hence its name). + + +.. _describing-warning-filters: + +Describing Warning Filters +~~~~~~~~~~~~~~~~~~~~~~~~~~ + The warnings filter is initialized by :option:`-W` options passed to the Python -interpreter command line. The interpreter saves the arguments for all -:option:`-W` options without interpretation in ``sys.warnoptions``; the -:mod:`warnings` module parses these when it is first imported (invalid options -are ignored, after printing a message to ``sys.stderr``). +interpreter command line and the :envvar:`PYTHONWARNINGS` environment variable. +The interpreter saves the arguments for all supplied entries without +interpretation in ``sys.warnoptions``; the :mod:`warnings` module parses these +when it is first imported (invalid options are ignored, after printing a +message to ``sys.stderr``). + +Individual warnings filters are specified as a sequence of fields separated by +colons:: + + action:message:category:module:line + +The meaning of each of these fields is as described in :ref:`warning-filter`. +When listing multiple filters on a single line (as for +:envvar:`PYTHONWARNINGS`), the individual filters are separated by commas,and +the filters listed later take precedence over those listed before them (as +they're applied left-to-right, and the most recently applied filters take +precedence over earlier ones). + +Commonly used warning filters apply to either all warnings, warnings in a +particular category, or warnings raised by particular modules or packages. +Some examples:: + + default # Show all warnings (even those ignored by default) + ignore # Ignore all warnings + error # Convert all warnings to errors + error::ResourceWarning # Treat ResourceWarning messages as errors + default::DeprecationWarning # Show DeprecationWarning messages + ignore,default:::mymodule # Only report warnings triggered by "mymodule" + error:::mymodule[.*] # Convert warnings to errors in "mymodule" + # and any subpackages of "mymodule" -Default Warning Filters -~~~~~~~~~~~~~~~~~~~~~~~ +.. _default-warning-filter: + +Default Warning Filter +~~~~~~~~~~~~~~~~~~~~~~ By default, Python installs several warning filters, which can be overridden by -the command-line options passed to :option:`-W` and calls to -:func:`filterwarnings`. +the :option:`-W` command-line option, the :envvar:`PYTHONWARNINGS` environment +variable and calls to :func:`filterwarnings`. -* :exc:`DeprecationWarning` and :exc:`PendingDeprecationWarning`, and - :exc:`ImportWarning` are ignored. +In regular release builds, the default warning filter has the following entries +(in order of precedence):: -* :exc:`BytesWarning` is ignored unless the :option:`-b` option is given once or - twice; in this case this warning is either printed (``-b``) or turned into an - exception (``-bb``). + default::DeprecationWarning:__main__ + ignore::DeprecationWarning + ignore::PendingDeprecationWarning + ignore::ImportWarning + ignore::ResourceWarning -* :exc:`ResourceWarning` is ignored unless Python was built in debug mode. +In debug builds, the list of default warning filters is empty. .. versionchanged:: 3.2 :exc:`DeprecationWarning` is now ignored by default in addition to :exc:`PendingDeprecationWarning`. +.. versionchanged:: 3.7 + :exc:`DeprecationWarning` is once again shown by default when triggered + directly by code in ``__main__``. + +.. versionchanged:: 3.7 + :exc:`BytesWarning` no longer appears in the default filter list and is + instead configured via :data:`sys.warnoptions` when :option:`-b` is specified + twice. + + +.. _warning-disable: + +Overriding the default filter +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Developers of applications written in Python may wish to hide *all* Python level +warnings from their users by default, and only display them when running tests +or otherwise working on the application. The :data:`sys.warnoptions` attribute +used to pass filter configurations to the interpreter can be used as a marker to +indicate whether or not warnings should be disabled:: + + import sys + + if not sys.warnoptions: + import warnings + warnings.simplefilter("ignore") + +Developers of test runners for Python code are advised to instead ensure that +*all* warnings are displayed by default for the code under test, using code +like:: + + import sys + + if not sys.warnoptions: + import os, warnings + warnings.simplefilter("default") # Change the filter in this process + os.environ["PYTHONWARNINGS"] = "default" # Also affect subprocesses + +Finally, developers of interactive shells that run user code in a namespace +other than ``__main__`` are advised to ensure that :exc:`DeprecationWarning` +messages are made visible by default, using code like the following (where +``user_ns`` is the module used to execute code entered interactively):: + + import warnings + warnings.filterwarnings("default", category=DeprecationWarning, + module=user_ns.get("__name__")) + .. _warning-suppress: @@ -191,7 +289,8 @@ Temporarily Suppressing Warnings -------------------------------- If you are using code that you know will raise a warning, such as a deprecated -function, but do not want to see the warning, then it is possible to suppress +function, but do not want to see the warning (even when warnings have been +explicitly configured via the command line), then it is possible to suppress the warning using the :class:`catch_warnings` context manager:: import warnings @@ -261,38 +360,30 @@ entries from the warnings list before each new operation). .. _warning-ignored: -Updating Code For New Versions of Python ----------------------------------------- +Updating Code For New Versions of Dependencies +---------------------------------------------- -Warnings that are only of interest to the developer are ignored by default. As -such you should make sure to test your code with typically ignored warnings -made visible. You can do this from the command-line by passing :option:`-Wd <-W>` -to the interpreter (this is shorthand for :option:`!-W default`). This enables -default handling for all warnings, including those that are ignored by default. -To change what action is taken for encountered warnings you simply change what -argument is passed to :option:`-W`, e.g. :option:`!-W error`. See the -:option:`-W` flag for more details on what is possible. +Warning categories that are primarily of interest to Python developers (rather +than end users of applications written in Python) are ignored by default. -To programmatically do the same as :option:`!-Wd`, use:: +Notably, this "ignored by default" list includes :exc:`DeprecationWarning` +(for every module except ``__main__``), which means developers should make sure +to test their code with typically ignored warnings made visible in order to +receive timely notifications of future breaking API changes (whether in the +standard library or third party packages). - warnings.simplefilter('default') +In the ideal case, the code will have a suitable test suite, and the test runner +will take care of implicitly enabling all warnings when running tests +(the test runner provided by the :mod:`unittest` module does this). -Make sure to execute this code as soon as possible. This prevents the -registering of what warnings have been raised from unexpectedly influencing how -future warnings are treated. - -Having certain warnings ignored by default is done to prevent a user from -seeing warnings that are only of interest to the developer. As you do not -necessarily have control over what interpreter a user uses to run their code, -it is possible that a new version of Python will be released between your -release cycles. The new interpreter release could trigger new warnings in your -code that were not there in an older interpreter, e.g. -:exc:`DeprecationWarning` for a module that you are using. While you as a -developer want to be notified that your code is using a deprecated module, to a -user this information is essentially noise and provides no benefit to them. - -The :mod:`unittest` module has been also updated to use the ``'default'`` -filter while running tests. +In less ideal cases, applications can be checked for use of deprecated +interfaces by passing :option:`-Wd <-W>` to the Python interpreter (this is +shorthand for :option:`!-W default`) or setting ``PYTHONWARNINGS=default`` in +the environment. This enables default handling for all warnings, including those +that are ignored by default. To change what action is taken for encountered +warnings you can change what argument is passed to :option:`-W` (e.g. +:option:`!-W error`). See the :option:`-W` flag for more details on what is +possible. .. _warning-functions: diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index a1d446902b8..b0ef3465b04 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -26,8 +26,8 @@ for implementing WSGI servers, a demo HTTP server that serves WSGI applications, and a validation tool that checks WSGI servers and applications for conformance to the WSGI specification (:pep:`3333`). -See https://wsgi.readthedocs.org/ for more information about WSGI, and links to -tutorials and other resources. +See `wsgi.readthedocs.io `_ for more information about WSGI, and links +to tutorials and other resources. .. XXX If you're just trying to write a web application... diff --git a/Doc/library/xml.dom.rst b/Doc/library/xml.dom.rst index de334af2363..18519a75a54 100644 --- a/Doc/library/xml.dom.rst +++ b/Doc/library/xml.dom.rst @@ -67,13 +67,13 @@ implementations are free to support the strict mapping from IDL). See section .. seealso:: - `Document Object Model (DOM) Level 2 Specification `_ + `Document Object Model (DOM) Level 2 Specification `_ The W3C recommendation upon which the Python DOM API is based. `Document Object Model (DOM) Level 1 Specification `_ The W3C recommendation for the DOM supported by :mod:`xml.dom.minidom`. - `Python Language Mapping Specification `_ + `Python Language Mapping Specification `_ This specifies the mapping from OMG IDL to Python. diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 5b8c776ed64..c0f2a89a3a1 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -130,10 +130,12 @@ ZipFile Objects --------------- -.. class:: ZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True) +.. class:: ZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True, \ + compresslevel=None) Open a ZIP file, where *file* can be a path to a file (a string), a file-like object or a :term:`path-like object`. + The *mode* parameter should be ``'r'`` to read an existing file, ``'w'`` to truncate and write a new file, ``'a'`` to append to an existing file, or ``'x'`` to exclusively create and write a new file. @@ -145,16 +147,27 @@ ZipFile Objects adding a ZIP archive to another file (such as :file:`python.exe`). If *mode* is ``'a'`` and the file does not exist at all, it is created. If *mode* is ``'r'`` or ``'a'``, the file should be seekable. + *compression* is the ZIP compression method to use when writing the archive, and should be :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2` or :const:`ZIP_LZMA`; unrecognized - values will cause :exc:`NotImplementedError` to be raised. If :const:`ZIP_DEFLATED`, - :const:`ZIP_BZIP2` or :const:`ZIP_LZMA` is specified but the corresponding module - (:mod:`zlib`, :mod:`bz2` or :mod:`lzma`) is not available, :exc:`RuntimeError` - is raised. The default is :const:`ZIP_STORED`. If *allowZip64* is - ``True`` (the default) zipfile will create ZIP files that use the ZIP64 - extensions when the zipfile is larger than 4 GiB. If it is false :mod:`zipfile` - will raise an exception when the ZIP file would require ZIP64 extensions. + values will cause :exc:`NotImplementedError` to be raised. If + :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2` or :const:`ZIP_LZMA` is specified + but the corresponding module (:mod:`zlib`, :mod:`bz2` or :mod:`lzma`) is not + available, :exc:`RuntimeError` is raised. The default is :const:`ZIP_STORED`. + + If *allowZip64* is ``True`` (the default) zipfile will create ZIP files that + use the ZIP64 extensions when the zipfile is larger than 4 GiB. If it is + ``false`` :mod:`zipfile` will raise an exception when the ZIP file would + require ZIP64 extensions. + + The *compresslevel* parameter controls the compression level to use when + writing files to the archive. + When using :const:`ZIP_STORED` or :const:`ZIP_LZMA` it has no effect. + When using :const:`ZIP_DEFLATED` integers ``0`` through ``9`` are accepted + (see :class:`zlib ` for more information). + When using :const:`ZIP_BZIP2` integers ``1`` through ``9`` are accepted + (see :class:`bz2 ` for more information). If the file is created with mode ``'w'``, ``'x'`` or ``'a'`` and then :meth:`closed ` without adding any files to the archive, the appropriate @@ -187,6 +200,9 @@ ZipFile Objects .. versionchanged:: 3.6.2 The *file* parameter accepts a :term:`path-like object`. + .. versionchanged:: 3.7 + Add the *compresslevel* parameter. + .. method:: ZipFile.close() @@ -230,9 +246,9 @@ ZipFile Objects With *mode* ``'r'`` the file-like object (``ZipExtFile``) is read-only and provides the following methods: :meth:`~io.BufferedIOBase.read`, :meth:`~io.IOBase.readline`, - :meth:`~io.IOBase.readlines`, :meth:`__iter__`, - :meth:`~iterator.__next__`. These objects can operate independently of - the ZipFile. + :meth:`~io.IOBase.readlines`, :meth:`~io.IOBase.seek`, + :meth:`~io.IOBase.tell`, :meth:`__iter__`, :meth:`~iterator.__next__`. + These objects can operate independently of the ZipFile. With ``mode='w'``, a writable file handle is returned, which supports the :meth:`~io.BufferedIOBase.write` method. While a writable file handle is open, @@ -351,13 +367,15 @@ ZipFile Objects :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. -.. method:: ZipFile.write(filename, arcname=None, compress_type=None) +.. method:: ZipFile.write(filename, arcname=None, compress_type=None, \ + compresslevel=None) Write the file named *filename* to the archive, giving it the archive name *arcname* (by default, this will be the same as *filename*, but without a drive letter and with leading path separators removed). If given, *compress_type* overrides the value given for the *compression* parameter to the constructor for - the new entry. + the new entry. Similarly, *compresslevel* will override the constructor if + given. The archive must be open with mode ``'w'``, ``'x'`` or ``'a'``. .. note:: @@ -383,7 +401,8 @@ ZipFile Objects a :exc:`RuntimeError` was raised. -.. method:: ZipFile.writestr(zinfo_or_arcname, data[, compress_type]) +.. method:: ZipFile.writestr(zinfo_or_arcname, data, compress_type=None, \ + compresslevel=None) Write the string *data* to the archive; *zinfo_or_arcname* is either the file name it will be given in the archive, or a :class:`ZipInfo` instance. If it's @@ -393,7 +412,8 @@ ZipFile Objects If given, *compress_type* overrides the value given for the *compression* parameter to the constructor for the new entry, or in the *zinfo_or_arcname* - (if that is a :class:`ZipInfo` instance). + (if that is a :class:`ZipInfo` instance). Similarly, *compresslevel* will + override the constructor if given. .. note:: @@ -471,7 +491,7 @@ The :class:`PyZipFile` constructor takes the same parameters as the :file:`\*.pyc` are added at the top level. If the directory is a package directory, then all :file:`\*.pyc` are added under the package name as a file path, and if any subdirectories are package directories, - all of these are added recursively. + all of these are added recursively in sorted order. *basename* is intended for internal use only. @@ -504,6 +524,9 @@ The :class:`PyZipFile` constructor takes the same parameters as the .. versionchanged:: 3.6.2 The *pathname* parameter accepts a :term:`path-like object`. + .. versionchanged:: 3.7 + Recursion sorts directory entries. + .. _zipinfo-objects: diff --git a/Doc/license.rst b/Doc/license.rst index cd018689f87..5f78240002f 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -87,7 +87,7 @@ PSF LICENSE AGREEMENT FOR PYTHON |release| analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python |release| alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of - copyright, i.e., "Copyright © 2001-2017 Python Software Foundation; All Rights + copyright, i.e., "Copyright © 2001-2018 Python Software Foundation; All Rights Reserved" are retained in Python |release| alone or in any derivative version prepared by Licensee. @@ -349,48 +349,6 @@ Project, http://www.wide.ad.jp/. :: SUCH DAMAGE. -Floating point exception control --------------------------------- - -The source for the :mod:`fpectl` module includes the following notice:: - - --------------------------------------------------------------------- - / Copyright (c) 1996. \ - | The Regents of the University of California. | - | All rights reserved. | - | | - | Permission to use, copy, modify, and distribute this software for | - | any purpose without fee is hereby granted, provided that this en- | - | tire notice is included in all copies of any software which is or | - | includes a copy or modification of this software and in all | - | copies of the supporting documentation for such software. | - | | - | This work was produced at the University of California, Lawrence | - | Livermore National Laboratory under contract no. W-7405-ENG-48 | - | between the U.S. Department of Energy and The Regents of the | - | University of California for the operation of UC LLNL. | - | | - | DISCLAIMER | - | | - | This software was prepared as an account of work sponsored by an | - | agency of the United States Government. Neither the United States | - | Government nor the University of California nor any of their em- | - | ployees, makes any warranty, express or implied, or assumes any | - | liability or responsibility for the accuracy, completeness, or | - | usefulness of any information, apparatus, product, or process | - | disclosed, or represents that its use would not infringe | - | privately-owned rights. Reference herein to any specific commer- | - | cial products, process, or service by trade name, trademark, | - | manufacturer, or otherwise, does not necessarily constitute or | - | imply its endorsement, recommendation, or favoring by the United | - | States Government or the University of California. The views and | - | opinions of authors expressed herein do not necessarily state or | - | reflect those of the United States Government or the University | - | of California, and shall not be used for advertising or product | - \ endorsement purposes. / - --------------------------------------------------------------------- - - Asynchronous socket services ---------------------------- diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index dca93624004..d7792f1eaf8 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -559,12 +559,14 @@ Parameters may have annotations of the form "``: expression``" following the parameter name. Any parameter may have an annotation even those of the form ``*identifier`` or ``**identifier``. Functions may have "return" annotation of the form "``-> expression``" after the parameter list. These annotations can be -any valid Python expression and are evaluated when the function definition is -executed. Annotations may be evaluated in a different order than they appear in -the source code. The presence of annotations does not change the semantics of a -function. The annotation values are available as values of a dictionary keyed -by the parameters' names in the :attr:`__annotations__` attribute of the -function object. +any valid Python expression. The presence of annotations does not change the +semantics of a function. The annotation values are available as values of +a dictionary keyed by the parameters' names in the :attr:`__annotations__` +attribute of the function object. If the ``annotations`` import from +:mod:`__future__` is used, annotations are preserved as strings at runtime which +enables postponed evaluation. Otherwise, they are evaluated when the function +definition is executed. In this case annotations may be evaluated in +a different order than they appear in the source code. .. index:: pair: lambda; expression @@ -587,6 +589,17 @@ access the local variables of the function containing the def. See section :pep:`3107` - Function Annotations The original specification for function annotations. + :pep:`484` - Type Hints + Definition of a standard meaning for annotations: type hints. + + :pep:`526` - Syntax for Variable Annotations + Ability to type hint variable declarations, including class + variables and instance variables + + :pep:`563` - Postponed Evaluation of Annotations + Support for forward references within annotations by preserving + annotations in a string form at runtime instead of eager evaluation. + .. _class: diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 153b58b4fbf..8420fb60bff 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1173,60 +1173,68 @@ Basic customization .. index:: single: destructor + single: finalizer statement: del Called when the instance is about to be destroyed. This is also called a - destructor. If a base class has a :meth:`__del__` method, the derived class's - :meth:`__del__` method, if any, must explicitly call it to ensure proper - deletion of the base class part of the instance. Note that it is possible - (though not recommended!) for the :meth:`__del__` method to postpone destruction - of the instance by creating a new reference to it. It may then be called at a - later time when this new reference is deleted. It is not guaranteed that - :meth:`__del__` methods are called for objects that still exist when the - interpreter exits. + finalizer or (improperly) a destructor. If a base class has a + :meth:`__del__` method, the derived class's :meth:`__del__` method, + if any, must explicitly call it to ensure proper deletion of the base + class part of the instance. + + It is possible (though not recommended!) for the :meth:`__del__` method + to postpone destruction of the instance by creating a new reference to + it. This is called object *resurrection*. It is implementation-dependent + whether :meth:`__del__` is called a second time when a resurrected object + is about to be destroyed; the current :term:`CPython` implementation + only calls it once. + + It is not guaranteed that :meth:`__del__` methods are called for objects + that still exist when the interpreter exits. .. note:: ``del x`` doesn't directly call ``x.__del__()`` --- the former decrements the reference count for ``x`` by one, and the latter is only called when - ``x``'s reference count reaches zero. Some common situations that may - prevent the reference count of an object from going to zero include: - circular references between objects (e.g., a doubly-linked list or a tree - data structure with parent and child pointers); a reference to the object - on the stack frame of a function that caught an exception (the traceback - stored in ``sys.exc_info()[2]`` keeps the stack frame alive); or a - reference to the object on the stack frame that raised an unhandled - exception in interactive mode (the traceback stored in - ``sys.last_traceback`` keeps the stack frame alive). The first situation - can only be remedied by explicitly breaking the cycles; the second can be - resolved by freeing the reference to the traceback object when it is no - longer useful, and the third can be resolved by storing ``None`` in - ``sys.last_traceback``. - Circular references which are garbage are detected and cleaned up when - the cyclic garbage collector is enabled (it's on by default). Refer to the - documentation for the :mod:`gc` module for more information about this - topic. + ``x``'s reference count reaches zero. + + .. impl-detail:: + It is possible for a reference cycle to prevent the reference count + of an object from going to zero. In this case, the cycle will be + later detected and deleted by the :term:`cyclic garbage collector + `. A common cause of reference cycles is when + an exception has been caught in a local variable. The frame's + locals then reference the exception, which references its own + traceback, which references the locals of all frames caught in the + traceback. + + .. seealso:: + Documentation for the :mod:`gc` module. .. warning:: Due to the precarious circumstances under which :meth:`__del__` methods are invoked, exceptions that occur during their execution are ignored, and a warning - is printed to ``sys.stderr`` instead. Also, when :meth:`__del__` is invoked in - response to a module being deleted (e.g., when execution of the program is - done), other globals referenced by the :meth:`__del__` method may already have - been deleted or in the process of being torn down (e.g. the import - machinery shutting down). For this reason, :meth:`__del__` methods - should do the absolute - minimum needed to maintain external invariants. Starting with version 1.5, - Python guarantees that globals whose name begins with a single underscore are - deleted from their module before other globals are deleted; if no other - references to such globals exist, this may help in assuring that imported - modules are still available at the time when the :meth:`__del__` method is - called. + is printed to ``sys.stderr`` instead. In particular: - .. index:: - single: repr() (built-in function); __repr__() (object method) + * :meth:`__del__` can be invoked when arbitrary code is being executed, + including from any arbitrary thread. If :meth:`__del__` needs to take + a lock or invoke any other blocking resource, it may deadlock as + the resource may already be taken by the code that gets interrupted + to execute :meth:`__del__`. + * :meth:`__del__` can be executed during interpreter shutdown. As a + consequence, the global variables it needs to access (including other + modules) may already have been deleted or set to ``None``. Python + guarantees that globals whose name begins with a single underscore + are deleted from their module before other globals are deleted; if + no other references to such globals exist, this may help in assuring + that imported modules are still available at the time when the + :meth:`__del__` method is called. + + + .. index:: + single: repr() (built-in function); __repr__() (object method) .. method:: object.__repr__(self) @@ -1455,10 +1463,12 @@ access (use of, assignment to, or deletion of ``x.name``) for class instances. .. method:: object.__getattr__(self, name) - Called when an attribute lookup has not found the attribute in the usual places - (i.e. it is not an instance attribute nor is it found in the class tree for - ``self``). ``name`` is the attribute name. This method should return the - (computed) attribute value or raise an :exc:`AttributeError` exception. + Called when the default attribute access fails with an :exc:`AttributeError` + (either :meth:`__getattribute__` raises an :exc:`AttributeError` because + *name* is not an instance attribute or an attribute in the class tree + for ``self``; or :meth:`__get__` of a *name* property raises + :exc:`AttributeError`). This method should either return the (computed) + attribute value or raise an :exc:`AttributeError` exception. Note that if the attribute is found through the normal mechanism, :meth:`__getattr__` is not called. (This is an intentional asymmetry between @@ -1512,6 +1522,62 @@ access (use of, assignment to, or deletion of ``x.name``) for class instances. returned. :func:`dir` converts the returned sequence to a list and sorts it. +Customizing module attribute access +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. index:: + single: __getattr__ (module attribute) + single: __dir__ (module attribute) + single: __class__ (module attribute) + +Special names ``__getattr__`` and ``__dir__`` can be also used to customize +access to module attributes. The ``__getattr__`` function at the module level +should accept one argument which is the name of an attribute and return the +computed value or raise an :exc:`AttributeError`. If an attribute is +not found on a module object through the normal lookup, i.e. +:meth:`object.__getattribute__`, then ``__getattr__`` is searched in +the module ``__dict__`` before raising an :exc:`AttributeError`. If found, +it is called with the attribute name and the result is returned. + +The ``__dir__`` function should accept no arguments, and return a list of +strings that represents the names accessible on module. If present, this +function overrides the standard :func:`dir` search on a module. + +For a more fine grained customization of the module behavior (setting +attributes, properties, etc.), one can set the ``__class__`` attribute of +a module object to a subclass of :class:`types.ModuleType`. For example:: + + import sys + from types import ModuleType + + class VerboseModule(ModuleType): + def __repr__(self): + return f'Verbose {self.__name__}' + + def __setattr__(self, attr, value): + print(f'Setting {attr}...') + setattr(self, attr, value) + + sys.modules[__name__].__class__ = VerboseModule + +.. note:: + Defining module ``__getattr__`` and setting module ``__class__`` only + affect lookups made using the attribute access syntax -- directly accessing + the module globals (whether by code within the module, or via a reference + to the module's globals dictionary) is unaffected. + +.. versionchanged:: 3.5 + ``__class__`` module attribute is now writable. + +.. versionadded:: 3.7 + ``__getattr__`` and ``__dir__`` module attributes. + +.. seealso:: + + :pep:`562` - Module __getattr__ and __dir__ + Describes the ``__getattr__`` and ``__dir__`` functions on modules. + + .. _descriptors: Implementing Descriptors diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 1cff8a52df9..151062ba175 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -183,8 +183,20 @@ by considering each of the :keyword:`for` or :keyword:`if` clauses a block, nesting from left to right, and evaluating the expression to produce an element each time the innermost block is reached. -Note that the comprehension is executed in a separate scope, so names assigned -to in the target list don't "leak" into the enclosing scope. +However, aside from the iterable expression in the leftmost :keyword:`for` clause, +the comprehension is executed in a separate implicitly nested scope. This ensures +that names assigned to in the target list don't "leak" into the enclosing scope. + +The iterable expression in the leftmost :keyword:`for` clause is evaluated +directly in the enclosing scope and then passed as an argument to the implictly +nested scope. Subsequent :keyword:`for` clauses and any filter condition in the +leftmost :keyword:`for` clause cannot be evaluated in the enclosing scope as +they may depend on the values obtained from the leftmost iterable. For example: +``[x*y for x in range(10) for y in range(x, x+10)]``. + +To ensure the comprehension always results in a container of the appropriate +type, ``yield`` and ``yield from`` expressions are prohibited in the implicitly +nested scope. Since Python 3.6, in an :keyword:`async def` function, an :keyword:`async for` clause may be used to iterate over a :term:`asynchronous iterator`. @@ -198,6 +210,13 @@ or :keyword:`await` expressions it is called an suspend the execution of the coroutine function in which it appears. See also :pep:`530`. +.. versionadded:: 3.6 + Asynchronous comprehensions were introduced. + +.. versionchanged:: 3.8 + ``yield`` and ``yield from`` prohibited in the implicitly nested scope. + + .. _lists: List displays @@ -316,27 +335,40 @@ brackets or curly braces. Variables used in the generator expression are evaluated lazily when the :meth:`~generator.__next__` method is called for the generator object (in the same -fashion as normal generators). However, the leftmost :keyword:`for` clause is -immediately evaluated, so that an error produced by it can be seen before any -other possible error in the code that handles the generator expression. -Subsequent :keyword:`for` clauses cannot be evaluated immediately since they -may depend on the previous :keyword:`for` loop. For example: ``(x*y for x in -range(10) for y in bar(x))``. +fashion as normal generators). However, the iterable expression in the +leftmost :keyword:`for` clause is immediately evaluated, so that an error +produced by it will be emitted at the point where the generator expression +is defined, rather than at the point where the first value is retrieved. +Subsequent :keyword:`for` clauses and any filter condition in the leftmost +:keyword:`for` clause cannot be evaluated in the enclosing scope as they may +depend on the values obtained from the leftmost iterable. For example: +``(x*y for x in range(10) for y in range(x, x+10))``. The parentheses can be omitted on calls with only one argument. See section :ref:`calls` for details. +To avoid interfering with the expected operation of the generator expression +itself, ``yield`` and ``yield from`` expressions are prohibited in the +implicitly defined generator. + If a generator expression contains either :keyword:`async for` clauses or :keyword:`await` expressions it is called an :dfn:`asynchronous generator expression`. An asynchronous generator expression returns a new asynchronous generator object, which is an asynchronous iterator (see :ref:`async-iterators`). +.. versionadded:: 3.6 + Asynchronous generator expressions were introduced. + .. versionchanged:: 3.7 Prior to Python 3.7, asynchronous generator expressions could only appear in :keyword:`async def` coroutines. Starting with 3.7, any function can use asynchronous generator expressions. +.. versionchanged:: 3.8 + ``yield`` and ``yield from`` prohibited in the implicitly nested scope. + + .. _yieldexpr: Yield expressions @@ -364,6 +396,14 @@ coroutine function to be an asynchronous generator. For example:: async def agen(): # defines an asynchronous generator function (PEP 525) yield 123 +Due to their side effects on the containing scope, ``yield`` expressions +are not permitted as part of the implicitly defined scopes used to +implement comprehensions and generator expressions. + +.. versionchanged:: 3.8 + Yield expressions prohibited in the implicitly nested scopes used to + implement comprehensions and generator expressions. + Generator functions are described below, while asynchronous generator functions are described separately in section :ref:`asynchronous-generator-functions`. diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 881e0aed32e..45d417295d0 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -675,6 +675,33 @@ Here are the exact rules used: :meth:`~importlib.abc.Loader.module_repr` method, if defined, before trying either approach described above. However, the method is deprecated. +.. _pyc-invalidation: + +Cached bytecode invalidation +---------------------------- + +Before Python loads cached bytecode from ``.pyc`` file, it checks whether the +cache is up-to-date with the source ``.py`` file. By default, Python does this +by storing the source's last-modified timestamp and size in the cache file when +writing it. At runtime, the import system then validates the cache file by +checking the stored metadata in the cache file against at source's +metadata. + +Python also supports "hash-based" cache files, which store a hash of the source +file's contents rather than its metadata. There are two variants of hash-based +``.pyc`` files: checked and unchecked. For checked hash-based ``.pyc`` files, +Python validates the cache file by hashing the source file and comparing the +resulting hash with the hash in the cache file. If a checked hash-based cache +file is found to be invalid, Python regenerates it and writes a new checked +hash-based cache file. For unchecked hash-based ``.pyc`` files, Python simply +assumes the cache file is valid if it exists. Hash-based ``.pyc`` files +validation behavior may be overridden with the :option:`--check-hash-based-pycs` +flag. + +.. versionchanged:: 3.7 + Added hash-based ``.pyc`` files. Previously, Python only supported + timestamp-based invalidation of bytecode caches. + The Path Based Finder ===================== diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index ee3fbc13e2b..c6d96ca612b 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -571,7 +571,7 @@ that a single backslash followed by a newline is interpreted as those two characters as part of the literal, *not* as a line continuation. -.. _string-catenation: +.. _string-concatenation: String literal concatenation ---------------------------- @@ -654,9 +654,11 @@ expression or conversion result. An empty string is passed when the format specifier is omitted. The formatted result is then included in the final value of the whole string. -Top-level format specifiers may include nested replacement fields. -These nested fields may include their own conversion fields and -format specifiers, but may not include more deeply-nested replacement fields. +Top-level format specifiers may include nested replacement fields. These nested +fields may include their own conversion fields and :ref:`format specifiers +`, but may not include more deeply-nested replacement fields. The +:ref:`format specifier mini-language ` is the same as that used by +the string .format() method. Formatted string literals may be concatenated, but replacement fields cannot be split across literals. @@ -674,7 +676,7 @@ Some examples of formatted string literals:: >>> f"result: {value:{width}.{precision}}" # nested fields 'result: 12.35' >>> today = datetime(year=2017, month=1, day=27) - >>> f"{today:%b %d, %Y}" # using date format specifier + >>> f"{today:%B %d, %Y}" # using date format specifier 'January 27, 2017' >>> number = 1024 >>> f"{number:#0x}" # using integer format specifier diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 8d17383853a..ef9a5f0dc85 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -853,12 +853,15 @@ can appear before a future statement are: * blank lines, and * other future statements. -.. XXX change this if future is cleaned out +The only feature in Python 3.7 that requires using the future statement is +``annotations``. -The features recognized by Python 3.0 are ``absolute_import``, ``division``, -``generators``, ``unicode_literals``, ``print_function``, ``nested_scopes`` and -``with_statement``. They are all redundant because they are always enabled, and -only kept for backwards compatibility. +All historical features enabled by the future statement are still recognized +by Python 3. The list includes ``absolute_import``, ``division``, +``generators``, ``generator_stop``, ``unicode_literals``, +``print_function``, ``nested_scopes`` and ``with_statement``. They are +all redundant because they are always enabled, and only kept for +backwards compatibility. A future statement is recognized and treated specially at compile time: Changes to the semantics of core constructs are often implemented by generating diff --git a/Doc/tools/static/switchers.js b/Doc/tools/static/switchers.js index c450f5eafff..8e0c5ea0092 100644 --- a/Doc/tools/static/switchers.js +++ b/Doc/tools/static/switchers.js @@ -10,7 +10,8 @@ '(?:release/\\d.\\d[\\x\\d\\.]*)']; var all_versions = { - '3.7': 'dev (3.7)', + '3.8': 'dev (3.8)', + '3.7': 'pre (3.7)', '3.6': '3.6', '3.5': '3.5', '2.7': '2.7', diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index d52f81b76b5..cfdd5266c51 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -259,12 +259,8 @@ tutorial/stdlib2,,:start,extra = data[start:start+extra_size] tutorial/stdlib2,,:start,"fields = struct.unpack('Lancelot @@ -327,3 +337,7 @@ whatsnew/changelog,,:end,str[start:end] library/binascii,,`,'`' library/uu,,`,'`' whatsnew/3.7,,`,'`' +whatsnew/3.7,,::,error::BytesWarning +whatsnew/changelog,,::,error::BytesWarning +whatsnew/changelog,,::,default::BytesWarning +whatsnew/changelog,,::,default::DeprecationWarning diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html index 9fa814f923b..2fdc7cb6a36 100644 --- a/Doc/tools/templates/indexsidebar.html +++ b/Doc/tools/templates/indexsidebar.html @@ -2,6 +2,7 @@

{% trans %}Download these documents{% endtrans %}

{% trans %}Docs for other versions{% endtrans %}

    +
  • {% trans %}Python 3.7 (pre-release){% endtrans %}
  • {% trans %}Python 3.6 (stable){% endtrans %}
  • {% trans %}Python 3.5 (stable){% endtrans %}
  • {% trans %}Python 2.7 (stable){% endtrans %}
    • diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 95141f94d6a..b8f1226e54a 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -876,9 +876,9 @@ Generator Expressions ===================== Some simple generators can be coded succinctly as expressions using a syntax -similar to list comprehensions but with parentheses instead of brackets. These -expressions are designed for situations where the generator is used right away -by an enclosing function. Generator expressions are more compact but less +similar to list comprehensions but with parentheses instead of square brackets. +These expressions are designed for situations where the generator is used right +away by an enclosing function. Generator expressions are more compact but less versatile than full generator definitions and tend to be more memory friendly than equivalent list comprehensions. @@ -892,10 +892,7 @@ Examples:: >>> sum(x*y for x,y in zip(xvec, yvec)) # dot product 260 - >>> from math import pi, sin - >>> sine_table = {x: sin(x*pi/180) for x in range(0, 91)} - - >>> unique_words = set(word for line in page for word in line.split()) + >>> unique_words = set(word for line in page for word in line.split()) >>> valedictorian = max((student.gpa, student.name) for student in graduates) diff --git a/Doc/tutorial/interactive.rst b/Doc/tutorial/interactive.rst index d73cfeb34f1..c0eb1feec4e 100644 --- a/Doc/tutorial/interactive.rst +++ b/Doc/tutorial/interactive.rst @@ -51,4 +51,4 @@ bpython_. .. _GNU Readline: https://tiswww.case.edu/php/chet/readline/rltop.html .. _IPython: https://ipython.org/ -.. _bpython: http://www.bpython-interpreter.org/ +.. _bpython: https://www.bpython-interpreter.org/ diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index bf7ce776417..3d57020dabe 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -10,13 +10,13 @@ Using the Python Interpreter Invoking the Interpreter ======================== -The Python interpreter is usually installed as :file:`/usr/local/bin/python3.7` +The Python interpreter is usually installed as :file:`/usr/local/bin/python3.8` on those machines where it is available; putting :file:`/usr/local/bin` in your Unix shell's search path makes it possible to start it by typing the command: .. code-block:: text - python3.7 + python3.8 to the shell. [#]_ Since the choice of the directory where the interpreter lives is an installation option, other places are possible; check with your local @@ -98,8 +98,8 @@ before printing the first prompt: .. code-block:: shell-session - $ python3.7 - Python 3.7 (default, Sep 16 2015, 09:25:04) + $ python3.8 + Python 3.8 (default, Sep 16 2015, 09:25:04) [GCC 4.8.2] on linux Type "help", "copyright", "credits" or "license" for more information. >>> diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 2fa894a533f..7176d819425 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -212,6 +212,13 @@ to each other are automatically concatenated. :: >>> 'Py' 'thon' 'Python' +This feature is particularly useful when you want to break long strings:: + + >>> text = ('Put several strings within parentheses ' + ... 'to have them joined together.') + >>> text + 'Put several strings within parentheses to have them joined together.' + This only works with two literals though, not with variables or expressions:: >>> prefix = 'Py' @@ -227,13 +234,6 @@ If you want to concatenate variables or a variable and a literal, use ``+``:: >>> prefix + 'thon' 'Python' -This feature is particularly useful when you want to break long strings:: - - >>> text = ('Put several strings within parentheses ' - ... 'to have them joined together.') - >>> text - 'Put several strings within parentheses to have them joined together.' - Strings can be *indexed* (subscripted), with the first character having index 0. There is no separate character type; a character is simply a string of size one:: diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 6ac29fc6025..4934d695180 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -15,7 +15,7 @@ operating system:: >>> import os >>> os.getcwd() # Return the current working directory - 'C:\\Python37' + 'C:\\Python38' >>> os.chdir('/server/accesslogs') # Change current working directory >>> os.system('mkdir today') # Run the command mkdir in the system shell 0 diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 99472314ea2..d2ac57b8e4d 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -278,7 +278,7 @@ applications include caching objects that are expensive to create:: Traceback (most recent call last): File "", line 1, in d['primary'] # entry was automatically removed - File "C:/python37/lib/weakref.py", line 46, in __getitem__ + File "C:/python38/lib/weakref.py", line 46, in __getitem__ o = self.data[key]() KeyError: 'primary' diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index b2269302cb4..1e9ed6e645a 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -210,9 +210,23 @@ Miscellaneous options import of source modules. See also :envvar:`PYTHONDONTWRITEBYTECODE`. +.. cmdoption:: --check-hash-based-pycs default|always|never + + Control the validation behavior of hash-based ``.pyc`` files. See + :ref:`pyc-invalidation`. When set to ``default``, checked and unchecked + hash-based bytecode cache files are validated according to their default + semantics. When set to ``always``, all hash-based ``.pyc`` files, whether + checked or unchecked, are validated against their corresponding source + file. When set to ``never``, hash-based ``.pyc`` files are not validated + against their corresponding source files. + + The semantics of timestamp-based ``.pyc`` files are unaffected by this + option. + + .. cmdoption:: -d - Turn on parser debugging output (for wizards only, depending on compilation + Turn on parser debugging output (for expert only, depending on compilation options). See also :envvar:`PYTHONDEBUG`. @@ -263,8 +277,9 @@ Miscellaneous options .. cmdoption:: -R - Kept for compatibility. On Python 3.3 and greater, hash randomization is - turned on by default. + Turn on hash randomization. This option only has an effect if the + :envvar:`PYTHONHASHSEED` environment variable is set to ``0``, since hash + randomization is enabled by default. On previous versions of Python, this option turns on hash randomization, so that the :meth:`__hash__` values of str, bytes and datetime @@ -280,6 +295,9 @@ Miscellaneous options :envvar:`PYTHONHASHSEED` allows you to set a fixed value for the hash seed secret. + .. versionchanged:: 3.7 + The option is no longer ignored. + .. versionadded:: 3.2.3 @@ -338,49 +356,27 @@ Miscellaneous options :option:`-W` options are ignored (though, a warning message is printed about invalid options when the first warning is issued). - Warnings can also be controlled from within a Python program using the + Warnings can also be controlled using the :envvar:`PYTHONWARNINGS` + environment variable and from within a Python program using the :mod:`warnings` module. - The simplest form of argument is one of the following action strings (or a - unique abbreviation): + The simplest settings apply a particular action unconditionally to all + warnings emitted by a process (even those that are otherwise ignored by + default):: - ``ignore`` - Ignore all warnings. - ``default`` - Explicitly request the default behavior (printing each warning once per - source line). - ``all`` - Print a warning each time it occurs (this may generate many messages if a - warning is triggered repeatedly for the same source line, such as inside a - loop). - ``module`` - Print each warning only the first time it occurs in each module. - ``once`` - Print each warning only the first time it occurs in the program. - ``error`` - Raise an exception instead of printing a warning message. + -Wdefault # Warn once per call location + -Werror # Convert to exceptions + -Walways # Warn every time + -Wmodule # Warn once per calling module + -Wonce # Warn once per Python process + -Wignore # Never warn - The full form of argument is:: + The action names can be abbreviated as desired (e.g. ``-Wi``, ``-Wd``, + ``-Wa``, ``-We``) and the interpreter will resolve them to the appropriate + action name. - action:message:category:module:line - - Here, *action* is as explained above but only applies to messages that match - the remaining fields. Empty fields match all values; trailing empty fields - may be omitted. The *message* field matches the start of the warning message - printed; this match is case-insensitive. The *category* field matches the - warning category. This must be a class name; the match tests whether the - actual warning category of the message is a subclass of the specified warning - category. The full class name must be given. The *module* field matches the - (fully-qualified) module name; this match is case-sensitive. The *line* - field matches the line number, where zero matches all line numbers and is - thus equivalent to an omitted line number. - - .. seealso:: - :mod:`warnings` -- the warnings module - - :pep:`230` -- Warning framework - - :envvar:`PYTHONWARNINGS` + See :ref:`warning-filter` and :ref:`describing-warning-filters` for more + details. .. cmdoption:: -x @@ -411,19 +407,22 @@ Miscellaneous options nested imports). Note that its output may be broken in multi-threaded application. Typical usage is ``python3 -X importtime -c 'import asyncio'``. See also :envvar:`PYTHONPROFILEIMPORTTIME`. - * ``-X dev`` enables the "developer mode": enable debug checks at runtime. - In short, ``python3 -X dev ...`` behaves as ``PYTHONMALLOC=debug PYTHONASYNCIODEBUG=1 python3 - -W default -X faulthandler ...``, except that the :envvar:`PYTHONMALLOC` - and :envvar:`PYTHONASYNCIODEBUG` environment variables are not set in - practice. Developer mode: + * ``-X dev``: enable CPython's "development mode", introducing additional + runtime checks which are too expensive to be enabled by default. It should + not be more verbose than the default if the code is correct: new warnings + are only emitted when an issue is detected. Effect of the developer mode: - * Add ``default`` warnings option. For example, display - :exc:`DeprecationWarning` and :exc:`ResourceWarning` warnings. + * Add ``default`` warning filter, as :option:`-W` ``default``. * Install debug hooks on memory allocators: see the :c:func:`PyMem_SetupDebugHooks` C function. * Enable the :mod:`faulthandler` module to dump the Python traceback on a crash. * Enable :ref:`asyncio debug mode `. + * Set the :attr:`~sys.flags.dev_mode` attribute of :attr:`sys.flags` to + ``True`` + + * ``-X utf8`` enables the UTF-8 mode, whereas ``-X utf8=0`` disables the + UTF-8 mode. It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -441,8 +440,7 @@ Miscellaneous options The ``-X showalloccount`` option. .. versionadded:: 3.7 - The ``-X importtime``, ``-X dev`` and :envvar:`PYTHONPROFILEIMPORTTIME` - options. + The ``-X importtime``, ``-X dev`` and ``-X utf8`` options. Options you shouldn't use @@ -639,7 +637,23 @@ conflict. This is equivalent to the :option:`-W` option. If set to a comma separated string, it is equivalent to specifying :option:`-W` multiple - times. + times, with filters later in the list taking precedence over those earlier + in the list. + + The simplest settings apply a particular action unconditionally to all + warnings emitted by a process (even those that are otherwise ignored by + default):: + + PYTHONWARNINGS=default # Warn once per call location + PYTHONWARNINGS=error # Convert to exceptions + PYTHONWARNINGS=always # Warn every time + PYTHONWARNINGS=module # Warn once per calling module + PYTHONWARNINGS=once # Warn once per Python process + PYTHONWARNINGS=ignore # Never warn + + See :ref:`warning-filter` and :ref:`describing-warning-filters` for more + details. + .. envvar:: PYTHONFAULTHANDLER @@ -686,6 +700,8 @@ conflict. Set the family of memory allocators used by Python: + * ``default``: use the :ref:`default memory allocators + `. * ``malloc``: use the :c:func:`malloc` function of the C library for all domains (:c:data:`PYMEM_DOMAIN_RAW`, :c:data:`PYMEM_DOMAIN_MEM`, :c:data:`PYMEM_DOMAIN_OBJ`). @@ -695,20 +711,17 @@ conflict. Install debug hooks: - * ``debug``: install debug hooks on top of the default memory allocator + * ``debug``: install debug hooks on top of the :ref:`default memory + allocators `. * ``malloc_debug``: same as ``malloc`` but also install debug hooks * ``pymalloc_debug``: same as ``pymalloc`` but also install debug hooks - When Python is compiled in release mode, the default is ``pymalloc``. When - compiled in debug mode, the default is ``pymalloc_debug`` and the debug hooks - are used automatically. + See the :ref:`default memory allocators ` and the + :c:func:`PyMem_SetupDebugHooks` function (install debug hooks on Python + memory allocators). - If Python is configured without ``pymalloc`` support, ``pymalloc`` and - ``pymalloc_debug`` are not available, the default is ``malloc`` in release - mode and ``malloc_debug`` in debug mode. - - See the :c:func:`PyMem_SetupDebugHooks` function for debug hooks on Python - memory allocators. + .. versionchanged:: 3.7 + Added the ``"default"`` allocator. .. versionadded:: 3.6 @@ -760,9 +773,7 @@ conflict. If set to the value ``0``, causes the main Python command line application to skip coercing the legacy ASCII-based C locale to a more capable UTF-8 - based alternative. Note that this setting is checked even when the - :option:`-E` or :option:`-I` options are used, as it is handled prior to - the processing of command line options. + based alternative. If this variable is *not* set, or is set to a value other than ``0``, and the current locale reported for the ``LC_CTYPE`` category is the default @@ -796,6 +807,22 @@ conflict. .. versionadded:: 3.7 See :pep:`538` for more details. + +.. envvar:: PYTHONDEVMODE + + If this environment variable is set to a non-empty string, enable the + CPython "development mode". See the :option:`-X` ``dev`` option. + + .. versionadded:: 3.7 + +.. envvar:: PYTHONUTF8 + + If set to ``1``, enable the UTF-8 mode. If set to ``0``, disable the UTF-8 + mode. Any other non-empty string cause an error. + + .. versionadded:: 3.7 + + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 8f1ac3f3fd3..a386728eaee 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -66,7 +66,7 @@ number of standard Unix command line editors, :program:`vim` and :program:`BBEdit` or :program:`TextWrangler` from Bare Bones Software (see http://www.barebones.com/products/bbedit/index.html) are good choices, as is :program:`TextMate` (see https://macromates.com/). Other editors include -:program:`Gvim` (http://macvim.org) and :program:`Aquamacs` +:program:`Gvim` (http://macvim-dev.github.io/macvim/) and :program:`Aquamacs` (http://aquamacs.org/). To run your script from the Terminal window you must make sure that @@ -117,7 +117,7 @@ The IDE MacPython ships with the standard IDLE development environment. A good introduction to using IDLE can be found at -https://hkn.eecs.berkeley.edu/~dyoo/python/idle_intro/index.html. +http://www.hashcollision.org/hkn/python/idle_intro/index.html. .. _mac-package-manager: @@ -149,7 +149,7 @@ X by Apple, and the latest version can be downloaded and installed from https://www.activestate.com; it can also be built from source. *wxPython* is another popular cross-platform GUI toolkit that runs natively on -Mac OS X. Packages and documentation are available from http://www.wxpython.org. +Mac OS X. Packages and documentation are available from https://www.wxpython.org. *PyQt* is another popular cross-platform GUI toolkit that runs natively on Mac OS X. More information can be found at diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index 8b96ebea396..ccdf84dcfa5 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -30,7 +30,7 @@ following links: for Debian users https://en.opensuse.org/Portal:Packaging for OpenSuse users - https://docs.fedoraproject.org/en-US/Fedora_Draft_Documentation/0.1/html/RPM_Guide/ch-creating-rpms.html + https://docs-old.fedoraproject.org/en-US/Fedora_Draft_Documentation/0.1/html/RPM_Guide/ch-creating-rpms.html for Fedora users http://www.slackbook.org/html/package-management-making-packages.html for Slackware users diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 3d47d7c5154..47d423f42d6 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -291,9 +291,9 @@ for detailed information about platforms with pre-compiled installers. by Mark Pilgrim, 2004, ISBN 1-59059-356-1 - `For Windows users `_ + `For Windows users `_ in "Installing Python" - in "`A Byte of Python `_" + in "`A Byte of Python `_" by Swaroop C H, 2003 @@ -307,11 +307,11 @@ key features: `ActivePython `_ Installer with multi-platform compatibility, documentation, PyWin32 -`Anaconda `_ +`Anaconda `_ Popular scientific modules (such as numpy, scipy and pandas) and the ``conda`` package manager. -`Canopy `_ +`Canopy `_ A "comprehensive Python analysis environment" with editors and other development tools. @@ -377,7 +377,7 @@ System variables, you need non-restricted access to your machine .. seealso:: - https://support.microsoft.com/kb/100843 + https://support.microsoft.com/en-us/help/100843/environment-variables-in-windows-nt Environment variables in Windows NT https://technet.microsoft.com/en-us/library/cc754250.aspx @@ -386,7 +386,7 @@ System variables, you need non-restricted access to your machine https://technet.microsoft.com/en-us/library/cc755104.aspx The SETX command, for permanently modifying environment variables - https://support.microsoft.com/kb/310519 + https://support.microsoft.com/en-us/help/310519/how-to-manage-environment-variables-in-windows-xp How To Manage Environment Variables in Windows XP https://www.chem.gla.ac.uk/~louis/software/faq/q1.html @@ -721,7 +721,7 @@ installation directory. So, if you had installed Python to :file:`C:\\Python\\Lib\\site-packages\\`. To completely override :data:`sys.path`, create a ``._pth`` file with the same -name as the DLL (``python36._pth``) or the executable (``python._pth``) and +name as the DLL (``python37._pth``) or the executable (``python._pth``) and specify one line for each path to add to :data:`sys.path`. The file based on the DLL name overrides the one based on the executable, which allows paths to be restricted for any program loading the runtime if desired. @@ -734,7 +734,7 @@ Import statements other than to ``site`` are not permitted, and arbitrary code cannot be specified. Note that ``.pth`` files (without leading underscore) will be processed normally -by the :mod:`site` module. +by the :mod:`site` module when ``import site`` has been specified. When no ``._pth`` file is found, this is how :data:`sys.path` is populated on Windows: @@ -796,7 +796,7 @@ following advice will prevent conflicts with other installations: environment variables, and also ignore :mod:`site` unless ``import site`` is listed. -* If you are loading :file:`python3.dll` or :file:`python36.dll` in your own +* If you are loading :file:`python3.dll` or :file:`python37.dll` in your own executable, explicitly call :c:func:`Py_SetPath` or (at least) :c:func:`Py_SetProgramName` before :c:func:`Py_Initialize`. @@ -871,7 +871,7 @@ shipped with PyWin32. It is an embeddable IDE with a built-in debugger. cx_Freeze --------- -`cx_Freeze `_ is a :mod:`distutils` +`cx_Freeze `_ is a :mod:`distutils` extension (see :ref:`extending-distutils`) which wraps Python scripts into executable Windows programs (:file:`{*}.exe` files). When you have done this, you can distribute your application without requiring your users to install @@ -930,7 +930,7 @@ directly accessed by end-users. When extracted, the embedded distribution is (almost) fully isolated from the user's system, including environment variables, system registry settings, and installed packages. The standard library is included as pre-compiled and -optimized ``.pyc`` files in a ZIP, and ``python3.dll``, ``python36.dll``, +optimized ``.pyc`` files in a ZIP, and ``python3.dll``, ``python37.dll``, ``python.exe`` and ``pythonw.exe`` are all provided. Tcl/tk (including all dependants, such as Idle), pip and the Python documentation are not included. diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index a0bb81a9748..0aa6003f4c4 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -957,7 +957,7 @@ New and Improved Modules # 'title': 'html2fo 0.3 (Default)'}, ... ] The :mod:`SimpleXMLRPCServer` module makes it easy to create straightforward - XML-RPC servers. See http://www.xmlrpc.com/ for more information about XML-RPC. + XML-RPC servers. See http://xmlrpc.scripting.com/ for more information about XML-RPC. * The new :mod:`hmac` module implements the HMAC algorithm described by :rfc:`2104`. (Contributed by Gerhard Häring.) diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index cebfb21156b..b5628a910ae 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -1955,7 +1955,7 @@ The RPM spec files, found in the :file:`Misc/RPM/` directory in the Python source distribution, were updated for 2.3. (Contributed by Sean Reifschneider.) Other new platforms now supported by Python include AtheOS -(http://atheos.cx/), GNU/Hurd, and OpenVMS. +(http://www.atheos.cx/), GNU/Hurd, and OpenVMS. .. ====================================================================== diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 4d482918313..15d35d384c4 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -330,7 +330,7 @@ statement, only the ``from ... import`` form. :pep:`328` - Imports: Multi-Line and Absolute/Relative PEP written by Aahz; implemented by Thomas Wouters. - https://pylib.readthedocs.org/ + https://pylib.readthedocs.io/ The py library by Holger Krekel, which contains the :mod:`py.std` package. .. ====================================================================== diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 45b0ab9f345..33d9f7ce43b 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -172,7 +172,7 @@ this edition of "What's New in Python" links to the bug/patch item for each change. Hosting of the Python bug tracker is kindly provided by -`Upfront Systems `__ +`Upfront Systems `__ of Stellenbosch, South Africa. Martin von Löwis put a lot of effort into importing existing bugs and patches from SourceForge; his scripts for this import operation are at diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 55392fc2be0..90ce06c3286 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -1545,7 +1545,7 @@ changes, or look through the Subversion logs for all the details. *ciphers* argument that's a string listing the encryption algorithms to be allowed; the format of the string is described `in the OpenSSL documentation - `__. + `__. (Added by Antoine Pitrou; :issue:`8322`.) Another change makes the extension load all of OpenSSL's ciphers and @@ -1809,7 +1809,7 @@ wish to read the Tcl/Tk manual page describing the Ttk theme engine, available at https://www.tcl.tk/man/tcl8.5/TkCmd/ttk_intro.htm. Some screenshots of the Python/Ttk code in use are at -http://code.google.com/p/python-ttk/wiki/Screenshots. +https://code.google.com/archive/p/python-ttk/wikis/Screenshots.wiki. The :mod:`ttk` module was written by Guilherme Polo and added in :issue:`2983`. An alternate version called ``Tile.py``, written by @@ -1832,8 +1832,8 @@ https://pypi.python.org/pypi/unittest2. When used from the command line, the module can automatically discover tests. It's not as fancy as `py.test `__ or -`nose `__, but provides a simple way -to run tests kept within a set of package directories. For example, +`nose `__, but provides a +simple way to run tests kept within a set of package directories. For example, the following command will search the :file:`test/` subdirectory for any importable test files named ``test*.py``:: diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 78d3105a5a4..2d740006a37 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -1643,7 +1643,7 @@ for secure (encrypted, authenticated) internet connections: * The :func:`ssl.wrap_socket` constructor function now takes a *ciphers* argument. The *ciphers* string lists the allowed encryption algorithms using the format described in the `OpenSSL documentation - `__. + `__. * When linked against recent versions of OpenSSL, the :mod:`ssl` module now supports the Server Name Indication extension to the TLS protocol, allowing diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 60469cd1d36..a355af7adb8 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -238,8 +238,8 @@ and the ``__annotations__`` attribute. and Guido van Rossum. Implemented by Ivan Levkivskyi. Tools that use or will use the new syntax: - `mypy `_, - `pytype `_, PyCharm, etc. + `mypy `_, + `pytype `_, PyCharm, etc. .. _whatsnew36-pep515: @@ -1568,7 +1568,7 @@ mark class variables. As introduced in :pep:`526`, a variable annotation wrapped in ClassVar indicates that a given attribute is intended to be used as a class variable and should not be set on instances of that class. (Contributed by Ivan Levkivskyi in `Github #280 -`_.) +`_.) A new :const:`~typing.TYPE_CHECKING` constant that is assumed to be ``True`` by the static type chekers, but is ``False`` at runtime. diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 71e8358a420..8d4772f973f 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -70,6 +70,7 @@ Summary -- Release highlights New Features ============ + .. _whatsnew37-pep538: PEP 538: Legacy C Locale Coercion @@ -107,6 +108,7 @@ locale remains active when the core interpreter is initialized. :pep:`538` -- Coercing the legacy C locale to a UTF-8 based locale PEP written and implemented by Nick Coghlan. + .. _whatsnew37-pep553: PEP 553: Built-in breakpoint() @@ -159,6 +161,75 @@ effort will be made to add such support. PEP written by Erik M. Bray; implementation by Masayuki Yamamoto. +PEP 562: Customization of access to module attributes +----------------------------------------------------- + +It is sometimes convenient to customize or otherwise have control over access +to module attributes. A typical example is managing deprecation warnings. +Typical workarounds are assigning ``__class__`` of a module object to +a custom subclass of :class:`types.ModuleType` or replacing the ``sys.modules`` +item with a custom wrapper instance. This procedure is now simplified by +recognizing ``__getattr__`` defined directly in a module that would act like +a normal ``__getattr__`` method, except that it will be defined on module +*instances*. + +.. seealso:: + + :pep:`562` -- Module ``__getattr__`` and ``__dir__`` + PEP written and implemented by Ivan Levkivskyi + + +PEP 563: Postponed evaluation of annotations +-------------------------------------------- + +The advent of type hints in Python uncovered two glaring usability issues +with the functionality of annotations added in :pep:`3107` and refined +further in :pep:`526`: + +* annotations could only use names which were already available in the + current scope, in other words they didn't support forward references + of any kind; and + +* annotating source code had adverse effects on startup time of Python + programs. + +Both of these issues are fixed by postponing the evaluation of +annotations. Instead of compiling code which executes expressions in +annotations at their definition time, the compiler stores the annotation +in a string form equivalent to the AST of the expression in question. +If needed, annotations can be resolved at runtime using +``typing.get_type_hints()``. In the common case where this is not +required, the annotations are cheaper to store (since short strings +are interned by the interpreter) and make startup time faster. + +Usability-wise, annotations now support forward references, making the +following syntax valid:: + + class C: + @classmethod + def from_string(cls, source: str) -> C: + ... + + def validate_b(self, obj: B) -> bool: + ... + + class B: + ... + +Since this change breaks compatibility, the new behavior can be enabled +on a per-module basis in Python 3.7 using a ``__future__`` import, like +this:: + + from __future__ import annotations + +It will become the default in Python 4.0. + +.. seealso:: + + :pep:`563` -- Postponed evaluation of annotations + PEP written and implemented by Łukasz Langa. + + PEP 564: Add new time functions with nanosecond resolution ---------------------------------------------------------- @@ -185,17 +256,125 @@ resolution on Linux and Windows. PEP written and implemented by Victor Stinner -New Developer Mode: -X dev --------------------------- +.. _whatsnew37-pep565: -Add a new "developer mode": ``-X dev`` command line option to enable debug -checks at runtime. +PEP 565: Show DeprecationWarning in ``__main__`` +------------------------------------------------ -In short, ``python3 -X dev ...`` behaves as ``PYTHONMALLOC=debug python3 -W -default -X faulthandler ...``, except that the PYTHONMALLOC environment -variable is not set in practice. +The default handling of :exc:`DeprecationWarning` has been changed such that +these warnings are once more shown by default, but only when the code +triggering them is running directly in the ``__main__`` module. As a result, +developers of single file scripts and those using Python interactively should +once again start seeing deprecation warnings for the APIs they use, but +deprecation warnings triggered by imported application, library and framework +modules will continue to be hidden by default. -See :option:`-X` ``dev`` for the details. +As a result of this change, the standard library now allows developers to choose +between three different deprecation warning behaviours: + +* :exc:`FutureWarning`: always displayed by default, recommended for warnings + intended to be seen by application end users (e.g. for deprecated application + configuration settings). +* :exc:`DeprecationWarning`: displayed by default only in ``__main__`` and when + running tests, recommended for warnings intended to be seen by other Python + developers where a version upgrade may result in changed behaviour or an + error. +* :exc:`PendingDeprecationWarning`: displayed by default only when running + tests, intended for cases where a future version upgrade will change the + warning category to :exc:`DeprecationWarning` or :exc:`FutureWarning`. + +Previously both :exc:`DeprecationWarning` and :exc:`PendingDeprecationWarning` +were only visible when running tests, which meant that developers primarily +writing single file scripts or using Python interactively could be surprised +by breaking changes in the APIs they used. + +.. seealso:: + + :pep:`565` -- Show DeprecationWarning in ``__main__`` + PEP written and implemented by Nick Coghlan + + +PEP 540: Add a new UTF-8 mode +----------------------------- + +Add a new UTF-8 mode to ignore the locale, use the UTF-8 encoding, and change +:data:`sys.stdin` and :data:`sys.stdout` error handlers to ``surrogateescape``. +This mode is enabled by default in the POSIX locale, but otherwise disabled by +default. + +The new :option:`-X` ``utf8`` command line option and :envvar:`PYTHONUTF8` +environment variable are added to control the UTF-8 mode. + +.. seealso:: + + :pep:`540` -- Add a new UTF-8 mode + PEP written and implemented by Victor Stinner + + +.. _whatsnew37-pep557: + +PEP 557: Data Classes +--------------------- + +Adds a new module ``dataclasses``. It provides a class decorator +``dataclass`` which inspects the class's variable annotations (see +:pep:`526`) and using them, adds methods such as ``__init__``, +``__repr__``, and ``__eq__`` to the class. It is similar to +``typing.NamedTuple``, but also works on classes with mutable +instances, among other features. + +For example:: + + @dataclass + class Point: + x: float + y: float + z: float = 0.0 + + p = Point(1.5, 2.5) + print(p) # produces "Point(x=1.5, y=2.5, z=0.0)" + +.. seealso:: + + :pep:`557` -- Data Classes + PEP written and implemented by Eric V. Smith + + +New Development Mode: -X dev +---------------------------- + +Add a new "development mode": :option:`-X` ``dev`` command line option and +:envvar:`PYTHONDEVMODE` environment variable to enable CPython's "development +mode", introducing additional runtime checks which are too expensive to be +enabled by default. See :option:`-X` ``dev`` documentation for the effects of +the development mode. + +Hash-based pycs +--------------- + +Python has traditionally checked the up-to-dateness of bytecode cache files +(i.e., ``.pyc`` files) by comparing the source metadata (last-modified timestamp +and size) with source metadata saved in the cache file header when it was +generated. While effective, this invalidation method has its drawbacks. When +filesystem timestamps are too coarse, Python can miss source updates, leading to +user confusion. Additionally, having a timestamp in the cache file is +problematic for `build reproduciblity `_ and +content-based build systems. + +:pep:`552` extends the pyc format to allow the hash of the source file to be +used for invalidation instead of the source timestamp. Such ``.pyc`` files are +called "hash-based". By default, Python still uses timestamp-based invalidation +and does not generate hash-based ``.pyc`` files at runtime. Hash-based ``.pyc`` +files may be generated with :mod:`py_compile` or :mod:`compileall`. + +Hash-based ``.pyc`` files come in two variants: checked and unchecked. Python +validates checked hash-based ``.pyc`` files against the corresponding source +files at runtime but doesn't do so for unchecked hash-based pycs. Unchecked +hash-based ``.pyc`` files are a useful performance optimization for environments +where a system external to Python (e.g., the build system) is responsible for +keeping ``.pyc`` files up-to-date. + +See :ref:`pyc-invalidation` for more information. Other Language Changes @@ -223,7 +402,18 @@ Other Language Changes New Modules =========== -* None yet. +importlib.resources +------------------- + +This module provides several new APIs and one new ABC for access to, opening, +and reading *resources* inside packages. Resources are roughly akin to files +inside of packages, but they needn't be actual files on the physical file +system. Module loaders can provide a :meth:`get_resource_reader()` function +which returns a :class:`importlib.abc.ResourceReader` instance to support this +new API. Built-in file path loaders and zip file loaders both support this. +(see the PyPI package +`importlib_resources `_ +as a compatible back port for older Python versions). Improved Modules @@ -265,8 +455,12 @@ is a list of strings, not bytes. contextlib ---------- -:func:`contextlib.asynccontextmanager` has been added. (Contributed by -Jelle Zijlstra in :issue:`29679`.) +:func:`~contextlib.asynccontextmanager` and +:class:`~contextlib.AbstractAsyncContextManager` have been added. (Contributed +by Jelle Zijlstra in :issue:`29679` and :issue:`30241`.) + +:class:`contextlib.AsyncExitStack` has been added. (Contributed by +Alexander Mohr and Ilya Kulakov in :issue:`29302`.) cProfile -------- @@ -283,6 +477,14 @@ Added support for the Blowfish method. The :func:`~crypt.mksalt` function now allows to specify the number of rounds for hashing. (Contributed by Serhiy Storchaka in :issue:`31702`.) +datetime +-------- + +Added the :func:`datetime.datetime.fromisoformat` method, which constructs a +:class:`datetime.datetime` object from a string in one of the formats output +by :func:`datetime.datetime.isoformat`. (Contributed by Paul Ganssle in +:issue:`15873`.) + dis --- @@ -298,6 +500,15 @@ README.rst is now included in the list of distutils standard READMEs and therefore included in source distributions. (Contributed by Ryan Gonzalez in :issue:`11913`.) +:class:`distutils.core.setup` now warns if the ``classifiers``, ``keywords`` +and ``platforms`` fields are not specified as a list or a string. +(Contributed by Berker Peksag in :issue:`19610`.) + +The ``upload`` command no longer tries to change CR end-of-line characters +to CRLF. This fixes a corruption issue with sdists that ended with a byte +equivalent to CR. +(Contributed by Bo Bayles in :issue:`32304`.) + http.client ----------- @@ -318,6 +529,19 @@ and the ``--directory`` to the command line of the module :mod:`~http.server`. With this parameter, the server serves the specified directory, by default it uses the current working directory. (Contributed by Stéphane Wirtel and Julien Palard in :issue:`28707`.) +hmac +---- + +The hmac module now has an optimized one-shot :func:`~hmac.digest` function, +which is up to three times faster than :func:`~hmac.HMAC`. +(Contributed by Christian Heimes in :issue:`32433`.) + +importlib +--------- + +The :class:`importlib.abc.ResourceReader` ABC was introduced to +support the loading of resource from packages. + locale ------ @@ -325,6 +549,10 @@ Added another argument *monetary* in :meth:`format_string` of :mod:`locale`. If *monetary* is true, the conversion uses monetary thousands separator and grouping strings. (Contributed by Garvit in :issue:`10379`.) +The :func:`locale.getpreferredencoding` function now always returns ``'UTF-8'`` +on Android or in the UTF-8 mode (:option:`-X` ``utf8`` option), the locale and +the *do_setlocale* argument are ignored. + math ---- @@ -351,6 +579,21 @@ pdb argument. If given, this is printed to the console just before debugging begins. (Contributed by Barry Warsaw in :issue:`31389`.) +pdb command line now accepts `-m module_name` as an alternative to +script file. (Contributed by Mario Corchero in :issue:`32206`.) + +py_compile +---------- + +:func:`py_compile.compile` -- and by extension, :mod:`compileall` -- now +respects the :envvar:`SOURCE_DATE_EPOCH` environment variable by +unconditionally creating ``.pyc`` files for hash-based validation. +This allows for guaranteeing +`reproducible builds `_ of ``.pyc`` +files when they are created eagerly. (Contributed by Bernhard M. Wiedemann +in :issue:`29708`.) + + re -- @@ -358,6 +601,42 @@ The flags :const:`re.ASCII`, :const:`re.LOCALE` and :const:`re.UNICODE` can be set within the scope of a group. (Contributed by Serhiy Storchaka in :issue:`31690`.) +:func:`re.split` now supports splitting on a pattern like ``r'\b'``, +``'^$'`` or ``(?=-)`` that matches an empty string. +(Contributed by Serhiy Storchaka in :issue:`25054`.) + +ssl +--- + +The ssl module now uses OpenSSL's builtin API instead of +:func:`~ssl.match_hostname` to check host name or IP address. Values +are validated during TLS handshake. Any cert validation error including +a failing host name match now raises :exc:`~ssl.SSLCertVerificationError` and +aborts the handshake with a proper TLS Alert message. The new exception +contains additional information. Host name validation can be customized +with :attr:`~ssl.SSLContext.host_flags`. +(Contributed by Christian Heimes in :issue:`31399`.) + +.. note:: + The improved host name check requires an OpenSSL 1.0.2 or 1.1 compatible + libssl. OpenSSL 0.9.8 and 1.0.1 are no longer supported. LibreSSL is + temporarily not supported until it gains the necessary OpenSSL 1.0.2 APIs. + +The ssl module no longer sends IP addresses in SNI TLS extension. +(Contributed by Christian Heimes in :issue:`32185`.) + +:func:`~ssl.match_hostname` no longer supports partial wildcards like +``www*.example.org``. :attr:`~ssl.SSLContext.host_flags` has partial +wildcard matching disabled by default. +(Contributed by Mandeep Singh in :issue:`23033` and Christian Heimes in +:issue:`31399`.) + +The default cipher suite selection of the ssl module now uses a blacklist +approach rather than a hard-coded whitelist. Python no longer re-enables +ciphers that have been blocked by OpenSSL security update. Default cipher +suite selection can be configured on compile time. +(Contributed by Christian Heimes in :issue:`31429`.) + string ------ @@ -365,6 +644,25 @@ string expression pattern for braced placeholders and non-braced placeholders separately. (Contributed by Barry Warsaw in :issue:`1198569`.) +subprocess +---------- + +On Windows the default for *close_fds* was changed from :const:`False` to +:const:`True` when redirecting the standard handles. It's now possible to set +*close_fds* to :const:`True` when redirecting the standard handles. See +:class:`subprocess.Popen`. + +This means that *close_fds* now defaults to :const:`True` on all supported +platforms. + +sys +--- + +Added :attr:`sys.flags.dev_mode` flag for the new development mode. + +Deprecated :func:`sys.set_coroutine_wrapper` and +:func:`sys.get_coroutine_wrapper`. + time ---- @@ -392,6 +690,21 @@ Added functions :func:`time.thread_time` and :func:`time.thread_time_ns` to get per-thread CPU time measurements. (Contributed by Antoine Pitrou in :issue:`32025`.) +unicodedata +----------- + +The internal :mod:`unicodedata` database has been upgraded to use `Unicode 10 +`_. (Contributed by Benjamin +Peterson.) + +unittest +-------- +Added new command-line option ``-k`` to filter tests to run with a substring or +Unix shell-like pattern. For example, ``python -m unittest -k foo`` runs the +tests ``foo_tests.SomeTest.test_something``, ``bar_tests.SomeTest.test_foo``, +but not ``bar_tests.FooTest.test_something``. + + unittest.mock ------------- @@ -404,20 +717,6 @@ children by preventing to get or set any new attribute on the sealed mock. The sealing process is performed recursively. (Contributed by Mario Corchero in :issue:`30541`.) -xmlrpc.server -------------- - -:meth:`register_function` of :class:`xmlrpc.server.SimpleXMLRPCDispatcher` and -its subclasses can be used as a decorator. (Contributed by Xiang Zhang in -:issue:`7769`.) - -unicodedata ------------ - -The internal :mod:`unicodedata` database has been upgraded to use `Unicode 10 -`_. (Contributed by Benjamin -Peterson.) - urllib.parse ------------ @@ -432,6 +731,29 @@ Function :func:`~uu.encode` now accepts an optional *backtick* keyword argument. When it's true, zeros are represented by ``'`'`` instead of spaces. (Contributed by Xiang Zhang in :issue:`30103`.) +warnings +-------- + +The initialization of the default warnings filters has changed as follows: + +* warnings enabled via command line options (including those for :option:`-b` + and the new CPython-specific ``-X dev`` option) are always passed to the + warnings machinery via the ``sys.warnoptions`` attribute. +* warnings filters enabled via the command line or the environment now have the + following precedence order: + + * the ``BytesWarning`` filter for :option:`-b` (or ``-bb``) + * any filters specified with :option:`-W` + * any filters specified with :envvar:`PYTHONWARNINGS` + * any other CPython specific filters (e.g. the ``default`` filter added + for the new ``-X dev`` mode) + * any implicit filters defined directly by the warnings machinery +* in CPython debug builds, all warnings are now displayed by default (the + implicit filter list is empty) + +(Contributed by Nick Coghlan and Victor Stinner in :issue:`20361`, +:issue:`32043`, and :issue:`32230`) + xml.etree --------- @@ -440,6 +762,13 @@ methods can now compare text of the current node with ``[. = "text"]``, not only text in children. Predicates also allow adding spaces for better readability. (Contributed by Stefan Behnel in :issue:`31648`.) +xmlrpc.server +------------- + +:meth:`register_function` of :class:`xmlrpc.server.SimpleXMLRPCDispatcher` and +its subclasses can be used as a decorator. (Contributed by Xiang Zhang in +:issue:`7769`.) + zipapp ------ @@ -490,13 +819,22 @@ Optimizations and :meth:`selectors.DevpollSelector.modify` may be around 10% faster under heavy loads. (Contributed by Giampaolo Rodola' in :issue:`30014`) +* Constant folding is moved from peephole optimizer to new AST optimizer. + (Contributed by Eugene Toder and INADA Naoki in :issue:`29469`) + Build and C API Changes ======================= +* :mod:`py_compile` and :mod:`compileall` now support the + :envvar:`SOURCE_DATE_EPOCH` environment variable by unconditionally + building ``.pyc`` files for hash verification instead of potentially + timestamp-based ``.pyc`` files. See the notes for the `py_compile`_ + improvement notes for more details. + * A full copy of libffi is no longer bundled for use when building the :mod:`_ctypes ` module on non-OSX UNIX platforms. An installed copy of libffi is now required when building ``_ctypes`` on such platforms. - Contributed by Zachary Ware in :issue:`27979`. + (Contributed by Zachary Ware in :issue:`27979`.) * The fields :c:member:`name` and :c:member:`doc` of structures :c:type:`PyMemberDef`, :c:type:`PyGetSetDef`, @@ -550,6 +888,23 @@ Other CPython Implementation Changes Deprecated ========== +* In Python 3.8, the abstract base classes in :mod:`collections.abc` will no + longer be exposed in the regular :mod:`collections` module. This will help + create a clearer distinction between the concrete classes and the abstract + base classes. + +* Yield expressions (both ``yield`` and ``yield from`` clauses) are now deprecated + in comprehensions and generator expressions (aside from the iterable expression + in the leftmost :keyword:`for` clause). This ensures that comprehensions + always immediately return a container of the appropriate type (rather than + potentially returning a :term:`generator iterator` object), while generator + expressions won't attempt to interleave their implicit output with the output + from any explicit yield expressions. + + In Python 3.7, such expressions emit :exc:`DeprecationWarning` when compiled, + in Python 3.8+ they will emit :exc:`SyntaxError`. (Contributed by Serhiy + Storchaka in :issue:`10544`.) + - Function :c:func:`PySlice_GetIndicesEx` is deprecated and replaced with a macro if ``Py_LIMITED_API`` is not set or set to the value between ``0x03050400`` and ``0x03060000`` (not including) or ``0x03060100`` or @@ -575,6 +930,9 @@ Deprecated - The :mod:`macpath` is now deprecated and will be removed in Python 3.8. +- The :class:`importlib.abc.ResourceLoader` ABC has been deprecated in + favour of :class:`importlib.abc.ResourceReader`. + Changes in the C API -------------------- @@ -599,8 +957,8 @@ Windows Only been used. If the specified version is not available py.exe will error exit. (Contributed by Steve Barnes in :issue:`30291`.) -- The launcher can be run as "py -0" to produce a list of the installed pythons, - *with default marked with an asterix*. Running "py -0p" will include the paths. +- The launcher can be run as ``py -0`` to produce a list of the installed pythons, + *with default marked with an asterisk*. Running ``py -0p`` will include the paths. If py is run with a version specifier that cannot be matched it will also print the *short form* list of available specifiers. (Contributed by Steve Barnes in :issue:`30362`.) @@ -608,6 +966,11 @@ Windows Only Removed ======= +Platform Support Removals +------------------------- + +* FreeBSD 9 and older are no longer supported. + API and Feature Removals ------------------------ @@ -653,6 +1016,12 @@ that may require changes to your code. Changes in Python behavior -------------------------- +* :pep:`479` is enabled for all code in Python 3.7, meaning that + :exc:`StopIteration` exceptions raised directly or indirectly in + coroutines and generators are transformed into :exc:`RuntimeError` + exceptions. + (Contributed by Yury Selivanov in :issue:`32670`.) + * Due to an oversight, earlier Python versions erroneously accepted the following syntax:: @@ -671,6 +1040,28 @@ Changes in Python behavior Changes in the Python API ------------------------- +* :meth:`socketserver.ThreadingMixIn.server_close` now waits until all + non-daemon threads complete. Use daemonic threads by setting + :data:`ThreadingMixIn.daemon_threads` to ``True`` to not wait until threads + complete. (Contributed by Victor Stinner in :issue:`31233`.) + +* :meth:`socketserver.ForkingMixIn.server_close` now waits until all + child processes complete. (Contributed by Victor Stinner in :issue:`31151`.) + +* The :func:`locale.localeconv` function now sets temporarily the ``LC_CTYPE`` + locale to the ``LC_NUMERIC`` locale in some cases. + +* The ``asyncio.windows_utils.socketpair()`` function has been + removed: use directly :func:`socket.socketpair` which is available on all + platforms since Python 3.5 (before, it wasn't available on Windows). + ``asyncio.windows_utils.socketpair()`` was just an alias to + ``socket.socketpair`` on Python 3.5 and newer. + +* :mod:`asyncio`: The module doesn't export :mod:`selectors` and + :mod:`_overlapped` modules as ``asyncio.selectors`` and + ``asyncio._overlapped``. Replace ``from asyncio import selectors`` with + ``import selectors`` for example. + * :meth:`pkgutil.walk_packages` now raises ValueError if *path* is a string. Previously an empty list was returned. (Contributed by Sanyam Khurana in :issue:`24744`.) @@ -725,8 +1116,51 @@ Changes in the Python API avoid a warning escape them with a backslash. (Contributed by Serhiy Storchaka in :issue:`30349`.) +* The result of splitting a string on a :mod:`regular expression ` + that could match an empty string has been changed. For example + splitting on ``r'\s*'`` will now split not only on whitespaces as it + did previously, but also on empty strings before all non-whitespace + characters and just before the end of the string. + The previous behavior can be restored by changing the pattern + to ``r'\s+'``. A :exc:`FutureWarning` was emitted for such patterns since + Python 3.5. + + For patterns that match both empty and non-empty strings, the result of + searching for all matches may also be changed in other cases. For example + in the string ``'a\n\n'``, the pattern ``r'(?m)^\s*?$'`` will not only + match empty strings at positions 2 and 3, but also the string ``'\n'`` at + positions 2--3. To match only blank lines, the pattern should be rewritten + as ``r'(?m)^[^\S\n]*$'``. + + :func:`re.sub()` now replaces empty matches adjacent to a previous + non-empty match. For example ``re.sub('x*', '-', 'abxd')`` returns now + ``'-a-b--d-'`` instead of ``'-a-b--d-'`` (the first minus between 'b' and + 'd' replaces 'x', and the second minus replaces an empty string between + 'x' and 'd'). + + (Contributed by Serhiy Storchaka in :issue:`25054` and :issue:`32308`.) + +* :class:`tracemalloc.Traceback` frames are now sorted from oldest to most + recent to be more consistent with :mod:`traceback`. + (Contributed by Jesse Bakker in :issue:`32121`.) + +* On OSes that support :const:`socket.SOCK_NONBLOCK` or + :const:`socket.SOCK_CLOEXEC` bit flags, the + :attr:`socket.type ` no longer has them applied. + Therefore, checks like ``if sock.type == socket.SOCK_STREAM`` + work as expected on all platforms. + (Contributed by Yury Selivanov in :issue:`32331`.) + .. _Unicode Technical Standard #18: https://unicode.org/reports/tr18/ +* On Windows the default for the *close_fds* argument of + :class:`subprocess.Popen` was changed from :const:`False` to :const:`True` + when redirecting the standard handles. If you previously depended on handles + being inherited when using :class:`subprocess.Popen` with standard io + redirection, you will have to pass ``close_fds=False`` to preserve the + previous behaviour, or use + :attr:`STARTUPINFO.lpAttributeList `. + Changes in the C API -------------------- @@ -747,6 +1181,8 @@ CPython bytecode changes * Added two new opcodes: :opcode:`LOAD_METHOD` and :opcode:`CALL_METHOD`. (Contributed by Yury Selivanov and INADA Naoki in :issue:`26110`.) +* Removed the STORE_ANNOTATION opcode. + Other CPython implementation changes ------------------------------------ @@ -761,6 +1197,18 @@ Other CPython implementation changes either in embedding applications, or in CPython itself. (Contributed by Nick Coghlan and Eric Snow as part of :issue:`22257`.) +* Due to changes in the way the default warnings filters are configured, + setting ``Py_BytesWarningFlag`` to a value greater than one is no longer + sufficient to both emit ``BytesWarning`` messages and have them converted + to exceptions. Instead, the flag must be set (to cause the warnings to be + emitted in the first place), and an explicit ``error::BytesWarning`` + warnings filter added to convert them to exceptions. + +* CPython' :mod:`ssl` module requires OpenSSL 1.0.2 or 1.1 compatible libssl. + OpenSSL 1.0.1 has reached end of lifetime on 2016-12-31 and is no longer + supported. LibreSSL is temporarily not supported as well. LibreSSL releases + up to version 2.6.4 are missing required OpenSSL 1.0.2 APIs. + Documentation ============= diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst new file mode 100644 index 00000000000..60f54a0561e --- /dev/null +++ b/Doc/whatsnew/3.8.rst @@ -0,0 +1,137 @@ +**************************** + What's New In Python 3.8 +**************************** + +:Release: |release| +:Date: |today| + +.. Rules for maintenance: + + * Anyone can add text to this document. Do not spend very much time + on the wording of your changes, because your text will probably + get rewritten to some degree. + + * The maintainer will go through Misc/NEWS periodically and add + changes; it's therefore more important to add your changes to + Misc/NEWS than to this file. + + * This is not a complete list of every single change; completeness + is the purpose of Misc/NEWS. Some changes I consider too small + or esoteric to include. If such a change is added to the text, + I'll just remove it. (This is another reason you shouldn't spend + too much time on writing your addition.) + + * If you want to draw your new text to the attention of the + maintainer, add 'XXX' to the beginning of the paragraph or + section. + + * It's OK to just add a fragmentary note about a change. For + example: "XXX Describe the transmogrify() function added to the + socket module." The maintainer will research the change and + write the necessary text. + + * You can comment out your additions if you like, but it's not + necessary (especially when a final release is some months away). + + * Credit the author of a patch or bugfix. Just the name is + sufficient; the e-mail address isn't necessary. + + * It's helpful to add the bug/patch number as a comment: + + XXX Describe the transmogrify() function added to the socket + module. + (Contributed by P.Y. Developer in :issue:`12345`.) + + This saves the maintainer the effort of going through the Mercurial log + when researching a change. + +This article explains the new features in Python 3.8, compared to 3.7. + +For full details, see the :source:`Misc/NEWS` file. + +.. note:: + + Prerelease users should be aware that this document is currently in draft + form. It will be updated substantially as Python 3.8 moves towards release, + so it's worth checking back even after reading earlier versions. + + +Summary -- Release highlights +============================= + +.. This section singles out the most important changes in Python 3.8. + Brevity is key. + + +.. PEP-sized items next. + + + +New Features +============ + + + +Other Language Changes +====================== + + + +New Modules +=========== + +* None yet. + + +Improved Modules +================ + + +Optimizations +============= + + +Build and C API Changes +======================= + + + +Deprecated +========== + + + +Removed +======= + + + +Porting to Python 3.8 +===================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + + +Changes in Python behavior +-------------------------- + +* Yield expressions (both ``yield`` and ``yield from`` clauses) are now disallowed + in comprehensions and generator expressions (aside from the iterable expression + in the leftmost :keyword:`for` clause). + (Contributed by Serhiy Storchaka in :issue:`10544`.) + + +Changes in the Python API +------------------------- + +* The :meth:`~tkinter.ttk.Treeview.selection` method of the + :class:`tkinter.ttk.Treeview` class no longer takes arguments. Using it with + arguments for changing the selection was deprecated in Python 3.6. Use + specialized methods like :meth:`~tkinter.ttk.Treeview.selection_set` for + changing the selection. (Contributed by Serhiy Storchaka in :issue:`31508`.) + +* A :mod:`dbm.dumb` database opened with flags ``'r'`` is now read-only. + :func:`dbm.dumb.open` with flags ``'r'`` and ``'w'`` no longer creates + a database if it does not exist. + (Contributed by Serhiy Storchaka in :issue:`32749`.) diff --git a/Doc/whatsnew/index.rst b/Doc/whatsnew/index.rst index 721dc03f42a..b1160c03982 100644 --- a/Doc/whatsnew/index.rst +++ b/Doc/whatsnew/index.rst @@ -11,6 +11,7 @@ anyone wishing to stay up-to-date after a new release. .. toctree:: :maxdepth: 2 + 3.8.rst 3.7.rst 3.6.rst 3.5.rst diff --git a/Include/Python.h b/Include/Python.h index 20051e7ff08..1feb1531cc9 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -35,6 +35,9 @@ #ifdef HAVE_UNISTD_H #include #endif +#ifdef HAVE_CRYPT_H +#include +#endif /* For size_t? */ #ifdef HAVE_STDDEF_H @@ -109,6 +112,7 @@ #include "pyerrors.h" #include "pystate.h" +#include "context.h" #include "pyarena.h" #include "modsupport.h" diff --git a/Include/abstract.h b/Include/abstract.h index 8969019e30c..3133cd10535 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -157,11 +157,11 @@ PyAPI_FUNC(PyObject *) PyObject_Call(PyObject *callable, #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject*) _PyStack_AsTuple( - PyObject **stack, + PyObject *const *stack, Py_ssize_t nargs); PyAPI_FUNC(PyObject*) _PyStack_AsTupleSlice( - PyObject **stack, + PyObject *const *stack, Py_ssize_t nargs, Py_ssize_t start, Py_ssize_t end); @@ -177,7 +177,7 @@ PyAPI_FUNC(PyObject*) _PyStack_AsTupleSlice( an exception, the caller is responsible to implement an explicit keys on kwnames. */ PyAPI_FUNC(PyObject *) _PyStack_AsDict( - PyObject **values, + PyObject *const *values, PyObject *kwnames); /* Convert (args, nargs, kwargs: dict) into a (stack, nargs, kwnames: tuple). @@ -192,10 +192,10 @@ PyAPI_FUNC(PyObject *) _PyStack_AsDict( The type of keyword keys is not checked, these checks should be done later (ex: _PyArg_ParseStackAndKeywords). */ PyAPI_FUNC(int) _PyStack_UnpackDict( - PyObject **args, + PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs, - PyObject ***p_stack, + PyObject *const **p_stack, PyObject **p_kwnames); /* Suggested size (number of positional arguments) for arrays of PyObject* @@ -224,7 +224,7 @@ PyAPI_FUNC(int) _PyObject_HasFastCall(PyObject *callable); error. */ PyAPI_FUNC(PyObject *) _PyObject_FastCallDict( PyObject *callable, - PyObject **args, + PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs); @@ -245,7 +245,7 @@ PyAPI_FUNC(PyObject *) _PyObject_FastCallDict( error. */ PyAPI_FUNC(PyObject *) _PyObject_FastCallKeywords( PyObject *callable, - PyObject **args, + PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames); @@ -264,7 +264,7 @@ PyAPI_FUNC(PyObject *) _PyObject_Call_Prepend( PyAPI_FUNC(PyObject *) _PyObject_FastCall_Prepend( PyObject *callable, PyObject *obj, - PyObject **args, + PyObject *const *args, Py_ssize_t nargs); PyAPI_FUNC(PyObject *) _Py_CheckFunctionResult(PyObject *callable, @@ -367,7 +367,7 @@ PyAPI_FUNC(PyObject *) _PyObject_CallMethodIdObjArgs( /* Implemented elsewhere: - long PyObject_Hash(PyObject *o); + Py_hash_t PyObject_Hash(PyObject *o); Compute and return the hash, hash_value, of an object, o. On failure, return -1. diff --git a/Include/ast.h b/Include/ast.h index 6a8c8165c05..639c4f82325 100644 --- a/Include/ast.h +++ b/Include/ast.h @@ -16,6 +16,15 @@ PyAPI_FUNC(mod_ty) PyAST_FromNodeObject( PyObject *filename, PyArena *arena); +#ifndef Py_LIMITED_API + +/* _PyAST_ExprAsUnicode is defined in ast_unparse.c */ +PyAPI_FUNC(PyObject *) _PyAST_ExprAsUnicode( + expr_ty e, + int omit_parens); + +#endif /* !Py_LIMITED_API */ + #ifdef __cplusplus } #endif diff --git a/Include/bitset.h b/Include/bitset.h index faeb41913df..b22fa77815c 100644 --- a/Include/bitset.h +++ b/Include/bitset.h @@ -7,7 +7,7 @@ extern "C" { /* Bitset interface */ -#define BYTE char +#define BYTE char typedef BYTE *bitset; @@ -18,13 +18,13 @@ int addbit(bitset bs, int ibit); /* Returns 0 if already set */ int samebitset(bitset bs1, bitset bs2, int nbits); void mergebitset(bitset bs1, bitset bs2, int nbits); -#define BITSPERBYTE (8*sizeof(BYTE)) -#define NBYTES(nbits) (((nbits) + BITSPERBYTE - 1) / BITSPERBYTE) +#define BITSPERBYTE (8*sizeof(BYTE)) +#define NBYTES(nbits) (((nbits) + BITSPERBYTE - 1) / BITSPERBYTE) -#define BIT2BYTE(ibit) ((ibit) / BITSPERBYTE) -#define BIT2SHIFT(ibit) ((ibit) % BITSPERBYTE) -#define BIT2MASK(ibit) (1 << BIT2SHIFT(ibit)) -#define BYTE2BIT(ibyte) ((ibyte) * BITSPERBYTE) +#define BIT2BYTE(ibit) ((ibit) / BITSPERBYTE) +#define BIT2SHIFT(ibit) ((ibit) % BITSPERBYTE) +#define BIT2MASK(ibit) (1 << BIT2SHIFT(ibit)) +#define BYTE2BIT(ibyte) ((ibyte) * BITSPERBYTE) #ifdef __cplusplus } diff --git a/Include/bytes_methods.h b/Include/bytes_methods.h index 7fa7540c38b..8434a50a4bb 100644 --- a/Include/bytes_methods.h +++ b/Include/bytes_methods.h @@ -9,6 +9,7 @@ extern PyObject* _Py_bytes_isspace(const char *cptr, Py_ssize_t len); extern PyObject* _Py_bytes_isalpha(const char *cptr, Py_ssize_t len); extern PyObject* _Py_bytes_isalnum(const char *cptr, Py_ssize_t len); +extern PyObject* _Py_bytes_isascii(const char *cptr, Py_ssize_t len); extern PyObject* _Py_bytes_isdigit(const char *cptr, Py_ssize_t len); extern PyObject* _Py_bytes_islower(const char *cptr, Py_ssize_t len); extern PyObject* _Py_bytes_isupper(const char *cptr, Py_ssize_t len); @@ -37,6 +38,7 @@ extern PyObject* _Py_bytes_maketrans(Py_buffer *frm, Py_buffer *to); extern const char _Py_isspace__doc__[]; extern const char _Py_isalpha__doc__[]; extern const char _Py_isalnum__doc__[]; +extern const char _Py_isascii__doc__[]; extern const char _Py_isdigit__doc__[]; extern const char _Py_islower__doc__[]; extern const char _Py_isupper__doc__[]; diff --git a/Include/bytesobject.h b/Include/bytesobject.h index 0f0bf9f369d..3fde4a221fd 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -52,9 +52,9 @@ PyAPI_FUNC(PyObject *) PyBytes_FromStringAndSize(const char *, Py_ssize_t); PyAPI_FUNC(PyObject *) PyBytes_FromString(const char *); PyAPI_FUNC(PyObject *) PyBytes_FromObject(PyObject *); PyAPI_FUNC(PyObject *) PyBytes_FromFormatV(const char*, va_list) - Py_GCC_ATTRIBUTE((format(printf, 1, 0))); + Py_GCC_ATTRIBUTE((format(printf, 1, 0))); PyAPI_FUNC(PyObject *) PyBytes_FromFormat(const char*, ...) - Py_GCC_ATTRIBUTE((format(printf, 1, 2))); + Py_GCC_ATTRIBUTE((format(printf, 1, 2))); PyAPI_FUNC(Py_ssize_t) PyBytes_Size(PyObject *); PyAPI_FUNC(char *) PyBytes_AsString(PyObject *); PyAPI_FUNC(PyObject *) PyBytes_Repr(PyObject *, int); @@ -72,8 +72,8 @@ PyAPI_FUNC(PyObject*) _PyBytes_FromHex( int use_bytearray); #endif PyAPI_FUNC(PyObject *) PyBytes_DecodeEscape(const char *, Py_ssize_t, - const char *, Py_ssize_t, - const char *); + const char *, Py_ssize_t, + const char *); #ifndef Py_LIMITED_API /* Helper for PyBytes_DecodeEscape that detects invalid escape chars. */ PyAPI_FUNC(PyObject *) _PyBytes_DecodeEscape(const char *, Py_ssize_t, @@ -132,10 +132,10 @@ PyAPI_FUNC(Py_ssize_t) _PyBytes_InsertThousandsGrouping(char *buffer, /* Flags used by string formatting */ #define F_LJUST (1<<0) -#define F_SIGN (1<<1) +#define F_SIGN (1<<1) #define F_BLANK (1<<2) -#define F_ALT (1<<3) -#define F_ZERO (1<<4) +#define F_ALT (1<<3) +#define F_ZERO (1<<4) #ifndef Py_LIMITED_API /* The _PyBytesWriter structure is big: it contains an embedded "stack buffer". diff --git a/Include/cellobject.h b/Include/cellobject.h index a0aa4d947c2..2f9b5b75d99 100644 --- a/Include/cellobject.h +++ b/Include/cellobject.h @@ -7,8 +7,8 @@ extern "C" { #endif typedef struct { - PyObject_HEAD - PyObject *ob_ref; /* Content of the cell or NULL when empty */ + PyObject_HEAD + PyObject *ob_ref; /* Content of the cell or NULL when empty */ } PyCellObject; PyAPI_DATA(PyTypeObject) PyCell_Type; diff --git a/Include/ceval.h b/Include/ceval.h index 70306b8bbd8..bce8a0beed8 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -31,6 +31,8 @@ PyAPI_FUNC(PyObject *) PyEval_CallMethod(PyObject *obj, #ifndef Py_LIMITED_API PyAPI_FUNC(void) PyEval_SetProfile(Py_tracefunc, PyObject *); PyAPI_FUNC(void) PyEval_SetTrace(Py_tracefunc, PyObject *); +PyAPI_FUNC(void) _PyEval_SetCoroutineOriginTrackingDepth(int new_depth); +PyAPI_FUNC(int) _PyEval_GetCoroutineOriginTrackingDepth(void); PyAPI_FUNC(void) _PyEval_SetCoroutineWrapper(PyObject *); PyAPI_FUNC(PyObject *) _PyEval_GetCoroutineWrapper(void); PyAPI_FUNC(void) _PyEval_SetAsyncGenFirstiter(PyObject *); diff --git a/Include/classobject.h b/Include/classobject.h index eeeb3e95a87..209f0f4a284 100644 --- a/Include/classobject.h +++ b/Include/classobject.h @@ -30,13 +30,13 @@ PyAPI_FUNC(PyObject *) PyMethod_Self(PyObject *); #define PyMethod_GET_FUNCTION(meth) \ (((PyMethodObject *)meth) -> im_func) #define PyMethod_GET_SELF(meth) \ - (((PyMethodObject *)meth) -> im_self) + (((PyMethodObject *)meth) -> im_self) PyAPI_FUNC(int) PyMethod_ClearFreeList(void); typedef struct { - PyObject_HEAD - PyObject *func; + PyObject_HEAD + PyObject *func; } PyInstanceMethodObject; PyAPI_DATA(PyTypeObject) PyInstanceMethod_Type; diff --git a/Include/code.h b/Include/code.h index 385258f93ce..2e661e8b36b 100644 --- a/Include/code.h +++ b/Include/code.h @@ -20,17 +20,17 @@ typedef uint16_t _Py_CODEUNIT; /* Bytecode object */ typedef struct { PyObject_HEAD - int co_argcount; /* #arguments, except *args */ - int co_kwonlyargcount; /* #keyword only arguments */ - int co_nlocals; /* #local variables */ - int co_stacksize; /* #entries needed for evaluation stack */ - int co_flags; /* CO_..., see below */ - int co_firstlineno; /* first source line number */ - PyObject *co_code; /* instruction opcodes */ - PyObject *co_consts; /* list (constants used) */ - PyObject *co_names; /* list of strings (names used) */ - PyObject *co_varnames; /* tuple of strings (local variable names) */ - PyObject *co_freevars; /* tuple of strings (free variable names) */ + int co_argcount; /* #arguments, except *args */ + int co_kwonlyargcount; /* #keyword only arguments */ + int co_nlocals; /* #local variables */ + int co_stacksize; /* #entries needed for evaluation stack */ + int co_flags; /* CO_..., see below */ + int co_firstlineno; /* first source line number */ + PyObject *co_code; /* instruction opcodes */ + PyObject *co_consts; /* list (constants used) */ + PyObject *co_names; /* list of strings (names used) */ + PyObject *co_varnames; /* tuple of strings (local variable names) */ + PyObject *co_freevars; /* tuple of strings (free variable names) */ PyObject *co_cellvars; /* tuple of strings (cell variable names) */ /* The rest aren't used in either hash or comparisons, except for co_name, used in both. This is done to preserve the name and line number @@ -38,11 +38,11 @@ typedef struct { would collapse identical functions/lambdas defined on different lines. */ Py_ssize_t *co_cell2arg; /* Maps cell vars which are arguments. */ - PyObject *co_filename; /* unicode (where it was loaded from) */ - PyObject *co_name; /* unicode (name, for reference) */ - PyObject *co_lnotab; /* string (encoding addr<->lineno mapping) See - Objects/lnotab_notes.txt for details. */ - void *co_zombieframe; /* for optimization only (see frameobject.c) */ + PyObject *co_filename; /* unicode (where it was loaded from) */ + PyObject *co_name; /* unicode (name, for reference) */ + PyObject *co_lnotab; /* string (encoding addr<->lineno mapping) See + Objects/lnotab_notes.txt for details. */ + void *co_zombieframe; /* for optimization only (see frameobject.c) */ PyObject *co_weakreflist; /* to support weakrefs to code objects */ /* Scratch space for extra data relating to the code object. Type is a void* to keep the format private in codeobject.c to force @@ -51,10 +51,10 @@ typedef struct { } PyCodeObject; /* Masks for co_flags above */ -#define CO_OPTIMIZED 0x0001 -#define CO_NEWLOCALS 0x0002 -#define CO_VARARGS 0x0004 -#define CO_VARKEYWORDS 0x0008 +#define CO_OPTIMIZED 0x0001 +#define CO_NEWLOCALS 0x0002 +#define CO_VARARGS 0x0004 +#define CO_VARKEYWORDS 0x0008 #define CO_NESTED 0x0010 #define CO_GENERATOR 0x0020 /* The CO_NOFREE flag is set if there are no free or cell variables. @@ -74,7 +74,7 @@ typedef struct { #if 0 #define CO_GENERATOR_ALLOWED 0x1000 #endif -#define CO_FUTURE_DIVISION 0x2000 +#define CO_FUTURE_DIVISION 0x2000 #define CO_FUTURE_ABSOLUTE_IMPORT 0x4000 /* do absolute imports by default */ #define CO_FUTURE_WITH_STATEMENT 0x8000 #define CO_FUTURE_PRINT_FUNCTION 0x10000 @@ -82,6 +82,7 @@ typedef struct { #define CO_FUTURE_BARRY_AS_BDFL 0x40000 #define CO_FUTURE_GENERATOR_STOP 0x80000 +#define CO_FUTURE_ANNOTATIONS 0x100000 /* This value is found in the co_cell2arg array when the associated cell variable does not correspond to an argument. */ @@ -101,9 +102,9 @@ PyAPI_DATA(PyTypeObject) PyCode_Type; /* Public interface */ PyAPI_FUNC(PyCodeObject *) PyCode_New( - int, int, int, int, int, PyObject *, PyObject *, - PyObject *, PyObject *, PyObject *, PyObject *, - PyObject *, PyObject *, int, PyObject *); + int, int, int, int, int, PyObject *, PyObject *, + PyObject *, PyObject *, PyObject *, PyObject *, + PyObject *, PyObject *, int, PyObject *); /* same as struct above */ /* Creates a new empty code object with the specified source location. */ diff --git a/Include/compile.h b/Include/compile.h index 3cc351d4098..edb961f4d72 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -16,7 +16,7 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *); #define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | \ CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | \ CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \ - CO_FUTURE_GENERATOR_STOP) + CO_FUTURE_GENERATOR_STOP | CO_FUTURE_ANNOTATIONS) #define PyCF_MASK_OBSOLETE (CO_NESTED) #define PyCF_SOURCE_IS_UTF8 0x0100 #define PyCF_DONT_IMPLY_DEDENT 0x0200 @@ -45,6 +45,7 @@ typedef struct { #define FUTURE_UNICODE_LITERALS "unicode_literals" #define FUTURE_BARRY_AS_BDFL "barry_as_FLUFL" #define FUTURE_GENERATOR_STOP "generator_stop" +#define FUTURE_ANNOTATIONS "annotations" struct _mod; /* Declare the existence of this type */ #define PyAST_Compile(mod, s, f, ar) PyAST_CompileEx(mod, s, f, -1, ar) @@ -75,6 +76,8 @@ PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name); #define PY_INVALID_STACK_EFFECT INT_MAX PyAPI_FUNC(int) PyCompile_OpcodeStackEffect(int opcode, int oparg); +PyAPI_FUNC(int) _PyAST_Optimize(struct _mod *, PyArena *arena, int optimize); + #ifdef __cplusplus } #endif diff --git a/Include/context.h b/Include/context.h new file mode 100644 index 00000000000..f872dceee0c --- /dev/null +++ b/Include/context.h @@ -0,0 +1,86 @@ +#ifndef Py_CONTEXT_H +#define Py_CONTEXT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_LIMITED_API + + +PyAPI_DATA(PyTypeObject) PyContext_Type; +typedef struct _pycontextobject PyContext; + +PyAPI_DATA(PyTypeObject) PyContextVar_Type; +typedef struct _pycontextvarobject PyContextVar; + +PyAPI_DATA(PyTypeObject) PyContextToken_Type; +typedef struct _pycontexttokenobject PyContextToken; + + +#define PyContext_CheckExact(o) (Py_TYPE(o) == &PyContext_Type) +#define PyContextVar_CheckExact(o) (Py_TYPE(o) == &PyContextVar_Type) +#define PyContextToken_CheckExact(o) (Py_TYPE(o) == &PyContextToken_Type) + + +PyAPI_FUNC(PyContext *) PyContext_New(void); +PyAPI_FUNC(PyContext *) PyContext_Copy(PyContext *); +PyAPI_FUNC(PyContext *) PyContext_CopyCurrent(void); + +PyAPI_FUNC(int) PyContext_Enter(PyContext *); +PyAPI_FUNC(int) PyContext_Exit(PyContext *); + + +/* Create a new context variable. + + default_value can be NULL. +*/ +PyAPI_FUNC(PyContextVar *) PyContextVar_New( + const char *name, PyObject *default_value); + + +/* Get a value for the variable. + + Returns -1 if an error occurred during lookup. + + Returns 0 if value either was or was not found. + + If value was found, *value will point to it. + If not, it will point to: + + - default_value, if not NULL; + - the default value of "var", if not NULL; + - NULL. + + '*value' will be a new ref, if not NULL. +*/ +PyAPI_FUNC(int) PyContextVar_Get( + PyContextVar *var, PyObject *default_value, PyObject **value); + + +/* Set a new value for the variable. + Returns NULL if an error occurs. +*/ +PyAPI_FUNC(PyContextToken *) PyContextVar_Set( + PyContextVar *var, PyObject *value); + + +/* Reset a variable to its previous value. + Returns 0 on sucess, -1 on error. +*/ +PyAPI_FUNC(int) PyContextVar_Reset( + PyContextVar *var, PyContextToken *token); + + +/* This method is exposed only for CPython tests. Don not use it. */ +PyAPI_FUNC(PyObject *) _PyContext_NewHamtForTests(void); + + +PyAPI_FUNC(int) PyContext_ClearFreeList(void); + + +#endif /* !Py_LIMITED_API */ + +#ifdef __cplusplus +} +#endif +#endif /* !Py_CONTEXT_H */ diff --git a/Include/datetime.h b/Include/datetime.h index 3bf35cbb7f2..059d5ecf7a2 100644 --- a/Include/datetime.h +++ b/Include/datetime.h @@ -155,12 +155,16 @@ typedef struct { PyTypeObject *DeltaType; PyTypeObject *TZInfoType; + /* singletons */ + PyObject *TimeZone_UTC; + /* constructors */ PyObject *(*Date_FromDate)(int, int, int, PyTypeObject*); PyObject *(*DateTime_FromDateAndTime)(int, int, int, int, int, int, int, PyObject*, PyTypeObject*); PyObject *(*Time_FromTime)(int, int, int, int, PyObject*, PyTypeObject*); PyObject *(*Delta_FromDelta)(int, int, int, int, PyTypeObject*); + PyObject *(*TimeZone_FromTimeZone)(PyObject *offset, PyObject *name); /* constructors for the DB API */ PyObject *(*DateTime_FromTimestamp)(PyObject*, PyObject*, PyObject*); @@ -202,6 +206,9 @@ static PyDateTime_CAPI *PyDateTimeAPI = NULL; #define PyDateTime_IMPORT \ PyDateTimeAPI = (PyDateTime_CAPI *)PyCapsule_Import(PyDateTime_CAPSULE_NAME, 0) +/* Macro for access to the UTC singleton */ +#define PyDateTime_TimeZone_UTC PyDateTimeAPI->TimeZone_UTC + /* Macros for type checking when not building the Python core. */ #define PyDate_Check(op) PyObject_TypeCheck(op, PyDateTimeAPI->DateType) #define PyDate_CheckExact(op) (Py_TYPE(op) == PyDateTimeAPI->DateType) @@ -242,6 +249,12 @@ static PyDateTime_CAPI *PyDateTimeAPI = NULL; PyDateTimeAPI->Delta_FromDelta(days, seconds, useconds, 1, \ PyDateTimeAPI->DeltaType) +#define PyTimeZone_FromOffset(offset) \ + PyDateTimeAPI->TimeZone_FromTimeZone(offset, NULL) + +#define PyTimeZone_FromOffsetAndName(offset, name) \ + PyDateTimeAPI->TimeZone_FromTimeZone(offset, name) + /* Macros supporting the DB API. */ #define PyDateTime_FromTimestamp(args) \ PyDateTimeAPI->DateTime_FromTimestamp( \ diff --git a/Include/descrobject.h b/Include/descrobject.h index cb43174838a..73bbb3fe54e 100644 --- a/Include/descrobject.h +++ b/Include/descrobject.h @@ -92,7 +92,7 @@ PyAPI_FUNC(PyObject *) PyDescr_NewGetSet(PyTypeObject *, #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyMethodDescr_FastCallKeywords( - PyObject *descrobj, PyObject **stack, Py_ssize_t nargs, PyObject *kwnames); + PyObject *descrobj, PyObject *const *stack, Py_ssize_t nargs, PyObject *kwnames); PyAPI_FUNC(PyObject *) PyDescr_NewWrapper(PyTypeObject *, struct wrapperbase *, void *); #define PyDescr_IsData(d) (Py_TYPE(d)->tp_descr_set != NULL) diff --git a/Include/errcode.h b/Include/errcode.h index 5946686c659..b37cd261d5e 100644 --- a/Include/errcode.h +++ b/Include/errcode.h @@ -13,24 +13,24 @@ extern "C" { the parser only returns E_EOF when it hits EOF immediately, and it never returns E_OK. */ -#define E_OK 10 /* No error */ -#define E_EOF 11 /* End Of File */ -#define E_INTR 12 /* Interrupted */ -#define E_TOKEN 13 /* Bad token */ -#define E_SYNTAX 14 /* Syntax error */ -#define E_NOMEM 15 /* Ran out of memory */ -#define E_DONE 16 /* Parsing complete */ -#define E_ERROR 17 /* Execution error */ -#define E_TABSPACE 18 /* Inconsistent mixing of tabs and spaces */ -#define E_OVERFLOW 19 /* Node had too many children */ -#define E_TOODEEP 20 /* Too many indentation levels */ -#define E_DEDENT 21 /* No matching outer block for dedent */ -#define E_DECODE 22 /* Error in decoding into Unicode */ -#define E_EOFS 23 /* EOF in triple-quoted string */ -#define E_EOLS 24 /* EOL in single-quoted string */ -#define E_LINECONT 25 /* Unexpected characters after a line continuation */ +#define E_OK 10 /* No error */ +#define E_EOF 11 /* End Of File */ +#define E_INTR 12 /* Interrupted */ +#define E_TOKEN 13 /* Bad token */ +#define E_SYNTAX 14 /* Syntax error */ +#define E_NOMEM 15 /* Ran out of memory */ +#define E_DONE 16 /* Parsing complete */ +#define E_ERROR 17 /* Execution error */ +#define E_TABSPACE 18 /* Inconsistent mixing of tabs and spaces */ +#define E_OVERFLOW 19 /* Node had too many children */ +#define E_TOODEEP 20 /* Too many indentation levels */ +#define E_DEDENT 21 /* No matching outer block for dedent */ +#define E_DECODE 22 /* Error in decoding into Unicode */ +#define E_EOFS 23 /* EOF in triple-quoted string */ +#define E_EOLS 24 /* EOL in single-quoted string */ +#define E_LINECONT 25 /* Unexpected characters after a line continuation */ #define E_IDENTIFIER 26 /* Invalid characters in identifier */ -#define E_BADSINGLE 27 /* Ill-formed single statement input */ +#define E_BADSINGLE 27 /* Ill-formed single statement input */ #ifdef __cplusplus } diff --git a/Include/eval.h b/Include/eval.h index 9541e106e29..2c1c2d0549a 100644 --- a/Include/eval.h +++ b/Include/eval.h @@ -12,19 +12,19 @@ PyAPI_FUNC(PyObject *) PyEval_EvalCode(PyObject *, PyObject *, PyObject *); PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyObject *co, PyObject *globals, PyObject *locals, - PyObject **args, int argc, - PyObject **kwds, int kwdc, - PyObject **defs, int defc, + PyObject *const *args, int argc, + PyObject *const *kwds, int kwdc, + PyObject *const *defs, int defc, PyObject *kwdefs, PyObject *closure); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyEval_EvalCodeWithName( PyObject *co, PyObject *globals, PyObject *locals, - PyObject **args, Py_ssize_t argcount, - PyObject **kwnames, PyObject **kwargs, + PyObject *const *args, Py_ssize_t argcount, + PyObject *const *kwnames, PyObject *const *kwargs, Py_ssize_t kwcount, int kwstep, - PyObject **defs, Py_ssize_t defcount, + PyObject *const *defs, Py_ssize_t defcount, PyObject *kwdefs, PyObject *closure, PyObject *name, PyObject *qualname); diff --git a/Include/fileobject.h b/Include/fileobject.h index 0b1678ee8da..89e8dd6a285 100644 --- a/Include/fileobject.h +++ b/Include/fileobject.h @@ -28,6 +28,10 @@ PyAPI_DATA(const char *) Py_FileSystemDefaultEncodeErrors; #endif PyAPI_DATA(int) Py_HasFileSystemDefaultEncoding; +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000 +PyAPI_DATA(int) Py_UTF8Mode; +#endif + /* Internal API The std printer acts as a preliminary sys.stderr until the new io diff --git a/Include/fileutils.h b/Include/fileutils.h index 900c70faad7..e4bf6d4db95 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -13,10 +13,51 @@ PyAPI_FUNC(wchar_t *) Py_DecodeLocale( PyAPI_FUNC(char*) Py_EncodeLocale( const wchar_t *text, size_t *error_pos); + +PyAPI_FUNC(char*) _Py_EncodeLocaleRaw( + const wchar_t *text, + size_t *error_pos); +#endif + +#ifdef Py_BUILD_CORE +PyAPI_FUNC(int) _Py_DecodeUTF8Ex( + const char *arg, + Py_ssize_t arglen, + wchar_t **wstr, + size_t *wlen, + const char **reason, + int surrogateescape); + +PyAPI_FUNC(int) _Py_EncodeUTF8Ex( + const wchar_t *text, + char **str, + size_t *error_pos, + const char **reason, + int raw_malloc, + int surrogateescape); + +PyAPI_FUNC(wchar_t*) _Py_DecodeUTF8_surrogateescape( + const char *arg, + Py_ssize_t arglen); + +PyAPI_FUNC(int) _Py_DecodeLocaleEx( + const char *arg, + wchar_t **wstr, + size_t *wlen, + const char **reason, + int current_locale, + int surrogateescape); + +PyAPI_FUNC(int) _Py_EncodeLocaleEx( + const wchar_t *text, + char **str, + size_t *error_pos, + const char **reason, + int current_locale, + int surrogateescape); #endif #ifndef Py_LIMITED_API - PyAPI_FUNC(PyObject *) _Py_device_encoding(int); #ifdef MS_WINDOWS @@ -111,6 +152,9 @@ PyAPI_FUNC(int) _Py_get_inheritable(int fd); PyAPI_FUNC(int) _Py_set_inheritable(int fd, int inheritable, int *atomic_flag_works); +PyAPI_FUNC(int) _Py_set_inheritable_async_safe(int fd, int inheritable, + int *atomic_flag_works); + PyAPI_FUNC(int) _Py_dup(int fd); #ifndef MS_WINDOWS @@ -119,6 +163,11 @@ PyAPI_FUNC(int) _Py_get_blocking(int fd); PyAPI_FUNC(int) _Py_set_blocking(int fd, int blocking); #endif /* !MS_WINDOWS */ +PyAPI_FUNC(int) _Py_GetLocaleconvNumeric( + PyObject **decimal_point, + PyObject **thousands_sep, + const char **grouping); + #endif /* Py_LIMITED_API */ #ifdef __cplusplus diff --git a/Include/funcobject.h b/Include/funcobject.h index 77bb8c39aeb..86674ac90a0 100644 --- a/Include/funcobject.h +++ b/Include/funcobject.h @@ -20,17 +20,17 @@ extern "C" { typedef struct { PyObject_HEAD - PyObject *func_code; /* A code object, the __code__ attribute */ - PyObject *func_globals; /* A dictionary (other mappings won't do) */ - PyObject *func_defaults; /* NULL or a tuple */ - PyObject *func_kwdefaults; /* NULL or a dict */ - PyObject *func_closure; /* NULL or a tuple of cell objects */ - PyObject *func_doc; /* The __doc__ attribute, can be anything */ - PyObject *func_name; /* The __name__ attribute, a string object */ - PyObject *func_dict; /* The __dict__ attribute, a dict or NULL */ - PyObject *func_weakreflist; /* List of weak references */ - PyObject *func_module; /* The __module__ attribute, can be anything */ - PyObject *func_annotations; /* Annotations, a dict or NULL */ + PyObject *func_code; /* A code object, the __code__ attribute */ + PyObject *func_globals; /* A dictionary (other mappings won't do) */ + PyObject *func_defaults; /* NULL or a tuple */ + PyObject *func_kwdefaults; /* NULL or a dict */ + PyObject *func_closure; /* NULL or a tuple of cell objects */ + PyObject *func_doc; /* The __doc__ attribute, can be anything */ + PyObject *func_name; /* The __name__ attribute, a string object */ + PyObject *func_dict; /* The __dict__ attribute, a dict or NULL */ + PyObject *func_weakreflist; /* List of weak references */ + PyObject *func_module; /* The __module__ attribute, can be anything */ + PyObject *func_annotations; /* Annotations, a dict or NULL */ PyObject *func_qualname; /* The qualified name */ /* Invariant: @@ -61,13 +61,13 @@ PyAPI_FUNC(int) PyFunction_SetAnnotations(PyObject *, PyObject *); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyFunction_FastCallDict( PyObject *func, - PyObject **args, + PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs); PyAPI_FUNC(PyObject *) _PyFunction_FastCallKeywords( PyObject *func, - PyObject **stack, + PyObject *const *stack, Py_ssize_t nargs, PyObject *kwnames); #endif @@ -77,17 +77,17 @@ PyAPI_FUNC(PyObject *) _PyFunction_FastCallKeywords( #define PyFunction_GET_CODE(func) \ (((PyFunctionObject *)func) -> func_code) #define PyFunction_GET_GLOBALS(func) \ - (((PyFunctionObject *)func) -> func_globals) + (((PyFunctionObject *)func) -> func_globals) #define PyFunction_GET_MODULE(func) \ - (((PyFunctionObject *)func) -> func_module) + (((PyFunctionObject *)func) -> func_module) #define PyFunction_GET_DEFAULTS(func) \ - (((PyFunctionObject *)func) -> func_defaults) + (((PyFunctionObject *)func) -> func_defaults) #define PyFunction_GET_KW_DEFAULTS(func) \ - (((PyFunctionObject *)func) -> func_kwdefaults) + (((PyFunctionObject *)func) -> func_kwdefaults) #define PyFunction_GET_CLOSURE(func) \ - (((PyFunctionObject *)func) -> func_closure) + (((PyFunctionObject *)func) -> func_closure) #define PyFunction_GET_ANNOTATIONS(func) \ - (((PyFunctionObject *)func) -> func_annotations) + (((PyFunctionObject *)func) -> func_annotations) /* The classmethod and staticmethod types lives here, too */ PyAPI_DATA(PyTypeObject) PyClassMethod_Type; diff --git a/Include/genobject.h b/Include/genobject.h index 87fbe17d4ab..16b983339cc 100644 --- a/Include/genobject.h +++ b/Include/genobject.h @@ -51,6 +51,7 @@ PyAPI_FUNC(void) _PyGen_Finalize(PyObject *self); #ifndef Py_LIMITED_API typedef struct { _PyGenObject_HEAD(cr) + PyObject *cr_origin; } PyCoroObject; PyAPI_DATA(PyTypeObject) PyCoro_Type; diff --git a/Include/grammar.h b/Include/grammar.h index f775f963817..e1703f4b360 100644 --- a/Include/grammar.h +++ b/Include/grammar.h @@ -12,58 +12,58 @@ extern "C" { /* A label of an arc */ typedef struct { - int lb_type; - char *lb_str; + int lb_type; + char *lb_str; } label; -#define EMPTY 0 /* Label number 0 is by definition the empty label */ +#define EMPTY 0 /* Label number 0 is by definition the empty label */ /* A list of labels */ typedef struct { - int ll_nlabels; - label *ll_label; + int ll_nlabels; + label *ll_label; } labellist; /* An arc from one state to another */ typedef struct { - short a_lbl; /* Label of this arc */ - short a_arrow; /* State where this arc goes to */ + short a_lbl; /* Label of this arc */ + short a_arrow; /* State where this arc goes to */ } arc; /* A state in a DFA */ typedef struct { - int s_narcs; - arc *s_arc; /* Array of arcs */ + int s_narcs; + arc *s_arc; /* Array of arcs */ /* Optional accelerators */ - int s_lower; /* Lowest label index */ - int s_upper; /* Highest label index */ - int *s_accel; /* Accelerator */ - int s_accept; /* Nonzero for accepting state */ + int s_lower; /* Lowest label index */ + int s_upper; /* Highest label index */ + int *s_accel; /* Accelerator */ + int s_accept; /* Nonzero for accepting state */ } state; /* A DFA */ typedef struct { - int d_type; /* Non-terminal this represents */ - char *d_name; /* For printing */ - int d_initial; /* Initial state */ - int d_nstates; - state *d_state; /* Array of states */ - bitset d_first; + int d_type; /* Non-terminal this represents */ + char *d_name; /* For printing */ + int d_initial; /* Initial state */ + int d_nstates; + state *d_state; /* Array of states */ + bitset d_first; } dfa; /* A grammar */ typedef struct { - int g_ndfas; - dfa *g_dfa; /* Array of DFAs */ - labellist g_ll; - int g_start; /* Start symbol of the grammar */ - int g_accel; /* Set if accelerators present */ + int g_ndfas; + dfa *g_dfa; /* Array of DFAs */ + labellist g_ll; + int g_start; /* Start symbol of the grammar */ + int g_accel; /* Set if accelerators present */ } grammar; /* FUNCTIONS */ diff --git a/Include/import.h b/Include/import.h index 26c4b1f18ef..ac3fc3bd28c 100644 --- a/Include/import.h +++ b/Include/import.h @@ -10,7 +10,7 @@ extern "C" { #ifndef Py_LIMITED_API PyAPI_FUNC(_PyInitError) _PyImportZip_Init(void); -PyMODINIT_FUNC PyInit_imp(void); +PyMODINIT_FUNC PyInit__imp(void); #endif /* !Py_LIMITED_API */ PyAPI_FUNC(long) PyImport_GetMagicNumber(void); PyAPI_FUNC(const char *) PyImport_GetMagicTag(void); diff --git a/Include/internal/context.h b/Include/internal/context.h new file mode 100644 index 00000000000..59f88f2614e --- /dev/null +++ b/Include/internal/context.h @@ -0,0 +1,41 @@ +#ifndef Py_INTERNAL_CONTEXT_H +#define Py_INTERNAL_CONTEXT_H + + +#include "internal/hamt.h" + + +struct _pycontextobject { + PyObject_HEAD + PyContext *ctx_prev; + PyHamtObject *ctx_vars; + PyObject *ctx_weakreflist; + int ctx_entered; +}; + + +struct _pycontextvarobject { + PyObject_HEAD + PyObject *var_name; + PyObject *var_default; + PyObject *var_cached; + uint64_t var_cached_tsid; + uint64_t var_cached_tsver; + Py_hash_t var_hash; +}; + + +struct _pycontexttokenobject { + PyObject_HEAD + PyContext *tok_ctx; + PyContextVar *tok_var; + PyObject *tok_oldval; + int tok_used; +}; + + +int _PyContext_Init(void); +void _PyContext_Fini(void); + + +#endif /* !Py_INTERNAL_CONTEXT_H */ diff --git a/Include/internal/hamt.h b/Include/internal/hamt.h new file mode 100644 index 00000000000..52488d0858d --- /dev/null +++ b/Include/internal/hamt.h @@ -0,0 +1,113 @@ +#ifndef Py_INTERNAL_HAMT_H +#define Py_INTERNAL_HAMT_H + + +#define _Py_HAMT_MAX_TREE_DEPTH 7 + + +#define PyHamt_Check(o) (Py_TYPE(o) == &_PyHamt_Type) + + +/* Abstract tree node. */ +typedef struct { + PyObject_HEAD +} PyHamtNode; + + +/* An HAMT immutable mapping collection. */ +typedef struct { + PyObject_HEAD + PyHamtNode *h_root; + PyObject *h_weakreflist; + Py_ssize_t h_count; +} PyHamtObject; + + +/* A struct to hold the state of depth-first traverse of the tree. + + HAMT is an immutable collection. Iterators will hold a strong reference + to it, and every node in the HAMT has strong references to its children. + + So for iterators, we can implement zero allocations and zero reference + inc/dec depth-first iteration. + + - i_nodes: an array of seven pointers to tree nodes + - i_level: the current node in i_nodes + - i_pos: an array of positions within nodes in i_nodes. +*/ +typedef struct { + PyHamtNode *i_nodes[_Py_HAMT_MAX_TREE_DEPTH]; + Py_ssize_t i_pos[_Py_HAMT_MAX_TREE_DEPTH]; + int8_t i_level; +} PyHamtIteratorState; + + +/* Base iterator object. + + Contains the iteration state, a pointer to the HAMT tree, + and a pointer to the 'yield function'. The latter is a simple + function that returns a key/value tuple for the 'Items' iterator, + just a key for the 'Keys' iterator, and a value for the 'Values' + iterator. +*/ +typedef struct { + PyObject_HEAD + PyHamtObject *hi_obj; + PyHamtIteratorState hi_iter; + binaryfunc hi_yield; +} PyHamtIterator; + + +PyAPI_DATA(PyTypeObject) _PyHamt_Type; +PyAPI_DATA(PyTypeObject) _PyHamt_ArrayNode_Type; +PyAPI_DATA(PyTypeObject) _PyHamt_BitmapNode_Type; +PyAPI_DATA(PyTypeObject) _PyHamt_CollisionNode_Type; +PyAPI_DATA(PyTypeObject) _PyHamtKeys_Type; +PyAPI_DATA(PyTypeObject) _PyHamtValues_Type; +PyAPI_DATA(PyTypeObject) _PyHamtItems_Type; + + +/* Create a new HAMT immutable mapping. */ +PyHamtObject * _PyHamt_New(void); + +/* Return a new collection based on "o", but with an additional + key/val pair. */ +PyHamtObject * _PyHamt_Assoc(PyHamtObject *o, PyObject *key, PyObject *val); + +/* Return a new collection based on "o", but without "key". */ +PyHamtObject * _PyHamt_Without(PyHamtObject *o, PyObject *key); + +/* Find "key" in the "o" collection. + + Return: + - -1: An error ocurred. + - 0: "key" wasn't found in "o". + - 1: "key" is in "o"; "*val" is set to its value (a borrowed ref). +*/ +int _PyHamt_Find(PyHamtObject *o, PyObject *key, PyObject **val); + +/* Check if "v" is equal to "w". + + Return: + - 0: v != w + - 1: v == w + - -1: An error occurred. +*/ +int _PyHamt_Eq(PyHamtObject *v, PyHamtObject *w); + +/* Return the size of "o"; equivalent of "len(o)". */ +Py_ssize_t _PyHamt_Len(PyHamtObject *o); + +/* Return a Keys iterator over "o". */ +PyObject * _PyHamt_NewIterKeys(PyHamtObject *o); + +/* Return a Values iterator over "o". */ +PyObject * _PyHamt_NewIterValues(PyHamtObject *o); + +/* Return a Items iterator over "o". */ +PyObject * _PyHamt_NewIterItems(PyHamtObject *o); + +int _PyHamt_Init(void); +void _PyHamt_Fini(void); + +#endif /* !Py_INTERNAL_HAMT_H */ diff --git a/Include/internal/hash.h b/Include/internal/hash.h new file mode 100644 index 00000000000..e14b80a7f27 --- /dev/null +++ b/Include/internal/hash.h @@ -0,0 +1,6 @@ +#ifndef Py_INTERNAL_HASH_H +#define Py_INTERNAL_HASH_H + +uint64_t _Py_KeyedHash(uint64_t, const char *, Py_ssize_t); + +#endif diff --git a/Include/internal/import.h b/Include/internal/import.h new file mode 100644 index 00000000000..4746e755755 --- /dev/null +++ b/Include/internal/import.h @@ -0,0 +1,6 @@ +#ifndef Py_INTERNAL_IMPORT_H +#define Py_INTERNAL_IMPORT_H + +extern const char *_Py_CheckHashBasedPycsMode; + +#endif diff --git a/Include/internal/mem.h b/Include/internal/mem.h index 471cdf45df2..a731e30e6af 100644 --- a/Include/internal/mem.h +++ b/Include/internal/mem.h @@ -7,54 +7,6 @@ extern "C" { #include "objimpl.h" #include "pymem.h" -#ifdef WITH_PYMALLOC -#include "internal/pymalloc.h" -#endif - -/* Low-level memory runtime state */ - -struct _pymem_runtime_state { - struct _allocator_runtime_state { - PyMemAllocatorEx mem; - PyMemAllocatorEx obj; - PyMemAllocatorEx raw; - } allocators; -#ifdef WITH_PYMALLOC - /* Array of objects used to track chunks of memory (arenas). */ - struct arena_object* arenas; - /* The head of the singly-linked, NULL-terminated list of available - arena_objects. */ - struct arena_object* unused_arena_objects; - /* The head of the doubly-linked, NULL-terminated at each end, - list of arena_objects associated with arenas that have pools - available. */ - struct arena_object* usable_arenas; - /* Number of slots currently allocated in the `arenas` vector. */ - unsigned int maxarenas; - /* Number of arenas allocated that haven't been free()'d. */ - size_t narenas_currently_allocated; - /* High water mark (max value ever seen) for - * narenas_currently_allocated. */ - size_t narenas_highwater; - /* Total number of times malloc() called to allocate an arena. */ - size_t ntimes_arena_allocated; - poolp usedpools[MAX_POOLS]; - Py_ssize_t num_allocated_blocks; -#endif /* WITH_PYMALLOC */ - size_t serialno; /* incremented on each debug {m,re}alloc */ -}; - -PyAPI_FUNC(void) _PyMem_Initialize(struct _pymem_runtime_state *); - - -/* High-level memory runtime state */ - -struct _pyobj_runtime_state { - PyObjectArenaAllocator allocator_arenas; -}; - -PyAPI_FUNC(void) _PyObject_Initialize(struct _pyobj_runtime_state *); - /* GC runtime state */ diff --git a/Include/internal/pygetopt.h b/Include/internal/pygetopt.h new file mode 100644 index 00000000000..8ef2ada72fe --- /dev/null +++ b/Include/internal/pygetopt.h @@ -0,0 +1,19 @@ +#ifndef Py_INTERNAL_PYGETOPT_H +#define Py_INTERNAL_PYGETOPT_H + +extern int _PyOS_opterr; +extern int _PyOS_optind; +extern wchar_t *_PyOS_optarg; + +extern void _PyOS_ResetGetOpt(void); + +typedef struct { + const wchar_t *name; + int has_arg; + int val; +} _PyOS_LongOption; + +extern int _PyOS_GetOpt(int argc, wchar_t **argv, wchar_t *optstring, + const _PyOS_LongOption *longopts, int *longindex); + +#endif /* !Py_INTERNAL_PYGETOPT_H */ diff --git a/Include/internal/pymalloc.h b/Include/internal/pymalloc.h deleted file mode 100644 index 723d9e7e671..00000000000 --- a/Include/internal/pymalloc.h +++ /dev/null @@ -1,443 +0,0 @@ - -/* An object allocator for Python. - - Here is an introduction to the layers of the Python memory architecture, - showing where the object allocator is actually used (layer +2), It is - called for every object allocation and deallocation (PyObject_New/Del), - unless the object-specific allocators implement a proprietary allocation - scheme (ex.: ints use a simple free list). This is also the place where - the cyclic garbage collector operates selectively on container objects. - - - Object-specific allocators - _____ ______ ______ ________ - [ int ] [ dict ] [ list ] ... [ string ] Python core | -+3 | <----- Object-specific memory -----> | <-- Non-object memory --> | - _______________________________ | | - [ Python's object allocator ] | | -+2 | ####### Object memory ####### | <------ Internal buffers ------> | - ______________________________________________________________ | - [ Python's raw memory allocator (PyMem_ API) ] | -+1 | <----- Python memory (under PyMem manager's control) ------> | | - __________________________________________________________________ - [ Underlying general-purpose allocator (ex: C library malloc) ] - 0 | <------ Virtual memory allocated for the python process -------> | - - ========================================================================= - _______________________________________________________________________ - [ OS-specific Virtual Memory Manager (VMM) ] --1 | <--- Kernel dynamic storage allocation & management (page-based) ---> | - __________________________________ __________________________________ - [ ] [ ] --2 | <-- Physical memory: ROM/RAM --> | | <-- Secondary storage (swap) --> | - -*/ -/*==========================================================================*/ - -/* A fast, special-purpose memory allocator for small blocks, to be used - on top of a general-purpose malloc -- heavily based on previous art. */ - -/* Vladimir Marangozov -- August 2000 */ - -/* - * "Memory management is where the rubber meets the road -- if we do the wrong - * thing at any level, the results will not be good. And if we don't make the - * levels work well together, we are in serious trouble." (1) - * - * (1) Paul R. Wilson, Mark S. Johnstone, Michael Neely, and David Boles, - * "Dynamic Storage Allocation: A Survey and Critical Review", - * in Proc. 1995 Int'l. Workshop on Memory Management, September 1995. - */ - -#ifndef Py_INTERNAL_PYMALLOC_H -#define Py_INTERNAL_PYMALLOC_H - -/* #undef WITH_MEMORY_LIMITS */ /* disable mem limit checks */ - -/*==========================================================================*/ - -/* - * Allocation strategy abstract: - * - * For small requests, the allocator sub-allocates blocks of memory. - * Requests greater than SMALL_REQUEST_THRESHOLD bytes are routed to the - * system's allocator. - * - * Small requests are grouped in size classes spaced 8 bytes apart, due - * to the required valid alignment of the returned address. Requests of - * a particular size are serviced from memory pools of 4K (one VMM page). - * Pools are fragmented on demand and contain free lists of blocks of one - * particular size class. In other words, there is a fixed-size allocator - * for each size class. Free pools are shared by the different allocators - * thus minimizing the space reserved for a particular size class. - * - * This allocation strategy is a variant of what is known as "simple - * segregated storage based on array of free lists". The main drawback of - * simple segregated storage is that we might end up with lot of reserved - * memory for the different free lists, which degenerate in time. To avoid - * this, we partition each free list in pools and we share dynamically the - * reserved space between all free lists. This technique is quite efficient - * for memory intensive programs which allocate mainly small-sized blocks. - * - * For small requests we have the following table: - * - * Request in bytes Size of allocated block Size class idx - * ---------------------------------------------------------------- - * 1-8 8 0 - * 9-16 16 1 - * 17-24 24 2 - * 25-32 32 3 - * 33-40 40 4 - * 41-48 48 5 - * 49-56 56 6 - * 57-64 64 7 - * 65-72 72 8 - * ... ... ... - * 497-504 504 62 - * 505-512 512 63 - * - * 0, SMALL_REQUEST_THRESHOLD + 1 and up: routed to the underlying - * allocator. - */ - -/*==========================================================================*/ - -/* - * -- Main tunable settings section -- - */ - -/* - * Alignment of addresses returned to the user. 8-bytes alignment works - * on most current architectures (with 32-bit or 64-bit address busses). - * The alignment value is also used for grouping small requests in size - * classes spaced ALIGNMENT bytes apart. - * - * You shouldn't change this unless you know what you are doing. - */ -#define ALIGNMENT 8 /* must be 2^N */ -#define ALIGNMENT_SHIFT 3 - -/* Return the number of bytes in size class I, as a uint. */ -#define INDEX2SIZE(I) (((unsigned int)(I) + 1) << ALIGNMENT_SHIFT) - -/* - * Max size threshold below which malloc requests are considered to be - * small enough in order to use preallocated memory pools. You can tune - * this value according to your application behaviour and memory needs. - * - * Note: a size threshold of 512 guarantees that newly created dictionaries - * will be allocated from preallocated memory pools on 64-bit. - * - * The following invariants must hold: - * 1) ALIGNMENT <= SMALL_REQUEST_THRESHOLD <= 512 - * 2) SMALL_REQUEST_THRESHOLD is evenly divisible by ALIGNMENT - * - * Although not required, for better performance and space efficiency, - * it is recommended that SMALL_REQUEST_THRESHOLD is set to a power of 2. - */ -#define SMALL_REQUEST_THRESHOLD 512 -#define NB_SMALL_SIZE_CLASSES (SMALL_REQUEST_THRESHOLD / ALIGNMENT) - -#if NB_SMALL_SIZE_CLASSES > 64 -#error "NB_SMALL_SIZE_CLASSES should be less than 64" -#endif /* NB_SMALL_SIZE_CLASSES > 64 */ - -/* - * The system's VMM page size can be obtained on most unices with a - * getpagesize() call or deduced from various header files. To make - * things simpler, we assume that it is 4K, which is OK for most systems. - * It is probably better if this is the native page size, but it doesn't - * have to be. In theory, if SYSTEM_PAGE_SIZE is larger than the native page - * size, then `POOL_ADDR(p)->arenaindex' could rarely cause a segmentation - * violation fault. 4K is apparently OK for all the platforms that python - * currently targets. - */ -#define SYSTEM_PAGE_SIZE (4 * 1024) -#define SYSTEM_PAGE_SIZE_MASK (SYSTEM_PAGE_SIZE - 1) - -/* - * Maximum amount of memory managed by the allocator for small requests. - */ -#ifdef WITH_MEMORY_LIMITS -#ifndef SMALL_MEMORY_LIMIT -#define SMALL_MEMORY_LIMIT (64 * 1024 * 1024) /* 64 MiB -- more? */ -#endif -#endif - -/* - * The allocator sub-allocates blocks of memory (called arenas) aligned - * on a page boundary. This is a reserved virtual address space for the - * current process (obtained through a malloc()/mmap() call). In no way this - * means that the memory arenas will be used entirely. A malloc() is - * usually an address range reservation for bytes, unless all pages within - * this space are referenced subsequently. So malloc'ing big blocks and not - * using them does not mean "wasting memory". It's an addressable range - * wastage... - * - * Arenas are allocated with mmap() on systems supporting anonymous memory - * mappings to reduce heap fragmentation. - */ -#define ARENA_SIZE (256 << 10) /* 256 KiB */ - -#ifdef WITH_MEMORY_LIMITS -#define MAX_ARENAS (SMALL_MEMORY_LIMIT / ARENA_SIZE) -#endif - -/* - * Size of the pools used for small blocks. Should be a power of 2, - * between 1K and SYSTEM_PAGE_SIZE, that is: 1k, 2k, 4k. - */ -#define POOL_SIZE SYSTEM_PAGE_SIZE /* must be 2^N */ -#define POOL_SIZE_MASK SYSTEM_PAGE_SIZE_MASK - -/* - * -- End of tunable settings section -- - */ - -/*==========================================================================*/ - -/* - * Locking - * - * To reduce lock contention, it would probably be better to refine the - * crude function locking with per size class locking. I'm not positive - * however, whether it's worth switching to such locking policy because - * of the performance penalty it might introduce. - * - * The following macros describe the simplest (should also be the fastest) - * lock object on a particular platform and the init/fini/lock/unlock - * operations on it. The locks defined here are not expected to be recursive - * because it is assumed that they will always be called in the order: - * INIT, [LOCK, UNLOCK]*, FINI. - */ - -/* - * Python's threads are serialized, so object malloc locking is disabled. - */ -#define SIMPLELOCK_DECL(lock) /* simple lock declaration */ -#define SIMPLELOCK_INIT(lock) /* allocate (if needed) and initialize */ -#define SIMPLELOCK_FINI(lock) /* free/destroy an existing lock */ -#define SIMPLELOCK_LOCK(lock) /* acquire released lock */ -#define SIMPLELOCK_UNLOCK(lock) /* release acquired lock */ - -/* When you say memory, my mind reasons in terms of (pointers to) blocks */ -typedef uint8_t pyblock; - -/* Pool for small blocks. */ -struct pool_header { - union { pyblock *_padding; - unsigned int count; } ref; /* number of allocated blocks */ - pyblock *freeblock; /* pool's free list head */ - struct pool_header *nextpool; /* next pool of this size class */ - struct pool_header *prevpool; /* previous pool "" */ - unsigned int arenaindex; /* index into arenas of base adr */ - unsigned int szidx; /* block size class index */ - unsigned int nextoffset; /* bytes to virgin block */ - unsigned int maxnextoffset; /* largest valid nextoffset */ -}; - -typedef struct pool_header *poolp; - -/* Record keeping for arenas. */ -struct arena_object { - /* The address of the arena, as returned by malloc. Note that 0 - * will never be returned by a successful malloc, and is used - * here to mark an arena_object that doesn't correspond to an - * allocated arena. - */ - uintptr_t address; - - /* Pool-aligned pointer to the next pool to be carved off. */ - pyblock* pool_address; - - /* The number of available pools in the arena: free pools + never- - * allocated pools. - */ - unsigned int nfreepools; - - /* The total number of pools in the arena, whether or not available. */ - unsigned int ntotalpools; - - /* Singly-linked list of available pools. */ - struct pool_header* freepools; - - /* Whenever this arena_object is not associated with an allocated - * arena, the nextarena member is used to link all unassociated - * arena_objects in the singly-linked `unused_arena_objects` list. - * The prevarena member is unused in this case. - * - * When this arena_object is associated with an allocated arena - * with at least one available pool, both members are used in the - * doubly-linked `usable_arenas` list, which is maintained in - * increasing order of `nfreepools` values. - * - * Else this arena_object is associated with an allocated arena - * all of whose pools are in use. `nextarena` and `prevarena` - * are both meaningless in this case. - */ - struct arena_object* nextarena; - struct arena_object* prevarena; -}; - -#define POOL_OVERHEAD _Py_SIZE_ROUND_UP(sizeof(struct pool_header), ALIGNMENT) - -#define DUMMY_SIZE_IDX 0xffff /* size class of newly cached pools */ - -/* Round pointer P down to the closest pool-aligned address <= P, as a poolp */ -#define POOL_ADDR(P) ((poolp)_Py_ALIGN_DOWN((P), POOL_SIZE)) - -/* Return total number of blocks in pool of size index I, as a uint. */ -#define NUMBLOCKS(I) \ - ((unsigned int)(POOL_SIZE - POOL_OVERHEAD) / INDEX2SIZE(I)) - -/*==========================================================================*/ - -/* - * This malloc lock - */ -SIMPLELOCK_DECL(_malloc_lock) -#define LOCK() SIMPLELOCK_LOCK(_malloc_lock) -#define UNLOCK() SIMPLELOCK_UNLOCK(_malloc_lock) -#define LOCK_INIT() SIMPLELOCK_INIT(_malloc_lock) -#define LOCK_FINI() SIMPLELOCK_FINI(_malloc_lock) - -/* - * Pool table -- headed, circular, doubly-linked lists of partially used pools. - -This is involved. For an index i, usedpools[i+i] is the header for a list of -all partially used pools holding small blocks with "size class idx" i. So -usedpools[0] corresponds to blocks of size 8, usedpools[2] to blocks of size -16, and so on: index 2*i <-> blocks of size (i+1)<freeblock points to -the start of a singly-linked list of free blocks within the pool. When a -block is freed, it's inserted at the front of its pool's freeblock list. Note -that the available blocks in a pool are *not* linked all together when a pool -is initialized. Instead only "the first two" (lowest addresses) blocks are -set up, returning the first such block, and setting pool->freeblock to a -one-block list holding the second such block. This is consistent with that -pymalloc strives at all levels (arena, pool, and block) never to touch a piece -of memory until it's actually needed. - -So long as a pool is in the used state, we're certain there *is* a block -available for allocating, and pool->freeblock is not NULL. If pool->freeblock -points to the end of the free list before we've carved the entire pool into -blocks, that means we simply haven't yet gotten to one of the higher-address -blocks. The offset from the pool_header to the start of "the next" virgin -block is stored in the pool_header nextoffset member, and the largest value -of nextoffset that makes sense is stored in the maxnextoffset member when a -pool is initialized. All the blocks in a pool have been passed out at least -once when and only when nextoffset > maxnextoffset. - - -Major obscurity: While the usedpools vector is declared to have poolp -entries, it doesn't really. It really contains two pointers per (conceptual) -poolp entry, the nextpool and prevpool members of a pool_header. The -excruciating initialization code below fools C so that - - usedpool[i+i] - -"acts like" a genuine poolp, but only so long as you only reference its -nextpool and prevpool members. The "- 2*sizeof(block *)" gibberish is -compensating for that a pool_header's nextpool and prevpool members -immediately follow a pool_header's first two members: - - union { block *_padding; - uint count; } ref; - block *freeblock; - -each of which consume sizeof(block *) bytes. So what usedpools[i+i] really -contains is a fudged-up pointer p such that *if* C believes it's a poolp -pointer, then p->nextpool and p->prevpool are both p (meaning that the headed -circular list is empty). - -It's unclear why the usedpools setup is so convoluted. It could be to -minimize the amount of cache required to hold this heavily-referenced table -(which only *needs* the two interpool pointer members of a pool_header). OTOH, -referencing code has to remember to "double the index" and doing so isn't -free, usedpools[0] isn't a strictly legal pointer, and we're crucially relying -on that C doesn't insert any padding anywhere in a pool_header at or before -the prevpool member. -**************************************************************************** */ - -#define MAX_POOLS (2 * ((NB_SMALL_SIZE_CLASSES + 7) / 8) * 8) - -/*========================================================================== -Arena management. - -`arenas` is a vector of arena_objects. It contains maxarenas entries, some of -which may not be currently used (== they're arena_objects that aren't -currently associated with an allocated arena). Note that arenas proper are -separately malloc'ed. - -Prior to Python 2.5, arenas were never free()'ed. Starting with Python 2.5, -we do try to free() arenas, and use some mild heuristic strategies to increase -the likelihood that arenas eventually can be freed. - -unused_arena_objects - - This is a singly-linked list of the arena_objects that are currently not - being used (no arena is associated with them). Objects are taken off the - head of the list in new_arena(), and are pushed on the head of the list in - PyObject_Free() when the arena is empty. Key invariant: an arena_object - is on this list if and only if its .address member is 0. - -usable_arenas - - This is a doubly-linked list of the arena_objects associated with arenas - that have pools available. These pools are either waiting to be reused, - or have not been used before. The list is sorted to have the most- - allocated arenas first (ascending order based on the nfreepools member). - This means that the next allocation will come from a heavily used arena, - which gives the nearly empty arenas a chance to be returned to the system. - In my unscientific tests this dramatically improved the number of arenas - that could be freed. - -Note that an arena_object associated with an arena all of whose pools are -currently in use isn't on either list. -*/ - -/* How many arena_objects do we initially allocate? - * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4 MiB before growing the - * `arenas` vector. - */ -#define INITIAL_ARENA_OBJECTS 16 - -#endif /* Py_INTERNAL_PYMALLOC_H */ diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index 67b4a516a76..2b60b25c19b 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -37,6 +37,107 @@ struct _gilstate_runtime_state { #define _PyGILState_check_enabled _PyRuntime.gilstate.check_enabled +typedef struct { + /* Full path to the Python program */ + wchar_t *program_full_path; + wchar_t *prefix; +#ifdef MS_WINDOWS + wchar_t *dll_path; +#else + wchar_t *exec_prefix; +#endif + /* Set by Py_SetPath(), or computed by _PyPathConfig_Init() */ + wchar_t *module_search_path; + /* Python program name */ + wchar_t *program_name; + /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */ + wchar_t *home; +} _PyPathConfig; + +#define _PyPathConfig_INIT {.module_search_path = NULL} +/* Note: _PyPathConfig_INIT sets other fields to 0/NULL */ + +PyAPI_DATA(_PyPathConfig) _Py_path_config; + +PyAPI_FUNC(_PyInitError) _PyPathConfig_Calculate( + _PyPathConfig *config, + const _PyCoreConfig *core_config); +PyAPI_FUNC(void) _PyPathConfig_Clear(_PyPathConfig *config); + + +/* interpreter state */ + +PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(PY_INT64_T); + + +/* cross-interpreter data */ + +struct _xid; + +// _PyCrossInterpreterData is similar to Py_buffer as an effectively +// opaque struct that holds data outside the object machinery. This +// is necessary to pass between interpreters in the same process. +typedef struct _xid { + // data is the cross-interpreter-safe derivation of a Python object + // (see _PyObject_GetCrossInterpreterData). It will be NULL if the + // new_object func (below) encodes the data. + void *data; + // obj is the Python object from which the data was derived. This + // is non-NULL only if the data remains bound to the object in some + // way, such that the object must be "released" (via a decref) when + // the data is released. In that case it is automatically + // incref'ed (to match the automatic decref when releaed). + PyObject *obj; + // interp is the ID of the owning interpreter of the original + // object. It corresponds to the active interpreter when + // _PyObject_GetCrossInterpreterData() was called. This should only + // be set by the cross-interpreter machinery. + // + // We use the ID rather than the PyInterpreterState to avoid issues + // with deleted interpreters. + int64_t interp; + // new_object is a function that returns a new object in the current + // interpreter given the data. The resulting object (a new + // reference) will be equivalent to the original object. This field + // is required. + PyObject *(*new_object)(struct _xid *); + // free is called when the data is released. If it is NULL then + // nothing will be done to free the data. For some types this is + // okay (e.g. bytes) and for those types this field should be set + // to NULL. However, for most the data was allocated just for + // cross-interpreter use, so it must be freed when + // _PyCrossInterpreterData_Release is called or the memory will + // leak. In that case, at the very least this field should be set + // to PyMem_RawFree (the default if not explicitly set to NULL). + // The call will happen with the original interpreter activated. + void (*free)(void *); +} _PyCrossInterpreterData; + +typedef int (*crossinterpdatafunc)(PyObject *, _PyCrossInterpreterData *); +PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); + +PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); +PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); +PyAPI_FUNC(void) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); + +/* cross-interpreter data registry */ + +/* For now we use a global registry of shareable classes. An + alternative would be to add a tp_* slot for a class's + crossinterpdatafunc. It would be simpler and more efficient. */ + +PyAPI_FUNC(int) _PyCrossInterpreterData_Register_Class(PyTypeObject *, crossinterpdatafunc); +PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); + +struct _xidregitem; + +struct _xidregitem { + PyTypeObject *cls; + crossinterpdatafunc getdata; + struct _xidregitem *next; +}; + + /* Full Python runtime state */ typedef struct pyruntimestate { @@ -58,15 +159,17 @@ typedef struct pyruntimestate { using a Python int. */ int64_t next_id; } interpreters; + // XXX Remove this field once we have a tp_* slot. + struct _xidregistry { + PyThread_type_lock mutex; + struct _xidregitem *head; + } xidregistry; #define NEXITFUNCS 32 void (*exitfuncs[NEXITFUNCS])(void); int nexitfuncs; - void (*pyexitfunc)(void); - struct _pyobj_runtime_state obj; struct _gc_runtime_state gc; - struct _pymem_runtime_state mem; struct _warnings_runtime_state warnings; struct _ceval_runtime_state ceval; struct _gilstate_runtime_state gilstate; @@ -75,6 +178,7 @@ typedef struct pyruntimestate { } _PyRuntimeState; #define _PyRuntimeState_INIT {.initialized = 0, .core_initialized = 0} +/* Note: _PyRuntimeState_INIT sets other fields to 0/NULL */ PyAPI_DATA(_PyRuntimeState) _PyRuntime; PyAPI_FUNC(_PyInitError) _PyRuntimeState_Init(_PyRuntimeState *); diff --git a/Include/longintrepr.h b/Include/longintrepr.h index a3b74b4f6dd..ff4155f9656 100644 --- a/Include/longintrepr.h +++ b/Include/longintrepr.h @@ -46,22 +46,22 @@ typedef uint32_t digit; typedef int32_t sdigit; /* signed variant of digit */ typedef uint64_t twodigits; typedef int64_t stwodigits; /* signed variant of twodigits */ -#define PyLong_SHIFT 30 -#define _PyLong_DECIMAL_SHIFT 9 /* max(e such that 10**e fits in a digit) */ -#define _PyLong_DECIMAL_BASE ((digit)1000000000) /* 10 ** DECIMAL_SHIFT */ +#define PyLong_SHIFT 30 +#define _PyLong_DECIMAL_SHIFT 9 /* max(e such that 10**e fits in a digit) */ +#define _PyLong_DECIMAL_BASE ((digit)1000000000) /* 10 ** DECIMAL_SHIFT */ #elif PYLONG_BITS_IN_DIGIT == 15 typedef unsigned short digit; typedef short sdigit; /* signed variant of digit */ typedef unsigned long twodigits; typedef long stwodigits; /* signed variant of twodigits */ -#define PyLong_SHIFT 15 -#define _PyLong_DECIMAL_SHIFT 4 /* max(e such that 10**e fits in a digit) */ -#define _PyLong_DECIMAL_BASE ((digit)10000) /* 10 ** DECIMAL_SHIFT */ +#define PyLong_SHIFT 15 +#define _PyLong_DECIMAL_SHIFT 4 /* max(e such that 10**e fits in a digit) */ +#define _PyLong_DECIMAL_BASE ((digit)10000) /* 10 ** DECIMAL_SHIFT */ #else #error "PYLONG_BITS_IN_DIGIT should be 15 or 30" #endif -#define PyLong_BASE ((digit)1 << PyLong_SHIFT) -#define PyLong_MASK ((digit)(PyLong_BASE - 1)) +#define PyLong_BASE ((digit)1 << PyLong_SHIFT) +#define PyLong_MASK ((digit)(PyLong_BASE - 1)) #if PyLong_SHIFT % 5 != 0 #error "longobject.c requires that PyLong_SHIFT be divisible by 5" @@ -69,12 +69,12 @@ typedef long stwodigits; /* signed variant of twodigits */ /* Long integer representation. The absolute value of a number is equal to - SUM(for i=0 through abs(ob_size)-1) ob_digit[i] * 2**(SHIFT*i) + SUM(for i=0 through abs(ob_size)-1) ob_digit[i] * 2**(SHIFT*i) Negative numbers are represented with ob_size < 0; zero is represented by ob_size == 0. In a normalized number, ob_digit[abs(ob_size)-1] (the most significant digit) is never zero. Also, in all cases, for all valid i, - 0 <= ob_digit[i] <= MASK. + 0 <= ob_digit[i] <= MASK. The allocation function takes care of allocating extra memory so that ob_digit[0] ... ob_digit[abs(ob_size)-1] are actually available. @@ -83,8 +83,8 @@ typedef long stwodigits; /* signed variant of twodigits */ */ struct _longobject { - PyObject_VAR_HEAD - digit ob_digit[1]; + PyObject_VAR_HEAD + digit ob_digit[1]; }; PyAPI_FUNC(PyLongObject *) _PyLong_New(Py_ssize_t); diff --git a/Include/methodobject.h b/Include/methodobject.h index 66e6a554b11..ea35d86bcd1 100644 --- a/Include/methodobject.h +++ b/Include/methodobject.h @@ -16,11 +16,11 @@ PyAPI_DATA(PyTypeObject) PyCFunction_Type; #define PyCFunction_Check(op) (Py_TYPE(op) == &PyCFunction_Type) typedef PyObject *(*PyCFunction)(PyObject *, PyObject *); -typedef PyObject *(*_PyCFunctionFast) (PyObject *, PyObject **, Py_ssize_t); +typedef PyObject *(*_PyCFunctionFast) (PyObject *, PyObject *const *, Py_ssize_t); typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *, PyObject *); typedef PyObject *(*_PyCFunctionFastWithKeywords) (PyObject *, - PyObject **, Py_ssize_t, + PyObject *const *, Py_ssize_t, PyObject *); typedef PyObject *(*PyNoArgsFunction)(PyObject *); @@ -43,12 +43,12 @@ PyAPI_FUNC(PyObject *) PyCFunction_Call(PyObject *, PyObject *, PyObject *); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyCFunction_FastCallDict(PyObject *func, - PyObject **args, + PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs); PyAPI_FUNC(PyObject *) _PyCFunction_FastCallKeywords(PyObject *func, - PyObject **stack, + PyObject *const *stack, Py_ssize_t nargs, PyObject *kwnames); #endif @@ -110,14 +110,14 @@ typedef struct { PyAPI_FUNC(PyObject *) _PyMethodDef_RawFastCallDict( PyMethodDef *method, PyObject *self, - PyObject **args, + PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs); PyAPI_FUNC(PyObject *) _PyMethodDef_RawFastCallKeywords( PyMethodDef *method, PyObject *self, - PyObject **args, + PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames); #endif diff --git a/Include/modsupport.h b/Include/modsupport.h index 73d86a94b95..a238bef03cb 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -52,7 +52,7 @@ PyAPI_FUNC(PyObject *) _Py_BuildValue_SizeT(const char *, ...); #ifndef Py_LIMITED_API PyAPI_FUNC(int) _PyArg_UnpackStack( - PyObject **args, + PyObject *const *args, Py_ssize_t nargs, const char *name, Py_ssize_t min, @@ -99,12 +99,12 @@ typedef struct _PyArg_Parser { PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywordsFast(PyObject *, PyObject *, struct _PyArg_Parser *, ...); PyAPI_FUNC(int) _PyArg_ParseStack( - PyObject **args, + PyObject *const *args, Py_ssize_t nargs, const char *format, ...); PyAPI_FUNC(int) _PyArg_ParseStackAndKeywords( - PyObject **args, + PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames, struct _PyArg_Parser *, diff --git a/Include/node.h b/Include/node.h index 654ad858230..40596dfecf1 100644 --- a/Include/node.h +++ b/Include/node.h @@ -8,12 +8,12 @@ extern "C" { #endif typedef struct _node { - short n_type; - char *n_str; - int n_lineno; - int n_col_offset; - int n_nchildren; - struct _node *n_child; + short n_type; + char *n_str; + int n_lineno; + int n_col_offset; + int n_nchildren; + struct _node *n_child; } node; PyAPI_FUNC(node *) PyNode_New(int type); @@ -25,12 +25,12 @@ PyAPI_FUNC(Py_ssize_t) _PyNode_SizeOf(node *n); #endif /* Node access functions */ -#define NCH(n) ((n)->n_nchildren) +#define NCH(n) ((n)->n_nchildren) -#define CHILD(n, i) (&(n)->n_child[i]) -#define RCHILD(n, i) (CHILD(n, NCH(n) + i)) -#define TYPE(n) ((n)->n_type) -#define STR(n) ((n)->n_str) +#define CHILD(n, i) (&(n)->n_child[i]) +#define RCHILD(n, i) (CHILD(n, NCH(n) + i)) +#define TYPE(n) ((n)->n_type) +#define STR(n) ((n)->n_str) #define LINENO(n) ((n)->n_lineno) /* Assert that the type of a node is what we expect */ diff --git a/Include/object.h b/Include/object.h index c65f948709f..c772deaf57d 100644 --- a/Include/object.h +++ b/Include/object.h @@ -539,6 +539,17 @@ PyAPI_FUNC(int) _PyObject_IsAbstract(PyObject *); PyAPI_FUNC(PyObject *) _PyObject_GetAttrId(PyObject *, struct _Py_Identifier *); PyAPI_FUNC(int) _PyObject_SetAttrId(PyObject *, struct _Py_Identifier *, PyObject *); PyAPI_FUNC(int) _PyObject_HasAttrId(PyObject *, struct _Py_Identifier *); +/* Replacements of PyObject_GetAttr() and _PyObject_GetAttrId() which + don't raise AttributeError. + + Return 1 and set *result != NULL if an attribute is found. + Return 0 and set *result == NULL if an attribute is not found; + an AttributeError is silenced. + Return -1 and set *result == NULL if an error other than AttributeError + is raised. +*/ +PyAPI_FUNC(int) _PyObject_LookupAttr(PyObject *, PyObject *, PyObject **); +PyAPI_FUNC(int) _PyObject_LookupAttrId(PyObject *, struct _Py_Identifier *, PyObject **); PyAPI_FUNC(PyObject **) _PyObject_GetDictPtr(PyObject *); #endif PyAPI_FUNC(PyObject *) PyObject_SelfIter(PyObject *); @@ -567,7 +578,7 @@ PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *); /* Same as PyObject_Generic{Get,Set}Attr, but passing the attributes dict as the last parameter. */ PyAPI_FUNC(PyObject *) -_PyObject_GenericGetAttrWithDict(PyObject *, PyObject *, PyObject *); +_PyObject_GenericGetAttrWithDict(PyObject *, PyObject *, PyObject *, int); PyAPI_FUNC(int) _PyObject_GenericSetAttrWithDict(PyObject *, PyObject *, PyObject *, PyObject *); @@ -728,7 +739,6 @@ PyAPI_FUNC(Py_ssize_t) _Py_GetRefTotal(void); /* Py_REF_DEBUG also controls the display of refcounts and memory block * allocations at the interactive prompt and at interpreter shutdown */ -PyAPI_FUNC(PyObject *) _PyDebug_XOptionShowRefCount(void); PyAPI_FUNC(void) _PyDebug_PrintTotalRefs(void); #else #define _Py_INC_REFTOTAL diff --git a/Include/objimpl.h b/Include/objimpl.h index 746f9c92134..057bb50cbda 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -56,7 +56,7 @@ must use the platform malloc heap(s), or shared memory, or C++ local storage or operator new), you must first allocate the object with your custom allocator, then pass its pointer to PyObject_{Init, InitVar} for filling in its Python- specific fields: reference count, type pointer, possibly others. You should -be aware that Python no control over these objects because they don't +be aware that Python has no control over these objects because they don't cooperate with the Python memory manager. Such objects may not be eligible for automatic garbage collection and you have to make sure that they are released accordingly whenever their destructor gets called (cf. the specific @@ -109,7 +109,7 @@ PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void); /* Macros */ #ifdef WITH_PYMALLOC #ifndef Py_LIMITED_API -PyAPI_FUNC(void) _PyObject_DebugMallocStats(FILE *out); +PyAPI_FUNC(int) _PyObject_DebugMallocStats(FILE *out); #endif /* #ifndef Py_LIMITED_API */ #endif diff --git a/Include/odictobject.h b/Include/odictobject.h index ff6ad64b8b4..8378dc4bfa4 100644 --- a/Include/odictobject.h +++ b/Include/odictobject.h @@ -6,6 +6,7 @@ extern "C" { /* OrderedDict */ +/* This API is optional and mostly redundant. */ #ifndef Py_LIMITED_API @@ -21,10 +22,6 @@ PyAPI_DATA(PyTypeObject) PyODictValues_Type; #define PyODict_CheckExact(op) (Py_TYPE(op) == &PyODict_Type) #define PyODict_SIZE(op) PyDict_GET_SIZE((op)) -#endif /* Py_LIMITED_API */ - -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000 - PyAPI_FUNC(PyObject *) PyODict_New(void); PyAPI_FUNC(int) PyODict_SetItem(PyObject *od, PyObject *key, PyObject *item); PyAPI_FUNC(int) PyODict_DelItem(PyObject *od, PyObject *key); diff --git a/Include/opcode.h b/Include/opcode.h index 99c3b0ef817..fc6cbf3a7af 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -99,7 +99,6 @@ extern "C" { #define LOAD_FAST 124 #define STORE_FAST 125 #define DELETE_FAST 126 -#define STORE_ANNOTATION 127 #define RAISE_VARARGS 130 #define CALL_FUNCTION 131 #define MAKE_FUNCTION 132 diff --git a/Include/parsetok.h b/Include/parsetok.h index 2acb8546727..c9407a3f702 100644 --- a/Include/parsetok.h +++ b/Include/parsetok.h @@ -21,13 +21,13 @@ typedef struct { } perrdetail; #if 0 -#define PyPARSE_YIELD_IS_KEYWORD 0x0001 +#define PyPARSE_YIELD_IS_KEYWORD 0x0001 #endif -#define PyPARSE_DONT_IMPLY_DEDENT 0x0002 +#define PyPARSE_DONT_IMPLY_DEDENT 0x0002 #if 0 -#define PyPARSE_WITH_IS_KEYWORD 0x0003 +#define PyPARSE_WITH_IS_KEYWORD 0x0003 #define PyPARSE_PRINT_IS_FUNCTION 0x0004 #define PyPARSE_UNICODE_LITERALS 0x0008 #endif diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 23e224d8c3d..68c29f58ab4 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -8,28 +8,28 @@ */ /* Values for PY_RELEASE_LEVEL */ -#define PY_RELEASE_LEVEL_ALPHA 0xA -#define PY_RELEASE_LEVEL_BETA 0xB -#define PY_RELEASE_LEVEL_GAMMA 0xC /* For release candidates */ -#define PY_RELEASE_LEVEL_FINAL 0xF /* Serial should be 0 here */ - /* Higher for patch releases */ +#define PY_RELEASE_LEVEL_ALPHA 0xA +#define PY_RELEASE_LEVEL_BETA 0xB +#define PY_RELEASE_LEVEL_GAMMA 0xC /* For release candidates */ +#define PY_RELEASE_LEVEL_FINAL 0xF /* Serial should be 0 here */ + /* Higher for patch releases */ /* Version parsed out into numeric values */ /*--start constants--*/ -#define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 7 -#define PY_MICRO_VERSION 0 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 2 +#define PY_MAJOR_VERSION 3 +#define PY_MINOR_VERSION 8 +#define PY_MICRO_VERSION 0 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA +#define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.7.0a2+" +#define PY_VERSION "3.8.0a0" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. Use this for numeric comparisons, e.g. #if PY_VERSION_HEX >= ... */ #define PY_VERSION_HEX ((PY_MAJOR_VERSION << 24) | \ - (PY_MINOR_VERSION << 16) | \ - (PY_MICRO_VERSION << 8) | \ - (PY_RELEASE_LEVEL << 4) | \ - (PY_RELEASE_SERIAL << 0)) + (PY_MINOR_VERSION << 16) | \ + (PY_MICRO_VERSION << 8) | \ + (PY_RELEASE_LEVEL << 4) | \ + (PY_RELEASE_SERIAL << 0)) diff --git a/Include/pgenheaders.h b/Include/pgenheaders.h index 4843de6c023..dbc5e0a5f13 100644 --- a/Include/pgenheaders.h +++ b/Include/pgenheaders.h @@ -10,9 +10,9 @@ extern "C" { #include "Python.h" PyAPI_FUNC(void) PySys_WriteStdout(const char *format, ...) - Py_GCC_ATTRIBUTE((format(printf, 1, 2))); + Py_GCC_ATTRIBUTE((format(printf, 1, 2))); PyAPI_FUNC(void) PySys_WriteStderr(const char *format, ...) - Py_GCC_ATTRIBUTE((format(printf, 1, 2))); + Py_GCC_ATTRIBUTE((format(printf, 1, 2))); #define addarc _Py_addarc #define addbit _Py_addbit diff --git a/Include/py_curses.h b/Include/py_curses.h index 597f419eaca..0eebc362a15 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -12,31 +12,15 @@ #endif #endif /* __APPLE__ */ -#ifdef __FreeBSD__ -/* -** On FreeBSD, [n]curses.h and stdlib.h/wchar.h use different guards -** against multiple definition of wchar_t and wint_t. -*/ -#ifdef _XOPEN_SOURCE_EXTENDED -#ifndef __FreeBSD_version -#include -#endif -#if __FreeBSD_version >= 500000 -#ifndef __wchar_t -#define __wchar_t -#endif -#ifndef __wint_t -#define __wint_t -#endif -#else -#ifndef _WCHAR_T -#define _WCHAR_T -#endif -#ifndef _WINT_T -#define _WINT_T -#endif -#endif -#endif +/* On FreeBSD, [n]curses.h and stdlib.h/wchar.h use different guards + against multiple definition of wchar_t and wint_t. */ +#if defined(__FreeBSD__) && defined(_XOPEN_SOURCE_EXTENDED) +# ifndef __wchar_t +# define __wchar_t +# endif +# ifndef __wint_t +# define __wint_t +# endif #endif #if !defined(HAVE_CURSES_IS_PAD) && defined(WINDOW_HAS_FLAGS) diff --git a/Include/pydebug.h b/Include/pydebug.h index d3b95966aa6..bd4aafe3b49 100644 --- a/Include/pydebug.h +++ b/Include/pydebug.h @@ -15,7 +15,6 @@ PyAPI_DATA(int) Py_InspectFlag; PyAPI_DATA(int) Py_OptimizeFlag; PyAPI_DATA(int) Py_NoSiteFlag; PyAPI_DATA(int) Py_BytesWarningFlag; -PyAPI_DATA(int) Py_UseClassExceptionsFlag; PyAPI_DATA(int) Py_FrozenFlag; PyAPI_DATA(int) Py_IgnoreEnvironmentFlag; PyAPI_DATA(int) Py_DontWriteBytecodeFlag; diff --git a/Include/pyfpe.h b/Include/pyfpe.h index f9a15e622ba..5a99e397931 100644 --- a/Include/pyfpe.h +++ b/Include/pyfpe.h @@ -1,176 +1,12 @@ #ifndef Py_PYFPE_H #define Py_PYFPE_H -#ifdef __cplusplus -extern "C" { -#endif -/* - --------------------------------------------------------------------- - / Copyright (c) 1996. \ - | The Regents of the University of California. | - | All rights reserved. | - | | - | Permission to use, copy, modify, and distribute this software for | - | any purpose without fee is hereby granted, provided that this en- | - | tire notice is included in all copies of any software which is or | - | includes a copy or modification of this software and in all | - | copies of the supporting documentation for such software. | - | | - | This work was produced at the University of California, Lawrence | - | Livermore National Laboratory under contract no. W-7405-ENG-48 | - | between the U.S. Department of Energy and The Regents of the | - | University of California for the operation of UC LLNL. | - | | - | DISCLAIMER | - | | - | This software was prepared as an account of work sponsored by an | - | agency of the United States Government. Neither the United States | - | Government nor the University of California nor any of their em- | - | ployees, makes any warranty, express or implied, or assumes any | - | liability or responsibility for the accuracy, completeness, or | - | usefulness of any information, apparatus, product, or process | - | disclosed, or represents that its use would not infringe | - | privately-owned rights. Reference herein to any specific commer- | - | cial products, process, or service by trade name, trademark, | - | manufacturer, or otherwise, does not necessarily constitute or | - | imply its endorsement, recommendation, or favoring by the United | - | States Government or the University of California. The views and | - | opinions of authors expressed herein do not necessarily state or | - | reflect those of the United States Government or the University | - | of California, and shall not be used for advertising or product | - \ endorsement purposes. / - --------------------------------------------------------------------- -*/ -/* - * Define macros for handling SIGFPE. - * Lee Busby, LLNL, November, 1996 - * busby1@llnl.gov - * - ********************************************* - * Overview of the system for handling SIGFPE: - * - * This file (Include/pyfpe.h) defines a couple of "wrapper" macros for - * insertion into your Python C code of choice. Their proper use is - * discussed below. The file Python/pyfpe.c defines a pair of global - * variables PyFPE_jbuf and PyFPE_counter which are used by the signal - * handler for SIGFPE to decide if a particular exception was protected - * by the macros. The signal handler itself, and code for enabling the - * generation of SIGFPE in the first place, is in a (new) Python module - * named fpectl. This module is standard in every respect. It can be loaded - * either statically or dynamically as you choose, and like any other - * Python module, has no effect until you import it. - * - * In the general case, there are three steps toward handling SIGFPE in any - * Python code: - * - * 1) Add the *_PROTECT macros to your C code as required to protect - * dangerous floating point sections. - * - * 2) Turn on the inclusion of the code by adding the ``--with-fpectl'' - * flag at the time you run configure. If the fpectl or other modules - * which use the *_PROTECT macros are to be dynamically loaded, be - * sure they are compiled with WANT_SIGFPE_HANDLER defined. - * - * 3) When python is built and running, import fpectl, and execute - * fpectl.turnon_sigfpe(). This sets up the signal handler and enables - * generation of SIGFPE whenever an exception occurs. From this point - * on, any properly trapped SIGFPE should result in the Python - * FloatingPointError exception. - * - * Step 1 has been done already for the Python kernel code, and should be - * done soon for the NumPy array package. Step 2 is usually done once at - * python install time. Python's behavior with respect to SIGFPE is not - * changed unless you also do step 3. Thus you can control this new - * facility at compile time, or run time, or both. - * - ******************************** - * Using the macros in your code: - * - * static PyObject *foobar(PyObject *self,PyObject *args) - * { - * .... - * PyFPE_START_PROTECT("Error in foobar", return 0) - * result = dangerous_op(somearg1, somearg2, ...); - * PyFPE_END_PROTECT(result) - * .... - * } - * - * If a floating point error occurs in dangerous_op, foobar returns 0 (NULL), - * after setting the associated value of the FloatingPointError exception to - * "Error in foobar". ``Dangerous_op'' can be a single operation, or a block - * of code, function calls, or any combination, so long as no alternate - * return is possible before the PyFPE_END_PROTECT macro is reached. - * - * The macros can only be used in a function context where an error return - * can be recognized as signaling a Python exception. (Generally, most - * functions that return a PyObject * will qualify.) - * - * Guido's original design suggestion for PyFPE_START_PROTECT and - * PyFPE_END_PROTECT had them open and close a local block, with a locally - * defined jmp_buf and jmp_buf pointer. This would allow recursive nesting - * of the macros. The Ansi C standard makes it clear that such local - * variables need to be declared with the "volatile" type qualifier to keep - * setjmp from corrupting their values. Some current implementations seem - * to be more restrictive. For example, the HPUX man page for setjmp says - * - * Upon the return from a setjmp() call caused by a longjmp(), the - * values of any non-static local variables belonging to the routine - * from which setjmp() was called are undefined. Code which depends on - * such values is not guaranteed to be portable. - * - * I therefore decided on a more limited form of nesting, using a counter - * variable (PyFPE_counter) to keep track of any recursion. If an exception - * occurs in an ``inner'' pair of macros, the return will apparently - * come from the outermost level. - * +/* These macros used to do something when Python was built with --with-fpectl, + * but support for that was dropped in 3.7. We continue to define them though, + * to avoid breaking API users. */ -#ifdef WANT_SIGFPE_HANDLER -#include -#include -#include -extern jmp_buf PyFPE_jbuf; -extern int PyFPE_counter; -extern double PyFPE_dummy(void *); - -#define PyFPE_START_PROTECT(err_string, leave_stmt) \ -if (!PyFPE_counter++ && setjmp(PyFPE_jbuf)) { \ - PyErr_SetString(PyExc_FloatingPointError, err_string); \ - PyFPE_counter = 0; \ - leave_stmt; \ -} - -/* - * This (following) is a heck of a way to decrement a counter. However, - * unless the macro argument is provided, code optimizers will sometimes move - * this statement so that it gets executed *before* the unsafe expression - * which we're trying to protect. That pretty well messes things up, - * of course. - * - * If the expression(s) you're trying to protect don't happen to return a - * value, you will need to manufacture a dummy result just to preserve the - * correct ordering of statements. Note that the macro passes the address - * of its argument (so you need to give it something which is addressable). - * If your expression returns multiple results, pass the last such result - * to PyFPE_END_PROTECT. - * - * Note that PyFPE_dummy returns a double, which is cast to int. - * This seeming insanity is to tickle the Floating Point Unit (FPU). - * If an exception has occurred in a preceding floating point operation, - * some architectures (notably Intel 80x86) will not deliver the interrupt - * until the *next* floating point operation. This is painful if you've - * already decremented PyFPE_counter. - */ -#define PyFPE_END_PROTECT(v) PyFPE_counter -= (int)PyFPE_dummy(&(v)); - -#else - #define PyFPE_START_PROTECT(err_string, leave_stmt) #define PyFPE_END_PROTECT(v) -#endif - -#ifdef __cplusplus -} -#endif #endif /* !Py_PYFPE_H */ diff --git a/Include/pygetopt.h b/Include/pygetopt.h deleted file mode 100644 index 962720c876c..00000000000 --- a/Include/pygetopt.h +++ /dev/null @@ -1,21 +0,0 @@ - -#ifndef Py_PYGETOPT_H -#define Py_PYGETOPT_H -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef Py_LIMITED_API -PyAPI_DATA(int) _PyOS_opterr; -PyAPI_DATA(int) _PyOS_optind; -PyAPI_DATA(wchar_t *) _PyOS_optarg; - -PyAPI_FUNC(void) _PyOS_ResetGetOpt(void); - -PyAPI_FUNC(int) _PyOS_GetOpt(int argc, wchar_t **argv, wchar_t *optstring); -#endif /* !Py_LIMITED_API */ - -#ifdef __cplusplus -} -#endif -#endif /* !Py_PYGETOPT_H */ diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index a75b77cc733..659c6df644e 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -7,19 +7,7 @@ extern "C" { #endif -PyAPI_FUNC(void) Py_SetProgramName(wchar_t *); -PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); - -PyAPI_FUNC(void) Py_SetPythonHome(wchar_t *); -PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); - #ifndef Py_LIMITED_API -/* Only used by applications that embed the interpreter and need to - * override the standard encoding determination mechanism - */ -PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding, - const char *errors); - typedef struct { const char *prefix; const char *msg; @@ -42,13 +30,44 @@ typedef struct { Don't abort() the process on such error. */ #define _Py_INIT_USER_ERR(MSG) \ (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 1} +#define _Py_INIT_NO_MEMORY() _Py_INIT_USER_ERR("memory allocation failed") #define _Py_INIT_FAILED(err) \ (err.msg != NULL) +#endif + + +PyAPI_FUNC(void) Py_SetProgramName(const wchar_t *); +PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); + +PyAPI_FUNC(void) Py_SetPythonHome(const wchar_t *); +PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void); + +#ifndef Py_LIMITED_API +/* Only used by applications that embed the interpreter and need to + * override the standard encoding determination mechanism + */ +PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding, + const char *errors); + /* PEP 432 Multi-phase initialization API (Private while provisional!) */ PyAPI_FUNC(_PyInitError) _Py_InitializeCore(const _PyCoreConfig *); PyAPI_FUNC(int) _Py_IsCoreInitialized(void); -PyAPI_FUNC(_PyInitError) _Py_ReadMainInterpreterConfig(_PyMainInterpreterConfig *); + +PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *); +PyAPI_FUNC(void) _PyCoreConfig_Clear(_PyCoreConfig *); +PyAPI_FUNC(int) _PyCoreConfig_Copy( + _PyCoreConfig *config, + const _PyCoreConfig *config2); + +PyAPI_FUNC(_PyInitError) _PyMainInterpreterConfig_Read( + _PyMainInterpreterConfig *config, + const _PyCoreConfig *core_config); +PyAPI_FUNC(void) _PyMainInterpreterConfig_Clear(_PyMainInterpreterConfig *); +PyAPI_FUNC(int) _PyMainInterpreterConfig_Copy( + _PyMainInterpreterConfig *config, + const _PyMainInterpreterConfig *config2); + PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *); #endif @@ -72,7 +91,7 @@ PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *); * exit functions. */ #ifndef Py_LIMITED_API -PyAPI_FUNC(void) _Py_PyAtExit(void (*func)(void)); +PyAPI_FUNC(void) _Py_PyAtExit(void (*func)(PyObject *), PyObject *); #endif PyAPI_FUNC(int) Py_AtExit(void (*func)(void)); @@ -87,15 +106,27 @@ PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *); /* Bootstrap __main__ (defined in Modules/main.c) */ PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv); +#ifdef Py_BUILD_CORE +PyAPI_FUNC(int) _Py_UnixMain(int argc, char **argv); +#endif /* In getpath.c */ PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void); PyAPI_FUNC(wchar_t *) Py_GetPrefix(void); PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void); PyAPI_FUNC(wchar_t *) Py_GetPath(void); +#ifdef Py_BUILD_CORE +PyAPI_FUNC(_PyInitError) _PyPathConfig_Init(const _PyCoreConfig *core_config); +PyAPI_FUNC(PyObject*) _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv); +PyAPI_FUNC(int) _Py_FindEnvConfigValue( + FILE *env_file, + const wchar_t *key, + wchar_t *value, + size_t value_size); +#endif PyAPI_FUNC(void) Py_SetPath(const wchar_t *); #ifdef MS_WINDOWS -int _Py_CheckPython3(); +int _Py_CheckPython3(void); #endif /* In their own files */ @@ -113,20 +144,35 @@ PyAPI_FUNC(const char *) _Py_gitversion(void); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyBuiltin_Init(void); PyAPI_FUNC(_PyInitError) _PySys_BeginInit(PyObject **sysmod); -PyAPI_FUNC(int) _PySys_EndInit(PyObject *sysdict); -PyAPI_FUNC(_PyInitError) _PyImport_Init(void); +PyAPI_FUNC(int) _PySys_EndInit(PyObject *sysdict, _PyMainInterpreterConfig *config); +PyAPI_FUNC(_PyInitError) _PyImport_Init(PyInterpreterState *interp); PyAPI_FUNC(void) _PyExc_Init(PyObject * bltinmod); PyAPI_FUNC(_PyInitError) _PyImportHooks_Init(void); PyAPI_FUNC(int) _PyFrame_Init(void); PyAPI_FUNC(int) _PyFloat_Init(void); PyAPI_FUNC(int) PyByteArray_Init(void); -PyAPI_FUNC(_PyInitError) _Py_HashRandomization_Init(_PyCoreConfig *core_config); +PyAPI_FUNC(_PyInitError) _Py_HashRandomization_Init(const _PyCoreConfig *); +#endif +#ifdef Py_BUILD_CORE +PyAPI_FUNC(int) _Py_ReadHashSeed( + const char *seed_text, + int *use_hash_seed, + unsigned long *hash_seed); #endif /* Various internal finalizers */ -#ifndef Py_LIMITED_API + +#ifdef Py_BUILD_CORE PyAPI_FUNC(void) _PyExc_Fini(void); PyAPI_FUNC(void) _PyImport_Fini(void); +PyAPI_FUNC(void) _PyImport_Fini2(void); +PyAPI_FUNC(void) _PyGC_DumpShutdownStats(void); +PyAPI_FUNC(void) _PyGC_Fini(void); +PyAPI_FUNC(void) _PyType_Fini(void); +PyAPI_FUNC(void) _Py_HashRandomization_Fini(void); +#endif /* Py_BUILD_CORE */ + +#ifndef Py_LIMITED_API PyAPI_FUNC(void) PyMethod_Fini(void); PyAPI_FUNC(void) PyFrame_Fini(void); PyAPI_FUNC(void) PyCFunction_Fini(void); @@ -138,15 +184,11 @@ PyAPI_FUNC(void) PyBytes_Fini(void); PyAPI_FUNC(void) PyByteArray_Fini(void); PyAPI_FUNC(void) PyFloat_Fini(void); PyAPI_FUNC(void) PyOS_FiniInterrupts(void); -PyAPI_FUNC(void) _PyGC_DumpShutdownStats(void); -PyAPI_FUNC(void) _PyGC_Fini(void); PyAPI_FUNC(void) PySlice_Fini(void); -PyAPI_FUNC(void) _PyType_Fini(void); -PyAPI_FUNC(void) _Py_HashRandomization_Fini(void); PyAPI_FUNC(void) PyAsyncGen_Fini(void); PyAPI_FUNC(int) _Py_IsFinalizing(void); -#endif +#endif /* !Py_LIMITED_API */ /* Signals */ typedef void (*PyOS_sighandler_t)(int); @@ -161,7 +203,7 @@ PyAPI_FUNC(int) _PyOS_URandomNonblock(void *buffer, Py_ssize_t size); /* Legacy locale support */ #ifndef Py_LIMITED_API -PyAPI_FUNC(void) _Py_CoerceLegacyLocale(void); +PyAPI_FUNC(void) _Py_CoerceLegacyLocale(const _PyCoreConfig *config); PyAPI_FUNC(int) _Py_LegacyLocaleDetected(void); PyAPI_FUNC(char *) _Py_SetLocaleFromEnv(int category); #endif diff --git a/Include/pymem.h b/Include/pymem.h index 928851a3d70..8ee0efddca7 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -21,9 +21,8 @@ PyAPI_FUNC(void) PyMem_RawFree(void *ptr); allocators. */ PyAPI_FUNC(int) _PyMem_SetupAllocators(const char *opt); -#ifdef WITH_PYMALLOC -PyAPI_FUNC(int) _PyMem_PymallocEnabled(void); -#endif +/* Try to get the allocators name set by _PyMem_SetupAllocators(). */ +PyAPI_FUNC(const char*) _PyMem_GetAllocatorsName(void); /* Track an allocated memory block in the tracemalloc module. Return 0 on success, return -1 on error (failed to allocate memory to store @@ -105,8 +104,14 @@ PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size); PyAPI_FUNC(void) PyMem_Free(void *ptr); #ifndef Py_LIMITED_API +/* strdup() using PyMem_RawMalloc() */ PyAPI_FUNC(char *) _PyMem_RawStrdup(const char *str); + +/* strdup() using PyMem_Malloc() */ PyAPI_FUNC(char *) _PyMem_Strdup(const char *str); + +/* wcsdup() using PyMem_RawMalloc() */ +PyAPI_FUNC(wchar_t*) _PyMem_RawWcsdup(const wchar_t *str); #endif /* Macros. */ @@ -224,7 +229,12 @@ PyAPI_FUNC(void) PyMem_SetupDebugHooks(void); #endif #ifdef Py_BUILD_CORE -PyAPI_FUNC(void) _PyMem_GetDefaultRawAllocator(PyMemAllocatorEx *alloc); +/* Set the memory allocator of the specified domain to the default. + Save the old allocator into *old_alloc if it's non-NULL. + Return on success, or return -1 if the domain is unknown. */ +PyAPI_FUNC(int) _PyMem_SetDefaultAllocator( + PyMemAllocatorDomain domain, + PyMemAllocatorEx *old_alloc); #endif #ifdef __cplusplus diff --git a/Include/pyport.h b/Include/pyport.h index 0e82543ac78..c1f4c7fbb52 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -564,18 +564,8 @@ extern char * _getpty(int *, int, mode_t, int); * workaround was provided by Tim Robbins of FreeBSD project. */ -#ifdef __FreeBSD__ -#include -#if (__FreeBSD_version >= 500040 && __FreeBSD_version < 602113) || \ - (__FreeBSD_version >= 700000 && __FreeBSD_version < 700054) || \ - (__FreeBSD_version >= 800000 && __FreeBSD_version < 800001) -# define _PY_PORT_CTYPE_UTF8_ISSUE -#endif -#endif - - #if defined(__APPLE__) -# define _PY_PORT_CTYPE_UTF8_ISSUE +# define _PY_PORT_CTYPE_UTF8_ISSUE #endif #ifdef _PY_PORT_CTYPE_UTF8_ISSUE @@ -784,7 +774,9 @@ extern _invalid_parameter_handler _Py_silent_invalid_parameter_handler; #endif /* Py_BUILD_CORE */ #ifdef __ANDROID__ -#include +/* The Android langinfo.h header is not used. */ +#undef HAVE_LANGINFO_H +#undef CODESET #endif /* Maximum value of the Windows DWORD type */ diff --git a/Include/pystate.h b/Include/pystate.h index 440122588f4..a19c1ed3b81 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -25,37 +25,85 @@ typedef PyObject* (*_PyFrameEvalFunction)(struct _frame *, int); typedef struct { - int ignore_environment; - int use_hash_seed; + int install_signal_handlers; /* Install signal handlers? -1 means unset */ + + int ignore_environment; /* -E, Py_IgnoreEnvironmentFlag */ + int use_hash_seed; /* PYTHONHASHSEED=x */ unsigned long hash_seed; + const char *allocator; /* Memory allocator: _PyMem_SetupAllocators() */ + int dev_mode; /* PYTHONDEVMODE, -X dev */ + int faulthandler; /* PYTHONFAULTHANDLER, -X faulthandler */ + int tracemalloc; /* PYTHONTRACEMALLOC, -X tracemalloc=N */ + int import_time; /* PYTHONPROFILEIMPORTTIME, -X importtime */ + int show_ref_count; /* -X showrefcount */ + int show_alloc_count; /* -X showalloccount */ + int dump_refs; /* PYTHONDUMPREFS */ + int malloc_stats; /* PYTHONMALLOCSTATS */ + int coerce_c_locale; /* PYTHONCOERCECLOCALE, -1 means unknown */ + int coerce_c_locale_warn; /* PYTHONCOERCECLOCALE=warn */ + int utf8_mode; /* PYTHONUTF8, -X utf8; -1 means unknown */ + + wchar_t *program_name; /* Program name, see also Py_GetProgramName() */ + int argc; /* Number of command line arguments, + -1 means unset */ + wchar_t **argv; /* Command line arguments */ + wchar_t *program; /* argv[0] or "" */ + + int nxoption; /* Number of -X options */ + wchar_t **xoptions; /* -X options */ + + int nwarnoption; /* Number of warnings options */ + wchar_t **warnoptions; /* Warnings options */ + + /* Path configuration inputs */ + wchar_t *module_search_path_env; /* PYTHONPATH environment variable */ + wchar_t *home; /* PYTHONHOME environment variable, + see also Py_SetPythonHome(). */ + + /* Path configuration outputs */ + int nmodule_search_path; /* Number of sys.path paths, + -1 means unset */ + wchar_t **module_search_paths; /* sys.path paths */ + wchar_t *executable; /* sys.executable */ + wchar_t *prefix; /* sys.prefix */ + wchar_t *base_prefix; /* sys.base_prefix */ + wchar_t *exec_prefix; /* sys.exec_prefix */ + wchar_t *base_exec_prefix; /* sys.base_exec_prefix */ + + /* Private fields */ int _disable_importlib; /* Needed by freeze_importlib */ - char *allocator; - int faulthandler; - int tracemalloc; /* Number of saved frames, 0=don't trace */ - int importtime; /* -X importtime */ } _PyCoreConfig; #define _PyCoreConfig_INIT \ - {.ignore_environment = 0, \ - .use_hash_seed = -1, \ - .hash_seed = 0, \ - ._disable_importlib = 0, \ - .allocator = NULL, \ - .faulthandler = 0, \ - .tracemalloc = 0, \ - .importtime = 0} + (_PyCoreConfig){ \ + .install_signal_handlers = -1, \ + .use_hash_seed = -1, \ + .coerce_c_locale = -1, \ + .utf8_mode = -1, \ + .argc = -1, \ + .nmodule_search_path = -1} +/* Note: _PyCoreConfig_INIT sets other fields to 0/NULL */ /* Placeholders while working on the new configuration API * * See PEP 432 for final anticipated contents - * - * For the moment, just handle the args to _Py_InitializeEx */ typedef struct { - int install_signal_handlers; + int install_signal_handlers; /* Install signal handlers? -1 means unset */ + PyObject *argv; /* sys.argv list, can be NULL */ + PyObject *executable; /* sys.executable str */ + PyObject *prefix; /* sys.prefix str */ + PyObject *base_prefix; /* sys.base_prefix str, can be NULL */ + PyObject *exec_prefix; /* sys.exec_prefix str */ + PyObject *base_exec_prefix; /* sys.base_exec_prefix str, can be NULL */ + PyObject *warnoptions; /* sys.warnoptions list, can be NULL */ + PyObject *xoptions; /* sys._xoptions dict, can be NULL */ + PyObject *module_search_path; /* sys.path list */ } _PyMainInterpreterConfig; -#define _PyMainInterpreterConfig_INIT {-1} +#define _PyMainInterpreterConfig_INIT \ + (_PyMainInterpreterConfig){.install_signal_handlers = -1} +/* Note: _PyMainInterpreterConfig_INIT sets other fields to 0/NULL */ typedef struct _is { @@ -106,8 +154,13 @@ typedef struct _is { PyObject *after_forkers_parent; PyObject *after_forkers_child; #endif + /* AtExit module */ + void (*pyexitfunc)(PyObject *); + PyObject *pyexitmodule; + + uint64_t tstate_next_unique_id; } PyInterpreterState; -#endif +#endif /* !Py_LIMITED_API */ /* State unique per thread */ @@ -129,7 +182,7 @@ typedef int (*Py_tracefunc)(PyObject *, struct _frame *, int, PyObject *); #define PyTrace_C_EXCEPTION 5 #define PyTrace_C_RETURN 6 #define PyTrace_OPCODE 7 -#endif +#endif /* Py_LIMITED_API */ #ifdef Py_LIMITED_API typedef struct _ts PyThreadState; @@ -225,16 +278,24 @@ typedef struct _ts { void (*on_delete)(void *); void *on_delete_data; + int coroutine_origin_tracking_depth; + PyObject *coroutine_wrapper; int in_coroutine_wrapper; PyObject *async_gen_firstiter; PyObject *async_gen_finalizer; + PyObject *context; + uint64_t context_ver; + + /* Unique thread state id. */ + uint64_t id; + /* XXX signal handlers should also be here */ } PyThreadState; -#endif +#endif /* !Py_LIMITED_API */ PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_New(void); @@ -359,7 +420,7 @@ PyAPI_FUNC(int) PyGILState_Check(void); Return NULL before _PyGILState_Init() is called and after _PyGILState_Fini() is called. */ PyAPI_FUNC(PyInterpreterState *) _PyGILState_GetInterpreterStateUnsafe(void); -#endif +#endif /* !Py_LIMITED_API */ /* The implementation of sys._current_frames() Returns a dict mapping diff --git a/Include/sliceobject.h b/Include/sliceobject.h index b24f2926f13..c238b099ea8 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -21,7 +21,7 @@ let these be any arbitrary python type. Py_None stands for omitted values. #ifndef Py_LIMITED_API typedef struct { PyObject_HEAD - PyObject *start, *stop, *step; /* not NULL */ + PyObject *start, *stop, *step; /* not NULL */ } PySliceObject; #endif diff --git a/Include/tupleobject.h b/Include/tupleobject.h index 98c26220ea0..72a7d8d5850 100644 --- a/Include/tupleobject.h +++ b/Include/tupleobject.h @@ -10,7 +10,7 @@ extern "C" { /* Another generally useful object type is a tuple of object pointers. For Python, this is an immutable type. C code can change the tuple items -(but not their number), and even use tuples are general-purpose arrays of +(but not their number), and even use tuples as general-purpose arrays of object references, but in general only brand new tuples should be mutated, not ones that might already have been exposed to Python code. diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 61e713be072..0274de6733a 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -1979,7 +1979,7 @@ PyAPI_FUNC(PyObject*) PyUnicode_Join( #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _PyUnicode_JoinArray( PyObject *separator, - PyObject **items, + PyObject *const *items, Py_ssize_t seqlen ); #endif /* Py_LIMITED_API */ diff --git a/Include/warnings.h b/Include/warnings.h index a3f83ff6967..a675bb5dfcb 100644 --- a/Include/warnings.h +++ b/Include/warnings.h @@ -56,6 +56,10 @@ PyErr_WarnExplicitFormat(PyObject *category, #define PyErr_Warn(category, msg) PyErr_WarnEx(category, msg, 1) #endif +#ifndef Py_LIMITED_API +void _PyErr_WarnUnawaitedCoroutine(PyObject *coro); +#endif + #ifdef __cplusplus } #endif diff --git a/LICENSE b/LICENSE index 529349e4b38..1afbedba92b 100644 --- a/LICENSE +++ b/LICENSE @@ -73,9 +73,9 @@ analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -2011, 2012, 2013, 2014, 2015, 2016, 2017 Python Software Foundation; All Rights -Reserved" are retained in Python alone or in any derivative version prepared by -Licensee. +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All +Rights Reserved" are retained in Python alone or in any derivative version +prepared by Licensee. 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and wants to make diff --git a/Lib/__future__.py b/Lib/__future__.py index 63b2be3524f..ce8bed7a643 100644 --- a/Lib/__future__.py +++ b/Lib/__future__.py @@ -57,13 +57,14 @@ all_feature_names = [ "unicode_literals", "barry_as_FLUFL", "generator_stop", + "annotations", ] __all__ = ["all_feature_names"] + all_feature_names -# The CO_xxx symbols are defined here under the same names used by -# compile.h, so that an editor search will find them here. However, -# they're not exported in __all__, because they don't really belong to +# The CO_xxx symbols are defined here under the same names defined in +# code.h and used by compile.h, so that an editor search will find them here. +# However, they're not exported in __all__, because they don't really belong to # this module. CO_NESTED = 0x0010 # nested_scopes CO_GENERATOR_ALLOWED = 0 # generators (obsolete, was 0x1000) @@ -74,6 +75,7 @@ CO_FUTURE_PRINT_FUNCTION = 0x10000 # print function CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals CO_FUTURE_BARRY_AS_BDFL = 0x40000 CO_FUTURE_GENERATOR_STOP = 0x80000 # StopIteration becomes RuntimeError in generators +CO_FUTURE_ANNOTATIONS = 0x100000 # annotations become strings at runtime class _Feature: def __init__(self, optionalRelease, mandatoryRelease, compiler_flag): @@ -132,9 +134,13 @@ unicode_literals = _Feature((2, 6, 0, "alpha", 2), CO_FUTURE_UNICODE_LITERALS) barry_as_FLUFL = _Feature((3, 1, 0, "alpha", 2), - (3, 9, 0, "alpha", 0), - CO_FUTURE_BARRY_AS_BDFL) + (3, 9, 0, "alpha", 0), + CO_FUTURE_BARRY_AS_BDFL) generator_stop = _Feature((3, 5, 0, "beta", 1), - (3, 7, 0, "alpha", 0), - CO_FUTURE_GENERATOR_STOP) + (3, 7, 0, "alpha", 0), + CO_FUTURE_GENERATOR_STOP) + +annotations = _Feature((3, 7, 0, "beta", 1), + (4, 0, 0, "alpha", 0), + CO_FUTURE_ANNOTATIONS) diff --git a/Lib/_bootlocale.py b/Lib/_bootlocale.py index 0c61b0d3a0f..3273a3b4225 100644 --- a/Lib/_bootlocale.py +++ b/Lib/_bootlocale.py @@ -9,6 +9,8 @@ import _locale if sys.platform.startswith("win"): def getpreferredencoding(do_setlocale=True): + if sys.flags.utf8_mode: + return 'UTF-8' return _locale._getdefaultlocale()[1] else: try: @@ -21,6 +23,8 @@ else: return 'UTF-8' else: def getpreferredencoding(do_setlocale=True): + if sys.flags.utf8_mode: + return 'UTF-8' # This path for legacy systems needs the more complex # getdefaultlocale() function, import the full locale module. import locale @@ -28,6 +32,8 @@ else: else: def getpreferredencoding(do_setlocale=True): assert not do_setlocale + if sys.flags.utf8_mode: + return 'UTF-8' result = _locale.nl_langinfo(_locale.CODESET) if not result and sys.platform == 'darwin': # nl_langinfo can return an empty string diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index e89e84bc081..dbe30dff1fe 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -746,7 +746,7 @@ class ItemsView(MappingView, Set): ItemsView.register(dict_items) -class ValuesView(MappingView): +class ValuesView(MappingView, Collection): __slots__ = () @@ -899,6 +899,9 @@ class Sequence(Reversible, Collection): def index(self, value, start=0, stop=None): '''S.index(value, [start, [stop]]) -> integer -- return first index of value. Raises ValueError if the value is not present. + + Supporting start and stop arguments is optional, but + recommended. ''' if start is not None and start < 0: start = max(len(self) + start, 0) diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index a1662bbd671..359690003fe 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -433,13 +433,11 @@ _rounding_modes = (ROUND_DOWN, ROUND_HALF_UP, ROUND_HALF_EVEN, ROUND_CEILING, # The getcontext() and setcontext() function manage access to a thread-local # current context. -import threading +import contextvars -local = threading.local() -if hasattr(local, '__decimal_context__'): - del local.__decimal_context__ +_current_context_var = contextvars.ContextVar('decimal_context') -def getcontext(_local=local): +def getcontext(): """Returns this thread's context. If this thread does not yet have a context, returns @@ -447,20 +445,20 @@ def getcontext(_local=local): New contexts are copies of DefaultContext. """ try: - return _local.__decimal_context__ - except AttributeError: + return _current_context_var.get() + except LookupError: context = Context() - _local.__decimal_context__ = context + _current_context_var.set(context) return context -def setcontext(context, _local=local): +def setcontext(context): """Set this thread's context to context.""" if context in (DefaultContext, BasicContext, ExtendedContext): context = context.copy() context.clear_flags() - _local.__decimal_context__ = context + _current_context_var.set(context) -del threading, local # Don't contaminate the namespace +del contextvars # Don't contaminate the namespace def localcontext(ctx=None): """Return a context manager for a copy of the supplied context diff --git a/Lib/_pyio.py b/Lib/_pyio.py index adf5d0ecbf6..c91a647a2f6 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -1182,7 +1182,6 @@ class BufferedWriter(_BufferedIOMixin): self.buffer_size = buffer_size self._write_buf = bytearray() self._write_lock = Lock() - _register_writer(self) def writable(self): return self.raw.writable() @@ -1939,10 +1938,7 @@ class TextIOWrapper(TextIOBase): # so that the signature can match the signature of the C version. def __init__(self, buffer, encoding=None, errors=None, newline=None, line_buffering=False, write_through=False): - if newline is not None and not isinstance(newline, str): - raise TypeError("illegal newline type: %r" % (type(newline),)) - if newline not in (None, "", "\n", "\r", "\r\n"): - raise ValueError("illegal newline value: %r" % (newline,)) + self._check_newline(newline) if encoding is None: try: encoding = os.device_encoding(buffer.fileno()) @@ -1972,22 +1968,38 @@ class TextIOWrapper(TextIOBase): raise ValueError("invalid errors: %r" % errors) self._buffer = buffer - self._encoding = encoding - self._errors = errors - self._readuniversal = not newline - self._readtranslate = newline is None - self._readnl = newline - self._writetranslate = newline != '' - self._writenl = newline or os.linesep - self._encoder = None - self._decoder = None self._decoded_chars = '' # buffer for text returned from decoder self._decoded_chars_used = 0 # offset into _decoded_chars for read() self._snapshot = None # info for reconstructing decoder state self._seekable = self._telling = self.buffer.seekable() self._has_read1 = hasattr(self.buffer, 'read1') + self._configure(encoding, errors, newline, + line_buffering, write_through) + + def _check_newline(self, newline): + if newline is not None and not isinstance(newline, str): + raise TypeError("illegal newline type: %r" % (type(newline),)) + if newline not in (None, "", "\n", "\r", "\r\n"): + raise ValueError("illegal newline value: %r" % (newline,)) + + def _configure(self, encoding=None, errors=None, newline=None, + line_buffering=False, write_through=False): + self._encoding = encoding + self._errors = errors + self._encoder = None + self._decoder = None self._b2cratio = 0.0 + self._readuniversal = not newline + self._readtranslate = newline is None + self._readnl = newline + self._writetranslate = newline != '' + self._writenl = newline or os.linesep + + self._line_buffering = line_buffering + self._write_through = write_through + + # don't write a BOM in the middle of a file if self._seekable and self.writable(): position = self.buffer.tell() if position != 0: @@ -1997,12 +2009,6 @@ class TextIOWrapper(TextIOBase): # Sometimes the encoder doesn't exist pass - self._configure(line_buffering, write_through) - - def _configure(self, line_buffering=False, write_through=False): - self._line_buffering = line_buffering - self._write_through = write_through - # self._snapshot is either None, or a tuple (dec_flags, next_input) # where dec_flags is the second (integer) item of the decoder state # and next_input is the chunk of input bytes that comes next after the @@ -2049,17 +2055,46 @@ class TextIOWrapper(TextIOBase): def buffer(self): return self._buffer - def reconfigure(self, *, line_buffering=None, write_through=None): + def reconfigure(self, *, + encoding=None, errors=None, newline=Ellipsis, + line_buffering=None, write_through=None): """Reconfigure the text stream with new parameters. This also flushes the stream. """ + if (self._decoder is not None + and (encoding is not None or errors is not None + or newline is not Ellipsis)): + raise UnsupportedOperation( + "It is not possible to set the encoding or newline of stream " + "after the first read") + + if errors is None: + if encoding is None: + errors = self._errors + else: + errors = 'strict' + elif not isinstance(errors, str): + raise TypeError("invalid errors: %r" % errors) + + if encoding is None: + encoding = self._encoding + else: + if not isinstance(encoding, str): + raise TypeError("invalid encoding: %r" % encoding) + + if newline is Ellipsis: + newline = self._readnl + self._check_newline(newline) + if line_buffering is None: line_buffering = self.line_buffering if write_through is None: write_through = self.write_through + self.flush() - self._configure(line_buffering, write_through) + self._configure(encoding, errors, newline, + line_buffering, write_through) def seekable(self): if self.closed: @@ -2587,26 +2622,3 @@ class StringIO(TextIOWrapper): def detach(self): # This doesn't make sense on StringIO. self._unsupported("detach") - - -# ____________________________________________________________ - -import atexit, weakref - -_all_writers = weakref.WeakSet() - -def _register_writer(w): - # keep weak-ref to buffered writer - _all_writers.add(w) - -def _flush_all_writers(): - # Ensure all buffered writers are flushed before proceeding with - # normal shutdown. Otherwise, if the underlying file objects get - # finalized before the buffered writer wrapping it then any buffered - # data will be lost. - for w in _all_writers: - try: - w.flush() - except: - pass -atexit.register(_flush_all_writers) diff --git a/Lib/_strptime.py b/Lib/_strptime.py index f5195af90c8..1be04850acf 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -470,7 +470,10 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): minutes = int(z[3:5]) seconds = int(z[5:7] or 0) gmtoff = (hours * 60 * 60) + (minutes * 60) + seconds - gmtoff_fraction = int(z[8:] or 0) + gmtoff_remainder = z[8:] + # Pad to always return microseconds. + gmtoff_remainder_padding = "0" * (6 - len(gmtoff_remainder)) + gmtoff_fraction = int(gmtoff_remainder + gmtoff_remainder_padding) if z.startswith("-"): gmtoff = -gmtoff gmtoff_fraction = -gmtoff_fraction diff --git a/Lib/abc.py b/Lib/abc.py index d13a0de89b4..9bdc36dce65 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -170,9 +170,11 @@ class ABCMeta(type): """Debug helper to print the ABC registry.""" print("Class: %s.%s" % (cls.__module__, cls.__qualname__), file=file) print("Inv.counter: %s" % ABCMeta._abc_invalidation_counter, file=file) - for name in sorted(cls.__dict__.keys()): + for name in cls.__dict__: if name.startswith("_abc_"): value = getattr(cls, name) + if isinstance(value, WeakSet): + value = set(value) print("%s: %r" % (name, value), file=file) def __instancecheck__(cls, instance): diff --git a/Lib/argparse.py b/Lib/argparse.py index d8bbd352fd3..e3da7f0443c 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -83,7 +83,6 @@ __all__ = [ ] -import collections as _collections import os as _os import re as _re import sys as _sys @@ -1084,7 +1083,7 @@ class _SubParsersAction(Action): self._prog_prefix = prog self._parser_class = parser_class - self._name_parser_map = _collections.OrderedDict() + self._name_parser_map = {} self._choices_actions = [] super(_SubParsersAction, self).__init__( diff --git a/Lib/ast.py b/Lib/ast.py index 070c2bee7f9..2ecb03f38bc 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -35,8 +35,6 @@ def parse(source, filename='', mode='exec'): return compile(source, filename, mode, PyCF_ONLY_AST) -_NUM_TYPES = (int, float, complex) - def literal_eval(node_or_string): """ Safely evaluate an expression node or a string containing a Python @@ -48,6 +46,21 @@ def literal_eval(node_or_string): node_or_string = parse(node_or_string, mode='eval') if isinstance(node_or_string, Expression): node_or_string = node_or_string.body + def _convert_num(node): + if isinstance(node, Constant): + if isinstance(node.value, (int, float, complex)): + return node.value + elif isinstance(node, Num): + return node.n + raise ValueError('malformed node or string: ' + repr(node)) + def _convert_signed_num(node): + if isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)): + operand = _convert_num(node.operand) + if isinstance(node.op, UAdd): + return + operand + else: + return - operand + return _convert_num(node) def _convert(node): if isinstance(node, Constant): return node.value @@ -62,26 +75,19 @@ def literal_eval(node_or_string): elif isinstance(node, Set): return set(map(_convert, node.elts)) elif isinstance(node, Dict): - return dict((_convert(k), _convert(v)) for k, v - in zip(node.keys, node.values)) + return dict(zip(map(_convert, node.keys), + map(_convert, node.values))) elif isinstance(node, NameConstant): return node.value - elif isinstance(node, UnaryOp) and isinstance(node.op, (UAdd, USub)): - operand = _convert(node.operand) - if isinstance(operand, _NUM_TYPES): - if isinstance(node.op, UAdd): - return + operand - else: - return - operand elif isinstance(node, BinOp) and isinstance(node.op, (Add, Sub)): - left = _convert(node.left) - right = _convert(node.right) - if isinstance(left, _NUM_TYPES) and isinstance(right, _NUM_TYPES): + left = _convert_signed_num(node.left) + right = _convert_num(node.right) + if isinstance(left, (int, float)) and isinstance(right, complex): if isinstance(node.op, Add): return left + right else: return left - right - raise ValueError('malformed node or string: ' + repr(node)) + return _convert_signed_num(node) return _convert(node_or_string) diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 011466b3e0d..23ea055912e 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -1,22 +1,9 @@ """The asyncio package, tracking PEP 3156.""" +# flake8: noqa + import sys -# The selectors module is in the stdlib in Python 3.4 but not in 3.3. -# Do this first, so the other submodules can use "from . import selectors". -# Prefer asyncio/selectors.py over the stdlib one, as ours may be newer. -try: - from . import selectors -except ImportError: - import selectors # Will also be exported. - -if sys.platform == 'win32': - # Similar thing for _overlapped. - try: - from . import _overlapped - except ImportError: - import _overlapped # Will also be exported. - # This relies on each of the submodules having an __all__ variable. from .base_events import * from .coroutines import * @@ -24,6 +11,7 @@ from .events import * from .futures import * from .locks import * from .protocols import * +from .runners import * from .queues import * from .streams import * from .subprocess import * @@ -36,6 +24,7 @@ __all__ = (base_events.__all__ + futures.__all__ + locks.__all__ + protocols.__all__ + + runners.__all__ + queues.__all__ + streams.__all__ + subprocess.__all__ + diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index bb04aef76e8..09eb440b0ef 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -29,15 +29,23 @@ import sys import warnings import weakref +try: + import ssl +except ImportError: # pragma: no cover + ssl = None + +from . import constants from . import coroutines from . import events from . import futures +from . import protocols +from . import sslproto from . import tasks -from .coroutines import coroutine +from . import transports from .log import logger -__all__ = ['BaseEventLoop'] +__all__ = 'BaseEventLoop', # Minimum number of _scheduled timer handles before cleanup of @@ -83,20 +91,6 @@ def _set_reuseport(sock): 'SO_REUSEPORT defined but not implemented.') -def _is_stream_socket(sock): - # Linux's socket.type is a bitmask that can include extra info - # about socket, therefore we can't do simple - # `sock_type == socket.SOCK_STREAM`. - return (sock.type & socket.SOCK_STREAM) == socket.SOCK_STREAM - - -def _is_dgram_socket(sock): - # Linux's socket.type is a bitmask that can include extra info - # about socket, therefore we can't do simple - # `sock_type == socket.SOCK_DGRAM`. - return (sock.type & socket.SOCK_DGRAM) == socket.SOCK_DGRAM - - def _ipaddr_info(host, port, family, type, proto): # Try to skip getaddrinfo if "host" is already an IP. Users might have # handled name resolution in their own code and pass in resolved IPs. @@ -108,11 +102,6 @@ def _ipaddr_info(host, port, family, type, proto): return None if type == socket.SOCK_STREAM: - # Linux only: - # getaddrinfo() can raise when socket.type is a bit mask. - # So if socket.type is a bit mask of SOCK_STREAM, and say - # SOCK_NONBLOCK, we simply return None, which will trigger - # a call to getaddrinfo() letting it process this request. proto = socket.IPPROTO_TCP elif type == socket.SOCK_DGRAM: proto = socket.IPPROTO_UDP @@ -158,59 +147,111 @@ def _ipaddr_info(host, port, family, type, proto): return None -def _ensure_resolved(address, *, family=0, type=socket.SOCK_STREAM, proto=0, - flags=0, loop): - host, port = address[:2] - info = _ipaddr_info(host, port, family, type, proto) - if info is not None: - # "host" is already a resolved IP. - fut = loop.create_future() - fut.set_result([info]) - return fut - else: - return loop.getaddrinfo(host, port, family=family, type=type, - proto=proto, flags=flags) - - def _run_until_complete_cb(fut): - exc = fut._exception - if (isinstance(exc, BaseException) - and not isinstance(exc, Exception)): - # Issue #22429: run_forever() already finished, no need to - # stop it. - return - fut._loop.stop() + if not fut.cancelled(): + exc = fut.exception() + if isinstance(exc, BaseException) and not isinstance(exc, Exception): + # Issue #22429: run_forever() already finished, no need to + # stop it. + return + futures._get_loop(fut).stop() + + + +class _SendfileFallbackProtocol(protocols.Protocol): + def __init__(self, transp): + if not isinstance(transp, transports._FlowControlMixin): + raise TypeError("transport should be _FlowControlMixin instance") + self._transport = transp + self._proto = transp.get_protocol() + self._should_resume_reading = transp.is_reading() + self._should_resume_writing = transp._protocol_paused + transp.pause_reading() + transp.set_protocol(self) + if self._should_resume_writing: + self._write_ready_fut = self._transport._loop.create_future() + else: + self._write_ready_fut = None + + async def drain(self): + if self._transport.is_closing(): + raise ConnectionError("Connection closed by peer") + fut = self._write_ready_fut + if fut is None: + return + await fut + + def connection_made(self, transport): + raise RuntimeError("Invalid state: " + "connection should have been established already.") + + def connection_lost(self, exc): + if self._write_ready_fut is not None: + # Never happens if peer disconnects after sending the whole content + # Thus disconnection is always an exception from user perspective + if exc is None: + self._write_ready_fut.set_exception( + ConnectionError("Connection is closed by peer")) + else: + self._write_ready_fut.set_exception(exc) + self._proto.connection_lost(exc) + + def pause_writing(self): + if self._write_ready_fut is not None: + return + self._write_ready_fut = self._transport._loop.create_future() + + def resume_writing(self): + if self._write_ready_fut is None: + return + self._write_ready_fut.set_result(False) + self._write_ready_fut = None + + def data_received(self, data): + raise RuntimeError("Invalid state: reading should be paused") + + def eof_received(self): + raise RuntimeError("Invalid state: reading should be paused") + + async def restore(self): + self._transport.set_protocol(self._proto) + if self._should_resume_reading: + self._transport.resume_reading() + if self._write_ready_fut is not None: + # Cancel the future. + # Basically it has no effect because protocol is switched back, + # no code should wait for it anymore. + self._write_ready_fut.cancel() + if self._should_resume_writing: + self._proto.resume_writing() class Server(events.AbstractServer): - def __init__(self, loop, sockets): + def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog, + ssl_handshake_timeout): self._loop = loop - self.sockets = sockets + self._sockets = sockets self._active_count = 0 self._waiters = [] + self._protocol_factory = protocol_factory + self._backlog = backlog + self._ssl_context = ssl_context + self._ssl_handshake_timeout = ssl_handshake_timeout + self._serving = False + self._serving_forever_fut = None def __repr__(self): - return '<%s sockets=%r>' % (self.__class__.__name__, self.sockets) + return f'<{self.__class__.__name__} sockets={self.sockets!r}>' def _attach(self): - assert self.sockets is not None + assert self._sockets is not None self._active_count += 1 def _detach(self): assert self._active_count > 0 self._active_count -= 1 - if self._active_count == 0 and self.sockets is None: - self._wakeup() - - def close(self): - sockets = self.sockets - if sockets is None: - return - self.sockets = None - for sock in sockets: - self._loop._stop_serving(sock) - if self._active_count == 0: + if self._active_count == 0 and self._sockets is None: self._wakeup() def _wakeup(self): @@ -220,13 +261,77 @@ class Server(events.AbstractServer): if not waiter.done(): waiter.set_result(waiter) - @coroutine - def wait_closed(self): - if self.sockets is None or self._waiters is None: + def _start_serving(self): + if self._serving: + return + self._serving = True + for sock in self._sockets: + sock.listen(self._backlog) + self._loop._start_serving( + self._protocol_factory, sock, self._ssl_context, + self, self._backlog, self._ssl_handshake_timeout) + + def get_loop(self): + return self._loop + + def is_serving(self): + return self._serving + + @property + def sockets(self): + if self._sockets is None: + return [] + return list(self._sockets) + + def close(self): + sockets = self._sockets + if sockets is None: + return + self._sockets = None + + for sock in sockets: + self._loop._stop_serving(sock) + + self._serving = False + + if (self._serving_forever_fut is not None and + not self._serving_forever_fut.done()): + self._serving_forever_fut.cancel() + self._serving_forever_fut = None + + if self._active_count == 0: + self._wakeup() + + async def start_serving(self): + self._start_serving() + + async def serve_forever(self): + if self._serving_forever_fut is not None: + raise RuntimeError( + f'server {self!r} is already being awaited on serve_forever()') + if self._sockets is None: + raise RuntimeError(f'server {self!r} is closed') + + self._start_serving() + self._serving_forever_fut = self._loop.create_future() + + try: + await self._serving_forever_fut + except futures.CancelledError: + try: + self.close() + await self.wait_closed() + finally: + raise + finally: + self._serving_forever_fut = None + + async def wait_closed(self): + if self._sockets is None or self._waiters is None: return waiter = self._loop.create_future() self._waiters.append(waiter) - yield from waiter + await waiter class BaseEventLoop(events.AbstractEventLoop): @@ -250,23 +355,20 @@ class BaseEventLoop(events.AbstractEventLoop): self.slow_callback_duration = 0.1 self._current_handle = None self._task_factory = None - self._coroutine_wrapper_set = False - - if hasattr(sys, 'get_asyncgen_hooks'): - # Python >= 3.6 - # A weak set of all asynchronous generators that are - # being iterated by the loop. - self._asyncgens = weakref.WeakSet() - else: - self._asyncgens = None + self._coroutine_origin_tracking_enabled = False + self._coroutine_origin_tracking_saved_depth = None + # A weak set of all asynchronous generators that are + # being iterated by the loop. + self._asyncgens = weakref.WeakSet() # Set to True when `loop.shutdown_asyncgens` is called. self._asyncgens_shutdown_called = False def __repr__(self): - return ('<%s running=%s closed=%s debug=%s>' - % (self.__class__.__name__, self.is_running(), - self.is_closed(), self.get_debug())) + return ( + f'<{self.__class__.__name__} running={self.is_running()} ' + f'closed={self.is_closed()} debug={self.get_debug()}>' + ) def create_future(self): """Create a Future object attached to the loop.""" @@ -309,9 +411,12 @@ class BaseEventLoop(events.AbstractEventLoop): """Create socket transport.""" raise NotImplementedError - def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter=None, - *, server_side=False, server_hostname=None, - extra=None, server=None): + def _make_ssl_transport( + self, rawsock, protocol, sslcontext, waiter=None, + *, server_side=False, server_hostname=None, + extra=None, server=None, + ssl_handshake_timeout=None, + call_connection_made=True): """Create SSL transport.""" raise NotImplementedError @@ -330,10 +435,9 @@ class BaseEventLoop(events.AbstractEventLoop): """Create write pipe transport.""" raise NotImplementedError - @coroutine - def _make_subprocess_transport(self, protocol, args, shell, - stdin, stdout, stderr, bufsize, - extra=None, **kwargs): + async def _make_subprocess_transport(self, protocol, args, shell, + stdin, stdout, stderr, bufsize, + extra=None, **kwargs): """Create subprocess transport.""" raise NotImplementedError @@ -365,18 +469,17 @@ class BaseEventLoop(events.AbstractEventLoop): def _asyncgen_firstiter_hook(self, agen): if self._asyncgens_shutdown_called: warnings.warn( - "asynchronous generator {!r} was scheduled after " - "loop.shutdown_asyncgens() call".format(agen), + f"asynchronous generator {agen!r} was scheduled after " + f"loop.shutdown_asyncgens() call", ResourceWarning, source=self) self._asyncgens.add(agen) - @coroutine - def shutdown_asyncgens(self): + async def shutdown_asyncgens(self): """Shutdown all active asynchronous generators.""" self._asyncgens_shutdown_called = True - if self._asyncgens is None or not len(self._asyncgens): + if not len(self._asyncgens): # If Python version is <3.6 or we don't have any asynchronous # generators alive. return @@ -384,17 +487,16 @@ class BaseEventLoop(events.AbstractEventLoop): closing_agens = list(self._asyncgens) self._asyncgens.clear() - shutdown_coro = tasks.gather( + results = await tasks.gather( *[ag.aclose() for ag in closing_agens], return_exceptions=True, loop=self) - results = yield from shutdown_coro for result, agen in zip(results, closing_agens): if isinstance(result, Exception): self.call_exception_handler({ - 'message': 'an error occurred during closing of ' - 'asynchronous generator {!r}'.format(agen), + 'message': f'an error occurred during closing of ' + f'asynchronous generator {agen!r}', 'exception': result, 'asyncgen': agen }) @@ -407,12 +509,12 @@ class BaseEventLoop(events.AbstractEventLoop): if events._get_running_loop() is not None: raise RuntimeError( 'Cannot run the event loop while another loop is running') - self._set_coroutine_wrapper(self._debug) + self._set_coroutine_origin_tracking(self._debug) self._thread_id = threading.get_ident() - if self._asyncgens is not None: - old_agen_hooks = sys.get_asyncgen_hooks() - sys.set_asyncgen_hooks(firstiter=self._asyncgen_firstiter_hook, - finalizer=self._asyncgen_finalizer_hook) + + old_agen_hooks = sys.get_asyncgen_hooks() + sys.set_asyncgen_hooks(firstiter=self._asyncgen_firstiter_hook, + finalizer=self._asyncgen_finalizer_hook) try: events._set_running_loop(self) while True: @@ -423,9 +525,8 @@ class BaseEventLoop(events.AbstractEventLoop): self._stopping = False self._thread_id = None events._set_running_loop(None) - self._set_coroutine_wrapper(False) - if self._asyncgens is not None: - sys.set_asyncgen_hooks(*old_agen_hooks) + self._set_coroutine_origin_tracking(False) + sys.set_asyncgen_hooks(*old_agen_hooks) def run_until_complete(self, future): """Run until the Future is done. @@ -500,7 +601,7 @@ class BaseEventLoop(events.AbstractEventLoop): def __del__(self): if not self.is_closed(): - warnings.warn("unclosed event loop %r" % self, ResourceWarning, + warnings.warn(f"unclosed event loop {self!r}", ResourceWarning, source=self) if not self.is_running(): self.close() @@ -518,7 +619,7 @@ class BaseEventLoop(events.AbstractEventLoop): """ return time.monotonic() - def call_later(self, delay, callback, *args): + def call_later(self, delay, callback, *args, context=None): """Arrange for a callback to be called at a given time. Return a Handle: an opaque object with a cancel() method that @@ -534,12 +635,13 @@ class BaseEventLoop(events.AbstractEventLoop): Any positional arguments after the callback will be passed to the callback when it is called. """ - timer = self.call_at(self.time() + delay, callback, *args) + timer = self.call_at(self.time() + delay, callback, *args, + context=context) if timer._source_traceback: del timer._source_traceback[-1] return timer - def call_at(self, when, callback, *args): + def call_at(self, when, callback, *args, context=None): """Like call_later(), but uses an absolute time. Absolute time corresponds to the event loop's time() method. @@ -548,14 +650,14 @@ class BaseEventLoop(events.AbstractEventLoop): if self._debug: self._check_thread() self._check_callback(callback, 'call_at') - timer = events.TimerHandle(when, callback, args, self) + timer = events.TimerHandle(when, callback, args, self, context) if timer._source_traceback: del timer._source_traceback[-1] heapq.heappush(self._scheduled, timer) timer._scheduled = True return timer - def call_soon(self, callback, *args): + def call_soon(self, callback, *args, context=None): """Arrange for a callback to be called as soon as possible. This operates as a FIFO queue: callbacks are called in the @@ -569,7 +671,7 @@ class BaseEventLoop(events.AbstractEventLoop): if self._debug: self._check_thread() self._check_callback(callback, 'call_soon') - handle = self._call_soon(callback, args) + handle = self._call_soon(callback, args, context) if handle._source_traceback: del handle._source_traceback[-1] return handle @@ -578,15 +680,14 @@ class BaseEventLoop(events.AbstractEventLoop): if (coroutines.iscoroutine(callback) or coroutines.iscoroutinefunction(callback)): raise TypeError( - "coroutines cannot be used with {}()".format(method)) + f"coroutines cannot be used with {method}()") if not callable(callback): raise TypeError( - 'a callable object was expected by {}(), got {!r}'.format( - method, callback)) + f'a callable object was expected by {method}(), ' + f'got {callback!r}') - - def _call_soon(self, callback, args): - handle = events.Handle(callback, args, self) + def _call_soon(self, callback, args, context): + handle = events.Handle(callback, args, self, context) if handle._source_traceback: del handle._source_traceback[-1] self._ready.append(handle) @@ -609,12 +710,12 @@ class BaseEventLoop(events.AbstractEventLoop): "Non-thread-safe operation invoked on an event loop other " "than the current one") - def call_soon_threadsafe(self, callback, *args): + def call_soon_threadsafe(self, callback, *args, context=None): """Like call_soon(), but thread-safe.""" self._check_closed() if self._debug: self._check_callback(callback, 'call_soon_threadsafe') - handle = self._call_soon(callback, args) + handle = self._call_soon(callback, args, context) if handle._source_traceback: del handle._source_traceback[-1] self._write_to_self() @@ -629,21 +730,22 @@ class BaseEventLoop(events.AbstractEventLoop): if executor is None: executor = concurrent.futures.ThreadPoolExecutor() self._default_executor = executor - return futures.wrap_future(executor.submit(func, *args), loop=self) + return futures.wrap_future( + executor.submit(func, *args), loop=self) def set_default_executor(self, executor): self._default_executor = executor def _getaddrinfo_debug(self, host, port, family, type, proto, flags): - msg = ["%s:%r" % (host, port)] + msg = [f"{host}:{port!r}"] if family: - msg.append('family=%r' % family) + msg.append(f'family={family!r}') if type: - msg.append('type=%r' % type) + msg.append(f'type={type!r}') if proto: - msg.append('proto=%r' % proto) + msg.append(f'proto={proto!r}') if flags: - msg.append('flags=%r' % flags) + msg.append(f'flags={flags!r}') msg = ', '.join(msg) logger.debug('Get address info %s', msg) @@ -651,30 +753,98 @@ class BaseEventLoop(events.AbstractEventLoop): addrinfo = socket.getaddrinfo(host, port, family, type, proto, flags) dt = self.time() - t0 - msg = ('Getting address info %s took %.3f ms: %r' - % (msg, dt * 1e3, addrinfo)) + msg = f'Getting address info {msg} took {dt * 1e3:.3f}ms: {addrinfo!r}' if dt >= self.slow_callback_duration: logger.info(msg) else: logger.debug(msg) return addrinfo - def getaddrinfo(self, host, port, *, - family=0, type=0, proto=0, flags=0): + async def getaddrinfo(self, host, port, *, + family=0, type=0, proto=0, flags=0): if self._debug: - return self.run_in_executor(None, self._getaddrinfo_debug, - host, port, family, type, proto, flags) + getaddr_func = self._getaddrinfo_debug else: - return self.run_in_executor(None, socket.getaddrinfo, - host, port, family, type, proto, flags) + getaddr_func = socket.getaddrinfo - def getnameinfo(self, sockaddr, flags=0): - return self.run_in_executor(None, socket.getnameinfo, sockaddr, flags) + return await self.run_in_executor( + None, getaddr_func, host, port, family, type, proto, flags) - @coroutine - def create_connection(self, protocol_factory, host=None, port=None, *, - ssl=None, family=0, proto=0, flags=0, sock=None, - local_addr=None, server_hostname=None): + async def getnameinfo(self, sockaddr, flags=0): + return await self.run_in_executor( + None, socket.getnameinfo, sockaddr, flags) + + async def sock_sendfile(self, sock, file, offset=0, count=None, + *, fallback=True): + if self._debug and sock.gettimeout() != 0: + raise ValueError("the socket must be non-blocking") + self._check_sendfile_params(sock, file, offset, count) + try: + return await self._sock_sendfile_native(sock, file, + offset, count) + except events.SendfileNotAvailableError as exc: + if not fallback: + raise + return await self._sock_sendfile_fallback(sock, file, + offset, count) + + async def _sock_sendfile_native(self, sock, file, offset, count): + # NB: sendfile syscall is not supported for SSL sockets and + # non-mmap files even if sendfile is supported by OS + raise events.SendfileNotAvailableError( + f"syscall sendfile is not available for socket {sock!r} " + "and file {file!r} combination") + + async def _sock_sendfile_fallback(self, sock, file, offset, count): + if offset: + file.seek(offset) + blocksize = min(count, 16384) if count else 16384 + buf = bytearray(blocksize) + total_sent = 0 + try: + while True: + if count: + blocksize = min(count - total_sent, blocksize) + if blocksize <= 0: + break + view = memoryview(buf)[:blocksize] + read = file.readinto(view) + if not read: + break # EOF + await self.sock_sendall(sock, view) + total_sent += read + return total_sent + finally: + if total_sent > 0 and hasattr(file, 'seek'): + file.seek(offset + total_sent) + + def _check_sendfile_params(self, sock, file, offset, count): + if 'b' not in getattr(file, 'mode', 'b'): + raise ValueError("file should be opened in binary mode") + if not sock.type == socket.SOCK_STREAM: + raise ValueError("only SOCK_STREAM type sockets are supported") + if count is not None: + if not isinstance(count, int): + raise TypeError( + "count must be a positive integer (got {!r})".format(count)) + if count <= 0: + raise ValueError( + "count must be a positive integer (got {!r})".format(count)) + if not isinstance(offset, int): + raise TypeError( + "offset must be a non-negative integer (got {!r})".format( + offset)) + if offset < 0: + raise ValueError( + "offset must be a non-negative integer (got {!r})".format( + offset)) + + async def create_connection( + self, protocol_factory, host=None, port=None, + *, ssl=None, family=0, + proto=0, flags=0, sock=None, + local_addr=None, server_hostname=None, + ssl_handshake_timeout=None): """Connect to a TCP server. Create a streaming transport connection to a given Internet host and @@ -705,30 +875,26 @@ class BaseEventLoop(events.AbstractEventLoop): 'when using ssl without a host') server_hostname = host + if ssl_handshake_timeout is not None and not ssl: + raise ValueError( + 'ssl_handshake_timeout is only meaningful with ssl') + if host is not None or port is not None: if sock is not None: raise ValueError( 'host/port and sock can not be specified at the same time') - f1 = _ensure_resolved((host, port), family=family, - type=socket.SOCK_STREAM, proto=proto, - flags=flags, loop=self) - fs = [f1] - if local_addr is not None: - f2 = _ensure_resolved(local_addr, family=family, - type=socket.SOCK_STREAM, proto=proto, - flags=flags, loop=self) - fs.append(f2) - else: - f2 = None - - yield from tasks.wait(fs, loop=self) - - infos = f1.result() + infos = await self._ensure_resolved( + (host, port), family=family, + type=socket.SOCK_STREAM, proto=proto, flags=flags, loop=self) if not infos: raise OSError('getaddrinfo() returned empty list') - if f2 is not None: - laddr_infos = f2.result() + + if local_addr is not None: + laddr_infos = await self._ensure_resolved( + local_addr, family=family, + type=socket.SOCK_STREAM, proto=proto, + flags=flags, loop=self) if not laddr_infos: raise OSError('getaddrinfo() returned empty list') @@ -737,17 +903,18 @@ class BaseEventLoop(events.AbstractEventLoop): try: sock = socket.socket(family=family, type=type, proto=proto) sock.setblocking(False) - if f2 is not None: + if local_addr is not None: for _, _, _, _, laddr in laddr_infos: try: sock.bind(laddr) break except OSError as exc: - exc = OSError( - exc.errno, 'error while ' - 'attempting to bind on address ' - '{!r}: {}'.format( - laddr, exc.strerror.lower())) + msg = ( + f'error while attempting to bind on ' + f'address {laddr!r}: ' + f'{exc.strerror.lower()}' + ) + exc = OSError(exc.errno, msg) exceptions.append(exc) else: sock.close() @@ -755,7 +922,7 @@ class BaseEventLoop(events.AbstractEventLoop): continue if self._debug: logger.debug("connect %r to %r", sock, address) - yield from self.sock_connect(sock, address) + await self.sock_connect(sock, address) except OSError as exc: if sock is not None: sock.close() @@ -783,7 +950,7 @@ class BaseEventLoop(events.AbstractEventLoop): if sock is None: raise ValueError( 'host and port was not specified and no sock specified') - if not _is_stream_socket(sock): + if sock.type != socket.SOCK_STREAM: # We allow AF_INET, AF_INET6, AF_UNIX as long as they # are SOCK_STREAM. # We support passing AF_UNIX sockets even though we have @@ -791,10 +958,11 @@ class BaseEventLoop(events.AbstractEventLoop): # Disallowing AF_UNIX in this method, breaks backwards # compatibility. raise ValueError( - 'A Stream Socket was expected, got {!r}'.format(sock)) + f'A Stream Socket was expected, got {sock!r}') - transport, protocol = yield from self._create_connection_transport( - sock, protocol_factory, ssl, server_hostname) + transport, protocol = await self._create_connection_transport( + sock, protocol_factory, ssl, server_hostname, + ssl_handshake_timeout=ssl_handshake_timeout) if self._debug: # Get the socket from the transport because SSL transport closes # the old socket and creates a new SSL socket @@ -803,9 +971,10 @@ class BaseEventLoop(events.AbstractEventLoop): sock, host, port, transport, protocol) return transport, protocol - @coroutine - def _create_connection_transport(self, sock, protocol_factory, ssl, - server_hostname, server_side=False): + async def _create_connection_transport( + self, sock, protocol_factory, ssl, + server_hostname, server_side=False, + ssl_handshake_timeout=None): sock.setblocking(False) @@ -815,29 +984,141 @@ class BaseEventLoop(events.AbstractEventLoop): sslcontext = None if isinstance(ssl, bool) else ssl transport = self._make_ssl_transport( sock, protocol, sslcontext, waiter, - server_side=server_side, server_hostname=server_hostname) + server_side=server_side, server_hostname=server_hostname, + ssl_handshake_timeout=ssl_handshake_timeout) else: transport = self._make_socket_transport(sock, protocol, waiter) try: - yield from waiter + await waiter except: transport.close() raise return transport, protocol - @coroutine - def create_datagram_endpoint(self, protocol_factory, - local_addr=None, remote_addr=None, *, - family=0, proto=0, flags=0, - reuse_address=None, reuse_port=None, - allow_broadcast=None, sock=None): + async def sendfile(self, transport, file, offset=0, count=None, + *, fallback=True): + """Send a file to transport. + + Return the total number of bytes which were sent. + + The method uses high-performance os.sendfile if available. + + file must be a regular file object opened in binary mode. + + offset tells from where to start reading the file. If specified, + count is the total number of bytes to transmit as opposed to + sending the file until EOF is reached. File position is updated on + return or also in case of error in which case file.tell() + can be used to figure out the number of bytes + which were sent. + + fallback set to True makes asyncio to manually read and send + the file when the platform does not support the sendfile syscall + (e.g. Windows or SSL socket on Unix). + + Raise SendfileNotAvailableError if the system does not support + sendfile syscall and fallback is False. + """ + if transport.is_closing(): + raise RuntimeError("Transport is closing") + mode = getattr(transport, '_sendfile_compatible', + constants._SendfileMode.UNSUPPORTED) + if mode is constants._SendfileMode.UNSUPPORTED: + raise RuntimeError( + f"sendfile is not supported for transport {transport!r}") + if mode is constants._SendfileMode.TRY_NATIVE: + try: + return await self._sendfile_native(transport, file, + offset, count) + except events.SendfileNotAvailableError as exc: + if not fallback: + raise + + if not fallback: + raise RuntimeError( + f"fallback is disabled and native sendfile is not " + f"supported for transport {transport!r}") + + return await self._sendfile_fallback(transport, file, + offset, count) + + async def _sendfile_native(self, transp, file, offset, count): + raise events.SendfileNotAvailableError( + "sendfile syscall is not supported") + + async def _sendfile_fallback(self, transp, file, offset, count): + if offset: + file.seek(offset) + blocksize = min(count, 16384) if count else 16384 + buf = bytearray(blocksize) + total_sent = 0 + proto = _SendfileFallbackProtocol(transp) + try: + while True: + if count: + blocksize = min(count - total_sent, blocksize) + if blocksize <= 0: + return total_sent + view = memoryview(buf)[:blocksize] + read = file.readinto(view) + if not read: + return total_sent # EOF + await proto.drain() + transp.write(view) + total_sent += read + finally: + if total_sent > 0 and hasattr(file, 'seek'): + file.seek(offset + total_sent) + await proto.restore() + + async def start_tls(self, transport, protocol, sslcontext, *, + server_side=False, + server_hostname=None, + ssl_handshake_timeout=None): + """Upgrade transport to TLS. + + Return a new transport that *protocol* should start using + immediately. + """ + if ssl is None: + raise RuntimeError('Python ssl module is not available') + + if not isinstance(sslcontext, ssl.SSLContext): + raise TypeError( + f'sslcontext is expected to be an instance of ssl.SSLContext, ' + f'got {sslcontext!r}') + + if not getattr(transport, '_start_tls_compatible', False): + raise TypeError( + f'transport {self!r} is not supported by start_tls()') + + waiter = self.create_future() + ssl_protocol = sslproto.SSLProtocol( + self, protocol, sslcontext, waiter, + server_side, server_hostname, + ssl_handshake_timeout=ssl_handshake_timeout, + call_connection_made=False) + + transport.set_protocol(ssl_protocol) + self.call_soon(ssl_protocol.connection_made, transport) + if not transport.is_reading(): + self.call_soon(transport.resume_reading) + + await waiter + return ssl_protocol._app_transport + + async def create_datagram_endpoint(self, protocol_factory, + local_addr=None, remote_addr=None, *, + family=0, proto=0, flags=0, + reuse_address=None, reuse_port=None, + allow_broadcast=None, sock=None): """Create datagram connection.""" if sock is not None: - if not _is_dgram_socket(sock): + if sock.type != socket.SOCK_DGRAM: raise ValueError( - 'A UDP Socket was expected, got {!r}'.format(sock)) + f'A UDP Socket was expected, got {sock!r}') if (local_addr or remote_addr or family or proto or flags or reuse_address or reuse_port or allow_broadcast): @@ -846,11 +1127,10 @@ class BaseEventLoop(events.AbstractEventLoop): family=family, proto=proto, flags=flags, reuse_address=reuse_address, reuse_port=reuse_port, allow_broadcast=allow_broadcast) - problems = ', '.join( - '{}={}'.format(k, v) for k, v in opts.items() if v) + problems = ', '.join(f'{k}={v}' for k, v in opts.items() if v) raise ValueError( - 'socket modifier keyword arguments can not be used ' - 'when sock is specified. ({})'.format(problems)) + f'socket modifier keyword arguments can not be used ' + f'when sock is specified. ({problems})') sock.setblocking(False) r_addr = None else: @@ -860,7 +1140,7 @@ class BaseEventLoop(events.AbstractEventLoop): addr_pairs_info = (((family, proto), (None, None)),) elif hasattr(socket, 'AF_UNIX') and family == socket.AF_UNIX: for addr in (local_addr, remote_addr): - if addr is not None and not isistance(addr, str): + if addr is not None and not isinstance(addr, str): raise TypeError('string is expected') addr_pairs_info = (((family, proto), (local_addr, remote_addr)), ) @@ -872,7 +1152,7 @@ class BaseEventLoop(events.AbstractEventLoop): assert isinstance(addr, tuple) and len(addr) == 2, ( '2-tuple is expected') - infos = yield from _ensure_resolved( + infos = await self._ensure_resolved( addr, family=family, type=socket.SOCK_DGRAM, proto=proto, flags=flags, loop=self) if not infos: @@ -918,7 +1198,7 @@ class BaseEventLoop(events.AbstractEventLoop): if local_addr: sock.bind(local_address) if remote_addr: - yield from self.sock_connect(sock, remote_address) + await self.sock_connect(sock, remote_address) r_addr = remote_address except OSError as exc: if sock is not None: @@ -948,36 +1228,49 @@ class BaseEventLoop(events.AbstractEventLoop): remote_addr, transport, protocol) try: - yield from waiter + await waiter except: transport.close() raise return transport, protocol - @coroutine - def _create_server_getaddrinfo(self, host, port, family, flags): - infos = yield from _ensure_resolved((host, port), family=family, + async def _ensure_resolved(self, address, *, + family=0, type=socket.SOCK_STREAM, + proto=0, flags=0, loop): + host, port = address[:2] + info = _ipaddr_info(host, port, family, type, proto) + if info is not None: + # "host" is already a resolved IP. + return [info] + else: + return await loop.getaddrinfo(host, port, family=family, type=type, + proto=proto, flags=flags) + + async def _create_server_getaddrinfo(self, host, port, family, flags): + infos = await self._ensure_resolved((host, port), family=family, type=socket.SOCK_STREAM, flags=flags, loop=self) if not infos: - raise OSError('getaddrinfo({!r}) returned empty list'.format(host)) + raise OSError(f'getaddrinfo({host!r}) returned empty list') return infos - @coroutine - def create_server(self, protocol_factory, host=None, port=None, - *, - family=socket.AF_UNSPEC, - flags=socket.AI_PASSIVE, - sock=None, - backlog=100, - ssl=None, - reuse_address=None, - reuse_port=None): + async def create_server( + self, protocol_factory, host=None, port=None, + *, + family=socket.AF_UNSPEC, + flags=socket.AI_PASSIVE, + sock=None, + backlog=100, + ssl=None, + reuse_address=None, + reuse_port=None, + ssl_handshake_timeout=None, + start_serving=True): """Create a TCP server. - The host parameter can be a string, in that case the TCP server is bound - to host and port. + The host parameter can be a string, in that case the TCP server is + bound to host and port. The host parameter can also be a sequence of strings and in that case the TCP server is bound to all hosts of the sequence. If a host @@ -991,6 +1284,11 @@ class BaseEventLoop(events.AbstractEventLoop): """ if isinstance(ssl, bool): raise TypeError('ssl argument must be an SSLContext or None') + + if ssl_handshake_timeout is not None and ssl is None: + raise ValueError( + 'ssl_handshake_timeout is only meaningful with ssl') + if host is not None or port is not None: if sock is not None: raise ValueError( @@ -1011,7 +1309,7 @@ class BaseEventLoop(events.AbstractEventLoop): fs = [self._create_server_getaddrinfo(host, port, family=family, flags=flags) for host in hosts] - infos = yield from tasks.gather(*fs, loop=self) + infos = await tasks.gather(*fs, loop=self) infos = set(itertools.chain.from_iterable(infos)) completed = False @@ -1054,22 +1352,26 @@ class BaseEventLoop(events.AbstractEventLoop): else: if sock is None: raise ValueError('Neither host/port nor sock were specified') - if not _is_stream_socket(sock): - raise ValueError( - 'A Stream Socket was expected, got {!r}'.format(sock)) + if sock.type != socket.SOCK_STREAM: + raise ValueError(f'A Stream Socket was expected, got {sock!r}') sockets = [sock] - server = Server(self, sockets) for sock in sockets: - sock.listen(backlog) sock.setblocking(False) - self._start_serving(protocol_factory, sock, ssl, server, backlog) + + server = Server(self, sockets, protocol_factory, + ssl, backlog, ssl_handshake_timeout) + if start_serving: + server._start_serving() + if self._debug: logger.info("%r is serving", server) return server - @coroutine - def connect_accepted_socket(self, protocol_factory, sock, *, ssl=None): + async def connect_accepted_socket( + self, protocol_factory, sock, + *, ssl=None, + ssl_handshake_timeout=None): """Handle an accepted connection. This is used by servers that accept connections outside of @@ -1078,12 +1380,16 @@ class BaseEventLoop(events.AbstractEventLoop): This method is a coroutine. When completed, the coroutine returns a (transport, protocol) pair. """ - if not _is_stream_socket(sock): - raise ValueError( - 'A Stream Socket was expected, got {!r}'.format(sock)) + if sock.type != socket.SOCK_STREAM: + raise ValueError(f'A Stream Socket was expected, got {sock!r}') - transport, protocol = yield from self._create_connection_transport( - sock, protocol_factory, ssl, '', server_side=True) + if ssl_handshake_timeout is not None and not ssl: + raise ValueError( + 'ssl_handshake_timeout is only meaningful with ssl') + + transport, protocol = await self._create_connection_transport( + sock, protocol_factory, ssl, '', server_side=True, + ssl_handshake_timeout=ssl_handshake_timeout) if self._debug: # Get the socket from the transport because SSL transport closes # the old socket and creates a new SSL socket @@ -1091,14 +1397,13 @@ class BaseEventLoop(events.AbstractEventLoop): logger.debug("%r handled: (%r, %r)", sock, transport, protocol) return transport, protocol - @coroutine - def connect_read_pipe(self, protocol_factory, pipe): + async def connect_read_pipe(self, protocol_factory, pipe): protocol = protocol_factory() waiter = self.create_future() transport = self._make_read_pipe_transport(pipe, protocol, waiter) try: - yield from waiter + await waiter except: transport.close() raise @@ -1108,14 +1413,13 @@ class BaseEventLoop(events.AbstractEventLoop): pipe.fileno(), transport, protocol) return transport, protocol - @coroutine - def connect_write_pipe(self, protocol_factory, pipe): + async def connect_write_pipe(self, protocol_factory, pipe): protocol = protocol_factory() waiter = self.create_future() transport = self._make_write_pipe_transport(pipe, protocol, waiter) try: - yield from waiter + await waiter except: transport.close() raise @@ -1128,21 +1432,23 @@ class BaseEventLoop(events.AbstractEventLoop): def _log_subprocess(self, msg, stdin, stdout, stderr): info = [msg] if stdin is not None: - info.append('stdin=%s' % _format_pipe(stdin)) + info.append(f'stdin={_format_pipe(stdin)}') if stdout is not None and stderr == subprocess.STDOUT: - info.append('stdout=stderr=%s' % _format_pipe(stdout)) + info.append(f'stdout=stderr={_format_pipe(stdout)}') else: if stdout is not None: - info.append('stdout=%s' % _format_pipe(stdout)) + info.append(f'stdout={_format_pipe(stdout)}') if stderr is not None: - info.append('stderr=%s' % _format_pipe(stderr)) + info.append(f'stderr={_format_pipe(stderr)}') logger.debug(' '.join(info)) - @coroutine - def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=False, shell=True, bufsize=0, - **kwargs): + async def subprocess_shell(self, protocol_factory, cmd, *, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=False, + shell=True, bufsize=0, + **kwargs): if not isinstance(cmd, (bytes, str)): raise ValueError("cmd must be a string") if universal_newlines: @@ -1157,17 +1463,16 @@ class BaseEventLoop(events.AbstractEventLoop): # (password) and may be too long debug_log = 'run shell command %r' % cmd self._log_subprocess(debug_log, stdin, stdout, stderr) - transport = yield from self._make_subprocess_transport( + transport = await self._make_subprocess_transport( protocol, cmd, True, stdin, stdout, stderr, bufsize, **kwargs) if self._debug: logger.info('%s: %r', debug_log, transport) return transport, protocol - @coroutine - def subprocess_exec(self, protocol_factory, program, *args, - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, universal_newlines=False, - shell=False, bufsize=0, **kwargs): + async def subprocess_exec(self, protocol_factory, program, *args, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, universal_newlines=False, + shell=False, bufsize=0, **kwargs): if universal_newlines: raise ValueError("universal_newlines must be False") if shell: @@ -1177,16 +1482,16 @@ class BaseEventLoop(events.AbstractEventLoop): popen_args = (program,) + args for arg in popen_args: if not isinstance(arg, (str, bytes)): - raise TypeError("program arguments must be " - "a bytes or text string, not %s" - % type(arg).__name__) + raise TypeError( + f"program arguments must be a bytes or text string, " + f"not {type(arg).__name__}") protocol = protocol_factory() if self._debug: # don't log parameters: they may contain sensitive information # (password) and may be too long - debug_log = 'execute program %r' % program + debug_log = f'execute program {program!r}' self._log_subprocess(debug_log, stdin, stdout, stderr) - transport = yield from self._make_subprocess_transport( + transport = await self._make_subprocess_transport( protocol, popen_args, False, stdin, stdout, stderr, bufsize, **kwargs) if self._debug: @@ -1211,8 +1516,8 @@ class BaseEventLoop(events.AbstractEventLoop): documentation for details about context). """ if handler is not None and not callable(handler): - raise TypeError('A callable object or None is expected, ' - 'got {!r}'.format(handler)) + raise TypeError(f'A callable object or None is expected, ' + f'got {handler!r}') self._exception_handler = handler def default_exception_handler(self, context): @@ -1240,10 +1545,11 @@ class BaseEventLoop(events.AbstractEventLoop): else: exc_info = False - if ('source_traceback' not in context - and self._current_handle is not None - and self._current_handle._source_traceback): - context['handle_traceback'] = self._current_handle._source_traceback + if ('source_traceback' not in context and + self._current_handle is not None and + self._current_handle._source_traceback): + context['handle_traceback'] = \ + self._current_handle._source_traceback log_lines = [message] for key in sorted(context): @@ -1260,7 +1566,7 @@ class BaseEventLoop(events.AbstractEventLoop): value += tb.rstrip() else: value = repr(value) - log_lines.append('{}: {}'.format(key, value)) + log_lines.append(f'{key}: {value}') logger.error('\n'.join(log_lines), exc_info=exc_info) @@ -1272,6 +1578,7 @@ class BaseEventLoop(events.AbstractEventLoop): - 'message': Error message; - 'exception' (optional): Exception object; - 'future' (optional): Future instance; + - 'task' (optional): Task instance; - 'handle' (optional): Handle instance; - 'protocol' (optional): Protocol instance; - 'transport' (optional): Transport instance; @@ -1431,38 +1738,20 @@ class BaseEventLoop(events.AbstractEventLoop): handle._run() handle = None # Needed to break cycles when an exception occurs. - def _set_coroutine_wrapper(self, enabled): - try: - set_wrapper = sys.set_coroutine_wrapper - get_wrapper = sys.get_coroutine_wrapper - except AttributeError: + def _set_coroutine_origin_tracking(self, enabled): + if bool(enabled) == bool(self._coroutine_origin_tracking_enabled): return - enabled = bool(enabled) - if self._coroutine_wrapper_set == enabled: - return - - wrapper = coroutines.debug_wrapper - current_wrapper = get_wrapper() - if enabled: - if current_wrapper not in (None, wrapper): - warnings.warn( - "loop.set_debug(True): cannot set debug coroutine " - "wrapper; another wrapper is already set %r" % - current_wrapper, RuntimeWarning) - else: - set_wrapper(wrapper) - self._coroutine_wrapper_set = True + self._coroutine_origin_tracking_saved_depth = ( + sys.get_coroutine_origin_tracking_depth()) + sys.set_coroutine_origin_tracking_depth( + constants.DEBUG_STACK_DEPTH) else: - if current_wrapper not in (None, wrapper): - warnings.warn( - "loop.set_debug(False): cannot unset debug coroutine " - "wrapper; another wrapper was set %r" % - current_wrapper, RuntimeWarning) - else: - set_wrapper(None) - self._coroutine_wrapper_set = False + sys.set_coroutine_origin_tracking_depth( + self._coroutine_origin_tracking_saved_depth) + + self._coroutine_origin_tracking_enabled = enabled def get_debug(self): return self._debug @@ -1471,4 +1760,4 @@ class BaseEventLoop(events.AbstractEventLoop): self._debug = enabled if self.is_running(): - self._set_coroutine_wrapper(enabled) + self.call_soon_threadsafe(self._set_coroutine_origin_tracking, enabled) diff --git a/Lib/asyncio/base_futures.py b/Lib/asyncio/base_futures.py index 01259a062e6..5182884e16d 100644 --- a/Lib/asyncio/base_futures.py +++ b/Lib/asyncio/base_futures.py @@ -1,9 +1,9 @@ -__all__ = [] +__all__ = () import concurrent.futures._base import reprlib -from . import events +from . import format_helpers Error = concurrent.futures._base.Error CancelledError = concurrent.futures.CancelledError @@ -38,17 +38,17 @@ def _format_callbacks(cb): cb = '' def format_cb(callback): - return events._format_callback_source(callback, ()) + return format_helpers._format_callback_source(callback, ()) if size == 1: - cb = format_cb(cb[0]) + cb = format_cb(cb[0][0]) elif size == 2: - cb = '{}, {}'.format(format_cb(cb[0]), format_cb(cb[1])) + cb = '{}, {}'.format(format_cb(cb[0][0]), format_cb(cb[1][0])) elif size > 2: - cb = '{}, <{} more>, {}'.format(format_cb(cb[0]), + cb = '{}, <{} more>, {}'.format(format_cb(cb[0][0]), size - 2, - format_cb(cb[-1])) - return 'cb=[%s]' % cb + format_cb(cb[-1][0])) + return f'cb=[{cb}]' def _future_repr_info(future): @@ -57,15 +57,15 @@ def _future_repr_info(future): info = [future._state.lower()] if future._state == _FINISHED: if future._exception is not None: - info.append('exception={!r}'.format(future._exception)) + info.append(f'exception={future._exception!r}') else: # use reprlib to limit the length of the output, especially # for very long strings result = reprlib.repr(future._result) - info.append('result={}'.format(result)) + info.append(f'result={result}') if future._callbacks: info.append(_format_callbacks(future._callbacks)) if future._source_traceback: frame = future._source_traceback[-1] - info.append('created at %s:%s' % (frame[0], frame[1])) + info.append(f'created at {frame[0]}:{frame[1]}') return info diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index cac8d962c0b..7c17066f8bb 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -4,7 +4,6 @@ import warnings from . import protocols from . import transports -from .coroutines import coroutine from .log import logger @@ -58,9 +57,9 @@ class BaseSubprocessTransport(transports.SubprocessTransport): if self._closed: info.append('closed') if self._pid is not None: - info.append('pid=%s' % self._pid) + info.append(f'pid={self.pid}') if self._returncode is not None: - info.append('returncode=%s' % self._returncode) + info.append(f'returncode={self._returncode}') elif self._pid is not None: info.append('running') else: @@ -68,19 +67,19 @@ class BaseSubprocessTransport(transports.SubprocessTransport): stdin = self._pipes.get(0) if stdin is not None: - info.append('stdin=%s' % stdin.pipe) + info.append(f'stdin={stdin.pipe}') stdout = self._pipes.get(1) stderr = self._pipes.get(2) if stdout is not None and stderr is stdout: - info.append('stdout=stderr=%s' % stdout.pipe) + info.append(f'stdout=stderr={stdout.pipe}') else: if stdout is not None: - info.append('stdout=%s' % stdout.pipe) + info.append(f'stdout={stdout.pipe}') if stderr is not None: - info.append('stderr=%s' % stderr.pipe) + info.append(f'stderr={stderr.pipe}') - return '<%s>' % ' '.join(info) + return '<{}>'.format(' '.join(info)) def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): raise NotImplementedError @@ -104,12 +103,13 @@ class BaseSubprocessTransport(transports.SubprocessTransport): continue proto.pipe.close() - if (self._proc is not None - # the child process finished? - and self._returncode is None - # the child process finished but the transport was not notified yet? - and self._proc.poll() is None - ): + if (self._proc is not None and + # has the child process finished? + self._returncode is None and + # the child process has finished, but the + # transport hasn't been notified yet? + self._proc.poll() is None): + if self._loop.get_debug(): logger.warning('Close running child process: kill %r', self) @@ -122,7 +122,7 @@ class BaseSubprocessTransport(transports.SubprocessTransport): def __del__(self): if not self._closed: - warnings.warn("unclosed transport %r" % self, ResourceWarning, + warnings.warn(f"unclosed transport {self!r}", ResourceWarning, source=self) self.close() @@ -154,26 +154,25 @@ class BaseSubprocessTransport(transports.SubprocessTransport): self._check_proc() self._proc.kill() - @coroutine - def _connect_pipes(self, waiter): + async def _connect_pipes(self, waiter): try: proc = self._proc loop = self._loop if proc.stdin is not None: - _, pipe = yield from loop.connect_write_pipe( + _, pipe = await loop.connect_write_pipe( lambda: WriteSubprocessPipeProto(self, 0), proc.stdin) self._pipes[0] = pipe if proc.stdout is not None: - _, pipe = yield from loop.connect_read_pipe( + _, pipe = await loop.connect_read_pipe( lambda: ReadSubprocessPipeProto(self, 1), proc.stdout) self._pipes[1] = pipe if proc.stderr is not None: - _, pipe = yield from loop.connect_read_pipe( + _, pipe = await loop.connect_read_pipe( lambda: ReadSubprocessPipeProto(self, 2), proc.stderr) self._pipes[2] = pipe @@ -208,8 +207,7 @@ class BaseSubprocessTransport(transports.SubprocessTransport): assert returncode is not None, returncode assert self._returncode is None, self._returncode if self._loop.get_debug(): - logger.info('%r exited with return code %r', - self, returncode) + logger.info('%r exited with return code %r', self, returncode) self._returncode = returncode if self._proc.returncode is None: # asyncio uses a child watcher: copy the status into the Popen @@ -224,8 +222,7 @@ class BaseSubprocessTransport(transports.SubprocessTransport): waiter.set_result(returncode) self._exit_waiters = None - @coroutine - def _wait(self): + async def _wait(self): """Wait until the process exit and return the process return code. This method is a coroutine.""" @@ -234,7 +231,7 @@ class BaseSubprocessTransport(transports.SubprocessTransport): waiter = self._loop.create_future() self._exit_waiters.append(waiter) - return (yield from waiter) + return await waiter def _try_finish(self): assert not self._finished @@ -266,8 +263,7 @@ class WriteSubprocessPipeProto(protocols.BaseProtocol): self.pipe = transport def __repr__(self): - return ('<%s fd=%s pipe=%r>' - % (self.__class__.__name__, self.fd, self.pipe)) + return f'<{self.__class__.__name__} fd={self.fd} pipe={self.pipe!r}>' def connection_lost(self, exc): self.disconnected = True diff --git a/Lib/asyncio/base_tasks.py b/Lib/asyncio/base_tasks.py index 5f34434c576..3ce51f6a986 100644 --- a/Lib/asyncio/base_tasks.py +++ b/Lib/asyncio/base_tasks.py @@ -13,10 +13,10 @@ def _task_repr_info(task): info[0] = 'cancelling' coro = coroutines._format_coroutine(task._coro) - info.insert(1, 'coro=<%s>' % coro) + info.insert(1, f'coro=<{coro}>') if task._fut_waiter is not None: - info.insert(2, 'wait_for=%r' % task._fut_waiter) + info.insert(2, f'wait_for={task._fut_waiter!r}') return info @@ -61,15 +61,15 @@ def _task_print_stack(task, limit, file): linecache.checkcache(filename) line = linecache.getline(filename, lineno, f.f_globals) extracted_list.append((filename, lineno, name, line)) + exc = task._exception if not extracted_list: - print('No stack for %r' % task, file=file) + print(f'No stack for {task!r}', file=file) elif exc is not None: - print('Traceback for %r (most recent call last):' % task, - file=file) + print(f'Traceback for {task!r} (most recent call last):', file=file) else: - print('Stack for %r (most recent call last):' % task, - file=file) + print(f'Stack for {task!r} (most recent call last):', file=file) + traceback.print_list(extracted_list, file=file) if exc is not None: for line in traceback.format_exception_only(exc.__class__, exc): diff --git a/Lib/asyncio/compat.py b/Lib/asyncio/compat.py deleted file mode 100644 index 520ec6870c8..00000000000 --- a/Lib/asyncio/compat.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Compatibility helpers for the different Python versions.""" - -import sys - -PY35 = sys.version_info >= (3, 5) -PY352 = sys.version_info >= (3, 5, 2) diff --git a/Lib/asyncio/constants.py b/Lib/asyncio/constants.py index 60ca0da75f8..739b0a70c13 100644 --- a/Lib/asyncio/constants.py +++ b/Lib/asyncio/constants.py @@ -1,4 +1,4 @@ -"""Constants.""" +import enum # After the connection is lost, log warnings after this many write()s. LOG_THRESHOLD_FOR_CONNLOST_WRITES = 5 @@ -8,5 +8,15 @@ ACCEPT_RETRY_DELAY = 1 # Number of stack entries to capture in debug mode. # The larger the number, the slower the operation in debug mode -# (see extract_stack() in events.py). +# (see extract_stack() in format_helpers.py). DEBUG_STACK_DEPTH = 10 + +# Number of seconds to wait for SSL handshake to complete +SSL_HANDSHAKE_TIMEOUT = 10.0 + +# The enum should be here to break circular dependencies between +# base_events and sslproto +class _SendfileMode(enum.Enum): + UNSUPPORTED = enum.auto() + TRY_NATIVE = enum.auto() + FALLBACK = enum.auto() diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index a87c9f9b0b6..c7fcd442558 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -1,98 +1,37 @@ -__all__ = ['coroutine', - 'iscoroutinefunction', 'iscoroutine'] +__all__ = 'coroutine', 'iscoroutinefunction', 'iscoroutine' +import collections.abc import functools import inspect -import opcode import os import sys import traceback import types -from . import compat -from . import constants -from . import events from . import base_futures +from . import constants +from . import format_helpers from .log import logger -# Opcode of "yield from" instruction -_YIELD_FROM = opcode.opmap['YIELD_FROM'] - - def _is_debug_mode(): # If you set _DEBUG to true, @coroutine will wrap the resulting # generator objects in a CoroWrapper instance (defined below). That # instance will log a message when the generator is never iterated - # over, which may happen when you forget to use "yield from" with a - # coroutine call. Note that the value of the _DEBUG flag is taken + # over, which may happen when you forget to use "await" or "yield from" + # with a coroutine call. + # Note that the value of the _DEBUG flag is taken # when the decorator is used, so to be of any use it must be set # before you define your coroutines. A downside of using this feature # is that tracebacks show entries for the CoroWrapper.__next__ method # when _DEBUG is true. - debug = (not sys.flags.ignore_environment and - bool(os.environ.get('PYTHONASYNCIODEBUG'))) - if hasattr(sys, '_xoptions') and 'dev' in sys._xoptions: - debug = True - return debug + return sys.flags.dev_mode or (not sys.flags.ignore_environment and + bool(os.environ.get('PYTHONASYNCIODEBUG'))) _DEBUG = _is_debug_mode() -try: - _types_coroutine = types.coroutine - _types_CoroutineType = types.CoroutineType -except AttributeError: - # Python 3.4 - _types_coroutine = None - _types_CoroutineType = None - -try: - _inspect_iscoroutinefunction = inspect.iscoroutinefunction -except AttributeError: - # Python 3.4 - _inspect_iscoroutinefunction = lambda func: False - -try: - from collections.abc import Coroutine as _CoroutineABC, \ - Awaitable as _AwaitableABC -except ImportError: - _CoroutineABC = _AwaitableABC = None - - -# Check for CPython issue #21209 -def has_yield_from_bug(): - class MyGen: - def __init__(self): - self.send_args = None - def __iter__(self): - return self - def __next__(self): - return 42 - def send(self, *what): - self.send_args = what - return None - def yield_from_gen(gen): - yield from gen - value = (1, 2, 3) - gen = MyGen() - coro = yield_from_gen(gen) - next(coro) - coro.send(value) - return gen.send_args != (value,) -_YIELD_FROM_BUG = has_yield_from_bug() -del has_yield_from_bug - - -def debug_wrapper(gen): - # This function is called from 'sys.set_coroutine_wrapper'. - # We only wrap here coroutines defined via 'async def' syntax. - # Generator-based coroutines are wrapped in @coroutine - # decorator. - return CoroWrapper(gen, None) - - class CoroWrapper: # Wrapper for coroutine object in _DEBUG mode. @@ -100,7 +39,7 @@ class CoroWrapper: assert inspect.isgenerator(gen) or inspect.iscoroutine(gen), gen self.gen = gen self.func = func # Used to unwrap @coroutine decorator - self._source_traceback = events.extract_stack(sys._getframe(1)) + self._source_traceback = format_helpers.extract_stack(sys._getframe(1)) self.__name__ = getattr(gen, '__name__', None) self.__qualname__ = getattr(gen, '__qualname__', None) @@ -108,8 +47,9 @@ class CoroWrapper: coro_repr = _format_coroutine(self) if self._source_traceback: frame = self._source_traceback[-1] - coro_repr += ', created at %s:%s' % (frame[0], frame[1]) - return '<%s %s>' % (self.__class__.__name__, coro_repr) + coro_repr += f', created at {frame[0]}:{frame[1]}' + + return f'<{self.__class__.__name__} {coro_repr}>' def __iter__(self): return self @@ -117,21 +57,8 @@ class CoroWrapper: def __next__(self): return self.gen.send(None) - if _YIELD_FROM_BUG: - # For for CPython issue #21209: using "yield from" and a custom - # generator, generator.send(tuple) unpacks the tuple instead of passing - # the tuple unchanged. Check if the caller is a generator using "yield - # from" to decide if the parameter should be unpacked or not. - def send(self, *value): - frame = sys._getframe() - caller = frame.f_back - assert caller.f_lasti >= 0 - if caller.f_code.co_code[caller.f_lasti] != _YIELD_FROM: - value = value[0] - return self.gen.send(value) - else: - def send(self, value): - return self.gen.send(value) + def send(self, value): + return self.gen.send(value) def throw(self, type, value=None, traceback=None): return self.gen.throw(type, value, traceback) @@ -151,44 +78,19 @@ class CoroWrapper: def gi_code(self): return self.gen.gi_code - if compat.PY35: + def __await__(self): + return self - def __await__(self): - cr_await = getattr(self.gen, 'cr_await', None) - if cr_await is not None: - raise RuntimeError( - "Cannot await on coroutine {!r} while it's " - "awaiting for {!r}".format(self.gen, cr_await)) - return self - - @property - def gi_yieldfrom(self): - return self.gen.gi_yieldfrom - - @property - def cr_await(self): - return self.gen.cr_await - - @property - def cr_running(self): - return self.gen.cr_running - - @property - def cr_code(self): - return self.gen.cr_code - - @property - def cr_frame(self): - return self.gen.cr_frame + @property + def gi_yieldfrom(self): + return self.gen.gi_yieldfrom def __del__(self): # Be careful accessing self.gen.frame -- self.gen might not exist. gen = getattr(self, 'gen', None) frame = getattr(gen, 'gi_frame', None) - if frame is None: - frame = getattr(gen, 'cr_frame', None) if frame is not None and frame.f_lasti == -1: - msg = '%r was never yielded from' % self + msg = f'{self!r} was never yielded from' tb = getattr(self, '_source_traceback', ()) if tb: tb = ''.join(traceback.format_list(tb)) @@ -205,11 +107,9 @@ def coroutine(func): If the coroutine is not yielded from before it is destroyed, an error message is logged. """ - if _inspect_iscoroutinefunction(func): + if inspect.iscoroutinefunction(func): # In Python 3.5 that's all we need to do for coroutines # defined with "async def". - # Wrapping in CoroWrapper will happen via - # 'sys.set_coroutine_wrapper' function. return func if inspect.isgeneratorfunction(func): @@ -219,25 +119,22 @@ def coroutine(func): def coro(*args, **kw): res = func(*args, **kw) if (base_futures.isfuture(res) or inspect.isgenerator(res) or - isinstance(res, CoroWrapper)): + isinstance(res, CoroWrapper)): res = yield from res - elif _AwaitableABC is not None: - # If 'func' returns an Awaitable (new in 3.5) we - # want to run it. + else: + # If 'res' is an awaitable, run it. try: await_meth = res.__await__ except AttributeError: pass else: - if isinstance(res, _AwaitableABC): + if isinstance(res, collections.abc.Awaitable): res = yield from await_meth() return res + coro = types.coroutine(coro) if not _DEBUG: - if _types_coroutine is None: - wrapper = coro - else: - wrapper = _types_coroutine(coro) + wrapper = coro else: @functools.wraps(func) def wrapper(*args, **kwds): @@ -262,22 +159,31 @@ _is_coroutine = object() def iscoroutinefunction(func): """Return True if func is a decorated coroutine function.""" - return (getattr(func, '_is_coroutine', None) is _is_coroutine or - _inspect_iscoroutinefunction(func)) + return (inspect.iscoroutinefunction(func) or + getattr(func, '_is_coroutine', None) is _is_coroutine) -_COROUTINE_TYPES = (types.GeneratorType, CoroWrapper) -if _CoroutineABC is not None: - _COROUTINE_TYPES += (_CoroutineABC,) -if _types_CoroutineType is not None: - # Prioritize native coroutine check to speed-up - # asyncio.iscoroutine. - _COROUTINE_TYPES = (_types_CoroutineType,) + _COROUTINE_TYPES +# Prioritize native coroutine check to speed-up +# asyncio.iscoroutine. +_COROUTINE_TYPES = (types.CoroutineType, types.GeneratorType, + collections.abc.Coroutine, CoroWrapper) +_iscoroutine_typecache = set() def iscoroutine(obj): """Return True if obj is a coroutine object.""" - return isinstance(obj, _COROUTINE_TYPES) + if type(obj) in _iscoroutine_typecache: + return True + + if isinstance(obj, _COROUTINE_TYPES): + # Just in case we don't want to cache more than 100 + # positive types. That shouldn't ever happen, unless + # someone stressing the system on purpose. + if len(_iscoroutine_typecache) < 100: + _iscoroutine_typecache.add(type(obj)) + return True + else: + return False def _format_coroutine(coro): @@ -290,7 +196,7 @@ def _format_coroutine(coro): coro_name = getattr( coro, '__qualname__', getattr(coro, '__name__', type(coro).__name__)) - coro_name = '{}()'.format(coro_name) + coro_name = f'{coro_name}()' running = False try: @@ -302,7 +208,7 @@ def _format_coroutine(coro): pass if running: - return '{} running'.format(coro_name) + return f'{coro_name} running' else: return coro_name @@ -311,12 +217,12 @@ def _format_coroutine(coro): func = coro.func coro_name = coro.__qualname__ if coro_name is not None: - coro_name = '{}()'.format(coro_name) + coro_name = f'{coro_name}()' else: func = coro if coro_name is None: - coro_name = events._format_callback(func, (), {}) + coro_name = format_helpers._format_callback(func, (), {}) try: coro_code = coro.gi_code @@ -333,22 +239,18 @@ def _format_coroutine(coro): if (isinstance(coro, CoroWrapper) and not inspect.isgeneratorfunction(coro.func) and coro.func is not None): - source = events._get_function_source(coro.func) + source = format_helpers._get_function_source(coro.func) if source is not None: filename, lineno = source if coro_frame is None: - coro_repr = ('%s done, defined at %s:%s' - % (coro_name, filename, lineno)) + coro_repr = f'{coro_name} done, defined at {filename}:{lineno}' else: - coro_repr = ('%s running, defined at %s:%s' - % (coro_name, filename, lineno)) + coro_repr = f'{coro_name} running, defined at {filename}:{lineno}' elif coro_frame is not None: lineno = coro_frame.f_lineno - coro_repr = ('%s running at %s:%s' - % (coro_name, filename, lineno)) + coro_repr = f'{coro_name} running at {filename}:{lineno}' else: lineno = coro_code.co_firstlineno - coro_repr = ('%s done, defined at %s:%s' - % (coro_name, filename, lineno)) + coro_repr = f'{coro_name} done, defined at {filename}:{lineno}' return coro_repr diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index f2f2e286342..fcca5d4cb34 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -1,110 +1,53 @@ """Event loop and event loop policy.""" -__all__ = ['AbstractEventLoopPolicy', - 'AbstractEventLoop', 'AbstractServer', - 'Handle', 'TimerHandle', - 'get_event_loop_policy', 'set_event_loop_policy', - 'get_event_loop', 'set_event_loop', 'new_event_loop', - 'get_child_watcher', 'set_child_watcher', - '_set_running_loop', '_get_running_loop', - ] +__all__ = ( + 'AbstractEventLoopPolicy', + 'AbstractEventLoop', 'AbstractServer', + 'Handle', 'TimerHandle', 'SendfileNotAvailableError', + 'get_event_loop_policy', 'set_event_loop_policy', + 'get_event_loop', 'set_event_loop', 'new_event_loop', + 'get_child_watcher', 'set_child_watcher', + '_set_running_loop', 'get_running_loop', + '_get_running_loop', +) -import functools -import inspect +import contextvars import os -import reprlib import socket import subprocess import sys import threading -import traceback -from . import constants +from . import format_helpers -def _get_function_source(func): - func = inspect.unwrap(func) - if inspect.isfunction(func): - code = func.__code__ - return (code.co_filename, code.co_firstlineno) - if isinstance(func, functools.partial): - return _get_function_source(func.func) - if isinstance(func, functools.partialmethod): - return _get_function_source(func.func) - return None +class SendfileNotAvailableError(RuntimeError): + """Sendfile syscall is not available. - -def _format_args_and_kwargs(args, kwargs): - """Format function arguments and keyword arguments. - - Special case for a single parameter: ('hello',) is formatted as ('hello'). + Raised if OS does not support senfile syscall for given socket or + file type. """ - # use reprlib to limit the length of the output - items = [] - if args: - items.extend(reprlib.repr(arg) for arg in args) - if kwargs: - items.extend('{}={}'.format(k, reprlib.repr(v)) - for k, v in kwargs.items()) - return '(' + ', '.join(items) + ')' - - -def _format_callback(func, args, kwargs, suffix=''): - if isinstance(func, functools.partial): - suffix = _format_args_and_kwargs(args, kwargs) + suffix - return _format_callback(func.func, func.args, func.keywords, suffix) - - if hasattr(func, '__qualname__'): - func_repr = getattr(func, '__qualname__') - elif hasattr(func, '__name__'): - func_repr = getattr(func, '__name__') - else: - func_repr = repr(func) - - func_repr += _format_args_and_kwargs(args, kwargs) - if suffix: - func_repr += suffix - return func_repr - -def _format_callback_source(func, args): - func_repr = _format_callback(func, args, None) - source = _get_function_source(func) - if source: - func_repr += ' at %s:%s' % source - return func_repr - - -def extract_stack(f=None, limit=None): - """Replacement for traceback.extract_stack() that only does the - necessary work for asyncio debug mode. - """ - if f is None: - f = sys._getframe().f_back - if limit is None: - # Limit the amount of work to a reasonable amount, as extract_stack() - # can be called for each coroutine and future in debug mode. - limit = constants.DEBUG_STACK_DEPTH - stack = traceback.StackSummary.extract(traceback.walk_stack(f), - limit=limit, - lookup_lines=False) - stack.reverse() - return stack class Handle: """Object returned by callback registration methods.""" __slots__ = ('_callback', '_args', '_cancelled', '_loop', - '_source_traceback', '_repr', '__weakref__') + '_source_traceback', '_repr', '__weakref__', + '_context') - def __init__(self, callback, args, loop): + def __init__(self, callback, args, loop, context=None): + if context is None: + context = contextvars.copy_context() + self._context = context self._loop = loop self._callback = callback self._args = args self._cancelled = False self._repr = None if self._loop.get_debug(): - self._source_traceback = extract_stack(sys._getframe(1)) + self._source_traceback = format_helpers.extract_stack( + sys._getframe(1)) else: self._source_traceback = None @@ -113,17 +56,18 @@ class Handle: if self._cancelled: info.append('cancelled') if self._callback is not None: - info.append(_format_callback_source(self._callback, self._args)) + info.append(format_helpers._format_callback_source( + self._callback, self._args)) if self._source_traceback: frame = self._source_traceback[-1] - info.append('created at %s:%s' % (frame[0], frame[1])) + info.append(f'created at {frame[0]}:{frame[1]}') return info def __repr__(self): if self._repr is not None: return self._repr info = self._repr_info() - return '<%s>' % ' '.join(info) + return '<{}>'.format(' '.join(info)) def cancel(self): if not self._cancelled: @@ -141,10 +85,11 @@ class Handle: def _run(self): try: - self._callback(*self._args) + self._context.run(self._callback, *self._args) except Exception as exc: - cb = _format_callback_source(self._callback, self._args) - msg = 'Exception in callback {}'.format(cb) + cb = format_helpers._format_callback_source( + self._callback, self._args) + msg = f'Exception in callback {cb}' context = { 'message': msg, 'exception': exc, @@ -161,9 +106,9 @@ class TimerHandle(Handle): __slots__ = ['_scheduled', '_when'] - def __init__(self, when, callback, args, loop): + def __init__(self, when, callback, args, loop, context=None): assert when is not None - super().__init__(callback, args, loop) + super().__init__(callback, args, loop, context) if self._source_traceback: del self._source_traceback[-1] self._when = when @@ -172,7 +117,7 @@ class TimerHandle(Handle): def _repr_info(self): info = super()._repr_info() pos = 2 if self._cancelled else 1 - info.insert(pos, 'when=%s' % self._when) + info.insert(pos, f'when={self._when}') return info def __hash__(self): @@ -211,17 +156,55 @@ class TimerHandle(Handle): self._loop._timer_handle_cancelled(self) super().cancel() + def when(self): + """Return a scheduled callback time. + + The time is an absolute timestamp, using the same time + reference as loop.time(). + """ + return self._when + class AbstractServer: """Abstract server returned by create_server().""" def close(self): """Stop serving. This leaves existing connections open.""" - return NotImplemented + raise NotImplementedError - def wait_closed(self): + def get_loop(self): + """Get the event loop the Server object is attached to.""" + raise NotImplementedError + + def is_serving(self): + """Return True if the server is accepting connections.""" + raise NotImplementedError + + async def start_serving(self): + """Start accepting connections. + + This method is idempotent, so it can be called when + the server is already being serving. + """ + raise NotImplementedError + + async def serve_forever(self): + """Start accepting connections until the coroutine is cancelled. + + The server is closed when the coroutine is cancelled. + """ + raise NotImplementedError + + async def wait_closed(self): """Coroutine to wait until service is closed.""" - return NotImplemented + raise NotImplementedError + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + self.close() + await self.wait_closed() class AbstractEventLoop: @@ -267,7 +250,7 @@ class AbstractEventLoop: """ raise NotImplementedError - def shutdown_asyncgens(self): + async def shutdown_asyncgens(self): """Shutdown all active asynchronous generators.""" raise NotImplementedError @@ -302,7 +285,7 @@ class AbstractEventLoop: def call_soon_threadsafe(self, callback, *args): raise NotImplementedError - def run_in_executor(self, executor, func, *args): + async def run_in_executor(self, executor, func, *args): raise NotImplementedError def set_default_executor(self, executor): @@ -310,21 +293,28 @@ class AbstractEventLoop: # Network I/O methods returning Futures. - def getaddrinfo(self, host, port, *, family=0, type=0, proto=0, flags=0): + async def getaddrinfo(self, host, port, *, + family=0, type=0, proto=0, flags=0): raise NotImplementedError - def getnameinfo(self, sockaddr, flags=0): + async def getnameinfo(self, sockaddr, flags=0): raise NotImplementedError - def create_connection(self, protocol_factory, host=None, port=None, *, - ssl=None, family=0, proto=0, flags=0, sock=None, - local_addr=None, server_hostname=None): + async def create_connection( + self, protocol_factory, host=None, port=None, + *, ssl=None, family=0, proto=0, + flags=0, sock=None, local_addr=None, + server_hostname=None, + ssl_handshake_timeout=None): raise NotImplementedError - def create_server(self, protocol_factory, host=None, port=None, *, - family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, - sock=None, backlog=100, ssl=None, reuse_address=None, - reuse_port=None): + async def create_server( + self, protocol_factory, host=None, port=None, + *, family=socket.AF_UNSPEC, + flags=socket.AI_PASSIVE, sock=None, backlog=100, + ssl=None, reuse_address=None, reuse_port=None, + ssl_handshake_timeout=None, + start_serving=True): """A coroutine which creates a TCP server bound to host and port. The return value is a Server object which can be used to stop @@ -332,8 +322,8 @@ class AbstractEventLoop: If host is an empty string or None all interfaces are assumed and a list of multiple sockets will be returned (most likely - one for IPv4 and another one for IPv6). The host parameter can also be a - sequence (e.g. list) of hosts to bind to. + one for IPv4 and another one for IPv6). The host parameter can also be + a sequence (e.g. list) of hosts to bind to. family can be set to either AF_INET or AF_INET6 to force the socket to use IPv4 or IPv6. If not set it will be determined @@ -359,16 +349,50 @@ class AbstractEventLoop: the same port as other existing endpoints are bound to, so long as they all set this flag when being created. This option is not supported on Windows. + + ssl_handshake_timeout is the time in seconds that an SSL server + will wait for completion of the SSL handshake before aborting the + connection. Default is 10s, longer timeouts may increase vulnerability + to DoS attacks (see https://support.f5.com/csp/article/K13834) + + start_serving set to True (default) causes the created server + to start accepting connections immediately. When set to False, + the user should await Server.start_serving() or Server.serve_forever() + to make the server to start accepting connections. """ raise NotImplementedError - def create_unix_connection(self, protocol_factory, path, *, - ssl=None, sock=None, - server_hostname=None): + async def sendfile(self, transport, file, offset=0, count=None, + *, fallback=True): + """Send a file through a transport. + + Return an amount of sent bytes. + """ raise NotImplementedError - def create_unix_server(self, protocol_factory, path, *, - sock=None, backlog=100, ssl=None): + async def start_tls(self, transport, protocol, sslcontext, *, + server_side=False, + server_hostname=None, + ssl_handshake_timeout=None): + """Upgrade a transport to TLS. + + Return a new transport that *protocol* should start using + immediately. + """ + raise NotImplementedError + + async def create_unix_connection( + self, protocol_factory, path=None, *, + ssl=None, sock=None, + server_hostname=None, + ssl_handshake_timeout=None): + raise NotImplementedError + + async def create_unix_server( + self, protocol_factory, path=None, *, + sock=None, backlog=100, ssl=None, + ssl_handshake_timeout=None, + start_serving=True): """A coroutine which creates a UNIX Domain Socket server. The return value is a Server object, which can be used to stop @@ -385,14 +409,22 @@ class AbstractEventLoop: ssl can be set to an SSLContext to enable SSL over the accepted connections. + + ssl_handshake_timeout is the time in seconds that an SSL server + will wait for the SSL handshake to complete (defaults to 10s). + + start_serving set to True (default) causes the created server + to start accepting connections immediately. When set to False, + the user should await Server.start_serving() or Server.serve_forever() + to make the server to start accepting connections. """ raise NotImplementedError - def create_datagram_endpoint(self, protocol_factory, - local_addr=None, remote_addr=None, *, - family=0, proto=0, flags=0, - reuse_address=None, reuse_port=None, - allow_broadcast=None, sock=None): + async def create_datagram_endpoint(self, protocol_factory, + local_addr=None, remote_addr=None, *, + family=0, proto=0, flags=0, + reuse_address=None, reuse_port=None, + allow_broadcast=None, sock=None): """A coroutine which creates a datagram endpoint. This method will try to establish the endpoint in the background. @@ -425,7 +457,7 @@ class AbstractEventLoop: # Pipes and subprocesses. - def connect_read_pipe(self, protocol_factory, pipe): + async def connect_read_pipe(self, protocol_factory, pipe): """Register read pipe in event loop. Set the pipe to non-blocking mode. protocol_factory should instantiate object with Protocol interface. @@ -438,7 +470,7 @@ class AbstractEventLoop: # close fd in pipe transport then close f and vise versa. raise NotImplementedError - def connect_write_pipe(self, protocol_factory, pipe): + async def connect_write_pipe(self, protocol_factory, pipe): """Register write pipe in event loop. protocol_factory should instantiate object with BaseProtocol interface. @@ -451,14 +483,18 @@ class AbstractEventLoop: # close fd in pipe transport then close f and vise versa. raise NotImplementedError - def subprocess_shell(self, protocol_factory, cmd, *, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - **kwargs): + async def subprocess_shell(self, protocol_factory, cmd, *, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + **kwargs): raise NotImplementedError - def subprocess_exec(self, protocol_factory, *args, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - **kwargs): + async def subprocess_exec(self, protocol_factory, *args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + **kwargs): raise NotImplementedError # Ready-based callback registration methods. @@ -480,19 +516,23 @@ class AbstractEventLoop: # Completion based I/O methods returning Futures. - def sock_recv(self, sock, nbytes): + async def sock_recv(self, sock, nbytes): raise NotImplementedError - def sock_recv_into(self, sock, buf): + async def sock_recv_into(self, sock, buf): raise NotImplementedError - def sock_sendall(self, sock, data): + async def sock_sendall(self, sock, data): raise NotImplementedError - def sock_connect(self, sock, address): + async def sock_connect(self, sock, address): raise NotImplementedError - def sock_accept(self, sock): + async def sock_accept(self, sock): + raise NotImplementedError + + async def sock_sendfile(self, sock, file, offset=0, count=None, + *, fallback=None): raise NotImplementedError # Signal handling. @@ -596,12 +636,14 @@ class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): This may be None or an instance of EventLoop. """ if (self._local._loop is None and - not self._local._set_called and - isinstance(threading.current_thread(), threading._MainThread)): + not self._local._set_called and + isinstance(threading.current_thread(), threading._MainThread)): self.set_event_loop(self.new_event_loop()) + if self._local._loop is None: raise RuntimeError('There is no current event loop in thread %r.' % threading.current_thread().name) + return self._local._loop def set_event_loop(self, loop): @@ -637,12 +679,25 @@ class _RunningLoop(threading.local): _running_loop = _RunningLoop() +def get_running_loop(): + """Return the running event loop. Raise a RuntimeError if there is none. + + This function is thread-specific. + """ + # NOTE: this function is implemented in C (see _asynciomodule.c) + loop = _get_running_loop() + if loop is None: + raise RuntimeError('no running event loop') + return loop + + def _get_running_loop(): """Return the running event loop or None. This is a low-level function intended to be used by event loops. This function is thread-specific. """ + # NOTE: this function is implemented in C (see _asynciomodule.c) running_loop, pid = _running_loop.loop_pid if running_loop is not None and pid == os.getpid(): return running_loop @@ -654,6 +709,7 @@ def _set_running_loop(loop): This is a low-level function intended to be used by event loops. This function is thread-specific. """ + # NOTE: this function is implemented in C (see _asynciomodule.c) _running_loop.loop_pid = (loop, os.getpid()) @@ -690,6 +746,7 @@ def get_event_loop(): If there is no running event loop set, the function will return the result of `get_event_loop_policy().get_event_loop()` call. """ + # NOTE: this function is implemented in C (see _asynciomodule.c) current_loop = _get_running_loop() if current_loop is not None: return current_loop @@ -715,3 +772,26 @@ def set_child_watcher(watcher): """Equivalent to calling get_event_loop_policy().set_child_watcher(watcher).""" return get_event_loop_policy().set_child_watcher(watcher) + + +# Alias pure-Python implementations for testing purposes. +_py__get_running_loop = _get_running_loop +_py__set_running_loop = _set_running_loop +_py_get_running_loop = get_running_loop +_py_get_event_loop = get_event_loop + + +try: + # get_event_loop() is one of the most frequently called + # functions in asyncio. Pure Python implementation is + # about 4 times slower than C-accelerated. + from _asyncio import (_get_running_loop, _set_running_loop, + get_running_loop, get_event_loop) +except ImportError: + pass +else: + # Alias C implementations for testing purposes. + _c__get_running_loop = _get_running_loop + _c__set_running_loop = _set_running_loop + _c_get_running_loop = get_running_loop + _c_get_event_loop = get_event_loop diff --git a/Lib/asyncio/format_helpers.py b/Lib/asyncio/format_helpers.py new file mode 100644 index 00000000000..39cfcee0c1c --- /dev/null +++ b/Lib/asyncio/format_helpers.py @@ -0,0 +1,75 @@ +import functools +import inspect +import reprlib +import traceback + +from . import constants + + +def _get_function_source(func): + func = inspect.unwrap(func) + if inspect.isfunction(func): + code = func.__code__ + return (code.co_filename, code.co_firstlineno) + if isinstance(func, functools.partial): + return _get_function_source(func.func) + if isinstance(func, functools.partialmethod): + return _get_function_source(func.func) + return None + + +def _format_callback_source(func, args): + func_repr = _format_callback(func, args, None) + source = _get_function_source(func) + if source: + func_repr += f' at {source[0]}:{source[1]}' + return func_repr + + +def _format_args_and_kwargs(args, kwargs): + """Format function arguments and keyword arguments. + + Special case for a single parameter: ('hello',) is formatted as ('hello'). + """ + # use reprlib to limit the length of the output + items = [] + if args: + items.extend(reprlib.repr(arg) for arg in args) + if kwargs: + items.extend(f'{k}={reprlib.repr(v)}' for k, v in kwargs.items()) + return '({})'.format(', '.join(items)) + + +def _format_callback(func, args, kwargs, suffix=''): + if isinstance(func, functools.partial): + suffix = _format_args_and_kwargs(args, kwargs) + suffix + return _format_callback(func.func, func.args, func.keywords, suffix) + + if hasattr(func, '__qualname__'): + func_repr = getattr(func, '__qualname__') + elif hasattr(func, '__name__'): + func_repr = getattr(func, '__name__') + else: + func_repr = repr(func) + + func_repr += _format_args_and_kwargs(args, kwargs) + if suffix: + func_repr += suffix + return func_repr + + +def extract_stack(f=None, limit=None): + """Replacement for traceback.extract_stack() that only does the + necessary work for asyncio debug mode. + """ + if f is None: + f = sys._getframe().f_back + if limit is None: + # Limit the amount of work to a reasonable amount, as extract_stack() + # can be called for each coroutine and future in debug mode. + limit = constants.DEBUG_STACK_DEPTH + stack = traceback.StackSummary.extract(traceback.walk_stack(f), + limit=limit, + lookup_lines=False) + stack.reverse() + return stack diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 472f2a8c74e..4a56f32c774 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -1,16 +1,18 @@ """A Future class similar to the one in PEP 3148.""" -__all__ = ['CancelledError', 'TimeoutError', 'InvalidStateError', - 'Future', 'wrap_future', 'isfuture'] +__all__ = ( + 'CancelledError', 'TimeoutError', 'InvalidStateError', + 'Future', 'wrap_future', 'isfuture', +) import concurrent.futures +import contextvars import logging import sys -import traceback from . import base_futures -from . import compat from . import events +from . import format_helpers CancelledError = base_futures.CancelledError @@ -59,12 +61,12 @@ class Future: # The value must also be not-None, to enable a subclass to declare # that it is not compatible by setting this to None. # - It is set by __iter__() below so that Task._step() can tell - # the difference between `yield from Future()` (correct) vs. + # the difference between + # `await Future()` or`yield from Future()` (correct) vs. # `yield Future()` (incorrect). _asyncio_future_blocking = False - _log_traceback = False # Used for Python 3.4 and later - _tb_logger = None # Used for Python 3.3 only + __log_traceback = False def __init__(self, *, loop=None): """Initialize the future. @@ -79,22 +81,24 @@ class Future: self._loop = loop self._callbacks = [] if self._loop.get_debug(): - self._source_traceback = events.extract_stack(sys._getframe(1)) + self._source_traceback = format_helpers.extract_stack( + sys._getframe(1)) _repr_info = base_futures._future_repr_info def __repr__(self): - return '<%s %s>' % (self.__class__.__name__, ' '.join(self._repr_info())) + return '<{} {}>'.format(self.__class__.__name__, + ' '.join(self._repr_info())) def __del__(self): - if not self._log_traceback: + if not self.__log_traceback: # set_exception() was not called, or result() or exception() # has consumed the exception return exc = self._exception context = { - 'message': ('%s exception was never retrieved' - % self.__class__.__name__), + 'message': + f'{self.__class__.__name__} exception was never retrieved', 'exception': exc, 'future': self, } @@ -102,6 +106,20 @@ class Future: context['source_traceback'] = self._source_traceback self._loop.call_exception_handler(context) + @property + def _log_traceback(self): + return self.__log_traceback + + @_log_traceback.setter + def _log_traceback(self, val): + if bool(val): + raise ValueError('_log_traceback can only be set to False') + self.__log_traceback = False + + def get_loop(self): + """Return the event loop the Future is bound to.""" + return self._loop + def cancel(self): """Cancel the future and schedule callbacks. @@ -109,14 +127,14 @@ class Future: change the future's state to cancelled, schedule the callbacks and return True. """ - self._log_traceback = False + self.__log_traceback = False if self._state != _PENDING: return False self._state = _CANCELLED - self._schedule_callbacks() + self.__schedule_callbacks() return True - def _schedule_callbacks(self): + def __schedule_callbacks(self): """Internal: Ask the event loop to call all callbacks. The callbacks are scheduled to be called as soon as possible. Also @@ -127,8 +145,8 @@ class Future: return self._callbacks[:] = [] - for callback in callbacks: - self._loop.call_soon(callback, self) + for callback, ctx in callbacks: + self._loop.call_soon(callback, self, context=ctx) def cancelled(self): """Return True if the future was cancelled.""" @@ -155,10 +173,7 @@ class Future: raise CancelledError if self._state != _FINISHED: raise InvalidStateError('Result is not ready.') - self._log_traceback = False - if self._tb_logger is not None: - self._tb_logger.clear() - self._tb_logger = None + self.__log_traceback = False if self._exception is not None: raise self._exception return self._result @@ -175,13 +190,10 @@ class Future: raise CancelledError if self._state != _FINISHED: raise InvalidStateError('Exception is not set.') - self._log_traceback = False - if self._tb_logger is not None: - self._tb_logger.clear() - self._tb_logger = None + self.__log_traceback = False return self._exception - def add_done_callback(self, fn): + def add_done_callback(self, fn, *, context=None): """Add a callback to be run when the future becomes done. The callback is called with a single argument - the future object. If @@ -189,9 +201,11 @@ class Future: scheduled with call_soon. """ if self._state != _PENDING: - self._loop.call_soon(fn, self) + self._loop.call_soon(fn, self, context=context) else: - self._callbacks.append(fn) + if context is None: + context = contextvars.copy_context() + self._callbacks.append((fn, context)) # New method not in PEP 3148. @@ -200,7 +214,9 @@ class Future: Returns the number of callbacks removed. """ - filtered_callbacks = [f for f in self._callbacks if f != fn] + filtered_callbacks = [(f, ctx) + for (f, ctx) in self._callbacks + if f != fn] removed_count = len(self._callbacks) - len(filtered_callbacks) if removed_count: self._callbacks[:] = filtered_callbacks @@ -218,7 +234,7 @@ class Future: raise InvalidStateError('{}: {!r}'.format(self._state, self)) self._result = result self._state = _FINISHED - self._schedule_callbacks() + self.__schedule_callbacks() def set_exception(self, exception): """Mark the future done and set an exception. @@ -235,24 +251,36 @@ class Future: "and cannot be raised into a Future") self._exception = exception self._state = _FINISHED - self._schedule_callbacks() - self._log_traceback = True + self.__schedule_callbacks() + self.__log_traceback = True - def __iter__(self): + def __await__(self): if not self.done(): self._asyncio_future_blocking = True yield self # This tells Task to wait for completion. - assert self.done(), "yield from wasn't used with future" + if not self.done(): + raise RuntimeError("await wasn't used with future") return self.result() # May raise too. - if compat.PY35: - __await__ = __iter__ # make compatible with 'await' expression + __iter__ = __await__ # make compatible with 'yield from'. # Needed for testing purposes. _PyFuture = Future +def _get_loop(fut): + # Tries to call Future.get_loop() if it's available. + # Otherwise fallbacks to using the old '_loop' property. + try: + get_loop = fut.get_loop + except AttributeError: + pass + else: + return get_loop() + return fut._loop + + def _set_result_unless_cancelled(fut, result): """Helper setting the result only if the future was not cancelled.""" if fut.cancelled(): @@ -308,8 +336,8 @@ def _chain_future(source, destination): if not isfuture(destination) and not isinstance(destination, concurrent.futures.Future): raise TypeError('A future is required for destination argument') - source_loop = source._loop if isfuture(source) else None - dest_loop = destination._loop if isfuture(destination) else None + source_loop = _get_loop(source) if isfuture(source) else None + dest_loop = _get_loop(destination) if isfuture(destination) else None def _set_state(future, other): if isfuture(future): @@ -339,7 +367,7 @@ def wrap_future(future, *, loop=None): if isfuture(future): return future assert isinstance(future, concurrent.futures.Future), \ - 'concurrent.futures.Future is expected, got {!r}'.format(future) + f'concurrent.futures.Future is expected, got {future!r}' if loop is None: loop = events.get_event_loop() new_future = loop.create_future() diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 92661830a06..508a2142d84 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -1,10 +1,10 @@ """Synchronization primitives.""" -__all__ = ['Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore'] +__all__ = ('Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore') import collections +import warnings -from . import compat from . import events from . import futures from .coroutines import coroutine @@ -23,6 +23,10 @@ class _ContextManager: with lock: + + Deprecated, use 'async with' statement: + async with lock: + """ def __init__(self, lock): @@ -64,26 +68,34 @@ class _ContextManagerMixin: # # finally: # lock.release() + # Deprecated, use 'async with' statement: + # async with lock: + # + warnings.warn("'with (yield from lock)' is deprecated " + "use 'async with lock' instead", + DeprecationWarning, stacklevel=2) yield from self.acquire() return _ContextManager(self) - if compat.PY35: + async def __acquire_ctx(self): + await self.acquire() + return _ContextManager(self) - def __await__(self): - # To make "with await lock" work. - yield from self.acquire() - return _ContextManager(self) + def __await__(self): + warnings.warn("'with await lock' is deprecated " + "use 'async with lock' instead", + DeprecationWarning, stacklevel=2) + # To make "with await lock" work. + return self.__acquire_ctx().__await__() - @coroutine - def __aenter__(self): - yield from self.acquire() - # We have no use for the "as ..." clause in the with - # statement for locks. - return None + async def __aenter__(self): + await self.acquire() + # We have no use for the "as ..." clause in the with + # statement for locks. + return None - @coroutine - def __aexit__(self, exc_type, exc, tb): - self.release() + async def __aexit__(self, exc_type, exc, tb): + self.release() class Lock(_ContextManagerMixin): @@ -108,16 +120,16 @@ class Lock(_ContextManagerMixin): release() call resets the state to unlocked; first coroutine which is blocked in acquire() is being processed. - acquire() is a coroutine and should be called with 'yield from'. + acquire() is a coroutine and should be called with 'await'. - Locks also support the context management protocol. '(yield from lock)' - should be used as the context manager expression. + Locks also support the asynchronous context management protocol. + 'async with lock' statement should be used. Usage: lock = Lock() ... - yield from lock + await lock.acquire() try: ... finally: @@ -127,13 +139,13 @@ class Lock(_ContextManagerMixin): lock = Lock() ... - with (yield from lock): + async with lock: ... Lock objects can be tested for locking state: if not lock.locked(): - yield from lock + await lock.acquire() else: # lock is acquired ... @@ -152,15 +164,14 @@ class Lock(_ContextManagerMixin): res = super().__repr__() extra = 'locked' if self._locked else 'unlocked' if self._waiters: - extra = '{},waiters:{}'.format(extra, len(self._waiters)) - return '<{} [{}]>'.format(res[1:-1], extra) + extra = f'{extra}, waiters:{len(self._waiters)}' + return f'<{res[1:-1]} [{extra}]>' def locked(self): """Return True if lock is acquired.""" return self._locked - @coroutine - def acquire(self): + async def acquire(self): """Acquire a lock. This method blocks until the lock is unlocked, then sets it to @@ -172,16 +183,22 @@ class Lock(_ContextManagerMixin): fut = self._loop.create_future() self._waiters.append(fut) + + # Finally block should be called before the CancelledError + # handling as we don't want CancelledError to call + # _wake_up_first() and attempt to wake up itself. try: - yield from fut - self._locked = True - return True + try: + await fut + finally: + self._waiters.remove(fut) except futures.CancelledError: if not self._locked: self._wake_up_first() raise - finally: - self._waiters.remove(fut) + + self._locked = True + return True def release(self): """Release a lock. @@ -201,11 +218,17 @@ class Lock(_ContextManagerMixin): raise RuntimeError('Lock is not acquired.') def _wake_up_first(self): - """Wake up the first waiter who isn't cancelled.""" - for fut in self._waiters: - if not fut.done(): - fut.set_result(True) - break + """Wake up the first waiter if it isn't done.""" + try: + fut = next(iter(self._waiters)) + except StopIteration: + return + + # .done() necessarily means that a waiter will wake up later on and + # either take the lock, or, if it was cancelled and lock wasn't + # taken already, will hit this again and wake up a new waiter. + if not fut.done(): + fut.set_result(True) class Event: @@ -229,8 +252,8 @@ class Event: res = super().__repr__() extra = 'set' if self._value else 'unset' if self._waiters: - extra = '{},waiters:{}'.format(extra, len(self._waiters)) - return '<{} [{}]>'.format(res[1:-1], extra) + extra = f'{extra}, waiters:{len(self._waiters)}' + return f'<{res[1:-1]} [{extra}]>' def is_set(self): """Return True if and only if the internal flag is true.""" @@ -254,8 +277,7 @@ class Event: to true again.""" self._value = False - @coroutine - def wait(self): + async def wait(self): """Block until the internal flag is true. If the internal flag is true on entry, return True @@ -268,7 +290,7 @@ class Event: fut = self._loop.create_future() self._waiters.append(fut) try: - yield from fut + await fut return True finally: self._waiters.remove(fut) @@ -307,11 +329,10 @@ class Condition(_ContextManagerMixin): res = super().__repr__() extra = 'locked' if self.locked() else 'unlocked' if self._waiters: - extra = '{},waiters:{}'.format(extra, len(self._waiters)) - return '<{} [{}]>'.format(res[1:-1], extra) + extra = f'{extra}, waiters:{len(self._waiters)}' + return f'<{res[1:-1]} [{extra}]>' - @coroutine - def wait(self): + async def wait(self): """Wait until notified. If the calling coroutine has not acquired the lock when this @@ -330,7 +351,7 @@ class Condition(_ContextManagerMixin): fut = self._loop.create_future() self._waiters.append(fut) try: - yield from fut + await fut return True finally: self._waiters.remove(fut) @@ -339,13 +360,12 @@ class Condition(_ContextManagerMixin): # Must reacquire lock even if wait is cancelled while True: try: - yield from self.acquire() + await self.acquire() break except futures.CancelledError: pass - @coroutine - def wait_for(self, predicate): + async def wait_for(self, predicate): """Wait until a predicate becomes true. The predicate should be a callable which result will be @@ -354,7 +374,7 @@ class Condition(_ContextManagerMixin): """ result = predicate() while not result: - yield from self.wait() + await self.wait() result = predicate() return result @@ -418,11 +438,10 @@ class Semaphore(_ContextManagerMixin): def __repr__(self): res = super().__repr__() - extra = 'locked' if self.locked() else 'unlocked,value:{}'.format( - self._value) + extra = 'locked' if self.locked() else f'unlocked, value:{self._value}' if self._waiters: - extra = '{},waiters:{}'.format(extra, len(self._waiters)) - return '<{} [{}]>'.format(res[1:-1], extra) + extra = f'{extra}, waiters:{len(self._waiters)}' + return f'<{res[1:-1]} [{extra}]>' def _wake_up_next(self): while self._waiters: @@ -435,8 +454,7 @@ class Semaphore(_ContextManagerMixin): """Returns True if semaphore can not be acquired immediately.""" return self._value == 0 - @coroutine - def acquire(self): + async def acquire(self): """Acquire a semaphore. If the internal counter is larger than zero on entry, @@ -449,7 +467,7 @@ class Semaphore(_ContextManagerMixin): fut = self._loop.create_future() self._waiters.append(fut) try: - yield from fut + await fut except: # See the similar code in Queue.get. fut.cancel() diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index e35d05b7bf0..10ca6f8967f 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -4,7 +4,7 @@ A proactor is a "notify-on-completion" multiplexer. Currently a proactor is only implemented on Windows with IOCP. """ -__all__ = ['BaseProactorEventLoop'] +__all__ = 'BaseProactorEventLoop', import socket import warnings @@ -12,6 +12,7 @@ import warnings from . import base_events from . import constants from . import futures +from . import protocols from . import sslproto from . import transports from .log import logger @@ -50,17 +51,16 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin, elif self._closing: info.append('closing') if self._sock is not None: - info.append('fd=%s' % self._sock.fileno()) + info.append(f'fd={self._sock.fileno()}') if self._read_fut is not None: - info.append('read=%s' % self._read_fut) + info.append(f'read={self._read_fut!r}') if self._write_fut is not None: - info.append("write=%r" % self._write_fut) + info.append(f'write={self._write_fut!r}') if self._buffer: - bufsize = len(self._buffer) - info.append('write_bufsize=%s' % bufsize) + info.append(f'write_bufsize={len(self._buffer)}') if self._eof_written: info.append('EOF written') - return '<%s>' % ' '.join(info) + return '<{}>'.format(' '.join(info)) def _set_extra(self, sock): self._extra['pipe'] = sock @@ -87,22 +87,24 @@ class _ProactorBasePipeTransport(transports._FlowControlMixin, def __del__(self): if self._sock is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning, + warnings.warn(f"unclosed transport {self!r}", ResourceWarning, source=self) self.close() def _fatal_error(self, exc, message='Fatal error on pipe transport'): - if isinstance(exc, base_events._FATAL_ERROR_IGNORE): - if self._loop.get_debug(): - logger.debug("%r: %s", self, message, exc_info=True) - else: - self._loop.call_exception_handler({ - 'message': message, - 'exception': exc, - 'transport': self, - 'protocol': self._protocol, - }) - self._force_close(exc) + try: + if isinstance(exc, base_events._FATAL_ERROR_IGNORE): + if self._loop.get_debug(): + logger.debug("%r: %s", self, message, exc_info=True) + else: + self._loop.call_exception_handler({ + 'message': message, + 'exception': exc, + 'transport': self, + 'protocol': self._protocol, + }) + finally: + self._force_close(exc) def _force_close(self, exc): if self._closing: @@ -151,38 +153,67 @@ class _ProactorReadPipeTransport(_ProactorBasePipeTransport, extra=None, server=None): super().__init__(loop, sock, protocol, waiter, extra, server) self._paused = False + + if protocols._is_buffered_protocol(protocol): + self._loop_reading = self._loop_reading__get_buffer + else: + self._loop_reading = self._loop_reading__data_received + self._loop.call_soon(self._loop_reading) + def is_reading(self): + return not self._paused and not self._closing + def pause_reading(self): - if self._closing: - raise RuntimeError('Cannot pause_reading() when closing') - if self._paused: - raise RuntimeError('Already paused') + if self._closing or self._paused: + return self._paused = True + + if self._read_fut is not None and not self._read_fut.done(): + self._read_fut.cancel() + self._read_fut = None + if self._loop.get_debug(): logger.debug("%r pauses reading", self) def resume_reading(self): - if not self._paused: - raise RuntimeError('Not paused') - self._paused = False - if self._closing: + if self._closing or not self._paused: return + self._paused = False self._loop.call_soon(self._loop_reading, self._read_fut) if self._loop.get_debug(): logger.debug("%r resumes reading", self) - def _loop_reading(self, fut=None): + def _loop_reading__on_eof(self): + if self._loop.get_debug(): + logger.debug("%r received EOF", self) + + try: + keep_open = self._protocol.eof_received() + except Exception as exc: + self._fatal_error( + exc, 'Fatal error: protocol.eof_received() call failed.') + return + + if not keep_open: + self.close() + + def _loop_reading__data_received(self, fut=None): if self._paused: return - data = None + data = None try: if fut is not None: assert self._read_fut is fut or (self._read_fut is None and self._closing) self._read_fut = None - data = fut.result() # deliver data later in "finally" clause + if fut.done(): + # deliver data later in "finally" clause + data = fut.result() + else: + # the future will be replaced by next proactor.recv call + fut.cancel() if self._closing: # since close() has been called we ignore any read data @@ -194,7 +225,7 @@ class _ProactorReadPipeTransport(_ProactorBasePipeTransport, return # reschedule a new read - self._read_fut = self._loop._proactor.recv(self._sock, 4096) + self._read_fut = self._loop._proactor.recv(self._sock, 32768) except ConnectionAbortedError as exc: if not self._closing: self._fatal_error(exc, 'Fatal read error on pipe transport') @@ -213,23 +244,94 @@ class _ProactorReadPipeTransport(_ProactorBasePipeTransport, finally: if data: self._protocol.data_received(data) - elif data is not None: - if self._loop.get_debug(): - logger.debug("%r received EOF", self) - keep_open = self._protocol.eof_received() - if not keep_open: - self.close() + elif data == b'': + self._loop_reading__on_eof() + + def _loop_reading__get_buffer(self, fut=None): + if self._paused: + return + + nbytes = None + if fut is not None: + assert self._read_fut is fut or (self._read_fut is None and + self._closing) + self._read_fut = None + try: + if fut.done(): + nbytes = fut.result() + else: + # the future will be replaced by next proactor.recv call + fut.cancel() + except ConnectionAbortedError as exc: + if not self._closing: + self._fatal_error( + exc, 'Fatal read error on pipe transport') + elif self._loop.get_debug(): + logger.debug("Read error on pipe transport while closing", + exc_info=True) + except ConnectionResetError as exc: + self._force_close(exc) + except OSError as exc: + self._fatal_error(exc, 'Fatal read error on pipe transport') + except futures.CancelledError: + if not self._closing: + raise + + if nbytes is not None: + if nbytes == 0: + # we got end-of-file so no need to reschedule a new read + self._loop_reading__on_eof() + else: + try: + self._protocol.buffer_updated(nbytes) + except Exception as exc: + self._fatal_error( + exc, + 'Fatal error: ' + 'protocol.buffer_updated() call failed.') + return + + if self._closing or nbytes == 0: + # since close() has been called we ignore any read data + return + + try: + buf = self._protocol.get_buffer() + except Exception as exc: + self._fatal_error( + exc, 'Fatal error: protocol.get_buffer() call failed.') + return + + try: + # schedule a new read + self._read_fut = self._loop._proactor.recv_into(self._sock, buf) + self._read_fut.add_done_callback(self._loop_reading) + except ConnectionAbortedError as exc: + if not self._closing: + self._fatal_error(exc, 'Fatal read error on pipe transport') + elif self._loop.get_debug(): + logger.debug("Read error on pipe transport while closing", + exc_info=True) + except ConnectionResetError as exc: + self._force_close(exc) + except OSError as exc: + self._fatal_error(exc, 'Fatal read error on pipe transport') + except futures.CancelledError: + if not self._closing: + raise class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport, transports.WriteTransport): """Transport for write pipes.""" + _start_tls_compatible = True + def write(self, data): if not isinstance(data, (bytes, bytearray, memoryview)): - msg = ("data argument must be a bytes-like object, not '%s'" % - type(data).__name__) - raise TypeError(msg) + raise TypeError( + f"data argument must be a bytes-like object, " + f"not {type(data).__name__}") if self._eof_written: raise RuntimeError('write_eof() already called') @@ -345,14 +447,18 @@ class _ProactorSocketTransport(_ProactorReadPipeTransport, transports.Transport): """Transport for connected sockets.""" + _sendfile_compatible = constants._SendfileMode.FALLBACK + def _set_extra(self, sock): self._extra['socket'] = sock + try: self._extra['sockname'] = sock.getsockname() except (socket.error, AttributeError): if self._loop.get_debug(): - logger.warning("getsockname() failed on %r", - sock, exc_info=True) + logger.warning( + "getsockname() failed on %r", sock, exc_info=True) + if 'peername' not in self._extra: try: self._extra['peername'] = sock.getpeername() @@ -389,11 +495,15 @@ class BaseProactorEventLoop(base_events.BaseEventLoop): return _ProactorSocketTransport(self, sock, protocol, waiter, extra, server) - def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter=None, - *, server_side=False, server_hostname=None, - extra=None, server=None): - ssl_protocol = sslproto.SSLProtocol(self, protocol, sslcontext, waiter, - server_side, server_hostname) + def _make_ssl_transport( + self, rawsock, protocol, sslcontext, waiter=None, + *, server_side=False, server_hostname=None, + extra=None, server=None, + ssl_handshake_timeout=None): + ssl_protocol = sslproto.SSLProtocol( + self, protocol, sslcontext, waiter, + server_side, server_hostname, + ssl_handshake_timeout=ssl_handshake_timeout) _ProactorSocketTransport(self, rawsock, ssl_protocol, extra=extra, server=server) return ssl_protocol._app_transport @@ -431,23 +541,20 @@ class BaseProactorEventLoop(base_events.BaseEventLoop): # Close the event loop super().close() - def sock_recv(self, sock, n): - return self._proactor.recv(sock, n) + async def sock_recv(self, sock, n): + return await self._proactor.recv(sock, n) - def sock_recv_into(self, sock, buf): - return self._proactor.recv_into(sock, buf) + async def sock_recv_into(self, sock, buf): + return await self._proactor.recv_into(sock, buf) - def sock_sendall(self, sock, data): - return self._proactor.send(sock, data) + async def sock_sendall(self, sock, data): + return await self._proactor.send(sock, data) - def sock_connect(self, sock, address): - return self._proactor.connect(sock, address) + async def sock_connect(self, sock, address): + return await self._proactor.connect(sock, address) - def sock_accept(self, sock): - return self._proactor.accept(sock) - - def _socketpair(self): - raise NotImplementedError + async def sock_accept(self, sock): + return await self._proactor.accept(sock) def _close_self_pipe(self): if self._self_reading_future is not None: @@ -461,7 +568,7 @@ class BaseProactorEventLoop(base_events.BaseEventLoop): def _make_self_pipe(self): # A self-socket, really. :-) - self._ssock, self._csock = self._socketpair() + self._ssock, self._csock = socket.socketpair() self._ssock.setblocking(False) self._csock.setblocking(False) self._internal_fds += 1 @@ -489,7 +596,8 @@ class BaseProactorEventLoop(base_events.BaseEventLoop): self._csock.send(b'\0') def _start_serving(self, protocol_factory, sock, - sslcontext=None, server=None, backlog=100): + sslcontext=None, server=None, backlog=100, + ssl_handshake_timeout=None): def loop(f=None): try: @@ -502,7 +610,8 @@ class BaseProactorEventLoop(base_events.BaseEventLoop): if sslcontext is not None: self._make_ssl_transport( conn, protocol, sslcontext, server_side=True, - extra={'peername': addr}, server=server) + extra={'peername': addr}, server=server, + ssl_handshake_timeout=ssl_handshake_timeout) else: self._make_socket_transport( conn, protocol, @@ -539,6 +648,8 @@ class BaseProactorEventLoop(base_events.BaseEventLoop): self._accept_futures.clear() def _stop_serving(self, sock): - self._stop_accept_futures() + future = self._accept_futures.pop(sock.fileno(), None) + if future: + future.cancel() self._proactor._stop_serving(sock) sock.close() diff --git a/Lib/asyncio/protocols.py b/Lib/asyncio/protocols.py index 80fcac9a82d..8904478f1ab 100644 --- a/Lib/asyncio/protocols.py +++ b/Lib/asyncio/protocols.py @@ -1,7 +1,9 @@ -"""Abstract Protocol class.""" +"""Abstract Protocol base classes.""" -__all__ = ['BaseProtocol', 'Protocol', 'DatagramProtocol', - 'SubprocessProtocol'] +__all__ = ( + 'BaseProtocol', 'Protocol', 'DatagramProtocol', + 'SubprocessProtocol', 'BufferedProtocol', +) class BaseProtocol: @@ -100,6 +102,57 @@ class Protocol(BaseProtocol): """ +class BufferedProtocol(BaseProtocol): + """Interface for stream protocol with manual buffer control. + + Important: this has been been added to asyncio in Python 3.7 + *on a provisional basis*! Consider it as an experimental API that + might be changed or removed in Python 3.8. + + Event methods, such as `create_server` and `create_connection`, + accept factories that return protocols that implement this interface. + + The idea of BufferedProtocol is that it allows to manually allocate + and control the receive buffer. Event loops can then use the buffer + provided by the protocol to avoid unnecessary data copies. This + can result in noticeable performance improvement for protocols that + receive big amounts of data. Sophisticated protocols can allocate + the buffer only once at creation time. + + State machine of calls: + + start -> CM [-> GB [-> BU?]]* [-> ER?] -> CL -> end + + * CM: connection_made() + * GB: get_buffer() + * BU: buffer_updated() + * ER: eof_received() + * CL: connection_lost() + """ + + def get_buffer(self): + """Called to allocate a new receive buffer. + + Must return an object that implements the + :ref:`buffer protocol `. + """ + + def buffer_updated(self, nbytes): + """Called when the buffer was updated with the received data. + + *nbytes* is the total number of bytes that were written to + the buffer. + """ + + def eof_received(self): + """Called when the other end calls write_eof() or equivalent. + + If this returns a false value (including None), the transport + will close itself. If it returns a true value, closing the + transport is up to the protocol. + """ + + class DatagramProtocol(BaseProtocol): """Interface for datagram protocol.""" @@ -132,3 +185,7 @@ class SubprocessProtocol(BaseProtocol): def process_exited(self): """Called when subprocess has exited.""" + + +def _is_buffered_protocol(proto): + return hasattr(proto, 'get_buffer') and not hasattr(proto, 'data_received') diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 1c66d67b041..2a8d3e76044 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -1,27 +1,19 @@ -"""Queues""" - -__all__ = ['Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty'] +__all__ = ('Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty') import collections import heapq -from . import compat from . import events from . import locks -from .coroutines import coroutine class QueueEmpty(Exception): - """Exception raised when Queue.get_nowait() is called on a Queue object - which is empty. - """ + """Raised when Queue.get_nowait() is called on an empty Queue.""" pass class QueueFull(Exception): - """Exception raised when the Queue.put_nowait() method is called on a Queue - object which is full. - """ + """Raised when the Queue.put_nowait() method is called on a full Queue.""" pass @@ -29,7 +21,7 @@ class Queue: """A queue, useful for coordinating producer and consumer coroutines. If maxsize is less than or equal to zero, the queue size is infinite. If it - is an integer greater than 0, then "yield from put()" will block when the + is an integer greater than 0, then "await put()" will block when the queue reaches maxsize, until an item is removed by get(). Unlike the standard library Queue, you can reliably know this Queue's size @@ -75,22 +67,21 @@ class Queue: break def __repr__(self): - return '<{} at {:#x} {}>'.format( - type(self).__name__, id(self), self._format()) + return f'<{type(self).__name__} at {id(self):#x} {self._format()}>' def __str__(self): - return '<{} {}>'.format(type(self).__name__, self._format()) + return f'<{type(self).__name__} {self._format()}>' def _format(self): - result = 'maxsize={!r}'.format(self._maxsize) + result = f'maxsize={self._maxsize!r}' if getattr(self, '_queue', None): - result += ' _queue={!r}'.format(list(self._queue)) + result += f' _queue={list(self._queue)!r}' if self._getters: - result += ' _getters[{}]'.format(len(self._getters)) + result += f' _getters[{len(self._getters)}]' if self._putters: - result += ' _putters[{}]'.format(len(self._putters)) + result += f' _putters[{len(self._putters)}]' if self._unfinished_tasks: - result += ' tasks={}'.format(self._unfinished_tasks) + result += f' tasks={self._unfinished_tasks}' return result def qsize(self): @@ -117,22 +108,26 @@ class Queue: else: return self.qsize() >= self._maxsize - @coroutine - def put(self, item): + async def put(self, item): """Put an item into the queue. Put an item into the queue. If the queue is full, wait until a free slot is available before adding item. - - This method is a coroutine. """ while self.full(): putter = self._loop.create_future() self._putters.append(putter) try: - yield from putter + await putter except: putter.cancel() # Just in case putter is not done yet. + try: + # Clean self._putters from canceled putters. + self._putters.remove(putter) + except ValueError: + # The putter could be removed from self._putters by a + # previous get_nowait call. + pass if not self.full() and not putter.cancelled(): # We were woken up by get_nowait(), but can't take # the call. Wake up the next in line. @@ -152,27 +147,25 @@ class Queue: self._finished.clear() self._wakeup_next(self._getters) - @coroutine - def get(self): + async def get(self): """Remove and return an item from the queue. If queue is empty, wait until an item is available. - - This method is a coroutine. """ while self.empty(): getter = self._loop.create_future() self._getters.append(getter) try: - yield from getter + await getter except: getter.cancel() # Just in case getter is not done yet. - try: + # Clean self._getters from canceled getters. self._getters.remove(getter) except ValueError: + # The getter could be removed from self._getters by a + # previous put_nowait call. pass - if not self.empty() and not getter.cancelled(): # We were woken up by put_nowait(), but can't take # the call. Wake up the next in line. @@ -211,8 +204,7 @@ class Queue: if self._unfinished_tasks == 0: self._finished.set() - @coroutine - def join(self): + async def join(self): """Block until all items in the queue have been gotten and processed. The count of unfinished tasks goes up whenever an item is added to the @@ -221,7 +213,7 @@ class Queue: When the count of unfinished tasks drops to zero, join() unblocks. """ if self._unfinished_tasks > 0: - yield from self._finished.wait() + await self._finished.wait() class PriorityQueue(Queue): @@ -251,9 +243,3 @@ class LifoQueue(Queue): def _get(self): return self._queue.pop() - - -if not compat.PY35: - JoinableQueue = Queue - """Deprecated alias for Queue.""" - __all__.append('JoinableQueue') diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py new file mode 100644 index 00000000000..bb54b725278 --- /dev/null +++ b/Lib/asyncio/runners.py @@ -0,0 +1,73 @@ +__all__ = 'run', + +from . import coroutines +from . import events +from . import tasks + + +def run(main, *, debug=False): + """Run a coroutine. + + This function runs the passed coroutine, taking care of + managing the asyncio event loop and finalizing asynchronous + generators. + + This function cannot be called when another asyncio event loop is + running in the same thread. + + If debug is True, the event loop will be run in debug mode. + + This function always creates a new event loop and closes it at the end. + It should be used as a main entry point for asyncio programs, and should + ideally only be called once. + + Example: + + async def main(): + await asyncio.sleep(1) + print('hello') + + asyncio.run(main()) + """ + if events._get_running_loop() is not None: + raise RuntimeError( + "asyncio.run() cannot be called from a running event loop") + + if not coroutines.iscoroutine(main): + raise ValueError("a coroutine was expected, got {!r}".format(main)) + + loop = events.new_event_loop() + try: + events.set_event_loop(loop) + loop.set_debug(debug) + return loop.run_until_complete(main) + finally: + try: + _cancel_all_tasks(loop) + loop.run_until_complete(loop.shutdown_asyncgens()) + finally: + events.set_event_loop(None) + loop.close() + + +def _cancel_all_tasks(loop): + to_cancel = [task for task in tasks.all_tasks(loop) + if not task.done()] + if not to_cancel: + return + + for task in to_cancel: + task.cancel() + + loop.run_until_complete( + tasks.gather(*to_cancel, loop=loop, return_exceptions=True)) + + for task in to_cancel: + if task.cancelled(): + continue + if task.exception() is not None: + loop.call_exception_handler({ + 'message': 'unhandled exception during asyncio.run() shutdown', + 'exception': task.exception(), + 'task': task, + }) diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 4baad48ce39..354bf9d1c2d 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -4,11 +4,12 @@ A selector is a "notify-when-ready" multiplexer. For a subclass which also includes support for signal handling, see the unix_events sub-module. """ -__all__ = ['BaseSelectorEventLoop'] +__all__ = 'BaseSelectorEventLoop', import collections import errno import functools +import selectors import socket import warnings import weakref @@ -21,10 +22,9 @@ from . import base_events from . import constants from . import events from . import futures -from . import selectors -from . import transports +from . import protocols from . import sslproto -from .coroutines import coroutine +from . import transports from .log import logger @@ -71,11 +71,15 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): return _SelectorSocketTransport(self, sock, protocol, waiter, extra, server) - def _make_ssl_transport(self, rawsock, protocol, sslcontext, waiter=None, - *, server_side=False, server_hostname=None, - extra=None, server=None): - ssl_protocol = sslproto.SSLProtocol(self, protocol, sslcontext, waiter, - server_side, server_hostname) + def _make_ssl_transport( + self, rawsock, protocol, sslcontext, waiter=None, + *, server_side=False, server_hostname=None, + extra=None, server=None, + ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT): + ssl_protocol = sslproto.SSLProtocol( + self, protocol, sslcontext, waiter, + server_side, server_hostname, + ssl_handshake_timeout=ssl_handshake_timeout) _SelectorSocketTransport(self, rawsock, ssl_protocol, extra=extra, server=server) return ssl_protocol._app_transport @@ -96,9 +100,6 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): self._selector.close() self._selector = None - def _socketpair(self): - raise NotImplementedError - def _close_self_pipe(self): self._remove_reader(self._ssock.fileno()) self._ssock.close() @@ -109,7 +110,7 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): def _make_self_pipe(self): # A self-socket, really. :-) - self._ssock, self._csock = self._socketpair() + self._ssock, self._csock = socket.socketpair() self._ssock.setblocking(False) self._csock.setblocking(False) self._internal_fds += 1 @@ -147,12 +148,16 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): exc_info=True) def _start_serving(self, protocol_factory, sock, - sslcontext=None, server=None, backlog=100): + sslcontext=None, server=None, backlog=100, + ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT): self._add_reader(sock.fileno(), self._accept_connection, - protocol_factory, sock, sslcontext, server, backlog) + protocol_factory, sock, sslcontext, server, backlog, + ssl_handshake_timeout) - def _accept_connection(self, protocol_factory, sock, - sslcontext=None, server=None, backlog=100): + def _accept_connection( + self, protocol_factory, sock, + sslcontext=None, server=None, backlog=100, + ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT): # This method is only called once for each event loop tick where the # listening socket has triggered an EVENT_READ. There may be multiple # connections waiting for an .accept() so it is called in a loop. @@ -183,18 +188,20 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): self.call_later(constants.ACCEPT_RETRY_DELAY, self._start_serving, protocol_factory, sock, sslcontext, server, - backlog) + backlog, ssl_handshake_timeout) else: raise # The event loop will catch, log and ignore it. else: extra = {'peername': addr} - accept = self._accept_connection2(protocol_factory, conn, extra, - sslcontext, server) + accept = self._accept_connection2( + protocol_factory, conn, extra, sslcontext, server, + ssl_handshake_timeout) self.create_task(accept) - @coroutine - def _accept_connection2(self, protocol_factory, conn, extra, - sslcontext=None, server=None): + async def _accept_connection2( + self, protocol_factory, conn, extra, + sslcontext=None, server=None, + ssl_handshake_timeout=constants.SSL_HANDSHAKE_TIMEOUT): protocol = None transport = None try: @@ -203,14 +210,15 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): if sslcontext: transport = self._make_ssl_transport( conn, protocol, sslcontext, waiter=waiter, - server_side=True, extra=extra, server=server) + server_side=True, extra=extra, server=server, + ssl_handshake_timeout=ssl_handshake_timeout) else: transport = self._make_socket_transport( conn, protocol, waiter=waiter, extra=extra, server=server) try: - yield from waiter + await waiter except: transport.close() raise @@ -219,8 +227,8 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): except Exception as exc: if self._debug: context = { - 'message': ('Error on transport creation ' - 'for incoming connection'), + 'message': + 'Error on transport creation for incoming connection', 'exception': exc, } if protocol is not None: @@ -236,8 +244,7 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): fileno = int(fileno.fileno()) except (AttributeError, TypeError, ValueError): # This code matches selectors._fileobj_to_fd function. - raise ValueError("Invalid file object: " - "{!r}".format(fd)) from None + raise ValueError(f"Invalid file object: {fd!r}") from None try: transport = self._transports[fileno] except KeyError: @@ -245,12 +252,12 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): else: if not transport.is_closing(): raise RuntimeError( - 'File descriptor {!r} is used by transport {!r}'.format( - fd, transport)) + f'File descriptor {fd!r} is used by transport ' + f'{transport!r}') def _add_reader(self, fd, callback, *args): self._check_closed() - handle = events.Handle(callback, args, self) + handle = events.Handle(callback, args, self, None) try: key = self._selector.get_key(fd) except KeyError: @@ -286,7 +293,7 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): def _add_writer(self, fd, callback, *args): self._check_closed() - handle = events.Handle(callback, args, self) + handle = events.Handle(callback, args, self, None) try: key = self._selector.get_key(fd) except KeyError: @@ -342,20 +349,18 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): self._ensure_fd_no_transport(fd) return self._remove_writer(fd) - def sock_recv(self, sock, n): + async def sock_recv(self, sock, n): """Receive data from the socket. The return value is a bytes object representing the data received. The maximum amount of data to be received at once is specified by nbytes. - - This method is a coroutine. """ if self._debug and sock.gettimeout() != 0: raise ValueError("the socket must be non-blocking") fut = self.create_future() self._sock_recv(fut, None, sock, n) - return fut + return await fut def _sock_recv(self, fut, registered_fd, sock, n): # _sock_recv() can add itself as an I/O callback if the operation can't @@ -378,26 +383,25 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): else: fut.set_result(data) - def sock_recv_into(self, sock, buf): + async def sock_recv_into(self, sock, buf): """Receive data from the socket. The received data is written into *buf* (a writable buffer). The return value is the number of bytes written. - - This method is a coroutine. """ if self._debug and sock.gettimeout() != 0: raise ValueError("the socket must be non-blocking") fut = self.create_future() self._sock_recv_into(fut, None, sock, buf) - return fut + return await fut def _sock_recv_into(self, fut, registered_fd, sock, buf): # _sock_recv_into() can add itself as an I/O callback if the operation - # can't be done immediately. Don't use it directly, call sock_recv_into(). + # can't be done immediately. Don't use it directly, call + # sock_recv_into(). if registered_fd is not None: # Remove the callback early. It should be rare that the - # selector says the fd is ready but the call still returns + # selector says the FD is ready but the call still returns # EAGAIN, and I am willing to take a hit in that case in # order to simplify the common case. self.remove_reader(registered_fd) @@ -413,7 +417,7 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): else: fut.set_result(nbytes) - def sock_sendall(self, sock, data): + async def sock_sendall(self, sock, data): """Send data to the socket. The socket must be connected to a remote socket. This method continues @@ -421,8 +425,6 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): error occurs. None is returned on success. On error, an exception is raised, and there is no way to determine how much data, if any, was successfully processed by the receiving end of the connection. - - This method is a coroutine. """ if self._debug and sock.gettimeout() != 0: raise ValueError("the socket must be non-blocking") @@ -431,7 +433,7 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): self._sock_sendall(fut, None, sock, data) else: fut.set_result(None) - return fut + return await fut def _sock_sendall(self, fut, registered_fd, sock, data): if registered_fd is not None: @@ -455,8 +457,7 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): fd = sock.fileno() self.add_writer(fd, self._sock_sendall, fut, fd, sock, data) - @coroutine - def sock_connect(self, sock, address): + async def sock_connect(self, sock, address): """Connect to a remote socket at address. This method is a coroutine. @@ -465,15 +466,13 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): raise ValueError("the socket must be non-blocking") if not hasattr(socket, 'AF_UNIX') or sock.family != socket.AF_UNIX: - resolved = base_events._ensure_resolved( + resolved = await self._ensure_resolved( address, family=sock.family, proto=sock.proto, loop=self) - if not resolved.done(): - yield from resolved - _, _, _, _, address = resolved.result()[0] + _, _, _, _, address = resolved[0] fut = self.create_future() self._sock_connect(fut, sock, address) - return (yield from fut) + return await fut def _sock_connect(self, fut, sock, address): fd = sock.fileno() @@ -503,7 +502,7 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): err = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) if err != 0: # Jump to any except clause below. - raise OSError(err, 'Connect call failed %s' % (address,)) + raise OSError(err, f'Connect call failed {address}') except (BlockingIOError, InterruptedError): # socket is still registered, the callback will be retried later pass @@ -512,21 +511,19 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): else: fut.set_result(None) - def sock_accept(self, sock): + async def sock_accept(self, sock): """Accept a connection. The socket must be bound to an address and listening for connections. The return value is a pair (conn, address) where conn is a new socket object usable to send and receive data on the connection, and address is the address bound to the socket on the other end of the connection. - - This method is a coroutine. """ if self._debug and sock.gettimeout() != 0: raise ValueError("the socket must be non-blocking") fut = self.create_future() self._sock_accept(fut, False, sock) - return fut + return await fut def _sock_accept(self, fut, registered, sock): fd = sock.fileno() @@ -544,6 +541,20 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): else: fut.set_result((conn, address)) + async def _sendfile_native(self, transp, file, offset, count): + del self._transports[transp._sock_fd] + resume_reading = transp.is_reading() + transp.pause_reading() + await transp._make_empty_waiter() + try: + return await self.sock_sendfile(transp._sock, file, offset, count, + fallback=False) + finally: + transp._reset_empty_waiter() + if resume_reading: + transp.resume_reading() + self._transports[transp._sock_fd] = transp + def _process_events(self, event_list): for key, mask in event_list: fileobj, (reader, writer) = key.fileobj, key.data @@ -602,7 +613,7 @@ class _SelectorTransport(transports._FlowControlMixin, info.append('closed') elif self._closing: info.append('closing') - info.append('fd=%s' % self._sock_fd) + info.append(f'fd={self._sock_fd}') # test if the transport was closed if self._loop is not None and not self._loop.is_closed(): polling = _test_selector_event(self._loop._selector, @@ -621,8 +632,8 @@ class _SelectorTransport(transports._FlowControlMixin, state = 'idle' bufsize = self.get_write_buffer_size() - info.append('write=<%s, bufsize=%s>' % (state, bufsize)) - return '<%s>' % ' '.join(info) + info.append(f'write=<{state}, bufsize={bufsize}>') + return '<{}>'.format(' '.join(info)) def abort(self): self._force_close(None) @@ -648,7 +659,7 @@ class _SelectorTransport(transports._FlowControlMixin, def __del__(self): if self._sock is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning, + warnings.warn(f"unclosed transport {self!r}", ResourceWarning, source=self) self._sock.close() @@ -698,11 +709,21 @@ class _SelectorTransport(transports._FlowControlMixin, class _SelectorSocketTransport(_SelectorTransport): + _start_tls_compatible = True + _sendfile_compatible = constants._SendfileMode.TRY_NATIVE + def __init__(self, loop, sock, protocol, waiter=None, extra=None, server=None): + + if protocols._is_buffered_protocol(protocol): + self._read_ready = self._read_ready__get_buffer + else: + self._read_ready = self._read_ready__data_received + super().__init__(loop, sock, protocol, extra, server) self._eof = False self._paused = False + self._empty_waiter = None # Disable the Nagle algorithm -- small writes will be # sent without waiting for the TCP ACK. This generally @@ -718,56 +739,102 @@ class _SelectorSocketTransport(_SelectorTransport): self._loop.call_soon(futures._set_result_unless_cancelled, waiter, None) + def is_reading(self): + return not self._paused and not self._closing + def pause_reading(self): - if self._closing: - raise RuntimeError('Cannot pause_reading() when closing') - if self._paused: - raise RuntimeError('Already paused') + if self._closing or self._paused: + return self._paused = True self._loop._remove_reader(self._sock_fd) if self._loop.get_debug(): logger.debug("%r pauses reading", self) def resume_reading(self): - if not self._paused: - raise RuntimeError('Not paused') - self._paused = False - if self._closing: + if self._closing or not self._paused: return + self._paused = False self._loop._add_reader(self._sock_fd, self._read_ready) if self._loop.get_debug(): logger.debug("%r resumes reading", self) - def _read_ready(self): + def _read_ready__get_buffer(self): + if self._conn_lost: + return + + try: + buf = self._protocol.get_buffer() + except Exception as exc: + self._fatal_error( + exc, 'Fatal error: protocol.get_buffer() call failed.') + return + + try: + nbytes = self._sock.recv_into(buf) + except (BlockingIOError, InterruptedError): + return + except Exception as exc: + self._fatal_error(exc, 'Fatal read error on socket transport') + return + + if not nbytes: + self._read_ready__on_eof() + return + + try: + self._protocol.buffer_updated(nbytes) + except Exception as exc: + self._fatal_error( + exc, 'Fatal error: protocol.buffer_updated() call failed.') + + def _read_ready__data_received(self): if self._conn_lost: return try: data = self._sock.recv(self.max_size) except (BlockingIOError, InterruptedError): - pass + return except Exception as exc: self._fatal_error(exc, 'Fatal read error on socket transport') + return + + if not data: + self._read_ready__on_eof() + return + + try: + self._protocol.data_received(data) + except Exception as exc: + self._fatal_error( + exc, 'Fatal error: protocol.data_received() call failed.') + + def _read_ready__on_eof(self): + if self._loop.get_debug(): + logger.debug("%r received EOF", self) + + try: + keep_open = self._protocol.eof_received() + except Exception as exc: + self._fatal_error( + exc, 'Fatal error: protocol.eof_received() call failed.') + return + + if keep_open: + # We're keeping the connection open so the + # protocol can write more, but we still can't + # receive more, so remove the reader callback. + self._loop._remove_reader(self._sock_fd) else: - if data: - self._protocol.data_received(data) - else: - if self._loop.get_debug(): - logger.debug("%r received EOF", self) - keep_open = self._protocol.eof_received() - if keep_open: - # We're keeping the connection open so the - # protocol can write more, but we still can't - # receive more, so remove the reader callback. - self._loop._remove_reader(self._sock_fd) - else: - self.close() + self.close() def write(self, data): if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError('data argument must be a bytes-like object, ' - 'not %r' % type(data).__name__) + raise TypeError(f'data argument must be a bytes-like object, ' + f'not {type(data).__name__!r}') if self._eof: raise RuntimeError('Cannot call write() after write_eof()') + if self._empty_waiter is not None: + raise RuntimeError('unable to write; sendfile is in progress') if not data: return @@ -810,12 +877,16 @@ class _SelectorSocketTransport(_SelectorTransport): self._loop._remove_writer(self._sock_fd) self._buffer.clear() self._fatal_error(exc, 'Fatal write error on socket transport') + if self._empty_waiter is not None: + self._empty_waiter.set_exception(exc) else: if n: del self._buffer[:n] self._maybe_resume_protocol() # May append to buffer. if not self._buffer: self._loop._remove_writer(self._sock_fd) + if self._empty_waiter is not None: + self._empty_waiter.set_result(None) if self._closing: self._call_connection_lost(None) elif self._eof: @@ -831,6 +902,23 @@ class _SelectorSocketTransport(_SelectorTransport): def can_write_eof(self): return True + def _call_connection_lost(self, exc): + super()._call_connection_lost(exc) + if self._empty_waiter is not None: + self._empty_waiter.set_exception( + ConnectionError("Connection is closed by peer")) + + def _make_empty_waiter(self): + if self._empty_waiter is not None: + raise RuntimeError("Empty waiter is already set") + self._empty_waiter = self._loop.create_future() + if not self._buffer: + self._empty_waiter.set_result(None) + return self._empty_waiter + + def _reset_empty_waiter(self): + self._empty_waiter = None + class _SelectorDatagramTransport(_SelectorTransport): @@ -868,14 +956,14 @@ class _SelectorDatagramTransport(_SelectorTransport): def sendto(self, data, addr=None): if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError('data argument must be a bytes-like object, ' - 'not %r' % type(data).__name__) + raise TypeError(f'data argument must be a bytes-like object, ' + f'not {type(data).__name__!r}') if not data: return if self._address and addr not in (None, self._address): - raise ValueError('Invalid address: must be None or %s' % - (self._address,)) + raise ValueError( + f'Invalid address: must be None or {self._address}') if self._conn_lost and self._address: if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: @@ -897,8 +985,8 @@ class _SelectorDatagramTransport(_SelectorTransport): self._protocol.error_received(exc) return except Exception as exc: - self._fatal_error(exc, - 'Fatal write error on datagram transport') + self._fatal_error( + exc, 'Fatal write error on datagram transport') return # Ensure that what we buffer is immutable. @@ -920,8 +1008,8 @@ class _SelectorDatagramTransport(_SelectorTransport): self._protocol.error_received(exc) return except Exception as exc: - self._fatal_error(exc, - 'Fatal write error on datagram transport') + self._fatal_error( + exc, 'Fatal write error on datagram transport') return self._maybe_resume_protocol() # May append to buffer. diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index c231eb58ee5..863b54313cc 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -6,6 +6,7 @@ except ImportError: # pragma: no cover ssl = None from . import base_events +from . import constants from . import protocols from . import transports from .log import logger @@ -281,6 +282,8 @@ class _SSLPipe(object): class _SSLProtocolTransport(transports._FlowControlMixin, transports.Transport): + _sendfile_compatible = constants._SendfileMode.FALLBACK + def __init__(self, loop, ssl_protocol): self._loop = loop # SSLProtocol instance @@ -313,10 +316,16 @@ class _SSLProtocolTransport(transports._FlowControlMixin, def __del__(self): if not self._closed: - warnings.warn("unclosed transport %r" % self, ResourceWarning, + warnings.warn(f"unclosed transport {self!r}", ResourceWarning, source=self) self.close() + def is_reading(self): + tr = self._ssl_protocol._transport + if tr is None: + raise RuntimeError('SSL transport has not been initialized yet') + return tr.is_reading() + def pause_reading(self): """Pause the receiving end. @@ -358,6 +367,11 @@ class _SSLProtocolTransport(transports._FlowControlMixin, """Return the current size of the write buffer.""" return self._ssl_protocol._transport.get_write_buffer_size() + @property + def _protocol_paused(self): + # Required for sendfile fallback pause_writing/resume_writing logic + return self._ssl_protocol._transport._protocol_paused + def write(self, data): """Write some data bytes to the transport. @@ -365,8 +379,8 @@ class _SSLProtocolTransport(transports._FlowControlMixin, to be sent out asynchronously. """ if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError("data: expecting a bytes-like instance, got {!r}" - .format(type(data).__name__)) + raise TypeError(f"data: expecting a bytes-like instance, " + f"got {type(data).__name__}") if not data: return self._ssl_protocol._write_appdata(data) @@ -394,12 +408,21 @@ class SSLProtocol(protocols.Protocol): def __init__(self, loop, app_protocol, sslcontext, waiter, server_side=False, server_hostname=None, - call_connection_made=True): + call_connection_made=True, + ssl_handshake_timeout=None): if ssl is None: raise RuntimeError('stdlib ssl module not available') + if ssl_handshake_timeout is None: + ssl_handshake_timeout = constants.SSL_HANDSHAKE_TIMEOUT + elif ssl_handshake_timeout <= 0: + raise ValueError( + f"ssl_handshake_timeout should be a positive number, " + f"got {ssl_handshake_timeout}") + if not sslcontext: - sslcontext = _create_transport_context(server_side, server_hostname) + sslcontext = _create_transport_context( + server_side, server_hostname) self._server_side = server_side if server_hostname and not server_side: @@ -427,6 +450,7 @@ class SSLProtocol(protocols.Protocol): # transport, ex: SelectorSocketTransport self._transport = None self._call_connection_made = call_connection_made + self._ssl_handshake_timeout = ssl_handshake_timeout def _wakeup_waiter(self, exc=None): if self._waiter is None: @@ -554,9 +578,18 @@ class SSLProtocol(protocols.Protocol): # the SSL handshake self._write_backlog.append((b'', 1)) self._loop.call_soon(self._process_write_backlog) + self._handshake_timeout_handle = \ + self._loop.call_later(self._ssl_handshake_timeout, + self._check_handshake_timeout) + + def _check_handshake_timeout(self): + if self._in_handshake is True: + logger.warning("%r stalled during handshake", self) + self._abort() def _on_handshake_complete(self, handshake_exc): self._in_handshake = False + self._handshake_timeout_handle.cancel() sslobj = self._sslpipe.ssl_object try: @@ -564,12 +597,6 @@ class SSLProtocol(protocols.Protocol): raise handshake_exc peercert = sslobj.getpeercert() - if not hasattr(self._sslcontext, 'check_hostname'): - # Verify hostname if requested, Python 3.4+ uses check_hostname - # and checks the hostname in do_handshake() - if (self._server_hostname - and self._sslcontext.verify_mode != ssl.CERT_NONE): - ssl.match_hostname(peercert, self._server_hostname) except BaseException as exc: if self._loop.get_debug(): if isinstance(exc, ssl.CertificateError): diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 30b751e9891..9a53ee48247 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -1,22 +1,19 @@ -"""Stream-related things.""" - -__all__ = ['StreamReader', 'StreamWriter', 'StreamReaderProtocol', - 'open_connection', 'start_server', - 'IncompleteReadError', - 'LimitOverrunError', - ] +__all__ = ( + 'StreamReader', 'StreamWriter', 'StreamReaderProtocol', + 'open_connection', 'start_server', + 'IncompleteReadError', 'LimitOverrunError', +) import socket if hasattr(socket, 'AF_UNIX'): - __all__.extend(['open_unix_connection', 'start_unix_server']) + __all__ += ('open_unix_connection', 'start_unix_server') from . import coroutines -from . import compat from . import events from . import protocols -from .coroutines import coroutine from .log import logger +from .tasks import sleep _DEFAULT_LIMIT = 2 ** 16 @@ -30,8 +27,8 @@ class IncompleteReadError(EOFError): - expected: total number of expected bytes (or None if unknown) """ def __init__(self, partial, expected): - super().__init__("%d bytes read on a total of %r expected bytes" - % (len(partial), expected)) + super().__init__(f'{len(partial)} bytes read on a total of ' + f'{expected!r} expected bytes') self.partial = partial self.expected = expected @@ -53,9 +50,8 @@ class LimitOverrunError(Exception): return type(self), (self.args[0], self.consumed) -@coroutine -def open_connection(host=None, port=None, *, - loop=None, limit=_DEFAULT_LIMIT, **kwds): +async def open_connection(host=None, port=None, *, + loop=None, limit=_DEFAULT_LIMIT, **kwds): """A wrapper for create_connection() returning a (reader, writer) pair. The reader returned is a StreamReader instance; the writer is a @@ -77,15 +73,14 @@ def open_connection(host=None, port=None, *, loop = events.get_event_loop() reader = StreamReader(limit=limit, loop=loop) protocol = StreamReaderProtocol(reader, loop=loop) - transport, _ = yield from loop.create_connection( + transport, _ = await loop.create_connection( lambda: protocol, host, port, **kwds) writer = StreamWriter(transport, protocol, reader, loop) return reader, writer -@coroutine -def start_server(client_connected_cb, host=None, port=None, *, - loop=None, limit=_DEFAULT_LIMIT, **kwds): +async def start_server(client_connected_cb, host=None, port=None, *, + loop=None, limit=_DEFAULT_LIMIT, **kwds): """Start a socket server, call back for each client connected. The first parameter, `client_connected_cb`, takes two parameters: @@ -116,28 +111,26 @@ def start_server(client_connected_cb, host=None, port=None, *, loop=loop) return protocol - return (yield from loop.create_server(factory, host, port, **kwds)) + return await loop.create_server(factory, host, port, **kwds) if hasattr(socket, 'AF_UNIX'): # UNIX Domain Sockets are supported on this platform - @coroutine - def open_unix_connection(path=None, *, - loop=None, limit=_DEFAULT_LIMIT, **kwds): + async def open_unix_connection(path=None, *, + loop=None, limit=_DEFAULT_LIMIT, **kwds): """Similar to `open_connection` but works with UNIX Domain Sockets.""" if loop is None: loop = events.get_event_loop() reader = StreamReader(limit=limit, loop=loop) protocol = StreamReaderProtocol(reader, loop=loop) - transport, _ = yield from loop.create_unix_connection( + transport, _ = await loop.create_unix_connection( lambda: protocol, path, **kwds) writer = StreamWriter(transport, protocol, reader, loop) return reader, writer - @coroutine - def start_unix_server(client_connected_cb, path=None, *, - loop=None, limit=_DEFAULT_LIMIT, **kwds): + async def start_unix_server(client_connected_cb, path=None, *, + loop=None, limit=_DEFAULT_LIMIT, **kwds): """Similar to `start_server` but works with UNIX Domain Sockets.""" if loop is None: loop = events.get_event_loop() @@ -148,14 +141,14 @@ if hasattr(socket, 'AF_UNIX'): loop=loop) return protocol - return (yield from loop.create_unix_server(factory, path, **kwds)) + return await loop.create_unix_server(factory, path, **kwds) class FlowControlMixin(protocols.Protocol): """Reusable flow control logic for StreamWriter.drain(). This implements the protocol methods pause_writing(), - resume_reading() and connection_lost(). If the subclass overrides + resume_writing() and connection_lost(). If the subclass overrides these it must call the super methods. StreamWriter.drain() must wait for _drain_helper() coroutine. @@ -204,8 +197,7 @@ class FlowControlMixin(protocols.Protocol): else: waiter.set_exception(exc) - @coroutine - def _drain_helper(self): + async def _drain_helper(self): if self._connection_lost: raise ConnectionResetError('Connection lost') if not self._paused: @@ -214,7 +206,7 @@ class FlowControlMixin(protocols.Protocol): assert waiter is None or waiter.cancelled() waiter = self._loop.create_future() self._drain_waiter = waiter - yield from waiter + await waiter class StreamReaderProtocol(FlowControlMixin, protocols.Protocol): @@ -232,6 +224,7 @@ class StreamReaderProtocol(FlowControlMixin, protocols.Protocol): self._stream_writer = None self._client_connected_cb = client_connected_cb self._over_ssl = False + self._closed = self._loop.create_future() def connection_made(self, transport): self._stream_reader.set_transport(transport) @@ -251,6 +244,11 @@ class StreamReaderProtocol(FlowControlMixin, protocols.Protocol): self._stream_reader.feed_eof() else: self._stream_reader.set_exception(exc) + if not self._closed.done(): + if exc is None: + self._closed.set_result(None) + else: + self._closed.set_exception(exc) super().connection_lost(exc) self._stream_reader = None self._stream_writer = None @@ -267,6 +265,13 @@ class StreamReaderProtocol(FlowControlMixin, protocols.Protocol): return False return True + def __del__(self): + # Prevent reports about unhandled exceptions. + # Better than self._closed._log_traceback = False hack + closed = self._closed + if closed.done() and not closed.cancelled(): + closed.exception() + class StreamWriter: """Wraps a Transport. @@ -287,10 +292,10 @@ class StreamWriter: self._loop = loop def __repr__(self): - info = [self.__class__.__name__, 'transport=%r' % self._transport] + info = [self.__class__.__name__, f'transport={self._transport!r}'] if self._reader is not None: - info.append('reader=%r' % self._reader) - return '<%s>' % ' '.join(info) + info.append(f'reader={self._reader!r}') + return '<{}>'.format(' '.join(info)) @property def transport(self): @@ -311,32 +316,36 @@ class StreamWriter: def close(self): return self._transport.close() + def is_closing(self): + return self._transport.is_closing() + + async def wait_closed(self): + await self._protocol._closed + def get_extra_info(self, name, default=None): return self._transport.get_extra_info(name, default) - @coroutine - def drain(self): + async def drain(self): """Flush the write buffer. The intended use is to write w.write(data) - yield from w.drain() + await w.drain() """ if self._reader is not None: exc = self._reader.exception() if exc is not None: raise exc - if self._transport is not None: - if self._transport.is_closing(): - # Yield to the event loop so connection_lost() may be - # called. Without this, _drain_helper() would return - # immediately, and code that calls - # write(...); yield from drain() - # in a loop would never call connection_lost(), so it - # would not see an error when the socket is closed. - yield - yield from self._protocol._drain_helper() + if self._transport.is_closing(): + # Yield to the event loop so connection_lost() may be + # called. Without this, _drain_helper() would return + # immediately, and code that calls + # write(...); await drain() + # in a loop would never call connection_lost(), so it + # would not see an error when the socket is closed. + await sleep(0, loop=self._loop) + await self._protocol._drain_helper() class StreamReader: @@ -363,20 +372,20 @@ class StreamReader: def __repr__(self): info = ['StreamReader'] if self._buffer: - info.append('%d bytes' % len(self._buffer)) + info.append(f'{len(self._buffer)} bytes') if self._eof: info.append('eof') if self._limit != _DEFAULT_LIMIT: - info.append('l=%d' % self._limit) + info.append(f'limit={self._limit}') if self._waiter: - info.append('w=%r' % self._waiter) + info.append(f'waiter={self._waiter!r}') if self._exception: - info.append('e=%r' % self._exception) + info.append(f'exception={self._exception!r}') if self._transport: - info.append('t=%r' % self._transport) + info.append(f'transport={self._transport!r}') if self._paused: info.append('paused') - return '<%s>' % ' '.join(info) + return '<{}>'.format(' '.join(info)) def exception(self): return self._exception @@ -437,8 +446,7 @@ class StreamReader: else: self._paused = True - @coroutine - def _wait_for_data(self, func_name): + async def _wait_for_data(self, func_name): """Wait until feed_data() or feed_eof() is called. If stream was paused, automatically resume it. @@ -448,8 +456,9 @@ class StreamReader: # would have an unexpected behaviour. It would not possible to know # which coroutine would get the next data. if self._waiter is not None: - raise RuntimeError('%s() called while another coroutine is ' - 'already waiting for incoming data' % func_name) + raise RuntimeError( + f'{func_name}() called while another coroutine is ' + f'already waiting for incoming data') assert not self._eof, '_wait_for_data after EOF' @@ -461,12 +470,11 @@ class StreamReader: self._waiter = self._loop.create_future() try: - yield from self._waiter + await self._waiter finally: self._waiter = None - @coroutine - def readline(self): + async def readline(self): """Read chunk of data from the stream until newline (b'\n') is found. On success, return chunk that ends with newline. If only partial @@ -485,7 +493,7 @@ class StreamReader: sep = b'\n' seplen = len(sep) try: - line = yield from self.readuntil(sep) + line = await self.readuntil(sep) except IncompleteReadError as e: return e.partial except LimitOverrunError as e: @@ -497,8 +505,7 @@ class StreamReader: raise ValueError(e.args[0]) return line - @coroutine - def readuntil(self, separator=b'\n'): + async def readuntil(self, separator=b'\n'): """Read data from the stream until ``separator`` is found. On success, the data and separator will be removed from the @@ -578,7 +585,7 @@ class StreamReader: raise IncompleteReadError(chunk, None) # _wait_for_data() will resume reading if stream was paused. - yield from self._wait_for_data('readuntil') + await self._wait_for_data('readuntil') if isep > self._limit: raise LimitOverrunError( @@ -589,8 +596,7 @@ class StreamReader: self._maybe_resume_transport() return bytes(chunk) - @coroutine - def read(self, n=-1): + async def read(self, n=-1): """Read up to `n` bytes from the stream. If n is not provided, or set to -1, read until EOF and return all read @@ -624,14 +630,14 @@ class StreamReader: # bytes. So just call self.read(self._limit) until EOF. blocks = [] while True: - block = yield from self.read(self._limit) + block = await self.read(self._limit) if not block: break blocks.append(block) return b''.join(blocks) if not self._buffer and not self._eof: - yield from self._wait_for_data('read') + await self._wait_for_data('read') # This will work right even if buffer is less than n bytes data = bytes(self._buffer[:n]) @@ -640,8 +646,7 @@ class StreamReader: self._maybe_resume_transport() return data - @coroutine - def readexactly(self, n): + async def readexactly(self, n): """Read exactly `n` bytes. Raise an IncompleteReadError if EOF is reached before `n` bytes can be @@ -671,7 +676,7 @@ class StreamReader: self._buffer.clear() raise IncompleteReadError(incomplete, n) - yield from self._wait_for_data('readexactly') + await self._wait_for_data('readexactly') if len(self._buffer) == n: data = bytes(self._buffer) @@ -685,9 +690,8 @@ class StreamReader: def __aiter__(self): return self - @coroutine - def __anext__(self): - val = yield from self.readline() + async def __anext__(self): + val = await self.readline() if val == b'': raise StopAsyncIteration return val diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py index 4c85466859f..90fc00de833 100644 --- a/Lib/asyncio/subprocess.py +++ b/Lib/asyncio/subprocess.py @@ -1,4 +1,4 @@ -__all__ = ['create_subprocess_exec', 'create_subprocess_shell'] +__all__ = 'create_subprocess_exec', 'create_subprocess_shell' import subprocess @@ -6,7 +6,6 @@ from . import events from . import protocols from . import streams from . import tasks -from .coroutines import coroutine from .log import logger @@ -30,12 +29,12 @@ class SubprocessStreamProtocol(streams.FlowControlMixin, def __repr__(self): info = [self.__class__.__name__] if self.stdin is not None: - info.append('stdin=%r' % self.stdin) + info.append(f'stdin={self.stdin!r}') if self.stdout is not None: - info.append('stdout=%r' % self.stdout) + info.append(f'stdout={self.stdout!r}') if self.stderr is not None: - info.append('stderr=%r' % self.stderr) - return '<%s>' % ' '.join(info) + info.append(f'stderr={self.stderr!r}') + return '<{}>'.format(' '.join(info)) def connection_made(self, transport): self._transport = transport @@ -84,7 +83,7 @@ class SubprocessStreamProtocol(streams.FlowControlMixin, reader = self.stderr else: reader = None - if reader != None: + if reader is not None: if exc is None: reader.feed_eof() else: @@ -115,18 +114,15 @@ class Process: self.pid = transport.get_pid() def __repr__(self): - return '<%s %s>' % (self.__class__.__name__, self.pid) + return f'<{self.__class__.__name__} {self.pid}>' @property def returncode(self): return self._transport.get_returncode() - @coroutine - def wait(self): - """Wait until the process exit and return the process return code. - - This method is a coroutine.""" - return (yield from self._transport._wait()) + async def wait(self): + """Wait until the process exit and return the process return code.""" + return await self._transport._wait() def send_signal(self, signal): self._transport.send_signal(signal) @@ -137,15 +133,14 @@ class Process: def kill(self): self._transport.kill() - @coroutine - def _feed_stdin(self, input): + async def _feed_stdin(self, input): debug = self._loop.get_debug() self.stdin.write(input) if debug: - logger.debug('%r communicate: feed stdin (%s bytes)', - self, len(input)) + logger.debug( + '%r communicate: feed stdin (%s bytes)', self, len(input)) try: - yield from self.stdin.drain() + await self.stdin.drain() except (BrokenPipeError, ConnectionResetError) as exc: # communicate() ignores BrokenPipeError and ConnectionResetError if debug: @@ -155,12 +150,10 @@ class Process: logger.debug('%r communicate: close stdin', self) self.stdin.close() - @coroutine - def _noop(self): + async def _noop(self): return None - @coroutine - def _read_stream(self, fd): + async def _read_stream(self, fd): transport = self._transport.get_pipe_transport(fd) if fd == 2: stream = self.stderr @@ -170,15 +163,14 @@ class Process: if self._loop.get_debug(): name = 'stdout' if fd == 1 else 'stderr' logger.debug('%r communicate: read %s', self, name) - output = yield from stream.read() + output = await stream.read() if self._loop.get_debug(): name = 'stdout' if fd == 1 else 'stderr' logger.debug('%r communicate: close %s', self, name) transport.close() return output - @coroutine - def communicate(self, input=None): + async def communicate(self, input=None): if input is not None: stdin = self._feed_stdin(input) else: @@ -191,36 +183,36 @@ class Process: stderr = self._read_stream(2) else: stderr = self._noop() - stdin, stdout, stderr = yield from tasks.gather(stdin, stdout, stderr, - loop=self._loop) - yield from self.wait() + stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr, + loop=self._loop) + await self.wait() return (stdout, stderr) -@coroutine -def create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None, - loop=None, limit=streams._DEFAULT_LIMIT, **kwds): +async def create_subprocess_shell(cmd, stdin=None, stdout=None, stderr=None, + loop=None, limit=streams._DEFAULT_LIMIT, + **kwds): if loop is None: loop = events.get_event_loop() protocol_factory = lambda: SubprocessStreamProtocol(limit=limit, loop=loop) - transport, protocol = yield from loop.subprocess_shell( - protocol_factory, - cmd, stdin=stdin, stdout=stdout, - stderr=stderr, **kwds) + transport, protocol = await loop.subprocess_shell( + protocol_factory, + cmd, stdin=stdin, stdout=stdout, + stderr=stderr, **kwds) return Process(transport, protocol, loop) -@coroutine -def create_subprocess_exec(program, *args, stdin=None, stdout=None, - stderr=None, loop=None, - limit=streams._DEFAULT_LIMIT, **kwds): + +async def create_subprocess_exec(program, *args, stdin=None, stdout=None, + stderr=None, loop=None, + limit=streams._DEFAULT_LIMIT, **kwds): if loop is None: loop = events.get_event_loop() protocol_factory = lambda: SubprocessStreamProtocol(limit=limit, loop=loop) - transport, protocol = yield from loop.subprocess_exec( - protocol_factory, - program, *args, - stdin=stdin, stdout=stdout, - stderr=stderr, **kwds) + transport, protocol = await loop.subprocess_exec( + protocol_factory, + program, *args, + stdin=stdin, stdout=stdout, + stderr=stderr, **kwds) return Process(transport, protocol, loop) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 52fef181cec..3590748c477 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -1,26 +1,46 @@ """Support for tasks, coroutines and the scheduler.""" -__all__ = ['Task', - 'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ALL_COMPLETED', - 'wait', 'wait_for', 'as_completed', 'sleep', 'async', - 'gather', 'shield', 'ensure_future', 'run_coroutine_threadsafe', - ] +__all__ = ( + 'Task', 'create_task', + 'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ALL_COMPLETED', + 'wait', 'wait_for', 'as_completed', 'sleep', + 'gather', 'shield', 'ensure_future', 'run_coroutine_threadsafe', + 'current_task', 'all_tasks', + '_register_task', '_unregister_task', '_enter_task', '_leave_task', +) import concurrent.futures +import contextvars import functools import inspect +import types import warnings import weakref from . import base_tasks -from . import compat from . import coroutines from . import events from . import futures from .coroutines import coroutine -class Task(futures.Future): +def current_task(loop=None): + """Return a currently executed task.""" + if loop is None: + loop = events.get_running_loop() + return _current_tasks.get(loop) + + +def all_tasks(loop=None): + """Return a set of all tasks for the loop.""" + if loop is None: + loop = events.get_event_loop() + return {t for t in _all_tasks if futures._get_loop(t) is loop} + + +class Task(futures._PyFuture): # Inherit Python Task implementation + # from a Python Future implementation. + """A coroutine wrapped in a Future.""" # An important invariant maintained while a Task not done: @@ -32,13 +52,6 @@ class Task(futures.Future): # _wakeup(). When _fut_waiter is not None, one of its callbacks # must be _wakeup(). - # Weak set containing all tasks alive. - _all_tasks = weakref.WeakSet() - - # Dictionary containing tasks that are currently active in - # all running event loops. {EventLoop: Task} - _current_tasks = {} - # If False, don't log a message if the task is destroyed whereas its # status is still pending _log_destroy_pending = True @@ -51,9 +64,13 @@ class Task(futures.Future): None is returned when called not in the context of a Task. """ + warnings.warn("Task.current_task() is deprecated, " + "use asyncio.current_task() instead", + PendingDeprecationWarning, + stacklevel=2) if loop is None: loop = events.get_event_loop() - return cls._current_tasks.get(loop) + return current_task(loop) @classmethod def all_tasks(cls, loop=None): @@ -61,20 +78,29 @@ class Task(futures.Future): By default all tasks for the current event loop are returned. """ - if loop is None: - loop = events.get_event_loop() - return {t for t in cls._all_tasks if t._loop is loop} + warnings.warn("Task.all_tasks() is deprecated, " + "use asyncio.all_tasks() instead", + PendingDeprecationWarning, + stacklevel=2) + return all_tasks(loop) def __init__(self, coro, *, loop=None): - assert coroutines.iscoroutine(coro), repr(coro) super().__init__(loop=loop) if self._source_traceback: del self._source_traceback[-1] - self._coro = coro - self._fut_waiter = None + if not coroutines.iscoroutine(coro): + # raise after Future.__init__(), attrs are required for __del__ + # prevent logging for pending task in __del__ + self._log_destroy_pending = False + raise TypeError(f"a coroutine was expected, got {coro!r}") + self._must_cancel = False - self._loop.call_soon(self._step) - self.__class__._all_tasks.add(self) + self._fut_waiter = None + self._coro = coro + self._context = contextvars.copy_context() + + self._loop.call_soon(self.__step, context=self._context) + _register_task(self) def __del__(self): if self._state == futures._PENDING and self._log_destroy_pending: @@ -85,11 +111,17 @@ class Task(futures.Future): if self._source_traceback: context['source_traceback'] = self._source_traceback self._loop.call_exception_handler(context) - futures.Future.__del__(self) + super().__del__() def _repr_info(self): return base_tasks._task_repr_info(self) + def set_result(self, result): + raise RuntimeError('Task does not support set_result operation') + + def set_exception(self, exception): + raise RuntimeError('Task does not support set_exception operation') + def get_stack(self, *, limit=None): """Return the list of stack frames for this task's coroutine. @@ -153,13 +185,14 @@ class Task(futures.Future): # catches and ignores the cancellation so we may have # to cancel it again later. return True - # It must be the case that self._step is already scheduled. + # It must be the case that self.__step is already scheduled. self._must_cancel = True return True - def _step(self, exc=None): - assert not self.done(), \ - '_step(): already done: {!r}, {!r}'.format(self, exc) + def __step(self, exc=None): + if self.done(): + raise futures.InvalidStateError( + f'_step(): already done: {self!r}, {exc!r}') if self._must_cancel: if not isinstance(exc, futures.CancelledError): exc = futures.CancelledError() @@ -167,7 +200,7 @@ class Task(futures.Future): coro = self._coro self._fut_waiter = None - self.__class__._current_tasks[self._loop] = self + _enter_task(self._loop, self) # Call either coro.throw(exc) or coro.send(None). try: if exc is None: @@ -180,73 +213,72 @@ class Task(futures.Future): if self._must_cancel: # Task is cancelled right before coro stops. self._must_cancel = False - self.set_exception(futures.CancelledError()) + super().set_exception(futures.CancelledError()) else: - self.set_result(exc.value) + super().set_result(exc.value) except futures.CancelledError: super().cancel() # I.e., Future.cancel(self). except Exception as exc: - self.set_exception(exc) + super().set_exception(exc) except BaseException as exc: - self.set_exception(exc) + super().set_exception(exc) raise else: blocking = getattr(result, '_asyncio_future_blocking', None) if blocking is not None: # Yielded Future must come from Future.__iter__(). - if result._loop is not self._loop: + if futures._get_loop(result) is not self._loop: + new_exc = RuntimeError( + f'Task {self!r} got Future ' + f'{result!r} attached to a different loop') self._loop.call_soon( - self._step, - RuntimeError( - 'Task {!r} got Future {!r} attached to a ' - 'different loop'.format(self, result))) + self.__step, new_exc, context=self._context) elif blocking: if result is self: + new_exc = RuntimeError( + f'Task cannot await on itself: {self!r}') self._loop.call_soon( - self._step, - RuntimeError( - 'Task cannot await on itself: {!r}'.format( - self))) + self.__step, new_exc, context=self._context) else: result._asyncio_future_blocking = False - result.add_done_callback(self._wakeup) + result.add_done_callback( + self.__wakeup, context=self._context) self._fut_waiter = result if self._must_cancel: if self._fut_waiter.cancel(): self._must_cancel = False else: + new_exc = RuntimeError( + f'yield was used instead of yield from ' + f'in task {self!r} with {result!r}') self._loop.call_soon( - self._step, - RuntimeError( - 'yield was used instead of yield from ' - 'in task {!r} with {!r}'.format(self, result))) + self.__step, new_exc, context=self._context) + elif result is None: # Bare yield relinquishes control for one event loop iteration. - self._loop.call_soon(self._step) + self._loop.call_soon(self.__step, context=self._context) elif inspect.isgenerator(result): # Yielding a generator is just wrong. + new_exc = RuntimeError( + f'yield was used instead of yield from for ' + f'generator in task {self!r} with {result}') self._loop.call_soon( - self._step, - RuntimeError( - 'yield was used instead of yield from for ' - 'generator in task {!r} with {}'.format( - self, result))) + self.__step, new_exc, context=self._context) else: # Yielding something else is an error. + new_exc = RuntimeError(f'Task got bad yield: {result!r}') self._loop.call_soon( - self._step, - RuntimeError( - 'Task got bad yield: {!r}'.format(result))) + self.__step, new_exc, context=self._context) finally: - self.__class__._current_tasks.pop(self._loop) + _leave_task(self._loop, self) self = None # Needed to break cycles when an exception occurs. - def _wakeup(self, future): + def __wakeup(self, future): try: future.result() except Exception as exc: # This may also be a cancellation. - self._step(exc) + self.__step(exc) else: # Don't pass the value of `future.result()` explicitly, # as `Future.__iter__` and `Future.__await__` don't need it. @@ -254,7 +286,7 @@ class Task(futures.Future): # Python eval loop would use `.send(value)` method call, # instead of `__next__()`, which is slower for futures # that return non-generator iterators from their `__iter__`. - self._step() + self.__step() self = None # Needed to break cycles when an exception occurs. @@ -270,6 +302,15 @@ else: Task = _CTask = _asyncio.Task +def create_task(coro): + """Schedule the execution of a coroutine object in a spawn task. + + Return a Task object. + """ + loop = events.get_running_loop() + return loop.create_task(coro) + + # wait() and as_completed() similar to those in PEP 3148. FIRST_COMPLETED = concurrent.futures.FIRST_COMPLETED @@ -277,8 +318,7 @@ FIRST_EXCEPTION = concurrent.futures.FIRST_EXCEPTION ALL_COMPLETED = concurrent.futures.ALL_COMPLETED -@coroutine -def wait(fs, *, loop=None, timeout=None, return_when=ALL_COMPLETED): +async def wait(fs, *, loop=None, timeout=None, return_when=ALL_COMPLETED): """Wait for the Futures and coroutines given by fs to complete. The sequence futures must not be empty. @@ -289,24 +329,24 @@ def wait(fs, *, loop=None, timeout=None, return_when=ALL_COMPLETED): Usage: - done, pending = yield from asyncio.wait(fs) + done, pending = await asyncio.wait(fs) Note: This does not raise TimeoutError! Futures that aren't done when the timeout occurs are returned in the second set. """ if futures.isfuture(fs) or coroutines.iscoroutine(fs): - raise TypeError("expect a list of futures, not %s" % type(fs).__name__) + raise TypeError(f"expect a list of futures, not {type(fs).__name__}") if not fs: raise ValueError('Set of coroutines/Futures is empty.') if return_when not in (FIRST_COMPLETED, FIRST_EXCEPTION, ALL_COMPLETED): - raise ValueError('Invalid return_when value: {}'.format(return_when)) + raise ValueError(f'Invalid return_when value: {return_when}') if loop is None: loop = events.get_event_loop() fs = {ensure_future(f, loop=loop) for f in set(fs)} - return (yield from _wait(fs, timeout, return_when, loop)) + return await _wait(fs, timeout, return_when, loop) def _release_waiter(waiter, *args): @@ -314,8 +354,7 @@ def _release_waiter(waiter, *args): waiter.set_result(None) -@coroutine -def wait_for(fut, timeout, *, loop=None): +async def wait_for(fut, timeout, *, loop=None): """Wait for the single Future or coroutine to complete, with timeout. Coroutine will be wrapped in Task. @@ -332,7 +371,7 @@ def wait_for(fut, timeout, *, loop=None): loop = events.get_event_loop() if timeout is None: - return (yield from fut) + return await fut if timeout <= 0: fut = ensure_future(fut, loop=loop) @@ -353,7 +392,7 @@ def wait_for(fut, timeout, *, loop=None): try: # wait until the future completes or the timeout try: - yield from waiter + await waiter except futures.CancelledError: fut.remove_done_callback(cb) fut.cancel() @@ -369,8 +408,7 @@ def wait_for(fut, timeout, *, loop=None): timeout_handle.cancel() -@coroutine -def _wait(fs, timeout, return_when, loop): +async def _wait(fs, timeout, return_when, loop): """Internal helper for wait() and wait_for(). The fs argument must be a collection of Futures. @@ -398,7 +436,7 @@ def _wait(fs, timeout, return_when, loop): f.add_done_callback(_on_completion) try: - yield from waiter + await waiter finally: if timeout_handle is not None: timeout_handle.cancel() @@ -424,16 +462,16 @@ def as_completed(fs, *, loop=None, timeout=None): This differs from PEP 3148; the proper way to use this is: for f in as_completed(fs): - result = yield from f # The 'yield from' may raise. + result = await f # The 'await' may raise. # Use result. - If a timeout is specified, the 'yield from' will raise + If a timeout is specified, the 'await' will raise TimeoutError when the timeout occurs before all Futures are done. Note: The futures 'f' are not necessarily members of fs. """ if futures.isfuture(fs) or coroutines.iscoroutine(fs): - raise TypeError("expect a list of futures, not %s" % type(fs).__name__) + raise TypeError(f"expect a list of futures, not {type(fs).__name__}") loop = loop if loop is not None else events.get_event_loop() todo = {ensure_future(f, loop=loop) for f in set(fs)} from .queues import Queue # Import here to avoid circular import problem. @@ -454,9 +492,8 @@ def as_completed(fs, *, loop=None, timeout=None): if not todo and timeout_handle is not None: timeout_handle.cancel() - @coroutine - def _wait_for_one(): - f = yield from done.get() + async def _wait_for_one(): + f = await done.get() if f is None: # Dummy value from _on_timeout(). raise futures.TimeoutError @@ -470,52 +507,43 @@ def as_completed(fs, *, loop=None, timeout=None): yield _wait_for_one() -@coroutine -def sleep(delay, result=None, *, loop=None): +@types.coroutine +def __sleep0(): + """Skip one event loop run cycle. + + This is a private helper for 'asyncio.sleep()', used + when the 'delay' is set to 0. It uses a bare 'yield' + expression (which Task.__step knows how to handle) + instead of creating a Future object. + """ + yield + + +async def sleep(delay, result=None, *, loop=None): """Coroutine that completes after a given time (in seconds).""" - if delay == 0: - yield + if delay <= 0: + await __sleep0() return result if loop is None: loop = events.get_event_loop() future = loop.create_future() - h = future._loop.call_later(delay, - futures._set_result_unless_cancelled, - future, result) + h = loop.call_later(delay, + futures._set_result_unless_cancelled, + future, result) try: - return (yield from future) + return await future finally: h.cancel() -def async_(coro_or_future, *, loop=None): - """Wrap a coroutine in a future. - - If the argument is a Future, it is returned directly. - - This function is deprecated in 3.5. Use asyncio.ensure_future() instead. - """ - - warnings.warn("asyncio.async() function is deprecated, use ensure_future()", - DeprecationWarning, - stacklevel=2) - - return ensure_future(coro_or_future, loop=loop) - -# Silence DeprecationWarning: -globals()['async'] = async_ -async_.__name__ = 'async' -del async_ - - def ensure_future(coro_or_future, *, loop=None): """Wrap a coroutine or an awaitable in a future. If the argument is a Future, it is returned directly. """ if futures.isfuture(coro_or_future): - if loop is not None and loop is not coro_or_future._loop: + if loop is not None and loop is not futures._get_loop(coro_or_future): raise ValueError('loop argument must agree with Future') return coro_or_future elif coroutines.iscoroutine(coro_or_future): @@ -525,7 +553,7 @@ def ensure_future(coro_or_future, *, loop=None): if task._source_traceback: del task._source_traceback[-1] return task - elif compat.PY35 and inspect.isawaitable(coro_or_future): + elif inspect.isawaitable(coro_or_future): return ensure_future(_wrap_awaitable(coro_or_future), loop=loop) else: raise TypeError('An asyncio.Future, a coroutine or an awaitable is ' @@ -565,8 +593,7 @@ class _GatheringFuture(futures.Future): def gather(*coros_or_futures, loop=None, return_exceptions=False): - """Return a future aggregating results from the given coroutines - or futures. + """Return a future aggregating results from the given coroutines/futures. Coroutines will be wrapped in a future and scheduled in the event loop. They will not necessarily be scheduled in the same order as @@ -595,56 +622,76 @@ def gather(*coros_or_futures, loop=None, return_exceptions=False): outer.set_result([]) return outer - arg_to_fut = {} - for arg in set(coros_or_futures): - if not futures.isfuture(arg): - fut = ensure_future(arg, loop=loop) - if loop is None: - loop = fut._loop - # The caller cannot control this future, the "destroy pending task" - # warning should not be emitted. - fut._log_destroy_pending = False - else: - fut = arg - if loop is None: - loop = fut._loop - elif fut._loop is not loop: - raise ValueError("futures are tied to different event loops") - arg_to_fut[arg] = fut - - children = [arg_to_fut[arg] for arg in coros_or_futures] - nchildren = len(children) - outer = _GatheringFuture(children, loop=loop) - nfinished = 0 - results = [None] * nchildren - - def _done_callback(i, fut): + def _done_callback(fut): nonlocal nfinished + nfinished += 1 + if outer.done(): if not fut.cancelled(): # Mark exception retrieved. fut.exception() return - if fut.cancelled(): - res = futures.CancelledError() - if not return_exceptions: - outer.set_exception(res) + if not return_exceptions: + if fut.cancelled(): + # Check if 'fut' is cancelled first, as + # 'fut.exception()' will *raise* a CancelledError + # instead of returning it. + exc = futures.CancelledError() + outer.set_exception(exc) return - elif fut._exception is not None: - res = fut.exception() # Mark exception retrieved. - if not return_exceptions: - outer.set_exception(res) - return - else: - res = fut._result - results[i] = res - nfinished += 1 - if nfinished == nchildren: + else: + exc = fut.exception() + if exc is not None: + outer.set_exception(exc) + return + + if nfinished == nfuts: + # All futures are done; create a list of results + # and set it to the 'outer' future. + results = [] + + for fut in children: + if fut.cancelled(): + # Check if 'fut' is cancelled first, as + # 'fut.exception()' will *raise* a CancelledError + # instead of returning it. + res = futures.CancelledError() + else: + res = fut.exception() + if res is None: + res = fut.result() + results.append(res) + outer.set_result(results) - for i, fut in enumerate(children): - fut.add_done_callback(functools.partial(_done_callback, i)) + arg_to_fut = {} + children = [] + nfuts = 0 + nfinished = 0 + for arg in coros_or_futures: + if arg not in arg_to_fut: + fut = ensure_future(arg, loop=loop) + if loop is None: + loop = futures._get_loop(fut) + if fut is not arg: + # 'arg' was not a Future, therefore, 'fut' is a new + # Future created specifically for 'arg'. Since the caller + # can't control it, disable the "destroy pending task" + # warning. + fut._log_destroy_pending = False + + nfuts += 1 + arg_to_fut[arg] = fut + fut.add_done_callback(_done_callback) + + else: + # There's a duplicate Future object in coros_or_futures. + fut = arg_to_fut[arg] + + children.append(fut) + + outer = _GatheringFuture(children, loop=loop) return outer @@ -653,11 +700,11 @@ def shield(arg, *, loop=None): The statement - res = yield from shield(something()) + res = await shield(something()) is exactly equivalent to the statement - res = yield from something() + res = await something() *except* that if the coroutine containing it is cancelled, the task running in something() is not cancelled. From the POV of @@ -670,7 +717,7 @@ def shield(arg, *, loop=None): you can combine shield() with a try/except clause, as follows: try: - res = yield from shield(something()) + res = await shield(something()) except CancelledError: res = None """ @@ -678,7 +725,7 @@ def shield(arg, *, loop=None): if inner.done(): # Shortcut. return inner - loop = inner._loop + loop = futures._get_loop(inner) outer = loop.create_future() def _done_callback(inner): @@ -720,3 +767,56 @@ def run_coroutine_threadsafe(coro, loop): loop.call_soon_threadsafe(callback) return future + + +# WeakSet containing all alive tasks. +_all_tasks = weakref.WeakSet() + +# Dictionary containing tasks that are currently active in +# all running event loops. {EventLoop: Task} +_current_tasks = {} + + +def _register_task(task): + """Register a new task in asyncio as executed by loop.""" + _all_tasks.add(task) + + +def _enter_task(loop, task): + current_task = _current_tasks.get(loop) + if current_task is not None: + raise RuntimeError(f"Cannot enter into task {task!r} while another " + f"task {current_task!r} is being executed.") + _current_tasks[loop] = task + + +def _leave_task(loop, task): + current_task = _current_tasks.get(loop) + if current_task is not task: + raise RuntimeError(f"Leaving task {task!r} does not match " + f"the current task {current_task!r}.") + del _current_tasks[loop] + + +def _unregister_task(task): + """Unregister a task.""" + _all_tasks.discard(task) + + +_py_register_task = _register_task +_py_unregister_task = _unregister_task +_py_enter_task = _enter_task +_py_leave_task = _leave_task + + +try: + from _asyncio import (_register_task, _unregister_task, + _enter_task, _leave_task, + _all_tasks, _current_tasks) +except ImportError: + pass +else: + _c_register_task = _register_task + _c_unregister_task = _unregister_task + _c_enter_task = _enter_task + _c_leave_task = _leave_task diff --git a/Lib/asyncio/transports.py b/Lib/asyncio/transports.py index a94079f433a..233bbb53cb6 100644 --- a/Lib/asyncio/transports.py +++ b/Lib/asyncio/transports.py @@ -1,8 +1,9 @@ """Abstract Transport class.""" -__all__ = ['BaseTransport', 'ReadTransport', 'WriteTransport', - 'Transport', 'DatagramTransport', 'SubprocessTransport', - ] +__all__ = ( + 'BaseTransport', 'ReadTransport', 'WriteTransport', + 'Transport', 'DatagramTransport', 'SubprocessTransport', +) class BaseTransport: @@ -43,6 +44,10 @@ class BaseTransport: class ReadTransport(BaseTransport): """Interface for read-only transports.""" + def is_reading(self): + """Return True if the transport is receiving.""" + raise NotImplementedError + def pause_reading(self): """Pause the receiving end. @@ -267,7 +272,7 @@ class _FlowControlMixin(Transport): def _maybe_resume_protocol(self): if (self._protocol_paused and - self.get_write_buffer_size() <= self._low_water): + self.get_write_buffer_size() <= self._low_water): self._protocol_paused = False try: self._protocol.resume_writing() @@ -285,14 +290,16 @@ class _FlowControlMixin(Transport): def _set_write_buffer_limits(self, high=None, low=None): if high is None: if low is None: - high = 64*1024 + high = 64 * 1024 else: - high = 4*low + high = 4 * low if low is None: low = high // 4 + if not high >= low >= 0: - raise ValueError('high (%r) must be >= low (%r) must be >= 0' % - (high, low)) + raise ValueError( + f'high ({high!r}) must be >= low ({low!r}) must be >= 0') + self._high_water = high self._low_water = low diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index bf682a1a98a..6cac137cacb 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -1,7 +1,9 @@ """Selector event loop for Unix with signal handling.""" import errno +import io import os +import selectors import signal import socket import stat @@ -18,16 +20,16 @@ from . import coroutines from . import events from . import futures from . import selector_events -from . import selectors from . import transports -from .coroutines import coroutine from .log import logger -__all__ = ['SelectorEventLoop', - 'AbstractChildWatcher', 'SafeChildWatcher', - 'FastChildWatcher', 'DefaultEventLoopPolicy', - ] +__all__ = ( + 'SelectorEventLoop', + 'AbstractChildWatcher', 'SafeChildWatcher', + 'FastChildWatcher', 'DefaultEventLoopPolicy', +) + if sys.platform == 'win32': # pragma: no cover raise ImportError('Signals are not really supported on Windows') @@ -38,13 +40,6 @@ def _sighandler_noop(signum, frame): pass -try: - _fspath = os.fspath -except AttributeError: - # Python 3.5 or earlier - _fspath = lambda path: path - - class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): """Unix event loop. @@ -55,13 +50,19 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): super().__init__(selector) self._signal_handlers = {} - def _socketpair(self): - return socket.socketpair() - def close(self): super().close() - for sig in list(self._signal_handlers): - self.remove_signal_handler(sig) + if not sys.is_finalizing(): + for sig in list(self._signal_handlers): + self.remove_signal_handler(sig) + else: + if self._signal_handlers: + warnings.warn(f"Closing the loop {self!r} " + f"on interpreter shutdown " + f"stage, skipping signal handlers removal", + ResourceWarning, + source=self) + self._signal_handlers.clear() def _process_self_data(self, data): for signum in data: @@ -76,8 +77,8 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): Raise ValueError if the signal number is invalid or uncatchable. Raise RuntimeError if there is a problem setting up the handler. """ - if (coroutines.iscoroutine(callback) - or coroutines.iscoroutinefunction(callback)): + if (coroutines.iscoroutine(callback) or + coroutines.iscoroutinefunction(callback)): raise TypeError("coroutines cannot be used " "with add_signal_handler()") self._check_signal(sig) @@ -91,7 +92,7 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): except (ValueError, OSError) as exc: raise RuntimeError(str(exc)) - handle = events.Handle(callback, args, self) + handle = events.Handle(callback, args, self, None) self._signal_handlers[sig] = handle try: @@ -111,7 +112,7 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): logger.info('set_wakeup_fd(-1) failed: %s', nexc) if exc.errno == errno.EINVAL: - raise RuntimeError('sig {} cannot be caught'.format(sig)) + raise RuntimeError(f'sig {sig} cannot be caught') else: raise @@ -145,7 +146,7 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): signal.signal(sig, handler) except OSError as exc: if exc.errno == errno.EINVAL: - raise RuntimeError('sig {} cannot be caught'.format(sig)) + raise RuntimeError(f'sig {sig} cannot be caught') else: raise @@ -164,11 +165,10 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): Raise RuntimeError if there is a problem setting up the handler. """ if not isinstance(sig, int): - raise TypeError('sig must be an int, not {!r}'.format(sig)) + raise TypeError(f'sig must be an int, not {sig!r}') if not (1 <= sig < signal.NSIG): - raise ValueError( - 'sig {} out of range(1, {})'.format(sig, signal.NSIG)) + raise ValueError(f'sig {sig} out of range(1, {signal.NSIG})') def _make_read_pipe_transport(self, pipe, protocol, waiter=None, extra=None): @@ -178,10 +178,9 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): extra=None): return _UnixWritePipeTransport(self, pipe, protocol, waiter, extra) - @coroutine - def _make_subprocess_transport(self, protocol, args, shell, - stdin, stdout, stderr, bufsize, - extra=None, **kwargs): + async def _make_subprocess_transport(self, protocol, args, shell, + stdin, stdout, stderr, bufsize, + extra=None, **kwargs): with events.get_child_watcher() as watcher: waiter = self.create_future() transp = _UnixSubprocessTransport(self, protocol, args, shell, @@ -192,29 +191,22 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): watcher.add_child_handler(transp.get_pid(), self._child_watcher_callback, transp) try: - yield from waiter - except Exception as exc: - # Workaround CPython bug #23353: using yield/yield-from in an - # except block of a generator doesn't clear properly - # sys.exc_info() - err = exc - else: - err = None - - if err is not None: + await waiter + except Exception: transp.close() - yield from transp._wait() - raise err + await transp._wait() + raise return transp def _child_watcher_callback(self, pid, returncode, transp): self.call_soon_threadsafe(transp._process_exited, returncode) - @coroutine - def create_unix_connection(self, protocol_factory, path, *, - ssl=None, sock=None, - server_hostname=None): + async def create_unix_connection( + self, protocol_factory, path=None, *, + ssl=None, sock=None, + server_hostname=None, + ssl_handshake_timeout=None): assert server_hostname is None or isinstance(server_hostname, str) if ssl: if server_hostname is None: @@ -223,16 +215,20 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): else: if server_hostname is not None: raise ValueError('server_hostname is only meaningful with ssl') + if ssl_handshake_timeout is not None: + raise ValueError( + 'ssl_handshake_timeout is only meaningful with ssl') if path is not None: if sock is not None: raise ValueError( 'path and sock can not be specified at the same time') + path = os.fspath(path) sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) try: sock.setblocking(False) - yield from self.sock_connect(sock, path) + await self.sock_connect(sock, path) except: sock.close() raise @@ -241,28 +237,34 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): if sock is None: raise ValueError('no path and sock were specified') if (sock.family != socket.AF_UNIX or - not base_events._is_stream_socket(sock)): + sock.type != socket.SOCK_STREAM): raise ValueError( - 'A UNIX Domain Stream Socket was expected, got {!r}' - .format(sock)) + f'A UNIX Domain Stream Socket was expected, got {sock!r}') sock.setblocking(False) - transport, protocol = yield from self._create_connection_transport( - sock, protocol_factory, ssl, server_hostname) + transport, protocol = await self._create_connection_transport( + sock, protocol_factory, ssl, server_hostname, + ssl_handshake_timeout=ssl_handshake_timeout) return transport, protocol - @coroutine - def create_unix_server(self, protocol_factory, path=None, *, - sock=None, backlog=100, ssl=None): + async def create_unix_server( + self, protocol_factory, path=None, *, + sock=None, backlog=100, ssl=None, + ssl_handshake_timeout=None, + start_serving=True): if isinstance(ssl, bool): raise TypeError('ssl argument must be an SSLContext or None') + if ssl_handshake_timeout is not None and not ssl: + raise ValueError( + 'ssl_handshake_timeout is only meaningful with ssl') + if path is not None: if sock is not None: raise ValueError( 'path and sock can not be specified at the same time') - path = _fspath(path) + path = os.fspath(path) sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # Check for abstract socket. `str` and `bytes` paths are supported. @@ -274,7 +276,8 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): pass except OSError as err: # Directory may have permissions only to create socket. - logger.error('Unable to check or remove stale UNIX socket %r: %r', path, err) + logger.error('Unable to check or remove stale UNIX socket ' + '%r: %r', path, err) try: sock.bind(path) @@ -283,7 +286,7 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): if exc.errno == errno.EADDRINUSE: # Let's improve the error message by adding # with what exact address it occurs. - msg = 'Address {!r} is already in use'.format(path) + msg = f'Address {path!r} is already in use' raise OSError(errno.EADDRINUSE, msg) from None else: raise @@ -296,28 +299,120 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): 'path was not specified, and no sock specified') if (sock.family != socket.AF_UNIX or - not base_events._is_stream_socket(sock)): + sock.type != socket.SOCK_STREAM): raise ValueError( - 'A UNIX Domain Stream Socket was expected, got {!r}' - .format(sock)) + f'A UNIX Domain Stream Socket was expected, got {sock!r}') - server = base_events.Server(self, [sock]) - sock.listen(backlog) sock.setblocking(False) - self._start_serving(protocol_factory, sock, ssl, server) + server = base_events.Server(self, [sock], protocol_factory, + ssl, backlog, ssl_handshake_timeout) + if start_serving: + server._start_serving() + return server + async def _sock_sendfile_native(self, sock, file, offset, count): + try: + os.sendfile + except AttributeError as exc: + raise events.SendfileNotAvailableError( + "os.sendfile() is not available") + try: + fileno = file.fileno() + except (AttributeError, io.UnsupportedOperation) as err: + raise events.SendfileNotAvailableError("not a regular file") + try: + fsize = os.fstat(fileno).st_size + except OSError as err: + raise events.SendfileNotAvailableError("not a regular file") + blocksize = count if count else fsize + if not blocksize: + return 0 # empty file -if hasattr(os, 'set_blocking'): - def _set_nonblocking(fd): - os.set_blocking(fd, False) -else: - import fcntl + fut = self.create_future() + self._sock_sendfile_native_impl(fut, None, sock, fileno, + offset, count, blocksize, 0) + return await fut - def _set_nonblocking(fd): - flags = fcntl.fcntl(fd, fcntl.F_GETFL) - flags = flags | os.O_NONBLOCK - fcntl.fcntl(fd, fcntl.F_SETFL, flags) + def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, + offset, count, blocksize, total_sent): + fd = sock.fileno() + if registered_fd is not None: + # Remove the callback early. It should be rare that the + # selector says the fd is ready but the call still returns + # EAGAIN, and I am willing to take a hit in that case in + # order to simplify the common case. + self.remove_writer(registered_fd) + if fut.cancelled(): + self._sock_sendfile_update_filepos(fileno, offset, total_sent) + return + if count: + blocksize = count - total_sent + if blocksize <= 0: + self._sock_sendfile_update_filepos(fileno, offset, total_sent) + fut.set_result(total_sent) + return + + try: + sent = os.sendfile(fd, fileno, offset, blocksize) + except (BlockingIOError, InterruptedError): + if registered_fd is None: + self._sock_add_cancellation_callback(fut, sock) + self.add_writer(fd, self._sock_sendfile_native_impl, fut, + fd, sock, fileno, + offset, count, blocksize, total_sent) + except OSError as exc: + if (registered_fd is not None and + exc.errno == errno.ENOTCONN and + type(exc) is not ConnectionError): + # If we have an ENOTCONN and this isn't a first call to + # sendfile(), i.e. the connection was closed in the middle + # of the operation, normalize the error to ConnectionError + # to make it consistent across all Posix systems. + new_exc = ConnectionError( + "socket is not connected", errno.ENOTCONN) + new_exc.__cause__ = exc + exc = new_exc + if total_sent == 0: + # We can get here for different reasons, the main + # one being 'file' is not a regular mmap(2)-like + # file, in which case we'll fall back on using + # plain send(). + err = events.SendfileNotAvailableError( + "os.sendfile call failed") + self._sock_sendfile_update_filepos(fileno, offset, total_sent) + fut.set_exception(err) + else: + self._sock_sendfile_update_filepos(fileno, offset, total_sent) + fut.set_exception(exc) + except Exception as exc: + self._sock_sendfile_update_filepos(fileno, offset, total_sent) + fut.set_exception(exc) + else: + if sent == 0: + # EOF + self._sock_sendfile_update_filepos(fileno, offset, total_sent) + fut.set_result(total_sent) + else: + offset += sent + total_sent += sent + if registered_fd is None: + self._sock_add_cancellation_callback(fut, sock) + self.add_writer(fd, self._sock_sendfile_native_impl, fut, + fd, sock, fileno, + offset, count, blocksize, total_sent) + + def _sock_sendfile_update_filepos(self, fileno, offset, total_sent): + if total_sent > 0: + os.lseek(fileno, offset, os.SEEK_SET) + + def _sock_add_cancellation_callback(self, fut, sock): + def cb(fut): + if fut.cancelled(): + fd = sock.fileno() + if fd != -1: + self.remove_writer(fd) + fut.add_done_callback(cb) class _UnixReadPipeTransport(transports.ReadTransport): @@ -342,7 +437,7 @@ class _UnixReadPipeTransport(transports.ReadTransport): self._protocol = None raise ValueError("Pipe transport is for pipes/sockets only.") - _set_nonblocking(self._fileno) + os.set_blocking(self._fileno, False) self._loop.call_soon(self._protocol.connection_made, self) # only start reading when connection_made() has been called @@ -359,12 +454,11 @@ class _UnixReadPipeTransport(transports.ReadTransport): info.append('closed') elif self._closing: info.append('closing') - info.append('fd=%s' % self._fileno) + info.append(f'fd={self._fileno}') selector = getattr(self._loop, '_selector', None) if self._pipe is not None and selector is not None: polling = selector_events._test_selector_event( - selector, - self._fileno, selectors.EVENT_READ) + selector, self._fileno, selectors.EVENT_READ) if polling: info.append('polling') else: @@ -373,7 +467,7 @@ class _UnixReadPipeTransport(transports.ReadTransport): info.append('open') else: info.append('closed') - return '<%s>' % ' '.join(info) + return '<{}>'.format(' '.join(info)) def _read_ready(self): try: @@ -414,7 +508,7 @@ class _UnixReadPipeTransport(transports.ReadTransport): def __del__(self): if self._pipe is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning, + warnings.warn(f"unclosed transport {self!r}", ResourceWarning, source=self) self._pipe.close() @@ -471,7 +565,7 @@ class _UnixWritePipeTransport(transports._FlowControlMixin, raise ValueError("Pipe transport is only for " "pipes, sockets and character devices") - _set_nonblocking(self._fileno) + os.set_blocking(self._fileno, False) self._loop.call_soon(self._protocol.connection_made, self) # On AIX, the reader trick (to be notified when the read end of the @@ -493,24 +587,23 @@ class _UnixWritePipeTransport(transports._FlowControlMixin, info.append('closed') elif self._closing: info.append('closing') - info.append('fd=%s' % self._fileno) + info.append(f'fd={self._fileno}') selector = getattr(self._loop, '_selector', None) if self._pipe is not None and selector is not None: polling = selector_events._test_selector_event( - selector, - self._fileno, selectors.EVENT_WRITE) + selector, self._fileno, selectors.EVENT_WRITE) if polling: info.append('polling') else: info.append('idle') bufsize = self.get_write_buffer_size() - info.append('bufsize=%s' % bufsize) + info.append(f'bufsize={bufsize}') elif self._pipe is not None: info.append('open') else: info.append('closed') - return '<%s>' % ' '.join(info) + return '<{}>'.format(' '.join(info)) def get_write_buffer_size(self): return len(self._buffer) @@ -611,7 +704,7 @@ class _UnixWritePipeTransport(transports._FlowControlMixin, def __del__(self): if self._pipe is not None: - warnings.warn("unclosed transport %r" % self, ResourceWarning, + warnings.warn(f"unclosed transport {self!r}", ResourceWarning, source=self) self._pipe.close() @@ -650,22 +743,6 @@ class _UnixWritePipeTransport(transports._FlowControlMixin, self._loop = None -if hasattr(os, 'set_inheritable'): - # Python 3.4 and newer - _set_inheritable = os.set_inheritable -else: - import fcntl - - def _set_inheritable(fd, inheritable): - cloexec_flag = getattr(fcntl, 'FD_CLOEXEC', 1) - - old = fcntl.fcntl(fd, fcntl.F_GETFD) - if not inheritable: - fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) - else: - fcntl.fcntl(fd, fcntl.F_SETFD, old & ~cloexec_flag) - - class _UnixSubprocessTransport(base_subprocess.BaseSubprocessTransport): def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): @@ -676,13 +753,7 @@ class _UnixSubprocessTransport(base_subprocess.BaseSubprocessTransport): # socket (which we use in order to detect closing of the # other end). Notably this is needed on AIX, and works # just fine on other platforms. - stdin, stdin_w = self._loop._socketpair() - - # Mark the write end of the stdin pipe as non-inheritable, - # needed by close_fds=False on Python 3.3 and older - # (Python 3.4 implements the PEP 446, socketpair returns - # non-inheritable sockets) - _set_inheritable(stdin_w.fileno(), False) + stdin, stdin_w = socket.socketpair() self._proc = subprocess.Popen( args, shell=shell, stdin=stdin, stdout=stdout, stderr=stderr, universal_newlines=False, bufsize=bufsize, **kwargs) @@ -1037,8 +1108,8 @@ class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): super().set_event_loop(loop) - if self._watcher is not None and \ - isinstance(threading.current_thread(), threading._MainThread): + if (self._watcher is not None and + isinstance(threading.current_thread(), threading._MainThread)): self._watcher.attach_loop(loop) def get_child_watcher(self): @@ -1061,5 +1132,6 @@ class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): self._watcher = watcher + SelectorEventLoop = _UnixSelectorEventLoop DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index 6045ba029e5..f91fcddb2aa 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -1,5 +1,6 @@ """Selector and proactor event loops for Windows.""" +import _overlapped import _winapi import errno import math @@ -14,14 +15,13 @@ from . import proactor_events from . import selector_events from . import tasks from . import windows_utils -from . import _overlapped -from .coroutines import coroutine from .log import logger -__all__ = ['SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor', - 'DefaultEventLoopPolicy', - ] +__all__ = ( + 'SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor', + 'DefaultEventLoopPolicy', +) NULL = 0 @@ -52,7 +52,7 @@ class _OverlappedFuture(futures.Future): info = super()._repr_info() if self._ov is not None: state = 'pending' if self._ov.pending else 'completed' - info.insert(1, 'overlapped=<%s, %#x>' % (state, self._ov.address)) + info.insert(1, f'overlapped=<{state}, {self._ov.address:#x}>') return info def _cancel_overlapped(self): @@ -108,12 +108,12 @@ class _BaseWaitHandleFuture(futures.Future): def _repr_info(self): info = super()._repr_info() - info.append('handle=%#x' % self._handle) + info.append(f'handle={self._handle:#x}') if self._handle is not None: state = 'signaled' if self._poll() else 'waiting' info.append(state) if self._wait_handle is not None: - info.append('wait_handle=%#x' % self._wait_handle) + info.append(f'wait_handle={self._wait_handle:#x}') return info def _unregister_wait_cb(self, fut): @@ -296,9 +296,6 @@ class PipeServer(object): class _WindowsSelectorEventLoop(selector_events.BaseSelectorEventLoop): """Windows version of selector event loop.""" - def _socketpair(self): - return windows_utils.socketpair() - class ProactorEventLoop(proactor_events.BaseProactorEventLoop): """Windows version of proactor event loop using IOCP.""" @@ -308,20 +305,15 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop): proactor = IocpProactor() super().__init__(proactor) - def _socketpair(self): - return windows_utils.socketpair() - - @coroutine - def create_pipe_connection(self, protocol_factory, address): + async def create_pipe_connection(self, protocol_factory, address): f = self._proactor.connect_pipe(address) - pipe = yield from f + pipe = await f protocol = protocol_factory() trans = self._make_duplex_pipe_transport(pipe, protocol, extra={'addr': address}) return trans, protocol - @coroutine - def start_serving_pipe(self, protocol_factory, address): + async def start_serving_pipe(self, protocol_factory, address): server = PipeServer(address) def loop_accept_pipe(f=None): @@ -367,28 +359,20 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop): self.call_soon(loop_accept_pipe) return [server] - @coroutine - def _make_subprocess_transport(self, protocol, args, shell, - stdin, stdout, stderr, bufsize, - extra=None, **kwargs): + async def _make_subprocess_transport(self, protocol, args, shell, + stdin, stdout, stderr, bufsize, + extra=None, **kwargs): waiter = self.create_future() transp = _WindowsSubprocessTransport(self, protocol, args, shell, stdin, stdout, stderr, bufsize, waiter=waiter, extra=extra, **kwargs) try: - yield from waiter - except Exception as exc: - # Workaround CPython bug #23353: using yield/yield-from in an - # except block of a generator doesn't clear properly sys.exc_info() - err = exc - else: - err = None - - if err is not None: + await waiter + except Exception: transp.close() - yield from transp._wait() - raise err + await transp._wait() + raise return transp @@ -441,7 +425,8 @@ class IocpProactor: try: return ov.getresult() except OSError as exc: - if exc.winerror == _overlapped.ERROR_NETNAME_DELETED: + if exc.winerror in (_overlapped.ERROR_NETNAME_DELETED, + _overlapped.ERROR_OPERATION_ABORTED): raise ConnectionResetError(*exc.args) else: raise @@ -463,7 +448,8 @@ class IocpProactor: try: return ov.getresult() except OSError as exc: - if exc.winerror == _overlapped.ERROR_NETNAME_DELETED: + if exc.winerror in (_overlapped.ERROR_NETNAME_DELETED, + _overlapped.ERROR_OPERATION_ABORTED): raise ConnectionResetError(*exc.args) else: raise @@ -482,7 +468,8 @@ class IocpProactor: try: return ov.getresult() except OSError as exc: - if exc.winerror == _overlapped.ERROR_NETNAME_DELETED: + if exc.winerror in (_overlapped.ERROR_NETNAME_DELETED, + _overlapped.ERROR_OPERATION_ABORTED): raise ConnectionResetError(*exc.args) else: raise @@ -504,11 +491,10 @@ class IocpProactor: conn.settimeout(listener.gettimeout()) return conn, conn.getpeername() - @coroutine - def accept_coro(future, conn): + async def accept_coro(future, conn): # Coroutine closing the accept socket if the future is cancelled try: - yield from future + await future except futures.CancelledError: conn.close() raise @@ -558,13 +544,12 @@ class IocpProactor: return self._register(ov, pipe, finish_accept_pipe) - @coroutine - def connect_pipe(self, address): + async def connect_pipe(self, address): delay = CONNECT_PIPE_INIT_DELAY while True: - # Unfortunately there is no way to do an overlapped connect to a pipe. - # Call CreateFile() in a loop until it doesn't fail with - # ERROR_PIPE_BUSY + # Unfortunately there is no way to do an overlapped connect to + # a pipe. Call CreateFile() in a loop until it doesn't fail with + # ERROR_PIPE_BUSY. try: handle = _overlapped.ConnectPipe(address) break @@ -574,7 +559,7 @@ class IocpProactor: # ConnectPipe() failed with ERROR_PIPE_BUSY: retry later delay = min(delay * 2, CONNECT_PIPE_MAX_DELAY) - yield from tasks.sleep(delay, loop=self._loop) + await tasks.sleep(delay, loop=self._loop) return windows_utils.PipeHandle(handle) @@ -729,7 +714,7 @@ class IocpProactor: f.set_result(value) self._results.append(f) - # Remove unregisted futures + # Remove unregistered futures for ov in self._unregistered: self._cache.pop(ov.address, None) self._unregistered.clear() diff --git a/Lib/asyncio/windows_utils.py b/Lib/asyncio/windows_utils.py index d65ea1790f0..9e22f6e0740 100644 --- a/Lib/asyncio/windows_utils.py +++ b/Lib/asyncio/windows_utils.py @@ -1,6 +1,4 @@ -""" -Various Windows specific bits and pieces -""" +"""Various Windows specific bits and pieces.""" import sys @@ -11,13 +9,12 @@ import _winapi import itertools import msvcrt import os -import socket import subprocess import tempfile import warnings -__all__ = ['socketpair', 'pipe', 'Popen', 'PIPE', 'PipeHandle'] +__all__ = 'pipe', 'Popen', 'PIPE', 'PipeHandle' # Constants/globals @@ -29,61 +26,14 @@ STDOUT = subprocess.STDOUT _mmap_counter = itertools.count() -if hasattr(socket, 'socketpair'): - # Since Python 3.5, socket.socketpair() is now also available on Windows - socketpair = socket.socketpair -else: - # Replacement for socket.socketpair() - def socketpair(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0): - """A socket pair usable as a self-pipe, for Windows. - - Origin: https://gist.github.com/4325783, by Geert Jansen. - Public domain. - """ - if family == socket.AF_INET: - host = '127.0.0.1' - elif family == socket.AF_INET6: - host = '::1' - else: - raise ValueError("Only AF_INET and AF_INET6 socket address " - "families are supported") - if type != socket.SOCK_STREAM: - raise ValueError("Only SOCK_STREAM socket type is supported") - if proto != 0: - raise ValueError("Only protocol zero is supported") - - # We create a connected TCP socket. Note the trick with setblocking(0) - # that prevents us from having to create a thread. - lsock = socket.socket(family, type, proto) - try: - lsock.bind((host, 0)) - lsock.listen(1) - # On IPv6, ignore flow_info and scope_id - addr, port = lsock.getsockname()[:2] - csock = socket.socket(family, type, proto) - try: - csock.setblocking(False) - try: - csock.connect((addr, port)) - except (BlockingIOError, InterruptedError): - pass - csock.setblocking(True) - ssock, _ = lsock.accept() - except: - csock.close() - raise - finally: - lsock.close() - return (ssock, csock) - - # Replacement for os.pipe() using handles instead of fds def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE): """Like os.pipe() but with overlapped support and using handles not fds.""" - address = tempfile.mktemp(prefix=r'\\.\pipe\python-pipe-%d-%d-' % - (os.getpid(), next(_mmap_counter))) + address = tempfile.mktemp( + prefix=r'\\.\pipe\python-pipe-{:d}-{:d}-'.format( + os.getpid(), next(_mmap_counter))) if duplex: openmode = _winapi.PIPE_ACCESS_DUPLEX @@ -138,10 +88,10 @@ class PipeHandle: def __repr__(self): if self._handle is not None: - handle = 'handle=%r' % self._handle + handle = f'handle={self._handle!r}' else: handle = 'closed' - return '<%s %s>' % (self.__class__.__name__, handle) + return f'<{self.__class__.__name__} {handle}>' @property def handle(self): @@ -159,7 +109,7 @@ class PipeHandle: def __del__(self): if self._handle is not None: - warnings.warn("unclosed %r" % self, ResourceWarning, + warnings.warn(f"unclosed {self!r}", ResourceWarning, source=self) self.close() diff --git a/Lib/bdb.py b/Lib/bdb.py index eb2fa4fb3e9..c6a10359ac6 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -3,10 +3,12 @@ import fnmatch import sys import os -from inspect import CO_GENERATOR +from inspect import CO_GENERATOR, CO_COROUTINE, CO_ASYNC_GENERATOR __all__ = ["BdbQuit", "Bdb", "Breakpoint"] +GENERATOR_AND_COROUTINE_FLAGS = CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR + class BdbQuit(Exception): """Exception to give up completely.""" @@ -127,7 +129,7 @@ class Bdb: # No need to trace this function return # None # Ignore call events in generator except when stepping. - if self.stopframe and frame.f_code.co_flags & CO_GENERATOR: + if self.stopframe and frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: return self.trace_dispatch self.user_call(frame, arg) if self.quitting: raise BdbQuit @@ -142,7 +144,7 @@ class Bdb: """ if self.stop_here(frame) or frame == self.returnframe: # Ignore return events in generator except when stepping. - if self.stopframe and frame.f_code.co_flags & CO_GENERATOR: + if self.stopframe and frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: return self.trace_dispatch try: self.frame_returning = frame @@ -166,7 +168,7 @@ class Bdb: # When stepping with next/until/return in a generator frame, skip # the internal StopIteration exception (with no traceback) # triggered by a subiterator run with the 'yield from' statement. - if not (frame.f_code.co_flags & CO_GENERATOR + if not (frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS and arg[0] is StopIteration and arg[2] is None): self.user_exception(frame, arg) if self.quitting: raise BdbQuit @@ -175,7 +177,7 @@ class Bdb: # next/until command at the last statement in the generator before the # exception. elif (self.stopframe and frame is not self.stopframe - and self.stopframe.f_code.co_flags & CO_GENERATOR + and self.stopframe.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS and arg[0] in (StopIteration, GeneratorExit)): self.user_exception(frame, arg) if self.quitting: raise BdbQuit @@ -309,7 +311,7 @@ class Bdb: def set_return(self, frame): """Stop when returning from the given frame.""" - if frame.f_code.co_flags & CO_GENERATOR: + if frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS: self._set_stopinfo(frame, None, -1) else: self._set_stopinfo(frame.f_back, frame) diff --git a/Lib/codecs.py b/Lib/codecs.py index 44618cbd2c4..a70ed20f2bc 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -480,15 +480,17 @@ class StreamReader(Codec): self.charbuffer = self._empty_charbuffer.join(self.linebuffer) self.linebuffer = None + if chars < 0: + # For compatibility with other read() methods that take a + # single argument + chars = size + # read until we get the required number of characters (if available) while True: # can the request be satisfied from the character buffer? if chars >= 0: if len(self.charbuffer) >= chars: break - elif size >= 0: - if len(self.charbuffer) >= size: - break # we need more data if size < 0: newdata = self.stream.read() diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 50cf8141731..f0b41fd9d10 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -17,12 +17,7 @@ list, set, and tuple. __all__ = ['deque', 'defaultdict', 'namedtuple', 'UserDict', 'UserList', 'UserString', 'Counter', 'OrderedDict', 'ChainMap'] -# For backwards compatibility, continue to make the collections ABCs -# available through the collections module. -from _collections_abc import * import _collections_abc -__all__ += _collections_abc.__all__ - from operator import itemgetter as _itemgetter, eq as _eq from keyword import iskeyword as _iskeyword import sys as _sys @@ -36,7 +31,7 @@ try: except ImportError: pass else: - MutableSequence.register(deque) + _collections_abc.MutableSequence.register(deque) try: from _collections import defaultdict @@ -44,22 +39,37 @@ except ImportError: pass +def __getattr__(name): + # For backwards compatibility, continue to make the collections ABCs + # through Python 3.6 available through the collections module. + # Note, no new collections ABCs were added in Python 3.7 + if name in _collections_abc.__all__: + obj = getattr(_collections_abc, name) + import warnings + warnings.warn("Using or importing the ABCs from 'collections' instead " + "of from 'collections.abc' is deprecated, " + "and in 3.8 it will stop working", + DeprecationWarning, stacklevel=2) + globals()[name] = obj + return obj + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') + ################################################################################ ### OrderedDict ################################################################################ -class _OrderedDictKeysView(KeysView): +class _OrderedDictKeysView(_collections_abc.KeysView): def __reversed__(self): yield from reversed(self._mapping) -class _OrderedDictItemsView(ItemsView): +class _OrderedDictItemsView(_collections_abc.ItemsView): def __reversed__(self): for key in reversed(self._mapping): yield (key, self._mapping[key]) -class _OrderedDictValuesView(ValuesView): +class _OrderedDictValuesView(_collections_abc.ValuesView): def __reversed__(self): for key in reversed(self._mapping): @@ -211,7 +221,7 @@ class OrderedDict(dict): size += sizeof(self.__root) * n # proxy objects return size - update = __update = MutableMapping.update + update = __update = _collections_abc.MutableMapping.update def keys(self): "D.keys() -> a set-like object providing a view on D's keys" @@ -225,7 +235,7 @@ class OrderedDict(dict): "D.values() -> an object providing a view on D's values" return _OrderedDictValuesView(self) - __ne__ = MutableMapping.__ne__ + __ne__ = _collections_abc.MutableMapping.__ne__ __marker = object() @@ -303,7 +313,7 @@ except ImportError: _nt_itemgetters = {} -def namedtuple(typename, field_names, *, rename=False, module=None): +def namedtuple(typename, field_names, *, rename=False, defaults=None, module=None): """Returns a new subclass of tuple with named fields. >>> Point = namedtuple('Point', ['x', 'y']) @@ -332,7 +342,8 @@ def namedtuple(typename, field_names, *, rename=False, module=None): if isinstance(field_names, str): field_names = field_names.replace(',', ' ').split() field_names = list(map(str, field_names)) - typename = str(typename) + typename = _sys.intern(str(typename)) + if rename: seen = set() for index, name in enumerate(field_names): @@ -342,6 +353,7 @@ def namedtuple(typename, field_names, *, rename=False, module=None): or name in seen): field_names[index] = f'_{index}' seen.add(name) + for name in [typename] + field_names: if type(name) is not str: raise TypeError('Type names and field names must be strings') @@ -351,6 +363,7 @@ def namedtuple(typename, field_names, *, rename=False, module=None): if _iskeyword(name): raise ValueError('Type names and field names cannot be a ' f'keyword: {name!r}') + seen = set() for name in field_names: if name.startswith('_') and not rename: @@ -360,6 +373,14 @@ def namedtuple(typename, field_names, *, rename=False, module=None): raise ValueError(f'Encountered duplicate field name: {name!r}') seen.add(name) + field_defaults = {} + if defaults is not None: + defaults = tuple(defaults) + if len(defaults) > len(field_names): + raise TypeError('Got more default values than field names') + field_defaults = dict(reversed(list(zip(reversed(field_names), + reversed(defaults))))) + # Variables used in the methods and docstrings field_names = tuple(map(_sys.intern, field_names)) num_fields = len(field_names) @@ -372,10 +393,12 @@ def namedtuple(typename, field_names, *, rename=False, module=None): s = f'def __new__(_cls, {arg_list}): return _tuple_new(_cls, ({arg_list}))' namespace = {'_tuple_new': tuple_new, '__name__': f'namedtuple_{typename}'} - # Note: exec() has the side-effect of interning the typename and field names + # Note: exec() has the side-effect of interning the field names exec(s, namespace) __new__ = namespace['__new__'] __new__.__doc__ = f'Create new instance of {typename}({arg_list})' + if defaults is not None: + __new__.__defaults__ = defaults @classmethod def _make(cls, iterable): @@ -420,6 +443,7 @@ def namedtuple(typename, field_names, *, rename=False, module=None): '__doc__': f'{typename}({arg_list})', '__slots__': (), '_fields': field_names, + '_fields_defaults': field_defaults, '__new__': __new__, '_make': _make, '_replace': _replace, @@ -618,7 +642,7 @@ class Counter(dict): raise TypeError('expected at most 1 arguments, got %d' % len(args)) iterable = args[0] if args else None if iterable is not None: - if isinstance(iterable, Mapping): + if isinstance(iterable, _collections_abc.Mapping): if self: self_get = self.get for elem, count in iterable.items(): @@ -655,7 +679,7 @@ class Counter(dict): iterable = args[0] if args else None if iterable is not None: self_get = self.get - if isinstance(iterable, Mapping): + if isinstance(iterable, _collections_abc.Mapping): for elem, count in iterable.items(): self[elem] = self_get(elem, 0) - count else: @@ -857,7 +881,7 @@ class Counter(dict): ### ChainMap ######################################################################## -class ChainMap(MutableMapping): +class ChainMap(_collections_abc.MutableMapping): ''' A ChainMap groups multiple dicts (or other mappings) together to create a single, updateable view. @@ -965,7 +989,7 @@ class ChainMap(MutableMapping): ### UserDict ################################################################################ -class UserDict(MutableMapping): +class UserDict(_collections_abc.MutableMapping): # Start by filling-out the abstract methods def __init__(*args, **kwargs): @@ -1032,7 +1056,7 @@ class UserDict(MutableMapping): ### UserList ################################################################################ -class UserList(MutableSequence): +class UserList(_collections_abc.MutableSequence): """A more or less complete user-defined wrapper around list objects.""" def __init__(self, initlist=None): self.data = [] @@ -1105,7 +1129,7 @@ class UserList(MutableSequence): ### UserString ################################################################################ -class UserString(Sequence): +class UserString(_collections_abc.Sequence): def __init__(self, seq): if isinstance(seq, str): self.data = seq @@ -1200,6 +1224,7 @@ class UserString(Sequence): return self.data.index(sub, start, end) def isalpha(self): return self.data.isalpha() def isalnum(self): return self.data.isalnum() + def isascii(self): return self.data.isascii() def isdecimal(self): return self.data.isdecimal() def isdigit(self): return self.data.isdigit() def isidentifier(self): return self.data.isidentifier() diff --git a/Lib/compileall.py b/Lib/compileall.py index 1c9ceb69309..72592126d74 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -52,7 +52,8 @@ def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0): maxlevels=maxlevels - 1, quiet=quiet) def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, - quiet=0, legacy=False, optimize=-1, workers=1): + quiet=0, legacy=False, optimize=-1, workers=1, + invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP): """Byte-compile all modules in the given directory tree. Arguments (only dir is required): @@ -67,6 +68,7 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, legacy: if True, produce legacy pyc paths instead of PEP 3147 paths optimize: optimization level or -1 for level of the interpreter workers: maximum number of parallel workers + invalidation_mode: how the up-to-dateness of the pyc will be checked """ if workers is not None and workers < 0: raise ValueError('workers must be greater or equal to 0') @@ -81,18 +83,20 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, ddir=ddir, force=force, rx=rx, quiet=quiet, legacy=legacy, - optimize=optimize), + optimize=optimize, + invalidation_mode=invalidation_mode), files) success = min(results, default=True) else: for file in files: if not compile_file(file, ddir, force, rx, quiet, - legacy, optimize): + legacy, optimize, invalidation_mode): success = False return success def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, - legacy=False, optimize=-1): + legacy=False, optimize=-1, + invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP): """Byte-compile one file. Arguments (only fullname is required): @@ -105,6 +109,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, no output with 2 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths optimize: optimization level or -1 for level of the interpreter + invalidation_mode: how the up-to-dateness of the pyc will be checked """ success = True if quiet < 2 and isinstance(fullname, os.PathLike): @@ -134,10 +139,10 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if not force: try: mtime = int(os.stat(fullname).st_mtime) - expect = struct.pack('<4sl', importlib.util.MAGIC_NUMBER, - mtime) + expect = struct.pack('<4sll', importlib.util.MAGIC_NUMBER, + 0, mtime) with open(cfile, 'rb') as chandle: - actual = chandle.read(8) + actual = chandle.read(12) if expect == actual: return success except OSError: @@ -146,7 +151,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, print('Compiling {!r}...'.format(fullname)) try: ok = py_compile.compile(fullname, cfile, dfile, True, - optimize=optimize) + optimize=optimize, + invalidation_mode=invalidation_mode) except py_compile.PyCompileError as err: success = False if quiet >= 2: @@ -175,7 +181,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, return success def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0, - legacy=False, optimize=-1): + legacy=False, optimize=-1, + invalidation_mode=py_compile.PycInvalidationMode.TIMESTAMP): """Byte-compile all module on sys.path. Arguments (all optional): @@ -186,6 +193,7 @@ def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0, quiet: as for compile_dir() (default 0) legacy: as for compile_dir() (default False) optimize: as for compile_dir() (default -1) + invalidation_mode: as for compiler_dir() """ success = True for dir in sys.path: @@ -193,9 +201,16 @@ def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0, if quiet < 2: print('Skipping current directory') else: - success = success and compile_dir(dir, maxlevels, None, - force, quiet=quiet, - legacy=legacy, optimize=optimize) + success = success and compile_dir( + dir, + maxlevels, + None, + force, + quiet=quiet, + legacy=legacy, + optimize=optimize, + invalidation_mode=invalidation_mode, + ) return success @@ -238,6 +253,11 @@ def main(): 'to the equivalent of -l sys.path')) parser.add_argument('-j', '--workers', default=1, type=int, help='Run compileall concurrently') + invalidation_modes = [mode.name.lower().replace('_', '-') + for mode in py_compile.PycInvalidationMode] + parser.add_argument('--invalidation-mode', default='timestamp', + choices=sorted(invalidation_modes), + help='How the pycs will be invalidated at runtime') args = parser.parse_args() compile_dests = args.compile_dest @@ -266,23 +286,29 @@ def main(): if args.workers is not None: args.workers = args.workers or None + ivl_mode = args.invalidation_mode.replace('-', '_').upper() + invalidation_mode = py_compile.PycInvalidationMode[ivl_mode] + success = True try: if compile_dests: for dest in compile_dests: if os.path.isfile(dest): if not compile_file(dest, args.ddir, args.force, args.rx, - args.quiet, args.legacy): + args.quiet, args.legacy, + invalidation_mode=invalidation_mode): success = False else: if not compile_dir(dest, maxlevels, args.ddir, args.force, args.rx, args.quiet, - args.legacy, workers=args.workers): + args.legacy, workers=args.workers, + invalidation_mode=invalidation_mode): success = False return success else: return compile_path(legacy=args.legacy, force=args.force, - quiet=args.quiet) + quiet=args.quiet, + invalidation_mode=invalidation_mode) except KeyboardInterrupt: if args.quiet < 2: print("\n[interrupted]") diff --git a/Lib/concurrent/futures/__init__.py b/Lib/concurrent/futures/__init__.py index ba8de163905..8434fcf4b5e 100644 --- a/Lib/concurrent/futures/__init__.py +++ b/Lib/concurrent/futures/__init__.py @@ -15,5 +15,38 @@ from concurrent.futures._base import (FIRST_COMPLETED, Executor, wait, as_completed) -from concurrent.futures.process import ProcessPoolExecutor -from concurrent.futures.thread import ThreadPoolExecutor + +__all__ = ( + 'FIRST_COMPLETED', + 'FIRST_EXCEPTION', + 'ALL_COMPLETED', + 'CancelledError', + 'TimeoutError', + 'BrokenExecutor', + 'Future', + 'Executor', + 'wait', + 'as_completed', + 'ProcessPoolExecutor', + 'ThreadPoolExecutor', +) + + +def __dir__(): + return __all__ + ('__author__', '__doc__') + + +def __getattr__(name): + global ProcessPoolExecutor, ThreadPoolExecutor + + if name == 'ProcessPoolExecutor': + from .process import ProcessPoolExecutor as pe + ProcessPoolExecutor = pe + return pe + + if name == 'ThreadPoolExecutor': + from .thread import ThreadPoolExecutor as te + ThreadPoolExecutor = te + return te + + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 35af65d0bee..aaa5151e017 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -8,10 +8,10 @@ The follow diagram and text describe the data-flow through the system: |======================= In-process =====================|== Out-of-process ==| +----------+ +----------+ +--------+ +-----------+ +---------+ -| | => | Work Ids | => | | => | Call Q | => | | -| | +----------+ | | +-----------+ | | -| | | ... | | | | ... | | | -| | | 6 | | | | 5, call() | | | +| | => | Work Ids | | | | Call Q | | Process | +| | +----------+ | | +-----------+ | Pool | +| | | ... | | | | ... | +---------+ +| | | 6 | => | | => | 5, call() | => | | | | | 7 | | | | ... | | | | Process | | ... | | Local | +-----------+ | Process | | Pool | +----------+ | Worker | | #1..n | @@ -52,6 +52,7 @@ import queue from queue import Full import multiprocessing as mp from multiprocessing.connection import wait +from multiprocessing.queues import Queue import threading import weakref from functools import partial @@ -72,16 +73,31 @@ import traceback # workers to exit when their work queues are empty and then waits until the # threads/processes finish. -_threads_queues = weakref.WeakKeyDictionary() +_threads_wakeups = weakref.WeakKeyDictionary() _global_shutdown = False + +class _ThreadWakeup: + __slot__ = ["_state"] + + def __init__(self): + self._reader, self._writer = mp.Pipe(duplex=False) + + def wakeup(self): + self._writer.send_bytes(b"") + + def clear(self): + while self._reader.poll(): + self._reader.recv_bytes() + + def _python_exit(): global _global_shutdown _global_shutdown = True - items = list(_threads_queues.items()) - for t, q in items: - q.put(None) - for t, q in items: + items = list(_threads_wakeups.items()) + for _, thread_wakeup in items: + thread_wakeup.wakeup() + for t, _ in items: t.join() # Controls how many more calls than processes will be queued in the call queue. @@ -90,6 +106,7 @@ def _python_exit(): # (Futures in the call queue cannot be cancelled). EXTRA_QUEUED_CALLS = 1 + # Hack to embed stringification of remote traceback in local traceback class _RemoteTraceback(Exception): @@ -132,6 +149,25 @@ class _CallItem(object): self.kwargs = kwargs +class _SafeQueue(Queue): + """Safe Queue set exception to the future object linked to a job""" + def __init__(self, max_size=0, *, ctx, pending_work_items): + self.pending_work_items = pending_work_items + super().__init__(max_size, ctx=ctx) + + def _on_queue_feeder_error(self, e, obj): + if isinstance(obj, _CallItem): + tb = traceback.format_exception(type(e), e, e.__traceback__) + e.__cause__ = _RemoteTraceback('\n"""\n{}"""'.format(''.join(tb))) + work_item = self.pending_work_items.pop(obj.work_id, None) + # work_item can be None if another process terminated. In this case, + # the queue_manager_thread fails all work_items with BrokenProcessPool + if work_item is not None: + work_item.future.set_exception(e) + else: + super()._on_queue_feeder_error(e, obj) + + def _get_chunks(*iterables, chunksize): """ Iterates over zip()ed iterables in chunks. """ it = zip(*iterables) @@ -152,6 +188,17 @@ def _process_chunk(fn, chunk): """ return [fn(*args) for args in chunk] + +def _sendback_result(result_queue, work_id, result=None, exception=None): + """Safely send back the given result or exception""" + try: + result_queue.put(_ResultItem(work_id, result=result, + exception=exception)) + except BaseException as e: + exc = _ExceptionWithTraceback(e, e.__traceback__) + result_queue.put(_ResultItem(work_id, exception=exc)) + + def _process_worker(call_queue, result_queue, initializer, initargs): """Evaluates calls from call_queue and places the results in result_queue. @@ -183,10 +230,9 @@ def _process_worker(call_queue, result_queue, initializer, initargs): r = call_item.fn(*call_item.args, **call_item.kwargs) except BaseException as e: exc = _ExceptionWithTraceback(e, e.__traceback__) - result_queue.put(_ResultItem(call_item.work_id, exception=exc)) + _sendback_result(result_queue, call_item.work_id, exception=exc) else: - result_queue.put(_ResultItem(call_item.work_id, - result=r)) + _sendback_result(result_queue, call_item.work_id, result=r) # Liberate the resource as soon as possible, to avoid holding onto # open files or shared memory that is not needed anymore @@ -230,12 +276,14 @@ def _add_call_item_to_queue(pending_work_items, del pending_work_items[work_id] continue + def _queue_management_worker(executor_reference, processes, pending_work_items, work_ids_queue, call_queue, - result_queue): + result_queue, + thread_wakeup): """Manages the communication between this process and the worker processes. This function is run in a local thread. @@ -253,6 +301,9 @@ def _queue_management_worker(executor_reference, derived from _WorkItems for processing by the process workers. result_queue: A ctx.SimpleQueue of _ResultItems generated by the process workers. + thread_wakeup: A _ThreadWakeup to allow waking up the + queue_manager_thread from the main Thread and avoid deadlocks + caused by permanently locked queues. """ executor = None @@ -261,10 +312,21 @@ def _queue_management_worker(executor_reference, or executor._shutdown_thread) def shutdown_worker(): - # This is an upper bound - nb_children_alive = sum(p.is_alive() for p in processes.values()) - for i in range(0, nb_children_alive): - call_queue.put_nowait(None) + # This is an upper bound on the number of children alive. + n_children_alive = sum(p.is_alive() for p in processes.values()) + n_children_to_stop = n_children_alive + n_sentinels_sent = 0 + # Send the right number of sentinels, to make sure all children are + # properly terminated. + while n_sentinels_sent < n_children_to_stop and n_children_alive > 0: + for i in range(n_children_to_stop - n_sentinels_sent): + try: + call_queue.put_nowait(None) + n_sentinels_sent += 1 + except Full: + break + n_children_alive = sum(p.is_alive() for p in processes.values()) + # Release the queue's resources as soon as possible. call_queue.close() # If .join() is not called on the created processes then @@ -272,19 +334,37 @@ def _queue_management_worker(executor_reference, for p in processes.values(): p.join() - reader = result_queue._reader + result_reader = result_queue._reader + wakeup_reader = thread_wakeup._reader + readers = [result_reader, wakeup_reader] while True: _add_call_item_to_queue(pending_work_items, work_ids_queue, call_queue) - sentinels = [p.sentinel for p in processes.values()] - assert sentinels - ready = wait([reader] + sentinels) - if reader in ready: - result_item = reader.recv() - else: + # Wait for a result to be ready in the result_queue while checking + # that all worker processes are still running, or for a wake up + # signal send. The wake up signals come either from new tasks being + # submitted, from the executor being shutdown/gc-ed, or from the + # shutdown of the python interpreter. + worker_sentinels = [p.sentinel for p in processes.values()] + ready = wait(readers + worker_sentinels) + + cause = None + is_broken = True + if result_reader in ready: + try: + result_item = result_reader.recv() + is_broken = False + except BaseException as e: + cause = traceback.format_exception(type(e), e, e.__traceback__) + + elif wakeup_reader in ready: + is_broken = False + result_item = None + thread_wakeup.clear() + if is_broken: # Mark the process pool broken so that submits fail right now. executor = executor_reference() if executor is not None: @@ -293,14 +373,15 @@ def _queue_management_worker(executor_reference, 'usable anymore') executor._shutdown_thread = True executor = None + bpe = BrokenProcessPool("A process in the process pool was " + "terminated abruptly while the future was " + "running or pending.") + if cause is not None: + bpe.__cause__ = _RemoteTraceback( + f"\n'''\n{''.join(cause)}'''") # All futures in flight must be marked failed for work_id, work_item in pending_work_items.items(): - work_item.future.set_exception( - BrokenProcessPool( - "A process in the process pool was " - "terminated abruptly while the future was " - "running or pending." - )) + work_item.future.set_exception(bpe) # Delete references to object. See issue16284 del work_item pending_work_items.clear() @@ -329,6 +410,9 @@ def _queue_management_worker(executor_reference, work_item.future.set_result(result_item.result) # Delete references to object. See issue16284 del work_item + # Delete reference to result_item + del result_item + # Check whether we should start shutting down. executor = executor_reference() # No more work items can be added if: @@ -348,8 +432,11 @@ def _queue_management_worker(executor_reference, pass executor = None + _system_limits_checked = False _system_limited = None + + def _check_system_limits(): global _system_limits_checked, _system_limited if _system_limits_checked: @@ -369,7 +456,8 @@ def _check_system_limits(): # minimum number of semaphores available # according to POSIX return - _system_limited = "system provides too few semaphores (%d available, 256 necessary)" % nsems_max + _system_limited = ("system provides too few semaphores (%d" + " available, 256 necessary)" % nsems_max) raise NotImplementedError(_system_limited) @@ -415,6 +503,7 @@ class ProcessPoolExecutor(_base.Executor): raise ValueError("max_workers must be greater than 0") self._max_workers = max_workers + if mp_context is None: mp_context = mp.get_context() self._mp_context = mp_context @@ -424,18 +513,9 @@ class ProcessPoolExecutor(_base.Executor): self._initializer = initializer self._initargs = initargs - # Make the call queue slightly larger than the number of processes to - # prevent the worker processes from idling. But don't make it too big - # because futures in the call queue cannot be cancelled. - queue_size = self._max_workers + EXTRA_QUEUED_CALLS - self._call_queue = mp_context.Queue(queue_size) - # Killed worker processes can produce spurious "broken pipe" - # tracebacks in the queue's own worker thread. But we detect killed - # processes anyway, so silence the tracebacks. - self._call_queue._ignore_epipe = True - self._result_queue = mp_context.SimpleQueue() - self._work_ids = queue.Queue() + # Management thread self._queue_management_thread = None + # Map of pids to processes self._processes = {} @@ -446,12 +526,39 @@ class ProcessPoolExecutor(_base.Executor): self._queue_count = 0 self._pending_work_items = {} + # Create communication channels for the executor + # Make the call queue slightly larger than the number of processes to + # prevent the worker processes from idling. But don't make it too big + # because futures in the call queue cannot be cancelled. + queue_size = self._max_workers + EXTRA_QUEUED_CALLS + self._call_queue = _SafeQueue( + max_size=queue_size, ctx=self._mp_context, + pending_work_items=self._pending_work_items) + # Killed worker processes can produce spurious "broken pipe" + # tracebacks in the queue's own worker thread. But we detect killed + # processes anyway, so silence the tracebacks. + self._call_queue._ignore_epipe = True + self._result_queue = mp_context.SimpleQueue() + self._work_ids = queue.Queue() + + # _ThreadWakeup is a communication channel used to interrupt the wait + # of the main loop of queue_manager_thread from another thread (e.g. + # when calling executor.submit or executor.shutdown). We do not use the + # _result_queue to send the wakeup signal to the queue_manager_thread + # as it could result in a deadlock if a worker process dies with the + # _result_queue write lock still acquired. + self._queue_management_thread_wakeup = _ThreadWakeup() + def _start_queue_management_thread(self): - # When the executor gets lost, the weakref callback will wake up - # the queue management thread. - def weakref_cb(_, q=self._result_queue): - q.put(None) if self._queue_management_thread is None: + # When the executor gets garbarge collected, the weakref callback + # will wake up the queue management thread so that it can terminate + # if there is no pending work item. + def weakref_cb(_, + thread_wakeup=self._queue_management_thread_wakeup): + mp.util.debug('Executor collected: triggering callback for' + ' QueueManager wakeup') + thread_wakeup.wakeup() # Start the processes so that their sentinels are known. self._adjust_process_count() self._queue_management_thread = threading.Thread( @@ -461,10 +568,13 @@ class ProcessPoolExecutor(_base.Executor): self._pending_work_items, self._work_ids, self._call_queue, - self._result_queue)) + self._result_queue, + self._queue_management_thread_wakeup), + name="QueueManagerThread") self._queue_management_thread.daemon = True self._queue_management_thread.start() - _threads_queues[self._queue_management_thread] = self._result_queue + _threads_wakeups[self._queue_management_thread] = \ + self._queue_management_thread_wakeup def _adjust_process_count(self): for _ in range(len(self._processes), self._max_workers): @@ -491,7 +601,7 @@ class ProcessPoolExecutor(_base.Executor): self._work_ids.put(self._queue_count) self._queue_count += 1 # Wake up queue management thread - self._result_queue.put(None) + self._queue_management_thread_wakeup.wakeup() self._start_queue_management_thread() return f @@ -531,7 +641,7 @@ class ProcessPoolExecutor(_base.Executor): self._shutdown_thread = True if self._queue_management_thread: # Wake up queue management thread - self._result_queue.put(None) + self._queue_management_thread_wakeup.wakeup() if wait: self._queue_management_thread.join() # To reduce the risk of opening too many files, remove references to diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py index 2e7100bc352..6e22950a157 100644 --- a/Lib/concurrent/futures/thread.py +++ b/Lib/concurrent/futures/thread.py @@ -128,7 +128,7 @@ class ThreadPoolExecutor(_base.Executor): raise TypeError("initializer must be a callable") self._max_workers = max_workers - self._work_queue = queue.Queue() + self._work_queue = queue.SimpleQueue() self._threads = set() self._broken = False self._shutdown = False diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 962cedab490..1ff8cdf1cec 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -5,8 +5,9 @@ import _collections_abc from collections import deque from functools import wraps -__all__ = ["asynccontextmanager", "contextmanager", "closing", - "AbstractContextManager", "ContextDecorator", "ExitStack", +__all__ = ["asynccontextmanager", "contextmanager", "closing", "nullcontext", + "AbstractContextManager", "AbstractAsyncContextManager", + "AsyncExitStack", "ContextDecorator", "ExitStack", "redirect_stdout", "redirect_stderr", "suppress"] @@ -30,6 +31,27 @@ class AbstractContextManager(abc.ABC): return NotImplemented +class AbstractAsyncContextManager(abc.ABC): + + """An abstract base class for asynchronous context managers.""" + + async def __aenter__(self): + """Return `self` upon entering the runtime context.""" + return self + + @abc.abstractmethod + async def __aexit__(self, exc_type, exc_value, traceback): + """Raise any exception triggered within the runtime context.""" + return None + + @classmethod + def __subclasshook__(cls, C): + if cls is AbstractAsyncContextManager: + return _collections_abc._check_methods(C, "__aenter__", + "__aexit__") + return NotImplemented + + class ContextDecorator(object): "A base class or mixin that enables context managers to work as decorators." @@ -83,6 +105,9 @@ class _GeneratorContextManager(_GeneratorContextManagerBase, return self.__class__(self.func, self.args, self.kwds) def __enter__(self): + # do not keep args and kwds alive unnecessarily + # they are only needed for recreation, which is not possible anymore + del self.args, self.kwds, self.func try: return next(self.gen) except StopIteration: @@ -136,7 +161,8 @@ class _GeneratorContextManager(_GeneratorContextManagerBase, raise RuntimeError("generator didn't stop after throw()") -class _AsyncGeneratorContextManager(_GeneratorContextManagerBase): +class _AsyncGeneratorContextManager(_GeneratorContextManagerBase, + AbstractAsyncContextManager): """Helper for @asynccontextmanager.""" async def __aenter__(self): @@ -342,85 +368,102 @@ class suppress(AbstractContextManager): return exctype is not None and issubclass(exctype, self._exceptions) -# Inspired by discussions on http://bugs.python.org/issue13585 -class ExitStack(AbstractContextManager): - """Context manager for dynamic management of a stack of exit callbacks +class _BaseExitStack: + """A base class for ExitStack and AsyncExitStack.""" - For example: + @staticmethod + def _create_exit_wrapper(cm, cm_exit): + def _exit_wrapper(exc_type, exc, tb): + return cm_exit(cm, exc_type, exc, tb) + return _exit_wrapper - with ExitStack() as stack: - files = [stack.enter_context(open(fname)) for fname in filenames] - # All opened files will automatically be closed at the end of - # the with statement, even if attempts to open files later - # in the list raise an exception + @staticmethod + def _create_cb_wrapper(callback, *args, **kwds): + def _exit_wrapper(exc_type, exc, tb): + callback(*args, **kwds) + return _exit_wrapper - """ def __init__(self): self._exit_callbacks = deque() def pop_all(self): - """Preserve the context stack by transferring it to a new instance""" + """Preserve the context stack by transferring it to a new instance.""" new_stack = type(self)() new_stack._exit_callbacks = self._exit_callbacks self._exit_callbacks = deque() return new_stack - def _push_cm_exit(self, cm, cm_exit): - """Helper to correctly register callbacks to __exit__ methods""" - def _exit_wrapper(*exc_details): - return cm_exit(cm, *exc_details) - _exit_wrapper.__self__ = cm - self.push(_exit_wrapper) - def push(self, exit): - """Registers a callback with the standard __exit__ method signature - - Can suppress exceptions the same way __exit__ methods can. + """Registers a callback with the standard __exit__ method signature. + Can suppress exceptions the same way __exit__ method can. Also accepts any object with an __exit__ method (registering a call - to the method instead of the object itself) + to the method instead of the object itself). """ # We use an unbound method rather than a bound method to follow - # the standard lookup behaviour for special methods + # the standard lookup behaviour for special methods. _cb_type = type(exit) + try: exit_method = _cb_type.__exit__ except AttributeError: - # Not a context manager, so assume its a callable - self._exit_callbacks.append(exit) + # Not a context manager, so assume it's a callable. + self._push_exit_callback(exit) else: self._push_cm_exit(exit, exit_method) - return exit # Allow use as a decorator - - def callback(self, callback, *args, **kwds): - """Registers an arbitrary callback and arguments. - - Cannot suppress exceptions. - """ - def _exit_wrapper(exc_type, exc, tb): - callback(*args, **kwds) - # We changed the signature, so using @wraps is not appropriate, but - # setting __wrapped__ may still help with introspection - _exit_wrapper.__wrapped__ = callback - self.push(_exit_wrapper) - return callback # Allow use as a decorator + return exit # Allow use as a decorator. def enter_context(self, cm): - """Enters the supplied context manager + """Enters the supplied context manager. If successful, also pushes its __exit__ method as a callback and returns the result of the __enter__ method. """ - # We look up the special methods on the type to match the with statement + # We look up the special methods on the type to match the with + # statement. _cm_type = type(cm) _exit = _cm_type.__exit__ result = _cm_type.__enter__(cm) self._push_cm_exit(cm, _exit) return result - def close(self): - """Immediately unwind the context stack""" - self.__exit__(None, None, None) + def callback(self, callback, *args, **kwds): + """Registers an arbitrary callback and arguments. + + Cannot suppress exceptions. + """ + _exit_wrapper = self._create_cb_wrapper(callback, *args, **kwds) + + # We changed the signature, so using @wraps is not appropriate, but + # setting __wrapped__ may still help with introspection. + _exit_wrapper.__wrapped__ = callback + self._push_exit_callback(_exit_wrapper) + return callback # Allow use as a decorator + + def _push_cm_exit(self, cm, cm_exit): + """Helper to correctly register callbacks to __exit__ methods.""" + _exit_wrapper = self._create_exit_wrapper(cm, cm_exit) + _exit_wrapper.__self__ = cm + self._push_exit_callback(_exit_wrapper, True) + + def _push_exit_callback(self, callback, is_sync=True): + self._exit_callbacks.append((is_sync, callback)) + + +# Inspired by discussions on http://bugs.python.org/issue13585 +class ExitStack(_BaseExitStack, AbstractContextManager): + """Context manager for dynamic management of a stack of exit callbacks. + + For example: + with ExitStack() as stack: + files = [stack.enter_context(open(fname)) for fname in filenames] + # All opened files will automatically be closed at the end of + # the with statement, even if attempts to open files later + # in the list raise an exception. + """ + + def __enter__(self): + return self def __exit__(self, *exc_details): received_exc = exc_details[0] is not None @@ -447,7 +490,8 @@ class ExitStack(AbstractContextManager): suppressed_exc = False pending_raise = False while self._exit_callbacks: - cb = self._exit_callbacks.pop() + is_sync, cb = self._exit_callbacks.pop() + assert is_sync try: if cb(*exc_details): suppressed_exc = True @@ -469,3 +513,165 @@ class ExitStack(AbstractContextManager): exc_details[1].__context__ = fixed_ctx raise return received_exc and suppressed_exc + + def close(self): + """Immediately unwind the context stack.""" + self.__exit__(None, None, None) + + +# Inspired by discussions on https://bugs.python.org/issue29302 +class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager): + """Async context manager for dynamic management of a stack of exit + callbacks. + + For example: + async with AsyncExitStack() as stack: + connections = [await stack.enter_async_context(get_connection()) + for i in range(5)] + # All opened connections will automatically be released at the + # end of the async with statement, even if attempts to open a + # connection later in the list raise an exception. + """ + + @staticmethod + def _create_async_exit_wrapper(cm, cm_exit): + async def _exit_wrapper(exc_type, exc, tb): + return await cm_exit(cm, exc_type, exc, tb) + return _exit_wrapper + + @staticmethod + def _create_async_cb_wrapper(callback, *args, **kwds): + async def _exit_wrapper(exc_type, exc, tb): + await callback(*args, **kwds) + return _exit_wrapper + + async def enter_async_context(self, cm): + """Enters the supplied async context manager. + + If successful, also pushes its __aexit__ method as a callback and + returns the result of the __aenter__ method. + """ + _cm_type = type(cm) + _exit = _cm_type.__aexit__ + result = await _cm_type.__aenter__(cm) + self._push_async_cm_exit(cm, _exit) + return result + + def push_async_exit(self, exit): + """Registers a coroutine function with the standard __aexit__ method + signature. + + Can suppress exceptions the same way __aexit__ method can. + Also accepts any object with an __aexit__ method (registering a call + to the method instead of the object itself). + """ + _cb_type = type(exit) + try: + exit_method = _cb_type.__aexit__ + except AttributeError: + # Not an async context manager, so assume it's a coroutine function + self._push_exit_callback(exit, False) + else: + self._push_async_cm_exit(exit, exit_method) + return exit # Allow use as a decorator + + def push_async_callback(self, callback, *args, **kwds): + """Registers an arbitrary coroutine function and arguments. + + Cannot suppress exceptions. + """ + _exit_wrapper = self._create_async_cb_wrapper(callback, *args, **kwds) + + # We changed the signature, so using @wraps is not appropriate, but + # setting __wrapped__ may still help with introspection. + _exit_wrapper.__wrapped__ = callback + self._push_exit_callback(_exit_wrapper, False) + return callback # Allow use as a decorator + + async def aclose(self): + """Immediately unwind the context stack.""" + await self.__aexit__(None, None, None) + + def _push_async_cm_exit(self, cm, cm_exit): + """Helper to correctly register coroutine function to __aexit__ + method.""" + _exit_wrapper = self._create_async_exit_wrapper(cm, cm_exit) + _exit_wrapper.__self__ = cm + self._push_exit_callback(_exit_wrapper, False) + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_details): + received_exc = exc_details[0] is not None + + # We manipulate the exception state so it behaves as though + # we were actually nesting multiple with statements + frame_exc = sys.exc_info()[1] + def _fix_exception_context(new_exc, old_exc): + # Context may not be correct, so find the end of the chain + while 1: + exc_context = new_exc.__context__ + if exc_context is old_exc: + # Context is already set correctly (see issue 20317) + return + if exc_context is None or exc_context is frame_exc: + break + new_exc = exc_context + # Change the end of the chain to point to the exception + # we expect it to reference + new_exc.__context__ = old_exc + + # Callbacks are invoked in LIFO order to match the behaviour of + # nested context managers + suppressed_exc = False + pending_raise = False + while self._exit_callbacks: + is_sync, cb = self._exit_callbacks.pop() + try: + if is_sync: + cb_suppress = cb(*exc_details) + else: + cb_suppress = await cb(*exc_details) + + if cb_suppress: + suppressed_exc = True + pending_raise = False + exc_details = (None, None, None) + except: + new_exc_details = sys.exc_info() + # simulate the stack of exceptions by setting the context + _fix_exception_context(new_exc_details[1], exc_details[1]) + pending_raise = True + exc_details = new_exc_details + if pending_raise: + try: + # bare "raise exc_details[1]" replaces our carefully + # set-up context + fixed_ctx = exc_details[1].__context__ + raise exc_details[1] + except BaseException: + exc_details[1].__context__ = fixed_ctx + raise + return received_exc and suppressed_exc + + +class nullcontext(AbstractContextManager): + """Context manager that does no additional processing. + + Used as a stand-in for a normal context manager, when a particular + block of code is only sometimes used with a normal context manager: + + cm = optional_cm if condition else nullcontext() + with cm: + # Perform operation, using optional_cm if condition is True + """ + + def __init__(self, enter_result=None): + self.enter_result = enter_result + + def __enter__(self): + return self.enter_result + + def __exit__(self, *excinfo): + pass diff --git a/Lib/contextvars.py b/Lib/contextvars.py new file mode 100644 index 00000000000..d78c80dfe6f --- /dev/null +++ b/Lib/contextvars.py @@ -0,0 +1,4 @@ +from _contextvars import Context, ContextVar, Token, copy_context + + +__all__ = ('Context', 'ContextVar', 'Token', 'copy_context') diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 972ea0ac879..61467739886 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -338,6 +338,14 @@ class CDLL(object): flags |= _FUNCFLAG_USE_ERRNO if use_last_error: flags |= _FUNCFLAG_USE_LASTERROR + if _sys.platform.startswith("aix"): + """When the name contains ".a(" and ends with ")", + e.g., "libFOO.a(libFOO.so)" - this is taken to be an + archive(member) syntax for dlopen(), and the mode is adjusted. + Otherwise, name is presented to dlopen() as a file argument. + """ + if name and name.endswith(")") and ".a(" in name: + mode |= ( _os.RTLD_MEMBER | _os.RTLD_NOW ) class _FuncPtr(_CFuncPtr): _flags_ = flags diff --git a/Lib/ctypes/_aix.py b/Lib/ctypes/_aix.py new file mode 100644 index 00000000000..463f60a2849 --- /dev/null +++ b/Lib/ctypes/_aix.py @@ -0,0 +1,331 @@ +""" +Lib/ctypes.util.find_library() support for AIX +Similar approach as done for Darwin support by using separate files +but unlike Darwin - no extension such as ctypes.macholib.* + +dlopen() is an interface to AIX initAndLoad() - primary documentation at: +https://www.ibm.com/support/knowledgecenter/en/ssw_aix_61/com.ibm.aix.basetrf1/dlopen.htm +https://www.ibm.com/support/knowledgecenter/en/ssw_aix_61/com.ibm.aix.basetrf1/load.htm + +AIX supports two styles for dlopen(): svr4 (System V Release 4) which is common on posix +platforms, but also a BSD style - aka SVR3. + +From AIX 5.3 Difference Addendum (December 2004) +2.9 SVR4 linking affinity +Nowadays, there are two major object file formats used by the operating systems: +XCOFF: The COFF enhanced by IBM and others. The original COFF (Common +Object File Format) was the base of SVR3 and BSD 4.2 systems. +ELF: Executable and Linking Format that was developed by AT&T and is a +base for SVR4 UNIX. + +While the shared library content is identical on AIX - one is located as a filepath name +(svr4 style) and the other is located as a member of an archive (and the archive +is located as a filepath name). + +The key difference arises when supporting multiple abi formats (i.e., 32 and 64 bit). +For svr4 either only one ABI is supported, or there are two directories, or there +are different file names. The most common solution for multiple ABI is multiple +directories. + +For the XCOFF (aka AIX) style - one directory (one archive file) is sufficient +as multiple shared libraries can be in the archive - even sharing the same name. +In documentation the archive is also referred to as the "base" and the shared +library object is referred to as the "member". + +For dlopen() on AIX (read initAndLoad()) the calls are similar. +Default activity occurs when no path information is provided. When path +information is provided dlopen() does not search any other directories. + +For SVR4 - the shared library name is the name of the file expected: libFOO.so +For AIX - the shared library is expressed as base(member). The search is for the +base (e.g., libFOO.a) and once the base is found the shared library - identified by +member (e.g., libFOO.so, or shr.o) is located and loaded. + +The mode bit RTLD_MEMBER tells initAndLoad() that it needs to use the AIX (SVR3) +naming style. +""" +__author__ = "Michael Felt " + +import re +from os import environ, path +from sys import executable +from ctypes import c_void_p, sizeof +from subprocess import Popen, PIPE, DEVNULL + +# Executable bit size - 32 or 64 +# Used to filter the search in an archive by size, e.g., -X64 +AIX_ABI = sizeof(c_void_p) * 8 + + +from sys import maxsize +def _last_version(libnames, sep): + def _num_version(libname): + # "libxyz.so.MAJOR.MINOR" => [MAJOR, MINOR] + parts = libname.split(sep) + nums = [] + try: + while parts: + nums.insert(0, int(parts.pop())) + except ValueError: + pass + return nums or [maxsize] + return max(reversed(libnames), key=_num_version) + +def get_ld_header(p): + # "nested-function, but placed at module level + ld_header = None + for line in p.stdout: + if line.startswith(('/', './', '../')): + ld_header = line + elif "INDEX" in line: + return ld_header.rstrip('\n') + return None + +def get_ld_header_info(p): + # "nested-function, but placed at module level + # as an ld_header was found, return known paths, archives and members + # these lines start with a digit + info = [] + for line in p.stdout: + if re.match("[0-9]", line): + info.append(line) + else: + # blank line (separator), consume line and end for loop + break + return info + +def get_ld_headers(file): + """ + Parse the header of the loader section of executable and archives + This function calls /usr/bin/dump -H as a subprocess + and returns a list of (ld_header, ld_header_info) tuples. + """ + # get_ld_headers parsing: + # 1. Find a line that starts with /, ./, or ../ - set as ld_header + # 2. If "INDEX" in occurs in a following line - return ld_header + # 3. get info (lines starting with [0-9]) + ldr_headers = [] + p = Popen(["/usr/bin/dump", f"-X{AIX_ABI}", "-H", file], + universal_newlines=True, stdout=PIPE, stderr=DEVNULL) + # be sure to read to the end-of-file - getting all entries + while True: + ld_header = get_ld_header(p) + if ld_header: + ldr_headers.append((ld_header, get_ld_header_info(p))) + else: + break + p.stdout.close() + p.wait + return ldr_headers + +def get_shared(ld_headers): + """ + extract the shareable objects from ld_headers + character "[" is used to strip off the path information. + Note: the "[" and "]" characters that are part of dump -H output + are not removed here. + """ + shared = [] + for (line, _) in ld_headers: + # potential member lines contain "[" + # otherwise, no processing needed + if "[" in line: + # Strip off trailing colon (:) + shared.append(line[line.index("["):-1]) + return shared + +def get_one_match(expr, lines): + """ + Must be only one match, otherwise result is None. + When there is a match, strip leading "[" and trailing "]" + """ + # member names in the ld_headers output are between square brackets + expr = rf'\[({expr})\]' + matches = list(filter(None, (re.search(expr, line) for line in lines))) + if len(matches) == 1: + return matches[0].group(1) + else: + return None + +# additional processing to deal with AIX legacy names for 64-bit members +def get_legacy(members): + """ + This routine provides historical aka legacy naming schemes started + in AIX4 shared library support for library members names. + e.g., in /usr/lib/libc.a the member name shr.o for 32-bit binary and + shr_64.o for 64-bit binary. + """ + if AIX_ABI == 64: + # AIX 64-bit member is one of shr64.o, shr_64.o, or shr4_64.o + expr = r'shr4?_?64\.o' + member = get_one_match(expr, members) + if member: + return member + else: + # 32-bit legacy names - both shr.o and shr4.o exist. + # shr.o is the preffered name so we look for shr.o first + # i.e., shr4.o is returned only when shr.o does not exist + for name in ['shr.o', 'shr4.o']: + member = get_one_match(re.escape(name), members) + if member: + return member + return None + +def get_version(name, members): + """ + Sort list of members and return highest numbered version - if it exists. + This function is called when an unversioned libFOO.a(libFOO.so) has + not been found. + + Versioning for the member name is expected to follow + GNU LIBTOOL conventions: the highest version (x, then X.y, then X.Y.z) + * find [libFoo.so.X] + * find [libFoo.so.X.Y] + * find [libFoo.so.X.Y.Z] + + Before the GNU convention became the standard scheme regardless of + binary size AIX packagers used GNU convention "as-is" for 32-bit + archive members but used an "distinguishing" name for 64-bit members. + This scheme inserted either 64 or _64 between libFOO and .so + - generally libFOO_64.so, but occasionally libFOO64.so + """ + # the expression ending for versions must start as + # '.so.[0-9]', i.e., *.so.[at least one digit] + # while multiple, more specific expressions could be specified + # to search for .so.X, .so.X.Y and .so.X.Y.Z + # after the first required 'dot' digit + # any combination of additional 'dot' digits pairs are accepted + # anything more than libFOO.so.digits.digits.digits + # should be seen as a member name outside normal expectations + exprs = [rf'lib{name}\.so\.[0-9]+[0-9.]*', + rf'lib{name}_?64\.so\.[0-9]+[0-9.]*'] + for expr in exprs: + versions = [] + for line in members: + m = re.search(expr, line) + if m: + versions.append(m.group(0)) + if versions: + return _last_version(versions, '.') + return None + +def get_member(name, members): + """ + Return an archive member matching the request in name. + Name is the library name without any prefix like lib, suffix like .so, + or version number. + Given a list of members find and return the most appropriate result + Priority is given to generic libXXX.so, then a versioned libXXX.so.a.b.c + and finally, legacy AIX naming scheme. + """ + # look first for a generic match - prepend lib and append .so + expr = rf'lib{name}\.so' + member = get_one_match(expr, members) + if member: + return member + elif AIX_ABI == 64: + expr = rf'lib{name}64\.so' + member = get_one_match(expr, members) + if member: + return member + # since an exact match with .so as suffix was not found + # look for a versioned name + # If a versioned name is not found, look for AIX legacy member name + member = get_version(name, members) + if member: + return member + else: + return get_legacy(members) + +def get_libpaths(): + """ + On AIX, the buildtime searchpath is stored in the executable. + as "loader header information". + The command /usr/bin/dump -H extracts this info. + Prefix searched libraries with LD_LIBRARY_PATH (preferred), + or LIBPATH if defined. These paths are appended to the paths + to libraries the python executable is linked with. + This mimics AIX dlopen() behavior. + """ + libpaths = environ.get("LD_LIBRARY_PATH") + if libpaths is None: + libpaths = environ.get("LIBPATH") + if libpaths is None: + libpaths = [] + else: + libpaths = libpaths.split(":") + objects = get_ld_headers(executable) + for (_, lines) in objects: + for line in lines: + # the second (optional) argument is PATH if it includes a / + path = line.split()[1] + if "/" in path: + libpaths.extend(path.split(":")) + return libpaths + +def find_shared(paths, name): + """ + paths is a list of directories to search for an archive. + name is the abbreviated name given to find_library(). + Process: search "paths" for archive, and if an archive is found + return the result of get_member(). + If an archive is not found then return None + """ + for dir in paths: + # /lib is a symbolic link to /usr/lib, skip it + if dir == "/lib": + continue + # "lib" is prefixed to emulate compiler name resolution, + # e.g., -lc to libc + base = f'lib{name}.a' + archive = path.join(dir, base) + if path.exists(archive): + members = get_shared(get_ld_headers(archive)) + member = get_member(re.escape(name), members) + if member != None: + return (base, member) + else: + return (None, None) + return (None, None) + +def find_library(name): + """AIX implementation of ctypes.util.find_library() + Find an archive member that will dlopen(). If not available, + also search for a file (or link) with a .so suffix. + + AIX supports two types of schemes that can be used with dlopen(). + The so-called SystemV Release4 (svr4) format is commonly suffixed + with .so while the (default) AIX scheme has the library (archive) + ending with the suffix .a + As an archive has multiple members (e.g., 32-bit and 64-bit) in one file + the argument passed to dlopen must include both the library and + the member names in a single string. + + find_library() looks first for an archive (.a) with a suitable member. + If no archive+member pair is found, look for a .so file. + """ + + libpaths = get_libpaths() + (base, member) = find_shared(libpaths, name) + if base != None: + return f"{base}({member})" + + # To get here, a member in an archive has not been found + # In other words, either: + # a) a .a file was not found + # b) a .a file did not have a suitable member + # So, look for a .so file + # Check libpaths for .so file + # Note, the installation must prepare a link from a .so + # to a versioned file + # This is common practice by GNU libtool on other platforms + soname = f"lib{name}.so" + for dir in libpaths: + # /lib is a symbolic link to /usr/lib, skip it + if dir == "/lib": + continue + shlib = path.join(dir, soname) + if path.exists(shlib): + return soname + # if we are here, we have not found anything plausible + return None diff --git a/Lib/ctypes/test/test_pep3118.py b/Lib/ctypes/test/test_pep3118.py index f3c0e23e53e..81e8ca7638f 100644 --- a/Lib/ctypes/test/test_pep3118.py +++ b/Lib/ctypes/test/test_pep3118.py @@ -188,7 +188,7 @@ native_types = [ (PackedPoint, "B", (), PackedPoint), (Point2, "T{ | No action: no method is added. | +# +---------+-----------------------------------------+ +# | add | Generated method is added. | +# +---------+-----------------------------------------+ +# | add* | Generated method is added only if the | +# | | existing attribute is None and if the | +# | | user supplied a __eq__ method in the | +# | | class definition. | +# +---------+-----------------------------------------+ +# | raise | TypeError is raised. | +# +---------+-----------------------------------------+ +# | None | Attribute is set to None. | +# +=========+=========================================+ + +# __init__ +# +# +--- init= parameter +# | +# v | | | +# | no | yes | <--- class has __init__ in __dict__? +# +=======+=======+=======+ +# | False | | | +# +-------+-------+-------+ +# | True | add | | <- the default +# +=======+=======+=======+ + +# __repr__ +# +# +--- repr= parameter +# | +# v | | | +# | no | yes | <--- class has __repr__ in __dict__? +# +=======+=======+=======+ +# | False | | | +# +-------+-------+-------+ +# | True | add | | <- the default +# +=======+=======+=======+ + + +# __setattr__ +# __delattr__ +# +# +--- frozen= parameter +# | +# v | | | +# | no | yes | <--- class has __setattr__ or __delattr__ in __dict__? +# +=======+=======+=======+ +# | False | | | <- the default +# +-------+-------+-------+ +# | True | add | raise | +# +=======+=======+=======+ +# Raise because not adding these methods would break the "frozen-ness" +# of the class. + +# __eq__ +# +# +--- eq= parameter +# | +# v | | | +# | no | yes | <--- class has __eq__ in __dict__? +# +=======+=======+=======+ +# | False | | | +# +-------+-------+-------+ +# | True | add | | <- the default +# +=======+=======+=======+ + +# __lt__ +# __le__ +# __gt__ +# __ge__ +# +# +--- order= parameter +# | +# v | | | +# | no | yes | <--- class has any comparison method in __dict__? +# +=======+=======+=======+ +# | False | | | <- the default +# +-------+-------+-------+ +# | True | add | raise | +# +=======+=======+=======+ +# Raise because to allow this case would interfere with using +# functools.total_ordering. + +# __hash__ + +# +------------------- hash= parameter +# | +----------- eq= parameter +# | | +--- frozen= parameter +# | | | +# v v v | | | +# | no | yes | <--- class has __hash__ in __dict__? +# +=========+=======+=======+========+========+ +# | 1 None | False | False | | | No __eq__, use the base class __hash__ +# +---------+-------+-------+--------+--------+ +# | 2 None | False | True | | | No __eq__, use the base class __hash__ +# +---------+-------+-------+--------+--------+ +# | 3 None | True | False | None | | <-- the default, not hashable +# +---------+-------+-------+--------+--------+ +# | 4 None | True | True | add | add* | Frozen, so hashable +# +---------+-------+-------+--------+--------+ +# | 5 False | False | False | | | +# +---------+-------+-------+--------+--------+ +# | 6 False | False | True | | | +# +---------+-------+-------+--------+--------+ +# | 7 False | True | False | | | +# +---------+-------+-------+--------+--------+ +# | 8 False | True | True | | | +# +---------+-------+-------+--------+--------+ +# | 9 True | False | False | add | add* | Has no __eq__, but hashable +# +---------+-------+-------+--------+--------+ +# |10 True | False | True | add | add* | Has no __eq__, but hashable +# +---------+-------+-------+--------+--------+ +# |11 True | True | False | add | add* | Not frozen, but hashable +# +---------+-------+-------+--------+--------+ +# |12 True | True | True | add | add* | Frozen, so hashable +# +=========+=======+=======+========+========+ +# For boxes that are blank, __hash__ is untouched and therefore +# inherited from the base class. If the base is object, then +# id-based hashing is used. +# Note that a class may have already __hash__=None if it specified an +# __eq__ method in the class body (not one that was created by +# @dataclass). + + +# Raised when an attempt is made to modify a frozen class. +class FrozenInstanceError(AttributeError): pass + +# A sentinel object for default values to signal that a +# default-factory will be used. +# This is given a nice repr() which will appear in the function +# signature of dataclasses' constructors. +class _HAS_DEFAULT_FACTORY_CLASS: + def __repr__(self): + return '' +_HAS_DEFAULT_FACTORY = _HAS_DEFAULT_FACTORY_CLASS() + +# A sentinel object to detect if a parameter is supplied or not. Use +# a class to give it a better repr. +class _MISSING_TYPE: + pass +MISSING = _MISSING_TYPE() + +# Since most per-field metadata will be unused, create an empty +# read-only proxy that can be shared among all fields. +_EMPTY_METADATA = types.MappingProxyType({}) + +# Markers for the various kinds of fields and pseudo-fields. +_FIELD = object() # An actual field. +_FIELD_CLASSVAR = object() # Not a field, but a ClassVar. +_FIELD_INITVAR = object() # Not a field, but an InitVar. + +# The name of an attribute on the class where we store the Field +# objects. Also used to check if a class is a Data Class. +_MARKER = '__dataclass_fields__' + +# The name of the function, that if it exists, is called at the end of +# __init__. +_POST_INIT_NAME = '__post_init__' + + +class _InitVarMeta(type): + def __getitem__(self, params): + return self + +class InitVar(metaclass=_InitVarMeta): + pass + + +# Instances of Field are only ever created from within this module, +# and only from the field() function, although Field instances are +# exposed externally as (conceptually) read-only objects. +# name and type are filled in after the fact, not in __init__. They're +# not known at the time this class is instantiated, but it's +# convenient if they're available later. +# When cls._MARKER is filled in with a list of Field objects, the name +# and type fields will have been populated. +class Field: + __slots__ = ('name', + 'type', + 'default', + 'default_factory', + 'repr', + 'hash', + 'init', + 'compare', + 'metadata', + '_field_type', # Private: not to be used by user code. + ) + + def __init__(self, default, default_factory, init, repr, hash, compare, + metadata): + self.name = None + self.type = None + self.default = default + self.default_factory = default_factory + self.init = init + self.repr = repr + self.hash = hash + self.compare = compare + self.metadata = (_EMPTY_METADATA + if metadata is None or len(metadata) == 0 else + types.MappingProxyType(metadata)) + self._field_type = None + + def __repr__(self): + return ('Field(' + f'name={self.name!r},' + f'type={self.type},' + f'default={self.default},' + f'default_factory={self.default_factory},' + f'init={self.init},' + f'repr={self.repr},' + f'hash={self.hash},' + f'compare={self.compare},' + f'metadata={self.metadata}' + ')') + + +# This function is used instead of exposing Field creation directly, +# so that a type checker can be told (via overloads) that this is a +# function whose type depends on its parameters. +def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, + hash=None, compare=True, metadata=None): + """Return an object to identify dataclass fields. + + default is the default value of the field. default_factory is a + 0-argument function called to initialize a field's value. If init + is True, the field will be a parameter to the class's __init__() + function. If repr is True, the field will be included in the + object's repr(). If hash is True, the field will be included in + the object's hash(). If compare is True, the field will be used in + comparison functions. metadata, if specified, must be a mapping + which is stored but not otherwise examined by dataclass. + + It is an error to specify both default and default_factory. + """ + + if default is not MISSING and default_factory is not MISSING: + raise ValueError('cannot specify both default and default_factory') + return Field(default, default_factory, init, repr, hash, compare, + metadata) + + +def _tuple_str(obj_name, fields): + # Return a string representing each field of obj_name as a tuple + # member. So, if fields is ['x', 'y'] and obj_name is "self", + # return "(self.x,self.y)". + + # Special case for the 0-tuple. + if not fields: + return '()' + # Note the trailing comma, needed if this turns out to be a 1-tuple. + return f'({",".join([f"{obj_name}.{f.name}" for f in fields])},)' + + +def _create_fn(name, args, body, *, globals=None, locals=None, + return_type=MISSING): + # Note that we mutate locals when exec() is called. Caller beware! + if locals is None: + locals = {} + return_annotation = '' + if return_type is not MISSING: + locals['_return_type'] = return_type + return_annotation = '->_return_type' + args = ','.join(args) + body = '\n'.join(f' {b}' for b in body) + + txt = f'def {name}({args}){return_annotation}:\n{body}' + + exec(txt, globals, locals) + return locals[name] + + +def _field_assign(frozen, name, value, self_name): + # If we're a frozen class, then assign to our fields in __init__ + # via object.__setattr__. Otherwise, just use a simple + # assignment. + # self_name is what "self" is called in this function: don't + # hard-code "self", since that might be a field name. + if frozen: + return f'object.__setattr__({self_name},{name!r},{value})' + return f'{self_name}.{name}={value}' + + +def _field_init(f, frozen, globals, self_name): + # Return the text of the line in the body of __init__ that will + # initialize this field. + + default_name = f'_dflt_{f.name}' + if f.default_factory is not MISSING: + if f.init: + # This field has a default factory. If a parameter is + # given, use it. If not, call the factory. + globals[default_name] = f.default_factory + value = (f'{default_name}() ' + f'if {f.name} is _HAS_DEFAULT_FACTORY ' + f'else {f.name}') + else: + # This is a field that's not in the __init__ params, but + # has a default factory function. It needs to be + # initialized here by calling the factory function, + # because there's no other way to initialize it. + + # For a field initialized with a default=defaultvalue, the + # class dict just has the default value + # (cls.fieldname=defaultvalue). But that won't work for a + # default factory, the factory must be called in __init__ + # and we must assign that to self.fieldname. We can't + # fall back to the class dict's value, both because it's + # not set, and because it might be different per-class + # (which, after all, is why we have a factory function!). + + globals[default_name] = f.default_factory + value = f'{default_name}()' + else: + # No default factory. + if f.init: + if f.default is MISSING: + # There's no default, just do an assignment. + value = f.name + elif f.default is not MISSING: + globals[default_name] = f.default + value = f.name + else: + # This field does not need initialization. Signify that to + # the caller by returning None. + return None + + # Only test this now, so that we can create variables for the + # default. However, return None to signify that we're not going + # to actually do the assignment statement for InitVars. + if f._field_type == _FIELD_INITVAR: + return None + + # Now, actually generate the field assignment. + return _field_assign(frozen, f.name, value, self_name) + + +def _init_param(f): + # Return the __init__ parameter string for this field. + # For example, the equivalent of 'x:int=3' (except instead of 'int', + # reference a variable set to int, and instead of '3', reference a + # variable set to 3). + if f.default is MISSING and f.default_factory is MISSING: + # There's no default, and no default_factory, just + # output the variable name and type. + default = '' + elif f.default is not MISSING: + # There's a default, this will be the name that's used to look it up. + default = f'=_dflt_{f.name}' + elif f.default_factory is not MISSING: + # There's a factory function. Set a marker. + default = '=_HAS_DEFAULT_FACTORY' + return f'{f.name}:_type_{f.name}{default}' + + +def _init_fn(fields, frozen, has_post_init, self_name): + # fields contains both real fields and InitVar pseudo-fields. + + # Make sure we don't have fields without defaults following fields + # with defaults. This actually would be caught when exec-ing the + # function source code, but catching it here gives a better error + # message, and future-proofs us in case we build up the function + # using ast. + seen_default = False + for f in fields: + # Only consider fields in the __init__ call. + if f.init: + if not (f.default is MISSING and f.default_factory is MISSING): + seen_default = True + elif seen_default: + raise TypeError(f'non-default argument {f.name!r} ' + 'follows default argument') + + globals = {'MISSING': MISSING, + '_HAS_DEFAULT_FACTORY': _HAS_DEFAULT_FACTORY} + + body_lines = [] + for f in fields: + # Do not initialize the pseudo-fields, only the real ones. + line = _field_init(f, frozen, globals, self_name) + if line is not None: + # line is None means that this field doesn't require + # initialization. Just skip it. + body_lines.append(line) + + # Does this class have a post-init function? + if has_post_init: + params_str = ','.join(f.name for f in fields + if f._field_type is _FIELD_INITVAR) + body_lines += [f'{self_name}.{_POST_INIT_NAME}({params_str})'] + + # If no body lines, use 'pass'. + if not body_lines: + body_lines = ['pass'] + + locals = {f'_type_{f.name}': f.type for f in fields} + return _create_fn('__init__', + [self_name] +[_init_param(f) for f in fields if f.init], + body_lines, + locals=locals, + globals=globals, + return_type=None) + + +def _repr_fn(fields): + return _create_fn('__repr__', + ['self'], + ['return self.__class__.__qualname__ + f"(' + + ', '.join([f"{f.name}={{self.{f.name}!r}}" + for f in fields]) + + ')"']) + + +def _frozen_setattr(self, name, value): + raise FrozenInstanceError(f'cannot assign to field {name!r}') + + +def _frozen_delattr(self, name): + raise FrozenInstanceError(f'cannot delete field {name!r}') + + +def _cmp_fn(name, op, self_tuple, other_tuple): + # Create a comparison function. If the fields in the object are + # named 'x' and 'y', then self_tuple is the string + # '(self.x,self.y)' and other_tuple is the string + # '(other.x,other.y)'. + + return _create_fn(name, + ['self', 'other'], + [ 'if other.__class__ is self.__class__:', + f' return {self_tuple}{op}{other_tuple}', + 'return NotImplemented']) + + +def _hash_fn(fields): + self_tuple = _tuple_str('self', fields) + return _create_fn('__hash__', + ['self'], + [f'return hash({self_tuple})']) + + +def _get_field(cls, a_name, a_type): + # Return a Field object, for this field name and type. ClassVars + # and InitVars are also returned, but marked as such (see + # f._field_type). + + # If the default value isn't derived from field, then it's + # only a normal default value. Convert it to a Field(). + default = getattr(cls, a_name, MISSING) + if isinstance(default, Field): + f = default + else: + f = field(default=default) + + # Assume it's a normal field until proven otherwise. + f._field_type = _FIELD + + # Only at this point do we know the name and the type. Set them. + f.name = a_name + f.type = a_type + + # If typing has not been imported, then it's impossible for + # any annotation to be a ClassVar. So, only look for ClassVar + # if typing has been imported. + typing = sys.modules.get('typing') + if typing is not None: + # This test uses a typing internal class, but it's the best + # way to test if this is a ClassVar. + if (type(a_type) is typing._GenericAlias and + a_type.__origin__ is typing.ClassVar): + # This field is a ClassVar, so it's not a field. + f._field_type = _FIELD_CLASSVAR + + if f._field_type is _FIELD: + # Check if this is an InitVar. + if a_type is InitVar: + # InitVars are not fields, either. + f._field_type = _FIELD_INITVAR + + # Validations for fields. This is delayed until now, instead of + # in the Field() constructor, since only here do we know the field + # name, which allows better error reporting. + + # Special restrictions for ClassVar and InitVar. + if f._field_type in (_FIELD_CLASSVAR, _FIELD_INITVAR): + if f.default_factory is not MISSING: + raise TypeError(f'field {f.name} cannot have a ' + 'default factory') + # Should I check for other field settings? default_factory + # seems the most serious to check for. Maybe add others. For + # example, how about init=False (or really, + # init=)? It makes no sense for + # ClassVar and InitVar to specify init=. + + # For real fields, disallow mutable defaults for known types. + if f._field_type is _FIELD and isinstance(f.default, (list, dict, set)): + raise ValueError(f'mutable default {type(f.default)} for field ' + f'{f.name} is not allowed: use default_factory') + + return f + + +def _find_fields(cls): + # Return a list of Field objects, in order, for this class (and no + # base classes). Fields are found from __annotations__ (which is + # guaranteed to be ordered). Default values are from class + # attributes, if a field has a default. If the default value is + # a Field(), then it contains additional info beyond (and + # possibly including) the actual default value. Pseudo-fields + # ClassVars and InitVars are included, despite the fact that + # they're not real fields. That's dealt with later. + + annotations = getattr(cls, '__annotations__', {}) + return [_get_field(cls, a_name, a_type) + for a_name, a_type in annotations.items()] + + +def _set_new_attribute(cls, name, value): + # Never overwrites an existing attribute. Returns True if the + # attribute already exists. + if name in cls.__dict__: + return True + setattr(cls, name, value) + return False + + +def _process_class(cls, repr, eq, order, hash, init, frozen): + # Now that dicts retain insertion order, there's no reason to use + # an ordered dict. I am leveraging that ordering here, because + # derived class fields overwrite base class fields, but the order + # is defined by the base class, which is found first. + fields = {} + + # Find our base classes in reverse MRO order, and exclude + # ourselves. In reversed order so that more derived classes + # override earlier field definitions in base classes. + for b in cls.__mro__[-1:0:-1]: + # Only process classes that have been processed by our + # decorator. That is, they have a _MARKER attribute. + base_fields = getattr(b, _MARKER, None) + if base_fields: + for f in base_fields.values(): + fields[f.name] = f + + # Now find fields in our class. While doing so, validate some + # things, and set the default values (as class attributes) + # where we can. + for f in _find_fields(cls): + fields[f.name] = f + + # If the class attribute (which is the default value for + # this field) exists and is of type 'Field', replace it + # with the real default. This is so that normal class + # introspection sees a real default value, not a Field. + if isinstance(getattr(cls, f.name, None), Field): + if f.default is MISSING: + # If there's no default, delete the class attribute. + # This happens if we specify field(repr=False), for + # example (that is, we specified a field object, but + # no default value). Also if we're using a default + # factory. The class attribute should not be set at + # all in the post-processed class. + delattr(cls, f.name) + else: + setattr(cls, f.name, f.default) + + # Remember all of the fields on our class (including bases). This + # marks this class as being a dataclass. + setattr(cls, _MARKER, fields) + + # We also need to check if a parent class is frozen: frozen has to + # be inherited down. + is_frozen = frozen or cls.__setattr__ is _frozen_setattr + + # Was this class defined with an __eq__? Used in __hash__ logic. + auto_hash_test= '__eq__' in cls.__dict__ and getattr(cls.__dict__, '__hash__', MISSING) is None + + # If we're generating ordering methods, we must be generating + # the eq methods. + if order and not eq: + raise ValueError('eq must be true if order is true') + + if init: + # Does this class have a post-init function? + has_post_init = hasattr(cls, _POST_INIT_NAME) + + # Include InitVars and regular fields (so, not ClassVars). + flds = [f for f in fields.values() + if f._field_type in (_FIELD, _FIELD_INITVAR)] + _set_new_attribute(cls, '__init__', + _init_fn(flds, + is_frozen, + has_post_init, + # The name to use for the "self" param + # in __init__. Use "self" if possible. + '__dataclass_self__' if 'self' in fields + else 'self', + )) + + # Get the fields as a list, and include only real fields. This is + # used in all of the following methods. + field_list = [f for f in fields.values() if f._field_type is _FIELD] + + if repr: + flds = [f for f in field_list if f.repr] + _set_new_attribute(cls, '__repr__', _repr_fn(flds)) + + if eq: + # Create _eq__ method. There's no need for a __ne__ method, + # since python will call __eq__ and negate it. + flds = [f for f in field_list if f.compare] + self_tuple = _tuple_str('self', flds) + other_tuple = _tuple_str('other', flds) + _set_new_attribute(cls, '__eq__', + _cmp_fn('__eq__', '==', + self_tuple, other_tuple)) + + if order: + # Create and set the ordering methods. + flds = [f for f in field_list if f.compare] + self_tuple = _tuple_str('self', flds) + other_tuple = _tuple_str('other', flds) + for name, op in [('__lt__', '<'), + ('__le__', '<='), + ('__gt__', '>'), + ('__ge__', '>='), + ]: + if _set_new_attribute(cls, name, + _cmp_fn(name, op, self_tuple, other_tuple)): + raise TypeError(f'Cannot overwrite attribute {name} ' + f'in {cls.__name__}. Consider using ' + 'functools.total_ordering') + + if is_frozen: + for name, fn in [('__setattr__', _frozen_setattr), + ('__delattr__', _frozen_delattr)]: + if _set_new_attribute(cls, name, fn): + raise TypeError(f'Cannot overwrite attribute {name} ' + f'in {cls.__name__}') + + # Decide if/how we're going to create a hash function. + # TODO: Move this table to module scope, so it's not recreated + # all the time. + generate_hash = {(None, False, False): ('', ''), + (None, False, True): ('', ''), + (None, True, False): ('none', ''), + (None, True, True): ('fn', 'fn-x'), + (False, False, False): ('', ''), + (False, False, True): ('', ''), + (False, True, False): ('', ''), + (False, True, True): ('', ''), + (True, False, False): ('fn', 'fn-x'), + (True, False, True): ('fn', 'fn-x'), + (True, True, False): ('fn', 'fn-x'), + (True, True, True): ('fn', 'fn-x'), + }[None if hash is None else bool(hash), # Force bool() if not None. + bool(eq), + bool(frozen)]['__hash__' in cls.__dict__] + # No need to call _set_new_attribute here, since we already know if + # we're overwriting a __hash__ or not. + if generate_hash == '': + # Do nothing. + pass + elif generate_hash == 'none': + cls.__hash__ = None + elif generate_hash in ('fn', 'fn-x'): + if generate_hash == 'fn' or auto_hash_test: + flds = [f for f in field_list + if (f.compare if f.hash is None else f.hash)] + cls.__hash__ = _hash_fn(flds) + else: + assert False, f"can't get here: {generate_hash}" + + if not getattr(cls, '__doc__'): + # Create a class doc-string. + cls.__doc__ = (cls.__name__ + + str(inspect.signature(cls)).replace(' -> None', '')) + + return cls + + +# _cls should never be specified by keyword, so start it with an +# underscore. The presence of _cls is used to detect if this +# decorator is being called with parameters or not. +def dataclass(_cls=None, *, init=True, repr=True, eq=True, order=False, + hash=None, frozen=False): + """Returns the same class as was passed in, with dunder methods + added based on the fields defined in the class. + + Examines PEP 526 __annotations__ to determine fields. + + If init is true, an __init__() method is added to the class. If + repr is true, a __repr__() method is added. If order is true, rich + comparison dunder methods are added. If hash is true, a __hash__() + method function is added. If frozen is true, fields may not be + assigned to after instance creation. + """ + + def wrap(cls): + return _process_class(cls, repr, eq, order, hash, init, frozen) + + # See if we're being called as @dataclass or @dataclass(). + if _cls is None: + # We're called with parens. + return wrap + + # We're called as @dataclass without parens. + return wrap(_cls) + + +def fields(class_or_instance): + """Return a tuple describing the fields of this dataclass. + + Accepts a dataclass or an instance of one. Tuple elements are of + type Field. + """ + + # Might it be worth caching this, per class? + try: + fields = getattr(class_or_instance, _MARKER) + except AttributeError: + raise TypeError('must be called with a dataclass type or instance') + + # Exclude pseudo-fields. Note that fields is sorted by insertion + # order, so the order of the tuple is as the fields were defined. + return tuple(f for f in fields.values() if f._field_type is _FIELD) + + +def _is_dataclass_instance(obj): + """Returns True if obj is an instance of a dataclass.""" + return not isinstance(obj, type) and hasattr(obj, _MARKER) + + +def is_dataclass(obj): + """Returns True if obj is a dataclass or an instance of a + dataclass.""" + return hasattr(obj, _MARKER) + + +def asdict(obj, *, dict_factory=dict): + """Return the fields of a dataclass instance as a new dictionary mapping + field names to field values. + + Example usage: + + @dataclass + class C: + x: int + y: int + + c = C(1, 2) + assert asdict(c) == {'x': 1, 'y': 2} + + If given, 'dict_factory' will be used instead of built-in dict. + The function applies recursively to field values that are + dataclass instances. This will also look into built-in containers: + tuples, lists, and dicts. + """ + if not _is_dataclass_instance(obj): + raise TypeError("asdict() should be called on dataclass instances") + return _asdict_inner(obj, dict_factory) + +def _asdict_inner(obj, dict_factory): + if _is_dataclass_instance(obj): + result = [] + for f in fields(obj): + value = _asdict_inner(getattr(obj, f.name), dict_factory) + result.append((f.name, value)) + return dict_factory(result) + elif isinstance(obj, (list, tuple)): + return type(obj)(_asdict_inner(v, dict_factory) for v in obj) + elif isinstance(obj, dict): + return type(obj)((_asdict_inner(k, dict_factory), _asdict_inner(v, dict_factory)) + for k, v in obj.items()) + else: + return deepcopy(obj) + + +def astuple(obj, *, tuple_factory=tuple): + """Return the fields of a dataclass instance as a new tuple of field values. + + Example usage:: + + @dataclass + class C: + x: int + y: int + + c = C(1, 2) + assert astuple(c) == (1, 2) + + If given, 'tuple_factory' will be used instead of built-in tuple. + The function applies recursively to field values that are + dataclass instances. This will also look into built-in containers: + tuples, lists, and dicts. + """ + + if not _is_dataclass_instance(obj): + raise TypeError("astuple() should be called on dataclass instances") + return _astuple_inner(obj, tuple_factory) + +def _astuple_inner(obj, tuple_factory): + if _is_dataclass_instance(obj): + result = [] + for f in fields(obj): + value = _astuple_inner(getattr(obj, f.name), tuple_factory) + result.append(value) + return tuple_factory(result) + elif isinstance(obj, (list, tuple)): + return type(obj)(_astuple_inner(v, tuple_factory) for v in obj) + elif isinstance(obj, dict): + return type(obj)((_astuple_inner(k, tuple_factory), _astuple_inner(v, tuple_factory)) + for k, v in obj.items()) + else: + return deepcopy(obj) + + +def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, + repr=True, eq=True, order=False, hash=None, frozen=False): + """Return a new dynamically created dataclass. + + The dataclass name will be 'cls_name'. 'fields' is an iterable + of either (name), (name, type) or (name, type, Field) objects. If type is + omitted, use the string 'typing.Any'. Field objects are created by + the equivalent of calling 'field(name, type [, Field-info])'. + + C = make_dataclass('C', ['x', ('y', int), ('z', int, field(init=False))], bases=(Base,)) + + is equivalent to: + + @dataclass + class C(Base): + x: 'typing.Any' + y: int + z: int = field(init=False) + + For the bases and namespace parameters, see the builtin type() function. + + The parameters init, repr, eq, order, hash, and frozen are passed to + dataclass(). + """ + + if namespace is None: + namespace = {} + else: + # Copy namespace since we're going to mutate it. + namespace = namespace.copy() + + anns = {} + for item in fields: + if isinstance(item, str): + name = item + tp = 'typing.Any' + elif len(item) == 2: + name, tp, = item + elif len(item) == 3: + name, tp, spec = item + namespace[name] = spec + anns[name] = tp + + namespace['__annotations__'] = anns + cls = type(cls_name, bases, namespace) + return dataclass(cls, init=init, repr=repr, eq=eq, order=order, + hash=hash, frozen=frozen) + +def replace(obj, **changes): + """Return a new object replacing specified fields with new values. + + This is especially useful for frozen classes. Example usage: + + @dataclass(frozen=True) + class C: + x: int + y: int + + c = C(1, 2) + c1 = replace(c, x=3) + assert c1.x == 3 and c1.y == 2 + """ + + # We're going to mutate 'changes', but that's okay because it's a new + # dict, even if called with 'replace(obj, **my_changes)'. + + if not _is_dataclass_instance(obj): + raise TypeError("replace() should be called on dataclass instances") + + # It's an error to have init=False fields in 'changes'. + # If a field is not in 'changes', read its value from the provided obj. + + for f in getattr(obj, _MARKER).values(): + if not f.init: + # Error if this field is specified in changes. + if f.name in changes: + raise ValueError(f'field {f.name} is declared with ' + 'init=False, it cannot be specified with ' + 'replace()') + continue + + if f.name not in changes: + changes[f.name] = getattr(obj, f.name) + + # Create the new object, which calls __init__() and __post_init__ + # (if defined), using all of the init fields we've added and/or + # left in 'changes'. + # If there are values supplied in changes that aren't fields, this + # will correctly raise a TypeError. + return obj.__class__(**changes) diff --git a/Lib/datetime.py b/Lib/datetime.py index 67d8600921c..8fa18a78932 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -173,6 +173,24 @@ def _format_time(hh, mm, ss, us, timespec='auto'): else: return fmt.format(hh, mm, ss, us) +def _format_offset(off): + s = '' + if off is not None: + if off.days < 0: + sign = "-" + off = -off + else: + sign = "+" + hh, mm = divmod(off, timedelta(hours=1)) + mm, ss = divmod(mm, timedelta(minutes=1)) + s += "%s%02d:%02d" % (sign, hh, mm) + if ss or ss.microseconds: + s += ":%02d" % ss.seconds + + if ss.microseconds: + s += '.%06d' % ss.microseconds + return s + # Correctly substitute for %z and %Z escapes in strftime formats. def _wrap_strftime(object, format, timetuple): # Don't call utcoffset() or tzname() unless actually needed. @@ -237,6 +255,102 @@ def _wrap_strftime(object, format, timetuple): newformat = "".join(newformat) return _time.strftime(newformat, timetuple) +# Helpers for parsing the result of isoformat() +def _parse_isoformat_date(dtstr): + # It is assumed that this function will only be called with a + # string of length exactly 10, and (though this is not used) ASCII-only + year = int(dtstr[0:4]) + if dtstr[4] != '-': + raise ValueError('Invalid date separator: %s' % dtstr[4]) + + month = int(dtstr[5:7]) + + if dtstr[7] != '-': + raise ValueError('Invalid date separator') + + day = int(dtstr[8:10]) + + return [year, month, day] + +def _parse_hh_mm_ss_ff(tstr): + # Parses things of the form HH[:MM[:SS[.fff[fff]]]] + len_str = len(tstr) + + time_comps = [0, 0, 0, 0] + pos = 0 + for comp in range(0, 3): + if (len_str - pos) < 2: + raise ValueError('Incomplete time component') + + time_comps[comp] = int(tstr[pos:pos+2]) + + pos += 2 + next_char = tstr[pos:pos+1] + + if not next_char or comp >= 2: + break + + if next_char != ':': + raise ValueError('Invalid time separator: %c' % next_char) + + pos += 1 + + if pos < len_str: + if tstr[pos] != '.': + raise ValueError('Invalid microsecond component') + else: + pos += 1 + + len_remainder = len_str - pos + if len_remainder not in (3, 6): + raise ValueError('Invalid microsecond component') + + time_comps[3] = int(tstr[pos:]) + if len_remainder == 3: + time_comps[3] *= 1000 + + return time_comps + +def _parse_isoformat_time(tstr): + # Format supported is HH[:MM[:SS[.fff[fff]]]][+HH:MM[:SS[.ffffff]]] + len_str = len(tstr) + if len_str < 2: + raise ValueError('Isoformat time too short') + + # This is equivalent to re.search('[+-]', tstr), but faster + tz_pos = (tstr.find('-') + 1 or tstr.find('+') + 1) + timestr = tstr[:tz_pos-1] if tz_pos > 0 else tstr + + time_comps = _parse_hh_mm_ss_ff(timestr) + + tzi = None + if tz_pos > 0: + tzstr = tstr[tz_pos:] + + # Valid time zone strings are: + # HH:MM len: 5 + # HH:MM:SS len: 8 + # HH:MM:SS.ffffff len: 15 + + if len(tzstr) not in (5, 8, 15): + raise ValueError('Malformed time zone string') + + tz_comps = _parse_hh_mm_ss_ff(tzstr) + if all(x == 0 for x in tz_comps): + tzi = timezone.utc + else: + tzsign = -1 if tstr[tz_pos - 1] == '-' else 1 + + td = timedelta(hours=tz_comps[0], minutes=tz_comps[1], + seconds=tz_comps[2], microseconds=tz_comps[3]) + + tzi = timezone(tzsign * td) + + time_comps.append(tzi) + + return time_comps + + # Just raise TypeError if the arg isn't None or a string. def _check_tzname(name): if name is not None and not isinstance(name, str): @@ -732,6 +846,19 @@ class date: y, m, d = _ord2ymd(n) return cls(y, m, d) + @classmethod + def fromisoformat(cls, date_string): + """Construct a date from the output of date.isoformat().""" + if not isinstance(date_string, str): + raise TypeError('fromisoformat: argument must be str') + + try: + assert len(date_string) == 10 + return cls(*_parse_isoformat_date(date_string)) + except Exception: + raise ValueError('Invalid isoformat string: %s' % date_string) + + # Conversions to string def __repr__(self): @@ -1190,22 +1317,10 @@ class time: # Conversion to string - def _tzstr(self, sep=":"): - """Return formatted timezone offset (+xx:xx) or None.""" + def _tzstr(self): + """Return formatted timezone offset (+xx:xx) or an empty string.""" off = self.utcoffset() - if off is not None: - if off.days < 0: - sign = "-" - off = -off - else: - sign = "+" - hh, mm = divmod(off, timedelta(hours=1)) - mm, ss = divmod(mm, timedelta(minutes=1)) - assert 0 <= hh < 24 - off = "%s%02d%s%02d" % (sign, hh, sep, mm) - if ss: - off += ':%02d' % ss.seconds - return off + return _format_offset(off) def __repr__(self): """Convert to formal string, for repr().""" @@ -1244,6 +1359,18 @@ class time: __str__ = isoformat + @classmethod + def fromisoformat(cls, time_string): + """Construct a time from the output of isoformat().""" + if not isinstance(time_string, str): + raise TypeError('fromisoformat: argument must be str') + + try: + return cls(*_parse_isoformat_time(time_string)) + except Exception: + raise ValueError('Invalid isoformat string: %s' % time_string) + + def strftime(self, fmt): """Format using strftime(). The date part of the timestamp passed to underlying strftime should not be used. @@ -1497,6 +1624,31 @@ class datetime(date): time.hour, time.minute, time.second, time.microsecond, tzinfo, fold=time.fold) + @classmethod + def fromisoformat(cls, date_string): + """Construct a datetime from the output of datetime.isoformat().""" + if not isinstance(date_string, str): + raise TypeError('fromisoformat: argument must be str') + + # Split this at the separator + dstr = date_string[0:10] + tstr = date_string[11:] + + try: + date_components = _parse_isoformat_date(dstr) + except ValueError: + raise ValueError('Invalid isoformat string: %s' % date_string) + + if tstr: + try: + time_components = _parse_isoformat_time(tstr) + except ValueError: + raise ValueError('Invalid isoformat string: %s' % date_string) + else: + time_components = [0, 0, 0, 0, None] + + return cls(*(date_components + time_components)) + def timetuple(self): "Return local time tuple compatible with time.localtime()." dst = self.dst() @@ -1673,18 +1825,10 @@ class datetime(date): self._microsecond, timespec)) off = self.utcoffset() - if off is not None: - if off.days < 0: - sign = "-" - off = -off - else: - sign = "+" - hh, mm = divmod(off, timedelta(hours=1)) - mm, ss = divmod(mm, timedelta(minutes=1)) - s += "%s%02d:%02d" % (sign, hh, mm) - if ss: - assert not ss.microseconds - s += ":%02d" % ss.seconds + tz = _format_offset(off) + if tz: + s += tz + return s def __repr__(self): @@ -2275,9 +2419,10 @@ else: _check_date_fields, _check_int_field, _check_time_fields, _check_tzinfo_arg, _check_tzname, _check_utc_offset, _cmp, _cmperror, _date_class, _days_before_month, _days_before_year, _days_in_month, - _format_time, _is_leap, _isoweek1monday, _math, _ord2ymd, - _time, _time_class, _tzinfo_class, _wrap_strftime, _ymd2ord, - _divide_and_round) + _format_time, _format_offset, _is_leap, _isoweek1monday, _math, + _ord2ymd, _time, _time_class, _tzinfo_class, _wrap_strftime, _ymd2ord, + _divide_and_round, _parse_isoformat_date, _parse_isoformat_time, + _parse_hh_mm_ss_ff) # XXX Since import * above excludes names that start with _, # docstring does not get overwritten. In the future, it may be # appropriate to maintain a single module level docstring and diff --git a/Lib/dbm/dumb.py b/Lib/dbm/dumb.py index 5064668c77e..e5c17f5ae2e 100644 --- a/Lib/dbm/dumb.py +++ b/Lib/dbm/dumb.py @@ -82,10 +82,7 @@ class _Database(collections.abc.MutableMapping): f = _io.open(self._datfile, 'r', encoding="Latin-1") except OSError: if flag not in ('c', 'n'): - import warnings - warnings.warn("The database file is missing, the " - "semantics of the 'c' flag will be used.", - DeprecationWarning, stacklevel=4) + raise with _io.open(self._datfile, 'w', encoding="Latin-1") as f: self._chmod(self._datfile) else: @@ -93,18 +90,15 @@ class _Database(collections.abc.MutableMapping): # Read directory file into the in-memory index dict. def _update(self, flag): + self._modified = False self._index = {} try: f = _io.open(self._dirfile, 'r', encoding="Latin-1") except OSError: - self._modified = not self._readonly if flag not in ('c', 'n'): - import warnings - warnings.warn("The index file is missing, the " - "semantics of the 'c' flag will be used.", - DeprecationWarning, stacklevel=4) + raise + self._modified = True else: - self._modified = False with f: for line in f: line = line.rstrip() @@ -191,9 +185,7 @@ class _Database(collections.abc.MutableMapping): def __setitem__(self, key, val): if self._readonly: - import warnings - warnings.warn('The database is opened for reading only', - DeprecationWarning, stacklevel=2) + raise ValueError('The database is opened for reading only') if isinstance(key, str): key = key.encode('utf-8') elif not isinstance(key, (bytes, bytearray)): @@ -230,9 +222,7 @@ class _Database(collections.abc.MutableMapping): def __delitem__(self, key): if self._readonly: - import warnings - warnings.warn('The database is opened for reading only', - DeprecationWarning, stacklevel=2) + raise ValueError('The database is opened for reading only') if isinstance(key, str): key = key.encode('utf-8') self._verify_open() @@ -323,7 +313,5 @@ def open(file, flag='c', mode=0o666): # Turn off any bits that are set in the umask mode = mode & (~um) if flag not in ('r', 'w', 'c', 'n'): - import warnings - warnings.warn("Flag must be one of 'r', 'w', 'c', or 'n'", - DeprecationWarning, stacklevel=2) + raise ValueError("Flag must be one of 'r', 'w', 'c', or 'n'") return _Database(file, mode, flag=flag) diff --git a/Lib/distutils/_msvccompiler.py b/Lib/distutils/_msvccompiler.py index ef1356b97d6..c9d3c6c6712 100644 --- a/Lib/distutils/_msvccompiler.py +++ b/Lib/distutils/_msvccompiler.py @@ -56,7 +56,7 @@ def _find_vc2015(): return best_version, best_dir def _find_vc2017(): - import _findvs + import _distutils_findvs import threading best_version = 0, # tuple for full version comparisons @@ -66,7 +66,7 @@ def _find_vc2017(): # initialize COM. all_packages = [] def _getall(): - all_packages.extend(_findvs.findall()) + all_packages.extend(_distutils_findvs.findall()) t = threading.Thread(target=_getall) t.start() t.join() diff --git a/Lib/distutils/command/bdist_wininst.py b/Lib/distutils/command/bdist_wininst.py index 6309c3e248c..0871a4f7d6b 100644 --- a/Lib/distutils/command/bdist_wininst.py +++ b/Lib/distutils/command/bdist_wininst.py @@ -337,11 +337,10 @@ class bdist_wininst(Command): # cross-building, so assume the latest version bv = '14.0' else: - bv = '.'.join(CRT_ASSEMBLY_VERSION.split('.', 2)[:2]) - if bv == '14.11': - # v141 and v140 are binary compatible, - # so keep using the 14.0 stub. - bv = '14.0' + # as far as we know, CRT is binary compatible based on + # the first field, so assume 'x.0' until proven otherwise + major = CRT_ASSEMBLY_VERSION.partition('.')[0] + bv = major + '.0' # wininst-x.y.exe is in the same directory as this file diff --git a/Lib/distutils/command/upload.py b/Lib/distutils/command/upload.py index 1fd574a9f15..f7752f9d12c 100644 --- a/Lib/distutils/command/upload.py +++ b/Lib/distutils/command/upload.py @@ -159,8 +159,6 @@ class upload(PyPIRCCommand): body.write(title.encode('utf-8')) body.write(b"\r\n\r\n") body.write(value) - if value and value[-1:] == b'\r': - body.write(b'\n') # write an extra newline (lurve Macs) body.write(end_boundary) body = body.getvalue() diff --git a/Lib/distutils/config.py b/Lib/distutils/config.py index bf8d8dd2f5a..2171abd6969 100644 --- a/Lib/distutils/config.py +++ b/Lib/distutils/config.py @@ -51,7 +51,6 @@ class PyPIRCCommand(Command): if os.path.exists(rc): self.announce('Using PyPI login from %s' % rc) repository = self.repository or self.DEFAULT_REPOSITORY - realm = self.realm or self.DEFAULT_REALM config = RawConfigParser() config.read(rc) diff --git a/Lib/distutils/dist.py b/Lib/distutils/dist.py index 62a24516cfa..6cf0a0d6632 100644 --- a/Lib/distutils/dist.py +++ b/Lib/distutils/dist.py @@ -27,6 +27,20 @@ from distutils.debug import DEBUG command_re = re.compile(r'^[a-zA-Z]([a-zA-Z0-9_]*)$') +def _ensure_list(value, fieldname): + if isinstance(value, str): + # a string containing comma separated values is okay. It will + # be converted to a list by Distribution.finalize_options(). + pass + elif not isinstance(value, list): + # passing a tuple or an iterator perhaps, warn and convert + typename = type(value).__name__ + msg = f"Warning: '{fieldname}' should be a list, got type '{typename}'" + log.log(log.WARN, msg) + value = list(value) + return value + + class Distribution: """The core of the Distutils. Most of the work hiding behind 'setup' is really done within a Distribution instance, which farms the work out @@ -257,10 +271,7 @@ Common commands: (see '--help-commands' for more) setattr(self, key, val) else: msg = "Unknown distribution option: %s" % repr(key) - if warnings is not None: - warnings.warn(msg) - else: - sys.stderr.write(msg + "\n") + warnings.warn(msg) # no-user-cfg is handled before other command line args # because other args override the config files, and this @@ -1188,12 +1199,21 @@ class DistributionMetadata: def get_keywords(self): return self.keywords or [] + def set_keywords(self, value): + self.keywords = _ensure_list(value, 'keywords') + def get_platforms(self): return self.platforms or ["UNKNOWN"] + def set_platforms(self, value): + self.platforms = _ensure_list(value, 'platforms') + def get_classifiers(self): return self.classifiers or [] + def set_classifiers(self, value): + self.classifiers = _ensure_list(value, 'classifiers') + def get_download_url(self): return self.download_url or "UNKNOWN" @@ -1205,7 +1225,7 @@ class DistributionMetadata: import distutils.versionpredicate for v in value: distutils.versionpredicate.VersionPredicate(v) - self.requires = value + self.requires = list(value) def get_provides(self): return self.provides or [] @@ -1224,7 +1244,7 @@ class DistributionMetadata: import distutils.versionpredicate for v in value: distutils.versionpredicate.VersionPredicate(v) - self.obsoletes = value + self.obsoletes = list(value) def fix_help_options(options): """Convert a 4-tuple 'help_options' list as found in various command diff --git a/Lib/distutils/tests/test_dist.py b/Lib/distutils/tests/test_dist.py index 1f104cef675..0a19f0fb627 100644 --- a/Lib/distutils/tests/test_dist.py +++ b/Lib/distutils/tests/test_dist.py @@ -11,7 +11,9 @@ from unittest import mock from distutils.dist import Distribution, fix_help_options, DistributionMetadata from distutils.cmd import Command -from test.support import TESTFN, captured_stdout, run_unittest +from test.support import ( + TESTFN, captured_stdout, captured_stderr, run_unittest +) from distutils.tests import support from distutils import log @@ -195,6 +197,13 @@ class DistributionTestCase(support.LoggingSilencer, self.assertEqual(dist.metadata.platforms, ['one', 'two']) self.assertEqual(dist.metadata.keywords, ['one', 'two']) + attrs = {'keywords': 'foo bar', + 'platforms': 'foo bar'} + dist = Distribution(attrs=attrs) + dist.finalize_options() + self.assertEqual(dist.metadata.platforms, ['foo bar']) + self.assertEqual(dist.metadata.keywords, ['foo bar']) + def test_get_command_packages(self): dist = Distribution() self.assertEqual(dist.command_packages, None) @@ -312,6 +321,13 @@ class MetadataTestCase(support.TempdirManager, support.EnvironGuard, "version": "1.0", "requires": ["my.pkg (splat)"]}) + def test_requires_to_list(self): + attrs = {"name": "package", + "requires": iter(["other"])} + dist = Distribution(attrs) + self.assertIsInstance(dist.metadata.requires, list) + + def test_obsoletes(self): attrs = {"name": "package", "version": "1.0", @@ -334,13 +350,69 @@ class MetadataTestCase(support.TempdirManager, support.EnvironGuard, "version": "1.0", "obsoletes": ["my.pkg (splat)"]}) + def test_obsoletes_to_list(self): + attrs = {"name": "package", + "obsoletes": iter(["other"])} + dist = Distribution(attrs) + self.assertIsInstance(dist.metadata.obsoletes, list) + def test_classifier(self): attrs = {'name': 'Boa', 'version': '3.0', 'classifiers': ['Programming Language :: Python :: 3']} dist = Distribution(attrs) + self.assertEqual(dist.get_classifiers(), + ['Programming Language :: Python :: 3']) meta = self.format_metadata(dist) self.assertIn('Metadata-Version: 1.1', meta) + def test_classifier_invalid_type(self): + attrs = {'name': 'Boa', 'version': '3.0', + 'classifiers': ('Programming Language :: Python :: 3',)} + with captured_stderr() as error: + d = Distribution(attrs) + # should have warning about passing a non-list + self.assertIn('should be a list', error.getvalue()) + # should be converted to a list + self.assertIsInstance(d.metadata.classifiers, list) + self.assertEqual(d.metadata.classifiers, + list(attrs['classifiers'])) + + def test_keywords(self): + attrs = {'name': 'Monty', 'version': '1.0', + 'keywords': ['spam', 'eggs', 'life of brian']} + dist = Distribution(attrs) + self.assertEqual(dist.get_keywords(), + ['spam', 'eggs', 'life of brian']) + + def test_keywords_invalid_type(self): + attrs = {'name': 'Monty', 'version': '1.0', + 'keywords': ('spam', 'eggs', 'life of brian')} + with captured_stderr() as error: + d = Distribution(attrs) + # should have warning about passing a non-list + self.assertIn('should be a list', error.getvalue()) + # should be converted to a list + self.assertIsInstance(d.metadata.keywords, list) + self.assertEqual(d.metadata.keywords, list(attrs['keywords'])) + + def test_platforms(self): + attrs = {'name': 'Monty', 'version': '1.0', + 'platforms': ['GNU/Linux', 'Some Evil Platform']} + dist = Distribution(attrs) + self.assertEqual(dist.get_platforms(), + ['GNU/Linux', 'Some Evil Platform']) + + def test_platforms_invalid_types(self): + attrs = {'name': 'Monty', 'version': '1.0', + 'platforms': ('GNU/Linux', 'Some Evil Platform')} + with captured_stderr() as error: + d = Distribution(attrs) + # should have warning about passing a non-list + self.assertIn('should be a list', error.getvalue()) + # should be converted to a list + self.assertIsInstance(d.metadata.platforms, list) + self.assertEqual(d.metadata.platforms, list(attrs['platforms'])) + def test_download_url(self): attrs = {'name': 'Boa', 'version': '3.0', 'download_url': 'http://example.org/boa'} diff --git a/Lib/distutils/tests/test_upload.py b/Lib/distutils/tests/test_upload.py index 2cb2f6ce938..c17d8e7d54e 100644 --- a/Lib/distutils/tests/test_upload.py +++ b/Lib/distutils/tests/test_upload.py @@ -143,6 +143,32 @@ class uploadTestCase(BasePyPIRCCommandTestCase): results = self.get_logs(INFO) self.assertEqual(results[-1], 75 * '-' + '\nxyzzy\n' + 75 * '-') + # bpo-32304: archives whose last byte was b'\r' were corrupted due to + # normalization intended for Mac OS 9. + def test_upload_correct_cr(self): + # content that ends with \r should not be modified. + tmp = self.mkdtemp() + path = os.path.join(tmp, 'xxx') + self.write_file(path, content='yy\r') + command, pyversion, filename = 'xxx', '2.6', path + dist_files = [(command, pyversion, filename)] + self.write_file(self.rc, PYPIRC_LONG_PASSWORD) + + # other fields that ended with \r used to be modified, now are + # preserved. + pkg_dir, dist = self.create_dist( + dist_files=dist_files, + description='long description\r' + ) + cmd = upload(dist) + cmd.show_response = 1 + cmd.ensure_finalized() + cmd.run() + + headers = dict(self.last_open.req.headers) + self.assertEqual(headers['Content-length'], '2172') + self.assertIn(b'long description\r', self.last_open.req.data) + def test_upload_fails(self): self.next_msg = "Not Found" self.next_code = 404 diff --git a/Lib/doctest.py b/Lib/doctest.py index 5e5bc21a038..c1d8a1db111 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1611,7 +1611,7 @@ class OutputChecker: '', want) # If a line in got contains only spaces, then remove the # spaces. - got = re.sub(r'(?m)^\s*?$', '', got) + got = re.sub(r'(?m)^[^\S\n]+$', '', got) if got == want: return True diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index b4737c806e1..d8becee1989 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -96,90 +96,6 @@ EXTENDED_ATTRIBUTE_ENDS = ATTRIBUTE_ENDS - set('%') def quote_string(value): return '"'+str(value).replace('\\', '\\\\').replace('"', r'\"')+'"' -# -# Accumulator for header folding -# - -class _Folded: - - def __init__(self, maxlen, policy): - self.maxlen = maxlen - self.policy = policy - self.lastlen = 0 - self.stickyspace = None - self.firstline = True - self.done = [] - self.current = [] - - def newline(self): - self.done.extend(self.current) - self.done.append(self.policy.linesep) - self.current.clear() - self.lastlen = 0 - - def finalize(self): - if self.current: - self.newline() - - def __str__(self): - return ''.join(self.done) - - def append(self, stoken): - self.current.append(stoken) - - def append_if_fits(self, token, stoken=None): - if stoken is None: - stoken = str(token) - l = len(stoken) - if self.stickyspace is not None: - stickyspace_len = len(self.stickyspace) - if self.lastlen + stickyspace_len + l <= self.maxlen: - self.current.append(self.stickyspace) - self.lastlen += stickyspace_len - self.current.append(stoken) - self.lastlen += l - self.stickyspace = None - self.firstline = False - return True - if token.has_fws: - ws = token.pop_leading_fws() - if ws is not None: - self.stickyspace += str(ws) - stickyspace_len += len(ws) - token._fold(self) - return True - if stickyspace_len and l + 1 <= self.maxlen: - margin = self.maxlen - l - if 0 < margin < stickyspace_len: - trim = stickyspace_len - margin - self.current.append(self.stickyspace[:trim]) - self.stickyspace = self.stickyspace[trim:] - stickyspace_len = trim - self.newline() - self.current.append(self.stickyspace) - self.current.append(stoken) - self.lastlen = l + stickyspace_len - self.stickyspace = None - self.firstline = False - return True - if not self.firstline: - self.newline() - self.current.append(self.stickyspace) - self.current.append(stoken) - self.stickyspace = None - self.firstline = False - return True - if self.lastlen + l <= self.maxlen: - self.current.append(stoken) - self.lastlen += l - return True - if l < self.maxlen: - self.newline() - self.current.append(stoken) - self.lastlen = l - return True - return False - # # TokenList and its subclasses # @@ -187,6 +103,8 @@ class _Folded: class TokenList(list): token_type = None + syntactic_break = True + ew_combine_allowed = True def __init__(self, *args, **kw): super().__init__(*args, **kw) @@ -207,84 +125,13 @@ class TokenList(list): def all_defects(self): return sum((x.all_defects for x in self), self.defects) - # - # Folding API - # - # parts(): - # - # return a list of objects that constitute the "higher level syntactic - # objects" specified by the RFC as the best places to fold a header line. - # The returned objects must include leading folding white space, even if - # this means mutating the underlying parse tree of the object. Each object - # is only responsible for returning *its* parts, and should not drill down - # to any lower level except as required to meet the leading folding white - # space constraint. - # - # _fold(folded): - # - # folded: the result accumulator. This is an instance of _Folded. - # (XXX: I haven't finished factoring this out yet, the folding code - # pretty much uses this as a state object.) When the folded.current - # contains as much text as will fit, the _fold method should call - # folded.newline. - # folded.lastlen: the current length of the test stored in folded.current. - # folded.maxlen: The maximum number of characters that may appear on a - # folded line. Differs from the policy setting in that "no limit" is - # represented by +inf, which means it can be used in the trivially - # logical fashion in comparisons. - # - # Currently no subclasses implement parts, and I think this will remain - # true. A subclass only needs to implement _fold when the generic version - # isn't sufficient. _fold will need to be implemented primarily when it is - # possible for encoded words to appear in the specialized token-list, since - # there is no generic algorithm that can know where exactly the encoded - # words are allowed. A _fold implementation is responsible for filling - # lines in the same general way that the top level _fold does. It may, and - # should, call the _fold method of sub-objects in a similar fashion to that - # of the top level _fold. - # - # XXX: I'm hoping it will be possible to factor the existing code further - # to reduce redundancy and make the logic clearer. - - @property - def parts(self): - klass = self.__class__ - this = [] - for token in self: - if token.startswith_fws(): - if this: - yield this[0] if len(this)==1 else klass(this) - this.clear() - end_ws = token.pop_trailing_ws() - this.append(token) - if end_ws: - yield klass(this) - this = [end_ws] - if this: - yield this[0] if len(this)==1 else klass(this) - def startswith_fws(self): return self[0].startswith_fws() - def pop_leading_fws(self): - if self[0].token_type == 'fws': - return self.pop(0) - return self[0].pop_leading_fws() - - def pop_trailing_ws(self): - if self[-1].token_type == 'cfws': - return self.pop(-1) - return self[-1].pop_trailing_ws() - @property - def has_fws(self): - for part in self: - if part.has_fws: - return True - return False - - def has_leading_comment(self): - return self[0].has_leading_comment() + def as_ew_allowed(self): + """True if all top level tokens of this part may be RFC2047 encoded.""" + return all(part.as_ew_allowed for part in self) @property def comments(self): @@ -294,69 +141,13 @@ class TokenList(list): return comments def fold(self, *, policy): - # max_line_length 0/None means no limit, ie: infinitely long. - maxlen = policy.max_line_length or float("+inf") - folded = _Folded(maxlen, policy) - self._fold(folded) - folded.finalize() - return str(folded) - - def as_encoded_word(self, charset): - # This works only for things returned by 'parts', which include - # the leading fws, if any, that should be used. - res = [] - ws = self.pop_leading_fws() - if ws: - res.append(ws) - trailer = self.pop(-1) if self[-1].token_type=='fws' else '' - res.append(_ew.encode(str(self), charset)) - res.append(trailer) - return ''.join(res) - - def cte_encode(self, charset, policy): - res = [] - for part in self: - res.append(part.cte_encode(charset, policy)) - return ''.join(res) - - def _fold(self, folded): - encoding = 'utf-8' if folded.policy.utf8 else 'ascii' - for part in self.parts: - tstr = str(part) - tlen = len(tstr) - try: - str(part).encode(encoding) - except UnicodeEncodeError: - if any(isinstance(x, errors.UndecodableBytesDefect) - for x in part.all_defects): - charset = 'unknown-8bit' - else: - # XXX: this should be a policy setting when utf8 is False. - charset = 'utf-8' - tstr = part.cte_encode(charset, folded.policy) - tlen = len(tstr) - if folded.append_if_fits(part, tstr): - continue - # Peel off the leading whitespace if any and make it sticky, to - # avoid infinite recursion. - ws = part.pop_leading_fws() - if ws is not None: - folded.stickyspace = str(ws) - if folded.append_if_fits(part): - continue - if part.has_fws: - part._fold(folded) - continue - # There are no fold points in this one; it is too long for a single - # line and can't be split...we just have to put it on its own line. - folded.append(tstr) - folded.newline() + return _refold_parse_tree(self, policy=policy) def pprint(self, indent=''): - print('\n'.join(self._pp(indent=''))) + print(self.ppstr(indent=indent)) def ppstr(self, indent=''): - return '\n'.join(self._pp(indent='')) + return '\n'.join(self._pp(indent=indent)) def _pp(self, indent=''): yield '{}{}/{}('.format( @@ -391,173 +182,11 @@ class UnstructuredTokenList(TokenList): token_type = 'unstructured' - def _fold(self, folded): - last_ew = None - encoding = 'utf-8' if folded.policy.utf8 else 'ascii' - for part in self.parts: - tstr = str(part) - is_ew = False - try: - str(part).encode(encoding) - except UnicodeEncodeError: - if any(isinstance(x, errors.UndecodableBytesDefect) - for x in part.all_defects): - charset = 'unknown-8bit' - else: - charset = 'utf-8' - if last_ew is not None: - # We've already done an EW, combine this one with it - # if there's room. - chunk = get_unstructured( - ''.join(folded.current[last_ew:]+[tstr])).as_encoded_word(charset) - oldlastlen = sum(len(x) for x in folded.current[:last_ew]) - schunk = str(chunk) - lchunk = len(schunk) - if oldlastlen + lchunk <= folded.maxlen: - del folded.current[last_ew:] - folded.append(schunk) - folded.lastlen = oldlastlen + lchunk - continue - tstr = part.as_encoded_word(charset) - is_ew = True - if folded.append_if_fits(part, tstr): - if is_ew: - last_ew = len(folded.current) - 1 - continue - if is_ew or last_ew: - # It's too big to fit on the line, but since we've - # got encoded words we can use encoded word folding. - part._fold_as_ew(folded) - continue - # Peel off the leading whitespace if any and make it sticky, to - # avoid infinite recursion. - ws = part.pop_leading_fws() - if ws is not None: - folded.stickyspace = str(ws) - if folded.append_if_fits(part): - continue - if part.has_fws: - part._fold(folded) - continue - # It can't be split...we just have to put it on its own line. - folded.append(tstr) - folded.newline() - last_ew = None - - def cte_encode(self, charset, policy): - res = [] - last_ew = None - for part in self: - spart = str(part) - try: - spart.encode('us-ascii') - res.append(spart) - except UnicodeEncodeError: - if last_ew is None: - res.append(part.cte_encode(charset, policy)) - last_ew = len(res) - else: - tl = get_unstructured(''.join(res[last_ew:] + [spart])) - res.append(tl.as_encoded_word(charset)) - return ''.join(res) - class Phrase(TokenList): token_type = 'phrase' - def _fold(self, folded): - # As with Unstructured, we can have pure ASCII with or without - # surrogateescape encoded bytes, or we could have unicode. But this - # case is more complicated, since we have to deal with the various - # sub-token types and how they can be composed in the face of - # unicode-that-needs-CTE-encoding, and the fact that if a token a - # comment that becomes a barrier across which we can't compose encoded - # words. - last_ew = None - encoding = 'utf-8' if folded.policy.utf8 else 'ascii' - for part in self.parts: - tstr = str(part) - tlen = len(tstr) - has_ew = False - try: - str(part).encode(encoding) - except UnicodeEncodeError: - if any(isinstance(x, errors.UndecodableBytesDefect) - for x in part.all_defects): - charset = 'unknown-8bit' - else: - charset = 'utf-8' - if last_ew is not None and not part.has_leading_comment(): - # We've already done an EW, let's see if we can combine - # this one with it. The last_ew logic ensures that all we - # have at this point is atoms, no comments or quoted - # strings. So we can treat the text between the last - # encoded word and the content of this token as - # unstructured text, and things will work correctly. But - # we have to strip off any trailing comment on this token - # first, and if it is a quoted string we have to pull out - # the content (we're encoding it, so it no longer needs to - # be quoted). - if part[-1].token_type == 'cfws' and part.comments: - remainder = part.pop(-1) - else: - remainder = '' - for i, token in enumerate(part): - if token.token_type == 'bare-quoted-string': - part[i] = UnstructuredTokenList(token[:]) - chunk = get_unstructured( - ''.join(folded.current[last_ew:]+[tstr])).as_encoded_word(charset) - schunk = str(chunk) - lchunk = len(schunk) - if last_ew + lchunk <= folded.maxlen: - del folded.current[last_ew:] - folded.append(schunk) - folded.lastlen = sum(len(x) for x in folded.current) - continue - tstr = part.as_encoded_word(charset) - tlen = len(tstr) - has_ew = True - if folded.append_if_fits(part, tstr): - if has_ew and not part.comments: - last_ew = len(folded.current) - 1 - elif part.comments or part.token_type == 'quoted-string': - # If a comment is involved we can't combine EWs. And if a - # quoted string is involved, it's not worth the effort to - # try to combine them. - last_ew = None - continue - part._fold(folded) - - def cte_encode(self, charset, policy): - res = [] - last_ew = None - is_ew = False - for part in self: - spart = str(part) - try: - spart.encode('us-ascii') - res.append(spart) - except UnicodeEncodeError: - is_ew = True - if last_ew is None: - if not part.comments: - last_ew = len(res) - res.append(part.cte_encode(charset, policy)) - elif not part.has_leading_comment(): - if part[-1].token_type == 'cfws' and part.comments: - remainder = part.pop(-1) - else: - remainder = '' - for i, token in enumerate(part): - if token.token_type == 'bare-quoted-string': - part[i] = UnstructuredTokenList(token[:]) - tl = get_unstructured(''.join(res[last_ew:] + [spart])) - res[last_ew:] = [tl.as_encoded_word(charset)] - if part.comments or (not is_ew and part.token_type == 'quoted-string'): - last_ew = None - return ''.join(res) - class Word(TokenList): token_type = 'word' @@ -567,9 +196,6 @@ class CFWSList(WhiteSpaceTokenList): token_type = 'cfws' - def has_leading_comment(self): - return bool(self.comments) - class Atom(TokenList): @@ -579,6 +205,7 @@ class Atom(TokenList): class Token(TokenList): token_type = 'token' + encode_as_ew = False class EncodedWord(TokenList): @@ -588,13 +215,6 @@ class EncodedWord(TokenList): charset = None lang = None - @property - def encoded(self): - if self.cte is not None: - return self.cte - _ew.encode(str(self), self.charset) - - class QuotedString(TokenList): @@ -810,7 +430,10 @@ class AngleAddr(TokenList): def addr_spec(self): for x in self: if x.token_type == 'addr-spec': - return x.addr_spec + if x.local_part: + return x.addr_spec + else: + return quote_string(x.local_part) + x.addr_spec else: return '<>' @@ -865,6 +488,7 @@ class InvalidMailbox(TokenList): class Domain(TokenList): token_type = 'domain' + as_ew_allowed = False @property def domain(self): @@ -879,11 +503,13 @@ class DotAtom(TokenList): class DotAtomText(TokenList): token_type = 'dot-atom-text' + as_ew_allowed = True class AddrSpec(TokenList): token_type = 'addr-spec' + as_ew_allowed = False @property def local_part(self): @@ -916,11 +542,13 @@ class AddrSpec(TokenList): class ObsLocalPart(TokenList): token_type = 'obs-local-part' + as_ew_allowed = False class DisplayName(Phrase): token_type = 'display-name' + ew_combine_allowed = False @property def display_name(self): @@ -960,6 +588,7 @@ class DisplayName(Phrase): class LocalPart(TokenList): token_type = 'local-part' + as_ew_allowed = False @property def value(self): @@ -995,6 +624,7 @@ class LocalPart(TokenList): class DomainLiteral(TokenList): token_type = 'domain-literal' + as_ew_allowed = False @property def domain(self): @@ -1081,6 +711,7 @@ class Value(TokenList): class MimeParameters(TokenList): token_type = 'mime-parameters' + syntactic_break = False @property def params(self): @@ -1165,6 +796,10 @@ class MimeParameters(TokenList): class ParameterizedHeaderValue(TokenList): + # Set this false so that the value doesn't wind up on a new line even + # if it and the parameters would fit there but not on the first line. + syntactic_break = False + @property def params(self): for token in reversed(self): @@ -1172,18 +807,11 @@ class ParameterizedHeaderValue(TokenList): return token.params return {} - @property - def parts(self): - if self and self[-1].token_type == 'mime-parameters': - # We don't want to start a new line if all of the params don't fit - # after the value, so unwrap the parameter list. - return TokenList(self[:-1] + self[-1]) - return TokenList(self).parts - class ContentType(ParameterizedHeaderValue): token_type = 'content-type' + as_ew_allowed = False maintype = 'text' subtype = 'plain' @@ -1191,40 +819,27 @@ class ContentType(ParameterizedHeaderValue): class ContentDisposition(ParameterizedHeaderValue): token_type = 'content-disposition' + as_ew_allowed = False content_disposition = None class ContentTransferEncoding(TokenList): token_type = 'content-transfer-encoding' + as_ew_allowed = False cte = '7bit' class HeaderLabel(TokenList): token_type = 'header-label' + as_ew_allowed = False class Header(TokenList): token_type = 'header' - def _fold(self, folded): - folded.append(str(self.pop(0))) - folded.lastlen = len(folded.current[0]) - # The first line of the header is different from all others: we don't - # want to start a new object on a new line if it has any fold points in - # it that would allow part of it to be on the first header line. - # Further, if the first fold point would fit on the new line, we want - # to do that, but if it doesn't we want to put it on the first line. - # Folded supports this via the stickyspace attribute. If this - # attribute is not None, it does the special handling. - folded.stickyspace = str(self.pop(0)) if self[0].token_type == 'cfws' else '' - rest = self.pop(0) - if self: - raise ValueError("Malformed Header token list") - rest._fold(folded) - # # Terminal classes and instances @@ -1232,6 +847,10 @@ class Header(TokenList): class Terminal(str): + as_ew_allowed = True + ew_combine_allowed = True + syntactic_break = True + def __new__(cls, value, token_type): self = super().__new__(cls, value) self.token_type = token_type @@ -1241,6 +860,9 @@ class Terminal(str): def __repr__(self): return "{}({})".format(self.__class__.__name__, super().__repr__()) + def pprint(self): + print(self.__class__.__name__ + '/' + self.token_type) + @property def all_defects(self): return list(self.defects) @@ -1254,29 +876,14 @@ class Terminal(str): '' if not self.defects else ' {}'.format(self.defects), )] - def cte_encode(self, charset, policy): - value = str(self) - try: - value.encode('us-ascii') - return value - except UnicodeEncodeError: - return _ew.encode(value, charset) - def pop_trailing_ws(self): # This terminates the recursion. return None - def pop_leading_fws(self): - # This terminates the recursion. - return None - @property def comments(self): return [] - def has_leading_comment(self): - return False - def __getnewargs__(self): return(str(self), self.token_type) @@ -1290,8 +897,6 @@ class WhiteSpaceTerminal(Terminal): def startswith_fws(self): return True - has_fws = True - class ValueTerminal(Terminal): @@ -1302,11 +907,6 @@ class ValueTerminal(Terminal): def startswith_fws(self): return False - has_fws = False - - def as_encoded_word(self, charset): - return _ew.encode(str(self), charset) - class EWWhiteSpaceTerminal(WhiteSpaceTerminal): @@ -1314,15 +914,9 @@ class EWWhiteSpaceTerminal(WhiteSpaceTerminal): def value(self): return '' - @property - def encoded(self): - return self[:] - def __str__(self): return '' - has_fws = True - # XXX these need to become classes and used as instances so # that a program can't change them in a parse tree and screw @@ -1573,6 +1167,9 @@ def get_bare_quoted_string(value): "expected '\"' but found '{}'".format(value)) bare_quoted_string = BareQuotedString() value = value[1:] + if value[0] == '"': + token, value = get_qcontent(value) + bare_quoted_string.append(token) while value and value[0] != '"': if value[0] in WSP: token, value = get_fws(value) @@ -2751,7 +2348,7 @@ def get_parameter(value): if value[0] != "'": raise errors.HeaderParseError("Expected RFC2231 char/lang encoding " "delimiter, but found {!r}".format(value)) - appendto.append(ValueTerminal("'", 'RFC2231 delimiter')) + appendto.append(ValueTerminal("'", 'RFC2231-delimiter')) value = value[1:] if value and value[0] != "'": token, value = get_attrtext(value) @@ -2760,7 +2357,7 @@ def get_parameter(value): if not value or value[0] != "'": raise errors.HeaderParseError("Expected RFC2231 char/lang encoding " "delimiter, but found {}".format(value)) - appendto.append(ValueTerminal("'", 'RFC2231 delimiter')) + appendto.append(ValueTerminal("'", 'RFC2231-delimiter')) value = value[1:] if remainder is not None: # Treat the rest of value as bare quoted string content. @@ -2965,3 +2562,255 @@ def parse_content_transfer_encoding_header(value): token, value = get_phrase(value) cte_header.append(token) return cte_header + + +# +# Header folding +# +# Header folding is complex, with lots of rules and corner cases. The +# following code does its best to obey the rules and handle the corner +# cases, but you can be sure there are few bugs:) +# +# This folder generally canonicalizes as it goes, preferring the stringified +# version of each token. The tokens contain information that supports the +# folder, including which tokens can be encoded in which ways. +# +# Folded text is accumulated in a simple list of strings ('lines'), each +# one of which should be less than policy.max_line_length ('maxlen'). +# + +def _steal_trailing_WSP_if_exists(lines): + wsp = '' + if lines and lines[-1] and lines[-1][-1] in WSP: + wsp = lines[-1][-1] + lines[-1] = lines[-1][:-1] + return wsp + +def _refold_parse_tree(parse_tree, *, policy): + """Return string of contents of parse_tree folded according to RFC rules. + + """ + # max_line_length 0/None means no limit, ie: infinitely long. + maxlen = policy.max_line_length or float("+inf") + encoding = 'utf-8' if policy.utf8 else 'us-ascii' + lines = [''] + last_ew = None + wrap_as_ew_blocked = 0 + want_encoding = False + end_ew_not_allowed = Terminal('', 'wrap_as_ew_blocked') + parts = list(parse_tree) + while parts: + part = parts.pop(0) + if part is end_ew_not_allowed: + wrap_as_ew_blocked -= 1 + continue + tstr = str(part) + try: + tstr.encode(encoding) + charset = encoding + except UnicodeEncodeError: + if any(isinstance(x, errors.UndecodableBytesDefect) + for x in part.all_defects): + charset = 'unknown-8bit' + else: + # If policy.utf8 is false this should really be taken from a + # 'charset' property on the policy. + charset = 'utf-8' + want_encoding = True + if part.token_type == 'mime-parameters': + # Mime parameter folding (using RFC2231) is extra special. + _fold_mime_parameters(part, lines, maxlen, encoding) + continue + if want_encoding and not wrap_as_ew_blocked: + if not part.as_ew_allowed: + want_encoding = False + last_ew = None + if part.syntactic_break: + encoded_part = part.fold(policy=policy)[:-1] # strip nl + if policy.linesep not in encoded_part: + # It fits on a single line + if len(encoded_part) > maxlen - len(lines[-1]): + # But not on this one, so start a new one. + newline = _steal_trailing_WSP_if_exists(lines) + # XXX what if encoded_part has no leading FWS? + lines.append(newline) + lines[-1] += encoded_part + continue + # Either this is not a major syntactic break, so we don't + # want it on a line by itself even if it fits, or it + # doesn't fit on a line by itself. Either way, fall through + # to unpacking the subparts and wrapping them. + if not hasattr(part, 'encode'): + # It's not a Terminal, do each piece individually. + parts = list(part) + parts + else: + # It's a terminal, wrap it as an encoded word, possibly + # combining it with previously encoded words if allowed. + last_ew = _fold_as_ew(tstr, lines, maxlen, last_ew, + part.ew_combine_allowed, charset) + want_encoding = False + continue + if len(tstr) <= maxlen - len(lines[-1]): + lines[-1] += tstr + continue + # This part is too long to fit. The RFC wants us to break at + # "major syntactic breaks", so unless we don't consider this + # to be one, check if it will fit on the next line by itself. + if (part.syntactic_break and + len(tstr) + 1 <= maxlen): + newline = _steal_trailing_WSP_if_exists(lines) + if newline or part.startswith_fws(): + lines.append(newline + tstr) + continue + if not hasattr(part, 'encode'): + # It's not a terminal, try folding the subparts. + newparts = list(part) + if not part.as_ew_allowed: + wrap_as_ew_blocked += 1 + newparts.append(end_ew_not_allowed) + parts = newparts + parts + continue + if part.as_ew_allowed and not wrap_as_ew_blocked: + # It doesn't need CTE encoding, but encode it anyway so we can + # wrap it. + parts.insert(0, part) + want_encoding = True + continue + # We can't figure out how to wrap, it, so give up. + newline = _steal_trailing_WSP_if_exists(lines) + if newline or part.startswith_fws(): + lines.append(newline + tstr) + else: + # We can't fold it onto the next line either... + lines[-1] += tstr + return policy.linesep.join(lines) + policy.linesep + +def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): + """Fold string to_encode into lines as encoded word, combining if allowed. + Return the new value for last_ew, or None if ew_combine_allowed is False. + + If there is already an encoded word in the last line of lines (indicated by + a non-None value for last_ew) and ew_combine_allowed is true, decode the + existing ew, combine it with to_encode, and re-encode. Otherwise, encode + to_encode. In either case, split to_encode as necessary so that the + encoded segments fit within maxlen. + + """ + if last_ew is not None and ew_combine_allowed: + to_encode = str( + get_unstructured(lines[-1][last_ew:] + to_encode)) + lines[-1] = lines[-1][:last_ew] + if to_encode[0] in WSP: + # We're joining this to non-encoded text, so don't encode + # the leading blank. + leading_wsp = to_encode[0] + to_encode = to_encode[1:] + if (len(lines[-1]) == maxlen): + lines.append(_steal_trailing_WSP_if_exists(lines)) + lines[-1] += leading_wsp + trailing_wsp = '' + if to_encode[-1] in WSP: + # Likewise for the trailing space. + trailing_wsp = to_encode[-1] + to_encode = to_encode[:-1] + new_last_ew = len(lines[-1]) if last_ew is None else last_ew + while to_encode: + remaining_space = maxlen - len(lines[-1]) + # The RFC2047 chrome takes up 7 characters plus the length + # of the charset name. + encode_as = 'utf-8' if charset == 'us-ascii' else charset + text_space = remaining_space - len(encode_as) - 7 + if text_space <= 0: + lines.append(' ') + # XXX We'll get an infinite loop here if maxlen is <= 7 + continue + first_part = to_encode[:text_space] + ew = _ew.encode(first_part, charset=encode_as) + excess = len(ew) - remaining_space + if excess > 0: + # encode always chooses the shortest encoding, so this + # is guaranteed to fit at this point. + first_part = first_part[:-excess] + ew = _ew.encode(first_part) + lines[-1] += ew + to_encode = to_encode[len(first_part):] + if to_encode: + lines.append(' ') + new_last_ew = len(lines[-1]) + lines[-1] += trailing_wsp + return new_last_ew if ew_combine_allowed else None + +def _fold_mime_parameters(part, lines, maxlen, encoding): + """Fold TokenList 'part' into the 'lines' list as mime parameters. + + Using the decoded list of parameters and values, format them according to + the RFC rules, including using RFC2231 encoding if the value cannot be + expressed in 'encoding' and/or the parameter+value is too long to fit + within 'maxlen'. + + """ + # Special case for RFC2231 encoding: start from decoded values and use + # RFC2231 encoding iff needed. + # + # Note that the 1 and 2s being added to the length calculations are + # accounting for the possibly-needed spaces and semicolons we'll be adding. + # + for name, value in part.params: + # XXX What if this ';' puts us over maxlen the first time through the + # loop? We should split the header value onto a newline in that case, + # but to do that we need to recognize the need earlier or reparse the + # header, so I'm going to ignore that bug for now. It'll only put us + # one character over. + if not lines[-1].rstrip().endswith(';'): + lines[-1] += ';' + charset = encoding + error_handler = 'strict' + try: + value.encode(encoding) + encoding_required = False + except UnicodeEncodeError: + encoding_required = True + if utils._has_surrogates(value): + charset = 'unknown-8bit' + error_handler = 'surrogateescape' + else: + charset = 'utf-8' + if encoding_required: + encoded_value = urllib.parse.quote( + value, safe='', errors=error_handler) + tstr = "{}*={}''{}".format(name, charset, encoded_value) + else: + tstr = '{}={}'.format(name, quote_string(value)) + if len(lines[-1]) + len(tstr) + 1 < maxlen: + lines[-1] = lines[-1] + ' ' + tstr + continue + elif len(tstr) + 2 <= maxlen: + lines.append(' ' + tstr) + continue + # We need multiple sections. We are allowed to mix encoded and + # non-encoded sections, but we aren't going to. We'll encode them all. + section = 0 + extra_chrome = charset + "''" + while value: + chrome_len = len(name) + len(str(section)) + 3 + len(extra_chrome) + if maxlen <= chrome_len + 3: + # We need room for the leading blank, the trailing semicolon, + # and at least one character of the value. If we don't + # have that, we'd be stuck, so in that case fall back to + # the RFC standard width. + maxlen = 78 + splitpoint = maxchars = maxlen - chrome_len - 2 + while True: + partial = value[:splitpoint] + encoded_value = urllib.parse.quote( + partial, safe='', errors=error_handler) + if len(encoded_value) <= maxchars: + break + splitpoint -= 1 + lines.append(" {}*{}*={}{}".format( + name, section, extra_chrome, encoded_value)) + extra_chrome = '' + section += 1 + value = value[splitpoint:] + if value: + lines[-1] += ';' diff --git a/Lib/email/headerregistry.py b/Lib/email/headerregistry.py index 81fee146dcc..00652049f2f 100644 --- a/Lib/email/headerregistry.py +++ b/Lib/email/headerregistry.py @@ -245,13 +245,16 @@ class BaseHeader(str): the header name and the ': ' separator. """ - # At some point we need to only put fws here if it was in the source. + # At some point we need to put fws here iif it was in the source. header = parser.Header([ parser.HeaderLabel([ parser.ValueTerminal(self.name, 'header-name'), parser.ValueTerminal(':', 'header-sep')]), - parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')]), - self._parse_tree]) + ]) + if self._parse_tree: + header.append( + parser.CFWSList([parser.WhiteSpaceTerminal(' ', 'fws')])) + header.append(self._parse_tree) return header.fold(policy=policy) diff --git a/Lib/email/quoprimime.py b/Lib/email/quoprimime.py index c543eb59ae7..94534f7ee1e 100644 --- a/Lib/email/quoprimime.py +++ b/Lib/email/quoprimime.py @@ -173,7 +173,7 @@ def body_encode(body, maxlinelen=76, eol=NL): if not body: return body - # quote speacial characters + # quote special characters body = body.translate(_QUOPRI_BODY_ENCODE_MAP) soft_break = '=' + eol diff --git a/Lib/encodings/__init__.py b/Lib/encodings/__init__.py index aa2fb7c2b93..025b7a8da3d 100644 --- a/Lib/encodings/__init__.py +++ b/Lib/encodings/__init__.py @@ -158,8 +158,9 @@ codecs.register(search_function) if sys.platform == 'win32': def _alias_mbcs(encoding): try: - import _bootlocale - if encoding == _bootlocale.getpreferredencoding(False): + import _winapi + ansi_code_page = "cp%s" % _winapi.GetACP() + if encoding == ansi_code_page: import encodings.mbcs return encodings.mbcs.getregentry() except ImportError: diff --git a/Lib/enum.py b/Lib/enum.py index fe7cb20fc06..e5fe6f3b94a 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -64,6 +64,7 @@ class _EnumDict(dict): super().__init__() self._member_names = [] self._last_values = [] + self._ignore = [] def __setitem__(self, key, value): """Changes anything not dundered or not a descriptor. @@ -77,17 +78,28 @@ class _EnumDict(dict): if _is_sunder(key): if key not in ( '_order_', '_create_pseudo_member_', - '_generate_next_value_', '_missing_', + '_generate_next_value_', '_missing_', '_ignore_', ): raise ValueError('_names_ are reserved for future Enum use') if key == '_generate_next_value_': setattr(self, '_generate_next_value', value) + elif key == '_ignore_': + if isinstance(value, str): + value = value.replace(',',' ').split() + else: + value = list(value) + self._ignore = value + already = set(value) & set(self._member_names) + if already: + raise ValueError('_ignore_ cannot specify already set names: %r' % (already, )) elif _is_dunder(key): if key == '__order__': key = '_order_' elif key in self._member_names: # descriptor overwriting an enum? raise TypeError('Attempted to reuse key: %r' % key) + elif key in self._ignore: + pass elif not _is_descriptor(value): if key in self: # enum overwriting a descriptor? @@ -124,6 +136,12 @@ class EnumMeta(type): # cannot be mixed with other types (int, float, etc.) if it has an # inherited __new__ unless a new __new__ is defined (or the resulting # class will fail). + # + # remove any keys listed in _ignore_ + classdict.setdefault('_ignore_', []).append('_ignore_') + ignore = classdict['_ignore_'] + for key in ignore: + classdict.pop(key, None) member_type, first_enum = metacls._get_mixins_(bases) __new__, save_new, use_args = metacls._find_new_(classdict, member_type, first_enum) diff --git a/Lib/functools.py b/Lib/functools.py index a51dddf8785..c8b79c2a7c2 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -793,7 +793,23 @@ def singledispatch(func): """ nonlocal cache_token if func is None: - return lambda f: register(cls, f) + if isinstance(cls, type): + return lambda f: register(cls, f) + ann = getattr(cls, '__annotations__', {}) + if not ann: + raise TypeError( + f"Invalid first argument to `register()`: {cls!r}. " + f"Use either `@register(some_class)` or plain `@register` " + f"on an annotated function." + ) + func = cls + + # only import typing if annotation parsing is necessary + from typing import get_type_hints + argname, cls = next(iter(get_type_hints(func).items())) + assert isinstance(cls, type), ( + f"Invalid annotation for {argname!r}. {cls!r} is not a class." + ) registry[cls] = func if cache_token is None and hasattr(cls, '__abstractmethods__'): cache_token = get_cache_token() diff --git a/Lib/hmac.py b/Lib/hmac.py index 121029aa670..93c084e4ae8 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -5,6 +5,13 @@ Implements the HMAC algorithm as described by RFC 2104. import warnings as _warnings from _operator import _compare_digest as compare_digest +try: + import _hashlib as _hashopenssl +except ImportError: + _hashopenssl = None + _openssl_md_meths = None +else: + _openssl_md_meths = frozenset(_hashopenssl.openssl_md_meth_names) import hashlib as _hashlib trans_5C = bytes((x ^ 0x5C) for x in range(256)) @@ -142,3 +149,38 @@ def new(key, msg = None, digestmod = None): method. """ return HMAC(key, msg, digestmod) + + +def digest(key, msg, digest): + """Fast inline implementation of HMAC + + key: key for the keyed hash object. + msg: input message + digest: A hash name suitable for hashlib.new() for best performance. *OR* + A hashlib constructor returning a new hash object. *OR* + A module supporting PEP 247. + + Note: key and msg must be a bytes or bytearray objects. + """ + if (_hashopenssl is not None and + isinstance(digest, str) and digest in _openssl_md_meths): + return _hashopenssl.hmac_digest(key, msg, digest) + + if callable(digest): + digest_cons = digest + elif isinstance(digest, str): + digest_cons = lambda d=b'': _hashlib.new(digest, d) + else: + digest_cons = lambda d=b'': digest.new(d) + + inner = digest_cons() + outer = digest_cons() + blocksize = getattr(inner, 'block_size', 64) + if len(key) > blocksize: + key = digest_cons(key).digest() + key = key + b'\x00' * (blocksize - len(key)) + inner.update(key.translate(trans_36)) + outer.update(key.translate(trans_5C)) + inner.update(msg) + outer.update(inner.digest()) + return outer.digest() diff --git a/Lib/http/client.py b/Lib/http/client.py index 70eadaed14e..1292db74784 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -538,7 +538,7 @@ class HTTPResponse(io.BufferedIOBase): chunk_left = self.chunk_left if not chunk_left: # Can be 0 or None if chunk_left is not None: - # We are at the end of chunk. dicard chunk end + # We are at the end of chunk, discard chunk end self._safe_read(2) # toss the CRLF at the end of the chunk try: chunk_left = self._read_next_chunk_size() @@ -1375,7 +1375,8 @@ else: if key_file or cert_file: context.load_cert_chain(cert_file, key_file) self._context = context - self._check_hostname = check_hostname + if check_hostname is not None: + self._context.check_hostname = check_hostname def connect(self): "Connect to a host on a given (SSL) port." @@ -1389,13 +1390,6 @@ else: self.sock = self._context.wrap_socket(self.sock, server_hostname=server_hostname) - if not self._context.check_hostname and self._check_hostname: - try: - ssl.match_hostname(self.sock.getpeercert(), server_hostname) - except Exception: - self.sock.shutdown(socket.SHUT_RDWR) - self.sock.close() - raise __all__.append("HTTPSConnection") diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index a7a1e9f61e1..d616a740d5f 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -1,8 +1,34 @@ -What's New in IDLE 3.7.0 +What's New in IDLE 3.7.0 (since 3.6.0) Released on 2018-06-18? -======================== +====================================== +bpo-32765: Update configdialog General tab create page docstring. +Add new widgets to the widget list. + +bpo-32207: Improve tk event exception tracebacks in IDLE. +When tk event handling is driven by IDLE's run loop, a confusing +and distracting queue.EMPTY traceback context is no longer added +to tk event exception tracebacks. The traceback is now the same +as when event handling is driven by user code. Patch based on +a suggestion by Serhiy Storchaka. + +bpo-32164: Delete unused file idlelib/tabbedpages.py. +Use of TabbedPageSet in configdialog was replaced by ttk.Notebook. + +bpo-32100: Fix old and new bugs in pathbrowser; improve tests. +Patch mostly by Cheryl Sabella. + +bpo-31860: The font sample in the settings dialog is now editable. +Edits persist while IDLE remains open. +Patch by Serhiy Storchake and Terry Jan Reedy. + +bpo-31858: Restrict shell prompt manipulaton to the shell. +Editor and output windows only see an empty last prompt line. This +simplifies the code and fixes a minor bug when newline is inserted. +Sys.ps1, if present, is read on Shell start-up, but is not set or changed. +Patch by Terry Jan Reedy. + bpo-28603: Fix a TypeError that caused a shell restart when printing a traceback that includes an exception that is unhashable. Patch by Zane Bitter. @@ -61,7 +87,7 @@ bpo-31421: Document how IDLE runs tkinter programs. IDLE calls tcl/tk update in the background in order to make live interaction and experimentatin with tkinter applications much easier. -bpo-bpo-31414: Fix tk entry box tests by deleting first. +bpo-31414: Fix tk entry box tests by deleting first. Adding to an int entry is not the same as deleting and inserting because int('') will fail. Patch by Terry Jan Reedy. @@ -271,9 +297,9 @@ Issue #28572: Add 10% to coverage of IDLE's test_configdialog. Update and augment description of the configuration system. -What's New in IDLE 3.6.0 +What's New in IDLE 3.6.0 (since 3.5.0) Released on 2016-12-23 -======================== +====================================== - Issue #15308: Add 'interrupt execution' (^C) to Shell menu. Patch by Roger Serwy, updated by Bayard Randel. diff --git a/Lib/idlelib/autocomplete_w.py b/Lib/idlelib/autocomplete_w.py index 7c3138f4040..12113f95e63 100644 --- a/Lib/idlelib/autocomplete_w.py +++ b/Lib/idlelib/autocomplete_w.py @@ -246,7 +246,7 @@ class AutoCompleteWindow: acw.wm_geometry("+%d+%d" % (new_x, new_y)) if platform.system().startswith('Windows'): - # See issue 15786. When on windows platform, Tk will misbehaive + # See issue 15786. When on windows platform, Tk will misbehave # to call winconfig_event multiple times, we need to prevent this, # otherwise mouse button double click will not be able to used. acw.unbind(WINCONFIG_SEQUENCE, self.winconfigid) diff --git a/Lib/idlelib/browser.py b/Lib/idlelib/browser.py index 79eaeb7eb45..447dafcc515 100644 --- a/Lib/idlelib/browser.py +++ b/Lib/idlelib/browser.py @@ -79,9 +79,6 @@ class ModuleBrowser: creating ModuleBrowserTreeItem as the rootnode for the tree and subsequently in the children. """ - global file_open - if not (_htest or _utest): - file_open = pyshell.flist.open self.master = master self.path = path self._htest = _htest @@ -95,9 +92,13 @@ class ModuleBrowser: def init(self): "Create browser tkinter widgets, including the tree." + global file_open root = self.master - # reset pyclbr + flist = (pyshell.flist if not (self._htest or self._utest) + else pyshell.PyShellFileList(root)) + file_open = flist.open pyclbr._modules.clear() + # create top self.top = top = ListedToplevel(root) top.protocol("WM_DELETE_WINDOW", self.close) @@ -107,6 +108,7 @@ class ModuleBrowser: (root.winfo_rootx(), root.winfo_rooty() + 200)) self.settitle() top.focus_set() + # create scrolled canvas theme = idleConf.CurrentTheme() background = idleConf.GetHighlight(theme, 'normal')['background'] @@ -236,8 +238,6 @@ def _module_browser(parent): # htest # def nested_in_class(): pass def closure(): class Nested_in_closure: pass - global file_open - file_open = pyshell.PyShellFileList(parent).open ModuleBrowser(parent, file, _htest=True) if __name__ == "__main__": diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 099f5262b1e..36ac4a23a52 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -11,8 +11,8 @@ Refer to comments in EditorWindow autoindent code for details. """ from tkinter import (Toplevel, Listbox, Text, Scale, Canvas, StringVar, BooleanVar, IntVar, TRUE, FALSE, - TOP, BOTTOM, RIGHT, LEFT, SOLID, GROOVE, NORMAL, DISABLED, - NONE, BOTH, X, Y, W, E, EW, NS, NSEW, NW, CENTER, + TOP, BOTTOM, RIGHT, LEFT, SOLID, GROOVE, + NONE, BOTH, X, Y, W, E, EW, NS, NSEW, NW, HORIZONTAL, VERTICAL, ANCHOR, ACTIVE, END) from tkinter.ttk import (Button, Checkbutton, Entry, Frame, Label, LabelFrame, OptionMenu, Notebook, Radiobutton, Scrollbar, Style) @@ -25,7 +25,6 @@ from idlelib.config_key import GetKeysDialog from idlelib.dynoption import DynOptionMenu from idlelib import macosx from idlelib.query import SectionName, HelpSource -from idlelib.tabbedpages import TabbedPageSet from idlelib.textview import view_text from idlelib.autocomplete import AutoComplete from idlelib.codecontext import CodeContext @@ -496,7 +495,7 @@ class FontPage(Frame): Changing any of the font vars invokes var_changed_font, which adds all 3 font options to changes and calls set_samples. Set_samples applies a new font constructed from the font vars to - font_sample and to highlight_sample on the hightlight page. + font_sample and to highlight_sample on the highlight page. Tabs: Enable users to change spaces entered for indent tabs. Changing indent_scale value with the mouse sets Var space_num, @@ -647,7 +646,7 @@ class FontPage(Frame): Called on font initialization and change events. Accesses font_name, font_size, and font_bold Variables. - Updates font_sample and hightlight page highlight_sample. + Updates font_sample and highlight page highlight_sample. """ font_name = self.font_name.get() font_weight = tkFont.BOLD if self.font_bold.get() else tkFont.NORMAL @@ -1443,7 +1442,7 @@ class KeysPage(Frame): self.bindingslist['xscrollcommand'] = scroll_target_x.set self.button_new_keys = Button( frame_custom, text='Get New Keys for Selection', - command=self.get_new_keys, state=DISABLED) + command=self.get_new_keys, state='disabled') # frame_key_sets. frames = [Frame(frame_key_sets, padding=2, borderwidth=0) for i in range(2)] @@ -1793,11 +1792,27 @@ class GenPage(Frame): (*)win_width_int: Entry - win_width win_height_title: Label (*)win_height_int: Entry - win_height + frame_autocomplete: Frame + auto_wait_title: Label + (*)auto_wait_int: Entry - autocomplete_wait + frame_paren1: Frame + paren_style_title: Label + (*)paren_style_type: OptionMenu - paren_style + frame_paren2: Frame + paren_time_title: Label + (*)paren_flash_time: Entry - flash_delay + (*)bell_on: Checkbutton - paren_bell frame_editor: LabelFrame frame_save: Frame run_save_title: Label (*)save_ask_on: Radiobutton - autosave (*)save_auto_on: Radiobutton - autosave + frame_format: Frame + format_width_title: Label + (*)format_width_int: Entry - format_width + frame_context: Frame + context_title: Label + (*)context_int: Entry - context_lines frame_help: LabelFrame frame_helplist: Frame frame_helplist_buttons: Frame diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 68450c921f2..b51c45c97e5 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -668,7 +668,7 @@ class EditorWindow(object): def open_path_browser(self, event=None): from idlelib import pathbrowser - pathbrowser.PathBrowser(self.flist) + pathbrowser.PathBrowser(self.root) return "break" def open_turtle_demo(self, event = None): diff --git a/Lib/idlelib/idle_test/test_browser.py b/Lib/idlelib/idle_test/test_browser.py index 59e03c5aab3..34eb332c1df 100644 --- a/Lib/idlelib/idle_test/test_browser.py +++ b/Lib/idlelib/idle_test/test_browser.py @@ -4,17 +4,19 @@ Coverage: 88% (Higher, because should exclude 3 lines that .coveragerc won't exclude.) """ -import os.path -import unittest -import pyclbr - -from idlelib import browser, filelist -from idlelib.tree import TreeNode -from test.support import requires -from unittest import mock -from tkinter import Tk -from idlelib.idle_test.mock_idle import Func from collections import deque +import os.path +import pyclbr +from tkinter import Tk + +from test.support import requires +import unittest +from unittest import mock +from idlelib.idle_test.mock_idle import Func + +from idlelib import browser +from idlelib import filelist +from idlelib.tree import TreeNode class ModuleBrowserTest(unittest.TestCase): @@ -29,6 +31,7 @@ class ModuleBrowserTest(unittest.TestCase): @classmethod def tearDownClass(cls): cls.mb.close() + cls.root.update_idletasks() cls.root.destroy() del cls.root, cls.mb @@ -38,6 +41,7 @@ class ModuleBrowserTest(unittest.TestCase): eq(mb.path, __file__) eq(pyclbr._modules, {}) self.assertIsInstance(mb.node, TreeNode) + self.assertIsNotNone(browser.file_open) def test_settitle(self): mb = self.mb @@ -151,10 +155,9 @@ class ModuleBrowserTreeItemTest(unittest.TestCase): self.assertEqual(sub0.name, 'f0') self.assertEqual(sub1.name, 'C0(base)') - - def test_ondoubleclick(self): + @mock.patch('idlelib.browser.file_open') + def test_ondoubleclick(self, fopen): mbt = self.mbt - fopen = browser.file_open = mock.Mock() with mock.patch('os.path.exists', return_value=False): mbt.OnDoubleClick() @@ -165,8 +168,6 @@ class ModuleBrowserTreeItemTest(unittest.TestCase): fopen.assert_called() fopen.called_with(fname) - del browser.file_open - class ChildBrowserTreeItemTest(unittest.TestCase): @@ -212,14 +213,13 @@ class ChildBrowserTreeItemTest(unittest.TestCase): eq(self.cbt_F1.GetSubList(), []) - def test_ondoubleclick(self): - fopen = browser.file_open = mock.Mock() + @mock.patch('idlelib.browser.file_open') + def test_ondoubleclick(self, fopen): goto = fopen.return_value.gotoline = mock.Mock() self.cbt_F1.OnDoubleClick() fopen.assert_called() goto.assert_called() goto.assert_called_with(self.cbt_F1.obj.lineno) - del browser.file_open # Failure test would have to raise OSError or AttributeError. diff --git a/Lib/idlelib/idle_test/test_pathbrowser.py b/Lib/idlelib/idle_test/test_pathbrowser.py index 813cbcc6316..74b716a3199 100644 --- a/Lib/idlelib/idle_test/test_pathbrowser.py +++ b/Lib/idlelib/idle_test/test_pathbrowser.py @@ -1,11 +1,68 @@ +""" Test idlelib.pathbrowser. +""" + + +import os.path +import pyclbr # for _modules +import sys # for sys.path +from tkinter import Tk + +from test.support import requires import unittest -import os -import sys -import idlelib +from idlelib.idle_test.mock_idle import Func + +import idlelib # for __file__ +from idlelib import browser from idlelib import pathbrowser +from idlelib.tree import TreeNode + class PathBrowserTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.pb = pathbrowser.PathBrowser(cls.root, _utest=True) + + @classmethod + def tearDownClass(cls): + cls.pb.close() + cls.root.update_idletasks() + cls.root.destroy() + del cls.root, cls.pb + + def test_init(self): + pb = self.pb + eq = self.assertEqual + eq(pb.master, self.root) + eq(pyclbr._modules, {}) + self.assertIsInstance(pb.node, TreeNode) + self.assertIsNotNone(browser.file_open) + + def test_settitle(self): + pb = self.pb + self.assertEqual(pb.top.title(), 'Path Browser') + self.assertEqual(pb.top.iconname(), 'Path Browser') + + def test_rootnode(self): + pb = self.pb + rn = pb.rootnode() + self.assertIsInstance(rn, pathbrowser.PathBrowserTreeItem) + + def test_close(self): + pb = self.pb + pb.top.destroy = Func() + pb.node.destroy = Func() + pb.close() + self.assertTrue(pb.top.destroy.called) + self.assertTrue(pb.node.destroy.called) + del pb.top.destroy, pb.node.destroy + + +class DirBrowserTreeItemTest(unittest.TestCase): + def test_DirBrowserTreeItem(self): # Issue16226 - make sure that getting a sublist works d = pathbrowser.DirBrowserTreeItem('') @@ -16,6 +73,9 @@ class PathBrowserTest(unittest.TestCase): self.assertEqual(d.ispackagedir(dir), True) self.assertEqual(d.ispackagedir(dir + '/Icons'), False) + +class PathBrowserTreeItemTest(unittest.TestCase): + def test_PathBrowserTreeItem(self): p = pathbrowser.PathBrowserTreeItem() self.assertEqual(p.GetText(), 'sys.path') @@ -23,5 +83,6 @@ class PathBrowserTest(unittest.TestCase): self.assertEqual(len(sub), len(sys.path)) self.assertEqual(type(sub[0]), pathbrowser.DirBrowserTreeItem) + if __name__ == '__main__': unittest.main(verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/test_query.py b/Lib/idlelib/idle_test/test_query.py index 1210afe70df..953f24f0a5a 100644 --- a/Lib/idlelib/idle_test/test_query.py +++ b/Lib/idlelib/idle_test/test_query.py @@ -162,7 +162,7 @@ class HelpsourceBrowsefileTest(unittest.TestCase): # Path is widget entry, either '' or something. # Func return is file dialog return, either '' or something. # Func return should override widget entry. - # We need all 4 combination to test all (most) code paths. + # We need all 4 combinations to test all (most) code paths. for path, func, result in ( ('', lambda a,b,c:'', ''), ('', lambda a,b,c: __file__, __file__), diff --git a/Lib/idlelib/idle_test/test_search.py b/Lib/idlelib/idle_test/test_search.py index 80fa93adf51..3ab72951efe 100644 --- a/Lib/idlelib/idle_test/test_search.py +++ b/Lib/idlelib/idle_test/test_search.py @@ -1,7 +1,7 @@ """Test SearchDialog class in idlelib.search.py""" # Does not currently test the event handler wrappers. -# A usage test should simulate clicks and check hilighting. +# A usage test should simulate clicks and check highlighting. # Tests need to be coordinated with SearchDialogBase tests # to avoid duplication. diff --git a/Lib/idlelib/pathbrowser.py b/Lib/idlelib/pathbrowser.py index c0aa2a15909..6de242d0000 100644 --- a/Lib/idlelib/pathbrowser.py +++ b/Lib/idlelib/pathbrowser.py @@ -3,19 +3,19 @@ import os import sys from idlelib.browser import ModuleBrowser, ModuleBrowserTreeItem -from idlelib.pyshell import PyShellFileList from idlelib.tree import TreeItem class PathBrowser(ModuleBrowser): - def __init__(self, flist, *, _htest=False, _utest=False): + def __init__(self, master, *, _htest=False, _utest=False): """ _htest - bool, change box location when running htest """ + self.master = master self._htest = _htest self._utest = _utest - self.init(flist) + self.init() def settitle(self): "Set window titles." @@ -100,8 +100,7 @@ class DirBrowserTreeItem(TreeItem): def _path_browser(parent): # htest # - flist = PyShellFileList(parent) - PathBrowser(flist, _htest=True) + PathBrowser(parent, _htest=True) parent.mainloop() if __name__ == "__main__": diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index 6440e673bf5..176fe3db743 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -134,13 +134,17 @@ def main(del_exitfunc=False): # exiting but got an extra KBI? Try again! continue try: - seq, request = rpc.request_queue.get(block=True, timeout=0.05) + request = rpc.request_queue.get(block=True, timeout=0.05) except queue.Empty: + request = None + # Issue 32207: calling handle_tk_events here adds spurious + # queue.Empty traceback to event handling exceptions. + if request: + seq, (method, args, kwargs) = request + ret = method(*args, **kwargs) + rpc.response_queue.put((seq, ret)) + else: handle_tk_events() - continue - method, args, kwargs = request - ret = method(*args, **kwargs) - rpc.response_queue.put((seq, ret)) except KeyboardInterrupt: if quitting: exit_now = True diff --git a/Lib/idlelib/tabbedpages.py b/Lib/idlelib/tabbedpages.py deleted file mode 100644 index 4186fa20131..00000000000 --- a/Lib/idlelib/tabbedpages.py +++ /dev/null @@ -1,498 +0,0 @@ -"""An implementation of tabbed pages using only standard Tkinter. - -Originally developed for use in IDLE. Based on tabpage.py. - -Classes exported: -TabbedPageSet -- A Tkinter implementation of a tabbed-page widget. -TabSet -- A widget containing tabs (buttons) in one or more rows. - -""" -from tkinter import * - -class InvalidNameError(Exception): pass -class AlreadyExistsError(Exception): pass - - -class TabSet(Frame): - """A widget containing tabs (buttons) in one or more rows. - - Only one tab may be selected at a time. - - """ - def __init__(self, page_set, select_command, - tabs=None, n_rows=1, max_tabs_per_row=5, - expand_tabs=False, **kw): - """Constructor arguments: - - select_command -- A callable which will be called when a tab is - selected. It is called with the name of the selected tab as an - argument. - - tabs -- A list of strings, the names of the tabs. Should be specified in - the desired tab order. The first tab will be the default and first - active tab. If tabs is None or empty, the TabSet will be initialized - empty. - - n_rows -- Number of rows of tabs to be shown. If n_rows <= 0 or is - None, then the number of rows will be decided by TabSet. See - _arrange_tabs() for details. - - max_tabs_per_row -- Used for deciding how many rows of tabs are needed, - when the number of rows is not constant. See _arrange_tabs() for - details. - - """ - Frame.__init__(self, page_set, **kw) - self.select_command = select_command - self.n_rows = n_rows - self.max_tabs_per_row = max_tabs_per_row - self.expand_tabs = expand_tabs - self.page_set = page_set - - self._tabs = {} - self._tab2row = {} - if tabs: - self._tab_names = list(tabs) - else: - self._tab_names = [] - self._selected_tab = None - self._tab_rows = [] - - self.padding_frame = Frame(self, height=2, - borderwidth=0, relief=FLAT, - background=self.cget('background')) - self.padding_frame.pack(side=TOP, fill=X, expand=False) - - self._arrange_tabs() - - def add_tab(self, tab_name): - """Add a new tab with the name given in tab_name.""" - if not tab_name: - raise InvalidNameError("Invalid Tab name: '%s'" % tab_name) - if tab_name in self._tab_names: - raise AlreadyExistsError("Tab named '%s' already exists" %tab_name) - - self._tab_names.append(tab_name) - self._arrange_tabs() - - def remove_tab(self, tab_name): - """Remove the tab named """ - if not tab_name in self._tab_names: - raise KeyError("No such Tab: '%s" % tab_name) - - self._tab_names.remove(tab_name) - self._arrange_tabs() - - def set_selected_tab(self, tab_name): - """Show the tab named as the selected one""" - if tab_name == self._selected_tab: - return - if tab_name is not None and tab_name not in self._tabs: - raise KeyError("No such Tab: '%s" % tab_name) - - # deselect the current selected tab - if self._selected_tab is not None: - self._tabs[self._selected_tab].set_normal() - self._selected_tab = None - - if tab_name is not None: - # activate the tab named tab_name - self._selected_tab = tab_name - tab = self._tabs[tab_name] - tab.set_selected() - # move the tab row with the selected tab to the bottom - tab_row = self._tab2row[tab] - tab_row.pack_forget() - tab_row.pack(side=TOP, fill=X, expand=0) - - def _add_tab_row(self, tab_names, expand_tabs): - if not tab_names: - return - - tab_row = Frame(self) - tab_row.pack(side=TOP, fill=X, expand=0) - self._tab_rows.append(tab_row) - - for tab_name in tab_names: - tab = TabSet.TabButton(tab_name, self.select_command, - tab_row, self) - if expand_tabs: - tab.pack(side=LEFT, fill=X, expand=True) - else: - tab.pack(side=LEFT) - self._tabs[tab_name] = tab - self._tab2row[tab] = tab_row - - # tab is the last one created in the above loop - tab.is_last_in_row = True - - def _reset_tab_rows(self): - while self._tab_rows: - tab_row = self._tab_rows.pop() - tab_row.destroy() - self._tab2row = {} - - def _arrange_tabs(self): - """ - Arrange the tabs in rows, in the order in which they were added. - - If n_rows >= 1, this will be the number of rows used. Otherwise the - number of rows will be calculated according to the number of tabs and - max_tabs_per_row. In this case, the number of rows may change when - adding/removing tabs. - - """ - # remove all tabs and rows - while self._tabs: - self._tabs.popitem()[1].destroy() - self._reset_tab_rows() - - if not self._tab_names: - return - - if self.n_rows is not None and self.n_rows > 0: - n_rows = self.n_rows - else: - # calculate the required number of rows - n_rows = (len(self._tab_names) - 1) // self.max_tabs_per_row + 1 - - # not expanding the tabs with more than one row is very ugly - expand_tabs = self.expand_tabs or n_rows > 1 - i = 0 # index in self._tab_names - for row_index in range(n_rows): - # calculate required number of tabs in this row - n_tabs = (len(self._tab_names) - i - 1) // (n_rows - row_index) + 1 - tab_names = self._tab_names[i:i + n_tabs] - i += n_tabs - self._add_tab_row(tab_names, expand_tabs) - - # re-select selected tab so it is properly displayed - selected = self._selected_tab - self.set_selected_tab(None) - if selected in self._tab_names: - self.set_selected_tab(selected) - - class TabButton(Frame): - """A simple tab-like widget.""" - - bw = 2 # borderwidth - - def __init__(self, name, select_command, tab_row, tab_set): - """Constructor arguments: - - name -- The tab's name, which will appear in its button. - - select_command -- The command to be called upon selection of the - tab. It is called with the tab's name as an argument. - - """ - Frame.__init__(self, tab_row, borderwidth=self.bw, relief=RAISED) - - self.name = name - self.select_command = select_command - self.tab_set = tab_set - self.is_last_in_row = False - - self.button = Radiobutton( - self, text=name, command=self._select_event, - padx=5, pady=1, takefocus=FALSE, indicatoron=FALSE, - highlightthickness=0, selectcolor='', borderwidth=0) - self.button.pack(side=LEFT, fill=X, expand=True) - - self._init_masks() - self.set_normal() - - def _select_event(self, *args): - """Event handler for tab selection. - - With TabbedPageSet, this calls TabbedPageSet.change_page, so that - selecting a tab changes the page. - - Note that this does -not- call set_selected -- it will be called by - TabSet.set_selected_tab, which should be called when whatever the - tabs are related to changes. - - """ - self.select_command(self.name) - return - - def set_selected(self): - """Assume selected look""" - self._place_masks(selected=True) - - def set_normal(self): - """Assume normal look""" - self._place_masks(selected=False) - - def _init_masks(self): - page_set = self.tab_set.page_set - background = page_set.pages_frame.cget('background') - # mask replaces the middle of the border with the background color - self.mask = Frame(page_set, borderwidth=0, relief=FLAT, - background=background) - # mskl replaces the bottom-left corner of the border with a normal - # left border - self.mskl = Frame(page_set, borderwidth=0, relief=FLAT, - background=background) - self.mskl.ml = Frame(self.mskl, borderwidth=self.bw, - relief=RAISED) - self.mskl.ml.place(x=0, y=-self.bw, - width=2*self.bw, height=self.bw*4) - # mskr replaces the bottom-right corner of the border with a normal - # right border - self.mskr = Frame(page_set, borderwidth=0, relief=FLAT, - background=background) - self.mskr.mr = Frame(self.mskr, borderwidth=self.bw, - relief=RAISED) - - def _place_masks(self, selected=False): - height = self.bw - if selected: - height += self.bw - - self.mask.place(in_=self, - relx=0.0, x=0, - rely=1.0, y=0, - relwidth=1.0, width=0, - relheight=0.0, height=height) - - self.mskl.place(in_=self, - relx=0.0, x=-self.bw, - rely=1.0, y=0, - relwidth=0.0, width=self.bw, - relheight=0.0, height=height) - - page_set = self.tab_set.page_set - if selected and ((not self.is_last_in_row) or - (self.winfo_rootx() + self.winfo_width() < - page_set.winfo_rootx() + page_set.winfo_width()) - ): - # for a selected tab, if its rightmost edge isn't on the - # rightmost edge of the page set, the right mask should be one - # borderwidth shorter (vertically) - height -= self.bw - - self.mskr.place(in_=self, - relx=1.0, x=0, - rely=1.0, y=0, - relwidth=0.0, width=self.bw, - relheight=0.0, height=height) - - self.mskr.mr.place(x=-self.bw, y=-self.bw, - width=2*self.bw, height=height + self.bw*2) - - # finally, lower the tab set so that all of the frames we just - # placed hide it - self.tab_set.lower() - - -class TabbedPageSet(Frame): - """A Tkinter tabbed-pane widget. - - Constains set of 'pages' (or 'panes') with tabs above for selecting which - page is displayed. Only one page will be displayed at a time. - - Pages may be accessed through the 'pages' attribute, which is a dictionary - of pages, using the name given as the key. A page is an instance of a - subclass of Tk's Frame widget. - - The page widgets will be created (and destroyed when required) by the - TabbedPageSet. Do not call the page's pack/place/grid/destroy methods. - - Pages may be added or removed at any time using the add_page() and - remove_page() methods. - - """ - - class Page(object): - """Abstract base class for TabbedPageSet's pages. - - Subclasses must override the _show() and _hide() methods. - - """ - uses_grid = False - - def __init__(self, page_set): - self.frame = Frame(page_set, borderwidth=2, relief=RAISED) - - def _show(self): - raise NotImplementedError - - def _hide(self): - raise NotImplementedError - - class PageRemove(Page): - """Page class using the grid placement manager's "remove" mechanism.""" - uses_grid = True - - def _show(self): - self.frame.grid(row=0, column=0, sticky=NSEW) - - def _hide(self): - self.frame.grid_remove() - - class PageLift(Page): - """Page class using the grid placement manager's "lift" mechanism.""" - uses_grid = True - - def __init__(self, page_set): - super(TabbedPageSet.PageLift, self).__init__(page_set) - self.frame.grid(row=0, column=0, sticky=NSEW) - self.frame.lower() - - def _show(self): - self.frame.lift() - - def _hide(self): - self.frame.lower() - - class PagePackForget(Page): - """Page class using the pack placement manager's "forget" mechanism.""" - def _show(self): - self.frame.pack(fill=BOTH, expand=True) - - def _hide(self): - self.frame.pack_forget() - - def __init__(self, parent, page_names=None, page_class=PageLift, - n_rows=1, max_tabs_per_row=5, expand_tabs=False, - **kw): - """Constructor arguments: - - page_names -- A list of strings, each will be the dictionary key to a - page's widget, and the name displayed on the page's tab. Should be - specified in the desired page order. The first page will be the default - and first active page. If page_names is None or empty, the - TabbedPageSet will be initialized empty. - - n_rows, max_tabs_per_row -- Parameters for the TabSet which will - manage the tabs. See TabSet's docs for details. - - page_class -- Pages can be shown/hidden using three mechanisms: - - * PageLift - All pages will be rendered one on top of the other. When - a page is selected, it will be brought to the top, thus hiding all - other pages. Using this method, the TabbedPageSet will not be resized - when pages are switched. (It may still be resized when pages are - added/removed.) - - * PageRemove - When a page is selected, the currently showing page is - hidden, and the new page shown in its place. Using this method, the - TabbedPageSet may resize when pages are changed. - - * PagePackForget - This mechanism uses the pack placement manager. - When a page is shown it is packed, and when it is hidden it is - unpacked (i.e. pack_forget). This mechanism may also cause the - TabbedPageSet to resize when the page is changed. - - """ - Frame.__init__(self, parent, **kw) - - self.page_class = page_class - self.pages = {} - self._pages_order = [] - self._current_page = None - self._default_page = None - - self.columnconfigure(0, weight=1) - self.rowconfigure(1, weight=1) - - self.pages_frame = Frame(self) - self.pages_frame.grid(row=1, column=0, sticky=NSEW) - if self.page_class.uses_grid: - self.pages_frame.columnconfigure(0, weight=1) - self.pages_frame.rowconfigure(0, weight=1) - - # the order of the following commands is important - self._tab_set = TabSet(self, self.change_page, n_rows=n_rows, - max_tabs_per_row=max_tabs_per_row, - expand_tabs=expand_tabs) - if page_names: - for name in page_names: - self.add_page(name) - self._tab_set.grid(row=0, column=0, sticky=NSEW) - - self.change_page(self._default_page) - - def add_page(self, page_name): - """Add a new page with the name given in page_name.""" - if not page_name: - raise InvalidNameError("Invalid TabPage name: '%s'" % page_name) - if page_name in self.pages: - raise AlreadyExistsError( - "TabPage named '%s' already exists" % page_name) - - self.pages[page_name] = self.page_class(self.pages_frame) - self._pages_order.append(page_name) - self._tab_set.add_tab(page_name) - - if len(self.pages) == 1: # adding first page - self._default_page = page_name - self.change_page(page_name) - - def remove_page(self, page_name): - """Destroy the page whose name is given in page_name.""" - if not page_name in self.pages: - raise KeyError("No such TabPage: '%s" % page_name) - - self._pages_order.remove(page_name) - - # handle removing last remaining, default, or currently shown page - if len(self._pages_order) > 0: - if page_name == self._default_page: - # set a new default page - self._default_page = self._pages_order[0] - else: - self._default_page = None - - if page_name == self._current_page: - self.change_page(self._default_page) - - self._tab_set.remove_tab(page_name) - page = self.pages.pop(page_name) - page.frame.destroy() - - def change_page(self, page_name): - """Show the page whose name is given in page_name.""" - if self._current_page == page_name: - return - if page_name is not None and page_name not in self.pages: - raise KeyError("No such TabPage: '%s'" % page_name) - - if self._current_page is not None: - self.pages[self._current_page]._hide() - self._current_page = None - - if page_name is not None: - self._current_page = page_name - self.pages[page_name]._show() - - self._tab_set.set_selected_tab(page_name) - - -def _tabbed_pages(parent): # htest # - top=Toplevel(parent) - x, y = map(int, parent.geometry().split('+')[1:]) - top.geometry("+%d+%d" % (x, y + 175)) - top.title("Test tabbed pages") - tabPage=TabbedPageSet(top, page_names=['Foobar','Baz'], n_rows=0, - expand_tabs=False, - ) - tabPage.pack(side=TOP, expand=TRUE, fill=BOTH) - Label(tabPage.pages['Foobar'].frame, text='Foo', pady=20).pack() - Label(tabPage.pages['Foobar'].frame, text='Bar', pady=20).pack() - Label(tabPage.pages['Baz'].frame, text='Baz').pack() - entryPgName=Entry(top) - buttonAdd=Button(top, text='Add Page', - command=lambda:tabPage.add_page(entryPgName.get())) - buttonRemove=Button(top, text='Remove Page', - command=lambda:tabPage.remove_page(entryPgName.get())) - labelPgName=Label(top, text='name of page to add/remove:') - buttonAdd.pack(padx=5, pady=5) - buttonRemove.pack(padx=5, pady=5) - labelPgName.pack(padx=5) - entryPgName.pack(padx=5) - -if __name__ == '__main__': - from idlelib.idle_test.htest import run - run(_tabbed_pages) diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 1c0b03bff8a..e1cece0b283 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1166,7 +1166,7 @@ class IMAP4: self.mo = cre.match(s) if __debug__: if self.mo is not None and self.debug >= 5: - self._mesg("\tmatched r'%r' => %r" % (cre.pattern, self.mo.groups())) + self._mesg("\tmatched %r => %r" % (cre.pattern, self.mo.groups())) return self.mo is not None diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 5df6aa040a2..2bdd1929b42 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -522,6 +522,18 @@ def _init_module_attrs(spec, module, *, override=False): loader = _NamespaceLoader.__new__(_NamespaceLoader) loader._path = spec.submodule_search_locations + spec.loader = loader + # While the docs say that module.__file__ is not set for + # built-in modules, and the code below will avoid setting it if + # spec.has_location is false, this is incorrect for namespace + # packages. Namespace packages have no location, but their + # __spec__.origin is None, and thus their module.__file__ + # should also be None for consistency. While a bit of a hack, + # this is the best place to ensure this consistency. + # + # See # https://docs.python.org/3/library/importlib.html#importlib.abc.Loader.load_module + # and bpo-32305 + module.__file__ = None try: module.__loader__ = loader except AttributeError: diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 41de8a7b863..f9a708c2913 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -6,12 +6,11 @@ such it requires the injection of specific modules and attributes in order to work. One should use importlib as the public-facing version of this module. """ -# -# IMPORTANT: Whenever making changes to this module, be sure to run -# a top-level make in order to get the frozen version of the module -# updated. Not doing so will result in the Makefile to fail for -# all others who don't have a ./python around to freeze the module -# in the early stages of compilation. +# IMPORTANT: Whenever making changes to this module, be sure to run a top-level +# `make regen-importlib` followed by `make` in order to get the frozen version +# of the module updated. Not doing so will result in the Makefile to fail for +# all others who don't have a ./python around to freeze the module in the early +# stages of compilation. # # See importlib._setup() for what is injected into the global namespace. @@ -242,6 +241,8 @@ _code_type = type(_write_atomic.__code__) # Python 3.6rc1 3379 (more thorough __class__ validation #23722) # Python 3.7a0 3390 (add LOAD_METHOD and CALL_METHOD opcodes) # Python 3.7a0 3391 (update GET_AITER #31709) +# Python 3.7a0 3392 (PEP 552: Deterministic pycs) +# Python 3.7a0 3393 (remove STORE_ANNOTATION opcode) # # MAGIC must change whenever the bytecode emitted by the compiler may no # longer be understood by older implementations of the eval loop (usually @@ -250,7 +251,7 @@ _code_type = type(_write_atomic.__code__) # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3391).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3393).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' @@ -429,63 +430,93 @@ def _find_module_shim(self, fullname): return loader -def _validate_bytecode_header(data, source_stats=None, name=None, path=None): - """Validate the header of the passed-in bytecode against source_stats (if - given) and returning the bytecode that can be compiled by compile(). +def _classify_pyc(data, name, exc_details): + """Perform basic validity checking of a pyc header and return the flags field, + which determines how the pyc should be further validated against the source. - All other arguments are used to enhance error reporting. + *data* is the contents of the pyc file. (Only the first 16 bytes are + required, though.) - ImportError is raised when the magic number is incorrect or the bytecode is - found to be stale. EOFError is raised when the data is found to be - truncated. + *name* is the name of the module being imported. It is used for logging. + + *exc_details* is a dictionary passed to ImportError if it raised for + improved debugging. + + ImportError is raised when the magic number is incorrect or when the flags + field is invalid. EOFError is raised when the data is found to be truncated. """ - exc_details = {} - if name is not None: - exc_details['name'] = name - else: - # To prevent having to make all messages have a conditional name. - name = '' - if path is not None: - exc_details['path'] = path magic = data[:4] - raw_timestamp = data[4:8] - raw_size = data[8:12] if magic != MAGIC_NUMBER: - message = 'bad magic number in {!r}: {!r}'.format(name, magic) + message = f'bad magic number in {name!r}: {magic!r}' _bootstrap._verbose_message('{}', message) raise ImportError(message, **exc_details) - elif len(raw_timestamp) != 4: - message = 'reached EOF while reading timestamp in {!r}'.format(name) + if len(data) < 16: + message = f'reached EOF while reading pyc header of {name!r}' _bootstrap._verbose_message('{}', message) raise EOFError(message) - elif len(raw_size) != 4: - message = 'reached EOF while reading size of source in {!r}'.format(name) + flags = _r_long(data[4:8]) + # Only the first two flags are defined. + if flags & ~0b11: + message = f'invalid flags {flags!r} in {name!r}' + raise ImportError(message, **exc_details) + return flags + + +def _validate_timestamp_pyc(data, source_mtime, source_size, name, + exc_details): + """Validate a pyc against the source last-modified time. + + *data* is the contents of the pyc file. (Only the first 16 bytes are + required.) + + *source_mtime* is the last modified timestamp of the source file. + + *source_size* is None or the size of the source file in bytes. + + *name* is the name of the module being imported. It is used for logging. + + *exc_details* is a dictionary passed to ImportError if it raised for + improved debugging. + + An ImportError is raised if the bytecode is stale. + + """ + if _r_long(data[8:12]) != (source_mtime & 0xFFFFFFFF): + message = f'bytecode is stale for {name!r}' _bootstrap._verbose_message('{}', message) - raise EOFError(message) - if source_stats is not None: - try: - source_mtime = int(source_stats['mtime']) - except KeyError: - pass - else: - if _r_long(raw_timestamp) != source_mtime: - message = 'bytecode is stale for {!r}'.format(name) - _bootstrap._verbose_message('{}', message) - raise ImportError(message, **exc_details) - try: - source_size = source_stats['size'] & 0xFFFFFFFF - except KeyError: - pass - else: - if _r_long(raw_size) != source_size: - raise ImportError('bytecode is stale for {!r}'.format(name), - **exc_details) - return data[12:] + raise ImportError(message, **exc_details) + if (source_size is not None and + _r_long(data[12:16]) != (source_size & 0xFFFFFFFF)): + raise ImportError(f'bytecode is stale for {name!r}', **exc_details) + + +def _validate_hash_pyc(data, source_hash, name, exc_details): + """Validate a hash-based pyc by checking the real source hash against the one in + the pyc header. + + *data* is the contents of the pyc file. (Only the first 16 bytes are + required.) + + *source_hash* is the importlib.util.source_hash() of the source file. + + *name* is the name of the module being imported. It is used for logging. + + *exc_details* is a dictionary passed to ImportError if it raised for + improved debugging. + + An ImportError is raised if the bytecode is stale. + + """ + if data[8:16] != source_hash: + raise ImportError( + f'hash in bytecode doesn\'t match hash of source {name!r}', + **exc_details, + ) def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None): - """Compile bytecode as returned by _validate_bytecode_header().""" + """Compile bytecode as found in a pyc.""" code = marshal.loads(data) if isinstance(code, _code_type): _bootstrap._verbose_message('code object from {!r}', bytecode_path) @@ -496,16 +527,28 @@ def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None): raise ImportError('Non-code object in {!r}'.format(bytecode_path), name=name, path=bytecode_path) -def _code_to_bytecode(code, mtime=0, source_size=0): - """Compile a code object into bytecode for writing out to a byte-compiled - file.""" + +def _code_to_timestamp_pyc(code, mtime=0, source_size=0): + "Produce the data for a timestamp-based pyc." data = bytearray(MAGIC_NUMBER) + data.extend(_w_long(0)) data.extend(_w_long(mtime)) data.extend(_w_long(source_size)) data.extend(marshal.dumps(code)) return data +def _code_to_hash_pyc(code, source_hash, checked=True): + "Produce the data for a hash-based pyc." + data = bytearray(MAGIC_NUMBER) + flags = 0b1 | checked << 1 + data.extend(_w_long(flags)) + assert len(source_hash) == 8 + data.extend(source_hash) + data.extend(marshal.dumps(code)) + return data + + def decode_source(source_bytes): """Decode bytes representing source code and return the string. @@ -751,6 +794,10 @@ class SourceLoader(_LoaderBasics): """ source_path = self.get_filename(fullname) source_mtime = None + source_bytes = None + source_hash = None + hash_based = False + check_source = True try: bytecode_path = cache_from_source(source_path) except NotImplementedError: @@ -767,10 +814,34 @@ class SourceLoader(_LoaderBasics): except OSError: pass else: + exc_details = { + 'name': fullname, + 'path': bytecode_path, + } try: - bytes_data = _validate_bytecode_header(data, - source_stats=st, name=fullname, - path=bytecode_path) + flags = _classify_pyc(data, fullname, exc_details) + bytes_data = memoryview(data)[16:] + hash_based = flags & 0b1 != 0 + if hash_based: + check_source = flags & 0b10 != 0 + if (_imp.check_hash_based_pycs != 'never' and + (check_source or + _imp.check_hash_based_pycs == 'always')): + source_bytes = self.get_data(source_path) + source_hash = _imp.source_hash( + _RAW_MAGIC_NUMBER, + source_bytes, + ) + _validate_hash_pyc(data, source_hash, fullname, + exc_details) + else: + _validate_timestamp_pyc( + data, + source_mtime, + st['size'], + fullname, + exc_details, + ) except (ImportError, EOFError): pass else: @@ -779,13 +850,19 @@ class SourceLoader(_LoaderBasics): return _compile_bytecode(bytes_data, name=fullname, bytecode_path=bytecode_path, source_path=source_path) - source_bytes = self.get_data(source_path) + if source_bytes is None: + source_bytes = self.get_data(source_path) code_object = self.source_to_code(source_bytes, source_path) _bootstrap._verbose_message('code object from {}', source_path) if (not sys.dont_write_bytecode and bytecode_path is not None and source_mtime is not None): - data = _code_to_bytecode(code_object, source_mtime, - len(source_bytes)) + if hash_based: + if source_hash is None: + source_hash = _imp.source_hash(source_bytes) + data = _code_to_hash_pyc(code_object, source_hash, check_source) + else: + data = _code_to_timestamp_pyc(code_object, source_mtime, + len(source_bytes)) try: self._cache_bytecode(source_path, bytecode_path, data) _bootstrap._verbose_message('wrote {!r}', bytecode_path) @@ -834,6 +911,33 @@ class FileLoader: with _io.FileIO(path, 'r') as file: return file.read() + # ResourceReader ABC API. + + @_check_name + def get_resource_reader(self, module): + if self.is_package(module): + return self + return None + + def open_resource(self, resource): + path = _path_join(_path_split(self.path)[0], resource) + return _io.FileIO(path, 'r') + + def resource_path(self, resource): + if not self.is_resource(resource): + raise FileNotFoundError + path = _path_join(_path_split(self.path)[0], resource) + return path + + def is_resource(self, name): + if path_sep in name: + return False + path = _path_join(_path_split(self.path)[0], name) + return _path_isfile(path) + + def contents(self): + return iter(_os.listdir(_path_split(self.path)[0])) + class SourceFileLoader(FileLoader, SourceLoader): @@ -887,8 +991,18 @@ class SourcelessFileLoader(FileLoader, _LoaderBasics): def get_code(self, fullname): path = self.get_filename(fullname) data = self.get_data(path) - bytes_data = _validate_bytecode_header(data, name=fullname, path=path) - return _compile_bytecode(bytes_data, name=fullname, bytecode_path=path) + # Call _classify_pyc to do basic validation of the pyc but ignore the + # result. There's no source to check against. + exc_details = { + 'name': fullname, + 'path': path, + } + _classify_pyc(data, fullname, exc_details) + return _compile_bytecode( + memoryview(data)[16:], + name=fullname, + bytecode_path=path, + ) def get_source(self, fullname): """Return None as there is no source code.""" @@ -1162,9 +1276,9 @@ class PathFinder: elif spec.loader is None: namespace_path = spec.submodule_search_locations if namespace_path: - # We found at least one namespace path. Return a - # spec which can create the namespace package. - spec.origin = 'namespace' + # We found at least one namespace path. Return a spec which + # can create the namespace package. + spec.origin = None spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec) return spec else: diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index d7cadf2ee74..d2f45093a45 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -340,3 +340,49 @@ class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLo """ _register(SourceLoader, machinery.SourceFileLoader) + + +class ResourceReader(metaclass=abc.ABCMeta): + + """Abstract base class to provide resource-reading support. + + Loaders that support resource reading are expected to implement + the ``get_resource_reader(fullname)`` method and have it either return None + or an object compatible with this ABC. + """ + + @abc.abstractmethod + def open_resource(self, resource): + """Return an opened, file-like object for binary reading. + + The 'resource' argument is expected to represent only a file name + and thus not contain any subdirectory components. + + If the resource cannot be found, FileNotFoundError is raised. + """ + raise FileNotFoundError + + @abc.abstractmethod + def resource_path(self, resource): + """Return the file system path to the specified resource. + + The 'resource' argument is expected to represent only a file name + and thus not contain any subdirectory components. + + If the resource does not exist on the file system, raise + FileNotFoundError. + """ + raise FileNotFoundError + + @abc.abstractmethod + def is_resource(self, name): + """Return True if the named 'name' is consider a resource.""" + raise FileNotFoundError + + @abc.abstractmethod + def contents(self): + """Return an iterator of strings over the contents of the package.""" + return iter([]) + + +_register(ResourceReader, machinery.SourceFileLoader) diff --git a/Lib/importlib/resources.py b/Lib/importlib/resources.py new file mode 100644 index 00000000000..c4f6bbde45f --- /dev/null +++ b/Lib/importlib/resources.py @@ -0,0 +1,330 @@ +import os +import tempfile + +from . import abc as resources_abc +from builtins import open as builtins_open +from contextlib import contextmanager, suppress +from importlib import import_module +from importlib.abc import ResourceLoader +from io import BytesIO, TextIOWrapper +from pathlib import Path +from types import ModuleType +from typing import Iterator, Optional, Set, Union # noqa: F401 +from typing import cast +from typing.io import BinaryIO, TextIO +from zipimport import ZipImportError + + +Package = Union[str, ModuleType] +Resource = Union[str, os.PathLike] + + +def _get_package(package) -> ModuleType: + """Take a package name or module object and return the module. + + If a name, the module is imported. If the passed or imported module + object is not a package, raise an exception. + """ + if hasattr(package, '__spec__'): + if package.__spec__.submodule_search_locations is None: + raise TypeError('{!r} is not a package'.format( + package.__spec__.name)) + else: + return package + else: + module = import_module(package) + if module.__spec__.submodule_search_locations is None: + raise TypeError('{!r} is not a package'.format(package)) + else: + return module + + +def _normalize_path(path) -> str: + """Normalize a path by ensuring it is a string. + + If the resulting string contains path separators, an exception is raised. + """ + str_path = str(path) + parent, file_name = os.path.split(str_path) + if parent: + raise ValueError('{!r} must be only a file name'.format(path)) + else: + return file_name + + +def _get_resource_reader( + package: ModuleType) -> Optional[resources_abc.ResourceReader]: + # Return the package's loader if it's a ResourceReader. We can't use + # a issubclass() check here because apparently abc.'s __subclasscheck__() + # hook wants to create a weak reference to the object, but + # zipimport.zipimporter does not support weak references, resulting in a + # TypeError. That seems terrible. + spec = package.__spec__ + if hasattr(spec.loader, 'get_resource_reader'): + return cast(resources_abc.ResourceReader, + spec.loader.get_resource_reader(spec.name)) + return None + + +def _check_location(package): + if package.__spec__.origin is None or not package.__spec__.has_location: + raise FileNotFoundError(f'Package has no location {package!r}') + + +def open_binary(package: Package, resource: Resource) -> BinaryIO: + """Return a file-like object opened for binary reading of the resource.""" + resource = _normalize_path(resource) + package = _get_package(package) + reader = _get_resource_reader(package) + if reader is not None: + return reader.open_resource(resource) + _check_location(package) + absolute_package_path = os.path.abspath(package.__spec__.origin) + package_path = os.path.dirname(absolute_package_path) + full_path = os.path.join(package_path, resource) + try: + return builtins_open(full_path, mode='rb') + except OSError: + # Just assume the loader is a resource loader; all the relevant + # importlib.machinery loaders are and an AttributeError for + # get_data() will make it clear what is needed from the loader. + loader = cast(ResourceLoader, package.__spec__.loader) + data = None + if hasattr(package.__spec__.loader, 'get_data'): + with suppress(OSError): + data = loader.get_data(full_path) + if data is None: + package_name = package.__spec__.name + message = '{!r} resource not found in {!r}'.format( + resource, package_name) + raise FileNotFoundError(message) + else: + return BytesIO(data) + + +def open_text(package: Package, + resource: Resource, + encoding: str = 'utf-8', + errors: str = 'strict') -> TextIO: + """Return a file-like object opened for text reading of the resource.""" + resource = _normalize_path(resource) + package = _get_package(package) + reader = _get_resource_reader(package) + if reader is not None: + return TextIOWrapper(reader.open_resource(resource), encoding, errors) + _check_location(package) + absolute_package_path = os.path.abspath(package.__spec__.origin) + package_path = os.path.dirname(absolute_package_path) + full_path = os.path.join(package_path, resource) + try: + return builtins_open( + full_path, mode='r', encoding=encoding, errors=errors) + except OSError: + # Just assume the loader is a resource loader; all the relevant + # importlib.machinery loaders are and an AttributeError for + # get_data() will make it clear what is needed from the loader. + loader = cast(ResourceLoader, package.__spec__.loader) + data = None + if hasattr(package.__spec__.loader, 'get_data'): + with suppress(OSError): + data = loader.get_data(full_path) + if data is None: + package_name = package.__spec__.name + message = '{!r} resource not found in {!r}'.format( + resource, package_name) + raise FileNotFoundError(message) + else: + return TextIOWrapper(BytesIO(data), encoding, errors) + + +def read_binary(package: Package, resource: Resource) -> bytes: + """Return the binary contents of the resource.""" + resource = _normalize_path(resource) + package = _get_package(package) + with open_binary(package, resource) as fp: + return fp.read() + + +def read_text(package: Package, + resource: Resource, + encoding: str = 'utf-8', + errors: str = 'strict') -> str: + """Return the decoded string of the resource. + + The decoding-related arguments have the same semantics as those of + bytes.decode(). + """ + resource = _normalize_path(resource) + package = _get_package(package) + with open_text(package, resource, encoding, errors) as fp: + return fp.read() + + +@contextmanager +def path(package: Package, resource: Resource) -> Iterator[Path]: + """A context manager providing a file path object to the resource. + + If the resource does not already exist on its own on the file system, + a temporary file will be created. If the file was created, the file + will be deleted upon exiting the context manager (no exception is + raised if the file was deleted prior to the context manager + exiting). + """ + resource = _normalize_path(resource) + package = _get_package(package) + reader = _get_resource_reader(package) + if reader is not None: + try: + yield Path(reader.resource_path(resource)) + return + except FileNotFoundError: + pass + else: + _check_location(package) + # Fall-through for both the lack of resource_path() *and* if + # resource_path() raises FileNotFoundError. + package_directory = Path(package.__spec__.origin).parent + file_path = package_directory / resource + if file_path.exists(): + yield file_path + else: + with open_binary(package, resource) as fp: + data = fp.read() + # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try' + # blocks due to the need to close the temporary file to work on + # Windows properly. + fd, raw_path = tempfile.mkstemp() + try: + os.write(fd, data) + os.close(fd) + yield Path(raw_path) + finally: + try: + os.remove(raw_path) + except FileNotFoundError: + pass + + +def is_resource(package: Package, name: str) -> bool: + """True if 'name' is a resource inside 'package'. + + Directories are *not* resources. + """ + package = _get_package(package) + _normalize_path(name) + reader = _get_resource_reader(package) + if reader is not None: + return reader.is_resource(name) + try: + package_contents = set(contents(package)) + except (NotADirectoryError, FileNotFoundError): + return False + if name not in package_contents: + return False + # Just because the given file_name lives as an entry in the package's + # contents doesn't necessarily mean it's a resource. Directories are not + # resources, so let's try to find out if it's a directory or not. + path = Path(package.__spec__.origin).parent / name + return path.is_file() + + +def contents(package: Package) -> Iterator[str]: + """Return the list of entries in 'package'. + + Note that not all entries are resources. Specifically, directories are + not considered resources. Use `is_resource()` on each entry returned here + to check if it is a resource or not. + """ + package = _get_package(package) + reader = _get_resource_reader(package) + if reader is not None: + yield from reader.contents() + return + # Is the package a namespace package? By definition, namespace packages + # cannot have resources. We could use _check_location() and catch the + # exception, but that's extra work, so just inline the check. + if package.__spec__.origin is None or not package.__spec__.has_location: + return [] + package_directory = Path(package.__spec__.origin).parent + yield from os.listdir(str(package_directory)) + + +# Private implementation of ResourceReader and get_resource_reader() for +# zipimport. Don't use these directly! We're implementing these in Python +# because 1) it's easier, 2) zipimport will likely get rewritten in Python +# itself at some point, so doing this all in C would just be a waste of +# effort. + +class _ZipImportResourceReader(resources_abc.ResourceReader): + """Private class used to support ZipImport.get_resource_reader(). + + This class is allowed to reference all the innards and private parts of + the zipimporter. + """ + + def __init__(self, zipimporter, fullname): + self.zipimporter = zipimporter + self.fullname = fullname + + def open_resource(self, resource): + path = f'{self.fullname}/{resource}' + try: + return BytesIO(self.zipimporter.get_data(path)) + except OSError: + raise FileNotFoundError + + def resource_path(self, resource): + # All resources are in the zip file, so there is no path to the file. + # Raising FileNotFoundError tells the higher level API to extract the + # binary data and create a temporary file. + raise FileNotFoundError + + def is_resource(self, name): + # Maybe we could do better, but if we can get the data, it's a + # resource. Otherwise it isn't. + path = f'{self.fullname}/{name}' + try: + self.zipimporter.get_data(path) + except OSError: + return False + return True + + def contents(self): + # This is a bit convoluted, because fullname will be a module path, + # but _files is a list of file names relative to the top of the + # archive's namespace. We want to compare file paths to find all the + # names of things inside the module represented by fullname. So we + # turn the module path of fullname into a file path relative to the + # top of the archive, and then we iterate through _files looking for + # names inside that "directory". + fullname_path = Path(self.zipimporter.get_filename(self.fullname)) + relative_path = fullname_path.relative_to(self.zipimporter.archive) + # Don't forget that fullname names a package, so its path will include + # __init__.py, which we want to ignore. + assert relative_path.name == '__init__.py' + package_path = relative_path.parent + subdirs_seen = set() + for filename in self.zipimporter._files: + try: + relative = Path(filename).relative_to(package_path) + except ValueError: + continue + # If the path of the file (which is relative to the top of the zip + # namespace), relative to the package given when the resource + # reader was created, has a parent, then it's a name in a + # subdirectory and thus we skip it. + parent_name = relative.parent.name + if len(parent_name) == 0: + yield relative.name + elif parent_name not in subdirs_seen: + subdirs_seen.add(parent_name) + yield parent_name + + +def _zipimport_get_resource_reader(zipimporter, fullname): + try: + if not zipimporter.is_package(fullname): + return None + except ZipImportError: + return None + return _ZipImportResourceReader(zipimporter, fullname) diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 41c74d4cc6c..9d0a90d2973 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -5,18 +5,25 @@ from ._bootstrap import _resolve_name from ._bootstrap import spec_from_loader from ._bootstrap import _find_spec from ._bootstrap_external import MAGIC_NUMBER +from ._bootstrap_external import _RAW_MAGIC_NUMBER from ._bootstrap_external import cache_from_source from ._bootstrap_external import decode_source from ._bootstrap_external import source_from_cache from ._bootstrap_external import spec_from_file_location from contextlib import contextmanager +import _imp import functools import sys import types import warnings +def source_hash(source_bytes): + "Return the hash of *source_bytes* as used in hash-based pyc files." + return _imp.source_hash(_RAW_MAGIC_NUMBER, source_bytes) + + def resolve_name(name, package): """Resolve a relative module name to an absolute one.""" if not name.startswith('.'): diff --git a/Lib/inspect.py b/Lib/inspect.py index 69f2b6209f6..bc97efe179c 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -32,7 +32,6 @@ __author__ = ('Ka-Ping Yee ', 'Yury Selivanov ') import abc -import ast import dis import collections.abc import enum @@ -457,10 +456,10 @@ def classify_class_attrs(cls): continue obj = get_obj if get_obj is not None else dict_obj # Classify the object or its descriptor. - if isinstance(dict_obj, staticmethod): + if isinstance(dict_obj, (staticmethod, types.BuiltinMethodType)): kind = "static method" obj = dict_obj - elif isinstance(dict_obj, classmethod): + elif isinstance(dict_obj, (classmethod, types.ClassMethodDescriptorType)): kind = "class method" obj = dict_obj elif isinstance(dict_obj, property): @@ -1381,7 +1380,7 @@ def getclosurevars(func): func = func.__func__ if not isfunction(func): - raise TypeError("'{!r}' is not a Python function".format(func)) + raise TypeError("{!r} is not a Python function".format(func)) code = func.__code__ # Nonlocal references are named in co_freevars and resolved @@ -1624,7 +1623,7 @@ def getgeneratorlocals(generator): bound values.""" if not isgenerator(generator): - raise TypeError("'{!r}' is not a Python generator".format(generator)) + raise TypeError("{!r} is not a Python generator".format(generator)) frame = getattr(generator, "gi_frame", None) if frame is not None: @@ -1940,6 +1939,9 @@ def _signature_fromstr(cls, obj, s, skip_bound_arg=True): """Private helper to parse content of '__text_signature__' and return a Signature based on it. """ + # Lazy import ast because it's relatively heavy and + # it's not used for other than this function. + import ast Parameter = cls._parameter_cls diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index 6d0511ebfe9..a5660099af7 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -76,7 +76,8 @@ Specializing JSON object encoding:: >>> def encode_complex(obj): ... if isinstance(obj, complex): ... return [obj.real, obj.imag] - ... raise TypeError(repr(obj) + " is not JSON serializable") + ... raise TypeError(f'Object of type {obj.__class__.__name__} ' + ... f'is not JSON serializable') ... >>> json.dumps(2 + 1j, default=encode_complex) '[2.0, 1.0]' @@ -344,8 +345,8 @@ def loads(s, *, encoding=None, cls=None, object_hook=None, parse_float=None, s, 0) else: if not isinstance(s, (bytes, bytearray)): - raise TypeError('the JSON object must be str, bytes or bytearray, ' - 'not {!r}'.format(s.__class__.__name__)) + raise TypeError(f'the JSON object must be str, bytes or bytearray, ' + f'not {s.__class__.__name__}') s = s.decode(detect_encoding(s), 'surrogatepass') if (cls is None and object_hook is None and diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 41a497c5da0..fb083ed61bb 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -176,8 +176,8 @@ class JSONEncoder(object): return JSONEncoder.default(self, o) """ - raise TypeError("Object of type '%s' is not JSON serializable" % - o.__class__.__name__) + raise TypeError(f'Object of type {o.__class__.__name__} ' + f'is not JSON serializable') def encode(self, o): """Return a JSON string representation of a Python data structure. @@ -373,7 +373,8 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, elif _skipkeys: continue else: - raise TypeError("key " + repr(key) + " is not a string") + raise TypeError(f'keys must be str, int, float, bool or None, ' + f'not {key.__class__.__name__}') if first: first = False else: diff --git a/Lib/json/tool.py b/Lib/json/tool.py index 4f3182c0c1e..5932f4ecded 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -11,7 +11,6 @@ Usage:: """ import argparse -import collections import json import sys @@ -34,11 +33,7 @@ def main(): sort_keys = options.sort_keys with infile: try: - if sort_keys: - obj = json.load(infile) - else: - obj = json.load(infile, - object_pairs_hook=collections.OrderedDict) + obj = json.load(infile) except ValueError as e: raise SystemExit(e) with outfile: diff --git a/Lib/lib2to3/fixes/fix_operator.py b/Lib/lib2to3/fixes/fix_operator.py index 0d82454023f..d303cd2018b 100644 --- a/Lib/lib2to3/fixes/fix_operator.py +++ b/Lib/lib2to3/fixes/fix_operator.py @@ -1,6 +1,6 @@ """Fixer for operator functions. -operator.isCallable(obj) -> hasattr(obj, '__call__') +operator.isCallable(obj) -> callable(obj) operator.sequenceIncludes(obj) -> operator.contains(obj) operator.isSequenceType(obj) -> isinstance(obj, collections.abc.Sequence) operator.isMappingType(obj) -> isinstance(obj, collections.abc.Mapping) @@ -49,11 +49,10 @@ class FixOperator(fixer_base.BaseFix): def _sequenceIncludes(self, node, results): return self._handle_rename(node, results, "contains") - @invocation("hasattr(%s, '__call__')") + @invocation("callable(%s)") def _isCallable(self, node, results): obj = results["obj"] - args = [obj.clone(), String(", "), String("'__call__'")] - return Call(Name("hasattr"), args, prefix=node.prefix) + return Call(Name("callable"), [obj.clone()], prefix=node.prefix) @invocation("operator.mul(%s)") def _repeat(self, node, results): diff --git a/Lib/lib2to3/patcomp.py b/Lib/lib2to3/patcomp.py index 0fefa9a3a42..f57f4954b26 100644 --- a/Lib/lib2to3/patcomp.py +++ b/Lib/lib2to3/patcomp.py @@ -12,7 +12,6 @@ __author__ = "Guido van Rossum " # Python imports import io -import os # Fairly local imports from .pgen2 import driver, literals, token, tokenize, parse, grammar @@ -21,10 +20,6 @@ from .pgen2 import driver, literals, token, tokenize, parse, grammar from . import pytree from . import pygram -# The pattern grammar file -_PATTERN_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), - "PatternGrammar.txt") - class PatternSyntaxError(Exception): pass @@ -42,13 +37,17 @@ def tokenize_wrapper(input): class PatternCompiler(object): - def __init__(self, grammar_file=_PATTERN_GRAMMAR_FILE): + def __init__(self, grammar_file=None): """Initializer. Takes an optional alternative filename for the pattern grammar. """ - self.grammar = driver.load_grammar(grammar_file) - self.syms = pygram.Symbols(self.grammar) + if grammar_file is None: + self.grammar = pygram.pattern_grammar + self.syms = pygram.pattern_symbols + else: + self.grammar = driver.load_grammar(grammar_file) + self.syms = pygram.Symbols(self.grammar) self.pygrammar = pygram.python_grammar self.pysyms = pygram.python_symbols self.driver = driver.Driver(self.grammar, convert=pattern_convert) diff --git a/Lib/lib2to3/pgen2/driver.py b/Lib/lib2to3/pgen2/driver.py index e5e4824c008..cbc58e759d5 100644 --- a/Lib/lib2to3/pgen2/driver.py +++ b/Lib/lib2to3/pgen2/driver.py @@ -20,6 +20,7 @@ import codecs import io import os import logging +import pkgutil import sys # Pgen imports @@ -140,6 +141,26 @@ def _newer(a, b): return os.path.getmtime(a) >= os.path.getmtime(b) +def load_packaged_grammar(package, grammar_source): + """Normally, loads a pickled grammar by doing + pkgutil.get_data(package, pickled_grammar) + where *pickled_grammar* is computed from *grammar_source* by adding the + Python version and using a ``.pickle`` extension. + + However, if *grammar_source* is an extant file, load_grammar(grammar_source) + is called instead. This facilitates using a packaged grammar file when needed + but preserves load_grammar's automatic regeneration behavior when possible. + + """ + if os.path.isfile(grammar_source): + return load_grammar(grammar_source) + pickled_name = _generate_pickle_name(os.path.basename(grammar_source)) + data = pkgutil.get_data(package, pickled_name) + g = grammar.Grammar() + g.loads(data) + return g + + def main(*args): """Main program, when run as a script: produce grammar pickle files. diff --git a/Lib/lib2to3/pgen2/grammar.py b/Lib/lib2to3/pgen2/grammar.py index c10dcfa9ac2..088c58bfa99 100644 --- a/Lib/lib2to3/pgen2/grammar.py +++ b/Lib/lib2to3/pgen2/grammar.py @@ -108,6 +108,10 @@ class Grammar(object): d = pickle.load(f) self.__dict__.update(d) + def loads(self, pkl): + """Load the grammar tables from a pickle bytes object.""" + self.__dict__.update(pickle.loads(pkl)) + def copy(self): """ Copy the grammar. diff --git a/Lib/lib2to3/pygram.py b/Lib/lib2to3/pygram.py index 01fa1087115..919624eb399 100644 --- a/Lib/lib2to3/pygram.py +++ b/Lib/lib2to3/pygram.py @@ -29,12 +29,12 @@ class Symbols(object): setattr(self, name, symbol) -python_grammar = driver.load_grammar(_GRAMMAR_FILE) +python_grammar = driver.load_packaged_grammar("lib2to3", _GRAMMAR_FILE) python_symbols = Symbols(python_grammar) python_grammar_no_print_statement = python_grammar.copy() del python_grammar_no_print_statement.keywords["print"] -pattern_grammar = driver.load_grammar(_PATTERN_GRAMMAR_FILE) +pattern_grammar = driver.load_packaged_grammar("lib2to3", _PATTERN_GRAMMAR_FILE) pattern_symbols = Symbols(pattern_grammar) diff --git a/Lib/lib2to3/tests/support.py b/Lib/lib2to3/tests/support.py index ae7cfe8ee27..fe084e8903f 100644 --- a/Lib/lib2to3/tests/support.py +++ b/Lib/lib2to3/tests/support.py @@ -15,7 +15,13 @@ test_dir = os.path.dirname(__file__) proj_dir = os.path.normpath(os.path.join(test_dir, "..")) grammar_path = os.path.join(test_dir, "..", "Grammar.txt") grammar = pgen2_driver.load_grammar(grammar_path) +grammar_no_print_statement = pgen2_driver.load_grammar(grammar_path) +del grammar_no_print_statement.keywords["print"] driver = pgen2_driver.Driver(grammar, convert=pytree.convert) +driver_no_print_statement = pgen2_driver.Driver( + grammar_no_print_statement, + convert=pytree.convert +) def parse_string(string): return driver.parse_string(reformat(string), debug=True) diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index e50b7dadae8..bfe7a23e700 100644 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -4409,7 +4409,7 @@ class Test_operator(FixerTestCase): def test_operator_isCallable(self): b = "operator.isCallable(x)" - a = "hasattr(x, '__call__')" + a = "callable(x)" self.check(b, a) def test_operator_sequenceIncludes(self): @@ -4468,7 +4468,7 @@ class Test_operator(FixerTestCase): def test_bare_isCallable(self): s = "isCallable(x)" - t = "You should use 'hasattr(x, '__call__')' here." + t = "You should use 'callable(x)' here." self.warns_unchanged(s, t) def test_bare_sequenceIncludes(self): diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index dc94a69036a..0cbba26bec0 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -8,11 +8,15 @@ test_grammar.py files from both Python 2 and Python 3. # Testing imports from . import support -from .support import driver +from .support import driver, driver_no_print_statement from test.support import verbose # Python imports +import difflib +import importlib +import operator import os +import pickle import shutil import subprocess import sys @@ -99,6 +103,18 @@ pgen2_driver.load_grammar(%r, save=True, force=True) finally: shutil.rmtree(tmpdir) + def test_load_packaged_grammar(self): + modname = __name__ + '.load_test' + class MyLoader: + def get_data(self, where): + return pickle.dumps({'elephant': 19}) + class MyModule: + __file__ = 'parsertestmodule' + __spec__ = importlib.util.spec_from_loader(modname, MyLoader()) + sys.modules[modname] = MyModule() + self.addCleanup(operator.delitem, sys.modules, modname) + g = pgen2_driver.load_packaged_grammar(modname, 'Grammar.txt') + self.assertEqual(g.elephant, 19) class GrammarTest(support.TestCase): @@ -398,8 +414,6 @@ class TestParserIdempotency(support.TestCase): """A cut-down version of pytree_idempotency.py.""" - # Issue 13125 - @unittest.expectedFailure def test_all_project_files(self): for filepath in support.all_project_files(): with open(filepath, "rb") as fp: @@ -410,13 +424,14 @@ class TestParserIdempotency(support.TestCase): source = fp.read() try: tree = driver.parse_string(source) - except ParseError as err: - if verbose > 0: - warnings.warn('ParseError on file %s (%s)' % (filepath, err)) - continue + except ParseError: + try: + tree = driver_no_print_statement.parse_string(source) + except ParseError as err: + self.fail('ParseError on file %s (%s)' % (filepath, err)) new = str(tree) - x = diff(filepath, new) - if x: + if new != source: + print(diff_texts(source, new, filepath)) self.fail("Idempotency failed: %s" % filepath) def test_extended_unpacking(self): @@ -466,17 +481,12 @@ class TestGeneratorExpressions(GrammarTest): self.validate("set(x for x in [],)") -def diff(fn, result): - try: - with open('@', 'w') as f: - f.write(str(result)) - fn = fn.replace('"', '\\"') - return subprocess.call(['diff', '-u', fn, '@'], stdout=(subprocess.DEVNULL if verbose < 1 else None)) - finally: - try: - os.remove("@") - except OSError: - pass +def diff_texts(a, b, filename): + a = a.splitlines() + b = b.splitlines() + return difflib.unified_diff(a, b, filename, filename, + "(original)", "(reserialized)", + lineterm="") if __name__ == '__main__': diff --git a/Lib/locale.py b/Lib/locale.py index f1d157d6f9c..18079e73ad6 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -617,6 +617,8 @@ if sys.platform.startswith("win"): # On Win32, this will return the ANSI code page def getpreferredencoding(do_setlocale = True): """Return the charset that the user is likely using.""" + if sys.flags.utf8_mode: + return 'UTF-8' import _bootlocale return _bootlocale.getpreferredencoding(False) else: @@ -634,6 +636,8 @@ else: def getpreferredencoding(do_setlocale = True): """Return the charset that the user is likely using, by looking at environment variables.""" + if sys.flags.utf8_mode: + return 'UTF-8' res = getdefaultlocale()[1] if res is None: # LANG not set, default conservatively to ASCII @@ -643,6 +647,8 @@ else: def getpreferredencoding(do_setlocale = True): """Return the charset that the user is likely using, according to the system configuration.""" + if sys.flags.utf8_mode: + return 'UTF-8' import _bootlocale if do_setlocale: oldloc = setlocale(LC_CTYPE) diff --git a/Lib/macpath.py b/Lib/macpath.py index f85a91435b9..aacf7235b01 100644 --- a/Lib/macpath.py +++ b/Lib/macpath.py @@ -1,5 +1,17 @@ """Pathname and path-related operations for the Macintosh.""" +# strings representing various path-related bits and pieces +# These are primarily for export; internally, they are hardcoded. +# Should be set before imports for resolving cyclic dependency. +curdir = ':' +pardir = '::' +extsep = '.' +sep = ':' +pathsep = '\n' +defpath = ':' +altsep = None +devnull = 'Dev:Null' + import os from stat import * import genericpath @@ -16,17 +28,6 @@ __all__ = ["normcase","isabs","join","splitdrive","split","splitext", "curdir","pardir","sep","pathsep","defpath","altsep","extsep", "devnull","realpath","supports_unicode_filenames"] -# strings representing various path-related bits and pieces -# These are primarily for export; internally, they are hardcoded. -curdir = ':' -pardir = '::' -extsep = '.' -sep = ':' -pathsep = '\n' -defpath = ':' -altsep = None -devnull = 'Dev:Null' - def _get_colon(path): if isinstance(path, bytes): return b':' diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 44112027cb9..c86dd6d1345 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -410,7 +410,7 @@ def _default_mime_types(): '.bat' : 'text/plain', '.bcpio' : 'application/x-bcpio', '.bin' : 'application/octet-stream', - '.bmp' : 'image/x-ms-bmp', + '.bmp' : 'image/bmp', '.c' : 'text/plain', '.cdf' : 'application/x-netcdf', '.cpio' : 'application/x-cpio', diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index e277ca72f8e..10320a74d94 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -287,11 +287,12 @@ class ModuleFinder: co = compile(fp.read()+'\n', pathname, 'exec') elif type == imp.PY_COMPILED: try: - marshal_data = importlib._bootstrap_external._validate_bytecode_header(fp.read()) + data = fp.read() + importlib._bootstrap_external._classify_pyc(data, fqname, {}) except ImportError as exc: self.msgout(2, "raise ImportError: " + str(exc), pathname) raise - co = marshal.loads(marshal_data) + co = marshal.loads(memoryview(data)[16:]) else: co = None m = self.add_module(fqname) diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index b1ee725fac6..3e9a0d6b486 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -156,7 +156,7 @@ class Pool(object): maxtasksperchild=None, context=None): self._ctx = context or get_context() self._setup_queues() - self._taskqueue = queue.Queue() + self._taskqueue = queue.SimpleQueue() self._cache = {} self._state = RUN self._maxtasksperchild = maxtasksperchild @@ -802,15 +802,18 @@ class ThreadPool(Pool): Pool.__init__(self, processes, initializer, initargs) def _setup_queues(self): - self._inqueue = queue.Queue() - self._outqueue = queue.Queue() + self._inqueue = queue.SimpleQueue() + self._outqueue = queue.SimpleQueue() self._quick_put = self._inqueue.put self._quick_get = self._outqueue.get @staticmethod def _help_stuff_finish(inqueue, task_handler, size): - # put sentinels at head of inqueue to make workers finish - with inqueue.not_empty: - inqueue.queue.clear() - inqueue.queue.extend([None] * size) - inqueue.not_empty.notify_all() + # drain inqueue, and put sentinels at its head to make workers finish + try: + while True: + inqueue.get(block=False) + except queue.Empty: + pass + for i in range(size): + inqueue.put(None) diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 328efbd95fe..d66d37a5c3e 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -160,9 +160,10 @@ class Queue(object): self._thread = threading.Thread( target=Queue._feed, args=(self._buffer, self._notempty, self._send_bytes, - self._wlock, self._writer.close, self._ignore_epipe), + self._wlock, self._writer.close, self._ignore_epipe, + self._on_queue_feeder_error), name='QueueFeederThread' - ) + ) self._thread.daemon = True debug('doing self._thread.start()') @@ -201,7 +202,8 @@ class Queue(object): notempty.notify() @staticmethod - def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe): + def _feed(buffer, notempty, send_bytes, writelock, close, ignore_epipe, + onerror): debug('starting thread to feed data to pipe') nacquire = notempty.acquire nrelease = notempty.release @@ -253,8 +255,17 @@ class Queue(object): info('error in queue thread: %s', e) return else: - import traceback - traceback.print_exc() + onerror(e, obj) + + @staticmethod + def _on_queue_feeder_error(e, obj): + """ + Private API hook called when feeding data in the background thread + raises an exception. For overriding by concurrent.futures. + """ + import traceback + traceback.print_exc() + _sentinel = object() diff --git a/Lib/multiprocessing/sharedctypes.py b/Lib/multiprocessing/sharedctypes.py index 066cf8f031d..6071707027b 100644 --- a/Lib/multiprocessing/sharedctypes.py +++ b/Lib/multiprocessing/sharedctypes.py @@ -78,7 +78,7 @@ def Value(typecode_or_type, *args, lock=True, ctx=None): ctx = ctx or get_context() lock = ctx.RLock() if not hasattr(lock, 'acquire'): - raise AttributeError("'%r' has no method 'acquire'" % lock) + raise AttributeError("%r has no method 'acquire'" % lock) return synchronized(obj, lock, ctx=ctx) def Array(typecode_or_type, size_or_initializer, *, lock=True, ctx=None): @@ -92,7 +92,7 @@ def Array(typecode_or_type, size_or_initializer, *, lock=True, ctx=None): ctx = ctx or get_context() lock = ctx.RLock() if not hasattr(lock, 'acquire'): - raise AttributeError("'%r' has no method 'acquire'" % lock) + raise AttributeError("%r has no method 'acquire'" % lock) return synchronized(obj, lock, ctx=ctx) def copy(obj): diff --git a/Lib/netrc.py b/Lib/netrc.py index baf8f1d99d9..f0ae48cfed9 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -23,10 +23,7 @@ class netrc: def __init__(self, file=None): default_netrc = file is None if file is None: - try: - file = os.path.join(os.environ['HOME'], ".netrc") - except KeyError: - raise OSError("Could not find .netrc: $HOME is not set") from None + file = os.path.join(os.path.expanduser("~"), ".netrc") self.hosts = {} self.macros = {} with open(file) as fp: diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 10d3f2dc35b..2182ec776cc 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -5,6 +5,18 @@ Instead of importing this module directly, import os and refer to this module as os.path. """ +# strings representing various path-related bits and pieces +# These are primarily for export; internally, they are hardcoded. +# Should be set before imports for resolving cyclic dependency. +curdir = '.' +pardir = '..' +extsep = '.' +sep = '\\' +pathsep = ';' +altsep = '/' +defpath = '.;C:\\bin' +devnull = 'nul' + import os import sys import stat @@ -19,17 +31,6 @@ __all__ = ["normcase","isabs","join","splitdrive","split","splitext", "extsep","devnull","realpath","supports_unicode_filenames","relpath", "samefile", "sameopenfile", "samestat", "commonpath"] -# strings representing various path-related bits and pieces -# These are primarily for export; internally, they are hardcoded. -curdir = '.' -pardir = '..' -extsep = '.' -sep = '\\' -pathsep = ';' -altsep = '/' -defpath = '.;C:\\bin' -devnull = 'nul' - def _get_bothseps(path): if isinstance(path, bytes): return b'\\/' diff --git a/Lib/opcode.py b/Lib/opcode.py index dffb38c314a..368472d9811 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -142,7 +142,7 @@ name_op('LOAD_NAME', 101) # Index in name list def_op('BUILD_TUPLE', 102) # Number of tuple items def_op('BUILD_LIST', 103) # Number of list items def_op('BUILD_SET', 104) # Number of set items -def_op('BUILD_MAP', 105) # Number of dict entries (upto 255) +def_op('BUILD_MAP', 105) # Number of dict entries name_op('LOAD_ATTR', 106) # Index in name list def_op('COMPARE_OP', 107) # Comparison operator hascompare.append(107) @@ -169,7 +169,6 @@ def_op('STORE_FAST', 125) # Local variable number haslocal.append(125) def_op('DELETE_FAST', 126) # Local variable number haslocal.append(126) -name_op('STORE_ANNOTATION', 127) # Index in name list def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) def_op('CALL_FUNCTION', 131) # #args diff --git a/Lib/os.py b/Lib/os.py index 4f9fdf5b0ea..499e6285618 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -891,7 +891,7 @@ If mode == P_WAIT return the process's exit code if it exits normally; otherwise return -SIG, where SIG is the signal that killed it. """ return _spawnvef(mode, file, args, env, execve) - # Note: spawnvp[e] is't currently supported on Windows + # Note: spawnvp[e] isn't currently supported on Windows def spawnvp(mode, file, args): """spawnvp(mode, file, args) -> integer diff --git a/Lib/pdb.py b/Lib/pdb.py index 8dd4dedb220..60bdb7675c8 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -1521,6 +1521,24 @@ class Pdb(bdb.Bdb, cmd.Cmd): return fullname return None + def _runmodule(self, module_name): + self._wait_for_mainpyfile = True + self._user_requested_quit = False + import runpy + mod_name, mod_spec, code = runpy._get_module_details(module_name) + self.mainpyfile = self.canonic(code.co_filename) + import __main__ + __main__.__dict__.clear() + __main__.__dict__.update({ + "__name__": "__main__", + "__file__": self.mainpyfile, + "__package__": mod_spec.parent, + "__loader__": mod_spec.loader, + "__spec__": mod_spec, + "__builtins__": __builtins__, + }) + self.run(code) + def _runscript(self, filename): # The script has to run in __main__ namespace (or imports from # __main__ will break). @@ -1620,9 +1638,11 @@ def help(): pydoc.pager(__doc__) _usage = """\ -usage: pdb.py [-c command] ... pyfile [arg] ... +usage: pdb.py [-c command] ... [-m module | pyfile] [arg] ... -Debug the Python program given by pyfile. +Debug the Python program given by pyfile. Alternatively, +an executable module or package to debug can be specified using +the -m switch. Initial commands are read from .pdbrc files in your home directory and in the current directory, if they exist. Commands supplied with @@ -1635,29 +1655,33 @@ To let the script run up to a given line X in the debugged file, use def main(): import getopt - opts, args = getopt.getopt(sys.argv[1:], 'hc:', ['--help', '--command=']) + opts, args = getopt.getopt(sys.argv[1:], 'mhc:', ['--help', '--command=']) if not args: print(_usage) sys.exit(2) commands = [] + run_as_module = False for opt, optarg in opts: if opt in ['-h', '--help']: print(_usage) sys.exit() elif opt in ['-c', '--command']: commands.append(optarg) + elif opt in ['-m']: + run_as_module = True mainpyfile = args[0] # Get script filename - if not os.path.exists(mainpyfile): + if not run_as_module and not os.path.exists(mainpyfile): print('Error:', mainpyfile, 'does not exist') sys.exit(1) sys.argv[:] = args # Hide "pdb.py" and pdb options from argument list # Replace pdb's dir with script's dir in front of module search path. - sys.path[0] = os.path.dirname(mainpyfile) + if not run_as_module: + sys.path[0] = os.path.dirname(mainpyfile) # Note on saving/restoring sys.argv: it's a good idea when sys.argv was # modified by the script being debugged. It's a bad idea when it was @@ -1667,7 +1691,10 @@ def main(): pdb.rcLines.extend(commands) while True: try: - pdb._runscript(mainpyfile) + if run_as_module: + pdb._runmodule(mainpyfile) + else: + pdb._runscript(mainpyfile) if pdb._user_requested_quit: break print("The program finished and will be restarted") diff --git a/Lib/pickle.py b/Lib/pickle.py index 350d4a46c06..e6d003787ba 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -183,6 +183,7 @@ __all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$", x)]) class _Framer: + _FRAME_SIZE_MIN = 4 _FRAME_SIZE_TARGET = 64 * 1024 def __init__(self, file_write): @@ -201,14 +202,25 @@ class _Framer: if self.current_frame: f = self.current_frame if f.tell() >= self._FRAME_SIZE_TARGET or force: - with f.getbuffer() as data: - n = len(data) - write = self.file_write - write(FRAME) - write(pack("= self._FRAME_SIZE_MIN: + # Issue a single call to the write method of the underlying + # file object for the frame opcode with the size of the + # frame. The concatenation is expected to be less expensive + # than issuing an additional call to write. + write(FRAME + pack("= 1 @@ -699,7 +727,9 @@ class _Pickler: if n <= 0xff: self.write(SHORT_BINBYTES + pack(" 0xffffffff and self.proto >= 4: - self.write(BINBYTES8 + pack("= self.framer._FRAME_SIZE_TARGET: + self._write_large_bytes(BINBYTES + pack("= 4: self.write(SHORT_BINUNICODE + pack(" 0xffffffff and self.proto >= 4: - self.write(BINUNICODE8 + pack("= self.framer._FRAME_SIZE_TARGET: + self._write_large_bytes(BINUNICODE + pack(" proto: proto = arg if pos == 0: - protoheader = p[pos: end_pos] + protoheader = p[pos:end_pos] else: opcodes.append((pos, end_pos)) else: @@ -2295,6 +2295,7 @@ def optimize(p): pickler.framer.start_framing() idx = 0 for op, arg in opcodes: + frameless = False if op is put: if arg not in newids: continue @@ -2305,8 +2306,12 @@ def optimize(p): data = pickler.get(newids[arg]) else: data = p[op:arg] - pickler.framer.commit_frame() - pickler.write(data) + frameless = len(data) > pickler.framer._FRAME_SIZE_TARGET + pickler.framer.commit_frame(force=frameless) + if frameless: + pickler.framer.file_write(data) + else: + pickler.write(data) pickler.framer.end_framing() return out.getvalue() diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index 9180eaed84d..8474a773e7c 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -46,7 +46,7 @@ def read_code(stream): if magic != importlib.util.MAGIC_NUMBER: return None - stream.read(8) # Skip timestamp and size + stream.read(12) # Skip rest of the header return marshal.load(stream) diff --git a/Lib/platform.py b/Lib/platform.py index cc2db9870d8..dc981ec144c 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -1202,9 +1202,6 @@ def _sys_version(sys_version=None): _, branch, revision = sys._git elif hasattr(sys, '_mercurial'): _, branch, revision = sys._mercurial - elif hasattr(sys, 'subversion'): - # sys.subversion was added in Python 2.5 - _, branch, revision = sys.subversion else: branch = '' revision = '' @@ -1259,7 +1256,7 @@ def python_branch(): """ Returns a string identifying the Python implementation branch. - For CPython this is the Subversion branch from which the + For CPython this is the SCM branch from which the Python binary was built. If not available, an empty string is returned. @@ -1273,7 +1270,7 @@ def python_revision(): """ Returns a string identifying the Python implementation revision. - For CPython this is the Subversion revision from which the + For CPython this is the SCM revision from which the Python binary was built. If not available, an empty string is returned. diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 2113a2dc57b..21ebec3f004 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -525,6 +525,8 @@ class InvalidFileException (ValueError): _BINARY_FORMAT = {1: 'B', 2: 'H', 4: 'L', 8: 'Q'} +_undefined = object() + class _BinaryPlistParser: """ Read or write a binary plist file, following the description of the binary @@ -555,7 +557,8 @@ class _BinaryPlistParser: ) = struct.unpack('>6xBBQQQ', trailer) self._fp.seek(offset_table_offset) self._object_offsets = self._read_ints(num_objects, offset_size) - return self._read_object(self._object_offsets[top_object]) + self._objects = [_undefined] * num_objects + return self._read_object(top_object) except (OSError, IndexError, struct.error, OverflowError, UnicodeDecodeError): @@ -584,62 +587,68 @@ class _BinaryPlistParser: def _read_refs(self, n): return self._read_ints(n, self._ref_size) - def _read_object(self, offset): + def _read_object(self, ref): """ - read the object at offset. + read the object by reference. May recursively read sub-objects (content of an array/dict/set) """ + result = self._objects[ref] + if result is not _undefined: + return result + + offset = self._object_offsets[ref] self._fp.seek(offset) token = self._fp.read(1)[0] tokenH, tokenL = token & 0xF0, token & 0x0F if token == 0x00: - return None + result = None elif token == 0x08: - return False + result = False elif token == 0x09: - return True + result = True # The referenced source code also mentions URL (0x0c, 0x0d) and # UUID (0x0e), but neither can be generated using the Cocoa libraries. elif token == 0x0f: - return b'' + result = b'' elif tokenH == 0x10: # int - return int.from_bytes(self._fp.read(1 << tokenL), - 'big', signed=tokenL >= 3) + result = int.from_bytes(self._fp.read(1 << tokenL), + 'big', signed=tokenL >= 3) elif token == 0x22: # real - return struct.unpack('>f', self._fp.read(4))[0] + result = struct.unpack('>f', self._fp.read(4))[0] elif token == 0x23: # real - return struct.unpack('>d', self._fp.read(8))[0] + result = struct.unpack('>d', self._fp.read(8))[0] elif token == 0x33: # date f = struct.unpack('>d', self._fp.read(8))[0] # timestamp 0 of binary plists corresponds to 1/1/2001 # (year of Mac OS X 10.0), instead of 1/1/1970. - return datetime.datetime(2001, 1, 1) + datetime.timedelta(seconds=f) + result = (datetime.datetime(2001, 1, 1) + + datetime.timedelta(seconds=f)) elif tokenH == 0x40: # data s = self._get_size(tokenL) if self._use_builtin_types: - return self._fp.read(s) + result = self._fp.read(s) else: - return Data(self._fp.read(s)) + result = Data(self._fp.read(s)) elif tokenH == 0x50: # ascii string s = self._get_size(tokenL) result = self._fp.read(s).decode('ascii') - return result + result = result elif tokenH == 0x60: # unicode string s = self._get_size(tokenL) - return self._fp.read(s * 2).decode('utf-16be') + result = self._fp.read(s * 2).decode('utf-16be') # tokenH == 0x80 is documented as 'UID' and appears to be used for # keyed-archiving, not in plists. @@ -647,8 +656,9 @@ class _BinaryPlistParser: elif tokenH == 0xA0: # array s = self._get_size(tokenL) obj_refs = self._read_refs(s) - return [self._read_object(self._object_offsets[x]) - for x in obj_refs] + result = [] + self._objects[ref] = result + result.extend(self._read_object(x) for x in obj_refs) # tokenH == 0xB0 is documented as 'ordset', but is not actually # implemented in the Apple reference code. @@ -661,12 +671,15 @@ class _BinaryPlistParser: key_refs = self._read_refs(s) obj_refs = self._read_refs(s) result = self._dict_type() + self._objects[ref] = result for k, o in zip(key_refs, obj_refs): - result[self._read_object(self._object_offsets[k]) - ] = self._read_object(self._object_offsets[o]) - return result + result[self._read_object(k)] = self._read_object(o) - raise InvalidFileException() + else: + raise InvalidFileException() + + self._objects[ref] = result + return result def _count_to_size(count): if count < 1 << 8: @@ -681,6 +694,8 @@ def _count_to_size(count): else: return 8 +_scalars = (str, int, float, datetime.datetime, bytes) + class _BinaryPlistWriter (object): def __init__(self, fp, sort_keys, skipkeys): self._fp = fp @@ -736,8 +751,7 @@ class _BinaryPlistWriter (object): # First check if the object is in the object table, not used for # containers to ensure that two subcontainers with the same contents # will be serialized as distinct values. - if isinstance(value, ( - str, int, float, datetime.datetime, bytes, bytearray)): + if isinstance(value, _scalars): if (type(value), value) in self._objtable: return @@ -745,15 +759,17 @@ class _BinaryPlistWriter (object): if (type(value.data), value.data) in self._objtable: return + elif id(value) in self._objidtable: + return + # Add to objectreference map refnum = len(self._objlist) self._objlist.append(value) - try: - if isinstance(value, Data): - self._objtable[(type(value.data), value.data)] = refnum - else: - self._objtable[(type(value), value)] = refnum - except TypeError: + if isinstance(value, _scalars): + self._objtable[(type(value), value)] = refnum + elif isinstance(value, Data): + self._objtable[(type(value.data), value.data)] = refnum + else: self._objidtable[id(value)] = refnum # And finally recurse into containers @@ -780,12 +796,11 @@ class _BinaryPlistWriter (object): self._flatten(o) def _getrefnum(self, value): - try: - if isinstance(value, Data): - return self._objtable[(type(value.data), value.data)] - else: - return self._objtable[(type(value), value)] - except TypeError: + if isinstance(value, _scalars): + return self._objtable[(type(value), value)] + elif isinstance(value, Data): + return self._objtable[(type(value.data), value.data)] + else: return self._objidtable[id(value)] def _write_size(self, token, size): diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 6dbdab27497..e92186c64e0 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -10,6 +10,18 @@ Some of this can actually be useful on non-Posix systems too, e.g. for manipulation of the pathname component of URLs. """ +# Strings representing various path-related bits and pieces. +# These are primarily for export; internally, they are hardcoded. +# Should be set before imports for resolving cyclic dependency. +curdir = '.' +pardir = '..' +extsep = '.' +sep = '/' +pathsep = ':' +defpath = ':/bin:/usr/bin' +altsep = None +devnull = '/dev/null' + import os import sys import stat @@ -25,16 +37,6 @@ __all__ = ["normcase","isabs","join","splitdrive","split","splitext", "devnull","realpath","supports_unicode_filenames","relpath", "commonpath"] -# Strings representing various path-related bits and pieces. -# These are primarily for export; internally, they are hardcoded. -curdir = '.' -pardir = '..' -extsep = '.' -sep = '/' -pathsep = ':' -defpath = ':/bin:/usr/bin' -altsep = None -devnull = '/dev/null' def _get_sep(path): if isinstance(path, bytes): diff --git a/Lib/pstats.py b/Lib/pstats.py index b7a20542a39..1b57d26b5a5 100644 --- a/Lib/pstats.py +++ b/Lib/pstats.py @@ -25,9 +25,32 @@ import os import time import marshal import re +from enum import Enum from functools import cmp_to_key -__all__ = ["Stats"] +__all__ = ["Stats", "SortKey"] + + +class SortKey(str, Enum): + CALLS = 'calls', 'ncalls' + CUMULATIVE = 'cumulative', 'cumtime' + FILENAME = 'filename', 'module' + LINE = 'line' + NAME = 'name' + NFL = 'nfl' + PCALLS = 'pcalls' + STDNAME = 'stdname' + TIME = 'time', 'tottime' + + def __new__(cls, *values): + obj = str.__new__(cls) + + obj._value_ = values[0] + for other_value in values[1:]: + cls._value2member_map_[other_value] = obj + obj._all_values = values + return obj + class Stats: """This class is used for creating reports from data generated by the @@ -49,13 +72,14 @@ class Stats: The sort_stats() method now processes some additional options (i.e., in addition to the old -1, 0, 1, or 2 that are respectively interpreted as - 'stdname', 'calls', 'time', and 'cumulative'). It takes an arbitrary number - of quoted strings to select the sort order. + 'stdname', 'calls', 'time', and 'cumulative'). It takes either an + arbitrary number of quoted strings or SortKey enum to select the sort + order. - For example sort_stats('time', 'name') sorts on the major key of 'internal - function time', and on the minor key of 'the name of the function'. Look at - the two tables in sort_stats() and get_sort_arg_defs(self) for more - examples. + For example sort_stats('time', 'name') or sort_stats(SortKey.TIME, + SortKey.NAME) sorts on the major key of 'internal function time', and on + the minor key of 'the name of the function'. Look at the two tables in + sort_stats() and get_sort_arg_defs(self) for more examples. All methods return self, so you can string together commands like: Stats('foo', 'goo').strip_dirs().sort_stats('calls').\ @@ -161,7 +185,6 @@ class Stats: "ncalls" : (((1,-1), ), "call count"), "cumtime" : (((3,-1), ), "cumulative time"), "cumulative": (((3,-1), ), "cumulative time"), - "file" : (((4, 1), ), "file name"), "filename" : (((4, 1), ), "file name"), "line" : (((5, 1), ), "line number"), "module" : (((4, 1), ), "file name"), @@ -202,12 +225,19 @@ class Stats: 0: "calls", 1: "time", 2: "cumulative"}[field[0]] ] + elif len(field) >= 2: + for arg in field[1:]: + if type(arg) != type(field[0]): + raise TypeError("Can't have mixed argument type") sort_arg_defs = self.get_sort_arg_defs() + sort_tuple = () self.sort_type = "" connector = "" for word in field: + if isinstance(word, SortKey): + word = word.value sort_tuple = sort_tuple + sort_arg_defs[word][0] self.sort_type += connector + sort_arg_defs[word][1] connector = ", " diff --git a/Lib/py_compile.py b/Lib/py_compile.py index 11c5b505cc6..16dc0a011ff 100644 --- a/Lib/py_compile.py +++ b/Lib/py_compile.py @@ -3,6 +3,7 @@ This module has intimate knowledge of the format of .pyc files. """ +import enum import importlib._bootstrap_external import importlib.machinery import importlib.util @@ -11,7 +12,7 @@ import os.path import sys import traceback -__all__ = ["compile", "main", "PyCompileError"] +__all__ = ["compile", "main", "PyCompileError", "PycInvalidationMode"] class PyCompileError(Exception): @@ -62,7 +63,14 @@ class PyCompileError(Exception): return self.msg -def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1): +class PycInvalidationMode(enum.Enum): + TIMESTAMP = 1 + CHECKED_HASH = 2 + UNCHECKED_HASH = 3 + + +def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1, + invalidation_mode=PycInvalidationMode.TIMESTAMP): """Byte-compile one Python source file to Python bytecode. :param file: The source file name. @@ -79,6 +87,7 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1): :param optimize: The optimization level for the compiler. Valid values are -1, 0, 1 and 2. A value of -1 means to use the optimization level of the current interpreter, as given by -O command line options. + :param invalidation_mode: :return: Path to the resulting byte compiled file. @@ -103,6 +112,8 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1): the resulting file would be regular and thus not the same type of file as it was previously. """ + if os.environ.get('SOURCE_DATE_EPOCH'): + invalidation_mode = PycInvalidationMode.CHECKED_HASH if cfile is None: if optimize >= 0: optimization = optimize if optimize >= 1 else '' @@ -136,9 +147,17 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1): os.makedirs(dirname) except FileExistsError: pass - source_stats = loader.path_stats(file) - bytecode = importlib._bootstrap_external._code_to_bytecode( + if invalidation_mode == PycInvalidationMode.TIMESTAMP: + source_stats = loader.path_stats(file) + bytecode = importlib._bootstrap_external._code_to_timestamp_pyc( code, source_stats['mtime'], source_stats['size']) + else: + source_hash = importlib.util.source_hash(source_bytes) + bytecode = importlib._bootstrap_external._code_to_hash_pyc( + code, + source_hash, + (invalidation_mode == PycInvalidationMode.CHECKED_HASH), + ) mode = importlib._bootstrap_external._calc_mode(file) importlib._bootstrap_external._write_atomic(cfile, bytecode, mode) return cfile diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 01f7a32f454..7d0b1d8aefb 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1952,7 +1952,7 @@ has the same effect as typing a particular string at the help> prompt. Welcome to Python {0}'s help utility! If this is your first time using Python, you should definitely check out -the tutorial on the Internet at http://docs.python.org/{0}/tutorial/. +the tutorial on the Internet at https://docs.python.org/{0}/tutorial/. Enter the name of any module, keyword, or topic to get help on writing Python programs and using Python modules. To quit this help utility and diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 2c52b548409..a979931e266 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Mon Oct 16 23:39:41 2017 +# Autogenerated by Sphinx on Tue Jan 30 18:36:07 2018 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -571,6 +571,77 @@ topics = {'assert': 'The "assert" statement\n' ' sorts it.\n' '\n' '\n' + 'Customizing module attribute access\n' + '===================================\n' + '\n' + 'Special names "__getattr__" and "__dir__" can be also ' + 'used to\n' + 'customize access to module attributes. The "__getattr__" ' + 'function at\n' + 'the module level should accept one argument which is the ' + 'name of an\n' + 'attribute and return the computed value or raise an ' + '"AttributeError".\n' + 'If an attribute is not found on a module object through ' + 'the normal\n' + 'lookup, i.e. "object.__getattribute__()", then ' + '"__getattr__" is\n' + 'searched in the module "__dict__" before raising an ' + '"AttributeError".\n' + 'If found, it is called with the attribute name and the ' + 'result is\n' + 'returned.\n' + '\n' + 'The "__dir__" function should accept no arguments, and ' + 'return a list\n' + 'of strings that represents the names accessible on ' + 'module. If present,\n' + 'this function overrides the standard "dir()" search on a ' + 'module.\n' + '\n' + 'For a more fine grained customization of the module ' + 'behavior (setting\n' + 'attributes, properties, etc.), one can set the ' + '"__class__" attribute\n' + 'of a module object to a subclass of "types.ModuleType". ' + 'For example:\n' + '\n' + ' import sys\n' + ' from types import ModuleType\n' + '\n' + ' class VerboseModule(ModuleType):\n' + ' def __repr__(self):\n' + " return f'Verbose {self.__name__}'\n" + '\n' + ' def __setattr__(self, attr, value):\n' + " print(f'Setting {attr}...')\n" + ' setattr(self, attr, value)\n' + '\n' + ' sys.modules[__name__].__class__ = VerboseModule\n' + '\n' + 'Note: Defining module "__getattr__" and setting module ' + '"__class__"\n' + ' only affect lookups made using the attribute access ' + 'syntax --\n' + ' directly accessing the module globals (whether by code ' + 'within the\n' + " module, or via a reference to the module's globals " + 'dictionary) is\n' + ' unaffected.\n' + '\n' + 'Changed in version 3.5: "__class__" module attribute is ' + 'now writable.\n' + '\n' + 'New in version 3.7: "__getattr__" and "__dir__" module ' + 'attributes.\n' + '\n' + 'See also:\n' + '\n' + ' **PEP 562** - Module __getattr__ and __dir__\n' + ' Describes the "__getattr__" and "__dir__" functions ' + 'on modules.\n' + '\n' + '\n' 'Implementing Descriptors\n' '========================\n' '\n' @@ -2453,17 +2524,23 @@ topics = {'assert': 'The "assert" statement\n' 'have\n' '"return" annotation of the form ""-> expression"" after the ' 'parameter\n' - 'list. These annotations can be any valid Python expression and ' - 'are\n' - 'evaluated when the function definition is executed. Annotations ' - 'may\n' - 'be evaluated in a different order than they appear in the source ' - 'code.\n' - 'The presence of annotations does not change the semantics of a\n' - 'function. The annotation values are available as values of a\n' - "dictionary keyed by the parameters' names in the " - '"__annotations__"\n' - 'attribute of the function object.\n' + 'list. These annotations can be any valid Python expression. ' + 'The\n' + 'presence of annotations does not change the semantics of a ' + 'function.\n' + 'The annotation values are available as values of a dictionary ' + 'keyed by\n' + 'the parameters\' names in the "__annotations__" attribute of ' + 'the\n' + 'function object. If the "annotations" import from "__future__" ' + 'is\n' + 'used, annotations are preserved as strings at runtime which ' + 'enables\n' + 'postponed evaluation. Otherwise, they are evaluated when the ' + 'function\n' + 'definition is executed. In this case annotations may be ' + 'evaluated in\n' + 'a different order than they appear in the source code.\n' '\n' 'It is also possible to create anonymous functions (functions not ' 'bound\n' @@ -2495,6 +2572,21 @@ topics = {'assert': 'The "assert" statement\n' ' **PEP 3107** - Function Annotations\n' ' The original specification for function annotations.\n' '\n' + ' **PEP 484** - Type Hints\n' + ' Definition of a standard meaning for annotations: type ' + 'hints.\n' + '\n' + ' **PEP 526** - Syntax for Variable Annotations\n' + ' Ability to type hint variable declarations, including ' + 'class\n' + ' variables and instance variables\n' + '\n' + ' **PEP 563** - Postponed Evaluation of Annotations\n' + ' Support for forward references within annotations by ' + 'preserving\n' + ' annotations in a string form at runtime instead of eager\n' + ' evaluation.\n' + '\n' '\n' 'Class definitions\n' '=================\n' @@ -2686,7 +2778,6 @@ topics = {'assert': 'The "assert" statement\n' ' mgr = (EXPR)\n' ' aexit = type(mgr).__aexit__\n' ' aenter = type(mgr).__aenter__(mgr)\n' - ' exc = True\n' '\n' ' VAR = await aenter\n' ' try:\n' @@ -2906,63 +2997,52 @@ topics = {'assert': 'The "assert" statement\n' '\n' ' Called when the instance is about to be destroyed. This ' 'is also\n' - ' called a destructor. If a base class has a "__del__()" ' - 'method, the\n' - ' derived class\'s "__del__()" method, if any, must ' - 'explicitly call it\n' - ' to ensure proper deletion of the base class part of the ' - 'instance.\n' - ' Note that it is possible (though not recommended!) for ' + ' called a finalizer or (improperly) a destructor. If a ' + 'base class\n' + ' has a "__del__()" method, the derived class\'s ' + '"__del__()" method,\n' + ' if any, must explicitly call it to ensure proper ' + 'deletion of the\n' + ' base class part of the instance.\n' + '\n' + ' It is possible (though not recommended!) for the ' + '"__del__()" method\n' + ' to postpone destruction of the instance by creating a ' + 'new reference\n' + ' to it. This is called object *resurrection*. It is\n' + ' implementation-dependent whether "__del__()" is called a ' + 'second\n' + ' time when a resurrected object is about to be destroyed; ' 'the\n' - ' "__del__()" method to postpone destruction of the ' - 'instance by\n' - ' creating a new reference to it. It may then be called ' - 'at a later\n' - ' time when this new reference is deleted. It is not ' - 'guaranteed that\n' - ' "__del__()" methods are called for objects that still ' - 'exist when\n' - ' the interpreter exits.\n' + ' current *CPython* implementation only calls it once.\n' + '\n' + ' It is not guaranteed that "__del__()" methods are called ' + 'for\n' + ' objects that still exist when the interpreter exits.\n' '\n' ' Note: "del x" doesn\'t directly call "x.__del__()" --- ' 'the former\n' ' decrements the reference count for "x" by one, and the ' 'latter is\n' - ' only called when "x"\'s reference count reaches zero. ' - 'Some common\n' - ' situations that may prevent the reference count of an ' - 'object from\n' - ' going to zero include: circular references between ' - 'objects (e.g.,\n' - ' a doubly-linked list or a tree data structure with ' - 'parent and\n' - ' child pointers); a reference to the object on the ' - 'stack frame of\n' - ' a function that caught an exception (the traceback ' - 'stored in\n' - ' "sys.exc_info()[2]" keeps the stack frame alive); or a ' + ' only called when "x"\'s reference count reaches zero.\n' + '\n' + ' **CPython implementation detail:** It is possible for a ' 'reference\n' - ' to the object on the stack frame that raised an ' - 'unhandled\n' - ' exception in interactive mode (the traceback stored ' - 'in\n' - ' "sys.last_traceback" keeps the stack frame alive). ' - 'The first\n' - ' situation can only be remedied by explicitly breaking ' - 'the cycles;\n' - ' the second can be resolved by freeing the reference to ' - 'the\n' - ' traceback object when it is no longer useful, and the ' - 'third can\n' - ' be resolved by storing "None" in "sys.last_traceback". ' - 'Circular\n' - ' references which are garbage are detected and cleaned ' - 'up when the\n' - " cyclic garbage collector is enabled (it's on by " - 'default). Refer\n' - ' to the documentation for the "gc" module for more ' - 'information\n' - ' about this topic.\n' + ' cycle to prevent the reference count of an object from ' + 'going to\n' + ' zero. In this case, the cycle will be later detected ' + 'and deleted\n' + ' by the *cyclic garbage collector*. A common cause of ' + 'reference\n' + ' cycles is when an exception has been caught in a local ' + 'variable.\n' + " The frame's locals then reference the exception, which " + 'references\n' + ' its own traceback, which references the locals of all ' + 'frames caught\n' + ' in the traceback.\n' + '\n' + ' See also: Documentation for the "gc" module.\n' '\n' ' Warning: Due to the precarious circumstances under ' 'which\n' @@ -2970,29 +3050,35 @@ topics = {'assert': 'The "assert" statement\n' 'during\n' ' their execution are ignored, and a warning is printed ' 'to\n' - ' "sys.stderr" instead. Also, when "__del__()" is ' - 'invoked in\n' - ' response to a module being deleted (e.g., when ' - 'execution of the\n' - ' program is done), other globals referenced by the ' + ' "sys.stderr" instead. In particular:\n' + '\n' + ' * "__del__()" can be invoked when arbitrary code is ' + 'being\n' + ' executed, including from any arbitrary thread. If ' '"__del__()"\n' - ' method may already have been deleted or in the process ' - 'of being\n' - ' torn down (e.g. the import machinery shutting down). ' - 'For this\n' - ' reason, "__del__()" methods should do the absolute ' - 'minimum needed\n' - ' to maintain external invariants. Starting with ' - 'version 1.5,\n' - ' Python guarantees that globals whose name begins with ' - 'a single\n' - ' underscore are deleted from their module before other ' - 'globals are\n' - ' deleted; if no other references to such globals exist, ' - 'this may\n' - ' help in assuring that imported modules are still ' - 'available at the\n' - ' time when the "__del__()" method is called.\n' + ' needs to take a lock or invoke any other blocking ' + 'resource, it\n' + ' may deadlock as the resource may already be taken by ' + 'the code\n' + ' that gets interrupted to execute "__del__()".\n' + '\n' + ' * "__del__()" can be executed during interpreter ' + 'shutdown. As\n' + ' a consequence, the global variables it needs to ' + 'access\n' + ' (including other modules) may already have been ' + 'deleted or set\n' + ' to "None". Python guarantees that globals whose name ' + 'begins\n' + ' with a single underscore are deleted from their ' + 'module before\n' + ' other globals are deleted; if no other references to ' + 'such\n' + ' globals exist, this may help in assuring that ' + 'imported modules\n' + ' are still available at the time when the "__del__()" ' + 'method is\n' + ' called.\n' '\n' 'object.__repr__(self)\n' '\n' @@ -3347,6 +3433,13 @@ topics = {'assert': 'The "assert" statement\n' 'executes\n' 'commands as if given in a ".pdbrc" file, see Debugger Commands.\n' '\n' + 'New in version 3.7: "pdb.py" now accepts a "-m" option that ' + 'execute\n' + 'modules similar to the way "python3 -m" does. As with a script, ' + 'the\n' + 'debugger will pause execution just before the first line of the\n' + 'module.\n' + '\n' 'The typical usage to break into the debugger from a running ' 'program is\n' 'to insert\n' @@ -3424,11 +3517,11 @@ topics = {'assert': 'The "assert" statement\n' ' hard-code a breakpoint at a given point in a program, even if ' 'the\n' ' code is not otherwise being debugged (e.g. when an assertion\n' - ' fails). If given, "header" is printed to the console just ' + ' fails). If given, *header* is printed to the console just ' 'before\n' ' debugging begins.\n' '\n' - ' New in version 3.7: The keyword-only argument "header".\n' + ' Changed in version 3.7: The keyword-only argument *header*.\n' '\n' 'pdb.post_mortem(traceback=None)\n' '\n' @@ -3688,33 +3781,30 @@ topics = {'assert': 'The "assert" statement\n' ' (com) end\n' ' (Pdb)\n' '\n' - ' To remove all commands from a breakpoint, type commands and ' - 'follow\n' - ' it immediately with "end"; that is, give no commands.\n' + ' To remove all commands from a breakpoint, type "commands" ' + 'and\n' + ' follow it immediately with "end"; that is, give no commands.\n' '\n' - ' With no *bpnumber* argument, commands refers to the last ' - 'breakpoint\n' - ' set.\n' + ' With no *bpnumber* argument, "commands" refers to the last\n' + ' breakpoint set.\n' '\n' ' You can use breakpoint commands to start your program up ' 'again.\n' - ' Simply use the continue command, or step, or any other ' - 'command that\n' - ' resumes execution.\n' + ' Simply use the "continue" command, or "step", or any other ' + 'command\n' + ' that resumes execution.\n' '\n' ' Specifying any command resuming execution (currently ' - 'continue,\n' - ' step, next, return, jump, quit and their abbreviations) ' - 'terminates\n' - ' the command list (as if that command was immediately followed ' - 'by\n' - ' end). This is because any time you resume execution (even ' - 'with a\n' - ' simple next or step), you may encounter another ' - 'breakpoint—which\n' - ' could have its own command list, leading to ambiguities about ' - 'which\n' - ' list to execute.\n' + '"continue",\n' + ' "step", "next", "return", "jump", "quit" and their ' + 'abbreviations)\n' + ' terminates the command "list" (as if that command was ' + 'immediately\n' + ' followed by end). This is because any time you resume ' + 'execution\n' + ' (even with a simple next or step), you may encounter another\n' + ' breakpoint—which could have its own command list, leading to\n' + ' ambiguities about which list to execute.\n' '\n' " If you use the 'silent' command in the command list, the " 'usual\n' @@ -5446,17 +5536,23 @@ topics = {'assert': 'The "assert" statement\n' 'have\n' '"return" annotation of the form ""-> expression"" after the ' 'parameter\n' - 'list. These annotations can be any valid Python expression and ' - 'are\n' - 'evaluated when the function definition is executed. Annotations ' - 'may\n' - 'be evaluated in a different order than they appear in the source ' - 'code.\n' - 'The presence of annotations does not change the semantics of a\n' - 'function. The annotation values are available as values of a\n' - "dictionary keyed by the parameters' names in the " - '"__annotations__"\n' - 'attribute of the function object.\n' + 'list. These annotations can be any valid Python expression. ' + 'The\n' + 'presence of annotations does not change the semantics of a ' + 'function.\n' + 'The annotation values are available as values of a dictionary ' + 'keyed by\n' + 'the parameters\' names in the "__annotations__" attribute of ' + 'the\n' + 'function object. If the "annotations" import from "__future__" ' + 'is\n' + 'used, annotations are preserved as strings at runtime which ' + 'enables\n' + 'postponed evaluation. Otherwise, they are evaluated when the ' + 'function\n' + 'definition is executed. In this case annotations may be ' + 'evaluated in\n' + 'a different order than they appear in the source code.\n' '\n' 'It is also possible to create anonymous functions (functions not ' 'bound\n' @@ -5486,7 +5582,22 @@ topics = {'assert': 'The "assert" statement\n' 'See also:\n' '\n' ' **PEP 3107** - Function Annotations\n' - ' The original specification for function annotations.\n', + ' The original specification for function annotations.\n' + '\n' + ' **PEP 484** - Type Hints\n' + ' Definition of a standard meaning for annotations: type ' + 'hints.\n' + '\n' + ' **PEP 526** - Syntax for Variable Annotations\n' + ' Ability to type hint variable declarations, including ' + 'class\n' + ' variables and instance variables\n' + '\n' + ' **PEP 563** - Postponed Evaluation of Annotations\n' + ' Support for forward references within annotations by ' + 'preserving\n' + ' annotations in a string form at runtime instead of eager\n' + ' evaluation.\n', 'global': 'The "global" statement\n' '**********************\n' '\n' @@ -5669,13 +5780,13 @@ topics = {'assert': 'The "assert" statement\n' 'They must\n' 'be spelled exactly as written here:\n' '\n' - ' False class finally is return\n' - ' None continue for lambda try\n' - ' True def from nonlocal while\n' - ' and del global not with\n' - ' as elif if or yield\n' - ' assert else import pass\n' - ' break except in raise\n' + ' False await else import pass\n' + ' None break except in raise\n' + ' True class finally is return\n' + ' and continue for lambda try\n' + ' as def from nonlocal while\n' + ' assert del global not with\n' + ' async elif if or yield\n' '\n' '\n' 'Reserved classes of identifiers\n' @@ -5959,11 +6070,16 @@ topics = {'assert': 'The "assert" statement\n' '\n' '* other future statements.\n' '\n' - 'The features recognized by Python 3.0 are "absolute_import",\n' - '"division", "generators", "unicode_literals", "print_function",\n' - '"nested_scopes" and "with_statement". They are all redundant ' - 'because\n' - 'they are always enabled, and only kept for backwards ' + 'The only feature in Python 3.7 that requires using the future\n' + 'statement is "annotations".\n' + '\n' + 'All historical features enabled by the future statement are still\n' + 'recognized by Python 3. The list includes "absolute_import",\n' + '"division", "generators", "generator_stop", "unicode_literals",\n' + '"print_function", "nested_scopes" and "with_statement". They are ' + 'all\n' + 'redundant because they are always enabled, and only kept for ' + 'backwards\n' 'compatibility.\n' '\n' 'A future statement is recognized and treated specially at compile\n' @@ -7564,91 +7680,87 @@ topics = {'assert': 'The "assert" statement\n' '\n' ' Called when the instance is about to be destroyed. This ' 'is also\n' - ' called a destructor. If a base class has a "__del__()" ' - 'method, the\n' - ' derived class\'s "__del__()" method, if any, must ' - 'explicitly call it\n' - ' to ensure proper deletion of the base class part of the ' - 'instance.\n' - ' Note that it is possible (though not recommended!) for ' + ' called a finalizer or (improperly) a destructor. If a ' + 'base class\n' + ' has a "__del__()" method, the derived class\'s ' + '"__del__()" method,\n' + ' if any, must explicitly call it to ensure proper deletion ' + 'of the\n' + ' base class part of the instance.\n' + '\n' + ' It is possible (though not recommended!) for the ' + '"__del__()" method\n' + ' to postpone destruction of the instance by creating a new ' + 'reference\n' + ' to it. This is called object *resurrection*. It is\n' + ' implementation-dependent whether "__del__()" is called a ' + 'second\n' + ' time when a resurrected object is about to be destroyed; ' 'the\n' - ' "__del__()" method to postpone destruction of the ' - 'instance by\n' - ' creating a new reference to it. It may then be called at ' - 'a later\n' - ' time when this new reference is deleted. It is not ' - 'guaranteed that\n' - ' "__del__()" methods are called for objects that still ' - 'exist when\n' - ' the interpreter exits.\n' + ' current *CPython* implementation only calls it once.\n' + '\n' + ' It is not guaranteed that "__del__()" methods are called ' + 'for\n' + ' objects that still exist when the interpreter exits.\n' '\n' ' Note: "del x" doesn\'t directly call "x.__del__()" --- ' 'the former\n' ' decrements the reference count for "x" by one, and the ' 'latter is\n' - ' only called when "x"\'s reference count reaches zero. ' - 'Some common\n' - ' situations that may prevent the reference count of an ' - 'object from\n' - ' going to zero include: circular references between ' - 'objects (e.g.,\n' - ' a doubly-linked list or a tree data structure with ' - 'parent and\n' - ' child pointers); a reference to the object on the stack ' - 'frame of\n' - ' a function that caught an exception (the traceback ' - 'stored in\n' - ' "sys.exc_info()[2]" keeps the stack frame alive); or a ' + ' only called when "x"\'s reference count reaches zero.\n' + '\n' + ' **CPython implementation detail:** It is possible for a ' 'reference\n' - ' to the object on the stack frame that raised an ' - 'unhandled\n' - ' exception in interactive mode (the traceback stored in\n' - ' "sys.last_traceback" keeps the stack frame alive). The ' - 'first\n' - ' situation can only be remedied by explicitly breaking ' - 'the cycles;\n' - ' the second can be resolved by freeing the reference to ' - 'the\n' - ' traceback object when it is no longer useful, and the ' - 'third can\n' - ' be resolved by storing "None" in "sys.last_traceback". ' - 'Circular\n' - ' references which are garbage are detected and cleaned ' - 'up when the\n' - " cyclic garbage collector is enabled (it's on by " - 'default). Refer\n' - ' to the documentation for the "gc" module for more ' - 'information\n' - ' about this topic.\n' + ' cycle to prevent the reference count of an object from ' + 'going to\n' + ' zero. In this case, the cycle will be later detected and ' + 'deleted\n' + ' by the *cyclic garbage collector*. A common cause of ' + 'reference\n' + ' cycles is when an exception has been caught in a local ' + 'variable.\n' + " The frame's locals then reference the exception, which " + 'references\n' + ' its own traceback, which references the locals of all ' + 'frames caught\n' + ' in the traceback.\n' + '\n' + ' See also: Documentation for the "gc" module.\n' '\n' ' Warning: Due to the precarious circumstances under which\n' ' "__del__()" methods are invoked, exceptions that occur ' 'during\n' ' their execution are ignored, and a warning is printed ' 'to\n' - ' "sys.stderr" instead. Also, when "__del__()" is invoked ' - 'in\n' - ' response to a module being deleted (e.g., when ' - 'execution of the\n' - ' program is done), other globals referenced by the ' + ' "sys.stderr" instead. In particular:\n' + '\n' + ' * "__del__()" can be invoked when arbitrary code is ' + 'being\n' + ' executed, including from any arbitrary thread. If ' '"__del__()"\n' - ' method may already have been deleted or in the process ' - 'of being\n' - ' torn down (e.g. the import machinery shutting down). ' - 'For this\n' - ' reason, "__del__()" methods should do the absolute ' - 'minimum needed\n' - ' to maintain external invariants. Starting with version ' - '1.5,\n' - ' Python guarantees that globals whose name begins with a ' - 'single\n' - ' underscore are deleted from their module before other ' - 'globals are\n' - ' deleted; if no other references to such globals exist, ' - 'this may\n' - ' help in assuring that imported modules are still ' - 'available at the\n' - ' time when the "__del__()" method is called.\n' + ' needs to take a lock or invoke any other blocking ' + 'resource, it\n' + ' may deadlock as the resource may already be taken by ' + 'the code\n' + ' that gets interrupted to execute "__del__()".\n' + '\n' + ' * "__del__()" can be executed during interpreter ' + 'shutdown. As\n' + ' a consequence, the global variables it needs to ' + 'access\n' + ' (including other modules) may already have been ' + 'deleted or set\n' + ' to "None". Python guarantees that globals whose name ' + 'begins\n' + ' with a single underscore are deleted from their ' + 'module before\n' + ' other globals are deleted; if no other references to ' + 'such\n' + ' globals exist, this may help in assuring that ' + 'imported modules\n' + ' are still available at the time when the "__del__()" ' + 'method is\n' + ' called.\n' '\n' 'object.__repr__(self)\n' '\n' @@ -8032,6 +8144,77 @@ topics = {'assert': 'The "assert" statement\n' ' sorts it.\n' '\n' '\n' + 'Customizing module attribute access\n' + '-----------------------------------\n' + '\n' + 'Special names "__getattr__" and "__dir__" can be also used ' + 'to\n' + 'customize access to module attributes. The "__getattr__" ' + 'function at\n' + 'the module level should accept one argument which is the ' + 'name of an\n' + 'attribute and return the computed value or raise an ' + '"AttributeError".\n' + 'If an attribute is not found on a module object through the ' + 'normal\n' + 'lookup, i.e. "object.__getattribute__()", then "__getattr__" ' + 'is\n' + 'searched in the module "__dict__" before raising an ' + '"AttributeError".\n' + 'If found, it is called with the attribute name and the ' + 'result is\n' + 'returned.\n' + '\n' + 'The "__dir__" function should accept no arguments, and ' + 'return a list\n' + 'of strings that represents the names accessible on module. ' + 'If present,\n' + 'this function overrides the standard "dir()" search on a ' + 'module.\n' + '\n' + 'For a more fine grained customization of the module behavior ' + '(setting\n' + 'attributes, properties, etc.), one can set the "__class__" ' + 'attribute\n' + 'of a module object to a subclass of "types.ModuleType". For ' + 'example:\n' + '\n' + ' import sys\n' + ' from types import ModuleType\n' + '\n' + ' class VerboseModule(ModuleType):\n' + ' def __repr__(self):\n' + " return f'Verbose {self.__name__}'\n" + '\n' + ' def __setattr__(self, attr, value):\n' + " print(f'Setting {attr}...')\n" + ' setattr(self, attr, value)\n' + '\n' + ' sys.modules[__name__].__class__ = VerboseModule\n' + '\n' + 'Note: Defining module "__getattr__" and setting module ' + '"__class__"\n' + ' only affect lookups made using the attribute access syntax ' + '--\n' + ' directly accessing the module globals (whether by code ' + 'within the\n' + " module, or via a reference to the module's globals " + 'dictionary) is\n' + ' unaffected.\n' + '\n' + 'Changed in version 3.5: "__class__" module attribute is now ' + 'writable.\n' + '\n' + 'New in version 3.7: "__getattr__" and "__dir__" module ' + 'attributes.\n' + '\n' + 'See also:\n' + '\n' + ' **PEP 562** - Module __getattr__ and __dir__\n' + ' Describes the "__getattr__" and "__dir__" functions on ' + 'modules.\n' + '\n' + '\n' 'Implementing Descriptors\n' '------------------------\n' '\n' @@ -9432,6 +9615,27 @@ topics = {'assert': 'The "assert" statement\n' ' formatting options that can be specified in format ' 'strings.\n' '\n' + ' Note: When formatting a number ("int", "float", "float" ' + 'and\n' + ' subclasses) with the "n" type (ex: ' + '"\'{:n}\'.format(1234)"), the\n' + ' function sets temporarily the "LC_CTYPE" locale to ' + 'the\n' + ' "LC_NUMERIC" locale to decode "decimal_point" and ' + '"thousands_sep"\n' + ' fields of "localeconv()" if they are non-ASCII or ' + 'longer than 1\n' + ' byte, and the "LC_NUMERIC" locale is different than ' + 'the\n' + ' "LC_CTYPE" locale. This temporary change affects ' + 'other threads.\n' + '\n' + ' Changed in version 3.7: When formatting a number with ' + 'the "n" type,\n' + ' the function sets temporarily the "LC_CTYPE" locale to ' + 'the\n' + ' "LC_NUMERIC" locale in some cases.\n' + '\n' 'str.format_map(mapping)\n' '\n' ' Similar to "str.format(**mapping)", except that ' @@ -9483,6 +9687,16 @@ topics = {'assert': 'The "assert" statement\n' 'Unicode\n' ' Standard.\n' '\n' + 'str.isascii()\n' + '\n' + ' Return true if the string is empty or all characters in ' + 'the string\n' + ' are ASCII, false otherwise. ASCII characters have code ' + 'points in\n' + ' the range U+0000-U+007F.\n' + '\n' + ' New in version 3.7.\n' + '\n' 'str.isdecimal()\n' '\n' ' Return true if all characters in the string are decimal ' @@ -12212,18 +12426,18 @@ topics = {'assert': 'The "assert" statement\n' ' sequence concatenation or repetition.\n' '\n' '8. "index" raises "ValueError" when *x* is not found in *s*. ' - 'When\n' - ' supported, the additional arguments to the index method ' - 'allow\n' - ' efficient searching of subsections of the sequence. Passing ' - 'the\n' - ' extra arguments is roughly equivalent to using ' - '"s[i:j].index(x)",\n' - ' only without copying any data and with the returned index ' - 'being\n' - ' relative to the start of the sequence rather than the start ' - 'of the\n' - ' slice.\n' + 'Not\n' + ' all implementations support passing the additional arguments ' + '*i*\n' + ' and *j*. These arguments allow efficient searching of ' + 'subsections\n' + ' of the sequence. Passing the extra arguments is roughly ' + 'equivalent\n' + ' to using "s[i:j].index(x)", only without copying any data and ' + 'with\n' + ' the returned index being relative to the start of the ' + 'sequence\n' + ' rather than the start of the slice.\n' '\n' '\n' 'Immutable Sequence Types\n' diff --git a/Lib/queue.py b/Lib/queue.py index c803b96deb0..ef07957781a 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -4,17 +4,26 @@ import threading from collections import deque from heapq import heappush, heappop from time import monotonic as time +try: + from _queue import SimpleQueue +except ImportError: + SimpleQueue = None -__all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue'] +__all__ = ['Empty', 'Full', 'Queue', 'PriorityQueue', 'LifoQueue', 'SimpleQueue'] -class Empty(Exception): - 'Exception raised by Queue.get(block=0)/get_nowait().' - pass + +try: + from _queue import Empty +except AttributeError: + class Empty(Exception): + 'Exception raised by Queue.get(block=0)/get_nowait().' + pass class Full(Exception): 'Exception raised by Queue.put(block=0)/put_nowait().' pass + class Queue: '''Create a queue object with a given maximum size. @@ -241,3 +250,72 @@ class LifoQueue(Queue): def _get(self): return self.queue.pop() + + +class _PySimpleQueue: + '''Simple, unbounded FIFO queue. + + This pure Python implementation is not reentrant. + ''' + # Note: while this pure Python version provides fairness + # (by using a threading.Semaphore which is itself fair, being based + # on threading.Condition), fairness is not part of the API contract. + # This allows the C version to use a different implementation. + + def __init__(self): + self._queue = deque() + self._count = threading.Semaphore(0) + + def put(self, item, block=True, timeout=None): + '''Put the item on the queue. + + The optional 'block' and 'timeout' arguments are ignored, as this method + never blocks. They are provided for compatibility with the Queue class. + ''' + self._queue.append(item) + self._count.release() + + def get(self, block=True, timeout=None): + '''Remove and return an item from the queue. + + If optional args 'block' is true and 'timeout' is None (the default), + block if necessary until an item is available. If 'timeout' is + a non-negative number, it blocks at most 'timeout' seconds and raises + the Empty exception if no item was available within that time. + Otherwise ('block' is false), return an item if one is immediately + available, else raise the Empty exception ('timeout' is ignored + in that case). + ''' + if timeout is not None and timeout < 0: + raise ValueError("'timeout' must be a non-negative number") + if not self._count.acquire(block, timeout): + raise Empty + return self._queue.popleft() + + def put_nowait(self, item): + '''Put an item into the queue without blocking. + + This is exactly equivalent to `put(item)` and is only provided + for compatibility with the Queue class. + ''' + return self.put(item, block=False) + + def get_nowait(self): + '''Remove and return an item from the queue without blocking. + + Only get an item if one is immediately available. Otherwise + raise the Empty exception. + ''' + return self.get(block=False) + + def empty(self): + '''Return True if the queue is empty, False otherwise (not reliable!).''' + return len(self._queue) == 0 + + def qsize(self): + '''Return the approximate size of the queue (not reliable!).''' + return len(self._queue) + + +if SimpleQueue is None: + SimpleQueue = _PySimpleQueue diff --git a/Lib/site.py b/Lib/site.py index 7dc1b041c19..950e7038a50 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -340,11 +340,6 @@ def getsitepackages(prefixes=None): else: sitepackages.append(prefix) sitepackages.append(os.path.join(prefix, "lib", "site-packages")) - # for framework builds *only* we add the standard Apple locations. - if sys.platform == "darwin" and sys._framework: - sitepackages.append( - os.path.join("/Library", sys._framework, - '%d.%d' % sys.version_info[:2], "site-packages")) return sitepackages def addsitepackages(known_paths, prefixes=None): diff --git a/Lib/smtplib.py b/Lib/smtplib.py index 5e422b704ad..b679875fd2c 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -933,6 +933,7 @@ class SMTP: from_addr = (msg[header_prefix + 'Sender'] if (header_prefix + 'Sender') in msg else msg[header_prefix + 'From']) + from_addr = email.utils.getaddresses([from_addr])[0][1] if to_addrs is None: addr_fields = [f for f in (msg[header_prefix + 'To'], msg[header_prefix + 'Bcc'], diff --git a/Lib/socket.py b/Lib/socket.py index 1ada24d3326..cfa605a22ad 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -136,11 +136,18 @@ class socket(_socket.socket): __slots__ = ["__weakref__", "_io_refs", "_closed"] - def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None): + def __init__(self, family=-1, type=-1, proto=-1, fileno=None): # For user code address family and type values are IntEnum members, but # for the underlying _socket.socket they're just integers. The # constructor of _socket.socket converts the given argument to an # integer automatically. + if fileno is None: + if family == -1: + family = AF_INET + if type == -1: + type = SOCK_STREAM + if proto == -1: + proto = 0 _socket.socket.__init__(self, family, type, proto, fileno) self._io_refs = 0 self._closed = False @@ -203,11 +210,7 @@ class socket(_socket.socket): For IP sockets, the address info is a pair (hostaddr, port). """ fd, addr = self._accept() - # If our type has the SOCK_NONBLOCK flag, we shouldn't pass it onto the - # new socket. We do not currently allow passing SOCK_NONBLOCK to - # accept4, so the returned socket is always blocking. - type = self.type & ~globals().get("SOCK_NONBLOCK", 0) - sock = socket(self.family, type, self.proto, fileno=fd) + sock = socket(self.family, self.type, self.proto, fileno=fd) # Issue #7995: if no default timeout is set and the listening # socket had a (non-zero) timeout, force the new socket in blocking # mode to override platform-specific socket flags inheritance. diff --git a/Lib/ssl.py b/Lib/ssl.py index 75caae0c440..b6161d0f178 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -115,6 +115,7 @@ except ImportError: from _ssl import HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN, HAS_TLSv1_3 +from _ssl import _DEFAULT_CIPHERS from _ssl import _OPENSSL_API_VERSION @@ -148,7 +149,6 @@ _IntEnum._convert( lambda name: name.startswith('CERT_'), source=_ssl) - PROTOCOL_SSLv23 = _SSLMethod.PROTOCOL_SSLv23 = _SSLMethod.PROTOCOL_TLS _PROTOCOL_NAMES = {value: name for name, value in _SSLMethod.__members__.items()} @@ -172,56 +172,15 @@ if _ssl.HAS_TLS_UNIQUE: else: CHANNEL_BINDING_TYPES = [] - -# Disable weak or insecure ciphers by default -# (OpenSSL's default setting is 'DEFAULT:!aNULL:!eNULL') -# Enable a better set of ciphers by default -# This list has been explicitly chosen to: -# * TLS 1.3 ChaCha20 and AES-GCM cipher suites -# * Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE) -# * Prefer ECDHE over DHE for better performance -# * Prefer AEAD over CBC for better performance and security -# * Prefer AES-GCM over ChaCha20 because most platforms have AES-NI -# (ChaCha20 needs OpenSSL 1.1.0 or patched 1.0.2) -# * Prefer any AES-GCM and ChaCha20 over any AES-CBC for better -# performance and security -# * Then Use HIGH cipher suites as a fallback -# * Disable NULL authentication, NULL encryption, 3DES and MD5 MACs -# for security reasons -_DEFAULT_CIPHERS = ( - 'TLS13-AES-256-GCM-SHA384:TLS13-CHACHA20-POLY1305-SHA256:' - 'TLS13-AES-128-GCM-SHA256:' - 'ECDH+AESGCM:ECDH+CHACHA20:DH+AESGCM:DH+CHACHA20:ECDH+AES256:DH+AES256:' - 'ECDH+AES128:DH+AES:ECDH+HIGH:DH+HIGH:RSA+AESGCM:RSA+AES:RSA+HIGH:' - '!aNULL:!eNULL:!MD5:!3DES' - ) - -# Restricted and more secure ciphers for the server side -# This list has been explicitly chosen to: -# * TLS 1.3 ChaCha20 and AES-GCM cipher suites -# * Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE) -# * Prefer ECDHE over DHE for better performance -# * Prefer AEAD over CBC for better performance and security -# * Prefer AES-GCM over ChaCha20 because most platforms have AES-NI -# * Prefer any AES-GCM and ChaCha20 over any AES-CBC for better -# performance and security -# * Then Use HIGH cipher suites as a fallback -# * Disable NULL authentication, NULL encryption, MD5 MACs, DSS, RC4, and -# 3DES for security reasons -_RESTRICTED_SERVER_CIPHERS = ( - 'TLS13-AES-256-GCM-SHA384:TLS13-CHACHA20-POLY1305-SHA256:' - 'TLS13-AES-128-GCM-SHA256:' - 'ECDH+AESGCM:ECDH+CHACHA20:DH+AESGCM:DH+CHACHA20:ECDH+AES256:DH+AES256:' - 'ECDH+AES128:DH+AES:ECDH+HIGH:DH+HIGH:RSA+AESGCM:RSA+AES:RSA+HIGH:' - '!aNULL:!eNULL:!MD5:!DSS:!RC4:!3DES' -) +HAS_NEVER_CHECK_COMMON_NAME = hasattr(_ssl, 'HOSTFLAG_NEVER_CHECK_SUBJECT') -class CertificateError(ValueError): - pass +_RESTRICTED_SERVER_CIPHERS = _DEFAULT_CIPHERS + +CertificateError = SSLCertVerificationError -def _dnsname_match(dn, hostname, max_wildcards=1): +def _dnsname_match(dn, hostname): """Matching according to RFC 6125, section 6.4.3 http://tools.ietf.org/html/rfc6125#section-6.4.3 @@ -233,7 +192,12 @@ def _dnsname_match(dn, hostname, max_wildcards=1): leftmost, *remainder = dn.split(r'.') wildcards = leftmost.count('*') - if wildcards > max_wildcards: + if wildcards == 1 and len(leftmost) > 1: + # Only match wildcard in leftmost segment. + raise CertificateError( + "wildcard can only be present in the leftmost segment: " + repr(dn)) + + if wildcards > 1: # Issue #17980: avoid denials of service by refusing more # than one wildcard per fragment. A survey of established # policy among SSL implementations showed it to be a @@ -389,8 +353,6 @@ class SSLContext(_SSLContext): def __new__(cls, protocol=PROTOCOL_TLS, *args, **kwargs): self = _SSLContext.__new__(cls, protocol) - if protocol != _SSLv2_IF_EXISTS: - self.set_ciphers(_DEFAULT_CIPHERS) return self def __init__(self, protocol=PROTOCOL_TLS): @@ -468,6 +430,23 @@ class SSLContext(_SSLContext): def options(self, value): super(SSLContext, SSLContext).options.__set__(self, value) + if hasattr(_ssl, 'HOSTFLAG_NEVER_CHECK_SUBJECT'): + @property + def hostname_checks_common_name(self): + ncs = self._host_flags & _ssl.HOSTFLAG_NEVER_CHECK_SUBJECT + return ncs != _ssl.HOSTFLAG_NEVER_CHECK_SUBJECT + + @hostname_checks_common_name.setter + def hostname_checks_common_name(self, value): + if value: + self._host_flags &= ~_ssl.HOSTFLAG_NEVER_CHECK_SUBJECT + else: + self._host_flags |= _ssl.HOSTFLAG_NEVER_CHECK_SUBJECT + else: + @property + def hostname_checks_common_name(self): + return True + @property def verify_flags(self): return VerifyFlags(super().verify_flags) @@ -509,8 +488,6 @@ def create_default_context(purpose=Purpose.SERVER_AUTH, *, cafile=None, # verify certs and host name in client mode context.verify_mode = CERT_REQUIRED context.check_hostname = True - elif purpose == Purpose.CLIENT_AUTH: - context.set_ciphers(_RESTRICTED_SERVER_CIPHERS) if cafile or capath or cadata: context.load_verify_locations(cafile, capath, cadata) @@ -694,11 +671,6 @@ class SSLObject: def do_handshake(self): """Start the SSL/TLS handshake.""" self._sslobj.do_handshake() - if self.context.check_hostname: - if not self.server_hostname: - raise ValueError("check_hostname needs server_hostname " - "argument") - match_hostname(self.getpeercert(), self.server_hostname) def unwrap(self): """Start the SSL shutdown handshake.""" @@ -781,17 +753,16 @@ class SSLSocket(socket): self.do_handshake_on_connect = do_handshake_on_connect self.suppress_ragged_eofs = suppress_ragged_eofs if sock is not None: - socket.__init__(self, - family=sock.family, - type=sock.type, - proto=sock.proto, - fileno=sock.fileno()) + super().__init__(family=sock.family, + type=sock.type, + proto=sock.proto, + fileno=sock.fileno()) self.settimeout(sock.gettimeout()) sock.detach() elif fileno is not None: - socket.__init__(self, fileno=fileno) + super().__init__(fileno=fileno) else: - socket.__init__(self, family=family, type=type, proto=proto) + super().__init__(family=family, type=type, proto=proto) # See if we are connected try: @@ -947,7 +918,7 @@ class SSLSocket(socket): self.__class__) return self._sslobj.write(data) else: - return socket.send(self, data, flags) + return super().send(data, flags) def sendto(self, data, flags_or_addr, addr=None): self._checkClosed() @@ -955,9 +926,9 @@ class SSLSocket(socket): raise ValueError("sendto not allowed on instances of %s" % self.__class__) elif addr is None: - return socket.sendto(self, data, flags_or_addr) + return super().sendto(data, flags_or_addr) else: - return socket.sendto(self, data, flags_or_addr, addr) + return super().sendto(data, flags_or_addr, addr) def sendmsg(self, *args, **kwargs): # Ensure programs don't send data unencrypted if they try to @@ -979,7 +950,7 @@ class SSLSocket(socket): v = self.send(byte_view[count:]) count += v else: - return socket.sendall(self, data, flags) + return super().sendall(data, flags) def sendfile(self, file, offset=0, count=None): """Send a file, possibly by using os.sendfile() if this is a @@ -1000,7 +971,7 @@ class SSLSocket(socket): self.__class__) return self.read(buflen) else: - return socket.recv(self, buflen, flags) + return super().recv(buflen, flags) def recv_into(self, buffer, nbytes=None, flags=0): self._checkClosed() @@ -1015,7 +986,7 @@ class SSLSocket(socket): self.__class__) return self.read(nbytes, buffer) else: - return socket.recv_into(self, buffer, nbytes, flags) + return super().recv_into(buffer, nbytes, flags) def recvfrom(self, buflen=1024, flags=0): self._checkClosed() @@ -1023,7 +994,7 @@ class SSLSocket(socket): raise ValueError("recvfrom not allowed on instances of %s" % self.__class__) else: - return socket.recvfrom(self, buflen, flags) + return super().recvfrom(buflen, flags) def recvfrom_into(self, buffer, nbytes=None, flags=0): self._checkClosed() @@ -1031,7 +1002,7 @@ class SSLSocket(socket): raise ValueError("recvfrom_into not allowed on instances of %s" % self.__class__) else: - return socket.recvfrom_into(self, buffer, nbytes, flags) + return super().recvfrom_into(buffer, nbytes, flags) def recvmsg(self, *args, **kwargs): raise NotImplementedError("recvmsg not allowed on instances of %s" % @@ -1051,7 +1022,7 @@ class SSLSocket(socket): def shutdown(self, how): self._checkClosed() self._sslobj = None - socket.shutdown(self, how) + super().shutdown(how) def unwrap(self): if self._sslobj: @@ -1063,7 +1034,7 @@ class SSLSocket(socket): def _real_close(self): self._sslobj = None - socket._real_close(self) + super()._real_close() def do_handshake(self, block=False): """Perform a TLS/SSL handshake.""" @@ -1088,10 +1059,10 @@ class SSLSocket(socket): session=self._session) try: if connect_ex: - rc = socket.connect_ex(self, addr) + rc = super().connect_ex(addr) else: rc = None - socket.connect(self, addr) + super().connect(addr) if not rc: self._connected = True if self.do_handshake_on_connect: @@ -1116,7 +1087,7 @@ class SSLSocket(socket): a tuple containing that new connection wrapped with a server-side SSL channel, and the address of the remote client.""" - newsock, addr = socket.accept(self) + newsock, addr = super().accept() newsock = self.context.wrap_socket(newsock, do_handshake_on_connect=self.do_handshake_on_connect, suppress_ragged_eofs=self.suppress_ragged_eofs, diff --git a/Lib/string.py b/Lib/string.py index a3e6d91bb4a..b9d6f5eb567 100644 --- a/Lib/string.py +++ b/Lib/string.py @@ -79,11 +79,11 @@ class Template(metaclass=_TemplateMetaclass): """A string class for supporting $-substitutions.""" delimiter = '$' - # r'[a-z]' matches to non-ASCII letters when used with IGNORECASE, - # but without ASCII flag. We can't add re.ASCII to flags because of - # backward compatibility. So we use local -i flag and [a-zA-Z] pattern. + # r'[a-z]' matches to non-ASCII letters when used with IGNORECASE, but + # without the ASCII flag. We can't add re.ASCII to flags because of + # backward compatibility. So we use the ?a local flag and [a-z] pattern. # See https://bugs.python.org/issue31672 - idpattern = r'(?-i:[_a-zA-Z][_a-zA-Z0-9]*)' + idpattern = r'(?a:[_a-z][_a-z0-9]*)' braceidpattern = None flags = _re.IGNORECASE diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 43be1f9bffa..2723bc9e427 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -128,12 +128,13 @@ if _mswindows: import _winapi class STARTUPINFO: def __init__(self, *, dwFlags=0, hStdInput=None, hStdOutput=None, - hStdError=None, wShowWindow=0): + hStdError=None, wShowWindow=0, lpAttributeList=None): self.dwFlags = dwFlags self.hStdInput = hStdInput self.hStdOutput = hStdOutput self.hStdError = hStdError self.wShowWindow = wShowWindow + self.lpAttributeList = lpAttributeList or {"handle_list": []} else: import _posixsubprocess import select @@ -241,7 +242,7 @@ def _optim_args_from_interpreter_flags(): def _args_from_interpreter_flags(): """Return a list of command-line arguments reproducing the current - settings in sys.flags and sys.warnoptions.""" + settings in sys.flags, sys.warnoptions and sys._xoptions.""" flag_opt_map = { 'debug': 'd', # 'inspect': 'i', @@ -260,8 +261,35 @@ def _args_from_interpreter_flags(): v = getattr(sys.flags, flag) if v > 0: args.append('-' + opt * v) - for opt in sys.warnoptions: + + # -W options + warnopts = sys.warnoptions[:] + bytes_warning = sys.flags.bytes_warning + xoptions = getattr(sys, '_xoptions', {}) + dev_mode = ('dev' in xoptions) + + if bytes_warning > 1: + warnopts.remove("error::BytesWarning") + elif bytes_warning: + warnopts.remove("default::BytesWarning") + if dev_mode: + warnopts.remove('default') + for opt in warnopts: args.append('-W' + opt) + + # -X options + if dev_mode: + args.extend(('-X', 'dev')) + for opt in ('faulthandler', 'tracemalloc', 'importtime', + 'showalloccount', 'showrefcount', 'utf8'): + if opt in xoptions: + value = xoptions[opt] + if value is True: + arg = opt + else: + arg = '%s=%s' % (opt, value) + args.extend(('-X', arg)) + return args @@ -276,9 +304,9 @@ def call(*popenargs, timeout=None, **kwargs): with Popen(*popenargs, **kwargs) as p: try: return p.wait(timeout=timeout) - except: + except: # Including KeyboardInterrupt, wait handled that. p.kill() - p.wait() + # We don't call p.wait() again as p.__exit__ does that for us. raise @@ -381,7 +409,8 @@ class CompletedProcess(object): self.stderr) -def run(*popenargs, input=None, timeout=None, check=False, **kwargs): +def run(*popenargs, + input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and @@ -414,6 +443,13 @@ def run(*popenargs, input=None, timeout=None, check=False, **kwargs): raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE + if capture_output: + if ('stdout' in kwargs) or ('stderr' in kwargs): + raise ValueError('stdout and stderr arguments may not be used ' + 'with capture_output.') + kwargs['stdout'] = PIPE + kwargs['stderr'] = PIPE + with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) @@ -422,9 +458,9 @@ def run(*popenargs, input=None, timeout=None, check=False, **kwargs): stdout, stderr = process.communicate() raise TimeoutExpired(process.args, timeout, output=stdout, stderr=stderr) - except: + except: # Including KeyboardInterrupt, communicate handled that. process.kill() - process.wait() + # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: @@ -550,9 +586,6 @@ def getoutput(cmd): return getstatusoutput(cmd)[1] -_PLATFORM_DEFAULT_CLOSE_FDS = object() - - class Popen(object): """ Execute a child program in a new process. @@ -603,7 +636,7 @@ class Popen(object): def __init__(self, args, bufsize=-1, executable=None, stdin=None, stdout=None, stderr=None, - preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS, + preexec_fn=None, close_fds=True, shell=False, cwd=None, env=None, universal_newlines=None, startupinfo=None, creationflags=0, restore_signals=True, start_new_session=False, @@ -628,21 +661,8 @@ class Popen(object): if preexec_fn is not None: raise ValueError("preexec_fn is not supported on Windows " "platforms") - any_stdio_set = (stdin is not None or stdout is not None or - stderr is not None) - if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: - if any_stdio_set: - close_fds = False - else: - close_fds = True - elif close_fds and any_stdio_set: - raise ValueError( - "close_fds is not supported on Windows platforms" - " if you redirect stdin/stdout/stderr") else: # POSIX - if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: - close_fds = True if pass_fds and not close_fds: warnings.warn("pass_fds overriding close_fds.", RuntimeWarning) close_fds = True @@ -702,6 +722,11 @@ class Popen(object): self.text_mode = encoding or errors or text or universal_newlines + # How long to resume waiting on a child after the first ^C. + # There is no right value for this. The purpose is to be polite + # yet remain good for interactive users trying to exit a tool. + self._sigint_wait_secs = 0.25 # 1/xkcd221.getRandomNumber() + self._closed_child_pipe_fds = False try: @@ -775,7 +800,7 @@ class Popen(object): def __enter__(self): return self - def __exit__(self, type, value, traceback): + def __exit__(self, exc_type, value, traceback): if self.stdout: self.stdout.close() if self.stderr: @@ -784,6 +809,22 @@ class Popen(object): if self.stdin: self.stdin.close() finally: + if exc_type == KeyboardInterrupt: + # https://bugs.python.org/issue25942 + # In the case of a KeyboardInterrupt we assume the SIGINT + # was also already sent to our child processes. We can't + # block indefinitely as that is not user friendly. + # If we have not already waited a brief amount of time in + # an interrupted .wait() or .communicate() call, do so here + # for consistency. + if self._sigint_wait_secs > 0: + try: + self._wait(timeout=self._sigint_wait_secs) + except TimeoutExpired: + pass + self._sigint_wait_secs = 0 # Note that this has been done. + return # resume the KeyboardInterrupt + # Wait for the process to terminate, to avoid zombies. self.wait() @@ -792,7 +833,7 @@ class Popen(object): # We didn't get to successfully create a child process. return if self.returncode is None: - # Not reading subprocess exit status creates a zombi process which + # Not reading subprocess exit status creates a zombie process which # is only destroyed at the parent python process exit _warn("subprocess %s is still running" % self.pid, ResourceWarning, source=self) @@ -877,6 +918,21 @@ class Popen(object): try: stdout, stderr = self._communicate(input, endtime, timeout) + except KeyboardInterrupt: + # https://bugs.python.org/issue25942 + # See the detailed comment in .wait(). + if timeout is not None: + sigint_timeout = min(self._sigint_wait_secs, + self._remaining_time(endtime)) + else: + sigint_timeout = self._sigint_wait_secs + self._sigint_wait_secs = 0 # nothing else should wait. + try: + self._wait(timeout=sigint_timeout) + except TimeoutExpired: + pass + raise # resume the KeyboardInterrupt + finally: self._communication_started = True @@ -907,6 +963,30 @@ class Popen(object): raise TimeoutExpired(self.args, orig_timeout) + def wait(self, timeout=None): + """Wait for child process to terminate; returns self.returncode.""" + if timeout is not None: + endtime = _time() + timeout + try: + return self._wait(timeout=timeout) + except KeyboardInterrupt: + # https://bugs.python.org/issue25942 + # The first keyboard interrupt waits briefly for the child to + # exit under the common assumption that it also received the ^C + # generated SIGINT and will exit rapidly. + if timeout is not None: + sigint_timeout = min(self._sigint_wait_secs, + self._remaining_time(endtime)) + else: + sigint_timeout = self._sigint_wait_secs + self._sigint_wait_secs = 0 # nothing else should wait. + try: + self._wait(timeout=sigint_timeout) + except TimeoutExpired: + pass + raise # resume the KeyboardInterrupt + + if _mswindows: # # Windows methods @@ -992,6 +1072,19 @@ class Popen(object): return Handle(h) + def _filter_handle_list(self, handle_list): + """Filter out console handles that can't be used + in lpAttributeList["handle_list"] and make sure the list + isn't empty. This also removes duplicate handles.""" + # An handle with it's lowest two bits set might be a special console + # handle that if passed in lpAttributeList["handle_list"], will + # cause it to fail. + return list({handle for handle in handle_list + if handle & 0x3 != 0x3 + or _winapi.GetFileType(handle) != + _winapi.FILE_TYPE_CHAR}) + + def _execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, @@ -1004,17 +1097,51 @@ class Popen(object): assert not pass_fds, "pass_fds not supported on Windows." if not isinstance(args, str): - args = list2cmdline(args) + try: + args = os.fsdecode(args) # os.PathLike -> str + except TypeError: # not an os.PathLike, must be a sequence. + args = list(args) + args[0] = os.fsdecode(args[0]) # os.PathLike -> str + args = list2cmdline(args) # Process startup details if startupinfo is None: startupinfo = STARTUPINFO() - if -1 not in (p2cread, c2pwrite, errwrite): + + use_std_handles = -1 not in (p2cread, c2pwrite, errwrite) + if use_std_handles: startupinfo.dwFlags |= _winapi.STARTF_USESTDHANDLES startupinfo.hStdInput = p2cread startupinfo.hStdOutput = c2pwrite startupinfo.hStdError = errwrite + attribute_list = startupinfo.lpAttributeList + have_handle_list = bool(attribute_list and + "handle_list" in attribute_list and + attribute_list["handle_list"]) + + # If we were given an handle_list or need to create one + if have_handle_list or (use_std_handles and close_fds): + if attribute_list is None: + attribute_list = startupinfo.lpAttributeList = {} + handle_list = attribute_list["handle_list"] = \ + list(attribute_list.get("handle_list", [])) + + if use_std_handles: + handle_list += [int(p2cread), int(c2pwrite), int(errwrite)] + + handle_list[:] = self._filter_handle_list(handle_list) + + if handle_list: + if not close_fds: + warnings.warn("startupinfo.lpAttributeList['handle_list'] " + "overriding close_fds", RuntimeWarning) + + # When using the handle_list we always request to inherit + # handles but the only handles that will be inherited are + # the ones in the handle_list + close_fds = False + if shell: startupinfo.dwFlags |= _winapi.STARTF_USESHOWWINDOW startupinfo.wShowWindow = _winapi.SW_HIDE @@ -1073,16 +1200,16 @@ class Popen(object): return self.returncode - def wait(self, timeout=None): - """Wait for child process to terminate. Returns returncode - attribute.""" + def _wait(self, timeout): + """Internal implementation of wait() on Windows.""" if timeout is None: timeout_millis = _winapi.INFINITE else: timeout_millis = int(timeout * 1000) if self.returncode is None: + # API note: Returns immediately if timeout_millis == 0. result = _winapi.WaitForSingleObject(self._handle, - timeout_millis) + timeout_millis) if result == _winapi.WAIT_TIMEOUT: raise TimeoutExpired(self.args, timeout) self.returncode = _winapi.GetExitCodeProcess(self._handle) @@ -1247,7 +1374,10 @@ class Popen(object): if isinstance(args, (str, bytes)): args = [args] else: - args = list(args) + try: + args = list(args) + except TypeError: # os.PathLike instead of a sequence? + args = [os.fsencode(args)] # os.PathLike -> [str] if shell: # On Android the default shell is at '/system/bin/sh'. @@ -1444,9 +1574,8 @@ class Popen(object): return (pid, sts) - def wait(self, timeout=None): - """Wait for child process to terminate. Returns returncode - attribute.""" + def _wait(self, timeout): + """Internal implementation of wait() on POSIX.""" if self.returncode is not None: return self.returncode diff --git a/Lib/tarfile.py b/Lib/tarfile.py index efc1f3b9f7f..a24ee42abf8 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1058,7 +1058,7 @@ class TarInfo(object): # The old GNU sparse format occupies some of the unused # space in the buffer for up to 4 sparse structures. - # Save the them for later processing in _proc_sparse(). + # Save them for later processing in _proc_sparse(). if obj.type == GNUTYPE_SPARSE: pos = 386 structs = [] @@ -1943,7 +1943,7 @@ class TarFile(object): elif tarinfo.isdir(): self.addfile(tarinfo) if recursive: - for f in os.listdir(name): + for f in sorted(os.listdir(name)): self.add(os.path.join(name, f), os.path.join(arcname, f), recursive, filter=filter) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index dbca2d89ed1..1e497a572a7 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -1029,6 +1029,43 @@ class _TestQueue(BaseTestCase): self.assertTrue(q.get(timeout=1.0)) close_queue(q) + def test_queue_feeder_on_queue_feeder_error(self): + # bpo-30006: verify feeder handles exceptions using the + # _on_queue_feeder_error hook. + if self.TYPE != 'processes': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + class NotSerializable(object): + """Mock unserializable object""" + def __init__(self): + self.reduce_was_called = False + self.on_queue_feeder_error_was_called = False + + def __reduce__(self): + self.reduce_was_called = True + raise AttributeError + + class SafeQueue(multiprocessing.queues.Queue): + """Queue with overloaded _on_queue_feeder_error hook""" + @staticmethod + def _on_queue_feeder_error(e, obj): + if (isinstance(e, AttributeError) and + isinstance(obj, NotSerializable)): + obj.on_queue_feeder_error_was_called = True + + not_serializable_obj = NotSerializable() + # The captured_stderr reduces the noise in the test report + with test.support.captured_stderr(): + q = SafeQueue(ctx=multiprocessing.get_context()) + q.put(not_serializable_obj) + + # Verify that q is still functionning correctly + q.put(True) + self.assertTrue(q.get(timeout=1.0)) + + # Assert that the serialization and the hook have been called correctly + self.assertTrue(not_serializable_obj.reduce_was_called) + self.assertTrue(not_serializable_obj.on_queue_feeder_error_was_called) # # # @@ -4115,7 +4152,7 @@ class TestNoForkBomb(unittest.TestCase): # class TestForkAwareThreadLock(unittest.TestCase): - # We recurisvely start processes. Issue #17555 meant that the + # We recursively start processes. Issue #17555 meant that the # after fork registry would get duplicate entries for the same # lock. The size of the registry at generation n was ~2**n. @@ -4167,7 +4204,7 @@ class TestCloseFds(unittest.TestCase): def close(self, fd): if WIN32: - socket.socket(fileno=fd).close() + socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd).close() else: os.close(fd) @@ -4365,7 +4402,7 @@ class TestSemaphoreTracker(unittest.TestCase): ''' r, w = os.pipe() p = subprocess.Popen([sys.executable, - '-c', cmd % (w, w)], + '-E', '-c', cmd % (w, w)], pass_fds=[w], stderr=subprocess.PIPE) os.close(w) diff --git a/Lib/test/allsans.pem b/Lib/test/allsans.pem index 3ee4f59513a..bf59f30abaa 100644 --- a/Lib/test/allsans.pem +++ b/Lib/test/allsans.pem @@ -1,37 +1,64 @@ -----BEGIN PRIVATE KEY----- -MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAOoy7/QOtTjQ0niE -6uDcTwtkC0R2Tvy1AjVnXohCntZfdzbTGDoYTgXSOLsP8A697jUiJ8VCePGH50xG -Z4DKnAF3a9O3a9nr2pLXb0iY3XOMv+YEBii7CfI+3oxFYgCl0sMgHzDD2ZTVYAsm -DWgLUVsE2gHEccRwrM2tPf2EgR+FAgMBAAECgYEA3qyfyYVSeTrTYxO93x6ZaVMu -A2IZp9zSxMQL9bKiI2GRj+cV2ebSCGbg2btFnD6qBor7FWsmYz+8g6FNN/9sY4az -61rMqMtQvLBe+7L8w70FeTze4qQ4Y1oQri0qD6tBWhDVlpnbI5Py9bkZKD67yVUk -elcEA/5x4PrYXkuqsAECQQD80NjT0mDvaY0JOOaQFSEpMv6QiUA8GGX8Xli7IoKb -tAolPG8rQBa+qSpcWfDMTrWw/aWHuMEEQoP/bVDH9W4FAkEA7SYQbBAKnojZ5A3G -kOHdV7aeivRQxQk/JN8Fb8oKB9Csvpv/BsuGxPKXHdhFa6CBTTsNRtHQw/szPo4l -xMIjgQJAPoMxqibR+0EBM6+TKzteSL6oPXsCnBl4Vk/J5vPgkbmR7KUl4+7j8N8J -b2554TrxKEN/w7CGYZRE6UrRd7ATNQJAWD7Yz41sli+wfPdPU2xo1BHljyl4wMk/ -EPZYbI/PCbdyAH/F935WyQTIjNeEhZc1Zkq6FwdOWw8ns3hrv3rKgQJAHXv1BqUa -czGPIFxX2TNoqtcl6/En4vrxVB1wzsfzkkDAg98kBl7qsF+S3qujSzKikjeaVbI2 -/CyWR2P3yLtOmA== +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQD59JyhPgYe7nhZ +Z2IGhaklNgtRkD+5BVs7lEWovYRBlXpPA6PuaHat25rI8EGYHmlufPherg2Qu6sC +GmZZKo7TgjlmDcwVS4hkebtFH7OZy5Il7Y2ZIdiK7Xp9Z0EPoqwacYowB0a8WhZY +I2Vm4EzCNKl6/htkwjgn2JXGizxvGt/1kNqP/GBAX+vjgeahOsn8jVh96KpFHJbS +g83cX4t8M7FJv7yNoDLvORHnvKCOXbQmr6ZMGcZN8PwS8awQ31khZTpEx+hCe+Pi +GzeOlxpZimXWDAGWA4tZ58Ka/QvO7VQbD5Ci166ODvvs+tEXfBUExtPcS+02IBJV +tzhBna9VAgMBAAECggEAPar9DccIqY76QEyCYcuOPLEFv9zP6+0HYj6lpQkE3U1s +vJvQURyS0zgQCy1Dca1nI6xPdsSIckHq4fzzbWJTlJlXYfdbd5GIGAn0iwxUOkiA +ST0/px0zmKsYgmH8KkhfH7MNfeX9rLCpPJuXA/eo2G03tzGEPqqwQhxsb2ygv2Qs +M7OqJz6RJu87K1Y+psWIv9+VhNVja0kvsg52QMK9mtp8layb54qLI5R5e09sIudq +RHegtnSOBo9kt32H9vWUFaF5PpYt4yks4KYI4ulKGWJGXHMDW4uHUaE/tjNQuYAX +DuDvjN+ECSJvigiUbu2k0xB2KYIb1fpcxlz/YBdADQKBgQD/Z2VtBUjOFnJKz00f +xN0akp7XPgd1yCb1/wZq9PQiGvzIAMDIplioTvjOjhOzPJaWD0GICNeypzQ48+0P +UsPIKbazpIZN6bZncr65plSpg0KANq46hbkPHOo8PHDa7yoxBUSPr8F7P1OCRkn6 ++QdgcnrAly7yfqO2ahAWOX7iCwKBgQD6ifXSCKfRF1GUb3Ws7S1rLxeBasWq+EmC +sUnck0S+AyaMkN+kZ5zejbN+NDuUMQ7+3wUIheTclUhzR0LP3+r5jjHsimJuvOml +wuV37F+Om5lD/Xx27NfbtRKn/bK6o0zDL8JB2eFB0N7Fh7hRYoUMdrpQs5sU91IC +pNYlAcLwHwKBgGvLK9eTf2LbvmksjRR3dgodD8UwfN2NGESC2iaSM+ehFEclajhF +XO3MRt6GwHHJhJTY44OSl9bjEvtmmAr7l34HfQDc04JWvZFzsGOSe/D/YTXT3jz8 +61ohjgrWR5tfjaMa4hDy0Oo/k/NLzzWJnT9rkbtvE3VtVZNLuHZo1dB5AoGBAMHO +wStV6MO1nzUNN+Gqo8zbY/qIJxsH8I26KaIJBk9azpJEa8yZHl+HDEffjgsoHCqL +STB7qzv7+0y53nRCClo8ZmBN+LEjUDcbWjl3z7/YnCpdR9ATjTP3kdQETCNWucXw +Bvy72CX6tqnlQG8soDGxEpXlKl2AqJ9E9icwgqUPAoGAL6xTDdgcYTbk9wxCd41l +NhHTSvLrGXLAzv61PCnlOJEJbuuezb2VW0ibsud5CA4Mi0tf9ET790XSOFd5nCjQ +6rr06AkjQsoFvjL1dO9EzVFPW0JrZ3C9y8ZOjdeAfPEmFL2T6VqmQ+IcCUNhSr39 +NBdKrboEFfnKanfbstekhAs= -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIIDcjCCAtugAwIBAgIJAN5dc9TOWjB7MA0GCSqGSIb3DQEBCwUAMF0xCzAJBgNV +MIIGMDCCBRigAwIBAgIJAJYf8T95ptq5MA0GCSqGSIb3DQEBCwUAMF0xCzAJBgNV BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2FsbHNhbnMwHhcNMTYwODA1 -MTAyMTExWhcNMjYwODAzMTAyMTExWjBdMQswCQYDVQQGEwJYWTEXMBUGA1UEBwwO +IFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2FsbHNhbnMwHhcNMTgwMTE5 +MTkwOTA3WhcNMjgwMTE3MTkwOTA3WjBdMQswCQYDVQQGEwJYWTEXMBUGA1UEBwwO Q2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0 -aW9uMRAwDgYDVQQDDAdhbGxzYW5zMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB -gQDqMu/0DrU40NJ4hOrg3E8LZAtEdk78tQI1Z16IQp7WX3c20xg6GE4F0ji7D/AO -ve41IifFQnjxh+dMRmeAypwBd2vTt2vZ69qS129ImN1zjL/mBAYouwnyPt6MRWIA -pdLDIB8ww9mU1WALJg1oC1FbBNoBxHHEcKzNrT39hIEfhQIDAQABo4IBODCCATQw -ggEwBgNVHREEggEnMIIBI4IHYWxsc2Fuc6AeBgMqAwSgFwwVc29tZSBvdGhlciBp -ZGVudGlmaWVyoDUGBisGAQUCAqArMCmgEBsOS0VSQkVST1MuUkVBTE2hFTAToAMC -AQGhDDAKGwh1c2VybmFtZYEQdXNlckBleGFtcGxlLm9yZ4IPd3d3LmV4YW1wbGUu -b3JnpGcwZTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMw -IQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEYMBYGA1UEAwwPZGly -bmFtZSBleGFtcGxlhhdodHRwczovL3d3dy5weXRob24ub3JnL4cEfwAAAYcQAAAA -AAAAAAAAAAAAAAAAAYgEKgMEBTANBgkqhkiG9w0BAQsFAAOBgQAy16h+F+nOmeiT -VWR0fc8F/j6FcadbLseAUaogcC15OGxCl4UYpLV88HBkABOoGCpP155qwWTwOrdG -iYPGJSusf1OnJEbvzFejZf6u078bPd9/ZL4VWLjv+FPGkjd+N+/OaqMvgj8Lu99f -3Y/C4S7YbHxxwff6C6l2Xli+q6gnuQ== +aW9uMRAwDgYDVQQDDAdhbGxzYW5zMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEA+fScoT4GHu54WWdiBoWpJTYLUZA/uQVbO5RFqL2EQZV6TwOj7mh2rdua +yPBBmB5pbnz4Xq4NkLurAhpmWSqO04I5Zg3MFUuIZHm7RR+zmcuSJe2NmSHYiu16 +fWdBD6KsGnGKMAdGvFoWWCNlZuBMwjSpev4bZMI4J9iVxos8bxrf9ZDaj/xgQF/r +44HmoTrJ/I1YfeiqRRyW0oPN3F+LfDOxSb+8jaAy7zkR57ygjl20Jq+mTBnGTfD8 +EvGsEN9ZIWU6RMfoQnvj4hs3jpcaWYpl1gwBlgOLWefCmv0Lzu1UGw+Qoteujg77 +7PrRF3wVBMbT3EvtNiASVbc4QZ2vVQIDAQABo4IC8TCCAu0wggEwBgNVHREEggEn +MIIBI4IHYWxsc2Fuc6AeBgMqAwSgFwwVc29tZSBvdGhlciBpZGVudGlmaWVyoDUG +BisGAQUCAqArMCmgEBsOS0VSQkVST1MuUkVBTE2hFTAToAMCAQGhDDAKGwh1c2Vy +bmFtZYEQdXNlckBleGFtcGxlLm9yZ4IPd3d3LmV4YW1wbGUub3JnpGcwZTELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEYMBYGA1UEAwwPZGlybmFtZSBleGFtcGxl +hhdodHRwczovL3d3dy5weXRob24ub3JnL4cEfwAAAYcQAAAAAAAAAAAAAAAAAAAA +AYgEKgMEBTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsG +AQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFH9ye3+WhBnHqNhtFu059bzY +SWM8MIGPBgNVHSMEgYcwgYSAFH9ye3+WhBnHqNhtFu059bzYSWM8oWGkXzBdMQsw +CQYDVQQGEwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5 +dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRAwDgYDVQQDDAdhbGxzYW5zggkAlh/x +P3mm2rkwgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rlc3Rj +YS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUFBzAB +hilodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBDBgNV +HR8EPDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3Rj +YS9yZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAQEAYwYJcerUPvnsP7e2 +HGp/It0OZ8Cvpt8Qf7A+NSPvJqkyKakl8zK/50iq/qQKH09CnfEae4rfXLdlYsvV +2PZYK0LDWnyTcHSJWAVJjlSFIFt3ig9FdHv9GYtSWWod66cZ0sEZOoF2IHZUGby+ +Qa+JQpmv5jEuGIZzjcsh6hSOou8ph7LsCsRdVlQqk8rM97vB7DAgh01vedlbolsq +JxsuPRydNFV/eWq3AgAWgZL3LdYYIAgaVOTnnd3xARw8DlT1q6+Lzc71GBXrRZYh +qgd+xC/K1812gMPImTX02bxpkhCuIdVd7cztWi8sdQmSgDEFdYMXo4NzlFTK8dlC +Y4wa3Q== -----END CERTIFICATE----- diff --git a/Lib/test/bad_getattr.py b/Lib/test/bad_getattr.py new file mode 100644 index 00000000000..16f901b13b8 --- /dev/null +++ b/Lib/test/bad_getattr.py @@ -0,0 +1,4 @@ +x = 1 + +__getattr__ = "Surprise!" +__dir__ = "Surprise again!" diff --git a/Lib/test/bad_getattr2.py b/Lib/test/bad_getattr2.py new file mode 100644 index 00000000000..0a52a53b54b --- /dev/null +++ b/Lib/test/bad_getattr2.py @@ -0,0 +1,7 @@ +def __getattr__(): + "Bad one" + +x = 1 + +def __dir__(bad_sig): + return [] diff --git a/Lib/test/bad_getattr3.py b/Lib/test/bad_getattr3.py new file mode 100644 index 00000000000..0d5f9266c71 --- /dev/null +++ b/Lib/test/bad_getattr3.py @@ -0,0 +1,5 @@ +def __getattr__(name): + if name != 'delgetattr': + raise AttributeError + del globals()['__getattr__'] + raise AttributeError diff --git a/Lib/test/capath/b1930218.0 b/Lib/test/capath/b1930218.0 index 373349cae05..07556ff9071 100644 --- a/Lib/test/capath/b1930218.0 +++ b/Lib/test/capath/b1930218.0 @@ -1,21 +1,21 @@ -----BEGIN CERTIFICATE----- -MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +MIIDbTCCAlWgAwIBAgIJAILtv0HIgJGbMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx -OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDZaFw0yODAxMTcx +OTA5MDZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI -hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV -q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/ -AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA -Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni -0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx -6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w -HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2 -2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB -AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4 -QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1 -Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O -JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR -f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf -9mmvtk57HVjsO6lTo15YyJ4= +hvcNAQEBBQADggEPADCCAQoCggEBAMMYaWvJRymYjrFWwi76Dl68I4CzB2Ik0kJb +8Uq/qcghdcjj5iwfhzxufBvtOTKVt0CyYEjDmhYI/m1niDQ7d3dwHHBa0R9fBCFU +uQzjQYUdWO4v7fMO79gjofpz+0wo4OXmTQsCUkmGx75+veZWdotwjgqPBjMgHXtb +qtDFG6ubzFQJPL/kQGbx+9b3Fp3EGdTD8v8HvG9aniUbAkql7EKWOnDSbJkrzr7o +0gHv1bqwz5Q+gtAB1ktxgAMKEkWGeYHYS9LotbcsbJpMihAQ5OT1386EkcrRRuCE +cxdm22lDeICDvhRN8T4a1mz13kXzOa+R1T1URL9BzHNoGvwk25ECAwEAAaNQME4w +HQYDVR0OBBYEFJrPz27rcT3bPPGuiGtWcgPLCKdIMB8GA1UdIwQYMBaAFJrPz27r +cT3bPPGuiGtWcgPLCKdIMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +ABAlyNwMVVzLg25573fsDY4MBsFLDNb3dVIhuBdKOIids3jEQvu4fBQ4EPus2hEA +W0KHXkWfbU5CpJoYBjkPRaaWiTLWWbPTjuOVtsSiS3QvZ8H7u/lybzdK5/RIM3Hf +uPXmQT/V1S8mCfgOkv9w6varWPuQBNZDLo+x+waradDcqPhbB/LUZh9j+F3BnkFE +u8nofeBG5KfIMl8xYuUcXIndt6JPng0TuF+xhFNMH84Z4QEAXr9BVZSppRPb8lnz +1k65nZ25CtmyGG18sfeWqr32+ZUPSm48fEZb39R47Jrc4uMB5oh3OZOcuipj+SVL +T6wIeTnGe98HNbrAwlC/WoE= -----END CERTIFICATE----- diff --git a/Lib/test/capath/ceff1710.0 b/Lib/test/capath/ceff1710.0 index 373349cae05..07556ff9071 100644 --- a/Lib/test/capath/ceff1710.0 +++ b/Lib/test/capath/ceff1710.0 @@ -1,21 +1,21 @@ -----BEGIN CERTIFICATE----- -MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +MIIDbTCCAlWgAwIBAgIJAILtv0HIgJGbMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx -OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDZaFw0yODAxMTcx +OTA5MDZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI -hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV -q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/ -AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA -Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni -0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx -6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w -HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2 -2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB -AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4 -QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1 -Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O -JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR -f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf -9mmvtk57HVjsO6lTo15YyJ4= +hvcNAQEBBQADggEPADCCAQoCggEBAMMYaWvJRymYjrFWwi76Dl68I4CzB2Ik0kJb +8Uq/qcghdcjj5iwfhzxufBvtOTKVt0CyYEjDmhYI/m1niDQ7d3dwHHBa0R9fBCFU +uQzjQYUdWO4v7fMO79gjofpz+0wo4OXmTQsCUkmGx75+veZWdotwjgqPBjMgHXtb +qtDFG6ubzFQJPL/kQGbx+9b3Fp3EGdTD8v8HvG9aniUbAkql7EKWOnDSbJkrzr7o +0gHv1bqwz5Q+gtAB1ktxgAMKEkWGeYHYS9LotbcsbJpMihAQ5OT1386EkcrRRuCE +cxdm22lDeICDvhRN8T4a1mz13kXzOa+R1T1URL9BzHNoGvwk25ECAwEAAaNQME4w +HQYDVR0OBBYEFJrPz27rcT3bPPGuiGtWcgPLCKdIMB8GA1UdIwQYMBaAFJrPz27r +cT3bPPGuiGtWcgPLCKdIMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +ABAlyNwMVVzLg25573fsDY4MBsFLDNb3dVIhuBdKOIids3jEQvu4fBQ4EPus2hEA +W0KHXkWfbU5CpJoYBjkPRaaWiTLWWbPTjuOVtsSiS3QvZ8H7u/lybzdK5/RIM3Hf +uPXmQT/V1S8mCfgOkv9w6varWPuQBNZDLo+x+waradDcqPhbB/LUZh9j+F3BnkFE +u8nofeBG5KfIMl8xYuUcXIndt6JPng0TuF+xhFNMH84Z4QEAXr9BVZSppRPb8lnz +1k65nZ25CtmyGG18sfeWqr32+ZUPSm48fEZb39R47Jrc4uMB5oh3OZOcuipj+SVL +T6wIeTnGe98HNbrAwlC/WoE= -----END CERTIFICATE----- diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index d0886c47bae..8f9ebddff5a 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -31,6 +31,8 @@ from datetime import timezone from datetime import date, datetime import time as _time +import _testcapi + # Needed by test_datetime import _strptime # @@ -49,7 +51,6 @@ OTHERSTUFF = (10, 34.5, "abc", {}, [], ()) INF = float("inf") NAN = float("nan") - ############################################################################# # module tests @@ -1553,6 +1554,50 @@ class TestDate(HarmlessMixedComparison, unittest.TestCase): self.assertEqual(dt1.toordinal(), dt2.toordinal()) self.assertEqual(dt2.newmeth(-7), dt1.year + dt1.month - 7) + def test_subclass_alternate_constructors(self): + # Test that alternate constructors call the constructor + class DateSubclass(self.theclass): + def __new__(cls, *args, **kwargs): + result = self.theclass.__new__(cls, *args, **kwargs) + result.extra = 7 + + return result + + args = (2003, 4, 14) + d_ord = 731319 # Equivalent ordinal date + d_isoformat = '2003-04-14' # Equivalent isoformat() + + base_d = DateSubclass(*args) + self.assertIsInstance(base_d, DateSubclass) + self.assertEqual(base_d.extra, 7) + + # Timestamp depends on time zone, so we'll calculate the equivalent here + ts = datetime.combine(base_d, time(0)).timestamp() + + test_cases = [ + ('fromordinal', (d_ord,)), + ('fromtimestamp', (ts,)), + ('fromisoformat', (d_isoformat,)), + ] + + for constr_name, constr_args in test_cases: + for base_obj in (DateSubclass, base_d): + # Test both the classmethod and method + with self.subTest(base_obj_type=type(base_obj), + constr_name=constr_name): + constr = getattr(base_obj, constr_name) + + dt = constr(*constr_args) + + # Test that it creates the right subclass + self.assertIsInstance(dt, DateSubclass) + + # Test that it's equal to the base object + self.assertEqual(dt, base_d) + + # Test that it called the constructor + self.assertEqual(dt.extra, 7) + def test_pickling_subclass_date(self): args = 6, 7, 23 @@ -1588,6 +1633,63 @@ class TestDate(HarmlessMixedComparison, unittest.TestCase): # blow up because other fields are insane. self.theclass(base[:2] + bytes([ord_byte]) + base[3:]) + def test_fromisoformat(self): + # Test that isoformat() is reversible + base_dates = [ + (1, 1, 1), + (1000, 2, 14), + (1900, 1, 1), + (2000, 2, 29), + (2004, 11, 12), + (2004, 4, 3), + (2017, 5, 30) + ] + + for dt_tuple in base_dates: + dt = self.theclass(*dt_tuple) + dt_str = dt.isoformat() + with self.subTest(dt_str=dt_str): + dt_rt = self.theclass.fromisoformat(dt.isoformat()) + + self.assertEqual(dt, dt_rt) + + def test_fromisoformat_subclass(self): + class DateSubclass(self.theclass): + pass + + dt = DateSubclass(2014, 12, 14) + + dt_rt = DateSubclass.fromisoformat(dt.isoformat()) + + self.assertIsInstance(dt_rt, DateSubclass) + + def test_fromisoformat_fails(self): + # Test that fromisoformat() fails on invalid values + bad_strs = [ + '', # Empty string + '009-03-04', # Not 10 characters + '123456789', # Not a date + '200a-12-04', # Invalid character in year + '2009-1a-04', # Invalid character in month + '2009-12-0a', # Invalid character in day + '2009-01-32', # Invalid day + '2009-02-29', # Invalid leap day + '20090228', # Valid ISO8601 output not from isoformat() + ] + + for bad_str in bad_strs: + with self.assertRaises(ValueError): + self.theclass.fromisoformat(bad_str) + + def test_fromisoformat_fails_typeerror(self): + # Test that fromisoformat fails when passed the wrong type + import io + + bad_types = [b'2009-03-01', None, io.StringIO('2009-03-01')] + for bad_type in bad_types: + with self.assertRaises(TypeError): + self.theclass.fromisoformat(bad_type) + ############################################################################# # datetime tests @@ -1675,6 +1777,36 @@ class TestDateTime(TestDate): t = self.theclass(2, 3, 2, tzinfo=tz) self.assertEqual(t.isoformat(), "0002-03-02T00:00:00+00:00:16") + def test_isoformat_timezone(self): + tzoffsets = [ + ('05:00', timedelta(hours=5)), + ('02:00', timedelta(hours=2)), + ('06:27', timedelta(hours=6, minutes=27)), + ('12:32:30', timedelta(hours=12, minutes=32, seconds=30)), + ('02:04:09.123456', timedelta(hours=2, minutes=4, seconds=9, microseconds=123456)) + ] + + tzinfos = [ + ('', None), + ('+00:00', timezone.utc), + ('+00:00', timezone(timedelta(0))), + ] + + tzinfos += [ + (prefix + expected, timezone(sign * td)) + for expected, td in tzoffsets + for prefix, sign in [('-', -1), ('+', 1)] + ] + + dt_base = self.theclass(2016, 4, 1, 12, 37, 9) + exp_base = '2016-04-01T12:37:09' + + for exp_tz, tzi in tzinfos: + dt = dt_base.replace(tzinfo=tzi) + exp = exp_base + exp_tz + with self.subTest(tzi=tzi): + assert dt.isoformat() == exp + def test_format(self): dt = self.theclass(2007, 9, 10, 4, 5, 1, 123) self.assertEqual(dt.__format__(''), str(dt)) @@ -1729,7 +1861,7 @@ class TestDateTime(TestDate): # Make sure comparison doesn't forget microseconds, and isn't done # via comparing a float timestamp (an IEEE double doesn't have enough - # precision to span microsecond resolution across years 1 thru 9999, + # precision to span microsecond resolution across years 1 through 9999, # so comparing via timestamp necessarily calls some distinct values # equal). dt1 = self.theclass(MAXYEAR, 12, 31, 23, 59, 59, 999998) @@ -2334,6 +2466,221 @@ class TestDateTime(TestDate): self.assertEqual(dt2.newmeth(-7), dt1.year + dt1.month + dt1.second - 7) + def test_subclass_alternate_constructors_datetime(self): + # Test that alternate constructors call the constructor + class DateTimeSubclass(self.theclass): + def __new__(cls, *args, **kwargs): + result = self.theclass.__new__(cls, *args, **kwargs) + result.extra = 7 + + return result + + args = (2003, 4, 14, 12, 30, 15, 123456) + d_isoformat = '2003-04-14T12:30:15.123456' # Equivalent isoformat() + utc_ts = 1050323415.123456 # UTC timestamp + + base_d = DateTimeSubclass(*args) + self.assertIsInstance(base_d, DateTimeSubclass) + self.assertEqual(base_d.extra, 7) + + # Timestamp depends on time zone, so we'll calculate the equivalent here + ts = base_d.timestamp() + + test_cases = [ + ('fromtimestamp', (ts,)), + # See https://bugs.python.org/issue32417 + # ('fromtimestamp', (ts, timezone.utc)), + ('utcfromtimestamp', (utc_ts,)), + ('fromisoformat', (d_isoformat,)), + ('strptime', (d_isoformat, '%Y-%m-%dT%H:%M:%S.%f')), + ('combine', (date(*args[0:3]), time(*args[3:]))), + ] + + for constr_name, constr_args in test_cases: + for base_obj in (DateTimeSubclass, base_d): + # Test both the classmethod and method + with self.subTest(base_obj_type=type(base_obj), + constr_name=constr_name): + constr = getattr(base_obj, constr_name) + + dt = constr(*constr_args) + + # Test that it creates the right subclass + self.assertIsInstance(dt, DateTimeSubclass) + + # Test that it's equal to the base object + self.assertEqual(dt, base_d.replace(tzinfo=None)) + + # Test that it called the constructor + self.assertEqual(dt.extra, 7) + + def test_fromisoformat_datetime(self): + # Test that isoformat() is reversible + base_dates = [ + (1, 1, 1), + (1900, 1, 1), + (2004, 11, 12), + (2017, 5, 30) + ] + + base_times = [ + (0, 0, 0, 0), + (0, 0, 0, 241000), + (0, 0, 0, 234567), + (12, 30, 45, 234567) + ] + + separators = [' ', 'T'] + + tzinfos = [None, timezone.utc, + timezone(timedelta(hours=-5)), + timezone(timedelta(hours=2))] + + dts = [self.theclass(*date_tuple, *time_tuple, tzinfo=tzi) + for date_tuple in base_dates + for time_tuple in base_times + for tzi in tzinfos] + + for dt in dts: + for sep in separators: + dtstr = dt.isoformat(sep=sep) + + with self.subTest(dtstr=dtstr): + dt_rt = self.theclass.fromisoformat(dtstr) + self.assertEqual(dt, dt_rt) + + def test_fromisoformat_timezone(self): + base_dt = self.theclass(2014, 12, 30, 12, 30, 45, 217456) + + tzoffsets = [ + timedelta(hours=5), timedelta(hours=2), + timedelta(hours=6, minutes=27), + timedelta(hours=12, minutes=32, seconds=30), + timedelta(hours=2, minutes=4, seconds=9, microseconds=123456) + ] + + tzoffsets += [-1 * td for td in tzoffsets] + + tzinfos = [None, timezone.utc, + timezone(timedelta(hours=0))] + + tzinfos += [timezone(td) for td in tzoffsets] + + for tzi in tzinfos: + dt = base_dt.replace(tzinfo=tzi) + dtstr = dt.isoformat() + + with self.subTest(tstr=dtstr): + dt_rt = self.theclass.fromisoformat(dtstr) + assert dt == dt_rt, dt_rt + + def test_fromisoformat_separators(self): + separators = [ + ' ', 'T', '\u007f', # 1-bit widths + '\u0080', 'ʁ', # 2-bit widths + 'ᛇ', '時', # 3-bit widths + '🐍' # 4-bit widths + ] + + for sep in separators: + dt = self.theclass(2018, 1, 31, 23, 59, 47, 124789) + dtstr = dt.isoformat(sep=sep) + + with self.subTest(dtstr=dtstr): + dt_rt = self.theclass.fromisoformat(dtstr) + self.assertEqual(dt, dt_rt) + + def test_fromisoformat_ambiguous(self): + # Test strings like 2018-01-31+12:15 (where +12:15 is not a time zone) + separators = ['+', '-'] + for sep in separators: + dt = self.theclass(2018, 1, 31, 12, 15) + dtstr = dt.isoformat(sep=sep) + + with self.subTest(dtstr=dtstr): + dt_rt = self.theclass.fromisoformat(dtstr) + self.assertEqual(dt, dt_rt) + + def test_fromisoformat_timespecs(self): + datetime_bases = [ + (2009, 12, 4, 8, 17, 45, 123456), + (2009, 12, 4, 8, 17, 45, 0)] + + tzinfos = [None, timezone.utc, + timezone(timedelta(hours=-5)), + timezone(timedelta(hours=2)), + timezone(timedelta(hours=6, minutes=27))] + + timespecs = ['hours', 'minutes', 'seconds', + 'milliseconds', 'microseconds'] + + for ip, ts in enumerate(timespecs): + for tzi in tzinfos: + for dt_tuple in datetime_bases: + if ts == 'milliseconds': + new_microseconds = 1000 * (dt_tuple[6] // 1000) + dt_tuple = dt_tuple[0:6] + (new_microseconds,) + + dt = self.theclass(*(dt_tuple[0:(4 + ip)]), tzinfo=tzi) + dtstr = dt.isoformat(timespec=ts) + with self.subTest(dtstr=dtstr): + dt_rt = self.theclass.fromisoformat(dtstr) + self.assertEqual(dt, dt_rt) + + def test_fromisoformat_fails_datetime(self): + # Test that fromisoformat() fails on invalid values + bad_strs = [ + '', # Empty string + '2009.04-19T03', # Wrong first separator + '2009-04.19T03', # Wrong second separator + '2009-04-19T0a', # Invalid hours + '2009-04-19T03:1a:45', # Invalid minutes + '2009-04-19T03:15:4a', # Invalid seconds + '2009-04-19T03;15:45', # Bad first time separator + '2009-04-19T03:15;45', # Bad second time separator + '2009-04-19T03:15:4500:00', # Bad time zone separator + '2009-04-19T03:15:45.2345', # Too many digits for milliseconds + '2009-04-19T03:15:45.1234567', # Too many digits for microseconds + '2009-04-19T03:15:45.123456+24:30', # Invalid time zone offset + '2009-04-19T03:15:45.123456-24:30', # Invalid negative offset + '2009-04-10ᛇᛇᛇᛇᛇ12:15', # Too many unicode separators + '2009-04-19T1', # Incomplete hours + '2009-04-19T12:3', # Incomplete minutes + '2009-04-19T12:30:4', # Incomplete seconds + '2009-04-19T12:', # Ends with time separator + '2009-04-19T12:30:', # Ends with time separator + '2009-04-19T12:30:45.', # Ends with time separator + '2009-04-19T12:30:45.123456+', # Ends with timzone separator + '2009-04-19T12:30:45.123456-', # Ends with timzone separator + '2009-04-19T12:30:45.123456-05:00a', # Extra text + '2009-04-19T12:30:45.123-05:00a', # Extra text + '2009-04-19T12:30:45-05:00a', # Extra text + ] + + for bad_str in bad_strs: + with self.subTest(bad_str=bad_str): + with self.assertRaises(ValueError): + self.theclass.fromisoformat(bad_str) + + def test_fromisoformat_utc(self): + dt_str = '2014-04-19T13:21:13+00:00' + dt = self.theclass.fromisoformat(dt_str) + + self.assertIs(dt.tzinfo, timezone.utc) + + def test_fromisoformat_subclass(self): + class DateTimeSubclass(self.theclass): + pass + + dt = DateTimeSubclass(2014, 12, 14, 9, 30, 45, 457390, + tzinfo=timezone(timedelta(hours=10, minutes=45))) + + dt_rt = DateTimeSubclass.fromisoformat(dt.isoformat()) + + self.assertEqual(dt, dt_rt) + self.assertIsInstance(dt_rt, DateTimeSubclass) + + class TestSubclassDateTime(TestDateTime): theclass = SubclassDatetime # Override tests not designed for subclass @@ -2517,6 +2864,36 @@ class TestTime(HarmlessMixedComparison, unittest.TestCase): self.assertEqual(t.isoformat(timespec='microseconds'), "12:34:56.000000") self.assertEqual(t.isoformat(timespec='auto'), "12:34:56") + def test_isoformat_timezone(self): + tzoffsets = [ + ('05:00', timedelta(hours=5)), + ('02:00', timedelta(hours=2)), + ('06:27', timedelta(hours=6, minutes=27)), + ('12:32:30', timedelta(hours=12, minutes=32, seconds=30)), + ('02:04:09.123456', timedelta(hours=2, minutes=4, seconds=9, microseconds=123456)) + ] + + tzinfos = [ + ('', None), + ('+00:00', timezone.utc), + ('+00:00', timezone(timedelta(0))), + ] + + tzinfos += [ + (prefix + expected, timezone(sign * td)) + for expected, td in tzoffsets + for prefix, sign in [('-', -1), ('+', 1)] + ] + + t_base = self.theclass(12, 37, 9) + exp_base = '12:37:09' + + for exp_tz, tzi in tzinfos: + t = t_base.replace(tzinfo=tzi) + exp = exp_base + exp_tz + with self.subTest(tzi=tzi): + assert t.isoformat() == exp + def test_1653736(self): # verify it doesn't accept extra keyword arguments t = self.theclass(second=1) @@ -3055,6 +3432,133 @@ class TestTimeTZ(TestTime, TZInfoBase, unittest.TestCase): t2 = t2.replace(tzinfo=Varies()) self.assertTrue(t1 < t2) # t1's offset counter still going up + def test_fromisoformat(self): + time_examples = [ + (0, 0, 0, 0), + (23, 59, 59, 999999), + ] + + hh = (9, 12, 20) + mm = (5, 30) + ss = (4, 45) + usec = (0, 245000, 678901) + + time_examples += list(itertools.product(hh, mm, ss, usec)) + + tzinfos = [None, timezone.utc, + timezone(timedelta(hours=2)), + timezone(timedelta(hours=6, minutes=27))] + + for ttup in time_examples: + for tzi in tzinfos: + t = self.theclass(*ttup, tzinfo=tzi) + tstr = t.isoformat() + + with self.subTest(tstr=tstr): + t_rt = self.theclass.fromisoformat(tstr) + self.assertEqual(t, t_rt) + + def test_fromisoformat_timezone(self): + base_time = self.theclass(12, 30, 45, 217456) + + tzoffsets = [ + timedelta(hours=5), timedelta(hours=2), + timedelta(hours=6, minutes=27), + timedelta(hours=12, minutes=32, seconds=30), + timedelta(hours=2, minutes=4, seconds=9, microseconds=123456) + ] + + tzoffsets += [-1 * td for td in tzoffsets] + + tzinfos = [None, timezone.utc, + timezone(timedelta(hours=0))] + + tzinfos += [timezone(td) for td in tzoffsets] + + for tzi in tzinfos: + t = base_time.replace(tzinfo=tzi) + tstr = t.isoformat() + + with self.subTest(tstr=tstr): + t_rt = self.theclass.fromisoformat(tstr) + assert t == t_rt, t_rt + + def test_fromisoformat_timespecs(self): + time_bases = [ + (8, 17, 45, 123456), + (8, 17, 45, 0) + ] + + tzinfos = [None, timezone.utc, + timezone(timedelta(hours=-5)), + timezone(timedelta(hours=2)), + timezone(timedelta(hours=6, minutes=27))] + + timespecs = ['hours', 'minutes', 'seconds', + 'milliseconds', 'microseconds'] + + for ip, ts in enumerate(timespecs): + for tzi in tzinfos: + for t_tuple in time_bases: + if ts == 'milliseconds': + new_microseconds = 1000 * (t_tuple[-1] // 1000) + t_tuple = t_tuple[0:-1] + (new_microseconds,) + + t = self.theclass(*(t_tuple[0:(1 + ip)]), tzinfo=tzi) + tstr = t.isoformat(timespec=ts) + with self.subTest(tstr=tstr): + t_rt = self.theclass.fromisoformat(tstr) + self.assertEqual(t, t_rt) + + def test_fromisoformat_fails(self): + bad_strs = [ + '', # Empty string + '12:', # Ends on a separator + '12:30:', # Ends on a separator + '12:30:15.', # Ends on a separator + '1', # Incomplete hours + '12:3', # Incomplete minutes + '12:30:1', # Incomplete seconds + '1a:30:45.334034', # Invalid character in hours + '12:a0:45.334034', # Invalid character in minutes + '12:30:a5.334034', # Invalid character in seconds + '12:30:45.1234', # Too many digits for milliseconds + '12:30:45.1234567', # Too many digits for microseconds + '12:30:45.123456+24:30', # Invalid time zone offset + '12:30:45.123456-24:30', # Invalid negative offset + '12：30：45', # Uses full-width unicode colons + '12:30:45․123456', # Uses \u2024 in place of decimal point + '12:30:45a', # Extra at tend of basic time + '12:30:45.123a', # Extra at end of millisecond time + '12:30:45.123456a', # Extra at end of microsecond time + '12:30:45.123456+12:00:30a', # Extra at end of full time + ] + + for bad_str in bad_strs: + with self.subTest(bad_str=bad_str): + with self.assertRaises(ValueError): + self.theclass.fromisoformat(bad_str) + + def test_fromisoformat_fails_typeerror(self): + # Test the fromisoformat fails when passed the wrong type + import io + + bad_types = [b'12:30:45', None, io.StringIO('12:30:45')] + + for bad_type in bad_types: + with self.assertRaises(TypeError): + self.theclass.fromisoformat(bad_type) + + def test_fromisoformat_subclass(self): + class TimeSubclass(self.theclass): + pass + + tsc = TimeSubclass(12, 14, 45, 203745, tzinfo=timezone.utc) + tsc_rt = TimeSubclass.fromisoformat(tsc.isoformat()) + + self.assertEqual(tsc, tsc_rt) + self.assertIsInstance(tsc_rt, TimeSubclass) + def test_subclass_timetz(self): class C(self.theclass): @@ -4941,6 +5445,185 @@ class ZoneInfoCompleteTest(unittest.TestSuite): class IranTest(ZoneInfoTest): zonename = 'Asia/Tehran' + +class CapiTest(unittest.TestCase): + def setUp(self): + # Since the C API is not present in the _Pure tests, skip all tests + if self.__class__.__name__.endswith('Pure'): + self.skipTest('Not relevant in pure Python') + + # This *must* be called, and it must be called first, so until either + # restriction is loosened, we'll call it as part of test setup + _testcapi.test_datetime_capi() + + def test_utc_capi(self): + for use_macro in (True, False): + capi_utc = _testcapi.get_timezone_utc_capi(use_macro) + + with self.subTest(use_macro=use_macro): + self.assertIs(capi_utc, timezone.utc) + + def test_timezones_capi(self): + est_capi, est_macro, est_macro_nn = _testcapi.make_timezones_capi() + + exp_named = timezone(timedelta(hours=-5), "EST") + exp_unnamed = timezone(timedelta(hours=-5)) + + cases = [ + ('est_capi', est_capi, exp_named), + ('est_macro', est_macro, exp_named), + ('est_macro_nn', est_macro_nn, exp_unnamed) + ] + + for name, tz_act, tz_exp in cases: + with self.subTest(name=name): + self.assertEqual(tz_act, tz_exp) + + dt1 = datetime(2000, 2, 4, tzinfo=tz_act) + dt2 = datetime(2000, 2, 4, tzinfo=tz_exp) + + self.assertEqual(dt1, dt2) + self.assertEqual(dt1.tzname(), dt2.tzname()) + + dt_utc = datetime(2000, 2, 4, 5, tzinfo=timezone.utc) + + self.assertEqual(dt1.astimezone(timezone.utc), dt_utc) + + def test_check_date(self): + class DateSubclass(date): + pass + + d = date(2011, 1, 1) + ds = DateSubclass(2011, 1, 1) + dt = datetime(2011, 1, 1) + + is_date = _testcapi.datetime_check_date + + # Check the ones that should be valid + self.assertTrue(is_date(d)) + self.assertTrue(is_date(dt)) + self.assertTrue(is_date(ds)) + self.assertTrue(is_date(d, True)) + + # Check that the subclasses do not match exactly + self.assertFalse(is_date(dt, True)) + self.assertFalse(is_date(ds, True)) + + # Check that various other things are not dates at all + args = [tuple(), list(), 1, '2011-01-01', + timedelta(1), timezone.utc, time(12, 00)] + for arg in args: + for exact in (True, False): + with self.subTest(arg=arg, exact=exact): + self.assertFalse(is_date(arg, exact)) + + def test_check_time(self): + class TimeSubclass(time): + pass + + t = time(12, 30) + ts = TimeSubclass(12, 30) + + is_time = _testcapi.datetime_check_time + + # Check the ones that should be valid + self.assertTrue(is_time(t)) + self.assertTrue(is_time(ts)) + self.assertTrue(is_time(t, True)) + + # Check that the subclass does not match exactly + self.assertFalse(is_time(ts, True)) + + # Check that various other things are not times + args = [tuple(), list(), 1, '2011-01-01', + timedelta(1), timezone.utc, date(2011, 1, 1)] + + for arg in args: + for exact in (True, False): + with self.subTest(arg=arg, exact=exact): + self.assertFalse(is_time(arg, exact)) + + def test_check_datetime(self): + class DateTimeSubclass(datetime): + pass + + dt = datetime(2011, 1, 1, 12, 30) + dts = DateTimeSubclass(2011, 1, 1, 12, 30) + + is_datetime = _testcapi.datetime_check_datetime + + # Check the ones that should be valid + self.assertTrue(is_datetime(dt)) + self.assertTrue(is_datetime(dts)) + self.assertTrue(is_datetime(dt, True)) + + # Check that the subclass does not match exactly + self.assertFalse(is_datetime(dts, True)) + + # Check that various other things are not datetimes + args = [tuple(), list(), 1, '2011-01-01', + timedelta(1), timezone.utc, date(2011, 1, 1)] + + for arg in args: + for exact in (True, False): + with self.subTest(arg=arg, exact=exact): + self.assertFalse(is_datetime(arg, exact)) + + def test_check_delta(self): + class TimeDeltaSubclass(timedelta): + pass + + td = timedelta(1) + tds = TimeDeltaSubclass(1) + + is_timedelta = _testcapi.datetime_check_delta + + # Check the ones that should be valid + self.assertTrue(is_timedelta(td)) + self.assertTrue(is_timedelta(tds)) + self.assertTrue(is_timedelta(td, True)) + + # Check that the subclass does not match exactly + self.assertFalse(is_timedelta(tds, True)) + + # Check that various other things are not timedeltas + args = [tuple(), list(), 1, '2011-01-01', + timezone.utc, date(2011, 1, 1), datetime(2011, 1, 1)] + + for arg in args: + for exact in (True, False): + with self.subTest(arg=arg, exact=exact): + self.assertFalse(is_timedelta(arg, exact)) + + def test_check_tzinfo(self): + class TZInfoSubclass(tzinfo): + pass + + tzi = tzinfo() + tzis = TZInfoSubclass() + tz = timezone(timedelta(hours=-5)) + + is_tzinfo = _testcapi.datetime_check_tzinfo + + # Check the ones that should be valid + self.assertTrue(is_tzinfo(tzi)) + self.assertTrue(is_tzinfo(tz)) + self.assertTrue(is_tzinfo(tzis)) + self.assertTrue(is_tzinfo(tzi, True)) + + # Check that the subclasses do not match exactly + self.assertFalse(is_tzinfo(tz, True)) + self.assertFalse(is_tzinfo(tzis, True)) + + # Check that various other things are not tzinfos + args = [tuple(), list(), 1, '2011-01-01', + date(2011, 1, 1), datetime(2011, 1, 1)] + + for arg in args: + for exact in (True, False): + with self.subTest(arg=arg, exact=exact): + self.assertFalse(is_tzinfo(arg, exact)) + def load_tests(loader, standard_tests, pattern): standard_tests.addTest(ZoneInfoCompleteTest()) return standard_tests diff --git a/Lib/test/good_getattr.py b/Lib/test/good_getattr.py new file mode 100644 index 00000000000..7d27de6262a --- /dev/null +++ b/Lib/test/good_getattr.py @@ -0,0 +1,11 @@ +x = 1 + +def __dir__(): + return ['a', 'b', 'c'] + +def __getattr__(name): + if name == "yolo": + raise AttributeError("Deprecated, use whatever instead") + return f"There is {name}" + +y = 2 diff --git a/Lib/test/idnsans.pem b/Lib/test/idnsans.pem new file mode 100644 index 00000000000..b4a771ce7ec --- /dev/null +++ b/Lib/test/idnsans.pem @@ -0,0 +1,136 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDeU8YQtyeEjPLA +SdMBPTS9QcuAZIJjbJRgr8nsRb767pbmWR9C1JuDy/Bz/AprFC6Om950fLn3pOqR +zDUWZX/qTe+o27i8u0Qzk06bhRkxAdTEoTfRcH/FkJaimJqeTt9rZqc+AGSNKM8o +4GyPW4IELnavmMB30+7rKIJMIpIn1a1k6MybJYdWNSuVqwArAVvRlj5qOiqX7KAS +otFRP8pz+Lgw3qREQzgnZz/bcScKd+5Uy4qMFPNOMjgW6nDV60ekNx0GT+59E/+8 +64GRq34rNVu2SN0XXcQh33R3LwwrvAdymaLyr1YyIRM5gLPxugxCIA0SYjG0YoGB +uUSwtNa7AgMBAAECggEBAJjxUGPXW1wYCja1km5byJgZVwEwI3J6E2igBWyAXm0J +DM3RqWu0DneQKA3h6NjYvV5lY5cG5nex/5vkuvB5SpHIo4GqBV/wA27ne0AJQ9cu +x0utDFUL6xnh6X5ZNKSK5a9gotRIOOPSmxAnswa7kKmHvSX3ExBbvxQOffQaJCk5 +0GHl6I/HltqVzMu4ICAo0NY0gw1n+hVKTo28KkJ9PL7X6v6H5yvZ3L6TkMytSvqf +9iVlYuIN66ToBtxaI4g2RiUJtA2hdT9IP7Wg4YD6Ptyih90zXz2wTzWppFem6UA9 +dePig94R9moj9ucuK0tx3kSATNo0op/XEx1e3OOtcQECgYEA/w7pNOPYgj7VMyYx +p4Lx4BOllzQts8mIBtUVZVQSJ2miun6DTalZVT2V3ayTuE0qhUHd1SHu9F77a9fQ +qaSUUY9elwXyfvcNCfhYVRJxyxirI4Z6ZCBwjpWOGSBB59NTeDhVnbkTlfE6guqS +3KRS1pfIQ6FCvGIrhjRZgHo1TGECgYEA3yXsospbOS7VeBj0UPSB87fp1QM+r48o +RflIsRzdsN9Ka2j6EiYpgKdbgXr80vkctYTK0dT8jrFSk81Y932CZezH2IWo8Meo +40qaFWMboNFBIC4yv6RSRxJMQfYsKnXC2trSnXH+qf55Trey4uZNMX7VJ+RFKExS +ieSWSbTWmJsCgYEAzo3yyoRiiEf+PKgHulLPMtp2VddJ07m30WCrLR5CfWyM/l8K +UtB8qg1v2s+x6aWEc9p9necXLwvkrNdgAqJoAw0KW1/TnILSKmrWjj6brRBTODfl +0kR7It128F4xQV7g0BE/NLX3aIytB+yT9t+Uvni5FBv6gbk26j5m5ScTFsECgYEA +hzrQYQcIqWq8av+Ub8r9Rdlal4BT6Mh0u5MKfmrj3mAzFUyU35LI6/J//cOum5vj +zg0fbHIKa98CEBgNpk4lS+dmZMz7SI92xedb4UIiaB7nvLzCfGj0g6WPGRo6QbED +2OVrZYbDsflJQm8ItYCjny8htf8b+gPmsTIZ8ajps6kCgYBnES8waDDAkL98lK28 +dcgnJXN+1UzeI6//If2uvDZEQ9tG/yMk2JYc84qZJLU5bRplMAjIQUVUcFWa+ZzV +ylnDhagAtiWkHPcElWHym9dH8CRuYM3OTDsApZ7yMB/ArCcZMIA35OvNf6uc4lNV +VD9VkaygPIg6ilv4npeTceqp8A== +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 82:ed:bf:41:c8:80:91:9f + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 19 19:09:07 2018 GMT + Not After : Nov 28 19:09:07 2027 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=idnsans + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:de:53:c6:10:b7:27:84:8c:f2:c0:49:d3:01:3d: + 34:bd:41:cb:80:64:82:63:6c:94:60:af:c9:ec:45: + be:fa:ee:96:e6:59:1f:42:d4:9b:83:cb:f0:73:fc: + 0a:6b:14:2e:8e:9b:de:74:7c:b9:f7:a4:ea:91:cc: + 35:16:65:7f:ea:4d:ef:a8:db:b8:bc:bb:44:33:93: + 4e:9b:85:19:31:01:d4:c4:a1:37:d1:70:7f:c5:90: + 96:a2:98:9a:9e:4e:df:6b:66:a7:3e:00:64:8d:28: + cf:28:e0:6c:8f:5b:82:04:2e:76:af:98:c0:77:d3: + ee:eb:28:82:4c:22:92:27:d5:ad:64:e8:cc:9b:25: + 87:56:35:2b:95:ab:00:2b:01:5b:d1:96:3e:6a:3a: + 2a:97:ec:a0:12:a2:d1:51:3f:ca:73:f8:b8:30:de: + a4:44:43:38:27:67:3f:db:71:27:0a:77:ee:54:cb: + 8a:8c:14:f3:4e:32:38:16:ea:70:d5:eb:47:a4:37: + 1d:06:4f:ee:7d:13:ff:bc:eb:81:91:ab:7e:2b:35: + 5b:b6:48:dd:17:5d:c4:21:df:74:77:2f:0c:2b:bc: + 07:72:99:a2:f2:af:56:32:21:13:39:80:b3:f1:ba: + 0c:42:20:0d:12:62:31:b4:62:81:81:b9:44:b0:b4: + d6:bb + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:idnsans, DNS:xn--knig-5qa.idn.pythontest.net, DNS:xn--knigsgsschen-lcb0w.idna2003.pythontest.net, DNS:xn--knigsgchen-b4a3dun.idna2008.pythontest.net, DNS:xn--nxasmq6b.idna2003.pythontest.net, DNS:xn--nxasmm1c.idna2008.pythontest.net + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 3B:F0:22:A0:1E:9B:CE:2A:7C:AE:B1:32:1B:B0:8E:3E:33:40:E3:FA + X509v3 Authority Key Identifier: + keyid:9A:CF:CF:6E:EB:71:3D:DB:3C:F1:AE:88:6B:56:72:03:CB:08:A7:48 + DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server + serial:82:ED:BF:41:C8:80:91:9B + + Authority Information Access: + CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer + OCSP - URI:http://testca.pythontest.net/testca/ocsp/ + + X509v3 CRL Distribution Points: + + Full Name: + URI:http://testca.pythontest.net/testca/revocation.crl + + Signature Algorithm: sha1WithRSAEncryption + 8b:1f:d7:e4:0d:15:76:b4:f5:87:33:de:b9:84:9b:f2:c1:9b: + c9:97:50:f7:18:33:ed:b7:60:83:be:bb:94:1c:49:39:ae:54: + 24:43:f7:85:d8:2a:8c:26:17:56:1e:a6:b7:63:c5:05:f1:6e: + f4:79:eb:fd:af:12:84:3c:28:4a:8f:b1:01:97:91:ba:18:2b: + ba:54:25:49:1b:5b:2e:1e:6b:33:2d:f5:07:2e:76:04:e0:a8: + 95:25:3f:cc:c8:26:c0:30:b6:90:d2:2b:e1:e2:13:b0:a8:76: + f0:06:90:b9:d5:28:6b:8a:e9:72:1a:ed:4f:7e:3c:37:2e:00: + aa:9b:f1:29:44:94:f2:dc:c8:31:5f:4c:2d:00:d3:5e:78:6c: + 68:fc:0e:1e:46:be:d8:2e:29:88:78:8e:7e:f5:50:c8:5c:5d: + 5f:4c:09:d5:51:07:40:be:9b:30:ed:a3:29:68:25:6b:88:69: + c7:43:35:54:2f:6e:9a:30:f1:d6:87:54:84:20:ef:a5:aa:33: + df:00:6a:87:a9:b4:d7:89:1f:e7:60:0d:01:60:66:11:61:3f: + d0:9f:86:37:cc:b3:b8:48:7e:1f:d2:7a:0f:02:e7:11:1d:dd: + 34:c4:0b:45:47:2b:05:37:dd:ee:6e:0e:1c:bd:de:24:42:50: + a4:07:af:e5 +-----BEGIN CERTIFICATE----- +MIIFvTCCBKWgAwIBAgIJAILtv0HIgJGfMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDdaFw0yNzExMjgx +OTA5MDdaMF0xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj +MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2lk +bnNhbnMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDeU8YQtyeEjPLA +SdMBPTS9QcuAZIJjbJRgr8nsRb767pbmWR9C1JuDy/Bz/AprFC6Om950fLn3pOqR +zDUWZX/qTe+o27i8u0Qzk06bhRkxAdTEoTfRcH/FkJaimJqeTt9rZqc+AGSNKM8o +4GyPW4IELnavmMB30+7rKIJMIpIn1a1k6MybJYdWNSuVqwArAVvRlj5qOiqX7KAS +otFRP8pz+Lgw3qREQzgnZz/bcScKd+5Uy4qMFPNOMjgW6nDV60ekNx0GT+59E/+8 +64GRq34rNVu2SN0XXcQh33R3LwwrvAdymaLyr1YyIRM5gLPxugxCIA0SYjG0YoGB +uUSwtNa7AgMBAAGjggKOMIICijCB4QYDVR0RBIHZMIHWggdpZG5zYW5zgh94bi0t +a25pZy01cWEuaWRuLnB5dGhvbnRlc3QubmV0gi54bi0ta25pZ3Nnc3NjaGVuLWxj +YjB3LmlkbmEyMDAzLnB5dGhvbnRlc3QubmV0gi54bi0ta25pZ3NnY2hlbi1iNGEz +ZHVuLmlkbmEyMDA4LnB5dGhvbnRlc3QubmV0giR4bi0tbnhhc21xNmIuaWRuYTIw +MDMucHl0aG9udGVzdC5uZXSCJHhuLS1ueGFzbW0xYy5pZG5hMjAwOC5weXRob250 +ZXN0Lm5ldDAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsG +AQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFDvwIqAem84qfK6xMhuwjj4z +QOP6MH0GA1UdIwR2MHSAFJrPz27rcT3bPPGuiGtWcgPLCKdIoVGkTzBNMQswCQYD +VQQGEwJYWTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0Ex +FjAUBgNVBAMMDW91ci1jYS1zZXJ2ZXKCCQCC7b9ByICRmzCBgwYIKwYBBQUHAQEE +dzB1MDwGCCsGAQUFBzAChjBodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rl +c3RjYS9weWNhY2VydC5jZXIwNQYIKwYBBQUHMAGGKWh0dHA6Ly90ZXN0Y2EucHl0 +aG9udGVzdC5uZXQvdGVzdGNhL29jc3AvMEMGA1UdHwQ8MDowOKA2oDSGMmh0dHA6 +Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3Jldm9jYXRpb24uY3JsMA0G +CSqGSIb3DQEBBQUAA4IBAQCLH9fkDRV2tPWHM965hJvywZvJl1D3GDPtt2CDvruU +HEk5rlQkQ/eF2CqMJhdWHqa3Y8UF8W70eev9rxKEPChKj7EBl5G6GCu6VCVJG1su +HmszLfUHLnYE4KiVJT/MyCbAMLaQ0ivh4hOwqHbwBpC51ShriulyGu1Pfjw3LgCq +m/EpRJTy3MgxX0wtANNeeGxo/A4eRr7YLimIeI5+9VDIXF1fTAnVUQdAvpsw7aMp +aCVriGnHQzVUL26aMPHWh1SEIO+lqjPfAGqHqbTXiR/nYA0BYGYRYT/Qn4Y3zLO4 +SH4f0noPAucRHd00xAtFRysFN93ubg4cvd4kQlCkB6/l +-----END CERTIFICATE----- diff --git a/Lib/test/keycert.passwd.pem b/Lib/test/keycert.passwd.pem index e90574881db..0ad69605519 100644 --- a/Lib/test/keycert.passwd.pem +++ b/Lib/test/keycert.passwd.pem @@ -1,33 +1,50 @@ -----BEGIN RSA PRIVATE KEY----- Proc-Type: 4,ENCRYPTED -DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A +DEK-Info: DES-EDE3-CBC,E74528136B90D2DD -kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c -u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA -AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr -Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+ -YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P -6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+ -noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1 -94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l -7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo -cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO -zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt -L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo -2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ== +WRHVD2PJXPqjFSHg92HURIsUzvsTE4a9oi0SC5yMBFKNWA5Z933gK3XTifp6jul5 +zpNYi8jBXZ2EqJJBxCuVcefmXSxL0q7CMej25TdIC4BVAFJVveeprHPUFkNB0IM1 +go5Lg4YofYqTCg3OE3k7WvfR3Zg1cRYxksDKO+WNZgWyKBex5X4vjOiyUqDl3GKt +kQXnkg1VgPV2Vrx93S9XNdELNRTguwf+XG0fkhtYhp/zCto8uKTgy5elK2P/ulGp +7fe6uj7h/uN9L7EOC6CjRkitywfeBUER739mOcGT4imSFJ9G27TCqPzj2ea3uuaf +/v1xhkQ4M6lNY/gcRfgVpCXhW43aAQV8XXQRMJTqLmz5Y5hYTKn+Ugq5vJ/ngyRM +lu1gUJnYYaemBTb4hbm6aBvnYK9mORa891Pmf+vxU9rYuQIdVAhvvXh4KBreSEBI +1AFy6dFKXl8ZKs6Wrq5wPefmFFkRmZ8OBiiq0fp2ApCRGZw6LsjCgwrRM38JiY7d +3OdsJpKvRYufgUyuuzUE0xA+E4yMvD48M9pPq2fC8O5giuGL1uEekQWXJuq+6ZRI +XYKIeSkuQALbX3RAzCPXTUEMtCYXKm/gxrrwJ+Bet4ob2amf3MX0uvWwOuAq++Fk +J0HFSBxrwvIWOhyQXOPuJdAN8PXA7dWOXfOgOMF0hQYqZCl3T4TiVZJbwVQtg1sN +dO7oAD5ZPHiKzvveZuB6k1FlBG8j0TyAC+44ChxkPDD3jF4dd6zGe62sDf85p4/d +W80gxJeD3xnDxG0ePPns+GuKUpUaWS7WvHtDpeFW1JEhvOqf8p1Li9a7RzWVo8ML +mGTdQgBIYIf6/fk69pFKl0nKtBU75KaunZz4nAmd9bNED4naDurMBg44u5TvODbJ +vgYIYXIYjNvONbskJatVrrTS8zch2NwVIjCi8L/hecwBXbIXzo1pECpc6BU7sQT8 ++i9sDKBeJcRipzfKZNHvnO19mUZaPCY8+a/f9c21DgKXz+bgLcJbohpSaeGM8Gfc +aZd3Vp9n3OJ3g2zQR1++HO9v1vR/wLELu6MeydkvMduHLmOPCn54gZ9z51ZNPAwa +qfFIsH+mLh9ks0H74ssF59uIlstkgB9zmZHv/Q0dK9ZfG/VEH6rSgdETWhZxhoMQ +Z92jXBEFT0zhI3rrIPNY+XS7eJCQIc1wc84Ea3cRk7SP+S1og3JtAxX56ykUwtkM +LQ/Dwwa6h1aqD0l2d5x1/BSdavtTuSegISRWQ4iOmSvEdlFP7H4g6RZk/okbLzMD +Evq5gNc7vlXhVawoQU8JCanJ5ZbbWnIRZfiXxBQS4lpYPKvJt4ML9z/x+82XxcXv +Z93N2Wep7wWW5OwS2LcQcOgZRDSIPompwo/0pMFGOS+5oort0ZDRHdmmGLjvBcCb +1KQmKQ4+8brI/3rjRzts6uDLjTGNxSCieNsnqhwHUv9Mg9WDSWupcGa+x27L89x3 +rObf6+3umcFLSjIzU8wuv1hx/e/y98Kv7BDBNYpAr6kVMrLnzYjAfJbBmqlxkzkQ +IgQzgrk2QZoTdgwR+S374NAMO0AE5IlO+/qa6qp2SORGTDX64I3UNw== -----END RSA PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV -BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw -MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH -Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k -YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw -gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 -6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt -pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw -FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd -BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G -lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 -CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +MIIDWTCCAkGgAwIBAgIJAPm6B21bar2bMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODAx +MTkxOTA5MDZaFw0yODAxMTcxOTA5MDZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +DA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAKvvsX2gEti4shve3iYMc+jE4Se7WHs1Bol2f21H8qNboDOFdeb1 +RKHjmq3exHpajywOUEgne9nKHJY/3f2phR4Y5klqG6liLgiSpVyRlcBGbeT2qEAj +9oLiLFUXLGfGDds2mTwivQDLJBWi51j7ff5k2Pr58fN5ugYMn24T9FNyn0moT+qj +SFoBNm58l9jrdkJSlgWfqPlbiMa+mqDn/SFtrwLF2Trbfzu42Sd9UdIzMaSSrzbN +sGm53pNhCh8KndWUQ8GPP2IsLPoUU4qAtmZuTxCx2S1cXrN9EkmT69tlOH84YfSn +96Ih9bWRc7M5y5bfVdEVM+fKQl3hBRf05qMCAwEAAaMYMBYwFAYDVR0RBA0wC4IJ +bG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IBAQAtQ8f37cCEk7/rAcbYR53ce3iK +Vpihb0U2ni1QjG9Tg9UIExkIGkwTiCm7kwQL+GEStBu9AG/QVrOjeTriRiddhWkk +ze8kRaI3AC/63t6Vh9Q1x6PESgeE4OtAO9JpJCf4GILglA789Y/b/GF8zJZQxR13 +qpB4ZwWw7gCBhdEW59u6CFeBmfDa58hM8lWvuVoRrTi7bjUeC6PAn5HVMzZSykhu +4HaUfBp6bKFjuym2+h/VvM1n8C3chjVSmutsLb6ELdD8IK0vPV/yf5+LN256zSsS +dyUZYd8XwQaioEMKdbhLvnehyzHiWfQIUR3BdhONxoIJhHv/EAo8eCkHHYIF -----END CERTIFICATE----- diff --git a/Lib/test/keycert.pem b/Lib/test/keycert.pem index 64318aa2e03..9545dcf4b94 100644 --- a/Lib/test/keycert.pem +++ b/Lib/test/keycert.pem @@ -1,31 +1,48 @@ -----BEGIN PRIVATE KEY----- -MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm -LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 -ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP -USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt -CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq -SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK -UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y -BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ -ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 -oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik -eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F -0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS -x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ -SPIXQuT8RMPDVNQ= +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCr77F9oBLYuLIb +3t4mDHPoxOEnu1h7NQaJdn9tR/KjW6AzhXXm9USh45qt3sR6Wo8sDlBIJ3vZyhyW +P939qYUeGOZJahupYi4IkqVckZXARm3k9qhAI/aC4ixVFyxnxg3bNpk8Ir0AyyQV +oudY+33+ZNj6+fHzeboGDJ9uE/RTcp9JqE/qo0haATZufJfY63ZCUpYFn6j5W4jG +vpqg5/0hba8Cxdk62387uNknfVHSMzGkkq82zbBpud6TYQofCp3VlEPBjz9iLCz6 +FFOKgLZmbk8QsdktXF6zfRJJk+vbZTh/OGH0p/eiIfW1kXOzOcuW31XRFTPnykJd +4QUX9OajAgMBAAECggEAHppmXDbuw9Z0FVPg9KLIysioTtsgz6VLiZIm8juZK4x2 +glUh/D7xvWL2uDXrgN+3lh7iGUW13LkFx5SMncbbo9TIwI57Z/XKvcnkVwquve+L +RfLFVc1Q5lD9lROv2rS86KTaN4LzYz3FKXi6dvMkpPAsUtfEQhMLkmISypQQq/1z +EJaqo7r85OjN7e0wKazlKZpOzJEa5FQLMVRjTRFhLFNbHXX/tAet2jw+umATKbw8 +hYgiuZ44TwSEd9JeIV/oSYWfI/3HetuYW0ru3caiztRF2NySNu8lcsWgNC7fIku9 +mcHjtSNzs91QN1Qlu7GQvvhpt6OWDirNDCW+49WGaQKBgQDg9SDhfF0jRYslgYbH +cqO4ggaFdHjrAAYpwnAgvanhFZL/zEqm5G1E7l/e2fCkJ9VOSFO0A208chvwMcr+ +dCjHE2tVdE81aQ2v/Eo83VdS1RcOV4Y75yPH48rMhxPaHvxWD/FFDbf0/P2mtPB7 +SZ3kIeZMkE1wxdaO3AKUbQoozwKBgQDDqYgg7kVtygyICE1mB8Hwp6nUxFTczG7y +4XcsDqMIrKmw+PbQluvkoHoStxeVrsTloDhkTjIrpmYLyAiazg+PUJdkd6xrfLSj +VV6X93W0S/1egEb1F1CGFxtk8v/PWH4K76EPL2vxXdxjywz3GWlrL9yDYaB2szzS +DqgwVMqx7QKBgDCD7UF0Bsoyl13RX3XoPXLvZ+SkR+e2q52Z94C4JskKVBeiwX7Y +yNAS8M4pBoMArDoj0xmBm69rlKbqtjLGbnzwrTdSzDpim7cWnBQgUFLm7gAD1Elb +AhZ8BCK0Bw4FnLoa2hfga4oEfdfUMgEE0W5/+SEOBgWKRUmuHUhRc911AoGAY2EN +YmSDYSM5wDIvVb5k9B3EtevOiqNPSw/XnsoEZtiEC/44JnQxdltIBY93bDBrk5IQ +cmoBM4h91kgQjshQwOMXMhFSwvmBKmCm/hrTbvMVytTutXfVD3ZXFKwT4DW7N0TF +ElhsxBh/YzRz7mG62JVjtFt2zDN3ld2Z8YpvtXUCgYEA4EJ4ObS5YyvcXAKHJFo6 +Fxmavyrf8LSm3MFA65uSnFvWukMVqqRMReQc5jvpxHKCis+XvnHzyOfL0gW9ZTi7 +tWGGbBi0TRJCa8BkvgngUZxOxUlMfg/7cVxOIB0TPoUSgxFd/+qVz4GZMvr0dPu7 +eAF7J/8ECVvb0wSPTUI1N3c= -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV -BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw -MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH -Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k -YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw -gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 -6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt -pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw -FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd -BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G -lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 -CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +MIIDWTCCAkGgAwIBAgIJAPm6B21bar2bMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODAx +MTkxOTA5MDZaFw0yODAxMTcxOTA5MDZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +DA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAKvvsX2gEti4shve3iYMc+jE4Se7WHs1Bol2f21H8qNboDOFdeb1 +RKHjmq3exHpajywOUEgne9nKHJY/3f2phR4Y5klqG6liLgiSpVyRlcBGbeT2qEAj +9oLiLFUXLGfGDds2mTwivQDLJBWi51j7ff5k2Pr58fN5ugYMn24T9FNyn0moT+qj +SFoBNm58l9jrdkJSlgWfqPlbiMa+mqDn/SFtrwLF2Trbfzu42Sd9UdIzMaSSrzbN +sGm53pNhCh8KndWUQ8GPP2IsLPoUU4qAtmZuTxCx2S1cXrN9EkmT69tlOH84YfSn +96Ih9bWRc7M5y5bfVdEVM+fKQl3hBRf05qMCAwEAAaMYMBYwFAYDVR0RBA0wC4IJ +bG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IBAQAtQ8f37cCEk7/rAcbYR53ce3iK +Vpihb0U2ni1QjG9Tg9UIExkIGkwTiCm7kwQL+GEStBu9AG/QVrOjeTriRiddhWkk +ze8kRaI3AC/63t6Vh9Q1x6PESgeE4OtAO9JpJCf4GILglA789Y/b/GF8zJZQxR13 +qpB4ZwWw7gCBhdEW59u6CFeBmfDa58hM8lWvuVoRrTi7bjUeC6PAn5HVMzZSykhu +4HaUfBp6bKFjuym2+h/VvM1n8C3chjVSmutsLb6ELdD8IK0vPV/yf5+LN256zSsS +dyUZYd8XwQaioEMKdbhLvnehyzHiWfQIUR3BdhONxoIJhHv/EAo8eCkHHYIF -----END CERTIFICATE----- diff --git a/Lib/test/keycert2.pem b/Lib/test/keycert2.pem index e8a9e082b31..bb5fa65a8ac 100644 --- a/Lib/test/keycert2.pem +++ b/Lib/test/keycert2.pem @@ -1,31 +1,49 @@ -----BEGIN PRIVATE KEY----- -MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBAJnsJZVrppL+W5I9 -zGQrrawWwE5QJpBK9nWw17mXrZ03R1cD9BamLGivVISbPlRlAVnZBEyh1ATpsB7d -CUQ+WHEvALquvx4+Yw5l+fXeiYRjrLRBYZuVy8yNtXzU3iWcGObcYRkUdiXdOyP7 -sLF2YZHRvQZpzgDBKkrraeQ81w21AgMBAAECgYBEm7n07FMHWlE+0kT0sXNsLYfy -YE+QKZnJw9WkaDN+zFEEPELkhZVt5BjsMraJr6v2fIEqF0gGGJPkbenffVq2B5dC -lWUOxvJHufMK4sM3Cp6s/gOp3LP+QkzVnvJSfAyZU6l+4PGX5pLdUsXYjPxgzjzL -S36tF7/2Uv1WePyLUQJBAMsPhYzUXOPRgmbhcJiqi9A9c3GO8kvSDYTCKt3VMnqz -HBn6MQ4VQasCD1F+7jWTI0FU/3vdw8non/Fj8hhYqZcCQQDCDRdvmZqDiZnpMqDq -L6ZSrLTVtMvZXZbgwForaAD9uHj51TME7+eYT7EG2YCgJTXJ4YvRJEnPNyskwdKt -vTSTAkEAtaaN/vyemEJ82BIGStwONNw0ILsSr5cZ9tBHzqiA/tipY+e36HRFiXhP -QcU9zXlxyWkDH8iz9DSAmE2jbfoqwwJANlMJ65E543cjIlitGcKLMnvtCCLcKpb7 -xSG0XJB6Lo11OKPJ66jp0gcFTSCY1Lx2CXVd+gfJrfwI1Pp562+bhwJBAJ9IfDPU -R8OpO9v1SGd8x33Owm7uXOpB9d63/T70AD1QOXjKUC4eXYbt0WWfWuny/RNPRuyh -w7DXSfUF+kPKolU= +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQC3ulRNfhbOAey/ +B+wIVYx+d5az7EV4riR6yi/qE6G+bxbTvay2pqySHtDweuaYSh2cVmcasBKKIFJm +rCD1zR8UmLb5i2XFIina1t3eePCuBZMrvZZwkzlQUSM1AZtjGOO/W0I3FwO6y645 +9xA5PduKI7SMYkH/VL3zE5W1JwMovv6bvNiT+GU5l6mB9ylCTgLpmUqoQhRqz/35 +zCzVyoh+ppDvVcpWYfvXywsXsgQwbAF0QJm8SSFi0TZm5ykv4WE16afQp08yuZS0 +3U4K3MJCa4rxO58edcxBopWYfQ29K3iINM8enRfr5q+u5mAAbALAEEvyFjgLWl/u +7arxn7bJAgMBAAECggEBAJfMt8KfHzBunrDnVrk8FayYGkfmOzAOkc1yKEx6k/TH +zFB+Mqlm5MaF95P5t3S0J+r36JBAUdEWC38RUNpF9BwMYYGlDxzlsTdCuGYL/q+J +o6NMLXQt7/jQUQqGnWAvPFzqhbcGqOo5R2ZVH25sEWv9PDuRI35XAepIkDTwWsfa +P6UcJJoP+4v9B++fb3sSL4zNwp1BqS4wxR8YTR0t1zQqOxJ5BGPw1J8aBMs1sq5t +qyosAQAT63kLrdqWotHaM26QxjqEQUMlh12XMWb5GdBXUxbvyGtEabsqskGa/f8B +RdHE437J8D8l+jxb2mZLzrlaH3dq2tbFGCe1rT8qLRECgYEA5CWIvoD/YnQydLGA +OlEhCSocqURuqcotg9Ev0nt/C60jkr/NHFLGppz9lhqjIDjixt3sIMGZMFzxRtwM +pSYal3XiR7rZuHau9iM35yDhpuytEiGbYy1ADakJRzY5jq/Qa8RfPP9Atua5xAeP +q6DiSnq9vhHv9G+O4MxzHBmrw9sCgYEAziiJWFthcwvuXn3Jv9xFYKEb/06puZAx +EgQCz/3rPzv5fmGD/sKVo1U/K4z/eA82DNeKG8QRTFJCxT8TCNRxOmGV7HdCYo/B +4BTNNvbKcdi3l0j75kKoADg+nt5CD5lz6gLG0GrUEnVO1y5HVfCTb3BEAfa36C85 +9i0sfQGiwysCgYEAuus9k8cgdct5oz3iLuVVSark/JGCkT2B+OOkaLChsDFUWeEm +7TOsaclpwldkmvvAYOplkZjMJ2GelE2pVo1XcAw3LkmaI5WpVyQXoxe/iQGT8qzy +IFlsh0Scw2lb0tmcyw6CcPk4TiHOxRrkzNrtS9QwLM+JZx0XVHptPPKTVc0CgYAu +j/VFYY5G/8Dc0qhIjyWUR48dQNUQtkJ/ASzpcT46z/7vznKTjbtiYpSb74KbyUO5 +7sygrM4DYOj3x+Eys1jHiNbly6HQxQtS4x/edCsRP5NntfI+9XsgYZOzKhvdjhki +F3J0DEzNxnUCIM+311hVaRPTJbgv1srOkTFlIoNydQKBgQC6/OHGaC/OewQqRlRK +Mg5KZm01/pk4iKrpA5nG7OTAeoa70NzXNtG8J3WnaJ4mWanNwNUOyRMAMrsUAy9q +EeGqHM5mMFpY4TeVuNLL21lu/x3KYw6mKL3Ctinn+JLAoYoqEy8deZnEA5/tjYlz +YhFBchnUicjoUN1chdpM6SpV2Q== -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- -MIICXTCCAcagAwIBAgIJAIO3upAG445fMA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV -BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMTDGZha2Vob3N0bmFtZTAeFw0x -MDEwMDkxNTAxMDBaFw0yMDEwMDYxNTAxMDBaMGIxCzAJBgNVBAYTAlhZMRcwFQYD -VQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZv -dW5kYXRpb24xFTATBgNVBAMTDGZha2Vob3N0bmFtZTCBnzANBgkqhkiG9w0BAQEF -AAOBjQAwgYkCgYEAmewllWumkv5bkj3MZCutrBbATlAmkEr2dbDXuZetnTdHVwP0 -FqYsaK9UhJs+VGUBWdkETKHUBOmwHt0JRD5YcS8Auq6/Hj5jDmX59d6JhGOstEFh -m5XLzI21fNTeJZwY5txhGRR2Jd07I/uwsXZhkdG9BmnOAMEqSutp5DzXDbUCAwEA -AaMbMBkwFwYDVR0RBBAwDoIMZmFrZWhvc3RuYW1lMA0GCSqGSIb3DQEBBQUAA4GB -AH+iMClLLGSaKWgwXsmdVo4FhTZZHo8Uprrtg3N9FxEeE50btpDVQysgRt5ias3K -m+bME9zbKwvbVWD5zZdjus4pDgzwF/iHyccL8JyYhxOvS/9zmvAtFXj/APIIbZFp -IT75d9f88ScIGEtknZQejnrdhB64tYki/EqluiuKBqKD +MIIDYjCCAkqgAwIBAgIJALJXRr8qF6oIMA0GCSqGSIb3DQEBCwUAMGIxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMMDGZha2Vob3N0bmFtZTAeFw0x +ODAxMTkxOTA5MDZaFw0yODAxMTcxOTA5MDZaMGIxCzAJBgNVBAYTAlhZMRcwFQYD +VQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZv +dW5kYXRpb24xFTATBgNVBAMMDGZha2Vob3N0bmFtZTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBALe6VE1+Fs4B7L8H7AhVjH53lrPsRXiuJHrKL+oTob5v +FtO9rLamrJIe0PB65phKHZxWZxqwEoogUmasIPXNHxSYtvmLZcUiKdrW3d548K4F +kyu9lnCTOVBRIzUBm2MY479bQjcXA7rLrjn3EDk924ojtIxiQf9UvfMTlbUnAyi+ +/pu82JP4ZTmXqYH3KUJOAumZSqhCFGrP/fnMLNXKiH6mkO9VylZh+9fLCxeyBDBs +AXRAmbxJIWLRNmbnKS/hYTXpp9CnTzK5lLTdTgrcwkJrivE7nx51zEGilZh9Db0r +eIg0zx6dF+vmr67mYABsAsAQS/IWOAtaX+7tqvGftskCAwEAAaMbMBkwFwYDVR0R +BBAwDoIMZmFrZWhvc3RuYW1lMA0GCSqGSIb3DQEBCwUAA4IBAQCZhHhGItpkqhEq +ntMRd6Hv0GoOJixNvgeMwK4NJSRT/no3OirtUTzccn46h+SWibSa2eVssAV+pAVJ +HbzkN/DH27A1mMx1zJL1ekcOKA1AF6MXhUnrUGXMqW36YNtzHfXJLrwvpLJ13OQg +/Kxo4Nw68bGzM+PyRtKU/mpgYyfcvwR+ZSeIDh1fvUZK/IEVCf8ub42GPVs5wPfv +M+k5aHxWTxeif3K1byTRzxHupYNG2yWO4XEdnBGOuOwzzN4/iQyNcsuQKeuKHGrt +YvIlG/ri04CQ7xISZCj74yjTZ+/A2bXre2mQXAHqKPumHL7cl34+erzbUaxYxbTE +u5FcOmLQ -----END CERTIFICATE----- diff --git a/Lib/test/keycert3.pem b/Lib/test/keycert3.pem index 5bfa62c4ca3..621eb08bb0c 100644 --- a/Lib/test/keycert3.pem +++ b/Lib/test/keycert3.pem @@ -1,73 +1,132 @@ -----BEGIN PRIVATE KEY----- -MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMLgD0kAKDb5cFyP -jbwNfR5CtewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM -9z2j1OlaN+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZ -aggEdkj1TsSsv1zWIYKlPIjlvhuxAgMBAAECgYA0aH+T2Vf3WOPv8KdkcJg6gCRe -yJKXOWgWRcicx/CUzOEsTxmFIDPLxqAWA3k7v0B+3vjGw5Y9lycV/5XqXNoQI14j -y09iNsumds13u5AKkGdTJnZhQ7UKdoVHfuP44ZdOv/rJ5/VD6F4zWywpe90pcbK+ -AWDVtusgGQBSieEl1QJBAOyVrUG5l2yoUBtd2zr/kiGm/DYyXlIthQO/A3/LngDW -5/ydGxVsT7lAVOgCsoT+0L4efTh90PjzW8LPQrPBWVMCQQDS3h/FtYYd5lfz+FNL -9CEe1F1w9l8P749uNUD0g317zv1tatIqVCsQWHfVHNdVvfQ+vSFw38OORO00Xqs9 -1GJrAkBkoXXEkxCZoy4PteheO/8IWWLGGr6L7di6MzFl1lIqwT6D8L9oaV2vynFT -DnKop0pa09Unhjyw57KMNmSE2SUJAkEArloTEzpgRmCq4IK2/NpCeGdHS5uqRlbh -1VIa/xGps7EWQl5Mn8swQDel/YP3WGHTjfx7pgSegQfkyaRtGpZ9OQJAa9Vumj8m -JAAtI0Bnga8hgQx7BhTQY4CadDxyiRGOGYhwUzYVCqkb2sbVRH9HnwUaJT7cWBY3 -RnJdHOMXWem7/w== +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDgV4G+Zzf2DT5n +oAisIGFhn/bz7Vn5WiXUqbDsxROJOh/7BtOlduZka0pPhFylGbnxS8l1kEWHRI2T +6hOoWzumB6ItKiN+T5J30lAvSyo7iwdFoAQ/S5nPXQfhNARQe/NEOhRtpcuNdyx4 +BWdPdPuJQA1ASNJCLwcLOoRxaLbKLvb2V5T5FCAkeNPtRvPuT4gKQItMmiHfAhoV +C8MZWF/GC0RukHINys5MwqeFexam8CznmQPMYrLdhmKTj3DTivCPoh97EDIFGlgZ +SCaaYDVQA+aqlo/q2pi52PFwC1KzhNEA7EeOqSwC1NQjwjHuhcnf9WbxrgTq2zh3 +rv5YEW2ZAgMBAAECggEAPfSMtTumPcJskIumuXp7yk02EyliZrWZqwBuBwVqHsS5 +nkbFXnXWrLbgn9MrDsFrE5NdgKUmPnQVMVs8sIr5jyGejSCNCs4I4iRn1pfIgwcj +K/xEEALd6GGF0pDd/CgvB5GOoLVf4KKf2kmLvWrOKJpSzoUN5A8+v8AaYYOMr4sC +czbvfGomzEIewEG+Rw9zOVUDlmwyEKPQZ47E7PQ+EEA7oeFdR+1Zj6eT9ndegf8B +54frySYCLRUCk/sHCpWhaJBtBrcpht7Y8CfY7hiH/7x866fvuLnYPz4YALtUb0wN +7zUCNS9ol3n4LbjFFKfZtiRjKaCBRzMjK0rz6ydFcQKBgQDyLI3oGbnW73vqcDe/ +6eR0w++fiCAVhfMs3AO/gOaJi2la2JHlJ5u+cIHQIOFwEhn6Zq0AtdmnFx1TS5IQ +C0MdXI0XoQQw7rEF8EJcvfe85Z0QxENVhzydtdb8QpJfnQGfBfLyQlaaRYzRRHB6 +VdYUHF3EIPVIhbjbghal+Qep/QKBgQDtJlRPHkWwTMevu0J0fYbWN1ywtVTFUR// +k7VyORSf8yuuSnaQRop4cbcqONxmDKH6Or1fl3NYBsAxtXkkOK1E2OZNo2sfQdRa +wpA7o7mPHRhztQFpT5vflp+8P6+PEFat8D04eBOhNwrwwfhiPjD4gv5KvN4XutRW +VWv/2pnmzQKBgHPvHGg2mJ7quvm6ixXW1MWJX1eSBToIjCe3lBvDi5nhIaiZ8Q4w +7gA3QA3xD7tlDwauzLeAVxgEmsdbcCs6GQEfY3QiYy1Bt4FOSZa4YrcNfSmfq1Rw +j3Y4rRjKjeQz96i3YlzToT3tecJc7zPBj+DEy6au2H3Fdn+vQURneWHJAoGBANG7 +XES8mRVaUh/wlM1BVsaNH8SIGfiHzqzRjV7/bGYpQTBbWpAuUrhCmaMVtpXqBjav +TFwGLVRkZAWSYRjPpy2ERenT5SE3rv61o6mbGrifGsj6A82HQmtzYsGx8SmtYXtj +REF0sKebbmmOooUAS379GrguYJzL9o6D7YfRZNrhAoGAVfb/tiFU4S67DSpYpQey +ULhgfsFpDByICY6Potsg67gVFf9jIaB83NPTx3u/r6sHFgxFw7lQsuZcgSuWMu7t +glzOXVIP11Y5sl5CJ5OsfeK1/0umMZF5MWPyAQCx/qrPlZL86vXjt24Y/VaOxsAi +CZYdyJsjgOrJrWoMbo5ta54= -----END PRIVATE KEY----- Certificate: Data: - Version: 1 (0x0) - Serial Number: 12723342612721443281 (0xb09264b1f2da21d1) + Version: 3 (0x2) + Serial Number: + 82:ed:bf:41:c8:80:91:9c Signature Algorithm: sha1WithRSAEncryption Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity - Not Before: Jan 4 19:47:07 2013 GMT - Not After : Nov 13 19:47:07 2022 GMT + Not Before: Jan 19 19:09:06 2018 GMT + Not After : Nov 28 19:09:06 2027 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost Subject Public Key Info: Public Key Algorithm: rsaEncryption - Public-Key: (1024 bit) + Public-Key: (2048 bit) Modulus: - 00:c2:e0:0f:49:00:28:36:f9:70:5c:8f:8d:bc:0d: - 7d:1e:42:b5:ec:1d:5c:2f:a4:31:70:16:0f:c0:cb: - c6:24:d3:be:13:16:ee:a5:67:97:03:a6:df:a9:99: - 96:cc:c7:2a:fb:11:7f:4e:65:4f:8a:5e:82:21:4c: - f7:3d:a3:d4:e9:5a:37:e7:22:fd:7e:cd:53:6d:93: - 34:de:9c:ad:84:a2:37:be:c5:8d:82:4f:e3:ae:23: - f3:be:a7:75:2c:72:0f:ea:f3:ca:cd:fc:e9:3f:b5: - af:56:99:6a:08:04:76:48:f5:4e:c4:ac:bf:5c:d6: - 21:82:a5:3c:88:e5:be:1b:b1 + 00:e0:57:81:be:67:37:f6:0d:3e:67:a0:08:ac:20: + 61:61:9f:f6:f3:ed:59:f9:5a:25:d4:a9:b0:ec:c5: + 13:89:3a:1f:fb:06:d3:a5:76:e6:64:6b:4a:4f:84: + 5c:a5:19:b9:f1:4b:c9:75:90:45:87:44:8d:93:ea: + 13:a8:5b:3b:a6:07:a2:2d:2a:23:7e:4f:92:77:d2: + 50:2f:4b:2a:3b:8b:07:45:a0:04:3f:4b:99:cf:5d: + 07:e1:34:04:50:7b:f3:44:3a:14:6d:a5:cb:8d:77: + 2c:78:05:67:4f:74:fb:89:40:0d:40:48:d2:42:2f: + 07:0b:3a:84:71:68:b6:ca:2e:f6:f6:57:94:f9:14: + 20:24:78:d3:ed:46:f3:ee:4f:88:0a:40:8b:4c:9a: + 21:df:02:1a:15:0b:c3:19:58:5f:c6:0b:44:6e:90: + 72:0d:ca:ce:4c:c2:a7:85:7b:16:a6:f0:2c:e7:99: + 03:cc:62:b2:dd:86:62:93:8f:70:d3:8a:f0:8f:a2: + 1f:7b:10:32:05:1a:58:19:48:26:9a:60:35:50:03: + e6:aa:96:8f:ea:da:98:b9:d8:f1:70:0b:52:b3:84: + d1:00:ec:47:8e:a9:2c:02:d4:d4:23:c2:31:ee:85: + c9:df:f5:66:f1:ae:04:ea:db:38:77:ae:fe:58:11: + 6d:99 Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:localhost + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 85:11:BE:16:47:04:D1:30:EE:86:8A:18:70:BE:A8:28:6F:82:3D:CE + X509v3 Authority Key Identifier: + keyid:9A:CF:CF:6E:EB:71:3D:DB:3C:F1:AE:88:6B:56:72:03:CB:08:A7:48 + DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server + serial:82:ED:BF:41:C8:80:91:9B + + Authority Information Access: + CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer + OCSP - URI:http://testca.pythontest.net/testca/ocsp/ + + X509v3 CRL Distribution Points: + + Full Name: + URI:http://testca.pythontest.net/testca/revocation.crl + Signature Algorithm: sha1WithRSAEncryption - 2f:42:5f:a3:09:2c:fa:51:88:c7:37:7f:ea:0e:63:f0:a2:9a: - e5:5a:e2:c8:20:f0:3f:60:bc:c8:0f:b6:c6:76:ce:db:83:93: - f5:a3:33:67:01:8e:04:cd:00:9a:73:fd:f3:35:86:fa:d7:13: - e2:46:c6:9d:c0:29:53:d4:a9:90:b8:77:4b:e6:83:76:e4:92: - d6:9c:50:cf:43:d0:c6:01:77:61:9a:de:9b:70:f7:72:cd:59: - 00:31:69:d9:b4:ca:06:9c:6d:c3:c7:80:8c:68:e6:b5:a2:f8: - ef:1d:bb:16:9f:77:77:ef:87:62:22:9b:4d:69:a4:3a:1a:f1: - 21:5e:8c:32:ac:92:fd:15:6b:18:c2:7f:15:0d:98:30:ca:75: - 8f:1a:71:df:da:1d:b2:ef:9a:e8:2d:2e:02:fd:4a:3c:aa:96: - 0b:06:5d:35:b3:3d:24:87:4b:e0:b0:58:60:2f:45:ac:2e:48: - 8a:b0:99:10:65:27:ff:cc:b1:d8:fd:bd:26:6b:b9:0c:05:2a: - f4:45:63:35:51:07:ed:83:85:fe:6f:69:cb:bb:40:a8:ae:b6: - 3b:56:4a:2d:a4:ed:6d:11:2c:4d:ed:17:24:fd:47:bc:d3:41: - a2:d3:06:fe:0c:90:d8:d8:94:26:c4:ff:cc:a1:d8:42:77:eb: - fc:a9:94:71 + 7f:a1:7e:3e:68:01:b0:32:b8:57:b8:03:68:13:13:b3:e3:f4: + 70:2f:15:e5:0f:87:b9:fd:e0:12:e3:16:f2:91:53:c7:4e:25: + af:ca:cb:a7:d9:9d:57:4d:bf:a2:80:d4:78:aa:04:31:fd:6d: + cc:6d:82:43:e9:62:16:0d:0e:26:8b:e7:f1:3d:57:5c:68:02: + 9c:2b:b6:c9:fd:62:2f:10:85:88:cc:44:a5:e7:a2:3e:89:f2: + 1f:02:6a:3f:d0:3c:6c:24:2d:bc:51:62:7a:ec:25:c5:86:87: + 77:35:8f:f9:7e:d0:17:3d:77:56:bf:1a:0c:be:09:78:ee:ea: + 73:97:65:60:94:91:35:b3:5c:46:8a:5e:6d:94:52:de:48:b7: + 1f:6c:28:79:7f:ff:08:8d:e4:7d:d0:b9:0b:7c:ae:c4:1d:2a: + a1:b3:50:11:82:03:5e:6c:e7:26:fa:05:32:39:07:83:49:b9: + a2:fa:04:da:0d:e5:ff:4c:db:97:d0:c3:a7:43:37:4c:16:de: + 3c:b5:e9:7e:82:d4:b3:10:df:d1:c1:66:72:9c:15:67:19:3b: + 7b:91:0a:82:07:67:c5:06:03:5f:80:54:08:81:8a:b1:5c:7c: + 4c:d2:07:38:92:eb:12:f5:71:ae:de:05:15:c8:e1:33:f0:e4: + 96:0f:0f:1e -----BEGIN CERTIFICATE----- -MIICpDCCAYwCCQCwkmSx8toh0TANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY -WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV -BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3 -WjBfMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV -BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRIwEAYDVQQDEwlsb2NhbGhv -c3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMLgD0kAKDb5cFyPjbwNfR5C -tewdXC+kMXAWD8DLxiTTvhMW7qVnlwOm36mZlszHKvsRf05lT4pegiFM9z2j1Ola -N+ci/X7NU22TNN6crYSiN77FjYJP464j876ndSxyD+rzys386T+1r1aZaggEdkj1 -TsSsv1zWIYKlPIjlvhuxAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAC9CX6MJLPpR -iMc3f+oOY/CimuVa4sgg8D9gvMgPtsZ2ztuDk/WjM2cBjgTNAJpz/fM1hvrXE+JG -xp3AKVPUqZC4d0vmg3bkktacUM9D0MYBd2Ga3ptw93LNWQAxadm0ygacbcPHgIxo -5rWi+O8duxafd3fvh2Iim01ppDoa8SFejDKskv0VaxjCfxUNmDDKdY8acd/aHbLv -mugtLgL9SjyqlgsGXTWzPSSHS+CwWGAvRawuSIqwmRBlJ//Msdj9vSZruQwFKvRF -YzVRB+2Dhf5vacu7QKiutjtWSi2k7W0RLE3tFyT9R7zTQaLTBv4MkNjYlCbE/8yh -2EJ36/yplHE= +MIIE8TCCA9mgAwIBAgIJAILtv0HIgJGcMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDZaFw0yNzExMjgx +OTA5MDZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj +MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxv +Y2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOBXgb5nN/YN +PmegCKwgYWGf9vPtWflaJdSpsOzFE4k6H/sG06V25mRrSk+EXKUZufFLyXWQRYdE +jZPqE6hbO6YHoi0qI35PknfSUC9LKjuLB0WgBD9Lmc9dB+E0BFB780Q6FG2ly413 +LHgFZ090+4lADUBI0kIvBws6hHFotsou9vZXlPkUICR40+1G8+5PiApAi0yaId8C +GhULwxlYX8YLRG6Qcg3KzkzCp4V7FqbwLOeZA8xist2GYpOPcNOK8I+iH3sQMgUa +WBlIJppgNVAD5qqWj+ramLnY8XALUrOE0QDsR46pLALU1CPCMe6Fyd/1ZvGuBOrb +OHeu/lgRbZkCAwEAAaOCAcAwggG8MBQGA1UdEQQNMAuCCWxvY2FsaG9zdDAOBgNV +HQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1Ud +EwEB/wQCMAAwHQYDVR0OBBYEFIURvhZHBNEw7oaKGHC+qChvgj3OMH0GA1UdIwR2 +MHSAFJrPz27rcT3bPPGuiGtWcgPLCKdIoVGkTzBNMQswCQYDVQQGEwJYWTEmMCQG +A1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91 +ci1jYS1zZXJ2ZXKCCQCC7b9ByICRmzCBgwYIKwYBBQUHAQEEdzB1MDwGCCsGAQUF +BzAChjBodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9weWNhY2Vy +dC5jZXIwNQYIKwYBBQUHMAGGKWh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQv +dGVzdGNhL29jc3AvMEMGA1UdHwQ8MDowOKA2oDSGMmh0dHA6Ly90ZXN0Y2EucHl0 +aG9udGVzdC5uZXQvdGVzdGNhL3Jldm9jYXRpb24uY3JsMA0GCSqGSIb3DQEBBQUA +A4IBAQB/oX4+aAGwMrhXuANoExOz4/RwLxXlD4e5/eAS4xbykVPHTiWvysun2Z1X +Tb+igNR4qgQx/W3MbYJD6WIWDQ4mi+fxPVdcaAKcK7bJ/WIvEIWIzESl56I+ifIf +Amo/0DxsJC28UWJ67CXFhod3NY/5ftAXPXdWvxoMvgl47upzl2VglJE1s1xGil5t +lFLeSLcfbCh5f/8IjeR90LkLfK7EHSqhs1ARggNebOcm+gUyOQeDSbmi+gTaDeX/ +TNuX0MOnQzdMFt48tel+gtSzEN/RwWZynBVnGTt7kQqCB2fFBgNfgFQIgYqxXHxM +0gc4kusS9XGu3gUVyOEz8OSWDw8e -----END CERTIFICATE----- diff --git a/Lib/test/keycert4.pem b/Lib/test/keycert4.pem index 53355c8a50c..b7df7f3f2c7 100644 --- a/Lib/test/keycert4.pem +++ b/Lib/test/keycert4.pem @@ -1,73 +1,132 @@ -----BEGIN PRIVATE KEY----- -MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAK5UQiMI5VkNs2Qv -L7gUaiDdFevNUXRjU4DHAe3ZzzYLZNE69h9gO9VCSS16tJ5fT5VEu0EZyGr0e3V2 -NkX0ZoU0Hc/UaY4qx7LHmn5SYZpIxhJnkf7SyHJK1zUaGlU0/LxYqIuGCtF5dqx1 -L2OQhEx1GM6RydHdgX69G64LXcY5AgMBAAECgYAhsRMfJkb9ERLMl/oG/5sLQu9L -pWDKt6+ZwdxzlZbggQ85CMYshjLKIod2DLL/sLf2x1PRXyRG131M1E3k8zkkz6de -R1uDrIN/x91iuYzfLQZGh8bMY7Yjd2eoroa6R/7DjpElGejLxOAaDWO0ST2IFQy9 -myTGS2jSM97wcXfsSQJBANP3jelJoS5X6BRjTSneY21wcocxVuQh8pXpErALVNsT -drrFTeaBuZp7KvbtnIM5g2WRNvaxLZlAY/hXPJvi6ncCQQDSix1cebml6EmPlEZS -Mm8gwI2F9ufUunwJmBJcz826Do0ZNGByWDAM/JQZH4FX4GfAFNuj8PUb+GQfadkx -i1DPAkEA0lVsNHojvuDsIo8HGuzarNZQT2beWjJ1jdxh9t7HrTx7LIps6rb/fhOK -Zs0R6gVAJaEbcWAPZ2tFyECInAdnsQJAUjaeXXjuxFkjOFym5PvqpvhpivEx78Bu -JPTr3rAKXmfGMxxfuOa0xK1wSyshP6ZR/RBn/+lcXPKubhHQDOegwwJAJF1DBQnN -+/tLmOPULtDwfP4Zixn+/8GmGOahFoRcu6VIGHmRilJTn6MOButw7Glv2YdeC6l/ -e83Gq6ffLVfKNQ== +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDH/76hZAZH4cSV +CmVZa5HEqKCjCKrcPwBECs9BS+3ibwN4x9NnFNP+tCeFGgJXl7WGFoeXgg3oK+1p +FsOWpsRHuF3BdqkCnShSydmT8bLaGHwKeL0cPxJP5T/uW7ezPKW2VWXGMwmwRaRJ +9dj2VCUu20vDZWSGFr9zjnjoJczBtH3RsVUgpK7euEHuQ5pIM9QSOaCo+5FPR7s7 +1nU7YqbFWtd+NhC8Og1G497B31DQlHciF6BRm6/cNGAmHaAErKUGBFdkGtFPHBn4 +vktoEg9fwxJAZLvGpoTZWrB4HRsRwVTmFdGvK+JXK225xF23AXRXp/snhSuSFeLj +E5cpyJJ7AgMBAAECggEAQOv527X2e/sDr0XSpHZQuT/r9UBpBlnFIlFH+fBF5k0X +GWv0ae/O6U1dzs0kmX57xG0n0ry6+vTXeleTYiH8cTOd66EzN9AAOO+hG29IGZf9 +HAEZkkO/FARc/mjzdtFnEYsjIHWM3ZWdwQx3Q28JKu6w51rQiN51g3NqOCGdF/uF +rE5XPKsKndn+nLHvsNuApFgUYZEwdrozgUueEgRaPTUCNhzotcA9eWoBdA24XNhk +x8Cm/bZWabXm7gBO75zl3Cu2F21ay+EuwyOZTsx6lZi6YX9/zo1mkO81Zi3tQk50 +NMEI0feLNwsdxTbmOcVJadjOgd+QVghlFyr5HGBWMQKBgQD3AH3rhnAo6tOyNkGN ++IzIU1MhUS452O7IavykUYO9sM24BVChpRtlI9Dpev4yE/q3BAO3+oWT3cJrN7/3 +iyo1dzAkpGvI65XWfElXFM4nLjEiZzx4W9fiPN91Oucpr0ED6+BZXTtz4gVm0TP/ +TUc2xvTB6EKvIyWmKOYEi0snxQKBgQDPSOjbz9jWOrC9XY7PmtLB6QJDDz7XSGVK +wzD+gDAPpAwhk58BEokdOhBx2Lwl8zMJi0CRHgH2vNvkRyhvUQ4UFzisrqann/Tw +klp5sw3iWC6ERC8z9zL7GfHs7sK3mOVeAdK6ffowPM3JrZ2vPusVBdr0MN3oZwki +CtNXqbY1PwKBgGheQNbAW6wubX0kB9chavtKmhm937Z5v4vYCSC1gOEqUAKt3EAx +L74wwBmn6rjmUE382EVpCgBM99WuHONQXmlxD1qsTw763LlgkuzE0cckcYaD8L06 +saHa7uDuHrcyYlpx1L5t8q0ol/e19i6uTKUMtGcq6OJwC3yGU4sgAIWxAoGBAMVq +qiQXm2vFL+jafxYoXUvDMJ1PmskMsTP4HOR2j8+FrOwZnVk3HxGP6HOVOPRn4JbZ +YiAT1Uj6a+7I+rCyINdvmlGUcTK6fFzW9oZryvBkjcD483/pkktmVWwTpa2YV/Ml +h16IdsyUTGYlDUYHhXtbPUJOfDpIT4F1j/0wrFGfAoGAO82BcUsehEUQE0xvQLIn +7QaFtUI5z19WW730jVuEobiYlh9Ka4DPbKMvka8MwyOxEwhk39gZQavmfG6+wZm+ +kjERU23LhHziJGWS2Um4yIhC7myKbWaLzjHEq72dszLpQku4BzE5fT60fxI7cURD +WGm/Z3Q2weS3ZGIoMj1RNPI= -----END PRIVATE KEY----- Certificate: Data: - Version: 1 (0x0) - Serial Number: 12723342612721443282 (0xb09264b1f2da21d2) + Version: 3 (0x2) + Serial Number: + 82:ed:bf:41:c8:80:91:9d Signature Algorithm: sha1WithRSAEncryption Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity - Not Before: Jan 4 19:47:07 2013 GMT - Not After : Nov 13 19:47:07 2022 GMT + Not Before: Jan 19 19:09:06 2018 GMT + Not After : Nov 28 19:09:06 2027 GMT Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=fakehostname Subject Public Key Info: Public Key Algorithm: rsaEncryption - Public-Key: (1024 bit) + Public-Key: (2048 bit) Modulus: - 00:ae:54:42:23:08:e5:59:0d:b3:64:2f:2f:b8:14: - 6a:20:dd:15:eb:cd:51:74:63:53:80:c7:01:ed:d9: - cf:36:0b:64:d1:3a:f6:1f:60:3b:d5:42:49:2d:7a: - b4:9e:5f:4f:95:44:bb:41:19:c8:6a:f4:7b:75:76: - 36:45:f4:66:85:34:1d:cf:d4:69:8e:2a:c7:b2:c7: - 9a:7e:52:61:9a:48:c6:12:67:91:fe:d2:c8:72:4a: - d7:35:1a:1a:55:34:fc:bc:58:a8:8b:86:0a:d1:79: - 76:ac:75:2f:63:90:84:4c:75:18:ce:91:c9:d1:dd: - 81:7e:bd:1b:ae:0b:5d:c6:39 + 00:c7:ff:be:a1:64:06:47:e1:c4:95:0a:65:59:6b: + 91:c4:a8:a0:a3:08:aa:dc:3f:00:44:0a:cf:41:4b: + ed:e2:6f:03:78:c7:d3:67:14:d3:fe:b4:27:85:1a: + 02:57:97:b5:86:16:87:97:82:0d:e8:2b:ed:69:16: + c3:96:a6:c4:47:b8:5d:c1:76:a9:02:9d:28:52:c9: + d9:93:f1:b2:da:18:7c:0a:78:bd:1c:3f:12:4f:e5: + 3f:ee:5b:b7:b3:3c:a5:b6:55:65:c6:33:09:b0:45: + a4:49:f5:d8:f6:54:25:2e:db:4b:c3:65:64:86:16: + bf:73:8e:78:e8:25:cc:c1:b4:7d:d1:b1:55:20:a4: + ae:de:b8:41:ee:43:9a:48:33:d4:12:39:a0:a8:fb: + 91:4f:47:bb:3b:d6:75:3b:62:a6:c5:5a:d7:7e:36: + 10:bc:3a:0d:46:e3:de:c1:df:50:d0:94:77:22:17: + a0:51:9b:af:dc:34:60:26:1d:a0:04:ac:a5:06:04: + 57:64:1a:d1:4f:1c:19:f8:be:4b:68:12:0f:5f:c3: + 12:40:64:bb:c6:a6:84:d9:5a:b0:78:1d:1b:11:c1: + 54:e6:15:d1:af:2b:e2:57:2b:6d:b9:c4:5d:b7:01: + 74:57:a7:fb:27:85:2b:92:15:e2:e3:13:97:29:c8: + 92:7b Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:fakehostname + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + F8:76:79:CB:11:85:F0:46:E5:95:E6:7E:69:CB:12:5E:4E:AA:EC:4D + X509v3 Authority Key Identifier: + keyid:9A:CF:CF:6E:EB:71:3D:DB:3C:F1:AE:88:6B:56:72:03:CB:08:A7:48 + DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server + serial:82:ED:BF:41:C8:80:91:9B + + Authority Information Access: + CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer + OCSP - URI:http://testca.pythontest.net/testca/ocsp/ + + X509v3 CRL Distribution Points: + + Full Name: + URI:http://testca.pythontest.net/testca/revocation.crl + Signature Algorithm: sha1WithRSAEncryption - ad:45:8a:8e:ef:c6:ef:04:41:5c:2c:4a:84:dc:02:76:0c:d0: - 66:0f:f0:16:04:58:4d:fd:68:b7:b8:d3:a8:41:a5:5c:3c:6f: - 65:3c:d1:f8:ce:43:35:e7:41:5f:53:3d:c9:2c:c3:7d:fc:56: - 4a:fa:47:77:38:9d:bb:97:28:0a:3b:91:19:7f:bc:74:ae:15: - 6b:bd:20:36:67:45:a5:1e:79:d7:75:e6:89:5c:6d:54:84:d1: - 95:d7:a7:b4:33:3c:af:37:c4:79:8f:5e:75:dc:75:c2:18:fb: - 61:6f:2d:dc:38:65:5b:ba:67:28:d0:88:d7:8d:b9:23:5a:8e: - e8:c6:bb:db:ce:d5:b8:41:2a:ce:93:08:b6:95:ad:34:20:18: - d5:3b:37:52:74:50:0b:07:2c:b0:6d:a4:4c:7b:f4:e0:fd:d1: - af:17:aa:20:cd:62:e3:f0:9d:37:69:db:41:bd:d4:1c:fb:53: - 20:da:88:9d:76:26:67:ce:01:90:a7:80:1d:a9:5b:39:73:68: - 54:0a:d1:2a:03:1b:8f:3c:43:5d:5d:c4:51:f1:a7:e7:11:da: - 31:2c:49:06:af:04:f4:b8:3c:99:c4:20:b9:06:36:a2:00:92: - 61:1d:0c:6d:24:05:e2:82:e1:47:db:a0:5f:ba:b9:fb:ba:fa: - 49:12:1e:ce + 6d:50:8d:fb:ee:4e:93:8b:eb:47:56:ba:38:cc:80:e1:9d:c7: + e1:9e:1f:9c:22:0c:d2:08:9b:ed:bf:31:d9:00:ee:af:8c:56: + 78:92:d1:7c:ba:4e:81:7f:82:1f:f4:68:99:86:91:c6:cb:57: + d3:b9:41:12:fa:75:53:fd:22:32:21:50:af:6b:4c:b1:34:36: + d1:a8:25:0a:d0:f0:f8:81:7d:69:58:6e:af:e3:d2:c4:32:87: + 79:d7:cd:ad:0c:56:f3:15:27:10:0c:f9:57:59:53:00:ed:af: + 5d:4d:07:86:7a:e5:f3:97:88:bc:86:b4:f1:17:46:33:55:28: + 66:7b:70:d3:a5:12:b9:4f:c7:ed:e6:13:20:2d:f0:9e:ec:17: + 64:cf:fd:13:14:1b:76:ba:64:ac:c5:51:b6:cd:13:0a:93:b1: + fd:43:09:a0:0b:44:6c:77:45:43:0b:e5:ed:70:b2:76:dc:08: + 4a:5b:73:5f:c1:fc:7f:63:70:f8:b9:ca:3c:98:06:5f:fd:98: + d1:e4:e6:61:5f:09:8f:6c:18:86:98:9c:cb:3f:73:7b:3f:38: + f5:a7:09:20:ee:a5:63:1c:ff:8b:a6:d1:8c:e8:f4:84:3d:99: + 38:0f:cc:e0:52:03:f9:18:05:23:76:39:de:52:ce:8e:fb:a6: + 6e:f5:4f:c3 -----BEGIN CERTIFICATE----- -MIICpzCCAY8CCQCwkmSx8toh0jANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJY +MIIE9zCCA9+gAwIBAgIJAILtv0HIgJGdMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDZaFw0yNzExMjgx +OTA5MDZaMGIxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj +MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMMDGZh +a2Vob3N0bmFtZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMf/vqFk +BkfhxJUKZVlrkcSooKMIqtw/AEQKz0FL7eJvA3jH02cU0/60J4UaAleXtYYWh5eC +Degr7WkWw5amxEe4XcF2qQKdKFLJ2ZPxstoYfAp4vRw/Ek/lP+5bt7M8pbZVZcYz +CbBFpEn12PZUJS7bS8NlZIYWv3OOeOglzMG0fdGxVSCkrt64Qe5Dmkgz1BI5oKj7 +kU9HuzvWdTtipsVa1342ELw6DUbj3sHfUNCUdyIXoFGbr9w0YCYdoASspQYEV2Qa +0U8cGfi+S2gSD1/DEkBku8amhNlasHgdGxHBVOYV0a8r4lcrbbnEXbcBdFen+yeF +K5IV4uMTlynIknsCAwEAAaOCAcMwggG/MBcGA1UdEQQQMA6CDGZha2Vob3N0bmFt +ZTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMC +MAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFPh2ecsRhfBG5ZXmfmnLEl5OquxNMH0G +A1UdIwR2MHSAFJrPz27rcT3bPPGuiGtWcgPLCKdIoVGkTzBNMQswCQYDVQQGEwJY WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV -BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTMwMTA0MTk0NzA3WhcNMjIxMTEzMTk0NzA3 -WjBiMQswCQYDVQQGEwJYWTEXMBUGA1UEBxMOQ2FzdGxlIEFudGhyYXgxIzAhBgNV -BAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMRUwEwYDVQQDEwxmYWtlaG9z -dG5hbWUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAK5UQiMI5VkNs2QvL7gU -aiDdFevNUXRjU4DHAe3ZzzYLZNE69h9gO9VCSS16tJ5fT5VEu0EZyGr0e3V2NkX0 -ZoU0Hc/UaY4qx7LHmn5SYZpIxhJnkf7SyHJK1zUaGlU0/LxYqIuGCtF5dqx1L2OQ -hEx1GM6RydHdgX69G64LXcY5AgMBAAEwDQYJKoZIhvcNAQEFBQADggEBAK1Fio7v -xu8EQVwsSoTcAnYM0GYP8BYEWE39aLe406hBpVw8b2U80fjOQzXnQV9TPcksw338 -Vkr6R3c4nbuXKAo7kRl/vHSuFWu9IDZnRaUeedd15olcbVSE0ZXXp7QzPK83xHmP -XnXcdcIY+2FvLdw4ZVu6ZyjQiNeNuSNajujGu9vO1bhBKs6TCLaVrTQgGNU7N1J0 -UAsHLLBtpEx79OD90a8XqiDNYuPwnTdp20G91Bz7UyDaiJ12JmfOAZCngB2pWzlz -aFQK0SoDG488Q11dxFHxp+cR2jEsSQavBPS4PJnEILkGNqIAkmEdDG0kBeKC4Ufb -oF+6ufu6+kkSHs4= +BAMMDW91ci1jYS1zZXJ2ZXKCCQCC7b9ByICRmzCBgwYIKwYBBQUHAQEEdzB1MDwG +CCsGAQUFBzAChjBodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9w +eWNhY2VydC5jZXIwNQYIKwYBBQUHMAGGKWh0dHA6Ly90ZXN0Y2EucHl0aG9udGVz +dC5uZXQvdGVzdGNhL29jc3AvMEMGA1UdHwQ8MDowOKA2oDSGMmh0dHA6Ly90ZXN0 +Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3Jldm9jYXRpb24uY3JsMA0GCSqGSIb3 +DQEBBQUAA4IBAQBtUI377k6Ti+tHVro4zIDhncfhnh+cIgzSCJvtvzHZAO6vjFZ4 +ktF8uk6Bf4If9GiZhpHGy1fTuUES+nVT/SIyIVCva0yxNDbRqCUK0PD4gX1pWG6v +49LEMod5182tDFbzFScQDPlXWVMA7a9dTQeGeuXzl4i8hrTxF0YzVShme3DTpRK5 +T8ft5hMgLfCe7Bdkz/0TFBt2umSsxVG2zRMKk7H9QwmgC0Rsd0VDC+XtcLJ23AhK +W3Nfwfx/Y3D4uco8mAZf/ZjR5OZhXwmPbBiGmJzLP3N7Pzj1pwkg7qVjHP+LptGM +6PSEPZk4D8zgUgP5GAUjdjneUs6O+6Zu9U/D -----END CERTIFICATE----- diff --git a/Lib/test/keycertecc.pem b/Lib/test/keycertecc.pem new file mode 100644 index 00000000000..deb484f9920 --- /dev/null +++ b/Lib/test/keycertecc.pem @@ -0,0 +1,96 @@ +-----BEGIN PRIVATE KEY----- +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDDe3QWmhZX07HZbntz4 +CFqAOaoYMdYwD7Z3WPNIc2zR7p4D6BMOa7NAWjLV5A7CUw6hZANiAAQ5IVKzLLz4 +LCfcpy6fMOp+jk5KwywsU3upPtjA6E3UetxPcfnnv+gghRyDAYLN2OVqZgLMEmUo +F1j1SM1QrbhHIuNcVxI9gPPMdumcNFSz/hqxrBRtA/8Z2gywczdNLjc= +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 82:ed:bf:41:c8:80:91:9e + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Jan 19 19:09:06 2018 GMT + Not After : Nov 28 19:09:06 2027 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost-ecc + Subject Public Key Info: + Public Key Algorithm: id-ecPublicKey + Public-Key: (384 bit) + pub: + 04:39:21:52:b3:2c:bc:f8:2c:27:dc:a7:2e:9f:30: + ea:7e:8e:4e:4a:c3:2c:2c:53:7b:a9:3e:d8:c0:e8: + 4d:d4:7a:dc:4f:71:f9:e7:bf:e8:20:85:1c:83:01: + 82:cd:d8:e5:6a:66:02:cc:12:65:28:17:58:f5:48: + cd:50:ad:b8:47:22:e3:5c:57:12:3d:80:f3:cc:76: + e9:9c:34:54:b3:fe:1a:b1:ac:14:6d:03:ff:19:da: + 0c:b0:73:37:4d:2e:37 + ASN1 OID: secp384r1 + NIST CURVE: P-384 + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:localhost-ecc + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 33:23:0E:15:04:83:2E:3D:BF:DA:81:6D:10:38:80:C3:C2:B0:A4:74 + X509v3 Authority Key Identifier: + keyid:9A:CF:CF:6E:EB:71:3D:DB:3C:F1:AE:88:6B:56:72:03:CB:08:A7:48 + DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server + serial:82:ED:BF:41:C8:80:91:9B + + Authority Information Access: + CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer + OCSP - URI:http://testca.pythontest.net/testca/ocsp/ + + X509v3 CRL Distribution Points: + + Full Name: + URI:http://testca.pythontest.net/testca/revocation.crl + + Signature Algorithm: sha1WithRSAEncryption + 3b:6f:97:af:7e:5f:e0:14:34:ed:57:7e:de:ce:c4:85:1e:aa: + 84:52:94:7c:e5:ce:e9:9c:88:8b:ad:b5:4d:16:ac:af:81:ea: + b8:a2:e2:50:2e:cb:e9:11:bd:1b:a6:3f:0c:a2:d7:7b:67:72: + b3:43:16:ad:c6:87:ac:6e:ac:47:78:ef:2f:8c:86:e8:9b:d1: + 43:8c:c1:7a:91:30:e9:14:d6:9f:41:8b:9b:0b:24:9b:78:86: + 11:8a:fc:2b:cd:c9:13:ee:90:4f:14:33:51:a3:c4:9e:d6:06: + 48:f5:41:12:af:f0:f2:71:40:78:f5:96:c2:5d:cf:e1:38:ff: + bf:10:eb:74:2f:c2:23:21:3e:27:f5:f1:f2:af:2c:62:82:31: + 00:c8:96:1b:c3:7e:8d:71:89:e7:40:b5:67:1a:33:fb:c0:8b: + 96:0c:36:78:25:27:82:d8:27:27:52:0f:f7:69:cd:ff:2b:92: + 10:d3:d2:0a:db:65:ed:af:90:eb:db:76:f3:8a:7a:13:9e:c6: + 33:57:15:42:06:13:d6:54:49:fa:84:a7:0e:1d:14:72:ca:19: + 8e:2b:aa:a4:02:54:3c:f6:1c:23:81:7a:59:54:b0:92:65:72: + c8:e5:ba:9f:03:4e:30:f2:4d:45:85:e3:35:a8:b1:68:58:b9: + 3b:20:a3:eb +-----BEGIN CERTIFICATE----- +MIIESzCCAzOgAwIBAgIJAILtv0HIgJGeMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDZaFw0yNzExMjgx +OTA5MDZaMGMxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj +MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFjAUBgNVBAMMDWxv +Y2FsaG9zdC1lY2MwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQ5IVKzLLz4LCfcpy6f +MOp+jk5KwywsU3upPtjA6E3UetxPcfnnv+gghRyDAYLN2OVqZgLMEmUoF1j1SM1Q +rbhHIuNcVxI9gPPMdumcNFSz/hqxrBRtA/8Z2gywczdNLjejggHEMIIBwDAYBgNV +HREEETAPgg1sb2NhbGhvc3QtZWNjMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAU +BggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUMyMO +FQSDLj2/2oFtEDiAw8KwpHQwfQYDVR0jBHYwdIAUms/PbutxPds88a6Ia1ZyA8sI +p0ihUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAILtv0HIgJGb +MIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0Y2EucHl0 +aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcwAYYpaHR0 +cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYDVR0fBDww +OjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcmV2 +b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQEFBQADggEBADtvl69+X+AUNO1Xft7OxIUe +qoRSlHzlzumciIuttU0WrK+B6rii4lAuy+kRvRumPwyi13tncrNDFq3Gh6xurEd4 +7y+Mhuib0UOMwXqRMOkU1p9Bi5sLJJt4hhGK/CvNyRPukE8UM1GjxJ7WBkj1QRKv +8PJxQHj1lsJdz+E4/78Q63QvwiMhPif18fKvLGKCMQDIlhvDfo1xiedAtWcaM/vA +i5YMNnglJ4LYJydSD/dpzf8rkhDT0grbZe2vkOvbdvOKehOexjNXFUIGE9ZUSfqE +pw4dFHLKGY4rqqQCVDz2HCOBellUsJJlcsjlup8DTjDyTUWF4zWosWhYuTsgo+s= +-----END CERTIFICATE----- diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 9871a28dbf2..ce01c8ce586 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -257,12 +257,12 @@ class Regrtest: if isinstance(test, unittest.TestSuite): self._list_cases(test) elif isinstance(test, unittest.TestCase): - if support._match_test(test): + if support.match_test(test): print(test.id()) def list_cases(self): support.verbose = False - support.match_tests = self.ns.match_tests + support.set_match_tests(self.ns.match_tests) for test in self.selected: abstest = get_abs_module(self.ns, test) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 18d5bd0511a..2ca9aa87644 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -135,11 +135,6 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): # Clear ABC registries, restoring previously saved ABC registries. abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__] abs_classes = filter(isabstract, abs_classes) - if 'typing' in sys.modules: - t = sys.modules['typing'] - # These classes require special treatment because they do not appear - # in direct subclasses of collections.abc classes - abs_classes = list(abs_classes) + [t.ChainMap, t.Counter, t.DefaultDict] for abc in abs_classes: for obj in abc.__subclasses__() + [abc]: obj._abc_registry = abcs.get(obj, WeakSet()).copy() diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index dbd463435c7..12bf422c902 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -102,7 +102,7 @@ def runtest(ns, test): if use_timeout: faulthandler.dump_traceback_later(ns.timeout, exit=True) try: - support.match_tests = ns.match_tests + support.set_match_tests(ns.match_tests) # reset the environment_altered flag to detect if a test altered # the environment support.environment_altered = False diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index ce9db9a1b8b..ed63fda20c1 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -53,10 +53,11 @@ class CommonTest(seq_tests.CommonTest): self.assertEqual(str(a2), "[0, 1, 2, [...], 3]") self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]") - l0 = [] + def test_repr_deep(self): + a = self.type2test([]) for i in range(sys.getrecursionlimit() + 100): - l0 = [l0] - self.assertRaises(RecursionError, repr, l0) + a = self.type2test([a]) + self.assertRaises(RecursionError, repr, a) def test_print(self): d = self.type2test(range(200)) diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index a1ea96d42ce..5b1f033c6f8 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -629,13 +629,14 @@ class BaseSemaphoreTests(BaseTestCase): sem = self.semtype(7) sem.acquire() N = 10 + sem_results = [] results1 = [] results2 = [] phase_num = 0 def f(): - sem.acquire() + sem_results.append(sem.acquire()) results1.append(phase_num) - sem.acquire() + sem_results.append(sem.acquire()) results2.append(phase_num) b = Bunch(f, 10) b.wait_for_started() @@ -659,6 +660,7 @@ class BaseSemaphoreTests(BaseTestCase): # Final release, to let the last thread finish sem.release() b.wait_for_finished() + self.assertEqual(sem_results, [True] * (6 + 7 + 6 + 1)) def test_try_acquire(self): sem = self.semtype(2) diff --git a/Lib/test/make_ssl_certs.py b/Lib/test/make_ssl_certs.py index 4d9f01ba6a8..b908c40c11e 100644 --- a/Lib/test/make_ssl_certs.py +++ b/Lib/test/make_ssl_certs.py @@ -2,14 +2,17 @@ and friends.""" import os +import pprint import shutil import tempfile from subprocess import * req_template = """ + [ default ] + base_url = http://testca.pythontest.net/testca + [req] distinguished_name = req_distinguished_name - x509_extensions = req_x509_extensions prompt = no [req_distinguished_name] @@ -18,9 +21,26 @@ req_template = """ O = Python Software Foundation CN = {hostname} - [req_x509_extensions] + [req_x509_extensions_simple] subjectAltName = @san + [req_x509_extensions_full] + subjectAltName = @san + keyUsage = critical,keyEncipherment,digitalSignature + extendedKeyUsage = serverAuth,clientAuth + basicConstraints = critical,CA:false + subjectKeyIdentifier = hash + authorityKeyIdentifier = keyid:always,issuer:always + authorityInfoAccess = @issuer_ocsp_info + crlDistributionPoints = @crl_info + + [ issuer_ocsp_info ] + caIssuers;URI.0 = $base_url/pycacert.cer + OCSP;URI.0 = $base_url/ocsp/ + + [ crl_info ] + URI.0 = $base_url/revocation.crl + [san] DNS.1 = {hostname} {extra_san} @@ -56,7 +76,6 @@ req_template = """ private_key = pycakey.pem serial = $dir/serial RANDFILE = $dir/.rand - policy = policy_match [ policy_match ] @@ -87,7 +106,9 @@ req_template = """ here = os.path.abspath(os.path.dirname(__file__)) -def make_cert_key(hostname, sign=False, extra_san=''): + +def make_cert_key(hostname, sign=False, extra_san='', + ext='req_x509_extensions_full', key='rsa:2048'): print("creating cert for " + hostname) tempnames = [] for i in range(3): @@ -99,7 +120,8 @@ def make_cert_key(hostname, sign=False, extra_san=''): with open(req_file, 'w') as f: f.write(req) args = ['req', '-new', '-days', '3650', '-nodes', - '-newkey', 'rsa:1024', '-keyout', key_file, + '-newkey', key, '-keyout', key_file, + '-extensions', ext, '-config', req_file] if sign: with tempfile.NamedTemporaryFile(delete=False) as f: @@ -112,8 +134,15 @@ def make_cert_key(hostname, sign=False, extra_san=''): check_call(['openssl'] + args) if sign: - args = ['ca', '-config', req_file, '-out', cert_file, '-outdir', 'cadir', - '-policy', 'policy_anything', '-batch', '-infiles', reqfile ] + args = [ + 'ca', + '-config', req_file, + '-extensions', ext, + '-out', cert_file, + '-outdir', 'cadir', + '-policy', 'policy_anything', + '-batch', '-infiles', reqfile + ] check_call(['openssl'] + args) @@ -157,9 +186,21 @@ def make_ca(): args = ['ca', '-config', t.name, '-gencrl', '-out', 'revocation.crl'] check_call(['openssl'] + args) + # capath hashes depend on subject! + check_call([ + 'openssl', 'x509', '-in', 'pycacert.pem', '-out', 'capath/ceff1710.0' + ]) + shutil.copy('capath/ceff1710.0', 'capath/b1930218.0') + + +def print_cert(path): + import _ssl + pprint.pprint(_ssl._test_decode_cert(path)) + + if __name__ == '__main__': os.chdir(here) - cert, key = make_cert_key('localhost') + cert, key = make_cert_key('localhost', ext='req_x509_extensions_simple') with open('ssl_cert.pem', 'w') as f: f.write(cert) with open('ssl_key.pem', 'w') as f: @@ -177,7 +218,7 @@ if __name__ == '__main__': # For certificate matching tests make_ca() - cert, key = make_cert_key('fakehostname') + cert, key = make_cert_key('fakehostname', ext='req_x509_extensions_simple') with open('keycert2.pem', 'w') as f: f.write(key) f.write(cert) @@ -192,6 +233,13 @@ if __name__ == '__main__': f.write(key) f.write(cert) + cert, key = make_cert_key( + 'localhost-ecc', True, key='param:secp384r1.pem' + ) + with open('keycertecc.pem', 'w') as f: + f.write(key) + f.write(cert) + extra_san = [ 'otherName.1 = 1.2.3.4;UTF8:some other identifier', 'otherName.2 = 1.3.6.1.5.2.2;SEQUENCE:princ_name', @@ -211,6 +259,24 @@ if __name__ == '__main__': f.write(key) f.write(cert) + extra_san = [ + # könig (king) + 'DNS.2 = xn--knig-5qa.idn.pythontest.net', + # königsgäßchen (king's alleyway) + 'DNS.3 = xn--knigsgsschen-lcb0w.idna2003.pythontest.net', + 'DNS.4 = xn--knigsgchen-b4a3dun.idna2008.pythontest.net', + # βόλοσ (marble) + 'DNS.5 = xn--nxasmq6b.idna2003.pythontest.net', + 'DNS.6 = xn--nxasmm1c.idna2008.pythontest.net', + ] + + # IDN SANS, signed + cert, key = make_cert_key('idnsans', True, extra_san='\n'.join(extra_san)) + with open('idnsans.pem', 'w') as f: + f.write(key) + f.write(cert) + unmake_ca() - print("\n\nPlease change the values in test_ssl.py, test_parse_cert function related to notAfter,notBefore and serialNumber") - check_call(['openssl','x509','-in','keycert.pem','-dates','-serial','-noout']) + print("update Lib/test/test_ssl.py and Lib/test/test_asyncio/util.py") + print_cert('keycert.pem') + print_cert('keycert3.pem') diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index ff82f4eb7d8..53f29f60538 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -1,6 +1,7 @@ # tests common to dict and UserDict import unittest import collections +import sys class BasicTestMappingProtocol(unittest.TestCase): @@ -619,6 +620,14 @@ class TestHashMappingProtocol(TestMappingProtocol): d = self._full_mapping({1: BadRepr()}) self.assertRaises(Exc, repr, d) + def test_repr_deep(self): + d = self._empty_mapping() + for i in range(sys.getrecursionlimit() + 100): + d0 = d + d = self._empty_mapping() + d[1] = d0 + self.assertRaises(RecursionError, repr, d) + def test_eq(self): self.assertEqual(self._empty_mapping(), self._empty_mapping()) self.assertEqual(self._full_mapping({1: 2}), diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index bf6116b2dfb..b84b87861d0 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -2037,27 +2037,54 @@ class AbstractPickleTests(unittest.TestCase): # Exercise framing (proto >= 4) for significant workloads + FRAME_SIZE_MIN = 4 FRAME_SIZE_TARGET = 64 * 1024 def check_frame_opcodes(self, pickled): """ Check the arguments of FRAME opcodes in a protocol 4+ pickle. + + Note that binary objects that are larger than FRAME_SIZE_TARGET are not + framed by default and are therefore considered a frame by themselves in + the following consistency check. """ - frame_opcode_size = 9 - last_arg = last_pos = None + frame_end = frameless_start = None + frameless_opcodes = {'BINBYTES', 'BINUNICODE', 'BINBYTES8', 'BINUNICODE8'} for op, arg, pos in pickletools.genops(pickled): - if op.name != 'FRAME': - continue - if last_pos is not None: - # The previous frame's size should be equal to the number - # of bytes up to the current frame. - frame_size = pos - last_pos - frame_opcode_size - self.assertEqual(frame_size, last_arg) - last_arg, last_pos = arg, pos - # The last frame's size should be equal to the number of bytes up - # to the pickle's end. - frame_size = len(pickled) - last_pos - frame_opcode_size - self.assertEqual(frame_size, last_arg) + if frame_end is not None: + self.assertLessEqual(pos, frame_end) + if pos == frame_end: + frame_end = None + + if frame_end is not None: # framed + self.assertNotEqual(op.name, 'FRAME') + if op.name in frameless_opcodes: + # Only short bytes and str objects should be written + # in a frame + self.assertLessEqual(len(arg), self.FRAME_SIZE_TARGET) + + else: # not framed + if (op.name == 'FRAME' or + (op.name in frameless_opcodes and + len(arg) > self.FRAME_SIZE_TARGET)): + # Frame or large bytes or str object + if frameless_start is not None: + # Only short data should be written outside of a frame + self.assertLess(pos - frameless_start, + self.FRAME_SIZE_MIN) + frameless_start = None + elif frameless_start is None and op.name != 'PROTO': + frameless_start = pos + + if op.name == 'FRAME': + self.assertGreaterEqual(arg, self.FRAME_SIZE_MIN) + frame_end = pos + 9 + arg + + pos = len(pickled) + if frame_end is not None: + self.assertEqual(frame_end, pos) + elif frameless_start is not None: + self.assertLess(pos - frameless_start, self.FRAME_SIZE_MIN) def test_framing_many_objects(self): obj = list(range(10**5)) @@ -2076,15 +2103,35 @@ class AbstractPickleTests(unittest.TestCase): def test_framing_large_objects(self): N = 1024 * 1024 - obj = [b'x' * N, b'y' * N, b'z' * N] + small_items = [[i] for i in range(10)] + obj = [b'x' * N, *small_items, b'y' * N, 'z' * N] for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): - with self.subTest(proto=proto): - pickled = self.dumps(obj, proto) - unpickled = self.loads(pickled) - self.assertEqual(obj, unpickled) - n_frames = count_opcode(pickle.FRAME, pickled) - self.assertGreaterEqual(n_frames, len(obj)) - self.check_frame_opcodes(pickled) + for fast in [False, True]: + with self.subTest(proto=proto, fast=fast): + if not fast: + # fast=False by default. + # This covers in-memory pickling with pickle.dumps(). + pickled = self.dumps(obj, proto) + else: + # Pickler is required when fast=True. + if not hasattr(self, 'pickler'): + continue + buf = io.BytesIO() + pickler = self.pickler(buf, protocol=proto) + pickler.fast = fast + pickler.dump(obj) + pickled = buf.getvalue() + unpickled = self.loads(pickled) + # More informative error message in case of failure. + self.assertEqual([len(x) for x in obj], + [len(x) for x in unpickled]) + # Perform full equality check if the lengths match. + self.assertEqual(obj, unpickled) + n_frames = count_opcode(pickle.FRAME, pickled) + # A single frame for small objects between + # first two large objects. + self.assertEqual(n_frames, 1) + self.check_frame_opcodes(pickled) def test_optional_frames(self): if pickle.HIGHEST_PROTOCOL < 4: @@ -2111,7 +2158,9 @@ class AbstractPickleTests(unittest.TestCase): frame_size = self.FRAME_SIZE_TARGET num_frames = 20 - obj = [bytes([i]) * frame_size for i in range(num_frames)] + # Large byte objects (dict values) intermitted with small objects + # (dict keys) + obj = {i: bytes([i]) * frame_size for i in range(num_frames)} for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): pickled = self.dumps(obj, proto) @@ -2125,6 +2174,71 @@ class AbstractPickleTests(unittest.TestCase): count_opcode(pickle.FRAME, pickled)) self.assertEqual(obj, self.loads(some_frames_pickle)) + def test_framed_write_sizes_with_delayed_writer(self): + class ChunkAccumulator: + """Accumulate pickler output in a list of raw chunks.""" + def __init__(self): + self.chunks = [] + def write(self, chunk): + self.chunks.append(chunk) + def concatenate_chunks(self): + return b"".join(self.chunks) + + for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): + objects = [(str(i).encode('ascii'), i % 42, {'i': str(i)}) + for i in range(int(1e4))] + # Add a large unique ASCII string + objects.append('0123456789abcdef' * + (self.FRAME_SIZE_TARGET // 16 + 1)) + + # Protocol 4 packs groups of small objects into frames and issues + # calls to write only once or twice per frame: + # The C pickler issues one call to write per-frame (header and + # contents) while Python pickler issues two calls to write: one for + # the frame header and one for the frame binary contents. + writer = ChunkAccumulator() + self.pickler(writer, proto).dump(objects) + + # Actually read the binary content of the chunks after the end + # of the call to dump: any memoryview passed to write should not + # be released otherwise this delayed access would not be possible. + pickled = writer.concatenate_chunks() + reconstructed = self.loads(pickled) + self.assertEqual(reconstructed, objects) + self.assertGreater(len(writer.chunks), 1) + + # memoryviews should own the memory. + del objects + support.gc_collect() + self.assertEqual(writer.concatenate_chunks(), pickled) + + n_frames = (len(pickled) - 1) // self.FRAME_SIZE_TARGET + 1 + # There should be at least one call to write per frame + self.assertGreaterEqual(len(writer.chunks), n_frames) + + # but not too many either: there can be one for the proto, + # one per-frame header, one per frame for the actual contents, + # and two for the header. + self.assertLessEqual(len(writer.chunks), 2 * n_frames + 3) + + chunk_sizes = [len(c) for c in writer.chunks] + large_sizes = [s for s in chunk_sizes + if s >= self.FRAME_SIZE_TARGET] + medium_sizes = [s for s in chunk_sizes + if 9 < s < self.FRAME_SIZE_TARGET] + small_sizes = [s for s in chunk_sizes if s <= 9] + + # Large chunks should not be too large: + for chunk_size in large_sizes: + self.assertLess(chunk_size, 2 * self.FRAME_SIZE_TARGET, + chunk_sizes) + # There shouldn't bee too many small chunks: the protocol header, + # the frame headers and the large string headers are written + # in small chunks. + self.assertLessEqual(len(small_sizes), + len(large_sizes) + len(medium_sizes) + 3, + chunk_sizes) + def test_nested_names(self): global Nested class Nested: diff --git a/Lib/test/pycacert.pem b/Lib/test/pycacert.pem index 09b1f3e08ae..850fa32aef7 100644 --- a/Lib/test/pycacert.pem +++ b/Lib/test/pycacert.pem @@ -1,78 +1,79 @@ Certificate: Data: Version: 3 (0x2) - Serial Number: 12723342612721443280 (0xb09264b1f2da21d0) + Serial Number: + 82:ed:bf:41:c8:80:91:9b Signature Algorithm: sha1WithRSAEncryption Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server Validity - Not Before: Jan 4 19:47:07 2013 GMT - Not After : Jan 2 19:47:07 2023 GMT + Not Before: Jan 19 19:09:06 2018 GMT + Not After : Jan 17 19:09:06 2028 GMT Subject: C=XY, O=Python Software Foundation CA, CN=our-ca-server Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: - 00:e7:de:e9:e3:0c:9f:00:b6:a1:fd:2b:5b:96:d2: - 6f:cc:e0:be:86:b9:20:5e:ec:03:7a:55:ab:ea:a4: - e9:f9:49:85:d2:66:d5:ed:c7:7a:ea:56:8e:2d:8f: - e7:42:e2:62:28:a9:9f:d6:1b:8e:eb:b5:b4:9c:9f: - 14:ab:df:e6:94:8b:76:1d:3e:6d:24:61:ed:0c:bf: - 00:8a:61:0c:df:5c:c8:36:73:16:00:cd:47:ba:6d: - a4:a4:74:88:83:23:0a:19:fc:09:a7:3c:4a:4b:d3: - e7:1d:2d:e4:ea:4c:54:21:f3:26:db:89:37:18:d4: - 02:bb:40:32:5f:a4:ff:2d:1c:f7:d4:bb:ec:8e:cf: - 5c:82:ac:e6:7c:08:6c:48:85:61:07:7f:25:e0:5c: - e0:bc:34:5f:e0:b9:04:47:75:c8:47:0b:8d:bc:d6: - c8:68:5f:33:83:62:d2:20:44:35:b1:ad:81:1a:8a: - cd:bc:35:b0:5c:8b:47:d6:18:e9:9c:18:97:cc:01: - 3c:29:cc:e8:1e:e4:e4:c1:b8:de:e7:c2:11:18:87: - 5a:93:34:d8:a6:25:f7:14:71:eb:e4:21:a2:d2:0f: - 2e:2e:d4:62:00:35:d3:d6:ef:5c:60:4b:4c:a9:14: - e2:dd:15:58:46:37:33:26:b7:e7:2e:5d:ed:42:e4: - c5:4d + 00:c3:18:69:6b:c9:47:29:98:8e:b1:56:c2:2e:fa: + 0e:5e:bc:23:80:b3:07:62:24:d2:42:5b:f1:4a:bf: + a9:c8:21:75:c8:e3:e6:2c:1f:87:3c:6e:7c:1b:ed: + 39:32:95:b7:40:b2:60:48:c3:9a:16:08:fe:6d:67: + 88:34:3b:77:77:70:1c:70:5a:d1:1f:5f:04:21:54: + b9:0c:e3:41:85:1d:58:ee:2f:ed:f3:0e:ef:d8:23: + a1:fa:73:fb:4c:28:e0:e5:e6:4d:0b:02:52:49:86: + c7:be:7e:bd:e6:56:76:8b:70:8e:0a:8f:06:33:20: + 1d:7b:5b:aa:d0:c5:1b:ab:9b:cc:54:09:3c:bf:e4: + 40:66:f1:fb:d6:f7:16:9d:c4:19:d4:c3:f2:ff:07: + bc:6f:5a:9e:25:1b:02:4a:a5:ec:42:96:3a:70:d2: + 6c:99:2b:ce:be:e8:d2:01:ef:d5:ba:b0:cf:94:3e: + 82:d0:01:d6:4b:71:80:03:0a:12:45:86:79:81:d8: + 4b:d2:e8:b5:b7:2c:6c:9a:4c:8a:10:10:e4:e4:f5: + df:ce:84:91:ca:d1:46:e0:84:73:17:66:db:69:43: + 78:80:83:be:14:4d:f1:3e:1a:d6:6c:f5:de:45:f3: + 39:af:91:d5:3d:54:44:bf:41:cc:73:68:1a:fc:24: + db:91 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: - BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B + 9A:CF:CF:6E:EB:71:3D:DB:3C:F1:AE:88:6B:56:72:03:CB:08:A7:48 X509v3 Authority Key Identifier: - keyid:BC:DD:62:D9:76:DA:1B:D2:54:6B:CF:E0:66:9B:1E:1E:7B:56:0C:0B + keyid:9A:CF:CF:6E:EB:71:3D:DB:3C:F1:AE:88:6B:56:72:03:CB:08:A7:48 X509v3 Basic Constraints: CA:TRUE Signature Algorithm: sha1WithRSAEncryption - 7d:0a:f5:cb:8d:d3:5d:bd:99:8e:f8:2b:0f:ba:eb:c2:d9:a6: - 27:4f:2e:7b:2f:0e:64:d8:1c:35:50:4e:ee:fc:90:b9:8d:6d: - a8:c5:c6:06:b0:af:f3:2d:bf:3b:b8:42:07:dd:18:7d:6d:95: - 54:57:85:18:60:47:2f:eb:78:1b:f9:e8:17:fd:5a:0d:87:17: - 28:ac:4c:6a:e6:bc:29:f4:f4:55:70:29:42:de:85:ea:ab:6c: - 23:06:64:30:75:02:8e:53:bc:5e:01:33:37:cc:1e:cd:b8:a4: - fd:ca:e4:5f:65:3b:83:1c:86:f1:55:02:a0:3a:8f:db:91:b7: - 40:14:b4:e7:8d:d2:ee:73:ba:e3:e5:34:2d:bc:94:6f:4e:24: - 06:f7:5f:8b:0e:a7:8e:6b:de:5e:75:f4:32:9a:50:b1:44:33: - 9a:d0:05:e2:78:82:ff:db:da:8a:63:eb:a9:dd:d1:bf:a0:61: - ad:e3:9e:8a:24:5d:62:0e:e7:4c:91:7f:ef:df:34:36:3b:2f: - 5d:f5:84:b2:2f:c4:6d:93:96:1a:6f:30:28:f1:da:12:9a:64: - b4:40:33:1d:bd:de:2b:53:a8:ea:be:d6:bc:4e:96:f5:44:fb: - 32:18:ae:d5:1f:f6:69:af:b6:4e:7b:1d:58:ec:3b:a9:53:a3: - 5e:58:c8:9e + 10:25:c8:dc:0c:55:5c:cb:83:6e:79:ef:77:ec:0d:8e:0c:06: + c1:4b:0c:d6:f7:75:52:21:b8:17:4a:38:88:9d:b3:78:c4:42: + fb:b8:7c:14:38:10:fb:ac:da:11:00:5b:42:87:5e:45:9f:6d: + 4e:42:a4:9a:18:06:39:0f:45:a6:96:89:32:d6:59:b3:d3:8e: + e3:95:b6:c4:a2:4b:74:2f:67:c1:fb:bb:f9:72:6f:37:4a:e7: + f4:48:33:71:df:b8:f5:e6:41:3f:d5:d5:2f:26:09:f8:0e:92: + ff:70:ea:f6:ab:58:fb:90:04:d6:43:2e:8f:b1:fb:06:ab:69: + d0:dc:a8:f8:5b:07:f2:d4:66:1f:63:f8:5d:c1:9e:41:44:bb: + c9:e8:7d:e0:46:e4:a7:c8:32:5f:31:62:e5:1c:5c:89:dd:b7: + a2:4f:9e:0d:13:b8:5f:b1:84:53:4c:1f:ce:19:e1:01:00:5e: + bf:41:55:94:a9:a5:13:db:f2:59:f3:d6:4e:b9:9d:9d:b9:0a: + d9:b2:18:6d:7c:b1:f7:96:aa:bd:f6:f9:95:0f:4a:6e:3c:7c: + 46:5b:df:d4:78:ec:9a:dc:e2:e3:01:e6:88:77:39:93:9c:ba: + 2a:63:f9:25:4b:4f:ac:08:79:39:c6:7b:df:07:35:ba:c0:c2: + 50:bf:5a:81 -----BEGIN CERTIFICATE----- -MIIDbTCCAlWgAwIBAgIJALCSZLHy2iHQMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV +MIIDbTCCAlWgAwIBAgIJAILtv0HIgJGbMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xMzAxMDQxOTQ3MDdaFw0yMzAxMDIx -OTQ3MDdaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDZaFw0yODAxMTcx +OTA5MDZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCASIwDQYJKoZI -hvcNAQEBBQADggEPADCCAQoCggEBAOfe6eMMnwC2of0rW5bSb8zgvoa5IF7sA3pV -q+qk6flJhdJm1e3HeupWji2P50LiYiipn9Ybjuu1tJyfFKvf5pSLdh0+bSRh7Qy/ -AIphDN9cyDZzFgDNR7ptpKR0iIMjChn8Cac8SkvT5x0t5OpMVCHzJtuJNxjUArtA -Ml+k/y0c99S77I7PXIKs5nwIbEiFYQd/JeBc4Lw0X+C5BEd1yEcLjbzWyGhfM4Ni -0iBENbGtgRqKzbw1sFyLR9YY6ZwYl8wBPCnM6B7k5MG43ufCERiHWpM02KYl9xRx -6+QhotIPLi7UYgA109bvXGBLTKkU4t0VWEY3Mya35y5d7ULkxU0CAwEAAaNQME4w -HQYDVR0OBBYEFLzdYtl22hvSVGvP4GabHh57VgwLMB8GA1UdIwQYMBaAFLzdYtl2 -2hvSVGvP4GabHh57VgwLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB -AH0K9cuN0129mY74Kw+668LZpidPLnsvDmTYHDVQTu78kLmNbajFxgawr/Mtvzu4 -QgfdGH1tlVRXhRhgRy/reBv56Bf9Wg2HFyisTGrmvCn09FVwKULeheqrbCMGZDB1 -Ao5TvF4BMzfMHs24pP3K5F9lO4MchvFVAqA6j9uRt0AUtOeN0u5zuuPlNC28lG9O -JAb3X4sOp45r3l519DKaULFEM5rQBeJ4gv/b2opj66nd0b+gYa3jnookXWIO50yR -f+/fNDY7L131hLIvxG2TlhpvMCjx2hKaZLRAMx293itTqOq+1rxOlvVE+zIYrtUf -9mmvtk57HVjsO6lTo15YyJ4= +hvcNAQEBBQADggEPADCCAQoCggEBAMMYaWvJRymYjrFWwi76Dl68I4CzB2Ik0kJb +8Uq/qcghdcjj5iwfhzxufBvtOTKVt0CyYEjDmhYI/m1niDQ7d3dwHHBa0R9fBCFU +uQzjQYUdWO4v7fMO79gjofpz+0wo4OXmTQsCUkmGx75+veZWdotwjgqPBjMgHXtb +qtDFG6ubzFQJPL/kQGbx+9b3Fp3EGdTD8v8HvG9aniUbAkql7EKWOnDSbJkrzr7o +0gHv1bqwz5Q+gtAB1ktxgAMKEkWGeYHYS9LotbcsbJpMihAQ5OT1386EkcrRRuCE +cxdm22lDeICDvhRN8T4a1mz13kXzOa+R1T1URL9BzHNoGvwk25ECAwEAAaNQME4w +HQYDVR0OBBYEFJrPz27rcT3bPPGuiGtWcgPLCKdIMB8GA1UdIwQYMBaAFJrPz27r +cT3bPPGuiGtWcgPLCKdIMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEB +ABAlyNwMVVzLg25573fsDY4MBsFLDNb3dVIhuBdKOIids3jEQvu4fBQ4EPus2hEA +W0KHXkWfbU5CpJoYBjkPRaaWiTLWWbPTjuOVtsSiS3QvZ8H7u/lybzdK5/RIM3Hf +uPXmQT/V1S8mCfgOkv9w6varWPuQBNZDLo+x+waradDcqPhbB/LUZh9j+F3BnkFE +u8nofeBG5KfIMl8xYuUcXIndt6JPng0TuF+xhFNMH84Z4QEAXr9BVZSppRPb8lnz +1k65nZ25CtmyGG18sfeWqr32+ZUPSm48fEZb39R47Jrc4uMB5oh3OZOcuipj+SVL +T6wIeTnGe98HNbrAwlC/WoE= -----END CERTIFICATE----- diff --git a/Lib/test/pycakey.pem b/Lib/test/pycakey.pem index fc6effefb21..16b75879d09 100644 --- a/Lib/test/pycakey.pem +++ b/Lib/test/pycakey.pem @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDn3unjDJ8AtqH9 -K1uW0m/M4L6GuSBe7AN6VavqpOn5SYXSZtXtx3rqVo4tj+dC4mIoqZ/WG47rtbSc -nxSr3+aUi3YdPm0kYe0MvwCKYQzfXMg2cxYAzUe6baSkdIiDIwoZ/AmnPEpL0+cd -LeTqTFQh8ybbiTcY1AK7QDJfpP8tHPfUu+yOz1yCrOZ8CGxIhWEHfyXgXOC8NF/g -uQRHdchHC4281shoXzODYtIgRDWxrYEais28NbBci0fWGOmcGJfMATwpzOge5OTB -uN7nwhEYh1qTNNimJfcUcevkIaLSDy4u1GIANdPW71xgS0ypFOLdFVhGNzMmt+cu -Xe1C5MVNAgMBAAECggEBAJPM7QuUrPn4cLN/Ysd15lwTWn9oHDFFgkYFvCs66gXE -ju/6Kx2BjWE4wTJby09AHM/MqB0DvguT7Mf1Q2j3tPQ1HZowg8OwRDleuwp6KIls -jBbhL0Jdl/5HC67ktWvZ9wNvO/wFG1rQfT6FVajf9LUbWEaSZbOG2SLhHfsHorzu -xjTJaI3bQ/0+79B1exwk5ruwhzFRd/XpY8hls7D/RfPIuHDlBghkW3N59KFWrf5h -6bNEh2THm0+IyGcGqs0FD+QCOXyvsjwSUswqrr2ctLREOeDcd5ReUjSxYgjcJRrm -J7ceIY/+uwDJxw/OlnmBvF6pQMkKwYW2gFztu+g2t4UCgYEA/9yo01Exz4crxXsy -tAlnDJM++nZcm07rtFjTKHUfKY/cCgNTa8udM0svnfwlid/dpgLsI38gx04HHC1i -EZ4acz+ToIWedLxM0nq73//xeRWEazOvCz1mMTZaMldahTWAyzN8qVK2B/625Yy4 -wNYWyweBBwEB8MzaCs73spksXOsCgYEA5/7wvhiofYGFAfMuANeJIwDL2OtBnoOv -mVNfCmi3GC38fzwyi5ZpskWDiS2woJ+LQfs9Qu4EcZbUFLd7gbeOvb5gmFUtYope -LitUUKunIR18MkQ+mQDBpQPQPhk4QJP5reCbWkrfTu7b5o/iS41s6fBTFmuzhLcT -C71vFdCyeKcCgYAiCCqYeOtELDmBOeLDmaCQRqGQ1N96dOPbCBmF/xYXBCCDYG/f -HaUaJnz96YTgstsbcrYP/p/Qgqtlbw/lQf9IpwMuzbcG1ejt8g89OyDWNyt2ytgU -iaUnFJCos3/Byh0Iah/BsdOueo2/OJl2ZMOBW80orlSgv86cs2y037TL4wKBgQDm -OOyW+MlbowhnIvfoBfwlLEkefnej4nKD6WRLZBcue5Qyf355X06Mhsc9foXlH+6G -D9h/bswiHNdhp6N82rdgPGiHQx/CxiUoE/+b/nvgNO5mw6qLE2EXbG1e8pAMJcyE -bHw+YkawggDfELI036fRj5gki8SeUz8nS1nNgElbyQKBgCRDX9Jh+MwSLu4QBWdt -/fi+lv3K6kun/fI7EOV1vCV/j871tICu7pu5BrOLxAHqoVfU9AUX299/2KjCb5pv -kjogiUK6qWCWBlfuqDNWGCoUGt1rhznUva0nNjSMy5rinBhhjpROZC2pw48lOluP -UuvXsaPph7GTqPuy4Kab12YC +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDDGGlryUcpmI6x +VsIu+g5evCOAswdiJNJCW/FKv6nIIXXI4+YsH4c8bnwb7TkylbdAsmBIw5oWCP5t +Z4g0O3d3cBxwWtEfXwQhVLkM40GFHVjuL+3zDu/YI6H6c/tMKODl5k0LAlJJhse+ +fr3mVnaLcI4KjwYzIB17W6rQxRurm8xUCTy/5EBm8fvW9xadxBnUw/L/B7xvWp4l +GwJKpexCljpw0myZK86+6NIB79W6sM+UPoLQAdZLcYADChJFhnmB2EvS6LW3LGya +TIoQEOTk9d/OhJHK0UbghHMXZttpQ3iAg74UTfE+GtZs9d5F8zmvkdU9VES/Qcxz +aBr8JNuRAgMBAAECggEAZHgv4hg3k45C/cSmH7caq2LMDb0kskAwH4hlzI7DipLg +q2Hh6Rsbc92aAG+8IvbC9ohl2VMSCQL8s667j9qH/XQ40QuT4kn2QIv2+FIYLcsd +Pxxjt+YbUf2XrvkHkwMCPqLJTkAVzFOijdGLThF83vZJz9oz4SRKyno8j2LSix68 +WEfnjdyWqYb0eS0luKrLHw+IL7bD5vfc/P0q6u31zJ9h8zEyN5EBCj5OxM/hD0VO +nObrp6r9Bs+xx+yRx+8J5Db6LPXggl5nBqsqrDKVDe6uTysYVgstqkfaDv1L78Vu +3BNdKPAdJ+ucPJrQufzFHBDIIN+Xwckf/09gdQagGQKBgQDnvFaOjZfqc6wL/kNK +tszQtedbdwP20L+EWdNEVsVWK1TOw36Pmkrp2AYLXMd7W1QQu0KukM89EFb84wKo +s4C9V/ch162mUhEAveaLioi7bMwMPIib2V6pHmYGG8nQVRvgkZVYx6ZtPEvWye1v +wmCzzxxK0gC6PQGxp8MSv9yXDwKBgQDXhe57ufc52pgJ+Agyl4PLkllIbG2DKQHG +LwY06v73jllirTpWBOBvN0NvEsI2Pj4aK/BXRNYN1PS7xi/3C6MVWxnOpBtbq3H5 +DwFb5mpfgJmhV6DZ6jMw7h3Yvy35ViKoiI9UK3eTmhkerH3DsILEje7jE9dGmIOJ +4oLa50JjXwKBgQDdTfyveMNasIrejTzATmC89Or0a22KuQIdKBddjSw5xXnhV8s2 +4temCJqFIV6UDLz0mZDt2vc+zqr0KOtyJrLMoAQv+qQoUPlR5wkTvAImU5luGiUw +CN+gzJoMPV93KMBNr1qcBVaHvWyDvCWXdF8beLABOBpfwUEr4xWlgzrruwKBgCvf +cr2zDJW1Xu/gkuKhn02ofA5XLC/gACF03yGUmNSSILYKp25tTba2HD8XJXvfTcsM +GL/bHmvwZuV2obr7nnYxdl5vX7ZYfzoBCPjJPew1BJEognD50PPr9R1zRYuVMjb2 +nZ63vn7IhsaMvIlCfExAzFljZ5ZSY6yE9LhVDVmnAoGBALOwMwpkm1drx5UNSJO7 +70Q8kYzg0oQhCo/7b6DWbAglDPSWQS5IA4rHYOwL3sE+69G2Exe+1454rVDisojW +XdSyA3svI/YQeom8R2LIM/ayCPxCc3/Dxy9+aQQT4lW3F0XQIxod/QsQJxpZIOnF +jOSPclypgV2X6dDOwDkd2Tgh -----END PRIVATE KEY----- diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 85e32a9ae7d..f1b02336f7e 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -56,6 +56,14 @@ def copy_attributes(info_add, obj, name_fmt, attributes, *, formatter=None): info_add(name, value) +def copy_attr(info_add, name, mod, attr_name): + try: + value = getattr(mod, attr_name) + except AttributeError: + return + info_add(name, value) + + def call_func(info_add, name, mod, func_name, *, formatter=None): try: func = getattr(mod, func_name) @@ -143,6 +151,11 @@ def collect_locale(info_add): info_add('locale.encoding', locale.getpreferredencoding(False)) +def collect_builtins(info_add): + info_add('builtins.float.float_format', float.__getformat__("float")) + info_add('builtins.float.double_format', float.__getformat__("double")) + + def collect_os(info_add): import os @@ -162,17 +175,16 @@ def collect_os(info_add): ) copy_attributes(info_add, os, 'os.%s', attributes, formatter=format_attr) - info_add("os.cwd", os.getcwd()) + call_func(info_add, 'os.cwd', os, 'getcwd') call_func(info_add, 'os.uid', os, 'getuid') call_func(info_add, 'os.gid', os, 'getgid') call_func(info_add, 'os.uname', os, 'uname') - if hasattr(os, 'getgroups'): - groups = os.getgroups() - groups = map(str, groups) - groups = ', '.join(groups) - info_add("os.groups", groups) + def format_groups(groups): + return ', '.join(map(str, groups)) + + call_func(info_add, 'os.groups', os, 'getgroups', formatter=format_groups) if hasattr(os, 'getlogin'): try: @@ -184,11 +196,7 @@ def collect_os(info_add): else: info_add("os.login", login) - if hasattr(os, 'cpu_count'): - cpu_count = os.cpu_count() - if cpu_count: - info_add('os.cpu_count', cpu_count) - + call_func(info_add, 'os.cpu_count', os, 'cpu_count') call_func(info_add, 'os.loadavg', os, 'getloadavg') # Get environment variables: filter to list @@ -219,7 +227,9 @@ def collect_os(info_add): ) for name, value in os.environ.items(): uname = name.upper() - if (uname in ENV_VARS or uname.startswith(("PYTHON", "LC_")) + if (uname in ENV_VARS + # Copy PYTHON* and LC_* variables + or uname.startswith(("PYTHON", "LC_")) # Visual Studio: VS140COMNTOOLS or (uname.startswith("VS") and uname.endswith("COMNTOOLS"))): info_add('os.environ[%s]' % name, value) @@ -305,6 +315,8 @@ def collect_tkinter(info_add): def collect_time(info_add): import time + info_add('time.time', time.time()) + attributes = ( 'altzone', 'daylight', @@ -313,12 +325,19 @@ def collect_time(info_add): ) copy_attributes(info_add, time, 'time.%s', attributes) - if not hasattr(time, 'get_clock_info'): + if hasattr(time, 'get_clock_info'): + for clock in ('time', 'perf_counter'): + tinfo = time.get_clock_info(clock) + info_add('time.get_clock_info(%s)' % clock, tinfo) + + +def collect_datetime(info_add): + try: + import datetime + except ImportError: return - for clock in ('time', 'perf_counter'): - tinfo = time.get_clock_info(clock) - info_add('time.%s' % clock, tinfo) + info_add('datetime.datetime.now', datetime.datetime.now()) def collect_sysconfig(info_add): @@ -331,7 +350,6 @@ def collect_sysconfig(info_add): 'CCSHARED', 'CFLAGS', 'CFLAGSFORSHARED', - 'PY_LDFLAGS', 'CONFIG_ARGS', 'HOST_GNU_TYPE', 'MACHDEP', @@ -339,6 +357,7 @@ def collect_sysconfig(info_add): 'OPT', 'PY_CFLAGS', 'PY_CFLAGS_NODIST', + 'PY_LDFLAGS', 'Py_DEBUG', 'Py_ENABLE_SHARED', 'SHELL', @@ -422,6 +441,54 @@ def collect_decimal(info_add): copy_attributes(info_add, _decimal, '_decimal.%s', attributes) +def collect_testcapi(info_add): + try: + import _testcapi + except ImportError: + return + + call_func(info_add, 'pymem.allocator', _testcapi, 'pymem_getallocatorsname') + copy_attr(info_add, 'pymem.with_pymalloc', _testcapi, 'WITH_PYMALLOC') + + +def collect_resource(info_add): + try: + import resource + except ImportError: + return + + limits = [attr for attr in dir(resource) if attr.startswith('RLIMIT_')] + for name in limits: + key = getattr(resource, name) + value = resource.getrlimit(key) + info_add('resource.%s' % name, value) + + +def collect_test_socket(info_add): + try: + from test import test_socket + except ImportError: + return + + # all check attributes like HAVE_SOCKET_CAN + attributes = [name for name in dir(test_socket) + if name.startswith('HAVE_')] + copy_attributes(info_add, test_socket, 'test_socket.%s', attributes) + + +def collect_test_support(info_add): + try: + from test import support + except ImportError: + return + + attributes = ('IPV6_ENABLED',) + copy_attributes(info_add, support, 'test_support.%s', attributes) + + call_func(info_add, 'test_support._is_gui_available', support, '_is_gui_available') + call_func(info_add, 'test_support.python_is_optimized', support, 'python_is_optimized') + + def collect_info(info): error = False info_add = info.add @@ -430,6 +497,7 @@ def collect_info(info): # collect_os() should be the first, to check the getrandom() status collect_os, + collect_builtins, collect_gdb, collect_locale, collect_platform, @@ -440,10 +508,17 @@ def collect_info(info): collect_sys, collect_sysconfig, collect_time, + collect_datetime, collect_tkinter, collect_zlib, collect_expat, collect_decimal, + collect_testcapi, + collect_resource, + + # Collecting from tests should be last as they have side effects. + collect_test_socket, + collect_test_support, ): try: collect_func(info_add) diff --git a/Lib/test/revocation.crl b/Lib/test/revocation.crl index 6d89b08ebea..53cb4b3721d 100644 --- a/Lib/test/revocation.crl +++ b/Lib/test/revocation.crl @@ -1,11 +1,11 @@ -----BEGIN X509 CRL----- MIIBpjCBjwIBATANBgkqhkiG9w0BAQUFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE CgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1j -YS1zZXJ2ZXIXDTEzMTEyMTE3MDg0N1oXDTIzMDkzMDE3MDg0N1qgDjAMMAoGA1Ud -FAQDAgEAMA0GCSqGSIb3DQEBBQUAA4IBAQCNJXC2mVKauEeN3LlQ3ZtM5gkH3ExH -+i4bmJjtJn497WwvvoIeUdrmVXgJQR93RtV37hZwN0SXMLlNmUZPH4rHhihayw4m -unCzVj/OhCCY7/TPjKuJ1O/0XhaLBpBVjQN7R/1ujoRKbSia/CD3vcn7Fqxzw7LK -fSRCKRGTj1CZiuxrphtFchwALXSiFDy9mr2ZKhImcyq1PydfgEzU78APpOkMQsIC -UNJ/cf3c9emzf+dUtcMEcejQ3mynBo4eIGg1EW42bz4q4hSjzQlKcBV0muw5qXhc -HOxH2iTFhQ7SrvVuK/dM14rYM4B5mSX3nRC1kNmXpS9j3wJDhuwmjHed +YS1zZXJ2ZXIXDTE4MDExOTE5MDkwNloXDTI3MTEyODE5MDkwNlqgDjAMMAoGA1Ud +FAQDAgEAMA0GCSqGSIb3DQEBBQUAA4IBAQBYVzH8n2LdyJJ/t8CpWjz652hZJ0sY +DeNYcwuTPzR9CbSwEwNbf0kY9bgWXGfoRD2SnnCnuNNJXO2MuXtxf6qYx2ZjhJm8 +qgxXs0Bz4agRswNMbumjHCmqIv1t88vbrO0+ItEZDK7RJVIMBtVJ0XYOHvD/IG/i +zqa1Fl3uCTvQbTJ2TrqzJeP/Vl40hOD+VdBBZK3j0r4AkCKU3tAiHYTGmHKhPxy1 +f8Yet+4SRMGp1BdDezTI1bICpSZhRJ4geW0UzuCZnXPW8IZzioUmdUBAmAMHPWFr +B0sTTc/ntD4jHG1/T5b0oiDMbXIbh5Iz9iQNcY0IbotkCw39h+K90wY6 -----END X509 CRL----- diff --git a/Lib/test/secp384r1.pem b/Lib/test/secp384r1.pem new file mode 100644 index 00000000000..eef7117af7a --- /dev/null +++ b/Lib/test/secp384r1.pem @@ -0,0 +1,7 @@ +$ openssl genpkey -genparam -algorithm EC -pkeyopt ec_paramgen_curve:secp384r1 -pkeyopt ec_param_enc:named_curve -text +-----BEGIN EC PARAMETERS----- +BgUrgQQAIg== +-----END EC PARAMETERS----- +ECDSA-Parameters: (384 bit) +ASN1 OID: secp384r1 +NIST CURVE: P-384 diff --git a/Lib/test/ssl_cert.pem b/Lib/test/ssl_cert.pem index 47a7d7e37e8..b1dd3f387f7 100644 --- a/Lib/test/ssl_cert.pem +++ b/Lib/test/ssl_cert.pem @@ -1,15 +1,20 @@ -----BEGIN CERTIFICATE----- -MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV -BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw -MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH -Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k -YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw -gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 -6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt -pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw -FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd -BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G -lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 -CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +MIIDWTCCAkGgAwIBAgIJAPm6B21bar2bMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODAx +MTkxOTA5MDZaFw0yODAxMTcxOTA5MDZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +DA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAKvvsX2gEti4shve3iYMc+jE4Se7WHs1Bol2f21H8qNboDOFdeb1 +RKHjmq3exHpajywOUEgne9nKHJY/3f2phR4Y5klqG6liLgiSpVyRlcBGbeT2qEAj +9oLiLFUXLGfGDds2mTwivQDLJBWi51j7ff5k2Pr58fN5ugYMn24T9FNyn0moT+qj +SFoBNm58l9jrdkJSlgWfqPlbiMa+mqDn/SFtrwLF2Trbfzu42Sd9UdIzMaSSrzbN +sGm53pNhCh8KndWUQ8GPP2IsLPoUU4qAtmZuTxCx2S1cXrN9EkmT69tlOH84YfSn +96Ih9bWRc7M5y5bfVdEVM+fKQl3hBRf05qMCAwEAAaMYMBYwFAYDVR0RBA0wC4IJ +bG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IBAQAtQ8f37cCEk7/rAcbYR53ce3iK +Vpihb0U2ni1QjG9Tg9UIExkIGkwTiCm7kwQL+GEStBu9AG/QVrOjeTriRiddhWkk +ze8kRaI3AC/63t6Vh9Q1x6PESgeE4OtAO9JpJCf4GILglA789Y/b/GF8zJZQxR13 +qpB4ZwWw7gCBhdEW59u6CFeBmfDa58hM8lWvuVoRrTi7bjUeC6PAn5HVMzZSykhu +4HaUfBp6bKFjuym2+h/VvM1n8C3chjVSmutsLb6ELdD8IK0vPV/yf5+LN256zSsS +dyUZYd8XwQaioEMKdbhLvnehyzHiWfQIUR3BdhONxoIJhHv/EAo8eCkHHYIF -----END CERTIFICATE----- diff --git a/Lib/test/ssl_key.passwd.pem b/Lib/test/ssl_key.passwd.pem index 2524672e70d..669c7ce3c81 100644 --- a/Lib/test/ssl_key.passwd.pem +++ b/Lib/test/ssl_key.passwd.pem @@ -1,18 +1,30 @@ -----BEGIN RSA PRIVATE KEY----- Proc-Type: 4,ENCRYPTED -DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A +DEK-Info: DES-EDE3-CBC,2D5DD30B9D440DBB -kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c -u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA -AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr -Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+ -YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P -6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+ -noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1 -94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l -7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo -cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO -zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt -L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo -2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ== +01gIwpy3XxPGsY0PoK59vAxdLhkVj3odO0Z1ULamUzIte6ThKL1HqnZiUlXpYKfK +XqHVVeQ1xouxiDRNFLJ4CqBG4HbRtqTkl+sfaNTVveL18lOOMAZy6W3dCGAnWOTZ +Z0RJyZlQxxjNQLuko4tIvFkrShXIgdiVFjwAhRU0KTUb7UQ2xfFA9R0Kfde30pzz +zSjb/OmYqAIhkdvafGXvJxzZAorQkU9akDh+uJ6cht5B/RGZsbKACYDSv2WSV5yW +r+fKVYcTup33r0Jj8hAD6fVY15K8BJknpkF9HfSlZnmmr2WDaffLokOOnCV/I1ie +WD7ENA7K//48km5D3Ogh2b2/0Iwuzjq8Mvd8aR39N9nINbGR+HNT85pawoo1S0W9 +pQTU4XTmxfXjtR2287C6XZyQ/tBwvNDMFPVhlxsGOdLYwoV5e/L1t1qIfkTlbuvd +JaMzOhSSLjiC156IFoH7PTPe+g75hw2b32XJURFGlaYknHF7P4BmCiwXOQYo5CCo +MQGGlw5qBCqODrIsc03wpL2jUzgvyPqLyaw395ITuSoGX+WO7vUQaGW0Tz/sOoTs +3pK+bTi2QmqZMe7xBOj07CYMMOo4QPrM6NpbObt+Jja2UXaxvKa9BwqCEQzA4pQZ +8ZHHfEWIaDffKTGkAlqm+S8qCtsrEZJhIn3aI/ikzK8v+YkWw6w+8t/tR1V8ET/s +CoYGIR7I8WhdfKAwgx2QT5bt1jkYKJyKPm4Iacp2mNh9gNFVq+JSKF318e7BrR3+ +wyqMkDxRYnov3ybtf6kiICxPREDqa6UG1xRq3SbWz6NnIF/1hoHs79YlSYbMfXNU +ffIxBaXNCcH6jM9duP2YRnO29jLwfLM/mmokTBBjyOBaKZia9GPa4naXoATXW3z+ +Xx4EKIUkKdb53kiV6NtEKMPialAnkeoHTEjyLPgaV8mCHLvGQbnxbYwvpPJH0e2f +CWgiw6ci4ROOzcZ7HJHIDUprwK0xRKn43hoI44fivlSHOFX6da6o3wIqhEUqMKwL +JQDS1GORRk1ndRXP+7Ub1dO+Vo/DqO1VcTr2o5RwZ1LWPnzLqbCG50mvTLH4djB+ ++hf6vlmnFC30N3yUFXWE5vS10nJHYP88dD9CB2RsaWzpxD9Zxl+PKcRsppen6HyO +u3b71a/TBOkJcI+lkOatEFvbuqzBAqhMceMctO+Dl55RFsbxfIw/IXZjdP0PYZ0C +t20DrIdBsvl9F/mfYpmkV4DF7yci78DqnRBcxylVNF2vwX7o+2fq/TsEwsHn3KnT +kvcF5Cq8Vr5C8ugWX8JfveNym0BjLu6Lr58qS4a6qCNGEGPFKyB+xkm4KEScbarQ +aLbEbfulMM7q9//sEOOLexIx7mNoLd29Xzn5hsLCAZLWX6wMq6JVJ/zbBOAHDbBT +yhi03yd5Kvw3swSt4QZj+uR3qTFwxkXUFiVvrSfxRZoyKsxsLr9Z7D8aoH9Rkb2L +6KjZ31nt9Drh7NJfh6ReANBW6INdDW0Y2mbzoDozLszAYjVfuUUEE76iJqXY0N4W +kNr0OQQTUtDpVk0AZZZvy17xV+rkqGgwlOqTvHbwFYEQvgwVz4EKUw== -----END RSA PRIVATE KEY----- diff --git a/Lib/test/ssl_key.pem b/Lib/test/ssl_key.pem index 3fd3bbd54a3..b63f38bc5cf 100644 --- a/Lib/test/ssl_key.pem +++ b/Lib/test/ssl_key.pem @@ -1,16 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANtb0+YrKuxevGpm -LrjaUhZSgz6zFAmuGFmKmUbdjmfv9zSmmdsQIksK++jK0Be9LeZy20j6ahOfuVa0 -ufEmPoP7Fy4hXegKZR9cCWcIe/A6H2xWF1IIJLRTLaU8ol/I7T+um5HD5AwAwNPP -USNU0Eegmvp+xxWu3NX2m1Veot85AgMBAAECgYA3ZdZ673X0oexFlq7AAmrutkHt -CL7LvwrpOiaBjhyTxTeSNWzvtQBkIU8DOI0bIazA4UreAFffwtvEuPmonDb3F+Iq -SMAu42XcGyVZEl+gHlTPU9XRX7nTOXVt+MlRRRxL6t9GkGfUAXI3XxJDXW3c0vBK -UL9xqD8cORXOfE06rQJBAP8mEX1ERkR64Ptsoe4281vjTlNfIbs7NMPkUnrn9N/Y -BLhjNIfQ3HFZG8BTMLfX7kCS9D593DW5tV4Z9BP/c6cCQQDcFzCcVArNh2JSywOQ -ZfTfRbJg/Z5Lt9Fkngv1meeGNPgIMLN8Sg679pAOOWmzdMO3V706rNPzSVMME7E5 -oPIfAkEA8pDddarP5tCvTTgUpmTFbakm0KoTZm2+FzHcnA4jRh+XNTjTOv98Y6Ik -eO5d1ZnKXseWvkZncQgxfdnMqqpj5wJAcNq/RVne1DbYlwWchT2Si65MYmmJ8t+F -0mcsULqjOnEMwf5e+ptq5LzwbyrHZYq5FNk7ocufPv/ZQrcSSC+cFwJBAKvOJByS -x56qyGeZLOQlWS2JS3KJo59XuLFGqcbgN9Om9xFa41Yb4N9NvplFivsvZdw3m1Q/ -SPIXQuT8RMPDVNQ= +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCr77F9oBLYuLIb +3t4mDHPoxOEnu1h7NQaJdn9tR/KjW6AzhXXm9USh45qt3sR6Wo8sDlBIJ3vZyhyW +P939qYUeGOZJahupYi4IkqVckZXARm3k9qhAI/aC4ixVFyxnxg3bNpk8Ir0AyyQV +oudY+33+ZNj6+fHzeboGDJ9uE/RTcp9JqE/qo0haATZufJfY63ZCUpYFn6j5W4jG +vpqg5/0hba8Cxdk62387uNknfVHSMzGkkq82zbBpud6TYQofCp3VlEPBjz9iLCz6 +FFOKgLZmbk8QsdktXF6zfRJJk+vbZTh/OGH0p/eiIfW1kXOzOcuW31XRFTPnykJd +4QUX9OajAgMBAAECggEAHppmXDbuw9Z0FVPg9KLIysioTtsgz6VLiZIm8juZK4x2 +glUh/D7xvWL2uDXrgN+3lh7iGUW13LkFx5SMncbbo9TIwI57Z/XKvcnkVwquve+L +RfLFVc1Q5lD9lROv2rS86KTaN4LzYz3FKXi6dvMkpPAsUtfEQhMLkmISypQQq/1z +EJaqo7r85OjN7e0wKazlKZpOzJEa5FQLMVRjTRFhLFNbHXX/tAet2jw+umATKbw8 +hYgiuZ44TwSEd9JeIV/oSYWfI/3HetuYW0ru3caiztRF2NySNu8lcsWgNC7fIku9 +mcHjtSNzs91QN1Qlu7GQvvhpt6OWDirNDCW+49WGaQKBgQDg9SDhfF0jRYslgYbH +cqO4ggaFdHjrAAYpwnAgvanhFZL/zEqm5G1E7l/e2fCkJ9VOSFO0A208chvwMcr+ +dCjHE2tVdE81aQ2v/Eo83VdS1RcOV4Y75yPH48rMhxPaHvxWD/FFDbf0/P2mtPB7 +SZ3kIeZMkE1wxdaO3AKUbQoozwKBgQDDqYgg7kVtygyICE1mB8Hwp6nUxFTczG7y +4XcsDqMIrKmw+PbQluvkoHoStxeVrsTloDhkTjIrpmYLyAiazg+PUJdkd6xrfLSj +VV6X93W0S/1egEb1F1CGFxtk8v/PWH4K76EPL2vxXdxjywz3GWlrL9yDYaB2szzS +DqgwVMqx7QKBgDCD7UF0Bsoyl13RX3XoPXLvZ+SkR+e2q52Z94C4JskKVBeiwX7Y +yNAS8M4pBoMArDoj0xmBm69rlKbqtjLGbnzwrTdSzDpim7cWnBQgUFLm7gAD1Elb +AhZ8BCK0Bw4FnLoa2hfga4oEfdfUMgEE0W5/+SEOBgWKRUmuHUhRc911AoGAY2EN +YmSDYSM5wDIvVb5k9B3EtevOiqNPSw/XnsoEZtiEC/44JnQxdltIBY93bDBrk5IQ +cmoBM4h91kgQjshQwOMXMhFSwvmBKmCm/hrTbvMVytTutXfVD3ZXFKwT4DW7N0TF +ElhsxBh/YzRz7mG62JVjtFt2zDN3ld2Z8YpvtXUCgYEA4EJ4ObS5YyvcXAKHJFo6 +Fxmavyrf8LSm3MFA65uSnFvWukMVqqRMReQc5jvpxHKCis+XvnHzyOfL0gW9ZTi7 +tWGGbBi0TRJCa8BkvgngUZxOxUlMfg/7cVxOIB0TPoUSgxFd/+qVz4GZMvr0dPu7 +eAF7J/8ECVvb0wSPTUI1N3c= -----END PRIVATE KEY----- diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index cd3ee48a92b..561b09a2d5e 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -909,6 +909,21 @@ class BaseTest: self.checkequal(False, 'abc\n', 'isalnum') self.checkraises(TypeError, 'abc', 'isalnum', 42) + def test_isascii(self): + self.checkequal(True, '', 'isascii') + self.checkequal(True, '\x00', 'isascii') + self.checkequal(True, '\x7f', 'isascii') + self.checkequal(True, '\x00\x7f', 'isascii') + self.checkequal(False, '\x80', 'isascii') + self.checkequal(False, '\xe9', 'isascii') + # bytes.isascii() and bytearray.isascii() has optimization which + # check 4 or 8 bytes at once. So check some alignments. + for p in range(8): + self.checkequal(True, ' '*p + '\x7f', 'isascii') + self.checkequal(False, ' '*p + '\x80', 'isascii') + self.checkequal(True, ' '*p + '\x7f' + ' '*8, 'isascii') + self.checkequal(False, ' '*p + '\x80' + ' '*8, 'isascii') + def test_isdigit(self): self.checkequal(False, '', 'isdigit') self.checkequal(False, 'a', 'isdigit') diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 527cf7fbf95..6c9e31a22c1 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -87,7 +87,7 @@ __all__ = [ "bigmemtest", "bigaddrspacetest", "cpython_only", "get_attribute", "requires_IEEE_754", "skip_unless_xattr", "requires_zlib", "anticipate_failure", "load_package_tests", "detect_api_mismatch", - "check__all__", "requires_android_level", "requires_multiprocessing_queue", + "check__all__", "skip_unless_bind_unix_socket", # sys "is_jython", "is_android", "check_impl_detail", "unix_shell", "setswitchinterval", @@ -278,7 +278,6 @@ max_memuse = 0 # Disable bigmem tests (they will still be run with # small sizes, to make sure they work.) real_max_memuse = 0 failfast = False -match_tests = None # _original_stdout is meant to hold stdout at the time regrtest began. # This may be "the real" stdout, or IDLE's emulation of stdout, or whatever. @@ -773,13 +772,7 @@ requires_lzma = unittest.skipUnless(lzma, 'requires lzma') is_jython = sys.platform.startswith('java') -try: - # constant used by requires_android_level() - _ANDROID_API_LEVEL = sys.getandroidapilevel() - is_android = True -except AttributeError: - # sys.getandroidapilevel() is only available on Android - is_android = False +is_android = hasattr(sys, 'getandroidapilevel') if sys.platform != 'win32': unix_shell = '/system/bin/sh' if is_android else '/bin/sh' @@ -1068,8 +1061,8 @@ def make_bad_fd(): file.close() unlink(TESTFN) -def check_syntax_error(testcase, statement, *, lineno=None, offset=None): - with testcase.assertRaises(SyntaxError) as cm: +def check_syntax_error(testcase, statement, errtext='', *, lineno=None, offset=None): + with testcase.assertRaisesRegex(SyntaxError, errtext) as cm: compile(statement, '', 'exec') err = cm.exception testcase.assertIsNotNone(err.lineno) @@ -1778,13 +1771,6 @@ def requires_resource(resource): else: return unittest.skip("resource {0!r} is not enabled".format(resource)) -def requires_android_level(level, reason): - if is_android and _ANDROID_API_LEVEL < level: - return unittest.skip('%s at Android API level %d' % - (reason, _ANDROID_API_LEVEL)) - else: - return _id - def cpython_only(test): """ Decorator for tests only applicable on CPython. @@ -1804,22 +1790,6 @@ def impl_detail(msg=None, **guards): msg = msg.format(' or '.join(guardnames)) return unittest.skip(msg) -_have_mp_queue = None -def requires_multiprocessing_queue(test): - """Skip decorator for tests that use multiprocessing.Queue.""" - global _have_mp_queue - if _have_mp_queue is None: - import multiprocessing - # Without a functioning shared semaphore implementation attempts to - # instantiate a Queue will result in an ImportError (issue #3770). - try: - multiprocessing.Queue() - _have_mp_queue = True - except ImportError: - _have_mp_queue = False - msg = "requires a functioning shared semaphore implementation" - return test if _have_mp_queue else unittest.skip(msg)(test) - def _parse_guards(guards): # Returns a tuple ({platform_name: run_me}, default_value) if not guards: @@ -1900,21 +1870,67 @@ def _run_suite(suite): raise TestFailed(err) -def _match_test(test): - global match_tests +# By default, don't filter tests +_match_test_func = None +_match_test_patterns = None - if match_tests is None: + +def match_test(test): + # Function used by support.run_unittest() and regrtest --list-cases + if _match_test_func is None: return True - test_id = test.id() + else: + return _match_test_func(test.id()) - for match_test in match_tests: - if fnmatch.fnmatchcase(test_id, match_test): - return True - for name in test_id.split("."): - if fnmatch.fnmatchcase(name, match_test): +def _is_full_match_test(pattern): + # If a pattern contains at least one dot, it's considered + # as a full test identifier. + # Example: 'test.test_os.FileTests.test_access'. + # + # Reject patterns which contain fnmatch patterns: '*', '?', '[...]' + # or '[!...]'. For example, reject 'test_access*'. + return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) + + +def set_match_tests(patterns): + global _match_test_func, _match_test_patterns + + if patterns == _match_test_patterns: + # No change: no need to recompile patterns. + return + + if not patterns: + func = None + # set_match_tests(None) behaves as set_match_tests(()) + patterns = () + elif all(map(_is_full_match_test, patterns)): + # Simple case: all patterns are full test identifier. + # The test.bisect utility only uses such full test identifiers. + func = set(patterns).__contains__ + else: + regex = '|'.join(map(fnmatch.translate, patterns)) + # The search *is* case sensitive on purpose: + # don't use flags=re.IGNORECASE + regex_match = re.compile(regex).match + + def match_test_regex(test_id): + if regex_match(test_id): + # The regex matchs the whole identifier like + # 'test.test_os.FileTests.test_access' return True - return False + else: + # Try to match parts of the test identifier. + # For example, split 'test.test_os.FileTests.test_access' + # into: 'test', 'test_os', 'FileTests' and 'test_access'. + return any(map(regex_match, test_id.split("."))) + + func = match_test_regex + + # Create a copy since patterns can be mutable and so modified later + _match_test_patterns = tuple(patterns) + _match_test_func = func + def run_unittest(*classes): @@ -1931,7 +1947,7 @@ def run_unittest(*classes): suite.addTest(cls) else: suite.addTest(unittest.makeSuite(cls)) - _filter_suite(suite, _match_test) + _filter_suite(suite, match_test) _run_suite(suite) #======================================================================= @@ -2387,6 +2403,28 @@ def skip_unless_xattr(test): msg = "no non-broken extended attribute support" return test if ok else unittest.skip(msg)(test) +_bind_nix_socket_error = None +def skip_unless_bind_unix_socket(test): + """Decorator for tests requiring a functional bind() for unix sockets.""" + if not hasattr(socket, 'AF_UNIX'): + return unittest.skip('No UNIX Sockets')(test) + global _bind_nix_socket_error + if _bind_nix_socket_error is None: + path = TESTFN + "can_bind_unix_socket" + with socket.socket(socket.AF_UNIX) as sock: + try: + sock.bind(path) + _bind_nix_socket_error = False + except OSError as e: + _bind_nix_socket_error = e + finally: + unlink(path) + if _bind_nix_socket_error: + msg = 'Requires a functional unix bind(): %s' % _bind_nix_socket_error + return unittest.skip(msg)(test) + else: + return test + def fs_is_case_insensitive(directory): """Detects if the file system for the specified directory is case-insensitive.""" @@ -2793,3 +2831,8 @@ class SaveSignals: def restore(self): for signum, handler in self.handlers.items(): self.signal.signal(signum, handler) + + +def with_pymalloc(): + import _testcapi + return _testcapi.WITH_PYMALLOC diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py new file mode 100644 index 00000000000..8d72ca20021 --- /dev/null +++ b/Lib/test/test__xxsubinterpreters.py @@ -0,0 +1,1120 @@ +import contextlib +import os +import pickle +from textwrap import dedent, indent +import threading +import unittest + +from test import support +from test.support import script_helper + +interpreters = support.import_module('_xxsubinterpreters') + + +def _captured_script(script): + r, w = os.pipe() + indented = script.replace('\n', '\n ') + wrapped = dedent(f""" + import contextlib + with open({w}, 'w') as chan: + with contextlib.redirect_stdout(chan): + {indented} + """) + return wrapped, open(r) + + +def _run_output(interp, request, shared=None): + script, chan = _captured_script(request) + with chan: + interpreters.run_string(interp, script, shared) + return chan.read() + + +@contextlib.contextmanager +def _running(interp): + r, w = os.pipe() + def run(): + interpreters.run_string(interp, dedent(f""" + # wait for "signal" + with open({r}) as chan: + chan.read() + """)) + + t = threading.Thread(target=run) + t.start() + + yield + + with open(w, 'w') as chan: + chan.write('done') + t.join() + + +class IsShareableTests(unittest.TestCase): + + def test_default_shareables(self): + shareables = [ + # singletons + None, + # builtin objects + b'spam', + ] + for obj in shareables: + with self.subTest(obj): + self.assertTrue( + interpreters.is_shareable(obj)) + + def test_not_shareable(self): + class Cheese: + def __init__(self, name): + self.name = name + def __str__(self): + return self.name + + class SubBytes(bytes): + """A subclass of a shareable type.""" + + not_shareables = [ + # singletons + True, + False, + NotImplemented, + ..., + # builtin types and objects + type, + object, + object(), + Exception(), + 42, + 100.0, + 'spam', + # user-defined types and objects + Cheese, + Cheese('Wensleydale'), + SubBytes(b'spam'), + ] + for obj in not_shareables: + with self.subTest(obj): + self.assertFalse( + interpreters.is_shareable(obj)) + + +class TestBase(unittest.TestCase): + + def tearDown(self): + for id in interpreters.list_all(): + if id == 0: # main + continue + try: + interpreters.destroy(id) + except RuntimeError: + pass # already destroyed + + for cid in interpreters.channel_list_all(): + try: + interpreters.channel_destroy(cid) + except interpreters.ChannelNotFoundError: + pass # already destroyed + + +class ListAllTests(TestBase): + + def test_initial(self): + main = interpreters.get_main() + ids = interpreters.list_all() + self.assertEqual(ids, [main]) + + def test_after_creating(self): + main = interpreters.get_main() + first = interpreters.create() + second = interpreters.create() + ids = interpreters.list_all() + self.assertEqual(ids, [main, first, second]) + + def test_after_destroying(self): + main = interpreters.get_main() + first = interpreters.create() + second = interpreters.create() + interpreters.destroy(first) + ids = interpreters.list_all() + self.assertEqual(ids, [main, second]) + + +class GetCurrentTests(TestBase): + + def test_main(self): + main = interpreters.get_main() + cur = interpreters.get_current() + self.assertEqual(cur, main) + + def test_subinterpreter(self): + main = interpreters.get_main() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + import _xxsubinterpreters as _interpreters + print(_interpreters.get_current()) + """)) + cur = int(out.strip()) + _, expected = interpreters.list_all() + self.assertEqual(cur, expected) + self.assertNotEqual(cur, main) + + +class GetMainTests(TestBase): + + def test_from_main(self): + [expected] = interpreters.list_all() + main = interpreters.get_main() + self.assertEqual(main, expected) + + def test_from_subinterpreter(self): + [expected] = interpreters.list_all() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + import _xxsubinterpreters as _interpreters + print(_interpreters.get_main()) + """)) + main = int(out.strip()) + self.assertEqual(main, expected) + + +class IsRunningTests(TestBase): + + def test_main(self): + main = interpreters.get_main() + self.assertTrue(interpreters.is_running(main)) + + def test_subinterpreter(self): + interp = interpreters.create() + self.assertFalse(interpreters.is_running(interp)) + + with _running(interp): + self.assertTrue(interpreters.is_running(interp)) + self.assertFalse(interpreters.is_running(interp)) + + def test_from_subinterpreter(self): + interp = interpreters.create() + out = _run_output(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + if _interpreters.is_running({interp}): + print(True) + else: + print(False) + """)) + self.assertEqual(out.strip(), 'True') + + def test_already_destroyed(self): + interp = interpreters.create() + interpreters.destroy(interp) + with self.assertRaises(RuntimeError): + interpreters.is_running(interp) + + def test_does_not_exist(self): + with self.assertRaises(RuntimeError): + interpreters.is_running(1_000_000) + + def test_bad_id(self): + with self.assertRaises(RuntimeError): + interpreters.is_running(-1) + + +class CreateTests(TestBase): + + def test_in_main(self): + id = interpreters.create() + + self.assertIn(id, interpreters.list_all()) + + @unittest.skip('enable this test when working on pystate.c') + def test_unique_id(self): + seen = set() + for _ in range(100): + id = interpreters.create() + interpreters.destroy(id) + seen.add(id) + + self.assertEqual(len(seen), 100) + + def test_in_thread(self): + lock = threading.Lock() + id = None + def f(): + nonlocal id + id = interpreters.create() + lock.acquire() + lock.release() + + t = threading.Thread(target=f) + with lock: + t.start() + t.join() + self.assertIn(id, interpreters.list_all()) + + def test_in_subinterpreter(self): + main, = interpreters.list_all() + id1 = interpreters.create() + out = _run_output(id1, dedent(""" + import _xxsubinterpreters as _interpreters + id = _interpreters.create() + print(id) + """)) + id2 = int(out.strip()) + + self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) + + def test_in_threaded_subinterpreter(self): + main, = interpreters.list_all() + id1 = interpreters.create() + id2 = None + def f(): + nonlocal id2 + out = _run_output(id1, dedent(""" + import _xxsubinterpreters as _interpreters + id = _interpreters.create() + print(id) + """)) + id2 = int(out.strip()) + + t = threading.Thread(target=f) + t.start() + t.join() + + self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) + + def test_after_destroy_all(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + ids = [] + for _ in range(3): + id = interpreters.create() + ids.append(id) + # Now destroy them. + for id in ids: + interpreters.destroy(id) + # Finally, create another. + id = interpreters.create() + self.assertEqual(set(interpreters.list_all()), before | {id}) + + def test_after_destroy_some(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + id1 = interpreters.create() + id2 = interpreters.create() + id3 = interpreters.create() + # Now destroy 2 of them. + interpreters.destroy(id1) + interpreters.destroy(id3) + # Finally, create another. + id = interpreters.create() + self.assertEqual(set(interpreters.list_all()), before | {id, id2}) + + +class DestroyTests(TestBase): + + def test_one(self): + id1 = interpreters.create() + id2 = interpreters.create() + id3 = interpreters.create() + self.assertIn(id2, interpreters.list_all()) + interpreters.destroy(id2) + self.assertNotIn(id2, interpreters.list_all()) + self.assertIn(id1, interpreters.list_all()) + self.assertIn(id3, interpreters.list_all()) + + def test_all(self): + before = set(interpreters.list_all()) + ids = set() + for _ in range(3): + id = interpreters.create() + ids.add(id) + self.assertEqual(set(interpreters.list_all()), before | ids) + for id in ids: + interpreters.destroy(id) + self.assertEqual(set(interpreters.list_all()), before) + + def test_main(self): + main, = interpreters.list_all() + with self.assertRaises(RuntimeError): + interpreters.destroy(main) + + def f(): + with self.assertRaises(RuntimeError): + interpreters.destroy(main) + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_already_destroyed(self): + id = interpreters.create() + interpreters.destroy(id) + with self.assertRaises(RuntimeError): + interpreters.destroy(id) + + def test_does_not_exist(self): + with self.assertRaises(RuntimeError): + interpreters.destroy(1_000_000) + + def test_bad_id(self): + with self.assertRaises(RuntimeError): + interpreters.destroy(-1) + + def test_from_current(self): + main, = interpreters.list_all() + id = interpreters.create() + script = dedent(f""" + import _xxsubinterpreters as _interpreters + try: + _interpreters.destroy({id}) + except RuntimeError: + pass + """) + + interpreters.run_string(id, script) + self.assertEqual(set(interpreters.list_all()), {main, id}) + + def test_from_sibling(self): + main, = interpreters.list_all() + id1 = interpreters.create() + id2 = interpreters.create() + script = dedent(""" + import _xxsubinterpreters as _interpreters + _interpreters.destroy({}) + """).format(id2) + interpreters.run_string(id1, script) + + self.assertEqual(set(interpreters.list_all()), {main, id1}) + + def test_from_other_thread(self): + id = interpreters.create() + def f(): + interpreters.destroy(id) + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_still_running(self): + main, = interpreters.list_all() + interp = interpreters.create() + with _running(interp): + with self.assertRaises(RuntimeError): + interpreters.destroy(interp) + self.assertTrue(interpreters.is_running(interp)) + + +class RunStringTests(TestBase): + + SCRIPT = dedent(""" + with open('{}', 'w') as out: + out.write('{}') + """) + FILENAME = 'spam' + + def setUp(self): + super().setUp() + self.id = interpreters.create() + self._fs = None + + def tearDown(self): + if self._fs is not None: + self._fs.close() + super().tearDown() + + @property + def fs(self): + if self._fs is None: + self._fs = FSFixture(self) + return self._fs + + def test_success(self): + script, file = _captured_script('print("it worked!", end="")') + with file: + interpreters.run_string(self.id, script) + out = file.read() + + self.assertEqual(out, 'it worked!') + + def test_in_thread(self): + script, file = _captured_script('print("it worked!", end="")') + with file: + def f(): + interpreters.run_string(self.id, script) + + t = threading.Thread(target=f) + t.start() + t.join() + out = file.read() + + self.assertEqual(out, 'it worked!') + + def test_create_thread(self): + script, file = _captured_script(""" + import threading + def f(): + print('it worked!', end='') + + t = threading.Thread(target=f) + t.start() + t.join() + """) + with file: + interpreters.run_string(self.id, script) + out = file.read() + + self.assertEqual(out, 'it worked!') + + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + def test_fork(self): + import tempfile + with tempfile.NamedTemporaryFile('w+') as file: + file.write('') + file.flush() + + expected = 'spam spam spam spam spam' + script = dedent(f""" + # (inspired by Lib/test/test_fork.py) + import os + pid = os.fork() + if pid == 0: # child + with open('{file.name}', 'w') as out: + out.write('{expected}') + # Kill the unittest runner in the child process. + os._exit(1) + else: + SHORT_SLEEP = 0.1 + import time + for _ in range(10): + spid, status = os.waitpid(pid, os.WNOHANG) + if spid == pid: + break + time.sleep(SHORT_SLEEP) + assert(spid == pid) + """) + interpreters.run_string(self.id, script) + + file.seek(0) + content = file.read() + self.assertEqual(content, expected) + + def test_already_running(self): + with _running(self.id): + with self.assertRaises(RuntimeError): + interpreters.run_string(self.id, 'print("spam")') + + def test_does_not_exist(self): + id = 0 + while id in interpreters.list_all(): + id += 1 + with self.assertRaises(RuntimeError): + interpreters.run_string(id, 'print("spam")') + + def test_error_id(self): + with self.assertRaises(RuntimeError): + interpreters.run_string(-1, 'print("spam")') + + def test_bad_id(self): + with self.assertRaises(TypeError): + interpreters.run_string('spam', 'print("spam")') + + def test_bad_script(self): + with self.assertRaises(TypeError): + interpreters.run_string(self.id, 10) + + def test_bytes_for_script(self): + with self.assertRaises(TypeError): + interpreters.run_string(self.id, b'print("spam")') + + @contextlib.contextmanager + def assert_run_failed(self, exctype, msg=None): + with self.assertRaises(interpreters.RunFailedError) as caught: + yield + if msg is None: + self.assertEqual(str(caught.exception).split(':')[0], + str(exctype)) + else: + self.assertEqual(str(caught.exception), + "{}: {}".format(exctype, msg)) + + def test_invalid_syntax(self): + with self.assert_run_failed(SyntaxError): + # missing close paren + interpreters.run_string(self.id, 'print("spam"') + + def test_failure(self): + with self.assert_run_failed(Exception, 'spam'): + interpreters.run_string(self.id, 'raise Exception("spam")') + + def test_SystemExit(self): + with self.assert_run_failed(SystemExit, '42'): + interpreters.run_string(self.id, 'raise SystemExit(42)') + + def test_sys_exit(self): + with self.assert_run_failed(SystemExit): + interpreters.run_string(self.id, dedent(""" + import sys + sys.exit() + """)) + + with self.assert_run_failed(SystemExit, '42'): + interpreters.run_string(self.id, dedent(""" + import sys + sys.exit(42) + """)) + + def test_with_shared(self): + r, w = os.pipe() + + shared = { + 'spam': b'ham', + 'eggs': b'-1', + 'cheddar': None, + } + script = dedent(f""" + eggs = int(eggs) + spam = 42 + result = spam + eggs + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """) + interpreters.run_string(self.id, script, shared) + with open(r, 'rb') as chan: + ns = pickle.load(chan) + + self.assertEqual(ns['spam'], 42) + self.assertEqual(ns['eggs'], -1) + self.assertEqual(ns['result'], 41) + self.assertIsNone(ns['cheddar']) + + def test_shared_overwrites(self): + interpreters.run_string(self.id, dedent(""" + spam = 'eggs' + ns1 = dict(vars()) + del ns1['__builtins__'] + """)) + + shared = {'spam': b'ham'} + script = dedent(f""" + ns2 = dict(vars()) + del ns2['__builtins__'] + """) + interpreters.run_string(self.id, script, shared) + + r, w = os.pipe() + script = dedent(f""" + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """) + interpreters.run_string(self.id, script) + with open(r, 'rb') as chan: + ns = pickle.load(chan) + + self.assertEqual(ns['ns1']['spam'], 'eggs') + self.assertEqual(ns['ns2']['spam'], b'ham') + self.assertEqual(ns['spam'], b'ham') + + def test_shared_overwrites_default_vars(self): + r, w = os.pipe() + + shared = {'__name__': b'not __main__'} + script = dedent(f""" + spam = 42 + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """) + interpreters.run_string(self.id, script, shared) + with open(r, 'rb') as chan: + ns = pickle.load(chan) + + self.assertEqual(ns['__name__'], b'not __main__') + + def test_main_reused(self): + r, w = os.pipe() + interpreters.run_string(self.id, dedent(f""" + spam = True + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + del ns, pickle, chan + """)) + with open(r, 'rb') as chan: + ns1 = pickle.load(chan) + + r, w = os.pipe() + interpreters.run_string(self.id, dedent(f""" + eggs = False + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """)) + with open(r, 'rb') as chan: + ns2 = pickle.load(chan) + + self.assertIn('spam', ns1) + self.assertNotIn('eggs', ns1) + self.assertIn('eggs', ns2) + self.assertIn('spam', ns2) + + def test_execution_namespace_is_main(self): + r, w = os.pipe() + + script = dedent(f""" + spam = 42 + + ns = dict(vars()) + ns['__builtins__'] = str(ns['__builtins__']) + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """) + interpreters.run_string(self.id, script) + with open(r, 'rb') as chan: + ns = pickle.load(chan) + + ns.pop('__builtins__') + ns.pop('__loader__') + self.assertEqual(ns, { + '__name__': '__main__', + '__annotations__': {}, + '__doc__': None, + '__package__': None, + '__spec__': None, + 'spam': 42, + }) + + def test_still_running_at_exit(self): + script = dedent(f""" + from textwrap import dedent + import threading + import _xxsubinterpreters as _interpreters + def f(): + _interpreters.run_string(id, dedent(''' + import time + # Give plenty of time for the main interpreter to finish. + time.sleep(1_000_000) + ''')) + + t = threading.Thread(target=f) + t.start() + """) + with support.temp_dir() as dirname: + filename = script_helper.make_script(dirname, 'interp', script) + with script_helper.spawn_python(filename) as proc: + retcode = proc.wait() + + self.assertEqual(retcode, 0) + + +class ChannelIDTests(TestBase): + + def test_default_kwargs(self): + cid = interpreters._channel_id(10, force=True) + + self.assertEqual(int(cid), 10) + self.assertEqual(cid.end, 'both') + + def test_with_kwargs(self): + cid = interpreters._channel_id(10, send=True, force=True) + self.assertEqual(cid.end, 'send') + + cid = interpreters._channel_id(10, send=True, recv=False, force=True) + self.assertEqual(cid.end, 'send') + + cid = interpreters._channel_id(10, recv=True, force=True) + self.assertEqual(cid.end, 'recv') + + cid = interpreters._channel_id(10, recv=True, send=False, force=True) + self.assertEqual(cid.end, 'recv') + + cid = interpreters._channel_id(10, send=True, recv=True, force=True) + self.assertEqual(cid.end, 'both') + + def test_coerce_id(self): + cid = interpreters._channel_id('10', force=True) + self.assertEqual(int(cid), 10) + + cid = interpreters._channel_id(10.0, force=True) + self.assertEqual(int(cid), 10) + + class Int(str): + def __init__(self, value): + self._value = value + def __int__(self): + return self._value + + cid = interpreters._channel_id(Int(10), force=True) + self.assertEqual(int(cid), 10) + + def test_bad_id(self): + for cid in [-1, 'spam']: + with self.subTest(cid): + with self.assertRaises(ValueError): + interpreters._channel_id(cid) + with self.assertRaises(OverflowError): + interpreters._channel_id(2**64) + with self.assertRaises(TypeError): + interpreters._channel_id(object()) + + def test_bad_kwargs(self): + with self.assertRaises(ValueError): + interpreters._channel_id(10, send=False, recv=False) + + def test_does_not_exist(self): + cid = interpreters.channel_create() + with self.assertRaises(interpreters.ChannelNotFoundError): + interpreters._channel_id(int(cid) + 1) # unforced + + def test_repr(self): + cid = interpreters._channel_id(10, force=True) + self.assertEqual(repr(cid), 'ChannelID(10)') + + cid = interpreters._channel_id(10, send=True, force=True) + self.assertEqual(repr(cid), 'ChannelID(10, send=True)') + + cid = interpreters._channel_id(10, recv=True, force=True) + self.assertEqual(repr(cid), 'ChannelID(10, recv=True)') + + cid = interpreters._channel_id(10, send=True, recv=True, force=True) + self.assertEqual(repr(cid), 'ChannelID(10)') + + def test_equality(self): + cid1 = interpreters.channel_create() + cid2 = interpreters._channel_id(int(cid1)) + cid3 = interpreters.channel_create() + + self.assertTrue(cid1 == cid1) + self.assertTrue(cid1 == cid2) + self.assertTrue(cid1 == int(cid1)) + self.assertFalse(cid1 == cid3) + + self.assertFalse(cid1 != cid1) + self.assertFalse(cid1 != cid2) + self.assertTrue(cid1 != cid3) + + +class ChannelTests(TestBase): + + def test_sequential_ids(self): + before = interpreters.channel_list_all() + id1 = interpreters.channel_create() + id2 = interpreters.channel_create() + id3 = interpreters.channel_create() + after = interpreters.channel_list_all() + + self.assertEqual(id2, int(id1) + 1) + self.assertEqual(id3, int(id2) + 1) + self.assertEqual(set(after) - set(before), {id1, id2, id3}) + + def test_ids_global(self): + id1 = interpreters.create() + out = _run_output(id1, dedent(""" + import _xxsubinterpreters as _interpreters + cid = _interpreters.channel_create() + print(int(cid)) + """)) + cid1 = int(out.strip()) + + id2 = interpreters.create() + out = _run_output(id2, dedent(""" + import _xxsubinterpreters as _interpreters + cid = _interpreters.channel_create() + print(int(cid)) + """)) + cid2 = int(out.strip()) + + self.assertEqual(cid2, int(cid1) + 1) + + #################### + + def test_drop_single_user(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_drop_interpreter(cid, send=True, recv=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_drop_multiple_users(self): + cid = interpreters.channel_create() + id1 = interpreters.create() + id2 = interpreters.create() + interpreters.run_string(id1, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_send({int(cid)}, b'spam') + """)) + out = _run_output(id2, dedent(f""" + import _xxsubinterpreters as _interpreters + obj = _interpreters.channel_recv({int(cid)}) + _interpreters.channel_drop_interpreter({int(cid)}) + print(repr(obj)) + """)) + interpreters.run_string(id1, dedent(f""" + _interpreters.channel_drop_interpreter({int(cid)}) + """)) + + self.assertEqual(out.strip(), "b'spam'") + + def test_drop_no_kwargs(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_drop_interpreter(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_drop_multiple_times(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_drop_interpreter(cid, send=True, recv=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_drop_interpreter(cid, send=True, recv=True) + + def test_drop_with_unused_items(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + interpreters.channel_drop_interpreter(cid, send=True, recv=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_drop_never_used(self): + cid = interpreters.channel_create() + interpreters.channel_drop_interpreter(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'spam') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_drop_by_unassociated_interp(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interp = interpreters.create() + interpreters.run_string(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_drop_interpreter({int(cid)}) + """)) + obj = interpreters.channel_recv(cid) + interpreters.channel_drop_interpreter(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + self.assertEqual(obj, b'spam') + + def test_drop_close_if_unassociated(self): + cid = interpreters.channel_create() + interp = interpreters.create() + interpreters.run_string(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + obj = _interpreters.channel_send({int(cid)}, b'spam') + _interpreters.channel_drop_interpreter({int(cid)}) + """)) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_drop_partially(self): + # XXX Is partial close too wierd/confusing? + cid = interpreters.channel_create() + interpreters.channel_send(cid, None) + interpreters.channel_recv(cid) + interpreters.channel_send(cid, b'spam') + interpreters.channel_drop_interpreter(cid, send=True) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + + def test_drop_used_multiple_times_by_single_user(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_drop_interpreter(cid, send=True, recv=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + #################### + + def test_close_single_user(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_close(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_multiple_users(self): + cid = interpreters.channel_create() + id1 = interpreters.create() + id2 = interpreters.create() + interpreters.run_string(id1, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_send({int(cid)}, b'spam') + """)) + interpreters.run_string(id2, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_recv({int(cid)}) + """)) + interpreters.channel_close(cid) + with self.assertRaises(interpreters.RunFailedError) as cm: + interpreters.run_string(id1, dedent(f""" + _interpreters.channel_send({int(cid)}, b'spam') + """)) + self.assertIn('ChannelClosedError', str(cm.exception)) + with self.assertRaises(interpreters.RunFailedError) as cm: + interpreters.run_string(id2, dedent(f""" + _interpreters.channel_send({int(cid)}, b'spam') + """)) + self.assertIn('ChannelClosedError', str(cm.exception)) + + def test_close_multiple_times(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_close(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_close(cid) + + def test_close_with_unused_items(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + interpreters.channel_close(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_never_used(self): + cid = interpreters.channel_create() + interpreters.channel_close(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'spam') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_by_unassociated_interp(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interp = interpreters.create() + interpreters.run_string(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_close({int(cid)}) + """)) + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_close(cid) + + def test_close_used_multiple_times_by_single_user(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_close(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + #################### + + def test_send_recv_main(self): + cid = interpreters.channel_create() + orig = b'spam' + interpreters.channel_send(cid, orig) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_same_interpreter(self): + id1 = interpreters.create() + out = _run_output(id1, dedent(""" + import _xxsubinterpreters as _interpreters + cid = _interpreters.channel_create() + orig = b'spam' + _interpreters.channel_send(cid, orig) + obj = _interpreters.channel_recv(cid) + assert obj is not orig + assert obj == orig + """)) + + def test_send_recv_different_interpreters(self): + cid = interpreters.channel_create() + id1 = interpreters.create() + out = _run_output(id1, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_send({int(cid)}, b'spam') + """)) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + + def test_send_not_found(self): + with self.assertRaises(interpreters.ChannelNotFoundError): + interpreters.channel_send(10, b'spam') + + def test_recv_not_found(self): + with self.assertRaises(interpreters.ChannelNotFoundError): + interpreters.channel_recv(10) + + def test_recv_empty(self): + cid = interpreters.channel_create() + with self.assertRaises(interpreters.ChannelEmptyError): + interpreters.channel_recv(cid) + + def test_run_string_arg(self): + cid = interpreters.channel_create() + interp = interpreters.create() + + out = _run_output(interp, dedent(""" + import _xxsubinterpreters as _interpreters + print(cid.end) + _interpreters.channel_send(cid, b'spam') + """), + dict(cid=cid.send)) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + self.assertEqual(out.strip(), 'send') + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index aa53503e3b5..67f363ad31f 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -551,14 +551,37 @@ class ASTHelpers_Test(unittest.TestCase): self.assertEqual(ast.literal_eval('{1, 2, 3}'), {1, 2, 3}) self.assertEqual(ast.literal_eval('b"hi"'), b"hi") self.assertRaises(ValueError, ast.literal_eval, 'foo()') + self.assertEqual(ast.literal_eval('6'), 6) + self.assertEqual(ast.literal_eval('+6'), 6) self.assertEqual(ast.literal_eval('-6'), -6) - self.assertEqual(ast.literal_eval('-6j+3'), 3-6j) self.assertEqual(ast.literal_eval('3.25'), 3.25) + self.assertEqual(ast.literal_eval('+3.25'), 3.25) + self.assertEqual(ast.literal_eval('-3.25'), -3.25) + self.assertEqual(repr(ast.literal_eval('-0.0')), '-0.0') + self.assertRaises(ValueError, ast.literal_eval, '++6') + self.assertRaises(ValueError, ast.literal_eval, '+True') + self.assertRaises(ValueError, ast.literal_eval, '2+3') - def test_literal_eval_issue4907(self): - self.assertEqual(ast.literal_eval('2j'), 2j) - self.assertEqual(ast.literal_eval('10 + 2j'), 10 + 2j) - self.assertEqual(ast.literal_eval('1.5 - 2j'), 1.5 - 2j) + def test_literal_eval_complex(self): + # Issue #4907 + self.assertEqual(ast.literal_eval('6j'), 6j) + self.assertEqual(ast.literal_eval('-6j'), -6j) + self.assertEqual(ast.literal_eval('6.75j'), 6.75j) + self.assertEqual(ast.literal_eval('-6.75j'), -6.75j) + self.assertEqual(ast.literal_eval('3+6j'), 3+6j) + self.assertEqual(ast.literal_eval('-3+6j'), -3+6j) + self.assertEqual(ast.literal_eval('3-6j'), 3-6j) + self.assertEqual(ast.literal_eval('-3-6j'), -3-6j) + self.assertEqual(ast.literal_eval('3.25+6.75j'), 3.25+6.75j) + self.assertEqual(ast.literal_eval('-3.25+6.75j'), -3.25+6.75j) + self.assertEqual(ast.literal_eval('3.25-6.75j'), 3.25-6.75j) + self.assertEqual(ast.literal_eval('-3.25-6.75j'), -3.25-6.75j) + self.assertEqual(ast.literal_eval('(3+6j)'), 3+6j) + self.assertRaises(ValueError, ast.literal_eval, '-6j+3') + self.assertRaises(ValueError, ast.literal_eval, '-6j+3j') + self.assertRaises(ValueError, ast.literal_eval, '3+-6j') + self.assertRaises(ValueError, ast.literal_eval, '3+(0+6j)') + self.assertRaises(ValueError, ast.literal_eval, '-(3+6j)') def test_bad_integer(self): # issue13436: Bad error message with invalid numeric values @@ -1077,11 +1100,11 @@ class ConstantTests(unittest.TestCase): ast.copy_location(new_left, binop.left) binop.left = new_left - new_right = ast.Constant(value=20) + new_right = ast.Constant(value=20j) ast.copy_location(new_right, binop.right) binop.right = new_right - self.assertEqual(ast.literal_eval(binop), 30) + self.assertEqual(ast.literal_eval(binop), 10+20j) def main(): diff --git a/Lib/test/test_asyncio/functional.py b/Lib/test/test_asyncio/functional.py new file mode 100644 index 00000000000..fbec462c1db --- /dev/null +++ b/Lib/test/test_asyncio/functional.py @@ -0,0 +1,276 @@ +import asyncio +import asyncio.events +import contextlib +import os +import pprint +import select +import socket +import tempfile +import threading + + +class FunctionalTestCaseMixin: + + def new_loop(self): + return asyncio.new_event_loop() + + def run_loop_briefly(self, *, delay=0.01): + self.loop.run_until_complete(asyncio.sleep(delay, loop=self.loop)) + + def loop_exception_handler(self, loop, context): + self.__unhandled_exceptions.append(context) + self.loop.default_exception_handler(context) + + def setUp(self): + self.loop = self.new_loop() + asyncio.set_event_loop(None) + + self.loop.set_exception_handler(self.loop_exception_handler) + self.__unhandled_exceptions = [] + + # Disable `_get_running_loop`. + self._old_get_running_loop = asyncio.events._get_running_loop + asyncio.events._get_running_loop = lambda: None + + def tearDown(self): + try: + self.loop.close() + + if self.__unhandled_exceptions: + print('Unexpected calls to loop.call_exception_handler():') + pprint.pprint(self.__unhandled_exceptions) + self.fail('unexpected calls to loop.call_exception_handler()') + + finally: + asyncio.events._get_running_loop = self._old_get_running_loop + asyncio.set_event_loop(None) + self.loop = None + + def tcp_server(self, server_prog, *, + family=socket.AF_INET, + addr=None, + timeout=5, + backlog=1, + max_clients=10): + + if addr is None: + if hasattr(socket, 'AF_UNIX') and family == socket.AF_UNIX: + with tempfile.NamedTemporaryFile() as tmp: + addr = tmp.name + else: + addr = ('127.0.0.1', 0) + + sock = socket.socket(family, socket.SOCK_STREAM) + + if timeout is None: + raise RuntimeError('timeout is required') + if timeout <= 0: + raise RuntimeError('only blocking sockets are supported') + sock.settimeout(timeout) + + try: + sock.bind(addr) + sock.listen(backlog) + except OSError as ex: + sock.close() + raise ex + + return TestThreadedServer( + self, sock, server_prog, timeout, max_clients) + + def tcp_client(self, client_prog, + family=socket.AF_INET, + timeout=10): + + sock = socket.socket(family, socket.SOCK_STREAM) + + if timeout is None: + raise RuntimeError('timeout is required') + if timeout <= 0: + raise RuntimeError('only blocking sockets are supported') + sock.settimeout(timeout) + + return TestThreadedClient( + self, sock, client_prog, timeout) + + def unix_server(self, *args, **kwargs): + if not hasattr(socket, 'AF_UNIX'): + raise NotImplementedError + return self.tcp_server(*args, family=socket.AF_UNIX, **kwargs) + + def unix_client(self, *args, **kwargs): + if not hasattr(socket, 'AF_UNIX'): + raise NotImplementedError + return self.tcp_client(*args, family=socket.AF_UNIX, **kwargs) + + @contextlib.contextmanager + def unix_sock_name(self): + with tempfile.TemporaryDirectory() as td: + fn = os.path.join(td, 'sock') + try: + yield fn + finally: + try: + os.unlink(fn) + except OSError: + pass + + def _abort_socket_test(self, ex): + try: + self.loop.stop() + finally: + self.fail(ex) + + +############################################################################## +# Socket Testing Utilities +############################################################################## + + +class TestSocketWrapper: + + def __init__(self, sock): + self.__sock = sock + + def recv_all(self, n): + buf = b'' + while len(buf) < n: + data = self.recv(n - len(buf)) + if data == b'': + raise ConnectionAbortedError + buf += data + return buf + + def start_tls(self, ssl_context, *, + server_side=False, + server_hostname=None): + + ssl_sock = ssl_context.wrap_socket( + self.__sock, server_side=server_side, + server_hostname=server_hostname, + do_handshake_on_connect=False) + + ssl_sock.do_handshake() + + self.__sock.close() + self.__sock = ssl_sock + + def __getattr__(self, name): + return getattr(self.__sock, name) + + def __repr__(self): + return '<{} {!r}>'.format(type(self).__name__, self.__sock) + + +class SocketThread(threading.Thread): + + def stop(self): + self._active = False + self.join() + + def __enter__(self): + self.start() + return self + + def __exit__(self, *exc): + self.stop() + + +class TestThreadedClient(SocketThread): + + def __init__(self, test, sock, prog, timeout): + threading.Thread.__init__(self, None, None, 'test-client') + self.daemon = True + + self._timeout = timeout + self._sock = sock + self._active = True + self._prog = prog + self._test = test + + def run(self): + try: + self._prog(TestSocketWrapper(self._sock)) + except Exception as ex: + self._test._abort_socket_test(ex) + + +class TestThreadedServer(SocketThread): + + def __init__(self, test, sock, prog, timeout, max_clients): + threading.Thread.__init__(self, None, None, 'test-server') + self.daemon = True + + self._clients = 0 + self._finished_clients = 0 + self._max_clients = max_clients + self._timeout = timeout + self._sock = sock + self._active = True + + self._prog = prog + + self._s1, self._s2 = socket.socketpair() + self._s1.setblocking(False) + + self._test = test + + def stop(self): + try: + if self._s2 and self._s2.fileno() != -1: + try: + self._s2.send(b'stop') + except OSError: + pass + finally: + super().stop() + + def run(self): + try: + with self._sock: + self._sock.setblocking(0) + self._run() + finally: + self._s1.close() + self._s2.close() + + def _run(self): + while self._active: + if self._clients >= self._max_clients: + return + + r, w, x = select.select( + [self._sock, self._s1], [], [], self._timeout) + + if self._s1 in r: + return + + if self._sock in r: + try: + conn, addr = self._sock.accept() + except BlockingIOError: + continue + except socket.timeout: + if not self._active: + return + else: + raise + else: + self._clients += 1 + conn.settimeout(self._timeout) + try: + with conn: + self._handle_client(conn) + except Exception as ex: + self._active = False + try: + raise + finally: + self._test._abort_socket_test(ex) + + def _handle_client(self, sock): + self._prog(TestSocketWrapper(sock)) + + @property + def addr(self): + return self._sock.getsockname() diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 98f2aef5627..ab6560c70b9 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -14,18 +14,10 @@ from unittest import mock import asyncio from asyncio import base_events from asyncio import constants -from asyncio import test_utils -try: - from test import support -except ImportError: - from asyncio import test_support as support -try: - from test.support.script_helper import assert_python_ok -except ImportError: - try: - from test.script_helper import assert_python_ok - except ImportError: - from asyncio.test_support import assert_python_ok +from asyncio import events +from test.test_asyncio import utils as test_utils +from test import support +from test.support.script_helper import assert_python_ok MOCK_ANY = mock.ANY @@ -116,13 +108,6 @@ class BaseEventTests(test_utils.TestCase): self.assertIsNone( base_events._ipaddr_info('::3%lo0', 1, INET6, STREAM, TCP)) - if hasattr(socket, 'SOCK_NONBLOCK'): - self.assertEqual( - None, - base_events._ipaddr_info( - '1.2.3.4', 1, INET, STREAM | socket.SOCK_NONBLOCK, TCP)) - - def test_port_parameter_types(self): # Test obscure kinds of arguments for "port". INET = socket.AF_INET @@ -207,14 +192,14 @@ class BaseEventLoopTests(test_utils.TestCase): self.assertRaises(RuntimeError, self.loop.run_until_complete, f) def test__add_callback_handle(self): - h = asyncio.Handle(lambda: False, (), self.loop) + h = asyncio.Handle(lambda: False, (), self.loop, None) self.loop._add_callback(h) self.assertFalse(self.loop._scheduled) self.assertIn(h, self.loop._ready) def test__add_callback_cancelled_handle(self): - h = asyncio.Handle(lambda: False, (), self.loop) + h = asyncio.Handle(lambda: False, (), self.loop, None) h.cancel() self.loop._add_callback(h) @@ -226,14 +211,6 @@ class BaseEventLoopTests(test_utils.TestCase): self.loop.set_default_executor(executor) self.assertIs(executor, self.loop._default_executor) - def test_getnameinfo(self): - sockaddr = mock.Mock() - self.loop.run_in_executor = mock.Mock() - self.loop.getnameinfo(sockaddr) - self.assertEqual( - (None, socket.getnameinfo, sockaddr, 0), - self.loop.run_in_executor.call_args[0]) - def test_call_soon(self): def cb(): pass @@ -354,31 +331,11 @@ class BaseEventLoopTests(test_utils.TestCase): # check disabled if debug mode is disabled test_thread(self.loop, False, create_loop=True) - def test_run_once_in_executor_plain(self): - def cb(): - pass - f = asyncio.Future(loop=self.loop) - executor = mock.Mock() - executor.submit.return_value = f - - self.loop.set_default_executor(executor) - - res = self.loop.run_in_executor(None, cb) - self.assertIs(f, res) - - executor = mock.Mock() - executor.submit.return_value = f - res = self.loop.run_in_executor(executor, cb) - self.assertIs(f, res) - self.assertTrue(executor.submit.called) - - f.cancel() # Don't complain about abandoned Future. - def test__run_once(self): h1 = asyncio.TimerHandle(time.monotonic() + 5.0, lambda: True, (), - self.loop) + self.loop, None) h2 = asyncio.TimerHandle(time.monotonic() + 10.0, lambda: True, (), - self.loop) + self.loop, None) h1.cancel() @@ -433,7 +390,7 @@ class BaseEventLoopTests(test_utils.TestCase): handle = loop.call_soon(lambda: True) h = asyncio.TimerHandle(time.monotonic() - 1, cb, (self.loop,), - self.loop) + self.loop, None) self.loop._process_events = mock.Mock() self.loop._scheduled.append(h) @@ -811,17 +768,20 @@ class BaseEventLoopTests(test_utils.TestCase): self.assertEqual(stdout.rstrip(), b'False') sts, stdout, stderr = assert_python_ok('-c', code, - PYTHONASYNCIODEBUG='') + PYTHONASYNCIODEBUG='', + PYTHONDEVMODE='') self.assertEqual(stdout.rstrip(), b'False') sts, stdout, stderr = assert_python_ok('-c', code, - PYTHONASYNCIODEBUG='1') + PYTHONASYNCIODEBUG='1', + PYTHONDEVMODE='') self.assertEqual(stdout.rstrip(), b'True') sts, stdout, stderr = assert_python_ok('-E', '-c', code, PYTHONASYNCIODEBUG='1') self.assertEqual(stdout.rstrip(), b'False') + # -X dev sts, stdout, stderr = assert_python_ok('-E', '-X', 'dev', '-c', code) self.assertEqual(stdout.rstrip(), b'True') @@ -1013,6 +973,12 @@ class BaseEventLoopWithSelectorTests(test_utils.TestCase): self.loop = asyncio.new_event_loop() self.set_event_loop(self.loop) + @mock.patch('socket.getnameinfo') + def test_getnameinfo(self, m_gai): + m_gai.side_effect = lambda *args: 42 + r = self.loop.run_until_complete(self.loop.getnameinfo(('abc', 123))) + self.assertEqual(r, 42) + @patch_socket def test_create_connection_multiple_errors(self, m_socket): @@ -1088,6 +1054,14 @@ class BaseEventLoopWithSelectorTests(test_utils.TestCase): 'A Stream Socket was expected'): self.loop.run_until_complete(coro) + def test_create_server_ssl_timeout_for_plain_socket(self): + coro = self.loop.create_server( + MyProto, 'example.com', 80, ssl_handshake_timeout=1) + with self.assertRaisesRegex( + ValueError, + 'ssl_handshake_timeout is only meaningful with ssl'): + self.loop.run_until_complete(coro) + @unittest.skipUnless(hasattr(socket, 'SOCK_NONBLOCK'), 'no socket.SOCK_NONBLOCK (linux only)') def test_create_server_stream_bittype(self): @@ -1125,9 +1099,7 @@ class BaseEventLoopWithSelectorTests(test_utils.TestCase): OSError, self.loop.run_until_complete, coro) def test_create_connection_connect_err(self): - @asyncio.coroutine - def getaddrinfo(*args, **kw): - yield from [] + async def getaddrinfo(*args, **kw): return [(2, 1, 6, '', ('107.6.106.82', 80))] def getaddrinfo_task(*args, **kwds): @@ -1316,7 +1288,8 @@ class BaseEventLoopWithSelectorTests(test_utils.TestCase): self.loop.getaddrinfo.side_effect = mock_getaddrinfo self.loop.sock_connect = mock.Mock() - self.loop.sock_connect.return_value = () + self.loop.sock_connect.return_value = self.loop.create_future() + self.loop.sock_connect.return_value.set_result(None) self.loop._make_ssl_transport = mock.Mock() class _SelectorTransportMock: @@ -1337,34 +1310,45 @@ class BaseEventLoopWithSelectorTests(test_utils.TestCase): self.loop._make_ssl_transport.side_effect = mock_make_ssl_transport ANY = mock.ANY + handshake_timeout = object() # First try the default server_hostname. self.loop._make_ssl_transport.reset_mock() - coro = self.loop.create_connection(MyProto, 'python.org', 80, ssl=True) + coro = self.loop.create_connection( + MyProto, 'python.org', 80, ssl=True, + ssl_handshake_timeout=handshake_timeout) transport, _ = self.loop.run_until_complete(coro) transport.close() self.loop._make_ssl_transport.assert_called_with( ANY, ANY, ANY, ANY, server_side=False, - server_hostname='python.org') + server_hostname='python.org', + ssl_handshake_timeout=handshake_timeout) # Next try an explicit server_hostname. self.loop._make_ssl_transport.reset_mock() - coro = self.loop.create_connection(MyProto, 'python.org', 80, ssl=True, - server_hostname='perl.com') + coro = self.loop.create_connection( + MyProto, 'python.org', 80, ssl=True, + server_hostname='perl.com', + ssl_handshake_timeout=handshake_timeout) transport, _ = self.loop.run_until_complete(coro) transport.close() self.loop._make_ssl_transport.assert_called_with( ANY, ANY, ANY, ANY, server_side=False, - server_hostname='perl.com') + server_hostname='perl.com', + ssl_handshake_timeout=handshake_timeout) # Finally try an explicit empty server_hostname. self.loop._make_ssl_transport.reset_mock() - coro = self.loop.create_connection(MyProto, 'python.org', 80, ssl=True, - server_hostname='') + coro = self.loop.create_connection( + MyProto, 'python.org', 80, ssl=True, + server_hostname='', + ssl_handshake_timeout=handshake_timeout) transport, _ = self.loop.run_until_complete(coro) transport.close() - self.loop._make_ssl_transport.assert_called_with(ANY, ANY, ANY, ANY, - server_side=False, - server_hostname='') + self.loop._make_ssl_transport.assert_called_with( + ANY, ANY, ANY, ANY, + server_side=False, + server_hostname='', + ssl_handshake_timeout=handshake_timeout) def test_create_connection_no_ssl_server_hostname_errors(self): # When not using ssl, server_hostname must be None. @@ -1387,6 +1371,14 @@ class BaseEventLoopWithSelectorTests(test_utils.TestCase): self.addCleanup(sock.close) self.assertRaises(ValueError, self.loop.run_until_complete, coro) + def test_create_connection_ssl_timeout_for_plain_socket(self): + coro = self.loop.create_connection( + MyProto, 'example.com', 80, ssl_handshake_timeout=1) + with self.assertRaisesRegex( + ValueError, + 'ssl_handshake_timeout is only meaningful with ssl'): + self.loop.run_until_complete(coro) + def test_create_server_empty_host(self): # if host is empty string use None instead host = object() @@ -1416,7 +1408,8 @@ class BaseEventLoopWithSelectorTests(test_utils.TestCase): def test_create_server_no_getaddrinfo(self): getaddrinfo = self.loop.getaddrinfo = mock.Mock() - getaddrinfo.return_value = [] + getaddrinfo.return_value = self.loop.create_future() + getaddrinfo.return_value.set_result(None) f = self.loop.create_server(MyProto, 'python.org', 0) self.assertRaises(OSError, self.loop.run_until_complete, f) @@ -1718,10 +1711,11 @@ class BaseEventLoopWithSelectorTests(test_utils.TestCase): self.assertTrue(m_log.error.called) self.assertFalse(sock.close.called) self.loop._remove_reader.assert_called_with(10) - self.loop.call_later.assert_called_with(constants.ACCEPT_RETRY_DELAY, - # self.loop._start_serving - mock.ANY, - MyProto, sock, None, None, mock.ANY) + self.loop.call_later.assert_called_with( + constants.ACCEPT_RETRY_DELAY, + # self.loop._start_serving + mock.ANY, + MyProto, sock, None, None, mock.ANY, mock.ANY) def test_call_coroutine(self): @asyncio.coroutine @@ -1742,7 +1736,8 @@ class BaseEventLoopWithSelectorTests(test_utils.TestCase): with self.assertRaises(TypeError): self.loop.call_at(self.loop.time() + 60, func) with self.assertRaises(TypeError): - self.loop.run_in_executor(None, func) + self.loop.run_until_complete( + self.loop.run_in_executor(None, func)) @mock.patch('asyncio.base_events.logger') def test_log_slow_callbacks(self, m_logger): @@ -1793,5 +1788,171 @@ class RunningLoopTests(unittest.TestCase): outer_loop.close() +class BaseLoopSockSendfileTests(test_utils.TestCase): + + DATA = b"12345abcde" * 16 * 1024 # 160 KiB + + class MyProto(asyncio.Protocol): + + def __init__(self, loop): + self.started = False + self.closed = False + self.data = bytearray() + self.fut = loop.create_future() + self.transport = None + + def connection_made(self, transport): + self.started = True + self.transport = transport + + def data_received(self, data): + self.data.extend(data) + + def connection_lost(self, exc): + self.closed = True + self.fut.set_result(None) + self.transport = None + + async def wait_closed(self): + await self.fut + + @classmethod + def setUpClass(cls): + with open(support.TESTFN, 'wb') as fp: + fp.write(cls.DATA) + super().setUpClass() + + @classmethod + def tearDownClass(cls): + support.unlink(support.TESTFN) + super().tearDownClass() + + def setUp(self): + from asyncio.selector_events import BaseSelectorEventLoop + # BaseSelectorEventLoop() has no native implementation + self.loop = BaseSelectorEventLoop() + self.set_event_loop(self.loop) + self.file = open(support.TESTFN, 'rb') + self.addCleanup(self.file.close) + super().setUp() + + def make_socket(self, blocking=False): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setblocking(blocking) + self.addCleanup(sock.close) + return sock + + def run_loop(self, coro): + return self.loop.run_until_complete(coro) + + def prepare(self): + sock = self.make_socket() + proto = self.MyProto(self.loop) + port = support.find_unused_port() + server = self.run_loop(self.loop.create_server( + lambda: proto, support.HOST, port)) + self.run_loop(self.loop.sock_connect(sock, (support.HOST, port))) + + def cleanup(): + server.close() + self.run_loop(server.wait_closed()) + sock.close() + if proto.transport is not None: + proto.transport.close() + self.run_loop(proto.wait_closed()) + + self.addCleanup(cleanup) + + return sock, proto + + def test__sock_sendfile_native_failure(self): + sock, proto = self.prepare() + + with self.assertRaisesRegex(events.SendfileNotAvailableError, + "sendfile is not available"): + self.run_loop(self.loop._sock_sendfile_native(sock, self.file, + 0, None)) + + self.assertEqual(proto.data, b'') + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_no_fallback(self): + sock, proto = self.prepare() + + with self.assertRaisesRegex(events.SendfileNotAvailableError, + "sendfile is not available"): + self.run_loop(self.loop.sock_sendfile(sock, self.file, + fallback=False)) + + self.assertEqual(self.file.tell(), 0) + self.assertEqual(proto.data, b'') + + def test_sock_sendfile_fallback(self): + sock, proto = self.prepare() + + ret = self.run_loop(self.loop.sock_sendfile(sock, self.file)) + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(self.file.tell(), len(self.DATA)) + self.assertEqual(proto.data, self.DATA) + + def test_sock_sendfile_fallback_offset_and_count(self): + sock, proto = self.prepare() + + ret = self.run_loop(self.loop.sock_sendfile(sock, self.file, + 1000, 2000)) + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(ret, 2000) + self.assertEqual(self.file.tell(), 3000) + self.assertEqual(proto.data, self.DATA[1000:3000]) + + def test_blocking_socket(self): + self.loop.set_debug(True) + sock = self.make_socket(blocking=True) + with self.assertRaisesRegex(ValueError, "must be non-blocking"): + self.run_loop(self.loop.sock_sendfile(sock, self.file)) + + def test_nonbinary_file(self): + sock = self.make_socket() + with open(support.TESTFN, 'r') as f: + with self.assertRaisesRegex(ValueError, "binary mode"): + self.run_loop(self.loop.sock_sendfile(sock, f)) + + def test_nonstream_socket(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setblocking(False) + self.addCleanup(sock.close) + with self.assertRaisesRegex(ValueError, "only SOCK_STREAM type"): + self.run_loop(self.loop.sock_sendfile(sock, self.file)) + + def test_notint_count(self): + sock = self.make_socket() + with self.assertRaisesRegex(TypeError, + "count must be a positive integer"): + self.run_loop(self.loop.sock_sendfile(sock, self.file, 0, 'count')) + + def test_negative_count(self): + sock = self.make_socket() + with self.assertRaisesRegex(ValueError, + "count must be a positive integer"): + self.run_loop(self.loop.sock_sendfile(sock, self.file, 0, -1)) + + def test_notint_offset(self): + sock = self.make_socket() + with self.assertRaisesRegex(TypeError, + "offset must be a non-negative integer"): + self.run_loop(self.loop.sock_sendfile(sock, self.file, 'offset')) + + def test_negative_offset(self): + sock = self.make_socket() + with self.assertRaisesRegex(ValueError, + "offset must be a non-negative integer"): + self.run_loop(self.loop.sock_sendfile(sock, self.file, -1)) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_buffered_proto.py b/Lib/test/test_asyncio/test_buffered_proto.py new file mode 100644 index 00000000000..22f9269e814 --- /dev/null +++ b/Lib/test/test_asyncio/test_buffered_proto.py @@ -0,0 +1,85 @@ +import asyncio +import unittest + +from test.test_asyncio import functional as func_tests + + +class ReceiveStuffProto(asyncio.BufferedProtocol): + def __init__(self, cb, con_lost_fut): + self.cb = cb + self.con_lost_fut = con_lost_fut + + def get_buffer(self): + self.buffer = bytearray(100) + return self.buffer + + def buffer_updated(self, nbytes): + self.cb(self.buffer[:nbytes]) + + def connection_lost(self, exc): + if exc is None: + self.con_lost_fut.set_result(None) + else: + self.con_lost_fut.set_exception(exc) + + +class BaseTestBufferedProtocol(func_tests.FunctionalTestCaseMixin): + + def new_loop(self): + raise NotImplementedError + + def test_buffered_proto_create_connection(self): + + NOISE = b'12345678+' * 1024 + + async def client(addr): + data = b'' + + def on_buf(buf): + nonlocal data + data += buf + if data == NOISE: + tr.write(b'1') + + conn_lost_fut = self.loop.create_future() + + tr, pr = await self.loop.create_connection( + lambda: ReceiveStuffProto(on_buf, conn_lost_fut), *addr) + + await conn_lost_fut + + async def on_server_client(reader, writer): + writer.write(NOISE) + await reader.readexactly(1) + writer.close() + await writer.wait_closed() + + srv = self.loop.run_until_complete( + asyncio.start_server( + on_server_client, '127.0.0.1', 0)) + + addr = srv.sockets[0].getsockname() + self.loop.run_until_complete( + asyncio.wait_for(client(addr), 5, loop=self.loop)) + + srv.close() + self.loop.run_until_complete(srv.wait_closed()) + + +class BufferedProtocolSelectorTests(BaseTestBufferedProtocol, + unittest.TestCase): + + def new_loop(self): + return asyncio.SelectorEventLoop() + + +@unittest.skipUnless(hasattr(asyncio, 'ProactorEventLoop'), 'Windows only') +class BufferedProtocolProactorTests(BaseTestBufferedProtocol, + unittest.TestCase): + + def new_loop(self): + return asyncio.ProactorEventLoop() + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_context.py b/Lib/test/test_asyncio/test_context.py new file mode 100644 index 00000000000..6abddd9f251 --- /dev/null +++ b/Lib/test/test_asyncio/test_context.py @@ -0,0 +1,29 @@ +import asyncio +import decimal +import unittest + + +class DecimalContextTest(unittest.TestCase): + + def test_asyncio_task_decimal_context(self): + async def fractions(t, precision, x, y): + with decimal.localcontext() as ctx: + ctx.prec = precision + a = decimal.Decimal(x) / decimal.Decimal(y) + await asyncio.sleep(t) + b = decimal.Decimal(x) / decimal.Decimal(y ** 2) + return a, b + + async def main(): + r1, r2 = await asyncio.gather( + fractions(0.1, 3, 1, 3), fractions(0.2, 6, 1, 3)) + + return r1, r2 + + r1, r2 = asyncio.run(main()) + + self.assertEqual(str(r1[0]), '0.333') + self.assertEqual(str(r1[1]), '0.111') + + self.assertEqual(str(r2[0]), '0.333333') + self.assertEqual(str(r2[1]), '0.111111') diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 1a8bc134296..f5995974c68 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -3,7 +3,6 @@ import collections.abc import concurrent.futures import functools -import gc import io import os import platform @@ -27,26 +26,14 @@ if sys.platform != 'win32': import tty import asyncio +from asyncio import base_events +from asyncio import constants from asyncio import coroutines +from asyncio import events from asyncio import proactor_events from asyncio import selector_events -from asyncio import sslproto -from asyncio import test_utils -try: - from test import support -except ImportError: - from asyncio import test_support as support - - -def data_file(filename): - if hasattr(support, 'TEST_HOME_DIR'): - fullname = os.path.join(support.TEST_HOME_DIR, filename) - if os.path.isfile(fullname): - return fullname - fullname = os.path.join(os.path.dirname(__file__), filename) - if os.path.isfile(fullname): - return fullname - raise FileNotFoundError(filename) +from test.test_asyncio import utils as test_utils +from test import support def osx_tiger(): @@ -67,21 +54,18 @@ def _test_get_event_loop_new_process__sub_proc(): return loop.run_until_complete(doit()) -ONLYCERT = data_file('ssl_cert.pem') -ONLYKEY = data_file('ssl_key.pem') -SIGNED_CERTFILE = data_file('keycert3.pem') -SIGNING_CA = data_file('pycacert.pem') -PEERCERT = {'serialNumber': 'B09264B1F2DA21D1', - 'version': 1, - 'subject': ((('countryName', 'XY'),), - (('localityName', 'Castle Anthrax'),), - (('organizationName', 'Python Software Foundation'),), - (('commonName', 'localhost'),)), - 'issuer': ((('countryName', 'XY'),), - (('organizationName', 'Python Software Foundation CA'),), - (('commonName', 'our-ca-server'),)), - 'notAfter': 'Nov 13 19:47:07 2022 GMT', - 'notBefore': 'Jan 4 19:47:07 2013 GMT'} +class CoroLike: + def send(self, v): + pass + + def throw(self, *exc): + pass + + def close(self): + pass + + def __await__(self): + pass class MyBaseProto(asyncio.Protocol): @@ -285,10 +269,10 @@ class EventLoopTestsMixin: self.assertTrue(0.08 <= t1-t0 <= 0.8, t1-t0) def test_run_until_complete_stopped(self): - @asyncio.coroutine - def cb(): + + async def cb(): self.loop.stop() - yield from asyncio.sleep(0.1, loop=self.loop) + await asyncio.sleep(0.1, loop=self.loop) task = cb() self.assertRaises(RuntimeError, self.loop.run_until_complete, task) @@ -363,7 +347,7 @@ class EventLoopTestsMixin: self.assertNotEqual(thread_id, threading.get_ident()) def test_reader_callback(self): - r, w = test_utils.socketpair() + r, w = socket.socketpair() r.setblocking(False) bytes_read = bytearray() @@ -391,7 +375,7 @@ class EventLoopTestsMixin: self.assertEqual(bytes_read, b'abcdef') def test_writer_callback(self): - r, w = test_utils.socketpair() + r, w = socket.socketpair() w.setblocking(False) def writer(data): @@ -470,7 +454,7 @@ class EventLoopTestsMixin: sock = socket.socket() self._basetest_sock_recv_into(httpd, sock) - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_unix_sock_client_ops(self): with test_utils.run_test_unix_server() as httpd: sock = socket.socket(socket.AF_UNIX) @@ -606,7 +590,7 @@ class EventLoopTestsMixin: lambda: MyProto(loop=self.loop), *httpd.address) self._basetest_create_connection(conn_fut) - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_create_unix_connection(self): # Issue #20682: On Mac OS X Tiger, getsockname() returns a # zero-length address for UNIX socket. @@ -736,8 +720,8 @@ class EventLoopTestsMixin: self._test_create_ssl_connection(httpd, create_connection, peername=httpd.address) + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_create_ssl_unix_connection(self): # Issue #20682: On Mac OS X Tiger, getsockname() returns a # zero-length address for UNIX socket. @@ -840,19 +824,21 @@ class EventLoopTestsMixin: 'SSL not supported with proactor event loops before Python 3.5' ) - server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - server_context.load_cert_chain(ONLYCERT, ONLYKEY) - if hasattr(server_context, 'check_hostname'): - server_context.check_hostname = False - server_context.verify_mode = ssl.CERT_NONE - - client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - if hasattr(server_context, 'check_hostname'): - client_context.check_hostname = False - client_context.verify_mode = ssl.CERT_NONE + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() self.test_connect_accepted_socket(server_context, client_context) + def test_connect_accepted_socket_ssl_timeout_for_plain_socket(self): + sock = socket.socket() + self.addCleanup(sock.close) + coro = self.loop.connect_accepted_socket( + MyProto, sock, ssl_handshake_timeout=1) + with self.assertRaisesRegex( + ValueError, + 'ssl_handshake_timeout is only meaningful with ssl'): + self.loop.run_until_complete(coro) + @mock.patch('asyncio.base_events.socket') def create_server_multiple_hosts(self, family, hosts, mock_sock): @asyncio.coroutine @@ -961,7 +947,7 @@ class EventLoopTestsMixin: return server, path - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_create_unix_server(self): proto = MyProto(loop=self.loop) server, path = self._make_unix_server(lambda: proto) @@ -1025,7 +1011,7 @@ class EventLoopTestsMixin: def test_create_server_ssl(self): proto = MyProto(loop=self.loop) server, host, port = self._make_ssl_server( - lambda: proto, ONLYCERT, ONLYKEY) + lambda: proto, test_utils.ONLYCERT, test_utils.ONLYKEY) f_c = self.loop.create_connection(MyBaseProto, host, port, ssl=test_utils.dummy_ssl_context()) @@ -1053,12 +1039,12 @@ class EventLoopTestsMixin: # stop serving server.close() + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_create_unix_server_ssl(self): proto = MyProto(loop=self.loop) server, path = self._make_ssl_unix_server( - lambda: proto, ONLYCERT, ONLYKEY) + lambda: proto, test_utils.ONLYCERT, test_utils.ONLYKEY) f_c = self.loop.create_unix_connection( MyBaseProto, path, ssl=test_utils.dummy_ssl_context(), @@ -1088,7 +1074,7 @@ class EventLoopTestsMixin: def test_create_server_ssl_verify_failed(self): proto = MyProto(loop=self.loop) server, host, port = self._make_ssl_server( - lambda: proto, SIGNED_CERTFILE) + lambda: proto, test_utils.SIGNED_CERTFILE) sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) sslcontext_client.options |= ssl.OP_NO_SSLv2 @@ -1113,12 +1099,12 @@ class EventLoopTestsMixin: self.assertIsNone(proto.transport) server.close() + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_create_unix_server_ssl_verify_failed(self): proto = MyProto(loop=self.loop) server, path = self._make_ssl_unix_server( - lambda: proto, SIGNED_CERTFILE) + lambda: proto, test_utils.SIGNED_CERTFILE) sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) sslcontext_client.options |= ssl.OP_NO_SSLv2 @@ -1147,13 +1133,13 @@ class EventLoopTestsMixin: def test_create_server_ssl_match_failed(self): proto = MyProto(loop=self.loop) server, host, port = self._make_ssl_server( - lambda: proto, SIGNED_CERTFILE) + lambda: proto, test_utils.SIGNED_CERTFILE) sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) sslcontext_client.options |= ssl.OP_NO_SSLv2 sslcontext_client.verify_mode = ssl.CERT_REQUIRED sslcontext_client.load_verify_locations( - cafile=SIGNING_CA) + cafile=test_utils.SIGNING_CA) if hasattr(sslcontext_client, 'check_hostname'): sslcontext_client.check_hostname = True @@ -1164,24 +1150,26 @@ class EventLoopTestsMixin: with test_utils.disable_logger(): with self.assertRaisesRegex( ssl.CertificateError, - "hostname '127.0.0.1' doesn't match 'localhost'"): + "IP address mismatch, certificate is not valid for " + "'127.0.0.1'"): self.loop.run_until_complete(f_c) # close connection - proto.transport.close() + # transport is None because TLS ALERT aborted the handshake + self.assertIsNone(proto.transport) server.close() + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_create_unix_server_ssl_verified(self): proto = MyProto(loop=self.loop) server, path = self._make_ssl_unix_server( - lambda: proto, SIGNED_CERTFILE) + lambda: proto, test_utils.SIGNED_CERTFILE) sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) sslcontext_client.options |= ssl.OP_NO_SSLv2 sslcontext_client.verify_mode = ssl.CERT_REQUIRED - sslcontext_client.load_verify_locations(cafile=SIGNING_CA) + sslcontext_client.load_verify_locations(cafile=test_utils.SIGNING_CA) if hasattr(sslcontext_client, 'check_hostname'): sslcontext_client.check_hostname = True @@ -1201,12 +1189,12 @@ class EventLoopTestsMixin: def test_create_server_ssl_verified(self): proto = MyProto(loop=self.loop) server, host, port = self._make_ssl_server( - lambda: proto, SIGNED_CERTFILE) + lambda: proto, test_utils.SIGNED_CERTFILE) sslcontext_client = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) sslcontext_client.options |= ssl.OP_NO_SSLv2 sslcontext_client.verify_mode = ssl.CERT_REQUIRED - sslcontext_client.load_verify_locations(cafile=SIGNING_CA) + sslcontext_client.load_verify_locations(cafile=test_utils.SIGNING_CA) if hasattr(sslcontext_client, 'check_hostname'): sslcontext_client.check_hostname = True @@ -1218,7 +1206,7 @@ class EventLoopTestsMixin: # extra info is available self.check_ssl_extra_info(client,peername=(host, port), - peercert=PEERCERT) + peercert=test_utils.PEERCERT) # close connection proto.transport.close() @@ -1424,9 +1412,8 @@ class EventLoopTestsMixin: rpipe, wpipe = os.pipe() pipeobj = io.open(rpipe, 'rb', 1024) - @asyncio.coroutine - def connect(): - t, p = yield from self.loop.connect_read_pipe( + async def connect(): + t, p = await self.loop.connect_read_pipe( lambda: proto, pipeobj) self.assertIs(p, proto) self.assertIs(t, proto.transport) @@ -1463,11 +1450,10 @@ class EventLoopTestsMixin: rpipeobj = io.open(rpipe, 'rb', 1024) wpipeobj = io.open(wpipe, 'w', 1024) - @asyncio.coroutine - def connect(): - read_transport, _ = yield from loop.connect_read_pipe( + async def connect(): + read_transport, _ = await loop.connect_read_pipe( lambda: read_proto, rpipeobj) - write_transport, _ = yield from loop.connect_write_pipe( + write_transport, _ = await loop.connect_write_pipe( lambda: write_proto, wpipeobj) return read_transport, write_transport @@ -1488,21 +1474,16 @@ class EventLoopTestsMixin: @unittest.skipUnless(sys.platform != 'win32', "Don't support pipes for Windows") - # select, poll and kqueue don't support character devices (PTY) on Mac OS X - # older than 10.6 (Snow Leopard) - @support.requires_mac_ver(10, 6) - # Issue #20495: The test hangs on FreeBSD 7.2 but pass on FreeBSD 9 - @support.requires_freebsd_version(8) + @unittest.skipIf(sys.platform == 'darwin', 'test hangs on MacOS') def test_read_pty_output(self): proto = MyReadPipeProto(loop=self.loop) master, slave = os.openpty() master_read_obj = io.open(master, 'rb', 0) - @asyncio.coroutine - def connect(): - t, p = yield from self.loop.connect_read_pipe(lambda: proto, - master_read_obj) + async def connect(): + t, p = await self.loop.connect_read_pipe(lambda: proto, + master_read_obj) self.assertIs(p, proto) self.assertIs(t, proto.transport) self.assertEqual(['INITIAL', 'CONNECTED'], proto.state) @@ -1568,7 +1549,7 @@ class EventLoopTestsMixin: @unittest.skipUnless(sys.platform != 'win32', "Don't support pipes for Windows") def test_write_pipe_disconnect_on_close(self): - rsock, wsock = test_utils.socketpair() + rsock, wsock = socket.socketpair() rsock.setblocking(False) pipeobj = io.open(wsock.detach(), 'wb', 1024) @@ -1706,18 +1687,17 @@ class EventLoopTestsMixin: self.assertEqual('CLOSED', write_proto.state) def test_prompt_cancellation(self): - r, w = test_utils.socketpair() + r, w = socket.socketpair() r.setblocking(False) - f = self.loop.sock_recv(r, 1) + f = self.loop.create_task(self.loop.sock_recv(r, 1)) ov = getattr(f, 'ov', None) if ov is not None: self.assertTrue(ov.pending) - @asyncio.coroutine - def main(): + async def main(): try: self.loop.call_soon(f.cancel) - yield from f + await f except asyncio.CancelledError: res = 'cancelled' else: @@ -1750,14 +1730,13 @@ class EventLoopTestsMixin: self.loop._run_once_counter = 0 self.loop._run_once = _run_once - @asyncio.coroutine - def wait(): + async def wait(): loop = self.loop - yield from asyncio.sleep(1e-2, loop=loop) - yield from asyncio.sleep(1e-4, loop=loop) - yield from asyncio.sleep(1e-6, loop=loop) - yield from asyncio.sleep(1e-8, loop=loop) - yield from asyncio.sleep(1e-10, loop=loop) + await asyncio.sleep(1e-2, loop=loop) + await asyncio.sleep(1e-4, loop=loop) + await asyncio.sleep(1e-6, loop=loop) + await asyncio.sleep(1e-8, loop=loop) + await asyncio.sleep(1e-10, loop=loop) self.loop.run_until_complete(wait()) # The ideal number of call is 12, but on some platforms, the selector @@ -1771,7 +1750,7 @@ class EventLoopTestsMixin: def test_remove_fds_after_closing(self): loop = self.create_event_loop() callback = lambda: None - r, w = test_utils.socketpair() + r, w = socket.socketpair() self.addCleanup(r.close) self.addCleanup(w.close) loop.add_reader(r, callback) @@ -1783,7 +1762,7 @@ class EventLoopTestsMixin: def test_add_fds_after_closing(self): loop = self.create_event_loop() callback = lambda: None - r, w = test_utils.socketpair() + r, w = socket.socketpair() self.addCleanup(r.close) self.addCleanup(w.close) loop.close() @@ -1826,13 +1805,18 @@ class EventLoopTestsMixin: self.loop.call_later(1.0, func) with self.assertRaises(RuntimeError): self.loop.call_at(self.loop.time() + .0, func) - with self.assertRaises(RuntimeError): - self.loop.run_in_executor(None, func) with self.assertRaises(RuntimeError): self.loop.create_task(coro) with self.assertRaises(RuntimeError): self.loop.add_signal_handler(signal.SIGTERM, func) + # run_in_executor test is tricky: the method is a coroutine, + # but run_until_complete cannot be called on closed loop. + # Thus iterate once explicitly. + with self.assertRaises(RuntimeError): + it = self.loop.run_in_executor(None, func).__await__() + next(it) + class SubprocessTestsMixin: @@ -2076,9 +2060,9 @@ class SubprocessTestsMixin: self.assertEqual(7, proto.returncode) def test_subprocess_exec_invalid_args(self): - @asyncio.coroutine - def connect(**kwds): - yield from self.loop.subprocess_exec( + + async def connect(**kwds): + await self.loop.subprocess_exec( asyncio.SubprocessProtocol, 'pwd', **kwds) @@ -2090,11 +2074,11 @@ class SubprocessTestsMixin: self.loop.run_until_complete(connect(shell=True)) def test_subprocess_shell_invalid_args(self): - @asyncio.coroutine - def connect(cmd=None, **kwds): + + async def connect(cmd=None, **kwds): if not cmd: cmd = 'pwd' - yield from self.loop.subprocess_shell( + await self.loop.subprocess_shell( asyncio.SubprocessProtocol, cmd, **kwds) @@ -2108,14 +2092,318 @@ class SubprocessTestsMixin: self.loop.run_until_complete(connect(shell=False)) +class MySendfileProto(MyBaseProto): + + def __init__(self, loop=None, close_after=0): + super().__init__(loop) + self.data = bytearray() + self.close_after = close_after + + def data_received(self, data): + self.data.extend(data) + super().data_received(data) + if self.close_after and self.nbytes >= self.close_after: + self.transport.close() + + +class SendfileMixin: + # Note: sendfile via SSL transport is equal to sendfile fallback + + DATA = b"12345abcde" * 160 * 1024 # 160 KiB + + @classmethod + def setUpClass(cls): + with open(support.TESTFN, 'wb') as fp: + fp.write(cls.DATA) + super().setUpClass() + + @classmethod + def tearDownClass(cls): + support.unlink(support.TESTFN) + super().tearDownClass() + + def setUp(self): + self.file = open(support.TESTFN, 'rb') + self.addCleanup(self.file.close) + super().setUp() + + def run_loop(self, coro): + return self.loop.run_until_complete(coro) + + def prepare(self, *, is_ssl=False, close_after=0): + port = support.find_unused_port() + srv_proto = MySendfileProto(loop=self.loop, close_after=close_after) + if is_ssl: + if not ssl: + self.skipTest("No ssl module") + srv_ctx = test_utils.simple_server_sslcontext() + cli_ctx = test_utils.simple_client_sslcontext() + else: + srv_ctx = None + cli_ctx = None + srv_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # reduce recv socket buffer size to test on relative small data sets + srv_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024) + srv_sock.bind((support.HOST, port)) + server = self.run_loop(self.loop.create_server( + lambda: srv_proto, sock=srv_sock, ssl=srv_ctx)) + + if is_ssl: + server_hostname = support.HOST + else: + server_hostname = None + cli_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # reduce send socket buffer size to test on relative small data sets + cli_sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024) + cli_sock.connect((support.HOST, port)) + cli_proto = MySendfileProto(loop=self.loop) + tr, pr = self.run_loop(self.loop.create_connection( + lambda: cli_proto, sock=cli_sock, + ssl=cli_ctx, server_hostname=server_hostname)) + + def cleanup(): + srv_proto.transport.close() + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.run_loop(cli_proto.done) + + server.close() + self.run_loop(server.wait_closed()) + + self.addCleanup(cleanup) + return srv_proto, cli_proto + + @unittest.skipIf(sys.platform == 'win32', "UDP sockets are not supported") + def test_sendfile_not_supported(self): + tr, pr = self.run_loop( + self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(loop=self.loop), + family=socket.AF_INET)) + try: + with self.assertRaisesRegex(RuntimeError, "not supported"): + self.run_loop( + self.loop.sendfile(tr, self.file)) + self.assertEqual(0, self.file.tell()) + finally: + # don't use self.addCleanup because it produces resource warning + tr.close() + + def test_sendfile(self): + srv_proto, cli_proto = self.prepare() + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.nbytes, len(self.DATA)) + self.assertEqual(srv_proto.data, self.DATA) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_force_fallback(self): + srv_proto, cli_proto = self.prepare() + + def sendfile_native(transp, file, offset, count): + # to raise SendfileNotAvailableError + return base_events.BaseEventLoop._sendfile_native( + self.loop, transp, file, offset, count) + + self.loop._sendfile_native = sendfile_native + + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.nbytes, len(self.DATA)) + self.assertEqual(srv_proto.data, self.DATA) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_force_unsupported_native(self): + if sys.platform == 'win32': + if isinstance(self.loop, asyncio.ProactorEventLoop): + self.skipTest("Fails on proactor event loop") + srv_proto, cli_proto = self.prepare() + + def sendfile_native(transp, file, offset, count): + # to raise SendfileNotAvailableError + return base_events.BaseEventLoop._sendfile_native( + self.loop, transp, file, offset, count) + + self.loop._sendfile_native = sendfile_native + + with self.assertRaisesRegex(events.SendfileNotAvailableError, + "not supported"): + self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file, + fallback=False)) + + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(srv_proto.nbytes, 0) + self.assertEqual(self.file.tell(), 0) + + def test_sendfile_ssl(self): + srv_proto, cli_proto = self.prepare(is_ssl=True) + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.nbytes, len(self.DATA)) + self.assertEqual(srv_proto.data, self.DATA) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_for_closing_transp(self): + srv_proto, cli_proto = self.prepare() + cli_proto.transport.close() + with self.assertRaisesRegex(RuntimeError, "is closing"): + self.run_loop(self.loop.sendfile(cli_proto.transport, self.file)) + self.run_loop(srv_proto.done) + self.assertEqual(srv_proto.nbytes, 0) + self.assertEqual(self.file.tell(), 0) + + def test_sendfile_pre_and_post_data(self): + srv_proto, cli_proto = self.prepare() + PREFIX = b'zxcvbnm' * 1024 + SUFFIX = b'0987654321' * 1024 + cli_proto.transport.write(PREFIX) + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + cli_proto.transport.write(SUFFIX) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.data, PREFIX + self.DATA + SUFFIX) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_ssl_pre_and_post_data(self): + srv_proto, cli_proto = self.prepare(is_ssl=True) + PREFIX = b'zxcvbnm' * 1024 + SUFFIX = b'0987654321' * 1024 + cli_proto.transport.write(PREFIX) + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + cli_proto.transport.write(SUFFIX) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.data, PREFIX + self.DATA + SUFFIX) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_partial(self): + srv_proto, cli_proto = self.prepare() + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file, 1000, 100)) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, 100) + self.assertEqual(srv_proto.nbytes, 100) + self.assertEqual(srv_proto.data, self.DATA[1000:1100]) + self.assertEqual(self.file.tell(), 1100) + + def test_sendfile_ssl_partial(self): + srv_proto, cli_proto = self.prepare(is_ssl=True) + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file, 1000, 100)) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, 100) + self.assertEqual(srv_proto.nbytes, 100) + self.assertEqual(srv_proto.data, self.DATA[1000:1100]) + self.assertEqual(self.file.tell(), 1100) + + def test_sendfile_close_peer_after_receiving(self): + srv_proto, cli_proto = self.prepare(close_after=len(self.DATA)) + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.nbytes, len(self.DATA)) + self.assertEqual(srv_proto.data, self.DATA) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_ssl_close_peer_after_receiving(self): + srv_proto, cli_proto = self.prepare(is_ssl=True, + close_after=len(self.DATA)) + ret = self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(srv_proto.nbytes, len(self.DATA)) + self.assertEqual(srv_proto.data, self.DATA) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_close_peer_in_middle_of_receiving(self): + srv_proto, cli_proto = self.prepare(close_after=1024) + with self.assertRaises(ConnectionError): + self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + self.run_loop(srv_proto.done) + + self.assertTrue(1024 <= srv_proto.nbytes < len(self.DATA), + srv_proto.nbytes) + self.assertTrue(1024 <= self.file.tell() < len(self.DATA), + self.file.tell()) + + def test_sendfile_fallback_close_peer_in_middle_of_receiving(self): + + def sendfile_native(transp, file, offset, count): + # to raise SendfileNotAvailableError + return base_events.BaseEventLoop._sendfile_native( + self.loop, transp, file, offset, count) + + self.loop._sendfile_native = sendfile_native + + srv_proto, cli_proto = self.prepare(close_after=1024) + with self.assertRaises(ConnectionError): + self.run_loop( + self.loop.sendfile(cli_proto.transport, self.file)) + self.run_loop(srv_proto.done) + + self.assertTrue(1024 <= srv_proto.nbytes < len(self.DATA), + srv_proto.nbytes) + self.assertTrue(1024 <= self.file.tell() < len(self.DATA), + self.file.tell()) + + @unittest.skipIf(not hasattr(os, 'sendfile'), + "Don't have native sendfile support") + def test_sendfile_prevents_bare_write(self): + srv_proto, cli_proto = self.prepare() + fut = self.loop.create_future() + + async def coro(): + fut.set_result(None) + return await self.loop.sendfile(cli_proto.transport, self.file) + + t = self.loop.create_task(coro()) + self.run_loop(fut) + with self.assertRaisesRegex(RuntimeError, + "sendfile is in progress"): + cli_proto.transport.write(b'data') + ret = self.run_loop(t) + self.assertEqual(ret, len(self.DATA)) + + def test_sendfile_no_fallback_for_fallback_transport(self): + transport = mock.Mock() + transport.is_closing.side_effect = lambda: False + transport._sendfile_compatible = constants._SendfileMode.FALLBACK + with self.assertRaisesRegex(RuntimeError, 'fallback is disabled'): + self.loop.run_until_complete( + self.loop.sendfile(transport, None, fallback=False)) + + if sys.platform == 'win32': - class SelectEventLoopTests(EventLoopTestsMixin, test_utils.TestCase): + class SelectEventLoopTests(EventLoopTestsMixin, + SendfileMixin, + test_utils.TestCase): def create_event_loop(self): return asyncio.SelectorEventLoop() class ProactorEventLoopTests(EventLoopTestsMixin, + SendfileMixin, SubprocessTestsMixin, test_utils.TestCase): @@ -2141,9 +2429,9 @@ if sys.platform == 'win32': def test_remove_fds_after_closing(self): raise unittest.SkipTest("IocpEventLoop does not have add_reader()") else: - from asyncio import selectors + import selectors - class UnixEventLoopTestsMixin(EventLoopTestsMixin): + class UnixEventLoopTestsMixin(EventLoopTestsMixin, SendfileMixin): def setUp(self): super().setUp() watcher = asyncio.SafeChildWatcher() @@ -2154,19 +2442,6 @@ else: asyncio.set_child_watcher(None) super().tearDown() - def test_get_event_loop_new_process(self): - async def main(): - pool = concurrent.futures.ProcessPoolExecutor() - result = await self.loop.run_in_executor( - pool, _test_get_event_loop_new_process__sub_proc) - pool.shutdown() - return result - - self.unpatch_get_running_loop() - - self.assertEqual( - self.loop.run_until_complete(main()), - 'hello') if hasattr(selectors, 'KqueueSelector'): class KqueueEventLoopTests(UnixEventLoopTestsMixin, @@ -2375,20 +2650,7 @@ class HandleTests(test_utils.TestCase): # collections.abc.Coroutine, but lack cr_core or gi_code attributes # (such as ones compiled with Cython). - class Coro: - def send(self, v): - pass - - def throw(self, *exc): - pass - - def close(self): - pass - - def __await__(self): - pass - - coro = Coro() + coro = CoroLike() coro.__name__ = 'AAA' self.assertTrue(asyncio.iscoroutine(coro)) self.assertEqual(coroutines._format_coroutine(coro), 'AAA()') @@ -2399,10 +2661,10 @@ class HandleTests(test_utils.TestCase): coro.cr_running = True self.assertEqual(coroutines._format_coroutine(coro), 'BBB() running') - coro = Coro() + coro = CoroLike() # Some coroutines might not have '__name__', such as # built-in async_gen.asend(). - self.assertEqual(coroutines._format_coroutine(coro), 'Coro()') + self.assertEqual(coroutines._format_coroutine(coro), 'CoroLike()') class TimerTests(unittest.TestCase): @@ -2417,6 +2679,12 @@ class TimerTests(unittest.TestCase): mock.Mock()) self.assertEqual(hash(h), hash(when)) + def test_when(self): + when = time.monotonic() + h = asyncio.TimerHandle(when, lambda: False, (), + mock.Mock()) + self.assertEqual(when, h.when()) + def test_timer(self): def callback(*args): return args @@ -2544,20 +2812,8 @@ class AbstractEventLoopTests(unittest.TestCase): NotImplementedError, loop.time) self.assertRaises( NotImplementedError, loop.call_soon_threadsafe, None) - self.assertRaises( - NotImplementedError, loop.run_in_executor, f, f) self.assertRaises( NotImplementedError, loop.set_default_executor, f) - self.assertRaises( - NotImplementedError, loop.getaddrinfo, 'localhost', 8080) - self.assertRaises( - NotImplementedError, loop.getnameinfo, ('localhost', 8080)) - self.assertRaises( - NotImplementedError, loop.create_connection, f) - self.assertRaises( - NotImplementedError, loop.create_server, f) - self.assertRaises( - NotImplementedError, loop.create_datagram_endpoint, f) self.assertRaises( NotImplementedError, loop.add_reader, 1, f) self.assertRaises( @@ -2566,33 +2822,12 @@ class AbstractEventLoopTests(unittest.TestCase): NotImplementedError, loop.add_writer, 1, f) self.assertRaises( NotImplementedError, loop.remove_writer, 1) - self.assertRaises( - NotImplementedError, loop.sock_recv, f, 10) - self.assertRaises( - NotImplementedError, loop.sock_recv_into, f, 10) - self.assertRaises( - NotImplementedError, loop.sock_sendall, f, 10) - self.assertRaises( - NotImplementedError, loop.sock_connect, f, f) - self.assertRaises( - NotImplementedError, loop.sock_accept, f) self.assertRaises( NotImplementedError, loop.add_signal_handler, 1, f) self.assertRaises( NotImplementedError, loop.remove_signal_handler, 1) self.assertRaises( NotImplementedError, loop.remove_signal_handler, 1) - self.assertRaises( - NotImplementedError, loop.connect_read_pipe, f, - mock.sentinel.pipe) - self.assertRaises( - NotImplementedError, loop.connect_write_pipe, f, - mock.sentinel.pipe) - self.assertRaises( - NotImplementedError, loop.subprocess_shell, f, - mock.sentinel) - self.assertRaises( - NotImplementedError, loop.subprocess_exec, f) self.assertRaises( NotImplementedError, loop.set_exception_handler, f) self.assertRaises( @@ -2604,6 +2839,51 @@ class AbstractEventLoopTests(unittest.TestCase): self.assertRaises( NotImplementedError, loop.set_debug, f) + def test_not_implemented_async(self): + + async def inner(): + f = mock.Mock() + loop = asyncio.AbstractEventLoop() + + with self.assertRaises(NotImplementedError): + await loop.run_in_executor(f, f) + with self.assertRaises(NotImplementedError): + await loop.getaddrinfo('localhost', 8080) + with self.assertRaises(NotImplementedError): + await loop.getnameinfo(('localhost', 8080)) + with self.assertRaises(NotImplementedError): + await loop.create_connection(f) + with self.assertRaises(NotImplementedError): + await loop.create_server(f) + with self.assertRaises(NotImplementedError): + await loop.create_datagram_endpoint(f) + with self.assertRaises(NotImplementedError): + await loop.sock_recv(f, 10) + with self.assertRaises(NotImplementedError): + await loop.sock_recv_into(f, 10) + with self.assertRaises(NotImplementedError): + await loop.sock_sendall(f, 10) + with self.assertRaises(NotImplementedError): + await loop.sock_connect(f, f) + with self.assertRaises(NotImplementedError): + await loop.sock_accept(f) + with self.assertRaises(NotImplementedError): + await loop.sock_sendfile(f, f) + with self.assertRaises(NotImplementedError): + await loop.sendfile(f, f) + with self.assertRaises(NotImplementedError): + await loop.connect_read_pipe(f, mock.sentinel.pipe) + with self.assertRaises(NotImplementedError): + await loop.connect_write_pipe(f, mock.sentinel.pipe) + with self.assertRaises(NotImplementedError): + await loop.subprocess_shell(f, mock.sentinel) + with self.assertRaises(NotImplementedError): + await loop.subprocess_exec(f) + + loop = asyncio.new_event_loop() + loop.run_until_complete(inner()) + loop.close() + class ProtocolsAbsTests(unittest.TestCase): @@ -2719,31 +2999,179 @@ class PolicyTests(unittest.TestCase): self.assertIs(policy, asyncio.get_event_loop_policy()) self.assertIsNot(policy, old_policy) + +class GetEventLoopTestsMixin: + + _get_running_loop_impl = None + _set_running_loop_impl = None + get_running_loop_impl = None + get_event_loop_impl = None + + def setUp(self): + self._get_running_loop_saved = events._get_running_loop + self._set_running_loop_saved = events._set_running_loop + self.get_running_loop_saved = events.get_running_loop + self.get_event_loop_saved = events.get_event_loop + + events._get_running_loop = type(self)._get_running_loop_impl + events._set_running_loop = type(self)._set_running_loop_impl + events.get_running_loop = type(self).get_running_loop_impl + events.get_event_loop = type(self).get_event_loop_impl + + asyncio._get_running_loop = type(self)._get_running_loop_impl + asyncio._set_running_loop = type(self)._set_running_loop_impl + asyncio.get_running_loop = type(self).get_running_loop_impl + asyncio.get_event_loop = type(self).get_event_loop_impl + + super().setUp() + + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + + if sys.platform != 'win32': + watcher = asyncio.SafeChildWatcher() + watcher.attach_loop(self.loop) + asyncio.set_child_watcher(watcher) + + def tearDown(self): + try: + if sys.platform != 'win32': + asyncio.set_child_watcher(None) + + super().tearDown() + finally: + self.loop.close() + asyncio.set_event_loop(None) + + events._get_running_loop = self._get_running_loop_saved + events._set_running_loop = self._set_running_loop_saved + events.get_running_loop = self.get_running_loop_saved + events.get_event_loop = self.get_event_loop_saved + + asyncio._get_running_loop = self._get_running_loop_saved + asyncio._set_running_loop = self._set_running_loop_saved + asyncio.get_running_loop = self.get_running_loop_saved + asyncio.get_event_loop = self.get_event_loop_saved + + if sys.platform != 'win32': + + def test_get_event_loop_new_process(self): + # Issue bpo-32126: The multiprocessing module used by + # ProcessPoolExecutor is not functional when the + # multiprocessing.synchronize module cannot be imported. + support.import_module('multiprocessing.synchronize') + + async def main(): + pool = concurrent.futures.ProcessPoolExecutor() + result = await self.loop.run_in_executor( + pool, _test_get_event_loop_new_process__sub_proc) + pool.shutdown() + return result + + self.assertEqual( + self.loop.run_until_complete(main()), + 'hello') + def test_get_event_loop_returns_running_loop(self): + class TestError(Exception): + pass + class Policy(asyncio.DefaultEventLoopPolicy): def get_event_loop(self): - raise NotImplementedError - - loop = None + raise TestError old_policy = asyncio.get_event_loop_policy() try: asyncio.set_event_loop_policy(Policy()) loop = asyncio.new_event_loop() + + with self.assertRaises(TestError): + asyncio.get_event_loop() + asyncio.set_event_loop(None) + with self.assertRaises(TestError): + asyncio.get_event_loop() + + with self.assertRaisesRegex(RuntimeError, 'no running'): + self.assertIs(asyncio.get_running_loop(), None) self.assertIs(asyncio._get_running_loop(), None) async def func(): self.assertIs(asyncio.get_event_loop(), loop) + self.assertIs(asyncio.get_running_loop(), loop) self.assertIs(asyncio._get_running_loop(), loop) loop.run_until_complete(func()) + + asyncio.set_event_loop(loop) + with self.assertRaises(TestError): + asyncio.get_event_loop() + + asyncio.set_event_loop(None) + with self.assertRaises(TestError): + asyncio.get_event_loop() + finally: asyncio.set_event_loop_policy(old_policy) if loop is not None: loop.close() + with self.assertRaisesRegex(RuntimeError, 'no running'): + self.assertIs(asyncio.get_running_loop(), None) + self.assertIs(asyncio._get_running_loop(), None) +class TestPyGetEventLoop(GetEventLoopTestsMixin, unittest.TestCase): + + _get_running_loop_impl = events._py__get_running_loop + _set_running_loop_impl = events._py__set_running_loop + get_running_loop_impl = events._py_get_running_loop + get_event_loop_impl = events._py_get_event_loop + + +try: + import _asyncio # NoQA +except ImportError: + pass +else: + + class TestCGetEventLoop(GetEventLoopTestsMixin, unittest.TestCase): + + _get_running_loop_impl = events._c__get_running_loop + _set_running_loop_impl = events._c__set_running_loop + get_running_loop_impl = events._c_get_running_loop + get_event_loop_impl = events._c_get_event_loop + + +class TestServer(unittest.TestCase): + + def test_get_loop(self): + loop = asyncio.new_event_loop() + self.addCleanup(loop.close) + proto = MyProto(loop) + server = loop.run_until_complete(loop.create_server(lambda: proto, '0.0.0.0', 0)) + self.assertEqual(server.get_loop(), loop) + server.close() + loop.run_until_complete(server.wait_closed()) + + +class TestAbstractServer(unittest.TestCase): + + def test_close(self): + with self.assertRaises(NotImplementedError): + events.AbstractServer().close() + + def test_wait_closed(self): + loop = asyncio.new_event_loop() + self.addCleanup(loop.close) + + with self.assertRaises(NotImplementedError): + loop.run_until_complete(events.AbstractServer().wait_closed()) + + def test_get_loop(self): + with self.assertRaises(NotImplementedError): + events.AbstractServer().get_loop() + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 4320a901f49..8c837ad6b62 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -9,12 +9,9 @@ import unittest from unittest import mock import asyncio -from asyncio import test_utils from asyncio import futures -try: - from test import support -except ImportError: - from asyncio import test_support as support +from test.test_asyncio import utils as test_utils +from test import support def _fakefunc(f): @@ -142,43 +139,63 @@ class BaseFutureTests: asyncio.set_event_loop(self.loop) f = self._new_future() self.assertIs(f._loop, self.loop) + self.assertIs(f.get_loop(), self.loop) def test_constructor_positional(self): # Make sure Future doesn't accept a positional argument self.assertRaises(TypeError, self._new_future, 42) def test_uninitialized(self): + # Test that C Future doesn't crash when Future.__init__() + # call was skipped. + fut = self.cls.__new__(self.cls, loop=self.loop) self.assertRaises(asyncio.InvalidStateError, fut.result) + fut = self.cls.__new__(self.cls, loop=self.loop) self.assertRaises(asyncio.InvalidStateError, fut.exception) + fut = self.cls.__new__(self.cls, loop=self.loop) with self.assertRaises((RuntimeError, AttributeError)): fut.set_result(None) + fut = self.cls.__new__(self.cls, loop=self.loop) with self.assertRaises((RuntimeError, AttributeError)): fut.set_exception(Exception) + fut = self.cls.__new__(self.cls, loop=self.loop) with self.assertRaises((RuntimeError, AttributeError)): fut.cancel() + fut = self.cls.__new__(self.cls, loop=self.loop) with self.assertRaises((RuntimeError, AttributeError)): fut.add_done_callback(lambda f: None) + fut = self.cls.__new__(self.cls, loop=self.loop) with self.assertRaises((RuntimeError, AttributeError)): fut.remove_done_callback(lambda f: None) - fut = self.cls.__new__(self.cls, loop=self.loop) - with self.assertRaises((RuntimeError, AttributeError)): - fut._schedule_callbacks() + fut = self.cls.__new__(self.cls, loop=self.loop) try: repr(fut) - except AttributeError: + except (RuntimeError, AttributeError): pass + fut = self.cls.__new__(self.cls, loop=self.loop) - fut.cancelled() - fut.done() - iter(fut) + try: + fut.__await__() + except RuntimeError: + pass + + fut = self.cls.__new__(self.cls, loop=self.loop) + try: + iter(fut) + except RuntimeError: + pass + + fut = self.cls.__new__(self.cls, loop=self.loop) + self.assertFalse(fut.cancelled()) + self.assertFalse(fut.done()) def test_cancel(self): f = self._new_future(loop=self.loop) @@ -249,30 +266,32 @@ class BaseFutureTests: self.loop.set_debug(True) f_pending_debug = self._new_future(loop=self.loop) frame = f_pending_debug._source_traceback[-1] - self.assertEqual(repr(f_pending_debug), - '' - % (frame[0], frame[1])) + self.assertEqual( + repr(f_pending_debug), + f'<{self.cls.__name__} pending created at {frame[0]}:{frame[1]}>') f_pending_debug.cancel() self.loop.set_debug(False) f_pending = self._new_future(loop=self.loop) - self.assertEqual(repr(f_pending), '') + self.assertEqual(repr(f_pending), f'<{self.cls.__name__} pending>') f_pending.cancel() f_cancelled = self._new_future(loop=self.loop) f_cancelled.cancel() - self.assertEqual(repr(f_cancelled), '') + self.assertEqual(repr(f_cancelled), f'<{self.cls.__name__} cancelled>') f_result = self._new_future(loop=self.loop) f_result.set_result(4) - self.assertEqual(repr(f_result), '') + self.assertEqual( + repr(f_result), f'<{self.cls.__name__} finished result=4>') self.assertEqual(f_result.result(), 4) exc = RuntimeError() f_exception = self._new_future(loop=self.loop) f_exception.set_exception(exc) - self.assertEqual(repr(f_exception), - '') + self.assertEqual( + repr(f_exception), + f'<{self.cls.__name__} finished exception=RuntimeError()>') self.assertIs(f_exception.exception(), exc) def func_repr(func): @@ -283,11 +302,12 @@ class BaseFutureTests: f_one_callbacks = self._new_future(loop=self.loop) f_one_callbacks.add_done_callback(_fakefunc) fake_repr = func_repr(_fakefunc) - self.assertRegex(repr(f_one_callbacks), - r'' % fake_repr) + self.assertRegex( + repr(f_one_callbacks), + r'<' + self.cls.__name__ + r' pending cb=\[%s\]>' % fake_repr) f_one_callbacks.cancel() self.assertEqual(repr(f_one_callbacks), - '') + f'<{self.cls.__name__} cancelled>') f_two_callbacks = self._new_future(loop=self.loop) f_two_callbacks.add_done_callback(first_cb) @@ -295,7 +315,7 @@ class BaseFutureTests: first_repr = func_repr(first_cb) last_repr = func_repr(last_cb) self.assertRegex(repr(f_two_callbacks), - r'' + r'<' + self.cls.__name__ + r' pending cb=\[%s, %s\]>' % (first_repr, last_repr)) f_many_callbacks = self._new_future(loop=self.loop) @@ -304,11 +324,12 @@ class BaseFutureTests: f_many_callbacks.add_done_callback(_fakefunc) f_many_callbacks.add_done_callback(last_cb) cb_regex = r'%s, <8 more>, %s' % (first_repr, last_repr) - self.assertRegex(repr(f_many_callbacks), - r'' % cb_regex) + self.assertRegex( + repr(f_many_callbacks), + r'<' + self.cls.__name__ + r' pending cb=\[%s\]>' % cb_regex) f_many_callbacks.cancel() self.assertEqual(repr(f_many_callbacks), - '') + f'<{self.cls.__name__} cancelled>') def test_copy_state(self): from asyncio.futures import _copy_future_state @@ -345,9 +366,15 @@ class BaseFutureTests: def test(): arg1, arg2 = coro() - self.assertRaises(AssertionError, test) + with self.assertRaisesRegex(RuntimeError, "await wasn't used"): + test() fut.cancel() + def test_log_traceback(self): + fut = self._new_future(loop=self.loop) + with self.assertRaisesRegex(ValueError, 'can only be set to False'): + fut._log_traceback = True + @mock.patch('asyncio.base_events.logger') def test_tb_logger_abandoned(self, m_log): fut = self._new_future(loop=self.loop) @@ -478,7 +505,7 @@ class BaseFutureTests: support.gc_collect() if sys.version_info >= (3, 4): - regex = r'^Future exception was never retrieved\n' + regex = f'^{self.cls.__name__} exception was never retrieved\n' exc_info = (type(exc), exc, exc.__traceback__) m_log.error.assert_called_once_with(mock.ANY, exc_info=exc_info) else: @@ -534,7 +561,22 @@ class BaseFutureTests: @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') class CFutureTests(BaseFutureTests, test_utils.TestCase): - cls = getattr(futures, '_CFuture') + try: + cls = futures._CFuture + except AttributeError: + cls = None + + +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +class CSubFutureTests(BaseFutureTests, test_utils.TestCase): + try: + class CSubFuture(futures._CFuture): + pass + + cls = CSubFuture + except AttributeError: + cls = None class PyFutureTests(BaseFutureTests, test_utils.TestCase): @@ -559,6 +601,76 @@ class BaseFutureDoneCallbackTests(): def _new_future(self): raise NotImplementedError + def test_callbacks_remove_first_callback(self): + bag = [] + f = self._new_future() + + cb1 = self._make_callback(bag, 42) + cb2 = self._make_callback(bag, 17) + cb3 = self._make_callback(bag, 100) + + f.add_done_callback(cb1) + f.add_done_callback(cb2) + f.add_done_callback(cb3) + + f.remove_done_callback(cb1) + f.remove_done_callback(cb1) + + self.assertEqual(bag, []) + f.set_result('foo') + + self.run_briefly() + + self.assertEqual(bag, [17, 100]) + self.assertEqual(f.result(), 'foo') + + def test_callbacks_remove_first_and_second_callback(self): + bag = [] + f = self._new_future() + + cb1 = self._make_callback(bag, 42) + cb2 = self._make_callback(bag, 17) + cb3 = self._make_callback(bag, 100) + + f.add_done_callback(cb1) + f.add_done_callback(cb2) + f.add_done_callback(cb3) + + f.remove_done_callback(cb1) + f.remove_done_callback(cb2) + f.remove_done_callback(cb1) + + self.assertEqual(bag, []) + f.set_result('foo') + + self.run_briefly() + + self.assertEqual(bag, [100]) + self.assertEqual(f.result(), 'foo') + + def test_callbacks_remove_third_callback(self): + bag = [] + f = self._new_future() + + cb1 = self._make_callback(bag, 42) + cb2 = self._make_callback(bag, 17) + cb3 = self._make_callback(bag, 100) + + f.add_done_callback(cb1) + f.add_done_callback(cb2) + f.add_done_callback(cb3) + + f.remove_done_callback(cb3) + f.remove_done_callback(cb3) + + self.assertEqual(bag, []) + f.set_result('foo') + + self.run_briefly() + + self.assertEqual(bag, [42, 17]) + self.assertEqual(f.result(), 'foo') + def test_callbacks_invoked_on_set_result(self): bag = [] f = self._new_future() @@ -681,6 +793,17 @@ class CFutureDoneCallbackTests(BaseFutureDoneCallbackTests, return futures._CFuture(loop=self.loop) +@unittest.skipUnless(hasattr(futures, '_CFuture'), + 'requires the C _asyncio module') +class CSubFutureDoneCallbackTests(BaseFutureDoneCallbackTests, + test_utils.TestCase): + + def _new_future(self): + class CSubFuture(futures._CFuture): + pass + return CSubFuture(loop=self.loop) + + class PyFutureDoneCallbackTests(BaseFutureDoneCallbackTests, test_utils.TestCase): diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index c85e8b1a32f..3e3dd799273 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -5,12 +5,12 @@ from unittest import mock import re import asyncio -from asyncio import test_utils +from test.test_asyncio import utils as test_utils STR_RGX_REPR = ( r'^<(?P.*?) object at (?P
    .*?)' r'\[(?P' - r'(set|unset|locked|unlocked)(,value:\d)?(,waiters:\d+)?' + r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?' r')\]>\Z' ) RGX_REPR = re.compile(STR_RGX_REPR) @@ -42,7 +42,8 @@ class LockTests(test_utils.TestCase): @asyncio.coroutine def acquire_lock(): - yield from lock + with self.assertWarns(DeprecationWarning): + yield from lock self.loop.run_until_complete(acquire_lock()) self.assertTrue(repr(lock).endswith('[locked]>')) @@ -53,7 +54,8 @@ class LockTests(test_utils.TestCase): @asyncio.coroutine def acquire_lock(): - return (yield from lock) + with self.assertWarns(DeprecationWarning): + return (yield from lock) res = self.loop.run_until_complete(acquire_lock()) @@ -63,27 +65,50 @@ class LockTests(test_utils.TestCase): lock.release() self.assertFalse(lock.locked()) + def test_lock_by_with_statement(self): + loop = asyncio.new_event_loop() # don't use TestLoop quirks + self.set_event_loop(loop) + primitives = [ + asyncio.Lock(loop=loop), + asyncio.Condition(loop=loop), + asyncio.Semaphore(loop=loop), + asyncio.BoundedSemaphore(loop=loop), + ] + + @asyncio.coroutine + def test(lock): + yield from asyncio.sleep(0.01, loop=loop) + self.assertFalse(lock.locked()) + with self.assertWarns(DeprecationWarning): + with (yield from lock) as _lock: + self.assertIs(_lock, None) + self.assertTrue(lock.locked()) + yield from asyncio.sleep(0.01, loop=loop) + self.assertTrue(lock.locked()) + self.assertFalse(lock.locked()) + + for primitive in primitives: + loop.run_until_complete(test(primitive)) + self.assertFalse(primitive.locked()) + def test_acquire(self): lock = asyncio.Lock(loop=self.loop) result = [] self.assertTrue(self.loop.run_until_complete(lock.acquire())) - @asyncio.coroutine - def c1(result): - if (yield from lock.acquire()): + async def c1(result): + if await lock.acquire(): result.append(1) return True - @asyncio.coroutine - def c2(result): - if (yield from lock.acquire()): + async def c2(result): + if await lock.acquire(): result.append(2) return True - @asyncio.coroutine - def c3(result): - if (yield from lock.acquire()): + async def c3(result): + if await lock.acquire(): result.append(3) return True @@ -145,12 +170,11 @@ class LockTests(test_utils.TestCase): # Setup: A has the lock, b and c are waiting. lock = asyncio.Lock(loop=self.loop) - @asyncio.coroutine - def lockit(name, blocker): - yield from lock.acquire() + async def lockit(name, blocker): + await lock.acquire() try: if blocker is not None: - yield from blocker + await blocker finally: lock.release() @@ -176,6 +200,56 @@ class LockTests(test_utils.TestCase): self.assertTrue(tb.cancelled()) self.assertTrue(tc.done()) + def test_cancel_release_race(self): + # Issue 32734 + # Acquire 4 locks, cancel second, release first + # and 2 locks are taken at once. + lock = asyncio.Lock(loop=self.loop) + lock_count = 0 + call_count = 0 + + async def lockit(): + nonlocal lock_count + nonlocal call_count + call_count += 1 + await lock.acquire() + lock_count += 1 + + async def lockandtrigger(): + await lock.acquire() + self.loop.call_soon(trigger) + + def trigger(): + t1.cancel() + lock.release() + + t0 = self.loop.create_task(lockandtrigger()) + t1 = self.loop.create_task(lockit()) + t2 = self.loop.create_task(lockit()) + t3 = self.loop.create_task(lockit()) + + # First loop acquires all + test_utils.run_briefly(self.loop) + self.assertTrue(t0.done()) + + # Second loop calls trigger + test_utils.run_briefly(self.loop) + # Third loop calls cancellation + test_utils.run_briefly(self.loop) + + # Make sure only one lock was taken + self.assertEqual(lock_count, 1) + # While 3 calls were made to lockit() + self.assertEqual(call_count, 3) + self.assertTrue(t1.cancelled() and t2.done()) + + # Cleanup the task that is stuck on acquire. + t3.cancel() + test_utils.run_briefly(self.loop) + self.assertTrue(t3.cancelled()) + + + def test_finished_waiter_cancelled(self): lock = asyncio.Lock(loop=self.loop) @@ -216,7 +290,8 @@ class LockTests(test_utils.TestCase): @asyncio.coroutine def acquire_lock(): - return (yield from lock) + with self.assertWarns(DeprecationWarning): + return (yield from lock) with self.loop.run_until_complete(acquire_lock()): self.assertTrue(lock.locked()) @@ -228,7 +303,8 @@ class LockTests(test_utils.TestCase): @asyncio.coroutine def acquire_lock(): - return (yield from lock) + with self.assertWarns(DeprecationWarning): + return (yield from lock) # This spells "yield from lock" outside a generator. cm = self.loop.run_until_complete(acquire_lock()) @@ -294,19 +370,16 @@ class EventTests(test_utils.TestCase): result = [] - @asyncio.coroutine - def c1(result): - if (yield from ev.wait()): + async def c1(result): + if await ev.wait(): result.append(1) - @asyncio.coroutine - def c2(result): - if (yield from ev.wait()): + async def c2(result): + if await ev.wait(): result.append(2) - @asyncio.coroutine - def c3(result): - if (yield from ev.wait()): + async def c3(result): + if await ev.wait(): result.append(3) t1 = asyncio.Task(c1(result), loop=self.loop) @@ -359,9 +432,8 @@ class EventTests(test_utils.TestCase): ev = asyncio.Event(loop=self.loop) result = [] - @asyncio.coroutine - def c1(result): - if (yield from ev.wait()): + async def c1(result): + if await ev.wait(): result.append(1) return True @@ -408,24 +480,21 @@ class ConditionTests(test_utils.TestCase): cond = asyncio.Condition(loop=self.loop) result = [] - @asyncio.coroutine - def c1(result): - yield from cond.acquire() - if (yield from cond.wait()): + async def c1(result): + await cond.acquire() + if await cond.wait(): result.append(1) return True - @asyncio.coroutine - def c2(result): - yield from cond.acquire() - if (yield from cond.wait()): + async def c2(result): + await cond.acquire() + if await cond.wait(): result.append(2) return True - @asyncio.coroutine - def c3(result): - yield from cond.acquire() - if (yield from cond.wait()): + async def c3(result): + await cond.acquire() + if await cond.wait(): result.append(3) return True @@ -522,10 +591,9 @@ class ConditionTests(test_utils.TestCase): result = [] - @asyncio.coroutine - def c1(result): - yield from cond.acquire() - if (yield from cond.wait_for(predicate)): + async def c1(result): + await cond.acquire() + if await cond.wait_for(predicate): result.append(1) cond.release() return True @@ -567,26 +635,23 @@ class ConditionTests(test_utils.TestCase): cond = asyncio.Condition(loop=self.loop) result = [] - @asyncio.coroutine - def c1(result): - yield from cond.acquire() - if (yield from cond.wait()): + async def c1(result): + await cond.acquire() + if await cond.wait(): result.append(1) cond.release() return True - @asyncio.coroutine - def c2(result): - yield from cond.acquire() - if (yield from cond.wait()): + async def c2(result): + await cond.acquire() + if await cond.wait(): result.append(2) cond.release() return True - @asyncio.coroutine - def c3(result): - yield from cond.acquire() - if (yield from cond.wait()): + async def c3(result): + await cond.acquire() + if await cond.wait(): result.append(3) cond.release() return True @@ -623,18 +688,16 @@ class ConditionTests(test_utils.TestCase): result = [] - @asyncio.coroutine - def c1(result): - yield from cond.acquire() - if (yield from cond.wait()): + async def c1(result): + await cond.acquire() + if await cond.wait(): result.append(1) cond.release() return True - @asyncio.coroutine - def c2(result): - yield from cond.acquire() - if (yield from cond.wait()): + async def c2(result): + await cond.acquire() + if await cond.wait(): result.append(2) cond.release() return True @@ -685,7 +748,8 @@ class ConditionTests(test_utils.TestCase): @asyncio.coroutine def acquire_cond(): - return (yield from cond) + with self.assertWarns(DeprecationWarning): + return (yield from cond) with self.loop.run_until_complete(acquire_cond()): self.assertTrue(cond.locked()) @@ -746,7 +810,7 @@ class SemaphoreTests(test_utils.TestCase): def test_repr(self): sem = asyncio.Semaphore(loop=self.loop) - self.assertTrue(repr(sem).endswith('[unlocked,value:1]>')) + self.assertTrue(repr(sem).endswith('[unlocked, value:1]>')) self.assertTrue(RGX_REPR.match(repr(sem))) self.loop.run_until_complete(sem.acquire()) @@ -768,7 +832,8 @@ class SemaphoreTests(test_utils.TestCase): @asyncio.coroutine def acquire_lock(): - return (yield from sem) + with self.assertWarns(DeprecationWarning): + return (yield from sem) res = self.loop.run_until_complete(acquire_lock()) @@ -791,27 +856,23 @@ class SemaphoreTests(test_utils.TestCase): self.assertTrue(self.loop.run_until_complete(sem.acquire())) self.assertFalse(sem.locked()) - @asyncio.coroutine - def c1(result): - yield from sem.acquire() + async def c1(result): + await sem.acquire() result.append(1) return True - @asyncio.coroutine - def c2(result): - yield from sem.acquire() + async def c2(result): + await sem.acquire() result.append(2) return True - @asyncio.coroutine - def c3(result): - yield from sem.acquire() + async def c3(result): + await sem.acquire() result.append(3) return True - @asyncio.coroutine - def c4(result): - yield from sem.acquire() + async def c4(result): + await sem.acquire() result.append(4) return True @@ -914,7 +975,8 @@ class SemaphoreTests(test_utils.TestCase): @asyncio.coroutine def acquire_lock(): - return (yield from sem) + with self.assertWarns(DeprecationWarning): + return (yield from sem) with self.loop.run_until_complete(acquire_lock()): self.assertFalse(sem.locked()) diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py index 77eb7cd6455..f2d588f5444 100644 --- a/Lib/test/test_asyncio/test_pep492.py +++ b/Lib/test/test_asyncio/test_pep492.py @@ -1,16 +1,29 @@ """Tests support for new syntax introduced by PEP 492.""" +import sys import types import unittest -try: - from test import support -except ImportError: - from asyncio import test_support as support +from test import support from unittest import mock import asyncio -from asyncio import test_utils +from test.test_asyncio import utils as test_utils + + +# Test that asyncio.iscoroutine() uses collections.abc.Coroutine +class FakeCoro: + def send(self, value): + pass + + def throw(self, typ, val=None, tb=None): + pass + + def close(self): + pass + + def __await__(self): + yield class BaseTest(test_utils.TestCase): @@ -59,12 +72,13 @@ class LockTests(BaseTest): async def test(lock): await asyncio.sleep(0.01, loop=self.loop) self.assertFalse(lock.locked()) - with await lock as _lock: - self.assertIs(_lock, None) - self.assertTrue(lock.locked()) - await asyncio.sleep(0.01, loop=self.loop) - self.assertTrue(lock.locked()) - self.assertFalse(lock.locked()) + with self.assertWarns(DeprecationWarning): + with await lock as _lock: + self.assertIs(_lock, None) + self.assertTrue(lock.locked()) + await asyncio.sleep(0.01, loop=self.loop) + self.assertTrue(lock.locked()) + self.assertFalse(lock.locked()) for primitive in primitives: self.loop.run_until_complete(test(primitive)) @@ -101,13 +115,6 @@ class CoroutineTests(BaseTest): finally: f.close() # silence warning - # Test that asyncio.iscoroutine() uses collections.abc.Coroutine - class FakeCoro: - def send(self, value): pass - def throw(self, typ, val=None, tb=None): pass - def close(self): pass - def __await__(self): yield - self.assertTrue(asyncio.iscoroutine(FakeCoro())) def test_iscoroutinefunction(self): @@ -142,35 +149,14 @@ class CoroutineTests(BaseTest): data = self.loop.run_until_complete(foo()) self.assertEqual(data, 'spam') - @mock.patch('asyncio.coroutines.logger') - def test_async_def_wrapped(self, m_log): - async def foo(): - pass + def test_debug_mode_manages_coroutine_origin_tracking(self): async def start(): - foo_coro = foo() - self.assertRegex( - repr(foo_coro), - r'') - - with support.check_warnings((r'.*foo.*was never', - RuntimeWarning)): - foo_coro = None - support.gc_collect() - self.assertTrue(m_log.error.called) - message = m_log.error.call_args[0][0] - self.assertRegex(message, - r'CoroWrapper.*foo.*was never') + self.assertTrue(sys.get_coroutine_origin_tracking_depth() > 0) + self.assertEqual(sys.get_coroutine_origin_tracking_depth(), 0) self.loop.set_debug(True) self.loop.run_until_complete(start()) - - async def start(): - foo_coro = foo() - task = asyncio.ensure_future(foo_coro, loop=self.loop) - self.assertRegex(repr(task), r'Task.*foo.*running') - - self.loop.run_until_complete(start()) - + self.assertEqual(sys.get_coroutine_origin_tracking_depth(), 0) def test_types_coroutine(self): def gen(): @@ -220,9 +206,9 @@ class CoroutineTests(BaseTest): t.cancel() self.loop.set_debug(True) - with self.assertRaisesRegex( - RuntimeError, - r'Cannot await.*test_double_await.*\bafunc\b.*while.*\bsleep\b'): + with self.assertRaises( + RuntimeError, + msg='coroutine is being awaited already'): self.loop.run_until_complete(runner()) diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py index 7a8b52352d6..f627dfce0e1 100644 --- a/Lib/test/test_asyncio/test_proactor_events.py +++ b/Lib/test/test_asyncio/test_proactor_events.py @@ -9,7 +9,7 @@ from asyncio.proactor_events import BaseProactorEventLoop from asyncio.proactor_events import _ProactorSocketTransport from asyncio.proactor_events import _ProactorWritePipeTransport from asyncio.proactor_events import _ProactorDuplexPipeTransport -from asyncio import test_utils +from test.test_asyncio import utils as test_utils def close_transport(transport): @@ -44,12 +44,12 @@ class ProactorSocketTransportTests(test_utils.TestCase): test_utils.run_briefly(self.loop) self.assertIsNone(fut.result()) self.protocol.connection_made(tr) - self.proactor.recv.assert_called_with(self.sock, 4096) + self.proactor.recv.assert_called_with(self.sock, 32768) def test_loop_reading(self): tr = self.socket_transport() tr._loop_reading() - self.loop._proactor.recv.assert_called_with(self.sock, 4096) + self.loop._proactor.recv.assert_called_with(self.sock, 32768) self.assertFalse(self.protocol.data_received.called) self.assertFalse(self.protocol.eof_received.called) @@ -60,7 +60,7 @@ class ProactorSocketTransportTests(test_utils.TestCase): tr = self.socket_transport() tr._read_fut = res tr._loop_reading(res) - self.loop._proactor.recv.assert_called_with(self.sock, 4096) + self.loop._proactor.recv.assert_called_with(self.sock, 32768) self.protocol.data_received.assert_called_with(b'data') def test_loop_reading_no_data(self): @@ -334,26 +334,36 @@ class ProactorSocketTransportTests(test_utils.TestCase): f = asyncio.Future(loop=self.loop) f.set_result(msg) futures.append(f) + self.loop._proactor.recv.side_effect = futures self.loop._run_once() self.assertFalse(tr._paused) + self.assertTrue(tr.is_reading()) self.loop._run_once() self.protocol.data_received.assert_called_with(b'data1') self.loop._run_once() self.protocol.data_received.assert_called_with(b'data2') + + tr.pause_reading() tr.pause_reading() self.assertTrue(tr._paused) + self.assertFalse(tr.is_reading()) for i in range(10): self.loop._run_once() self.protocol.data_received.assert_called_with(b'data2') + + tr.resume_reading() tr.resume_reading() self.assertFalse(tr._paused) + self.assertTrue(tr.is_reading()) self.loop._run_once() self.protocol.data_received.assert_called_with(b'data3') self.loop._run_once() self.protocol.data_received.assert_called_with(b'data4') tr.close() + self.assertFalse(tr.is_reading()) + def pause_writing_transport(self, high): tr = self.socket_transport() @@ -423,7 +433,7 @@ class ProactorSocketTransportTests(test_utils.TestCase): def test_dont_pause_writing(self): tr = self.pause_writing_transport(high=4) - # write a large chunk which completes immedialty, + # write a large chunk which completes immediately, # it should not pause writing fut = asyncio.Future(loop=self.loop) fut.set_result(None) @@ -434,6 +444,197 @@ class ProactorSocketTransportTests(test_utils.TestCase): self.assertFalse(self.protocol.pause_writing.called) +class ProactorSocketTransportBufferedProtoTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.addCleanup(self.loop.close) + self.proactor = mock.Mock() + self.loop._proactor = self.proactor + + self.protocol = test_utils.make_test_protocol(asyncio.BufferedProtocol) + self.buf = mock.Mock() + self.protocol.get_buffer.side_effect = lambda: self.buf + + self.sock = mock.Mock(socket.socket) + + def socket_transport(self, waiter=None): + transport = _ProactorSocketTransport(self.loop, self.sock, + self.protocol, waiter=waiter) + self.addCleanup(close_transport, transport) + return transport + + def test_ctor(self): + fut = asyncio.Future(loop=self.loop) + tr = self.socket_transport(waiter=fut) + test_utils.run_briefly(self.loop) + self.assertIsNone(fut.result()) + self.protocol.connection_made(tr) + self.proactor.recv_into.assert_called_with(self.sock, self.buf) + + def test_loop_reading(self): + tr = self.socket_transport() + tr._loop_reading() + self.loop._proactor.recv_into.assert_called_with(self.sock, self.buf) + self.assertTrue(self.protocol.get_buffer.called) + self.assertFalse(self.protocol.buffer_updated.called) + self.assertFalse(self.protocol.eof_received.called) + + def test_get_buffer_error(self): + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + + self.loop.call_exception_handler = mock.Mock() + self.protocol.get_buffer.side_effect = LookupError() + + transport._loop_reading() + + self.assertTrue(transport._fatal_error.called) + self.assertTrue(self.protocol.get_buffer.called) + self.assertFalse(self.protocol.buffer_updated.called) + + def test_buffer_updated_error(self): + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + + self.loop.call_exception_handler = mock.Mock() + self.protocol.buffer_updated.side_effect = LookupError() + + res = asyncio.Future(loop=self.loop) + res.set_result(10) + transport._read_fut = res + transport._loop_reading(res) + + self.assertTrue(transport._fatal_error.called) + self.assertFalse(self.protocol.get_buffer.called) + self.assertTrue(self.protocol.buffer_updated.called) + + def test_loop_eof_received_error(self): + res = asyncio.Future(loop=self.loop) + res.set_result(0) + + self.protocol.eof_received.side_effect = LookupError() + + tr = self.socket_transport() + tr._fatal_error = mock.Mock() + + tr.close = mock.Mock() + tr._read_fut = res + tr._loop_reading(res) + self.assertFalse(self.loop._proactor.recv_into.called) + self.assertTrue(self.protocol.eof_received.called) + self.assertTrue(tr._fatal_error.called) + + def test_loop_reading_data(self): + res = asyncio.Future(loop=self.loop) + res.set_result(4) + + tr = self.socket_transport() + tr._read_fut = res + tr._loop_reading(res) + self.loop._proactor.recv_into.assert_called_with(self.sock, self.buf) + self.protocol.buffer_updated.assert_called_with(4) + + def test_loop_reading_no_data(self): + res = asyncio.Future(loop=self.loop) + res.set_result(0) + + tr = self.socket_transport() + self.assertRaises(AssertionError, tr._loop_reading, res) + + tr.close = mock.Mock() + tr._read_fut = res + tr._loop_reading(res) + self.assertFalse(self.loop._proactor.recv_into.called) + self.assertTrue(self.protocol.eof_received.called) + self.assertTrue(tr.close.called) + + def test_loop_reading_aborted(self): + err = self.loop._proactor.recv_into.side_effect = \ + ConnectionAbortedError() + + tr = self.socket_transport() + tr._fatal_error = mock.Mock() + tr._loop_reading() + tr._fatal_error.assert_called_with( + err, 'Fatal read error on pipe transport') + + def test_loop_reading_aborted_closing(self): + self.loop._proactor.recv.side_effect = ConnectionAbortedError() + + tr = self.socket_transport() + tr._closing = True + tr._fatal_error = mock.Mock() + tr._loop_reading() + self.assertFalse(tr._fatal_error.called) + + def test_loop_reading_aborted_is_fatal(self): + self.loop._proactor.recv_into.side_effect = ConnectionAbortedError() + tr = self.socket_transport() + tr._closing = False + tr._fatal_error = mock.Mock() + tr._loop_reading() + self.assertTrue(tr._fatal_error.called) + + def test_loop_reading_conn_reset_lost(self): + err = self.loop._proactor.recv_into.side_effect = ConnectionResetError() + + tr = self.socket_transport() + tr._closing = False + tr._fatal_error = mock.Mock() + tr._force_close = mock.Mock() + tr._loop_reading() + self.assertFalse(tr._fatal_error.called) + tr._force_close.assert_called_with(err) + + def test_loop_reading_exception(self): + err = self.loop._proactor.recv_into.side_effect = OSError() + + tr = self.socket_transport() + tr._fatal_error = mock.Mock() + tr._loop_reading() + tr._fatal_error.assert_called_with( + err, 'Fatal read error on pipe transport') + + def test_pause_resume_reading(self): + tr = self.socket_transport() + futures = [] + for msg in [10, 20, 30, 40, 0]: + f = asyncio.Future(loop=self.loop) + f.set_result(msg) + futures.append(f) + + self.loop._proactor.recv_into.side_effect = futures + self.loop._run_once() + self.assertFalse(tr._paused) + self.assertTrue(tr.is_reading()) + self.loop._run_once() + self.protocol.buffer_updated.assert_called_with(10) + self.loop._run_once() + self.protocol.buffer_updated.assert_called_with(20) + + tr.pause_reading() + tr.pause_reading() + self.assertTrue(tr._paused) + self.assertFalse(tr.is_reading()) + for i in range(10): + self.loop._run_once() + self.protocol.buffer_updated.assert_called_with(20) + + tr.resume_reading() + tr.resume_reading() + self.assertFalse(tr._paused) + self.assertTrue(tr.is_reading()) + self.loop._run_once() + self.protocol.buffer_updated.assert_called_with(30) + self.loop._run_once() + self.protocol.buffer_updated.assert_called_with(40) + tr.close() + + self.assertFalse(tr.is_reading()) + + class BaseProactorEventLoopTests(test_utils.TestCase): def setUp(self): @@ -444,15 +645,13 @@ class BaseProactorEventLoopTests(test_utils.TestCase): self.ssock, self.csock = mock.Mock(), mock.Mock() - class EventLoop(BaseProactorEventLoop): - def _socketpair(s): - return (self.ssock, self.csock) - - self.loop = EventLoop(self.proactor) + with mock.patch('asyncio.proactor_events.socket.socketpair', + return_value=(self.ssock, self.csock)): + self.loop = BaseProactorEventLoop(self.proactor) self.set_event_loop(self.loop) @mock.patch.object(BaseProactorEventLoop, 'call_soon') - @mock.patch.object(BaseProactorEventLoop, '_socketpair') + @mock.patch('asyncio.proactor_events.socket.socketpair') def test_ctor(self, socketpair, call_soon): ssock, csock = socketpair.return_value = ( mock.Mock(), mock.Mock()) @@ -485,35 +684,6 @@ class BaseProactorEventLoopTests(test_utils.TestCase): self.loop.close() self.assertFalse(self.loop._close_self_pipe.called) - def test_sock_recv(self): - self.loop.sock_recv(self.sock, 1024) - self.proactor.recv.assert_called_with(self.sock, 1024) - - def test_sock_recv_into(self): - buf = bytearray(10) - self.loop.sock_recv_into(self.sock, buf) - self.proactor.recv_into.assert_called_with(self.sock, buf) - - def test_sock_sendall(self): - self.loop.sock_sendall(self.sock, b'data') - self.proactor.send.assert_called_with(self.sock, b'data') - - def test_sock_connect(self): - self.loop.sock_connect(self.sock, ('1.2.3.4', 123)) - self.proactor.connect.assert_called_with(self.sock, ('1.2.3.4', 123)) - - def test_sock_accept(self): - self.loop.sock_accept(self.sock) - self.proactor.accept.assert_called_with(self.sock) - - def test_socketpair(self): - class EventLoop(BaseProactorEventLoop): - # override the destructor to not log a ResourceWarning - def __del__(self): - pass - self.assertRaises( - NotImplementedError, EventLoop, self.proactor) - def test_make_socket_transport(self): tr = self.loop._make_socket_transport(self.sock, asyncio.Protocol()) self.assertIsInstance(tr, _ProactorSocketTransport) @@ -588,10 +758,21 @@ class BaseProactorEventLoopTests(test_utils.TestCase): self.assertTrue(self.sock.close.called) def test_stop_serving(self): - sock = mock.Mock() - self.loop._stop_serving(sock) - self.assertTrue(sock.close.called) - self.proactor._stop_serving.assert_called_with(sock) + sock1 = mock.Mock() + future1 = mock.Mock() + sock2 = mock.Mock() + future2 = mock.Mock() + self.loop._accept_futures = { + sock1.fileno(): future1, + sock2.fileno(): future2 + } + + self.loop._stop_serving(sock1) + self.assertTrue(sock1.close.called) + self.assertTrue(future1.cancel.called) + self.proactor._stop_serving.assert_called_with(sock1) + self.assertFalse(sock2.close.called) + self.assertFalse(future2.cancel.called) if __name__ == '__main__': diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index 2137cde6f48..efe719ed39a 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -4,7 +4,7 @@ import unittest from unittest import mock import asyncio -from asyncio import test_utils +from test.test_asyncio import utils as test_utils class _QueueTestBase(test_utils.TestCase): @@ -36,27 +36,25 @@ class QueueBasicTests(_QueueTestBase): id_is_present = hex(id(q)) in fn(q) self.assertEqual(expect_id, id_is_present) - @asyncio.coroutine - def add_getter(): + async def add_getter(): q = asyncio.Queue(loop=loop) # Start a task that waits to get. asyncio.Task(q.get(), loop=loop) # Let it start waiting. - yield from asyncio.sleep(0.1, loop=loop) + await asyncio.sleep(0.1, loop=loop) self.assertTrue('_getters[1]' in fn(q)) # resume q.get coroutine to finish generator q.put_nowait(0) loop.run_until_complete(add_getter()) - @asyncio.coroutine - def add_putter(): + async def add_putter(): q = asyncio.Queue(maxsize=1, loop=loop) q.put_nowait(1) # Start a task that waits to put. asyncio.Task(q.put(2), loop=loop) # Let it start waiting. - yield from asyncio.sleep(0.1, loop=loop) + await asyncio.sleep(0.1, loop=loop) self.assertTrue('_putters[1]' in fn(q)) # resume q.put coroutine to finish generator q.get_nowait() @@ -125,24 +123,22 @@ class QueueBasicTests(_QueueTestBase): self.assertEqual(2, q.maxsize) have_been_put = [] - @asyncio.coroutine - def putter(): + async def putter(): for i in range(3): - yield from q.put(i) + await q.put(i) have_been_put.append(i) return True - @asyncio.coroutine - def test(): + async def test(): t = asyncio.Task(putter(), loop=loop) - yield from asyncio.sleep(0.01, loop=loop) + await asyncio.sleep(0.01, loop=loop) # The putter is blocked after putting two items. self.assertEqual([0, 1], have_been_put) self.assertEqual(0, q.get_nowait()) # Let the putter resume and put last item. - yield from asyncio.sleep(0.01, loop=loop) + await asyncio.sleep(0.01, loop=loop) self.assertEqual([0, 1, 2], have_been_put) self.assertEqual(1, q.get_nowait()) self.assertEqual(2, q.get_nowait()) @@ -160,9 +156,8 @@ class QueueGetTests(_QueueTestBase): q = asyncio.Queue(loop=self.loop) q.put_nowait(1) - @asyncio.coroutine - def queue_get(): - return (yield from q.get()) + async def queue_get(): + return await q.get() res = self.loop.run_until_complete(queue_get()) self.assertEqual(1, res) @@ -192,21 +187,19 @@ class QueueGetTests(_QueueTestBase): started = asyncio.Event(loop=loop) finished = False - @asyncio.coroutine - def queue_get(): + async def queue_get(): nonlocal finished started.set() - res = yield from q.get() + res = await q.get() finished = True return res - @asyncio.coroutine - def queue_put(): + async def queue_put(): loop.call_later(0.01, q.put_nowait, 1) queue_get_task = asyncio.Task(queue_get(), loop=loop) - yield from started.wait() + await started.wait() self.assertFalse(finished) - res = yield from queue_get_task + res = await queue_get_task self.assertTrue(finished) return res @@ -236,16 +229,14 @@ class QueueGetTests(_QueueTestBase): q = asyncio.Queue(loop=loop) - @asyncio.coroutine - def queue_get(): - return (yield from asyncio.wait_for(q.get(), 0.051, loop=loop)) + async def queue_get(): + return await asyncio.wait_for(q.get(), 0.051, loop=loop) - @asyncio.coroutine - def test(): + async def test(): get_task = asyncio.Task(queue_get(), loop=loop) - yield from asyncio.sleep(0.01, loop=loop) # let the task start + await asyncio.sleep(0.01, loop=loop) # let the task start q.put_nowait(1) - return (yield from get_task) + return await get_task self.assertEqual(1, loop.run_until_complete(test())) self.assertAlmostEqual(0.06, loop.time()) @@ -275,15 +266,13 @@ class QueueGetTests(_QueueTestBase): def test_why_are_getters_waiting(self): # From issue #268. - @asyncio.coroutine - def consumer(queue, num_expected): + async def consumer(queue, num_expected): for _ in range(num_expected): - yield from queue.get() + await queue.get() - @asyncio.coroutine - def producer(queue, num_items): + async def producer(queue, num_items): for i in range(num_items): - yield from queue.put(i) + await queue.put(i) queue_size = 1 producer_num_items = 5 @@ -301,10 +290,10 @@ class QueueGetTests(_QueueTestBase): yield 0.2 self.loop = self.new_test_loop(a_generator) - @asyncio.coroutine - def consumer(queue): + + async def consumer(queue): try: - item = yield from asyncio.wait_for(queue.get(), 0.1, loop=self.loop) + item = await asyncio.wait_for(queue.get(), 0.1, loop=self.loop) except asyncio.TimeoutError: pass @@ -318,10 +307,9 @@ class QueuePutTests(_QueueTestBase): def test_blocking_put(self): q = asyncio.Queue(loop=self.loop) - @asyncio.coroutine - def queue_put(): + async def queue_put(): # No maxsize, won't block. - yield from q.put(1) + await q.put(1) self.loop.run_until_complete(queue_put()) @@ -338,21 +326,19 @@ class QueuePutTests(_QueueTestBase): started = asyncio.Event(loop=loop) finished = False - @asyncio.coroutine - def queue_put(): + async def queue_put(): nonlocal finished started.set() - yield from q.put(1) - yield from q.put(2) + await q.put(1) + await q.put(2) finished = True - @asyncio.coroutine - def queue_get(): + async def queue_get(): loop.call_later(0.01, q.get_nowait) queue_put_task = asyncio.Task(queue_put(), loop=loop) - yield from started.wait() + await started.wait() self.assertFalse(finished) - yield from queue_put_task + await queue_put_task self.assertTrue(finished) loop.run_until_complete(queue_get()) @@ -464,24 +450,22 @@ class QueuePutTests(_QueueTestBase): self.assertRaises(asyncio.QueueFull, q.put_nowait, 3) q = asyncio.Queue(maxsize=1.3, loop=self.loop) - @asyncio.coroutine - def queue_put(): - yield from q.put(1) - yield from q.put(2) + + async def queue_put(): + await q.put(1) + await q.put(2) self.assertTrue(q.full()) self.loop.run_until_complete(queue_put()) def test_put_cancelled(self): q = asyncio.Queue(loop=self.loop) - @asyncio.coroutine - def queue_put(): - yield from q.put(1) + async def queue_put(): + await q.put(1) return True - @asyncio.coroutine - def test(): - return (yield from q.get()) + async def test(): + return await q.get() t = asyncio.Task(queue_put(), loop=self.loop) self.assertEqual(1, self.loop.run_until_complete(test())) @@ -520,13 +504,11 @@ class QueuePutTests(_QueueTestBase): queue = asyncio.Queue(2, loop=self.loop) - @asyncio.coroutine - def putter(item): - yield from queue.put(item) + async def putter(item): + await queue.put(item) - @asyncio.coroutine - def getter(): - yield + async def getter(): + await asyncio.sleep(0, loop=self.loop) num = queue.qsize() for _ in range(num): item = queue.get_nowait() @@ -538,6 +520,56 @@ class QueuePutTests(_QueueTestBase): self.loop.run_until_complete( asyncio.gather(getter(), t0, t1, t2, t3, loop=self.loop)) + def test_cancelled_puts_not_being_held_in_self_putters(self): + def a_generator(): + yield 0.01 + yield 0.1 + + loop = self.new_test_loop(a_generator) + + # Full queue. + queue = asyncio.Queue(loop=loop, maxsize=1) + queue.put_nowait(1) + + # Task waiting for space to put an item in the queue. + put_task = loop.create_task(queue.put(1)) + loop.run_until_complete(asyncio.sleep(0.01, loop=loop)) + + # Check that the putter is correctly removed from queue._putters when + # the task is canceled. + self.assertEqual(len(queue._putters), 1) + put_task.cancel() + with self.assertRaises(asyncio.CancelledError): + loop.run_until_complete(put_task) + self.assertEqual(len(queue._putters), 0) + + def test_cancelled_put_silence_value_error_exception(self): + def gen(): + yield 0.01 + yield 0.1 + + loop = self.new_test_loop(gen) + + # Full Queue. + queue = asyncio.Queue(1, loop=loop) + queue.put_nowait(1) + + # Task waiting for space to put a item in the queue. + put_task = loop.create_task(queue.put(1)) + loop.run_until_complete(asyncio.sleep(0.01, loop=loop)) + + # get_nowait() remove the future of put_task from queue._putters. + queue.get_nowait() + # When canceled, queue.put is going to remove its future from + # self._putters but it was removed previously by queue.get_nowait(). + put_task.cancel() + + # The ValueError exception triggered by queue._putters.remove(putter) + # inside queue.put should be silenced. + # If the ValueError is silenced we should catch a CancelledError. + with self.assertRaises(asyncio.CancelledError): + loop.run_until_complete(put_task) + class LifoQueueTests(_QueueTestBase): @@ -580,21 +612,19 @@ class _QueueJoinTestMixin: # Join the queue and assert all items have been processed. running = True - @asyncio.coroutine - def worker(): + async def worker(): nonlocal accumulator while running: - item = yield from q.get() + item = await q.get() accumulator += item q.task_done() - @asyncio.coroutine - def test(): + async def test(): tasks = [asyncio.Task(worker(), loop=self.loop) for index in range(2)] - yield from q.join() + await q.join() return tasks tasks = self.loop.run_until_complete(test()) @@ -612,10 +642,9 @@ class _QueueJoinTestMixin: # Test that a queue join()s successfully, and before anything else # (done twice for insurance). - @asyncio.coroutine - def join(): - yield from q.join() - yield from q.join() + async def join(): + await q.join() + await q.join() self.loop.run_until_complete(join()) diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py new file mode 100644 index 00000000000..3b58ddee443 --- /dev/null +++ b/Lib/test/test_asyncio/test_runners.py @@ -0,0 +1,179 @@ +import asyncio +import unittest + +from unittest import mock +from . import utils as test_utils + + +class TestPolicy(asyncio.AbstractEventLoopPolicy): + + def __init__(self, loop_factory): + self.loop_factory = loop_factory + self.loop = None + + def get_event_loop(self): + # shouldn't ever be called by asyncio.run() + raise RuntimeError + + def new_event_loop(self): + return self.loop_factory() + + def set_event_loop(self, loop): + if loop is not None: + # we want to check if the loop is closed + # in BaseTest.tearDown + self.loop = loop + + +class BaseTest(unittest.TestCase): + + def new_loop(self): + loop = asyncio.BaseEventLoop() + loop._process_events = mock.Mock() + loop._selector = mock.Mock() + loop._selector.select.return_value = () + loop.shutdown_ag_run = False + + async def shutdown_asyncgens(): + loop.shutdown_ag_run = True + loop.shutdown_asyncgens = shutdown_asyncgens + + return loop + + def setUp(self): + super().setUp() + + policy = TestPolicy(self.new_loop) + asyncio.set_event_loop_policy(policy) + + def tearDown(self): + policy = asyncio.get_event_loop_policy() + if policy.loop is not None: + self.assertTrue(policy.loop.is_closed()) + self.assertTrue(policy.loop.shutdown_ag_run) + + asyncio.set_event_loop_policy(None) + super().tearDown() + + +class RunTests(BaseTest): + + def test_asyncio_run_return(self): + async def main(): + await asyncio.sleep(0) + return 42 + + self.assertEqual(asyncio.run(main()), 42) + + def test_asyncio_run_raises(self): + async def main(): + await asyncio.sleep(0) + raise ValueError('spam') + + with self.assertRaisesRegex(ValueError, 'spam'): + asyncio.run(main()) + + def test_asyncio_run_only_coro(self): + for o in {1, lambda: None}: + with self.subTest(obj=o), \ + self.assertRaisesRegex(ValueError, + 'a coroutine was expected'): + asyncio.run(o) + + def test_asyncio_run_debug(self): + async def main(expected): + loop = asyncio.get_event_loop() + self.assertIs(loop.get_debug(), expected) + + asyncio.run(main(False)) + asyncio.run(main(True), debug=True) + + def test_asyncio_run_from_running_loop(self): + async def main(): + coro = main() + try: + asyncio.run(coro) + finally: + coro.close() # Suppress ResourceWarning + + with self.assertRaisesRegex(RuntimeError, + 'cannot be called from a running'): + asyncio.run(main()) + + def test_asyncio_run_cancels_hanging_tasks(self): + lo_task = None + + async def leftover(): + await asyncio.sleep(0.1) + + async def main(): + nonlocal lo_task + lo_task = asyncio.create_task(leftover()) + return 123 + + self.assertEqual(asyncio.run(main()), 123) + self.assertTrue(lo_task.done()) + + def test_asyncio_run_reports_hanging_tasks_errors(self): + lo_task = None + call_exc_handler_mock = mock.Mock() + + async def leftover(): + try: + await asyncio.sleep(0.1) + except asyncio.CancelledError: + 1 / 0 + + async def main(): + loop = asyncio.get_running_loop() + loop.call_exception_handler = call_exc_handler_mock + + nonlocal lo_task + lo_task = asyncio.create_task(leftover()) + return 123 + + self.assertEqual(asyncio.run(main()), 123) + self.assertTrue(lo_task.done()) + + call_exc_handler_mock.assert_called_with({ + 'message': test_utils.MockPattern(r'asyncio.run.*shutdown'), + 'task': lo_task, + 'exception': test_utils.MockInstanceOf(ZeroDivisionError) + }) + + def test_asyncio_run_closes_gens_after_hanging_tasks_errors(self): + spinner = None + lazyboy = None + + class FancyExit(Exception): + pass + + async def fidget(): + while True: + yield 1 + await asyncio.sleep(1) + + async def spin(): + nonlocal spinner + spinner = fidget() + try: + async for the_meaning_of_life in spinner: # NoQA + pass + except asyncio.CancelledError: + 1 / 0 + + async def main(): + loop = asyncio.get_running_loop() + loop.call_exception_handler = mock.Mock() + + nonlocal lazyboy + lazyboy = asyncio.create_task(spin()) + raise FancyExit + + with self.assertRaises(FancyExit): + asyncio.run(main()) + + self.assertTrue(lazyboy.done()) + + self.assertIsNone(spinner.ag_frame) + self.assertFalse(spinner.ag_running) diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index e67afcc7204..219ab0eb5b8 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -1,6 +1,7 @@ """Tests for selector_events.py""" import errno +import selectors import socket import unittest from unittest import mock @@ -10,12 +11,12 @@ except ImportError: ssl = None import asyncio -from asyncio import selectors -from asyncio import test_utils from asyncio.selector_events import BaseSelectorEventLoop from asyncio.selector_events import _SelectorTransport from asyncio.selector_events import _SelectorSocketTransport from asyncio.selector_events import _SelectorDatagramTransport +from asyncio.selector_events import _set_nodelay +from test.test_asyncio import utils as test_utils MOCK_ANY = mock.ANY @@ -79,10 +80,23 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): with test_utils.disable_logger(): transport = self.loop._make_ssl_transport( m, asyncio.Protocol(), m, waiter) + + with self.assertRaisesRegex(RuntimeError, + r'SSL transport.*not.*initialized'): + transport.is_reading() + # execute the handshake while the logger is disabled # to ignore SSL handshake failure test_utils.run_briefly(self.loop) + self.assertTrue(transport.is_reading()) + transport.pause_reading() + transport.pause_reading() + self.assertFalse(transport.is_reading()) + transport.resume_reading() + transport.resume_reading() + self.assertTrue(transport.is_reading()) + # Sanity check class_name = transport.__class__.__name__ self.assertIn("ssl", class_name.lower()) @@ -154,9 +168,6 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): self.loop.close() self.assertIsNone(self.loop._selector) - def test_socketpair(self): - self.assertRaises(NotImplementedError, self.loop._socketpair) - def test_read_from_self_tryagain(self): self.loop._ssock.recv.side_effect = BlockingIOError self.assertIsNone(self.loop._read_from_self()) @@ -179,18 +190,29 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): sock = test_utils.mock_nonblocking_socket() self.loop._sock_recv = mock.Mock() - f = self.loop.sock_recv(sock, 1024) - self.assertIsInstance(f, asyncio.Future) - self.loop._sock_recv.assert_called_with(f, None, sock, 1024) + f = self.loop.create_task(self.loop.sock_recv(sock, 1024)) + self.loop.run_until_complete(asyncio.sleep(0.01, loop=self.loop)) + + self.assertEqual(self.loop._sock_recv.call_args[0][1:], + (None, sock, 1024)) + + f.cancel() + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(f) def test_sock_recv_reconnection(self): sock = mock.Mock() sock.fileno.return_value = 10 sock.recv.side_effect = BlockingIOError + sock.gettimeout.return_value = 0.0 self.loop.add_reader = mock.Mock() self.loop.remove_reader = mock.Mock() - fut = self.loop.sock_recv(sock, 1024) + fut = self.loop.create_task( + self.loop.sock_recv(sock, 1024)) + + self.loop.run_until_complete(asyncio.sleep(0.01, loop=self.loop)) + callback = self.loop.add_reader.call_args[0][1] params = self.loop.add_reader.call_args[0][2:] @@ -200,6 +222,8 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): sock.recv.side_effect = OSError(9) callback(*params) + self.loop.run_until_complete(asyncio.sleep(0.01, loop=self.loop)) + self.assertIsInstance(fut.exception(), OSError) self.assertEqual((10,), self.loop.remove_reader.call_args[0]) @@ -247,18 +271,26 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): sock = test_utils.mock_nonblocking_socket() self.loop._sock_sendall = mock.Mock() - f = self.loop.sock_sendall(sock, b'data') - self.assertIsInstance(f, asyncio.Future) + f = self.loop.create_task( + self.loop.sock_sendall(sock, b'data')) + + self.loop.run_until_complete(asyncio.sleep(0.01, loop=self.loop)) + self.assertEqual( - (f, None, sock, b'data'), - self.loop._sock_sendall.call_args[0]) + (None, sock, b'data'), + self.loop._sock_sendall.call_args[0][1:]) + + f.cancel() + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(f) def test_sock_sendall_nodata(self): sock = test_utils.mock_nonblocking_socket() self.loop._sock_sendall = mock.Mock() - f = self.loop.sock_sendall(sock, b'') - self.assertIsInstance(f, asyncio.Future) + f = self.loop.create_task(self.loop.sock_sendall(sock, b'')) + self.loop.run_until_complete(asyncio.sleep(0, loop=self.loop)) + self.assertTrue(f.done()) self.assertIsNone(f.result()) self.assertFalse(self.loop._sock_sendall.called) @@ -267,10 +299,14 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): sock = mock.Mock() sock.fileno.return_value = 10 sock.send.side_effect = BlockingIOError + sock.gettimeout.return_value = 0.0 self.loop.add_writer = mock.Mock() self.loop.remove_writer = mock.Mock() - fut = self.loop.sock_sendall(sock, b'data') + fut = self.loop.create_task(self.loop.sock_sendall(sock, b'data')) + + self.loop.run_until_complete(asyncio.sleep(0.01, loop=self.loop)) + callback = self.loop.add_writer.call_args[0][1] params = self.loop.add_writer.call_args[0][2:] @@ -280,6 +316,8 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): sock.send.side_effect = OSError(9) callback(*params) + self.loop.run_until_complete(asyncio.sleep(0.01, loop=self.loop)) + self.assertIsInstance(fut.exception(), OSError) self.assertEqual((10,), self.loop.remove_writer.call_args[0]) @@ -403,17 +441,17 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): def test_sock_connect_resolve_using_socket_params(self, m_gai): addr = ('need-resolution.com', 8080) sock = test_utils.mock_nonblocking_socket() - m_gai.side_effect = (None, None, None, None, ('127.0.0.1', 0)) - m_gai._is_coroutine = False + + m_gai.side_effect = \ + lambda *args: [(None, None, None, None, ('127.0.0.1', 0))] + con = self.loop.create_task(self.loop.sock_connect(sock, addr)) - while not m_gai.called: - self.loop._run_once() + self.loop.run_until_complete(con) m_gai.assert_called_with( addr[0], addr[1], sock.family, sock.type, sock.proto, 0) - con.cancel() - with self.assertRaises(asyncio.CancelledError): - self.loop.run_until_complete(con) + self.loop.run_until_complete(con) + sock.connect.assert_called_with(('127.0.0.1', 0)) def test__sock_connect(self): f = asyncio.Future(loop=self.loop) @@ -488,10 +526,15 @@ class BaseSelectorEventLoopTests(test_utils.TestCase): sock = test_utils.mock_nonblocking_socket() self.loop._sock_accept = mock.Mock() - f = self.loop.sock_accept(sock) - self.assertIsInstance(f, asyncio.Future) - self.assertEqual( - (f, False, sock), self.loop._sock_accept.call_args[0]) + f = self.loop.create_task(self.loop.sock_accept(sock)) + self.loop.run_until_complete(asyncio.sleep(0.01, loop=self.loop)) + + self.assertFalse(self.loop._sock_accept.call_args[0][1]) + self.assertIs(self.loop._sock_accept.call_args[0][2], sock) + + f.cancel() + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(f) def test__sock_accept(self): f = asyncio.Future(loop=self.loop) @@ -864,15 +907,52 @@ class SelectorSocketTransportTests(test_utils.TestCase): tr = self.socket_transport() test_utils.run_briefly(self.loop) self.assertFalse(tr._paused) + self.assertTrue(tr.is_reading()) self.loop.assert_reader(7, tr._read_ready) + + tr.pause_reading() tr.pause_reading() self.assertTrue(tr._paused) - self.assertFalse(7 in self.loop.readers) + self.assertFalse(tr.is_reading()) + self.loop.assert_no_reader(7) + + tr.resume_reading() tr.resume_reading() self.assertFalse(tr._paused) + self.assertTrue(tr.is_reading()) self.loop.assert_reader(7, tr._read_ready) - with self.assertRaises(RuntimeError): - tr.resume_reading() + + tr.close() + self.assertFalse(tr.is_reading()) + self.loop.assert_no_reader(7) + + def test_read_eof_received_error(self): + transport = self.socket_transport() + transport.close = mock.Mock() + transport._fatal_error = mock.Mock() + + self.loop.call_exception_handler = mock.Mock() + + self.protocol.eof_received.side_effect = LookupError() + + self.sock.recv.return_value = b'' + transport._read_ready() + + self.protocol.eof_received.assert_called_with() + self.assertTrue(transport._fatal_error.called) + + def test_data_received_error(self): + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + + self.loop.call_exception_handler = mock.Mock() + self.protocol.data_received.side_effect = LookupError() + + self.sock.recv.return_value = b'data' + transport._read_ready() + + self.assertTrue(transport._fatal_error.called) + self.assertTrue(self.protocol.data_received.called) def test_read_ready(self): transport = self.socket_transport() @@ -1177,6 +1257,149 @@ class SelectorSocketTransportTests(test_utils.TestCase): remove_writer.assert_called_with(self.sock_fd) +class SelectorSocketTransportBufferedProtocolTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + + self.protocol = test_utils.make_test_protocol(asyncio.BufferedProtocol) + self.buf = mock.Mock() + self.protocol.get_buffer.side_effect = lambda: self.buf + + self.sock = mock.Mock(socket.socket) + self.sock_fd = self.sock.fileno.return_value = 7 + + def socket_transport(self, waiter=None): + transport = _SelectorSocketTransport(self.loop, self.sock, + self.protocol, waiter=waiter) + self.addCleanup(close_transport, transport) + return transport + + def test_ctor(self): + waiter = asyncio.Future(loop=self.loop) + tr = self.socket_transport(waiter=waiter) + self.loop.run_until_complete(waiter) + + self.loop.assert_reader(7, tr._read_ready) + test_utils.run_briefly(self.loop) + self.protocol.connection_made.assert_called_with(tr) + + def test_get_buffer_error(self): + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + + self.loop.call_exception_handler = mock.Mock() + self.protocol.get_buffer.side_effect = LookupError() + + transport._read_ready() + + self.assertTrue(transport._fatal_error.called) + self.assertTrue(self.protocol.get_buffer.called) + self.assertFalse(self.protocol.buffer_updated.called) + + def test_buffer_updated_error(self): + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + + self.loop.call_exception_handler = mock.Mock() + self.protocol.buffer_updated.side_effect = LookupError() + + self.sock.recv_into.return_value = 10 + transport._read_ready() + + self.assertTrue(transport._fatal_error.called) + self.assertTrue(self.protocol.get_buffer.called) + self.assertTrue(self.protocol.buffer_updated.called) + + def test_read_eof_received_error(self): + transport = self.socket_transport() + transport.close = mock.Mock() + transport._fatal_error = mock.Mock() + + self.loop.call_exception_handler = mock.Mock() + + self.protocol.eof_received.side_effect = LookupError() + + self.sock.recv_into.return_value = 0 + transport._read_ready() + + self.protocol.eof_received.assert_called_with() + self.assertTrue(transport._fatal_error.called) + + def test_read_ready(self): + transport = self.socket_transport() + + self.sock.recv_into.return_value = 10 + transport._read_ready() + + self.protocol.get_buffer.assert_called_with() + self.protocol.buffer_updated.assert_called_with(10) + + def test_read_ready_eof(self): + transport = self.socket_transport() + transport.close = mock.Mock() + + self.sock.recv_into.return_value = 0 + transport._read_ready() + + self.protocol.eof_received.assert_called_with() + transport.close.assert_called_with() + + def test_read_ready_eof_keep_open(self): + transport = self.socket_transport() + transport.close = mock.Mock() + + self.sock.recv_into.return_value = 0 + self.protocol.eof_received.return_value = True + transport._read_ready() + + self.protocol.eof_received.assert_called_with() + self.assertFalse(transport.close.called) + + @mock.patch('logging.exception') + def test_read_ready_tryagain(self, m_exc): + self.sock.recv_into.side_effect = BlockingIOError + + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + transport._read_ready() + + self.assertFalse(transport._fatal_error.called) + + @mock.patch('logging.exception') + def test_read_ready_tryagain_interrupted(self, m_exc): + self.sock.recv_into.side_effect = InterruptedError + + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + transport._read_ready() + + self.assertFalse(transport._fatal_error.called) + + @mock.patch('logging.exception') + def test_read_ready_conn_reset(self, m_exc): + err = self.sock.recv_into.side_effect = ConnectionResetError() + + transport = self.socket_transport() + transport._force_close = mock.Mock() + with test_utils.disable_logger(): + transport._read_ready() + transport._force_close.assert_called_with(err) + + @mock.patch('logging.exception') + def test_read_ready_err(self, m_exc): + err = self.sock.recv_into.side_effect = OSError() + + transport = self.socket_transport() + transport._fatal_error = mock.Mock() + transport._read_ready() + + transport._fatal_error.assert_called_with( + err, + 'Fatal read error on socket transport') + + class SelectorDatagramTransportTests(test_utils.TestCase): def setUp(self): @@ -1464,5 +1687,31 @@ class SelectorDatagramTransportTests(test_utils.TestCase): 'Fatal error on transport\nprotocol:.*\ntransport:.*'), exc_info=(ConnectionRefusedError, MOCK_ANY, MOCK_ANY)) + +class TestSelectorUtils(test_utils.TestCase): + def check_set_nodelay(self, sock): + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) + self.assertFalse(opt) + + _set_nodelay(sock) + + opt = sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) + self.assertTrue(opt) + + @unittest.skipUnless(hasattr(socket, 'TCP_NODELAY'), + 'need socket.TCP_NODELAY') + def test_set_nodelay(self): + sock = socket.socket(family=socket.AF_INET, type=socket.SOCK_STREAM, + proto=socket.IPPROTO_TCP) + with sock: + self.check_set_nodelay(sock) + + sock = socket.socket(family=socket.AF_INET, type=socket.SOCK_STREAM, + proto=socket.IPPROTO_TCP) + with sock: + sock.setblocking(False) + self.check_set_nodelay(sock) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_server.py b/Lib/test/test_asyncio/test_server.py new file mode 100644 index 00000000000..034293cb3f5 --- /dev/null +++ b/Lib/test/test_asyncio/test_server.py @@ -0,0 +1,128 @@ +import asyncio +import socket +import time +import threading +import unittest + +from test import support +from test.test_asyncio import utils as test_utils +from test.test_asyncio import functional as func_tests + + +class BaseStartServer(func_tests.FunctionalTestCaseMixin): + + def new_loop(self): + raise NotImplementedError + + def test_start_server_1(self): + HELLO_MSG = b'1' * 1024 * 5 + b'\n' + + def client(sock, addr): + for i in range(10): + time.sleep(0.2) + if srv.is_serving(): + break + else: + raise RuntimeError + + sock.settimeout(2) + sock.connect(addr) + sock.send(HELLO_MSG) + sock.recv_all(1) + sock.close() + + async def serve(reader, writer): + await reader.readline() + main_task.cancel() + writer.write(b'1') + writer.close() + await writer.wait_closed() + + async def main(srv): + async with srv: + await srv.serve_forever() + + srv = self.loop.run_until_complete(asyncio.start_server( + serve, support.HOSTv4, 0, loop=self.loop, start_serving=False)) + + self.assertFalse(srv.is_serving()) + + main_task = self.loop.create_task(main(srv)) + + addr = srv.sockets[0].getsockname() + with self.assertRaises(asyncio.CancelledError): + with self.tcp_client(lambda sock: client(sock, addr)): + self.loop.run_until_complete(main_task) + + self.assertEqual(srv.sockets, []) + + self.assertIsNone(srv._sockets) + self.assertIsNone(srv._waiters) + self.assertFalse(srv.is_serving()) + + with self.assertRaisesRegex(RuntimeError, r'is closed'): + self.loop.run_until_complete(srv.serve_forever()) + + +class SelectorStartServerTests(BaseStartServer, unittest.TestCase): + + def new_loop(self): + return asyncio.SelectorEventLoop() + + @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'no Unix sockets') + def test_start_unix_server_1(self): + HELLO_MSG = b'1' * 1024 * 5 + b'\n' + started = threading.Event() + + def client(sock, addr): + sock.settimeout(2) + started.wait(5) + sock.connect(addr) + sock.send(HELLO_MSG) + sock.recv_all(1) + sock.close() + + async def serve(reader, writer): + await reader.readline() + main_task.cancel() + writer.write(b'1') + writer.close() + await writer.wait_closed() + + async def main(srv): + async with srv: + self.assertFalse(srv.is_serving()) + await srv.start_serving() + self.assertTrue(srv.is_serving()) + started.set() + await srv.serve_forever() + + with test_utils.unix_socket_path() as addr: + srv = self.loop.run_until_complete(asyncio.start_unix_server( + serve, addr, loop=self.loop, start_serving=False)) + + main_task = self.loop.create_task(main(srv)) + + with self.assertRaises(asyncio.CancelledError): + with self.unix_client(lambda sock: client(sock, addr)): + self.loop.run_until_complete(main_task) + + self.assertEqual(srv.sockets, []) + + self.assertIsNone(srv._sockets) + self.assertIsNone(srv._waiters) + self.assertFalse(srv.is_serving()) + + with self.assertRaisesRegex(RuntimeError, r'is closed'): + self.loop.run_until_complete(srv.serve_forever()) + + +@unittest.skipUnless(hasattr(asyncio, 'ProactorEventLoop'), 'Windows only') +class ProactorStartServerTests(BaseStartServer, unittest.TestCase): + + def new_loop(self): + return asyncio.ProactorEventLoop() + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index f573ae8fe77..c4c30bedb4e 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -1,6 +1,8 @@ """Tests for asyncio/sslproto.py.""" +import os import logging +import time import unittest from unittest import mock try: @@ -11,7 +13,9 @@ except ImportError: import asyncio from asyncio import log from asyncio import sslproto -from asyncio import test_utils +from asyncio import tasks +from test.test_asyncio import utils as test_utils +from test.test_asyncio import functional as func_tests @unittest.skipIf(ssl is None, 'No ssl module') @@ -25,7 +29,8 @@ class SslProtoHandshakeTests(test_utils.TestCase): def ssl_protocol(self, waiter=None): sslcontext = test_utils.dummy_ssl_context() app_proto = asyncio.Protocol() - proto = sslproto.SSLProtocol(self.loop, app_proto, sslcontext, waiter) + proto = sslproto.SSLProtocol(self.loop, app_proto, sslcontext, waiter, + ssl_handshake_timeout=0.1) self.assertIs(proto._app_transport.get_protocol(), app_proto) self.addCleanup(proto._app_transport.close) return proto @@ -63,6 +68,32 @@ class SslProtoHandshakeTests(test_utils.TestCase): with test_utils.disable_logger(): self.loop.run_until_complete(handshake_fut) + def test_handshake_timeout(self): + # bpo-29970: Check that a connection is aborted if handshake is not + # completed in timeout period, instead of remaining open indefinitely + ssl_proto = self.ssl_protocol() + transport = self.connection_made(ssl_proto) + + with test_utils.disable_logger(): + self.loop.run_until_complete(tasks.sleep(0.2, loop=self.loop)) + self.assertTrue(transport.abort.called) + + def test_handshake_timeout_zero(self): + sslcontext = test_utils.dummy_ssl_context() + app_proto = mock.Mock() + waiter = mock.Mock() + with self.assertRaisesRegex(ValueError, 'a positive number'): + sslproto.SSLProtocol(self.loop, app_proto, sslcontext, waiter, + ssl_handshake_timeout=0) + + def test_handshake_timeout_negative(self): + sslcontext = test_utils.dummy_ssl_context() + app_proto = mock.Mock() + waiter = mock.Mock() + with self.assertRaisesRegex(ValueError, 'a positive number'): + sslproto.SSLProtocol(self.loop, app_proto, sslcontext, waiter, + ssl_handshake_timeout=-10) + def test_eof_received_waiter(self): waiter = asyncio.Future(loop=self.loop) ssl_proto = self.ssl_protocol(waiter) @@ -130,5 +161,164 @@ class SslProtoHandshakeTests(test_utils.TestCase): self.assertIs(ssl_proto._app_protocol, new_app_proto) +############################################################################## +# Start TLS Tests +############################################################################## + + +class BaseStartTLS(func_tests.FunctionalTestCaseMixin): + + def new_loop(self): + raise NotImplementedError + + def test_start_tls_client_1(self): + HELLO_MSG = b'1' * 1024 * 1024 + + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + + def serve(sock): + sock.settimeout(5) + + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.start_tls(server_context, server_side=True) + + sock.sendall(b'O') + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + sock.close() + + class ClientProto(asyncio.Protocol): + def __init__(self, on_data, on_eof): + self.on_data = on_data + self.on_eof = on_eof + self.con_made_cnt = 0 + + def connection_made(proto, tr): + proto.con_made_cnt += 1 + # Ensure connection_made gets called only once. + self.assertEqual(proto.con_made_cnt, 1) + + def data_received(self, data): + self.on_data.set_result(data) + + def eof_received(self): + self.on_eof.set_result(True) + + async def client(addr): + await asyncio.sleep(0.5, loop=self.loop) + + on_data = self.loop.create_future() + on_eof = self.loop.create_future() + + tr, proto = await self.loop.create_connection( + lambda: ClientProto(on_data, on_eof), *addr) + + tr.write(HELLO_MSG) + new_tr = await self.loop.start_tls(tr, proto, client_context) + + self.assertEqual(await on_data, b'O') + new_tr.write(HELLO_MSG) + await on_eof + + new_tr.close() + + with self.tcp_server(serve) as srv: + self.loop.run_until_complete( + asyncio.wait_for(client(srv.addr), loop=self.loop, timeout=10)) + + def test_start_tls_server_1(self): + HELLO_MSG = b'1' * 1024 * 1024 + + server_context = test_utils.simple_server_sslcontext() + client_context = test_utils.simple_client_sslcontext() + + def client(sock, addr): + time.sleep(0.5) + sock.settimeout(5) + + sock.connect(addr) + data = sock.recv_all(len(HELLO_MSG)) + self.assertEqual(len(data), len(HELLO_MSG)) + + sock.start_tls(client_context) + sock.sendall(HELLO_MSG) + sock.close() + + class ServerProto(asyncio.Protocol): + def __init__(self, on_con, on_eof): + self.on_con = on_con + self.on_eof = on_eof + self.data = b'' + + def connection_made(self, tr): + self.on_con.set_result(tr) + + def data_received(self, data): + self.data += data + + def eof_received(self): + self.on_eof.set_result(1) + + async def main(): + tr = await on_con + tr.write(HELLO_MSG) + + self.assertEqual(proto.data, b'') + + new_tr = await self.loop.start_tls( + tr, proto, server_context, + server_side=True) + + await on_eof + self.assertEqual(proto.data, HELLO_MSG) + new_tr.close() + + server.close() + await server.wait_closed() + + on_con = self.loop.create_future() + on_eof = self.loop.create_future() + proto = ServerProto(on_con, on_eof) + + server = self.loop.run_until_complete( + self.loop.create_server( + lambda: proto, '127.0.0.1', 0)) + addr = server.sockets[0].getsockname() + + with self.tcp_client(lambda sock: client(sock, addr)): + self.loop.run_until_complete( + asyncio.wait_for(main(), loop=self.loop, timeout=10)) + + def test_start_tls_wrong_args(self): + async def main(): + with self.assertRaisesRegex(TypeError, 'SSLContext, got'): + await self.loop.start_tls(None, None, None) + + sslctx = test_utils.simple_server_sslcontext() + with self.assertRaisesRegex(TypeError, 'is not supported'): + await self.loop.start_tls(None, None, sslctx) + + self.loop.run_until_complete(main()) + + +@unittest.skipIf(ssl is None, 'No ssl module') +class SelectorStartTLSTests(BaseStartTLS, unittest.TestCase): + + def new_loop(self): + return asyncio.SelectorEventLoop() + + +@unittest.skipIf(ssl is None, 'No ssl module') +@unittest.skipUnless(hasattr(asyncio, 'ProactorEventLoop'), 'Windows only') +@unittest.skipIf(os.environ.get('APPVEYOR'), 'XXX: issue 32458') +class ProactorStartTLSTests(BaseStartTLS, unittest.TestCase): + + def new_loop(self): + return asyncio.ProactorEventLoop() + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 6d16d200796..63fa13f79e2 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -9,16 +9,17 @@ import sys import threading import unittest from unittest import mock +from test import support try: import ssl except ImportError: ssl = None import asyncio -from asyncio import test_utils +from test.test_asyncio import utils as test_utils -class StreamReaderTests(test_utils.TestCase): +class StreamTests(test_utils.TestCase): DATA = b'line1\nline2\nline3\n' @@ -57,7 +58,7 @@ class StreamReaderTests(test_utils.TestCase): loop=self.loop) self._basetest_open_connection(conn_fut) - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_open_unix_connection(self): with test_utils.run_test_unix_server() as httpd: conn_fut = asyncio.open_unix_connection(httpd.address, @@ -86,8 +87,8 @@ class StreamReaderTests(test_utils.TestCase): self._basetest_open_connection_no_loop_ssl(conn_fut) + @support.skip_unless_bind_unix_socket @unittest.skipIf(ssl is None, 'No ssl module') - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') def test_open_unix_connection_no_loop_ssl(self): with test_utils.run_test_unix_server(use_ssl=True) as httpd: conn_fut = asyncio.open_unix_connection( @@ -113,7 +114,7 @@ class StreamReaderTests(test_utils.TestCase): loop=self.loop) self._basetest_open_connection_error(conn_fut) - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_open_unix_connection_error(self): with test_utils.run_test_unix_server() as httpd: conn_fut = asyncio.open_unix_connection(httpd.address, @@ -570,11 +571,10 @@ class StreamReaderTests(test_utils.TestCase): self.server = None self.loop = loop - @asyncio.coroutine - def handle_client(self, client_reader, client_writer): - data = yield from client_reader.readline() + async def handle_client(self, client_reader, client_writer): + data = await client_reader.readline() client_writer.write(data) - yield from client_writer.drain() + await client_writer.drain() client_writer.close() def start(self): @@ -607,14 +607,13 @@ class StreamReaderTests(test_utils.TestCase): self.loop.run_until_complete(self.server.wait_closed()) self.server = None - @asyncio.coroutine - def client(addr): - reader, writer = yield from asyncio.open_connection( + async def client(addr): + reader, writer = await asyncio.open_connection( *addr, loop=self.loop) # send a line writer.write(b"hello world!\n") # read it back - msgback = yield from reader.readline() + msgback = await reader.readline() writer.close() return msgback @@ -634,7 +633,7 @@ class StreamReaderTests(test_utils.TestCase): server.stop() self.assertEqual(msg, b"hello world!\n") - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'No UNIX Sockets') + @support.skip_unless_bind_unix_socket def test_start_unix_server(self): class MyServer: @@ -644,11 +643,10 @@ class StreamReaderTests(test_utils.TestCase): self.loop = loop self.path = path - @asyncio.coroutine - def handle_client(self, client_reader, client_writer): - data = yield from client_reader.readline() + async def handle_client(self, client_reader, client_writer): + data = await client_reader.readline() client_writer.write(data) - yield from client_writer.drain() + await client_writer.drain() client_writer.close() def start(self): @@ -673,14 +671,13 @@ class StreamReaderTests(test_utils.TestCase): self.loop.run_until_complete(self.server.wait_closed()) self.server = None - @asyncio.coroutine - def client(path): - reader, writer = yield from asyncio.open_unix_connection( + async def client(path): + reader, writer = await asyncio.open_unix_connection( path, loop=self.loop) # send a line writer.write(b"hello world!\n") # read it back - msgback = yield from reader.readline() + msgback = await reader.readline() writer.close() return msgback @@ -781,14 +778,13 @@ os.close(fd) clt, _ = sock.accept() clt.close() - @asyncio.coroutine - def client(host, port): - reader, writer = yield from asyncio.open_connection( + async def client(host, port): + reader, writer = await asyncio.open_connection( host, port, loop=self.loop) while True: writer.write(b"foo\n") - yield from writer.drain() + await writer.drain() # Start the server thread and wait for it to be listening. thread = threading.Thread(target=server) @@ -810,7 +806,7 @@ os.close(fd) def test___repr__nondefault_limit(self): stream = asyncio.StreamReader(loop=self.loop, limit=123) - self.assertEqual("", repr(stream)) + self.assertEqual("", repr(stream)) def test___repr__eof(self): stream = asyncio.StreamReader(loop=self.loop) @@ -826,14 +822,15 @@ os.close(fd) stream = asyncio.StreamReader(loop=self.loop) exc = RuntimeError() stream.set_exception(exc) - self.assertEqual("", repr(stream)) + self.assertEqual("", + repr(stream)) def test___repr__waiter(self): stream = asyncio.StreamReader(loop=self.loop) stream._waiter = asyncio.Future(loop=self.loop) self.assertRegex( repr(stream), - r">") + r">") stream._waiter.set_result(None) self.loop.run_until_complete(stream._waiter) stream._waiter = None @@ -844,7 +841,7 @@ os.close(fd) stream._transport = mock.Mock() stream._transport.__repr__ = mock.Mock() stream._transport.__repr__.return_value = "" - self.assertEqual(">", repr(stream)) + self.assertEqual(">", repr(stream)) def test_IncompleteReadError_pickleable(self): e = asyncio.IncompleteReadError(b'abc', 10) @@ -863,6 +860,35 @@ os.close(fd) self.assertEqual(str(e), str(e2)) self.assertEqual(e.consumed, e2.consumed) + def test_wait_closed_on_close(self): + with test_utils.run_test_server() as httpd: + rd, wr = self.loop.run_until_complete( + asyncio.open_connection(*httpd.address, loop=self.loop)) + + wr.write(b'GET / HTTP/1.0\r\n\r\n') + f = rd.readline() + data = self.loop.run_until_complete(f) + self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') + f = rd.read() + data = self.loop.run_until_complete(f) + self.assertTrue(data.endswith(b'\r\n\r\nTest message')) + self.assertFalse(wr.is_closing()) + wr.close() + self.assertTrue(wr.is_closing()) + self.loop.run_until_complete(wr.wait_closed()) + + def test_wait_closed_on_close_with_unread_data(self): + with test_utils.run_test_server() as httpd: + rd, wr = self.loop.run_until_complete( + asyncio.open_connection(*httpd.address, loop=self.loop)) + + wr.write(b'GET / HTTP/1.0\r\n\r\n') + f = rd.readline() + data = self.loop.run_until_complete(f) + self.assertEqual(data, b'HTTP/1.0 200 OK\r\n') + wr.close() + self.loop.run_until_complete(wr.wait_closed()) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index e8822c36698..81b08d6c292 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -7,11 +7,9 @@ from unittest import mock import asyncio from asyncio import base_subprocess from asyncio import subprocess -from asyncio import test_utils -try: - from test import support -except ImportError: - from asyncio import test_support as support +from test.test_asyncio import utils as test_utils +from test import support + if sys.platform != 'win32': from asyncio import unix_events @@ -81,9 +79,8 @@ class SubprocessMixin: def test_stdin_stdout(self): args = PROGRAM_CAT - @asyncio.coroutine - def run(data): - proc = yield from asyncio.create_subprocess_exec( + async def run(data): + proc = await asyncio.create_subprocess_exec( *args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, @@ -91,12 +88,12 @@ class SubprocessMixin: # feed data proc.stdin.write(data) - yield from proc.stdin.drain() + await proc.stdin.drain() proc.stdin.close() # get output and exitcode - data = yield from proc.stdout.read() - exitcode = yield from proc.wait() + data = await proc.stdout.read() + exitcode = await proc.wait() return (exitcode, data) task = run(b'some data') @@ -108,14 +105,13 @@ class SubprocessMixin: def test_communicate(self): args = PROGRAM_CAT - @asyncio.coroutine - def run(data): - proc = yield from asyncio.create_subprocess_exec( + async def run(data): + proc = await asyncio.create_subprocess_exec( *args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, loop=self.loop) - stdout, stderr = yield from proc.communicate(data) + stdout, stderr = await proc.communicate(data) return proc.returncode, stdout task = run(b'some data') @@ -178,14 +174,13 @@ class SubprocessMixin: loop=self.loop) proc = self.loop.run_until_complete(create) - @asyncio.coroutine - def send_signal(proc): + async def send_signal(proc): # basic synchronization to wait until the program is sleeping - line = yield from proc.stdout.readline() + line = await proc.stdout.readline() self.assertEqual(line, b'sleeping\n') proc.send_signal(signal.SIGHUP) - returncode = (yield from proc.wait()) + returncode = await proc.wait() return returncode returncode = self.loop.run_until_complete(send_signal(proc)) @@ -208,10 +203,9 @@ class SubprocessMixin: def test_stdin_broken_pipe(self): proc, large_data = self.prepare_broken_pipe_test() - @asyncio.coroutine - def write_stdin(proc, data): + async def write_stdin(proc, data): proc.stdin.write(data) - yield from proc.stdin.drain() + await proc.stdin.drain() coro = write_stdin(proc, large_data) # drain() must raise BrokenPipeError or ConnectionResetError @@ -232,8 +226,7 @@ class SubprocessMixin: limit = 10 size = (limit * 2 + 1) - @asyncio.coroutine - def test_pause_reading(): + async def test_pause_reading(): code = '\n'.join(( 'import sys', 'sys.stdout.write("x" * %s)' % size, @@ -242,16 +235,15 @@ class SubprocessMixin: connect_read_pipe = self.loop.connect_read_pipe - @asyncio.coroutine - def connect_read_pipe_mock(*args, **kw): - transport, protocol = yield from connect_read_pipe(*args, **kw) + async def connect_read_pipe_mock(*args, **kw): + transport, protocol = await connect_read_pipe(*args, **kw) transport.pause_reading = mock.Mock() transport.resume_reading = mock.Mock() return (transport, protocol) self.loop.connect_read_pipe = connect_read_pipe_mock - proc = yield from asyncio.create_subprocess_exec( + proc = await asyncio.create_subprocess_exec( sys.executable, '-c', code, stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, @@ -259,7 +251,7 @@ class SubprocessMixin: loop=self.loop) stdout_transport = proc._transport.get_pipe_transport(1) - stdout, stderr = yield from proc.communicate() + stdout, stderr = await proc.communicate() # The child process produced more than limit bytes of output, # the stream reader transport should pause the protocol to not @@ -277,18 +269,17 @@ class SubprocessMixin: def test_stdin_not_inheritable(self): # asyncio issue #209: stdin must not be inheritable, otherwise # the Process.communicate() hangs - @asyncio.coroutine - def len_message(message): + async def len_message(message): code = 'import sys; data = sys.stdin.read(); print(len(data))' - proc = yield from asyncio.create_subprocess_exec( + proc = await asyncio.create_subprocess_exec( sys.executable, '-c', code, stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, close_fds=False, loop=self.loop) - stdout, stderr = yield from proc.communicate(message) - exitcode = yield from proc.wait() + stdout, stderr = await proc.communicate(message) + exitcode = await proc.wait() return (stdout, exitcode) output, exitcode = self.loop.run_until_complete(len_message(b'abc')) @@ -296,18 +287,18 @@ class SubprocessMixin: self.assertEqual(exitcode, 0) def test_empty_input(self): - @asyncio.coroutine - def empty_input(): + + async def empty_input(): code = 'import sys; data = sys.stdin.read(); print(len(data))' - proc = yield from asyncio.create_subprocess_exec( + proc = await asyncio.create_subprocess_exec( sys.executable, '-c', code, stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, close_fds=False, loop=self.loop) - stdout, stderr = yield from proc.communicate(b'') - exitcode = yield from proc.wait() + stdout, stderr = await proc.communicate(b'') + exitcode = await proc.wait() return (stdout, exitcode) output, exitcode = self.loop.run_until_complete(empty_input()) @@ -317,9 +308,8 @@ class SubprocessMixin: def test_cancel_process_wait(self): # Issue #23140: cancel Process.wait() - @asyncio.coroutine - def cancel_wait(): - proc = yield from asyncio.create_subprocess_exec( + async def cancel_wait(): + proc = await asyncio.create_subprocess_exec( *PROGRAM_BLOCKED, loop=self.loop) @@ -327,7 +317,7 @@ class SubprocessMixin: task = self.loop.create_task(proc.wait()) self.loop.call_soon(task.cancel) try: - yield from task + await task except asyncio.CancelledError: pass @@ -336,20 +326,20 @@ class SubprocessMixin: # Kill the process and wait until it is done proc.kill() - yield from proc.wait() + await proc.wait() self.loop.run_until_complete(cancel_wait()) def test_cancel_make_subprocess_transport_exec(self): - @asyncio.coroutine - def cancel_make_transport(): + + async def cancel_make_transport(): coro = asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, loop=self.loop) task = self.loop.create_task(coro) self.loop.call_soon(task.cancel) try: - yield from task + await task except asyncio.CancelledError: pass @@ -359,15 +349,15 @@ class SubprocessMixin: self.loop.run_until_complete(cancel_make_transport()) def test_cancel_post_init(self): - @asyncio.coroutine - def cancel_make_transport(): + + async def cancel_make_transport(): coro = self.loop.subprocess_exec(asyncio.SubprocessProtocol, *PROGRAM_BLOCKED) task = self.loop.create_task(coro) self.loop.call_soon(task.cancel) try: - yield from task + await task except asyncio.CancelledError: pass @@ -378,11 +368,11 @@ class SubprocessMixin: test_utils.run_briefly(self.loop) def test_close_kill_running(self): - @asyncio.coroutine - def kill_running(): + + async def kill_running(): create = self.loop.subprocess_exec(asyncio.SubprocessProtocol, *PROGRAM_BLOCKED) - transport, protocol = yield from create + transport, protocol = await create kill_called = False def kill(): @@ -395,7 +385,7 @@ class SubprocessMixin: proc.kill = kill returncode = transport.get_returncode() transport.close() - yield from transport._wait() + await transport._wait() return (returncode, kill_called) # Ignore "Close running child process: kill ..." log @@ -408,11 +398,11 @@ class SubprocessMixin: test_utils.run_briefly(self.loop) def test_close_dont_kill_finished(self): - @asyncio.coroutine - def kill_running(): + + async def kill_running(): create = self.loop.subprocess_exec(asyncio.SubprocessProtocol, *PROGRAM_BLOCKED) - transport, protocol = yield from create + transport, protocol = await create proc = transport.get_extra_info('subprocess') # kill the process (but asyncio is not notified immediately) @@ -444,8 +434,8 @@ class SubprocessMixin: # Unlike SafeChildWatcher, FastChildWatcher does not pop the # callbacks if waitpid() is called elsewhere. Let's clear them # manually to avoid a warning when the watcher is detached. - if sys.platform != 'win32' and \ - isinstance(self, SubprocessFastWatcherTests): + if (sys.platform != 'win32' and + isinstance(self, SubprocessFastWatcherTests)): asyncio.get_child_watcher()._callbacks.clear() def test_popen_error(self): @@ -467,8 +457,8 @@ class SubprocessMixin: self.assertEqual(warns, []) def test_read_stdout_after_process_exit(self): - @asyncio.coroutine - def execute(): + + async def execute(): code = '\n'.join(['import sys', 'for _ in range(64):', ' sys.stdout.write("x" * 4096)', @@ -480,11 +470,11 @@ class SubprocessMixin: stdout=asyncio.subprocess.PIPE, loop=self.loop) - process = yield from fut + process = await fut while True: - data = yield from process.stdout.read(65536) + data = await process.stdout.read(65536) if data: - yield from asyncio.sleep(0.3, loop=self.loop) + await asyncio.sleep(0.3, loop=self.loop) else: break diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index f66f7f1e170..19cf1e667fc 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -2,12 +2,14 @@ import collections import contextlib +import contextvars import functools import gc import io -import os +import random import re import sys +import textwrap import types import unittest import weakref @@ -17,22 +19,9 @@ import asyncio from asyncio import coroutines from asyncio import futures from asyncio import tasks -from asyncio import test_utils -try: - from test import support -except ImportError: - from asyncio import test_support as support -try: - from test.support.script_helper import assert_python_ok -except ImportError: - try: - from test.script_helper import assert_python_ok - except ImportError: - from asyncio.test_support import assert_python_ok - - -PY34 = (sys.version_info >= (3, 4)) -PY35 = (sys.version_info >= (3, 5)) +from test.test_asyncio import utils as test_utils +from test import support +from test.support.script_helper import assert_python_ok @asyncio.coroutine @@ -75,6 +64,20 @@ class Dummy: pass +class CoroLikeObject: + def send(self, v): + raise StopIteration(42) + + def throw(self, *exc): + pass + + def close(self): + pass + + def __await__(self): + return self + + class BaseTaskTests: Task = None @@ -110,9 +113,8 @@ class BaseTaskTests: other_loop = asyncio.new_event_loop() fut = self.new_future(other_loop) - @asyncio.coroutine - def run(fut): - yield from fut + async def run(fut): + await fut try: with self.assertRaisesRegex(RuntimeError, @@ -122,9 +124,9 @@ class BaseTaskTests: other_loop.close() def test_task_awaits_on_itself(self): - @asyncio.coroutine - def test(): - yield from task + + async def test(): + await task task = asyncio.ensure_future(test(), loop=self.loop) @@ -141,6 +143,7 @@ class BaseTaskTests: self.assertTrue(t.done()) self.assertEqual(t.result(), 'ok') self.assertIs(t._loop, self.loop) + self.assertIs(t.get_loop(), self.loop) loop = asyncio.new_event_loop() self.set_event_loop(loop) @@ -209,7 +212,6 @@ class BaseTaskTests: t = asyncio.ensure_future(t_orig, loop=self.loop) self.assertIs(t, t_orig) - @unittest.skipUnless(PY35, 'need python 3.5 or later') def test_ensure_future_awaitable(self): class Aw: def __init__(self, coro): @@ -234,12 +236,10 @@ class BaseTaskTests: def test_get_stack(self): T = None - @asyncio.coroutine - def foo(): - yield from bar() + async def foo(): + await bar() - @asyncio.coroutine - def bar(): + async def bar(): # test get_stack() f = T.get_stack(limit=1) try: @@ -254,11 +254,10 @@ class BaseTaskTests: tb = file.read() self.assertRegex(tb, r'foo\(\) running') - @asyncio.coroutine - def runner(): + async def runner(): nonlocal T T = asyncio.ensure_future(foo(), loop=self.loop) - yield from T + await T self.loop.run_until_complete(runner()) @@ -272,9 +271,8 @@ class BaseTaskTests: # test coroutine function self.assertEqual(notmuch.__name__, 'notmuch') - if PY35: - self.assertRegex(notmuch.__qualname__, - r'\w+.test_task_repr..notmuch') + self.assertRegex(notmuch.__qualname__, + r'\w+.test_task_repr..notmuch') self.assertEqual(notmuch.__module__, __name__) filename, lineno = test_utils.get_function_source(notmuch) @@ -282,14 +280,9 @@ class BaseTaskTests: # test coroutine object gen = notmuch() - if coroutines._DEBUG or PY35: - coro_qualname = 'BaseTaskTests.test_task_repr..notmuch' - else: - coro_qualname = 'notmuch' + coro_qualname = 'BaseTaskTests.test_task_repr..notmuch' self.assertEqual(gen.__name__, 'notmuch') - if PY35: - self.assertEqual(gen.__qualname__, - coro_qualname) + self.assertEqual(gen.__qualname__, coro_qualname) # test pending Task t = self.new_task(self.loop, gen) @@ -332,28 +325,21 @@ class BaseTaskTests: # test coroutine function self.assertEqual(notmuch.__name__, 'notmuch') - if PY35: - self.assertRegex(notmuch.__qualname__, - r'\w+.test_task_repr_coro_decorator' - r'\.\.notmuch') + self.assertRegex(notmuch.__qualname__, + r'\w+.test_task_repr_coro_decorator' + r'\.\.notmuch') self.assertEqual(notmuch.__module__, __name__) # test coroutine object gen = notmuch() - if coroutines._DEBUG or PY35: - # On Python >= 3.5, generators now inherit the name of the - # function, as expected, and have a qualified name (__qualname__ - # attribute). - coro_name = 'notmuch' - coro_qualname = ('BaseTaskTests.test_task_repr_coro_decorator' - '..notmuch') - else: - # On Python < 3.5, generators inherit the name of the code, not of - # the function. See: http://bugs.python.org/issue21205 - coro_name = coro_qualname = 'coro' + # On Python >= 3.5, generators now inherit the name of the + # function, as expected, and have a qualified name (__qualname__ + # attribute). + coro_name = 'notmuch' + coro_qualname = ('BaseTaskTests.test_task_repr_coro_decorator' + '..notmuch') self.assertEqual(gen.__name__, coro_name) - if PY35: - self.assertEqual(gen.__qualname__, coro_qualname) + self.assertEqual(gen.__qualname__, coro_qualname) # test repr(CoroWrapper) if coroutines._DEBUG: @@ -392,9 +378,8 @@ class BaseTaskTests: def test_task_repr_wait_for(self): self.loop.set_debug(False) - @asyncio.coroutine - def wait_for(fut): - return (yield from fut) + async def wait_for(fut): + return await fut fut = self.new_future(self.loop) task = self.new_task(self.loop, wait_for(fut)) @@ -411,9 +396,8 @@ class BaseTaskTests: with set_coroutine_debug(True): self.loop.set_debug(True) - @asyncio.coroutine - def func(x, y): - yield from asyncio.sleep(0) + async def func(x, y): + await asyncio.sleep(0) partial_func = asyncio.coroutine(functools.partial(func, 1)) task = self.loop.create_task(partial_func(2)) @@ -430,18 +414,16 @@ class BaseTaskTests: self.assertRegex(coro_repr, expected) def test_task_basics(self): - @asyncio.coroutine - def outer(): - a = yield from inner1() - b = yield from inner2() + + async def outer(): + a = await inner1() + b = await inner2() return a+b - @asyncio.coroutine - def inner1(): + async def inner1(): return 42 - @asyncio.coroutine - def inner2(): + async def inner2(): return 1000 t = outer() @@ -456,9 +438,8 @@ class BaseTaskTests: loop = self.new_test_loop(gen) - @asyncio.coroutine - def task(): - yield from asyncio.sleep(10.0, loop=loop) + async def task(): + await asyncio.sleep(10.0, loop=loop) return 12 t = self.new_task(loop, task()) @@ -488,9 +469,8 @@ class BaseTaskTests: def test_cancel_inner_future(self): f = self.new_future(self.loop) - @asyncio.coroutine - def task(): - yield from f + async def task(): + await f return 12 t = self.new_task(self.loop, task()) @@ -504,9 +484,8 @@ class BaseTaskTests: def test_cancel_both_task_and_inner_future(self): f = self.new_future(self.loop) - @asyncio.coroutine - def task(): - yield from f + async def task(): + await f return 12 t = self.new_task(self.loop, task()) @@ -526,11 +505,10 @@ class BaseTaskTests: fut1 = self.new_future(self.loop) fut2 = self.new_future(self.loop) - @asyncio.coroutine - def task(): - yield from fut1 + async def task(): + await fut1 try: - yield from fut2 + await fut2 except asyncio.CancelledError: return 42 @@ -551,14 +529,13 @@ class BaseTaskTests: fut2 = self.new_future(self.loop) fut3 = self.new_future(self.loop) - @asyncio.coroutine - def task(): - yield from fut1 + async def task(): + await fut1 try: - yield from fut2 + await fut2 except asyncio.CancelledError: pass - res = yield from fut3 + res = await fut3 return res t = self.new_task(self.loop, task()) @@ -581,12 +558,11 @@ class BaseTaskTests: loop = asyncio.new_event_loop() self.set_event_loop(loop) - @asyncio.coroutine - def task(): + async def task(): t.cancel() self.assertTrue(t._must_cancel) # White-box test. # The sleep should be cancelled immediately. - yield from asyncio.sleep(100, loop=loop) + await asyncio.sleep(100, loop=loop) return 12 t = self.new_task(loop, task()) @@ -628,14 +604,11 @@ class BaseTaskTests: loop = self.new_test_loop(gen) x = 0 - waiters = [] - @asyncio.coroutine - def task(): + async def task(): nonlocal x while x < 10: - waiters.append(asyncio.sleep(0.1, loop=loop)) - yield from waiters[-1] + await asyncio.sleep(0.1, loop=loop) x += 1 if x == 2: loop.stop() @@ -649,12 +622,18 @@ class BaseTaskTests: self.assertEqual(x, 2) self.assertAlmostEqual(0.3, loop.time()) - # close generators - for w in waiters: - w.close() t.cancel() self.assertRaises(asyncio.CancelledError, loop.run_until_complete, t) + def test_log_traceback(self): + async def coro(): + pass + + task = self.new_task(self.loop, coro()) + with self.assertRaisesRegex(ValueError, 'can only be set to False'): + task._log_traceback = True + self.loop.run_until_complete(task) + def test_wait_for_timeout_less_then_0_or_0_future_done(self): def gen(): when = yield @@ -704,12 +683,11 @@ class BaseTaskTests: foo_running = None - @asyncio.coroutine - def foo(): + async def foo(): nonlocal foo_running foo_running = True try: - yield from asyncio.sleep(0.2, loop=loop) + await asyncio.sleep(0.2, loop=loop) finally: foo_running = False return 'done' @@ -738,12 +716,11 @@ class BaseTaskTests: foo_running = None - @asyncio.coroutine - def foo(): + async def foo(): nonlocal foo_running foo_running = True try: - yield from asyncio.sleep(0.2, loop=loop) + await asyncio.sleep(0.2, loop=loop) finally: foo_running = False return 'done' @@ -781,9 +758,8 @@ class BaseTaskTests: loop = self.new_test_loop(gen) - @asyncio.coroutine - def foo(): - yield from asyncio.sleep(0.2, loop=loop) + async def foo(): + await asyncio.sleep(0.2, loop=loop) return 'done' asyncio.set_event_loop(loop) @@ -827,9 +803,8 @@ class BaseTaskTests: a = self.new_task(loop, asyncio.sleep(0.1, loop=loop)) b = self.new_task(loop, asyncio.sleep(0.15, loop=loop)) - @asyncio.coroutine - def foo(): - done, pending = yield from asyncio.wait([b, a], loop=loop) + async def foo(): + done, pending = await asyncio.wait([b, a], loop=loop) self.assertEqual(done, set([a, b])) self.assertEqual(pending, set()) return 42 @@ -857,9 +832,8 @@ class BaseTaskTests: a = self.new_task(loop, asyncio.sleep(0.01, loop=loop)) b = self.new_task(loop, asyncio.sleep(0.015, loop=loop)) - @asyncio.coroutine - def foo(): - done, pending = yield from asyncio.wait([b, a]) + async def foo(): + done, pending = await asyncio.wait([b, a]) self.assertEqual(done, set([a, b])) self.assertEqual(pending, set()) return 42 @@ -871,6 +845,7 @@ class BaseTaskTests: self.assertEqual(res, 42) def test_wait_duplicate_coroutines(self): + @asyncio.coroutine def coro(s): return s @@ -1000,9 +975,8 @@ class BaseTaskTests: # first_exception, exception during waiting a = self.new_task(loop, asyncio.sleep(10.0, loop=loop)) - @asyncio.coroutine - def exc(): - yield from asyncio.sleep(0.01, loop=loop) + async def exc(): + await asyncio.sleep(0.01, loop=loop) raise ZeroDivisionError('err') b = self.new_task(loop, exc()) @@ -1038,9 +1012,8 @@ class BaseTaskTests: b = self.new_task(loop, sleeper()) - @asyncio.coroutine - def foo(): - done, pending = yield from asyncio.wait([b, a], loop=loop) + async def foo(): + done, pending = await asyncio.wait([b, a], loop=loop) self.assertEqual(len(done), 2) self.assertEqual(pending, set()) errors = set(f for f in done if f.exception() is not None) @@ -1068,9 +1041,8 @@ class BaseTaskTests: a = self.new_task(loop, asyncio.sleep(0.1, loop=loop)) b = self.new_task(loop, asyncio.sleep(0.15, loop=loop)) - @asyncio.coroutine - def foo(): - done, pending = yield from asyncio.wait([b, a], timeout=0.11, + async def foo(): + done, pending = await asyncio.wait([b, a], timeout=0.11, loop=loop) self.assertEqual(done, set([a])) self.assertEqual(pending, set([b])) @@ -1164,17 +1136,16 @@ class BaseTaskTests: loop = self.new_test_loop(gen) - a = asyncio.sleep(0.1, 'a', loop=loop) - b = asyncio.sleep(0.15, 'b', loop=loop) + a = loop.create_task(asyncio.sleep(0.1, 'a', loop=loop)) + b = loop.create_task(asyncio.sleep(0.15, 'b', loop=loop)) - @asyncio.coroutine - def foo(): + async def foo(): values = [] for f in asyncio.as_completed([a, b], timeout=0.12, loop=loop): if values: loop.advance_time(0.02) try: - v = yield from f + v = await f values.append((1, v)) except asyncio.TimeoutError as exc: values.append((2, exc)) @@ -1202,10 +1173,9 @@ class BaseTaskTests: a = asyncio.sleep(0.01, 'a', loop=loop) - @asyncio.coroutine - def foo(): + async def foo(): for f in asyncio.as_completed([a], timeout=1, loop=loop): - v = yield from f + v = await f self.assertEqual(v, 'a') loop.run_until_complete(self.new_task(loop, foo())) @@ -1373,17 +1343,23 @@ class BaseTaskTests: self.assertIsNone(task._fut_waiter) self.assertTrue(fut.cancelled()) - def test_step_in_completed_task(self): + def test_task_set_methods(self): @asyncio.coroutine def notmuch(): return 'ko' gen = notmuch() task = self.new_task(self.loop, gen) - task.set_result('ok') - self.assertRaises(AssertionError, task._step) - gen.close() + with self.assertRaisesRegex(RuntimeError, 'not support set_result'): + task.set_result('ok') + + with self.assertRaisesRegex(RuntimeError, 'not support set_exception'): + task.set_exception(ValueError()) + + self.assertEqual( + self.loop.run_until_complete(task), + 'ko') def test_step_result(self): @asyncio.coroutine @@ -1403,9 +1379,9 @@ class BaseTaskTests: self.cb_added = False super().__init__(*args, **kwds) - def add_done_callback(self, fn): + def add_done_callback(self, *args, **kwargs): self.cb_added = True - super().add_done_callback(fn) + super().add_done_callback(*args, **kwargs) fut = Fut(loop=self.loop) result = None @@ -1426,17 +1402,6 @@ class BaseTaskTests: self.assertTrue(t.done()) self.assertIsNone(t.result()) - def test_step_with_baseexception(self): - @asyncio.coroutine - def notmutch(): - raise BaseException() - - task = self.new_task(self.loop, notmutch()) - self.assertRaises(BaseException, task._step) - - self.assertTrue(task.done()) - self.assertIsInstance(task.exception(), BaseException) - def test_baseexception_during_cancel(self): def gen(): @@ -1549,55 +1514,69 @@ class BaseTaskTests: self.assertEqual(res, 'test') self.assertIsNone(t2.result()) - def test_current_task(self): + + def test_current_task_deprecated(self): Task = self.__class__.Task - self.assertIsNone(Task.current_task(loop=self.loop)) + with self.assertWarns(PendingDeprecationWarning): + self.assertIsNone(Task.current_task(loop=self.loop)) - @asyncio.coroutine - def coro(loop): - self.assertTrue(Task.current_task(loop=loop) is task) + async def coro(loop): + with self.assertWarns(PendingDeprecationWarning): + self.assertIs(Task.current_task(loop=loop), task) # See http://bugs.python.org/issue29271 for details: asyncio.set_event_loop(loop) try: - self.assertIs(Task.current_task(None), task) - self.assertIs(Task.current_task(), task) + with self.assertWarns(PendingDeprecationWarning): + self.assertIs(Task.current_task(None), task) + with self.assertWarns(PendingDeprecationWarning): + self.assertIs(Task.current_task(), task) finally: asyncio.set_event_loop(None) task = self.new_task(self.loop, coro(self.loop)) self.loop.run_until_complete(task) - self.assertIsNone(Task.current_task(loop=self.loop)) + with self.assertWarns(PendingDeprecationWarning): + self.assertIsNone(Task.current_task(loop=self.loop)) + + def test_current_task(self): + self.assertIsNone(asyncio.current_task(loop=self.loop)) + + async def coro(loop): + self.assertIs(asyncio.current_task(loop=loop), task) + + self.assertIs(asyncio.current_task(None), task) + self.assertIs(asyncio.current_task(), task) + + task = self.new_task(self.loop, coro(self.loop)) + self.loop.run_until_complete(task) + self.assertIsNone(asyncio.current_task(loop=self.loop)) def test_current_task_with_interleaving_tasks(self): - Task = self.__class__.Task - - self.assertIsNone(Task.current_task(loop=self.loop)) + self.assertIsNone(asyncio.current_task(loop=self.loop)) fut1 = self.new_future(self.loop) fut2 = self.new_future(self.loop) - @asyncio.coroutine - def coro1(loop): - self.assertTrue(Task.current_task(loop=loop) is task1) - yield from fut1 - self.assertTrue(Task.current_task(loop=loop) is task1) + async def coro1(loop): + self.assertTrue(asyncio.current_task(loop=loop) is task1) + await fut1 + self.assertTrue(asyncio.current_task(loop=loop) is task1) fut2.set_result(True) - @asyncio.coroutine - def coro2(loop): - self.assertTrue(Task.current_task(loop=loop) is task2) + async def coro2(loop): + self.assertTrue(asyncio.current_task(loop=loop) is task2) fut1.set_result(True) - yield from fut2 - self.assertTrue(Task.current_task(loop=loop) is task2) + await fut2 + self.assertTrue(asyncio.current_task(loop=loop) is task2) task1 = self.new_task(self.loop, coro1(self.loop)) task2 = self.new_task(self.loop, coro2(self.loop)) self.loop.run_until_complete(asyncio.wait((task1, task2), loop=self.loop)) - self.assertIsNone(Task.current_task(loop=self.loop)) + self.assertIsNone(asyncio.current_task(loop=self.loop)) # Some thorough tests for cancellation propagation through # coroutines, tasks and wait(). @@ -1607,22 +1586,20 @@ class BaseTaskTests: proof = 0 waiter = self.new_future(self.loop) - @asyncio.coroutine - def inner(): + async def inner(): nonlocal proof try: - yield from waiter + await waiter except asyncio.CancelledError: proof += 1 raise else: self.fail('got past sleep() in inner()') - @asyncio.coroutine - def outer(): + async def outer(): nonlocal proof try: - yield from inner() + await inner() except asyncio.CancelledError: proof += 100 # Expect this path. else: @@ -1641,16 +1618,14 @@ class BaseTaskTests: proof = 0 waiter = self.new_future(self.loop) - @asyncio.coroutine - def inner(): + async def inner(): nonlocal proof - yield from waiter + await waiter proof += 1 - @asyncio.coroutine - def outer(): + async def outer(): nonlocal proof - d, p = yield from asyncio.wait([inner()], loop=self.loop) + d, p = await asyncio.wait([inner()], loop=self.loop) proof += 100 f = asyncio.ensure_future(outer(), loop=self.loop) @@ -1697,16 +1672,14 @@ class BaseTaskTests: proof = 0 waiter = self.new_future(self.loop) - @asyncio.coroutine - def inner(): + async def inner(): nonlocal proof - yield from waiter + await waiter proof += 1 - @asyncio.coroutine - def outer(): + async def outer(): nonlocal proof - yield from asyncio.shield(inner(), loop=self.loop) + await asyncio.shield(inner(), loop=self.loop) proof += 100 f = asyncio.ensure_future(outer(), loop=self.loop) @@ -1890,8 +1863,16 @@ class BaseTaskTests: self.assertIsInstance(exception, Exception) self.assertEqual(exception.args, ("foo", )) - @unittest.skipUnless(PY34, - 'need python 3.4 or later') + def test_all_tasks_deprecated(self): + Task = self.__class__.Task + + async def coro(): + with self.assertWarns(PendingDeprecationWarning): + assert Task.all_tasks(self.loop) == {t} + + t = self.new_task(self.loop, coro()) + self.loop.run_until_complete(t) + def test_log_destroyed_pending_task(self): Task = self.__class__.Task @@ -1911,13 +1892,13 @@ class BaseTaskTests: coro = kill_me(self.loop) task = asyncio.ensure_future(coro, loop=self.loop) - self.assertEqual(Task.all_tasks(loop=self.loop), {task}) + self.assertEqual(asyncio.all_tasks(loop=self.loop), {task}) # See http://bugs.python.org/issue29271 for details: asyncio.set_event_loop(self.loop) try: - self.assertEqual(Task.all_tasks(), {task}) - self.assertEqual(Task.all_tasks(None), {task}) + self.assertEqual(asyncio.all_tasks(), {task}) + self.assertEqual(asyncio.all_tasks(None), {task}) finally: asyncio.set_event_loop(None) @@ -1934,7 +1915,7 @@ class BaseTaskTests: # no more reference to kill_me() task: the task is destroyed by the GC support.gc_collect() - self.assertEqual(Task.all_tasks(loop=self.loop), set()) + self.assertEqual(asyncio.all_tasks(loop=self.loop), set()) mock_handler.assert_called_with(self.loop, { 'message': 'Task was destroyed but it is pending!', @@ -2101,7 +2082,7 @@ class BaseTaskTests: @mock.patch('asyncio.base_events.logger') def test_error_in_call_soon(self, m_log): - def call_soon(callback, *args): + def call_soon(callback, *args, **kwargs): raise ValueError self.loop.call_soon = call_soon @@ -2118,7 +2099,158 @@ class BaseTaskTests: message = m_log.error.call_args[0][0] self.assertIn('Task was destroyed but it is pending', message) - self.assertEqual(self.Task.all_tasks(self.loop), set()) + self.assertEqual(asyncio.all_tasks(self.loop), set()) + + def test_create_task_with_noncoroutine(self): + with self.assertRaisesRegex(TypeError, + "a coroutine was expected, got 123"): + self.new_task(self.loop, 123) + + # test it for the second time to ensure that caching + # in asyncio.iscoroutine() doesn't break things. + with self.assertRaisesRegex(TypeError, + "a coroutine was expected, got 123"): + self.new_task(self.loop, 123) + + def test_create_task_with_oldstyle_coroutine(self): + + @asyncio.coroutine + def coro(): + pass + + task = self.new_task(self.loop, coro()) + self.assertIsInstance(task, self.Task) + self.loop.run_until_complete(task) + + # test it for the second time to ensure that caching + # in asyncio.iscoroutine() doesn't break things. + task = self.new_task(self.loop, coro()) + self.assertIsInstance(task, self.Task) + self.loop.run_until_complete(task) + + def test_create_task_with_async_function(self): + + async def coro(): + pass + + task = self.new_task(self.loop, coro()) + self.assertIsInstance(task, self.Task) + self.loop.run_until_complete(task) + + # test it for the second time to ensure that caching + # in asyncio.iscoroutine() doesn't break things. + task = self.new_task(self.loop, coro()) + self.assertIsInstance(task, self.Task) + self.loop.run_until_complete(task) + + def test_create_task_with_asynclike_function(self): + task = self.new_task(self.loop, CoroLikeObject()) + self.assertIsInstance(task, self.Task) + self.assertEqual(self.loop.run_until_complete(task), 42) + + # test it for the second time to ensure that caching + # in asyncio.iscoroutine() doesn't break things. + task = self.new_task(self.loop, CoroLikeObject()) + self.assertIsInstance(task, self.Task) + self.assertEqual(self.loop.run_until_complete(task), 42) + + def test_bare_create_task(self): + + async def inner(): + return 1 + + async def coro(): + task = asyncio.create_task(inner()) + self.assertIsInstance(task, self.Task) + ret = await task + self.assertEqual(1, ret) + + self.loop.run_until_complete(coro()) + + def test_context_1(self): + cvar = contextvars.ContextVar('cvar', default='nope') + + async def sub(): + await asyncio.sleep(0.01, loop=loop) + self.assertEqual(cvar.get(), 'nope') + cvar.set('something else') + + async def main(): + self.assertEqual(cvar.get(), 'nope') + subtask = self.new_task(loop, sub()) + cvar.set('yes') + self.assertEqual(cvar.get(), 'yes') + await subtask + self.assertEqual(cvar.get(), 'yes') + + loop = asyncio.new_event_loop() + try: + task = self.new_task(loop, main()) + loop.run_until_complete(task) + finally: + loop.close() + + def test_context_2(self): + cvar = contextvars.ContextVar('cvar', default='nope') + + async def main(): + def fut_on_done(fut): + # This change must not pollute the context + # of the "main()" task. + cvar.set('something else') + + self.assertEqual(cvar.get(), 'nope') + + for j in range(2): + fut = self.new_future(loop) + fut.add_done_callback(fut_on_done) + cvar.set(f'yes{j}') + loop.call_soon(fut.set_result, None) + await fut + self.assertEqual(cvar.get(), f'yes{j}') + + for i in range(3): + # Test that task passed its context to add_done_callback: + cvar.set(f'yes{i}-{j}') + await asyncio.sleep(0.001, loop=loop) + self.assertEqual(cvar.get(), f'yes{i}-{j}') + + loop = asyncio.new_event_loop() + try: + task = self.new_task(loop, main()) + loop.run_until_complete(task) + finally: + loop.close() + + self.assertEqual(cvar.get(), 'nope') + + def test_context_3(self): + # Run 100 Tasks in parallel, each modifying cvar. + + cvar = contextvars.ContextVar('cvar', default=-1) + + async def sub(num): + for i in range(10): + cvar.set(num + i) + await asyncio.sleep( + random.uniform(0.001, 0.05), loop=loop) + self.assertEqual(cvar.get(), num + i) + + async def main(): + tasks = [] + for i in range(100): + task = loop.create_task(sub(random.randint(0, 10))) + tasks.append(task) + + await asyncio.gather(*tasks, loop=loop) + + loop = asyncio.new_event_loop() + try: + loop.run_until_complete(main()) + finally: + loop.close() + + self.assertEqual(cvar.get(), -1) def add_subclass_tests(cls): @@ -2133,22 +2265,12 @@ def add_subclass_tests(cls): self.calls = collections.defaultdict(lambda: 0) super().__init__(*args, **kwargs) - def _schedule_callbacks(self): - self.calls['_schedule_callbacks'] += 1 - return super()._schedule_callbacks() - - def add_done_callback(self, *args): + def add_done_callback(self, *args, **kwargs): self.calls['add_done_callback'] += 1 - return super().add_done_callback(*args) + return super().add_done_callback(*args, **kwargs) class Task(CommonFuture, BaseTask): - def _step(self, *args): - self.calls['_step'] += 1 - return super()._step(*args) - - def _wakeup(self, *args): - self.calls['_wakeup'] += 1 - return super()._wakeup(*args) + pass class Future(CommonFuture, BaseFuture): pass @@ -2168,12 +2290,11 @@ def add_subclass_tests(cls): self.assertEqual( dict(task.calls), - {'_step': 2, '_wakeup': 1, 'add_done_callback': 1, - '_schedule_callbacks': 1}) + {'add_done_callback': 1}) self.assertEqual( dict(fut.calls), - {'add_done_callback': 1, '_schedule_callbacks': 1}) + {'add_done_callback': 1}) # Add patched Task & Future back to the test case cls.Task = Task @@ -2190,24 +2311,101 @@ def add_subclass_tests(cls): return cls -@unittest.skipUnless(hasattr(futures, '_CFuture'), +class SetMethodsTest: + + def test_set_result_causes_invalid_state(self): + Future = type(self).Future + self.loop.call_exception_handler = exc_handler = mock.Mock() + + async def foo(): + await asyncio.sleep(0.1, loop=self.loop) + return 10 + + coro = foo() + task = self.new_task(self.loop, coro) + Future.set_result(task, 'spam') + + self.assertEqual( + self.loop.run_until_complete(task), + 'spam') + + exc_handler.assert_called_once() + exc = exc_handler.call_args[0][0]['exception'] + with self.assertRaisesRegex(asyncio.InvalidStateError, + r'step\(\): already done'): + raise exc + + coro.close() + + def test_set_exception_causes_invalid_state(self): + class MyExc(Exception): + pass + + Future = type(self).Future + self.loop.call_exception_handler = exc_handler = mock.Mock() + + async def foo(): + await asyncio.sleep(0.1, loop=self.loop) + return 10 + + coro = foo() + task = self.new_task(self.loop, coro) + Future.set_exception(task, MyExc()) + + with self.assertRaises(MyExc): + self.loop.run_until_complete(task) + + exc_handler.assert_called_once() + exc = exc_handler.call_args[0][0]['exception'] + with self.assertRaisesRegex(asyncio.InvalidStateError, + r'step\(\): already done'): + raise exc + + coro.close() + + +@unittest.skipUnless(hasattr(futures, '_CFuture') and + hasattr(tasks, '_CTask'), 'requires the C _asyncio module') -class CTask_CFuture_Tests(BaseTaskTests, test_utils.TestCase): +class CTask_CFuture_Tests(BaseTaskTests, SetMethodsTest, + test_utils.TestCase): + Task = getattr(tasks, '_CTask', None) Future = getattr(futures, '_CFuture', None) +@unittest.skipUnless(hasattr(futures, '_CFuture') and + hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +@add_subclass_tests +class CTask_CFuture_SubclassTests(BaseTaskTests, test_utils.TestCase): + + Task = getattr(tasks, '_CTask', None) + Future = getattr(futures, '_CFuture', None) + + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +@add_subclass_tests +class CTaskSubclass_PyFuture_Tests(BaseTaskTests, test_utils.TestCase): + + Task = getattr(tasks, '_CTask', None) + Future = futures._PyFuture + + @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') @add_subclass_tests -class CTask_CFuture_SubclassTests(BaseTaskTests, test_utils.TestCase): - Task = getattr(tasks, '_CTask', None) +class PyTask_CFutureSubclass_Tests(BaseTaskTests, test_utils.TestCase): + Future = getattr(futures, '_CFuture', None) + Task = tasks._PyTask -@unittest.skipUnless(hasattr(futures, '_CFuture'), +@unittest.skipUnless(hasattr(tasks, '_CTask'), 'requires the C _asyncio module') class CTask_PyFuture_Tests(BaseTaskTests, test_utils.TestCase): + Task = getattr(tasks, '_CTask', None) Future = futures._PyFuture @@ -2215,11 +2413,14 @@ class CTask_PyFuture_Tests(BaseTaskTests, test_utils.TestCase): @unittest.skipUnless(hasattr(futures, '_CFuture'), 'requires the C _asyncio module') class PyTask_CFuture_Tests(BaseTaskTests, test_utils.TestCase): + Task = tasks._PyTask Future = getattr(futures, '_CFuture', None) -class PyTask_PyFuture_Tests(BaseTaskTests, test_utils.TestCase): +class PyTask_PyFuture_Tests(BaseTaskTests, SetMethodsTest, + test_utils.TestCase): + Task = tasks._PyTask Future = futures._PyFuture @@ -2230,6 +2431,189 @@ class PyTask_PyFuture_SubclassTests(BaseTaskTests, test_utils.TestCase): Future = futures._PyFuture +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class CTask_Future_Tests(test_utils.TestCase): + + def test_foobar(self): + class Fut(asyncio.Future): + @property + def get_loop(self): + raise AttributeError + + async def coro(): + await fut + return 'spam' + + self.loop = asyncio.new_event_loop() + try: + fut = Fut(loop=self.loop) + self.loop.call_later(0.1, fut.set_result, 1) + task = asyncio.Task(coro(), loop=self.loop) + res = self.loop.run_until_complete(task) + finally: + self.loop.close() + + self.assertEqual(res, 'spam') + + +class BaseTaskIntrospectionTests: + _register_task = None + _unregister_task = None + _enter_task = None + _leave_task = None + + def test__register_task_1(self): + class TaskLike: + @property + def _loop(self): + return loop + + task = TaskLike() + loop = mock.Mock() + + self.assertEqual(asyncio.all_tasks(loop), set()) + self._register_task(task) + self.assertEqual(asyncio.all_tasks(loop), {task}) + self._unregister_task(task) + + def test__register_task_2(self): + class TaskLike: + def get_loop(self): + return loop + + task = TaskLike() + loop = mock.Mock() + + self.assertEqual(asyncio.all_tasks(loop), set()) + self._register_task(task) + self.assertEqual(asyncio.all_tasks(loop), {task}) + self._unregister_task(task) + + def test__enter_task(self): + task = mock.Mock() + loop = mock.Mock() + self.assertIsNone(asyncio.current_task(loop)) + self._enter_task(loop, task) + self.assertIs(asyncio.current_task(loop), task) + self._leave_task(loop, task) + + def test__enter_task_failure(self): + task1 = mock.Mock() + task2 = mock.Mock() + loop = mock.Mock() + self._enter_task(loop, task1) + with self.assertRaises(RuntimeError): + self._enter_task(loop, task2) + self.assertIs(asyncio.current_task(loop), task1) + self._leave_task(loop, task1) + + def test__leave_task(self): + task = mock.Mock() + loop = mock.Mock() + self._enter_task(loop, task) + self._leave_task(loop, task) + self.assertIsNone(asyncio.current_task(loop)) + + def test__leave_task_failure1(self): + task1 = mock.Mock() + task2 = mock.Mock() + loop = mock.Mock() + self._enter_task(loop, task1) + with self.assertRaises(RuntimeError): + self._leave_task(loop, task2) + self.assertIs(asyncio.current_task(loop), task1) + self._leave_task(loop, task1) + + def test__leave_task_failure2(self): + task = mock.Mock() + loop = mock.Mock() + with self.assertRaises(RuntimeError): + self._leave_task(loop, task) + self.assertIsNone(asyncio.current_task(loop)) + + def test__unregister_task(self): + task = mock.Mock() + loop = mock.Mock() + task.get_loop = lambda: loop + self._register_task(task) + self._unregister_task(task) + self.assertEqual(asyncio.all_tasks(loop), set()) + + def test__unregister_task_not_registered(self): + task = mock.Mock() + loop = mock.Mock() + self._unregister_task(task) + self.assertEqual(asyncio.all_tasks(loop), set()) + + +class PyIntrospectionTests(unittest.TestCase, BaseTaskIntrospectionTests): + _register_task = staticmethod(tasks._py_register_task) + _unregister_task = staticmethod(tasks._py_unregister_task) + _enter_task = staticmethod(tasks._py_enter_task) + _leave_task = staticmethod(tasks._py_leave_task) + + +@unittest.skipUnless(hasattr(tasks, '_c_register_task'), + 'requires the C _asyncio module') +class CIntrospectionTests(unittest.TestCase, BaseTaskIntrospectionTests): + if hasattr(tasks, '_c_register_task'): + _register_task = staticmethod(tasks._c_register_task) + _unregister_task = staticmethod(tasks._c_unregister_task) + _enter_task = staticmethod(tasks._c_enter_task) + _leave_task = staticmethod(tasks._c_leave_task) + else: + _register_task = _unregister_task = _enter_task = _leave_task = None + + +class BaseCurrentLoopTests: + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + + def tearDown(self): + self.loop.close() + asyncio.set_event_loop(None) + super().tearDown() + + def new_task(self, coro): + raise NotImplementedError + + def test_current_task_no_running_loop(self): + self.assertIsNone(asyncio.current_task(loop=self.loop)) + + def test_current_task_no_running_loop_implicit(self): + with self.assertRaises(RuntimeError): + asyncio.current_task() + + def test_current_task_with_implicit_loop(self): + async def coro(): + self.assertIs(asyncio.current_task(loop=self.loop), task) + + self.assertIs(asyncio.current_task(None), task) + self.assertIs(asyncio.current_task(), task) + + task = self.new_task(coro()) + self.loop.run_until_complete(task) + self.assertIsNone(asyncio.current_task(loop=self.loop)) + + +class PyCurrentLoopTests(BaseCurrentLoopTests, unittest.TestCase): + + def new_task(self, coro): + return tasks._PyTask(coro, loop=self.loop) + + +@unittest.skipUnless(hasattr(tasks, '_CTask'), + 'requires the C _asyncio module') +class CCurrentLoopTests(BaseCurrentLoopTests, unittest.TestCase): + + def new_task(self, coro): + return getattr(tasks, '_CTask')(coro, loop=self.loop) + + class GenericTaskTests(test_utils.TestCase): def test_future_subclass(self): @@ -2323,33 +2707,31 @@ class GatherTestsBase: self.assertEqual(fut.result(), [3, 1, exc, exc2]) def test_env_var_debug(self): - aio_path = os.path.dirname(os.path.dirname(asyncio.__file__)) - code = '\n'.join(( 'import asyncio.coroutines', 'print(asyncio.coroutines._DEBUG)')) # Test with -E to not fail if the unit test was run with # PYTHONASYNCIODEBUG set to a non-empty string - sts, stdout, stderr = assert_python_ok('-E', '-c', code, - PYTHONPATH=aio_path) + sts, stdout, stderr = assert_python_ok('-E', '-c', code) self.assertEqual(stdout.rstrip(), b'False') sts, stdout, stderr = assert_python_ok('-c', code, PYTHONASYNCIODEBUG='', - PYTHONPATH=aio_path) + PYTHONDEVMODE='') self.assertEqual(stdout.rstrip(), b'False') sts, stdout, stderr = assert_python_ok('-c', code, PYTHONASYNCIODEBUG='1', - PYTHONPATH=aio_path) + PYTHONDEVMODE='') self.assertEqual(stdout.rstrip(), b'True') sts, stdout, stderr = assert_python_ok('-E', '-c', code, PYTHONASYNCIODEBUG='1', - PYTHONPATH=aio_path) + PYTHONDEVMODE='') self.assertEqual(stdout.rstrip(), b'False') + # -X dev sts, stdout, stderr = assert_python_ok('-E', '-X', 'dev', '-c', code) self.assertEqual(stdout.rstrip(), b'True') @@ -2553,7 +2935,7 @@ class RunCoroutineThreadsafeTests(test_utils.TestCase): if fail: raise RuntimeError("Fail!") if cancel: - asyncio.tasks.Task.current_task(self.loop).cancel() + asyncio.current_task(self.loop).cancel() yield return a + b @@ -2599,7 +2981,7 @@ class RunCoroutineThreadsafeTests(test_utils.TestCase): self.loop.run_until_complete(future) test_utils.run_briefly(self.loop) # Check that there's no pending task (add has been cancelled) - for task in asyncio.Task.all_tasks(self.loop): + for task in asyncio.all_tasks(self.loop): self.assertTrue(task.done()) def test_run_coroutine_threadsafe_task_cancelled(self): @@ -2613,19 +2995,25 @@ class RunCoroutineThreadsafeTests(test_utils.TestCase): def test_run_coroutine_threadsafe_task_factory_exception(self): """Test coroutine submission from a tread to an event loop when the task factory raise an exception.""" - # Schedule the target - future = self.loop.run_in_executor( + + def task_factory(loop, coro): + raise NameError + + run = self.loop.run_in_executor( None, lambda: self.target(advance_coro=True)) - # Set corrupted task factory - self.loop.set_task_factory(lambda loop, coro: wrong_name) + # Set exception handler callback = test_utils.MockCallback() self.loop.set_exception_handler(callback) + + # Set corrupted task factory + self.loop.set_task_factory(task_factory) + # Run event loop with self.assertRaises(NameError) as exc_context: - self.loop.run_until_complete(future) + self.loop.run_until_complete(run) + # Check exceptions - self.assertIn('wrong_name', exc_context.exception.args[0]) self.assertEqual(len(callback.call_args_list), 1) (loop, context), kwargs = callback.call_args self.assertEqual(context['exception'], exc_context.exception) @@ -2661,5 +3049,63 @@ class SleepTests(test_utils.TestCase): self.assertEqual(result, 11) +class CompatibilityTests(test_utils.TestCase): + # Tests for checking a bridge between old-styled coroutines + # and async/await syntax + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(None) + + def tearDown(self): + self.loop.close() + self.loop = None + super().tearDown() + + def test_yield_from_awaitable(self): + + @asyncio.coroutine + def coro(): + yield from asyncio.sleep(0, loop=self.loop) + return 'ok' + + result = self.loop.run_until_complete(coro()) + self.assertEqual('ok', result) + + def test_await_old_style_coro(self): + + @asyncio.coroutine + def coro1(): + return 'ok1' + + @asyncio.coroutine + def coro2(): + yield from asyncio.sleep(0, loop=self.loop) + return 'ok2' + + async def inner(): + return await asyncio.gather(coro1(), coro2(), loop=self.loop) + + result = self.loop.run_until_complete(inner()) + self.assertEqual(['ok1', 'ok2'], result) + + def test_debug_mode_interop(self): + # https://bugs.python.org/issue32636 + code = textwrap.dedent(""" + import asyncio + + async def native_coro(): + pass + + @asyncio.coroutine + def old_style_coro(): + yield from native_coro() + + asyncio.run(old_style_coro()) + """) + assert_python_ok("-c", code, PYTHONASYNCIODEBUG="1") + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_transports.py b/Lib/test/test_asyncio/test_transports.py index 3b6e3d67075..7de9b3e81a3 100644 --- a/Lib/test/test_asyncio/test_transports.py +++ b/Lib/test/test_asyncio/test_transports.py @@ -42,6 +42,7 @@ class TransportTests(unittest.TestCase): self.assertRaises(NotImplementedError, transport.can_write_eof) self.assertRaises(NotImplementedError, transport.pause_reading) self.assertRaises(NotImplementedError, transport.resume_reading) + self.assertRaises(NotImplementedError, transport.is_reading) self.assertRaises(NotImplementedError, transport.close) self.assertRaises(NotImplementedError, transport.abort) diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index d558842a1f8..5bd76d30d2d 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1,6 +1,7 @@ """Tests for unix_events.py.""" import collections +import contextlib import errno import io import os @@ -13,6 +14,7 @@ import tempfile import threading import unittest from unittest import mock +from test import support if sys.platform == 'win32': raise unittest.SkipTest('UNIX only') @@ -20,8 +22,10 @@ if sys.platform == 'win32': import asyncio from asyncio import log -from asyncio import test_utils +from asyncio import base_events +from asyncio import events from asyncio import unix_events +from test.test_asyncio import utils as test_utils MOCK_ANY = mock.ANY @@ -76,9 +80,8 @@ class SelectorEventLoopSignalTests(test_utils.TestCase): def test_add_signal_handler_coroutine_error(self, m_signal): m_signal.NSIG = signal.NSIG - @asyncio.coroutine - def simple_coroutine(): - yield from [] + async def simple_coroutine(): + pass # callback must not be a coroutine function coro_func = simple_coroutine @@ -229,6 +232,23 @@ class SelectorEventLoopSignalTests(test_utils.TestCase): self.assertEqual(len(self.loop._signal_handlers), 0) m_signal.set_wakeup_fd.assert_called_once_with(-1) + @mock.patch('asyncio.unix_events.sys') + @mock.patch('asyncio.unix_events.signal') + def test_close_on_finalizing(self, m_signal, m_sys): + m_signal.NSIG = signal.NSIG + self.loop.add_signal_handler(signal.SIGHUP, lambda: True) + + self.assertEqual(len(self.loop._signal_handlers), 1) + m_sys.is_finalizing.return_value = True + m_signal.signal.reset_mock() + + with self.assertWarnsRegex(ResourceWarning, + "skipping signal handlers removal"): + self.loop.close() + + self.assertEqual(len(self.loop._signal_handlers), 0) + self.assertFalse(m_signal.signal.called) + @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'UNIX Sockets are not supported') @@ -239,6 +259,7 @@ class SelectorEventLoopUnixSocketTests(test_utils.TestCase): self.loop = asyncio.SelectorEventLoop() self.set_event_loop(self.loop) + @support.skip_unless_bind_unix_socket def test_create_unix_server_existing_path_sock(self): with test_utils.unix_socket_path() as path: sock = socket.socket(socket.AF_UNIX) @@ -251,7 +272,7 @@ class SelectorEventLoopUnixSocketTests(test_utils.TestCase): srv.close() self.loop.run_until_complete(srv.wait_closed()) - @unittest.skipUnless(hasattr(os, 'fspath'), 'no os.fspath') + @support.skip_unless_bind_unix_socket def test_create_unix_server_pathlib(self): with test_utils.unix_socket_path() as path: path = pathlib.Path(path) @@ -260,6 +281,15 @@ class SelectorEventLoopUnixSocketTests(test_utils.TestCase): srv.close() self.loop.run_until_complete(srv.wait_closed()) + def test_create_unix_connection_pathlib(self): + with test_utils.unix_socket_path() as path: + path = pathlib.Path(path) + coro = self.loop.create_unix_connection(lambda: None, path) + with self.assertRaises(FileNotFoundError): + # If pathlib.Path wasn't supported, the exception would be + # different. + self.loop.run_until_complete(coro) + def test_create_unix_server_existing_path_nonsock(self): with tempfile.NamedTemporaryFile() as file: coro = self.loop.create_unix_server(lambda: None, file.name) @@ -300,6 +330,7 @@ class SelectorEventLoopUnixSocketTests(test_utils.TestCase): @unittest.skipUnless(hasattr(socket, 'SOCK_NONBLOCK'), 'no socket.SOCK_NONBLOCK (linux only)') + @support.skip_unless_bind_unix_socket def test_create_unix_server_path_stream_bittype(self): sock = socket.socket( socket.AF_UNIX, socket.SOCK_STREAM | socket.SOCK_NONBLOCK) @@ -316,10 +347,18 @@ class SelectorEventLoopUnixSocketTests(test_utils.TestCase): finally: os.unlink(fn) + def test_create_unix_server_ssl_timeout_with_plain_sock(self): + coro = self.loop.create_unix_server(lambda: None, path='spam', + ssl_handshake_timeout=1) + with self.assertRaisesRegex( + ValueError, + 'ssl_handshake_timeout is only meaningful with ssl'): + self.loop.run_until_complete(coro) + def test_create_unix_connection_path_inetsock(self): sock = socket.socket() with sock: - coro = self.loop.create_unix_connection(lambda: None, path=None, + coro = self.loop.create_unix_connection(lambda: None, sock=sock) with self.assertRaisesRegex(ValueError, 'A UNIX Domain Stream.*was expected'): @@ -372,6 +411,267 @@ class SelectorEventLoopUnixSocketTests(test_utils.TestCase): self.loop.run_until_complete(coro) + def test_create_unix_connection_ssl_timeout_with_plain_sock(self): + coro = self.loop.create_unix_connection(lambda: None, path='spam', + ssl_handshake_timeout=1) + with self.assertRaisesRegex( + ValueError, + 'ssl_handshake_timeout is only meaningful with ssl'): + self.loop.run_until_complete(coro) + + +@unittest.skipUnless(hasattr(os, 'sendfile'), + 'sendfile is not supported') +class SelectorEventLoopUnixSockSendfileTests(test_utils.TestCase): + DATA = b"12345abcde" * 16 * 1024 # 160 KiB + + class MyProto(asyncio.Protocol): + + def __init__(self, loop): + self.started = False + self.closed = False + self.data = bytearray() + self.fut = loop.create_future() + self.transport = None + + def connection_made(self, transport): + self.started = True + self.transport = transport + + def data_received(self, data): + self.data.extend(data) + + def connection_lost(self, exc): + self.closed = True + self.fut.set_result(None) + + async def wait_closed(self): + await self.fut + + @classmethod + def setUpClass(cls): + with open(support.TESTFN, 'wb') as fp: + fp.write(cls.DATA) + super().setUpClass() + + @classmethod + def tearDownClass(cls): + support.unlink(support.TESTFN) + super().tearDownClass() + + def setUp(self): + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + self.file = open(support.TESTFN, 'rb') + self.addCleanup(self.file.close) + super().setUp() + + def make_socket(self, blocking=False): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setblocking(blocking) + self.addCleanup(sock.close) + return sock + + def run_loop(self, coro): + return self.loop.run_until_complete(coro) + + def prepare(self): + sock = self.make_socket() + proto = self.MyProto(self.loop) + port = support.find_unused_port() + server = self.run_loop(self.loop.create_server( + lambda: proto, support.HOST, port)) + self.run_loop(self.loop.sock_connect(sock, (support.HOST, port))) + + def cleanup(): + if proto.transport is not None: + # can be None if the task was cancelled before + # connection_made callback + proto.transport.close() + self.run_loop(proto.wait_closed()) + + server.close() + self.run_loop(server.wait_closed()) + + self.addCleanup(cleanup) + + return sock, proto + + def test_sock_sendfile_success(self): + sock, proto = self.prepare() + ret = self.run_loop(self.loop.sock_sendfile(sock, self.file)) + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(ret, len(self.DATA)) + self.assertEqual(proto.data, self.DATA) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sock_sendfile_with_offset_and_count(self): + sock, proto = self.prepare() + ret = self.run_loop(self.loop.sock_sendfile(sock, self.file, + 1000, 2000)) + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(proto.data, self.DATA[1000:3000]) + self.assertEqual(self.file.tell(), 3000) + self.assertEqual(ret, 2000) + + def test_sock_sendfile_not_available(self): + sock, proto = self.prepare() + with mock.patch('asyncio.unix_events.os', spec=[]): + with self.assertRaisesRegex(events.SendfileNotAvailableError, + "os[.]sendfile[(][)] is not available"): + self.run_loop(self.loop._sock_sendfile_native(sock, self.file, + 0, None)) + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_not_a_file(self): + sock, proto = self.prepare() + f = object() + with self.assertRaisesRegex(events.SendfileNotAvailableError, + "not a regular file"): + self.run_loop(self.loop._sock_sendfile_native(sock, f, + 0, None)) + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_iobuffer(self): + sock, proto = self.prepare() + f = io.BytesIO() + with self.assertRaisesRegex(events.SendfileNotAvailableError, + "not a regular file"): + self.run_loop(self.loop._sock_sendfile_native(sock, f, + 0, None)) + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_not_regular_file(self): + sock, proto = self.prepare() + f = mock.Mock() + f.fileno.return_value = -1 + with self.assertRaisesRegex(events.SendfileNotAvailableError, + "not a regular file"): + self.run_loop(self.loop._sock_sendfile_native(sock, f, + 0, None)) + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_zero_size(self): + sock, proto = self.prepare() + fname = support.TESTFN + '.suffix' + with open(fname, 'wb') as f: + pass # make zero sized file + f = open(fname, 'rb') + self.addCleanup(f.close) + self.addCleanup(support.unlink, fname) + ret = self.run_loop(self.loop._sock_sendfile_native(sock, f, + 0, None)) + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(ret, 0) + self.assertEqual(self.file.tell(), 0) + + def test_sock_sendfile_mix_with_regular_send(self): + buf = b'1234567890' * 1024 * 1024 # 10 MB + sock, proto = self.prepare() + self.run_loop(self.loop.sock_sendall(sock, buf)) + ret = self.run_loop(self.loop.sock_sendfile(sock, self.file)) + self.run_loop(self.loop.sock_sendall(sock, buf)) + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(ret, len(self.DATA)) + expected = buf + self.DATA + buf + self.assertEqual(proto.data, expected) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sock_sendfile_cancel1(self): + sock, proto = self.prepare() + + fut = self.loop.create_future() + fileno = self.file.fileno() + self.loop._sock_sendfile_native_impl(fut, None, sock, fileno, + 0, None, len(self.DATA), 0) + fut.cancel() + with contextlib.suppress(asyncio.CancelledError): + self.run_loop(fut) + with self.assertRaises(KeyError): + self.loop._selector.get_key(sock) + + def test_sock_sendfile_cancel2(self): + sock, proto = self.prepare() + + fut = self.loop.create_future() + fileno = self.file.fileno() + self.loop._sock_sendfile_native_impl(fut, None, sock, fileno, + 0, None, len(self.DATA), 0) + fut.cancel() + self.loop._sock_sendfile_native_impl(fut, sock.fileno(), sock, fileno, + 0, None, len(self.DATA), 0) + with self.assertRaises(KeyError): + self.loop._selector.get_key(sock) + + def test_sock_sendfile_blocking_error(self): + sock, proto = self.prepare() + + fileno = self.file.fileno() + fut = mock.Mock() + fut.cancelled.return_value = False + with mock.patch('os.sendfile', side_effect=BlockingIOError()): + self.loop._sock_sendfile_native_impl(fut, None, sock, fileno, + 0, None, len(self.DATA), 0) + key = self.loop._selector.get_key(sock) + self.assertIsNotNone(key) + fut.add_done_callback.assert_called_once_with(mock.ANY) + + def test_sock_sendfile_os_error_first_call(self): + sock, proto = self.prepare() + + fileno = self.file.fileno() + fut = self.loop.create_future() + with mock.patch('os.sendfile', side_effect=OSError()): + self.loop._sock_sendfile_native_impl(fut, None, sock, fileno, + 0, None, len(self.DATA), 0) + with self.assertRaises(KeyError): + self.loop._selector.get_key(sock) + exc = fut.exception() + self.assertIsInstance(exc, events.SendfileNotAvailableError) + self.assertEqual(0, self.file.tell()) + + def test_sock_sendfile_os_error_next_call(self): + sock, proto = self.prepare() + + fileno = self.file.fileno() + fut = self.loop.create_future() + err = OSError() + with mock.patch('os.sendfile', side_effect=err): + self.loop._sock_sendfile_native_impl(fut, sock.fileno(), + sock, fileno, + 1000, None, len(self.DATA), + 1000) + with self.assertRaises(KeyError): + self.loop._selector.get_key(sock) + exc = fut.exception() + self.assertIs(exc, err) + self.assertEqual(1000, self.file.tell()) + + def test_sock_sendfile_exception(self): + sock, proto = self.prepare() + + fileno = self.file.fileno() + fut = self.loop.create_future() + err = events.SendfileNotAvailableError() + with mock.patch('os.sendfile', side_effect=err): + self.loop._sock_sendfile_native_impl(fut, sock.fileno(), + sock, fileno, + 1000, None, len(self.DATA), + 1000) + with self.assertRaises(KeyError): + self.loop._selector.get_key(sock) + exc = fut.exception() + self.assertIs(exc, err) + self.assertEqual(1000, self.file.tell()) + class UnixReadPipeTransportTests(test_utils.TestCase): @@ -382,7 +682,7 @@ class UnixReadPipeTransportTests(test_utils.TestCase): self.pipe = mock.Mock(spec_set=io.RawIOBase) self.pipe.fileno.return_value = 5 - blocking_patcher = mock.patch('asyncio.unix_events._set_nonblocking') + blocking_patcher = mock.patch('os.set_blocking') blocking_patcher.start() self.addCleanup(blocking_patcher.stop) @@ -532,7 +832,7 @@ class UnixWritePipeTransportTests(test_utils.TestCase): self.pipe = mock.Mock(spec_set=io.RawIOBase) self.pipe.fileno.return_value = 5 - blocking_patcher = mock.patch('asyncio.unix_events._set_nonblocking') + blocking_patcher = mock.patch('os.set_blocking') blocking_patcher.start() self.addCleanup(blocking_patcher.stop) diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py index c72eef1afdb..e4ff7fc7dd4 100644 --- a/Lib/test/test_asyncio/test_windows_events.py +++ b/Lib/test/test_asyncio/test_windows_events.py @@ -1,4 +1,5 @@ import os +import socket import sys import unittest from unittest import mock @@ -6,12 +7,12 @@ from unittest import mock if sys.platform != 'win32': raise unittest.SkipTest('Windows only') +import _overlapped import _winapi import asyncio -from asyncio import _overlapped -from asyncio import test_utils from asyncio import windows_events +from test.test_asyncio import utils as test_utils class UpperProto(asyncio.Protocol): @@ -36,9 +37,9 @@ class ProactorTests(test_utils.TestCase): self.set_event_loop(self.loop) def test_close(self): - a, b = self.loop._socketpair() + a, b = socket.socketpair() trans = self.loop._make_socket_transport(a, asyncio.Protocol()) - f = asyncio.ensure_future(self.loop.sock_recv(b, 100)) + f = asyncio.ensure_future(self.loop.sock_recv(b, 100), loop=self.loop) trans.close() self.loop.run_until_complete(f) self.assertEqual(f.result(), b'') @@ -55,14 +56,14 @@ class ProactorTests(test_utils.TestCase): res = self.loop.run_until_complete(self._test_pipe()) self.assertEqual(res, 'done') - def _test_pipe(self): + async def _test_pipe(self): ADDRESS = r'\\.\pipe\_test_pipe-%s' % os.getpid() with self.assertRaises(FileNotFoundError): - yield from self.loop.create_pipe_connection( + await self.loop.create_pipe_connection( asyncio.Protocol, ADDRESS) - [server] = yield from self.loop.start_serving_pipe( + [server] = await self.loop.start_serving_pipe( UpperProto, ADDRESS) self.assertIsInstance(server, windows_events.PipeServer) @@ -71,7 +72,7 @@ class ProactorTests(test_utils.TestCase): stream_reader = asyncio.StreamReader(loop=self.loop) protocol = asyncio.StreamReaderProtocol(stream_reader, loop=self.loop) - trans, proto = yield from self.loop.create_pipe_connection( + trans, proto = await self.loop.create_pipe_connection( lambda: protocol, ADDRESS) self.assertIsInstance(trans, asyncio.Transport) self.assertEqual(protocol, proto) @@ -81,14 +82,14 @@ class ProactorTests(test_utils.TestCase): w.write('lower-{}\n'.format(i).encode()) for i, (r, w) in enumerate(clients): - response = yield from r.readline() + response = await r.readline() self.assertEqual(response, 'LOWER-{}\n'.format(i).encode()) w.close() server.close() with self.assertRaises(FileNotFoundError): - yield from self.loop.create_pipe_connection( + await self.loop.create_pipe_connection( asyncio.Protocol, ADDRESS) return 'done' @@ -96,7 +97,8 @@ class ProactorTests(test_utils.TestCase): def test_connect_pipe_cancel(self): exc = OSError() exc.winerror = _overlapped.ERROR_PIPE_BUSY - with mock.patch.object(_overlapped, 'ConnectPipe', side_effect=exc) as connect: + with mock.patch.object(_overlapped, 'ConnectPipe', + side_effect=exc) as connect: coro = self.loop._proactor.connect_pipe('pipe_address') task = self.loop.create_task(coro) diff --git a/Lib/test/test_asyncio/test_windows_utils.py b/Lib/test/test_asyncio/test_windows_utils.py index d48b8bcbb08..9fc38586ab0 100644 --- a/Lib/test/test_asyncio/test_windows_utils.py +++ b/Lib/test/test_asyncio/test_windows_utils.py @@ -1,72 +1,17 @@ """Tests for window_utils""" -import socket import sys import unittest import warnings -from unittest import mock if sys.platform != 'win32': raise unittest.SkipTest('Windows only') +import _overlapped import _winapi -from asyncio import _overlapped from asyncio import windows_utils -try: - from test import support -except ImportError: - from asyncio import test_support as support - - -class WinsocketpairTests(unittest.TestCase): - - def check_winsocketpair(self, ssock, csock): - csock.send(b'xxx') - self.assertEqual(b'xxx', ssock.recv(1024)) - csock.close() - ssock.close() - - def test_winsocketpair(self): - ssock, csock = windows_utils.socketpair() - self.check_winsocketpair(ssock, csock) - - @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 not supported or enabled') - def test_winsocketpair_ipv6(self): - ssock, csock = windows_utils.socketpair(family=socket.AF_INET6) - self.check_winsocketpair(ssock, csock) - - @unittest.skipIf(hasattr(socket, 'socketpair'), - 'socket.socketpair is available') - @mock.patch('asyncio.windows_utils.socket') - def test_winsocketpair_exc(self, m_socket): - m_socket.AF_INET = socket.AF_INET - m_socket.SOCK_STREAM = socket.SOCK_STREAM - m_socket.socket.return_value.getsockname.return_value = ('', 12345) - m_socket.socket.return_value.accept.return_value = object(), object() - m_socket.socket.return_value.connect.side_effect = OSError() - - self.assertRaises(OSError, windows_utils.socketpair) - - def test_winsocketpair_invalid_args(self): - self.assertRaises(ValueError, - windows_utils.socketpair, family=socket.AF_UNSPEC) - self.assertRaises(ValueError, - windows_utils.socketpair, type=socket.SOCK_DGRAM) - self.assertRaises(ValueError, - windows_utils.socketpair, proto=1) - - @unittest.skipIf(hasattr(socket, 'socketpair'), - 'socket.socketpair is available') - @mock.patch('asyncio.windows_utils.socket') - def test_winsocketpair_close(self, m_socket): - m_socket.AF_INET = socket.AF_INET - m_socket.SOCK_STREAM = socket.SOCK_STREAM - sock = mock.Mock() - m_socket.socket.return_value = sock - sock.bind.side_effect = OSError - self.assertRaises(OSError, windows_utils.socketpair) - self.assertTrue(sock.close.called) +from test import support class PipeTests(unittest.TestCase): diff --git a/Lib/asyncio/test_utils.py b/Lib/test/test_asyncio/utils.py similarity index 82% rename from Lib/asyncio/test_utils.py rename to Lib/test/test_asyncio/utils.py index c3ddfe37563..96dfe2f85b4 100644 --- a/Lib/asyncio/test_utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -6,6 +6,7 @@ import io import logging import os import re +import selectors import socket import socketserver import sys @@ -25,20 +26,62 @@ try: except ImportError: # pragma: no cover ssl = None -from . import base_events -from . import events -from . import futures -from . import selectors -from . import tasks -from .coroutines import coroutine -from .log import logger +from asyncio import base_events +from asyncio import events +from asyncio import format_helpers +from asyncio import futures +from asyncio import tasks +from asyncio.log import logger from test import support -if sys.platform == 'win32': # pragma: no cover - from .windows_utils import socketpair -else: - from socket import socketpair # pragma: no cover +def data_file(filename): + if hasattr(support, 'TEST_HOME_DIR'): + fullname = os.path.join(support.TEST_HOME_DIR, filename) + if os.path.isfile(fullname): + return fullname + fullname = os.path.join(os.path.dirname(__file__), filename) + if os.path.isfile(fullname): + return fullname + raise FileNotFoundError(filename) + + +ONLYCERT = data_file('ssl_cert.pem') +ONLYKEY = data_file('ssl_key.pem') +SIGNED_CERTFILE = data_file('keycert3.pem') +SIGNING_CA = data_file('pycacert.pem') +PEERCERT = { + 'OCSP': ('http://testca.pythontest.net/testca/ocsp/',), + 'caIssuers': ('http://testca.pythontest.net/testca/pycacert.cer',), + 'crlDistributionPoints': ('http://testca.pythontest.net/testca/revocation.crl',), + 'issuer': ((('countryName', 'XY'),), + (('organizationName', 'Python Software Foundation CA'),), + (('commonName', 'our-ca-server'),)), + 'notAfter': 'Nov 28 19:09:06 2027 GMT', + 'notBefore': 'Jan 19 19:09:06 2018 GMT', + 'serialNumber': '82EDBF41C880919C', + 'subject': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'subjectAltName': (('DNS', 'localhost'),), + 'version': 3 +} + + +def simple_server_sslcontext(): + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(ONLYCERT, ONLYKEY) + server_context.check_hostname = False + server_context.verify_mode = ssl.CERT_NONE + return server_context + + +def simple_client_sslcontext(): + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + return client_context def dummy_ssl_context(): @@ -49,8 +92,7 @@ def dummy_ssl_context(): def run_briefly(loop): - @coroutine - def once(): + async def once(): pass gen = once() t = loop.create_task(gen) @@ -323,7 +365,7 @@ class TestLoop(base_events.BaseEventLoop): raise AssertionError("Time generator is not finished") def _add_reader(self, fd, callback, *args): - self.readers[fd] = events.Handle(callback, args, self) + self.readers[fd] = events.Handle(callback, args, self, None) def _remove_reader(self, fd): self.remove_reader_count[fd] += 1 @@ -334,15 +376,22 @@ class TestLoop(base_events.BaseEventLoop): return False def assert_reader(self, fd, callback, *args): - assert fd in self.readers, 'fd {} is not registered'.format(fd) + if fd not in self.readers: + raise AssertionError(f'fd {fd} is not registered') handle = self.readers[fd] - assert handle._callback == callback, '{!r} != {!r}'.format( - handle._callback, callback) - assert handle._args == args, '{!r} != {!r}'.format( - handle._args, args) + if handle._callback != callback: + raise AssertionError( + f'unexpected callback: {handle._callback} != {callback}') + if handle._args != args: + raise AssertionError( + f'unexpected callback args: {handle._args} != {args}') + + def assert_no_reader(self, fd): + if fd in self.readers: + raise AssertionError(f'fd {fd} is registered') def _add_writer(self, fd, callback, *args): - self.writers[fd] = events.Handle(callback, args, self) + self.writers[fd] = events.Handle(callback, args, self, None) def _remove_writer(self, fd): self.remove_writer_count[fd] += 1 @@ -408,9 +457,9 @@ class TestLoop(base_events.BaseEventLoop): self.advance_time(advance) self._timers = [] - def call_at(self, when, callback, *args): + def call_at(self, when, callback, *args, context=None): self._timers.append(when) - return super().call_at(when, callback, *args) + return super().call_at(when, callback, *args, context=context) def _process_events(self, event_list): return @@ -436,8 +485,16 @@ class MockPattern(str): return bool(re.search(str(self), other, re.S)) +class MockInstanceOf: + def __init__(self, type): + self._type = type + + def __eq__(self, other): + return isinstance(other, self._type) + + def get_function_source(func): - source = events._get_function_source(func) + source = format_helpers._get_function_source(func) if source is None: raise ValueError("unable to get the source of %r" % (func,)) return source diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py index ee0c3b371f8..694ddffd687 100644 --- a/Lib/test/test_asyncore.py +++ b/Lib/test/test_asyncore.py @@ -726,14 +726,10 @@ class BaseTestAPI: def test_create_socket(self): s = asyncore.dispatcher() s.create_socket(self.family) + self.assertEqual(s.socket.type, socket.SOCK_STREAM) self.assertEqual(s.socket.family, self.family) - SOCK_NONBLOCK = getattr(socket, 'SOCK_NONBLOCK', 0) - sock_type = socket.SOCK_STREAM | SOCK_NONBLOCK - if hasattr(socket, 'SOCK_CLOEXEC'): - self.assertIn(s.socket.type, - (sock_type | socket.SOCK_CLOEXEC, sock_type)) - else: - self.assertEqual(s.socket.type, sock_type) + self.assertEqual(s.socket.gettimeout(), 0) + self.assertFalse(s.socket.get_inheritable()) def test_bind(self): if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: diff --git a/Lib/test/test_atexit.py b/Lib/test/test_atexit.py index 1d0b018aafa..3105f6c3781 100644 --- a/Lib/test/test_atexit.py +++ b/Lib/test/test_atexit.py @@ -2,7 +2,9 @@ import sys import unittest import io import atexit +import os from test import support +from test.support import script_helper ### helpers def h1(): @@ -152,6 +154,21 @@ class GeneralTest(unittest.TestCase): atexit._run_exitfuncs() self.assertEqual(l, [5]) + def test_shutdown(self): + # Actually test the shutdown mechanism in a subprocess + code = """if 1: + import atexit + + def f(msg): + print(msg) + + atexit.register(f, "one") + atexit.register(f, "two") + """ + res = script_helper.assert_python_ok("-c", code) + self.assertEqual(res.out.decode().splitlines(), ["two", "one"]) + self.assertFalse(res.err) + @support.cpython_only class SubinterpreterTest(unittest.TestCase): @@ -187,6 +204,24 @@ class SubinterpreterTest(unittest.TestCase): self.assertEqual(ret, 0) self.assertEqual(atexit._ncallbacks(), n) + def test_callback_on_subinterpreter_teardown(self): + # This tests if a callback is called on + # subinterpreter teardown. + expected = b"The test has passed!" + r, w = os.pipe() + + code = r"""if 1: + import os + import atexit + def callback(): + os.write({:d}, b"The test has passed!") + atexit.register(callback) + """.format(w) + ret = support.run_in_subinterp(code) + os.close(w) + self.assertEqual(os.read(r, len(expected)), expected) + os.close(r) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_baseexception.py b/Lib/test/test_baseexception.py index c055ee3d83c..c32468269a7 100644 --- a/Lib/test/test_baseexception.py +++ b/Lib/test/test_baseexception.py @@ -163,7 +163,7 @@ class UsageTests(unittest.TestCase): self.raise_fails("spam") def test_catch_non_BaseException(self): - # Tryinng to catch an object that does not inherit from BaseException + # Trying to catch an object that does not inherit from BaseException # is not allowed. class NonBaseException(object): pass diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 0a61c054444..8f91bc9bf91 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -337,16 +337,16 @@ class BuiltinTest(unittest.TestCase): try: assert False except AssertionError: - return (True, f.__doc__, debug_enabled) + return (True, f.__doc__, debug_enabled, __debug__) else: - return (False, f.__doc__, debug_enabled) + return (False, f.__doc__, debug_enabled, __debug__) ''' def f(): """doc""" - values = [(-1, __debug__, f.__doc__, __debug__), - (0, True, 'doc', True), - (1, False, 'doc', False), - (2, False, None, False)] - for optval, assertval, docstring, debugval in values: + values = [(-1, __debug__, f.__doc__, __debug__, __debug__), + (0, True, 'doc', True, True), + (1, False, 'doc', False, False), + (2, False, None, False, False)] + for optval, *expected in values: # test both direct compilation and compilation via AST codeobjs = [] codeobjs.append(compile(codestr, "", "exec", optimize=optval)) @@ -356,7 +356,7 @@ class BuiltinTest(unittest.TestCase): ns = {} exec(code, ns) rv = ns['f']() - self.assertEqual(rv, (assertval, docstring, debugval)) + self.assertEqual(rv, tuple(expected)) def test_delattr(self): sys.spam = 1 @@ -1022,6 +1022,7 @@ class BuiltinTest(unittest.TestCase): self.assertRaises(ValueError, open, 'a\x00b') self.assertRaises(ValueError, open, b'a\x00b') + @unittest.skipIf(sys.flags.utf8_mode, "utf-8 mode is enabled") def test_open_default_encoding(self): old_environ = dict(os.environ) try: diff --git a/Lib/test/test_c_locale_coercion.py b/Lib/test/test_c_locale_coercion.py index 2a22739fb0f..1db293b9c37 100644 --- a/Lib/test/test_c_locale_coercion.py +++ b/Lib/test/test_c_locale_coercion.py @@ -14,30 +14,51 @@ from test.support.script_helper import ( interpreter_requires_environment, ) +# Set the list of ways we expect to be able to ask for the "C" locale +EXPECTED_C_LOCALE_EQUIVALENTS = ["C", "invalid.ascii"] + # Set our expectation for the default encoding used in the C locale # for the filesystem encoding and the standard streams +EXPECTED_C_LOCALE_STREAM_ENCODING = "ascii" +EXPECTED_C_LOCALE_FS_ENCODING = "ascii" -# While most *nix platforms default to ASCII in the C locale, some use a -# different encoding. -if sys.platform.startswith("aix"): - C_LOCALE_STREAM_ENCODING = "iso8859-1" -elif test.support.is_android: - C_LOCALE_STREAM_ENCODING = "utf-8" -else: - C_LOCALE_STREAM_ENCODING = "ascii" +# Set our expectation for the default locale used when none is specified +EXPECT_COERCION_IN_DEFAULT_LOCALE = True -# FS encoding is UTF-8 on macOS, other *nix platforms use the locale encoding -if sys.platform == "darwin": - C_LOCALE_FS_ENCODING = "utf-8" -else: - C_LOCALE_FS_ENCODING = C_LOCALE_STREAM_ENCODING +# Apply some platform dependent overrides +if sys.platform.startswith("linux"): + if test.support.is_android: + # Android defaults to using UTF-8 for all system interfaces + EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8" + EXPECTED_C_LOCALE_FS_ENCODING = "utf-8" + else: + # Linux distros typically alias the POSIX locale directly to the C + # locale. + # TODO: Once https://bugs.python.org/issue30672 is addressed, we'll be + # able to check this case unconditionally + EXPECTED_C_LOCALE_EQUIVALENTS.append("POSIX") +elif sys.platform.startswith("aix"): + # AIX uses iso8859-1 in the C locale, other *nix platforms use ASCII + EXPECTED_C_LOCALE_STREAM_ENCODING = "iso8859-1" + EXPECTED_C_LOCALE_FS_ENCODING = "iso8859-1" +elif sys.platform == "darwin": + # FS encoding is UTF-8 on macOS + EXPECTED_C_LOCALE_FS_ENCODING = "utf-8" +elif sys.platform == "cygwin": + # Cygwin defaults to using C.UTF-8 + # TODO: Work out a robust dynamic test for this that doesn't rely on + # CPython's own locale handling machinery + EXPECT_COERCION_IN_DEFAULT_LOCALE = False -# Note that the above is probably still wrong in some cases, such as: +# Note that the above expectations are still wrong in some cases, such as: # * Windows when PYTHONLEGACYWINDOWSFSENCODING is set -# * AIX and any other platforms that use latin-1 in the C locale +# * Any platform other than AIX that uses latin-1 in the C locale +# * Any Linux distro where POSIX isn't a simple alias for the C locale +# * Any Linux distro where the default locale is something other than "C" # # Options for dealing with this: -# * Don't set PYTHON_COERCE_C_LOCALE on such platforms (e.g. Windows doesn't) +# * Don't set the PY_COERCE_C_LOCALE preprocessor definition on +# such platforms (e.g. it isn't set on Windows) # * Fix the test expectations to match the actual platform behaviour # In order to get the warning messages to match up as expected, the candidate @@ -47,7 +68,7 @@ _C_UTF8_LOCALES = ("C.UTF-8", "C.utf8", "UTF-8") # There's no reliable cross-platform way of checking locale alias # lists, so the only way of knowing which of these locales will work # is to try them with locale.setlocale(). We do that in a subprocess -# to avoid altering the locale of the test runner. +# in setUpModule() below to avoid altering the locale of the test runner. # # If the relevant locale module attributes exist, and we're not on a platform # where we expect it to always succeed, we also check that @@ -65,7 +86,7 @@ def _set_locale_in_subprocess(locale_name): # If there's no valid CODESET, we expect coercion to be skipped cmd_fmt += "; import sys; sys.exit(not locale.nl_langinfo(locale.CODESET))" cmd = cmd_fmt.format(locale_name) - result, py_cmd = run_python_until_end("-c", cmd, __isolated=True) + result, py_cmd = run_python_until_end("-c", cmd, PYTHONCOERCECLOCALE='') return result.rc == 0 @@ -130,8 +151,7 @@ class EncodingDetails(_EncodingDetails): that. """ result, py_cmd = run_python_until_end( - "-c", cls.CHILD_PROCESS_SCRIPT, - __isolated=True, + "-X", "utf8=0", "-c", cls.CHILD_PROCESS_SCRIPT, **env_vars ) if not result.rc == 0: @@ -217,8 +237,9 @@ class _LocaleHandlingTestCase(unittest.TestCase): class LocaleConfigurationTests(_LocaleHandlingTestCase): # Test explicit external configuration via the process environment - def setUpClass(): - # This relies on setupModule() having been run, so it can't be + @classmethod + def setUpClass(cls): + # This relies on setUpModule() having been run, so it can't be # handled via the @unittest.skipUnless decorator if not AVAILABLE_TARGETS: raise unittest.SkipTest("No C-with-UTF-8 locale available") @@ -236,6 +257,7 @@ class LocaleConfigurationTests(_LocaleHandlingTestCase): "LANG": "", "LC_CTYPE": "", "LC_ALL": "", + "PYTHONCOERCECLOCALE": "", } for env_var in ("LANG", "LC_CTYPE"): for locale_to_set in AVAILABLE_TARGETS: @@ -284,8 +306,8 @@ class LocaleCoercionTests(_LocaleHandlingTestCase): if not AVAILABLE_TARGETS: # Locale coercion is disabled when there aren't any target locales - fs_encoding = C_LOCALE_FS_ENCODING - stream_encoding = C_LOCALE_STREAM_ENCODING + fs_encoding = EXPECTED_C_LOCALE_FS_ENCODING + stream_encoding = EXPECTED_C_LOCALE_STREAM_ENCODING coercion_expected = False if expected_warnings: expected_warnings = [LEGACY_LOCALE_WARNING] @@ -294,43 +316,50 @@ class LocaleCoercionTests(_LocaleHandlingTestCase): "LANG": "", "LC_CTYPE": "", "LC_ALL": "", + "PYTHONCOERCECLOCALE": "", } base_var_dict.update(extra_vars) - for env_var in ("LANG", "LC_CTYPE"): - for locale_to_set in ("", "C", "POSIX", "invalid.ascii"): - # XXX (ncoghlan): *BSD platforms don't behave as expected in the - # POSIX locale, so we skip that for now - # See https://bugs.python.org/issue30672 for discussion - if locale_to_set == "POSIX": - continue + if coerce_c_locale is not None: + base_var_dict["PYTHONCOERCECLOCALE"] = coerce_c_locale - # Platforms using UTF-8 in the C locale do not print - # CLI_COERCION_WARNING when all the locale envt variables are - # not set or set to the empty string. + # Check behaviour for the default locale + with self.subTest(default_locale=True, + PYTHONCOERCECLOCALE=coerce_c_locale): + if EXPECT_COERCION_IN_DEFAULT_LOCALE: _expected_warnings = expected_warnings - for _env_var in base_var_dict: - if base_var_dict[_env_var]: - break - else: - if (C_LOCALE_STREAM_ENCODING == "utf-8" and - locale_to_set == "" and coerce_c_locale == "warn"): - _expected_warnings = None + _coercion_expected = coercion_expected + else: + _expected_warnings = None + _coercion_expected = False + # On Android CLI_COERCION_WARNING is not printed when all the + # locale environment variables are undefined or empty. When + # this code path is run with environ['LC_ALL'] == 'C', then + # LEGACY_LOCALE_WARNING is printed. + if (test.support.is_android and + _expected_warnings == [CLI_COERCION_WARNING]): + _expected_warnings = None + self._check_child_encoding_details(base_var_dict, + fs_encoding, + stream_encoding, + _expected_warnings, + _coercion_expected) + # Check behaviour for explicitly configured locales + for locale_to_set in EXPECTED_C_LOCALE_EQUIVALENTS: + for env_var in ("LANG", "LC_CTYPE"): with self.subTest(env_var=env_var, nominal_locale=locale_to_set, PYTHONCOERCECLOCALE=coerce_c_locale): var_dict = base_var_dict.copy() var_dict[env_var] = locale_to_set - if coerce_c_locale is not None: - var_dict["PYTHONCOERCECLOCALE"] = coerce_c_locale # Check behaviour on successful coercion self._check_child_encoding_details(var_dict, fs_encoding, stream_encoding, - _expected_warnings, + expected_warnings, coercion_expected) - def test_test_PYTHONCOERCECLOCALE_not_set(self): + def test_PYTHONCOERCECLOCALE_not_set(self): # This should coerce to the first available target locale by default self._check_c_locale_coercion("utf-8", "utf-8", coerce_c_locale=None) @@ -349,27 +378,27 @@ class LocaleCoercionTests(_LocaleHandlingTestCase): def test_PYTHONCOERCECLOCALE_set_to_zero(self): # The setting "0" should result in the locale coercion being disabled - self._check_c_locale_coercion(C_LOCALE_FS_ENCODING, - C_LOCALE_STREAM_ENCODING, + self._check_c_locale_coercion(EXPECTED_C_LOCALE_FS_ENCODING, + EXPECTED_C_LOCALE_STREAM_ENCODING, coerce_c_locale="0", coercion_expected=False) # Setting LC_ALL=C shouldn't make any difference to the behaviour - self._check_c_locale_coercion(C_LOCALE_FS_ENCODING, - C_LOCALE_STREAM_ENCODING, + self._check_c_locale_coercion(EXPECTED_C_LOCALE_FS_ENCODING, + EXPECTED_C_LOCALE_STREAM_ENCODING, coerce_c_locale="0", LC_ALL="C", coercion_expected=False) def test_LC_ALL_set_to_C(self): # Setting LC_ALL should render the locale coercion ineffective - self._check_c_locale_coercion(C_LOCALE_FS_ENCODING, - C_LOCALE_STREAM_ENCODING, + self._check_c_locale_coercion(EXPECTED_C_LOCALE_FS_ENCODING, + EXPECTED_C_LOCALE_STREAM_ENCODING, coerce_c_locale=None, LC_ALL="C", coercion_expected=False) # And result in a warning about a lack of locale compatibility - self._check_c_locale_coercion(C_LOCALE_FS_ENCODING, - C_LOCALE_STREAM_ENCODING, + self._check_c_locale_coercion(EXPECTED_C_LOCALE_FS_ENCODING, + EXPECTED_C_LOCALE_STREAM_ENCODING, coerce_c_locale="warn", LC_ALL="C", expected_warnings=[LEGACY_LOCALE_WARNING], diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index bb5b2a3b9f0..2a6de3c5aa9 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -1,10 +1,9 @@ # Run the _testcapi module tests (tests for the Python/C API): by defn, # these are all functions _testcapi exports whose name begins with 'test_'. -from collections import namedtuple, OrderedDict +from collections import OrderedDict import os import pickle -import platform import random import re import subprocess @@ -420,180 +419,6 @@ class Test6012(unittest.TestCase): self.assertEqual(_testcapi.argparsing("Hello", "World"), 1) -class EmbeddingTests(unittest.TestCase): - def setUp(self): - here = os.path.abspath(__file__) - basepath = os.path.dirname(os.path.dirname(os.path.dirname(here))) - exename = "_testembed" - if sys.platform.startswith("win"): - ext = ("_d" if "_d" in sys.executable else "") + ".exe" - exename += ext - exepath = os.path.dirname(sys.executable) - else: - exepath = os.path.join(basepath, "Programs") - self.test_exe = exe = os.path.join(exepath, exename) - if not os.path.exists(exe): - self.skipTest("%r doesn't exist" % exe) - # This is needed otherwise we get a fatal error: - # "Py_Initialize: Unable to get the locale encoding - # LookupError: no codec search functions registered: can't find encoding" - self.oldcwd = os.getcwd() - os.chdir(basepath) - - def tearDown(self): - os.chdir(self.oldcwd) - - def run_embedded_interpreter(self, *args, env=None): - """Runs a test in the embedded interpreter""" - cmd = [self.test_exe] - cmd.extend(args) - if env is not None and sys.platform == 'win32': - # Windows requires at least the SYSTEMROOT environment variable to - # start Python. - env = env.copy() - env['SYSTEMROOT'] = os.environ['SYSTEMROOT'] - - p = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - env=env) - (out, err) = p.communicate() - self.assertEqual(p.returncode, 0, - "bad returncode %d, stderr is %r" % - (p.returncode, err)) - return out, err - - def run_repeated_init_and_subinterpreters(self): - out, err = self.run_embedded_interpreter("repeated_init_and_subinterpreters") - self.assertEqual(err, "") - - # The output from _testembed looks like this: - # --- Pass 0 --- - # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728 - # interp 1 <0x1d4f690>, thread state <0x1d35350>: id(modules) = 139650431165784 - # interp 2 <0x1d5a690>, thread state <0x1d99ed0>: id(modules) = 139650413140368 - # interp 3 <0x1d4f690>, thread state <0x1dc3340>: id(modules) = 139650412862200 - # interp 0 <0x1cf9330>, thread state <0x1cf9700>: id(modules) = 139650431942728 - # --- Pass 1 --- - # ... - - interp_pat = (r"^interp (\d+) <(0x[\dA-F]+)>, " - r"thread state <(0x[\dA-F]+)>: " - r"id\(modules\) = ([\d]+)$") - Interp = namedtuple("Interp", "id interp tstate modules") - - numloops = 0 - current_run = [] - for line in out.splitlines(): - if line == "--- Pass {} ---".format(numloops): - self.assertEqual(len(current_run), 0) - if support.verbose: - print(line) - numloops += 1 - continue - - self.assertLess(len(current_run), 5) - match = re.match(interp_pat, line) - if match is None: - self.assertRegex(line, interp_pat) - - # Parse the line from the loop. The first line is the main - # interpreter and the 3 afterward are subinterpreters. - interp = Interp(*match.groups()) - if support.verbose: - print(interp) - self.assertTrue(interp.interp) - self.assertTrue(interp.tstate) - self.assertTrue(interp.modules) - current_run.append(interp) - - # The last line in the loop should be the same as the first. - if len(current_run) == 5: - main = current_run[0] - self.assertEqual(interp, main) - yield current_run - current_run = [] - - def test_subinterps_main(self): - for run in self.run_repeated_init_and_subinterpreters(): - main = run[0] - - self.assertEqual(main.id, '0') - - def test_subinterps_different_ids(self): - for run in self.run_repeated_init_and_subinterpreters(): - main, *subs, _ = run - - mainid = int(main.id) - for i, sub in enumerate(subs): - self.assertEqual(sub.id, str(mainid + i + 1)) - - def test_subinterps_distinct_state(self): - for run in self.run_repeated_init_and_subinterpreters(): - main, *subs, _ = run - - if '0x0' in main: - # XXX Fix on Windows (and other platforms): something - # is going on with the pointers in Programs/_testembed.c. - # interp.interp is 0x0 and interp.modules is the same - # between interpreters. - raise unittest.SkipTest('platform prints pointers as 0x0') - - for sub in subs: - # A new subinterpreter may have the same - # PyInterpreterState pointer as a previous one if - # the earlier one has already been destroyed. So - # we compare with the main interpreter. The same - # applies to tstate. - self.assertNotEqual(sub.interp, main.interp) - self.assertNotEqual(sub.tstate, main.tstate) - self.assertNotEqual(sub.modules, main.modules) - - def test_forced_io_encoding(self): - # Checks forced configuration of embedded interpreter IO streams - env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape") - out, err = self.run_embedded_interpreter("forced_io_encoding", env=env) - if support.verbose > 1: - print() - print(out) - print(err) - expected_stream_encoding = "utf-8" - expected_errors = "surrogateescape" - expected_output = '\n'.join([ - "--- Use defaults ---", - "Expected encoding: default", - "Expected errors: default", - "stdin: {in_encoding}:{errors}", - "stdout: {out_encoding}:{errors}", - "stderr: {out_encoding}:backslashreplace", - "--- Set errors only ---", - "Expected encoding: default", - "Expected errors: ignore", - "stdin: {in_encoding}:ignore", - "stdout: {out_encoding}:ignore", - "stderr: {out_encoding}:backslashreplace", - "--- Set encoding only ---", - "Expected encoding: latin-1", - "Expected errors: default", - "stdin: latin-1:{errors}", - "stdout: latin-1:{errors}", - "stderr: latin-1:backslashreplace", - "--- Set encoding and errors ---", - "Expected encoding: latin-1", - "Expected errors: replace", - "stdin: latin-1:replace", - "stdout: latin-1:replace", - "stderr: latin-1:backslashreplace"]) - expected_output = expected_output.format( - in_encoding=expected_stream_encoding, - out_encoding=expected_stream_encoding, - errors=expected_errors) - # This is useful if we ever trip over odd platform behaviour - self.maxDiff = None - self.assertEqual(out.strip(), expected_output) - - class SkipitemTest(unittest.TestCase): def test_skipitem(self): @@ -829,8 +654,7 @@ class PyMemMallocDebugTests(PyMemDebugTests): PYTHONMALLOC = 'malloc_debug' -@unittest.skipUnless(sysconfig.get_config_var('WITH_PYMALLOC') == 1, - 'need pymalloc') +@unittest.skipUnless(support.with_pymalloc(), 'need pymalloc') class PyMemPymallocDebugTests(PyMemDebugTests): PYTHONMALLOC = 'pymalloc_debug' diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index ecc01f27795..841cac9171c 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -595,5 +595,54 @@ class ClassTests(unittest.TestCase): with self.assertRaises(TypeError): type.__setattr__(A, b'x', None) + def testConstructorErrorMessages(self): + # bpo-31506: Improves the error message logic for object_new & object_init + + # Class without any method overrides + class C: + pass + + with self.assertRaisesRegex(TypeError, r'C\(\) takes no arguments'): + C(42) + + with self.assertRaisesRegex(TypeError, r'C\(\) takes no arguments'): + C.__new__(C, 42) + + with self.assertRaisesRegex(TypeError, r'C\(\).__init__\(\) takes no arguments'): + C().__init__(42) + + with self.assertRaisesRegex(TypeError, r'C\(\) takes no arguments'): + object.__new__(C, 42) + + with self.assertRaisesRegex(TypeError, r'C\(\).__init__\(\) takes no arguments'): + object.__init__(C(), 42) + + # Class with both `__init__` & `__new__` method overridden + class D: + def __new__(cls, *args, **kwargs): + super().__new__(cls, *args, **kwargs) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + with self.assertRaisesRegex(TypeError, r'object.__new__\(\) takes no argument'): + D(42) + + with self.assertRaisesRegex(TypeError, r'object.__new__\(\) takes no argument'): + D.__new__(D, 42) + + with self.assertRaisesRegex(TypeError, r'object.__new__\(\) takes no argument'): + object.__new__(D, 42) + + # Class that only overrides __init__ + class E: + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + with self.assertRaisesRegex(TypeError, r'object.__init__\(\) takes no argument'): + E().__init__(42) + + with self.assertRaisesRegex(TypeError, r'object.__init__\(\) takes no argument'): + object.__init__(E(), 42) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_cmd.py b/Lib/test/test_cmd.py index 99a483be43b..96e0c30da32 100644 --- a/Lib/test/test_cmd.py +++ b/Lib/test/test_cmd.py @@ -51,7 +51,7 @@ class samplecmdclass(cmd.Cmd): Test for the function completedefault(): >>> mycmd.completedefault() - This is the completedefault methode + This is the completedefault method >>> mycmd.completenames("a") ['add'] @@ -140,7 +140,7 @@ class samplecmdclass(cmd.Cmd): print("Hello from postloop") def completedefault(self, *ignored): - print("This is the completedefault methode") + print("This is the completedefault method") def complete_command(self): print("complete command") diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 3dbe75f0a09..a6b663403f4 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -5,6 +5,7 @@ import os import subprocess import sys +import sysconfig import tempfile import unittest from test import support @@ -13,6 +14,11 @@ from test.support.script_helper import ( interpreter_requires_environment ) + +# Debug build? +Py_DEBUG = hasattr(sys, "gettotalrefcount") + + # XXX (ncoghlan): Move to script_helper and make consistent with run_python def _kill_python_and_exit_code(p): data = kill_python(p) @@ -96,7 +102,7 @@ class CmdLineTest(unittest.TestCase): # "-X showrefcount" shows the refcount, but only in debug builds rc, out, err = run_python('-X', 'showrefcount', '-c', code) self.assertEqual(out.rstrip(), b"{'showrefcount': True}") - if hasattr(sys, 'gettotalrefcount'): # debug build + if Py_DEBUG: self.assertRegex(err, br'^\[\d+ refs, \d+ blocks\]') else: self.assertEqual(err, b'') @@ -426,8 +432,16 @@ class CmdLineTest(unittest.TestCase): # Verify that sys.flags contains hash_randomization code = 'import sys; print("random is", sys.flags.hash_randomization)' - rc, out, err = assert_python_ok('-c', code) - self.assertEqual(rc, 0) + rc, out, err = assert_python_ok('-c', code, PYTHONHASHSEED='') + self.assertIn(b'random is 1', out) + + rc, out, err = assert_python_ok('-c', code, PYTHONHASHSEED='random') + self.assertIn(b'random is 1', out) + + rc, out, err = assert_python_ok('-c', code, PYTHONHASHSEED='0') + self.assertIn(b'random is 0', out) + + rc, out, err = assert_python_ok('-R', '-c', code, PYTHONHASHSEED='0') self.assertIn(b'random is 1', out) def test_del___main__(self): @@ -507,14 +521,18 @@ class CmdLineTest(unittest.TestCase): with self.subTest(envar_value=value): assert_python_ok('-c', code, **env_vars) - def run_xdev(self, code, check_exitcode=True): + def run_xdev(self, *args, check_exitcode=True, xdev=True): env = dict(os.environ) env.pop('PYTHONWARNINGS', None) + env.pop('PYTHONDEVMODE', None) # Force malloc() to disable the debug hooks which are enabled # by default for Python compiled in debug mode env['PYTHONMALLOC'] = 'malloc' - args = (sys.executable, '-X', 'dev', '-c', code) + if xdev: + args = (sys.executable, '-X', 'dev', *args) + else: + args = (sys.executable, *args) proc = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -525,28 +543,167 @@ class CmdLineTest(unittest.TestCase): return proc.stdout.rstrip() def test_xdev(self): - out = self.run_xdev("import sys; print(sys.warnoptions)") - self.assertEqual(out, "['default']") + # sys.flags.dev_mode + code = "import sys; print(sys.flags.dev_mode)" + out = self.run_xdev("-c", code, xdev=False) + self.assertEqual(out, "False") + out = self.run_xdev("-c", code) + self.assertEqual(out, "True") + # Warnings + code = ("import warnings; " + "print(' '.join('%s::%s' % (f[0], f[2].__name__) " + "for f in warnings.filters))") + if Py_DEBUG: + expected_filters = "default::Warning" + else: + expected_filters = ("default::Warning " + "default::DeprecationWarning " + "ignore::DeprecationWarning " + "ignore::PendingDeprecationWarning " + "ignore::ImportWarning " + "ignore::ResourceWarning") + + out = self.run_xdev("-c", code) + self.assertEqual(out, expected_filters) + + out = self.run_xdev("-b", "-c", code) + self.assertEqual(out, f"default::BytesWarning {expected_filters}") + + out = self.run_xdev("-bb", "-c", code) + self.assertEqual(out, f"error::BytesWarning {expected_filters}") + + out = self.run_xdev("-Werror", "-c", code) + self.assertEqual(out, f"error::Warning {expected_filters}") + + # Memory allocator debug hooks try: import _testcapi except ImportError: pass else: - code = "import _testcapi; _testcapi.pymem_api_misuse()" + code = "import _testcapi; print(_testcapi.pymem_getallocatorsname())" with support.SuppressCrashReport(): - out = self.run_xdev(code, check_exitcode=False) - self.assertIn("Debug memory block at address p=", out) + out = self.run_xdev("-c", code, check_exitcode=False) + if support.with_pymalloc(): + alloc_name = "pymalloc_debug" + else: + alloc_name = "malloc_debug" + self.assertEqual(out, alloc_name) + # Faulthandler try: import faulthandler except ImportError: pass else: code = "import faulthandler; print(faulthandler.is_enabled())" - out = self.run_xdev(code) + out = self.run_xdev("-c", code) self.assertEqual(out, "True") + def check_warnings_filters(self, cmdline_option, envvar, use_pywarning=False): + if use_pywarning: + code = ("import sys; from test.support import import_fresh_module; " + "warnings = import_fresh_module('warnings', blocked=['_warnings']); ") + else: + code = "import sys, warnings; " + code += ("print(' '.join('%s::%s' % (f[0], f[2].__name__) " + "for f in warnings.filters))") + args = (sys.executable, '-W', cmdline_option, '-bb', '-c', code) + env = dict(os.environ) + env.pop('PYTHONDEVMODE', None) + env["PYTHONWARNINGS"] = envvar + proc = subprocess.run(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + env=env) + self.assertEqual(proc.returncode, 0, proc) + return proc.stdout.rstrip() + + def test_warnings_filter_precedence(self): + expected_filters = ("error::BytesWarning " + "once::UserWarning " + "always::UserWarning") + if not Py_DEBUG: + expected_filters += (" " + "default::DeprecationWarning " + "ignore::DeprecationWarning " + "ignore::PendingDeprecationWarning " + "ignore::ImportWarning " + "ignore::ResourceWarning") + + out = self.check_warnings_filters("once::UserWarning", + "always::UserWarning") + self.assertEqual(out, expected_filters) + + out = self.check_warnings_filters("once::UserWarning", + "always::UserWarning", + use_pywarning=True) + self.assertEqual(out, expected_filters) + + def check_pythonmalloc(self, env_var, name): + code = 'import _testcapi; print(_testcapi.pymem_getallocatorsname())' + env = dict(os.environ) + env.pop('PYTHONDEVMODE', None) + if env_var is not None: + env['PYTHONMALLOC'] = env_var + else: + env.pop('PYTHONMALLOC', None) + args = (sys.executable, '-c', code) + proc = subprocess.run(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + env=env) + self.assertEqual(proc.stdout.rstrip(), name) + self.assertEqual(proc.returncode, 0) + + def test_pythonmalloc(self): + # Test the PYTHONMALLOC environment variable + pymalloc = support.with_pymalloc() + if pymalloc: + default_name = 'pymalloc_debug' if Py_DEBUG else 'pymalloc' + default_name_debug = 'pymalloc_debug' + else: + default_name = 'malloc_debug' if Py_DEBUG else 'malloc' + default_name_debug = 'malloc_debug' + + tests = [ + (None, default_name), + ('debug', default_name_debug), + ('malloc', 'malloc'), + ('malloc_debug', 'malloc_debug'), + ] + if pymalloc: + tests.extend(( + ('pymalloc', 'pymalloc'), + ('pymalloc_debug', 'pymalloc_debug'), + )) + + for env_var, name in tests: + with self.subTest(env_var=env_var, name=name): + self.check_pythonmalloc(env_var, name) + + def test_pythondevmode_env(self): + # Test the PYTHONDEVMODE environment variable + code = "import sys; print(sys.flags.dev_mode)" + env = dict(os.environ) + env.pop('PYTHONDEVMODE', None) + args = (sys.executable, '-c', code) + + proc = subprocess.run(args, stdout=subprocess.PIPE, + universal_newlines=True, env=env) + self.assertEqual(proc.stdout.rstrip(), 'False') + self.assertEqual(proc.returncode, 0, proc) + + env['PYTHONDEVMODE'] = '1' + proc = subprocess.run(args, stdout=subprocess.PIPE, + universal_newlines=True, env=env) + self.assertEqual(proc.stdout.rstrip(), 'True') + self.assertEqual(proc.returncode, 0, proc) + + class IgnoreEnvironmentTest(unittest.TestCase): def run_ignoring_vars(self, predicate, **env_vars): diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 90cb584ac40..55faf4c4279 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -102,6 +102,7 @@ consts: ('None',) """ +import inspect import sys import threading import unittest @@ -130,6 +131,10 @@ def dump(co): print("%s: %s" % (attr, getattr(co, "co_" + attr))) print("consts:", tuple(consts(co.co_consts))) +# Needed for test_closure_injection below +# Defined at global scope to avoid implicitly closing over __class__ +def external_getitem(self, i): + return f"Foreign getitem: {super().__getitem__(i)}" class CodeTest(unittest.TestCase): @@ -141,6 +146,46 @@ class CodeTest(unittest.TestCase): self.assertEqual(co.co_name, "funcname") self.assertEqual(co.co_firstlineno, 15) + @cpython_only + def test_closure_injection(self): + # From https://bugs.python.org/issue32176 + from types import FunctionType, CodeType + + def create_closure(__class__): + return (lambda: __class__).__closure__ + + def new_code(c): + '''A new code object with a __class__ cell added to freevars''' + return CodeType( + c.co_argcount, c.co_kwonlyargcount, c.co_nlocals, + c.co_stacksize, c.co_flags, c.co_code, c.co_consts, c.co_names, + c.co_varnames, c.co_filename, c.co_name, c.co_firstlineno, + c.co_lnotab, c.co_freevars + ('__class__',), c.co_cellvars) + + def add_foreign_method(cls, name, f): + code = new_code(f.__code__) + assert not f.__closure__ + closure = create_closure(cls) + defaults = f.__defaults__ + setattr(cls, name, FunctionType(code, globals(), name, defaults, closure)) + + class List(list): + pass + + add_foreign_method(List, "__getitem__", external_getitem) + + # Ensure the closure injection actually worked + function = List.__getitem__ + class_ref = function.__closure__[0].cell_contents + self.assertIs(class_ref, List) + + # Ensure the code correctly indicates it accesses a free variable + self.assertFalse(function.__code__.co_flags & inspect.CO_NOFREE, + hex(function.__code__.co_flags)) + + # Ensure the zero-arg super() call in the injected method works + obj = List([1, 2, 3]) + self.assertEqual(obj[0], "Foreign getitem: 1") def isinterned(s): return s is sys.intern(('_' + s + '_')[1:-1]) diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 6a3e9932656..e2e7463389e 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -1032,7 +1032,7 @@ class CodecCallbackTest(unittest.TestCase): def mutating(exc): if isinstance(exc, UnicodeDecodeError): - exc.object[:] = b"" + exc.object = b"" return ("\u4242", 0) else: raise TypeError("don't know how to handle %r" % exc) @@ -1042,8 +1042,59 @@ class CodecCallbackTest(unittest.TestCase): with test.support.check_warnings(): # unicode-internal has been deprecated for (encoding, data) in baddata: - with self.assertRaises(TypeError): - data.decode(encoding, "test.replacing") + self.assertEqual(data.decode(encoding, "test.mutating"), "\u4242") + + # issue32583 + def test_crashing_decode_handler(self): + # better generating one more character to fill the extra space slot + # so in debug build it can steadily fail + def forward_shorter_than_end(exc): + if isinstance(exc, UnicodeDecodeError): + # size one character, 0 < forward < exc.end + return ('\ufffd', exc.start+1) + else: + raise TypeError("don't know how to handle %r" % exc) + codecs.register_error( + "test.forward_shorter_than_end", forward_shorter_than_end) + + self.assertEqual( + b'\xd8\xd8\xd8\xd8\xd8\x00\x00\x00'.decode( + 'utf-16-le', 'test.forward_shorter_than_end'), + '\ufffd\ufffd\ufffd\ufffd\xd8\x00' + ) + self.assertEqual( + b'\xd8\xd8\xd8\xd8\x00\xd8\x00\x00'.decode( + 'utf-16-be', 'test.forward_shorter_than_end'), + '\ufffd\ufffd\ufffd\ufffd\xd8\x00' + ) + self.assertEqual( + b'\x11\x11\x11\x11\x11\x00\x00\x00\x00\x00\x00'.decode( + 'utf-32-le', 'test.forward_shorter_than_end'), + '\ufffd\ufffd\ufffd\u1111\x00' + ) + self.assertEqual( + b'\x11\x11\x11\x00\x00\x11\x11\x00\x00\x00\x00'.decode( + 'utf-32-be', 'test.forward_shorter_than_end'), + '\ufffd\ufffd\ufffd\u1111\x00' + ) + + def replace_with_long(exc): + if isinstance(exc, UnicodeDecodeError): + exc.object = b"\x00" * 8 + return ('\ufffd', exc.start) + else: + raise TypeError("don't know how to handle %r" % exc) + codecs.register_error("test.replace_with_long", replace_with_long) + + self.assertEqual( + b'\x00'.decode('utf-16', 'test.replace_with_long'), + '\ufffd\x00\x00\x00\x00' + ) + self.assertEqual( + b'\x00'.decode('utf-32', 'test.replace_with_long'), + '\ufffd\x00\x00' + ) + def test_fake_error_class(self): handlers = [ diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index de6868a46c4..a59a5e21358 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -5,6 +5,7 @@ import locale import sys import unittest import encodings +from unittest import mock from test import support @@ -196,19 +197,33 @@ class ReadTest(MixInCheckStateHandling): self.assertEqual(f.read(), ''.join(lines[1:])) self.assertEqual(f.read(), '') + # Issue #32110: Test readline() followed by read(n) + f = getreader() + self.assertEqual(f.readline(), lines[0]) + self.assertEqual(f.read(1), lines[1][0]) + self.assertEqual(f.read(0), '') + self.assertEqual(f.read(100), data[len(lines[0]) + 1:][:100]) + # Issue #16636: Test readline() followed by readlines() f = getreader() self.assertEqual(f.readline(), lines[0]) self.assertEqual(f.readlines(), lines[1:]) self.assertEqual(f.read(), '') - # Test read() followed by read() + # Test read(n) followed by read() f = getreader() self.assertEqual(f.read(size=40, chars=5), data[:5]) self.assertEqual(f.read(), data[5:]) self.assertEqual(f.read(), '') - # Issue #12446: Test read() followed by readlines() + # Issue #32110: Test read(n) followed by read(n) + f = getreader() + self.assertEqual(f.read(size=40, chars=5), data[:5]) + self.assertEqual(f.read(1), data[5]) + self.assertEqual(f.read(0), '') + self.assertEqual(f.read(100), data[6:106]) + + # Issue #12446: Test read(n) followed by readlines() f = getreader() self.assertEqual(f.read(size=40, chars=5), data[:5]) self.assertEqual(f.readlines(), [lines[0][5:]] + lines[1:]) @@ -3166,16 +3181,9 @@ class CodePageTest(unittest.TestCase): def test_mbcs_alias(self): # Check that looking up our 'default' codepage will return # mbcs when we don't have a more specific one available - import _bootlocale - def _get_fake_codepage(*a): - return 'cp123' - old_getpreferredencoding = _bootlocale.getpreferredencoding - _bootlocale.getpreferredencoding = _get_fake_codepage - try: + with mock.patch('_winapi.GetACP', return_value=123): codec = codecs.lookup('cp123') self.assertEqual(codec.name, 'mbcs') - finally: - _bootlocale.getpreferredencoding = old_getpreferredencoding class ASCIITest(unittest.TestCase): diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 7e106affbe0..a55239e5730 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -216,6 +216,57 @@ class TestNamedTuple(unittest.TestCase): self.assertRaises(TypeError, Point._make, [11]) # catch too few args self.assertRaises(TypeError, Point._make, [11, 22, 33]) # catch too many args + def test_defaults(self): + Point = namedtuple('Point', 'x y', defaults=(10, 20)) # 2 defaults + self.assertEqual(Point._fields_defaults, {'x': 10, 'y': 20}) + self.assertEqual(Point(1, 2), (1, 2)) + self.assertEqual(Point(1), (1, 20)) + self.assertEqual(Point(), (10, 20)) + + Point = namedtuple('Point', 'x y', defaults=(20,)) # 1 default + self.assertEqual(Point._fields_defaults, {'y': 20}) + self.assertEqual(Point(1, 2), (1, 2)) + self.assertEqual(Point(1), (1, 20)) + + Point = namedtuple('Point', 'x y', defaults=()) # 0 defaults + self.assertEqual(Point._fields_defaults, {}) + self.assertEqual(Point(1, 2), (1, 2)) + with self.assertRaises(TypeError): + Point(1) + + with self.assertRaises(TypeError): # catch too few args + Point() + with self.assertRaises(TypeError): # catch too many args + Point(1, 2, 3) + with self.assertRaises(TypeError): # too many defaults + Point = namedtuple('Point', 'x y', defaults=(10, 20, 30)) + with self.assertRaises(TypeError): # non-iterable defaults + Point = namedtuple('Point', 'x y', defaults=10) + with self.assertRaises(TypeError): # another non-iterable default + Point = namedtuple('Point', 'x y', defaults=False) + + Point = namedtuple('Point', 'x y', defaults=None) # default is None + self.assertEqual(Point._fields_defaults, {}) + self.assertIsNone(Point.__new__.__defaults__, None) + self.assertEqual(Point(10, 20), (10, 20)) + with self.assertRaises(TypeError): # catch too few args + Point(10) + + Point = namedtuple('Point', 'x y', defaults=[10, 20]) # allow non-tuple iterable + self.assertEqual(Point._fields_defaults, {'x': 10, 'y': 20}) + self.assertEqual(Point.__new__.__defaults__, (10, 20)) + self.assertEqual(Point(1, 2), (1, 2)) + self.assertEqual(Point(1), (1, 20)) + self.assertEqual(Point(), (10, 20)) + + Point = namedtuple('Point', 'x y', defaults=iter([10, 20])) # allow plain iterator + self.assertEqual(Point._fields_defaults, {'x': 10, 'y': 20}) + self.assertEqual(Point.__new__.__defaults__, (10, 20)) + self.assertEqual(Point(1, 2), (1, 2)) + self.assertEqual(Point(1), (1, 20)) + self.assertEqual(Point(), (10, 20)) + + @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_factory_doc_attr(self): @@ -558,7 +609,7 @@ class TestOneTrickPonyABCs(ABCTestCase): c = new_coro() self.assertIsInstance(c, Awaitable) - c.close() # awoid RuntimeWarning that coro() was not awaited + c.close() # avoid RuntimeWarning that coro() was not awaited class CoroLike: pass Coroutine.register(CoroLike) @@ -608,7 +659,7 @@ class TestOneTrickPonyABCs(ABCTestCase): c = new_coro() self.assertIsInstance(c, Coroutine) - c.close() # awoid RuntimeWarning that coro() was not awaited + c.close() # avoid RuntimeWarning that coro() was not awaited class CoroLike: def send(self, value): @@ -792,13 +843,13 @@ class TestOneTrickPonyABCs(ABCTestCase): self.assertFalse(issubclass(type(x), Collection), repr(type(x))) # Check some non-collection iterables non_col_iterables = [_test_gen(), iter(b''), iter(bytearray()), - (x for x in []), dict().values()] + (x for x in [])] for x in non_col_iterables: self.assertNotIsInstance(x, Collection) self.assertFalse(issubclass(type(x), Collection), repr(type(x))) # Check some collections samples = [set(), frozenset(), dict(), bytes(), str(), tuple(), - list(), dict().keys(), dict().items()] + list(), dict().keys(), dict().items(), dict().values()] for x in samples: self.assertIsInstance(x, Collection) self.assertTrue(issubclass(type(x), Collection), repr(type(x))) @@ -1615,7 +1666,7 @@ class TestCollectionABCs(ABCTestCase): '__len__', '__getitem__', '__setitem__', '__delitem__', 'insert') def test_MutableSequence_mixins(self): - # Test the mixins of MutableSequence by creating a miminal concrete + # Test the mixins of MutableSequence by creating a minimal concrete # class inherited from it. class MutableSequenceSubclass(MutableSequence): def __init__(self): diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index b4a52a53105..4617a126abe 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -35,6 +35,7 @@ class TestSpecifics(unittest.TestCase): import builtins prev = builtins.__debug__ setattr(builtins, '__debug__', 'sure') + self.assertEqual(__debug__, prev) setattr(builtins, '__debug__', prev) def test_argument_handling(self): @@ -670,8 +671,13 @@ if 1: compile("42", PathLike("test_compile_pathlike"), "single") + def test_stack_overflow(self): + # bpo-31113: Stack overflow when compile a long sequence of + # complex statements. + compile("if a: b\n" * 200000, "", "exec") -class TestStackSize(unittest.TestCase): + +class TestExpressionStackSize(unittest.TestCase): # These tests check that the computed stack size for a code object # stays within reasonable bounds (see issue #21523 for an example # dysfunction). @@ -709,5 +715,294 @@ class TestStackSize(unittest.TestCase): self.check_stack_size(code) +class TestStackSizeStability(unittest.TestCase): + # Check that repeating certain snippets doesn't increase the stack size + # beyond what a single snippet requires. + + def check_stack_size(self, snippet, async_=False): + def compile_snippet(i): + ns = {} + script = """def func():\n""" + i * snippet + if async_: + script = "async " + script + code = compile(script, "