Merge.

Doc/library/email.parser.rst

@@ -181,7 +181,7 @@ have the same API as the :class:`Parser` and :class:`BytesParser` classes.
    .. versionchanged:: 3.3
       Removed the *strict* argument.  Added the *policy* keyword.
 
-   .. method:: parse(fp, headeronly=False)
+   .. method:: parse(fp, headersonly=False)
 
       Read all the data from the binary file-like object *fp*, parse the
       resulting bytes, and return the message object.  *fp* must support

Doc/library/subprocess.rst

@@ -406,12 +406,18 @@ functions.
 
       Read the `Security Considerations`_ section before using ``shell=True``.
 
-   *bufsize* will be supplied as the corresponding argument to the :func:`open`
+   *bufsize* will be supplied as the corresponding argument to the
-   function when creating the stdin/stdout/stderr pipe file objects: :const:`0`
+   :func:`open` function when creating the stdin/stdout/stderr pipe
-   means unbuffered (read and write are one system call and can return short),
+   file objects:
-   :const:`1` means line buffered, any other positive value means use a buffer
+
-   of approximately that size.  A negative bufsize (the default) means the
+   - :const:`0` means unbuffered (read and write are one
-   system default of io.DEFAULT_BUFFER_SIZE will be used.
+     system call and can return short)
+   - :const:`1` means line buffered
+     (only usable if ``universal_newlines=True`` i.e., in a text mode)
+   - any other positive value means use a buffer of approximately that
+     size
+   - negative bufsize (the default) means the system default of
+     io.DEFAULT_BUFFER_SIZE will be used.
 
    .. versionchanged:: 3.3.1
       *bufsize* now defaults to -1 to enable buffering by default to match the

Lib/sre_parse.py

@@ -94,33 +94,45 @@ class SubPattern:
         self.data = data
         self.width = None
     def dump(self, level=0):
-        nl = 1
+        nl = True
         seqtypes = (tuple, list)
         for op, av in self.data:
-            print(level*"  " + op, end=' '); nl = 0
+            print(level*"  " + op, end='')
-            if op == "in":
+            if op == IN:
                 # member sublanguage
-                print(); nl = 1
+                print()
                 for op, a in av:
                     print((level+1)*"  " + op, a)
-            elif op == "branch":
+            elif op == BRANCH:
-                print(); nl = 1
+                print()
-                i = 0
+                for i, a in enumerate(av[1]):
-                for a in av[1]:
+                    if i:
-                    if i > 0:
                         print(level*"  " + "or")
-                    a.dump(level+1); nl = 1
+                    a.dump(level+1)
-                    i = i + 1
+            elif op == GROUPREF_EXISTS:
+                condgroup, item_yes, item_no = av
+                print('', condgroup)
+                item_yes.dump(level+1)
+                if item_no:
+                    print(level*"  " + "else")
+                    item_no.dump(level+1)
             elif isinstance(av, seqtypes):
+                nl = False
                 for a in av:
                     if isinstance(a, SubPattern):
-                        if not nl: print()
+                        if not nl:
-                        a.dump(level+1); nl = 1
+                            print()
+                        a.dump(level+1)
+                        nl = True
                     else:
-                        print(a, end=' ') ; nl = 0
+                        if not nl:
+                            print(' ', end='')
+                        print(a, end='')
+                        nl = False
+                if not nl:
+                    print()
             else:
-                print(av, end=' ') ; nl = 0
+                print('', av)
-            if not nl: print()
     def __repr__(self):
         return repr(self.data)
     def __len__(self):

Lib/subprocess.py

@@ -837,7 +837,8 @@ class Popen(object):
         if p2cwrite != -1:
             self.stdin = io.open(p2cwrite, 'wb', bufsize)
             if universal_newlines:
-                self.stdin = io.TextIOWrapper(self.stdin, write_through=True)
+                self.stdin = io.TextIOWrapper(self.stdin, write_through=True,
+                                              line_buffering=(bufsize == 1))
         if c2pread != -1:
             self.stdout = io.open(c2pread, 'rb', bufsize)
             if universal_newlines:

Lib/test/test_re.py

@@ -1203,16 +1203,33 @@ class ReTests(unittest.TestCase):
                 self.assertEqual(m.group(2), "y")
 
     def test_debug_flag(self):
+        pat = r'(\.)(?:[ch]|py)(?(1)$|: )'
         with captured_stdout() as out:
-            re.compile('foo', re.DEBUG)
+            re.compile(pat, re.DEBUG)
-        self.assertEqual(out.getvalue().splitlines(),
+        dump = '''\
-                         ['literal 102 ', 'literal 111 ', 'literal 111 '])
+subpattern 1
+  literal 46
+subpattern None
+  branch
+    in
+      literal 99
+      literal 104
+  or
+    literal 112
+    literal 121
+subpattern None
+  groupref_exists 1
+    at at_end
+  else
+    literal 58
+    literal 32
+'''
+        self.assertEqual(out.getvalue(), dump)
         # Debug output is output again even a second time (bypassing
         # the cache -- issue #20426).
         with captured_stdout() as out:
-            re.compile('foo', re.DEBUG)
+            re.compile(pat, re.DEBUG)
-        self.assertEqual(out.getvalue().splitlines(),
+        self.assertEqual(out.getvalue(), dump)
-                         ['literal 102 ', 'literal 111 ', 'literal 111 '])
 
     def test_keyword_parameters(self):
         # Issue #20283: Accepting the string keyword parameter.

Lib/test/test_subprocess.py

@@ -1008,6 +1008,39 @@ class ProcessTestCase(BaseTestCase):
         p = subprocess.Popen([sys.executable, "-c", "pass"], bufsize=None)
         self.assertEqual(p.wait(), 0)
 
+    def _test_bufsize_equal_one(self, line, expected, universal_newlines):
+        # subprocess may deadlock with bufsize=1, see issue #21332
+        with subprocess.Popen([sys.executable, "-c", "import sys;"
+                               "sys.stdout.write(sys.stdin.readline());"
+                               "sys.stdout.flush()"],
+                              stdin=subprocess.PIPE,
+                              stdout=subprocess.PIPE,
+                              stderr=subprocess.DEVNULL,
+                              bufsize=1,
+                              universal_newlines=universal_newlines) as p:
+            p.stdin.write(line) # expect that it flushes the line in text mode
+            os.close(p.stdin.fileno()) # close it without flushing the buffer
+            read_line = p.stdout.readline()
+            try:
+                p.stdin.close()
+            except OSError:
+                pass
+            p.stdin = None
+        self.assertEqual(p.returncode, 0)
+        self.assertEqual(read_line, expected)
+
+    def test_bufsize_equal_one_text_mode(self):
+        # line is flushed in text mode with bufsize=1.
+        # we should get the full line in return
+        line = "line\n"
+        self._test_bufsize_equal_one(line, line, universal_newlines=True)
+
+    def test_bufsize_equal_one_binary_mode(self):
+        # line is not flushed in binary mode with bufsize=1.
+        # we should get empty response
+        line = b'line' + os.linesep.encode() # assume ascii-based locale
+        self._test_bufsize_equal_one(line, b'', universal_newlines=False)
+
     def test_leaking_fds_on_error(self):
         # see bug #5179: Popen leaks file descriptors to PIPEs if
         # the child fails to execute; this will eventually exhaust

Lib/test/test_threading.py

@@ -4,7 +4,7 @@ Tests for the threading module.
 
 import test.support
 from test.support import verbose, strip_python_stderr, import_module, cpython_only
-from test.script_helper import assert_python_ok
+from test.script_helper import assert_python_ok, assert_python_failure
 
 import random
 import re
@@ -15,7 +15,6 @@ import time
 import unittest
 import weakref
 import os
-from test.script_helper import assert_python_ok, assert_python_failure
 import subprocess
 
 from test import lock_tests
@@ -962,6 +961,88 @@ class ThreadingExceptionTests(BaseTestCase):
         self.assertEqual(p.returncode, 0, "Unexpected error: " + stderr.decode())
         self.assertEqual(data, expected_output)
 
+    def test_print_exception(self):
+        script = r"""if True:
+            import threading
+            import time
+
+            running = False
+            def run():
+                global running
+                running = True
+                while running:
+                    time.sleep(0.01)
+                1/0
+            t = threading.Thread(target=run)
+            t.start()
+            while not running:
+                time.sleep(0.01)
+            running = False
+            t.join()
+            """
+        rc, out, err = assert_python_ok("-c", script)
+        self.assertEqual(out, b'')
+        err = err.decode()
+        self.assertIn("Exception in thread", err)
+        self.assertIn("Traceback (most recent call last):", err)
+        self.assertIn("ZeroDivisionError", err)
+        self.assertNotIn("Unhandled exception", err)
+
+    def test_print_exception_stderr_is_none_1(self):
+        script = r"""if True:
+            import sys
+            import threading
+            import time
+
+            running = False
+            def run():
+                global running
+                running = True
+                while running:
+                    time.sleep(0.01)
+                1/0
+            t = threading.Thread(target=run)
+            t.start()
+            while not running:
+                time.sleep(0.01)
+            sys.stderr = None
+            running = False
+            t.join()
+            """
+        rc, out, err = assert_python_ok("-c", script)
+        self.assertEqual(out, b'')
+        err = err.decode()
+        self.assertIn("Exception in thread", err)
+        self.assertIn("Traceback (most recent call last):", err)
+        self.assertIn("ZeroDivisionError", err)
+        self.assertNotIn("Unhandled exception", err)
+
+    def test_print_exception_stderr_is_none_2(self):
+        script = r"""if True:
+            import sys
+            import threading
+            import time
+
+            running = False
+            def run():
+                global running
+                running = True
+                while running:
+                    time.sleep(0.01)
+                1/0
+            sys.stderr = None
+            t = threading.Thread(target=run)
+            t.start()
+            while not running:
+                time.sleep(0.01)
+            running = False
+            t.join()
+            """
+        rc, out, err = assert_python_ok("-c", script)
+        self.assertEqual(out, b'')
+        self.assertNotIn("Unhandled exception", err.decode())
+
+
 class TimerTests(BaseTestCase):
 
     def setUp(self):

Lib/threading.py

@@ -248,7 +248,7 @@ class Condition:
 
     def _is_owned(self):
         # Return True if lock is owned by current_thread.
-        # This method is called only if __lock doesn't have _is_owned().
+        # This method is called only if _lock doesn't have _is_owned().
         if self._lock.acquire(0):
             self._lock.release()
             return False
@@ -749,12 +749,12 @@ class Thread:
 
     """
 
-    __initialized = False
+    _initialized = False
     # Need to store a reference to sys.exc_info for printing
     # out exceptions when a thread tries to use a global var. during interp.
     # shutdown and thus raises an exception about trying to perform some
     # operation on/with a NoneType
-    __exc_info = _sys.exc_info
+    _exc_info = _sys.exc_info
     # Keep sys.exc_clear too to clear the exception just before
     # allowing .join() to return.
     #XXX __exc_clear = _sys.exc_clear
@@ -926,10 +926,10 @@ class Thread:
                 # shutdown) use self._stderr.  Otherwise still use sys (as in
                 # _sys) in case sys.stderr was redefined since the creation of
                 # self.
-                if _sys:
+                if _sys and _sys.stderr is not None:
-                    _sys.stderr.write("Exception in thread %s:\n%s\n" %
+                    print("Exception in thread %s:\n%s" %
-                                      (self.name, _format_exc()))
+                          (self.name, _format_exc()), file=self._stderr)
-                else:
+                elif self._stderr is not None:
                     # Do the best job possible w/o a huge amt. of code to
                     # approximate a traceback (code ideas from
                     # Lib/traceback.py)
@@ -957,7 +957,7 @@ class Thread:
                 # test_threading.test_no_refcycle_through_target when
                 # the exception keeps the target alive past when we
                 # assert that it's dead.
-                #XXX self.__exc_clear()
+                #XXX self._exc_clear()
                 pass
         finally:
             with _active_limbo_lock:

Misc/NEWS

@@ -13,6 +13,15 @@ Core and Builtins
 Library
 -------
 
+- Issue #22415: Fixed debugging output of the GROUPREF_EXISTS opcode in the re
+  module.  Removed trailing spaces in debugging output.
+
+- Issue #22423: Unhandled exception in thread no longer causes unhandled
+  AttributeError when sys.stderr is None.
+
+- Issue #21332: Ensure that ``bufsize=1`` in subprocess.Popen() selects
+  line buffering, rather than block buffering.  Patch by Akira Li.
+
 
 What's New in Python 3.4.2rc1?
 ==============================