merged

Lib/importlib/_bootstrap.py

@@ -722,6 +722,56 @@ class FrozenImporter:
         return _imp.init_frozen(fullname)
 
 
+class WindowsRegistryImporter:
+
+    """Meta path import for modules declared in the Windows registry.
+    """
+
+    REGISTRY_KEY = (
+        "Software\\Python\\PythonCore\\{sys_version}"
+        "\\Modules\\{fullname}")
+    REGISTRY_KEY_DEBUG = (
+        "Software\\Python\\PythonCore\\{sys_version}"
+        "\\Modules\\{fullname}\\Debug")
+    DEBUG_BUILD = False  # Changed in _setup()
+
+    @classmethod
+    def _open_registry(cls, key):
+        try:
+            return _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, key)
+        except WindowsError:
+            return _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, key)
+
+    @classmethod
+    def _search_registry(cls, fullname):
+        if cls.DEBUG_BUILD:
+            registry_key = cls.REGISTRY_KEY_DEBUG
+        else:
+            registry_key = cls.REGISTRY_KEY
+        key = registry_key.format(fullname=fullname,
+                                  sys_version=sys.version[:3])
+        try:
+            with cls._open_registry(key) as hkey:
+                filepath = _winreg.QueryValue(hkey, "")
+        except WindowsError:
+            return None
+        return filepath
+
+    @classmethod
+    def find_module(cls, fullname, path=None):
+        """Find module named in the registry."""
+        filepath = cls._search_registry(fullname)
+        if filepath is None:
+            return None
+        try:
+            _os.stat(filepath)
+        except OSError:
+            return None
+        for loader, suffixes, _ in _get_supported_file_loaders():
+            if filepath.endswith(tuple(suffixes)):
+                return loader(fullname, filepath)
+
+
 class _LoaderBasics:
 
     """Base class of common code needed by both SourceLoader and
@@ -1422,7 +1472,7 @@ def _find_and_load_unlocked(name, import_):
     parent = name.rpartition('.')[0]
     if parent:
         if parent not in sys.modules:
-            import_(parent)
+            _recursive_import(import_, parent)
         # Crazy side-effects!
         if name in sys.modules:
             return sys.modules[name]
@@ -1500,6 +1550,12 @@ def _gcd_import(name, package=None, level=0):
     _lock_unlock_module(name)
     return module
 
+def _recursive_import(import_, name):
+    """Common exit point for recursive calls to the import machinery
+
+    This simplifies the process of stripping importlib from tracebacks
+    """
+    return import_(name)
 
 def _handle_fromlist(module, fromlist, import_):
     """Figure out what __import__ should return.
@@ -1519,7 +1575,8 @@ def _handle_fromlist(module, fromlist, import_):
                 fromlist.extend(module.__all__)
         for x in fromlist:
             if not hasattr(module, x):
-                import_('{}.{}'.format(module.__name__, x))
+                _recursive_import(import_,
+                                  '{}.{}'.format(module.__name__, x))
     return module
 
 
@@ -1538,6 +1595,17 @@ def _calc___package__(globals):
     return package
 
 
+def _get_supported_file_loaders():
+    """Returns a list of file-based module loaders.
+
+    Each item is a tuple (loader, suffixes, allow_packages).
+    """
+    extensions = ExtensionFileLoader, _imp.extension_suffixes(), False
+    source = SourceFileLoader, SOURCE_SUFFIXES, True
+    bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES, True
+    return [extensions, source, bytecode]
+
+
 def __import__(name, globals={}, locals={}, fromlist=[], level=0):
     """Import a module.
 
@@ -1620,6 +1688,10 @@ def _setup(sys_module, _imp_module):
         thread_module = None
     weakref_module = BuiltinImporter.load_module('_weakref')
 
+    if builtin_os == 'nt':
+        winreg_module = BuiltinImporter.load_module('winreg')
+        setattr(self_module, '_winreg', winreg_module)
+
     setattr(self_module, '_os', os_module)
     setattr(self_module, '_thread', thread_module)
     setattr(self_module, '_weakref', weakref_module)
@@ -1629,14 +1701,17 @@ def _setup(sys_module, _imp_module):
     setattr(self_module, '_relax_case', _make_relax_case())
     if builtin_os == 'nt':
         SOURCE_SUFFIXES.append('.pyw')
+        if '_d.pyd' in _imp.extension_suffixes():
+            WindowsRegistryImporter.DEBUG_BUILD = True
 
 
 def _install(sys_module, _imp_module):
     """Install importlib as the implementation of import."""
     _setup(sys_module, _imp_module)
-    extensions = ExtensionFileLoader, _imp_module.extension_suffixes(), False
+    supported_loaders = _get_supported_file_loaders()
-    source = SourceFileLoader, SOURCE_SUFFIXES, True
-    bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES, True
-    supported_loaders = [extensions, source, bytecode]
     sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)])
-    sys.meta_path.extend([BuiltinImporter, FrozenImporter, PathFinder])
+    sys.meta_path.append(BuiltinImporter)
+    sys.meta_path.append(FrozenImporter)
+    if _os.__name__ == 'nt':
+        sys.meta_path.append(WindowsRegistryImporter)
+    sys.meta_path.append(PathFinder)

Lib/test/support.py

@@ -25,6 +25,7 @@ import fnmatch
 import logging.handlers
 import struct
 import tempfile
+import _testcapi
 
 try:
     import _thread, threading
@@ -1082,6 +1083,33 @@ def python_is_optimized():
     return final_opt != '' and final_opt != '-O0'
 
 
+_header = 'nP'
+_align = '0n'
+if hasattr(sys, "gettotalrefcount"):
+    _header = '2P' + _header
+    _align = '0P'
+_vheader = _header + 'n'
+
+def calcobjsize(fmt):
+    return struct.calcsize(_header + fmt + _align)
+
+def calcvobjsize(fmt):
+    return struct.calcsize(_vheader + fmt + _align)
+
+
+_TPFLAGS_HAVE_GC = 1<<14
+_TPFLAGS_HEAPTYPE = 1<<9
+
+def check_sizeof(test, o, size):
+    result = sys.getsizeof(o)
+    # add GC header size
+    if ((type(o) == type) and (o.__flags__ & _TPFLAGS_HEAPTYPE) or\
+        ((type(o) != type) and (type(o).__flags__ & _TPFLAGS_HAVE_GC))):
+        size += _testcapi.SIZEOF_PYGC_HEAD
+    msg = 'wrong size for %s: got %d, expected %d' \
+            % (type(o), result, size)
+    test.assertEqual(result, size, msg)
+
 #=======================================================================
 # Decorator for running a function in a different locale, correctly resetting
 # it afterwards.

Lib/test/test_import.py

@@ -844,6 +844,74 @@ class ImportTracebackTests(unittest.TestCase):
             self.fail("ZeroDivisionError should have been raised")
         self.assert_traceback(tb, [__file__, 'foo.py', 'bar.py'])
 
+    # A few more examples from issue #15425
+    def test_syntax_error(self):
+        self.create_module("foo", "invalid syntax is invalid")
+        try:
+            import foo
+        except SyntaxError as e:
+            tb = e.__traceback__
+        else:
+            self.fail("SyntaxError should have been raised")
+        self.assert_traceback(tb, [__file__])
+
+    def _setup_broken_package(self, parent, child):
+        pkg_name = "_parent_foo"
+        def cleanup():
+            rmtree(pkg_name)
+            unload(pkg_name)
+        os.mkdir(pkg_name)
+        self.addCleanup(cleanup)
+        # Touch the __init__.py
+        init_path = os.path.join(pkg_name, '__init__.py')
+        with open(init_path, 'w') as f:
+            f.write(parent)
+        bar_path = os.path.join(pkg_name, 'bar.py')
+        with open(bar_path, 'w') as f:
+            f.write(child)
+        importlib.invalidate_caches()
+        return init_path, bar_path
+
+    def test_broken_submodule(self):
+        init_path, bar_path = self._setup_broken_package("", "1/0")
+        try:
+            import _parent_foo.bar
+        except ZeroDivisionError as e:
+            tb = e.__traceback__
+        else:
+            self.fail("ZeroDivisionError should have been raised")
+        self.assert_traceback(tb, [__file__, bar_path])
+
+    def test_broken_from(self):
+        init_path, bar_path = self._setup_broken_package("", "1/0")
+        try:
+            from _parent_foo import bar
+        except ZeroDivisionError as e:
+            tb = e.__traceback__
+        else:
+            self.fail("ImportError should have been raised")
+        self.assert_traceback(tb, [__file__, bar_path])
+
+    def test_broken_parent(self):
+        init_path, bar_path = self._setup_broken_package("1/0", "")
+        try:
+            import _parent_foo.bar
+        except ZeroDivisionError as e:
+            tb = e.__traceback__
+        else:
+            self.fail("ZeroDivisionError should have been raised")
+        self.assert_traceback(tb, [__file__, init_path])
+
+    def test_broken_parent_from(self):
+        init_path, bar_path = self._setup_broken_package("1/0", "")
+        try:
+            from _parent_foo import bar
+        except ZeroDivisionError as e:
+            tb = e.__traceback__
+        else:
+            self.fail("ZeroDivisionError should have been raised")
+        self.assert_traceback(tb, [__file__, init_path])
+
     @cpython_only
     def test_import_bug(self):
         # We simulate a bug in importlib and check that it's not stripped

Lib/test/test_io.py

@@ -802,6 +802,20 @@ class CommonBufferedTests:
             buf.raw = x
 
 
+class SizeofTest:
+
+    @support.cpython_only
+    def test_sizeof(self):
+        bufsize1 = 4096
+        bufsize2 = 8192
+        rawio = self.MockRawIO()
+        bufio = self.tp(rawio, buffer_size=bufsize1)
+        size = sys.getsizeof(bufio) - bufsize1
+        rawio = self.MockRawIO()
+        bufio = self.tp(rawio, buffer_size=bufsize2)
+        self.assertEqual(sys.getsizeof(bufio), size + bufsize2)
+
+
 class BufferedReaderTest(unittest.TestCase, CommonBufferedTests):
     read_mode = "rb"
 
@@ -999,7 +1013,7 @@ class BufferedReaderTest(unittest.TestCase, CommonBufferedTests):
                              "failed for {}: {} != 0".format(n, rawio._extraneous_reads))
 
 
-class CBufferedReaderTest(BufferedReaderTest):
+class CBufferedReaderTest(BufferedReaderTest, SizeofTest):
     tp = io.BufferedReader
 
     def test_constructor(self):
@@ -1260,7 +1274,7 @@ class BufferedWriterTest(unittest.TestCase, CommonBufferedTests):
             self.tp(self.MockRawIO(), 8, 12)
 
 
-class CBufferedWriterTest(BufferedWriterTest):
+class CBufferedWriterTest(BufferedWriterTest, SizeofTest):
     tp = io.BufferedWriter
 
     def test_constructor(self):
@@ -1650,7 +1664,7 @@ class BufferedRandomTest(BufferedReaderTest, BufferedWriterTest):
     # You can't construct a BufferedRandom over a non-seekable stream.
     test_unseekable = None
 
-class CBufferedRandomTest(BufferedRandomTest):
+class CBufferedRandomTest(BufferedRandomTest, SizeofTest):
     tp = io.BufferedRandom
 
     def test_constructor(self):

Lib/test/test_struct.py

@@ -3,7 +3,7 @@ import unittest
 import struct
 import sys
 
-from test.support import run_unittest
+from test import support
 
 ISBIGENDIAN = sys.byteorder == "big"
 IS32BIT = sys.maxsize == 0x7fffffff
@@ -572,18 +572,29 @@ class StructTest(unittest.TestCase):
         s = struct.Struct('i')
         s.__init__('ii')
 
-    def test_sizeof(self):
+    def check_sizeof(self, format_str, number_of_codes):
-        self.assertGreater(sys.getsizeof(struct.Struct('BHILfdspP')),
+        # The size of 'PyStructObject'
-                           sys.getsizeof(struct.Struct('B')))
+        totalsize = support.calcobjsize('2n3P')
-        self.assertGreater(sys.getsizeof(struct.Struct('123B')),
+        # The size taken up by the 'formatcode' dynamic array
-                                sys.getsizeof(struct.Struct('B')))
+        totalsize += struct.calcsize('P2n0P') * (number_of_codes + 1)
-        self.assertGreater(sys.getsizeof(struct.Struct('B' * 1234)),
+        support.check_sizeof(self, struct.Struct(format_str), totalsize)
-                                sys.getsizeof(struct.Struct('123B')))
+
-        self.assertGreater(sys.getsizeof(struct.Struct('1234B')),
+    @support.cpython_only
-                                sys.getsizeof(struct.Struct('123B')))
+    def test__sizeof__(self):
+        for code in integer_codes:
+            self.check_sizeof(code, 1)
+        self.check_sizeof('BHILfdspP', 9)
+        self.check_sizeof('B' * 1234, 1234)
+        self.check_sizeof('fd', 2)
+        self.check_sizeof('xxxxxxxxxxxxxx', 0)
+        self.check_sizeof('100H', 100)
+        self.check_sizeof('187s', 1)
+        self.check_sizeof('20p', 1)
+        self.check_sizeof('0s', 1)
+        self.check_sizeof('0c', 0)
 
 def test_main():
-    run_unittest(StructTest)
+    support.run_unittest(StructTest)
 
 if __name__ == '__main__':
     test_main()

Lib/test/test_sys.py

@@ -612,22 +612,8 @@ class SysModuleTest(unittest.TestCase):
 
 class SizeofTest(unittest.TestCase):
 
-    TPFLAGS_HAVE_GC = 1<<14
-    TPFLAGS_HEAPTYPE = 1<<9
-
     def setUp(self):
-        self.c = len(struct.pack('c', b' '))
+        self.P = struct.calcsize('P')
-        self.H = len(struct.pack('H', 0))
-        self.i = len(struct.pack('i', 0))
-        self.l = len(struct.pack('l', 0))
-        self.P = len(struct.pack('P', 0))
-        # due to missing size_t information from struct, it is assumed that
-        # sizeof(Py_ssize_t) = sizeof(void*)
-        self.header = 'PP'
-        self.vheader = self.header + 'P'
-        if hasattr(sys, "gettotalrefcount"):
-            self.header += '2P'
-            self.vheader += '2P'
         self.longdigit = sys.int_info.sizeof_digit
         import _testcapi
         self.gc_headsize = _testcapi.SIZEOF_PYGC_HEAD
@@ -637,129 +623,108 @@ class SizeofTest(unittest.TestCase):
         self.file.close()
         test.support.unlink(test.support.TESTFN)
 
-    def check_sizeof(self, o, size):
+    check_sizeof = test.support.check_sizeof
-        result = sys.getsizeof(o)
-        # add GC header size
-        if ((type(o) == type) and (o.__flags__ & self.TPFLAGS_HEAPTYPE) or\
-           ((type(o) != type) and (type(o).__flags__ & self.TPFLAGS_HAVE_GC))):
-            size += self.gc_headsize
-        msg = 'wrong size for %s: got %d, expected %d' \
-              % (type(o), result, size)
-        self.assertEqual(result, size, msg)
-
-    def calcsize(self, fmt):
-        """Wrapper around struct.calcsize which enforces the alignment of the
-        end of a structure to the alignment requirement of pointer.
-
-        Note: This wrapper should only be used if a pointer member is included
-        and no member with a size larger than a pointer exists.
-        """
-        return struct.calcsize(fmt + '0P')
 
     def test_gc_head_size(self):
         # Check that the gc header size is added to objects tracked by the gc.
-        h = self.header
+        vsize = test.support.calcvobjsize
-        vh = self.vheader
-        size = self.calcsize
         gc_header_size = self.gc_headsize
         # bool objects are not gc tracked
-        self.assertEqual(sys.getsizeof(True), size(vh) + self.longdigit)
+        self.assertEqual(sys.getsizeof(True), vsize('') + self.longdigit)
         # but lists are
-        self.assertEqual(sys.getsizeof([]), size(vh + 'PP') + gc_header_size)
+        self.assertEqual(sys.getsizeof([]), vsize('Pn') + gc_header_size)
 
     def test_default(self):
-        h = self.header
+        size = test.support.calcvobjsize
-        vh = self.vheader
+        self.assertEqual(sys.getsizeof(True), size('') + self.longdigit)
-        size = self.calcsize
+        self.assertEqual(sys.getsizeof(True, -1), size('') + self.longdigit)
-        self.assertEqual(sys.getsizeof(True), size(vh) + self.longdigit)
-        self.assertEqual(sys.getsizeof(True, -1), size(vh) + self.longdigit)
 
     def test_objecttypes(self):
         # check all types defined in Objects/
-        h = self.header
+        size = test.support.calcobjsize
-        vh = self.vheader
+        vsize = test.support.calcvobjsize
-        size = self.calcsize
         check = self.check_sizeof
         # bool
-        check(True, size(vh) + self.longdigit)
+        check(True, vsize('') + self.longdigit)
         # buffer
         # XXX
         # builtin_function_or_method
-        check(len, size(h + '3P'))
+        check(len, size('3P')) # XXX check layout
         # bytearray
         samples = [b'', b'u'*100000]
         for sample in samples:
             x = bytearray(sample)
-            check(x, size(vh + 'iPP') + x.__alloc__() * self.c)
+            check(x, vsize('inP') + x.__alloc__())
         # bytearray_iterator
-        check(iter(bytearray()), size(h + 'PP'))
+        check(iter(bytearray()), size('nP'))
         # cell
         def get_cell():
             x = 42
             def inner():
                 return x
             return inner
-        check(get_cell().__closure__[0], size(h + 'P'))
+        check(get_cell().__closure__[0], size('P'))
         # code
-        check(get_cell().__code__, size(h + '5i9Pi3P'))
+        check(get_cell().__code__, size('5i9Pi3P'))
-        check(get_cell.__code__, size(h + '5i9Pi3P'))
+        check(get_cell.__code__, size('5i9Pi3P'))
         def get_cell2(x):
             def inner():
                 return x
             return inner
-        check(get_cell2.__code__, size(h + '5i9Pi3P') + 1)
+        check(get_cell2.__code__, size('5i9Pi3P') + 1)
         # complex
-        check(complex(0,1), size(h + '2d'))
+        check(complex(0,1), size('2d'))
         # method_descriptor (descriptor object)
-        check(str.lower, size(h + '3PP'))
+        check(str.lower, size('3PP'))
         # classmethod_descriptor (descriptor object)
         # XXX
         # member_descriptor (descriptor object)
         import datetime
-        check(datetime.timedelta.days, size(h + '3PP'))
+        check(datetime.timedelta.days, size('3PP'))
         # getset_descriptor (descriptor object)
         import collections
-        check(collections.defaultdict.default_factory, size(h + '3PP'))
+        check(collections.defaultdict.default_factory, size('3PP'))
         # wrapper_descriptor (descriptor object)
-        check(int.__add__, size(h + '3P2P'))
+        check(int.__add__, size('3P2P'))
         # method-wrapper (descriptor object)
-        check({}.__iter__, size(h + '2P'))
+        check({}.__iter__, size('2P'))
         # dict
-        check({}, size(h + '3P' + '4P' + 8*'P2P'))
+        check({}, size('n2P' + '2nPn' + 8*'n2P'))
         longdict = {1:1, 2:2, 3:3, 4:4, 5:5, 6:6, 7:7, 8:8}
-        check(longdict, size(h + '3P' + '4P') + 16*size('P2P'))
+        check(longdict, size('n2P' + '2nPn') + 16*struct.calcsize('n2P'))
         # dictionary-keyiterator
-        check({}.keys(), size(h + 'P'))
+        check({}.keys(), size('P'))
         # dictionary-valueiterator
-        check({}.values(), size(h + 'P'))
+        check({}.values(), size('P'))
         # dictionary-itemiterator
-        check({}.items(), size(h + 'P'))
+        check({}.items(), size('P'))
+        # dictionary iterator
+        check(iter({}), size('P2nPn'))
         # dictproxy
         class C(object): pass
-        check(C.__dict__, size(h + 'P'))
+        check(C.__dict__, size('P'))
         # BaseException
-        check(BaseException(), size(h + '5Pi'))
+        check(BaseException(), size('5Pi'))
         # UnicodeEncodeError
-        check(UnicodeEncodeError("", "", 0, 0, ""), size(h + '5Pi 2P2PP'))
+        check(UnicodeEncodeError("", "", 0, 0, ""), size('5Pi 2P2nP'))
         # UnicodeDecodeError
-        # XXX
+        check(UnicodeDecodeError("", b"", 0, 0, ""), size('5Pi 2P2nP'))
-#        check(UnicodeDecodeError("", "", 0, 0, ""), size(h + '5P2PP'))
         # UnicodeTranslateError
-        check(UnicodeTranslateError("", 0, 1, ""), size(h + '5Pi 2P2PP'))
+        check(UnicodeTranslateError("", 0, 1, ""), size('5Pi 2P2nP'))
         # ellipses
-        check(Ellipsis, size(h + ''))
+        check(Ellipsis, size(''))
         # EncodingMap
         import codecs, encodings.iso8859_3
         x = codecs.charmap_build(encodings.iso8859_3.decoding_table)
-        check(x, size(h + '32B2iB'))
+        check(x, size('32B2iB'))
         # enumerate
-        check(enumerate([]), size(h + 'l3P'))
+        check(enumerate([]), size('n3P'))
         # reverse
-        check(reversed(''), size(h + 'PP'))
+        check(reversed(''), size('nP'))
         # float
-        check(float(0), size(h + 'd'))
+        check(float(0), size('d'))
         # sys.floatinfo
-        check(sys.float_info, size(vh) + self.P * len(sys.float_info))
+        check(sys.float_info, vsize('') + self.P * len(sys.float_info))
         # frame
         import inspect
         CO_MAXBLOCKS = 20
@@ -768,10 +733,10 @@ class SizeofTest(unittest.TestCase):
         nfrees = len(x.f_code.co_freevars)
         extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\
                   ncells + nfrees - 1
-        check(x, size(vh + '12P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
+        check(x, vsize('12P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
         # function
         def func(): pass
-        check(func, size(h + '12P'))
+        check(func, size('12P'))
         class c():
             @staticmethod
             def foo():
@@ -780,68 +745,68 @@ class SizeofTest(unittest.TestCase):
             def bar(cls):
                 pass
             # staticmethod
-            check(foo, size(h + 'PP'))
+            check(foo, size('PP'))
             # classmethod
-            check(bar, size(h + 'PP'))
+            check(bar, size('PP'))
         # generator
         def get_gen(): yield 1
-        check(get_gen(), size(h + 'Pi2P'))
+        check(get_gen(), size('Pb2P'))
         # iterator
-        check(iter('abc'), size(h + 'lP'))
+        check(iter('abc'), size('lP'))
         # callable-iterator
         import re
-        check(re.finditer('',''), size(h + '2P'))
+        check(re.finditer('',''), size('2P'))
         # list
         samples = [[], [1,2,3], ['1', '2', '3']]
         for sample in samples:
-            check(sample, size(vh + 'PP') + len(sample)*self.P)
+            check(sample, vsize('Pn') + len(sample)*self.P)
         # sortwrapper (list)
         # XXX
         # cmpwrapper (list)
         # XXX
         # listiterator (list)
-        check(iter([]), size(h + 'lP'))
+        check(iter([]), size('lP'))
         # listreverseiterator (list)
-        check(reversed([]), size(h + 'lP'))
+        check(reversed([]), size('nP'))
         # long
-        check(0, size(vh))
+        check(0, vsize(''))
-        check(1, size(vh) + self.longdigit)
+        check(1, vsize('') + self.longdigit)
-        check(-1, size(vh) + self.longdigit)
+        check(-1, vsize('') + self.longdigit)
         PyLong_BASE = 2**sys.int_info.bits_per_digit
-        check(int(PyLong_BASE), size(vh) + 2*self.longdigit)
+        check(int(PyLong_BASE), vsize('') + 2*self.longdigit)
-        check(int(PyLong_BASE**2-1), size(vh) + 2*self.longdigit)
+        check(int(PyLong_BASE**2-1), vsize('') + 2*self.longdigit)
-        check(int(PyLong_BASE**2), size(vh) + 3*self.longdigit)
+        check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit)
         # memoryview
-        check(memoryview(b''), size(h + 'PPiP4P2i5P3c2P'))
+        check(memoryview(b''), size('Pnin 2P2n2i5P 3cPn'))
         # module
-        check(unittest, size(h + '3P'))
+        check(unittest, size('PnP'))
         # None
-        check(None, size(h + ''))
+        check(None, size(''))
         # NotImplementedType
-        check(NotImplemented, size(h))
+        check(NotImplemented, size(''))
         # object
-        check(object(), size(h + ''))
+        check(object(), size(''))
         # property (descriptor object)
         class C(object):
             def getx(self): return self.__x
             def setx(self, value): self.__x = value
             def delx(self): del self.__x
             x = property(getx, setx, delx, "")
-            check(x, size(h + '4Pi'))
+            check(x, size('4Pi'))
         # PyCapsule
         # XXX
         # rangeiterator
-        check(iter(range(1)), size(h + '4l'))
+        check(iter(range(1)), size('4l'))
         # reverse
-        check(reversed(''), size(h + 'PP'))
+        check(reversed(''), size('nP'))
         # range
-        check(range(1), size(h + '4P'))
+        check(range(1), size('4P'))
-        check(range(66000), size(h + '4P'))
+        check(range(66000), size('4P'))
         # set
         # frozenset
         PySet_MINSIZE = 8
         samples = [[], range(10), range(50)]
-        s = size(h + '3P2P' + PySet_MINSIZE*'lP' + 'lP')
+        s = size('3n2P' + PySet_MINSIZE*'nP' + 'nP')
         for sample in samples:
             minused = len(sample)
             if minused == 0: tmp = 1
@@ -855,31 +820,31 @@ class SizeofTest(unittest.TestCase):
                 check(set(sample), s)
                 check(frozenset(sample), s)
             else:
-                check(set(sample), s + newsize*struct.calcsize('lP'))
+                check(set(sample), s + newsize*struct.calcsize('nP'))
-                check(frozenset(sample), s + newsize*struct.calcsize('lP'))
+                check(frozenset(sample), s + newsize*struct.calcsize('nP'))
         # setiterator
-        check(iter(set()), size(h + 'P3P'))
+        check(iter(set()), size('P3n'))
         # slice
-        check(slice(0), size(h + '3P'))
+        check(slice(0), size('3P'))
         # super
-        check(super(int), size(h + '3P'))
+        check(super(int), size('3P'))
         # tuple
-        check((), size(vh))
+        check((), vsize(''))
-        check((1,2,3), size(vh) + 3*self.P)
+        check((1,2,3), vsize('') + 3*self.P)
         # type
         # static type: PyTypeObject
-        s = size(vh + 'P2P15Pl4PP9PP11PI')
+        s = vsize('P2n15Pl4Pn9Pn11PI')
         check(int, s)
         # (PyTypeObject + PyNumberMethods + PyMappingMethods +
         #  PySequenceMethods + PyBufferProcs + 4P)
-        s = size(vh + 'P2P15Pl4PP9PP11PI') + size('34P 3P 10P 2P 4P')
+        s = vsize('P2n15Pl4Pn9Pn11PI') + struct.calcsize('34P 3P 10P 2P 4P')
         # Separate block for PyDictKeysObject with 4 entries
-        s += size("PPPP") + 4*size("PPP")
+        s += struct.calcsize("2nPn") + 4*struct.calcsize("n2P")
         # class
         class newstyleclass(object): pass
         check(newstyleclass, s)
         # dict with shared keys
-        check(newstyleclass().__dict__, size(h+"PPP4P"))
+        check(newstyleclass().__dict__, size('n2P' + '2nPn'))
         # unicode
         # each tuple contains a string and its expected character size
         # don't put any static strings here, as they may contain
@@ -887,8 +852,8 @@ class SizeofTest(unittest.TestCase):
         samples = ['1'*100, '\xff'*50,
                    '\u0100'*40, '\uffff'*100,
                    '\U00010000'*30, '\U0010ffff'*100]
-        asciifields = h + "PPiP"
+        asciifields = "nniP"
-        compactfields = asciifields + "PPP"
+        compactfields = asciifields + "nPn"
         unicodefields = compactfields + "P"
         for s in samples:
             maxchar = ord(max(s))
@@ -912,32 +877,31 @@ class SizeofTest(unittest.TestCase):
         # TODO: add check that forces layout of unicodefields
         # weakref
         import weakref
-        check(weakref.ref(int), size(h + '2Pl2P'))
+        check(weakref.ref(int), size('2Pn2P'))
         # weakproxy
         # XXX
         # weakcallableproxy
-        check(weakref.proxy(int), size(h + '2Pl2P'))
+        check(weakref.proxy(int), size('2Pn2P'))
 
     def test_pythontypes(self):
         # check all types defined in Python/
-        h = self.header
+        size = test.support.calcobjsize
-        vh = self.vheader
+        vsize = test.support.calcvobjsize
-        size = self.calcsize
         check = self.check_sizeof
         # _ast.AST
         import _ast
-        check(_ast.AST(), size(h + 'P'))
+        check(_ast.AST(), size('P'))
         try:
             raise TypeError
         except TypeError:
             tb = sys.exc_info()[2]
             # traceback
             if tb != None:
-                check(tb, size(h + '2P2i'))
+                check(tb, size('2P2i'))
         # symtable entry
         # XXX
         # sys.flags
-        check(sys.flags, size(vh) + self.P * len(sys.flags))
+        check(sys.flags, vsize('') + self.P * len(sys.flags))
 
 
 def test_main():

Lib/test/test_xml_etree_c.py

@@ -41,38 +41,30 @@ class TestAcceleratorImported(unittest.TestCase):
 
 
 @unittest.skipUnless(cET, 'requires _elementtree')
+@support.cpython_only
 class SizeofTest(unittest.TestCase):
     def setUp(self):
-        import _testcapi
+        self.elementsize = support.calcobjsize('5P')
-        gc_headsize = _testcapi.SIZEOF_PYGC_HEAD
-        # object header
-        header = 'PP'
-        if hasattr(sys, "gettotalrefcount"):
-            # debug header
-            header = 'PP' + header
-        # fields
-        element = header + '5P'
-        self.elementsize = gc_headsize + struct.calcsize(element)
         # extra
         self.extra = struct.calcsize('PiiP4P')
 
+    check_sizeof = support.check_sizeof
+
     def test_element(self):
         e = cET.Element('a')
-        self.assertEqual(sys.getsizeof(e), self.elementsize)
+        self.check_sizeof(e, self.elementsize)
 
     def test_element_with_attrib(self):
         e = cET.Element('a', href='about:')
-        self.assertEqual(sys.getsizeof(e),
+        self.check_sizeof(e, self.elementsize + self.extra)
-                         self.elementsize + self.extra)
 
     def test_element_with_children(self):
         e = cET.Element('a')
         for i in range(5):
             cET.SubElement(e, 'span')
         # should have space for 8 children now
-        self.assertEqual(sys.getsizeof(e),
+        self.check_sizeof(e, self.elementsize + self.extra +
-                         self.elementsize + self.extra +
+                             struct.calcsize('8P'))
-                         struct.calcsize('8P'))
 
 def test_main():
     from test import test_xml_etree, test_xml_etree_c

Misc/NEWS

@@ -10,6 +10,11 @@ What's New in Python 3.3.0 Beta 2?
 Core and Builtins
 -----------------
 
+- Issue #15425: Eliminated traceback noise from more situations involving
+  importlib
+
+- Issue #14578: Support modules registered in the Windows registry again.
+
 - Issue #15466: Stop using TYPE_INT64 in marshal, to make importlib.h
   (and other byte code files) equal between 32-bit and 64-bit systems.
 
@@ -236,6 +241,9 @@ Documentation
 Tests
 -----
 
+- Issue #15467: Move helpers for __sizeof__ tests into test_support.
+  Patch by Serhiy Storchaka.
+
 - Issue #15320: Make iterating the list of tests thread-safe when running
   tests in multiprocess mode. Patch by Chris Jerdonek.
 
@@ -329,6 +337,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #15487: Add a __sizeof__ implementation for buffered I/O objects.
+  Patch by Serhiy Storchaka.
+
 - Issue #15187: Bugfix: remove temporary directories test_shutil was leaving
   behind.
 

Modules/_freeze_importlib.c

@@ -21,6 +21,13 @@ static struct _frozen _PyImport_FrozenModules[] = {
     {0, 0, 0} /* sentinel */
 };
 
+#ifndef MS_WINDOWS
+/* On Windows, this links with the regular pythonXY.dll, so this variable comes
+   from frozen.obj. In the Makefile, frozen.o is not linked into this executable,
+   so we define the variable here. */
+struct _frozen *PyImport_FrozenModules;
+#endif
+
 const char header[] = "/* Auto-generated by Modules/_freeze_importlib.c */";
 
 int
@@ -91,8 +98,8 @@ main(int argc, char *argv[])
     data_size = PyBytes_GET_SIZE(marshalled);
 
     /* Open the file in text mode. The hg checkout should be using the eol extension,
-       which in turn should cause the existing file to use CRLF */
+       which in turn should cause the EOL style match the C library's text mode */
-    outfile = fopen(outpath, "wt");
+    outfile = fopen(outpath, "w");
     if (outfile == NULL) {
         fprintf(stderr, "cannot open '%s' for writing\n", outpath);
         return 1;

Modules/_io/bufferedio.c

@@ -398,6 +398,17 @@ buffered_dealloc(buffered *self)
     Py_TYPE(self)->tp_free((PyObject *)self);
 }
 
+static PyObject *
+buffered_sizeof(buffered *self, void *unused)
+{
+    Py_ssize_t res;
+
+    res = sizeof(buffered);
+    if (self->buffer)
+        res += self->buffer_size;
+    return PyLong_FromSsize_t(res);
+}
+
 static int
 buffered_traverse(buffered *self, visitproc visit, void *arg)
 {
@@ -1699,6 +1710,7 @@ static PyMethodDef bufferedreader_methods[] = {
     {"seek", (PyCFunction)buffered_seek, METH_VARARGS},
     {"tell", (PyCFunction)buffered_tell, METH_NOARGS},
     {"truncate", (PyCFunction)buffered_truncate, METH_VARARGS},
+    {"__sizeof__", (PyCFunction)buffered_sizeof, METH_NOARGS},
     {NULL, NULL}
 };
 
@@ -2079,6 +2091,7 @@ static PyMethodDef bufferedwriter_methods[] = {
     {"flush", (PyCFunction)buffered_flush, METH_NOARGS},
     {"seek", (PyCFunction)buffered_seek, METH_VARARGS},
     {"tell", (PyCFunction)buffered_tell, METH_NOARGS},
+    {"__sizeof__", (PyCFunction)buffered_sizeof, METH_NOARGS},
     {NULL, NULL}
 };
 
@@ -2470,6 +2483,7 @@ static PyMethodDef bufferedrandom_methods[] = {
     {"readline", (PyCFunction)buffered_readline, METH_VARARGS},
     {"peek", (PyCFunction)buffered_peek, METH_VARARGS},
     {"write", (PyCFunction)bufferedwriter_write, METH_VARARGS},
+    {"__sizeof__", (PyCFunction)buffered_sizeof, METH_NOARGS},
     {NULL, NULL}
 };
 

Modules/_struct.c

@@ -1756,15 +1756,11 @@ PyDoc_STRVAR(s_sizeof__doc__,
 "S.__sizeof__() -> size of S in memory, in bytes");
 
 static PyObject *
-s_sizeof(PyStructObject *self)
+s_sizeof(PyStructObject *self, void *unused)
 {
     Py_ssize_t size;
-    formatcode *code;
 
-    size = sizeof(PyStructObject) + sizeof(formatcode);
+    size = sizeof(PyStructObject) + sizeof(formatcode) * (self->s_len + 1);
-    for (code = self->s_codes; code->fmtdef != NULL; code++) {
-        size += sizeof(formatcode);
-    }
     return PyLong_FromSsize_t(size);
 }
 

PCbuild/pcbuild.sln

@@ -603,19 +603,13 @@ Global
 		{1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|x64.ActiveCfg = Release|Win32
 		{1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|x64.Build.0 = Release|Win32
 		{19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|Win32.ActiveCfg = Debug|Win32
-		{19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|Win32.Build.0 = Debug|Win32
 		{19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|x64.ActiveCfg = Debug|x64
-		{19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|x64.Build.0 = Debug|x64
 		{19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|Win32.ActiveCfg = Release|Win32
-		{19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|Win32.Build.0 = Release|Win32
 		{19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGInstrument|x64.ActiveCfg = Release|Win32
 		{19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|Win32.ActiveCfg = Release|Win32
-		{19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|Win32.Build.0 = Release|Win32
 		{19C0C13F-47CA-4432-AFF3-799A296A4DDC}.PGUpdate|x64.ActiveCfg = Release|Win32
 		{19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|Win32.ActiveCfg = Release|Win32
-		{19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|Win32.Build.0 = Release|Win32
 		{19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|x64.ActiveCfg = Release|x64
-		{19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Release|x64.Build.0 = Release|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

Python/import.c

@@ -1154,14 +1154,27 @@ remove_importlib_frames(void)
 {
     const char *importlib_filename = "<frozen importlib._bootstrap>";
     const char *exec_funcname = "_exec_module";
+    const char *get_code_funcname = "get_code";
+    const char *recursive_import = "_recursive_import";
     int always_trim = 0;
+    int trim_get_code = 0;
     int in_importlib = 0;
     PyObject *exception, *value, *base_tb, *tb;
     PyObject **prev_link, **outer_link = NULL;
 
     /* Synopsis: if it's an ImportError, we trim all importlib chunks
-       from the traceback.  Otherwise, we trim only those chunks which
+       from the traceback.  If it's a SyntaxError, we trim any chunks that
-       end with a call to "_exec_module". */
+       end with a call to "get_code", We always trim chunks
+       which end with a call to "_exec_module". */
+
+    /* Thanks to issue 15425, we also strip any chunk ending with
+     * _recursive_import. This is used when making a recursive call to the
+     * full import machinery which means the inner stack gets stripped early
+     * and the normal heuristics won't fire properly for outer frames. A
+     * more elegant mechanism would be nice, as this one can misfire if
+     * builtins.__import__ has been replaced with a custom implementation.
+     * However, the current approach at least gets the job done.
+     */
 
     PyErr_Fetch(&exception, &value, &base_tb);
     if (!exception || Py_VerboseFlag)
@@ -1169,6 +1182,9 @@ remove_importlib_frames(void)
     if (PyType_IsSubtype((PyTypeObject *) exception,
                          (PyTypeObject *) PyExc_ImportError))
         always_trim = 1;
+    if (PyType_IsSubtype((PyTypeObject *) exception,
+                         (PyTypeObject *) PyExc_SyntaxError))
+        trim_get_code = 1;
 
     prev_link = &base_tb;
     tb = base_tb;
@@ -1191,8 +1207,14 @@ remove_importlib_frames(void)
 
         if (in_importlib &&
             (always_trim ||
-             PyUnicode_CompareWithASCIIString(code->co_name,
+             (PyUnicode_CompareWithASCIIString(code->co_name,
-                                              exec_funcname) == 0)) {
+                                              exec_funcname) == 0) ||
+             (PyUnicode_CompareWithASCIIString(code->co_name,
+                                              recursive_import) == 0) ||
+             (trim_get_code &&
+              PyUnicode_CompareWithASCIIString(code->co_name,
+                                              get_code_funcname) == 0)
+           )) {
             PyObject *tmp = *outer_link;
             *outer_link = next;
             Py_XINCREF(next);