merge

Doc/Makefile

@@ -187,6 +187,10 @@ autobuild-dev:
 	make update
 	make dist SPHINXOPTS='-A daily=1 -A versionswitcher=1'
 
+# for quick rebuilds (HTML only)
+autobuild-html:
+	make html SPHINXOPTS='-A daily=1 -A versionswitcher=1'
+
 # for stable releases: only build if not in pre-release stage (alpha, beta, rc)
 autobuild-stable:
 	@case $(DISTVERSION) in *[abc]*) \
@@ -194,3 +198,4 @@ autobuild-stable:
 		exit 1;; \
 	esac
 	@make autobuild-dev
+

Doc/library/json.rst

@@ -116,7 +116,10 @@ Using json.tool from the shell to validate and pretty-print::
 Basic Usage
 -----------
 
-.. function:: dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, cls=None, indent=None, separators=None, default=None, **kw)
+.. function:: dump(obj, fp, skipkeys=False, ensure_ascii=True, \
+                   check_circular=True, allow_nan=True, cls=None, \
+                   indent=None, separators=None, default=None, \
+                   sort_keys=False, **kw)
 
    Serialize *obj* as a JSON formatted stream to *fp* (a ``.write()``-supporting
    :term:`file-like object`).
@@ -159,12 +162,18 @@ Basic Usage
    *default(obj)* is a function that should return a serializable version of
    *obj* or raise :exc:`TypeError`.  The default simply raises :exc:`TypeError`.
 
+   If *sort_keys* is ``True`` (default: ``False``), then the output of
+   dictionaries will be sorted by key.
+
    To use a custom :class:`JSONEncoder` subclass (e.g. one that overrides the
    :meth:`default` method to serialize additional types), specify it with the
    *cls* kwarg; otherwise :class:`JSONEncoder` is used.
 
 
-.. function:: dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, cls=None, indent=None, separators=None, default=None, **kw)
+.. function:: dumps(obj, skipkeys=False, ensure_ascii=True, \
+                    check_circular=True, allow_nan=True, cls=None, \
+                    indent=None, separators=None, default=None, \
+                    sort_keys=False, **kw)
 
    Serialize *obj* to a JSON formatted :class:`str`.  The arguments have the
    same meaning as in :func:`dump`.

Doc/library/shutil.rst

@@ -70,7 +70,8 @@ Directory and files operations
       Now returns *dst*.
 
    .. versionchanged:: 3.4
-      Raise :exc:`SameFileError` instead of :exc:`Error`.
+      Raise :exc:`SameFileError` instead of :exc:`Error`.  Since the former is
+      a subclass of the latter, this change is backward compatible.
 
 
 .. exception:: SameFileError

Doc/library/subprocess.rst

@@ -341,6 +341,10 @@ default values. The arguments that are most commonly needed are:
       from this vulnerability; see the Note in the :class:`Popen` constructor
       documentation for helpful hints in getting ``shell=False`` to work.
 
+      When using ``shell=True``, :func:`shlex.quote` can be used to properly
+      escape whitespace and shell metacharacters in strings that are going to
+      be used to construct shell commands.
+
 These options, along with all of the other options, are described in more
 detail in the :class:`Popen` constructor documentation.
 

Doc/tools/sphinxext/static/version_switch.js

@@ -43,19 +43,21 @@
 
     if (new_url != url) {
       // check beforehand if url exists, else redirect to version's start page
-      $.get(new_url, function() {
-        window.location.href = new_url;
-      }).error(function() {
-        window.location.href = 'http://docs.python.org/' + selected;
+      $.ajax({
+        url: new_url,
+        success: function() {
+           window.location.href = new_url;
+        },
+        error: function() {
+           window.location.href = 'http://docs.python.org/' + selected;
+        }
       });
     }
   }
 
   $(document).ready(function() {
-    var version = DOCUMENTATION_OPTIONS.VERSION.split('.'),
-        release = DOCUMENTATION_OPTIONS.RELEASE || DOCUMENTATION_OPTIONS.VERSION;
-
-    version = version[0] + '.' + version[1];
+    var release = DOCUMENTATION_OPTIONS.VERSION;
+    var version = release.substr(0, 3);
     var select = build_select(version, release);
 
     $('.version_switcher_placeholder').html(select);

Doc/whatsnew/2.6.rst

@@ -7,8 +7,6 @@
 .. XXX add trademark info for Apple, Microsoft, SourceForge.
 
 :Author: A.M. Kuchling (amk at amk.ca)
-:Release: |release|
-:Date: |today|
 
 .. $Id$
    Rules for maintenance:

Doc/whatsnew/2.7.rst

@@ -3,8 +3,6 @@
 ****************************
 
 :Author: A.M. Kuchling (amk at amk.ca)
-:Release: |release|
-:Date: |today|
 
 ..  hyperlink all the methods & functions.
 

Doc/whatsnew/3.0.rst

@@ -5,8 +5,6 @@
 .. XXX Add trademark info for Apple, Microsoft.
 
 :Author: Guido van Rossum
-:Release: |release|
-:Date: |today|
 
 .. $Id$
    Rules for maintenance:

Doc/whatsnew/3.1.rst

@@ -3,8 +3,6 @@
 ****************************
 
 :Author: Raymond Hettinger
-:Release: |release|
-:Date: |today|
 
 .. $Id$
    Rules for maintenance:

Doc/whatsnew/3.2.rst

@@ -3,8 +3,6 @@
 ****************************
 
 :Author: Raymond Hettinger
-:Release: |release|
-:Date: |today|
 
 .. $Id$
    Rules for maintenance:

Doc/whatsnew/3.3.rst

@@ -2,9 +2,6 @@
   What's New In Python 3.3
 ****************************
 
-:Release: |release|
-:Date: |today|
-
 .. Rules for maintenance:
 
    * Anyone can add text to this document.  Do not spend very much time

Doc/whatsnew/3.4.rst

@@ -2,8 +2,8 @@
   What's New In Python 3.4
 ****************************
 
-:Release: |release|
-:Date: |today|
+.. :Author: Someone <email>
+   (uncomment if there is a principal author)
 
 .. Rules for maintenance:
 

Include/pyport.h

@@ -866,7 +866,7 @@ extern pid_t forkpty(int *, char *, struct termios *, struct winsize *);
 #endif
 
 /*
- * Convenient macros to deal with endianess of the platform. WORDS_BIGENDIAN is
+ * Convenient macros to deal with endianness of the platform. WORDS_BIGENDIAN is
  * detected by configure and defined in pyconfig.h. The code in pyconfig.h
  * also also takes care of Apple's universal builds.
  */

Lib/json/__init__.py

@@ -122,7 +122,7 @@ _default_encoder = JSONEncoder(
 
 def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
         allow_nan=True, cls=None, indent=None, separators=None,
-        default=None, **kw):
+        default=None, sort_keys=False, **kw):
     """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a
     ``.write()``-supporting file-like object).
 
@@ -155,6 +155,9 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
     ``default(obj)`` is a function that should return a serializable version
     of obj or raise TypeError. The default simply raises TypeError.
 
+    If *sort_keys* is ``True`` (default: ``False``), then the output of
+    dictionaries will be sorted by key.
+
     To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
     ``.default()`` method to serialize additional types), specify it with
     the ``cls`` kwarg; otherwise ``JSONEncoder`` is used.
@@ -164,7 +167,7 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
     if (not skipkeys and ensure_ascii and
         check_circular and allow_nan and
         cls is None and indent is None and separators is None and
-        default is None and not kw):
+        default is None and not sort_keys and not kw):
         iterable = _default_encoder.iterencode(obj)
     else:
         if cls is None:
@@ -172,7 +175,7 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
         iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii,
             check_circular=check_circular, allow_nan=allow_nan, indent=indent,
             separators=separators,
-            default=default, **kw).iterencode(obj)
+            default=default, sort_keys=sort_keys, **kw).iterencode(obj)
     # could accelerate with writelines in some versions of Python, at
     # a debuggability cost
     for chunk in iterable:
@@ -181,7 +184,7 @@ def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
 
 def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
         allow_nan=True, cls=None, indent=None, separators=None,
-        default=None, **kw):
+        default=None, sort_keys=False, **kw):
     """Serialize ``obj`` to a JSON formatted ``str``.
 
     If ``skipkeys`` is false then ``dict`` keys that are not basic types
@@ -213,6 +216,9 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
     ``default(obj)`` is a function that should return a serializable version
     of obj or raise TypeError. The default simply raises TypeError.
 
+    If *sort_keys* is ``True`` (default: ``False``), then the output of
+    dictionaries will be sorted by key.
+
     To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
     ``.default()`` method to serialize additional types), specify it with
     the ``cls`` kwarg; otherwise ``JSONEncoder`` is used.
@@ -222,14 +228,14 @@ def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
     if (not skipkeys and ensure_ascii and
         check_circular and allow_nan and
         cls is None and indent is None and separators is None and
-        default is None and not kw):
+        default is None and not sort_keys and not kw):
         return _default_encoder.encode(obj)
     if cls is None:
         cls = JSONEncoder
     return cls(
         skipkeys=skipkeys, ensure_ascii=ensure_ascii,
         check_circular=check_circular, allow_nan=allow_nan, indent=indent,
-        separators=separators, default=default,
+        separators=separators, default=default, sort_keys=sort_keys,
         **kw).encode(obj)
 
 

Lib/mimetypes.py

@@ -378,12 +378,14 @@ def _default_mime_types():
         '.taz': '.tar.gz',
         '.tz': '.tar.gz',
         '.tbz2': '.tar.bz2',
+        '.txz': '.tar.xz',
         }
 
     encodings_map = {
         '.gz': 'gzip',
         '.Z': 'compress',
         '.bz2': 'bzip2',
+        '.xz': 'xz',
         }
 
     # Before adding new types, make sure they are either registered with IANA,

Lib/test/test_bz2.py

@@ -640,6 +640,7 @@ class BZ2DecompressorTest(BaseTest):
         bz2d = BZ2Decompressor()
         text = bz2d.decompress(self.DATA)
         self.assertRaises(EOFError, bz2d.decompress, b"anything")
+        self.assertRaises(EOFError, bz2d.decompress, b"")
 
     @bigmemtest(size=_4G + 100, memuse=3)
     def testDecompress4G(self, size):

Lib/test/test_mimetypes.py

@@ -22,6 +22,8 @@ class MimeTypesTestCase(unittest.TestCase):
         eq(self.db.guess_type("foo.tgz"), ("application/x-tar", "gzip"))
         eq(self.db.guess_type("foo.tar.gz"), ("application/x-tar", "gzip"))
         eq(self.db.guess_type("foo.tar.Z"), ("application/x-tar", "compress"))
+        eq(self.db.guess_type("foo.tar.bz2"), ("application/x-tar", "bzip2"))
+        eq(self.db.guess_type("foo.tar.xz"), ("application/x-tar", "xz"))
 
     def test_data_urls(self):
         eq = self.assertEqual

Lib/test/test_shutil.py

@@ -1223,6 +1223,8 @@ class TestShutil(unittest.TestCase):
         src_file = os.path.join(src_dir, 'foo')
         write_file(src_file, 'foo')
         self.assertRaises(SameFileError, shutil.copyfile, src_file, src_file)
+        # But Error should work too, to stay backward compatible.
+        self.assertRaises(Error, shutil.copyfile, src_file, src_file)
 
     def test_copytree_return_value(self):
         # copytree returns its destination path.

Lib/test/test_unicode.py

@@ -981,6 +981,21 @@ class UnicodeTest(string_tests.CommonTest,
         self.assertRaises(ValueError, '{}'.format_map, 'a')
         self.assertRaises(ValueError, '{a} {}'.format_map, {"a" : 2, "b" : 1})
 
+    def test_format_huge_precision(self):
+        format_string = ".{}f".format(sys.maxsize + 1)
+        with self.assertRaises(ValueError):
+            result = format(2.34, format_string)
+
+    def test_format_huge_width(self):
+        format_string = "{}f".format(sys.maxsize + 1)
+        with self.assertRaises(ValueError):
+            result = format(2.34, format_string)
+
+    def test_format_huge_item_number(self):
+        format_string = "{{{}:.6f}}".format(sys.maxsize + 1)
+        with self.assertRaises(ValueError):
+            result = format_string.format(2.34)
+
     def test_format_auto_numbering(self):
         class C:
             def __init__(self, x=100):
@@ -1069,6 +1084,18 @@ class UnicodeTest(string_tests.CommonTest,
         self.assertEqual('%.1s' % "a\xe9\u20ac", 'a')
         self.assertEqual('%.2s' % "a\xe9\u20ac", 'a\xe9')
 
+    @support.cpython_only
+    def test_formatting_huge_precision(self):
+        from _testcapi import INT_MAX
+        format_string = "%.{}f".format(INT_MAX + 1)
+        with self.assertRaises(ValueError):
+            result = format_string % 2.34
+
+    def test_formatting_huge_width(self):
+        format_string = "%{}f".format(sys.maxsize + 1)
+        with self.assertRaises(ValueError):
+            result = format_string % 2.34
+
     def test_startswith_endswith_errors(self):
         for meth in ('foo'.startswith, 'foo'.endswith):
             with self.assertRaises(TypeError) as cm:

Lib/test/test_urllib2net.py

@@ -157,12 +157,12 @@ class OtherNetworkTests(unittest.TestCase):
 ##             self._test_urls(urls, self._extra_handlers()+[bauth, dauth])
 
     def test_urlwithfrag(self):
-        urlwith_frag = "http://docs.python.org/glossary.html#glossary"
+        urlwith_frag = "http://docs.python.org/2/glossary.html#glossary"
         with support.transient_internet(urlwith_frag):
             req = urllib.request.Request(urlwith_frag)
             res = urllib.request.urlopen(req)
             self.assertEqual(res.geturl(),
-                    "http://docs.python.org/glossary.html#glossary")
+                    "http://docs.python.org/2/glossary.html#glossary")
 
     def test_custom_headers(self):
         url = "http://www.example.com"

Lib/venv/__init__.py

@@ -313,11 +313,17 @@ class EnvBuilder:
                     mode = 'wb'
                 else:
                     mode = 'w'
-                    data = data.decode('utf-8')
-                    data = self.replace_variables(data, context)
-                with open(dstfile, mode) as f:
-                    f.write(data)
-                shutil.copymode(srcfile, dstfile)
+                    try:
+                        data = data.decode('utf-8')
+                        data = self.replace_variables(data, context)
+                    except UnicodeDecodeError as e:
+                        data = None
+                        logger.warning('unable to copy script %r, '
+                                       'may be binary: %s', srcfile, e)
+                if data is not None:
+                    with open(dstfile, mode) as f:
+                        f.write(data)
+                    shutil.copymode(srcfile, dstfile)
 
 
 def create(env_dir, system_site_packages=False, clear=False, symlinks=False):

Misc/NEWS

@@ -20,7 +20,7 @@ Core and Builtins
   user that created them, even when the source file is read-only.
 
 - Issue #16166: Add PY_LITTLE_ENDIAN and PY_BIG_ENDIAN macros and unified
-  endianess detection and handling.
+  endianness detection and handling.
 
 - Issue #15958: bytes.join and bytearray.join now accept arbitrary buffer
   objects.
@@ -62,6 +62,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #16316: mimetypes now recognizes the .xz and .txz (.tar.xz) extensions.
+  Patch by Serhiy Storchaka.
+
 - Issue #12890: cgitb no longer prints spurious <p> tags in text
   mode when the logdir option is specified.
 

Tools/msi/msi.py

@@ -416,7 +416,7 @@ def add_ui(db):
               ("VerdanaRed9", "Verdana", 9, 255, 0),
              ])
 
-    compileargs = r'-Wi "[TARGETDIR]Lib\compileall.py" -f -x "bad_coding|badsyntax|site-packages|py2_|lib2to3\\tests" "[TARGETDIR]Lib"'
+    compileargs = r'-Wi "[TARGETDIR]Lib\compileall.py" -f -x "bad_coding|badsyntax|site-packages|py2_|lib2to3\\tests|venv\\scripts" "[TARGETDIR]Lib"'
     lib2to3args = r'-c "import lib2to3.pygram, lib2to3.patcomp;lib2to3.patcomp.PatternCompiler()"'
     # See "CustomAction Table"
     add_data(db, "CustomAction", [