Bug #1473625: stop cPickle making float dumps locale dependent in protocol 0.

On the way, add a decorator to test_support to facilitate running single
test functions in different locales with automatic cleanup.

Lib/test/pickletester.py

@@ -4,7 +4,8 @@ import cPickle
 import pickletools
 import copy_reg
 
-from test.test_support import TestFailed, have_unicode, TESTFN
+from test.test_support import TestFailed, have_unicode, TESTFN, \
+                              run_with_locale
 
 # Tests that try a number of pickle protocols should have a
 #     for proto in protocols:
@@ -527,6 +528,11 @@ class AbstractPickleTests(unittest.TestCase):
             got = self.loads(p)
             self.assertEqual(n, got)
 
+    @run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
+    def test_float_format(self):
+        # make sure that floats are formatted locale independent
+        self.assertEqual(self.dumps(1.2)[0:3], 'F1.')
+
     def test_reduce(self):
         pass
 

Lib/test/test_builtin.py

@@ -1,7 +1,8 @@
 # Python test set -- built-in functions
 
 import test.test_support, unittest
-from test.test_support import fcmp, have_unicode, TESTFN, unlink, run_unittest
+from test.test_support import fcmp, have_unicode, TESTFN, unlink, \
+                              run_unittest, run_with_locale
 from operator import neg
 
 import sys, warnings, cStringIO, random, UserDict
@@ -554,33 +555,20 @@ class BuiltinTest(unittest.TestCase):
             # Implementation limitation in PyFloat_FromString()
             self.assertRaises(ValueError, float, unicode("1"*10000))
 
+    @run_with_locale('LC_NUMERIC', 'fr_FR', 'de_DE')
     def test_float_with_comma(self):
         # set locale to something that doesn't use '.' for the decimal point
-        try:
+        import locale
-            import locale
+        if not locale.localeconv()['decimal_point'] == ',':
-            orig_locale = locale.setlocale(locale.LC_NUMERIC)
-            locale.setlocale(locale.LC_NUMERIC, 'fr_FR')
-        except:
-            # if we can't set the locale, just ignore this test
             return
 
-        try:
+        self.assertEqual(float("  3,14  "), 3.14)
-            self.assertEqual(locale.localeconv()['decimal_point'], ',')
+        self.assertEqual(float("  +3,14  "), 3.14)
-        except:
+        self.assertEqual(float("  -3,14  "), -3.14)
-            # this test is worthless, just skip it and reset the locale
+        self.assertRaises(ValueError, float, "  0x3.1  ")
-            locale.setlocale(locale.LC_NUMERIC, orig_locale)
+        self.assertRaises(ValueError, float, "  -0x3.p-1  ")
-            return
+        self.assertEqual(float("  25.e-1  "), 2.5)
-
+        self.assertEqual(fcmp(float("  .25e-1  "), .025), 0)
-        try:
-            self.assertEqual(float("  3,14  "), 3.14)
-            self.assertEqual(float("  +3,14  "), 3.14)
-            self.assertEqual(float("  -3,14  "), -3.14)
-            self.assertRaises(ValueError, float, "  0x3.1  ")
-            self.assertRaises(ValueError, float, "  -0x3.p-1  ")
-            self.assertEqual(float("  25.e-1  "), 2.5)
-            self.assertEqual(fcmp(float("  .25e-1  "), .025), 0)
-        finally:
-            locale.setlocale(locale.LC_NUMERIC, orig_locale)
 
     def test_floatconversion(self):
         # Make sure that calls to __float__() work properly

Lib/test/test_logging.py

@@ -28,6 +28,7 @@ import select
 import os, sys, string, struct, types, cPickle, cStringIO
 import socket, tempfile, threading, time
 import logging, logging.handlers, logging.config
+from test.test_support import run_with_locale
 
 BANNER = "-- %-10s %-6s ---------------------------------------------------\n"
 
@@ -657,19 +658,11 @@ def test_main_inner():
             pass
         rootLogger.removeHandler(hdlr)
 
+# Set the locale to the platform-dependent default.  I have no idea
+# why the test does this, but in any case we save the current locale
+# first and restore it at the end.
+@run_with_locale('LC_ALL', '')
 def test_main():
-    import locale
-    # Set the locale to the platform-dependent default.  I have no idea
-    # why the test does this, but in any case we save the current locale
-    # first so we can restore it at the end.
-    try:
-        original_locale = locale.setlocale(locale.LC_ALL)
-        locale.setlocale(locale.LC_ALL, '')
-    except (ValueError, locale.Error):
-        # this happens on a Solaris box which only supports "C" locale
-        # or a Mac OS X box which supports very little locale stuff at all
-        original_locale = None
-
     # Save and restore the original root logger level across the tests.
     # Otherwise, e.g., if any test using cookielib runs after test_logging,
     # cookielib's debug-level logger tries to log messages, leading to
@@ -681,8 +674,6 @@ def test_main():
     try:
         test_main_inner()
     finally:
-        if original_locale is not None:
-            locale.setlocale(locale.LC_ALL, original_locale)
         root_logger.setLevel(original_logging_level)
 
 if __name__ == "__main__":

Lib/test/test_support.py

@@ -251,6 +251,42 @@ def open_urlresource(url):
     fn, _ = urllib.urlretrieve(url, filename)
     return open(fn)
 
+#=======================================================================
+# Decorator for running a function in a different locale, correctly resetting
+# it afterwards.
+
+def run_with_locale(catstr, *locales):
+    def decorator(func):
+        def inner(*args, **kwds):
+            try:
+                import locale
+                category = getattr(locale, catstr)
+                orig_locale = locale.setlocale(category)
+            except AttributeError:
+                # if the test author gives us an invalid category string
+                raise
+            except:
+                # cannot retrieve original locale, so do nothing
+                locale = orig_locale = None
+            else:
+                for loc in locales:
+                    try:
+                        locale.setlocale(category, loc)
+                        break
+                    except:
+                        pass
+
+            # now run the function, resetting the locale on exceptions
+            try:
+                return func(*args, **kwds)
+            finally:
+                if locale and orig_locale:
+                    locale.setlocale(category, orig_locale)
+        inner.func_name = func.func_name
+        inner.__doc__ = func.__doc__
+        return inner
+    return decorator
+
 #=======================================================================
 # Big-memory-test support. Separate from 'resources' because memory use should be configurable.
 

Lib/test/test_unicode.py

@@ -410,20 +410,11 @@ class UnicodeTest(
             def __str__(self):
                 return u'\u1234'
         self.assertEqual('%s' % Wrapper(), u'\u1234')
-
+    
+    @test_support.run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
     def test_format_float(self):
-        try:
+        # should not format with a comma, but always with C locale
-            import locale
+        self.assertEqual(u'1.0', u'%.1f' % 1.0)
-            orig_locale = locale.setlocale(locale.LC_ALL)
-            locale.setlocale(locale.LC_ALL, 'de_DE')
-        except (ImportError, locale.Error):
-            return # skip if we can't set locale
-
-        try:
-            # should not format with a comma, but always with C locale
-            self.assertEqual(u'1.0', u'%.1f' % 1.0)
-        finally:
-            locale.setlocale(locale.LC_ALL, orig_locale)
 
     def test_constructor(self):
         # unicode(obj) tests (this maps to PyObject_Unicode() at C level)

Modules/cPickle.c

@@ -1151,7 +1151,9 @@ save_float(Picklerobject *self, PyObject *args)
 	else {
 		char c_str[250];
 		c_str[0] = FLOAT;
-		PyOS_snprintf(c_str + 1, sizeof(c_str) - 1, "%.17g\n", x);
+		PyOS_ascii_formatd(c_str + 1, sizeof(c_str) - 2, "%.17g", x);
+		/* Extend the formatted string with a newline character */
+		strcat(c_str, "\n");
 
 		if (self->write_func(self, c_str, strlen(c_str)) < 0)
 			return -1;