ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626

Include/parsetok.h

@@ -29,6 +29,8 @@ typedef struct {
 #define PyPARSE_UNICODE_LITERALS        0x0008
 #endif
 
+#define PyPARSE_IGNORE_COOKIE 0x0010
+
 PyAPI_FUNC(node *) PyParser_ParseString(const char *, grammar *, int,
                                               perrdetail *);
 PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int,

Include/pythonrun.h

@@ -12,6 +12,7 @@ extern "C" {
 #define PyCF_SOURCE_IS_UTF8  0x0100
 #define PyCF_DONT_IMPLY_DEDENT 0x0200
 #define PyCF_ONLY_AST 0x0400
+#define PyCF_IGNORE_COOKIE 0x0800
 
 typedef struct {
 	int cf_flags;  /* bitmask of CO_xxx flags relevant to future */

Lib/test/test_coding.py

@@ -17,10 +17,10 @@ class CodingTest(unittest.TestCase):
 
         path = os.path.dirname(__file__)
         filename = os.path.join(path, module_name + '.py')
-        fp = open(filename, encoding='utf-8')
+        fp = open(filename, "rb")
-        text = fp.read()
+        bytes = fp.read()
         fp.close()
-        self.assertRaises(SyntaxError, compile, text, filename, 'exec')
+        self.assertRaises(SyntaxError, compile, bytes, filename, 'exec')
 
     def test_exec_valid_coding(self):
         d = {}

Lib/test/test_pep263.py

@@ -30,6 +30,12 @@ class PEP263Test(unittest.TestCase):
         else:
             self.fail()
 
+    def test_issue4626(self):
+        c = compile("# coding=latin-1\n\u00c6 = '\u00c6'", "dummy", "exec")
+        d = {}
+        exec(c, d)
+        self.assertEquals(d['\xc6'], '\xc6')
+
 def test_main():
     support.run_unittest(PEP263Test)
 

Misc/NEWS

@@ -19,6 +19,9 @@ Core and Builtins
 - Issue #5249: time.strftime returned malformed string when format string
   contained non ascii character on windows.
 
+- Issue #4626: compile(), exec(), and eval() ignore the coding cookie if the
+  source has already been decoded into str.
+
 - Issue #5186: Reduce hash collisions for objects with no __hash__ method by
   rotating the object pointer by 4 bits to the right.
 

Parser/parsetok.c

@@ -49,7 +49,11 @@ PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename,
 
 	initerr(err_ret, filename);
 
-	if ((tok = PyTokenizer_FromString(s)) == NULL) {
+	if (*flags & PyPARSE_IGNORE_COOKIE)
+		tok = PyTokenizer_FromUTF8(s);
+	else
+		tok = PyTokenizer_FromString(s);
+	if (tok == NULL) {
 		err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;
 		return NULL;
 	}

Parser/tokenizer.c

@@ -715,6 +715,28 @@ PyTokenizer_FromString(const char *str)
 	return tok;
 }
 
+struct tok_state *
+PyTokenizer_FromUTF8(const char *str)
+{
+	struct tok_state *tok = tok_new();
+	if (tok == NULL)
+		return NULL;
+	tok->decoding_state = STATE_RAW;
+	tok->read_coding_spec = 1;
+	tok->enc = NULL;
+	tok->str = str;
+	tok->encoding = (char *)PyMem_MALLOC(6);
+	if (!tok->encoding) {
+		PyTokenizer_Free(tok);
+		return NULL;
+	}
+	strcpy(tok->encoding, "utf-8");
+
+	/* XXX: constify members. */
+	tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
+	return tok;
+}
+
 
 /* Set up tokenizer for file */
 

Parser/tokenizer.h

@@ -61,6 +61,7 @@ struct tok_state {
 };
 
 extern struct tok_state *PyTokenizer_FromString(const char *);
+extern struct tok_state *PyTokenizer_FromUTF8(const char *);
 extern struct tok_state *PyTokenizer_FromFile(FILE *, char*,
 					      char *, char *);
 extern void PyTokenizer_Free(struct tok_state *);

Python/bltinmodule.c

@@ -494,12 +494,13 @@ PyDoc_STR(
 
 
 static char *
-source_as_string(PyObject *cmd, char *funcname, char *what)
+source_as_string(PyObject *cmd, char *funcname, char *what, PyCompilerFlags *cf)
 {
 	char *str;
 	Py_ssize_t size;
 
 	if (PyUnicode_Check(cmd)) {
+		cf->cf_flags |= PyCF_IGNORE_COOKIE;
 		cmd = _PyUnicode_AsDefaultEncodedString(cmd, NULL);
 		if (cmd == NULL)
 			return NULL;
@@ -591,7 +592,7 @@ builtin_compile(PyObject *self, PyObject *args, PyObject *kwds)
 		return result;
 	}
 
-	str = source_as_string(cmd, "compile", "string, bytes, AST or code");
+	str = source_as_string(cmd, "compile", "string, bytes, AST or code", &cf);
 	if (str == NULL)
 		return NULL;
 
@@ -703,14 +704,14 @@ builtin_eval(PyObject *self, PyObject *args)
 		return PyEval_EvalCode((PyCodeObject *) cmd, globals, locals);
 	}
 
-	str = source_as_string(cmd, "eval", "string, bytes or code");
+	cf.cf_flags = PyCF_SOURCE_IS_UTF8;
+	str = source_as_string(cmd, "eval", "string, bytes or code", &cf);
 	if (str == NULL)
 		return NULL;
 
 	while (*str == ' ' || *str == '\t')
 		str++;
 
-	cf.cf_flags = PyCF_SOURCE_IS_UTF8;
 	(void)PyEval_MergeCompilerFlags(&cf);
 	result = PyRun_StringFlags(str, Py_eval_input, globals, locals, &cf);
 	Py_XDECREF(tmp);
@@ -779,12 +780,13 @@ builtin_exec(PyObject *self, PyObject *args)
 		v = PyEval_EvalCode((PyCodeObject *) prog, globals, locals);
 	}
 	else {
-		char *str = source_as_string(prog, "exec",
+		char *str;
-					     "string, bytes or code");
 		PyCompilerFlags cf;
+		cf.cf_flags = PyCF_SOURCE_IS_UTF8;
+		str = source_as_string(prog, "exec",
+					     "string, bytes or code", &cf);
 		if (str == NULL)
 			return NULL;
-		cf.cf_flags = PyCF_SOURCE_IS_UTF8;
 		if (PyEval_MergeCompilerFlags(&cf))
 			v = PyRun_StringFlags(str, Py_file_input, globals,
 					      locals, &cf);

Python/pythonrun.c

@@ -1002,9 +1002,17 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flag
 }
 
 /* compute parser flags based on compiler flags */
-#define PARSER_FLAGS(flags) \
+static int PARSER_FLAGS(PyCompilerFlags *flags)
-	((flags) ? ((((flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? \
+{
-		      PyPARSE_DONT_IMPLY_DEDENT : 0)) : 0)
+	int parser_flags = 0;
+	if (!flags)
+		return 0;
+	if (flags->cf_flags & PyCF_DONT_IMPLY_DEDENT)
+		parser_flags |= PyPARSE_DONT_IMPLY_DEDENT;
+	if (flags->cf_flags & PyCF_IGNORE_COOKIE)
+		parser_flags |= PyPARSE_IGNORE_COOKIE;
+	return parser_flags;
+}
 
 #if 0
 /* Keep an example of flags with future keyword support. */