Add functions PyUnicode_Append() and PyUnicode_AppendAndDel() that mirror

PyString_Concat() and PyString_ConcatAndDel() (the name PyUnicode_Concat()
was already taken).

Change PyObject_Repr() to always return a unicode object.

Update all repr implementations to return unicode objects.

Add a function PyObject_ReprStr8() that calls PyObject_Repr() and converts
the result to an 8bit string.

Use PyObject_ReprStr8() where using PyObject_Repr() can't be done
straightforward.

Modules/_codecsmodule.c

@@ -161,27 +161,63 @@ static PyObject *
 escape_encode(PyObject *self,
 	      PyObject *args)
 {
+	static const char *hexdigits = "0123456789abcdef";
 	PyObject *str;
+	Py_ssize_t size;
+	Py_ssize_t newsize;
 	const char *errors = NULL;
-	char *buf;
-	Py_ssize_t len;
+	PyObject *v;
 
 	if (!PyArg_ParseTuple(args, "O!|z:escape_encode",
 			      &PyString_Type, &str, &errors))
 		return NULL;
 
-	str = PyString_Repr(str, 0);
-	if (!str)
-		return NULL;
+	size = PyUnicode_GET_SIZE(str);
+	newsize = 4*size;
+	if (newsize > PY_SSIZE_T_MAX || newsize / 4 != size) {
+		PyErr_SetString(PyExc_OverflowError,
+			"string is too large to encode");
+			return NULL;
+	}
+	v = PyBytes_FromStringAndSize(NULL, newsize);
 
-	/* The string will be quoted. Unquote, similar to unicode-escape. */
-	buf = PyString_AS_STRING (str);
-	len = PyString_GET_SIZE (str);
-	memmove(buf, buf+1, len-2);
-	if (_PyString_Resize(&str, len-2) < 0)
+	if (v == NULL) {
 		return NULL;
+	}
+	else {
+		register Py_ssize_t i;
+		register char c;
+		register char *p = PyBytes_AS_STRING(v);
+
+		for (i = 0; i < size; i++) {
+			/* There's at least enough room for a hex escape */
+			assert(newsize - (p - PyBytes_AS_STRING(v)) >= 4);
+			c = PyString_AS_STRING(str)[i];
+			if (c == '\'' || c == '\\')
+				*p++ = '\\', *p++ = c;
+			else if (c == '\t')
+				*p++ = '\\', *p++ = 't';
+			else if (c == '\n')
+				*p++ = '\\', *p++ = 'n';
+			else if (c == '\r')
+				*p++ = '\\', *p++ = 'r';
+			else if (c < ' ' || c >= 0x7f) {
+				*p++ = '\\';
+				*p++ = 'x';
+				*p++ = hexdigits[(c & 0xf0) >> 4];
+				*p++ = hexdigits[c & 0xf];
+			}
+			else
+				*p++ = c;
+		}
+		*p = '\0';
+		if (PyBytes_Resize(v, (p - PyBytes_AS_STRING(v)))) {
+			Py_DECREF(v);
+			return NULL;
+		}
+	}
 	
-	return codec_tuple(str, PyString_Size(str));
+	return codec_tuple(v, PyBytes_Size(v));
 }
 
 /* --- Decoder ------------------------------------------------------------ */

Modules/_collectionsmodule.c

@@ -611,14 +611,14 @@ PyDoc_STRVAR(reduce_doc, "Return state information for pickling.");
 static PyObject *
 deque_repr(PyObject *deque)
 {
-	PyObject *aslist, *result, *fmt;
+	PyObject *aslist, *result;
 	int i;
 
 	i = Py_ReprEnter(deque);
 	if (i != 0) {
 		if (i < 0)
 			return NULL;
-		return PyString_FromString("[...]");
+		return PyUnicode_FromString("[...]");
 	}
 
 	aslist = PySequence_List(deque);
@@ -627,14 +627,14 @@ deque_repr(PyObject *deque)
 		return NULL;
 	}
 
-	fmt = PyString_FromString("deque(%r)");
-	if (fmt == NULL) {
+	result = PyUnicode_FromString("deque(");
+	if (result == NULL) {
 		Py_DECREF(aslist);
 		Py_ReprLeave(deque);
 		return NULL;
 	}
-	result = PyString_Format(fmt, aslist);
-	Py_DECREF(fmt);
+	PyUnicode_AppendAndDel(&result, PyObject_Repr(aslist));
+	PyUnicode_AppendAndDel(&result, PyUnicode_FromString(")"));
 	Py_DECREF(aslist);
 	Py_ReprLeave(deque);
 	return result;
@@ -1215,18 +1215,18 @@ defdict_repr(defdictobject *dd)
 	if (baserepr == NULL)
 		return NULL;
 	if (dd->default_factory == NULL)
-		defrepr = PyString_FromString("None");
+		defrepr = PyUnicode_FromString("None");
 	else
 		defrepr = PyObject_Repr(dd->default_factory);
 	if (defrepr == NULL) {
 		Py_DECREF(baserepr);
 		return NULL;
 	}
-	result = PyString_FromFormat("defaultdict(%s, %s)",
-				     PyString_AS_STRING(defrepr),
-				     PyString_AS_STRING(baserepr));
-	Py_DECREF(defrepr);
-	Py_DECREF(baserepr);
+	result = PyUnicode_FromString("defaultdict(");
+	PyUnicode_AppendAndDel(&result, defrepr);
+	PyUnicode_AppendAndDel(&result, PyUnicode_FromString(", "));
+	PyUnicode_AppendAndDel(&result, baserepr);
+	PyUnicode_AppendAndDel(&result, PyUnicode_FromString(")"));
 	return result;
 }
 

Modules/_ctypes/_ctypes.c

@@ -3446,12 +3446,12 @@ CFuncPtr_repr(CFuncPtrObject *self)
 {
 #ifdef MS_WIN32
 	if (self->index)
-		return PyString_FromFormat("<COM method offset %d: %s at %p>",
+		return PyUnicode_FromFormat("<COM method offset %d: %s at %p>",
 					   self->index - 0x1000,
 					   self->ob_type->tp_name,
 					   self);
 #endif
-	return PyString_FromFormat("<%s object at %p>",
+	return PyUnicode_FromFormat("<%s object at %p>",
 				   self->ob_type->tp_name,
 				   self);
 }
@@ -4081,12 +4081,12 @@ Simple_repr(CDataObject *self)
 	static PyObject *format;
 
 	if (self->ob_type->tp_base != &Simple_Type) {
-		return PyString_FromFormat("<%s object at %p>",
+		return PyUnicode_FromFormat("<%s object at %p>",
 					   self->ob_type->tp_name, self);
 	}
 
 	if (format == NULL) {
-		format = PyString_FromString("%s(%r)");
+		format = PyUnicode_FromString("%s(%r)");
 		if (format == NULL)
 			return NULL;
 	}
@@ -4095,7 +4095,7 @@ Simple_repr(CDataObject *self)
 	if (val == NULL)
 		return NULL;
 
-	name = PyString_FromString(self->ob_type->tp_name);
+	name = PyUnicode_FromString(self->ob_type->tp_name);
 	if (name == NULL) {
 		Py_DECREF(val);
 		return NULL;
@@ -4107,7 +4107,7 @@ Simple_repr(CDataObject *self)
 	if (args == NULL)
 		return NULL;
 
-	result = PyString_Format(format, args);
+	result = PyUnicode_Format(format, args);
 	Py_DECREF(args);
 	return result;
 }

Modules/_ctypes/callproc.c

@@ -370,7 +370,7 @@ PyCArg_repr(PyCArgObject *self)
 			self->tag, (long)self);
 		break;
 	}
-	return PyString_FromString(buffer);
+	return PyUnicode_FromString(buffer);
 }
 
 static PyMemberDef PyCArgType_members[] = {

Modules/_ctypes/cfield.c

@@ -275,7 +275,7 @@ CField_repr(CFieldObject *self)
 	name = ((PyTypeObject *)self->proto)->tp_name;
 
 	if (bits)
-		result = PyString_FromFormat(
+		result = PyUnicode_FromFormat(
 #if (PY_VERSION_HEX < 0x02050000)
 			"<Field type=%s, ofs=%d:%d, bits=%d>",
 #else
@@ -283,7 +283,7 @@ CField_repr(CFieldObject *self)
 #endif
 			name, self->offset, size, bits);
 	else
-		result = PyString_FromFormat(
+		result = PyUnicode_FromFormat(
 #if (PY_VERSION_HEX < 0x02050000)
 			"<Field type=%s, ofs=%d, size=%d>",
 #else

Modules/_elementtree.c

@@ -1121,12 +1121,12 @@ element_repr(ElementObject* self)
     PyObject* repr;
     char buffer[100];
     
-    repr = PyString_FromString("<Element ");
+    repr = PyUnicode_FromString("<Element ");
 
-    PyString_ConcatAndDel(&repr, PyObject_Repr(self->tag));
+    PyUnicode_AppendAndDel(&repr, PyObject_Repr(self->tag));
 
     sprintf(buffer, " at %p>", self);
-    PyString_ConcatAndDel(&repr, PyString_FromString(buffer));
+    PyUnicode_AppendAndDel(&repr, PyUnicode_FromString(buffer));
 
     return repr;
 }

Modules/_fileio.c

@@ -580,9 +580,9 @@ static PyObject *
 fileio_repr(PyFileIOObject *self)
 {
         if (self->fd < 0)
-		return PyString_FromFormat("_fileio._FileIO(-1)");
+		return PyUnicode_FromFormat("_fileio._FileIO(-1)");
 
-	return PyString_FromFormat("_fileio._FileIO(%d, '%s')",
+	return PyUnicode_FromFormat("_fileio._FileIO(%d, '%s')",
 				   self->fd, mode_string(self));
 }
 

Modules/_hashopenssl.c

@@ -221,7 +221,7 @@ EVP_repr(PyObject *self)
     char buf[100];
     PyOS_snprintf(buf, sizeof(buf), "<%s HASH object @ %p>",
             PyString_AsString(((EVPobject *)self)->name), self);
-    return PyString_FromString(buf);
+    return PyUnicode_FromString(buf);
 }
 
 #if HASH_OBJ_CONSTRUCTOR

Modules/_tkinter.c

@@ -812,7 +812,7 @@ PyTclObject_repr(PyTclObject *self)
 	char buf[50];
 	PyOS_snprintf(buf, 50, "<%s object at %p>",
 		      self->value->typePtr->name, self->value);
-	return PyString_FromString(buf);
+	return PyUnicode_FromString(buf);
 }
 
 static int
@@ -2375,7 +2375,7 @@ Tktt_Repr(PyObject *self)
 
 	PyOS_snprintf(buf, sizeof(buf), "<tktimertoken at %p%s>", v,
 	                v->func == NULL ? ", handler deleted" : "");
-	return PyString_FromString(buf);
+	return PyUnicode_FromString(buf);
 }
 
 static PyObject *

Modules/arraymodule.c

@@ -1574,7 +1574,7 @@ array_repr(arrayobject *a)
 	typecode = a->ob_descr->typecode;
 	if (len == 0) {
 		PyOS_snprintf(buf, sizeof(buf), "array('%c')", typecode);
-		return PyString_FromString(buf);
+		return PyUnicode_FromString(buf);
 	}
 		
 	if (typecode == 'c')
@@ -1587,9 +1587,9 @@ array_repr(arrayobject *a)
 	Py_XDECREF(v);
 
 	PyOS_snprintf(buf, sizeof(buf), "array('%c', ", typecode);
-	s = PyString_FromString(buf);
-	PyString_ConcatAndDel(&s, t);
-	PyString_ConcatAndDel(&s, PyString_FromString(")"));
+	s = PyUnicode_FromString(buf);
+	PyUnicode_AppendAndDel(&s, t);
+	PyUnicode_AppendAndDel(&s, PyUnicode_FromString(")"));
 	return s;
 }
 

Modules/cPickle.c

@@ -1122,7 +1122,7 @@ save_long(Picklerobject *self, PyObject *args)
 	/* proto < 2:  write the repr and newline.  This is quadratic-time
 	 * (in the number of digits), in both directions.
 	 */
-	if (!( repr = PyObject_Repr(args)))
+	if (!( repr = PyObject_ReprStr8(args)))
 		goto finally;
 
 	if ((size = PyString_Size(repr)) < 0)
@@ -1189,7 +1189,7 @@ save_string(Picklerobject *self, PyObject *args, int doput)
 
 		static char string = STRING;
 
-		if (!( repr = PyObject_Repr(args)))
+		if (!( repr = PyObject_ReprStr8(args)))
 			return -1;
 
 		if ((len = PyString_Size(repr)) < 0)

Modules/datetimemodule.c

@@ -1044,27 +1044,27 @@ append_keyword_tzinfo(PyObject *repr, PyObject *tzinfo)
 {
 	PyObject *temp;
 
-	assert(PyString_Check(repr));
+	assert(PyUnicode_Check(repr));
 	assert(tzinfo);
 	if (tzinfo == Py_None)
 		return repr;
 	/* Get rid of the trailing ')'. */
-	assert(PyString_AsString(repr)[PyString_Size(repr)-1] == ')');
-	temp = PyString_FromStringAndSize(PyString_AsString(repr),
-					  PyString_Size(repr) - 1);
+	assert(PyUnicode_AS_UNICODE(repr)[PyUnicode_GET_SIZE(repr)-1] == ')');
+	temp = PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(repr),
+					  PyUnicode_GET_SIZE(repr) - 1);
 	Py_DECREF(repr);
 	if (temp == NULL)
 		return NULL;
 	repr = temp;
 
 	/* Append ", tzinfo=". */
-	PyString_ConcatAndDel(&repr, PyString_FromString(", tzinfo="));
+	PyUnicode_AppendAndDel(&repr, PyUnicode_FromString(", tzinfo="));
 
 	/* Append repr(tzinfo). */
-	PyString_ConcatAndDel(&repr, PyObject_Repr(tzinfo));
+	PyUnicode_AppendAndDel(&repr, PyObject_Repr(tzinfo));
 
 	/* Add a closing paren. */
-	PyString_ConcatAndDel(&repr, PyString_FromString(")"));
+	PyUnicode_AppendAndDel(&repr, PyUnicode_FromString(")"));
 	return repr;
 }
 
@@ -1972,18 +1972,18 @@ static PyObject *
 delta_repr(PyDateTime_Delta *self)
 {
 	if (GET_TD_MICROSECONDS(self) != 0)
-		return PyString_FromFormat("%s(%d, %d, %d)",
+		return PyUnicode_FromFormat("%s(%d, %d, %d)",
 					   self->ob_type->tp_name,
 					   GET_TD_DAYS(self),
 					   GET_TD_SECONDS(self),
 					   GET_TD_MICROSECONDS(self));
 	if (GET_TD_SECONDS(self) != 0)
-		return PyString_FromFormat("%s(%d, %d)",
+		return PyUnicode_FromFormat("%s(%d, %d)",
 					   self->ob_type->tp_name,
 					   GET_TD_DAYS(self),
 					   GET_TD_SECONDS(self));
 
-	return PyString_FromFormat("%s(%d)",
+	return PyUnicode_FromFormat("%s(%d)",
 				   self->ob_type->tp_name,
 				   GET_TD_DAYS(self));
 }
@@ -2410,7 +2410,7 @@ date_repr(PyDateTime_Date *self)
 		      type_name,
 		      GET_YEAR(self), GET_MONTH(self), GET_DAY(self));
 
-	return PyString_FromString(buffer);
+	return PyUnicode_FromString(buffer);
 }
 
 static PyObject *
@@ -3131,7 +3131,7 @@ time_repr(PyDateTime_Time *self)
 	else
 		PyOS_snprintf(buffer, sizeof(buffer),
 			      "%s(%d, %d)", type_name, h, m);
-	result = PyString_FromString(buffer);
+	result = PyUnicode_FromString(buffer);
 	if (result != NULL && HASTZINFO(self))
 		result = append_keyword_tzinfo(result, self->tzinfo);
 	return result;
@@ -4043,7 +4043,7 @@ datetime_repr(PyDateTime_DateTime *self)
 			      GET_YEAR(self), GET_MONTH(self), GET_DAY(self),
 			      DATE_GET_HOUR(self), DATE_GET_MINUTE(self));
 	}
-	baserepr = PyString_FromString(buffer);
+	baserepr = PyUnicode_FromString(buffer);
 	if (baserepr == NULL || ! HASTZINFO(self))
 		return baserepr;
 	return append_keyword_tzinfo(baserepr, self->tzinfo);

Modules/itertoolsmodule.c

@@ -2084,7 +2084,7 @@ count_next(countobject *lz)
 static PyObject *
 count_repr(countobject *lz)
 {
-	return PyString_FromFormat("count(%zd)", lz->cnt);
+	return PyUnicode_FromFormat("count(%zd)", lz->cnt);
 }
 
 PyDoc_STRVAR(count_doc,
@@ -2396,11 +2396,11 @@ repeat_repr(repeatobject *ro)
 		return NULL;
 
 	if (ro->cnt == -1)
-		result = PyString_FromFormat("repeat(%s)",
-			PyString_AS_STRING(objrepr));
+		result = PyUnicode_FromFormat("repeat(%U)",
+			objrepr);
 	else
-		result = PyString_FromFormat("repeat(%s, %zd)",
-			PyString_AS_STRING(objrepr), ro->cnt);
+		result = PyUnicode_FromFormat("repeat(%U, %zd)",
+			objrepr, ro->cnt);
 	Py_DECREF(objrepr);
 	return result;
 }	

Modules/socketmodule.c

@@ -2865,7 +2865,7 @@ sock_repr(PySocketSockObject *s)
 		(long)s->sock_fd, s->sock_family,
 		s->sock_type,
 		s->sock_proto);
-	return PyString_FromString(buf);
+	return PyUnicode_FromString(buf);
 }
 
 

Modules/zipimport.c

@@ -203,7 +203,7 @@ zipimporter_repr(ZipImporter *self)
 		PyOS_snprintf(buf, sizeof(buf),
 			      "<zipimporter object \"%.300s\">",
 			      archive);
-	return PyString_FromString(buf);
+	return PyUnicode_FromString(buf);
 }
 
 /* return fullname.split(".")[-1] */