mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-11-01 18:51:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 21

Implement PEP 393.

Browse source
This commit is contained in:
Martin v. Löwis 2011-09-28 07:41:54 +02:00
parent 48d49497c5
commit d63a3b8beb
102 changed files with 8153 additions and 5431 deletions
Download patch file Download diff file Expand all files Collapse all files

4
Objects/abstract.c
View file

@ -1379,9 +1379,7 @@ PyNumber_Long(PyObject *o)
PyBytes_GET_SIZE(o));
if (PyUnicode_Check(o))
/* The above check is done in PyLong_FromUnicode(). */
return PyLong_FromUnicode(PyUnicode_AS_UNICODE(o),
PyUnicode_GET_SIZE(o),
10);
return PyLong_FromUnicodeObject(o, 10);
if (!PyObject_AsCharBuffer(o, &buffer, &buffer_len))
return long_from_string(buffer, buffer_len);

147
Objects/bytearrayobject.c
View file

@ -854,83 +854,79 @@ bytearray_repr(PyByteArrayObject *self)
const char *quote_prefix = "bytearray(b";
const char *quote_postfix = ")";
Py_ssize_t length = Py_SIZE(self);
/* 14 == strlen(quote_prefix) + 2 + strlen(quote_postfix) */
/* 15 == strlen(quote_prefix) + 2 + strlen(quote_postfix) + 1 */
size_t newsize;
PyObject *v;
if (length > (PY_SSIZE_T_MAX - 14) / 4) {
register Py_ssize_t i;
register char c;
register char *p;
int quote;
char *test, *start;
char *buffer;
if (length > (PY_SSIZE_T_MAX - 15) / 4) {
PyErr_SetString(PyExc_OverflowError,
"bytearray object is too large to make repr");
return NULL;
}
newsize = 14 + 4 * length;
v = PyUnicode_FromUnicode(NULL, newsize);
if (v == NULL) {
newsize = 15 + length * 4;
buffer = PyMem_Malloc(newsize);
if (buffer == NULL) {
PyErr_NoMemory();
return NULL;
}
else {
register Py_ssize_t i;
register Py_UNICODE c;
register Py_UNICODE *p;
int quote;
/* Figure out which quote to use; single is preferred */
quote = '\'';
{
char *test, *start;
start = PyByteArray_AS_STRING(self);
for (test = start; test < start+length; ++test) {
if (*test == '"') {
quote = '\''; /* back to single */
goto decided;
}
else if (*test == '\'')
quote = '"';
}
decided:
;
/* Figure out which quote to use; single is preferred */
quote = '\'';
start = PyByteArray_AS_STRING(self);
for (test = start; test < start+length; ++test) {
if (*test == '"') {
quote = '\''; /* back to single */
break;
}
p = PyUnicode_AS_UNICODE(v);
while (*quote_prefix)
*p++ = *quote_prefix++;
*p++ = quote;
for (i = 0; i < length; i++) {
/* There's at least enough room for a hex escape
and a closing quote. */
assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 5);
c = self->ob_bytes[i];
if (c == '\'' || c == '\\')
*p++ = '\\', *p++ = c;
else if (c == '\t')
*p++ = '\\', *p++ = 't';
else if (c == '\n')
*p++ = '\\', *p++ = 'n';
else if (c == '\r')
*p++ = '\\', *p++ = 'r';
else if (c == 0)
*p++ = '\\', *p++ = 'x', *p++ = '0', *p++ = '0';
else if (c < ' ' || c >= 0x7f) {
*p++ = '\\';
*p++ = 'x';
*p++ = hexdigits[(c & 0xf0) >> 4];
*p++ = hexdigits[c & 0xf];
}
else
*p++ = c;
}
assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 1);
*p++ = quote;
while (*quote_postfix) {
*p++ = *quote_postfix++;
}
*p = '\0';
if (PyUnicode_Resize(&v, (p - PyUnicode_AS_UNICODE(v)))) {
Py_DECREF(v);
return NULL;
}
return v;
else if (*test == '\'')
quote = '"';
}
p = buffer;
while (*quote_prefix)
*p++ = *quote_prefix++;
*p++ = quote;
for (i = 0; i < length; i++) {
/* There's at least enough room for a hex escape
and a closing quote. */
assert(newsize - (p - buffer) >= 5);
c = self->ob_bytes[i];
if (c == '\'' || c == '\\')
*p++ = '\\', *p++ = c;
else if (c == '\t')
*p++ = '\\', *p++ = 't';
else if (c == '\n')
*p++ = '\\', *p++ = 'n';
else if (c == '\r')
*p++ = '\\', *p++ = 'r';
else if (c == 0)
*p++ = '\\', *p++ = 'x', *p++ = '0', *p++ = '0';
else if (c < ' ' || c >= 0x7f) {
*p++ = '\\';
*p++ = 'x';
*p++ = hexdigits[(c & 0xf0) >> 4];
*p++ = hexdigits[c & 0xf];
}
else
*p++ = c;
}
assert(newsize - (p - buffer) >= 1);
*p++ = quote;
while (*quote_postfix) {
*p++ = *quote_postfix++;
}
v = PyUnicode_DecodeASCII(buffer, p - buffer, NULL);
PyMem_Free(buffer);
return v;
}
static PyObject *
@ -1034,6 +1030,8 @@ bytearray_dealloc(PyByteArrayObject *self)
/* -------------------------------------------------------------------- */
/* Methods */
#define FASTSEARCH fastsearch
#define STRINGLIB(F) stringlib_##F
#define STRINGLIB_CHAR char
#define STRINGLIB_LEN PyByteArray_GET_SIZE
#define STRINGLIB_STR PyByteArray_AS_STRING
@ -2651,15 +2649,20 @@ bytearray_fromhex(PyObject *cls, PyObject *args)
{
PyObject *newbytes, *hexobj;
char *buf;
Py_UNICODE *hex;
Py_ssize_t hexlen, byteslen, i, j;
int top, bot;
void *data;
unsigned int kind;
if (!PyArg_ParseTuple(args, "U:fromhex", &hexobj))
return NULL;
assert(PyUnicode_Check(hexobj));
hexlen = PyUnicode_GET_SIZE(hexobj);
hex = PyUnicode_AS_UNICODE(hexobj);
if (PyUnicode_READY(hexobj))
return NULL;
kind = PyUnicode_KIND(hexobj);
data = PyUnicode_DATA(hexobj);
hexlen = PyUnicode_GET_LENGTH(hexobj);
byteslen = hexlen/2; /* This overestimates if there are spaces */
newbytes = PyByteArray_FromStringAndSize(NULL, byteslen);
if (!newbytes)
@ -2667,12 +2670,12 @@ bytearray_fromhex(PyObject *cls, PyObject *args)
buf = PyByteArray_AS_STRING(newbytes);
for (i = j = 0; i < hexlen; i += 2) {
/* skip over spaces in the input */
while (hex[i] == ' ')
while (PyUnicode_READ(kind, data, i) == ' ')
i++;
if (i >= hexlen)
break;
top = hex_digit_to_int(hex[i]);
bot = hex_digit_to_int(hex[i+1]);
top = hex_digit_to_int(PyUnicode_READ(kind, data, i));
bot = hex_digit_to_int(PyUnicode_READ(kind, data, i+1));
if (top == -1 || bot == -1) {
PyErr_Format(PyExc_ValueError,
"non-hexadecimal number found in "

125
Objects/bytesobject.c
View file

@ -566,74 +566,68 @@ PyBytes_Repr(PyObject *obj, int smartquotes)
{
static const char *hexdigits = "0123456789abcdef";
register PyBytesObject* op = (PyBytesObject*) obj;
Py_ssize_t length = Py_SIZE(op);
size_t newsize;
Py_ssize_t i, length = Py_SIZE(op);
size_t newsize, squotes, dquotes;
PyObject *v;
if (length > (PY_SSIZE_T_MAX - 3) / 4) {
unsigned char quote, *s, *p;
/* Compute size of output string */
squotes = dquotes = 0;
newsize = 3; /* b'' */
s = (unsigned char*)op->ob_sval;
for (i = 0; i < length; i++) {
switch(s[i]) {
case '\'': squotes++; newsize++; break;
case '"': dquotes++; newsize++; break;
case '\\': case '\t': case '\n': case '\r':
newsize += 2; break; /* \C */
default:
if (s[i] < ' ' || s[i] >= 0x7f)
newsize += 4; /* \xHH */
else
newsize++;
}
}
quote = '\'';
if (smartquotes && squotes && !dquotes)
quote = '"';
if (squotes && quote == '\'')
newsize += squotes;
if (newsize > (PY_SSIZE_T_MAX - sizeof(PyUnicodeObject) - 1)) {
PyErr_SetString(PyExc_OverflowError,
"bytes object is too large to make repr");
return NULL;
}
newsize = 3 + 4 * length;
v = PyUnicode_FromUnicode(NULL, newsize);
v = PyUnicode_New(newsize, 127);
if (v == NULL) {
return NULL;
}
else {
register Py_ssize_t i;
register Py_UNICODE c;
register Py_UNICODE *p = PyUnicode_AS_UNICODE(v);
int quote;
p = PyUnicode_1BYTE_DATA(v);
/* Figure out which quote to use; single is preferred */
quote = '\'';
if (smartquotes) {
char *test, *start;
start = PyBytes_AS_STRING(op);
for (test = start; test < start+length; ++test) {
if (*test == '"') {
quote = '\''; /* back to single */
goto decided;
}
else if (*test == '\'')
quote = '"';
}
decided:
;
*p++ = 'b', *p++ = quote;
for (i = 0; i < length; i++) {
unsigned char c = op->ob_sval[i];
if (c == quote || c == '\\')
*p++ = '\\', *p++ = c;
else if (c == '\t')
*p++ = '\\', *p++ = 't';
else if (c == '\n')
*p++ = '\\', *p++ = 'n';
else if (c == '\r')
*p++ = '\\', *p++ = 'r';
else if (c < ' ' || c >= 0x7f) {
*p++ = '\\';
*p++ = 'x';
*p++ = hexdigits[(c & 0xf0) >> 4];
*p++ = hexdigits[c & 0xf];
}
*p++ = 'b', *p++ = quote;
for (i = 0; i < length; i++) {
/* There's at least enough room for a hex escape
and a closing quote. */
assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 5);
c = op->ob_sval[i];
if (c == quote || c == '\\')
*p++ = '\\', *p++ = c;
else if (c == '\t')
*p++ = '\\', *p++ = 't';
else if (c == '\n')
*p++ = '\\', *p++ = 'n';
else if (c == '\r')
*p++ = '\\', *p++ = 'r';
else if (c < ' ' || c >= 0x7f) {
*p++ = '\\';
*p++ = 'x';
*p++ = hexdigits[(c & 0xf0) >> 4];
*p++ = hexdigits[c & 0xf];
}
else
*p++ = c;
}
assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 1);
*p++ = quote;
*p = '\0';
if (PyUnicode_Resize(&v, (p - PyUnicode_AS_UNICODE(v)))) {
Py_DECREF(v);
return NULL;
}
return v;
else
*p++ = c;
}
*p++ = quote;
return v;
}
static PyObject *
@ -2356,15 +2350,20 @@ bytes_fromhex(PyObject *cls, PyObject *args)
{
PyObject *newstring, *hexobj;
char *buf;
Py_UNICODE *hex;
Py_ssize_t hexlen, byteslen, i, j;
int top, bot;
void *data;
unsigned int kind;
if (!PyArg_ParseTuple(args, "U:fromhex", &hexobj))
return NULL;
assert(PyUnicode_Check(hexobj));
hexlen = PyUnicode_GET_SIZE(hexobj);
hex = PyUnicode_AS_UNICODE(hexobj);
if (PyUnicode_READY(hexobj))
return NULL;
kind = PyUnicode_KIND(hexobj);
data = PyUnicode_DATA(hexobj);
hexlen = PyUnicode_GET_LENGTH(hexobj);
byteslen = hexlen/2; /* This overestimates if there are spaces */
newstring = PyBytes_FromStringAndSize(NULL, byteslen);
if (!newstring)
@ -2372,12 +2371,12 @@ bytes_fromhex(PyObject *cls, PyObject *args)
buf = PyBytes_AS_STRING(newstring);
for (i = j = 0; i < hexlen; i += 2) {
/* skip over spaces in the input */
while (hex[i] == ' ')
while (PyUnicode_READ(kind, data, i) == ' ')
i++;
if (i >= hexlen)
break;
top = hex_digit_to_int(hex[i]);
bot = hex_digit_to_int(hex[i+1]);
top = hex_digit_to_int(PyUnicode_READ(kind, data, i));
bot = hex_digit_to_int(PyUnicode_READ(kind, data, i+1));
if (top == -1 || bot == -1) {
PyErr_Format(PyExc_ValueError,
"non-hexadecimal number found in "

15
Objects/codeobject.c
View file

@ -8,19 +8,24 @@
/* all_name_chars(s): true iff all chars in s are valid NAME_CHARS */
static int
all_name_chars(Py_UNICODE *s)
all_name_chars(PyObject *o)
{
static char ok_name_char[256];
static unsigned char *name_chars = (unsigned char *)NAME_CHARS;
PyUnicodeObject *u = (PyUnicodeObject *)o;
const unsigned char *s;
if (!PyUnicode_Check(o) || PyUnicode_READY(u) == -1 ||
PyUnicode_MAX_CHAR_VALUE(u) >= 128)
return 0;
if (ok_name_char[*name_chars] == 0) {
unsigned char *p;
for (p = name_chars; *p; p++)
ok_name_char[*p] = 1;
}
s = PyUnicode_1BYTE_DATA(u);
while (*s) {
if (*s >= 128)
return 0;
if (ok_name_char[*s++] == 0)
return 0;
}
@ -77,9 +82,7 @@ PyCode_New(int argcount, int kwonlyargcount,
/* Intern selected string constants */
for (i = PyTuple_GET_SIZE(consts); --i >= 0; ) {
PyObject *v = PyTuple_GetItem(consts, i);
if (!PyUnicode_Check(v))
continue;
if (!all_name_chars(PyUnicode_AS_UNICODE(v)))
if (!all_name_chars(v))
continue;
PyUnicode_InternInPlace(&PyTuple_GET_ITEM(consts, i));
}

19
Objects/complexobject.c
View file

@ -702,9 +702,8 @@ complex__format__(PyObject* self, PyObject* args)
if (!PyArg_ParseTuple(args, "U:__format__", &format_spec))
return NULL;
return _PyComplex_FormatAdvanced(self,
PyUnicode_AS_UNICODE(format_spec),
PyUnicode_GET_SIZE(format_spec));
return _PyComplex_FormatAdvanced(self, format_spec, 0,
PyUnicode_GET_LENGTH(format_spec));
}
#if 0
@ -755,20 +754,10 @@ complex_subtype_from_string(PyTypeObject *type, PyObject *v)
Py_ssize_t len;
if (PyUnicode_Check(v)) {
Py_ssize_t i, buflen = PyUnicode_GET_SIZE(v);
Py_UNICODE *bufptr;
s_buffer = PyUnicode_TransformDecimalToASCII(
PyUnicode_AS_UNICODE(v), buflen);
s_buffer = _PyUnicode_TransformDecimalAndSpaceToASCII(v);
if (s_buffer == NULL)
return NULL;
/* Replace non-ASCII whitespace with ' ' */
bufptr = PyUnicode_AS_UNICODE(s_buffer);
for (i = 0; i < buflen; i++) {
Py_UNICODE ch = bufptr[i];
if (ch > 127 && Py_UNICODE_ISSPACE(ch))
bufptr[i] = ' ';
}
s = _PyUnicode_AsStringAndSize(s_buffer, &len);
s = PyUnicode_AsUTF8AndSize(s_buffer, &len);
if (s == NULL)
goto error;
}

20
Objects/dictobject.c
View file

@ -710,7 +710,7 @@ PyDict_GetItem(PyObject *op, PyObject *key)
if (!PyDict_Check(op))
return NULL;
if (!PyUnicode_CheckExact(key) ||
(hash = ((PyUnicodeObject *) key)->hash) == -1)
(hash = ((PyASCIIObject *) key)->hash) == -1)
{
hash = PyObject_Hash(key);
if (hash == -1) {
@ -762,7 +762,7 @@ PyDict_GetItemWithError(PyObject *op, PyObject *key)
return NULL;
}
if (!PyUnicode_CheckExact(key) ||
(hash = ((PyUnicodeObject *) key)->hash) == -1)
(hash = ((PyASCIIObject *) key)->hash) == -1)
{
hash = PyObject_Hash(key);
if (hash == -1) {
@ -797,7 +797,7 @@ PyDict_SetItem(register PyObject *op, PyObject *key, PyObject *value)
assert(value);
mp = (PyDictObject *)op;
if (!PyUnicode_CheckExact(key) ||
(hash = ((PyUnicodeObject *) key)->hash) == -1)
(hash = ((PyASCIIObject *) key)->hash) == -1)
{
hash = PyObject_Hash(key);
if (hash == -1)
@ -842,7 +842,7 @@ PyDict_DelItem(PyObject *op, PyObject *key)
}
assert(key);
if (!PyUnicode_CheckExact(key) ||
(hash = ((PyUnicodeObject *) key)->hash) == -1) {
(hash = ((PyASCIIObject *) key)->hash) == -1) {
hash = PyObject_Hash(key);
if (hash == -1)
return -1;
@ -1122,7 +1122,7 @@ dict_subscript(PyDictObject *mp, register PyObject *key)
PyDictEntry *ep;
assert(mp->ma_table != NULL);
if (!PyUnicode_CheckExact(key) ||
(hash = ((PyUnicodeObject *) key)->hash) == -1) {
(hash = ((PyASCIIObject *) key)->hash) == -1) {
hash = PyObject_Hash(key);
if (hash == -1)
return NULL;
@ -1726,7 +1726,7 @@ dict_contains(register PyDictObject *mp, PyObject *key)
PyDictEntry *ep;
if (!PyUnicode_CheckExact(key) ||
(hash = ((PyUnicodeObject *) key)->hash) == -1) {
(hash = ((PyASCIIObject *) key)->hash) == -1) {
hash = PyObject_Hash(key);
if (hash == -1)
return NULL;
@ -1750,7 +1750,7 @@ dict_get(register PyDictObject *mp, PyObject *args)
return NULL;
if (!PyUnicode_CheckExact(key) ||
(hash = ((PyUnicodeObject *) key)->hash) == -1) {
(hash = ((PyASCIIObject *) key)->hash) == -1) {
hash = PyObject_Hash(key);
if (hash == -1)
return NULL;
@ -1779,7 +1779,7 @@ dict_setdefault(register PyDictObject *mp, PyObject *args)
return NULL;
if (!PyUnicode_CheckExact(key) ||
(hash = ((PyUnicodeObject *) key)->hash) == -1) {
(hash = ((PyASCIIObject *) key)->hash) == -1) {
hash = PyObject_Hash(key);
if (hash == -1)
return NULL;
@ -1824,7 +1824,7 @@ dict_pop(PyDictObject *mp, PyObject *args)
return NULL;
}
if (!PyUnicode_CheckExact(key) ||
(hash = ((PyUnicodeObject *) key)->hash) == -1) {
(hash = ((PyASCIIObject *) key)->hash) == -1) {
hash = PyObject_Hash(key);
if (hash == -1)
return NULL;
@ -2033,7 +2033,7 @@ PyDict_Contains(PyObject *op, PyObject *key)
PyDictEntry *ep;
if (!PyUnicode_CheckExact(key) ||
(hash = ((PyUnicodeObject *) key)->hash) == -1) {
(hash = ((PyASCIIObject *) key)->hash) == -1) {
hash = PyObject_Hash(key);
if (hash == -1)
return -1;

26
Objects/exceptions.c
View file

@ -962,21 +962,18 @@ SyntaxError_traverse(PySyntaxErrorObject *self, visitproc visit, void *arg)
static PyObject*
my_basename(PyObject *name)
{
Py_UNICODE *unicode;
Py_ssize_t i, size, offset;
unicode = PyUnicode_AS_UNICODE(name);
size = PyUnicode_GET_SIZE(name);
int kind = PyUnicode_KIND(name);
void *data = PyUnicode_DATA(name);
size = PyUnicode_GET_LENGTH(name);
offset = 0;
for(i=0; i < size; i++) {
if (unicode[i] == SEP)
if (PyUnicode_READ(kind, data, i) == SEP)
offset = i + 1;
}
if (offset != 0) {
return PyUnicode_FromUnicode(
PyUnicode_AS_UNICODE(name) + offset,
size - offset);
} else {
if (offset != 0)
return PyUnicode_Substring(name, offset, size);
else {
Py_INCREF(name);
return name;
}
@ -1712,6 +1709,7 @@ static PyTypeObject _PyExc_UnicodeTranslateError = {
};
PyObject *PyExc_UnicodeTranslateError = (PyObject *)&_PyExc_UnicodeTranslateError;
/* Deprecated. */
PyObject *
PyUnicodeTranslateError_Create(
const Py_UNICODE *object, Py_ssize_t length,
@ -1721,6 +1719,14 @@ PyUnicodeTranslateError_Create(
object, length, start, end, reason);
}
PyObject *
_PyUnicodeTranslateError_Create(
PyObject *object,
Py_ssize_t start, Py_ssize_t end, const char *reason)
{
return PyObject_CallFunction(PyExc_UnicodeTranslateError, "Ons",
object, start, end, reason);
}
/*
* AssertionError extends Exception

17
Objects/fileobject.c
View file

@ -103,23 +103,18 @@ PyFile_GetLine(PyObject *f, int n)
}
}
if (n < 0 && result != NULL && PyUnicode_Check(result)) {
Py_UNICODE *s = PyUnicode_AS_UNICODE(result);
Py_ssize_t len = PyUnicode_GET_SIZE(result);
Py_ssize_t len = PyUnicode_GET_LENGTH(result);
if (len == 0) {
Py_DECREF(result);
result = NULL;
PyErr_SetString(PyExc_EOFError,
"EOF when reading a line");
}
else if (s[len-1] == '\n') {
if (result->ob_refcnt == 1)
PyUnicode_Resize(&result, len-1);
else {
PyObject *v;
v = PyUnicode_FromUnicode(s, len-1);
Py_DECREF(result);
result = v;
}
else if (PyUnicode_READ_CHAR(result, len-1) == '\n') {
PyObject *v;
v = PyUnicode_Substring(result, 0, len-1);
Py_DECREF(result);
result = v;
}
}
return result;

19
Objects/floatobject.c
View file

@ -174,20 +174,10 @@ PyFloat_FromString(PyObject *v)
PyObject *result = NULL;
if (PyUnicode_Check(v)) {
Py_ssize_t i, buflen = PyUnicode_GET_SIZE(v);
Py_UNICODE *bufptr;
s_buffer = PyUnicode_TransformDecimalToASCII(
PyUnicode_AS_UNICODE(v), buflen);
s_buffer = _PyUnicode_TransformDecimalAndSpaceToASCII(v);
if (s_buffer == NULL)
return NULL;
/* Replace non-ASCII whitespace with ' ' */
bufptr = PyUnicode_AS_UNICODE(s_buffer);
for (i = 0; i < buflen; i++) {
Py_UNICODE ch = bufptr[i];
if (ch > 127 && Py_UNICODE_ISSPACE(ch))
bufptr[i] = ' ';
}
s = _PyUnicode_AsStringAndSize(s_buffer, &len);
s = PyUnicode_AsUTF8AndSize(s_buffer, &len);
if (s == NULL) {
Py_DECREF(s_buffer);
return NULL;
@ -1741,9 +1731,8 @@ float__format__(PyObject *self, PyObject *args)
if (!PyArg_ParseTuple(args, "U:__format__", &format_spec))
return NULL;
return _PyFloat_FormatAdvanced(self,
PyUnicode_AS_UNICODE(format_spec),
PyUnicode_GET_SIZE(format_spec));
return _PyFloat_FormatAdvanced(self, format_spec, 0,
PyUnicode_GET_LENGTH(format_spec));
}
PyDoc_STRVAR(float__format__doc,

82
Objects/longobject.c
View file

@ -1551,7 +1551,7 @@ long_to_decimal_string(PyObject *aa)
PyObject *str;
Py_ssize_t size, strlen, size_a, i, j;
digit *pout, *pin, rem, tenpow;
Py_UNICODE *p;
unsigned char *p;
int negative;
a = (PyLongObject *)aa;
@ -1619,14 +1619,15 @@ long_to_decimal_string(PyObject *aa)
tenpow *= 10;
strlen++;
}
str = PyUnicode_FromUnicode(NULL, strlen);
str = PyUnicode_New(strlen, '9');
if (str == NULL) {
Py_DECREF(scratch);
return NULL;
}
/* fill the string right-to-left */
p = PyUnicode_AS_UNICODE(str) + strlen;
assert(PyUnicode_KIND(str) == PyUnicode_1BYTE_KIND);
p = PyUnicode_1BYTE_DATA(str) + strlen;
*p = '\0';
/* pout[0] through pout[size-2] contribute exactly
_PyLong_DECIMAL_SHIFT digits each */
@ -1649,7 +1650,7 @@ long_to_decimal_string(PyObject *aa)
*--p = '-';
/* check we've counted correctly */
assert(p == PyUnicode_AS_UNICODE(str));
assert(p == PyUnicode_1BYTE_DATA(str));
Py_DECREF(scratch);
return (PyObject *)str;
}
@ -1662,10 +1663,12 @@ PyObject *
_PyLong_Format(PyObject *aa, int base)
{
register PyLongObject *a = (PyLongObject *)aa;
PyObject *str;
PyObject *v;
Py_ssize_t i, sz;
Py_ssize_t size_a;
Py_UNICODE *p, sign = '\0';
char *p;
char sign = '\0';
char *buffer;
int bits;
assert(base == 2 || base == 8 || base == 10 || base == 16);
@ -1695,7 +1698,7 @@ _PyLong_Format(PyObject *aa, int base)
}
/* compute length of output string: allow 2 characters for prefix and
1 for possible '-' sign. */
if (size_a > (PY_SSIZE_T_MAX - 3) / PyLong_SHIFT) {
if (size_a > (PY_SSIZE_T_MAX - 3) / PyLong_SHIFT / sizeof(Py_UCS4)) {
PyErr_SetString(PyExc_OverflowError,
"int is too large to format");
return NULL;
@ -1704,11 +1707,12 @@ _PyLong_Format(PyObject *aa, int base)
is safe from overflow */
sz = 3 + (size_a * PyLong_SHIFT + (bits - 1)) / bits;
assert(sz >= 0);
str = PyUnicode_FromUnicode(NULL, sz);
if (str == NULL)
buffer = PyMem_Malloc(sz);
if (buffer == NULL) {
PyErr_NoMemory();
return NULL;
p = PyUnicode_AS_UNICODE(str) + sz;
*p = '\0';
}
p = &buffer[sz];
if (Py_SIZE(a) < 0)
sign = '-';
@ -1724,10 +1728,10 @@ _PyLong_Format(PyObject *aa, int base)
accumbits += PyLong_SHIFT;
assert(accumbits >= bits);
do {
Py_UNICODE cdigit;
cdigit = (Py_UNICODE)(accum & (base - 1));
char cdigit;
cdigit = (char)(accum & (base - 1));
cdigit += (cdigit < 10) ? '0' : 'a'-10;
assert(p > PyUnicode_AS_UNICODE(str));
assert(p > buffer);
*--p = cdigit;
accumbits -= bits;
accum >>= bits;
@ -1744,19 +1748,9 @@ _PyLong_Format(PyObject *aa, int base)
*--p = '0';
if (sign)
*--p = sign;
if (p != PyUnicode_AS_UNICODE(str)) {
Py_UNICODE *q = PyUnicode_AS_UNICODE(str);
assert(p > q);
do {
} while ((*q++ = *p++) != '\0');
q--;
if (PyUnicode_Resize(&str,(Py_ssize_t) (q -
PyUnicode_AS_UNICODE(str)))) {
Py_DECREF(str);
return NULL;
}
}
return (PyObject *)str;
v = PyUnicode_DecodeASCII(p, &buffer[sz] - p, NULL);
PyMem_Free(buffer);
return v;
}
/* Table of digit values for 8-bit string -> integer conversion.
@ -2133,24 +2127,27 @@ digit beyond the first.
PyObject *
PyLong_FromUnicode(Py_UNICODE *u, Py_ssize_t length, int base)
{
PyObject *v, *unicode = PyUnicode_FromUnicode(u, length);
if (unicode == NULL)
return NULL;
v = PyLong_FromUnicodeObject(unicode, base);
Py_DECREF(unicode);
return v;
}
PyObject *
PyLong_FromUnicodeObject(PyObject *u, int base)
{
PyObject *result;
PyObject *asciidig;
char *buffer, *end;
Py_ssize_t i, buflen;
Py_UNICODE *ptr;
Py_ssize_t buflen;
asciidig = PyUnicode_TransformDecimalToASCII(u, length);
asciidig = _PyUnicode_TransformDecimalAndSpaceToASCII(u);
if (asciidig == NULL)
return NULL;
/* Replace non-ASCII whitespace with ' ' */
ptr = PyUnicode_AS_UNICODE(asciidig);
for (i = 0; i < length; i++) {
Py_UNICODE ch = ptr[i];
if (ch > 127 && Py_UNICODE_ISSPACE(ch))
ptr[i] = ' ';
}
buffer = _PyUnicode_AsStringAndSize(asciidig, &buflen);
buffer = PyUnicode_AsUTF8AndSize(asciidig, &buflen);
if (buffer == NULL) {
Py_DECREF(asciidig);
return NULL;
@ -4144,9 +4141,7 @@ long_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
}
if (PyUnicode_Check(x))
return PyLong_FromUnicode(PyUnicode_AS_UNICODE(x),
PyUnicode_GET_SIZE(x),
(int)base);
return PyLong_FromUnicodeObject(x, (int)base);
else if (PyByteArray_Check(x) || PyBytes_Check(x)) {
/* Since PyLong_FromString doesn't have a length parameter,
* check here for possible NULs in the string. */
@ -4228,9 +4223,8 @@ long__format__(PyObject *self, PyObject *args)
if (!PyArg_ParseTuple(args, "U:__format__", &format_spec))
return NULL;
return _PyLong_FormatAdvanced(self,
PyUnicode_AS_UNICODE(format_spec),
PyUnicode_GET_SIZE(format_spec));
return _PyLong_FormatAdvanced(self, format_spec, 0,
PyUnicode_GET_LENGTH(format_spec));
}
/* Return a pair (q, r) such that a = b * q + r, and

9
Objects/moduleobject.c
View file

@ -285,8 +285,8 @@ _PyModule_Clear(PyObject *m)
pos = 0;
while (PyDict_Next(d, &pos, &key, &value)) {
if (value != Py_None && PyUnicode_Check(key)) {
Py_UNICODE *u = PyUnicode_AS_UNICODE(key);
if (u[0] == '_' && u[1] != '_') {
if (PyUnicode_READ_CHAR(key, 0) == '_' &&
PyUnicode_READ_CHAR(key, 1) != '_') {
if (Py_VerboseFlag > 1) {
const char *s = _PyUnicode_AsString(key);
if (s != NULL)
@ -303,9 +303,8 @@ _PyModule_Clear(PyObject *m)
pos = 0;
while (PyDict_Next(d, &pos, &key, &value)) {
if (value != Py_None && PyUnicode_Check(key)) {
Py_UNICODE *u = PyUnicode_AS_UNICODE(key);
if (u[0] != '_'
|| PyUnicode_CompareWithASCIIString(key, "__builtins__") != 0)
if (PyUnicode_READ_CHAR(key, 0) != '_' ||
PyUnicode_CompareWithASCIIString(key, "__builtins__") != 0)
{
if (Py_VerboseFlag > 1) {
const char *s = _PyUnicode_AsString(key);

10
Objects/object.c
View file

@ -295,9 +295,7 @@ PyObject_Print(PyObject *op, FILE *fp, int flags)
}
else if (PyUnicode_Check(s)) {
PyObject *t;
t = PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(s),
PyUnicode_GET_SIZE(s),
"backslashreplace");
t = PyUnicode_AsEncodedString(s, "utf-8", "backslashreplace");
if (t == NULL)
ret = 0;
else {
@ -439,11 +437,7 @@ PyObject_ASCII(PyObject *v)
return NULL;
/* repr is guaranteed to be a PyUnicode object by PyObject_Repr */
ascii = PyUnicode_EncodeASCII(
PyUnicode_AS_UNICODE(repr),
PyUnicode_GET_SIZE(repr),
"backslashreplace");
ascii = _PyUnicode_AsASCIIString(repr, "backslashreplace");
Py_DECREF(repr);
if (ascii == NULL)
return NULL;

42
Objects/setobject.c
View file

@ -386,7 +386,7 @@ set_add_key(register PySetObject *so, PyObject *key)
register Py_ssize_t n_used;
if (!PyUnicode_CheckExact(key) ||
(hash = ((PyUnicodeObject *) key)->hash) == -1) {
(hash = ((PyASCIIObject *) key)->hash) == -1) {
hash = PyObject_Hash(key);
if (hash == -1)
return -1;
@ -434,7 +434,7 @@ set_discard_key(PySetObject *so, PyObject *key)
assert (PyAnySet_Check(so));
if (!PyUnicode_CheckExact(key) ||
(hash = ((PyUnicodeObject *) key)->hash) == -1) {
(hash = ((PyASCIIObject *) key)->hash) == -1) {
hash = PyObject_Hash(key);
if (hash == -1)
return -1;
@ -579,11 +579,8 @@ set_dealloc(PySetObject *so)
static PyObject *
set_repr(PySetObject *so)
{
PyObject *keys, *result=NULL;
Py_UNICODE *u;
PyObject *result=NULL, *keys, *listrepr, *tmp;
int status = Py_ReprEnter((PyObject*)so);
PyObject *listrepr;
Py_ssize_t newsize;
if (status != 0) {
if (status < 0)
@ -601,31 +598,24 @@ set_repr(PySetObject *so)
if (keys == NULL)
goto done;
/* repr(keys)[1:-1] */
listrepr = PyObject_Repr(keys);
Py_DECREF(keys);
if (listrepr == NULL)
goto done;
newsize = PyUnicode_GET_SIZE(listrepr);
result = PyUnicode_FromUnicode(NULL, newsize);
if (result == NULL)
goto done;
u = PyUnicode_AS_UNICODE(result);
*u++ = '{';
/* Omit the brackets from the listrepr */
Py_UNICODE_COPY(u, PyUnicode_AS_UNICODE(listrepr)+1,
newsize-2);
u += newsize-2;
*u++ = '}';
tmp = PyUnicode_Substring(listrepr, 1, PyUnicode_GET_LENGTH(listrepr)-1);
Py_DECREF(listrepr);
if (tmp == NULL)
goto done;
listrepr = tmp;
if (Py_TYPE(so) != &PySet_Type) {
PyObject *tmp = PyUnicode_FromFormat("%s(%U)",
Py_TYPE(so)->tp_name,
result);
Py_DECREF(result);
result = tmp;
}
if (Py_TYPE(so) != &PySet_Type)
result = PyUnicode_FromFormat("%s({%U})",
Py_TYPE(so)->tp_name,
listrepr);
else
result = PyUnicode_FromFormat("{%U}", listrepr);
Py_DECREF(listrepr);
done:
Py_ReprLeave((PyObject*)so);
return result;
@ -684,7 +674,7 @@ set_contains_key(PySetObject *so, PyObject *key)
setentry *entry;
if (!PyUnicode_CheckExact(key) ||
(hash = ((PyUnicodeObject *) key)->hash) == -1) {
(hash = ((PyASCIIObject *) key)->hash) == -1) {
hash = PyObject_Hash(key);
if (hash == -1)
return -1;

9
Objects/stringlib/count.h
View file

@ -1,14 +1,11 @@
/* stringlib: count implementation */
#ifndef STRINGLIB_COUNT_H
#define STRINGLIB_COUNT_H
#ifndef STRINGLIB_FASTSEARCH_H
#error must include "stringlib/fastsearch.h" before including this module
#endif
Py_LOCAL_INLINE(Py_ssize_t)
stringlib_count(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
STRINGLIB(count)(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
const STRINGLIB_CHAR* sub, Py_ssize_t sub_len,
Py_ssize_t maxcount)
{
@ -19,7 +16,7 @@ stringlib_count(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
if (sub_len == 0)
return (str_len < maxcount) ? str_len + 1 : maxcount;
count = fastsearch(str, str_len, sub, sub_len, maxcount, FAST_COUNT);
count = FASTSEARCH(str, str_len, sub, sub_len, maxcount, FAST_COUNT);
if (count < 0)
return 0; /* no match */
@ -27,4 +24,4 @@ stringlib_count(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
return count;
}
#endif

25
Objects/stringlib/eq.h
View file

@ -9,13 +9,26 @@ unicode_eq(PyObject *aa, PyObject *bb)
register PyUnicodeObject *a = (PyUnicodeObject *)aa;
register PyUnicodeObject *b = (PyUnicodeObject *)bb;
if (a->length != b->length)
if (PyUnicode_READY(a) == -1 || PyUnicode_READY(b) == -1) {
assert(0 && "unicode_eq ready fail");
return 0;
if (a->length == 0)
return 1;
if (a->str[0] != b->str[0])
}
if (PyUnicode_GET_LENGTH(a) != PyUnicode_GET_LENGTH(b))
return 0;
if (a->length == 1)
if (PyUnicode_GET_LENGTH(a) == 0)
return 1;
return memcmp(a->str, b->str, a->length * sizeof(Py_UNICODE)) == 0;
if (PyUnicode_KIND(a) != PyUnicode_KIND(b))
return 0;
/* Just comparing the first byte is enough to see if a and b differ.
* If they are 2 byte or 4 byte character most differences will happen in
* the lower bytes anyways.
*/
if (PyUnicode_1BYTE_DATA(a)[0] != PyUnicode_1BYTE_DATA(b)[0])
return 0;
if (PyUnicode_KIND(a) == PyUnicode_1BYTE_KIND &&
PyUnicode_GET_LENGTH(a) == 1)
return 1;
return memcmp(PyUnicode_1BYTE_DATA(a), PyUnicode_1BYTE_DATA(b),
PyUnicode_GET_LENGTH(a) * PyUnicode_CHARACTER_SIZE(a)) == 0;
}

4
Objects/stringlib/fastsearch.h
View file

@ -1,6 +1,5 @@
/* stringlib: fastsearch implementation */
#ifndef STRINGLIB_FASTSEARCH_H
#define STRINGLIB_FASTSEARCH_H
/* fast search/count implementation, based on a mix between boyer-
@ -34,7 +33,7 @@
((mask & (1UL << ((ch) & (STRINGLIB_BLOOM_WIDTH -1)))))
Py_LOCAL_INLINE(Py_ssize_t)
fastsearch(const STRINGLIB_CHAR* s, Py_ssize_t n,
FASTSEARCH(const STRINGLIB_CHAR* s, Py_ssize_t n,
const STRINGLIB_CHAR* p, Py_ssize_t m,
Py_ssize_t maxcount, int mode)
{
@ -157,4 +156,3 @@ fastsearch(const STRINGLIB_CHAR* s, Py_ssize_t n,
return count;
}
#endif

31
Objects/stringlib/find.h
View file

@ -1,14 +1,11 @@
/* stringlib: find/index implementation */
#ifndef STRINGLIB_FIND_H
#define STRINGLIB_FIND_H
#ifndef STRINGLIB_FASTSEARCH_H
#error must include "stringlib/fastsearch.h" before including this module
#endif
Py_LOCAL_INLINE(Py_ssize_t)
stringlib_find(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
STRINGLIB(find)(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
const STRINGLIB_CHAR* sub, Py_ssize_t sub_len,
Py_ssize_t offset)
{
@ -19,7 +16,7 @@ stringlib_find(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
if (sub_len == 0)
return offset;
pos = fastsearch(str, str_len, sub, sub_len, -1, FAST_SEARCH);
pos = FASTSEARCH(str, str_len, sub, sub_len, -1, FAST_SEARCH);
if (pos >= 0)
pos += offset;
@ -28,7 +25,7 @@ stringlib_find(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
}
Py_LOCAL_INLINE(Py_ssize_t)
stringlib_rfind(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
STRINGLIB(rfind)(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
const STRINGLIB_CHAR* sub, Py_ssize_t sub_len,
Py_ssize_t offset)
{
@ -39,7 +36,7 @@ stringlib_rfind(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
if (sub_len == 0)
return str_len + offset;
pos = fastsearch(str, str_len, sub, sub_len, -1, FAST_RSEARCH);
pos = FASTSEARCH(str, str_len, sub, sub_len, -1, FAST_RSEARCH);
if (pos >= 0)
pos += offset;
@ -63,29 +60,29 @@ stringlib_rfind(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
}
Py_LOCAL_INLINE(Py_ssize_t)
stringlib_find_slice(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
STRINGLIB(find_slice)(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
const STRINGLIB_CHAR* sub, Py_ssize_t sub_len,
Py_ssize_t start, Py_ssize_t end)
{
ADJUST_INDICES(start, end, str_len);
return stringlib_find(str + start, end - start, sub, sub_len, start);
return STRINGLIB(find)(str + start, end - start, sub, sub_len, start);
}
Py_LOCAL_INLINE(Py_ssize_t)
stringlib_rfind_slice(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
STRINGLIB(rfind_slice)(const STRINGLIB_CHAR* str, Py_ssize_t str_len,
const STRINGLIB_CHAR* sub, Py_ssize_t sub_len,
Py_ssize_t start, Py_ssize_t end)
{
ADJUST_INDICES(start, end, str_len);
return stringlib_rfind(str + start, end - start, sub, sub_len, start);
return STRINGLIB(rfind)(str + start, end - start, sub, sub_len, start);
}
#ifdef STRINGLIB_WANT_CONTAINS_OBJ
Py_LOCAL_INLINE(int)
stringlib_contains_obj(PyObject* str, PyObject* sub)
STRINGLIB(contains_obj)(PyObject* str, PyObject* sub)
{
return stringlib_find(
return STRINGLIB(find)(
STRINGLIB_STR(str), STRINGLIB_LEN(str),
STRINGLIB_STR(sub), STRINGLIB_LEN(sub), 0
) != -1;
@ -105,7 +102,7 @@ is ok.
#define FORMAT_BUFFER_SIZE 50
Py_LOCAL_INLINE(int)
stringlib_parse_args_finds(const char * function_name, PyObject *args,
STRINGLIB(parse_args_finds)(const char * function_name, PyObject *args,
PyObject **subobj,
Py_ssize_t *start, Py_ssize_t *end)
{
@ -153,13 +150,13 @@ after finishing using the substring, must DECREF it).
*/
Py_LOCAL_INLINE(int)
stringlib_parse_args_finds_unicode(const char * function_name, PyObject *args,
STRINGLIB(parse_args_finds_unicode)(const char * function_name, PyObject *args,
PyUnicodeObject **substring,
Py_ssize_t *start, Py_ssize_t *end)
{
PyObject *tmp_substring;
if(stringlib_parse_args_finds(function_name, args, &tmp_substring,
if(STRINGLIB(parse_args_finds)(function_name, args, &tmp_substring,
start, end)) {
tmp_substring = PyUnicode_FromObject(tmp_substring);
if (!tmp_substring)
@ -171,5 +168,3 @@ stringlib_parse_args_finds_unicode(const char * function_name, PyObject *args,
}
#endif /* STRINGLIB_IS_UNICODE */
#endif /* STRINGLIB_FIND_H */

1516
Objects/stringlib/formatter.h
View file

File diff suppressed because it is too large Load diff

27
Objects/stringlib/localeutil.h
View file

@ -1,8 +1,5 @@
/* stringlib: locale related helpers implementation */
#ifndef STRINGLIB_LOCALEUTIL_H
#define STRINGLIB_LOCALEUTIL_H
#include <locale.h>
#define MAX(x, y) ((x) < (y) ? (y) : (x))
@ -12,10 +9,10 @@ typedef struct {
const char *grouping;
char previous;
Py_ssize_t i; /* Where we're currently pointing in grouping. */
} GroupGenerator;
} STRINGLIB(GroupGenerator);
static void
_GroupGenerator_init(GroupGenerator *self, const char *grouping)
STRINGLIB(GroupGenerator_init)(STRINGLIB(GroupGenerator) *self, const char *grouping)
{
self->grouping = grouping;
self->i = 0;
@ -24,7 +21,7 @@ _GroupGenerator_init(GroupGenerator *self, const char *grouping)
/* Returns the next grouping, or 0 to signify end. */
static Py_ssize_t
_GroupGenerator_next(GroupGenerator *self)
STRINGLIB(GroupGenerator_next)(STRINGLIB(GroupGenerator) *self)
{
/* Note that we don't really do much error checking here. If a
grouping string contains just CHAR_MAX, for example, then just
@ -48,13 +45,11 @@ _GroupGenerator_next(GroupGenerator *self)
/* Fill in some digits, leading zeros, and thousands separator. All
are optional, depending on when we're called. */
static void
fill(STRINGLIB_CHAR **digits_end, STRINGLIB_CHAR **buffer_end,
STRINGLIB(fill)(STRINGLIB_CHAR **digits_end, STRINGLIB_CHAR **buffer_end,
Py_ssize_t n_chars, Py_ssize_t n_zeros, const char* thousands_sep,
Py_ssize_t thousands_sep_len)
{
#if STRINGLIB_IS_UNICODE
Py_ssize_t i;
#endif
if (thousands_sep) {
*buffer_end -= thousands_sep_len;
@ -76,7 +71,8 @@ fill(STRINGLIB_CHAR **digits_end, STRINGLIB_CHAR **buffer_end,
memcpy(*buffer_end, *digits_end, n_chars * sizeof(STRINGLIB_CHAR));
*buffer_end -= n_zeros;
STRINGLIB_FILL(*buffer_end, '0', n_zeros);
for (i = 0; i < n_zeros; i++)
(*buffer_end)[i] = '0';
}
/**
@ -133,15 +129,15 @@ _Py_InsertThousandsGrouping(STRINGLIB_CHAR *buffer,
be looked at */
/* A generator that returns all of the grouping widths, until it
returns 0. */
GroupGenerator groupgen;
_GroupGenerator_init(&groupgen, grouping);
STRINGLIB(GroupGenerator) groupgen;
STRINGLIB(GroupGenerator_init)(&groupgen, grouping);
if (buffer) {
buffer_end = buffer + n_buffer;
digits_end = digits + n_digits;
}
while ((l = _GroupGenerator_next(&groupgen)) > 0) {
while ((l = STRINGLIB(GroupGenerator_next)(&groupgen)) > 0) {
l = MIN(l, MAX(MAX(remaining, min_width), 1));
n_zeros = MAX(0, l - remaining);
n_chars = MAX(0, MIN(remaining, l));
@ -153,7 +149,7 @@ _Py_InsertThousandsGrouping(STRINGLIB_CHAR *buffer,
if (buffer) {
/* Copy into the output buffer. */
fill(&digits_end, &buffer_end, n_chars, n_zeros,
STRINGLIB(fill)(&digits_end, &buffer_end, n_chars, n_zeros,
use_separator ? thousands_sep : NULL, thousands_sep_len);
}
@ -180,7 +176,7 @@ _Py_InsertThousandsGrouping(STRINGLIB_CHAR *buffer,
count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars;
if (buffer) {
/* Copy into the output buffer. */
fill(&digits_end, &buffer_end, n_chars, n_zeros,
STRINGLIB(fill)(&digits_end, &buffer_end, n_chars, n_zeros,
use_separator ? thousands_sep : NULL, thousands_sep_len);
}
}
@ -209,4 +205,3 @@ _Py_InsertThousandsGroupingLocale(STRINGLIB_CHAR *buffer,
return _Py_InsertThousandsGrouping(buffer, n_buffer, digits, n_digits,
min_width, grouping, thousands_sep);
}
#endif /* STRINGLIB_LOCALEUTIL_H */

12
Objects/stringlib/partition.h
View file

@ -1,14 +1,11 @@
/* stringlib: partition implementation */
#ifndef STRINGLIB_PARTITION_H
#define STRINGLIB_PARTITION_H
#ifndef STRINGLIB_FASTSEARCH_H
#error must include "stringlib/fastsearch.h" before including this module
#endif
Py_LOCAL_INLINE(PyObject*)
stringlib_partition(PyObject* str_obj,
STRINGLIB(partition)(PyObject* str_obj,
const STRINGLIB_CHAR* str, Py_ssize_t str_len,
PyObject* sep_obj,
const STRINGLIB_CHAR* sep, Py_ssize_t sep_len)
@ -25,7 +22,7 @@ stringlib_partition(PyObject* str_obj,
if (!out)
return NULL;
pos = fastsearch(str, str_len, sep, sep_len, -1, FAST_SEARCH);
pos = FASTSEARCH(str, str_len, sep, sep_len, -1, FAST_SEARCH);
if (pos < 0) {
#if STRINGLIB_MUTABLE
@ -58,7 +55,7 @@ stringlib_partition(PyObject* str_obj,
}
Py_LOCAL_INLINE(PyObject*)
stringlib_rpartition(PyObject* str_obj,
STRINGLIB(rpartition)(PyObject* str_obj,
const STRINGLIB_CHAR* str, Py_ssize_t str_len,
PyObject* sep_obj,
const STRINGLIB_CHAR* sep, Py_ssize_t sep_len)
@ -75,7 +72,7 @@ stringlib_rpartition(PyObject* str_obj,
if (!out)
return NULL;
pos = fastsearch(str, str_len, sep, sep_len, -1, FAST_RSEARCH);
pos = FASTSEARCH(str, str_len, sep, sep_len, -1, FAST_RSEARCH);
if (pos < 0) {
#if STRINGLIB_MUTABLE
@ -107,4 +104,3 @@ stringlib_rpartition(PyObject* str_obj,
return out;
}
#endif

26
Objects/stringlib/split.h
View file

@ -1,8 +1,5 @@
/* stringlib: split implementation */
#ifndef STRINGLIB_SPLIT_H
#define STRINGLIB_SPLIT_H
#ifndef STRINGLIB_FASTSEARCH_H
#error must include "stringlib/fastsearch.h" before including this module
#endif
@ -54,7 +51,7 @@
#define FIX_PREALLOC_SIZE(list) Py_SIZE(list) = count
Py_LOCAL_INLINE(PyObject *)
stringlib_split_whitespace(PyObject* str_obj,
STRINGLIB(split_whitespace)(PyObject* str_obj,
const STRINGLIB_CHAR* str, Py_ssize_t str_len,
Py_ssize_t maxcount)
{
@ -102,7 +99,7 @@ stringlib_split_whitespace(PyObject* str_obj,
}
Py_LOCAL_INLINE(PyObject *)
stringlib_split_char(PyObject* str_obj,
STRINGLIB(split_char)(PyObject* str_obj,
const STRINGLIB_CHAR* str, Py_ssize_t str_len,
const STRINGLIB_CHAR ch,
Py_ssize_t maxcount)
@ -145,7 +142,7 @@ stringlib_split_char(PyObject* str_obj,
}
Py_LOCAL_INLINE(PyObject *)
stringlib_split(PyObject* str_obj,
STRINGLIB(split)(PyObject* str_obj,
const STRINGLIB_CHAR* str, Py_ssize_t str_len,
const STRINGLIB_CHAR* sep, Py_ssize_t sep_len,
Py_ssize_t maxcount)
@ -158,7 +155,7 @@ stringlib_split(PyObject* str_obj,
return NULL;
}
else if (sep_len == 1)
return stringlib_split_char(str_obj, str, str_len, sep[0], maxcount);
return STRINGLIB(split_char)(str_obj, str, str_len, sep[0], maxcount);
list = PyList_New(PREALLOC_SIZE(maxcount));
if (list == NULL)
@ -166,7 +163,7 @@ stringlib_split(PyObject* str_obj,
i = j = 0;
while (maxcount-- > 0) {
pos = fastsearch(str+i, str_len-i, sep, sep_len, -1, FAST_SEARCH);
pos = FASTSEARCH(str+i, str_len-i, sep, sep_len, -1, FAST_SEARCH);
if (pos < 0)
break;
j = i + pos;
@ -193,7 +190,7 @@ stringlib_split(PyObject* str_obj,
}
Py_LOCAL_INLINE(PyObject *)
stringlib_rsplit_whitespace(PyObject* str_obj,
STRINGLIB(rsplit_whitespace)(PyObject* str_obj,
const STRINGLIB_CHAR* str, Py_ssize_t str_len,
Py_ssize_t maxcount)
{
@ -243,7 +240,7 @@ stringlib_rsplit_whitespace(PyObject* str_obj,
}
Py_LOCAL_INLINE(PyObject *)
stringlib_rsplit_char(PyObject* str_obj,
STRINGLIB(rsplit_char)(PyObject* str_obj,
const STRINGLIB_CHAR* str, Py_ssize_t str_len,
const STRINGLIB_CHAR ch,
Py_ssize_t maxcount)
@ -287,7 +284,7 @@ stringlib_rsplit_char(PyObject* str_obj,
}
Py_LOCAL_INLINE(PyObject *)
stringlib_rsplit(PyObject* str_obj,
STRINGLIB(rsplit)(PyObject* str_obj,
const STRINGLIB_CHAR* str, Py_ssize_t str_len,
const STRINGLIB_CHAR* sep, Py_ssize_t sep_len,
Py_ssize_t maxcount)
@ -300,7 +297,7 @@ stringlib_rsplit(PyObject* str_obj,
return NULL;
}
else if (sep_len == 1)
return stringlib_rsplit_char(str_obj, str, str_len, sep[0], maxcount);
return STRINGLIB(rsplit_char)(str_obj, str, str_len, sep[0], maxcount);
list = PyList_New(PREALLOC_SIZE(maxcount));
if (list == NULL)
@ -308,7 +305,7 @@ stringlib_rsplit(PyObject* str_obj,
j = str_len;
while (maxcount-- > 0) {
pos = fastsearch(str, j, sep, sep_len, -1, FAST_RSEARCH);
pos = FASTSEARCH(str, j, sep, sep_len, -1, FAST_RSEARCH);
if (pos < 0)
break;
SPLIT_ADD(str, pos + sep_len, j);
@ -336,7 +333,7 @@ stringlib_rsplit(PyObject* str_obj,
}
Py_LOCAL_INLINE(PyObject *)
stringlib_splitlines(PyObject* str_obj,
STRINGLIB(splitlines)(PyObject* str_obj,
const STRINGLIB_CHAR* str, Py_ssize_t str_len,
int keepends)
{
@ -391,4 +388,3 @@ stringlib_splitlines(PyObject* str_obj,
return NULL;
}
#endif

2
Objects/stringlib/stringdefs.h
View file

@ -6,6 +6,8 @@
compiled as unicode. */
#define STRINGLIB_IS_UNICODE 0
#define FASTSEARCH fastsearch
#define STRINGLIB(F) stringlib_##F
#define STRINGLIB_OBJECT PyBytesObject
#define STRINGLIB_CHAR char
#define STRINGLIB_TYPE_NAME "string"

35
Objects/stringlib/ucs1lib.h Normal file
View file

@ -0,0 +1,35 @@
/* this is sort of a hack. there's at least one place (formatting
floats) where some stringlib code takes a different path if it's
compiled as unicode. */
#define STRINGLIB_IS_UNICODE 1
#define FASTSEARCH ucs1lib_fastsearch
#define STRINGLIB(F) ucs1lib_##F
#define STRINGLIB_OBJECT PyUnicodeObject
#define STRINGLIB_CHAR Py_UCS1
#define STRINGLIB_TYPE_NAME "unicode"
#define STRINGLIB_PARSE_CODE "U"
#define STRINGLIB_EMPTY unicode_empty
#define STRINGLIB_ISSPACE Py_UNICODE_ISSPACE
#define STRINGLIB_ISLINEBREAK BLOOM_LINEBREAK
#define STRINGLIB_ISDECIMAL Py_UNICODE_ISDECIMAL
#define STRINGLIB_TODECIMAL Py_UNICODE_TODECIMAL
#define STRINGLIB_TOUPPER Py_UNICODE_TOUPPER
#define STRINGLIB_TOLOWER Py_UNICODE_TOLOWER
#define STRINGLIB_FILL Py_UNICODE_FILL
#define STRINGLIB_STR PyUnicode_1BYTE_DATA
#define STRINGLIB_LEN PyUnicode_GET_LENGTH
#define STRINGLIB_NEW PyUnicode_FromUCS1
#define STRINGLIB_RESIZE not_supported
#define STRINGLIB_CHECK PyUnicode_Check
#define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact
#define STRINGLIB_GROUPING _PyUnicode_InsertThousandsGrouping
#define STRINGLIB_GROUPING_LOCALE _PyUnicode_InsertThousandsGroupingLocale
#define STRINGLIB_TOSTR PyObject_Str
#define STRINGLIB_TOASCII PyObject_ASCII
#define _Py_InsertThousandsGrouping _PyUnicode_ucs1_InsertThousandsGrouping
#define _Py_InsertThousandsGroupingLocale _PyUnicode_ucs1_InsertThousandsGroupingLocale

34
Objects/stringlib/ucs2lib.h Normal file
View file

@ -0,0 +1,34 @@
/* this is sort of a hack. there's at least one place (formatting
floats) where some stringlib code takes a different path if it's
compiled as unicode. */
#define STRINGLIB_IS_UNICODE 1
#define FASTSEARCH ucs2lib_fastsearch
#define STRINGLIB(F) ucs2lib_##F
#define STRINGLIB_OBJECT PyUnicodeObject
#define STRINGLIB_CHAR Py_UCS2
#define STRINGLIB_TYPE_NAME "unicode"
#define STRINGLIB_PARSE_CODE "U"
#define STRINGLIB_EMPTY unicode_empty
#define STRINGLIB_ISSPACE Py_UNICODE_ISSPACE
#define STRINGLIB_ISLINEBREAK BLOOM_LINEBREAK
#define STRINGLIB_ISDECIMAL Py_UNICODE_ISDECIMAL
#define STRINGLIB_TODECIMAL Py_UNICODE_TODECIMAL
#define STRINGLIB_TOUPPER Py_UNICODE_TOUPPER
#define STRINGLIB_TOLOWER Py_UNICODE_TOLOWER
#define STRINGLIB_FILL Py_UNICODE_FILL
#define STRINGLIB_STR PyUnicode_1BYTE_DATA
#define STRINGLIB_LEN PyUnicode_GET_LENGTH
#define STRINGLIB_NEW PyUnicode_FromUCS2
#define STRINGLIB_RESIZE not_supported
#define STRINGLIB_CHECK PyUnicode_Check
#define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact
#define STRINGLIB_GROUPING _PyUnicode_InsertThousandsGrouping
#define STRINGLIB_GROUPING_LOCALE _PyUnicode_InsertThousandsGroupingLocale
#define STRINGLIB_TOSTR PyObject_Str
#define STRINGLIB_TOASCII PyObject_ASCII
#define _Py_InsertThousandsGrouping _PyUnicode_ucs2_InsertThousandsGrouping
#define _Py_InsertThousandsGroupingLocale _PyUnicode_ucs2_InsertThousandsGroupingLocale

34
Objects/stringlib/ucs4lib.h Normal file
View file

@ -0,0 +1,34 @@
/* this is sort of a hack. there's at least one place (formatting
floats) where some stringlib code takes a different path if it's
compiled as unicode. */
#define STRINGLIB_IS_UNICODE 1
#define FASTSEARCH ucs4lib_fastsearch
#define STRINGLIB(F) ucs4lib_##F
#define STRINGLIB_OBJECT PyUnicodeObject
#define STRINGLIB_CHAR Py_UCS4
#define STRINGLIB_TYPE_NAME "unicode"
#define STRINGLIB_PARSE_CODE "U"
#define STRINGLIB_EMPTY unicode_empty
#define STRINGLIB_ISSPACE Py_UNICODE_ISSPACE
#define STRINGLIB_ISLINEBREAK BLOOM_LINEBREAK
#define STRINGLIB_ISDECIMAL Py_UNICODE_ISDECIMAL
#define STRINGLIB_TODECIMAL Py_UNICODE_TODECIMAL
#define STRINGLIB_TOUPPER Py_UNICODE_TOUPPER
#define STRINGLIB_TOLOWER Py_UNICODE_TOLOWER
#define STRINGLIB_FILL Py_UNICODE_FILL
#define STRINGLIB_STR PyUnicode_1BYTE_DATA
#define STRINGLIB_LEN PyUnicode_GET_LENGTH
#define STRINGLIB_NEW PyUnicode_FromUCS4
#define STRINGLIB_RESIZE not_supported
#define STRINGLIB_CHECK PyUnicode_Check
#define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact
#define STRINGLIB_GROUPING _PyUnicode_InsertThousandsGrouping
#define STRINGLIB_GROUPING_LOCALE _PyUnicode_InsertThousandsGroupingLocale
#define STRINGLIB_TOSTR PyObject_Str
#define STRINGLIB_TOASCII PyObject_ASCII
#define _Py_InsertThousandsGrouping _PyUnicode_ucs4_InsertThousandsGrouping
#define _Py_InsertThousandsGroupingLocale _PyUnicode_ucs4_InsertThousandsGroupingLocale

10
Objects/stringlib/undef.h Normal file
View file

@ -0,0 +1,10 @@
#undef FASTSEARCH
#undef STRINGLIB
#undef STRINGLIB_CHAR
#undef STRINGLIB_STR
#undef STRINGLIB_LEN
#undef STRINGLIB_NEW
#undef STRINGLIB_RESIZE
#undef _Py_InsertThousandsGrouping
#undef _Py_InsertThousandsGroupingLocale

385
Objects/stringlib/string_format.h → Objects/stringlib/unicode_format.h
View file

@ -1,16 +1,8 @@
/*
string_format.h -- implementation of string.format().
It uses the Objects/stringlib conventions, so that it can be
compiled for both unicode and string objects.
unicode_format.h -- implementation of str.format().
*/
/* Defines for Python 2.6 compatibility */
#if PY_VERSION_HEX < 0x03000000
#define PyLong_FromSsize_t _PyLong_FromSsize_t
#endif
/* Defines for more efficiently reallocating the string buffer */
#define INITIAL_SIZE_INCREMENT 100
#define SIZE_MULTIPLIER 2
@ -26,8 +18,8 @@
unicode pointers.
*/
typedef struct {
STRINGLIB_CHAR *ptr;
STRINGLIB_CHAR *end;
PyObject *str; /* borrowed reference */
Py_ssize_t start, end;
} SubString;
@ -64,34 +56,32 @@ AutoNumber_Init(AutoNumber *auto_number)
/* fill in a SubString from a pointer and length */
Py_LOCAL_INLINE(void)
SubString_init(SubString *str, STRINGLIB_CHAR *p, Py_ssize_t len)
SubString_init(SubString *str, PyObject *s, int start, int end)
{
str->ptr = p;
if (p == NULL)
str->end = NULL;
else
str->end = str->ptr + len;
str->str = s;
str->start = start;
str->end = end;
}
/* return a new string. if str->ptr is NULL, return None */
/* return a new string. if str->str is NULL, return None */
Py_LOCAL_INLINE(PyObject *)
SubString_new_object(SubString *str)
{
if (str->ptr == NULL) {
if (str->str == NULL) {
Py_INCREF(Py_None);
return Py_None;
}
return STRINGLIB_NEW(str->ptr, str->end - str->ptr);
return PyUnicode_Substring(str->str, str->start, str->end);
}
/* return a new string. if str->ptr is NULL, return None */
/* return a new string. if str->str is NULL, return None */
Py_LOCAL_INLINE(PyObject *)
SubString_new_object_or_empty(SubString *str)
{
if (str->ptr == NULL) {
return STRINGLIB_NEW(NULL, 0);
if (str->str == NULL) {
return PyUnicode_FromUnicode(NULL, 0);
}
return STRINGLIB_NEW(str->ptr, str->end - str->ptr);
return SubString_new_object(str);
}
/* Return 1 if an error has been detected switching between automatic
@ -125,9 +115,10 @@ autonumber_state_error(AutoNumberState state, int field_name_is_empty)
/************************************************************************/
typedef struct {
STRINGLIB_CHAR *ptr;
STRINGLIB_CHAR *end;
PyObject *obj;
char *data;
Py_UCS4 maxchar;
unsigned int kind;
Py_ssize_t pos, size;
Py_ssize_t size_increment;
} OutputString;
@ -135,12 +126,16 @@ typedef struct {
static int
output_initialize(OutputString *output, Py_ssize_t size)
{
output->obj = STRINGLIB_NEW(NULL, size);
if (output->obj == NULL)
output->data = PyMem_Malloc(size);
if (output->data == NULL) {
PyErr_NoMemory();
return 0;
}
output->ptr = STRINGLIB_STR(output->obj);
output->end = STRINGLIB_LEN(output->obj) + output->ptr;
output->maxchar = 127;
output->kind = PyUnicode_1BYTE_KIND;
output->pos = 0;
output->size = size;
output->size_increment = INITIAL_SIZE_INCREMENT;
return 1;
@ -155,20 +150,51 @@ output_initialize(OutputString *output, Py_ssize_t size)
static int
output_extend(OutputString *output, Py_ssize_t count)
{
STRINGLIB_CHAR *startptr = STRINGLIB_STR(output->obj);
Py_ssize_t curlen = output->ptr - startptr;
Py_ssize_t maxlen = curlen + count + output->size_increment;
Py_ssize_t maxlen = output->size + count + output->size_increment;
if (STRINGLIB_RESIZE(&output->obj, maxlen) < 0)
output->data = PyMem_Realloc(output->data, maxlen << (output->kind-1));
output->size = maxlen;
if (output->data == 0) {
PyErr_NoMemory();
return 0;
startptr = STRINGLIB_STR(output->obj);
output->ptr = startptr + curlen;
output->end = startptr + maxlen;
}
if (output->size_increment < MAX_SIZE_INCREMENT)
output->size_increment *= SIZE_MULTIPLIER;
return 1;
}
static int
output_widen(OutputString *output, Py_UCS4 maxchar)
{
int kind;
void *data;
Py_ssize_t i;
if (maxchar <= output->maxchar)
return 1;
if (maxchar < 256) {
output->maxchar = 255;
return 1;
}
if (maxchar < 65536) {
output->maxchar = 65535;
kind = 2;
}
else {
output->maxchar = 1<<21;
kind = 3;
}
data = PyMem_Malloc(output->size << (kind-1));
if (data == 0)
return 0;
for (i = 0; i < output->size; i++)
PyUnicode_WRITE(kind, data, i,
PyUnicode_READ(output->kind, output->data, i));
PyMem_Free(output->data);
output->data = data;
output->kind = kind;
return 1;
}
/*
output_data dumps characters into our output string
buffer.
@ -179,12 +205,25 @@ output_extend(OutputString *output, Py_ssize_t count)
1 for success.
*/
static int
output_data(OutputString *output, const STRINGLIB_CHAR *s, Py_ssize_t count)
output_data(OutputString *output, PyObject *s, Py_ssize_t start, Py_ssize_t end)
{
if ((count > output->end - output->ptr) && !output_extend(output, count))
Py_ssize_t i;
int kind;
if ((output->pos + end - start > output->size) &&
!output_extend(output, end - start))
return 0;
memcpy(output->ptr, s, count * sizeof(STRINGLIB_CHAR));
output->ptr += count;
kind = PyUnicode_KIND(s);
if (PyUnicode_MAX_CHAR_VALUE(s) > output->maxchar) {
Py_UCS4 maxchar = output->maxchar;
for (i = start; i < end; i++)
if (PyUnicode_READ(kind, PyUnicode_DATA(s), i) > maxchar)
maxchar = PyUnicode_READ(kind, PyUnicode_DATA(s), i);
if (!output_widen(output, maxchar))
return 0;
}
for (i = start; i < end; i++)
PyUnicode_WRITE(output->kind, output->data, output->pos++,
PyUnicode_READ(kind, PyUnicode_DATA(s), i));
return 1;
}
@ -197,15 +236,14 @@ get_integer(const SubString *str)
{
Py_ssize_t accumulator = 0;
Py_ssize_t digitval;
Py_ssize_t oldaccumulator;
STRINGLIB_CHAR *p;
Py_ssize_t i;
/* empty string is an error */
if (str->ptr >= str->end)
if (str->start >= str->end)
return -1;
for (p = str->ptr; p < str->end; p++) {
digitval = STRINGLIB_TODECIMAL(*p);
for (i = str->start; i < str->end; i++) {
digitval = Py_UNICODE_TODECIMAL(PyUnicode_READ_CHAR(str->str, i));
if (digitval < 0)
return -1;
/*
@ -280,34 +318,36 @@ typedef struct {
lifetime of the iterator. can be empty */
SubString str;
/* pointer to where we are inside field_name */
STRINGLIB_CHAR *ptr;
/* index to where we are inside field_name */
Py_ssize_t index;
} FieldNameIterator;
static int
FieldNameIterator_init(FieldNameIterator *self, STRINGLIB_CHAR *ptr,
Py_ssize_t len)
FieldNameIterator_init(FieldNameIterator *self, PyObject *s,
Py_ssize_t start, Py_ssize_t end)
{
SubString_init(&self->str, ptr, len);
self->ptr = self->str.ptr;
SubString_init(&self->str, s, start, end);
self->index = start;
return 1;
}
static int
_FieldNameIterator_attr(FieldNameIterator *self, SubString *name)
{
STRINGLIB_CHAR c;
Py_UCS4 c;
name->ptr = self->ptr;
name->str = self->str.str;
name->start = self->index;
/* return everything until '.' or '[' */
while (self->ptr < self->str.end) {
switch (c = *self->ptr++) {
while (self->index < self->str.end) {
c = PyUnicode_READ_CHAR(self->str.str, self->index++);
switch (c) {
case '[':
case '.':
/* backup so that we this character will be seen next time */
self->ptr--;
self->index--;
break;
default:
continue;
@ -315,7 +355,7 @@ _FieldNameIterator_attr(FieldNameIterator *self, SubString *name)
break;
}
/* end of string is okay */
name->end = self->ptr;
name->end = self->index;
return 1;
}
@ -323,13 +363,15 @@ static int
_FieldNameIterator_item(FieldNameIterator *self, SubString *name)
{
int bracket_seen = 0;
STRINGLIB_CHAR c;
Py_UCS4 c;
name->ptr = self->ptr;
name->str = self->str.str;
name->start = self->index;
/* return everything until ']' */
while (self->ptr < self->str.end) {
switch (c = *self->ptr++) {
while (self->index < self->str.end) {
c = PyUnicode_READ_CHAR(self->str.str, self->index++);
switch (c) {
case ']':
bracket_seen = 1;
break;
@ -346,7 +388,7 @@ _FieldNameIterator_item(FieldNameIterator *self, SubString *name)
/* end of string is okay */
/* don't include the ']' */
name->end = self->ptr-1;
name->end = self->index-1;
return 1;
}
@ -356,10 +398,10 @@ FieldNameIterator_next(FieldNameIterator *self, int *is_attribute,
Py_ssize_t *name_idx, SubString *name)
{
/* check at end of input */
if (self->ptr >= self->str.end)
if (self->index >= self->str.end)
return 1;
switch (*self->ptr++) {
switch (PyUnicode_READ_CHAR(self->str.str, self->index++)) {
case '.':
*is_attribute = 1;
if (_FieldNameIterator_attr(self, name) == 0)
@ -382,7 +424,7 @@ FieldNameIterator_next(FieldNameIterator *self, int *is_attribute,
}
/* empty string is an error */
if (name->ptr == name->end) {
if (name->start == name->end) {
PyErr_SetString(PyExc_ValueError, "Empty attribute in format string");
return 0;
}
@ -398,24 +440,23 @@ FieldNameIterator_next(FieldNameIterator *self, int *is_attribute,
'rest' is an iterator to return the rest
*/
static int
field_name_split(STRINGLIB_CHAR *ptr, Py_ssize_t len, SubString *first,
field_name_split(PyObject *str, Py_ssize_t start, Py_ssize_t end, SubString *first,
Py_ssize_t *first_idx, FieldNameIterator *rest,
AutoNumber *auto_number)
{
STRINGLIB_CHAR c;
STRINGLIB_CHAR *p = ptr;
STRINGLIB_CHAR *end = ptr + len;
Py_UCS4 c;
Py_ssize_t i = start;
int field_name_is_empty;
int using_numeric_index;
/* find the part up until the first '.' or '[' */
while (p < end) {
switch (c = *p++) {
while (i < end) {
switch (c = PyUnicode_READ_CHAR(str, i++)) {
case '[':
case '.':
/* backup so that we this character is available to the
"rest" iterator */
p--;
i--;
break;
default:
continue;
@ -424,15 +465,15 @@ field_name_split(STRINGLIB_CHAR *ptr, Py_ssize_t len, SubString *first,
}
/* set up the return values */
SubString_init(first, ptr, p - ptr);
FieldNameIterator_init(rest, p, end - p);
SubString_init(first, str, start, i);
FieldNameIterator_init(rest, str, i, end);
/* see if "first" is an integer, in which case it's used as an index */
*first_idx = get_integer(first);
if (*first_idx == -1 && PyErr_Occurred())
return 0;
field_name_is_empty = first->ptr >= first->end;
field_name_is_empty = first->start >= first->end;
/* If the field name is omitted or if we have a numeric index
specified, then we're doing numeric indexing into args. */
@ -487,7 +528,7 @@ get_field_object(SubString *input, PyObject *args, PyObject *kwargs,
Py_ssize_t index;
FieldNameIterator rest;
if (!field_name_split(input->ptr, input->end - input->ptr, &first,
if (!field_name_split(input->str, input->start, input->end, &first,
&index, &rest, auto_number)) {
goto error;
}
@ -576,12 +617,8 @@ render_field(PyObject *fieldobj, SubString *format_spec, OutputString *output)
int ok = 0;
PyObject *result = NULL;
PyObject *format_spec_object = NULL;
PyObject *(*formatter)(PyObject *, STRINGLIB_CHAR *, Py_ssize_t) = NULL;
STRINGLIB_CHAR* format_spec_start = format_spec->ptr ?
format_spec->ptr : NULL;
Py_ssize_t format_spec_len = format_spec->ptr ?
format_spec->end - format_spec->ptr : 0;
PyObject *(*formatter)(PyObject *, PyObject *, Py_ssize_t, Py_ssize_t) = NULL;
/* If we know the type exactly, skip the lookup of __format__ and just
call the formatter directly. */
if (PyUnicode_CheckExact(fieldobj))
@ -597,39 +634,28 @@ render_field(PyObject *fieldobj, SubString *format_spec, OutputString *output)
if (formatter) {
/* we know exactly which formatter will be called when __format__ is
looked up, so call it directly, instead. */
result = formatter(fieldobj, format_spec_start, format_spec_len);
result = formatter(fieldobj, format_spec->str,
format_spec->start, format_spec->end);
}
else {
/* We need to create an object out of the pointers we have, because
__format__ takes a string/unicode object for format_spec. */
format_spec_object = STRINGLIB_NEW(format_spec_start,
format_spec_len);
if (format_spec->str)
format_spec_object = PyUnicode_Substring(format_spec->str,
format_spec->start,
format_spec->end);
else
format_spec_object = PyUnicode_New(0, 0);
if (format_spec_object == NULL)
goto done;
result = PyObject_Format(fieldobj, format_spec_object);
}
if (result == NULL)
if (result == NULL || PyUnicode_READY(result) == -1)
goto done;
#if PY_VERSION_HEX >= 0x03000000
assert(PyUnicode_Check(result));
#else
assert(PyBytes_Check(result) || PyUnicode_Check(result));
/* Convert result to our type. We could be str, and result could
be unicode */
{
PyObject *tmp = STRINGLIB_TOSTR(result);
if (tmp == NULL)
goto done;
Py_DECREF(result);
result = tmp;
}
#endif
ok = output_data(output,
STRINGLIB_STR(result), STRINGLIB_LEN(result));
ok = output_data(output, result, 0, PyUnicode_GET_LENGTH(result));
done:
Py_XDECREF(format_spec_object);
Py_XDECREF(result);
@ -638,23 +664,24 @@ done:
static int
parse_field(SubString *str, SubString *field_name, SubString *format_spec,
STRINGLIB_CHAR *conversion)
Py_UCS4 *conversion)
{
/* Note this function works if the field name is zero length,
which is good. Zero length field names are handled later, in
field_name_split. */
STRINGLIB_CHAR c = 0;
Py_UCS4 c = 0;
/* initialize these, as they may be empty */
*conversion = '\0';
SubString_init(format_spec, NULL, 0);
SubString_init(format_spec, NULL, 0, 0);
/* Search for the field name. it's terminated by the end of
the string, or a ':' or '!' */
field_name->ptr = str->ptr;
while (str->ptr < str->end) {
switch (c = *(str->ptr++)) {
field_name->str = str->str;
field_name->start = str->start;
while (str->start < str->end) {
switch ((c = PyUnicode_READ_CHAR(str->str, str->start++))) {
case ':':
case '!':
break;
@ -667,26 +694,27 @@ parse_field(SubString *str, SubString *field_name, SubString *format_spec,
if (c == '!' || c == ':') {
/* we have a format specifier and/or a conversion */
/* don't include the last character */
field_name->end = str->ptr-1;
field_name->end = str->start-1;
/* the format specifier is the rest of the string */
format_spec->ptr = str->ptr;
format_spec->str = str->str;
format_spec->start = str->start;
format_spec->end = str->end;
/* see if there's a conversion specifier */
if (c == '!') {
/* there must be another character present */
if (format_spec->ptr >= format_spec->end) {
if (format_spec->start >= format_spec->end) {
PyErr_SetString(PyExc_ValueError,
"end of format while looking for conversion "
"specifier");
return 0;
}
*conversion = *(format_spec->ptr++);
*conversion = PyUnicode_READ_CHAR(format_spec->str, format_spec->start++);
/* if there is another character, it must be a colon */
if (format_spec->ptr < format_spec->end) {
c = *(format_spec->ptr++);
if (format_spec->start < format_spec->end) {
c = PyUnicode_READ_CHAR(format_spec->str, format_spec->start++);
if (c != ':') {
PyErr_SetString(PyExc_ValueError,
"expected ':' after format specifier");
@ -697,7 +725,7 @@ parse_field(SubString *str, SubString *field_name, SubString *format_spec,
}
else
/* end of string, there's no format_spec or conversion */
field_name->end = str->ptr;
field_name->end = str->start;
return 1;
}
@ -716,9 +744,10 @@ typedef struct {
} MarkupIterator;
static int
MarkupIterator_init(MarkupIterator *self, STRINGLIB_CHAR *ptr, Py_ssize_t len)
MarkupIterator_init(MarkupIterator *self, PyObject *str,
Py_ssize_t start, Py_ssize_t end)
{
SubString_init(&self->str, ptr, len);
SubString_init(&self->str, str, start, end);
return 1;
}
@ -727,30 +756,30 @@ MarkupIterator_init(MarkupIterator *self, STRINGLIB_CHAR *ptr, Py_ssize_t len)
static int
MarkupIterator_next(MarkupIterator *self, SubString *literal,
int *field_present, SubString *field_name,
SubString *format_spec, STRINGLIB_CHAR *conversion,
SubString *format_spec, Py_UCS4 *conversion,
int *format_spec_needs_expanding)
{
int at_end;
STRINGLIB_CHAR c = 0;
STRINGLIB_CHAR *start;
Py_UCS4 c = 0;
Py_ssize_t start;
int count;
Py_ssize_t len;
int markup_follows = 0;
/* initialize all of the output variables */
SubString_init(literal, NULL, 0);
SubString_init(field_name, NULL, 0);
SubString_init(format_spec, NULL, 0);
SubString_init(literal, NULL, 0, 0);
SubString_init(field_name, NULL, 0, 0);
SubString_init(format_spec, NULL, 0, 0);
*conversion = '\0';
*format_spec_needs_expanding = 0;
*field_present = 0;
/* No more input, end of iterator. This is the normal exit
path. */
if (self->str.ptr >= self->str.end)
if (self->str.start >= self->str.end)
return 1;
start = self->str.ptr;
start = self->str.start;
/* First read any literal text. Read until the end of string, an
escaped '{' or '}', or an unescaped '{'. In order to never
@ -759,8 +788,8 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal,
including the brace, but no format object. The next time
through, we'll return the rest of the literal, skipping past
the second consecutive brace. */
while (self->str.ptr < self->str.end) {
switch (c = *(self->str.ptr++)) {
while (self->str.start < self->str.end) {
switch (c = PyUnicode_READ_CHAR(self->str.str, self->str.start++)) {
case '{':
case '}':
markup_follows = 1;
@ -771,10 +800,12 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal,
break;
}
at_end = self->str.ptr >= self->str.end;
len = self->str.ptr - start;
at_end = self->str.start >= self->str.end;
len = self->str.start - start;
if ((c == '}') && (at_end || (c != *self->str.ptr))) {
if ((c == '}') && (at_end ||
(c != PyUnicode_READ_CHAR(self->str.str,
self->str.start)))) {
PyErr_SetString(PyExc_ValueError, "Single '}' encountered "
"in format string");
return 0;
@ -785,10 +816,10 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal,
return 0;
}
if (!at_end) {
if (c == *self->str.ptr) {
if (c == PyUnicode_READ_CHAR(self->str.str, self->str.start)) {
/* escaped } or {, skip it in the input. there is no
markup object following us, just this literal text */
self->str.ptr++;
self->str.start++;
markup_follows = 0;
}
else
@ -796,7 +827,8 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal,
}
/* record the literal text */
literal->ptr = start;
literal->str = self->str.str;
literal->start = start;
literal->end = start + len;
if (!markup_follows)
@ -808,12 +840,12 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal,
*field_present = 1;
count = 1;
start = self->str.ptr;
start = self->str.start;
/* we know we can't have a zero length string, so don't worry
about that case */
while (self->str.ptr < self->str.end) {
switch (c = *(self->str.ptr++)) {
while (self->str.start < self->str.end) {
switch (c = PyUnicode_READ_CHAR(self->str.str, self->str.start++)) {
case '{':
/* the format spec needs to be recursively expanded.
this is an optimization, and not strictly needed */
@ -826,7 +858,7 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal,
/* we're done. parse and get out */
SubString s;
SubString_init(&s, start, self->str.ptr - 1 - start);
SubString_init(&s, self->str.str, start, self->str.start - 1);
if (parse_field(&s, field_name, format_spec, conversion) == 0)
return 0;
@ -845,7 +877,7 @@ MarkupIterator_next(MarkupIterator *self, SubString *literal,
/* do the !r or !s conversion on obj */
static PyObject *
do_conversion(PyObject *obj, STRINGLIB_CHAR conversion)
do_conversion(PyObject *obj, Py_UCS4 conversion)
{
/* XXX in pre-3.0, do we need to convert this to unicode, since it
might have returned a string? */
@ -853,11 +885,9 @@ do_conversion(PyObject *obj, STRINGLIB_CHAR conversion)
case 'r':
return PyObject_Repr(obj);
case 's':
return STRINGLIB_TOSTR(obj);
#if PY_VERSION_HEX >= 0x03000000
return PyObject_Str(obj);
case 'a':
return STRINGLIB_TOASCII(obj);
#endif
return PyObject_ASCII(obj);
default:
if (conversion > 32 && conversion < 127) {
/* It's the ASCII subrange; casting to char is safe
@ -889,7 +919,7 @@ do_conversion(PyObject *obj, STRINGLIB_CHAR conversion)
static int
output_markup(SubString *field_name, SubString *format_spec,
int format_spec_needs_expanding, STRINGLIB_CHAR conversion,
int format_spec_needs_expanding, Py_UCS4 conversion,
OutputString *output, PyObject *args, PyObject *kwargs,
int recursion_depth, AutoNumber *auto_number)
{
@ -906,7 +936,7 @@ output_markup(SubString *field_name, SubString *format_spec,
if (conversion != '\0') {
tmp = do_conversion(fieldobj, conversion);
if (tmp == NULL)
if (tmp == NULL || PyUnicode_READY(tmp) == -1)
goto done;
/* do the assignment, transferring ownership: fieldobj = tmp */
@ -919,14 +949,13 @@ output_markup(SubString *field_name, SubString *format_spec,
if (format_spec_needs_expanding) {
tmp = build_string(format_spec, args, kwargs, recursion_depth-1,
auto_number);
if (tmp == NULL)
if (tmp == NULL || PyUnicode_READY(tmp) == -1)
goto done;
/* note that in the case we're expanding the format string,
tmp must be kept around until after the call to
render_field. */
SubString_init(&expanded_format_spec,
STRINGLIB_STR(tmp), STRINGLIB_LEN(tmp));
SubString_init(&expanded_format_spec, tmp, 0, PyUnicode_GET_LENGTH(tmp));
actual_format_spec = &expanded_format_spec;
}
else
@ -961,14 +990,14 @@ do_markup(SubString *input, PyObject *args, PyObject *kwargs,
SubString literal;
SubString field_name;
SubString format_spec;
STRINGLIB_CHAR conversion;
Py_UCS4 conversion;
MarkupIterator_init(&iter, input->ptr, input->end - input->ptr);
MarkupIterator_init(&iter, input->str, input->start, input->end);
while ((result = MarkupIterator_next(&iter, &literal, &field_present,
&field_name, &format_spec,
&conversion,
&format_spec_needs_expanding)) == 2) {
if (!output_data(output, literal.ptr, literal.end - literal.ptr))
if (!output_data(output, literal.str, literal.start, literal.end))
return 0;
if (field_present)
if (!output_markup(&field_name, &format_spec,
@ -990,9 +1019,8 @@ build_string(SubString *input, PyObject *args, PyObject *kwargs,
{
OutputString output;
PyObject *result = NULL;
Py_ssize_t count;
output.obj = NULL; /* needed so cleanup code always works */
output.data = NULL; /* needed so cleanup code always works */
/* check the recursion level */
if (recursion_depth <= 0) {
@ -1004,7 +1032,7 @@ build_string(SubString *input, PyObject *args, PyObject *kwargs,
/* initial size is the length of the format string, plus the size
increment. seems like a reasonable default */
if (!output_initialize(&output,
input->end - input->ptr +
input->end - input->start +
INITIAL_SIZE_INCREMENT))
goto done;
@ -1013,17 +1041,14 @@ build_string(SubString *input, PyObject *args, PyObject *kwargs,
goto done;
}
count = output.ptr - STRINGLIB_STR(output.obj);
if (STRINGLIB_RESIZE(&output.obj, count) < 0) {
result = PyUnicode_New(output.pos, output.maxchar);
if (!result)
goto done;
}
/* transfer ownership to result */
result = output.obj;
output.obj = NULL;
memcpy(PyUnicode_DATA(result), output.data, output.pos << (output.kind-1));
done:
Py_XDECREF(output.obj);
if (output.data)
PyMem_Free(output.data);
return result;
}
@ -1045,8 +1070,11 @@ do_string_format(PyObject *self, PyObject *args, PyObject *kwargs)
AutoNumber auto_number;
if (PyUnicode_READY(self) == -1)
return NULL;
AutoNumber_Init(&auto_number);
SubString_init(&input, STRINGLIB_STR(self), STRINGLIB_LEN(self));
SubString_init(&input, self, 0, PyUnicode_GET_LENGTH(self));
return build_string(&input, args, kwargs, recursion_depth, &auto_number);
}
@ -1069,7 +1097,7 @@ do_string_format_map(PyObject *self, PyObject *obj)
typedef struct {
PyObject_HEAD
STRINGLIB_OBJECT *str;
PyUnicodeObject *str;
MarkupIterator it_markup;
} formatteriterobject;
@ -1095,7 +1123,7 @@ formatteriter_next(formatteriterobject *it)
SubString literal;
SubString field_name;
SubString format_spec;
STRINGLIB_CHAR conversion;
Py_UCS4 conversion;
int format_spec_needs_expanding;
int field_present;
int result = MarkupIterator_next(&it->it_markup, &literal, &field_present,
@ -1139,7 +1167,8 @@ formatteriter_next(formatteriterobject *it)
Py_INCREF(conversion_str);
}
else
conversion_str = STRINGLIB_NEW(&conversion, 1);
conversion_str = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND,
&conversion, 1);
if (conversion_str == NULL)
goto done;
@ -1196,7 +1225,7 @@ static PyTypeObject PyFormatterIter_Type = {
describing the parsed elements. It's a wrapper around
stringlib/string_format.h's MarkupIterator */
static PyObject *
formatter_parser(PyObject *ignored, STRINGLIB_OBJECT *self)
formatter_parser(PyObject *ignored, PyUnicodeObject *self)
{
formatteriterobject *it;
@ -1205,6 +1234,9 @@ formatter_parser(PyObject *ignored, STRINGLIB_OBJECT *self)
return NULL;
}
if (PyUnicode_READY(self) == -1)
return NULL;
it = PyObject_New(formatteriterobject, &PyFormatterIter_Type);
if (it == NULL)
return NULL;
@ -1214,10 +1246,7 @@ formatter_parser(PyObject *ignored, STRINGLIB_OBJECT *self)
it->str = self;
/* initialize the contained MarkupIterator */
MarkupIterator_init(&it->it_markup,
STRINGLIB_STR(self),
STRINGLIB_LEN(self));
MarkupIterator_init(&it->it_markup, (PyObject*)self, 0, PyUnicode_GET_LENGTH(self));
return (PyObject *)it;
}
@ -1234,7 +1263,7 @@ formatter_parser(PyObject *ignored, STRINGLIB_OBJECT *self)
typedef struct {
PyObject_HEAD
STRINGLIB_OBJECT *str;
PyUnicodeObject *str;
FieldNameIterator it_field;
} fieldnameiterobject;
@ -1336,7 +1365,7 @@ static PyTypeObject PyFieldNameIter_Type = {
field_name_split. The iterator it returns is a
FieldNameIterator */
static PyObject *
formatter_field_name_split(PyObject *ignored, STRINGLIB_OBJECT *self)
formatter_field_name_split(PyObject *ignored, PyUnicodeObject *self)
{
SubString first;
Py_ssize_t first_idx;
@ -1350,6 +1379,9 @@ formatter_field_name_split(PyObject *ignored, STRINGLIB_OBJECT *self)
return NULL;
}
if (PyUnicode_READY(self) == -1)
return NULL;
it = PyObject_New(fieldnameiterobject, &PyFieldNameIter_Type);
if (it == NULL)
return NULL;
@ -1361,8 +1393,7 @@ formatter_field_name_split(PyObject *ignored, STRINGLIB_OBJECT *self)
/* Pass in auto_number = NULL. We'll return an empty string for
first_obj in that case. */
if (!field_name_split(STRINGLIB_STR(self),
STRINGLIB_LEN(self),
if (!field_name_split((PyObject*)self, 0, PyUnicode_GET_LENGTH(self),
&first, &first_idx, &it->it_field, NULL))
goto done;

2
Objects/stringlib/unicodedefs.h
View file

@ -6,6 +6,8 @@
compiled as unicode. */
#define STRINGLIB_IS_UNICODE 1
#define FASTSEARCH fastsearch
#define STRINGLIB(F) stringlib_##F
#define STRINGLIB_OBJECT PyUnicodeObject
#define STRINGLIB_CHAR Py_UNICODE
#define STRINGLIB_TYPE_NAME "unicode"

18
Objects/typeobject.c
View file

@ -20,10 +20,11 @@
>> (8*sizeof(unsigned int) - MCACHE_SIZE_EXP))
#define MCACHE_HASH_METHOD(type, name) \
MCACHE_HASH((type)->tp_version_tag, \
((PyUnicodeObject *)(name))->hash)
((PyASCIIObject *)(name))->hash)
#define MCACHE_CACHEABLE_NAME(name) \
PyUnicode_CheckExact(name) && \
PyUnicode_GET_SIZE(name) <= MCACHE_MAX_ATTR_SIZE
PyUnicode_READY(name) != -1 && \
PyUnicode_GET_LENGTH(name) <= MCACHE_MAX_ATTR_SIZE
struct method_cache_entry {
unsigned int version;
@ -3489,7 +3490,7 @@ object_format(PyObject *self, PyObject *args)
if (self_as_str != NULL) {
/* Issue 7994: If we're converting to a string, we
should reject format specifications */
if (PyUnicode_GET_SIZE(format_spec) > 0) {
if (PyUnicode_GET_LENGTH(format_spec) > 0) {
if (PyErr_WarnEx(PyExc_DeprecationWarning,
"object.__format__ with a non-empty format "
"string is deprecated", 1) < 0) {
@ -5122,14 +5123,21 @@ slot_tp_str(PyObject *self)
return res;
}
else {
PyObject *ress;
/* PyObject *ress; */
PyErr_Clear();
res = slot_tp_repr(self);
if (!res)
return NULL;
/* XXX this is non-sensical. Why should we return
a bytes object from __str__. Is this code even
used? - mvl */
assert(0);
return res;
/*
ress = _PyUnicode_AsDefaultEncodedString(res);
Py_DECREF(res);
return ress;
*/
}
}
@ -6206,7 +6214,7 @@ super_getattro(PyObject *self, PyObject *name)
/* We want __class__ to return the class of the super object
(i.e. super, or a subclass), not the class of su->obj. */
skip = (PyUnicode_Check(name) &&
PyUnicode_GET_SIZE(name) == 9 &&
PyUnicode_GET_LENGTH(name) == 9 &&
PyUnicode_CompareWithASCIIString(name, "__class__") == 0);
}

5868
Objects/unicodeobject.c
View file

File diff suppressed because it is too large Load diff

91
Objects/uniops.h Normal file
View file

@ -0,0 +1,91 @@
size_t
UNIOP(strlen)(const UNIOP_t *u)
{
int res = 0;
while(*u++)
res++;
return res;
}
UNIOP_t*
UNIOP(strcpy)(UNIOP_t *s1, const UNIOP_t *s2)
{
UNIOP_t *u = s1;
while ((*u++ = *s2++));
return s1;
}
UNIOP_t*
UNIOP(strncpy)(UNIOP_t *s1, const UNIOP_t *s2, size_t n)
{
UNIOP_t *u = s1;
while ((*u++ = *s2++))
if (n-- == 0)
break;
return s1;
}
UNIOP_t*
UNIOP(strcat)(UNIOP_t *s1, const UNIOP_t *s2)
{
UNIOP_t *u1 = s1;
u1 += UNIOP(strlen(u1));
UNIOP(strcpy(u1, s2));
return s1;
}
int
UNIOP(strcmp)(const UNIOP_t *s1, const UNIOP_t *s2)
{
while (*s1 && *s2 && *s1 == *s2)
s1++, s2++;
if (*s1 && *s2)
return (*s1 < *s2) ? -1 : +1;
if (*s1)
return 1;
if (*s2)
return -1;
return 0;
}
int
UNIOP(strncmp)(const UNIOP_t *s1, const UNIOP_t *s2, size_t n)
{
register UNIOP_t u1, u2;
for (; n != 0; n--) {
u1 = *s1;
u2 = *s2;
if (u1 != u2)
return (u1 < u2) ? -1 : +1;
if (u1 == '\0')
return 0;
s1++;
s2++;
}
return 0;
}
UNIOP_t*
UNIOP(strchr)(const UNIOP_t *s, UNIOP_t c)
{
const UNIOP_t *p;
for (p = s; *p; p++)
if (*p == c)
return (UNIOP_t*)p;
return NULL;
}
UNIOP_t*
UNIOP(strrchr)(const UNIOP_t *s, UNIOP_t c)
{
const UNIOP_t *p;
p = s + UNIOP(strlen)(s);
while (p != s) {
p--;
if (*p == c)
return (UNIOP_t*)p;
}
return NULL;
}