mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-10 00:43:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 18

Remove now duplicate code in _json.c; instead, reuse the new private lib

Browse source
This commit is contained in:
Antoine Pitrou 2011-10-06 19:09:51 +02:00
parent c61c8d7a5e
commit 90c30e87be
1 changed files with 22 additions and 123 deletions
Download patch file Download diff file Expand all files Collapse all files

145
Modules/_json.c
View file

@ -75,17 +75,6 @@ static PyMemberDef encoder_members[] = {
{NULL} {NULL}
}; };
/*
* A two-level accumulator of unicode objects that avoids both the overhead
* of keeping a huge number of small separate objects, and the quadratic
* behaviour of using a naive repeated concatenation scheme.
*/
typedef struct {
PyObject *large; /* A list of previously accumulated large strings */
PyObject *small; /* Pending small strings */
} accumulator;
static PyObject * static PyObject *
join_list_unicode(PyObject *lst) join_list_unicode(PyObject *lst)
{ {
@ -99,96 +88,6 @@ join_list_unicode(PyObject *lst)
return PyUnicode_Join(sep, lst); return PyUnicode_Join(sep, lst);
} }
static int
init_accumulator(accumulator *acc)
{
acc->large = PyList_New(0);
if (acc->large == NULL)
return -1;
acc->small = PyList_New(0);
if (acc->small == NULL) {
Py_CLEAR(acc->large);
return -1;
}
return 0;
}
static int
flush_accumulator(accumulator *acc)
{
Py_ssize_t nsmall = PyList_GET_SIZE(acc->small);
if (nsmall) {
int ret;
PyObject *joined = join_list_unicode(acc->small);
if (joined == NULL)
return -1;
if (PyList_SetSlice(acc->small, 0, nsmall, NULL)) {
Py_DECREF(joined);
return -1;
}
ret = PyList_Append(acc->large, joined);
Py_DECREF(joined);
return ret;
}
return 0;
}
static int
accumulate_unicode(accumulator *acc, PyObject *obj)
{
int ret;
Py_ssize_t nsmall;
PyObject *joined;
assert(PyUnicode_Check(obj));
if (PyList_Append(acc->small, obj))
return -1;
nsmall = PyList_GET_SIZE(acc->small);
/* Each item in a list of unicode objects has an overhead (in 64-bit
* builds) of:
* - 8 bytes for the list slot
* - 56 bytes for the header of the unicode object
* that is, 64 bytes. 100000 such objects waste more than 6MB
* compared to a single concatenated string.
*/
if (nsmall < 100000)
return 0;
joined = join_list_unicode(acc->small);
if (joined == NULL)
return -1;
if (PyList_SetSlice(acc->small, 0, nsmall, NULL)) {
Py_DECREF(joined);
return -1;
}
ret = PyList_Append(acc->large, joined);
Py_DECREF(joined);
return ret;
}
static PyObject *
finish_accumulator(accumulator *acc)
{
int ret;
PyObject *res;
ret = flush_accumulator(acc);
Py_CLEAR(acc->small);
if (ret) {
Py_CLEAR(acc->large);
return NULL;
}
res = acc->large;
acc->large = NULL;
return res;
}
static void
destroy_accumulator(accumulator *acc)
{
Py_CLEAR(acc->small);
Py_CLEAR(acc->large);
}
/* Forward decls */ /* Forward decls */
static PyObject * static PyObject *
@ -217,11 +116,11 @@ encoder_dealloc(PyObject *self);
static int static int
encoder_clear(PyObject *self); encoder_clear(PyObject *self);
static int static int
encoder_listencode_list(PyEncoderObject *s, accumulator *acc, PyObject *seq, Py_ssize_t indent_level); encoder_listencode_list(PyEncoderObject *s, _PyAccu *acc, PyObject *seq, Py_ssize_t indent_level);
static int static int
encoder_listencode_obj(PyEncoderObject *s, accumulator *acc, PyObject *obj, Py_ssize_t indent_level); encoder_listencode_obj(PyEncoderObject *s, _PyAccu *acc, PyObject *obj, Py_ssize_t indent_level);
static int static int
encoder_listencode_dict(PyEncoderObject *s, accumulator *acc, PyObject *dct, Py_ssize_t indent_level); encoder_listencode_dict(PyEncoderObject *s, _PyAccu *acc, PyObject *dct, Py_ssize_t indent_level);
static PyObject * static PyObject *
_encoded_const(PyObject *obj); _encoded_const(PyObject *obj);
static void static void
@ -1383,20 +1282,20 @@ encoder_call(PyObject *self, PyObject *args, PyObject *kwds)
PyObject *obj; PyObject *obj;
Py_ssize_t indent_level; Py_ssize_t indent_level;
PyEncoderObject *s; PyEncoderObject *s;
accumulator acc; _PyAccu acc;
assert(PyEncoder_Check(self)); assert(PyEncoder_Check(self));
s = (PyEncoderObject *)self; s = (PyEncoderObject *)self;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:_iterencode", kwlist, if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:_iterencode", kwlist,
&obj, _convertPyInt_AsSsize_t, &indent_level)) &obj, _convertPyInt_AsSsize_t, &indent_level))
return NULL; return NULL;
if (init_accumulator(&acc)) if (_PyAccu_Init(&acc))
return NULL; return NULL;
if (encoder_listencode_obj(s, &acc, obj, indent_level)) { if (encoder_listencode_obj(s, &acc, obj, indent_level)) {
destroy_accumulator(&acc); _PyAccu_Destroy(&acc);
return NULL; return NULL;
} }
return finish_accumulator(&acc); return _PyAccu_FinishAsList(&acc);
} }
static PyObject * static PyObject *
@ -1468,16 +1367,16 @@ encoder_encode_string(PyEncoderObject *s, PyObject *obj)
} }
static int static int
_steal_accumulate(accumulator *acc, PyObject *stolen) _steal_accumulate(_PyAccu *acc, PyObject *stolen)
{ {
/* Append stolen and then decrement its reference count */ /* Append stolen and then decrement its reference count */
int rval = accumulate_unicode(acc, stolen); int rval = _PyAccu_Accumulate(acc, stolen);
Py_DECREF(stolen); Py_DECREF(stolen);
return rval; return rval;
} }
static int static int
encoder_listencode_obj(PyEncoderObject *s, accumulator *acc, encoder_listencode_obj(PyEncoderObject *s, _PyAccu *acc,
PyObject *obj, Py_ssize_t indent_level) PyObject *obj, Py_ssize_t indent_level)
{ {
/* Encode Python object obj to a JSON term */ /* Encode Python object obj to a JSON term */
@ -1570,7 +1469,7 @@ encoder_listencode_obj(PyEncoderObject *s, accumulator *acc,
} }
static int static int
encoder_listencode_dict(PyEncoderObject *s, accumulator *acc, encoder_listencode_dict(PyEncoderObject *s, _PyAccu *acc,
PyObject *dct, Py_ssize_t indent_level) PyObject *dct, Py_ssize_t indent_level)
{ {
/* Encode Python dict dct a JSON term */ /* Encode Python dict dct a JSON term */
@ -1593,7 +1492,7 @@ encoder_listencode_dict(PyEncoderObject *s, accumulator *acc,
return -1; return -1;
} }
if (Py_SIZE(dct) == 0) if (Py_SIZE(dct) == 0)
return accumulate_unicode(acc, empty_dict); return _PyAccu_Accumulate(acc, empty_dict);
if (s->markers != Py_None) { if (s->markers != Py_None) {
int has_key; int has_key;
@ -1611,7 +1510,7 @@ encoder_listencode_dict(PyEncoderObject *s, accumulator *acc,
} }
} }
if (accumulate_unicode(acc, open_dict)) if (_PyAccu_Accumulate(acc, open_dict))
goto bail; goto bail;
if (s->indent != Py_None) { if (s->indent != Py_None) {
@ -1698,7 +1597,7 @@ encoder_listencode_dict(PyEncoderObject *s, accumulator *acc,
} }
if (idx) { if (idx) {
if (accumulate_unicode(acc, s->item_separator)) if (_PyAccu_Accumulate(acc, s->item_separator))
goto bail; goto bail;
} }
@ -1706,12 +1605,12 @@ encoder_listencode_dict(PyEncoderObject *s, accumulator *acc,
Py_CLEAR(kstr); Py_CLEAR(kstr);
if (encoded == NULL) if (encoded == NULL)
goto bail; goto bail;
if (accumulate_unicode(acc, encoded)) { if (_PyAccu_Accumulate(acc, encoded)) {
Py_DECREF(encoded); Py_DECREF(encoded);
goto bail; goto bail;
} }
Py_DECREF(encoded); Py_DECREF(encoded);
if (accumulate_unicode(acc, s->key_separator)) if (_PyAccu_Accumulate(acc, s->key_separator))
goto bail; goto bail;
value = PyTuple_GET_ITEM(item, 1); value = PyTuple_GET_ITEM(item, 1);
@ -1735,7 +1634,7 @@ encoder_listencode_dict(PyEncoderObject *s, accumulator *acc,
yield '\n' + (' ' * (_indent * _current_indent_level)) yield '\n' + (' ' * (_indent * _current_indent_level))
}*/ }*/
if (accumulate_unicode(acc, close_dict)) if (_PyAccu_Accumulate(acc, close_dict))
goto bail; goto bail;
return 0; return 0;
@ -1749,7 +1648,7 @@ bail:
static int static int
encoder_listencode_list(PyEncoderObject *s, accumulator *acc, encoder_listencode_list(PyEncoderObject *s, _PyAccu *acc,
PyObject *seq, Py_ssize_t indent_level) PyObject *seq, Py_ssize_t indent_level)
{ {
/* Encode Python list seq to a JSON term */ /* Encode Python list seq to a JSON term */
@ -1776,7 +1675,7 @@ encoder_listencode_list(PyEncoderObject *s, accumulator *acc,
num_items = PySequence_Fast_GET_SIZE(s_fast); num_items = PySequence_Fast_GET_SIZE(s_fast);
if (num_items == 0) { if (num_items == 0) {
Py_DECREF(s_fast); Py_DECREF(s_fast);
return accumulate_unicode(acc, empty_array); return _PyAccu_Accumulate(acc, empty_array);
} }
if (s->markers != Py_None) { if (s->markers != Py_None) {
@ -1796,7 +1695,7 @@ encoder_listencode_list(PyEncoderObject *s, accumulator *acc,
} }
seq_items = PySequence_Fast_ITEMS(s_fast); seq_items = PySequence_Fast_ITEMS(s_fast);
if (accumulate_unicode(acc, open_array)) if (_PyAccu_Accumulate(acc, open_array))
goto bail; goto bail;
if (s->indent != Py_None) { if (s->indent != Py_None) {
/* TODO: DOES NOT RUN */ /* TODO: DOES NOT RUN */
@ -1810,7 +1709,7 @@ encoder_listencode_list(PyEncoderObject *s, accumulator *acc,
for (i = 0; i < num_items; i++) { for (i = 0; i < num_items; i++) {
PyObject *obj = seq_items[i]; PyObject *obj = seq_items[i];
if (i) { if (i) {
if (accumulate_unicode(acc, s->item_separator)) if (_PyAccu_Accumulate(acc, s->item_separator))
goto bail; goto bail;
} }
if (encoder_listencode_obj(s, acc, obj, indent_level)) if (encoder_listencode_obj(s, acc, obj, indent_level))
@ -1828,7 +1727,7 @@ encoder_listencode_list(PyEncoderObject *s, accumulator *acc,
yield '\n' + (' ' * (_indent * _current_indent_level)) yield '\n' + (' ' * (_indent * _current_indent_level))
}*/ }*/
if (accumulate_unicode(acc, close_array)) if (_PyAccu_Accumulate(acc, close_array))
goto bail; goto bail;
Py_DECREF(s_fast); Py_DECREF(s_fast);
return 0; return 0;