mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-17 04:08:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

gh-126004: Fix positions handling in codecs.backslashreplace_errors (#127676)

Browse source 
This fixes how `PyCodec_BackslashReplaceErrors` handles the `start` and `end`
attributes of `UnicodeError` objects via the `_PyUnicodeError_GetParams` helper.
This commit is contained in:
Bénédikt Tran 2025-01-23 14:28:33 +01:00 • committed by GitHub
parent 5c9a63f62c
commit 25a614a502
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 71 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Lib/test/test_capi/test_codecs.py
View file

@ -849,7 +849,8 @@ class CAPICodecErrors(unittest.TestCase):
def test_codec_backslashreplace_errors_handler(self): def test_codec_backslashreplace_errors_handler(self):
handler = _testcapi.codec_backslashreplace_errors handler = _testcapi.codec_backslashreplace_errors
self.do_test_codec_errors_handler(handler, self.all_unicode_errors) self.do_test_codec_errors_handler(handler, self.all_unicode_errors,
safe=True)
def test_codec_namereplace_errors_handler(self): def test_codec_namereplace_errors_handler(self):
handler = _testlimitedcapi.codec_namereplace_errors handler = _testlimitedcapi.codec_namereplace_errors

3
Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-11-30-58.gh-issue-126004.-p8MAS.rst Normal file
View file

@ -0,0 +1,3 @@
Fix handling of :attr:`UnicodeError.start` and :attr:`UnicodeError.end`
values in the :func:`codecs.backslashreplace_errors` error handler. Patch by
Bénédikt Tran.

133
Python/codecs.c
View file

@ -864,108 +864,107 @@ PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc)
PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc) PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc)
{ {
PyObject *object; PyObject *obj;
Py_ssize_t i; Py_ssize_t objlen, start, end, slen;
Py_ssize_t start;
Py_ssize_t end;
PyObject *res;
Py_UCS1 *outp;
int ressize;
Py_UCS4 c;
if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeDecodeError)) { if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeDecodeError)) {
const unsigned char *p; if (_PyUnicodeError_GetParams(exc,
if (PyUnicodeDecodeError_GetStart(exc, &start)) &obj, &objlen,
return NULL; &start, &end, &slen, true) < 0)
if (PyUnicodeDecodeError_GetEnd(exc, &end)) {
return NULL;
if (!(object = PyUnicodeDecodeError_GetObject(exc)))
return NULL;
p = (const unsigned char*)PyBytes_AS_STRING(object);
res = PyUnicode_New(4 * (end - start), 127);
if (res == NULL) {
Py_DECREF(object);
return NULL; return NULL;
} }
outp = PyUnicode_1BYTE_DATA(res); PyObject *res = PyUnicode_New(4 * slen, 127);
for (i = start; i < end; i++, outp += 4) { if (res == NULL) {
unsigned char c = p[i]; Py_DECREF(obj);
return NULL;
}
Py_UCS1 *outp = PyUnicode_1BYTE_DATA(res);
const unsigned char *p = (const unsigned char *)PyBytes_AS_STRING(obj);
for (Py_ssize_t i = start; i < end; i++, outp += 4) {
const unsigned char ch = p[i];
outp[0] = '\\'; outp[0] = '\\';
outp[1] = 'x'; outp[1] = 'x';
outp[2] = Py_hexdigits[(c>>4)&0xf]; outp[2] = Py_hexdigits[(ch >> 4) & 0xf];
outp[3] = Py_hexdigits[c&0xf]; outp[3] = Py_hexdigits[ch & 0xf];
} }
assert(_PyUnicode_CheckConsistency(res, 1)); assert(_PyUnicode_CheckConsistency(res, 1));
Py_DECREF(object); Py_DECREF(obj);
return Py_BuildValue("(Nn)", res, end); return Py_BuildValue("(Nn)", res, end);
} }
if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) {
if (PyUnicodeEncodeError_GetStart(exc, &start)) if (
return NULL; PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)
if (PyUnicodeEncodeError_GetEnd(exc, &end)) || PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeTranslateError)
return NULL; ) {
if (!(object = PyUnicodeEncodeError_GetObject(exc))) if (_PyUnicodeError_GetParams(exc,
return NULL; &obj, &objlen,
} &start, &end, &slen, false) < 0)
else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeTranslateError)) { {
if (PyUnicodeTranslateError_GetStart(exc, &start))
return NULL;
if (PyUnicodeTranslateError_GetEnd(exc, &end))
return NULL;
if (!(object = PyUnicodeTranslateError_GetObject(exc)))
return NULL; return NULL;
}
} }
else { else {
wrong_exception_type(exc); wrong_exception_type(exc);
return NULL; return NULL;
} }
if (end - start > PY_SSIZE_T_MAX / (1+1+8)) // The number of characters that each character 'ch' contributes
end = start + PY_SSIZE_T_MAX / (1+1+8); // in the result is 1 + 1 + k, where k >= min{t >= 1 | 16^t > ch}
for (i = start, ressize = 0; i < end; ++i) { // and will be formatted as "\\" + ('U'|'u'|'x') + HEXDIGITS,
// where the number of hexdigits is either 2, 4, or 8 (not 6).
// Since the Unicode range is below 10^7, we choose k = 8 whence
// each "block" requires at most 1 + 1 + 8 characters.
if (slen > PY_SSIZE_T_MAX / (1 + 1 + 8)) {
end = start + PY_SSIZE_T_MAX / (1 + 1 + 8);
end = Py_MIN(end, objlen);
slen = Py_MAX(0, end - start);
}
Py_ssize_t ressize = 0;
for (Py_ssize_t i = start; i < end; ++i) {
/* object is guaranteed to be "ready" */ /* object is guaranteed to be "ready" */
c = PyUnicode_READ_CHAR(object, i); Py_UCS4 c = PyUnicode_READ_CHAR(obj, i);
if (c >= 0x10000) { if (c >= 0x10000) {
ressize += 1+1+8; ressize += 1 + 1 + 8;
} }
else if (c >= 0x100) { else if (c >= 0x100) {
ressize += 1+1+4; ressize += 1 + 1 + 4;
}
else {
ressize += 1 + 1 + 2;
} }
else
ressize += 1+1+2;
} }
res = PyUnicode_New(ressize, 127); PyObject *res = PyUnicode_New(ressize, 127);
if (res == NULL) { if (res == NULL) {
Py_DECREF(object); Py_DECREF(obj);
return NULL; return NULL;
} }
outp = PyUnicode_1BYTE_DATA(res); Py_UCS1 *outp = PyUnicode_1BYTE_DATA(res);
for (i = start; i < end; ++i) { for (Py_ssize_t i = start; i < end; ++i) {
c = PyUnicode_READ_CHAR(object, i); Py_UCS4 c = PyUnicode_READ_CHAR(obj, i);
*outp++ = '\\'; *outp++ = '\\';
if (c >= 0x00010000) { if (c >= 0x00010000) {
*outp++ = 'U'; *outp++ = 'U';
*outp++ = Py_hexdigits[(c>>28)&0xf]; *outp++ = Py_hexdigits[(c >> 28) & 0xf];
*outp++ = Py_hexdigits[(c>>24)&0xf]; *outp++ = Py_hexdigits[(c >> 24) & 0xf];
*outp++ = Py_hexdigits[(c>>20)&0xf]; *outp++ = Py_hexdigits[(c >> 20) & 0xf];
*outp++ = Py_hexdigits[(c>>16)&0xf]; *outp++ = Py_hexdigits[(c >> 16) & 0xf];
*outp++ = Py_hexdigits[(c>>12)&0xf]; *outp++ = Py_hexdigits[(c >> 12) & 0xf];
*outp++ = Py_hexdigits[(c>>8)&0xf]; *outp++ = Py_hexdigits[(c >> 8) & 0xf];
} }
else if (c >= 0x100) { else if (c >= 0x100) {
*outp++ = 'u'; *outp++ = 'u';
*outp++ = Py_hexdigits[(c>>12)&0xf]; *outp++ = Py_hexdigits[(c >> 12) & 0xf];
*outp++ = Py_hexdigits[(c>>8)&0xf]; *outp++ = Py_hexdigits[(c >> 8) & 0xf];
} }
else else {
*outp++ = 'x'; *outp++ = 'x';
*outp++ = Py_hexdigits[(c>>4)&0xf]; }
*outp++ = Py_hexdigits[c&0xf]; *outp++ = Py_hexdigits[(c >> 4) & 0xf];
*outp++ = Py_hexdigits[c & 0xf];
} }
assert(_PyUnicode_CheckConsistency(res, 1)); assert(_PyUnicode_CheckConsistency(res, 1));
Py_DECREF(object); Py_DECREF(obj);
return Py_BuildValue("(Nn)", res, end); return Py_BuildValue("(Nn)", res, end);
} }