mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-25 11:15:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

bpo-36311: Fixes decoding multibyte characters around chunk boundaries and improves decoding performance (GH-15083)

Browse source
This commit is contained in:
Steve Dower 2019-08-21 16:22:33 -07:00 • committed by GitHub
parent df0c21ff46
commit 7ebdda0dbe
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 29 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

16
Objects/unicodeobject.c
View file

@ -7186,6 +7186,12 @@ PyUnicode_AsASCIIString(PyObject *unicode)
#define NEED_RETRY
#endif
/* INT_MAX is the theoretical largest chunk (or INT_MAX / 2 when
transcoding from UTF-16), but INT_MAX / 4 perfoms better in
both cases also and avoids partial characters overrunning the
length limit in MultiByteToWideChar on Windows */
#define DECODING_CHUNK_SIZE (INT_MAX/4)
#ifndef WC_ERR_INVALID_CHARS
# define WC_ERR_INVALID_CHARS 0x0080
#endif
@ -7422,8 +7428,8 @@ decode_code_page_stateful(int code_page,
do
{
#ifdef NEED_RETRY
if (size > INT_MAX) {
chunk_size = INT_MAX;
if (size > DECODING_CHUNK_SIZE) {
chunk_size = DECODING_CHUNK_SIZE;
final = 0;
done = 0;
}
@ -7827,10 +7833,8 @@ encode_code_page(int code_page,
do
{
#ifdef NEED_RETRY
/* UTF-16 encoding may double the size, so use only INT_MAX/2
chunks. */
if (len > INT_MAX/2) {
chunk_len = INT_MAX/2;
if (len > DECODING_CHUNK_SIZE) {
chunk_len = DECODING_CHUNK_SIZE;
done = 0;
}
else