mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-23 18:24:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

bpo-36311: Fixes decoding multibyte characters around chunk boundaries and improves decoding performance (GH-15083)

Browse source 
(cherry picked from commit 7ebdda0dbe)

Co-authored-by: Steve Dower <steve.dower@python.org>
This commit is contained in:
Miss Islington (bot) 2019-08-21 16:53:56 -07:00 • committed by GitHub
parent 9eb3d54639
commit f93c15aedc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 29 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

16
Objects/unicodeobject.c
View file

@ -7118,6 +7118,12 @@ PyUnicode_AsASCIIString(PyObject *unicode)
#define NEED_RETRY
#endif
/* INT_MAX is the theoretical largest chunk (or INT_MAX / 2 when
transcoding from UTF-16), but INT_MAX / 4 perfoms better in
both cases also and avoids partial characters overrunning the
length limit in MultiByteToWideChar on Windows */
#define DECODING_CHUNK_SIZE (INT_MAX/4)
#ifndef WC_ERR_INVALID_CHARS
# define WC_ERR_INVALID_CHARS 0x0080
#endif
@ -7354,8 +7360,8 @@ decode_code_page_stateful(int code_page,
do
{
#ifdef NEED_RETRY
if (size > INT_MAX) {
chunk_size = INT_MAX;
if (size > DECODING_CHUNK_SIZE) {
chunk_size = DECODING_CHUNK_SIZE;
final = 0;
done = 0;
}
@ -7759,10 +7765,8 @@ encode_code_page(int code_page,
do
{
#ifdef NEED_RETRY
/* UTF-16 encoding may double the size, so use only INT_MAX/2
chunks. */
if (len > INT_MAX/2) {
chunk_len = INT_MAX/2;
if (len > DECODING_CHUNK_SIZE) {
chunk_len = DECODING_CHUNK_SIZE;
done = 0;
}
else