mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-28 12:45:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

bpo-29240: readline now ignores the UTF-8 Mode (#5145)

Browse source 
Add new fuctions ignoring the UTF-8 mode:

* _Py_DecodeCurrentLocale()
* _Py_EncodeCurrentLocale()
* _PyUnicode_DecodeCurrentLocaleAndSize()
* _PyUnicode_EncodeCurrentLocale()

Modify the readline module to use these functions.

Re-enable test_readline.test_nonascii().
This commit is contained in:
Victor Stinner 2018-01-10 22:46:15 +01:00 • committed by GitHub
parent f80c0ca133
commit 2cba6b8579
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 126 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

62
Objects/unicodeobject.c
View file

@ -3395,8 +3395,8 @@ locale_error_handler(const char *errors, int *surrogateescape)
}
}
PyObject *
PyUnicode_EncodeLocale(PyObject *unicode, const char *errors)
static PyObject *
unicode_encode_locale(PyObject *unicode, const char *errors, int current_locale)
{
Py_ssize_t wlen, wlen2;
wchar_t *wstr;
@ -3423,7 +3423,12 @@ PyUnicode_EncodeLocale(PyObject *unicode, const char *errors)
/* "surrogateescape" error handler */
char *str;
str = Py_EncodeLocale(wstr, &error_pos);
if (current_locale) {
str = _Py_EncodeCurrentLocale(wstr, &error_pos);
}
else {
str = Py_EncodeLocale(wstr, &error_pos);
}
if (str == NULL) {
if (error_pos == (size_t)-1) {
PyErr_NoMemory();
@ -3437,7 +3442,12 @@ PyUnicode_EncodeLocale(PyObject *unicode, const char *errors)
PyMem_Free(wstr);
bytes = PyBytes_FromString(str);
PyMem_Free(str);
if (current_locale) {
PyMem_RawFree(str);
}
else {
PyMem_Free(str);
}
}
else {
/* strict mode */
@ -3502,6 +3512,18 @@ encode_error:
return NULL;
}
PyObject *
PyUnicode_EncodeLocale(PyObject *unicode, const char *errors)
{
return unicode_encode_locale(unicode, errors, 0);
}
PyObject *
_PyUnicode_EncodeCurrentLocale(PyObject *unicode, const char *errors)
{
return unicode_encode_locale(unicode, errors, 1);
}
PyObject *
PyUnicode_EncodeFSDefault(PyObject *unicode)
{
@ -3524,7 +3546,8 @@ PyUnicode_EncodeFSDefault(PyObject *unicode)
Py_FileSystemDefaultEncodeErrors);
}
else {
return PyUnicode_EncodeLocale(unicode, Py_FileSystemDefaultEncodeErrors);
return unicode_encode_locale(unicode,
Py_FileSystemDefaultEncodeErrors, 0);
}
#endif
}
@ -3695,9 +3718,9 @@ mbstowcs_errorpos(const char *str, size_t len)
return 0;
}
PyObject*
PyUnicode_DecodeLocaleAndSize(const char *str, Py_ssize_t len,
const char *errors)
static PyObject*
unicode_decode_locale(const char *str, Py_ssize_t len, const char *errors,
int current_locale)
{
wchar_t smallbuf[256];
size_t smallbuf_len = Py_ARRAY_LENGTH(smallbuf);
@ -3719,7 +3742,12 @@ PyUnicode_DecodeLocaleAndSize(const char *str, Py_ssize_t len,
if (surrogateescape) {
/* "surrogateescape" error handler */
wstr = Py_DecodeLocale(str, &wlen);
if (current_locale) {
wstr = _Py_DecodeCurrentLocale(str, &wlen);
}
else {
wstr = Py_DecodeLocale(str, &wlen);
}
if (wstr == NULL) {
if (wlen == (size_t)-1)
PyErr_NoMemory();
@ -3794,11 +3822,25 @@ decode_error:
return NULL;
}
PyObject*
PyUnicode_DecodeLocaleAndSize(const char *str, Py_ssize_t len,
const char *errors)
{
return unicode_decode_locale(str, len, errors, 0);
}
PyObject*
_PyUnicode_DecodeCurrentLocaleAndSize(const char *str, Py_ssize_t len,
const char *errors)
{
return unicode_decode_locale(str, len, errors, 1);
}
PyObject*
PyUnicode_DecodeLocale(const char *str, const char *errors)
{
Py_ssize_t size = (Py_ssize_t)strlen(str);
return PyUnicode_DecodeLocaleAndSize(str, size, errors);
return unicode_decode_locale(str, size, errors, 0);
}