mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-29 05:05:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

bpo-34485: Add _PyCoreConfig.stdio_encoding (GH-8881)

Browse source 
* Add stdio_encoding and stdio_errors fields to _PyCoreConfig.
* Add unit tests on stdio_encoding and stdio_errors.
This commit is contained in:
Victor Stinner 2018-08-29 11:47:29 +02:00 • committed by GitHub
parent 177d921c8c
commit dfe0dc7453
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 265 additions and 135 deletions
Download patch file Download diff file Expand all files Collapse all files

181
Python/coreconfig.c
View file

@ -1,6 +1,9 @@
#include "Python.h"
#include "internal/pystate.h"
#include <locale.h>
#ifdef HAVE_LANGINFO_H
# include <langinfo.h>
#endif
#define DECODE_LOCALE_ERR(NAME, LEN) \
@ -89,8 +92,8 @@ _Py_wstrlist_copy(int len, wchar_t **list)
* mechanism that attempts to figure out an appropriate IO encoding
*/
char *_Py_StandardStreamEncoding = NULL;
char *_Py_StandardStreamErrors = NULL;
static char *_Py_StandardStreamEncoding = NULL;
static char *_Py_StandardStreamErrors = NULL;
int
Py_SetStandardStreamEncoding(const char *encoding, const char *errors)
@ -205,6 +208,9 @@ _PyCoreConfig_Clear(_PyCoreConfig *config)
CLEAR(config->dll_path);
#endif
CLEAR(config->base_exec_prefix);
CLEAR(config->stdio_encoding);
CLEAR(config->stdio_errors);
#undef CLEAR
#undef CLEAR_WSTRLIST
}
@ -216,6 +222,15 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2)
_PyCoreConfig_Clear(config);
#define COPY_ATTR(ATTR) config->ATTR = config2->ATTR
#define COPY_STR_ATTR(ATTR) \
do { \
if (config2->ATTR != NULL) { \
config->ATTR = _PyMem_RawStrdup(config2->ATTR); \
if (config->ATTR == NULL) { \
return -1; \
} \
} \
} while (0)
#define COPY_WSTR_ATTR(ATTR) \
do { \
if (config2->ATTR != NULL) { \
@ -287,6 +302,8 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2)
COPY_ATTR(quiet);
COPY_ATTR(user_site_directory);
COPY_ATTR(buffered_stdio);
COPY_STR_ATTR(stdio_encoding);
COPY_STR_ATTR(stdio_errors);
#ifdef MS_WINDOWS
COPY_ATTR(legacy_windows_fs_encoding);
COPY_ATTR(legacy_windows_stdio);
@ -932,6 +949,161 @@ config_init_locale(_PyCoreConfig *config)
}
static const char *
get_stdio_errors(const _PyCoreConfig *config)
{
#ifndef MS_WINDOWS
const char *loc = setlocale(LC_CTYPE, NULL);
if (loc != NULL) {
/* surrogateescape is the default in the legacy C and POSIX locales */
if (strcmp(loc, "C") == 0 || strcmp(loc, "POSIX") == 0) {
return "surrogateescape";
}
#ifdef PY_COERCE_C_LOCALE
/* surrogateescape is the default in locale coercion target locales */
if (_Py_IsLocaleCoercionTarget(loc)) {
return "surrogateescape";
}
#endif
}
return "strict";
#else
/* On Windows, always use surrogateescape by default */
return "surrogateescape";
#endif
}
_PyInitError
_Py_get_locale_encoding(char **locale_encoding)
{
#ifdef MS_WINDOWS
char encoding[20];
PyOS_snprintf(encoding, sizeof(encoding), "cp%d", GetACP());
#elif defined(__ANDROID__)
const char *encoding = "UTF-8";
#else
const char *encoding = nl_langinfo(CODESET);
if (!encoding || encoding[0] == '\0') {
return _Py_INIT_USER_ERR("failed to get the locale encoding: "
"nl_langinfo(CODESET) failed");
}
#endif
*locale_encoding = _PyMem_RawStrdup(encoding);
if (*locale_encoding == NULL) {
return _Py_INIT_NO_MEMORY();
}
return _Py_INIT_OK();
}
static _PyInitError
config_init_stdio_encoding(_PyCoreConfig *config)
{
/* If Py_SetStandardStreamEncoding() have been called, use these
parameters. */
if (config->stdio_encoding == NULL && _Py_StandardStreamEncoding != NULL) {
config->stdio_encoding = _PyMem_RawStrdup(_Py_StandardStreamEncoding);
if (config->stdio_encoding == NULL) {
return _Py_INIT_NO_MEMORY();
}
}
if (config->stdio_errors == NULL && _Py_StandardStreamErrors != NULL) {
config->stdio_errors = _PyMem_RawStrdup(_Py_StandardStreamErrors);
if (config->stdio_errors == NULL) {
return _Py_INIT_NO_MEMORY();
}
}
if (config->stdio_encoding != NULL && config->stdio_errors != NULL) {
return _Py_INIT_OK();
}
/* PYTHONIOENCODING environment variable */
const char *opt = _PyCoreConfig_GetEnv(config, "PYTHONIOENCODING");
if (opt) {
char *pythonioencoding = _PyMem_RawStrdup(opt);
if (pythonioencoding == NULL) {
return _Py_INIT_NO_MEMORY();
}
char *err = strchr(pythonioencoding, ':');
if (err) {
*err = '\0';
err++;
if (!err[0]) {
err = NULL;
}
}
/* Does PYTHONIOENCODING contain an encoding? */
if (pythonioencoding[0]) {
if (config->stdio_encoding == NULL) {
config->stdio_encoding = _PyMem_RawStrdup(pythonioencoding);
if (config->stdio_encoding == NULL) {
PyMem_RawFree(pythonioencoding);
return _Py_INIT_NO_MEMORY();
}
}
/* If the encoding is set but not the error handler,
use "strict" error handler by default.
PYTHONIOENCODING=latin1 behaves as
PYTHONIOENCODING=latin1:strict. */
if (!err) {
err = "strict";
}
}
if (config->stdio_errors == NULL && err != NULL) {
config->stdio_errors = _PyMem_RawStrdup(err);
if (config->stdio_errors == NULL) {
PyMem_RawFree(pythonioencoding);
return _Py_INIT_NO_MEMORY();
}
}
PyMem_RawFree(pythonioencoding);
}
/* UTF-8 Mode uses UTF-8/surrogateescape */
if (config->utf8_mode) {
if (config->stdio_encoding == NULL) {
config->stdio_encoding = _PyMem_RawStrdup("utf-8");
if (config->stdio_encoding == NULL) {
return _Py_INIT_NO_MEMORY();
}
}
if (config->stdio_errors == NULL) {
config->stdio_errors = _PyMem_RawStrdup("surrogateescape");
if (config->stdio_errors == NULL) {
return _Py_INIT_NO_MEMORY();
}
}
}
/* Choose the default error handler based on the current locale. */
if (config->stdio_encoding == NULL) {
_PyInitError err = _Py_get_locale_encoding(&config->stdio_encoding);
if (_Py_INIT_FAILED(err)) {
return err;
}
}
if (config->stdio_errors == NULL) {
const char *errors = get_stdio_errors(config);
config->stdio_errors = _PyMem_RawStrdup(errors);
if (config->stdio_errors == NULL) {
return _Py_INIT_NO_MEMORY();
}
}
return _Py_INIT_OK();
}
/* Read configuration settings from standard locations
*
* This function doesn't make any changes to the interpreter state - it
@ -1044,6 +1216,11 @@ _PyCoreConfig_Read(_PyCoreConfig *config)
config->argc = 0;
}
err = config_init_stdio_encoding(config);
if (_Py_INIT_FAILED(err)) {
return err;
}
assert(config->coerce_c_locale >= 0);
assert(config->use_environment >= 0);

160
Python/pylifecycle.c
View file

@ -184,27 +184,6 @@ error:
return NULL;
}
static _PyInitError
get_locale_encoding(char **locale_encoding)
{
#ifdef MS_WINDOWS
char encoding[20];
PyOS_snprintf(encoding, sizeof(encoding), "cp%d", GetACP());
#elif defined(__ANDROID__)
const char *encoding = "UTF-8";
#else
const char *encoding = nl_langinfo(CODESET);
if (!encoding || encoding[0] == '\0') {
return _Py_INIT_USER_ERR("failed to get the locale encoding: "
"nl_langinfo(CODESET) failed");
}
#endif
*locale_encoding = _PyMem_RawStrdup(encoding);
if (*locale_encoding == NULL) {
return _Py_INIT_NO_MEMORY();
}
return _Py_INIT_OK();
}
static _PyInitError
initimport(PyInterpreterState *interp, PyObject *sysmod)
@ -340,35 +319,20 @@ static _LocaleCoercionTarget _TARGET_LOCALES[] = {
{NULL}
};
static const char *
get_stdio_errors(void)
int
_Py_IsLocaleCoercionTarget(const char *ctype_loc)
{
#ifndef MS_WINDOWS
const char *ctype_loc = setlocale(LC_CTYPE, NULL);
if (ctype_loc != NULL) {
/* surrogateescape is the default in the legacy C and POSIX locales */
if (strcmp(ctype_loc, "C") == 0 || strcmp(ctype_loc, "POSIX") == 0) {
return "surrogateescape";
const _LocaleCoercionTarget *target = NULL;
for (target = _TARGET_LOCALES; target->locale_name; target++) {
if (strcmp(ctype_loc, target->locale_name) == 0) {
return 1;
}
#ifdef PY_COERCE_C_LOCALE
/* surrogateescape is the default in locale coercion target locales */
const _LocaleCoercionTarget *target = NULL;
for (target = _TARGET_LOCALES; target->locale_name; target++) {
if (strcmp(ctype_loc, target->locale_name) == 0) {
return "surrogateescape";
}
}
#endif
}
return "strict";
#else
/* On Windows, always use surrogateescape by default */
return "surrogateescape";
#endif
return 0;
}
#ifdef PY_COERCE_C_LOCALE
static const char C_LOCALE_COERCION_WARNING[] =
"Python detected LC_CTYPE=C: LC_CTYPE coerced to %.20s (set another locale "
@ -1533,8 +1497,10 @@ initfsencoding(PyInterpreterState *interp)
Py_HasFileSystemDefaultEncoding = 1;
}
else {
extern _PyInitError _Py_get_locale_encoding(char **locale_encoding);
char *locale_encoding;
_PyInitError err = get_locale_encoding(&locale_encoding);
_PyInitError err = _Py_get_locale_encoding(&locale_encoding);
if (_Py_INIT_FAILED(err)) {
return err;
}
@ -1740,13 +1706,16 @@ init_sys_streams(PyInterpreterState *interp)
PyObject *std = NULL;
int fd;
PyObject * encoding_attr;
char *pythonioencoding = NULL;
const char *encoding, *errors;
char *locale_encoding = NULL;
char *codec_name = NULL;
_PyInitError res = _Py_INIT_OK();
extern char *_Py_StandardStreamEncoding;
extern char *_Py_StandardStreamErrors;
_PyCoreConfig *config = &interp->core_config;
char *codec_name = get_codec_name(config->stdio_encoding);
if (codec_name == NULL) {
return _Py_INIT_ERR("failed to get the Python codec name "
"of the stdio encoding");
}
PyMem_RawFree(config->stdio_encoding);
config->stdio_encoding = codec_name;
/* Hack to avoid a nasty recursion issue when Python is invoked
in verbose mode: pre-import the Latin-1 and UTF-8 codecs */
@ -1778,85 +1747,15 @@ init_sys_streams(PyInterpreterState *interp)
}
Py_DECREF(wrapper);
encoding = _Py_StandardStreamEncoding;
errors = _Py_StandardStreamErrors;
if (!encoding || !errors) {
char *opt = Py_GETENV("PYTHONIOENCODING");
if (opt && opt[0] != '\0') {
char *err;
pythonioencoding = _PyMem_Strdup(opt);
if (pythonioencoding == NULL) {
PyErr_NoMemory();
goto error;
}
err = strchr(pythonioencoding, ':');
if (err) {
*err = '\0';
err++;
if (!err[0]) {
err = NULL;
}
}
/* Does PYTHONIOENCODING contain an encoding? */
if (pythonioencoding[0]) {
if (!encoding) {
encoding = pythonioencoding;
}
/* If the encoding is set but not the error handler,
use "strict" error handler by default.
PYTHONIOENCODING=latin1 behaves as
PYTHONIOENCODING=latin1:strict. */
if (!err) {
err = "strict";
}
}
if (!errors && err != NULL) {
errors = err;
}
}
if (interp->core_config.utf8_mode) {
if (!encoding) {
encoding = "utf-8";
}
if (!errors) {
errors = "surrogateescape";
}
}
if (!errors) {
/* Choose the default error handler based on the current locale */
errors = get_stdio_errors();
}
}
if (encoding == NULL) {
_PyInitError err = get_locale_encoding(&locale_encoding);
if (_Py_INIT_FAILED(err)) {
return err;
}
encoding = locale_encoding;
}
codec_name = get_codec_name(encoding);
if (codec_name == NULL) {
PyErr_SetString(PyExc_RuntimeError,
"failed to get the Python codec name "
"of stdio encoding");
goto error;
}
encoding = codec_name;
/* Set sys.stdin */
fd = fileno(stdin);
/* Under some conditions stdin, stdout and stderr may not be connected
* and fileno() may point to an invalid file descriptor. For example
* GUI apps don't have valid standard streams by default.
*/
std = create_stdio(iomod, fd, 0, "<stdin>", encoding, errors);
std = create_stdio(iomod, fd, 0, "<stdin>",
config->stdio_encoding,
config->stdio_errors);
if (std == NULL)
goto error;
PySys_SetObject("__stdin__", std);
@ -1865,7 +1764,9 @@ init_sys_streams(PyInterpreterState *interp)
/* Set sys.stdout */
fd = fileno(stdout);
std = create_stdio(iomod, fd, 1, "<stdout>", encoding, errors);
std = create_stdio(iomod, fd, 1, "<stdout>",
config->stdio_encoding,
config->stdio_errors);
if (std == NULL)
goto error;
PySys_SetObject("__stdout__", std);
@ -1875,7 +1776,9 @@ init_sys_streams(PyInterpreterState *interp)
#if 1 /* Disable this if you have trouble debugging bootstrap stuff */
/* Set sys.stderr, replaces the preliminary stderr */
fd = fileno(stderr);
std = create_stdio(iomod, fd, 1, "<stderr>", encoding, "backslashreplace");
std = create_stdio(iomod, fd, 1, "<stderr>",
config->stdio_encoding,
"backslashreplace");
if (std == NULL)
goto error;
@ -1911,9 +1814,6 @@ error:
done:
_Py_ClearStandardStreamEncoding();
PyMem_RawFree(locale_encoding);
PyMem_RawFree(codec_name);
PyMem_Free(pythonioencoding);
Py_XDECREF(bimod);
Py_XDECREF(iomod);
return res;