bpo-36301: Add _PyWstrList structure (GH-12343)

Replace messy _Py_wstrlist_xxx() functions with a new clean
_PyWstrList structure and new _PyWstrList_xxx() functions.

Changes:

* Add _PyCoreConfig.use_module_search_paths to decide if
  _PyCoreConfig.module_search_paths should be computed or not, to
  support empty search path list.
* _PyWstrList_Clear() sets length to 0 and items to NULL, whereas
  _Py_wstrlist_clear() only freed memory.
* _PyWstrList_Append() returns an int, whereas _Py_wstrlist_append()
  returned _PyInitError.
* _PyWstrList uses Py_ssize_t for the length, instead of int.
* Replace (int, wchar_t**) with _PyWstrList in:

  * _PyPreConfig
  * _PyCoreConfig
  * _PyPreCmdline
  * _PyCmdline

* Replace "int orig_argv; wchar_t **orig_argv;"
  with "_PyWstrList orig_argv".
* _PyCmdline and _PyPreCmdline now also copy wchar_argv.
* Rename _PyArgv_Decode() to _PyArgv_AsWstrList().
* PySys_SetArgvEx() now pass the fixed (argc, argv) to
  _PyPathConfig_ComputeArgv0() (don't pass negative argc or NULL
  argv).
* _PyOS_GetOpt() uses Py_ssize_t
This commit is contained in:
Victor Stinner 2019-03-15 15:08:05 +01:00 committed by GitHub
parent 86082c22d2
commit 74f6568bbd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 303 additions and 345 deletions

View file

@ -202,81 +202,128 @@ fail:
}
/* --- _Py_wstrlist ----------------------------------------------- */
/* --- _PyWstrList ------------------------------------------------ */
#ifndef NDEBUG
int
_PyWstrList_CheckConsistency(const _PyWstrList *list)
{
assert(list->length >= 0);
if (list->length != 0) {
assert(list->items != NULL);
}
for (Py_ssize_t i = 0; i < list->length; i++) {
assert(list->items[i] != NULL);
}
return 1;
}
#endif /* Py_DEBUG */
void
_Py_wstrlist_clear(int len, wchar_t **list)
_PyWstrList_Clear(_PyWstrList *list)
{
for (int i=0; i < len; i++) {
PyMem_RawFree(list[i]);
assert(_PyWstrList_CheckConsistency(list));
for (Py_ssize_t i=0; i < list->length; i++) {
PyMem_RawFree(list->items[i]);
}
PyMem_RawFree(list);
PyMem_RawFree(list->items);
list->length = 0;
list->items = NULL;
}
wchar_t**
_Py_wstrlist_copy(int len, wchar_t * const *list)
int
_PyWstrList_Copy(_PyWstrList *list, const _PyWstrList *list2)
{
assert((len > 0 && list != NULL) || len == 0);
size_t size = len * sizeof(list[0]);
wchar_t **list_copy = PyMem_RawMalloc(size);
if (list_copy == NULL) {
return NULL;
assert(_PyWstrList_CheckConsistency(list));
assert(_PyWstrList_CheckConsistency(list2));
if (list2->length == 0) {
_PyWstrList_Clear(list);
return 0;
}
for (int i=0; i < len; i++) {
wchar_t* arg = _PyMem_RawWcsdup(list[i]);
if (arg == NULL) {
_Py_wstrlist_clear(i, list_copy);
return NULL;
_PyWstrList copy = _PyWstrList_INIT;
size_t size = list2->length * sizeof(list2->items[0]);
copy.items = PyMem_RawMalloc(size);
if (copy.items == NULL) {
return -1;
}
for (Py_ssize_t i=0; i < list2->length; i++) {
wchar_t *item = _PyMem_RawWcsdup(list2->items[i]);
if (item == NULL) {
_PyWstrList_Clear(&copy);
return -1;
}
list_copy[i] = arg;
copy.items[i] = item;
copy.length = i + 1;
}
return list_copy;
_PyWstrList_Clear(list);
*list = copy;
return 0;
}
_PyInitError
_Py_wstrlist_append(int *len, wchar_t ***list, const wchar_t *str)
int
_PyWstrList_Append(_PyWstrList *list, const wchar_t *item)
{
if (*len == INT_MAX) {
/* len+1 would overflow */
return _Py_INIT_NO_MEMORY();
}
wchar_t *str2 = _PyMem_RawWcsdup(str);
if (str2 == NULL) {
return _Py_INIT_NO_MEMORY();
if (list->length == PY_SSIZE_T_MAX) {
/* lenght+1 would overflow */
return -1;
}
size_t size = (*len + 1) * sizeof(list[0]);
wchar_t **list2 = (wchar_t **)PyMem_RawRealloc(*list, size);
if (list2 == NULL) {
PyMem_RawFree(str2);
return _Py_INIT_NO_MEMORY();
wchar_t *item2 = _PyMem_RawWcsdup(item);
if (item2 == NULL) {
return -1;
}
list2[*len] = str2;
*list = list2;
(*len)++;
return _Py_INIT_OK();
size_t size = (list->length + 1) * sizeof(list->items[0]);
wchar_t **items2 = (wchar_t **)PyMem_RawRealloc(list->items, size);
if (items2 == NULL) {
PyMem_RawFree(item2);
return -1;
}
items2[list->length] = item2;
list->items = items2;
list->length++;
return 0;
}
static int
_PyWstrList_Extend(_PyWstrList *list, const _PyWstrList *list2)
{
for (Py_ssize_t i = 0; i < list2->length; i++) {
if (_PyWstrList_Append(list, list2->items[i])) {
return -1;
}
}
return 0;
}
PyObject*
_Py_wstrlist_as_pylist(int len, wchar_t **list)
_PyWstrList_AsList(const _PyWstrList *list)
{
assert(list != NULL || len < 1);
assert(_PyWstrList_CheckConsistency(list));
PyObject *pylist = PyList_New(len);
PyObject *pylist = PyList_New(list->length);
if (pylist == NULL) {
return NULL;
}
for (int i = 0; i < len; i++) {
PyObject *v = PyUnicode_FromWideChar(list[i], -1);
if (v == NULL) {
for (Py_ssize_t i = 0; i < list->length; i++) {
PyObject *item = PyUnicode_FromWideChar(list->items[i], -1);
if (item == NULL) {
Py_DECREF(pylist);
return NULL;
}
PyList_SET_ITEM(pylist, i, v);
PyList_SET_ITEM(pylist, i, item);
}
return pylist;
}
@ -369,8 +416,7 @@ _Py_ClearStandardStreamEncoding(void)
/* --- Py_GetArgcArgv() ------------------------------------------- */
/* For Py_GetArgcArgv(); set by _Py_SetArgcArgv() */
static int orig_argc = 0;
static wchar_t **orig_argv = NULL;
static _PyWstrList orig_argv = {.length = 0, .items = NULL};
void
@ -379,32 +425,22 @@ _Py_ClearArgcArgv(void)
PyMemAllocatorEx old_alloc;
_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
_Py_wstrlist_clear(orig_argc, orig_argv);
orig_argc = 0;
orig_argv = NULL;
_PyWstrList_Clear(&orig_argv);
PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
}
static int
_Py_SetArgcArgv(int argc, wchar_t * const *argv)
_Py_SetArgcArgv(Py_ssize_t argc, wchar_t * const *argv)
{
const _PyWstrList argv_list = {.length = argc, .items = (wchar_t **)argv};
int res;
PyMemAllocatorEx old_alloc;
_PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
wchar_t **argv_copy = _Py_wstrlist_copy(argc, argv);
if (argv_copy != NULL) {
_Py_ClearArgcArgv();
orig_argc = argc;
orig_argv = argv_copy;
res = 0;
}
else {
res = -1;
}
res = _PyWstrList_Copy(&orig_argv, &argv_list);
PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
return res;
@ -416,8 +452,8 @@ _Py_SetArgcArgv(int argc, wchar_t * const *argv)
void
Py_GetArgcArgv(int *argc, wchar_t ***argv)
{
*argc = orig_argc;
*argv = orig_argv;
*argc = (int)orig_argv.length;
*argv = orig_argv.items;
}
@ -439,12 +475,6 @@ _PyCoreConfig_Clear(_PyCoreConfig *config)
PyMem_RawFree(ATTR); \
ATTR = NULL; \
} while (0)
#define CLEAR_WSTRLIST(LEN, LIST) \
do { \
_Py_wstrlist_clear(LEN, LIST); \
LEN = 0; \
LIST = NULL; \
} while (0)
CLEAR(config->pycache_prefix);
CLEAR(config->module_search_path_env);
@ -452,13 +482,11 @@ _PyCoreConfig_Clear(_PyCoreConfig *config)
CLEAR(config->program_name);
CLEAR(config->program);
CLEAR_WSTRLIST(config->argc, config->argv);
config->argc = -1;
CLEAR_WSTRLIST(config->nwarnoption, config->warnoptions);
CLEAR_WSTRLIST(config->nxoption, config->xoptions);
CLEAR_WSTRLIST(config->nmodule_search_path, config->module_search_paths);
config->nmodule_search_path = -1;
_PyWstrList_Clear(&config->argv);
_PyWstrList_Clear(&config->warnoptions);
_PyWstrList_Clear(&config->xoptions);
_PyWstrList_Clear(&config->module_search_paths);
config->use_module_search_paths = 0;
CLEAR(config->executable);
CLEAR(config->prefix);
@ -477,7 +505,6 @@ _PyCoreConfig_Clear(_PyCoreConfig *config)
CLEAR(config->run_module);
CLEAR(config->run_filename);
#undef CLEAR
#undef CLEAR_WSTRLIST
}
@ -509,15 +536,11 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2)
} \
} \
} while (0)
#define COPY_WSTRLIST(LEN, LIST) \
#define COPY_WSTRLIST(LIST) \
do { \
if (config2->LIST != NULL) { \
config->LIST = _Py_wstrlist_copy(config2->LEN, config2->LIST); \
if (config->LIST == NULL) { \
return -1; \
} \
if (_PyWstrList_Copy(&config->LIST, &config2->LIST) < 0 ) { \
return -1; \
} \
config->LEN = config2->LEN; \
} while (0)
COPY_ATTR(install_signal_handlers);
@ -538,10 +561,11 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2)
COPY_WSTR_ATTR(program_name);
COPY_WSTR_ATTR(program);
COPY_WSTRLIST(argc, argv);
COPY_WSTRLIST(nwarnoption, warnoptions);
COPY_WSTRLIST(nxoption, xoptions);
COPY_WSTRLIST(nmodule_search_path, module_search_paths);
COPY_WSTRLIST(argv);
COPY_WSTRLIST(warnoptions);
COPY_WSTRLIST(xoptions);
COPY_WSTRLIST(module_search_paths);
COPY_ATTR(use_module_search_paths);
COPY_WSTR_ATTR(executable);
COPY_WSTR_ATTR(prefix);
@ -817,7 +841,7 @@ config_init_executable(_PyCoreConfig *config)
static const wchar_t*
config_get_xoption(const _PyCoreConfig *config, wchar_t *name)
{
return _Py_get_xoption(config->nxoption, config->xoptions, name);
return _Py_get_xoption(&config->xoptions, name);
}
@ -1427,9 +1451,6 @@ _PyCoreConfig_Read(_PyCoreConfig *config, const _PyPreConfig *preconfig)
if (config->tracemalloc < 0) {
config->tracemalloc = 0;
}
if (config->argc < 0) {
config->argc = 0;
}
if (config->filesystem_encoding == NULL || config->filesystem_errors == NULL) {
err = config_init_fs_encoding(config);
@ -1449,6 +1470,7 @@ _PyCoreConfig_Read(_PyCoreConfig *config, const _PyPreConfig *preconfig)
assert(config->stdio_encoding != NULL);
assert(config->stdio_errors != NULL);
assert(config->_check_hash_pycs_mode != NULL);
assert(_PyWstrList_CheckConsistency(&config->argv));
return _Py_INIT_OK();
}
@ -1546,8 +1568,8 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config)
: (Py_INCREF(Py_None), Py_None))
#define SET_ITEM_WSTR(ATTR) \
SET_ITEM(#ATTR, FROM_WSTRING(config->ATTR))
#define SET_ITEM_WSTRLIST(NOPTION, OPTIONS) \
SET_ITEM(#OPTIONS, _Py_wstrlist_as_pylist(config->NOPTION, config->OPTIONS))
#define SET_ITEM_WSTRLIST(LIST) \
SET_ITEM(#LIST, _PyWstrList_AsList(&config->LIST))
SET_ITEM_INT(install_signal_handlers);
SET_ITEM_INT(use_hash_seed);
@ -1563,13 +1585,13 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config)
SET_ITEM_STR(filesystem_errors);
SET_ITEM_WSTR(pycache_prefix);
SET_ITEM_WSTR(program_name);
SET_ITEM_WSTRLIST(argc, argv);
SET_ITEM_WSTRLIST(argv);
SET_ITEM_WSTR(program);
SET_ITEM_WSTRLIST(nxoption, xoptions);
SET_ITEM_WSTRLIST(nwarnoption, warnoptions);
SET_ITEM_WSTRLIST(xoptions);
SET_ITEM_WSTRLIST(warnoptions);
SET_ITEM_WSTR(module_search_path_env);
SET_ITEM_WSTR(home);
SET_ITEM_WSTRLIST(nmodule_search_path, module_search_paths);
SET_ITEM_WSTRLIST(module_search_paths);
SET_ITEM_WSTR(executable);
SET_ITEM_WSTR(prefix);
SET_ITEM_WSTR(base_prefix);
@ -1622,13 +1644,9 @@ fail:
/* --- _PyCmdline ------------------------------------------------- */
typedef struct {
const _PyArgv *args;
int argc;
wchar_t **argv;
int nwarnoption; /* Number of -W command line options */
wchar_t **warnoptions; /* Command line -W options */
int nenv_warnoption; /* Number of PYTHONWARNINGS environment variables */
wchar_t **env_warnoptions; /* PYTHONWARNINGS environment variables */
_PyWstrList argv;
_PyWstrList warnoptions; /* Command line -W options */
_PyWstrList env_warnoptions; /* PYTHONWARNINGS environment variables */
int print_help; /* -h, -? options */
int print_version; /* -V option */
} _PyCmdline;
@ -1637,18 +1655,9 @@ typedef struct {
static void
cmdline_clear(_PyCmdline *cmdline)
{
_Py_wstrlist_clear(cmdline->nwarnoption, cmdline->warnoptions);
cmdline->nwarnoption = 0;
cmdline->warnoptions = NULL;
_Py_wstrlist_clear(cmdline->nenv_warnoption, cmdline->env_warnoptions);
cmdline->nenv_warnoption = 0;
cmdline->env_warnoptions = NULL;
if (cmdline->args->use_bytes_argv && cmdline->argv != NULL) {
_Py_wstrlist_clear(cmdline->args->argc, cmdline->argv);
}
cmdline->argv = NULL;
_PyWstrList_Clear(&cmdline->warnoptions);
_PyWstrList_Clear(&cmdline->env_warnoptions);
_PyWstrList_Clear(&cmdline->argv);
}
@ -1659,11 +1668,10 @@ static _PyInitError
config_parse_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline,
int *need_usage)
{
_PyInitError err;
_PyOS_ResetGetOpt();
do {
int longindex = -1;
int c = _PyOS_GetOpt(cmdline->args->argc, cmdline->argv, &longindex);
int c = _PyOS_GetOpt(cmdline->argv.length, cmdline->argv.items, &longindex);
if (c == EOF) {
break;
}
@ -1775,20 +1783,14 @@ config_parse_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline,
break;
case 'W':
err = _Py_wstrlist_append(&cmdline->nwarnoption,
&cmdline->warnoptions,
_PyOS_optarg);
if (_Py_INIT_FAILED(err)) {
return err;
if (_PyWstrList_Append(&cmdline->warnoptions, _PyOS_optarg) < 0) {
return _Py_INIT_NO_MEMORY();
}
break;
case 'X':
err = _Py_wstrlist_append(&config->nxoption,
&config->xoptions,
_PyOS_optarg);
if (_Py_INIT_FAILED(err)) {
return err;
if (_PyWstrList_Append(&config->xoptions, _PyOS_optarg) < 0) {
return _Py_INIT_NO_MEMORY();
}
break;
@ -1810,10 +1812,10 @@ config_parse_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline,
} while (1);
if (config->run_command == NULL && config->run_module == NULL
&& _PyOS_optind < cmdline->args->argc
&& wcscmp(cmdline->argv[_PyOS_optind], L"-") != 0)
&& _PyOS_optind < cmdline->argv.length
&& wcscmp(cmdline->argv.items[_PyOS_optind], L"-") != 0)
{
config->run_filename = _PyMem_RawWcsdup(cmdline->argv[_PyOS_optind]);
config->run_filename = _PyMem_RawWcsdup(cmdline->argv.items[_PyOS_optind]);
if (config->run_filename == NULL) {
return _Py_INIT_NO_MEMORY();
}
@ -1858,12 +1860,9 @@ cmdline_init_env_warnoptions(_PyCmdline *cmdline, const _PyCoreConfig *config)
warning != NULL;
warning = WCSTOK(NULL, L",", &context))
{
_PyInitError err = _Py_wstrlist_append(&cmdline->nenv_warnoption,
&cmdline->env_warnoptions,
warning);
if (_Py_INIT_FAILED(err)) {
if (_PyWstrList_Append(&cmdline->env_warnoptions, warning) < 0) {
PyMem_RawFree(env);
return err;
return _Py_INIT_NO_MEMORY();
}
}
PyMem_RawFree(env);
@ -1875,8 +1874,8 @@ static _PyInitError
config_init_program(_PyCoreConfig *config, const _PyCmdline *cmdline)
{
wchar_t *program;
if (cmdline->args->argc >= 1 && cmdline->argv != NULL) {
program = cmdline->argv[0];
if (cmdline->argv.length >= 1) {
program = cmdline->argv.items[0];
}
else {
program = L"";
@ -1890,28 +1889,10 @@ config_init_program(_PyCoreConfig *config, const _PyCmdline *cmdline)
}
static _PyInitError
config_add_warnings_optlist(_PyCoreConfig *config,
int len, wchar_t * const *options)
{
for (int i = 0; i < len; i++) {
_PyInitError err = _Py_wstrlist_append(&config->nwarnoption,
&config->warnoptions,
options[i]);
if (_Py_INIT_FAILED(err)) {
return err;
}
}
return _Py_INIT_OK();
}
static _PyInitError
config_init_warnoptions(_PyCoreConfig *config, const _PyCmdline *cmdline)
{
_PyInitError err;
assert(config->nwarnoption == 0);
assert(config->warnoptions.length == 0);
/* The priority order for warnings configuration is (highest precedence
* first):
@ -1929,26 +1910,17 @@ config_init_warnoptions(_PyCoreConfig *config, const _PyCmdline *cmdline)
*/
if (config->preconfig.dev_mode) {
err = _Py_wstrlist_append(&config->nwarnoption,
&config->warnoptions,
L"default");
if (_Py_INIT_FAILED(err)) {
return err;
if (_PyWstrList_Append(&config->warnoptions, L"default")) {
return _Py_INIT_NO_MEMORY();
}
}
err = config_add_warnings_optlist(config,
cmdline->nenv_warnoption,
cmdline->env_warnoptions);
if (_Py_INIT_FAILED(err)) {
return err;
if (_PyWstrList_Extend(&config->warnoptions, &cmdline->env_warnoptions) < 0) {
return _Py_INIT_NO_MEMORY();
}
err = config_add_warnings_optlist(config,
cmdline->nwarnoption,
cmdline->warnoptions);
if (_Py_INIT_FAILED(err)) {
return err;
if (_PyWstrList_Extend(&config->warnoptions, &cmdline->warnoptions) < 0) {
return _Py_INIT_NO_MEMORY();
}
/* If the bytes_warning_flag isn't set, bytesobject.c and bytearrayobject.c
@ -1956,18 +1928,15 @@ config_init_warnoptions(_PyCoreConfig *config, const _PyCmdline *cmdline)
* case.
*/
if (config->bytes_warning) {
wchar_t *filter;
const wchar_t *filter;
if (config->bytes_warning> 1) {
filter = L"error::BytesWarning";
}
else {
filter = L"default::BytesWarning";
}
err = _Py_wstrlist_append(&config->nwarnoption,
&config->warnoptions,
filter);
if (_Py_INIT_FAILED(err)) {
return err;
if (_PyWstrList_Append(&config->warnoptions, filter)) {
return _Py_INIT_NO_MEMORY();
}
}
return _Py_INIT_OK();
@ -1977,23 +1946,24 @@ config_init_warnoptions(_PyCoreConfig *config, const _PyCmdline *cmdline)
static _PyInitError
config_init_argv(_PyCoreConfig *config, const _PyCmdline *cmdline)
{
/* Copy argv to be able to modify it (to force -c/-m) */
int argc = cmdline->args->argc - _PyOS_optind;
wchar_t **argv;
_PyWstrList wargv = _PyWstrList_INIT;
if (argc <= 0 || cmdline->argv == NULL) {
/* Copy argv to be able to modify it (to force -c/-m) */
if (cmdline->argv.length <= _PyOS_optind) {
/* Ensure at least one (empty) argument is seen */
static wchar_t *empty_argv[1] = {L""};
argc = 1;
argv = _Py_wstrlist_copy(1, empty_argv);
if (_PyWstrList_Append(&wargv, L"") < 0) {
return _Py_INIT_NO_MEMORY();
}
}
else {
argv = _Py_wstrlist_copy(argc, &cmdline->argv[_PyOS_optind]);
}
if (argv == NULL) {
return _Py_INIT_NO_MEMORY();
_PyWstrList slice;
slice.length = cmdline->argv.length - _PyOS_optind;
slice.items = &cmdline->argv.items[_PyOS_optind];
if (_PyWstrList_Copy(&wargv, &slice) < 0) {
return _Py_INIT_NO_MEMORY();
}
}
assert(wargv.length >= 1);
wchar_t *arg0 = NULL;
if (config->run_command != NULL) {
@ -2007,17 +1977,16 @@ config_init_argv(_PyCoreConfig *config, const _PyCmdline *cmdline)
if (arg0 != NULL) {
arg0 = _PyMem_RawWcsdup(arg0);
if (arg0 == NULL) {
_Py_wstrlist_clear(argc, argv);
_PyWstrList_Clear(&wargv);
return _Py_INIT_NO_MEMORY();
}
assert(argc >= 1);
PyMem_RawFree(argv[0]);
argv[0] = arg0;
PyMem_RawFree(wargv.items[0]);
wargv.items[0] = arg0;
}
config->argc = argc;
config->argv = argv;
_PyWstrList_Clear(&config->argv);
config->argv = wargv;
return _Py_INIT_OK();
}
@ -2097,7 +2066,7 @@ config_from_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline,
return err;
}
if (_Py_SetArgcArgv(cmdline->args->argc, cmdline->argv) < 0) {
if (_Py_SetArgcArgv(cmdline->argv.length, cmdline->argv.items) < 0) {
return _Py_INIT_NO_MEMORY();
}
return _Py_INIT_OK();
@ -2117,9 +2086,8 @@ _PyCoreConfig_ReadFromArgv(_PyCoreConfig *config, const _PyArgv *args,
_PyCmdline cmdline;
memset(&cmdline, 0, sizeof(cmdline));
cmdline.args = args;
err = _PyArgv_Decode(cmdline.args, &cmdline.argv);
err = _PyArgv_AsWstrList(args, &cmdline.argv);
if (_Py_INIT_FAILED(err)) {
goto done;
}