Change command line processing API to use wchar_t.

Fixes #2128.

Python/frozenmain.c

@@ -2,6 +2,7 @@
 /* Python interpreter main program for frozen scripts */
 
 #include "Python.h"
+#include <locale.h>
 
 #ifdef MS_WINDOWS
 extern void PyWinFreeze_ExeInit(void);
@@ -15,9 +16,13 @@ int
 Py_FrozenMain(int argc, char **argv)
 {
 	char *p;
-	int n, sts;
+	int i, n, sts;
 	int inspect = 0;
 	int unbuffered = 0;
+	char *oldloc;
+	wchar_t **argv_copy = PyMem_Malloc(sizeof(wchar_t*)*argc);
+	/* We need a second copies, as Python might modify the first one. */
+	wchar_t **argv_copy2 = PyMem_Malloc(sizeof(wchar_t*)*argc);
 
 	Py_FrozenFlag = 1; /* Suppress errors from getpath.c */
 
@@ -32,10 +37,33 @@ Py_FrozenMain(int argc, char **argv)
 		setbuf(stderr, (char *)NULL);
 	}
 
+	if (!argv_copy) {
+		fprintf(stderr, "out of memory");
+		return 1;
+	}
+
+	oldloc = setlocale(LC_ALL, NULL);
+	setlocale(LC_ALL, "");
+	for (i = 0; i < argc; i++) {
+		size_t argsize = mbstowcs(NULL, argv[i], 0);
+		if (argsize == (size_t)-1) {
+			fprintf(stderr, "Could not convert argument %d to string", i);
+			return 1;
+		}
+		argv_copy[i] = PyMem_Malloc((argsize+1)*sizeof(wchar_t));
+		argv_copy2[i] = argv_copy[i];
+		if (!argv_copy[i]) {
+			fprintf(stderr, "out of memory");
+			return 1;
+		}
+		mbstowcs(argv_copy[i], argv[i], argsize+1);
+	}
+	setlocale(LC_ALL, oldloc);
+
 #ifdef MS_WINDOWS
 	PyInitFrozenExtensions();
 #endif /* MS_WINDOWS */
-	Py_SetProgramName(argv[0]);
+	Py_SetProgramName(argv_copy[0]);
 	Py_Initialize();
 #ifdef MS_WINDOWS
 	PyWinFreeze_ExeInit();
@@ -45,7 +73,7 @@ Py_FrozenMain(int argc, char **argv)
 		fprintf(stderr, "Python %s\n%s\n",
 			Py_GetVersion(), Py_GetCopyright());
 
-	PySys_SetArgv(argc, argv);
+	PySys_SetArgv(argc, argv_copy);
 
 	n = PyImport_ImportFrozenModule("__main__");
 	if (n == 0)
@@ -64,5 +92,10 @@ Py_FrozenMain(int argc, char **argv)
 	PyWinFreeze_ExeTerm();
 #endif
 	Py_Finalize();
+	for (i = 0; i < argc; i++) {
+		PyMem_Free(argv_copy2[i]);
+	}
+	PyMem_Free(argv_copy);
+	PyMem_Free(argv_copy2);
 	return sts;
 }

Python/getopt.c

@@ -27,8 +27,11 @@
 /* Modified to support --help and --version, as well as /? on Windows
  * by Georg Brandl. */
 
+#include <Python.h>
 #include <stdio.h>
 #include <string.h>
+#include <wchar.h>
+#include <pygetopt.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -36,40 +39,40 @@ extern "C" {
 
 int _PyOS_opterr = 1;          /* generate error messages */
 int _PyOS_optind = 1;          /* index into argv array   */
-char *_PyOS_optarg = NULL;     /* optional argument       */
+wchar_t *_PyOS_optarg = NULL;     /* optional argument       */
 
-int _PyOS_GetOpt(int argc, char **argv, char *optstring)
+int _PyOS_GetOpt(int argc, wchar_t **argv, wchar_t *optstring)
 {
-	static char *opt_ptr = "";
-	char *ptr;
-	int option;
+	static wchar_t *opt_ptr = L"";
+	wchar_t *ptr;
+	wchar_t option;
 
 	if (*opt_ptr == '\0') {
 
 		if (_PyOS_optind >= argc)
 			return -1;
 #ifdef MS_WINDOWS
-		else if (strcmp(argv[_PyOS_optind], "/?") == 0) {
+		else if (wcscmp(argv[_PyOS_optind], L"/?") == 0) {
 			++_PyOS_optind;
 			return 'h';
 		}
 #endif
 
-		else if (argv[_PyOS_optind][0] != '-' ||
-		         argv[_PyOS_optind][1] == '\0' /* lone dash */ )
+		else if (argv[_PyOS_optind][0] != L'-' ||
+		         argv[_PyOS_optind][1] == L'\0' /* lone dash */ )
 			return -1;
 
-		else if (strcmp(argv[_PyOS_optind], "--") == 0) {
+		else if (wcscmp(argv[_PyOS_optind], L"--") == 0) {
 			++_PyOS_optind;
 			return -1;
 		}
 
-		else if (strcmp(argv[_PyOS_optind], "--help") == 0) {
+		else if (wcscmp(argv[_PyOS_optind], L"--help") == 0) {
 			++_PyOS_optind;
 			return 'h';
 		}
 
-		else if (strcmp(argv[_PyOS_optind], "--version") == 0) {
+		else if (wcscmp(argv[_PyOS_optind], L"--version") == 0) {
 			++_PyOS_optind;
 			return 'V';
 		}
@@ -78,27 +81,27 @@ int _PyOS_GetOpt(int argc, char **argv, char *optstring)
 		opt_ptr = &argv[_PyOS_optind++][1]; 
 	}
 
-	if ( (option = *opt_ptr++) == '\0')
+	if ( (option = *opt_ptr++) == L'\0')
 		return -1;
 	
-	if ((ptr = strchr(optstring, option)) == NULL) {
+	if ((ptr = wcschr(optstring, option)) == NULL) {
 		if (_PyOS_opterr)
-			fprintf(stderr, "Unknown option: -%c\n", option);
+		  fprintf(stderr, "Unknown option: -%c\n", (char)option);
 
 		return '_';
 	}
 
-	if (*(ptr + 1) == ':') {
-		if (*opt_ptr != '\0') {
+	if (*(ptr + 1) == L':') {
+		if (*opt_ptr != L'\0') {
 			_PyOS_optarg  = opt_ptr;
-			opt_ptr = "";
+			opt_ptr = L"";
 		}
 
 		else {
 			if (_PyOS_optind >= argc) {
 				if (_PyOS_opterr)
 					fprintf(stderr,
-			    "Argument expected for the -%c option\n", option);
+						"Argument expected for the -%c option\n", (char)option);
 				return '_';
 			}
 

Python/pythonrun.c

@@ -17,6 +17,7 @@
 #include "ast.h"
 #include "eval.h"
 #include "marshal.h"
+#include "osdefs.h"
 
 #ifdef HAVE_SIGNAL_H
 #include <signal.h>
@@ -30,6 +31,7 @@
 #ifdef MS_WINDOWS
 #undef BYTE
 #include "windows.h"
+#define PATH_MAX MAXPATHLEN
 #endif
 
 #ifndef Py_REF_DEBUG
@@ -44,7 +46,7 @@
 extern "C" {
 #endif
 
-extern char *Py_GetPath(void);
+extern wchar_t *Py_GetPath(void);
 
 extern grammar _PyParser_Grammar; /* From graminit.c */
 
@@ -646,35 +648,43 @@ Py_EndInterpreter(PyThreadState *tstate)
 	PyInterpreterState_Delete(interp);
 }
 
-static char *progname = "python";
+static wchar_t *progname = L"python";
 
 void
-Py_SetProgramName(char *pn)
+Py_SetProgramName(wchar_t *pn)
 {
 	if (pn && *pn)
 		progname = pn;
 }
 
-char *
+wchar_t *
 Py_GetProgramName(void)
 {
 	return progname;
 }
 
-static char *default_home = NULL;
+static wchar_t *default_home = NULL;
+static wchar_t env_home[PATH_MAX+1];
 
 void
-Py_SetPythonHome(char *home)
+Py_SetPythonHome(wchar_t *home)
 {
 	default_home = home;
 }
 
-char *
+wchar_t *
 Py_GetPythonHome(void)
 {
-	char *home = default_home;
-	if (home == NULL && !Py_IgnoreEnvironmentFlag)
-		home = Py_GETENV("PYTHONHOME");
+	wchar_t *home = default_home;
+	if (home == NULL && !Py_IgnoreEnvironmentFlag) {
+		char* chome = Py_GETENV("PYTHONHOME");
+		if (chome) {
+			size_t r = mbstowcs(env_home, chome, PATH_MAX+1);
+			if (r != (size_t)-1 && r <= PATH_MAX)
+				home = env_home;
+		}
+
+	}
 	return home;
 }
 

Python/sysmodule.c

@@ -882,7 +882,7 @@ PySys_ResetWarnOptions(void)
 }
 
 void
-PySys_AddWarnOption(const char *s)
+PySys_AddWarnOption(const wchar_t *s)
 {
 	PyObject *str;
 
@@ -892,7 +892,7 @@ PySys_AddWarnOption(const char *s)
 		if (warnoptions == NULL)
 			return;
 	}
-	str = PyUnicode_FromString(s);
+	str = PyUnicode_FromWideChar(s, -1);
 	if (str != NULL) {
 		PyList_Append(warnoptions, str);
 		Py_DECREF(str);
@@ -1222,12 +1222,12 @@ _PySys_Init(void)
 	SET_SYS_FROM_STRING("platform",
 			    PyUnicode_FromString(Py_GetPlatform()));
 	SET_SYS_FROM_STRING("executable",
-			    PyUnicode_DecodeFSDefault(
-				Py_GetProgramFullPath()));
+			    PyUnicode_FromWideChar(
+				   Py_GetProgramFullPath(), -1));
 	SET_SYS_FROM_STRING("prefix",
-			    PyUnicode_DecodeFSDefault(Py_GetPrefix()));
+			    PyUnicode_FromWideChar(Py_GetPrefix(), -1));
 	SET_SYS_FROM_STRING("exec_prefix",
-		   	    PyUnicode_DecodeFSDefault(Py_GetExecPrefix()));
+		   	    PyUnicode_FromWideChar(Py_GetExecPrefix(), -1));
 	SET_SYS_FROM_STRING("maxsize",
 			    PyLong_FromSsize_t(PY_SSIZE_T_MAX));
 	SET_SYS_FROM_STRING("float_info",
@@ -1280,15 +1280,15 @@ _PySys_Init(void)
 }
 
 static PyObject *
-makepathobject(const char *path, int delim)
+makepathobject(const wchar_t *path, wchar_t delim)
 {
 	int i, n;
-	const char *p;
+	const wchar_t *p;
 	PyObject *v, *w;
 
 	n = 1;
 	p = path;
-	while ((p = strchr(p, delim)) != NULL) {
+	while ((p = wcschr(p, delim)) != NULL) {
 		n++;
 		p++;
 	}
@@ -1296,10 +1296,10 @@ makepathobject(const char *path, int delim)
 	if (v == NULL)
 		return NULL;
 	for (i = 0; ; i++) {
-		p = strchr(path, delim);
+		p = wcschr(path, delim);
 		if (p == NULL)
-			p = strchr(path, '\0'); /* End of string */
-		w = PyUnicode_DecodeFSDefaultAndSize(path, (Py_ssize_t) (p - path));
+			p = wcschr(path, L'\0'); /* End of string */
+		w = PyUnicode_FromWideChar(path, (Py_ssize_t)(p - path));
 		if (w == NULL) {
 			Py_DECREF(v);
 			return NULL;
@@ -1313,7 +1313,7 @@ makepathobject(const char *path, int delim)
 }
 
 void
-PySys_SetPath(const char *path)
+PySys_SetPath(const wchar_t *path)
 {
 	PyObject *v;
 	if ((v = makepathobject(path, DELIM)) == NULL)
@@ -1324,12 +1324,12 @@ PySys_SetPath(const char *path)
 }
 
 static PyObject *
-makeargvobject(int argc, char **argv)
+makeargvobject(int argc, wchar_t **argv)
 {
 	PyObject *av;
 	if (argc <= 0 || argv == NULL) {
 		/* Ensure at least one (empty) argument is seen */
-		static char *empty_argv[1] = {""};
+		static wchar_t *empty_argv[1] = {L""};
 		argv = empty_argv;
 		argc = 1;
 	}
@@ -1351,7 +1351,7 @@ makeargvobject(int argc, char **argv)
 			} else
 				v = PyUnicode_FromString(argv[i]);
 #else
-			PyObject *v = PyUnicode_FromString(argv[i]);
+			PyObject *v = PyUnicode_FromWideChar(argv[i], -1);
 #endif
 			if (v == NULL) {
 				Py_DECREF(av);
@@ -1364,13 +1364,38 @@ makeargvobject(int argc, char **argv)
 	return av;
 }
 
+#ifdef HAVE_REALPATH
+static wchar_t*
+_wrealpath(const wchar_t *path, wchar_t *resolved_path)
+{
+	char cpath[PATH_MAX];
+	char cresolved_path[PATH_MAX];
+	char *res;
+	size_t r;
+	r = wcstombs(cpath, path, PATH_MAX);
+	if (r == (size_t)-1 || r >= PATH_MAX) {
+		errno = EINVAL;
+		return NULL;
+	}
+	res = realpath(cpath, cresolved_path);
+	if (res == NULL)
+		return NULL;
+	r = mbstowcs(resolved_path, cresolved_path, PATH_MAX);
+	if (r == (size_t)-1 || r >= PATH_MAX) {
+		errno = EINVAL;
+		return NULL;
+	}
+	return resolved_path;
+}
+#endif
+
 void
-PySys_SetArgv(int argc, char **argv)
+PySys_SetArgv(int argc, wchar_t **argv)
 {
 #if defined(HAVE_REALPATH)
-	char fullpath[MAXPATHLEN];
+	wchar_t fullpath[MAXPATHLEN];
 #elif defined(MS_WINDOWS)
-	char fullpath[MAX_PATH];
+	wchar_t fullpath[MAX_PATH];
 #endif
 	PyObject *av = makeargvobject(argc, argv);
 	PyObject *path = PySys_GetObject("path");
@@ -1379,53 +1404,54 @@ PySys_SetArgv(int argc, char **argv)
 	if (PySys_SetObject("argv", av) != 0)
 		Py_FatalError("can't assign sys.argv");
 	if (path != NULL) {
-		char *argv0 = argv[0];
-		char *p = NULL;
+		wchar_t *argv0 = argv[0];
+		wchar_t *p = NULL;
 		Py_ssize_t n = 0;
 		PyObject *a;
+		extern int _Py_wreadlink(const wchar_t *, wchar_t *, size_t);
 #ifdef HAVE_READLINK
-		char link[MAXPATHLEN+1];
-		char argv0copy[2*MAXPATHLEN+1];
+		wchar_t link[MAXPATHLEN+1];
+		wchar_t argv0copy[2*MAXPATHLEN+1];
 		int nr = 0;
-		if (argc > 0 && argv0 != NULL && strcmp(argv0, "-c") != 0)
-			nr = readlink(argv0, link, MAXPATHLEN);
+		if (argc > 0 && argv0 != NULL && wcscmp(argv0, L"-c") != 0)
+			nr = _Py_wreadlink(argv0, link, MAXPATHLEN);
 		if (nr > 0) {
 			/* It's a symlink */
 			link[nr] = '\0';
 			if (link[0] == SEP)
 				argv0 = link; /* Link to absolute path */
-			else if (strchr(link, SEP) == NULL)
+			else if (wcschr(link, SEP) == NULL)
 				; /* Link without path */
 			else {
 				/* Must join(dirname(argv0), link) */
-				char *q = strrchr(argv0, SEP);
+				wchar_t *q = wcsrchr(argv0, SEP);
 				if (q == NULL)
 					argv0 = link; /* argv0 without path */
 				else {
 					/* Must make a copy */
-					strcpy(argv0copy, argv0);
-					q = strrchr(argv0copy, SEP);
-					strcpy(q+1, link);
+					wcscpy(argv0copy, argv0);
+					q = wcsrchr(argv0copy, SEP);
+					wcscpy(q+1, link);
 					argv0 = argv0copy;
 				}
 			}
 		}
 #endif /* HAVE_READLINK */
 #if SEP == '\\' /* Special case for MS filename syntax */
-		if (argc > 0 && argv0 != NULL && strcmp(argv0, "-c") != 0) {
-			char *q;
+		if (argc > 0 && argv0 != NULL && wcscmp(argv0, L"-c") != 0) {
+			wchar_t *q;
 #ifdef MS_WINDOWS
-			char *ptemp;
-			if (GetFullPathName(argv0,
-					   sizeof(fullpath),
+			wchar_t *ptemp;
+			if (GetFullPathNameW(argv0,
+					   sizeof(fullpath)/sizeof(fullpath[0]),
 					   fullpath,
 					   &ptemp)) {
 				argv0 = fullpath;
 			}
 #endif
-			p = strrchr(argv0, SEP);
+			p = wcsrchr(argv0, SEP);
 			/* Test for alternate separator */
-			q = strrchr(p ? p : argv0, '/');
+			q = wcsrchr(p ? p : argv0, '/');
 			if (q != NULL)
 				p = q;
 			if (p != NULL) {
@@ -1435,13 +1461,13 @@ PySys_SetArgv(int argc, char **argv)
 			}
 		}
 #else /* All other filename syntaxes */
-		if (argc > 0 && argv0 != NULL && strcmp(argv0, "-c") != 0) {
+		if (argc > 0 && argv0 != NULL && wcscmp(argv0, L"-c") != 0) {
 #if defined(HAVE_REALPATH)
-			if (realpath(argv0, fullpath)) {
+			if (_wrealpath(argv0, fullpath)) {
 				argv0 = fullpath;
 			}
 #endif
-			p = strrchr(argv0, SEP);
+			p = wcsrchr(argv0, SEP);
 		}
 		if (p != NULL) {
 			n = p + 1 - argv0;
@@ -1451,7 +1477,7 @@ PySys_SetArgv(int argc, char **argv)
 #endif /* Unix */
 		}
 #endif /* All others */
-		a = PyUnicode_FromStringAndSize(argv0, n);
+		a = PyUnicode_FromWideChar(argv0, n);
 		if (a == NULL)
 			Py_FatalError("no mem for sys.path insertion");
 		if (PyList_Insert(path, 0, a) < 0)