Patch #683592 revisited, after discussions with MvL:

- Implement the behavior as specified in PEP 277, meaning os.listdir()
  will only return unicode strings if it is _called_ with a unicode
  argument.
- And then return only unicode, don't attempt to convert to ASCII.
- Don't switch on Py_FileSystemDefaultEncoding, but simply use the
  default encoding if Py_FileSystemDefaultEncoding is NULL. This means
  os.listdir() can now raise UnicodeDecodeError if the default encoding
  can't represent the directory entry. (This seems better than silcencing
  the error and fall back to a byte string.)
- Attempted to decribe the above in Doc/lib/libos.tex.
- Reworded the Misc/NEWS items to reflect the current situation.

This checkin also fixes bug #696261, which was due to os.listdir() not
using Py_FileSystemDefaultEncoding, like all file system calls are
supposed to.

Modules/posixmodule.c

@@ -1775,7 +1775,13 @@ posix_listdir(PyObject *self, PyObject *args)
 	PyObject *d, *v;
 	DIR *dirp;
 	struct dirent *ep;
-	if (!PyArg_ParseTuple(args, "s:listdir", &name))
+	int arg_is_unicode = 1;
+
+	if (!PyArg_ParseTuple(args, "U:listdir", &v)) {
+		arg_is_unicode = 0;
+		PyErr_Clear();
+	}
+	if (!PyArg_ParseTuple(args, "et:listdir", Py_FileSystemDefaultEncoding, &name))
 		return NULL;
 	if ((dirp = opendir(name)) == NULL) {
 		return posix_error_with_filename(name);
@@ -1796,7 +1802,7 @@ posix_listdir(PyObject *self, PyObject *args)
 			break;
 		}
 #ifdef Py_USING_UNICODE
-		if (Py_FileSystemDefaultEncoding != NULL) {
+		if (arg_is_unicode) {
 			PyObject *w;
 
 			w = PyUnicode_FromEncodedObject(v,
@@ -1809,14 +1815,6 @@ posix_listdir(PyObject *self, PyObject *args)
 				d = NULL;
 				break;
 			}
-			/* attempt to convert to ASCII */
-			w = PyUnicode_AsASCIIString(v);
-			if (w != NULL) {
-				Py_DECREF(v);
-				v = w;
-			}
-			else
-				PyErr_Clear();
 		}
 #endif
 		if (PyList_Append(d, v) != 0) {