gh-127350: Add Py_fopen() and Py_fclose() functions (#127821)

Doc/c-api/sys.rst

@@ -216,6 +216,38 @@ Operating System Utilities
       The function now uses the UTF-8 encoding on Windows if
       :c:member:`PyPreConfig.legacy_windows_fs_encoding` is zero.
 
+.. c:function:: FILE* Py_fopen(PyObject *path, const char *mode)
+
+   Similar to :c:func:`!fopen`, but *path* is a Python object and
+   an exception is set on error.
+
+   *path* must be a :class:`str` object, a :class:`bytes` object,
+   or a :term:`path-like object`.
+
+   On success, return the new file pointer.
+   On error, set an exception and return ``NULL``.
+
+   The file must be closed by :c:func:`Py_fclose` rather than calling directly
+   :c:func:`!fclose`.
+
+   The file descriptor is created non-inheritable (:pep:`446`).
+
+   The caller must hold the GIL.
+
+   .. versionadded:: next
+
+
+.. c:function:: int Py_fclose(FILE *file)
+
+   Close a file that was opened by :c:func:`Py_fopen`.
+
+   On success, return ``0``.
+   On error, return ``EOF`` and ``errno`` is set to indicate the error.
+   In either case, any further access (including another call to
+   :c:func:`Py_fclose`) to the stream results in undefined behavior.
+
+   .. versionadded:: next
+
 
 .. _systemfunctions:
 

Doc/whatsnew/3.14.rst

@@ -1237,6 +1237,12 @@ New features
   :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT`
   events, respectively.
 
+* Add :c:func:`Py_fopen` function to open a file. Similar to the
+  :c:func:`!fopen` function, but the *path* parameter is a Python object and an
+  exception is set on error. Add also :c:func:`Py_fclose` function to close a
+  file.
+  (Contributed by Victor Stinner in :gh:`127350`.)
+
 
 Porting to Python 3.14
 ----------------------

Include/cpython/fileutils.h

@@ -2,7 +2,13 @@
 #  error "this header file must not be included directly"
 #endif
 
-// Used by _testcapi which must not use the internal C API
+PyAPI_FUNC(FILE*) Py_fopen(
-PyAPI_FUNC(FILE*) _Py_fopen_obj(
     PyObject *path,
     const char *mode);
+
+// Deprecated alias to Py_fopen() kept for backward compatibility
+Py_DEPRECATED(3.14) PyAPI_FUNC(FILE*) _Py_fopen_obj(
+    PyObject *path,
+    const char *mode);
+
+PyAPI_FUNC(int) Py_fclose(FILE *file);

Lib/test/test_capi/test_file.py

@@ -0,0 +1,67 @@
+import os
+import unittest
+from test import support
+from test.support import import_helper, os_helper
+
+_testcapi = import_helper.import_module('_testcapi')
+
+
+class CAPIFileTest(unittest.TestCase):
+    def test_py_fopen(self):
+        # Test Py_fopen() and Py_fclose()
+
+        with open(__file__, "rb") as fp:
+            source = fp.read()
+
+        for filename in (__file__, os.fsencode(__file__)):
+            with self.subTest(filename=filename):
+                data = _testcapi.py_fopen(filename, "rb")
+                self.assertEqual(data, source[:256])
+
+                data = _testcapi.py_fopen(os_helper.FakePath(filename), "rb")
+                self.assertEqual(data, source[:256])
+
+        filenames = [
+            os_helper.TESTFN,
+            os.fsencode(os_helper.TESTFN),
+        ]
+        # TESTFN_UNDECODABLE cannot be used to create a file on macOS/WASI.
+        if os_helper.TESTFN_UNENCODABLE is not None:
+            filenames.append(os_helper.TESTFN_UNENCODABLE)
+        for filename in filenames:
+            with self.subTest(filename=filename):
+                try:
+                    with open(filename, "wb") as fp:
+                        fp.write(source)
+
+                    data = _testcapi.py_fopen(filename, "rb")
+                    self.assertEqual(data, source[:256])
+                finally:
+                    os_helper.unlink(filename)
+
+        # embedded null character/byte in the filename
+        with self.assertRaises(ValueError):
+            _testcapi.py_fopen("a\x00b", "rb")
+        with self.assertRaises(ValueError):
+            _testcapi.py_fopen(b"a\x00b", "rb")
+
+        # non-ASCII mode failing with "Invalid argument"
+        with self.assertRaises(OSError):
+            _testcapi.py_fopen(__file__, "\xe9")
+
+        # invalid filename type
+        for invalid_type in (123, object()):
+            with self.subTest(filename=invalid_type):
+                with self.assertRaises(TypeError):
+                    _testcapi.py_fopen(invalid_type, "rb")
+
+        if support.MS_WINDOWS:
+            with self.assertRaises(OSError):
+                # On Windows, the file mode is limited to 10 characters
+                _testcapi.py_fopen(__file__, "rt+, ccs=UTF-8")
+
+        # CRASHES py_fopen(__file__, None)
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_ssl.py

@@ -1325,8 +1325,7 @@ class ContextTests(unittest.TestCase):
     def test_load_dh_params(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
         ctx.load_dh_params(DHFILE)
-        if os.name != 'nt':
+        ctx.load_dh_params(BYTES_DHFILE)
-            ctx.load_dh_params(BYTES_DHFILE)
         self.assertRaises(TypeError, ctx.load_dh_params)
         self.assertRaises(TypeError, ctx.load_dh_params, None)
         with self.assertRaises(FileNotFoundError) as cm:

Misc/NEWS.d/next/C_API/2024-12-11-13-01-26.gh-issue-127350.uEBZZ4.rst

@@ -0,0 +1,5 @@
+Add :c:func:`Py_fopen` function to open a file. Similar to the :c:func:`!fopen`
+function, but the *path* parameter is a Python object and an exception is set
+on error. Add also :c:func:`Py_fclose` function to close a file, function
+needed for Windows support.
+Patch by Victor Stinner.

Modules/_ssl.c

@@ -4377,7 +4377,7 @@ _ssl__SSLContext_load_dh_params_impl(PySSLContext *self, PyObject *filepath)
     FILE *f;
     DH *dh;
 
-    f = _Py_fopen_obj(filepath, "rb");
+    f = Py_fopen(filepath, "rb");
     if (f == NULL)
         return NULL;
 

Modules/_ssl/debughelpers.c

@@ -180,8 +180,8 @@ _PySSLContext_set_keylog_filename(PySSLContext *self, PyObject *arg, void *c) {
         return 0;
     }
 
-    /* _Py_fopen_obj() also checks that arg is of proper type. */
+    /* Py_fopen() also checks that arg is of proper type. */
-    fp = _Py_fopen_obj(arg, "a" PY_STDIOTEXTMODE);
+    fp = Py_fopen(arg, "a" PY_STDIOTEXTMODE);
     if (fp == NULL)
         return -1;
 

Modules/_testcapi/clinic/file.c.h

@@ -0,0 +1,48 @@
+/*[clinic input]
+preserve
+[clinic start generated code]*/
+
+#include "pycore_modsupport.h"    // _PyArg_CheckPositional()
+
+PyDoc_STRVAR(_testcapi_py_fopen__doc__,
+"py_fopen($module, path, mode, /)\n"
+"--\n"
+"\n"
+"Call Py_fopen(), fread(256) and Py_fclose(). Return read bytes.");
+
+#define _TESTCAPI_PY_FOPEN_METHODDEF    \
+    {"py_fopen", _PyCFunction_CAST(_testcapi_py_fopen), METH_FASTCALL, _testcapi_py_fopen__doc__},
+
+static PyObject *
+_testcapi_py_fopen_impl(PyObject *module, PyObject *path, const char *mode);
+
+static PyObject *
+_testcapi_py_fopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
+{
+    PyObject *return_value = NULL;
+    PyObject *path;
+    const char *mode;
+
+    if (!_PyArg_CheckPositional("py_fopen", nargs, 2, 2)) {
+        goto exit;
+    }
+    path = args[0];
+    if (!PyUnicode_Check(args[1])) {
+        _PyArg_BadArgument("py_fopen", "argument 2", "str", args[1]);
+        goto exit;
+    }
+    Py_ssize_t mode_length;
+    mode = PyUnicode_AsUTF8AndSize(args[1], &mode_length);
+    if (mode == NULL) {
+        goto exit;
+    }
+    if (strlen(mode) != (size_t)mode_length) {
+        PyErr_SetString(PyExc_ValueError, "embedded null character");
+        goto exit;
+    }
+    return_value = _testcapi_py_fopen_impl(module, path, mode);
+
+exit:
+    return return_value;
+}
+/*[clinic end generated code: output=c9fe964c3e5a0c32 input=a9049054013a1b77]*/

Modules/_testcapi/file.c

@@ -1,8 +1,43 @@
+// clinic/file.c.h uses internal pycore_modsupport.h API
+#define PYTESTCAPI_NEED_INTERNAL_API
+
 #include "parts.h"
 #include "util.h"
+#include "clinic/file.c.h"
 
+/*[clinic input]
+module _testcapi
+[clinic start generated code]*/
+/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6361033e795369fc]*/
+
+/*[clinic input]
+_testcapi.py_fopen
+
+    path: object
+    mode: str
+    /
+
+Call Py_fopen(), fread(256) and Py_fclose(). Return read bytes.
+[clinic start generated code]*/
+
+static PyObject *
+_testcapi_py_fopen_impl(PyObject *module, PyObject *path, const char *mode)
+/*[clinic end generated code: output=5a900af000f759de input=d7e7b8f0fd151953]*/
+{
+    FILE *fp = Py_fopen(path, mode);
+    if (fp == NULL) {
+        return NULL;
+    }
+
+    char buffer[256];
+    size_t size = fread(buffer, 1, Py_ARRAY_LENGTH(buffer), fp);
+    Py_fclose(fp);
+
+    return PyBytes_FromStringAndSize(buffer, size);
+}
 
 static PyMethodDef test_methods[] = {
+    _TESTCAPI_PY_FOPEN_METHODDEF
     {NULL},
 };
 

Modules/_testcapi/object.c

@@ -15,7 +15,7 @@ call_pyobject_print(PyObject *self, PyObject * args)
         return NULL;
     }
 
-    fp = _Py_fopen_obj(filename, "w+");
+    fp = Py_fopen(filename, "w+");
 
     if (Py_IsTrue(print_raw)) {
         flags = Py_PRINT_RAW;
@@ -41,7 +41,7 @@ pyobject_print_null(PyObject *self, PyObject *args)
         return NULL;
     }
 
-    fp = _Py_fopen_obj(filename, "w+");
+    fp = Py_fopen(filename, "w+");
 
     if (PyObject_Print(NULL, fp, 0) < 0) {
         fclose(fp);
@@ -72,7 +72,7 @@ pyobject_print_noref_object(PyObject *self, PyObject *args)
         return NULL;
     }
 
-    fp = _Py_fopen_obj(filename, "w+");
+    fp = Py_fopen(filename, "w+");
 
     if (PyObject_Print(test_string, fp, 0) < 0){
         fclose(fp);
@@ -103,7 +103,7 @@ pyobject_print_os_error(PyObject *self, PyObject *args)
     }
 
     // open file in read mode to induce OSError
-    fp = _Py_fopen_obj(filename, "r");
+    fp = Py_fopen(filename, "r");
 
     if (PyObject_Print(test_string, fp, 0) < 0) {
         fclose(fp);

Modules/_testcapimodule.c

@@ -1744,7 +1744,7 @@ pymarshal_write_long_to_file(PyObject* self, PyObject *args)
                           &value, &filename, &version))
         return NULL;
 
-    fp = _Py_fopen_obj(filename, "wb");
+    fp = Py_fopen(filename, "wb");
     if (fp == NULL) {
         PyErr_SetFromErrno(PyExc_OSError);
         return NULL;
@@ -1769,7 +1769,7 @@ pymarshal_write_object_to_file(PyObject* self, PyObject *args)
                           &obj, &filename, &version))
         return NULL;
 
-    fp = _Py_fopen_obj(filename, "wb");
+    fp = Py_fopen(filename, "wb");
     if (fp == NULL) {
         PyErr_SetFromErrno(PyExc_OSError);
         return NULL;
@@ -1793,7 +1793,7 @@ pymarshal_read_short_from_file(PyObject* self, PyObject *args)
     if (!PyArg_ParseTuple(args, "O:pymarshal_read_short_from_file", &filename))
         return NULL;
 
-    fp = _Py_fopen_obj(filename, "rb");
+    fp = Py_fopen(filename, "rb");
     if (fp == NULL) {
         PyErr_SetFromErrno(PyExc_OSError);
         return NULL;
@@ -1818,7 +1818,7 @@ pymarshal_read_long_from_file(PyObject* self, PyObject *args)
     if (!PyArg_ParseTuple(args, "O:pymarshal_read_long_from_file", &filename))
         return NULL;
 
-    fp = _Py_fopen_obj(filename, "rb");
+    fp = Py_fopen(filename, "rb");
     if (fp == NULL) {
         PyErr_SetFromErrno(PyExc_OSError);
         return NULL;
@@ -1840,7 +1840,7 @@ pymarshal_read_last_object_from_file(PyObject* self, PyObject *args)
     if (!PyArg_ParseTuple(args, "O:pymarshal_read_last_object_from_file", &filename))
         return NULL;
 
-    FILE *fp = _Py_fopen_obj(filename, "rb");
+    FILE *fp = Py_fopen(filename, "rb");
     if (fp == NULL) {
         PyErr_SetFromErrno(PyExc_OSError);
         return NULL;
@@ -1863,7 +1863,7 @@ pymarshal_read_object_from_file(PyObject* self, PyObject *args)
     if (!PyArg_ParseTuple(args, "O:pymarshal_read_object_from_file", &filename))
         return NULL;
 
-    FILE *fp = _Py_fopen_obj(filename, "rb");
+    FILE *fp = Py_fopen(filename, "rb");
     if (fp == NULL) {
         PyErr_SetFromErrno(PyExc_OSError);
         return NULL;

Modules/main.c

@@ -370,7 +370,7 @@ pymain_run_file_obj(PyObject *program_name, PyObject *filename,
         return pymain_exit_err_print();
     }
 
-    FILE *fp = _Py_fopen_obj(filename, "rb");
+    FILE *fp = Py_fopen(filename, "rb");
     if (fp == NULL) {
         // Ignore the OSError
         PyErr_Clear();
@@ -465,7 +465,7 @@ pymain_run_startup(PyConfig *config, int *exitcode)
         goto error;
     }
 
-    FILE *fp = _Py_fopen_obj(startup, "r");
+    FILE *fp = Py_fopen(startup, "r");
     if (fp == NULL) {
         int save_errno = errno;
         PyErr_Clear();

Python/errors.c

@@ -1981,7 +1981,7 @@ _PyErr_ProgramDecodedTextObject(PyObject *filename, int lineno, const char* enco
         return NULL;
     }
 
-    FILE *fp = _Py_fopen_obj(filename, "r" PY_STDIOTEXTMODE);
+    FILE *fp = Py_fopen(filename, "r" PY_STDIOTEXTMODE);
     if (fp == NULL) {
         PyErr_Clear();
         return NULL;

Python/fileutils.c

@@ -1748,8 +1748,10 @@ _Py_wfopen(const wchar_t *path, const wchar_t *mode)
 }
 
 
-/* Open a file. Call _wfopen() on Windows, or encode the path to the filesystem
+/* Open a file.
-   encoding and call fopen() otherwise.
+
+   On Windows, if 'path' is a Unicode string, call _wfopen(). Otherwise, encode
+   the path to the filesystem encoding and call fopen().
 
    Return the new file object on success. Raise an exception and return NULL
    on error.
@@ -1762,32 +1764,32 @@ _Py_wfopen(const wchar_t *path, const wchar_t *mode)
    Release the GIL to call _wfopen() or fopen(). The caller must hold
    the GIL. */
 FILE*
-_Py_fopen_obj(PyObject *path, const char *mode)
+Py_fopen(PyObject *path, const char *mode)
 {
-    FILE *f;
-    int async_err = 0;
-#ifdef MS_WINDOWS
-    wchar_t wmode[10];
-    int usize;
-
     assert(PyGILState_Check());
 
     if (PySys_Audit("open", "Osi", path, mode, 0) < 0) {
         return NULL;
     }
-    if (!PyUnicode_Check(path)) {
+
-        PyErr_Format(PyExc_TypeError,
+    FILE *f;
-                     "str file path expected under Windows, got %R",
+    int async_err = 0;
-                     Py_TYPE(path));
+    int saved_errno;
+#ifdef MS_WINDOWS
+    PyObject *unicode;
+    if (!PyUnicode_FSDecoder(path, &unicode)) {
         return NULL;
     }
 
-    wchar_t *wpath = PyUnicode_AsWideCharString(path, NULL);
+    wchar_t *wpath = PyUnicode_AsWideCharString(unicode, NULL);
-    if (wpath == NULL)
+    Py_DECREF(unicode);
+    if (wpath == NULL) {
         return NULL;
+    }
 
-    usize = MultiByteToWideChar(CP_ACP, 0, mode, -1,
+    wchar_t wmode[10];
-                                wmode, Py_ARRAY_LENGTH(wmode));
+    int usize = MultiByteToWideChar(CP_ACP, 0, mode, -1,
+                                    wmode, Py_ARRAY_LENGTH(wmode));
     if (usize == 0) {
         PyErr_SetFromWindowsErr(0);
         PyMem_Free(wpath);
@@ -1796,26 +1798,20 @@ _Py_fopen_obj(PyObject *path, const char *mode)
 
     do {
         Py_BEGIN_ALLOW_THREADS
+        _Py_BEGIN_SUPPRESS_IPH
         f = _wfopen(wpath, wmode);
+        _Py_END_SUPPRESS_IPH
         Py_END_ALLOW_THREADS
     } while (f == NULL
              && errno == EINTR && !(async_err = PyErr_CheckSignals()));
-    int saved_errno = errno;
+    saved_errno = errno;
     PyMem_Free(wpath);
 #else
     PyObject *bytes;
-    const char *path_bytes;
+    if (!PyUnicode_FSConverter(path, &bytes)) {
-
-    assert(PyGILState_Check());
-
-    if (!PyUnicode_FSConverter(path, &bytes))
-        return NULL;
-    path_bytes = PyBytes_AS_STRING(bytes);
-
-    if (PySys_Audit("open", "Osi", path, mode, 0) < 0) {
-        Py_DECREF(bytes);
         return NULL;
     }
+    const char *path_bytes = PyBytes_AS_STRING(bytes);
 
     do {
         Py_BEGIN_ALLOW_THREADS
@@ -1823,11 +1819,13 @@ _Py_fopen_obj(PyObject *path, const char *mode)
         Py_END_ALLOW_THREADS
     } while (f == NULL
              && errno == EINTR && !(async_err = PyErr_CheckSignals()));
-    int saved_errno = errno;
+    saved_errno = errno;
     Py_DECREF(bytes);
 #endif
-    if (async_err)
+
+    if (async_err) {
         return NULL;
+    }
 
     if (f == NULL) {
         errno = saved_errno;
@@ -1842,6 +1840,27 @@ _Py_fopen_obj(PyObject *path, const char *mode)
     return f;
 }
 
+
+// Deprecated alias to Py_fopen() kept for backward compatibility
+FILE*
+_Py_fopen_obj(PyObject *path, const char *mode)
+{
+    return Py_fopen(path, mode);
+}
+
+
+// Call fclose().
+//
+// On Windows, files opened by Py_fopen() in the Python DLL must be closed by
+// the Python DLL to use the same C runtime version. Otherwise, calling
+// fclose() directly can cause undefined behavior.
+int
+Py_fclose(FILE *file)
+{
+    return fclose(file);
+}
+
+
 /* Read count bytes from fd into buf.
 
    On success, return the number of read bytes, it can be lower than count.

Python/import.c

@@ -4688,7 +4688,7 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file)
      * code relies on fp still being open. */
     FILE *fp;
     if (file != NULL) {
-        fp = _Py_fopen_obj(info.filename, "r");
+        fp = Py_fopen(info.filename, "r");
         if (fp == NULL) {
             goto finally;
         }

Python/pythonrun.c

@@ -467,7 +467,7 @@ _PyRun_SimpleFileObject(FILE *fp, PyObject *filename, int closeit,
             fclose(fp);
         }
 
-        pyc_fp = _Py_fopen_obj(filename, "rb");
+        pyc_fp = Py_fopen(filename, "rb");
         if (pyc_fp == NULL) {
             fprintf(stderr, "python: Can't reopen .pyc file\n");
             goto done;

Python/sysmodule.c

@@ -2356,7 +2356,7 @@ static PyObject *
 sys__dump_tracelets_impl(PyObject *module, PyObject *outpath)
 /*[clinic end generated code: output=a7fe265e2bc3b674 input=5bff6880cd28ffd1]*/
 {
-    FILE *out = _Py_fopen_obj(outpath, "wb");
+    FILE *out = Py_fopen(outpath, "wb");
     if (out == NULL) {
         return NULL;
     }