bpo-43244: Add pycore_compile.h header file (GH-25000)

Remove the compiler functions using "struct _mod" type, because the
public AST C API was removed:

* PyAST_Compile()
* PyAST_CompileEx()
* PyAST_CompileObject()
* PyFuture_FromAST()
* PyFuture_FromASTObject()

These functions were undocumented and excluded from the limited C API.

Rename functions:

* PyAST_CompileObject() => _PyAST_Compile()
* PyFuture_FromASTObject() => _PyFuture_FromAST()

Moreover, _PyFuture_FromAST() is no longer exported (replace
PyAPI_FUNC() with extern). _PyAST_Compile() remains exported for
test_peg_generator.

Remove also compatibility functions:

* PyAST_Compile()
* PyAST_CompileEx()
* PyFuture_FromAST()

Doc/whatsnew/3.10.rst

@@ -1399,3 +1399,15 @@ Removed
   ``Yield`` macro which was conflict with the ``Yield`` name used by the
   Windows ``<winbase.h>`` header. Use the Python :mod:`ast` module instead.
   (Contributed by Victor Stinner in :issue:`43244`.)
+
+* Remove the compiler functions using ``struct _mod`` type, because the public
+  AST C API was removed:
+
+  * ``PyAST_Compile()``
+  * ``PyAST_CompileEx()``
+  * ``PyAST_CompileObject()``
+  * ``PyFuture_FromAST()``
+  * ``PyFuture_FromASTObject()``
+
+  These functions were undocumented and excluded from the limited C API.
+  (Contributed by Victor Stinner in :issue:`43244`.)

Include/cpython/compile.h

@@ -47,40 +47,6 @@ typedef struct {
 #define FUTURE_GENERATOR_STOP "generator_stop"
 #define FUTURE_ANNOTATIONS "annotations"
 
-struct _mod;   // Type defined in pycore_ast.h
-
-#define PyAST_Compile(mod, s, f, ar) PyAST_CompileEx(mod, s, f, -1, ar)
-PyAPI_FUNC(PyCodeObject *) PyAST_CompileEx(
-    struct _mod *mod,
-    const char *filename,       /* decoded from the filesystem encoding */
-    PyCompilerFlags *flags,
-    int optimize,
-    PyArena *arena);
-PyAPI_FUNC(PyCodeObject *) PyAST_CompileObject(
-    struct _mod *mod,
-    PyObject *filename,
-    PyCompilerFlags *flags,
-    int optimize,
-    PyArena *arena);
-PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(
-    struct _mod * mod,
-    const char *filename        /* decoded from the filesystem encoding */
-    );
-PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromASTObject(
-    struct _mod * mod,
-    PyObject *filename
-    );
-
-/* _Py_Mangle is defined in compile.c */
-PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name);
-
 #define PY_INVALID_STACK_EFFECT INT_MAX
 PyAPI_FUNC(int) PyCompile_OpcodeStackEffect(int opcode, int oparg);
 PyAPI_FUNC(int) PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump);
-
-typedef struct {
-    int optimize;
-    int ff_features;
-} _PyASTOptimizeState;
-
-PyAPI_FUNC(int) _PyAST_Optimize(struct _mod *, PyArena *arena, _PyASTOptimizeState *state);

Include/internal/pycore_compile.h

@@ -0,0 +1,40 @@
+#ifndef Py_INTERNAL_COMPILE_H
+#define Py_INTERNAL_COMPILE_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+struct _mod;   // Type defined in pycore_ast.h
+
+// Export the symbol for test_peg_generator (built as a library)
+PyAPI_FUNC(PyCodeObject*) _PyAST_Compile(
+    struct _mod *mod,
+    PyObject *filename,
+    PyCompilerFlags *flags,
+    int optimize,
+    PyArena *arena);
+extern PyFutureFeatures* _PyFuture_FromAST(
+    struct _mod * mod,
+    PyObject *filename
+    );
+
+extern PyObject* _Py_Mangle(PyObject *p, PyObject *name);
+
+typedef struct {
+    int optimize;
+    int ff_features;
+} _PyASTOptimizeState;
+
+extern int _PyAST_Optimize(
+    struct _mod *,
+    PyArena *arena,
+    _PyASTOptimizeState *state);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_COMPILE_H */

Makefile.pre.in

@@ -1146,6 +1146,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_call.h \
 		$(srcdir)/Include/internal/pycore_ceval.h \
 		$(srcdir)/Include/internal/pycore_code.h \
+		$(srcdir)/Include/internal/pycore_compile.h \
 		$(srcdir)/Include/internal/pycore_condvar.h \
 		$(srcdir)/Include/internal/pycore_context.h \
 		$(srcdir)/Include/internal/pycore_dtoa.h \

Misc/NEWS.d/next/C API/2021-03-23-20-53-41.bpo-43244.VK3sLH.rst

@@ -0,0 +1,11 @@
+Remove the compiler functions using ``struct _mod`` type, because the public
+AST C API was removed:
+
+* ``PyAST_Compile()``
+* ``PyAST_CompileEx()``
+* ``PyAST_CompileObject()``
+* ``PyFuture_FromAST()``
+* ``PyFuture_FromASTObject()``
+
+These functions were undocumented and excluded from the limited C API.
+Patch by Victor Stinner.

Objects/typeobject.c

@@ -2,6 +2,7 @@
 
 #include "Python.h"
 #include "pycore_call.h"
+#include "pycore_compile.h"       // _Py_Mangle()
 #include "pycore_initconfig.h"
 #include "pycore_object.h"
 #include "pycore_pyerrors.h"

PCbuild/pythoncore.vcxproj

@@ -183,6 +183,7 @@
     <ClInclude Include="..\Include\internal\pycore_call.h" />
     <ClInclude Include="..\Include\internal\pycore_ceval.h" />
     <ClInclude Include="..\Include\internal\pycore_code.h" />
+    <ClInclude Include="..\Include\internal\pycore_compile.h" />
     <ClInclude Include="..\Include\internal\pycore_condvar.h" />
     <ClInclude Include="..\Include\internal\pycore_context.h" />
     <ClInclude Include="..\Include\internal\pycore_dtoa.h" />

PCbuild/pythoncore.vcxproj.filters

@@ -510,6 +510,9 @@
     <ClInclude Include="..\Include\internal\pycore_code.h">
       <Filter>Include\internal</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\internal\pycore_compile.h">
+      <Filter>Include\internal</Filter>
+    </ClInclude>
     <ClInclude Include="..\Include\internal\pycore_condvar.h">
       <Filter>Include\internal</Filter>
     </ClInclude>

Python/ast_opt.c

@@ -1,6 +1,7 @@
 /* AST Optimizer */
 #include "Python.h"
 #include "pycore_ast.h"           // _PyAST_GetDocString()
+#include "pycore_compile.h"       // _PyASTOptimizeState
 
 
 static int

Python/bltinmodule.c

@@ -4,6 +4,7 @@
 #include <ctype.h>
 #include "pycore_ast.h"           // _PyAST_Validate()
 #undef Yield   /* undefine macro conflicting with <winbase.h> */
+#include "pycore_compile.h"       // _PyAST_Compile()
 #include "pycore_object.h"        // _Py_AddToAllObjects()
 #include "pycore_pyerrors.h"      // _PyErr_NoMemory()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
@@ -839,8 +840,8 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename,
                 PyArena_Free(arena);
                 goto error;
             }
-            result = (PyObject*)PyAST_CompileObject(mod, filename,
+            result = (PyObject*)_PyAST_Compile(mod, filename,
-                                                    &cf, optimize, arena);
+                                               &cf, optimize, arena);
             PyArena_Free(arena);
         }
         goto finally;

Python/compile.c

@@ -1,7 +1,7 @@
 /*
  * This file compiles an abstract syntax tree (AST) into Python bytecode.
  *
- * The primary entry point is PyAST_Compile(), which returns a
+ * The primary entry point is _PyAST_Compile(), which returns a
  * PyCodeObject.  The compiler makes several passes to build the code
  * object:
  *   1. Checks for future statements.  See future.c
@@ -23,6 +23,7 @@
 
 #include "Python.h"
 #include "pycore_ast.h"           // _PyAST_GetDocString()
+#include "pycore_compile.h"       // _PyFuture_FromAST()
 #include "pycore_pymem.h"         // _PyMem_IsPtrFreed()
 #include "pycore_long.h"          // _PyLong_GetZero()
 #include "pycore_symtable.h"      // PySTEntryObject
@@ -350,8 +351,8 @@ compiler_init(struct compiler *c)
 }
 
 PyCodeObject *
-PyAST_CompileObject(mod_ty mod, PyObject *filename, PyCompilerFlags *flags,
+_PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *flags,
-                   int optimize, PyArena *arena)
+               int optimize, PyArena *arena)
 {
     struct compiler c;
     PyCodeObject *co = NULL;
@@ -373,7 +374,7 @@ PyAST_CompileObject(mod_ty mod, PyObject *filename, PyCompilerFlags *flags,
     Py_INCREF(filename);
     c.c_filename = filename;
     c.c_arena = arena;
-    c.c_future = PyFuture_FromASTObject(mod, filename);
+    c.c_future = _PyFuture_FromAST(mod, filename);
     if (c.c_future == NULL)
         goto finally;
     if (!flags) {
@@ -409,21 +410,6 @@ PyAST_CompileObject(mod_ty mod, PyObject *filename, PyCompilerFlags *flags,
     return co;
 }
 
-PyCodeObject *
-PyAST_CompileEx(mod_ty mod, const char *filename_str, PyCompilerFlags *flags,
-                int optimize, PyArena *arena)
-{
-    PyObject *filename;
-    PyCodeObject *co;
-    filename = PyUnicode_DecodeFSDefault(filename_str);
-    if (filename == NULL)
-        return NULL;
-    co = PyAST_CompileObject(mod, filename, flags, optimize, arena);
-    Py_DECREF(filename);
-    return co;
-
-}
-
 static void
 compiler_free(struct compiler *c)
 {
@@ -6758,15 +6744,6 @@ assemble(struct compiler *c, int addNone)
     return co;
 }
 
-#undef PyAST_Compile
-PyCodeObject *
-PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags,
-              PyArena *arena)
-{
-    return PyAST_CompileEx(mod, filename, flags, -1, arena);
-}
-
-
 /* Replace LOAD_CONST c1, LOAD_CONST c2 ... LOAD_CONST cn, BUILD_TUPLE n
    with    LOAD_CONST (c1, c2, ... cn).
    The consts table must still be in list form so that the

Python/future.c

@@ -116,7 +116,7 @@ future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename)
 
 
 PyFutureFeatures *
-PyFuture_FromASTObject(mod_ty mod, PyObject *filename)
+_PyFuture_FromAST(mod_ty mod, PyObject *filename)
 {
     PyFutureFeatures *ff;
 
@@ -134,18 +134,3 @@ PyFuture_FromASTObject(mod_ty mod, PyObject *filename)
     }
     return ff;
 }
-
-
-PyFutureFeatures *
-PyFuture_FromAST(mod_ty mod, const char *filename_str)
-{
-    PyFutureFeatures *ff;
-    PyObject *filename;
-
-    filename = PyUnicode_DecodeFSDefault(filename_str);
-    if (filename == NULL)
-        return NULL;
-    ff = PyFuture_FromASTObject(mod, filename);
-    Py_DECREF(filename);
-    return ff;
-}

Python/pythonrun.c

@@ -13,6 +13,7 @@
 #include "pycore_ast.h"           // PyAST_mod2obj
 #undef Yield   /* undefine macro conflicting with <winbase.h> */
 
+#include "pycore_compile.h"       // _PyAST_Compile()
 #include "pycore_interp.h"        // PyInterpreterState.importlib
 #include "pycore_object.h"        // _PyDebug_PrintTotalRefs()
 #include "pycore_pyerrors.h"      // _PyErr_Fetch
@@ -1224,7 +1225,7 @@ run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals,
             PyCompilerFlags *flags, PyArena *arena)
 {
     PyThreadState *tstate = _PyThreadState_GET();
-    PyCodeObject *co = PyAST_CompileObject(mod, filename, flags, -1, arena);
+    PyCodeObject *co = _PyAST_Compile(mod, filename, flags, -1, arena);
     if (co == NULL)
         return NULL;
 
@@ -1301,7 +1302,7 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start,
         PyArena_Free(arena);
         return result;
     }
-    co = PyAST_CompileObject(mod, filename, flags, optimize, arena);
+    co = _PyAST_Compile(mod, filename, flags, optimize, arena);
     PyArena_Free(arena);
     return (PyObject *)co;
 }

Python/symtable.c

@@ -1,8 +1,9 @@
 #include "Python.h"
 #include "pycore_ast.h"           // identifier, stmt_ty
+#undef Yield   /* undefine macro conflicting with <winbase.h> */
+#include "pycore_compile.h"       // _Py_Mangle()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "pycore_symtable.h"      // PySTEntryObject
-#undef Yield   /* undefine macro conflicting with <winbase.h> */
 #include "structmember.h"         // PyMemberDef
 
 /* error strings used for warnings */

Tools/peg_generator/peg_extension/peg_extension.c

@@ -1,4 +1,6 @@
 #include "pegen.h"
+#include "pycore_compile.h"       // _PyAST_Compile()
+
 
 PyObject *
 _build_return_object(mod_ty module, int mode, PyObject *filename_ob, PyArena *arena)
@@ -6,7 +8,7 @@ _build_return_object(mod_ty module, int mode, PyObject *filename_ob, PyArena *ar
     PyObject *result = NULL;
 
     if (mode == 2) {
-        result = (PyObject *)PyAST_CompileObject(module, filename_ob, NULL, -1, arena);
+        result = (PyObject *)_PyAST_Compile(module, filename_ob, NULL, -1, arena);
     } else if (mode == 1) {
         result = PyAST_mod2obj(module);
     } else {