bpo-43244: Remove ast.h, asdl.h, Python-ast.h headers (GH-24933)

These functions were undocumented and excluded from the limited C
API.

Most names defined by these header files were not prefixed by "Py"
and so could create names conflicts. For example, Python-ast.h
defined a "Yield" macro which was conflict with the "Yield" name used
by the Windows <winbase.h> header.

Use the Python ast module instead.

* Move Include/asdl.h to Include/internal/pycore_asdl.h.
* Move Include/Python-ast.h to Include/internal/pycore_ast.h.
* Remove ast.h header file.
* pycore_symtable.h no longer includes Python-ast.h.

Parser/asdl_c.py

@@ -1445,13 +1445,12 @@ get_ast_state(void)
 }
 """)
 
-    # f-string for {mod.name}
-    f.write(f"""
-// Include {mod.name}-ast.h after pycore_interp.h to avoid conflicts
-// with the Yield macro redefined by <winbase.h>
-#include "{mod.name}-ast.h"
-#include "structmember.h"
-""")
+    print(textwrap.dedent("""
+        // Include pycore_ast.h after pycore_interp.h to avoid conflicts
+        // with the Yield macro redefined by <winbase.h>
+        #include "pycore_ast.h"
+        #include "structmember.h"
+    """).rstrip(), file=f)
 
     generate_ast_fini(module_state, f)
 
@@ -1465,33 +1464,49 @@ get_ast_state(void)
     f.write('};\n\n')
 
 def write_header(mod, f):
-    f.write('#ifndef Py_PYTHON_AST_H\n')
-    f.write('#define Py_PYTHON_AST_H\n')
-    f.write('#ifdef __cplusplus\n')
-    f.write('extern "C" {\n')
-    f.write('#endif\n')
-    f.write('\n')
-    f.write('#ifndef Py_LIMITED_API\n')
-    f.write('#include "asdl.h"\n')
-    f.write('\n')
-    f.write('#undef Yield   /* undefine macro conflicting with <winbase.h> */\n')
-    f.write('\n')
+    f.write(textwrap.dedent("""
+        #ifndef Py_INTERNAL_AST_H
+        #define Py_INTERNAL_AST_H
+        #ifdef __cplusplus
+        extern "C" {
+        #endif
+
+        #ifndef Py_BUILD_CORE
+        #  error "this header requires Py_BUILD_CORE define"
+        #endif
+
+        #include "pycore_asdl.h"
+
+        #undef Yield   /* undefine macro conflicting with <winbase.h> */
+
+    """).lstrip())
     c = ChainOfVisitors(TypeDefVisitor(f),
                         SequenceDefVisitor(f),
                         StructVisitor(f))
     c.visit(mod)
     f.write("// Note: these macros affect function definitions, not only call sites.\n")
     PrototypeVisitor(f).visit(mod)
-    f.write("\n")
-    f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
-    f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
-    f.write("int PyAST_Check(PyObject* obj);\n")
-    f.write("#endif /* !Py_LIMITED_API */\n")
-    f.write('\n')
-    f.write('#ifdef __cplusplus\n')
-    f.write('}\n')
-    f.write('#endif\n')
-    f.write('#endif /* !Py_PYTHON_AST_H */\n')
+    f.write(textwrap.dedent("""
+
+        PyObject* PyAST_mod2obj(mod_ty t);
+        mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);
+        int PyAST_Check(PyObject* obj);
+
+        extern int _PyAST_Validate(mod_ty);
+
+        /* _PyAST_ExprAsUnicode is defined in ast_unparse.c */
+        extern PyObject* _PyAST_ExprAsUnicode(expr_ty);
+
+        /* Return the borrowed reference to the first literal string in the
+           sequence of statements or NULL if it doesn't start from a literal string.
+           Doesn't set exception. */
+        extern PyObject* _PyAST_GetDocString(asdl_stmt_seq *);
+
+        #ifdef __cplusplus
+        }
+        #endif
+        #endif /* !Py_INTERNAL_AST_H */
+    """))
 
 
 def write_internal_h_header(mod, f):

Parser/pegen.c

@@ -5,7 +5,6 @@
 
 #include "pegen.h"
 #include "string_parser.h"
-#include "ast.h"
 
 PyObject *
 _PyPegen_new_type_comment(Parser *p, char *s)

Parser/pegen.h

@@ -4,7 +4,7 @@
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
 #include <token.h>
-#include <Python-ast.h>
+#include <pycore_ast.h>
 
 #if 0
 #define PyPARSE_YIELD_IS_KEYWORD        0x0001
@@ -101,7 +101,7 @@ typedef struct {
     arg_ty kwarg;
 } StarEtc;
 
-typedef struct { operator_ty kind; } AugOperator; 
+typedef struct { operator_ty kind; } AugOperator;
 typedef struct {
     void *element;
     int is_keyword;

Parser/string_parser.h

@@ -2,7 +2,7 @@
 #define STRINGS_H
 
 #include <Python.h>
-#include <Python-ast.h>
+#include <pycore_ast.h>
 #include "pegen.h"
 
 #define EXPRLIST_N_CACHED  64