mirror of
https://github.com/python/cpython.git
synced 2025-08-31 22:18:28 +00:00
Merge ast-branch to head
This change implements a new bytecode compiler, based on a transformation of the parse tree to an abstract syntax defined in Parser/Python.asdl. The compiler implementation is not complete, but it is in stable enough shape to run the entire test suite excepting two disabled tests.
This commit is contained in:
Jeremy Hylton
parent
2cb94aba12
commit
3e0055f8c6
54 changed files with 13675 additions and 6810 deletions
|
|
@ -3,13 +3,16 @@
|
|||
|
||||
#include "Python.h"
|
||||
|
||||
#include "Python-ast.h"
|
||||
#include "grammar.h"
|
||||
#include "node.h"
|
||||
#include "token.h"
|
||||
#include "parsetok.h"
|
||||
#include "errcode.h"
|
||||
#include "code.h"
|
||||
#include "compile.h"
|
||||
#include "symtable.h"
|
||||
#include "ast.h"
|
||||
#include "eval.h"
|
||||
#include "marshal.h"
|
||||
|
||||
|
|
@ -32,9 +35,9 @@ extern grammar _PyParser_Grammar; /* From graminit.c */
|
|||
/* Forward */
|
||||
static void initmain(void);
|
||||
static void initsite(void);
|
||||
static PyObject *run_err_node(node *, const char *, PyObject *, PyObject *,
|
||||
static PyObject *run_err_mod(mod_ty, const char *, PyObject *, PyObject *,
|
||||
PyCompilerFlags *);
|
||||
static PyObject *run_node(node *, const char *, PyObject *, PyObject *,
|
||||
static PyObject *run_mod(mod_ty, const char *, PyObject *, PyObject *,
|
||||
PyCompilerFlags *);
|
||||
static PyObject *run_pyc_file(FILE *, const char *, PyObject *, PyObject *,
|
||||
PyCompilerFlags *);
|
||||
|
|
@ -634,25 +637,7 @@ initsite(void)
|
|||
/* Parse input from a file and execute it */
|
||||
|
||||
int
|
||||
PyRun_AnyFile(FILE *fp, const char *filename)
|
||||
{
|
||||
return PyRun_AnyFileExFlags(fp, filename, 0, NULL);
|
||||
}
|
||||
|
||||
int
|
||||
PyRun_AnyFileFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)
|
||||
{
|
||||
return PyRun_AnyFileExFlags(fp, filename, 0, flags);
|
||||
}
|
||||
|
||||
int
|
||||
PyRun_AnyFileEx(FILE *fp, const char *filename, int closeit)
|
||||
{
|
||||
return PyRun_AnyFileExFlags(fp, filename, closeit, NULL);
|
||||
}
|
||||
|
||||
int
|
||||
PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit,
|
||||
PyRun_AnyFileExFlags(FILE *fp, char *filename, int closeit,
|
||||
PyCompilerFlags *flags)
|
||||
{
|
||||
if (filename == NULL)
|
||||
|
|
@ -667,12 +652,6 @@ PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit,
|
|||
return PyRun_SimpleFileExFlags(fp, filename, closeit, flags);
|
||||
}
|
||||
|
||||
int
|
||||
PyRun_InteractiveLoop(FILE *fp, const char *filename)
|
||||
{
|
||||
return PyRun_InteractiveLoopFlags(fp, filename, NULL);
|
||||
}
|
||||
|
||||
int
|
||||
PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)
|
||||
{
|
||||
|
|
@ -708,12 +687,6 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flag
|
|||
}
|
||||
}
|
||||
|
||||
int
|
||||
PyRun_InteractiveOne(FILE *fp, const char *filename)
|
||||
{
|
||||
return PyRun_InteractiveOneFlags(fp, filename, NULL);
|
||||
}
|
||||
|
||||
/* compute parser flags based on compiler flags */
|
||||
#define PARSER_FLAGS(flags) \
|
||||
(((flags) && (flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? \
|
||||
|
|
@ -723,9 +696,9 @@ int
|
|||
PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)
|
||||
{
|
||||
PyObject *m, *d, *v, *w;
|
||||
node *n;
|
||||
perrdetail err;
|
||||
mod_ty mod;
|
||||
char *ps1 = "", *ps2 = "";
|
||||
int errcode = 0;
|
||||
|
||||
v = PySys_GetObject("ps1");
|
||||
if (v != NULL) {
|
||||
|
|
@ -743,26 +716,25 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags
|
|||
else if (PyString_Check(w))
|
||||
ps2 = PyString_AsString(w);
|
||||
}
|
||||
n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
|
||||
Py_single_input, ps1, ps2, &err,
|
||||
PARSER_FLAGS(flags));
|
||||
mod = PyParser_ASTFromFile(fp, filename,
|
||||
Py_single_input, ps1, ps2,
|
||||
flags, &errcode);
|
||||
Py_XDECREF(v);
|
||||
Py_XDECREF(w);
|
||||
if (n == NULL) {
|
||||
if (err.error == E_EOF) {
|
||||
if (err.text)
|
||||
PyMem_DEL(err.text);
|
||||
if (mod == NULL) {
|
||||
if (errcode == E_EOF) {
|
||||
PyErr_Clear();
|
||||
return E_EOF;
|
||||
}
|
||||
err_input(&err);
|
||||
PyErr_Print();
|
||||
return err.error;
|
||||
return -1;
|
||||
}
|
||||
m = PyImport_AddModule("__main__");
|
||||
if (m == NULL)
|
||||
return -1;
|
||||
d = PyModule_GetDict(m);
|
||||
v = run_node(n, filename, d, d, flags);
|
||||
v = run_mod(mod, filename, d, d, flags);
|
||||
free_mod(mod);
|
||||
if (v == NULL) {
|
||||
PyErr_Print();
|
||||
return -1;
|
||||
|
|
@ -773,12 +745,6 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags
|
|||
return 0;
|
||||
}
|
||||
|
||||
int
|
||||
PyRun_SimpleFile(FILE *fp, const char *filename)
|
||||
{
|
||||
return PyRun_SimpleFileEx(fp, filename, 0);
|
||||
}
|
||||
|
||||
/* Check whether a file maybe a pyc file: Look at the extension,
|
||||
the file type, and, if we may close it, at the first few bytes. */
|
||||
|
||||
|
|
@ -819,12 +785,6 @@ maybe_pyc_file(FILE *fp, const char* filename, const char* ext, int closeit)
|
|||
return 0;
|
||||
}
|
||||
|
||||
int
|
||||
PyRun_SimpleFileEx(FILE *fp, const char *filename, int closeit)
|
||||
{
|
||||
return PyRun_SimpleFileExFlags(fp, filename, closeit, NULL);
|
||||
}
|
||||
|
||||
int
|
||||
PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit,
|
||||
PyCompilerFlags *flags)
|
||||
|
|
@ -873,12 +833,6 @@ PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit,
|
|||
return 0;
|
||||
}
|
||||
|
||||
int
|
||||
PyRun_SimpleString(const char *command)
|
||||
{
|
||||
return PyRun_SimpleStringFlags(command, NULL);
|
||||
}
|
||||
|
||||
int
|
||||
PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags)
|
||||
{
|
||||
|
|
@ -1054,6 +1008,8 @@ PyErr_PrintEx(int set_sys_last_vars)
|
|||
handle_system_exit();
|
||||
}
|
||||
PyErr_Fetch(&exception, &v, &tb);
|
||||
if (exception == NULL)
|
||||
return;
|
||||
PyErr_NormalizeException(&exception, &v, &tb);
|
||||
if (exception == NULL)
|
||||
return;
|
||||
|
|
@ -1195,74 +1151,48 @@ void PyErr_Display(PyObject *exception, PyObject *value, PyObject *tb)
|
|||
}
|
||||
|
||||
PyObject *
|
||||
PyRun_String(const char *str, int start, PyObject *globals, PyObject *locals)
|
||||
PyRun_StringFlags(const char *str, int start, PyObject *globals,
|
||||
PyObject *locals, PyCompilerFlags *flags)
|
||||
{
|
||||
return run_err_node(PyParser_SimpleParseString(str, start),
|
||||
"<string>", globals, locals, NULL);
|
||||
}
|
||||
|
||||
PyObject *
|
||||
PyRun_File(FILE *fp, const char *filename, int start, PyObject *globals,
|
||||
PyObject *locals)
|
||||
{
|
||||
return PyRun_FileEx(fp, filename, start, globals, locals, 0);
|
||||
}
|
||||
|
||||
PyObject *
|
||||
PyRun_FileEx(FILE *fp, const char *filename, int start, PyObject *globals,
|
||||
PyObject *locals, int closeit)
|
||||
{
|
||||
node *n = PyParser_SimpleParseFile(fp, filename, start);
|
||||
if (closeit)
|
||||
fclose(fp);
|
||||
return run_err_node(n, filename, globals, locals, NULL);
|
||||
}
|
||||
|
||||
PyObject *
|
||||
PyRun_StringFlags(const char *str, int start, PyObject *globals, PyObject *locals,
|
||||
PyCompilerFlags *flags)
|
||||
{
|
||||
return run_err_node(PyParser_SimpleParseStringFlags(
|
||||
str, start, PARSER_FLAGS(flags)),
|
||||
"<string>", globals, locals, flags);
|
||||
}
|
||||
|
||||
PyObject *
|
||||
PyRun_FileFlags(FILE *fp, const char *filename, int start, PyObject *globals,
|
||||
PyObject *locals, PyCompilerFlags *flags)
|
||||
{
|
||||
return PyRun_FileExFlags(fp, filename, start, globals, locals, 0,
|
||||
flags);
|
||||
PyObject *ret;
|
||||
mod_ty mod = PyParser_ASTFromString(str, "<string>", start, flags);
|
||||
ret = run_err_mod(mod, "<string>", globals, locals, flags);
|
||||
free_mod(mod);
|
||||
return ret;
|
||||
}
|
||||
|
||||
PyObject *
|
||||
PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals,
|
||||
PyObject *locals, int closeit, PyCompilerFlags *flags)
|
||||
{
|
||||
node *n = PyParser_SimpleParseFileFlags(fp, filename, start,
|
||||
PARSER_FLAGS(flags));
|
||||
PyObject *ret;
|
||||
mod_ty mod = PyParser_ASTFromFile(fp, filename, start, 0, 0,
|
||||
flags, NULL);
|
||||
if (mod == NULL)
|
||||
return NULL;
|
||||
if (closeit)
|
||||
fclose(fp);
|
||||
return run_err_node(n, filename, globals, locals, flags);
|
||||
ret = run_err_mod(mod, filename, globals, locals, flags);
|
||||
free_mod(mod);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static PyObject *
|
||||
run_err_node(node *n, const char *filename, PyObject *globals, PyObject *locals,
|
||||
PyCompilerFlags *flags)
|
||||
run_err_mod(mod_ty mod, const char *filename, PyObject *globals,
|
||||
PyObject *locals, PyCompilerFlags *flags)
|
||||
{
|
||||
if (n == NULL)
|
||||
if (mod == NULL)
|
||||
return NULL;
|
||||
return run_node(n, filename, globals, locals, flags);
|
||||
return run_mod(mod, filename, globals, locals, flags);
|
||||
}
|
||||
|
||||
static PyObject *
|
||||
run_node(node *n, const char *filename, PyObject *globals, PyObject *locals,
|
||||
run_mod(mod_ty mod, const char *filename, PyObject *globals, PyObject *locals,
|
||||
PyCompilerFlags *flags)
|
||||
{
|
||||
PyCodeObject *co;
|
||||
PyObject *v;
|
||||
co = PyNode_CompileFlags(n, filename, flags);
|
||||
PyNode_Free(n);
|
||||
co = PyAST_Compile(mod, filename, flags);
|
||||
if (co == NULL)
|
||||
return NULL;
|
||||
v = PyEval_EvalCode(co, globals, locals);
|
||||
|
|
@ -1271,8 +1201,8 @@ run_node(node *n, const char *filename, PyObject *globals, PyObject *locals,
|
|||
}
|
||||
|
||||
static PyObject *
|
||||
run_pyc_file(FILE *fp, const char *filename, PyObject *globals, PyObject *locals,
|
||||
PyCompilerFlags *flags)
|
||||
run_pyc_file(FILE *fp, const char *filename, PyObject *globals,
|
||||
PyObject *locals, PyCompilerFlags *flags)
|
||||
{
|
||||
PyCodeObject *co;
|
||||
PyObject *v;
|
||||
|
|
@ -1302,42 +1232,78 @@ run_pyc_file(FILE *fp, const char *filename, PyObject *globals, PyObject *locals
|
|||
return v;
|
||||
}
|
||||
|
||||
PyObject *
|
||||
Py_CompileString(const char *str, const char *filename, int start)
|
||||
{
|
||||
return Py_CompileStringFlags(str, filename, start, NULL);
|
||||
}
|
||||
|
||||
PyObject *
|
||||
Py_CompileStringFlags(const char *str, const char *filename, int start,
|
||||
PyCompilerFlags *flags)
|
||||
{
|
||||
node *n;
|
||||
mod_ty mod;
|
||||
PyCodeObject *co;
|
||||
|
||||
n = PyParser_SimpleParseStringFlagsFilename(str, filename, start,
|
||||
PARSER_FLAGS(flags));
|
||||
if (n == NULL)
|
||||
mod = PyParser_ASTFromString(str, filename, start, flags);
|
||||
if (mod == NULL)
|
||||
return NULL;
|
||||
co = PyNode_CompileFlags(n, filename, flags);
|
||||
PyNode_Free(n);
|
||||
co = PyAST_Compile(mod, filename, flags);
|
||||
free_mod(mod);
|
||||
return (PyObject *)co;
|
||||
}
|
||||
|
||||
struct symtable *
|
||||
Py_SymtableString(const char *str, const char *filename, int start)
|
||||
{
|
||||
node *n;
|
||||
mod_ty mod;
|
||||
struct symtable *st;
|
||||
n = PyParser_SimpleParseStringFlagsFilename(str, filename,
|
||||
start, 0);
|
||||
if (n == NULL)
|
||||
|
||||
mod = PyParser_ASTFromString(str, filename, start, NULL);
|
||||
if (mod == NULL)
|
||||
return NULL;
|
||||
st = PyNode_CompileSymtable(n, filename);
|
||||
PyNode_Free(n);
|
||||
st = PySymtable_Build(mod, filename, 0);
|
||||
free_mod(mod);
|
||||
return st;
|
||||
}
|
||||
|
||||
/* Preferred access to parser is through AST. */
|
||||
mod_ty
|
||||
PyParser_ASTFromString(const char *s, const char *filename, int start,
|
||||
PyCompilerFlags *flags)
|
||||
{
|
||||
node *n;
|
||||
mod_ty mod;
|
||||
perrdetail err;
|
||||
n = PyParser_ParseStringFlagsFilename(s, filename, &_PyParser_Grammar,
|
||||
start, &err,
|
||||
PARSER_FLAGS(flags));
|
||||
if (n) {
|
||||
mod = PyAST_FromNode(n, flags, filename);
|
||||
PyNode_Free(n);
|
||||
return mod;
|
||||
}
|
||||
else {
|
||||
err_input(&err);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
mod_ty
|
||||
PyParser_ASTFromFile(FILE *fp, const char *filename, int start, char *ps1,
|
||||
char *ps2, PyCompilerFlags *flags, int *errcode)
|
||||
{
|
||||
node *n;
|
||||
mod_ty mod;
|
||||
perrdetail err;
|
||||
n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar, start,
|
||||
ps1, ps2, &err, PARSER_FLAGS(flags));
|
||||
if (n) {
|
||||
mod = PyAST_FromNode(n, flags, filename);
|
||||
PyNode_Free(n);
|
||||
return mod;
|
||||
}
|
||||
else {
|
||||
err_input(&err);
|
||||
if (errcode)
|
||||
*errcode = err.error;
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/* Simplified interface to parsefile -- return node or set exception */
|
||||
|
||||
node *
|
||||
|
|
@ -1349,15 +1315,10 @@ PyParser_SimpleParseFileFlags(FILE *fp, const char *filename, int start, int fla
|
|||
(char *)0, (char *)0, &err, flags);
|
||||
if (n == NULL)
|
||||
err_input(&err);
|
||||
|
||||
return n;
|
||||
}
|
||||
|
||||
node *
|
||||
PyParser_SimpleParseFile(FILE *fp, const char *filename, int start)
|
||||
{
|
||||
return PyParser_SimpleParseFileFlags(fp, filename, start, 0);
|
||||
}
|
||||
|
||||
/* Simplified interface to parsestring -- return node or set exception */
|
||||
|
||||
node *
|
||||
|
|
@ -1372,12 +1333,6 @@ PyParser_SimpleParseStringFlags(const char *str, int start, int flags)
|
|||
return n;
|
||||
}
|
||||
|
||||
node *
|
||||
PyParser_SimpleParseString(const char *str, int start)
|
||||
{
|
||||
return PyParser_SimpleParseStringFlags(str, start, 0);
|
||||
}
|
||||
|
||||
node *
|
||||
PyParser_SimpleParseStringFlagsFilename(const char *str, const char *filename,
|
||||
int start, int flags)
|
||||
|
|
@ -1418,12 +1373,6 @@ err_input(perrdetail *err)
|
|||
PyObject* u = NULL;
|
||||
char *msg = NULL;
|
||||
errtype = PyExc_SyntaxError;
|
||||
v = Py_BuildValue("(ziiz)", err->filename,
|
||||
err->lineno, err->offset, err->text);
|
||||
if (err->text != NULL) {
|
||||
PyMem_DEL(err->text);
|
||||
err->text = NULL;
|
||||
}
|
||||
switch (err->error) {
|
||||
case E_SYNTAX:
|
||||
errtype = PyExc_IndentationError;
|
||||
|
|
@ -1450,11 +1399,9 @@ err_input(perrdetail *err)
|
|||
case E_INTR:
|
||||
if (!PyErr_Occurred())
|
||||
PyErr_SetNone(PyExc_KeyboardInterrupt);
|
||||
Py_XDECREF(v);
|
||||
return;
|
||||
case E_NOMEM:
|
||||
PyErr_NoMemory();
|
||||
Py_XDECREF(v);
|
||||
return;
|
||||
case E_EOF:
|
||||
msg = "unexpected EOF while parsing";
|
||||
|
|
@ -1498,7 +1445,15 @@ err_input(perrdetail *err)
|
|||
msg = "unknown parsing error";
|
||||
break;
|
||||
}
|
||||
w = Py_BuildValue("(sO)", msg, v);
|
||||
v = Py_BuildValue("(ziiz)", err->filename,
|
||||
err->lineno, err->offset, err->text);
|
||||
if (err->text != NULL) {
|
||||
PyMem_DEL(err->text);
|
||||
err->text = NULL;
|
||||
}
|
||||
w = NULL;
|
||||
if (v != NULL)
|
||||
w = Py_BuildValue("(sO)", msg, v);
|
||||
Py_XDECREF(u);
|
||||
Py_XDECREF(v);
|
||||
PyErr_SetObject(errtype, w);
|
||||
|
|
@ -1687,3 +1642,20 @@ PyOS_setsig(int sig, PyOS_sighandler_t handler)
|
|||
return oldhandler;
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Deprecated C API functions still provided for binary compatiblity */
|
||||
|
||||
#undef PyParser_SimpleParseFile
|
||||
#undef PyParser_SimpleParseString
|
||||
|
||||
node *
|
||||
PyParser_SimpleParseFile(FILE *fp, const char *filename, int start)
|
||||
{
|
||||
return PyParser_SimpleParseFileFlags(fp, filename, start, 0);
|
||||
}
|
||||
|
||||
node *
|
||||
PyParser_SimpleParseString(const char *str, int start)
|
||||
{
|
||||
return PyParser_SimpleParseStringFlags(str, start, 0);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue