Merge from ast-arena. This reduces the code in Python/ast.c by ~300 lines,

simplifies a lot of error handling code, and fixes many memory leaks.

Python/asdl.c

@@ -2,17 +2,18 @@
 #include "asdl.h"
 
 asdl_seq *
-asdl_seq_new(int size)
+asdl_seq_new(int size, PyArena *arena)
 {
 	asdl_seq *seq = NULL;
 	size_t n = sizeof(asdl_seq) +
 			(size ? (sizeof(void *) * (size - 1)) : 0);
 
-	seq = (asdl_seq *)PyObject_Malloc(n);
+	seq = (asdl_seq *)malloc(n);
 	if (!seq) {
 		PyErr_NoMemory();
 		return NULL;
 	}
+        PyArena_AddMallocPointer(arena, (void *)seq);
 	memset(seq, 0, n);
 	seq->size = size;
 	return seq;
@@ -21,6 +22,4 @@ asdl_seq_new(int size)
 void
 asdl_seq_free(asdl_seq *seq)
 {
-	PyObject_Free(seq);
 }
-

Python/compile.c

@@ -23,6 +23,7 @@
 
 #include "Python-ast.h"
 #include "node.h"
+#include "pyarena.h"
 #include "ast.h"
 #include "code.h"
 #include "compile.h"
@@ -148,6 +149,7 @@ struct compiler {
         struct compiler_unit *u; /* compiler state for current block */
 	PyObject *c_stack;       /* Python list holding compiler_unit ptrs */
 	char *c_encoding;	 /* source encoding (a borrowed reference) */
+        PyArena *c_arena;        /* pointer to memory allocation arena */
 };
 
 struct assembler {
@@ -243,7 +245,8 @@ compiler_init(struct compiler *c)
 }
 
 PyCodeObject *
-PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags)
+PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags,
+              PyArena *arena)
 {
 	struct compiler c;
 	PyCodeObject *co = NULL;
@@ -259,6 +262,7 @@ PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags)
 	if (!compiler_init(&c))
 		goto error;
 	c.c_filename = filename;
+        c.c_arena = arena;
 	c.c_future = PyFuture_FromAST(mod, filename);
 	if (c.c_future == NULL)
 		goto error;
@@ -292,12 +296,13 @@ PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags)
 PyCodeObject *
 PyNode_Compile(struct _node *n, const char *filename)
 {
-	PyCodeObject *co;
-	mod_ty mod = PyAST_FromNode(n, NULL, filename);
-	if (!mod)
-		return NULL;
-	co = PyAST_Compile(mod, filename, NULL);
-	free_mod(mod);
+	PyCodeObject *co = NULL;
+        PyArena *arena;
+        arena = PyArena_New();
+	mod_ty mod = PyAST_FromNode(n, NULL, filename, arena);
+	if (mod)
+		co = PyAST_Compile(mod, filename, NULL, arena);
+        PyArena_Free(arena);
 	return co;
 }
 
@@ -3404,7 +3409,7 @@ compiler_augassign(struct compiler *c, stmt_ty s)
 	switch (e->kind) {
                 case Attribute_kind:
 		auge = Attribute(e->v.Attribute.value, e->v.Attribute.attr,
-				 AugLoad, e->lineno);
+				 AugLoad, e->lineno, c->c_arena);
                 if (auge == NULL)
                     return 0;
 		VISIT(c, expr, auge);
@@ -3412,11 +3417,10 @@ compiler_augassign(struct compiler *c, stmt_ty s)
 		ADDOP(c, inplace_binop(c, s->v.AugAssign.op));
 		auge->v.Attribute.ctx = AugStore;
 		VISIT(c, expr, auge);
-		free(auge);
 		break;
 	case Subscript_kind:
 		auge = Subscript(e->v.Subscript.value, e->v.Subscript.slice,
-				 AugLoad, e->lineno);
+				 AugLoad, e->lineno, c->c_arena);
                 if (auge == NULL)
                     return 0;
 		VISIT(c, expr, auge);
@@ -3424,7 +3428,6 @@ compiler_augassign(struct compiler *c, stmt_ty s)
 		ADDOP(c, inplace_binop(c, s->v.AugAssign.op));
                 auge->v.Subscript.ctx = AugStore;
 		VISIT(c, expr, auge);
-		free(auge);
                 break;
 	case Name_kind:
 		VISIT(c, expr, s->v.AugAssign.target);

Python/import.c

@@ -4,6 +4,7 @@
 #include "Python.h"
 
 #include "Python-ast.h"
+#include "pyarena.h"
 #include "pythonrun.h"
 #include "errcode.h"
 #include "marshal.h"
@@ -773,13 +774,14 @@ parse_source_module(const char *pathname, FILE *fp)
 {
 	PyCodeObject *co = NULL;
 	mod_ty mod;
+        PyArena *arena = PyArena_New();
 
 	mod = PyParser_ASTFromFile(fp, pathname, Py_file_input, 0, 0, 0, 
-				   NULL);
+				   NULL, arena);
 	if (mod) {
-		co = PyAST_Compile(mod, pathname, NULL);
-		free_mod(mod);
+		co = PyAST_Compile(mod, pathname, NULL, arena);
 	}
+        PyArena_Free(arena);
 	return co;
 }
 

Python/pyarena.c

@@ -0,0 +1,133 @@
+#include "Python.h"
+#include "pyarena.h"
+
+/* An arena list is a linked list that can store either pointers or
+   PyObjects.  The type is clear from context.
+ */
+
+typedef struct _arena_list {
+  struct _arena_list *al_next;
+  void *al_pointer;
+} PyArenaList;
+
+/* There are two linked lists in an arena, one for malloc pointers and
+   one for PyObject.  For each list, there is a pointer to the head
+   and to the tail.  The head is used to free the list.  The tail is
+   used to add a new element to the list.
+
+   The list always keeps one un-used node at the end of the list.
+*/
+
+struct _arena {
+  PyArenaList *a_malloc_head;
+  PyArenaList *a_malloc_tail;
+  PyArenaList *a_object_head;
+  PyArenaList *a_object_tail;
+};
+
+static PyArenaList*
+PyArenaList_New(void) 
+{
+  PyArenaList *alist = (PyArenaList *)malloc(sizeof(PyArenaList));
+  if (!alist)
+    return NULL;
+
+  alist->al_next = NULL;
+  alist->al_pointer = NULL;
+  return alist;
+}
+
+static void
+PyArenaList_FreeObject(PyArenaList *alist) 
+{
+  if (!alist)
+    return;
+
+  while (alist) {
+    PyArenaList *prev;
+    Py_XDECREF((PyObject *)alist->al_pointer);
+    alist->al_pointer = NULL;
+    prev = alist;
+    alist = alist->al_next;
+    free(prev);
+  }
+}
+
+static void
+PyArenaList_FreeMalloc(PyArenaList *alist)
+{
+  if (!alist)
+    return;
+
+  while (alist) {
+    PyArenaList *prev;
+    if (alist->al_pointer) {
+      free(alist->al_pointer);
+    }
+    alist->al_pointer = NULL;
+    prev = alist;
+    alist = alist->al_next;
+    free(prev);
+  }
+}
+
+
+PyArena *
+PyArena_New()
+{
+  PyArena* arena = (PyArena *)malloc(sizeof(PyArena));
+  if (!arena)
+    return NULL;
+
+  arena->a_object_head = PyArenaList_New();
+  arena->a_object_tail = arena->a_object_head;
+  arena->a_malloc_head = PyArenaList_New();
+  arena->a_malloc_tail = arena->a_malloc_head;
+  return arena;
+}
+
+void
+PyArena_Free(PyArena *arena)
+{
+  assert(arena);
+  PyArenaList_FreeObject(arena->a_object_head);
+  PyArenaList_FreeMalloc(arena->a_malloc_head);
+  free(arena);
+}
+
+void *
+PyArena_Malloc(PyArena *arena, size_t size) 
+{
+  /* A better implementation might actually use an arena.  The current
+     approach is just a trivial implementation of the API that allows
+     it to be tested.
+  */
+  void *p;
+  assert(size != 0);
+  p = malloc(size);
+  PyArena_AddMallocPointer(arena, p);
+  return p;
+}
+
+int
+PyArena_AddMallocPointer(PyArena *arena, void *pointer) 
+{
+  assert(pointer);
+  PyArenaList *tail = arena->a_malloc_tail;
+  assert(tail->al_pointer != pointer);
+  tail->al_next = PyArenaList_New();
+  tail->al_pointer = pointer;
+  arena->a_malloc_tail = tail->al_next;
+  return 1;
+}
+
+int
+PyArena_AddPyObject(PyArena *arena, PyObject *pointer) 
+{
+  assert(pointer);
+  PyArenaList *tail = arena->a_object_tail;
+  tail->al_next = PyArenaList_New();
+  tail->al_pointer = pointer;
+  arena->a_object_tail = tail->al_next;
+  return 1;
+}

Python/pythonrun.c

@@ -12,6 +12,7 @@
 #include "code.h"
 #include "compile.h"
 #include "symtable.h"
+#include "pyarena.h"
 #include "ast.h"
 #include "eval.h"
 #include "marshal.h"
@@ -36,9 +37,9 @@ extern grammar _PyParser_Grammar; /* From graminit.c */
 static void initmain(void);
 static void initsite(void);
 static PyObject *run_err_mod(mod_ty, const char *, PyObject *, PyObject *,
-			      PyCompilerFlags *);
+			      PyCompilerFlags *, PyArena *arena);
 static PyObject *run_mod(mod_ty, const char *, PyObject *, PyObject *,
-			  PyCompilerFlags *);
+			  PyCompilerFlags *, PyArena *);
 static PyObject *run_pyc_file(FILE *, const char *, PyObject *, PyObject *,
 			      PyCompilerFlags *);
 static void err_input(perrdetail *);
@@ -697,6 +698,7 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags
 {
 	PyObject *m, *d, *v, *w;
 	mod_ty mod;
+        PyArena *arena;
 	char *ps1 = "", *ps2 = "";
 	int errcode = 0;
 
@@ -716,12 +718,14 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags
 		else if (PyString_Check(w))
 			ps2 = PyString_AsString(w);
 	}
+        arena = PyArena_New();
 	mod = PyParser_ASTFromFile(fp, filename, 
 				   Py_single_input, ps1, ps2,
-				   flags, &errcode);
+				   flags, &errcode, arena);
 	Py_XDECREF(v);
 	Py_XDECREF(w);
 	if (mod == NULL) {
+        	PyArena_Free(arena);
 		if (errcode == E_EOF) {
 			PyErr_Clear();
 			return E_EOF;
@@ -730,11 +734,13 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags
 		return -1;
 	}
 	m = PyImport_AddModule("__main__");
-	if (m == NULL)
+	if (m == NULL) {
+        	PyArena_Free(arena);
 		return -1;
+	}
 	d = PyModule_GetDict(m);
-	v = run_mod(mod, filename, d, d, flags);
-	free_mod(mod);
+	v = run_mod(mod, filename, d, d, flags, arena);
+        PyArena_Free(arena);
 	if (v == NULL) {
 		PyErr_Print();
 		return -1;
@@ -1155,9 +1161,11 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals,
 		  PyObject *locals, PyCompilerFlags *flags)
 {
 	PyObject *ret;
-	mod_ty mod = PyParser_ASTFromString(str, "<string>", start, flags);
-	ret = run_err_mod(mod, "<string>", globals, locals, flags);
-	free_mod(mod);
+        PyArena *arena = PyArena_New();
+	mod_ty mod = PyParser_ASTFromString(str, "<string>", start, flags,
+                                            arena);
+	ret = run_err_mod(mod, "<string>", globals, locals, flags, arena);
+        PyArena_Free(arena);
 	return ret;
 }
 
@@ -1166,33 +1174,36 @@ PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals,
 		  PyObject *locals, int closeit, PyCompilerFlags *flags)
 {
 	PyObject *ret;
+        PyArena *arena = PyArena_New();
 	mod_ty mod = PyParser_ASTFromFile(fp, filename, start, 0, 0,
-					  flags, NULL);
-	if (mod == NULL)
+					  flags, NULL, arena);
+	if (mod == NULL) {
+                PyArena_Free(arena);
 		return NULL;
+        }
 	if (closeit)
 		fclose(fp);
-	ret = run_err_mod(mod, filename, globals, locals, flags);
-	free_mod(mod);
+	ret = run_err_mod(mod, filename, globals, locals, flags, arena);
+        PyArena_Free(arena);
 	return ret;
 }
 
 static PyObject *
 run_err_mod(mod_ty mod, const char *filename, PyObject *globals, 
-	    PyObject *locals, PyCompilerFlags *flags)
+	    PyObject *locals, PyCompilerFlags *flags, PyArena *arena)
 {
 	if (mod == NULL)
 		return  NULL;
-	return run_mod(mod, filename, globals, locals, flags);
+	return run_mod(mod, filename, globals, locals, flags, arena);
 }
 
 static PyObject *
 run_mod(mod_ty mod, const char *filename, PyObject *globals, PyObject *locals,
-	 PyCompilerFlags *flags)
+	 PyCompilerFlags *flags, PyArena *arena)
 {
 	PyCodeObject *co;
 	PyObject *v;
-	co = PyAST_Compile(mod, filename, flags);
+	co = PyAST_Compile(mod, filename, flags, arena);
 	if (co == NULL)
 		return NULL;
 	v = PyEval_EvalCode(co, globals, locals);
@@ -1236,43 +1247,45 @@ PyObject *
 Py_CompileStringFlags(const char *str, const char *filename, int start,
 		      PyCompilerFlags *flags)
 {
-	mod_ty mod;
 	PyCodeObject *co;
-	mod = PyParser_ASTFromString(str, filename, start, flags);
-	if (mod == NULL)
+        PyArena *arena = PyArena_New();
+	mod_ty mod = PyParser_ASTFromString(str, filename, start, flags, arena);
+	if (mod == NULL) {
+                PyArena_Free(arena);
 		return NULL;
-	co = PyAST_Compile(mod, filename, flags);
-	free_mod(mod);
+        }
+	co = PyAST_Compile(mod, filename, flags, arena);
+        PyArena_Free(arena);
 	return (PyObject *)co;
 }
 
 struct symtable *
 Py_SymtableString(const char *str, const char *filename, int start)
 {
-	mod_ty mod;
 	struct symtable *st;
-
-	mod = PyParser_ASTFromString(str, filename, start, NULL);
-	if (mod == NULL)
+        PyArena *arena = PyArena_New();
+	mod_ty mod = PyParser_ASTFromString(str, filename, start, NULL, arena);
+	if (mod == NULL) {
+                PyArena_Free(arena);
 		return NULL;
+        }
 	st = PySymtable_Build(mod, filename, 0);
-	free_mod(mod);
+        PyArena_Free(arena);
 	return st;
 }
 
 /* Preferred access to parser is through AST. */
 mod_ty
 PyParser_ASTFromString(const char *s, const char *filename, int start, 
-		       PyCompilerFlags *flags)
+		       PyCompilerFlags *flags, PyArena *arena)
 {
-	node *n;
 	mod_ty mod;
 	perrdetail err;
-	n = PyParser_ParseStringFlagsFilename(s, filename, &_PyParser_Grammar,
-					      start, &err, 
-					      PARSER_FLAGS(flags));
+	node *n = PyParser_ParseStringFlagsFilename(s, filename,
+					&_PyParser_Grammar, start, &err, 
+					PARSER_FLAGS(flags));
 	if (n) {
-		mod = PyAST_FromNode(n, flags, filename);
+		mod = PyAST_FromNode(n, flags, filename, arena);
 		PyNode_Free(n);
 		return mod;
 	}
@@ -1284,15 +1297,15 @@ PyParser_ASTFromString(const char *s, const char *filename, int start,
 
 mod_ty
 PyParser_ASTFromFile(FILE *fp, const char *filename, int start, char *ps1, 
-		     char *ps2, PyCompilerFlags *flags, int *errcode)
+		     char *ps2, PyCompilerFlags *flags, int *errcode,
+                     PyArena *arena)
 {
-	node *n;
 	mod_ty mod;
 	perrdetail err;
-	n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar, start, 
-				    ps1, ps2, &err, PARSER_FLAGS(flags));
+	node *n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
+				start, ps1, ps2, &err, PARSER_FLAGS(flags));
 	if (n) {
-		mod = PyAST_FromNode(n, flags, filename);
+		mod = PyAST_FromNode(n, flags, filename, arena);
 		PyNode_Free(n);
 		return mod;
 	}
@@ -1309,10 +1322,9 @@ PyParser_ASTFromFile(FILE *fp, const char *filename, int start, char *ps1,
 node *
 PyParser_SimpleParseFileFlags(FILE *fp, const char *filename, int start, int flags)
 {
-	node *n;
 	perrdetail err;
-	n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar, start,
-					(char *)0, (char *)0, &err, flags);
+	node *n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
+					  start, NULL, NULL, &err, flags);
 	if (n == NULL)
 		err_input(&err);
 		
@@ -1324,10 +1336,9 @@ PyParser_SimpleParseFileFlags(FILE *fp, const char *filename, int start, int fla
 node *
 PyParser_SimpleParseStringFlags(const char *str, int start, int flags)
 {
-	node *n;
 	perrdetail err;
-	n = PyParser_ParseStringFlags(str, &_PyParser_Grammar, start, &err,
-				      flags);
+	node *n = PyParser_ParseStringFlags(str, &_PyParser_Grammar,
+					    start, &err, flags);
 	if (n == NULL)
 		err_input(&err);
 	return n;
@@ -1337,12 +1348,9 @@ node *
 PyParser_SimpleParseStringFlagsFilename(const char *str, const char *filename,
 					int start, int flags)
 {
-	node *n;
 	perrdetail err;
-
-	n = PyParser_ParseStringFlagsFilename(str, filename,
-					      &_PyParser_Grammar,
-					      start, &err, flags);
+	node *n = PyParser_ParseStringFlagsFilename(str, filename,
+				&_PyParser_Grammar, start, &err, flags);
 	if (n == NULL)
 		err_input(&err);
 	return n;
@@ -1351,8 +1359,7 @@ PyParser_SimpleParseStringFlagsFilename(const char *str, const char *filename,
 node *
 PyParser_SimpleParseStringFilename(const char *str, const char *filename, int start)
 {
-	return PyParser_SimpleParseStringFlagsFilename(str, filename,
-						       start, 0);
+	return PyParser_SimpleParseStringFlagsFilename(str, filename, start, 0);
 }
 
 /* May want to move a more generalized form of this to parsetok.c or