Provisional implementation of PEP 3104.

Add nonlocal_stmt to Grammar and Nonlocal node to AST.  They both
parallel the definitions for globals.  The symbol table treats
variables declared as nonlocal just like variables that are free
implicitly.

This change is missing the language spec changes, but makes some
decisions about what the spec should say via the unittests.  The PEP
is silent on a number of decisions, so we should review those before
claiming that nonlocal is complete.

Thomas Wouters made the grammer and ast changes.  Jeremy Hylton added
the symbol table changes and the tests.  Pete Shinners and Neal
Norwitz helped review the code.

Grammar/Grammar

@@ -39,7 +39,7 @@ vfplist: vfpdef (',' vfpdef)* [',']
 stmt: simple_stmt | compound_stmt
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
-             import_stmt | global_stmt | assert_stmt)
+             import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
 expr_stmt: testlist (augassign (yield_expr|testlist) |
                      ('=' (yield_expr|testlist))*)
 augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
@@ -63,6 +63,7 @@ import_as_names: import_as_name (',' import_as_name)* [',']
 dotted_as_names: dotted_as_name (',' dotted_as_name)*
 dotted_name: NAME ('.' NAME)*
 global_stmt: 'global' NAME (',' NAME)*
+nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
 assert_stmt: 'assert' test [',' test]
 
 compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef

Include/Python-ast.h

@@ -66,7 +66,8 @@ enum _stmt_kind {FunctionDef_kind=1, ClassDef_kind=2, Return_kind=3,
                   While_kind=8, If_kind=9, With_kind=10, Raise_kind=11,
                   TryExcept_kind=12, TryFinally_kind=13, Assert_kind=14,
                   Import_kind=15, ImportFrom_kind=16, Global_kind=17,
-                  Expr_kind=18, Pass_kind=19, Break_kind=20, Continue_kind=21};
+                  Nonlocal_kind=18, Expr_kind=19, Pass_kind=20, Break_kind=21,
+                  Continue_kind=22};
 struct _stmt {
         enum _stmt_kind kind;
         union {
@@ -164,6 +165,10 @@ struct _stmt {
                         asdl_seq *names;
                 } Global;
                 
+                struct {
+                        asdl_seq *names;
+                } Nonlocal;
+                
                 struct {
                         expr_ty value;
                 } Expr;
@@ -422,6 +427,9 @@ stmt_ty _Py_ImportFrom(identifier module, asdl_seq * names, int level, int
 #define Global(a0, a1, a2, a3) _Py_Global(a0, a1, a2, a3)
 stmt_ty _Py_Global(asdl_seq * names, int lineno, int col_offset, PyArena
                    *arena);
+#define Nonlocal(a0, a1, a2, a3) _Py_Nonlocal(a0, a1, a2, a3)
+stmt_ty _Py_Nonlocal(asdl_seq * names, int lineno, int col_offset, PyArena
+                     *arena);
 #define Expr(a0, a1, a2, a3) _Py_Expr(a0, a1, a2, a3)
 stmt_ty _Py_Expr(expr_ty value, int lineno, int col_offset, PyArena *arena);
 #define Pass(a0, a1, a2) _Py_Pass(a0, a1, a2)

Include/graminit.h

@@ -35,53 +35,54 @@
 #define dotted_as_names 290
 #define dotted_name 291
 #define global_stmt 292
-#define assert_stmt 293
+#define nonlocal_stmt 293
-#define compound_stmt 294
+#define assert_stmt 294
-#define if_stmt 295
+#define compound_stmt 295
-#define while_stmt 296
+#define if_stmt 296
-#define for_stmt 297
+#define while_stmt 297
-#define try_stmt 298
+#define for_stmt 298
-#define with_stmt 299
+#define try_stmt 299
-#define with_var 300
+#define with_stmt 300
-#define except_clause 301
+#define with_var 301
-#define suite 302
+#define except_clause 302
-#define testlist_safe 303
+#define suite 303
-#define old_test 304
+#define testlist_safe 304
-#define old_lambdef 305
+#define old_test 305
-#define test 306
+#define old_lambdef 306
-#define or_test 307
+#define test 307
-#define and_test 308
+#define or_test 308
-#define not_test 309
+#define and_test 309
-#define comparison 310
+#define not_test 310
-#define comp_op 311
+#define comparison 311
-#define expr 312
+#define comp_op 312
-#define xor_expr 313
+#define expr 313
-#define and_expr 314
+#define xor_expr 314
-#define shift_expr 315
+#define and_expr 315
-#define arith_expr 316
+#define shift_expr 316
-#define term 317
+#define arith_expr 317
-#define factor 318
+#define term 318
-#define power 319
+#define factor 319
-#define atom 320
+#define power 320
-#define listmaker 321
+#define atom 321
-#define testlist_gexp 322
+#define listmaker 322
-#define lambdef 323
+#define testlist_gexp 323
-#define trailer 324
+#define lambdef 324
-#define subscriptlist 325
+#define trailer 325
-#define subscript 326
+#define subscriptlist 326
-#define sliceop 327
+#define subscript 327
-#define exprlist 328
+#define sliceop 328
-#define testlist 329
+#define exprlist 329
-#define dictsetmaker 330
+#define testlist 330
-#define classdef 331
+#define dictsetmaker 331
-#define arglist 332
+#define classdef 332
-#define argument 333
+#define arglist 333
-#define list_iter 334
+#define argument 334
-#define list_for 335
+#define list_iter 335
-#define list_if 336
+#define list_for 336
-#define gen_iter 337
+#define list_if 337
-#define gen_for 338
+#define gen_iter 338
-#define gen_if 339
+#define gen_for 339
-#define testlist1 340
+#define gen_if 340
-#define encoding_decl 341
+#define testlist1 341
-#define yield_expr 342
+#define encoding_decl 342
+#define yield_expr 343

Include/symtable.h

@@ -64,23 +64,24 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
 #define DEF_GLOBAL 1           /* global stmt */
 #define DEF_LOCAL 2            /* assignment in code block */
 #define DEF_PARAM 2<<1         /* formal parameter */
-#define USE 2<<2               /* name is used */
+#define DEF_NONLOCAL 2<<2      /* nonlocal stmt */
-#define DEF_STAR 2<<3          /* parameter is star arg */
+#define USE 2<<3               /* name is used */
-#define DEF_DOUBLESTAR 2<<4    /* parameter is star-star arg */
+#define DEF_STAR 2<<4          /* parameter is star arg */
-#define DEF_INTUPLE 2<<5       /* name defined in tuple in parameters */
+#define DEF_DOUBLESTAR 2<<5    /* parameter is star-star arg */
-#define DEF_FREE 2<<6          /* name used but not defined in nested block */
+#define DEF_INTUPLE 2<<6       /* name defined in tuple in parameters */
-#define DEF_FREE_GLOBAL 2<<7   /* free variable is actually implicit global */
+#define DEF_FREE 2<<7          /* name used but not defined in nested block */
-#define DEF_FREE_CLASS 2<<8    /* free variable from class's method */
+#define DEF_FREE_GLOBAL 2<<8   /* free variable is actually implicit global */
-#define DEF_IMPORT 2<<9        /* assignment occurred via import */
+#define DEF_FREE_CLASS 2<<9    /* free variable from class's method */
+#define DEF_IMPORT 2<<10        /* assignment occurred via import */
 
 #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT)
 
 /* GLOBAL_EXPLICIT and GLOBAL_IMPLICIT are used internally by the symbol
    table.  GLOBAL is returned from PyST_GetScope() for either of them. 
-   It is stored in ste_symbols at bits 12-14.
+   It is stored in ste_symbols at bits 12-15.
 */
 #define SCOPE_OFF 11
-#define SCOPE_MASK 7
+#define SCOPE_MASK (DEF_GLOBAL | DEF_LOCAL | DEF_PARAM | DEF_NONLOCAL)
 
 #define LOCAL 1
 #define GLOBAL_EXPLICIT 2

Lib/test/test_scope.py

@@ -555,6 +555,90 @@ self.assert_(X.passed)
 
         f(4)()
 
+    def testNonLocalFunction(self):
+
+        def f(x):
+            def inc():
+                nonlocal x
+                x += 1
+                return x
+            def dec():
+                nonlocal x
+                x -= 1
+                return x
+            return inc, dec
+
+        inc, dec = f(0)
+        self.assertEqual(inc(), 1)
+        self.assertEqual(inc(), 2)
+        self.assertEqual(dec(), 1)
+        self.assertEqual(dec(), 0)
+
+    def testNonLocalMethod(self):
+
+        def f(x):
+            class c:
+                def inc(self):
+                    nonlocal x
+                    x += 1
+                    return x
+                def dec(self):
+                    nonlocal x
+                    x -= 1
+                    return x
+            return c()
+
+        c = f(0)
+        self.assertEqual(c.inc(), 1)
+        self.assertEqual(c.inc(), 2)
+        self.assertEqual(c.dec(), 1)
+        self.assertEqual(c.dec(), 0)
+
+    def testNonLocalClass(self):
+
+        def f(x):
+            class c:
+                nonlocal x
+                x += 1
+                def get(self):
+                    return x
+            return c()
+
+        c = f(0)
+        self.assertEqual(c.get(), 1)
+        self.assert_("x" not in c.__class__.__dict__)
+        
+
+    def testNonLocalGenerator(self):
+
+        def f(x):
+            def g(y):
+                nonlocal x
+                for i in range(y):
+                    x += 1
+                    yield x
+            return g
+
+        g = f(0)
+        self.assertEqual(list(g(5)), [1, 2, 3, 4, 5])
+
+    def testNestedNonLocal(self):
+
+        def f(x):
+            def g():
+                nonlocal x
+                x -= 2
+                def h():
+                    nonlocal x
+                    x += 4
+                    return x
+                return h
+            return g
+
+        g = f(1)
+        h = g()
+        self.assertEqual(h(), 3)
+
 
 def test_main():
     run_unittest(ScopeTests)

Lib/test/test_syntax.py

@@ -367,6 +367,46 @@ build.  The number of blocks must be greater than CO_MAXBLOCKS.  SF #1565514
      ...
    SystemError: too many statically nested blocks
 
+Misuse of the nonlocal statement can lead to a few unique syntax errors.
+
+   >>> def f(x):
+   ...     nonlocal x
+   Traceback (most recent call last):
+     ...
+   SyntaxError: name 'x' is local and nonlocal
+   
+   >>> def f():
+   ...     global x
+   ...     nonlocal x
+   Traceback (most recent call last):
+     ...
+   SyntaxError: name 'x' is nonlocal and global
+
+   >>> def f():
+   ...     nonlocal x
+   Traceback (most recent call last):
+     ...
+   SyntaxError: no binding for nonlocal 'x' found
+
+TODO(jhylton): Figure out how to test SyntaxWarning with doctest.
+
+##   >>> def f(x):
+##   ...     def f():
+##   ...         print(x)
+##   ...         nonlocal x
+##   Traceback (most recent call last):
+##     ...
+##   SyntaxWarning: name 'x' is assigned to before nonlocal declaration
+   
+##   >>> def f():
+##   ...     x = 1
+##   ...     nonlocal x
+##   Traceback (most recent call last):
+##     ...
+##   SyntaxWarning: name 'x' is assigned to before nonlocal declaration
+
+  
+
 """
 
 import re

Parser/Python.asdl

@@ -34,6 +34,7 @@ module Python version "$Revision$"
 	      | ImportFrom(identifier module, alias* names, int? level)
 
 	      | Global(identifier* names)
+	      | Nonlocal(identifier* names)
 	      | Expr(expr value)
 	      | Pass | Break | Continue
 

Python/Python-ast.c

@@ -131,6 +131,10 @@ static PyTypeObject *Global_type;
 static char *Global_fields[]={
         "names",
 };
+static PyTypeObject *Nonlocal_type;
+static char *Nonlocal_fields[]={
+        "names",
+};
 static PyTypeObject *Expr_type;
 static char *Expr_fields[]={
         "value",
@@ -507,6 +511,8 @@ static int init_types(void)
         if (!ImportFrom_type) return 0;
         Global_type = make_type("Global", stmt_type, Global_fields, 1);
         if (!Global_type) return 0;
+        Nonlocal_type = make_type("Nonlocal", stmt_type, Nonlocal_fields, 1);
+        if (!Nonlocal_type) return 0;
         Expr_type = make_type("Expr", stmt_type, Expr_fields, 1);
         if (!Expr_type) return 0;
         Pass_type = make_type("Pass", stmt_type, NULL, 0);
@@ -1145,6 +1151,20 @@ Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
         return p;
 }
 
+stmt_ty
+Nonlocal(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
+{
+        stmt_ty p;
+        p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
+        if (!p)
+                return NULL;
+        p->kind = Nonlocal_kind;
+        p->v.Nonlocal.names = names;
+        p->lineno = lineno;
+        p->col_offset = col_offset;
+        return p;
+}
+
 stmt_ty
 Expr(expr_ty value, int lineno, int col_offset, PyArena *arena)
 {
@@ -2197,6 +2217,15 @@ ast2obj_stmt(void* _o)
                         goto failed;
                 Py_DECREF(value);
                 break;
+        case Nonlocal_kind:
+                result = PyType_GenericNew(Nonlocal_type, NULL, NULL);
+                if (!result) goto failed;
+                value = ast2obj_list(o->v.Nonlocal.names, ast2obj_identifier);
+                if (!value) goto failed;
+                if (PyObject_SetAttrString(result, "names", value) == -1)
+                        goto failed;
+                Py_DECREF(value);
+                break;
         case Expr_kind:
                 result = PyType_GenericNew(Expr_type, NULL, NULL);
                 if (!result) goto failed;
@@ -3049,6 +3078,8 @@ init_ast(void)
             0) return;
         if (PyDict_SetItemString(d, "Global", (PyObject*)Global_type) < 0)
             return;
+        if (PyDict_SetItemString(d, "Nonlocal", (PyObject*)Nonlocal_type) < 0)
+            return;
         if (PyDict_SetItemString(d, "Expr", (PyObject*)Expr_type) < 0) return;
         if (PyDict_SetItemString(d, "Pass", (PyObject*)Pass_type) < 0) return;
         if (PyDict_SetItemString(d, "Break", (PyObject*)Break_type) < 0) return;

Python/ast.c

@@ -2556,6 +2556,27 @@ ast_for_global_stmt(struct compiling *c, const node *n)
     return Global(s, LINENO(n), n->n_col_offset, c->c_arena);
 }
 
+static stmt_ty
+ast_for_nonlocal_stmt(struct compiling *c, const node *n)
+{
+    /* nonlocal_stmt: 'nonlocal' NAME (',' NAME)* */
+    identifier name;
+    asdl_seq *s;
+    int i;
+
+    REQ(n, nonlocal_stmt);
+    s = asdl_seq_new(NCH(n) / 2, c->c_arena);
+    if (!s)
+        return NULL;
+    for (i = 1; i < NCH(n); i += 2) {
+        name = NEW_IDENTIFIER(CHILD(n, i));
+        if (!name)
+            return NULL;
+        asdl_seq_SET(s, i / 2, name);
+    }
+    return Nonlocal(s, LINENO(n), n->n_col_offset, c->c_arena);
+}
+
 static stmt_ty
 ast_for_assert_stmt(struct compiling *c, const node *n)
 {
@@ -3063,8 +3084,8 @@ ast_for_stmt(struct compiling *c, const node *n)
     if (TYPE(n) == small_stmt) {
         REQ(n, small_stmt);
         n = CHILD(n, 0);
-        /* small_stmt: expr_stmt | del_stmt | pass_stmt
+        /* small_stmt: expr_stmt | del_stmt | pass_stmt | flow_stmt
-                     | flow_stmt | import_stmt | global_stmt | assert_stmt
+                  | import_stmt | global_stmt | nonlocal_stmt | assert_stmt
         */
         switch (TYPE(n)) {
             case expr_stmt:
@@ -3079,6 +3100,8 @@ ast_for_stmt(struct compiling *c, const node *n)
                 return ast_for_import_stmt(c, n);
             case global_stmt:
                 return ast_for_global_stmt(c, n);
+            case nonlocal_stmt:
+                return ast_for_nonlocal_stmt(c, n);
             case assert_stmt:
                 return ast_for_assert_stmt(c, n);
             default:

Python/compile.c

@@ -2239,6 +2239,7 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s)
 	case ImportFrom_kind:
 		return compiler_from_import(c, s);
 	case Global_kind:
+	case Nonlocal_kind:
 		break;
 	case Expr_kind:
 		if (c->c_interactive && c->c_nestlevel <= 1) {

Python/symtable.c

@@ -8,9 +8,15 @@
 #define GLOBAL_AFTER_ASSIGN \
 "name '%.400s' is assigned to before global declaration"
 
+#define NONLOCAL_AFTER_ASSIGN \
+"name '%.400s' is assigned to before nonlocal declaration"
+
 #define GLOBAL_AFTER_USE \
 "name '%.400s' is used prior to global declaration"
 
+#define NONLOCAL_AFTER_USE \
+"name '%.400s' is used prior to nonlocal declaration"
+
 #define IMPORT_STAR_WARNING "import * only allowed at module level"
 
 #define RETURN_VAL_IN_GENERATOR \
@@ -328,6 +334,8 @@ PyST_GetScope(PySTEntryObject *ste, PyObject *name)
    block, the name is treated as global until it is assigned to; then it
    is treated as a local.
 
+   TODO(jhylton): Discuss nonlocal
+
    The symbol table requires two passes to determine the scope of each name.
    The first pass collects raw facts from the AST: the name is a parameter 
    here, the name is used by not defined here, etc.  The second pass analyzes
@@ -378,6 +386,12 @@ analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags,
 				     PyString_AS_STRING(name));
 			return 0;
 		}
+                if (flags & DEF_NONLOCAL) {
+			PyErr_Format(PyExc_SyntaxError,
+				     "name '%s' is nonlocal and global",
+				     PyString_AS_STRING(name));
+			return 0;
+                }
 		SET_SCOPE(dict, name, GLOBAL_EXPLICIT);
 		if (PyDict_SetItem(global, name, Py_None) < 0)
 			return 0;
@@ -387,6 +401,24 @@ analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags,
 		}
 		return 1;
 	}
+        if (flags & DEF_NONLOCAL) {
+		if (flags & DEF_PARAM) {
+			PyErr_Format(PyExc_SyntaxError,
+				     "name '%s' is local and nonlocal",
+				     PyString_AS_STRING(name));
+			return 0;
+		}
+                if (!PyDict_GetItem(bound, name)) {
+                        PyErr_Format(PyExc_SyntaxError,
+                                     "no binding for nonlocal '%s' found",
+				     PyString_AS_STRING(name));
+                                     
+                        return 0;
+                }
+                SET_SCOPE(dict, name, FREE);
+                ste->ste_free = 1;
+                return PyDict_SetItem(free, name, Py_None) >= 0;
+        }
 	if (flags & DEF_BOUND) {
 		SET_SCOPE(dict, name, LOCAL);
 		if (PyDict_SetItem(local, name, Py_None) < 0)
@@ -405,24 +437,19 @@ analyze_name(PySTEntryObject *ste, PyObject *dict, PyObject *name, long flags,
 	if (bound && PyDict_GetItem(bound, name)) {
 		SET_SCOPE(dict, name, FREE);
 		ste->ste_free = 1;
-		if (PyDict_SetItem(free, name, Py_None) < 0)
+		return PyDict_SetItem(free, name, Py_None) >= 0;
-			return 0;
-		return 1;
 	}
 	/* If a parent has a global statement, then call it global
 	   explicit?  It could also be global implicit.
 	 */
-	else if (global && PyDict_GetItem(global, name)) {
+	if (global && PyDict_GetItem(global, name)) {
 		SET_SCOPE(dict, name, GLOBAL_EXPLICIT);
 		return 1;
 	}
-	else {
+	if (ste->ste_nested)
-		if (ste->ste_nested)
+		ste->ste_free = 1;
-			ste->ste_free = 1;
+	SET_SCOPE(dict, name, GLOBAL_IMPLICIT);
-		SET_SCOPE(dict, name, GLOBAL_IMPLICIT);
+	return 1;
-		return 1;
-	}
-	return 0; /* Can't get here */
 }
 
 #undef SET_SCOPE
@@ -782,6 +809,7 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag)
 	long val;
 	PyObject *mangled = _Py_Mangle(st->st_private, name);
 
+
 	if (!mangled)
 		return 0;
 	dict = st->st_cur->ste_symbols;
@@ -1075,6 +1103,33 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
 		}
 		break;
 	}
+        case Nonlocal_kind: {
+		int i;
+		asdl_seq *seq = s->v.Nonlocal.names;
+		for (i = 0; i < asdl_seq_LEN(seq); i++) {
+			identifier name = (identifier)asdl_seq_GET(seq, i);
+			char *c_name = PyString_AS_STRING(name);
+			long cur = symtable_lookup(st, name);
+			if (cur < 0)
+				return 0;
+			if (cur & (DEF_LOCAL | USE)) {
+				char buf[256];
+				if (cur & DEF_LOCAL) 
+					PyOS_snprintf(buf, sizeof(buf),
+						      NONLOCAL_AFTER_ASSIGN,
+						      c_name);
+				else
+					PyOS_snprintf(buf, sizeof(buf),
+						      NONLOCAL_AFTER_USE,
+						      c_name);
+				if (!symtable_warn(st, buf, s->lineno))
+                                    return 0;
+			}
+			if (!symtable_add_def(st, name, DEF_NONLOCAL))
+				return 0;
+		}
+		break;
+	}
         case Expr_kind:
 		VISIT(st, expr, s->v.Expr.value);
 		break;