#4617: Previously it was illegal to delete a name from the local

namespace if it occurs as a free variable in a nested block.  This limitation
of the compiler has been lifted, and a new opcode introduced (DELETE_DEREF).

This sample was valid in 2.6, but fails to compile in 3.x without this change::

   >>> def f():
   ...     def print_error():
   ...        print(e)
   ...     try:
   ...        something
   ...     except Exception as e:
   ...        print_error()
   ...        # implicit "del e" here


This sample has always been invalid in Python, and now works::

   >>> def outer(x):
   ...     def inner():
   ...        return x
   ...     inner()
   ...     del x

There is no need to bump the PYC magic number: the new opcode is used
for code that did not compile before.

Doc/library/dis.rst

@@ -723,6 +723,12 @@ the more significant byte last.
    storage.
 
 
+.. opcode:: DELETE_DEREF (i)
+
+   Empties the cell contained in slot *i* of the cell and free variable storage.
+   Used by the :keyword:`del` statement.
+
+
 .. opcode:: SET_LINENO (lineno)
 
    This opcode is obsolete.

Doc/reference/simple_stmts.rst

@@ -388,11 +388,6 @@ namespace, depending on whether the name occurs in a :keyword:`global` statement
 in the same code block.  If the name is unbound, a :exc:`NameError` exception
 will be raised.
 
-.. index:: pair: free; variable
-
-It is illegal to delete a name from the local namespace if it occurs as a free
-variable in a nested block.
-
 .. index:: pair: attribute; deletion
 
 Deletion of attribute references, subscriptions and slicings is passed to the
@@ -400,6 +395,10 @@ primary object involved; deletion of a slicing is in general equivalent to
 assignment of an empty slice of the right type (but even this is determined by
 the sliced object).
 
+.. versionchanged:: 3.2
+   Previously it was illegal to delete a name from the local namespace if it
+   occurs as a free variable in a nested block.
+
 
 .. _return:
 

Doc/whatsnew/3.2.rst

@@ -240,6 +240,31 @@ Some smaller changes made to the core Python language are:
 
   (See :issue:`8188`.)
 
+* Previously it was illegal to delete a name from the local namespace if it
+  occurs as a free variable in a nested block::
+
+   >>> def outer(x):
+   ...     def inner():
+   ...        return x
+   ...     inner()
+   ...     del x
+
+  This is now allowed.  Remember that the target of an :keyword:`except` clause
+  is cleared, so this code which used to work with Python 2.6, raised a
+  :exc:`SyntaxError` with Python 3.1 and now works again::
+
+   >>> def f():
+   ...     def print_error():
+   ...        print(e)
+   ...     try:
+   ...        something
+   ...     except Exception as e:
+   ...        print_error()
+   ...        # implicit "del e" here
+
+  (See :issue:`4617`.)
+
+
 New, Improved, and Deprecated Modules
 =====================================
 

Include/opcode.h

@@ -123,6 +123,7 @@ extern "C" {
 #define LOAD_CLOSURE    135     /* Load free variable from closure */
 #define LOAD_DEREF      136     /* Load and dereference from closure cell */ 
 #define STORE_DEREF     137     /* Store into cell */ 
+#define DELETE_DEREF    138     /* Delete closure cell */ 
 
 /* The next 3 opcodes must be contiguous and satisfy
    (CALL_FUNCTION_VAR - CALL_FUNCTION) & 3 == 1  */

Lib/opcode.py

@@ -161,6 +161,8 @@ def_op('LOAD_DEREF', 136)
 hasfree.append(136)
 def_op('STORE_DEREF', 137)
 hasfree.append(137)
+def_op('DELETE_DEREF', 138)
+hasfree.append(138)
 
 def_op('CALL_FUNCTION_VAR', 140)     # #args + (#kwargs << 8)
 def_op('CALL_FUNCTION_KW', 141)      # #args + (#kwargs << 8)

Lib/test/test_exceptions.py

@@ -526,6 +526,17 @@ class ExceptionTests(unittest.TestCase):
         obj = wr()
         self.assertTrue(obj is None, "%s" % obj)
 
+    def test_exception_target_in_nested_scope(self):
+        # issue 4617: This used to raise a SyntaxError
+        # "can not delete variable 'e' referenced in nested scope"
+        def print_error():
+            e
+        try:
+            something
+        except Exception as e:
+            print_error()
+            # implicit "del e" here
+
     def test_generator_leaking(self):
         # Test that generator exception state doesn't leak into the calling
         # frame

Lib/test/test_scope.py

@@ -217,13 +217,6 @@ class ScopeTests(unittest.TestCase):
                     return f
             """)
 
-        check_syntax_error(self, """if 1:
-            def f(x):
-                def g():
-                    return x
-                del x # can't del name
-            """)
-
         check_syntax_error(self, """if 1:
             def f():
                 def g():
@@ -272,6 +265,28 @@ class ScopeTests(unittest.TestCase):
         self.assertRaises(UnboundLocalError, errorInOuter)
         self.assertRaises(NameError, errorInInner)
 
+    def testUnboundLocal_AfterDel(self):
+        # #4617: It is now legal to delete a cell variable.
+        # The following functions must obviously compile,
+        # and give the correct error when accessing the deleted name.
+        def errorInOuter():
+            y = 1
+            del y
+            print(y)
+            def inner():
+                return y
+
+        def errorInInner():
+            def inner():
+                return y
+            y = 1
+            del y
+            inner()
+
+        self.assertRaises(UnboundLocalError, errorInOuter)
+        self.assertRaises(NameError, errorInInner)
+
+    def testUnboundLocal_AugAssign(self):
         # test for bug #1501934: incorrect LOAD/STORE_GLOBAL generation
         exec("""if 1:
             global_x = 1

Lib/test/test_syntax.py

@@ -564,15 +564,6 @@ class SyntaxTestCase(unittest.TestCase):
     def test_break_outside_loop(self):
         self._check_error("break", "outside loop")
 
-    def test_delete_deref(self):
-        source = """if 1:
-            def foo(x):
-              def bar():
-                print(x)
-              del x
-            """
-        self._check_error(source, "nested scope")
-
     def test_unexpected_indent(self):
         self._check_error("foo()\n bar()\n", "unexpected indent",
                           subclass=IndentationError)

Misc/NEWS

@@ -10,6 +10,10 @@ What's New in Python 3.2 Alpha 3?
 Core and Builtins
 -----------------
 
+- Issue #4617: Previously it was illegal to delete a name from the local
+  namespace if it occurs as a free variable in a nested block.  This limitation
+  of the compiler has been lifted, and a new opcode introduced (DELETE_DEREF).
+
 - Issue #9804: ascii() now always represents unicode surrogate pairs as
   a single ``\UXXXXXXXX``, regardless of whether the character is printable
   or not.  Also, the "backslashreplace" error handler now joins surrogate

Python/ceval.c

@@ -135,6 +135,7 @@ static PyObject * cmp_outcome(int, PyObject *, PyObject *);
 static PyObject * import_from(PyObject *, PyObject *);
 static int import_all_from(PyObject *, PyObject *);
 static void format_exc_check_arg(PyObject *, const char *, PyObject *);
+static void format_exc_unbound(PyCodeObject *co, int oparg);
 static PyObject * unicode_concatenate(PyObject *, PyObject *,
                                       PyFrameObject *, unsigned char *);
 static PyObject * special_lookup(PyObject *, char *, PyObject **);
@@ -2143,6 +2144,16 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
                 );
             break;
 
+        TARGET(DELETE_DEREF)
+            x = freevars[oparg];
+            if (PyCell_GET(x) != NULL) {
+                PyCell_Set(x, NULL);
+                continue;
+            }
+            err = -1;
+            format_exc_unbound(co, oparg);
+            break;
+
         TARGET(LOAD_CLOSURE)
             x = freevars[oparg];
             Py_INCREF(x);
@@ -2158,22 +2169,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
                 DISPATCH();
             }
             err = -1;
-            /* Don't stomp existing exception */
+            format_exc_unbound(co, oparg);
-            if (PyErr_Occurred())
-                break;
-            if (oparg < PyTuple_GET_SIZE(co->co_cellvars)) {
-                v = PyTuple_GET_ITEM(co->co_cellvars,
-                                     oparg);
-                format_exc_check_arg(
-                       PyExc_UnboundLocalError,
-                       UNBOUNDLOCAL_ERROR_MSG,
-                       v);
-            } else {
-                v = PyTuple_GET_ITEM(co->co_freevars, oparg -
-                    PyTuple_GET_SIZE(co->co_cellvars));
-                format_exc_check_arg(PyExc_NameError,
-                                     UNBOUNDFREE_ERROR_MSG, v);
-            }
             break;
 
         TARGET(STORE_DEREF)
@@ -4352,6 +4348,28 @@ format_exc_check_arg(PyObject *exc, const char *format_str, PyObject *obj)
     PyErr_Format(exc, format_str, obj_str);
 }
 
+static void
+format_exc_unbound(PyCodeObject *co, int oparg)
+{
+    PyObject *name;
+    /* Don't stomp existing exception */
+    if (PyErr_Occurred())
+        return;
+    if (oparg < PyTuple_GET_SIZE(co->co_cellvars)) {
+        name = PyTuple_GET_ITEM(co->co_cellvars,
+                                oparg);
+        format_exc_check_arg(
+            PyExc_UnboundLocalError,
+            UNBOUNDLOCAL_ERROR_MSG,
+            name);
+    } else {
+        name = PyTuple_GET_ITEM(co->co_freevars, oparg -
+                                PyTuple_GET_SIZE(co->co_cellvars));
+        format_exc_check_arg(PyExc_NameError,
+                             UNBOUNDFREE_ERROR_MSG, name);
+    }
+}
+
 static PyObject *
 unicode_concatenate(PyObject *v, PyObject *w,
                    PyFrameObject *f, unsigned char *next_instr)

Python/compile.c

@@ -857,6 +857,8 @@ opcode_stack_effect(int opcode, int oparg)
             return 1;
         case STORE_DEREF:
             return -1;
+        case DELETE_DEREF:
+            return 0;
         default:
             fprintf(stderr, "opcode = %d\n", opcode);
             Py_FatalError("opcode_stack_effect()");
@@ -2506,13 +2508,7 @@ compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx)
         case AugLoad:
         case AugStore:
             break;
-        case Del:
+        case Del: op = DELETE_DEREF; break;
-            PyErr_Format(PyExc_SyntaxError,
-                         "can not delete variable '%S' referenced "
-                         "in nested scope",
-                         name);
-            Py_DECREF(mangled);
-            return 0;
         case Param:
         default:
             PyErr_SetString(PyExc_SystemError,

Python/opcode_targets.h

@@ -137,7 +137,7 @@ static void *opcode_targets[256] = {
     &&TARGET_LOAD_CLOSURE,
     &&TARGET_LOAD_DEREF,
     &&TARGET_STORE_DEREF,
-    &&_unknown_opcode,
+    &&TARGET_DELETE_DEREF,
     &&_unknown_opcode,
     &&TARGET_CALL_FUNCTION_VAR,
     &&TARGET_CALL_FUNCTION_KW,