Merged revisions 65715,65724,65726,65732,65736-65739,65775 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r65715 | benjamin.peterson | 2008-08-16 16:04:16 -0500 (Sat, 16 Aug 2008) | 1 line

  add some documentation for symtable
........
  r65724 | benjamin.peterson | 2008-08-16 17:11:33 -0500 (Sat, 16 Aug 2008) | 2 lines

  include filename and line number in SyntaxError
........
  r65726 | georg.brandl | 2008-08-16 17:37:05 -0500 (Sat, 16 Aug 2008) | 2 lines

  Review symtable docs.
........
  r65732 | benjamin.peterson | 2008-08-16 18:29:40 -0500 (Sat, 16 Aug 2008) | 1 line

  PySTEntry's constructor is static; there's no point in a fancy API name
........
  r65736 | benjamin.peterson | 2008-08-16 20:09:17 -0500 (Sat, 16 Aug 2008) | 1 line

  expose PySTEntry.nested so the symtable module will work
........
  r65737 | benjamin.peterson | 2008-08-16 20:17:15 -0500 (Sat, 16 Aug 2008) | 1 line

  a few improvements
........
  r65738 | benjamin.peterson | 2008-08-16 20:27:30 -0500 (Sat, 16 Aug 2008) | 1 line

  fix compile errors
........
  r65739 | benjamin.peterson | 2008-08-16 21:23:43 -0500 (Sat, 16 Aug 2008) | 1 line

  uhh PySTEntry->ste_unoptimized has to be exposed too
........
  r65775 | benjamin.peterson | 2008-08-17 12:13:26 -0500 (Sun, 17 Aug 2008) | 5 lines

  get the symtable module back in working order
  - Fix broken functions
  - Add (hopefully) extensive tests
  - Modernize a little
........

Doc/library/language.rst

@@ -16,6 +16,7 @@ These modules include:
 
    parser.rst
    ast.rst
+   symtable.rst
    symbol.rst
    token.rst
    keyword.rst

Doc/library/symtable.rst

@@ -0,0 +1,187 @@
+:mod:`symtable` --- Access to the compiler's symbol tables
+==========================================================
+
+.. module:: symtable
+   :synopsis: Interface to the compiler's internal symbol tables.
+
+.. moduleauthor:: Jeremy Hylton <jeremy@alum.mit.edu>
+.. sectionauthor:: Benjamin Peterson
+
+
+Symbol tables are generated by the compiler from AST just before bytecode is
+generated.  The symbol table is responsible for calculating the scope of every
+identifier in the code.  :mod:`symtable` provides an interface to examine these
+tables.
+
+
+Generating Symbol Tables
+------------------------
+
+.. function:: symtable(code, filename, compile_type)
+
+   Return the toplevel :class:`SymbolTable` for the Python source *code*.
+   *filename* is the name of the file containing the code.  *compile_type* is
+   like the *mode* argument to :func:`compile`.
+
+
+Examining Symbol Tables
+-----------------------
+
+.. class:: SymbolTable
+
+   A namespace table for a block.  The constructor is not public.
+
+   .. method:: get_type()
+
+      Return the type of the symbol table.  Possible values are ``'class'``,
+      ``'module'``, and ``'function'``.
+
+   .. method:: get_id()
+
+      Return the table's identifier.
+
+   .. method:: get_name()
+
+      Return the table's name.  This is the name of the class if the table is
+      for a class, the name of the function if the table is for a function, or
+      ``'top'`` if the table is global (:meth:`get_type` returns ``'module'``).
+
+   .. method:: get_lineno()
+
+      Return the number of the first line in the block this table represents.
+
+   .. method:: is_optimized()
+
+      Return ``True`` if the locals in this table can be optimized.
+
+   .. method:: is_nested()
+
+      Return ``True`` if the block is a nested class or function.
+
+   .. method:: has_children()
+
+      Return ``True`` if the block has nested namespaces within it.  These can
+      be obtained with :meth:`get_children`.
+
+   .. method:: has_exec()
+
+      Return ``True`` if the block uses ``exec``.
+
+   .. method:: has_import_start()
+
+      Return ``True`` if the block uses a starred from-import.
+
+   .. method:: get_identifiers()
+
+      Return a list of names of symbols in this table.
+
+   .. method:: lookup(name)
+
+      Lookup *name* in the table and return a :class:`Symbol` instance.
+
+   .. method:: get_symbols()
+
+      Return a list of :class:`Symbol` instances for names in the table.
+
+   .. method:: get_children()
+
+      Return a list of the nested symbol tables.
+
+
+.. class:: Function
+
+   A namespace for a function or method.  This class inherits
+   :class:`SymbolTable`.
+
+   .. method:: get_parameters()
+
+      Return a tuple containing names of parameters to this function.
+
+   .. method:: get_locals()
+
+      Return a tuple containing names of locals in this function.
+
+   .. method:: get_globals()
+
+      Return a tuple containing names of globals in this function.
+
+   .. method:: get_frees()
+
+      Return a tuple containing names of free variables in this function.
+
+
+.. class:: Class
+
+   A namespace of a class.  This class inherits :class:`SymbolTable`.
+
+   .. method:: get_methods()
+
+      Return a tuple containing the names of methods declared in the class.
+
+
+.. class:: Symbol
+
+   An entry in a :class:`SymbolTable` corresponding to an identifier in the
+   source.  The constructor is not public.
+
+   .. method:: get_name()
+
+      Return the symbol's name.
+
+   .. method:: is_referenced()
+
+      Return ``True`` if the symbol is used in its block.
+
+   .. method:: is_imported()
+
+      Return ``True`` if the symbol is created from an import statement.
+
+   .. method:: is_parameter()
+
+      Return ``True`` if the symbol is a parameter.
+
+   .. method:: is_global()
+
+      Return ``True`` if the symbol is global.
+
+   .. method:: is_vararg()
+
+      Return ``True`` if the symbol is a star arg (receives varargs).
+
+   .. method:: is_kewordarg()
+
+      Return ``True`` if the symbol is a two-star arg (receives keyword
+      arguments).
+
+   .. method:: is_local()
+
+      Return ``True`` if the symbol is local to its block.
+
+   .. method:: is_free()
+
+      Return ``True`` if the symbol is referenced in its block, but not assigned
+      to.
+
+   .. method:: is_assigned()
+
+      Return ``True`` if the symbol is assigned to in its block.
+
+   .. method:: is_namespace()
+
+      Return ``True`` if name binding introduces new namespace.
+
+      If the name is used as the target of a function or class statement, this
+      will be true.
+
+      Note that a single name can be bound to multiple objects.  If the result
+      is ``True``, the name may also be bound to other objects, like an int or
+      list, that does not introduce a new namespace.
+
+   .. method:: get_namespaces()
+
+      Return a list of namespaces bound to this name.
+
+   .. method:: get_namespace()
+
+      Return the namespace bound to this name.  If more than one namespace is
+      bound, a :exc:`ValueError` is raised.

Include/symtable.h

@@ -36,7 +36,7 @@ typedef struct _symtable_entry {
 	PyObject *ste_children;  /* list of child blocks */
 	_Py_block_ty ste_type;   /* module, class, or function */
 	int ste_unoptimized;     /* false if namespace is optimized */
-	unsigned ste_nested : 1;      /* true if block is nested */
+	int ste_nested;      /* true if block is nested */
 	unsigned ste_free : 1;        /* true if block has free variables */
 	unsigned ste_child_free : 1;  /* true if a child block has free vars,
 				         including free refs to globals */

Lib/symtable.py

@@ -1,10 +1,11 @@
 """Interface to the compiler's internal symbol tables"""
 
 import _symtable
-from _symtable import USE, DEF_GLOBAL, DEF_LOCAL, DEF_PARAM, \
+from _symtable import (USE, DEF_GLOBAL, DEF_LOCAL, DEF_PARAM,
-     DEF_STAR, DEF_DOUBLESTAR, DEF_INTUPLE, DEF_FREE, \
+     DEF_STAR, DEF_DOUBLESTAR, DEF_INTUPLE, DEF_FREE,
-     DEF_FREE_GLOBAL, DEF_FREE_CLASS, DEF_IMPORT, DEF_BOUND, \
+     DEF_FREE_GLOBAL, DEF_FREE_CLASS, DEF_IMPORT, DEF_BOUND,
-     OPT_IMPORT_STAR
+     OPT_IMPORT_STAR, SCOPE_OFF, SCOPE_MASK, FREE,
+     GLOBAL_IMPLICIT, GLOBAL_EXPLICIT)
 
 import weakref
 
@@ -38,15 +39,9 @@ class SymbolTableFactory:
 
 newSymbolTable = SymbolTableFactory()
 
-def is_free(flags):
-    if (flags & (USE | DEF_FREE)) \
-       and (flags & (DEF_LOCAL | DEF_PARAM | DEF_GLOBAL)):
-        return True
-    if flags & DEF_FREE_CLASS:
-        return True
-    return False
 
-class SymbolTable:
+class SymbolTable(object):
+
     def __init__(self, raw_table, filename):
         self._table = raw_table
         self._filename = filename
@@ -59,10 +54,11 @@ class SymbolTable:
             kind = "%s " % self.__class__.__name__
 
         if self._table.name == "global":
-            return "<%sSymbolTable for module %s>" % (kind, self._filename)
+            return "<{0}SymbolTable for module {1}>".format(kind, self._filename)
         else:
-            return "<%sSymbolTable for %s in %s>" % (kind, self._table.name,
+            return "<{0}SymbolTable for {1} in {2}>".format(kind,
-                                                     self._filename)
+                                                            self._table.name,
+                                                            self._filename)
 
     def get_type(self):
         if self._table.type == _symtable.TYPE_MODULE:
@@ -72,7 +68,7 @@ class SymbolTable:
         if self._table.type == _symtable.TYPE_CLASS:
             return "class"
         assert self._table.type in (1, 2, 3), \
-               "unexpected type: %s" % self._table.type
+               "unexpected type: {0}".format(self._table.type)
 
     def get_id(self):
         return self._table.id
@@ -124,6 +120,7 @@ class SymbolTable:
         return [newSymbolTable(st, self._filename)
                 for st in self._table.children]
 
+
 class Function(SymbolTable):
 
     # Default values for instance variables
@@ -148,15 +145,18 @@ class Function(SymbolTable):
 
     def get_globals(self):
         if self.__globals is None:
-            glob = DEF_GLOBAL | DEF_FREE_GLOBAL
+            glob = (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT)
-            self.__globals = self.__idents_matching(lambda x:x & glob)
+            test = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) in glob
+            self.__globals = self.__idents_matching(test)
         return self.__globals
 
     def get_frees(self):
         if self.__frees is None:
+            is_free = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) == FREE
             self.__frees = self.__idents_matching(is_free)
         return self.__frees
 
+
 class Class(SymbolTable):
 
     __methods = None
@@ -169,14 +169,17 @@ class Class(SymbolTable):
             self.__methods = tuple(d)
         return self.__methods
 
-class Symbol:
+
+class Symbol(object):
+
     def __init__(self, name, flags, namespaces=None):
         self.__name = name
         self.__flags = flags
+        self.__scope = (flags >> SCOPE_OFF) & SCOPE_MASK # like PyST_GetScope()
         self.__namespaces = namespaces or ()
 
     def __repr__(self):
-        return "<symbol '%s'>" % self.__name
+        return "<symbol {0!r}>".format(self.__name)
 
     def get_name(self):
         return self.__name
@@ -188,8 +191,7 @@ class Symbol:
         return bool(self.__flags & DEF_PARAM)
 
     def is_global(self):
-        return bool((self.__flags & DEF_GLOBAL)
+        return bool(self.__scope in (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT))
-                    or (self.__flags & DEF_FREE_GLOBAL))
 
     def is_vararg(self):
         return bool(self.__flags & DEF_STAR)
@@ -201,12 +203,7 @@ class Symbol:
         return bool(self.__flags & DEF_BOUND)
 
     def is_free(self):
-        if (self.__flags & (USE | DEF_FREE)) \
+        return bool(self.__scope == FREE)
-            and (self.__flags & (DEF_LOCAL | DEF_PARAM | DEF_GLOBAL)):
-            return True
-        if self.__flags & DEF_FREE_CLASS:
-            return True
-        return False
 
     def is_imported(self):
         return bool(self.__flags & DEF_IMPORT)

Lib/test/test_symtable.py

@@ -1,31 +1,161 @@
-from test import support
+"""
-
+Test the API of the symtable module.
+"""
 import symtable
 import unittest
 
+from test import support
 
-## XXX
-## Test disabled because symtable module needs to be rewritten for new compiler
 
-##vereq(symbols[0].name, "global")
+TEST_CODE = """
-##vereq(len([ste for ste in symbols.values() if ste.name == "f"]), 1)
+import sys
+
+glob = 42
+
+class Mine:
+    instance_var = 24
+    def a_method(p1, p2):
+        pass
+
+def spam(a, b, *var, **kw):
+    global bar
+    bar = 47
+    x = 23
+    glob
+    def internal():
+        return x
+    return internal
+
+def foo():
+    pass
+
+def namespace_test(): pass
+def namespace_test(): pass
+"""
+
+
+def find_block(block, name):
+    for ch in block.get_children():
+        if ch.get_name() == name:
+            return ch
 
-### Bug tickler: SyntaxError file name correct whether error raised
-### while parsing or building symbol table.
-##def checkfilename(brokencode):
-##    try:
-##        _symtable.symtable(brokencode, "spam", "exec")
-##    except SyntaxError, e:
-##        vereq(e.filename, "spam")
-##    else:
-##        raise TestFailed("no SyntaxError for %r" % (brokencode,))
-##checkfilename("def f(x): foo)(")  # parse-time
-##checkfilename("def f(x): global x")  # symtable-build-time
 
 class SymtableTest(unittest.TestCase):
-    def test_invalid_args(self):
+
-        self.assertRaises(TypeError, symtable.symtable, "42")
+    top = symtable.symtable(TEST_CODE, "?", "exec")
-        self.assertRaises(ValueError, symtable.symtable, "42", "?", "")
+    # These correspond to scopes in TEST_CODE
+    Mine = find_block(top, "Mine")
+    a_method = find_block(Mine, "a_method")
+    spam = find_block(top, "spam")
+    internal = find_block(spam, "internal")
+    foo = find_block(top, "foo")
+
+    def test_type(self):
+        self.assertEqual(self.top.get_type(), "module")
+        self.assertEqual(self.Mine.get_type(), "class")
+        self.assertEqual(self.a_method.get_type(), "function")
+        self.assertEqual(self.spam.get_type(), "function")
+        self.assertEqual(self.internal.get_type(), "function")
+
+    def test_optimized(self):
+        self.assertFalse(self.top.is_optimized())
+        self.assertFalse(self.top.has_exec())
+
+        self.assertTrue(self.spam.is_optimized())
+
+    def test_nested(self):
+        self.assertFalse(self.top.is_nested())
+        self.assertFalse(self.Mine.is_nested())
+        self.assertFalse(self.spam.is_nested())
+        self.assertTrue(self.internal.is_nested())
+
+    def test_children(self):
+        self.assertTrue(self.top.has_children())
+        self.assertTrue(self.Mine.has_children())
+        self.assertFalse(self.foo.has_children())
+
+    def test_lineno(self):
+        self.assertEqual(self.top.get_lineno(), 0)
+        self.assertEqual(self.spam.get_lineno(), 11)
+
+    def test_function_info(self):
+        func = self.spam
+        self.assertEqual(func.get_parameters(), ("a", "b", "kw", "var"))
+        self.assertEqual(func.get_locals(),
+                         ("a", "b", "bar", "glob", "internal", "kw", "var", "x"))
+        self.assertEqual(func.get_globals(), ("bar", "glob"))
+        self.assertEqual(self.internal.get_frees(), ("x",))
+
+    def test_globals(self):
+        self.assertTrue(self.spam.lookup("glob").is_global())
+        self.assertTrue(self.spam.lookup("bar").is_global())
+        self.assertFalse(self.internal.lookup("x").is_global())
+        self.assertFalse(self.Mine.lookup("instance_var").is_global())
+
+    def test_local(self):
+        self.assertTrue(self.spam.lookup("x").is_local())
+        self.assertFalse(self.internal.lookup("x").is_local())
+
+    def test_referenced(self):
+        self.assertTrue(self.internal.lookup("x").is_referenced())
+        self.assertTrue(self.spam.lookup("internal").is_referenced())
+        self.assertFalse(self.spam.lookup("x").is_referenced())
+
+    def test_parameters(self):
+        for sym in ("a", "var", "kw"):
+            self.assertTrue(self.spam.lookup(sym).is_parameter())
+        self.assertFalse(self.spam.lookup("x").is_parameter())
+
+    def test_symbol_lookup(self):
+        self.assertEqual(len(self.top.get_identifiers()),
+                         len(self.top.get_symbols()))
+
+        self.assertRaises(KeyError, self.top.lookup, "not_here")
+
+    def test_namespaces(self):
+        self.assertTrue(self.top.lookup("Mine").is_namespace())
+        self.assertTrue(self.Mine.lookup("a_method").is_namespace())
+        self.assertTrue(self.top.lookup("spam").is_namespace())
+        self.assertTrue(self.spam.lookup("internal").is_namespace())
+        self.assertTrue(self.top.lookup("namespace_test").is_namespace())
+        self.assertFalse(self.spam.lookup("x").is_namespace())
+
+        self.assert_(self.top.lookup("spam").get_namespace() is self.spam)
+        ns_test = self.top.lookup("namespace_test")
+        self.assertEqual(len(ns_test.get_namespaces()), 2)
+        self.assertRaises(ValueError, ns_test.get_namespace)
+
+    def test_assigned(self):
+        self.assertTrue(self.spam.lookup("x").is_assigned())
+        self.assertTrue(self.spam.lookup("bar").is_assigned())
+        self.assertTrue(self.top.lookup("spam").is_assigned())
+        self.assertTrue(self.Mine.lookup("a_method").is_assigned())
+        self.assertFalse(self.internal.lookup("x").is_assigned())
+
+    def test_imported(self):
+        self.assertTrue(self.top.lookup("sys").is_imported())
+
+    def test_name(self):
+        self.assertEqual(self.top.get_name(), "top")
+        self.assertEqual(self.spam.get_name(), "spam")
+        self.assertEqual(self.spam.lookup("x").get_name(), "x")
+        self.assertEqual(self.Mine.get_name(), "Mine")
+
+    def test_class_info(self):
+        self.assertEqual(self.Mine.get_methods(), ('a_method',))
+
+    def test_filename_correct(self):
+        ### Bug tickler: SyntaxError file name correct whether error raised
+        ### while parsing or building symbol table.
+        def checkfilename(brokencode):
+            try:
+                symtable.symtable(brokencode, "spam", "exec")
+            except SyntaxError as e:
+                self.assertEqual(e.filename, "spam")
+            else:
+                self.fail("no SyntaxError for %r" % (brokencode,))
+        checkfilename("def f(x): foo)(")  # parse-time
+        checkfilename("def f(x): global x")  # symtable-build-time
 
     def test_eval(self):
         symbols = symtable.symtable("42", "?", "eval")

Modules/symtablemodule.c

@@ -92,6 +92,10 @@ PyInit__symtable(void)
 	PyModule_AddIntConstant(m, "GLOBAL_IMPLICIT", GLOBAL_IMPLICIT);
 	PyModule_AddIntConstant(m, "FREE", FREE);
 	PyModule_AddIntConstant(m, "CELL", CELL);
+
+	PyModule_AddIntConstant(m, "SCOPE_OFF", SCOPE_OFFSET);
+	PyModule_AddIntConstant(m, "SCOPE_MASK", SCOPE_MASK);
+
 	if (PyErr_Occurred()) {
 		Py_DECREF(m);
 		m = 0;

Python/symtable.c

@@ -22,9 +22,9 @@
 #define RETURN_VAL_IN_GENERATOR \
     "'return' with argument inside generator"
 
-/* XXX(nnorwitz): change name since static? */
+
 static PySTEntryObject *
-PySTEntry_New(struct symtable *st, identifier name, _Py_block_ty block,
+ste_new(struct symtable *st, identifier name, _Py_block_ty block,
 	      void *key, int lineno)
 {
 	PySTEntryObject *ste = NULL;
@@ -114,6 +114,8 @@ static PyMemberDef ste_memberlist[] = {
 	{"symbols",  T_OBJECT, OFF(ste_symbols), READONLY},
 	{"varnames", T_OBJECT, OFF(ste_varnames), READONLY},
 	{"children", T_OBJECT, OFF(ste_children), READONLY},
+        {"optimized",T_INT,    OFF(ste_unoptimized), READONLY},
+	{"nested",   T_INT,    OFF(ste_nested), READONLY},
 	{"type",     T_INT,    OFF(ste_type), READONLY},
 	{"lineno",   T_INT,    OFF(ste_lineno), READONLY},
 	{NULL}
@@ -388,6 +390,9 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags,
 			PyErr_Format(PyExc_SyntaxError,
 			            "name '%U' is parameter and global",
 			            name);
+			PyErr_SyntaxLocation(ste->ste_table->st_filename,
+					     ste->ste_lineno);
+			
 			return 0;
 		}
                 if (flags & DEF_NONLOCAL) {
@@ -788,7 +793,7 @@ symtable_warn(struct symtable *st, char *msg, int lineno)
 	return 1;
 }
 
-/* symtable_enter_block() gets a reference via PySTEntry_New().
+/* symtable_enter_block() gets a reference via ste_new.
    This reference is released when the block is exited, via the DECREF
    in symtable_exit_block().
 */
@@ -825,7 +830,7 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
 		}
 		Py_DECREF(st->st_cur);
 	}
-	st->st_cur = PySTEntry_New(st, name, block, ast, lineno);
+	st->st_cur = ste_new(st, name, block, ast, lineno);
 	if (st->st_cur == NULL)
 		return 0;
 	if (name == GET_IDENTIFIER(top))