Improve symbol table processing.

This commit is contained in:
Windel Bouwman 2019-08-13 21:13:16 +02:00
parent 6128b2b46e
commit 3b876205e5
2 changed files with 306 additions and 187 deletions

View file

@ -8,7 +8,9 @@
use crate::error::{CompileError, CompileErrorType}; use crate::error::{CompileError, CompileErrorType};
use crate::output_stream::{CodeObjectStream, OutputStream}; use crate::output_stream::{CodeObjectStream, OutputStream};
use crate::peephole::PeepholeOptimizer; use crate::peephole::PeepholeOptimizer;
use crate::symboltable::{make_symbol_table, statements_to_symbol_table, Symbol, SymbolScope}; use crate::symboltable::{
make_symbol_table, statements_to_symbol_table, Symbol, SymbolScope, SymbolTable,
};
use num_complex::Complex64; use num_complex::Complex64;
use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Varargs}; use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Varargs};
use rustpython_parser::{ast, parser}; use rustpython_parser::{ast, parser};
@ -18,7 +20,7 @@ type BasicOutputStream = PeepholeOptimizer<CodeObjectStream>;
/// Main structure holding the state of compilation. /// Main structure holding the state of compilation.
struct Compiler<O: OutputStream = BasicOutputStream> { struct Compiler<O: OutputStream = BasicOutputStream> {
output_stack: Vec<O>, output_stack: Vec<O>,
scope_stack: Vec<SymbolScope>, symbol_table_stack: Vec<SymbolTable>,
nxt_label: usize, nxt_label: usize,
source_path: Option<String>, source_path: Option<String>,
current_source_location: ast::Location, current_source_location: ast::Location,
@ -123,7 +125,7 @@ impl<O: OutputStream> Compiler<O> {
fn new(optimize: u8) -> Self { fn new(optimize: u8) -> Self {
Compiler { Compiler {
output_stack: Vec::new(), output_stack: Vec::new(),
scope_stack: Vec::new(), symbol_table_stack: Vec::new(),
nxt_label: 0, nxt_label: 0,
source_path: None, source_path: None,
current_source_location: ast::Location::default(), current_source_location: ast::Location::default(),
@ -158,10 +160,10 @@ impl<O: OutputStream> Compiler<O> {
fn compile_program( fn compile_program(
&mut self, &mut self,
program: &ast::Program, program: &ast::Program,
symbol_scope: SymbolScope, symbol_table: SymbolTable,
) -> Result<(), CompileError> { ) -> Result<(), CompileError> {
let size_before = self.output_stack.len(); let size_before = self.output_stack.len();
self.scope_stack.push(symbol_scope); self.symbol_table_stack.push(symbol_table);
self.compile_statements(&program.statements)?; self.compile_statements(&program.statements)?;
assert_eq!(self.output_stack.len(), size_before); assert_eq!(self.output_stack.len(), size_before);
@ -176,9 +178,9 @@ impl<O: OutputStream> Compiler<O> {
fn compile_program_single( fn compile_program_single(
&mut self, &mut self,
program: &ast::Program, program: &ast::Program,
symbol_scope: SymbolScope, symbol_table: SymbolTable,
) -> Result<(), CompileError> { ) -> Result<(), CompileError> {
self.scope_stack.push(symbol_scope); self.symbol_table_stack.push(symbol_table);
let mut emitted_return = false; let mut emitted_return = false;
@ -215,9 +217,9 @@ impl<O: OutputStream> Compiler<O> {
fn compile_statement_eval( fn compile_statement_eval(
&mut self, &mut self,
statements: &[ast::Statement], statements: &[ast::Statement],
symbol_table: SymbolScope, symbol_table: SymbolTable,
) -> Result<(), CompileError> { ) -> Result<(), CompileError> {
self.scope_stack.push(symbol_table); self.symbol_table_stack.push(symbol_table);
for statement in statements { for statement in statements {
if let ast::StatementType::Expression { ref expression } = statement.node { if let ast::StatementType::Expression { ref expression } = statement.node {
self.compile_expression(expression)?; self.compile_expression(expression)?;
@ -241,12 +243,11 @@ impl<O: OutputStream> Compiler<O> {
fn scope_for_name(&self, name: &str) -> bytecode::NameScope { fn scope_for_name(&self, name: &str) -> bytecode::NameScope {
let symbol = self.lookup_name(name); let symbol = self.lookup_name(name);
if symbol.is_global { match symbol.scope {
bytecode::NameScope::Global SymbolScope::Global => bytecode::NameScope::Global,
} else if symbol.is_nonlocal { SymbolScope::Nonlocal => bytecode::NameScope::NonLocal,
bytecode::NameScope::NonLocal SymbolScope::Unknown => bytecode::NameScope::Local,
} else { SymbolScope::Local => bytecode::NameScope::Local,
bytecode::NameScope::Local
} }
} }
@ -1897,22 +1898,27 @@ impl<O: OutputStream> Compiler<O> {
// Scope helpers: // Scope helpers:
fn enter_scope(&mut self) { fn enter_scope(&mut self) {
// println!("Enter scope {:?}", self.scope_stack); // println!("Enter scope {:?}", self.symbol_table_stack);
// Enter first subscope! // Enter first subscope!
let scope = self.scope_stack.last_mut().unwrap().sub_scopes.remove(0); let table = self
self.scope_stack.push(scope); .symbol_table_stack
.last_mut()
.unwrap()
.sub_tables
.remove(0);
self.symbol_table_stack.push(table);
} }
fn leave_scope(&mut self) { fn leave_scope(&mut self) {
// println!("Leave scope {:?}", self.scope_stack); // println!("Leave scope {:?}", self.symbol_table_stack);
let scope = self.scope_stack.pop().unwrap(); let table = self.symbol_table_stack.pop().unwrap();
assert!(scope.sub_scopes.is_empty()); assert!(table.sub_tables.is_empty());
} }
fn lookup_name(&self, name: &str) -> &Symbol { fn lookup_name(&self, name: &str) -> &Symbol {
// println!("Looking up {:?}", name); // println!("Looking up {:?}", name);
let scope = self.scope_stack.last().unwrap(); let symbol_table = self.symbol_table_stack.last().unwrap();
scope.lookup(name).expect( symbol_table.lookup(name).expect(
"The symbol must be present in the symbol table, even when it is undefined in python.", "The symbol must be present in the symbol table, even when it is undefined in python.",
) )
} }

View file

@ -12,47 +12,59 @@ use indexmap::map::IndexMap;
use rustpython_parser::ast; use rustpython_parser::ast;
use rustpython_parser::location::Location; use rustpython_parser::location::Location;
pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolScope, SymbolTableError> { pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolTable, SymbolTableError> {
let mut builder: SymbolTableBuilder = Default::default(); let mut builder: SymbolTableBuilder = Default::default();
builder.enter_scope(); builder.prepare();
builder.scan_program(program)?; builder.scan_program(program)?;
assert_eq!(builder.scopes.len(), 1); builder.finish()
let symbol_table = builder.scopes.pop().unwrap();
analyze_symbol_table(&symbol_table, None)?;
Ok(symbol_table)
} }
pub fn statements_to_symbol_table( pub fn statements_to_symbol_table(
statements: &[ast::Statement], statements: &[ast::Statement],
) -> Result<SymbolScope, SymbolTableError> { ) -> Result<SymbolTable, SymbolTableError> {
let mut builder: SymbolTableBuilder = Default::default(); let mut builder: SymbolTableBuilder = Default::default();
builder.enter_scope(); builder.prepare();
builder.scan_statements(statements)?; builder.scan_statements(statements)?;
assert_eq!(builder.scopes.len(), 1); builder.finish()
let symbol_table = builder.scopes.pop().unwrap();
analyze_symbol_table(&symbol_table, None)?;
Ok(symbol_table)
} }
/// Captures all symbols in the current scope, and has a list of subscopes in this scope. /// Captures all symbols in the current scope, and has a list of subscopes in this scope.
#[derive(Clone, Default)] #[derive(Clone, Default)]
pub struct SymbolScope { pub struct SymbolTable {
/// A set of symbols present on this scope level. /// A set of symbols present on this scope level.
pub symbols: IndexMap<String, Symbol>, pub symbols: IndexMap<String, Symbol>,
/// A list of subscopes in the order as found in the /// A list of subscopes in the order as found in the
/// AST nodes. /// AST nodes.
pub sub_scopes: Vec<SymbolScope>, pub sub_tables: Vec<SymbolTable>,
} }
impl SymbolTable {
fn new() -> Self {
SymbolTable {
symbols: Default::default(),
sub_tables: vec![],
}
}
}
/// Indicator for a single symbol what the scope of this symbol is.
/// The scope can be unknown, which is unfortunate, but not impossible.
#[derive(Debug, Clone)]
pub enum SymbolScope {
Global,
Nonlocal,
Local,
Unknown,
}
/// A single symbol in a table. Has various properties such as the scope
/// of the symbol, and also the various uses of the symbol.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct Symbol { pub struct Symbol {
pub name: String, pub name: String,
pub is_global: bool, // pub table: SymbolTableRef,
pub is_local: bool, pub scope: SymbolScope,
pub is_nonlocal: bool,
pub is_param: bool, pub is_param: bool,
pub is_referenced: bool, pub is_referenced: bool,
pub is_assigned: bool, pub is_assigned: bool,
@ -64,9 +76,8 @@ impl Symbol {
fn new(name: &str) -> Self { fn new(name: &str) -> Self {
Symbol { Symbol {
name: name.to_string(), name: name.to_string(),
is_global: false, // table,
is_local: false, scope: SymbolScope::Unknown,
is_nonlocal: false,
is_param: false, is_param: false,
is_referenced: false, is_referenced: false,
is_assigned: false, is_assigned: false,
@ -74,6 +85,22 @@ impl Symbol {
is_free: false, is_free: false,
} }
} }
pub fn is_global(&self) -> bool {
if let SymbolScope::Global = self.scope {
true
} else {
false
}
}
pub fn is_local(&self) -> bool {
if let SymbolScope::Local = self.scope {
true
} else {
false
}
}
} }
#[derive(Debug)] #[derive(Debug)]
@ -93,19 +120,19 @@ impl From<SymbolTableError> for CompileError {
type SymbolTableResult = Result<(), SymbolTableError>; type SymbolTableResult = Result<(), SymbolTableError>;
impl SymbolScope { impl SymbolTable {
pub fn lookup(&self, name: &str) -> Option<&Symbol> { pub fn lookup(&self, name: &str) -> Option<&Symbol> {
self.symbols.get(name) self.symbols.get(name)
} }
} }
impl std::fmt::Debug for SymbolScope { impl std::fmt::Debug for SymbolTable {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!( write!(
f, f,
"SymbolScope({:?} symbols, {:?} sub scopes)", "SymbolTable({:?} symbols, {:?} sub scopes)",
self.symbols.len(), self.symbols.len(),
self.sub_scopes.len() self.sub_tables.len()
) )
} }
} }
@ -113,75 +140,127 @@ impl std::fmt::Debug for SymbolScope {
/* Perform some sort of analysis on nonlocals, globals etc.. /* Perform some sort of analysis on nonlocals, globals etc..
See also: https://github.com/python/cpython/blob/master/Python/symtable.c#L410 See also: https://github.com/python/cpython/blob/master/Python/symtable.c#L410
*/ */
fn analyze_symbol_table( fn analyze_symbol_table(symbol_table: &mut SymbolTable) -> SymbolTableResult {
symbol_scope: &SymbolScope, let mut analyzer = SymbolTableAnalyzer::default();
parent_symbol_scope: Option<&SymbolScope>, analyzer.analyze_symbol_table(symbol_table)
) -> SymbolTableResult {
// Analyze sub scopes:
for sub_scope in &symbol_scope.sub_scopes {
analyze_symbol_table(&sub_scope, Some(symbol_scope))?;
}
// Analyze symbols:
for symbol in symbol_scope.symbols.values() {
analyze_symbol(symbol, parent_symbol_scope)?;
}
Ok(())
} }
fn analyze_symbol(symbol: &Symbol, parent_symbol_scope: Option<&SymbolScope>) -> SymbolTableResult { /// Symbol table analysis. Can be used to analyze a fully
if symbol.is_nonlocal { /// build symbol table structure. It will mark variables
// check if name is defined in parent scope! /// as local variables for example.
if let Some(parent_symbol_scope) = parent_symbol_scope { #[derive(Default)]
if !parent_symbol_scope.symbols.contains_key(&symbol.name) { struct SymbolTableAnalyzer {
return Err(SymbolTableError { tables: Vec<SymbolTable>,
error: format!("no binding for nonlocal '{}' found", symbol.name), }
location: Default::default(),
}); impl SymbolTableAnalyzer {
} fn analyze_symbol_table(&mut self, symbol_table: &mut SymbolTable) -> SymbolTableResult {
} else { // Store a copy to determine the parent.
return Err(SymbolTableError { // TODO: this should be improved to resolve this clone action.
error: format!( self.tables.push(symbol_table.clone());
"nonlocal {} defined at place without an enclosing scope",
symbol.name // Analyze sub scopes:
), for sub_table in &mut symbol_table.sub_tables {
location: Default::default(), self.analyze_symbol_table(sub_table)?;
});
} }
self.tables.pop();
// Analyze symbols:
for symbol in symbol_table.symbols.values_mut() {
self.analyze_symbol(symbol)?;
}
Ok(())
} }
// TODO: add more checks for globals fn analyze_symbol(&self, symbol: &mut Symbol) -> SymbolTableResult {
match symbol.scope {
SymbolScope::Nonlocal => {
// check if name is defined in parent table!
let parent_symbol_table: Option<&SymbolTable> = self.tables.last();
// symbol.table.borrow().parent.clone();
Ok(()) if let Some(table) = parent_symbol_table {
if !table.symbols.contains_key(&symbol.name) {
return Err(SymbolTableError {
error: format!("no binding for nonlocal '{}' found", symbol.name),
location: Default::default(),
});
}
} else {
return Err(SymbolTableError {
error: format!(
"nonlocal {} defined at place without an enclosing scope",
symbol.name
),
location: Default::default(),
});
}
}
SymbolScope::Global => {
// TODO: add more checks for globals?
}
SymbolScope::Local => {
// all is well
}
SymbolScope::Unknown => {
if symbol.is_assigned {
symbol.scope = SymbolScope::Local;
}
}
}
Ok(())
}
} }
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
enum SymbolRole { enum SymbolUsage {
Global, Global,
Nonlocal, Nonlocal,
Used, Used,
Assigned, Assigned,
Parameter,
} }
#[derive(Default)] #[derive(Default)]
struct SymbolTableBuilder { struct SymbolTableBuilder {
// Scope stack. // Scope stack.
scopes: Vec<SymbolScope>, tables: Vec<SymbolTable>,
}
/// Enum to indicate in what mode an expression
/// was used.
/// In cpython this is stored in the AST, but I think this
/// is not logical, since it is not context free.
enum ExpressionContext {
Load,
Store,
Delete,
} }
impl SymbolTableBuilder { impl SymbolTableBuilder {
fn enter_scope(&mut self) { fn prepare(&mut self) {
let scope = Default::default(); let table = SymbolTable::new();
self.scopes.push(scope); self.tables.push(table);
// self.work_scopes.push(Default::default());
} }
fn leave_scope(&mut self) { fn finish(&mut self) -> Result<SymbolTable, SymbolTableError> {
// Pop scope and add to subscopes of parent scope. assert_eq!(self.tables.len(), 1);
// let work_scope = self.work_scopes.pop().unwrap(); let mut symbol_table = self.tables.pop().unwrap();
let scope = self.scopes.pop().unwrap(); analyze_symbol_table(&mut symbol_table)?;
self.scopes.last_mut().unwrap().sub_scopes.push(scope); Ok(symbol_table)
}
fn enter_block(&mut self) {
// let parent = Some(self.tables.last().unwrap().clone());
let table = SymbolTable::new();
self.tables.push(table);
}
fn leave_block(&mut self) {
// Pop symbol table and add to sub table of parent table.
let table = self.tables.pop().unwrap();
self.tables.last_mut().unwrap().sub_tables.push(table);
} }
fn scan_program(&mut self, program: &ast::Program) -> SymbolTableResult { fn scan_program(&mut self, program: &ast::Program) -> SymbolTableResult {
@ -204,7 +283,7 @@ impl SymbolTableBuilder {
} }
fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult { fn scan_parameter(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
self.register_name(&parameter.arg, SymbolRole::Assigned) self.register_name(&parameter.arg, SymbolUsage::Parameter)
} }
fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult { fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult {
@ -216,7 +295,7 @@ impl SymbolTableBuilder {
fn scan_parameter_annotation(&mut self, parameter: &ast::Parameter) -> SymbolTableResult { fn scan_parameter_annotation(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
if let Some(annotation) = &parameter.annotation { if let Some(annotation) = &parameter.annotation {
self.scan_expression(&annotation)?; self.scan_expression(&annotation, &ExpressionContext::Load)?;
} }
Ok(()) Ok(())
} }
@ -226,12 +305,12 @@ impl SymbolTableBuilder {
match &statement.node { match &statement.node {
Global { names } => { Global { names } => {
for name in names { for name in names {
self.register_name(name, SymbolRole::Global)?; self.register_name(name, SymbolUsage::Global)?;
} }
} }
Nonlocal { names } => { Nonlocal { names } => {
for name in names { for name in names {
self.register_name(name, SymbolRole::Nonlocal)?; self.register_name(name, SymbolUsage::Nonlocal)?;
} }
} }
FunctionDef { FunctionDef {
@ -242,14 +321,14 @@ impl SymbolTableBuilder {
returns, returns,
.. ..
} => { } => {
self.scan_expressions(decorator_list)?; self.scan_expressions(decorator_list, &ExpressionContext::Load)?;
self.register_name(name, SymbolRole::Assigned)?; self.register_name(name, SymbolUsage::Assigned)?;
if let Some(expression) = returns { if let Some(expression) = returns {
self.scan_expression(expression)?; self.scan_expression(expression, &ExpressionContext::Load)?;
} }
self.enter_function(args)?; self.enter_function(args)?;
self.scan_statements(body)?; self.scan_statements(body)?;
self.leave_scope(); self.leave_block();
} }
ClassDef { ClassDef {
name, name,
@ -258,19 +337,21 @@ impl SymbolTableBuilder {
keywords, keywords,
decorator_list, decorator_list,
} => { } => {
self.register_name(name, SymbolRole::Assigned)?; self.register_name(name, SymbolUsage::Assigned)?;
self.enter_scope(); self.enter_block();
self.scan_statements(body)?; self.scan_statements(body)?;
self.leave_scope(); self.leave_block();
self.scan_expressions(bases)?; self.scan_expressions(bases, &ExpressionContext::Load)?;
for keyword in keywords { for keyword in keywords {
self.scan_expression(&keyword.value)?; self.scan_expression(&keyword.value, &ExpressionContext::Load)?;
} }
self.scan_expressions(decorator_list)?; self.scan_expressions(decorator_list, &ExpressionContext::Load)?;
}
Expression { expression } => {
self.scan_expression(expression, &ExpressionContext::Load)?
} }
Expression { expression } => self.scan_expression(expression)?,
If { test, body, orelse } => { If { test, body, orelse } => {
self.scan_expression(test)?; self.scan_expression(test, &ExpressionContext::Load)?;
self.scan_statements(body)?; self.scan_statements(body)?;
if let Some(code) = orelse { if let Some(code) = orelse {
self.scan_statements(code)?; self.scan_statements(code)?;
@ -283,15 +364,15 @@ impl SymbolTableBuilder {
orelse, orelse,
.. ..
} => { } => {
self.scan_expression(target)?; self.scan_expression(target, &ExpressionContext::Store)?;
self.scan_expression(iter)?; self.scan_expression(iter, &ExpressionContext::Load)?;
self.scan_statements(body)?; self.scan_statements(body)?;
if let Some(code) = orelse { if let Some(code) = orelse {
self.scan_statements(code)?; self.scan_statements(code)?;
} }
} }
While { test, body, orelse } => { While { test, body, orelse } => {
self.scan_expression(test)?; self.scan_expression(test, &ExpressionContext::Load)?;
self.scan_statements(body)?; self.scan_statements(body)?;
if let Some(code) = orelse { if let Some(code) = orelse {
self.scan_statements(code)?; self.scan_statements(code)?;
@ -304,51 +385,51 @@ impl SymbolTableBuilder {
for name in names { for name in names {
if let Some(alias) = &name.alias { if let Some(alias) = &name.alias {
// `import mymodule as myalias` // `import mymodule as myalias`
self.register_name(alias, SymbolRole::Assigned)?; self.register_name(alias, SymbolUsage::Assigned)?;
} else { } else {
// `import module` // `import module`
self.register_name(&name.symbol, SymbolRole::Assigned)?; self.register_name(&name.symbol, SymbolUsage::Assigned)?;
} }
} }
} }
Return { value } => { Return { value } => {
if let Some(expression) = value { if let Some(expression) = value {
self.scan_expression(expression)?; self.scan_expression(expression, &ExpressionContext::Load)?;
} }
} }
Assert { test, msg } => { Assert { test, msg } => {
self.scan_expression(test)?; self.scan_expression(test, &ExpressionContext::Load)?;
if let Some(expression) = msg { if let Some(expression) = msg {
self.scan_expression(expression)?; self.scan_expression(expression, &ExpressionContext::Load)?;
} }
} }
Delete { targets } => { Delete { targets } => {
self.scan_expressions(targets)?; self.scan_expressions(targets, &ExpressionContext::Delete)?;
} }
Assign { targets, value } => { Assign { targets, value } => {
self.scan_expressions(targets)?; self.scan_expressions(targets, &ExpressionContext::Store)?;
self.scan_expression(value)?; self.scan_expression(value, &ExpressionContext::Load)?;
} }
AugAssign { target, value, .. } => { AugAssign { target, value, .. } => {
self.scan_expression(target)?; self.scan_expression(target, &ExpressionContext::Store)?;
self.scan_expression(value)?; self.scan_expression(value, &ExpressionContext::Load)?;
} }
AnnAssign { AnnAssign {
target, target,
annotation, annotation,
value, value,
} => { } => {
self.scan_expression(target)?; self.scan_expression(target, &ExpressionContext::Store)?;
self.scan_expression(annotation)?; self.scan_expression(annotation, &ExpressionContext::Load)?;
if let Some(value) = value { if let Some(value) = value {
self.scan_expression(value)?; self.scan_expression(value, &ExpressionContext::Load)?;
} }
} }
With { items, body, .. } => { With { items, body, .. } => {
for item in items { for item in items {
self.scan_expression(&item.context_expr)?; self.scan_expression(&item.context_expr, &ExpressionContext::Load)?;
if let Some(expression) = &item.optional_vars { if let Some(expression) = &item.optional_vars {
self.scan_expression(expression)?; self.scan_expression(expression, &ExpressionContext::Store)?;
} }
} }
self.scan_statements(body)?; self.scan_statements(body)?;
@ -362,10 +443,10 @@ impl SymbolTableBuilder {
self.scan_statements(body)?; self.scan_statements(body)?;
for handler in handlers { for handler in handlers {
if let Some(expression) = &handler.typ { if let Some(expression) = &handler.typ {
self.scan_expression(expression)?; self.scan_expression(expression, &ExpressionContext::Load)?;
} }
if let Some(name) = &handler.name { if let Some(name) = &handler.name {
self.register_name(name, SymbolRole::Assigned)?; self.register_name(name, SymbolUsage::Assigned)?;
} }
self.scan_statements(&handler.body)?; self.scan_statements(&handler.body)?;
} }
@ -378,94 +459,102 @@ impl SymbolTableBuilder {
} }
Raise { exception, cause } => { Raise { exception, cause } => {
if let Some(expression) = exception { if let Some(expression) = exception {
self.scan_expression(expression)?; self.scan_expression(expression, &ExpressionContext::Load)?;
} }
if let Some(expression) = cause { if let Some(expression) = cause {
self.scan_expression(expression)?; self.scan_expression(expression, &ExpressionContext::Load)?;
} }
} }
} }
Ok(()) Ok(())
} }
fn scan_expressions(&mut self, expressions: &[ast::Expression]) -> SymbolTableResult { fn scan_expressions(
&mut self,
expressions: &[ast::Expression],
context: &ExpressionContext,
) -> SymbolTableResult {
for expression in expressions { for expression in expressions {
self.scan_expression(expression)?; self.scan_expression(expression, context)?;
} }
Ok(()) Ok(())
} }
fn scan_expression(&mut self, expression: &ast::Expression) -> SymbolTableResult { fn scan_expression(
&mut self,
expression: &ast::Expression,
context: &ExpressionContext,
) -> SymbolTableResult {
use ast::ExpressionType::*; use ast::ExpressionType::*;
match &expression.node { match &expression.node {
Binop { a, b, .. } => { Binop { a, b, .. } => {
self.scan_expression(a)?; self.scan_expression(a, context)?;
self.scan_expression(b)?; self.scan_expression(b, context)?;
} }
BoolOp { values, .. } => { BoolOp { values, .. } => {
self.scan_expressions(values)?; self.scan_expressions(values, context)?;
} }
Compare { vals, .. } => { Compare { vals, .. } => {
self.scan_expressions(vals)?; self.scan_expressions(vals, context)?;
} }
Subscript { a, b } => { Subscript { a, b } => {
self.scan_expression(a)?; self.scan_expression(a, context)?;
self.scan_expression(b)?; self.scan_expression(b, context)?;
} }
Attribute { value, .. } => { Attribute { value, .. } => {
self.scan_expression(value)?; self.scan_expression(value, context)?;
} }
Dict { elements } => { Dict { elements } => {
for (key, value) in elements { for (key, value) in elements {
if let Some(key) = key { if let Some(key) = key {
self.scan_expression(key)?; self.scan_expression(key, context)?;
} else { } else {
// dict unpacking marker // dict unpacking marker
} }
self.scan_expression(value)?; self.scan_expression(value, context)?;
} }
} }
Await { value } => { Await { value } => {
self.scan_expression(value)?; self.scan_expression(value, context)?;
} }
Yield { value } => { Yield { value } => {
if let Some(expression) = value { if let Some(expression) = value {
self.scan_expression(expression)?; self.scan_expression(expression, context)?;
} }
} }
YieldFrom { value } => { YieldFrom { value } => {
self.scan_expression(value)?; self.scan_expression(value, context)?;
} }
Unop { a, .. } => { Unop { a, .. } => {
self.scan_expression(a)?; self.scan_expression(a, context)?;
} }
True | False | None | Ellipsis => {} True | False | None | Ellipsis => {}
Number { .. } => {} Number { .. } => {}
Starred { value } => { Starred { value } => {
self.scan_expression(value)?; self.scan_expression(value, context)?;
} }
Bytes { .. } => {} Bytes { .. } => {}
Tuple { elements } | Set { elements } | List { elements } | Slice { elements } => { Tuple { elements } | Set { elements } | List { elements } | Slice { elements } => {
self.scan_expressions(elements)?; self.scan_expressions(elements, &ExpressionContext::Load)?;
} }
Comprehension { kind, generators } => { Comprehension { kind, generators } => {
match **kind { match **kind {
ast::ComprehensionKind::GeneratorExpression { ref element } ast::ComprehensionKind::GeneratorExpression { ref element }
| ast::ComprehensionKind::List { ref element } | ast::ComprehensionKind::List { ref element }
| ast::ComprehensionKind::Set { ref element } => { | ast::ComprehensionKind::Set { ref element } => {
self.scan_expression(element)?; self.scan_expression(element, &ExpressionContext::Load)?;
} }
ast::ComprehensionKind::Dict { ref key, ref value } => { ast::ComprehensionKind::Dict { ref key, ref value } => {
self.scan_expression(&key)?; self.scan_expression(&key, &ExpressionContext::Load)?;
self.scan_expression(&value)?; self.scan_expression(&value, &ExpressionContext::Load)?;
} }
} }
for generator in generators { for generator in generators {
self.scan_expression(&generator.target)?; self.scan_expression(&generator.target, &ExpressionContext::Store)?;
self.scan_expression(&generator.iter)?; self.scan_expression(&generator.iter, &ExpressionContext::Load)?;
for if_expr in &generator.ifs { for if_expr in &generator.ifs {
self.scan_expression(if_expr)?; self.scan_expression(if_expr, &ExpressionContext::Load)?;
} }
} }
} }
@ -474,27 +563,38 @@ impl SymbolTableBuilder {
args, args,
keywords, keywords,
} => { } => {
self.scan_expression(function)?; self.scan_expression(function, &ExpressionContext::Load)?;
self.scan_expressions(args)?; self.scan_expressions(args, &ExpressionContext::Load)?;
for keyword in keywords { for keyword in keywords {
self.scan_expression(&keyword.value)?; self.scan_expression(&keyword.value, &ExpressionContext::Load)?;
} }
} }
String { value } => { String { value } => {
self.scan_string_group(value)?; self.scan_string_group(value)?;
} }
Identifier { name } => { Identifier { name } => {
self.register_name(name, SymbolRole::Used)?; // Determine the contextual usage of this symbol:
match context {
ExpressionContext::Delete => {
self.register_name(name, SymbolUsage::Used)?;
}
ExpressionContext::Load => {
self.register_name(name, SymbolUsage::Used)?;
}
ExpressionContext::Store => {
self.register_name(name, SymbolUsage::Assigned)?;
}
}
} }
Lambda { args, body } => { Lambda { args, body } => {
self.enter_function(args)?; self.enter_function(args)?;
self.scan_expression(body)?; self.scan_expression(body, &ExpressionContext::Load)?;
self.leave_scope(); self.leave_block();
} }
IfExpression { test, body, orelse } => { IfExpression { test, body, orelse } => {
self.scan_expression(test)?; self.scan_expression(test, &ExpressionContext::Load)?;
self.scan_expression(body)?; self.scan_expression(body, &ExpressionContext::Load)?;
self.scan_expression(orelse)?; self.scan_expression(orelse, &ExpressionContext::Load)?;
} }
} }
Ok(()) Ok(())
@ -502,10 +602,10 @@ impl SymbolTableBuilder {
fn enter_function(&mut self, args: &ast::Parameters) -> SymbolTableResult { fn enter_function(&mut self, args: &ast::Parameters) -> SymbolTableResult {
// Evaluate eventual default parameters: // Evaluate eventual default parameters:
self.scan_expressions(&args.defaults)?; self.scan_expressions(&args.defaults, &ExpressionContext::Load)?;
for kw_default in &args.kw_defaults { for kw_default in &args.kw_defaults {
if let Some(expression) = kw_default { if let Some(expression) = kw_default {
self.scan_expression(&expression)?; self.scan_expression(&expression, &ExpressionContext::Load)?;
} }
} }
@ -519,7 +619,7 @@ impl SymbolTableBuilder {
self.scan_parameter_annotation(name)?; self.scan_parameter_annotation(name)?;
} }
self.enter_scope(); self.enter_block();
// Fill scope with parameter names: // Fill scope with parameter names:
self.scan_parameters(&args.args)?; self.scan_parameters(&args.args)?;
@ -537,7 +637,7 @@ impl SymbolTableBuilder {
match group { match group {
ast::StringGroup::Constant { .. } => {} ast::StringGroup::Constant { .. } => {}
ast::StringGroup::FormattedValue { value, .. } => { ast::StringGroup::FormattedValue { value, .. } => {
self.scan_expression(value)?; self.scan_expression(value, &ExpressionContext::Load)?;
} }
ast::StringGroup::Joined { values } => { ast::StringGroup::Joined { values } => {
for subgroup in values { for subgroup in values {
@ -549,22 +649,22 @@ impl SymbolTableBuilder {
} }
#[allow(clippy::single_match)] #[allow(clippy::single_match)]
fn register_name(&mut self, name: &str, role: SymbolRole) -> SymbolTableResult { fn register_name(&mut self, name: &str, role: SymbolUsage) -> SymbolTableResult {
let scope_depth = self.scopes.len(); let scope_depth = self.tables.len();
let current_scope = self.scopes.last_mut().unwrap(); let table = self.tables.last_mut().unwrap();
let location = Default::default(); let location = Default::default();
// Some checks: // Some checks:
if current_scope.symbols.contains_key(name) { if table.symbols.contains_key(name) {
// Role already set.. // Role already set..
match role { match role {
SymbolRole::Global => { SymbolUsage::Global => {
return Err(SymbolTableError { return Err(SymbolTableError {
error: format!("name '{}' is used prior to global declaration", name), error: format!("name '{}' is used prior to global declaration", name),
location, location,
}) })
} }
SymbolRole::Nonlocal => { SymbolUsage::Nonlocal => {
return Err(SymbolTableError { return Err(SymbolTableError {
error: format!("name '{}' is used prior to nonlocal declaration", name), error: format!("name '{}' is used prior to nonlocal declaration", name),
location, location,
@ -578,7 +678,7 @@ impl SymbolTableBuilder {
// Some more checks: // Some more checks:
match role { match role {
SymbolRole::Nonlocal => { SymbolUsage::Nonlocal => {
if scope_depth < 2 { if scope_depth < 2 {
return Err(SymbolTableError { return Err(SymbolTableError {
error: format!("cannot define nonlocal '{}' at top level.", name), error: format!("cannot define nonlocal '{}' at top level.", name),
@ -592,25 +692,38 @@ impl SymbolTableBuilder {
} }
// Insert symbol when required: // Insert symbol when required:
if !current_scope.symbols.contains_key(name) { if !table.symbols.contains_key(name) {
let symbol = Symbol::new(name); let symbol = Symbol::new(name);
current_scope.symbols.insert(name.to_string(), symbol); table.symbols.insert(name.to_string(), symbol);
} }
// Set proper flags on symbol: // Set proper flags on symbol:
let symbol = current_scope.symbols.get_mut(name).unwrap(); let symbol = table.symbols.get_mut(name).unwrap();
match role { match role {
SymbolRole::Nonlocal => { SymbolUsage::Nonlocal => {
symbol.is_nonlocal = true; if let SymbolScope::Unknown = symbol.scope {
symbol.scope = SymbolScope::Nonlocal;
} else {
return Err(SymbolTableError {
error: format!("Symbol {} scope cannot be set to nonlocal, since its scope was already determined otherwise.", name),
location,
});
}
} }
SymbolRole::Assigned => { SymbolUsage::Parameter | SymbolUsage::Assigned => {
symbol.is_assigned = true; symbol.is_assigned = true;
// symbol.is_local = true;
} }
SymbolRole::Global => { SymbolUsage::Global => {
symbol.is_global = true; if let SymbolScope::Unknown = symbol.scope {
symbol.scope = SymbolScope::Global;
} else {
return Err(SymbolTableError {
error: format!("Symbol {} scope cannot be set to global, since its scope was already determined otherwise.", name),
location,
});
}
} }
SymbolRole::Used => { SymbolUsage::Used => {
symbol.is_referenced = true; symbol.is_referenced = true;
} }
} }