mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-07-21 12:05:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Split the ast from the parser, remove compiler dep on parser

Browse source
This commit is contained in:
Noah 2020-11-02 12:37:19 -06:00
parent 3f88b08aaa
commit e9095a741d
5 changed files with 153 additions and 225 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Cargo.toml
View file

@ -11,10 +11,11 @@ edition = "2018"
indexmap = "1.0" indexmap = "1.0"
itertools = "0.9" itertools = "0.9"
rustpython-bytecode = { path = "../bytecode", version = "0.1.1" } rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
rustpython-parser = { path = "../parser", version = "0.1.1" } rustpython-ast = { path = "../ast" }
num-complex = { version = "0.3", features = ["serde"] } num-complex = { version = "0.3", features = ["serde"] }
log = "0.4" log = "0.4"
arrayvec = "0.5" arrayvec = "0.5"
[dev-dependencies] [dev-dependencies]
rustpython-parser = { path = "../parser" }
insta = "1.1" insta = "1.1"

46
src/compile.rs
View file

@ -12,8 +12,8 @@ use crate::symboltable::{
}; };
use itertools::Itertools; use itertools::Itertools;
use num_complex::Complex64; use num_complex::Complex64;
use rustpython_ast as ast;
use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Label}; use rustpython_bytecode::bytecode::{self, CallType, CodeObject, Instruction, Label};
use rustpython_parser::{ast, parser};
type CompileResult<T> = Result<T, CompileError>; type CompileResult<T> = Result<T, CompileError>;
@ -61,31 +61,6 @@ impl CompileContext {
} }
} }
/// Compile a given sourcecode into a bytecode object.
pub fn compile(
source: &str,
mode: Mode,
source_path: String,
opts: CompileOpts,
) -> CompileResult<CodeObject> {
let to_compile_error =
|parse_error| CompileError::from_parse_error(parse_error, source_path.clone());
match mode {
Mode::Exec => {
let ast = parser::parse_program(source).map_err(to_compile_error)?;
compile_program(ast, source_path, opts)
}
Mode::Eval => {
let statement = parser::parse_statement(source).map_err(to_compile_error)?;
compile_statement_eval(statement, source_path, opts)
}
Mode::Single => {
let ast = parser::parse_program(source).map_err(to_compile_error)?;
compile_program_single(ast, source_path, opts)
}
}
}
/// A helper function for the shared code of the different compile functions /// A helper function for the shared code of the different compile functions
fn with_compiler( fn with_compiler(
source_path: String, source_path: String,
@ -106,8 +81,10 @@ pub fn compile_program(
source_path: String, source_path: String,
opts: CompileOpts, opts: CompileOpts,
) -> CompileResult<CodeObject> { ) -> CompileResult<CodeObject> {
let symbol_table = make_symbol_table(&ast) let symbol_table = match make_symbol_table(&ast) {
.map_err(|e| CompileError::from_symbol_table_error(e, source_path.clone()))?; Ok(x) => x,
Err(e) => return Err(e.into_compile_error(source_path)),
};
with_compiler(source_path, opts, |compiler| { with_compiler(source_path, opts, |compiler| {
compiler.compile_program(&ast, symbol_table) compiler.compile_program(&ast, symbol_table)
}) })
@ -119,8 +96,10 @@ pub fn compile_statement_eval(
source_path: String, source_path: String,
opts: CompileOpts, opts: CompileOpts,
) -> CompileResult<CodeObject> { ) -> CompileResult<CodeObject> {
let symbol_table = statements_to_symbol_table(&statement) let symbol_table = match statements_to_symbol_table(&statement) {
.map_err(|e| CompileError::from_symbol_table_error(e, source_path.clone()))?; Ok(x) => x,
Err(e) => return Err(e.into_compile_error(source_path)),
};
with_compiler(source_path, opts, |compiler| { with_compiler(source_path, opts, |compiler| {
compiler.compile_statement_eval(&statement, symbol_table) compiler.compile_statement_eval(&statement, symbol_table)
}) })
@ -132,8 +111,10 @@ pub fn compile_program_single(
source_path: String, source_path: String,
opts: CompileOpts, opts: CompileOpts,
) -> CompileResult<CodeObject> { ) -> CompileResult<CodeObject> {
let symbol_table = make_symbol_table(&ast) let symbol_table = match make_symbol_table(&ast) {
.map_err(|e| CompileError::from_symbol_table_error(e, source_path.clone()))?; Ok(x) => x,
Err(e) => return Err(e.into_compile_error(source_path)),
};
with_compiler(source_path, opts, |compiler| { with_compiler(source_path, opts, |compiler| {
compiler.compile_program_single(&ast, symbol_table) compiler.compile_program_single(&ast, symbol_table)
}) })
@ -165,7 +146,6 @@ impl Compiler {
error, error,
location, location,
source_path: self.source_path.clone(), source_path: self.source_path.clone(),
statement: None,
} }
} }

112
src/error.rs
View file

@ -1,34 +1,17 @@
use rustpython_parser::error::{LexicalErrorType, ParseError, ParseErrorType}; use rustpython_ast::Location;
use rustpython_parser::location::Location;
use rustpython_parser::token::Tok;
use std::error::Error; use std::error::Error;
use std::fmt; use std::fmt;
#[derive(Debug)] #[derive(Debug)]
pub struct CompileError { pub struct CompileError {
pub statement: Option<String>,
pub error: CompileErrorType, pub error: CompileErrorType,
pub location: Location, pub location: Location,
pub source_path: String, pub source_path: String,
} }
impl CompileError {
pub fn from_parse_error(parse_error: ParseError, source_path: String) -> Self {
Self {
statement: None,
error: CompileErrorType::Parse(parse_error.error),
location: parse_error.location,
source_path,
}
}
pub fn update_statement_info(&mut self, statement: String) {
self.statement = Some(statement);
}
}
#[derive(Debug)] #[derive(Debug)]
#[non_exhaustive]
pub enum CompileErrorType { pub enum CompileErrorType {
/// Invalid assignment, cannot store value in target. /// Invalid assignment, cannot store value in target.
Assign(&'static str), Assign(&'static str),
@ -36,8 +19,6 @@ pub enum CompileErrorType {
Delete(&'static str), Delete(&'static str),
/// Expected an expression got a statement /// Expected an expression got a statement
ExpectExpr, ExpectExpr,
/// Parser error
Parse(ParseErrorType),
SyntaxError(String), SyntaxError(String),
/// Multiple `*` detected /// Multiple `*` detected
MultipleStarArgs, MultipleStarArgs,
@ -57,74 +38,43 @@ pub enum CompileErrorType {
InvalidFutureFeature(String), InvalidFutureFeature(String),
} }
impl CompileError { impl fmt::Display for CompileErrorType {
pub fn is_indentation_error(&self) -> bool { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
if let CompileErrorType::Parse(parse) = &self.error { match self {
match parse { CompileErrorType::Assign(target) => write!(f, "can't assign to {}", target),
ParseErrorType::Lexical(LexicalErrorType::IndentationError) => true, CompileErrorType::Delete(target) => write!(f, "can't delete {}", target),
ParseErrorType::UnrecognizedToken(token, expected) => { CompileErrorType::ExpectExpr => write!(f, "Expecting expression, got statement"),
*token == Tok::Indent || expected.clone() == Some("Indent".to_owned()) CompileErrorType::SyntaxError(err) => write!(f, "{}", err.as_str()),
CompileErrorType::MultipleStarArgs => {
write!(f, "two starred expressions in assignment")
}
CompileErrorType::InvalidStarExpr => write!(f, "can't use starred expression here"),
CompileErrorType::InvalidBreak => write!(f, "'break' outside loop"),
CompileErrorType::InvalidContinue => write!(f, "'continue' outside loop"),
CompileErrorType::InvalidReturn => write!(f, "'return' outside function"),
CompileErrorType::InvalidYield => write!(f, "'yield' outside function"),
CompileErrorType::InvalidYieldFrom => write!(f, "'yield from' outside function"),
CompileErrorType::InvalidAwait => write!(f, "'await' outside async function"),
CompileErrorType::AsyncYieldFrom => write!(f, "'yield from' inside async function"),
CompileErrorType::AsyncReturnValue => {
write!(f, "'return' with value inside async generator")
}
CompileErrorType::InvalidFuturePlacement => write!(
f,
"from __future__ imports must occur at the beginning of the file"
),
CompileErrorType::InvalidFutureFeature(feat) => {
write!(f, "future feature {} is not defined", feat)
} }
_ => false,
} }
} else {
false
} }
} }
pub fn is_tab_error(&self) -> bool { impl Error for CompileErrorType {}
if let CompileErrorType::Parse(parse) = &self.error {
if let ParseErrorType::Lexical(lex) = parse {
if let LexicalErrorType::TabError = lex {
return true;
}
}
}
false
}
}
impl fmt::Display for CompileError { impl fmt::Display for CompileError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let error_desc = match &self.error { write!(f, "{} at {}", self.error, self.location)
CompileErrorType::Assign(target) => format!("can't assign to {}", target),
CompileErrorType::Delete(target) => format!("can't delete {}", target),
CompileErrorType::ExpectExpr => "Expecting expression, got statement".to_owned(),
CompileErrorType::Parse(err) => err.to_string(),
CompileErrorType::SyntaxError(err) => err.to_string(),
CompileErrorType::MultipleStarArgs => {
"two starred expressions in assignment".to_owned()
}
CompileErrorType::InvalidStarExpr => "can't use starred expression here".to_owned(),
CompileErrorType::InvalidBreak => "'break' outside loop".to_owned(),
CompileErrorType::InvalidContinue => "'continue' outside loop".to_owned(),
CompileErrorType::InvalidReturn => "'return' outside function".to_owned(),
CompileErrorType::InvalidYield => "'yield' outside function".to_owned(),
CompileErrorType::InvalidYieldFrom => "'yield from' outside function".to_owned(),
CompileErrorType::InvalidAwait => "'await' outside async function".to_owned(),
CompileErrorType::AsyncYieldFrom => "'yield from' inside async function".to_owned(),
CompileErrorType::AsyncReturnValue => {
"'return' with value inside async generator".to_owned()
}
CompileErrorType::InvalidFuturePlacement => {
"from __future__ imports must occur at the beginning of the file".to_owned()
}
CompileErrorType::InvalidFutureFeature(feat) => {
format!("future feature {} is not defined", feat)
}
};
if let Some(statement) = &self.statement {
if self.location.column() > 0 {
if let Some(line) = statement.lines().nth(self.location.row() - 1) {
// visualize the error, when location and statement are provided
return write!(f, "{}", self.location.visualize(line, &error_desc));
}
}
}
// print line number
write!(f, "{} at {}", error_desc, self.location)
} }
} }

11
src/mode.rs
View file

@ -1,5 +1,3 @@
use rustpython_parser::parser;
#[derive(Clone, Copy)] #[derive(Clone, Copy)]
pub enum Mode { pub enum Mode {
Exec, Exec,
@ -19,15 +17,6 @@ impl std::str::FromStr for Mode {
} }
} }
impl Mode {
pub fn to_parser_mode(self) -> parser::Mode {
match self {
Mode::Exec | Mode::Single => parser::Mode::Program,
Mode::Eval => parser::Mode::Statement,
}
}
}
#[derive(Debug)] #[derive(Debug)]
pub struct ModeParseError { pub struct ModeParseError {
_priv: (), _priv: (),

206
src/symboltable.rs
View file

@ -9,12 +9,11 @@ Inspirational file: https://github.com/python/cpython/blob/master/Python/symtabl
use crate::error::{CompileError, CompileErrorType}; use crate::error::{CompileError, CompileErrorType};
use indexmap::map::IndexMap; use indexmap::map::IndexMap;
use rustpython_parser::ast; use rustpython_ast::{self as ast, Location};
use rustpython_parser::location::Location;
use std::fmt; use std::fmt;
pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolTable, SymbolTableError> { pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolTable, SymbolTableError> {
let mut builder: SymbolTableBuilder = Default::default(); let mut builder = SymbolTableBuilder::default();
builder.prepare(); builder.prepare();
builder.scan_program(program)?; builder.scan_program(program)?;
builder.finish() builder.finish()
@ -23,7 +22,7 @@ pub fn make_symbol_table(program: &ast::Program) -> Result<SymbolTable, SymbolTa
pub fn statements_to_symbol_table( pub fn statements_to_symbol_table(
statements: &[ast::Statement], statements: &[ast::Statement],
) -> Result<SymbolTable, SymbolTableError> { ) -> Result<SymbolTable, SymbolTableError> {
let mut builder: SymbolTableBuilder = Default::default(); let mut builder = SymbolTableBuilder::default();
builder.prepare(); builder.prepare();
builder.scan_statements(statements)?; builder.scan_statements(statements)?;
builder.finish() builder.finish()
@ -59,7 +58,7 @@ impl SymbolTable {
typ, typ,
line_number, line_number,
is_nested, is_nested,
symbols: Default::default(), symbols: IndexMap::new(),
sub_tables: vec![], sub_tables: vec![],
} }
} }
@ -150,12 +149,11 @@ pub struct SymbolTableError {
location: Location, location: Location,
} }
impl CompileError { impl SymbolTableError {
pub fn from_symbol_table_error(error: SymbolTableError, source_path: String) -> Self { pub fn into_compile_error(self, source_path: String) -> CompileError {
CompileError { CompileError {
statement: None, error: CompileErrorType::SyntaxError(self.error),
error: CompileErrorType::SyntaxError(error.error), location: self.location,
location: error.location,
source_path, source_path,
} }
} }
@ -238,7 +236,8 @@ impl<'a> SymbolTableAnalyzer<'a> {
if scope_depth < 2 || !self.found_in_outer_scope(symbol) { if scope_depth < 2 || !self.found_in_outer_scope(symbol) {
return Err(SymbolTableError { return Err(SymbolTableError {
error: format!("no binding for nonlocal '{}' found", symbol.name), error: format!("no binding for nonlocal '{}' found", symbol.name),
location: Default::default(), // TODO: accurate location info, somehow
location: Location::default(),
}); });
} }
} else { } else {
@ -247,7 +246,8 @@ impl<'a> SymbolTableAnalyzer<'a> {
"nonlocal {} defined at place without an enclosing scope", "nonlocal {} defined at place without an enclosing scope",
symbol.name symbol.name
), ),
location: Default::default(), // TODO: accurate location info, somehow
location: Location::default(),
}); });
} }
} }
@ -312,7 +312,8 @@ impl<'a> SymbolTableAnalyzer<'a> {
"assignment expression cannot rebind comprehension iteration variable {}", "assignment expression cannot rebind comprehension iteration variable {}",
symbol.name symbol.name
), ),
location: Default::default(), // TODO: accurate location info, somehow
location: Location::default(),
}); });
} }
@ -324,7 +325,8 @@ impl<'a> SymbolTableAnalyzer<'a> {
// named expressions are forbidden in comprehensions on class scope // named expressions are forbidden in comprehensions on class scope
return Err(SymbolTableError { return Err(SymbolTableError {
error: "assignment expression within a comprehension cannot be used in a class body".to_string(), error: "assignment expression within a comprehension cannot be used in a class body".to_string(),
location: Default::default(), // TODO: accurate location info, somehow
location: Location::default(),
} ); } );
} }
SymbolTableType::Function => { SymbolTableType::Function => {
@ -356,7 +358,8 @@ impl<'a> SymbolTableAnalyzer<'a> {
if parent_symbol.is_iter { if parent_symbol.is_iter {
return Err(SymbolTableError { return Err(SymbolTableError {
error: format!("assignment expression cannot rebind comprehension iteration variable {}", symbol.name), error: format!("assignment expression cannot rebind comprehension iteration variable {}", symbol.name),
location: Default::default(), // TODO: accurate location info, somehow
location: Location::default(),
}); });
} }
@ -405,6 +408,7 @@ struct SymbolTableBuilder {
/// was used. /// was used.
/// In cpython this is stored in the AST, but I think this /// In cpython this is stored in the AST, but I think this
/// is not logical, since it is not context free. /// is not logical, since it is not context free.
#[derive(Copy, Clone)]
enum ExpressionContext { enum ExpressionContext {
Load, Load,
Store, Store,
@ -466,7 +470,7 @@ impl SymbolTableBuilder {
} else { } else {
SymbolUsage::Parameter SymbolUsage::Parameter
}; };
self.register_name(&parameter.arg, usage) self.register_name(&parameter.arg, usage, parameter.location)
} }
fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult { fn scan_parameters_annotations(&mut self, parameters: &[ast::Parameter]) -> SymbolTableResult {
@ -478,22 +482,23 @@ impl SymbolTableBuilder {
fn scan_parameter_annotation(&mut self, parameter: &ast::Parameter) -> SymbolTableResult { fn scan_parameter_annotation(&mut self, parameter: &ast::Parameter) -> SymbolTableResult {
if let Some(annotation) = &parameter.annotation { if let Some(annotation) = &parameter.annotation {
self.scan_expression(&annotation, &ExpressionContext::Load)?; self.scan_expression(&annotation, ExpressionContext::Load)?;
} }
Ok(()) Ok(())
} }
fn scan_statement(&mut self, statement: &ast::Statement) -> SymbolTableResult { fn scan_statement(&mut self, statement: &ast::Statement) -> SymbolTableResult {
use ast::StatementType::*; use ast::StatementType::*;
let location = statement.location;
match &statement.node { match &statement.node {
Global { names } => { Global { names } => {
for name in names { for name in names {
self.register_name(name, SymbolUsage::Global)?; self.register_name(name, SymbolUsage::Global, location)?;
} }
} }
Nonlocal { names } => { Nonlocal { names } => {
for name in names { for name in names {
self.register_name(name, SymbolUsage::Nonlocal)?; self.register_name(name, SymbolUsage::Nonlocal, location)?;
} }
} }
FunctionDef { FunctionDef {
@ -504,12 +509,12 @@ impl SymbolTableBuilder {
returns, returns,
.. ..
} => { } => {
self.scan_expressions(decorator_list, &ExpressionContext::Load)?; self.scan_expressions(decorator_list, ExpressionContext::Load)?;
self.register_name(name, SymbolUsage::Assigned)?; self.register_name(name, SymbolUsage::Assigned, location)?;
if let Some(expression) = returns { if let Some(expression) = returns {
self.scan_expression(expression, &ExpressionContext::Load)?; self.scan_expression(expression, ExpressionContext::Load)?;
} }
self.enter_function(name, args, statement.location.row())?; self.enter_function(name, args, location.row())?;
self.scan_statements(body)?; self.scan_statements(body)?;
self.leave_scope(); self.leave_scope();
} }
@ -520,24 +525,24 @@ impl SymbolTableBuilder {
keywords, keywords,
decorator_list, decorator_list,
} => { } => {
self.enter_scope(name, SymbolTableType::Class, statement.location.row()); self.enter_scope(name, SymbolTableType::Class, location.row());
self.register_name("__module__", SymbolUsage::Assigned)?; self.register_name("__module__", SymbolUsage::Assigned, location)?;
self.register_name("__qualname__", SymbolUsage::Assigned)?; self.register_name("__qualname__", SymbolUsage::Assigned, location)?;
self.register_name("__doc__", SymbolUsage::Assigned)?; self.register_name("__doc__", SymbolUsage::Assigned, location)?;
self.scan_statements(body)?; self.scan_statements(body)?;
self.leave_scope(); self.leave_scope();
self.scan_expressions(bases, &ExpressionContext::Load)?; self.scan_expressions(bases, ExpressionContext::Load)?;
for keyword in keywords { for keyword in keywords {
self.scan_expression(&keyword.value, &ExpressionContext::Load)?; self.scan_expression(&keyword.value, ExpressionContext::Load)?;
} }
self.scan_expressions(decorator_list, &ExpressionContext::Load)?; self.scan_expressions(decorator_list, ExpressionContext::Load)?;
self.register_name(name, SymbolUsage::Assigned)?; self.register_name(name, SymbolUsage::Assigned, location)?;
} }
Expression { expression } => { Expression { expression } => {
self.scan_expression(expression, &ExpressionContext::Load)? self.scan_expression(expression, ExpressionContext::Load)?
} }
If { test, body, orelse } => { If { test, body, orelse } => {
self.scan_expression(test, &ExpressionContext::Load)?; self.scan_expression(test, ExpressionContext::Load)?;
self.scan_statements(body)?; self.scan_statements(body)?;
if let Some(code) = orelse { if let Some(code) = orelse {
self.scan_statements(code)?; self.scan_statements(code)?;
@ -550,15 +555,15 @@ impl SymbolTableBuilder {
orelse, orelse,
.. ..
} => { } => {
self.scan_expression(target, &ExpressionContext::Store)?; self.scan_expression(target, ExpressionContext::Store)?;
self.scan_expression(iter, &ExpressionContext::Load)?; self.scan_expression(iter, ExpressionContext::Load)?;
self.scan_statements(body)?; self.scan_statements(body)?;
if let Some(code) = orelse { if let Some(code) = orelse {
self.scan_statements(code)?; self.scan_statements(code)?;
} }
} }
While { test, body, orelse } => { While { test, body, orelse } => {
self.scan_expression(test, &ExpressionContext::Load)?; self.scan_expression(test, ExpressionContext::Load)?;
self.scan_statements(body)?; self.scan_statements(body)?;
if let Some(code) = orelse { if let Some(code) = orelse {
self.scan_statements(code)?; self.scan_statements(code)?;
@ -571,37 +576,38 @@ impl SymbolTableBuilder {
for name in names { for name in names {
if let Some(alias) = &name.alias { if let Some(alias) = &name.alias {
// `import mymodule as myalias` // `import mymodule as myalias`
self.register_name(alias, SymbolUsage::Imported)?; self.register_name(alias, SymbolUsage::Imported, location)?;
} else { } else {
// `import module` // `import module`
self.register_name( self.register_name(
name.symbol.split('.').next().unwrap(), name.symbol.split('.').next().unwrap(),
SymbolUsage::Imported, SymbolUsage::Imported,
location,
)?; )?;
} }
} }
} }
Return { value } => { Return { value } => {
if let Some(expression) = value { if let Some(expression) = value {
self.scan_expression(expression, &ExpressionContext::Load)?; self.scan_expression(expression, ExpressionContext::Load)?;
} }
} }
Assert { test, msg } => { Assert { test, msg } => {
self.scan_expression(test, &ExpressionContext::Load)?; self.scan_expression(test, ExpressionContext::Load)?;
if let Some(expression) = msg { if let Some(expression) = msg {
self.scan_expression(expression, &ExpressionContext::Load)?; self.scan_expression(expression, ExpressionContext::Load)?;
} }
} }
Delete { targets } => { Delete { targets } => {
self.scan_expressions(targets, &ExpressionContext::Delete)?; self.scan_expressions(targets, ExpressionContext::Delete)?;
} }
Assign { targets, value } => { Assign { targets, value } => {
self.scan_expressions(targets, &ExpressionContext::Store)?; self.scan_expressions(targets, ExpressionContext::Store)?;
self.scan_expression(value, &ExpressionContext::Load)?; self.scan_expression(value, ExpressionContext::Load)?;
} }
AugAssign { target, value, .. } => { AugAssign { target, value, .. } => {
self.scan_expression(target, &ExpressionContext::Store)?; self.scan_expression(target, ExpressionContext::Store)?;
self.scan_expression(value, &ExpressionContext::Load)?; self.scan_expression(value, ExpressionContext::Load)?;
} }
AnnAssign { AnnAssign {
target, target,
@ -610,20 +616,20 @@ impl SymbolTableBuilder {
} => { } => {
// https://github.com/python/cpython/blob/master/Python/symtable.c#L1233 // https://github.com/python/cpython/blob/master/Python/symtable.c#L1233
if let ast::ExpressionType::Identifier { ref name } = target.node { if let ast::ExpressionType::Identifier { ref name } = target.node {
self.register_name(name, SymbolUsage::AnnotationAssigned)?; self.register_name(name, SymbolUsage::AnnotationAssigned, location)?;
} else { } else {
self.scan_expression(target, &ExpressionContext::Store)?; self.scan_expression(target, ExpressionContext::Store)?;
} }
self.scan_expression(annotation, &ExpressionContext::Load)?; self.scan_expression(annotation, ExpressionContext::Load)?;
if let Some(value) = value { if let Some(value) = value {
self.scan_expression(value, &ExpressionContext::Load)?; self.scan_expression(value, ExpressionContext::Load)?;
} }
} }
With { items, body, .. } => { With { items, body, .. } => {
for item in items { for item in items {
self.scan_expression(&item.context_expr, &ExpressionContext::Load)?; self.scan_expression(&item.context_expr, ExpressionContext::Load)?;
if let Some(expression) = &item.optional_vars { if let Some(expression) = &item.optional_vars {
self.scan_expression(expression, &ExpressionContext::Store)?; self.scan_expression(expression, ExpressionContext::Store)?;
} }
} }
self.scan_statements(body)?; self.scan_statements(body)?;
@ -637,10 +643,10 @@ impl SymbolTableBuilder {
self.scan_statements(body)?; self.scan_statements(body)?;
for handler in handlers { for handler in handlers {
if let Some(expression) = &handler.typ { if let Some(expression) = &handler.typ {
self.scan_expression(expression, &ExpressionContext::Load)?; self.scan_expression(expression, ExpressionContext::Load)?;
} }
if let Some(name) = &handler.name { if let Some(name) = &handler.name {
self.register_name(name, SymbolUsage::Assigned)?; self.register_name(name, SymbolUsage::Assigned, location)?;
} }
self.scan_statements(&handler.body)?; self.scan_statements(&handler.body)?;
} }
@ -653,10 +659,10 @@ impl SymbolTableBuilder {
} }
Raise { exception, cause } => { Raise { exception, cause } => {
if let Some(expression) = exception { if let Some(expression) = exception {
self.scan_expression(expression, &ExpressionContext::Load)?; self.scan_expression(expression, ExpressionContext::Load)?;
} }
if let Some(expression) = cause { if let Some(expression) = cause {
self.scan_expression(expression, &ExpressionContext::Load)?; self.scan_expression(expression, ExpressionContext::Load)?;
} }
} }
} }
@ -666,7 +672,7 @@ impl SymbolTableBuilder {
fn scan_expressions( fn scan_expressions(
&mut self, &mut self,
expressions: &[ast::Expression], expressions: &[ast::Expression],
context: &ExpressionContext, context: ExpressionContext,
) -> SymbolTableResult { ) -> SymbolTableResult {
for expression in expressions { for expression in expressions {
self.scan_expression(expression, context)?; self.scan_expression(expression, context)?;
@ -677,9 +683,10 @@ impl SymbolTableBuilder {
fn scan_expression( fn scan_expression(
&mut self, &mut self,
expression: &ast::Expression, expression: &ast::Expression,
context: &ExpressionContext, context: ExpressionContext,
) -> SymbolTableResult { ) -> SymbolTableResult {
use ast::ExpressionType::*; use ast::ExpressionType::*;
let location = expression.location;
match &expression.node { match &expression.node {
Binop { a, b, .. } => { Binop { a, b, .. } => {
self.scan_expression(a, context)?; self.scan_expression(a, context)?;
@ -692,11 +699,11 @@ impl SymbolTableBuilder {
self.scan_expressions(vals, context)?; self.scan_expressions(vals, context)?;
} }
Subscript { a, b } => { Subscript { a, b } => {
self.scan_expression(a, &ExpressionContext::Load)?; self.scan_expression(a, ExpressionContext::Load)?;
self.scan_expression(b, &ExpressionContext::Load)?; self.scan_expression(b, ExpressionContext::Load)?;
} }
Attribute { value, .. } => { Attribute { value, .. } => {
self.scan_expression(value, &ExpressionContext::Load)?; self.scan_expression(value, ExpressionContext::Load)?;
} }
Dict { elements } => { Dict { elements } => {
for (key, value) in elements { for (key, value) in elements {
@ -740,41 +747,37 @@ impl SymbolTableBuilder {
ast::ComprehensionKind::Dict { .. } => "dictcomp", ast::ComprehensionKind::Dict { .. } => "dictcomp",
}; };
self.enter_scope( self.enter_scope(scope_name, SymbolTableType::Comprehension, location.row());
scope_name,
SymbolTableType::Comprehension,
expression.location.row(),
);
// Register the passed argument to the generator function as the name ".0" // Register the passed argument to the generator function as the name ".0"
self.register_name(".0", SymbolUsage::Parameter)?; self.register_name(".0", SymbolUsage::Parameter, location)?;
match **kind { match **kind {
ast::ComprehensionKind::GeneratorExpression { ref element } ast::ComprehensionKind::GeneratorExpression { ref element }
| ast::ComprehensionKind::List { ref element } | ast::ComprehensionKind::List { ref element }
| ast::ComprehensionKind::Set { ref element } => { | ast::ComprehensionKind::Set { ref element } => {
self.scan_expression(element, &ExpressionContext::Load)?; self.scan_expression(element, ExpressionContext::Load)?;
} }
ast::ComprehensionKind::Dict { ref key, ref value } => { ast::ComprehensionKind::Dict { ref key, ref value } => {
self.scan_expression(&key, &ExpressionContext::Load)?; self.scan_expression(&key, ExpressionContext::Load)?;
self.scan_expression(&value, &ExpressionContext::Load)?; self.scan_expression(&value, ExpressionContext::Load)?;
} }
} }
let mut is_first_generator = true; let mut is_first_generator = true;
for generator in generators { for generator in generators {
self.scan_expression(&generator.target, &ExpressionContext::Iter)?; self.scan_expression(&generator.target, ExpressionContext::Iter)?;
if is_first_generator { if is_first_generator {
is_first_generator = false; is_first_generator = false;
} else { } else {
self.scan_expression( self.scan_expression(
&generator.iter, &generator.iter,
&ExpressionContext::IterDefinitionExp, ExpressionContext::IterDefinitionExp,
)?; )?;
} }
for if_expr in &generator.ifs { for if_expr in &generator.ifs {
self.scan_expression(if_expr, &ExpressionContext::Load)?; self.scan_expression(if_expr, ExpressionContext::Load)?;
} }
} }
@ -782,25 +785,25 @@ impl SymbolTableBuilder {
// The first iterable is passed as an argument into the created function: // The first iterable is passed as an argument into the created function:
assert!(!generators.is_empty()); assert!(!generators.is_empty());
self.scan_expression(&generators[0].iter, &ExpressionContext::IterDefinitionExp)?; self.scan_expression(&generators[0].iter, ExpressionContext::IterDefinitionExp)?;
} }
Call { Call {
function, function,
args, args,
keywords, keywords,
} => { } => {
match *context { match context {
ExpressionContext::IterDefinitionExp => { ExpressionContext::IterDefinitionExp => {
self.scan_expression(function, &ExpressionContext::IterDefinitionExp)?; self.scan_expression(function, ExpressionContext::IterDefinitionExp)?;
} }
_ => { _ => {
self.scan_expression(function, &ExpressionContext::Load)?; self.scan_expression(function, ExpressionContext::Load)?;
} }
} }
self.scan_expressions(args, &ExpressionContext::Load)?; self.scan_expressions(args, ExpressionContext::Load)?;
for keyword in keywords { for keyword in keywords {
self.scan_expression(&keyword.value, &ExpressionContext::Load)?; self.scan_expression(&keyword.value, ExpressionContext::Load)?;
} }
} }
String { value } => { String { value } => {
@ -810,48 +813,49 @@ impl SymbolTableBuilder {
// Determine the contextual usage of this symbol: // Determine the contextual usage of this symbol:
match context { match context {
ExpressionContext::Delete => { ExpressionContext::Delete => {
self.register_name(name, SymbolUsage::Used)?; self.register_name(name, SymbolUsage::Used, location)?;
} }
ExpressionContext::Load | ExpressionContext::IterDefinitionExp => { ExpressionContext::Load | ExpressionContext::IterDefinitionExp => {
self.register_name(name, SymbolUsage::Used)?; self.register_name(name, SymbolUsage::Used, location)?;
} }
ExpressionContext::Store => { ExpressionContext::Store => {
self.register_name(name, SymbolUsage::Assigned)?; self.register_name(name, SymbolUsage::Assigned, location)?;
} }
ExpressionContext::Iter => { ExpressionContext::Iter => {
self.register_name(name, SymbolUsage::Iter)?; self.register_name(name, SymbolUsage::Iter, location)?;
} }
} }
} }
Lambda { args, body } => { Lambda { args, body } => {
self.enter_function("lambda", args, expression.location.row())?; self.enter_function("lambda", args, expression.location.row())?;
match *context { match context {
ExpressionContext::IterDefinitionExp => { ExpressionContext::IterDefinitionExp => {
self.scan_expression(body, &ExpressionContext::IterDefinitionExp)?; self.scan_expression(body, ExpressionContext::IterDefinitionExp)?;
} }
_ => { _ => {
self.scan_expression(body, &ExpressionContext::Load)?; self.scan_expression(body, ExpressionContext::Load)?;
} }
} }
self.leave_scope(); self.leave_scope();
} }
IfExpression { test, body, orelse } => { IfExpression { test, body, orelse } => {
self.scan_expression(test, &ExpressionContext::Load)?; self.scan_expression(test, ExpressionContext::Load)?;
self.scan_expression(body, &ExpressionContext::Load)?; self.scan_expression(body, ExpressionContext::Load)?;
self.scan_expression(orelse, &ExpressionContext::Load)?; self.scan_expression(orelse, ExpressionContext::Load)?;
} }
NamedExpression { left, right } => { NamedExpression { left, right } => {
// named expressions are not allowed in the definiton of // named expressions are not allowed in the definiton of
// comprehension iterator definitions // comprehension iterator definitions
if let ExpressionContext::IterDefinitionExp = *context { if let ExpressionContext::IterDefinitionExp = context {
return Err(SymbolTableError { return Err(SymbolTableError {
error: "assignment expression cannot be used in a comprehension iterable expression".to_string(), error: "assignment expression cannot be used in a comprehension iterable expression".to_string(),
location: Default::default(), // TODO: accurate location info, somehow
location: Location::default(),
}); });
} }
self.scan_expression(right, &ExpressionContext::Load)?; self.scan_expression(right, ExpressionContext::Load)?;
// special handling for assigned identifier in named expressions // special handling for assigned identifier in named expressions
// that are used in comprehensions. This required to correctly // that are used in comprehensions. This required to correctly
@ -860,15 +864,15 @@ impl SymbolTableBuilder {
if let Identifier { name } = &left.node { if let Identifier { name } = &left.node {
let table = self.tables.last().unwrap(); let table = self.tables.last().unwrap();
if table.typ == SymbolTableType::Comprehension { if table.typ == SymbolTableType::Comprehension {
self.register_name(name, SymbolUsage::AssignedNamedExprInCompr)?; self.register_name(name, SymbolUsage::AssignedNamedExprInCompr, location)?;
} else { } else {
// omit one recursion. When the handling of an store changes for // omit one recursion. When the handling of an store changes for
// Identifiers this needs adapted - more forward safe would be // Identifiers this needs adapted - more forward safe would be
// calling scan_expression directly. // calling scan_expression directly.
self.register_name(name, SymbolUsage::Assigned)?; self.register_name(name, SymbolUsage::Assigned, location)?;
} }
} else { } else {
self.scan_expression(left, &ExpressionContext::Store)?; self.scan_expression(left, ExpressionContext::Store)?;
} }
} }
} }
@ -882,10 +886,10 @@ impl SymbolTableBuilder {
line_number: usize, line_number: usize,
) -> SymbolTableResult { ) -> SymbolTableResult {
// Evaluate eventual default parameters: // Evaluate eventual default parameters:
self.scan_expressions(&args.defaults, &ExpressionContext::Load)?; self.scan_expressions(&args.defaults, ExpressionContext::Load)?;
for kw_default in &args.kw_defaults { for kw_default in &args.kw_defaults {
if let Some(expression) = kw_default { if let Some(expression) = kw_default {
self.scan_expression(&expression, &ExpressionContext::Load)?; self.scan_expression(&expression, ExpressionContext::Load)?;
} }
} }
@ -917,7 +921,7 @@ impl SymbolTableBuilder {
match group { match group {
ast::StringGroup::Constant { .. } => {} ast::StringGroup::Constant { .. } => {}
ast::StringGroup::FormattedValue { value, spec, .. } => { ast::StringGroup::FormattedValue { value, spec, .. } => {
self.scan_expression(value, &ExpressionContext::Load)?; self.scan_expression(value, ExpressionContext::Load)?;
if let Some(spec) = spec { if let Some(spec) = spec {
self.scan_string_group(spec)?; self.scan_string_group(spec)?;
} }
@ -931,10 +935,14 @@ impl SymbolTableBuilder {
Ok(()) Ok(())
} }
fn register_name(&mut self, name: &str, role: SymbolUsage) -> SymbolTableResult { fn register_name(
&mut self,
name: &str,
role: SymbolUsage,
location: Location,
) -> SymbolTableResult {
let scope_depth = self.tables.len(); let scope_depth = self.tables.len();
let table = self.tables.last_mut().unwrap(); let table = self.tables.last_mut().unwrap();
let location = Default::default();
// Some checks for the symbol that present on this scope level: // Some checks for the symbol that present on this scope level:
if let Some(symbol) = table.symbols.get(name) { if let Some(symbol) = table.symbols.get(name) {