mirror of
https://github.com/astral-sh/ruff.git
synced 2025-08-04 02:38:25 +00:00
Refactor semantic syntax error scope handling (#17314)
## Summary Based on the discussion in https://github.com/astral-sh/ruff/pull/17298#discussion_r2033975460, we decided to move the scope handling out of the `SemanticSyntaxChecker` and into the `SemanticSyntaxContext` trait. This PR implements that refactor by: - Reverting all of the `Checkpoint` and `in_async_context` code in the `SemanticSyntaxChecker` - Adding four new methods to the `SemanticSyntaxContext` trait - `in_async_context`: matches `SemanticModel::in_async_context` and only detects the nearest enclosing function - `in_sync_comprehension`: uses the new `is_async` tracking on `Generator` scopes to detect any enclosing sync comprehension - `in_module_scope`: reports whether we're at the top-level scope - `in_notebook`: reports whether we're in a Jupyter notebook - In-lining the `TestContext` directly into the `SemanticSyntaxCheckerVisitor` - This allows modifying the context as the visitor traverses the AST, which wasn't possible before One potential question here is "why not add a single method returning a `Scope` or `Scopes` to the context?" The main reason is that the `Scope` type is defined in the `ruff_python_semantic` crate, which is not currently a dependency of the parser. It also doesn't appear to be used in red-knot. So it seemed best to use these more granular methods instead of trying to access `Scope` in `ruff_python_parser` (and red-knot). ## Test Plan Existing parser and linter tests.
This commit is contained in:
Brent Westbrook
GitHub
parent
c87e3ccb2f
commit
144484d46c
7 changed files with 255 additions and 238 deletions
|
|
@ -7,9 +7,9 @@ use std::path::Path;
|
|||
use ruff_annotate_snippets::{Level, Renderer, Snippet};
|
||||
use ruff_python_ast::visitor::source_order::{walk_module, SourceOrderVisitor, TraversalSignal};
|
||||
use ruff_python_ast::visitor::Visitor;
|
||||
use ruff_python_ast::{AnyNodeRef, Mod, PythonVersion};
|
||||
use ruff_python_ast::{self as ast, AnyNodeRef, Mod, PythonVersion};
|
||||
use ruff_python_parser::semantic_errors::{
|
||||
SemanticSyntaxCheckerVisitor, SemanticSyntaxContext, SemanticSyntaxError,
|
||||
SemanticSyntaxChecker, SemanticSyntaxContext, SemanticSyntaxError,
|
||||
};
|
||||
use ruff_python_parser::{parse_unchecked, Mode, ParseErrorType, ParseOptions, Token};
|
||||
use ruff_source_file::{LineIndex, OneIndexed, SourceCode};
|
||||
|
|
@ -88,15 +88,14 @@ fn test_valid_syntax(input_path: &Path) {
|
|||
|
||||
let parsed = parsed.try_into_module().expect("Parsed with Mode::Module");
|
||||
|
||||
let mut visitor = SemanticSyntaxCheckerVisitor::new(
|
||||
TestContext::new(&source).with_python_version(options.target_version()),
|
||||
);
|
||||
let mut visitor =
|
||||
SemanticSyntaxCheckerVisitor::new(&source).with_python_version(options.target_version());
|
||||
|
||||
for stmt in parsed.suite() {
|
||||
visitor.visit_stmt(stmt);
|
||||
}
|
||||
|
||||
let semantic_syntax_errors = visitor.into_context().diagnostics.into_inner();
|
||||
let semantic_syntax_errors = visitor.into_diagnostics();
|
||||
|
||||
if !semantic_syntax_errors.is_empty() {
|
||||
let mut message = "Expected no semantic syntax errors for a valid program:\n".to_string();
|
||||
|
|
@ -184,15 +183,14 @@ fn test_invalid_syntax(input_path: &Path) {
|
|||
|
||||
let parsed = parsed.try_into_module().expect("Parsed with Mode::Module");
|
||||
|
||||
let mut visitor = SemanticSyntaxCheckerVisitor::new(
|
||||
TestContext::new(&source).with_python_version(options.target_version()),
|
||||
);
|
||||
let mut visitor =
|
||||
SemanticSyntaxCheckerVisitor::new(&source).with_python_version(options.target_version());
|
||||
|
||||
for stmt in parsed.suite() {
|
||||
visitor.visit_stmt(stmt);
|
||||
}
|
||||
|
||||
let semantic_syntax_errors = visitor.into_context().diagnostics.into_inner();
|
||||
let semantic_syntax_errors = visitor.into_diagnostics();
|
||||
|
||||
assert!(
|
||||
parsed.has_syntax_errors() || !semantic_syntax_errors.is_empty(),
|
||||
|
|
@ -462,19 +460,28 @@ impl<'ast> SourceOrderVisitor<'ast> for ValidateAstVisitor<'ast> {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TestContext<'a> {
|
||||
enum Scope {
|
||||
Module,
|
||||
Function { is_async: bool },
|
||||
Comprehension { is_async: bool },
|
||||
}
|
||||
|
||||
struct SemanticSyntaxCheckerVisitor<'a> {
|
||||
checker: SemanticSyntaxChecker,
|
||||
diagnostics: RefCell<Vec<SemanticSyntaxError>>,
|
||||
python_version: PythonVersion,
|
||||
source: &'a str,
|
||||
scopes: Vec<Scope>,
|
||||
}
|
||||
|
||||
impl<'a> TestContext<'a> {
|
||||
impl<'a> SemanticSyntaxCheckerVisitor<'a> {
|
||||
fn new(source: &'a str) -> Self {
|
||||
Self {
|
||||
checker: SemanticSyntaxChecker::new(),
|
||||
diagnostics: RefCell::default(),
|
||||
python_version: PythonVersion::default(),
|
||||
source,
|
||||
scopes: vec![Scope::Module],
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -483,9 +490,19 @@ impl<'a> TestContext<'a> {
|
|||
self.python_version = python_version;
|
||||
self
|
||||
}
|
||||
|
||||
fn into_diagnostics(self) -> Vec<SemanticSyntaxError> {
|
||||
self.diagnostics.into_inner()
|
||||
}
|
||||
|
||||
fn with_semantic_checker(&mut self, f: impl FnOnce(&mut SemanticSyntaxChecker, &Self)) {
|
||||
let mut checker = std::mem::take(&mut self.checker);
|
||||
f(&mut checker, self);
|
||||
self.checker = checker;
|
||||
}
|
||||
}
|
||||
|
||||
impl SemanticSyntaxContext for TestContext<'_> {
|
||||
impl SemanticSyntaxContext for SemanticSyntaxCheckerVisitor<'_> {
|
||||
fn seen_docstring_boundary(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
|
@ -509,4 +526,73 @@ impl SemanticSyntaxContext for TestContext<'_> {
|
|||
fn global(&self, _name: &str) -> Option<TextRange> {
|
||||
None
|
||||
}
|
||||
|
||||
fn in_async_context(&self) -> bool {
|
||||
for scope in &self.scopes {
|
||||
if let Scope::Function { is_async } = scope {
|
||||
return *is_async;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn in_sync_comprehension(&self) -> bool {
|
||||
for scope in &self.scopes {
|
||||
if let Scope::Comprehension { is_async: false } = scope {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn in_module_scope(&self) -> bool {
|
||||
self.scopes
|
||||
.last()
|
||||
.is_some_and(|scope| matches!(scope, Scope::Module))
|
||||
}
|
||||
|
||||
fn in_notebook(&self) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
impl Visitor<'_> for SemanticSyntaxCheckerVisitor<'_> {
|
||||
fn visit_stmt(&mut self, stmt: &ast::Stmt) {
|
||||
self.with_semantic_checker(|semantic, context| semantic.visit_stmt(stmt, context));
|
||||
match stmt {
|
||||
ast::Stmt::FunctionDef(ast::StmtFunctionDef { is_async, .. }) => {
|
||||
self.scopes.push(Scope::Function {
|
||||
is_async: *is_async,
|
||||
});
|
||||
ast::visitor::walk_stmt(self, stmt);
|
||||
self.scopes.pop().unwrap();
|
||||
}
|
||||
_ => {
|
||||
ast::visitor::walk_stmt(self, stmt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn visit_expr(&mut self, expr: &ast::Expr) {
|
||||
self.with_semantic_checker(|semantic, context| semantic.visit_expr(expr, context));
|
||||
match expr {
|
||||
ast::Expr::Lambda(_) => {
|
||||
self.scopes.push(Scope::Function { is_async: false });
|
||||
ast::visitor::walk_expr(self, expr);
|
||||
self.scopes.pop().unwrap();
|
||||
}
|
||||
ast::Expr::ListComp(ast::ExprListComp { generators, .. })
|
||||
| ast::Expr::SetComp(ast::ExprSetComp { generators, .. })
|
||||
| ast::Expr::DictComp(ast::ExprDictComp { generators, .. }) => {
|
||||
self.scopes.push(Scope::Comprehension {
|
||||
is_async: generators.iter().any(|gen| gen.is_async),
|
||||
});
|
||||
ast::visitor::walk_expr(self, expr);
|
||||
self.scopes.pop().unwrap();
|
||||
}
|
||||
_ => {
|
||||
ast::visitor::walk_expr(self, expr);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,20 +7,20 @@ input_file: crates/ruff_python_parser/resources/inline/ok/nested_async_comprehen
|
|||
```
|
||||
Module(
|
||||
ModModule {
|
||||
range: 0..259,
|
||||
range: 0..181,
|
||||
body: [
|
||||
FunctionDef(
|
||||
StmtFunctionDef {
|
||||
range: 87..159,
|
||||
range: 44..116,
|
||||
is_async: true,
|
||||
decorator_list: [],
|
||||
name: Identifier {
|
||||
id: Name("f"),
|
||||
range: 97..98,
|
||||
range: 54..55,
|
||||
},
|
||||
type_params: None,
|
||||
parameters: Parameters {
|
||||
range: 98..100,
|
||||
range: 55..57,
|
||||
posonlyargs: [],
|
||||
args: [],
|
||||
vararg: None,
|
||||
|
|
@ -31,43 +31,43 @@ Module(
|
|||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
range: 106..127,
|
||||
range: 63..84,
|
||||
value: ListComp(
|
||||
ExprListComp {
|
||||
range: 106..127,
|
||||
range: 63..84,
|
||||
elt: Name(
|
||||
ExprName {
|
||||
range: 107..108,
|
||||
range: 64..65,
|
||||
id: Name("_"),
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
generators: [
|
||||
Comprehension {
|
||||
range: 109..126,
|
||||
range: 66..83,
|
||||
target: Name(
|
||||
ExprName {
|
||||
range: 113..114,
|
||||
range: 70..71,
|
||||
id: Name("n"),
|
||||
ctx: Store,
|
||||
},
|
||||
),
|
||||
iter: Call(
|
||||
ExprCall {
|
||||
range: 118..126,
|
||||
range: 75..83,
|
||||
func: Name(
|
||||
ExprName {
|
||||
range: 118..123,
|
||||
range: 75..80,
|
||||
id: Name("range"),
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
arguments: Arguments {
|
||||
range: 123..126,
|
||||
range: 80..83,
|
||||
args: [
|
||||
NumberLiteral(
|
||||
ExprNumberLiteral {
|
||||
range: 124..125,
|
||||
range: 81..82,
|
||||
value: Int(
|
||||
3,
|
||||
),
|
||||
|
|
@ -88,43 +88,43 @@ Module(
|
|||
),
|
||||
Expr(
|
||||
StmtExpr {
|
||||
range: 132..159,
|
||||
range: 89..116,
|
||||
value: ListComp(
|
||||
ExprListComp {
|
||||
range: 132..159,
|
||||
range: 89..116,
|
||||
elt: Name(
|
||||
ExprName {
|
||||
range: 133..134,
|
||||
range: 90..91,
|
||||
id: Name("_"),
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
generators: [
|
||||
Comprehension {
|
||||
range: 135..158,
|
||||
range: 92..115,
|
||||
target: Name(
|
||||
ExprName {
|
||||
range: 145..146,
|
||||
range: 102..103,
|
||||
id: Name("n"),
|
||||
ctx: Store,
|
||||
},
|
||||
),
|
||||
iter: Call(
|
||||
ExprCall {
|
||||
range: 150..158,
|
||||
range: 107..115,
|
||||
func: Name(
|
||||
ExprName {
|
||||
range: 150..155,
|
||||
range: 107..112,
|
||||
id: Name("range"),
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
arguments: Arguments {
|
||||
range: 155..158,
|
||||
range: 112..115,
|
||||
args: [
|
||||
NumberLiteral(
|
||||
ExprNumberLiteral {
|
||||
range: 156..157,
|
||||
range: 113..114,
|
||||
value: Int(
|
||||
3,
|
||||
),
|
||||
|
|
@ -148,16 +148,16 @@ Module(
|
|||
),
|
||||
FunctionDef(
|
||||
StmtFunctionDef {
|
||||
range: 195..258,
|
||||
range: 117..180,
|
||||
is_async: true,
|
||||
decorator_list: [],
|
||||
name: Identifier {
|
||||
id: Name("f"),
|
||||
range: 205..206,
|
||||
range: 127..128,
|
||||
},
|
||||
type_params: None,
|
||||
parameters: Parameters {
|
||||
range: 206..208,
|
||||
range: 128..130,
|
||||
posonlyargs: [],
|
||||
args: [],
|
||||
vararg: None,
|
||||
|
|
@ -168,16 +168,16 @@ Module(
|
|||
body: [
|
||||
FunctionDef(
|
||||
StmtFunctionDef {
|
||||
range: 214..226,
|
||||
range: 136..148,
|
||||
is_async: false,
|
||||
decorator_list: [],
|
||||
name: Identifier {
|
||||
id: Name("g"),
|
||||
range: 218..219,
|
||||
range: 140..141,
|
||||
},
|
||||
type_params: None,
|
||||
parameters: Parameters {
|
||||
range: 219..221,
|
||||
range: 141..143,
|
||||
posonlyargs: [],
|
||||
args: [],
|
||||
vararg: None,
|
||||
|
|
@ -188,10 +188,10 @@ Module(
|
|||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
range: 223..226,
|
||||
range: 145..148,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
range: 223..226,
|
||||
range: 145..148,
|
||||
},
|
||||
),
|
||||
},
|
||||
|
|
@ -201,43 +201,43 @@ Module(
|
|||
),
|
||||
Expr(
|
||||
StmtExpr {
|
||||
range: 231..258,
|
||||
range: 153..180,
|
||||
value: ListComp(
|
||||
ExprListComp {
|
||||
range: 231..258,
|
||||
range: 153..180,
|
||||
elt: Name(
|
||||
ExprName {
|
||||
range: 232..233,
|
||||
range: 154..155,
|
||||
id: Name("_"),
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
generators: [
|
||||
Comprehension {
|
||||
range: 234..257,
|
||||
range: 156..179,
|
||||
target: Name(
|
||||
ExprName {
|
||||
range: 244..245,
|
||||
range: 166..167,
|
||||
id: Name("n"),
|
||||
ctx: Store,
|
||||
},
|
||||
),
|
||||
iter: Call(
|
||||
ExprCall {
|
||||
range: 249..257,
|
||||
range: 171..179,
|
||||
func: Name(
|
||||
ExprName {
|
||||
range: 249..254,
|
||||
range: 171..176,
|
||||
id: Name("range"),
|
||||
ctx: Load,
|
||||
},
|
||||
),
|
||||
arguments: Arguments {
|
||||
range: 254..257,
|
||||
range: 176..179,
|
||||
args: [
|
||||
NumberLiteral(
|
||||
ExprNumberLiteral {
|
||||
range: 255..256,
|
||||
range: 177..178,
|
||||
value: Int(
|
||||
3,
|
||||
),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue