language-servers/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-03 10:23:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

red-knot: Use parse_unchecked to get all parse errors (#11725)

Browse source
This commit is contained in:
Micha Reiser 2024-06-04 08:04:48 +02:00 • committed by GitHub
parent 0c75548146
commit 64165bee43
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 58 additions and 125 deletions
Download patch file Download diff file Expand all files Collapse all files

24
Cargo.lock generated
View file

@ -1789,7 +1789,6 @@ dependencies = [
"rustc-hash",
"smol_str",
"tempfile",
"textwrap",
"tracing",
"tracing-subscriber",
"tracing-tree",
@ -2690,12 +2689,6 @@ version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
[[package]]
name = "smawk"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c"
[[package]]
name = "smol_str"
version = "0.2.2"
@ -2845,17 +2838,6 @@ dependencies = [
"test-case-core",
]
[[package]]
name = "textwrap"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9"
dependencies = [
"smawk",
"unicode-linebreak",
"unicode-width",
]
[[package]]
name = "thiserror"
version = "1.0.61"
@ -3111,12 +3093,6 @@ version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
[[package]]
name = "unicode-linebreak"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f"
[[package]]
name = "unicode-normalization"
version = "0.1.23"

1
crates/red_knot/Cargo.toml
View file

@ -35,7 +35,6 @@ tracing-subscriber = { workspace = true }
tracing-tree = { workspace = true }
[dev-dependencies]
textwrap = { version = "0.16.1" }
tempfile = { workspace = true }
[lints]

13
crates/red_knot/src/lint.rs
View file

@ -5,12 +5,13 @@ use std::time::Duration;
use ruff_python_ast::visitor::Visitor;
use ruff_python_ast::{ModModule, StringLiteral};
use ruff_python_parser::Parsed;
use crate::cache::KeyValueCache;
use crate::db::{LintDb, LintJar, QueryResult};
use crate::files::FileId;
use crate::module::ModuleName;
use crate::parse::{parse, Parsed};
use crate::parse::parse;
use crate::source::{source_text, Source};
use crate::symbols::{
resolve_global_symbol, symbol_table, Definition, GlobalSymbolId, SymbolId, SymbolTable,
@ -40,7 +41,7 @@ pub(crate) fn lint_syntax(db: &dyn LintDb, file_id: FileId) -> QueryResult<Diagn
let parsed = parse(db.upcast(), *file_id)?;
if parsed.errors().is_empty() {
let ast = parsed.ast();
let ast = parsed.syntax();
let mut visitor = SyntaxLintVisitor {
diagnostics,
@ -86,7 +87,7 @@ pub(crate) fn lint_semantic(db: &dyn LintDb, file_id: FileId) -> QueryResult<Dia
let context = SemanticLintContext {
file_id: *file_id,
source,
parsed,
parsed: &parsed,
symbols,
db,
diagnostics: RefCell::new(Vec::new()),
@ -194,7 +195,7 @@ fn lint_bad_overrides(context: &SemanticLintContext) -> QueryResult<()> {
pub struct SemanticLintContext<'a> {
file_id: FileId,
source: Source,
parsed: Parsed,
parsed: &'a Parsed<ModModule>,
symbols: Arc<SymbolTable>,
db: &'a dyn LintDb,
diagnostics: RefCell<Vec<String>>,
@ -209,8 +210,8 @@ impl<'a> SemanticLintContext<'a> {
self.file_id
}
pub fn ast(&self) -> &ModModule {
self.parsed.ast()
pub fn ast(&self) -> &'a ModModule {
self.parsed.syntax()
}
pub fn symbols(&self) -> &SymbolTable {

72
crates/red_knot/src/parse.rs
View file

@ -1,87 +1,33 @@
use std::ops::{Deref, DerefMut};
use std::sync::Arc;
use ruff_python_ast as ast;
use ruff_python_parser::{Mode, ParseError};
use ruff_text_size::{Ranged, TextRange};
use ruff_python_ast::ModModule;
use ruff_python_parser::Parsed;
use crate::cache::KeyValueCache;
use crate::db::{QueryResult, SourceDb};
use crate::files::FileId;
use crate::source::source_text;
#[derive(Debug, Clone, PartialEq)]
pub struct Parsed {
inner: Arc<ParsedInner>,
}
#[derive(Debug, PartialEq)]
struct ParsedInner {
ast: ast::ModModule,
errors: Vec<ParseError>,
}
impl Parsed {
fn new(ast: ast::ModModule, errors: Vec<ParseError>) -> Self {
Self {
inner: Arc::new(ParsedInner { ast, errors }),
}
}
pub(crate) fn from_text(text: &str) -> Self {
let result = ruff_python_parser::parse(text, Mode::Module);
let (module, errors) = match result {
Ok(parsed) => match parsed.into_syntax() {
ast::Mod::Module(module) => (module, vec![]),
ast::Mod::Expression(expression) => (
ast::ModModule {
range: expression.range(),
body: vec![ast::Stmt::Expr(ast::StmtExpr {
range: expression.range(),
value: expression.body,
})],
},
vec![],
),
},
Err(errors) => (
ast::ModModule {
range: TextRange::default(),
body: Vec::new(),
},
vec![errors],
),
};
Parsed::new(module, errors)
}
pub fn ast(&self) -> &ast::ModModule {
&self.inner.ast
}
pub fn errors(&self) -> &[ParseError] {
&self.inner.errors
}
}
#[tracing::instrument(level = "debug", skip(db))]
pub(crate) fn parse(db: &dyn SourceDb, file_id: FileId) -> QueryResult<Parsed> {
pub(crate) fn parse(db: &dyn SourceDb, file_id: FileId) -> QueryResult<Arc<Parsed<ModModule>>> {
let jar = db.jar()?;
jar.parsed.get(&file_id, |file_id| {
let source = source_text(db, *file_id)?;
Ok(Parsed::from_text(source.text()))
Ok(Arc::new(ruff_python_parser::parse_unchecked_source(
source.text(),
source.kind().into(),
)))
})
}
#[derive(Debug, Default)]
pub struct ParsedStorage(KeyValueCache<FileId, Parsed>);
pub struct ParsedStorage(KeyValueCache<FileId, Arc<Parsed<ModModule>>>);
impl Deref for ParsedStorage {
type Target = KeyValueCache<FileId, Parsed>;
type Target = KeyValueCache<FileId, Arc<Parsed<ModModule>>>;
fn deref(&self) -> &Self::Target {
&self.0

10
crates/red_knot/src/source.rs
View file

@ -53,6 +53,16 @@ pub enum SourceKind {
IpyNotebook(Arc<Notebook>),
}
impl<'a> From<&'a SourceKind> for PySourceType {
fn from(value: &'a SourceKind) -> Self {
match value {
SourceKind::Python(_) => PySourceType::Python,
SourceKind::Stub(_) => PySourceType::Stub,
SourceKind::IpyNotebook(_) => PySourceType::Ipynb,
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct Source {
kind: SourceKind,

43
crates/red_knot/src/symbols.rs
View file

@ -28,7 +28,7 @@ pub fn symbol_table(db: &dyn SemanticDb, file_id: FileId) -> QueryResult<Arc<Sym
jar.symbol_tables.get(&file_id, |_| {
let parsed = parse(db.upcast(), file_id)?;
Ok(Arc::from(SymbolTable::from_ast(parsed.ast())))
Ok(Arc::from(SymbolTable::from_ast(parsed.syntax())))
})
}
@ -903,13 +903,16 @@ mod tests {
use crate::symbols::{ScopeKind, SymbolFlags, SymbolTable};
mod from_ast {
use crate::parse::Parsed;
use crate::symbols::{Definition, ScopeKind, SymbolId, SymbolIterator, SymbolTable};
use ruff_python_ast as ast;
use textwrap::dedent;
use ruff_python_ast::ModModule;
use ruff_python_parser::{Mode, Parsed};
fn parse(code: &str) -> Parsed {
Parsed::from_text(&dedent(code))
use crate::symbols::{Definition, ScopeKind, SymbolId, SymbolIterator, SymbolTable};
fn parse(code: &str) -> Parsed<ModModule> {
ruff_python_parser::parse_unchecked(code, Mode::Module)
.try_into_module()
.unwrap()
}
fn names<I>(it: SymbolIterator<I>) -> Vec<&str>
@ -924,14 +927,14 @@ mod tests {
#[test]
fn empty() {
let parsed = parse("");
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()).len(), 0);
}
#[test]
fn simple() {
let parsed = parse("x");
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["x"]);
assert_eq!(
table
@ -944,7 +947,7 @@ mod tests {
#[test]
fn annotation_only() {
let parsed = parse("x: int");
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["int", "x"]);
// TODO record definition
}
@ -952,7 +955,7 @@ mod tests {
#[test]
fn import() {
let parsed = parse("import foo");
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["foo"]);
assert_eq!(
table
@ -965,21 +968,21 @@ mod tests {
#[test]
fn import_sub() {
let parsed = parse("import foo.bar");
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["foo"]);
}
#[test]
fn import_as() {
let parsed = parse("import foo.bar as baz");
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["baz"]);
}
#[test]
fn import_from() {
let parsed = parse("from bar import foo");
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["foo"]);
assert_eq!(
table
@ -999,7 +1002,7 @@ mod tests {
#[test]
fn assign() {
let parsed = parse("x = foo");
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["foo", "x"]);
assert_eq!(
table
@ -1025,7 +1028,7 @@ mod tests {
y = 2
",
);
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["C", "y"]);
let scopes = table.root_child_scope_ids();
assert_eq!(scopes.len(), 1);
@ -1050,7 +1053,7 @@ mod tests {
y = 2
",
);
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["func", "y"]);
let scopes = table.root_child_scope_ids();
assert_eq!(scopes.len(), 1);
@ -1076,7 +1079,7 @@ mod tests {
y = 2
",
);
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["func"]);
let scopes = table.root_child_scope_ids();
assert_eq!(scopes.len(), 2);
@ -1104,7 +1107,7 @@ mod tests {
x = 1
",
);
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["func"]);
let scopes = table.root_child_scope_ids();
assert_eq!(scopes.len(), 1);
@ -1130,7 +1133,7 @@ mod tests {
x = 1
",
);
let table = SymbolTable::from_ast(parsed.ast());
let table = SymbolTable::from_ast(parsed.syntax());
assert_eq!(names(table.root_symbols()), vec!["C"]);
let scopes = table.root_child_scope_ids();
assert_eq!(scopes.len(), 1);
@ -1157,7 +1160,7 @@ mod tests {
#[test]
fn reachability_trivial() {
let parsed = parse("x = 1; x");
let ast = parsed.ast();
let ast = parsed.syntax();
let table = SymbolTable::from_ast(ast);
let x_sym = table
.root_symbol_id_by_name("x")

11
crates/red_knot/src/types/infer.rs
View file

@ -107,7 +107,7 @@ pub fn infer_definition_type(
Ok(ty)
} else {
let parsed = parse(db.upcast(), file_id)?;
let ast = parsed.ast();
let ast = parsed.syntax();
let table = symbol_table(db, file_id)?;
let node = node_key.resolve_unwrap(ast.as_any_node_ref());
@ -127,7 +127,7 @@ pub fn infer_definition_type(
Ok(ty)
} else {
let parsed = parse(db.upcast(), file_id)?;
let ast = parsed.ast();
let ast = parsed.syntax();
let table = symbol_table(db, file_id)?;
let node = node_key
.resolve(ast.as_any_node_ref())
@ -154,14 +154,14 @@ pub fn infer_definition_type(
}
Definition::Assignment(node_key) => {
let parsed = parse(db.upcast(), file_id)?;
let ast = parsed.ast();
let ast = parsed.syntax();
let node = node_key.resolve_unwrap(ast.as_any_node_ref());
// TODO handle unpacking assignment correctly (here and for AnnotatedAssignment case, below)
infer_expr_type(db, file_id, &node.value)
}
Definition::AnnotatedAssignment(node_key) => {
let parsed = parse(db.upcast(), file_id)?;
let ast = parsed.ast();
let ast = parsed.syntax();
let node = node_key.resolve_unwrap(ast.as_any_node_ref());
// TODO actually look at the annotation
let Some(value) = &node.value else {
@ -213,7 +213,6 @@ fn infer_expr_type(db: &dyn SemanticDb, file_id: FileId, expr: &ast::Expr) -> Qu
#[cfg(test)]
mod tests {
use textwrap::dedent;
use crate::db::tests::TestDb;
use crate::db::{HasJar, SemanticJar};
@ -251,7 +250,7 @@ mod tests {
fn write_to_path(case: &TestCase, relpath: &str, contents: &str) -> anyhow::Result<()> {
let path = case.src.path().join(relpath);
std::fs::write(path, dedent(contents))?;
std::fs::write(path, contents)?;
Ok(())
}

9
crates/ruff_python_parser/src/lib.rs
View file

@ -64,7 +64,6 @@
//! [parsing]: https://en.wikipedia.org/wiki/Parsing
//! [lexer]: crate::lexer
use std::cell::OnceCell;
use std::ops::Deref;
pub use crate::error::{FStringErrorType, ParseError, ParseErrorType};
@ -308,7 +307,7 @@ impl Parsed<Mod> {
/// returns [`None`].
///
/// [`Some(Parsed<ModModule>)`]: Some
fn try_into_module(self) -> Option<Parsed<ModModule>> {
pub fn try_into_module(self) -> Option<Parsed<ModModule>> {
match self.syntax {
Mod::Module(module) => Some(Parsed {
syntax: module,
@ -327,7 +326,7 @@ impl Parsed<Mod> {
/// Otherwise, it returns [`None`].
///
/// [`Some(Parsed<ModExpression>)`]: Some
fn try_into_expression(self) -> Option<Parsed<ModExpression>> {
pub fn try_into_expression(self) -> Option<Parsed<ModExpression>> {
match self.syntax {
Mod::Module(_) => None,
Mod::Expression(expression) => Some(Parsed {
@ -370,14 +369,14 @@ pub struct Tokens {
raw: Vec<Token>,
/// Index of the first [`TokenKind::Unknown`] token or the length of the token vector.
first_unknown_or_len: OnceCell<usize>,
first_unknown_or_len: std::sync::OnceLock<usize>,
}
impl Tokens {
pub(crate) fn new(tokens: Vec<Token>) -> Tokens {
Tokens {
raw: tokens,
first_unknown_or_len: OnceCell::new(),
first_unknown_or_len: std::sync::OnceLock::new(),
}
}