From 78806361fd5b23a814d3d16af36b04304e586952 Mon Sep 17 00:00:00 2001 From: Brent Westbrook <36778786+ntBre@users.noreply.github.com> Date: Tue, 25 Feb 2025 23:03:48 -0500 Subject: [PATCH] Start detecting version-related syntax errors in the parser (#16090) ## Summary This PR builds on the changes in #16220 to pass a target Python version to the parser. It also adds the `Parser::unsupported_syntax_errors` field, which collects version-related syntax errors while parsing. These syntax errors are then turned into `Message`s in ruff (in preview mode). This PR only detects one syntax error (`match` statement before Python 3.10), but it has been pretty quick to extend to several other simple errors (see #16308 for example). ## Test Plan The current tests are CLI tests in the linter crate, but these could be supplemented with inline parser tests after #16357. I also tested the display of these syntax errors in VS Code: ![image](https://github.com/user-attachments/assets/062b4441-740e-46c3-887c-a954049ef26e) ![image](https://github.com/user-attachments/assets/101f55b8-146c-4d59-b6b0-922f19bcd0fa) --------- Co-authored-by: Alex Waygood --- crates/ruff/src/cache.rs | 7 +- crates/ruff/tests/lint.rs | 74 +++++++++++++++++ crates/ruff_linter/src/linter.rs | 66 ++++++++++++--- crates/ruff_linter/src/message/mod.rs | 14 +++- crates/ruff_linter/src/rules/pyflakes/mod.rs | 9 ++- crates/ruff_linter/src/test.rs | 14 +++- crates/ruff_python_parser/src/error.rs | 47 ++++++++++- crates/ruff_python_parser/src/lib.rs | 22 ++++- crates/ruff_python_parser/src/parser/mod.rs | 12 +++ .../ruff_python_parser/src/parser/options.rs | 20 ++++- .../src/parser/statement.rs | 15 +++- crates/ruff_server/src/lint.rs | 81 ++++++++++++++++--- crates/ruff_wasm/src/lib.rs | 1 + python/py-fuzzer/fuzz.py | 11 ++- 14 files changed, 356 insertions(+), 37 deletions(-) diff --git a/crates/ruff/src/cache.rs b/crates/ruff/src/cache.rs index 40b495189a..2ae5e82850 100644 --- a/crates/ruff/src/cache.rs +++ b/crates/ruff/src/cache.rs @@ -586,6 +586,7 @@ mod tests { use anyhow::Result; use filetime::{set_file_mtime, FileTime}; use itertools::Itertools; + use ruff_linter::settings::LinterSettings; use test_case::test_case; use ruff_cache::CACHE_DIR_NAME; @@ -593,7 +594,7 @@ mod tests { use ruff_linter::package::PackageRoot; use ruff_linter::settings::flags; use ruff_linter::settings::types::UnsafeFixes; - use ruff_python_ast::PySourceType; + use ruff_python_ast::{PySourceType, PythonVersion}; use ruff_workspace::Settings; use crate::cache::{self, FileCache, FileCacheData, FileCacheKey}; @@ -611,6 +612,10 @@ mod tests { let settings = Settings { cache_dir, + linter: LinterSettings { + unresolved_target_version: PythonVersion::PY310, + ..Default::default() + }, ..Settings::default() }; diff --git a/crates/ruff/tests/lint.rs b/crates/ruff/tests/lint.rs index e3348cb22f..15591314ce 100644 --- a/crates/ruff/tests/lint.rs +++ b/crates/ruff/tests/lint.rs @@ -2627,3 +2627,77 @@ class A(Generic[T]): " ); } + +#[test] +fn match_before_py310() { + // ok on 3.10 + assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME)) + .args(STDIN_BASE_OPTIONS) + .args(["--stdin-filename", "test.py"]) + .arg("--target-version=py310") + .arg("-") + .pass_stdin( + r#" +match 2: + case 1: + print("it's one") +"# + ), + @r" + success: true + exit_code: 0 + ----- stdout ----- + All checks passed! + + ----- stderr ----- + " + ); + + // ok on 3.9 without preview + assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME)) + .args(STDIN_BASE_OPTIONS) + .args(["--stdin-filename", "test.py"]) + .arg("--target-version=py39") + .arg("-") + .pass_stdin( + r#" +match 2: + case 1: + print("it's one") +"# + ), + @r" + success: true + exit_code: 0 + ----- stdout ----- + All checks passed! + + ----- stderr ----- + " + ); + + // syntax error on 3.9 with preview + assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME)) + .args(STDIN_BASE_OPTIONS) + .args(["--stdin-filename", "test.py"]) + .arg("--target-version=py39") + .arg("--preview") + .arg("-") + .pass_stdin( + r#" +match 2: + case 1: + print("it's one") +"# + ), + @r" + success: false + exit_code: 1 + ----- stdout ----- + test.py:2:1: SyntaxError: Cannot use `match` statement on Python 3.9 (syntax was added in Python 3.10) + Found 1 error. + + ----- stderr ----- + " + ); +} diff --git a/crates/ruff_linter/src/linter.rs b/crates/ruff_linter/src/linter.rs index 4179c3b381..be0362f9b1 100644 --- a/crates/ruff_linter/src/linter.rs +++ b/crates/ruff_linter/src/linter.rs @@ -10,10 +10,10 @@ use rustc_hash::FxHashMap; use ruff_diagnostics::Diagnostic; use ruff_notebook::Notebook; -use ruff_python_ast::{ModModule, PySourceType}; +use ruff_python_ast::{ModModule, PySourceType, PythonVersion}; use ruff_python_codegen::Stylist; use ruff_python_index::Indexer; -use ruff_python_parser::{ParseError, Parsed}; +use ruff_python_parser::{ParseError, ParseOptions, Parsed, UnsupportedSyntaxError}; use ruff_source_file::SourceFileBuilder; use ruff_text_size::Ranged; @@ -71,6 +71,7 @@ pub fn check_path( source_kind: &SourceKind, source_type: PySourceType, parsed: &Parsed, + target_version: PythonVersion, ) -> Vec { // Aggregate all diagnostics. let mut diagnostics = vec![]; @@ -104,8 +105,6 @@ pub fn check_path( )); } - let target_version = settings.resolve_target_version(path); - // Run the filesystem-based rules. if settings .rules @@ -335,7 +334,8 @@ pub fn add_noqa_to_path( settings: &LinterSettings, ) -> Result { // Parse once. - let parsed = ruff_python_parser::parse_unchecked_source(source_kind.source_code(), source_type); + let target_version = settings.resolve_target_version(path); + let parsed = parse_unchecked_source(source_kind, source_type, target_version); // Map row and column locations to byte slices (lazily). let locator = Locator::new(source_kind.source_code()); @@ -367,6 +367,7 @@ pub fn add_noqa_to_path( source_kind, source_type, &parsed, + target_version, ); // Add any missing `# noqa` pragmas. @@ -393,7 +394,8 @@ pub fn lint_only( source_type: PySourceType, source: ParseSource, ) -> LinterResult { - let parsed = source.into_parsed(source_kind, source_type); + let target_version = settings.resolve_target_version(path); + let parsed = source.into_parsed(source_kind, source_type, target_version); // Map row and column locations to byte slices (lazily). let locator = Locator::new(source_kind.source_code()); @@ -425,12 +427,20 @@ pub fn lint_only( source_kind, source_type, &parsed, + target_version, ); + let syntax_errors = if settings.preview.is_enabled() { + parsed.unsupported_syntax_errors() + } else { + &[] + }; + LinterResult { messages: diagnostics_to_messages( diagnostics, parsed.errors(), + syntax_errors, path, &locator, &directives, @@ -443,6 +453,7 @@ pub fn lint_only( fn diagnostics_to_messages( diagnostics: Vec, parse_errors: &[ParseError], + unsupported_syntax_errors: &[UnsupportedSyntaxError], path: &Path, locator: &Locator, directives: &Directives, @@ -461,6 +472,9 @@ fn diagnostics_to_messages( parse_errors .iter() .map(|parse_error| Message::from_parse_error(parse_error, locator, file.deref().clone())) + .chain(unsupported_syntax_errors.iter().map(|syntax_error| { + Message::from_unsupported_syntax_error(syntax_error, file.deref().clone()) + })) .chain(diagnostics.into_iter().map(|diagnostic| { let noqa_offset = directives.noqa_line_for.resolve(diagnostic.start()); Message::from_diagnostic(diagnostic, file.deref().clone(), noqa_offset) @@ -491,11 +505,12 @@ pub fn lint_fix<'a>( // Track whether the _initial_ source code is valid syntax. let mut is_valid_syntax = false; + let target_version = settings.resolve_target_version(path); + // Continuously fix until the source code stabilizes. loop { // Parse once. - let parsed = - ruff_python_parser::parse_unchecked_source(transformed.source_code(), source_type); + let parsed = parse_unchecked_source(&transformed, source_type, target_version); // Map row and column locations to byte slices (lazily). let locator = Locator::new(transformed.source_code()); @@ -527,6 +542,7 @@ pub fn lint_fix<'a>( &transformed, source_type, &parsed, + target_version, ); if iterations == 0 { @@ -573,11 +589,18 @@ pub fn lint_fix<'a>( report_failed_to_converge_error(path, transformed.source_code(), &diagnostics); } + let syntax_errors = if settings.preview.is_enabled() { + parsed.unsupported_syntax_errors() + } else { + &[] + }; + return Ok(FixerResult { result: LinterResult { messages: diagnostics_to_messages( diagnostics, parsed.errors(), + syntax_errors, path, &locator, &directives, @@ -680,16 +703,35 @@ pub enum ParseSource { impl ParseSource { /// Consumes the [`ParseSource`] and returns the parsed [`Parsed`], parsing the source code if /// necessary. - fn into_parsed(self, source_kind: &SourceKind, source_type: PySourceType) -> Parsed { + fn into_parsed( + self, + source_kind: &SourceKind, + source_type: PySourceType, + target_version: PythonVersion, + ) -> Parsed { match self { - ParseSource::None => { - ruff_python_parser::parse_unchecked_source(source_kind.source_code(), source_type) - } + ParseSource::None => parse_unchecked_source(source_kind, source_type, target_version), ParseSource::Precomputed(parsed) => parsed, } } } +/// Like [`ruff_python_parser::parse_unchecked_source`] but with an additional [`PythonVersion`] +/// argument. +fn parse_unchecked_source( + source_kind: &SourceKind, + source_type: PySourceType, + target_version: PythonVersion, +) -> Parsed { + let options = ParseOptions::from(source_type).with_target_version(target_version); + // SAFETY: Safe because `PySourceType` always parses to a `ModModule`. See + // `ruff_python_parser::parse_unchecked_source`. We use `parse_unchecked` (and thus + // have to unwrap) in order to pass the `PythonVersion` via `ParseOptions`. + ruff_python_parser::parse_unchecked(source_kind.source_code(), options) + .try_into_module() + .expect("PySourceType always parses into a module") +} + #[cfg(test)] mod tests { use std::path::Path; diff --git a/crates/ruff_linter/src/message/mod.rs b/crates/ruff_linter/src/message/mod.rs index 52de250c4c..09d96bb5c6 100644 --- a/crates/ruff_linter/src/message/mod.rs +++ b/crates/ruff_linter/src/message/mod.rs @@ -16,7 +16,7 @@ pub use pylint::PylintEmitter; pub use rdjson::RdjsonEmitter; use ruff_diagnostics::{Diagnostic, DiagnosticKind, Fix}; use ruff_notebook::NotebookIndex; -use ruff_python_parser::ParseError; +use ruff_python_parser::{ParseError, UnsupportedSyntaxError}; use ruff_source_file::{SourceFile, SourceLocation}; use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; pub use sarif::SarifEmitter; @@ -121,6 +121,18 @@ impl Message { }) } + /// Create a [`Message`] from the given [`UnsupportedSyntaxError`]. + pub fn from_unsupported_syntax_error( + unsupported_syntax_error: &UnsupportedSyntaxError, + file: SourceFile, + ) -> Message { + Message::SyntaxError(SyntaxErrorMessage { + message: format!("SyntaxError: {unsupported_syntax_error}"), + range: unsupported_syntax_error.range, + file, + }) + } + pub const fn as_diagnostic_message(&self) -> Option<&DiagnosticMessage> { match self { Message::Diagnostic(m) => Some(m), diff --git a/crates/ruff_linter/src/rules/pyflakes/mod.rs b/crates/ruff_linter/src/rules/pyflakes/mod.rs index 83f0387b96..4f00f15b8d 100644 --- a/crates/ruff_linter/src/rules/pyflakes/mod.rs +++ b/crates/ruff_linter/src/rules/pyflakes/mod.rs @@ -11,6 +11,7 @@ mod tests { use anyhow::Result; use regex::Regex; + use ruff_python_parser::ParseOptions; use rustc_hash::FxHashMap; use test_case::test_case; @@ -744,8 +745,11 @@ mod tests { let source_type = PySourceType::default(); let source_kind = SourceKind::Python(contents.to_string()); let settings = LinterSettings::for_rules(Linter::Pyflakes.rules()); - let parsed = - ruff_python_parser::parse_unchecked_source(source_kind.source_code(), source_type); + let options = + ParseOptions::from(source_type).with_target_version(settings.unresolved_target_version); + let parsed = ruff_python_parser::parse_unchecked(source_kind.source_code(), options) + .try_into_module() + .expect("PySourceType always parses into a module"); let locator = Locator::new(&contents); let stylist = Stylist::from_tokens(parsed.tokens(), locator.contents()); let indexer = Indexer::from_tokens(parsed.tokens(), locator.contents()); @@ -767,6 +771,7 @@ mod tests { &source_kind, source_type, &parsed, + settings.unresolved_target_version, ); diagnostics.sort_by_key(Ranged::start); let actual = diagnostics diff --git a/crates/ruff_linter/src/test.rs b/crates/ruff_linter/src/test.rs index 6ded54565e..d15f49ed20 100644 --- a/crates/ruff_linter/src/test.rs +++ b/crates/ruff_linter/src/test.rs @@ -16,7 +16,7 @@ use ruff_notebook::NotebookError; use ruff_python_ast::PySourceType; use ruff_python_codegen::Stylist; use ruff_python_index::Indexer; -use ruff_python_parser::ParseError; +use ruff_python_parser::{ParseError, ParseOptions}; use ruff_python_trivia::textwrap::dedent; use ruff_source_file::SourceFileBuilder; use ruff_text_size::Ranged; @@ -110,7 +110,11 @@ pub(crate) fn test_contents<'a>( settings: &LinterSettings, ) -> (Vec, Cow<'a, SourceKind>) { let source_type = PySourceType::from(path); - let parsed = ruff_python_parser::parse_unchecked_source(source_kind.source_code(), source_type); + let target_version = settings.resolve_target_version(path); + let options = ParseOptions::from(source_type).with_target_version(target_version); + let parsed = ruff_python_parser::parse_unchecked(source_kind.source_code(), options.clone()) + .try_into_module() + .expect("PySourceType always parses into a module"); let locator = Locator::new(source_kind.source_code()); let stylist = Stylist::from_tokens(parsed.tokens(), locator.contents()); let indexer = Indexer::from_tokens(parsed.tokens(), locator.contents()); @@ -134,6 +138,7 @@ pub(crate) fn test_contents<'a>( source_kind, source_type, &parsed, + target_version, ); let source_has_errors = !parsed.is_valid(); @@ -174,7 +179,9 @@ pub(crate) fn test_contents<'a>( transformed = Cow::Owned(transformed.updated(fixed_contents, &source_map)); let parsed = - ruff_python_parser::parse_unchecked_source(transformed.source_code(), source_type); + ruff_python_parser::parse_unchecked(transformed.source_code(), options.clone()) + .try_into_module() + .expect("PySourceType always parses into a module"); let locator = Locator::new(transformed.source_code()); let stylist = Stylist::from_tokens(parsed.tokens(), locator.contents()); let indexer = Indexer::from_tokens(parsed.tokens(), locator.contents()); @@ -197,6 +204,7 @@ pub(crate) fn test_contents<'a>( &transformed, source_type, &parsed, + target_version, ); if !parsed.is_valid() && !source_has_errors { diff --git a/crates/ruff_python_parser/src/error.rs b/crates/ruff_python_parser/src/error.rs index 98efdf52e2..0c5435e91f 100644 --- a/crates/ruff_python_parser/src/error.rs +++ b/crates/ruff_python_parser/src/error.rs @@ -1,5 +1,6 @@ -use std::fmt; +use std::fmt::{self, Display}; +use ruff_python_ast::PythonVersion; use ruff_text_size::TextRange; use crate::TokenKind; @@ -426,6 +427,50 @@ impl std::fmt::Display for LexicalErrorType { } } +/// Represents a version-related syntax error detected during parsing. +/// +/// An example of a version-related error is the use of a `match` statement before Python 3.10, when +/// it was first introduced. See [`UnsupportedSyntaxErrorKind`] for other kinds of errors. +#[derive(Debug, PartialEq, Clone)] +pub struct UnsupportedSyntaxError { + pub kind: UnsupportedSyntaxErrorKind, + pub range: TextRange, + /// The target [`PythonVersion`] for which this error was detected. + /// + /// This is different from the version reported by the + /// [`minimum_version`](UnsupportedSyntaxError::minimum_version) method, which is the earliest + /// allowed version for this piece of syntax. The `target_version` is primarily used for + /// user-facing error messages. + pub target_version: PythonVersion, +} + +impl UnsupportedSyntaxError { + /// The earliest allowed version for the syntax associated with this error. + pub const fn minimum_version(&self) -> PythonVersion { + match self.kind { + UnsupportedSyntaxErrorKind::MatchBeforePy310 => PythonVersion::PY310, + } + } +} + +impl Display for UnsupportedSyntaxError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.kind { + UnsupportedSyntaxErrorKind::MatchBeforePy310 => write!( + f, + "Cannot use `match` statement on Python {} (syntax was added in Python {})", + self.target_version, + self.minimum_version(), + ), + } + } +} + +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum UnsupportedSyntaxErrorKind { + MatchBeforePy310, +} + #[cfg(target_pointer_width = "64")] mod sizes { use crate::error::{LexicalError, LexicalErrorType}; diff --git a/crates/ruff_python_parser/src/lib.rs b/crates/ruff_python_parser/src/lib.rs index 61db67ddad..c2c21adec8 100644 --- a/crates/ruff_python_parser/src/lib.rs +++ b/crates/ruff_python_parser/src/lib.rs @@ -67,7 +67,10 @@ use std::iter::FusedIterator; use std::ops::Deref; -pub use crate::error::{FStringErrorType, LexicalErrorType, ParseError, ParseErrorType}; +pub use crate::error::{ + FStringErrorType, LexicalErrorType, ParseError, ParseErrorType, UnsupportedSyntaxError, + UnsupportedSyntaxErrorKind, +}; pub use crate::parser::ParseOptions; pub use crate::token::{Token, TokenKind}; @@ -305,6 +308,7 @@ pub struct Parsed { syntax: T, tokens: Tokens, errors: Vec, + unsupported_syntax_errors: Vec, } impl Parsed { @@ -323,6 +327,11 @@ impl Parsed { &self.errors } + /// Returns a list of version-related syntax errors found during parsing. + pub fn unsupported_syntax_errors(&self) -> &[UnsupportedSyntaxError] { + &self.unsupported_syntax_errors + } + /// Consumes the [`Parsed`] output and returns the contained syntax node. pub fn into_syntax(self) -> T { self.syntax @@ -334,12 +343,18 @@ impl Parsed { } /// Returns `true` if the parsed source code is valid i.e., it has no syntax errors. + /// + /// Note that this does not include version-related + /// [`unsupported_syntax_errors`](Parsed::unsupported_syntax_errors). pub fn is_valid(&self) -> bool { self.errors.is_empty() } /// Returns the [`Parsed`] output as a [`Result`], returning [`Ok`] if it has no syntax errors, /// or [`Err`] containing the first [`ParseError`] encountered. + /// + /// Note that any [`unsupported_syntax_errors`](Parsed::unsupported_syntax_errors) will not + /// cause [`Err`] to be returned. pub fn as_result(&self) -> Result<&Parsed, &[ParseError]> { if self.is_valid() { Ok(self) @@ -350,6 +365,9 @@ impl Parsed { /// Consumes the [`Parsed`] output and returns a [`Result`] which is [`Ok`] if it has no syntax /// errors, or [`Err`] containing the first [`ParseError`] encountered. + /// + /// Note that any [`unsupported_syntax_errors`](Parsed::unsupported_syntax_errors) will not + /// cause [`Err`] to be returned. pub(crate) fn into_result(self) -> Result, ParseError> { if self.is_valid() { Ok(self) @@ -373,6 +391,7 @@ impl Parsed { syntax: module, tokens: self.tokens, errors: self.errors, + unsupported_syntax_errors: self.unsupported_syntax_errors, }), Mod::Expression(_) => None, } @@ -392,6 +411,7 @@ impl Parsed { syntax: expression, tokens: self.tokens, errors: self.errors, + unsupported_syntax_errors: self.unsupported_syntax_errors, }), } } diff --git a/crates/ruff_python_parser/src/parser/mod.rs b/crates/ruff_python_parser/src/parser/mod.rs index 951727667f..e7382f6e0b 100644 --- a/crates/ruff_python_parser/src/parser/mod.rs +++ b/crates/ruff_python_parser/src/parser/mod.rs @@ -5,6 +5,7 @@ use bitflags::bitflags; use ruff_python_ast::{Mod, ModExpression, ModModule}; use ruff_text_size::{Ranged, TextRange, TextSize}; +use crate::error::UnsupportedSyntaxError; use crate::parser::expression::ExpressionContext; use crate::parser::progress::{ParserProgress, TokenId}; use crate::token::TokenValue; @@ -35,6 +36,9 @@ pub(crate) struct Parser<'src> { /// Stores all the syntax errors found during the parsing. errors: Vec, + /// Stores non-fatal syntax errors found during parsing, such as version-related errors. + unsupported_syntax_errors: Vec, + /// Options for how the code will be parsed. options: ParseOptions, @@ -70,6 +74,7 @@ impl<'src> Parser<'src> { options, source, errors: Vec::new(), + unsupported_syntax_errors: Vec::new(), tokens, recovery_context: RecoveryContext::empty(), prev_token_end: TextSize::new(0), @@ -166,6 +171,7 @@ impl<'src> Parser<'src> { syntax, tokens: Tokens::new(tokens), errors: parse_errors, + unsupported_syntax_errors: self.unsupported_syntax_errors, }; } @@ -197,6 +203,7 @@ impl<'src> Parser<'src> { syntax, tokens: Tokens::new(tokens), errors: merged, + unsupported_syntax_errors: self.unsupported_syntax_errors, } } @@ -658,6 +665,7 @@ impl<'src> Parser<'src> { ParserCheckpoint { tokens: self.tokens.checkpoint(), errors_position: self.errors.len(), + unsupported_syntax_errors_position: self.unsupported_syntax_errors.len(), current_token_id: self.current_token_id, prev_token_end: self.prev_token_end, recovery_context: self.recovery_context, @@ -669,6 +677,7 @@ impl<'src> Parser<'src> { let ParserCheckpoint { tokens, errors_position, + unsupported_syntax_errors_position, current_token_id, prev_token_end, recovery_context, @@ -676,6 +685,8 @@ impl<'src> Parser<'src> { self.tokens.rewind(tokens); self.errors.truncate(errors_position); + self.unsupported_syntax_errors + .truncate(unsupported_syntax_errors_position); self.current_token_id = current_token_id; self.prev_token_end = prev_token_end; self.recovery_context = recovery_context; @@ -685,6 +696,7 @@ impl<'src> Parser<'src> { struct ParserCheckpoint { tokens: TokenSourceCheckpoint, errors_position: usize, + unsupported_syntax_errors_position: usize, current_token_id: TokenId, prev_token_end: TextSize, recovery_context: RecoveryContext, diff --git a/crates/ruff_python_parser/src/parser/options.rs b/crates/ruff_python_parser/src/parser/options.rs index 27a87a32ba..6258216d88 100644 --- a/crates/ruff_python_parser/src/parser/options.rs +++ b/crates/ruff_python_parser/src/parser/options.rs @@ -1,4 +1,4 @@ -use ruff_python_ast::PySourceType; +use ruff_python_ast::{PySourceType, PythonVersion}; use crate::{AsMode, Mode}; @@ -20,15 +20,28 @@ use crate::{AsMode, Mode}; /// /// let options = ParseOptions::from(PySourceType::Python); /// ``` -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct ParseOptions { /// Specify the mode in which the code will be parsed. pub(crate) mode: Mode, + /// Target version for detecting version-related syntax errors. + pub(crate) target_version: PythonVersion, +} + +impl ParseOptions { + #[must_use] + pub fn with_target_version(mut self, target_version: PythonVersion) -> Self { + self.target_version = target_version; + self + } } impl From for ParseOptions { fn from(mode: Mode) -> Self { - Self { mode } + Self { + mode, + target_version: PythonVersion::default(), + } } } @@ -36,6 +49,7 @@ impl From for ParseOptions { fn from(source_type: PySourceType) -> Self { Self { mode: source_type.as_mode(), + target_version: PythonVersion::default(), } } } diff --git a/crates/ruff_python_parser/src/parser/statement.rs b/crates/ruff_python_parser/src/parser/statement.rs index e76fc08915..5e5bee4ce2 100644 --- a/crates/ruff_python_parser/src/parser/statement.rs +++ b/crates/ruff_python_parser/src/parser/statement.rs @@ -5,7 +5,8 @@ use rustc_hash::{FxBuildHasher, FxHashSet}; use ruff_python_ast::name::Name; use ruff_python_ast::{ - self as ast, ExceptHandler, Expr, ExprContext, IpyEscapeKind, Operator, Stmt, WithItem, + self as ast, ExceptHandler, Expr, ExprContext, IpyEscapeKind, Operator, PythonVersion, Stmt, + WithItem, }; use ruff_text_size::{Ranged, TextSize}; @@ -16,7 +17,7 @@ use crate::parser::{ }; use crate::token::{TokenKind, TokenValue}; use crate::token_set::TokenSet; -use crate::{Mode, ParseErrorType}; +use crate::{Mode, ParseErrorType, UnsupportedSyntaxError, UnsupportedSyntaxErrorKind}; use super::expression::ExpressionContext; use super::Parenthesized; @@ -2257,11 +2258,21 @@ impl<'src> Parser<'src> { let start = self.node_start(); self.bump(TokenKind::Match); + let match_range = self.node_range(start); + let subject = self.parse_match_subject_expression(); self.expect(TokenKind::Colon); let cases = self.parse_match_body(); + if self.options.target_version < PythonVersion::PY310 { + self.unsupported_syntax_errors.push(UnsupportedSyntaxError { + kind: UnsupportedSyntaxErrorKind::MatchBeforePy310, + range: match_range, + target_version: self.options.target_version, + }); + } + ast::StmtMatch { subject: Box::new(subject), cases, diff --git a/crates/ruff_server/src/lint.rs b/crates/ruff_server/src/lint.rs index 9b9e8a06d2..53cff99ad3 100644 --- a/crates/ruff_server/src/lint.rs +++ b/crates/ruff_server/src/lint.rs @@ -24,7 +24,7 @@ use ruff_linter::{ use ruff_notebook::Notebook; use ruff_python_codegen::Stylist; use ruff_python_index::Indexer; -use ruff_python_parser::ParseError; +use ruff_python_parser::{ParseError, ParseOptions, UnsupportedSyntaxError}; use ruff_source_file::LineIndex; use ruff_text_size::{Ranged, TextRange}; @@ -94,8 +94,18 @@ pub(crate) fn check( let source_type = query.source_type(); + let target_version = if let Some(path) = &document_path { + settings.linter.resolve_target_version(path) + } else { + settings.linter.unresolved_target_version + }; + + let parse_options = ParseOptions::from(source_type).with_target_version(target_version); + // Parse once. - let parsed = ruff_python_parser::parse_unchecked_source(source_kind.source_code(), source_type); + let parsed = ruff_python_parser::parse_unchecked(source_kind.source_code(), parse_options) + .try_into_module() + .expect("PySourceType always parses to a ModModule"); // Map row and column locations to byte slices (lazily). let locator = Locator::new(source_kind.source_code()); @@ -122,6 +132,7 @@ pub(crate) fn check( &source_kind, source_type, &parsed, + target_version, ); let noqa_edits = generate_noqa_edits( @@ -164,14 +175,25 @@ pub(crate) fn check( let lsp_diagnostics = lsp_diagnostics.chain( show_syntax_errors .then(|| { - parsed.errors().iter().map(|parse_error| { - parse_error_to_lsp_diagnostic( - parse_error, - &source_kind, - locator.to_index(), - encoding, - ) - }) + parsed + .errors() + .iter() + .map(|parse_error| { + parse_error_to_lsp_diagnostic( + parse_error, + &source_kind, + locator.to_index(), + encoding, + ) + }) + .chain(parsed.unsupported_syntax_errors().iter().map(|error| { + unsupported_syntax_error_to_lsp_diagnostic( + error, + &source_kind, + locator.to_index(), + encoding, + ) + })) }) .into_iter() .flatten(), @@ -350,6 +372,45 @@ fn parse_error_to_lsp_diagnostic( ) } +fn unsupported_syntax_error_to_lsp_diagnostic( + unsupported_syntax_error: &UnsupportedSyntaxError, + source_kind: &SourceKind, + index: &LineIndex, + encoding: PositionEncoding, +) -> (usize, lsp_types::Diagnostic) { + let range: lsp_types::Range; + let cell: usize; + + if let Some(notebook_index) = source_kind.as_ipy_notebook().map(Notebook::index) { + NotebookRange { cell, range } = unsupported_syntax_error.range.to_notebook_range( + source_kind.source_code(), + index, + notebook_index, + encoding, + ); + } else { + cell = usize::default(); + range = unsupported_syntax_error + .range + .to_range(source_kind.source_code(), index, encoding); + } + + ( + cell, + lsp_types::Diagnostic { + range, + severity: Some(lsp_types::DiagnosticSeverity::ERROR), + tags: None, + code: None, + code_description: None, + source: Some(DIAGNOSTIC_NAME.into()), + message: format!("SyntaxError: {unsupported_syntax_error}"), + related_information: None, + data: None, + }, + ) +} + fn diagnostic_edit_range( range: TextRange, source_kind: &SourceKind, diff --git a/crates/ruff_wasm/src/lib.rs b/crates/ruff_wasm/src/lib.rs index 52b25b3794..0006abf7fa 100644 --- a/crates/ruff_wasm/src/lib.rs +++ b/crates/ruff_wasm/src/lib.rs @@ -196,6 +196,7 @@ impl Workspace { &source_kind, source_type, &parsed, + self.settings.linter.unresolved_target_version, ); let source_code = locator.to_source_code(); diff --git a/python/py-fuzzer/fuzz.py b/python/py-fuzzer/fuzz.py index c4713f4028..181b69770b 100644 --- a/python/py-fuzzer/fuzz.py +++ b/python/py-fuzzer/fuzz.py @@ -58,7 +58,16 @@ def redknot_contains_bug(code: str, *, red_knot_executable: Path) -> bool: def ruff_contains_bug(code: str, *, ruff_executable: Path) -> bool: """Return `True` if the code triggers a parser error.""" completed_process = subprocess.run( - [ruff_executable, "check", "--config", "lint.select=[]", "--no-cache", "-"], + [ + ruff_executable, + "check", + "--config", + "lint.select=[]", + "--no-cache", + "--target-version", + "py313", + "-", + ], capture_output=True, text=True, input=code,