Emit LexError for dedent to incorrect level (#7638)

crates/ruff_python_formatter/resources/test/fixtures/ruff/expression/binary.py

@@ -366,7 +366,7 @@ if (
 ):
     pass
 
- z = (
+z = (
                  a
                  +
                  # a: extracts this comment
@@ -377,7 +377,7 @@ if (
                          x and y
                      )
              )
- )
+)
 
 z = (
     (

crates/ruff_python_formatter/tests/snapshots/format@expression__binary.py.snap

@@ -372,7 +372,7 @@ if (
 ):
     pass
 
- z = (
+z = (
                  a
                  +
                  # a: extracts this comment
@@ -383,7 +383,7 @@ if (
                          x and y
                      )
              )
- )
+)
 
 z = (
     (

crates/ruff_python_parser/src/lexer.rs

@@ -556,10 +556,22 @@ impl<'source> Lexer<'source> {
     pub fn next_token(&mut self) -> LexResult {
         // Return dedent tokens until the current indentation level matches the indentation of the next token.
         if let Some(indentation) = self.pending_indentation.take() {
-            if let Ok(Ordering::Greater) = self.indentations.current().try_compare(indentation) {
+            match self.indentations.current().try_compare(indentation) {
-                self.pending_indentation = Some(indentation);
+                Ok(Ordering::Greater) => {
-                self.indentations.pop();
+                    self.pending_indentation = Some(indentation);
-                return Ok((Tok::Dedent, TextRange::empty(self.offset())));
+                    let offset = self.offset();
+                    self.indentations.dedent_one(indentation).map_err(|_| {
+                        LexicalError::new(LexicalErrorType::IndentationError, offset)
+                    })?;
+                    return Ok((Tok::Dedent, TextRange::empty(offset)));
+                }
+                Ok(_) => {}
+                Err(_) => {
+                    return Err(LexicalError::new(
+                        LexicalErrorType::IndentationError,
+                        self.offset(),
+                    ));
+                }
             }
         }
 
@@ -690,9 +702,12 @@ impl<'source> Lexer<'source> {
         let token = match self.indentations.current().try_compare(indentation) {
             // Dedent
             Ok(Ordering::Greater) => {
-                self.indentations.pop();
                 self.pending_indentation = Some(indentation);
 
+                self.indentations.dedent_one(indentation).map_err(|_| {
+                    LexicalError::new(LexicalErrorType::IndentationError, self.offset())
+                })?;
+
                 Some((Tok::Dedent, TextRange::empty(self.offset())))
             }
 
@@ -700,7 +715,7 @@ impl<'source> Lexer<'source> {
 
             // Indent
             Ok(Ordering::Less) => {
-                self.indentations.push(indentation);
+                self.indentations.indent(indentation);
                 Some((Tok::Indent, self.token_range()))
             }
             Err(_) => {
@@ -732,7 +747,7 @@ impl<'source> Lexer<'source> {
             Ok((Tok::Newline, TextRange::empty(self.offset())))
         }
         // Next, flush the indentation stack to zero.
-        else if self.indentations.pop().is_some() {
+        else if self.indentations.dedent().is_some() {
             Ok((Tok::Dedent, TextRange::empty(self.offset())))
         } else {
             Ok((Tok::EndOfFile, TextRange::empty(self.offset())))
@@ -1678,4 +1693,16 @@ def f(arg=%timeit a = b):
             result => panic!("Expected an error token but found {result:?}"),
         }
     }
+
+    #[test]
+    fn tet_too_low_dedent() {
+        let tokens: Vec<_> = lex(
+            r#"if True:
+    pass
+  pass"#,
+            Mode::Module,
+        )
+        .collect();
+        assert_debug_snapshot!(tokens);
+    }
 }

crates/ruff_python_parser/src/lexer/indentation.rs

@@ -90,13 +90,33 @@ pub(super) struct Indentations {
 }
 
 impl Indentations {
-    pub(super) fn push(&mut self, indent: Indentation) {
+    pub(super) fn indent(&mut self, indent: Indentation) {
         debug_assert_eq!(self.current().try_compare(indent), Ok(Ordering::Less));
 
         self.stack.push(indent);
     }
 
-    pub(super) fn pop(&mut self) -> Option<Indentation> {
+    /// Dedent one level to eventually reach `new_indentation`.
+    ///
+    /// Returns `Err` if the `new_indentation` is greater than the new current indentation level.
+    pub(super) fn dedent_one(
+        &mut self,
+        new_indentation: Indentation,
+    ) -> Result<Option<Indentation>, UnexpectedIndentation> {
+        let previous = self.dedent();
+
+        match new_indentation.try_compare(*self.current())? {
+            Ordering::Less | Ordering::Equal => Ok(previous),
+            // ```python
+            // if True:
+            //     pass
+            //   pass <- The indentation is greater than the expected indent of 0.
+            // ```
+            Ordering::Greater => Err(UnexpectedIndentation),
+        }
+    }
+
+    pub(super) fn dedent(&mut self) -> Option<Indentation> {
         self.stack.pop()
     }
 

crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__tet_too_low_dedent.snap

@@ -0,0 +1,66 @@
+---
+source: crates/ruff_python_parser/src/lexer.rs
+expression: tokens
+---
+[
+    Ok(
+        (
+            If,
+            0..2,
+        ),
+    ),
+    Ok(
+        (
+            True,
+            3..7,
+        ),
+    ),
+    Ok(
+        (
+            Colon,
+            7..8,
+        ),
+    ),
+    Ok(
+        (
+            Newline,
+            8..9,
+        ),
+    ),
+    Ok(
+        (
+            Indent,
+            9..13,
+        ),
+    ),
+    Ok(
+        (
+            Pass,
+            13..17,
+        ),
+    ),
+    Ok(
+        (
+            Newline,
+            17..18,
+        ),
+    ),
+    Err(
+        LexicalError {
+            error: IndentationError,
+            location: 20,
+        },
+    ),
+    Ok(
+        (
+            Pass,
+            20..24,
+        ),
+    ),
+    Ok(
+        (
+            Newline,
+            24..24,
+        ),
+    ),
+]