Remove type parameter from parse_* methods (#9466)

crates/ruff_python_parser/src/invalid.rs

@@ -687,7 +687,7 @@ mod tests {
 
         let src = r"!foo = 42";
         let tokens = crate::lexer::lex(src, Mode::Ipython);
-        let ast = crate::parse_tokens(tokens, src, Mode::Ipython);
+        let ast = crate::parse_tokens(tokens.collect(), src, Mode::Ipython);
         insta::assert_debug_snapshot!(ast);
     }
 

crates/ruff_python_parser/src/lib.rs

@@ -85,7 +85,7 @@
 //!    return bool(i & 1)
 //! "#;
 //! let tokens = lex(python_source, Mode::Module);
-//! let ast = parse_tokens(tokens, python_source, Mode::Module);
+//! let ast = parse_tokens(tokens.collect(), python_source, Mode::Module);
 //!
 //! assert!(ast.is_ok());
 //! ```
@@ -110,8 +110,8 @@
 //! [lexer]: crate::lexer
 
 pub use parser::{
-    parse, parse_expression, parse_expression_starts_at, parse_ok_tokens, parse_program,
-    parse_starts_at, parse_suite, parse_tokens, ParseError, ParseErrorType,
+    parse, parse_expression, parse_expression_starts_at, parse_program, parse_starts_at,
+    parse_suite, parse_tokens, ParseError, ParseErrorType,
 };
 use ruff_python_ast::{Mod, PySourceType, Suite};
 pub use string::FStringErrorType;
@@ -128,6 +128,7 @@ mod parser;
 mod soft_keywords;
 mod string;
 mod token;
+mod token_source;
 pub mod typing;
 
 /// Collect tokens up to and including the first error.
@@ -145,7 +146,7 @@ pub fn tokenize(contents: &str, mode: Mode) -> Vec<LexResult> {
 
 /// Parse a full Python program from its tokens.
 pub fn parse_program_tokens(
-    lxr: Vec<LexResult>,
+    tokens: Vec<LexResult>,
     source: &str,
     is_jupyter_notebook: bool,
 ) -> anyhow::Result<Suite, ParseError> {
@@ -154,7 +155,7 @@ pub fn parse_program_tokens(
     } else {
         Mode::Module
     };
-    match parse_tokens(lxr, source, mode)? {
+    match parse_tokens(tokens, source, mode)? {
         Mod::Module(m) => Ok(m.body),
         Mod::Expression(_) => unreachable!("Mode::Module doesn't return other variant"),
     }

crates/ruff_python_parser/src/parser.rs

@@ -14,15 +14,7 @@
 
 use itertools::Itertools;
 pub(super) use lalrpop_util::ParseError as LalrpopError;
-use ruff_text_size::{Ranged, TextRange, TextSize};
 
-use crate::lexer::{lex, lex_starts_at, Spanned};
-use crate::{
-    lexer::{self, LexResult, LexicalError, LexicalErrorType},
-    python,
-    token::Tok,
-    Mode,
-};
 use ruff_python_ast::{
     Expr, ExprAttribute, ExprAwait, ExprBinOp, ExprBoolOp, ExprBooleanLiteral, ExprBytesLiteral,
     ExprCall, ExprCompare, ExprDict, ExprDictComp, ExprEllipsisLiteral, ExprFString,
@@ -31,6 +23,16 @@ use ruff_python_ast::{
     ExprStarred, ExprStringLiteral, ExprSubscript, ExprTuple, ExprUnaryOp, ExprYield,
     ExprYieldFrom, Mod, ModModule, Suite,
 };
+use ruff_text_size::{Ranged, TextRange, TextSize};
+
+use crate::lexer::{lex, lex_starts_at, LexResult};
+use crate::token_source::TokenSource;
+use crate::{
+    lexer::{self, LexicalError, LexicalErrorType},
+    python,
+    token::Tok,
+    Mode,
+};
 
 /// Parse a full Python program usually consisting of multiple lines.
 ///
@@ -54,7 +56,7 @@ use ruff_python_ast::{
 /// ```
 pub fn parse_program(source: &str) -> Result<ModModule, ParseError> {
     let lexer = lex(source, Mode::Module);
-    match parse_tokens(lexer, source, Mode::Module)? {
+    match parse_tokens(lexer.collect(), source, Mode::Module)? {
         Mod::Module(m) => Ok(m),
         Mod::Expression(_) => unreachable!("Mode::Module doesn't return other variant"),
     }
@@ -82,7 +84,7 @@ pub fn parse_suite(source: &str) -> Result<Suite, ParseError> {
 /// ```
 pub fn parse_expression(source: &str) -> Result<Expr, ParseError> {
     let lexer = lex(source, Mode::Expression);
-    match parse_tokens(lexer, source, Mode::Expression)? {
+    match parse_tokens(lexer.collect(), source, Mode::Expression)? {
         Mod::Expression(expression) => Ok(*expression.body),
         Mod::Module(_m) => unreachable!("Mode::Expression doesn't return other variant"),
     }
@@ -107,7 +109,7 @@ pub fn parse_expression(source: &str) -> Result<Expr, ParseError> {
 /// ```
 pub fn parse_expression_starts_at(source: &str, offset: TextSize) -> Result<Expr, ParseError> {
     let lexer = lex_starts_at(source, Mode::Module, offset);
-    match parse_tokens(lexer, source, Mode::Expression)? {
+    match parse_tokens(lexer.collect(), source, Mode::Expression)? {
         Mod::Expression(expression) => Ok(*expression.body),
         Mod::Module(_m) => unreachable!("Mode::Expression doesn't return other variant"),
     }
@@ -188,7 +190,7 @@ pub fn parse(source: &str, mode: Mode) -> Result<Mod, ParseError> {
 /// ```
 pub fn parse_starts_at(source: &str, mode: Mode, offset: TextSize) -> Result<Mod, ParseError> {
     let lxr = lexer::lex_starts_at(source, mode, offset);
-    parse_tokens(lxr, source, mode)
+    parse_tokens(lxr.collect(), source, mode)
 }
 
 /// Parse an iterator of [`LexResult`]s using the specified [`Mode`].
@@ -204,48 +206,12 @@ pub fn parse_starts_at(source: &str, mode: Mode, offset: TextSize) -> Result<Mod
 /// use ruff_python_parser::{lexer::lex, Mode, parse_tokens};
 ///
 /// let source = "1 + 2";
-/// let expr = parse_tokens(lex(source, Mode::Expression), source, Mode::Expression);
+/// let expr = parse_tokens(lex(source, Mode::Expression).collect(), source, Mode::Expression);
 /// assert!(expr.is_ok());
 /// ```
-pub fn parse_tokens(
-    lxr: impl IntoIterator<Item = LexResult>,
-    source: &str,
-    mode: Mode,
-) -> Result<Mod, ParseError> {
-    let lxr = lxr.into_iter();
-
-    parse_filtered_tokens(
-        lxr.filter_ok(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline)),
-        source,
-        mode,
-    )
-}
-
-/// Parse tokens into an AST like [`parse_tokens`], but we already know all tokens are valid.
-pub fn parse_ok_tokens(
-    lxr: impl IntoIterator<Item = Spanned>,
-    source: &str,
-    mode: Mode,
-) -> Result<Mod, ParseError> {
-    let lxr = lxr
-        .into_iter()
-        .filter(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
+pub fn parse_tokens(tokens: Vec<LexResult>, source: &str, mode: Mode) -> Result<Mod, ParseError> {
     let marker_token = (Tok::start_marker(mode), TextRange::default());
-    let lexer = std::iter::once(marker_token)
-        .chain(lxr)
-        .map(|(t, range)| (range.start(), t, range.end()));
-    python::TopParser::new()
-        .parse(source, mode, lexer)
-        .map_err(parse_error_from_lalrpop)
-}
-
-fn parse_filtered_tokens(
-    lxr: impl IntoIterator<Item = LexResult>,
-    source: &str,
-    mode: Mode,
-) -> Result<Mod, ParseError> {
-    let marker_token = (Tok::start_marker(mode), TextRange::default());
-    let lexer = std::iter::once(Ok(marker_token)).chain(lxr);
+    let lexer = std::iter::once(Ok(marker_token)).chain(TokenSource::new(tokens));
     python::TopParser::new()
         .parse(
             source,
@@ -597,9 +563,10 @@ impl From<ExprSlice> for ParenthesizedExpr {
 
 #[cfg(target_pointer_width = "64")]
 mod size_assertions {
-    use crate::parser::ParenthesizedExpr;
     use static_assertions::assert_eq_size;
 
+    use crate::parser::ParenthesizedExpr;
+
     assert_eq_size!(ParenthesizedExpr, [u8; 88]);
 }
 
@@ -1475,7 +1442,7 @@ a = 1
     "
         .trim();
         let lxr = lexer::lex_starts_at(source, Mode::Ipython, TextSize::default());
-        let parse_err = parse_tokens(lxr, source, Mode::Module).unwrap_err();
+        let parse_err = parse_tokens(lxr.collect(), source, Mode::Module).unwrap_err();
         assert_eq!(
             parse_err.to_string(),
             "IPython escape commands are only allowed in `Mode::Ipython` at byte offset 6"

crates/ruff_python_parser/src/token_source.rs

@@ -0,0 +1,46 @@
+use crate::lexer::LexResult;
+use crate::Tok;
+use std::iter::FusedIterator;
+
+#[derive(Clone, Debug)]
+pub(crate) struct TokenSource {
+    tokens: std::vec::IntoIter<LexResult>,
+}
+
+impl TokenSource {
+    pub(crate) fn new(tokens: Vec<LexResult>) -> Self {
+        Self {
+            tokens: tokens.into_iter(),
+        }
+    }
+}
+
+impl FromIterator<LexResult> for TokenSource {
+    #[inline]
+    fn from_iter<T: IntoIterator<Item = LexResult>>(iter: T) -> Self {
+        Self::new(Vec::from_iter(iter))
+    }
+}
+
+impl Iterator for TokenSource {
+    type Item = LexResult;
+
+    #[inline]
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            let next = self.tokens.next()?;
+
+            if is_trivia(&next) {
+                continue;
+            }
+
+            break Some(next);
+        }
+    }
+}
+
+impl FusedIterator for TokenSource {}
+
+const fn is_trivia(result: &LexResult) -> bool {
+    matches!(result, Ok((Tok::Comment(_) | Tok::NonLogicalNewline, _)))
+}