Build CommentRanges outside the parser (#11792)

## Summary

This PR updates the parser to remove building the `CommentRanges` and
instead it'll be built by the linter and the formatter when it's
required.

For the linter, it'll be built and owned by the `Indexer` while for the
formatter it'll be built from the `Tokens` struct and passed as an
argument.

## Test Plan

`cargo insta test`

crates/ruff_python_formatter/src/cli.rs

@@ -8,6 +8,7 @@ use clap::{command, Parser, ValueEnum};
 use ruff_formatter::SourceCode;
 use ruff_python_ast::PySourceType;
 use ruff_python_parser::{parse, AsMode};
+use ruff_python_trivia::CommentRanges;
 use ruff_text_size::Ranged;
 
 use crate::comments::collect_comments;
@@ -62,14 +63,15 @@ pub fn format_and_debug_print(source: &str, cli: &Cli, source_path: &Path) -> Re
         });
 
     let source_code = SourceCode::new(source);
-    let formatted = format_module_ast(&parsed, source, options).context("Failed to format node")?;
+    let comment_ranges = CommentRanges::from(parsed.tokens());
+    let formatted = format_module_ast(&parsed, &comment_ranges, source, options)
+        .context("Failed to format node")?;
     if cli.print_ir {
         println!("{}", formatted.document().display(source_code));
     }
     if cli.print_comments {
         // Print preceding, following and enclosing nodes
-        let decorated_comments =
-            collect_comments(parsed.syntax(), source_code, parsed.comment_ranges());
+        let decorated_comments = collect_comments(parsed.syntax(), source_code, &comment_ranges);
         if !decorated_comments.is_empty() {
             println!("# Comment decoration: Range, Preceding, Following, Enclosing, Comment");
         }

crates/ruff_python_formatter/src/comments/mod.rs

@@ -482,11 +482,13 @@ mod tests {
     use ruff_formatter::SourceCode;
     use ruff_python_ast::{Mod, PySourceType};
     use ruff_python_parser::{parse, AsMode, Parsed};
+    use ruff_python_trivia::CommentRanges;
 
     use crate::comments::Comments;
 
     struct CommentsTestCase<'a> {
         parsed: Parsed<Mod>,
+        comment_ranges: CommentRanges,
         source_code: SourceCode<'a>,
     }
 
@@ -496,19 +498,17 @@ mod tests {
             let source_type = PySourceType::Python;
             let parsed =
                 parse(source, source_type.as_mode()).expect("Expect source to be valid Python");
+            let comment_ranges = CommentRanges::from(parsed.tokens());
 
             CommentsTestCase {
                 parsed,
+                comment_ranges,
                 source_code,
             }
         }
 
         fn to_comments(&self) -> Comments {
-            Comments::from_ast(
-                self.parsed.syntax(),
-                self.source_code,
-                self.parsed.comment_ranges(),
-            )
+            Comments::from_ast(self.parsed.syntax(), self.source_code, &self.comment_ranges)
         }
     }
 

crates/ruff_python_formatter/src/expression/parentheses.rs

@@ -444,6 +444,7 @@ impl Format<PyFormatContext<'_>> for FormatEmptyParenthesized<'_> {
 mod tests {
     use ruff_python_ast::ExpressionRef;
     use ruff_python_parser::parse_expression;
+    use ruff_python_trivia::CommentRanges;
 
     use crate::expression::parentheses::is_expression_parenthesized;
 
@@ -453,7 +454,7 @@ mod tests {
         let parsed = parse_expression(expression).unwrap();
         assert!(!is_expression_parenthesized(
             ExpressionRef::from(parsed.expr()),
-            parsed.comment_ranges(),
+            &CommentRanges::default(),
             expression
         ));
     }

crates/ruff_python_formatter/src/lib.rs

@@ -114,17 +114,19 @@ pub fn format_module_source(
 ) -> Result<Printed, FormatModuleError> {
     let source_type = options.source_type();
     let parsed = parse(source, source_type.as_mode())?;
-    let formatted = format_module_ast(&parsed, source, options)?;
+    let comment_ranges = CommentRanges::from(parsed.tokens());
+    let formatted = format_module_ast(&parsed, &comment_ranges, source, options)?;
     Ok(formatted.print()?)
 }
 
 pub fn format_module_ast<'a>(
     parsed: &'a Parsed<Mod>,
+    comment_ranges: &'a CommentRanges,
     source: &'a str,
     options: PyFormatOptions,
 ) -> FormatResult<Formatted<PyFormatContext<'a>>> {
     let source_code = SourceCode::new(source);
-    let comments = Comments::from_ast(parsed.syntax(), source_code, parsed.comment_ranges());
+    let comments = Comments::from_ast(parsed.syntax(), source_code, comment_ranges);
     let locator = Locator::new(source);
 
     let formatted = format!(
@@ -155,6 +157,7 @@ mod tests {
 
     use ruff_python_ast::PySourceType;
     use ruff_python_parser::{parse, AsMode};
+    use ruff_python_trivia::CommentRanges;
     use ruff_text_size::{TextRange, TextSize};
 
     use crate::{format_module_ast, format_module_source, format_range, PyFormatOptions};
@@ -199,8 +202,9 @@ def main() -> None:
         // Parse the AST.
         let source_path = "code_inline.py";
         let parsed = parse(source, source_type.as_mode()).unwrap();
+        let comment_ranges = CommentRanges::from(parsed.tokens());
         let options = PyFormatOptions::from_extension(Path::new(source_path));
-        let formatted = format_module_ast(&parsed, source, options).unwrap();
+        let formatted = format_module_ast(&parsed, &comment_ranges, source, options).unwrap();
 
         // Uncomment the `dbg` to print the IR.
         // Use `dbg_write!(f, []) instead of `write!(f, [])` in your formatting code to print some IR

crates/ruff_python_formatter/src/range.rs

@@ -7,7 +7,9 @@ use ruff_formatter::{
 use ruff_python_ast::visitor::source_order::{walk_body, SourceOrderVisitor, TraversalSignal};
 use ruff_python_ast::{AnyNodeRef, Stmt, StmtMatch, StmtTry};
 use ruff_python_parser::{parse, AsMode};
-use ruff_python_trivia::{indentation_at_offset, BackwardsTokenizer, SimpleToken, SimpleTokenKind};
+use ruff_python_trivia::{
+    indentation_at_offset, BackwardsTokenizer, CommentRanges, SimpleToken, SimpleTokenKind,
+};
 use ruff_source_file::Locator;
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
 
@@ -74,7 +76,8 @@ pub fn format_range(
 
     let parsed = parse(source, options.source_type().as_mode())?;
     let source_code = SourceCode::new(source);
-    let comments = Comments::from_ast(parsed.syntax(), source_code, parsed.comment_ranges());
+    let comment_ranges = CommentRanges::from(parsed.tokens());
+    let comments = Comments::from_ast(parsed.syntax(), source_code, &comment_ranges);
 
     let mut context = PyFormatContext::new(
         options.with_source_map_generation(SourceMapGeneration::Enabled),

crates/ruff_python_formatter/src/statement/suite.rs

@@ -831,6 +831,7 @@ impl Format<PyFormatContext<'_>> for SuiteChildStatement<'_> {
 mod tests {
     use ruff_formatter::format;
     use ruff_python_parser::parse_module;
+    use ruff_python_trivia::CommentRanges;
 
     use crate::comments::Comments;
     use crate::prelude::*;
@@ -860,11 +861,12 @@ def trailing_func():
 ";
 
         let parsed = parse_module(source).unwrap();
+        let comment_ranges = CommentRanges::from(parsed.tokens());
 
         let context = PyFormatContext::new(
             PyFormatOptions::default(),
             source,
-            Comments::from_ranges(parsed.comment_ranges()),
+            Comments::from_ranges(&comment_ranges),
             parsed.tokens(),
         );
 

crates/ruff_python_formatter/src/string/docstring.rs

@@ -9,6 +9,7 @@ use itertools::Itertools;
 
 use ruff_formatter::printer::SourceMapGeneration;
 use ruff_python_ast::{str::Quote, StringFlags};
+use ruff_python_trivia::CommentRanges;
 use {once_cell::sync::Lazy, regex::Regex};
 use {
     ruff_formatter::{write, FormatOptions, IndentStyle, LineWidth, Printed},
@@ -1552,8 +1553,9 @@ fn docstring_format_source(
 
     let source_type = options.source_type();
     let parsed = ruff_python_parser::parse(source, source_type.as_mode())?;
+    let comment_ranges = CommentRanges::from(parsed.tokens());
     let source_code = ruff_formatter::SourceCode::new(source);
-    let comments = crate::Comments::from_ast(parsed.syntax(), source_code, parsed.comment_ranges());
+    let comments = crate::Comments::from_ast(parsed.syntax(), source_code, &comment_ranges);
     let locator = Locator::new(source);
 
     let ctx = PyFormatContext::new(options, locator.contents(), comments, parsed.tokens())