Pass ParserOptions to the parser (#16220)

## Summary

This is part of the preparation for detecting syntax errors in the
parser from https://github.com/astral-sh/ruff/pull/16090/. As suggested
in [this
comment](https://github.com/astral-sh/ruff/pull/16090/#discussion_r1953084509),
I started working on a `ParseOptions` struct that could be stored in the
parser. For this initial refactor, I only made it hold the existing
`Mode` option, but for syntax errors, we will also need it to have a
`PythonVersion`. For that use case, I'm picturing something like a
`ParseOptions::with_python_version` method, so you can extend the
current calls to something like

```rust
ParseOptions::from(mode).with_python_version(settings.target_version)
```

But I thought it was worth adding `ParseOptions` alone without changing
any other behavior first.

Most of the diff is just updating call sites taking `Mode` to take
`ParseOptions::from(Mode)` or those taking `PySourceType`s to take
`ParseOptions::from(PySourceType)`. The interesting changes are in the
new `parser/options.rs` file and smaller parts of `parser/mod.rs` and
`ruff_python_parser/src/lib.rs`.

## Test Plan

Existing tests, this should not change any behavior.

crates/ruff_python_formatter/src/cli.rs

@@ -7,7 +7,7 @@ use clap::{command, Parser, ValueEnum};
 
 use ruff_formatter::SourceCode;
 use ruff_python_ast::PySourceType;
-use ruff_python_parser::{parse, AsMode};
+use ruff_python_parser::{parse, ParseOptions};
 use ruff_python_trivia::CommentRanges;
 use ruff_text_size::Ranged;
 
@@ -48,7 +48,7 @@ pub fn format_and_debug_print(source: &str, cli: &Cli, source_path: &Path) -> Re
     let source_type = PySourceType::from(source_path);
 
     // Parse the AST.
-    let parsed = parse(source, source_type.as_mode()).context("Syntax error in input")?;
+    let parsed = parse(source, ParseOptions::from(source_type)).context("Syntax error in input")?;
 
     let options = PyFormatOptions::from_extension(source_path)
         .with_preview(if cli.preview {

crates/ruff_python_formatter/src/comments/mod.rs

@@ -514,7 +514,7 @@ mod tests {
 
     use ruff_formatter::SourceCode;
     use ruff_python_ast::{Mod, PySourceType};
-    use ruff_python_parser::{parse, AsMode, Parsed};
+    use ruff_python_parser::{parse, ParseOptions, Parsed};
     use ruff_python_trivia::CommentRanges;
 
     use crate::comments::Comments;
@@ -529,8 +529,8 @@ mod tests {
         fn from_code(source: &'a str) -> Self {
             let source_code = SourceCode::new(source);
             let source_type = PySourceType::Python;
-            let parsed =
-                parse(source, source_type.as_mode()).expect("Expect source to be valid Python");
+            let parsed = parse(source, ParseOptions::from(source_type))
+                .expect("Expect source to be valid Python");
             let comment_ranges = CommentRanges::from(parsed.tokens());
 
             CommentsTestCase {

crates/ruff_python_formatter/src/lib.rs

@@ -5,7 +5,7 @@ pub use range::format_range;
 use ruff_formatter::prelude::*;
 use ruff_formatter::{format, write, FormatError, Formatted, PrintError, Printed, SourceCode};
 use ruff_python_ast::{AnyNodeRef, Mod};
-use ruff_python_parser::{parse, AsMode, ParseError, Parsed};
+use ruff_python_parser::{parse, ParseError, ParseOptions, Parsed};
 use ruff_python_trivia::CommentRanges;
 use ruff_text_size::Ranged;
 
@@ -112,7 +112,7 @@ pub fn format_module_source(
     options: PyFormatOptions,
 ) -> Result<Printed, FormatModuleError> {
     let source_type = options.source_type();
-    let parsed = parse(source, source_type.as_mode())?;
+    let parsed = parse(source, ParseOptions::from(source_type))?;
     let comment_ranges = CommentRanges::from(parsed.tokens());
     let formatted = format_module_ast(&parsed, &comment_ranges, source, options)?;
     Ok(formatted.print()?)
@@ -154,7 +154,7 @@ mod tests {
     use insta::assert_snapshot;
 
     use ruff_python_ast::PySourceType;
-    use ruff_python_parser::{parse, AsMode};
+    use ruff_python_parser::{parse, ParseOptions};
     use ruff_python_trivia::CommentRanges;
     use ruff_text_size::{TextRange, TextSize};
 
@@ -199,7 +199,7 @@ def main() -> None:
 
         // Parse the AST.
         let source_path = "code_inline.py";
-        let parsed = parse(source, source_type.as_mode()).unwrap();
+        let parsed = parse(source, ParseOptions::from(source_type)).unwrap();
         let comment_ranges = CommentRanges::from(parsed.tokens());
         let options = PyFormatOptions::from_extension(Path::new(source_path));
         let formatted = format_module_ast(&parsed, &comment_ranges, source, options).unwrap();

crates/ruff_python_formatter/src/range.rs

@@ -6,7 +6,7 @@ use ruff_formatter::{
 };
 use ruff_python_ast::visitor::source_order::{walk_body, SourceOrderVisitor, TraversalSignal};
 use ruff_python_ast::{AnyNodeRef, Stmt, StmtMatch, StmtTry};
-use ruff_python_parser::{parse, AsMode};
+use ruff_python_parser::{parse, ParseOptions};
 use ruff_python_trivia::{
     indentation_at_offset, BackwardsTokenizer, CommentRanges, SimpleToken, SimpleTokenKind,
 };
@@ -73,7 +73,7 @@ pub fn format_range(
 
     assert_valid_char_boundaries(range, source);
 
-    let parsed = parse(source, options.source_type().as_mode())?;
+    let parsed = parse(source, ParseOptions::from(options.source_type()))?;
     let source_code = SourceCode::new(source);
     let comment_ranges = CommentRanges::from(parsed.tokens());
     let comments = Comments::from_ast(parsed.syntax(), source_code, &comment_ranges);

crates/ruff_python_formatter/src/string/docstring.rs

@@ -11,6 +11,7 @@ use regex::Regex;
 
 use ruff_formatter::printer::SourceMapGeneration;
 use ruff_python_ast::{str::Quote, AnyStringFlags, StringFlags};
+use ruff_python_parser::ParseOptions;
 use ruff_python_trivia::CommentRanges;
 use {
     ruff_formatter::{write, FormatOptions, IndentStyle, LineWidth, Printed},
@@ -492,8 +493,6 @@ impl<'src> DocstringLinePrinter<'_, '_, '_, 'src> {
         &mut self,
         kind: &mut CodeExampleKind<'_>,
     ) -> FormatResult<Option<Vec<OutputDocstringLine<'static>>>> {
-        use ruff_python_parser::AsMode;
-
         let line_width = match self.f.options().docstring_code_line_width() {
             DocstringCodeLineWidth::Fixed(width) => width,
             DocstringCodeLineWidth::Dynamic => {
@@ -570,7 +569,8 @@ impl<'src> DocstringLinePrinter<'_, '_, '_, 'src> {
                 std::format!(r#""""{}""""#, printed.as_code())
             }
         };
-        let result = ruff_python_parser::parse(&wrapped, self.f.options().source_type().as_mode());
+        let result =
+            ruff_python_parser::parse(&wrapped, ParseOptions::from(self.f.options().source_type()));
         // If the resulting code is not valid, then reset and pass through
         // the docstring lines as-is.
         if result.is_err() {
@@ -1580,10 +1580,8 @@ fn docstring_format_source(
     docstring_quote_style: Quote,
     source: &str,
 ) -> Result<Printed, FormatModuleError> {
-    use ruff_python_parser::AsMode;
-
     let source_type = options.source_type();
-    let parsed = ruff_python_parser::parse(source, source_type.as_mode())?;
+    let parsed = ruff_python_parser::parse(source, ParseOptions::from(source_type))?;
     let comment_ranges = CommentRanges::from(parsed.tokens());
     let source_code = ruff_formatter::SourceCode::new(source);
     let comments = crate::Comments::from_ast(parsed.syntax(), source_code, &comment_ranges);