Pass ParserOptions to the parser (#16220)

## Summary

This is part of the preparation for detecting syntax errors in the
parser from https://github.com/astral-sh/ruff/pull/16090/. As suggested
in [this
comment](https://github.com/astral-sh/ruff/pull/16090/#discussion_r1953084509),
I started working on a `ParseOptions` struct that could be stored in the
parser. For this initial refactor, I only made it hold the existing
`Mode` option, but for syntax errors, we will also need it to have a
`PythonVersion`. For that use case, I'm picturing something like a
`ParseOptions::with_python_version` method, so you can extend the
current calls to something like

```rust
ParseOptions::from(mode).with_python_version(settings.target_version)
```

But I thought it was worth adding `ParseOptions` alone without changing
any other behavior first.

Most of the diff is just updating call sites taking `Mode` to take
`ParseOptions::from(Mode)` or those taking `PySourceType`s to take
`ParseOptions::from(PySourceType)`. The interesting changes are in the
new `parser/options.rs` file and smaller parts of `parser/mod.rs` and
`ruff_python_parser/src/lib.rs`.

## Test Plan

Existing tests, this should not change any behavior.
This commit is contained in:
Brent Westbrook 2025-02-19 10:50:50 -05:00 committed by GitHub
parent cfc6941d5c
commit 97d0659ce3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
25 changed files with 148 additions and 93 deletions

View file

@ -13,8 +13,11 @@ use crate::token_source::{TokenSource, TokenSourceCheckpoint};
use crate::{Mode, ParseError, ParseErrorType, TokenKind};
use crate::{Parsed, Tokens};
pub use crate::parser::options::ParseOptions;
mod expression;
mod helpers;
mod options;
mod pattern;
mod progress;
mod recovery;
@ -32,8 +35,8 @@ pub(crate) struct Parser<'src> {
/// Stores all the syntax errors found during the parsing.
errors: Vec<ParseError>,
/// Specify the mode in which the code will be parsed.
mode: Mode,
/// Options for how the code will be parsed.
options: ParseOptions,
/// The ID of the current token. This is used to track the progress of the parser
/// to avoid infinite loops when the parser is stuck.
@ -51,16 +54,20 @@ pub(crate) struct Parser<'src> {
impl<'src> Parser<'src> {
/// Create a new parser for the given source code.
pub(crate) fn new(source: &'src str, mode: Mode) -> Self {
Parser::new_starts_at(source, mode, TextSize::new(0))
pub(crate) fn new(source: &'src str, options: ParseOptions) -> Self {
Parser::new_starts_at(source, TextSize::new(0), options)
}
/// Create a new parser for the given source code which starts parsing at the given offset.
pub(crate) fn new_starts_at(source: &'src str, mode: Mode, start_offset: TextSize) -> Self {
let tokens = TokenSource::from_source(source, mode, start_offset);
pub(crate) fn new_starts_at(
source: &'src str,
start_offset: TextSize,
options: ParseOptions,
) -> Self {
let tokens = TokenSource::from_source(source, options.mode, start_offset);
Parser {
mode,
options,
source,
errors: Vec::new(),
tokens,
@ -73,7 +80,7 @@ impl<'src> Parser<'src> {
/// Consumes the [`Parser`] and returns the parsed [`Parsed`].
pub(crate) fn parse(mut self) -> Parsed<Mod> {
let syntax = match self.mode {
let syntax = match self.options.mode {
Mode::Expression | Mode::ParenthesizedExpression => {
Mod::Expression(self.parse_single_expression())
}