Use CommentRanges in backwards lexing (#7360)

## Summary

The tokenizer was split into a forward and a backwards tokenizer. The
backwards tokenizer uses the same names as the forwards ones (e.g.
`next_token`). The backwards tokenizer gets the comment ranges that we
already built to skip comments.

---------

Co-authored-by: Micha Reiser <micha@reiser.io>
This commit is contained in:
konsti 2023-09-16 05:21:45 +02:00 committed by GitHub
parent 1f6e1485f9
commit 2cbe1733c8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
41 changed files with 744 additions and 628 deletions

View file

@ -16,10 +16,10 @@ use ruff_formatter::{FormatResult, Formatted, LineWidth};
use ruff_python_ast::{Mod, PySourceType};
use ruff_python_codegen::Stylist;
use ruff_python_formatter::{format_node, pretty_comments, PyFormatContext, PyFormatOptions};
use ruff_python_index::{CommentRanges, CommentRangesBuilder, Indexer};
use ruff_python_index::{CommentRangesBuilder, Indexer};
use ruff_python_parser::lexer::LexResult;
use ruff_python_parser::AsMode;
use ruff_python_parser::{parse_tokens, Mode};
use ruff_python_parser::{parse_tokens, AsMode, Mode};
use ruff_python_trivia::CommentRanges;
use ruff_source_file::{Locator, SourceLocation};
use ruff_text_size::Ranged;
use ruff_workspace::configuration::Configuration;
@ -116,7 +116,7 @@ impl Workspace {
Ok(Workspace { settings })
}
#[wasm_bindgen(js_name=defaultSettings)]
#[wasm_bindgen(js_name = defaultSettings)]
pub fn default_settings() -> Result<JsValue, Error> {
serde_wasm_bindgen::to_value(&Options {
// Propagate defaults.