mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-20 02:20:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Approximate tokens len (#9546)

Browse source
This commit is contained in:
Micha Reiser 2024-01-19 17:39:37 +01:00 • committed by GitHub
parent b3a6f0ce81
commit 47ad7b4500
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 38 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

4
crates/ruff_wasm/src/lib.rs
View file

@ -17,7 +17,7 @@ use ruff_python_codegen::Stylist;
use ruff_python_formatter::{format_module_ast, pretty_comments, PyFormatContext, QuoteStyle};
use ruff_python_index::{CommentRangesBuilder, Indexer};
use ruff_python_parser::lexer::LexResult;
use ruff_python_parser::{parse_tokens, AsMode, Mode};
use ruff_python_parser::{parse_tokens, tokenize_all, AsMode, Mode};
use ruff_python_trivia::CommentRanges;
use ruff_source_file::{Locator, SourceLocation};
use ruff_text_size::Ranged;
@ -272,7 +272,7 @@ struct ParsedModule<'a> {
impl<'a> ParsedModule<'a> {
fn from_source(source_code: &'a str) -> Result<Self, Error> {
let tokens: Vec<_> = ruff_python_parser::lexer::lex(source_code, Mode::Module).collect();
let tokens: Vec<_> = tokenize_all(source_code, Mode::Module);
let mut comment_ranges = CommentRangesBuilder::default();
for (token, range) in tokens.iter().flatten() {