From 262e768fd3886c04471e9d2dbd398df2ff6e6981 Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Wed, 22 Feb 2023 15:22:06 +0100 Subject: [PATCH] refactor(ruff): Implement `doc_lines_from_tokens` as iterator (#3124) This is a nit refactor... It implements the extraction of document lines as an iterator instead of a Vector to avoid the extra allocation. --- crates/ruff/src/doc_lines.rs | 64 +++++++++++++++++++-------- crates/ruff/src/fs.rs | 11 ----- crates/ruff/src/linter.rs | 2 +- crates/ruff/src/settings/pyproject.rs | 5 +-- crates/ruff/src/test.rs | 4 +- crates/ruff_cli/src/diagnostics.rs | 2 +- 6 files changed, 51 insertions(+), 37 deletions(-) diff --git a/crates/ruff/src/doc_lines.rs b/crates/ruff/src/doc_lines.rs index b849d33efc..f185a65f64 100644 --- a/crates/ruff/src/doc_lines.rs +++ b/crates/ruff/src/doc_lines.rs @@ -3,32 +3,58 @@ use rustpython_parser::ast::{Constant, ExprKind, Stmt, StmtKind, Suite}; use rustpython_parser::lexer::{LexResult, Tok}; +use std::iter::FusedIterator; use crate::ast::visitor; use crate::ast::visitor::Visitor; /// Extract doc lines (standalone comments) from a token sequence. -pub fn doc_lines_from_tokens(lxr: &[LexResult]) -> Vec { - let mut doc_lines: Vec = Vec::default(); - let mut prev: Option = None; - for (start, tok, end) in lxr.iter().flatten() { - if matches!(tok, Tok::Indent | Tok::Dedent | Tok::Newline) { - continue; - } - if matches!(tok, Tok::Comment(..)) { - if let Some(prev) = prev { - if start.row() > prev { - doc_lines.push(start.row()); - } - } else { - doc_lines.push(start.row()); - } - } - prev = Some(end.row()); - } - doc_lines +pub fn doc_lines_from_tokens(lxr: &[LexResult]) -> DocLines { + DocLines::new(lxr) } +pub struct DocLines<'a> { + inner: std::iter::Flatten>, + prev: Option, +} + +impl<'a> DocLines<'a> { + fn new(lxr: &'a [LexResult]) -> Self { + Self { + inner: lxr.iter().flatten(), + prev: None, + } + } +} + +impl Iterator for DocLines<'_> { + type Item = usize; + + fn next(&mut self) -> Option { + loop { + let (start, tok, end) = self.inner.next()?; + + match tok { + Tok::Indent | Tok::Dedent | Tok::Newline => continue, + Tok::Comment(..) => { + if let Some(prev) = self.prev { + if start.row() > prev { + break Some(start.row()); + } + } else { + break Some(start.row()); + } + } + _ => {} + } + + self.prev = Some(end.row()); + } + } +} + +impl FusedIterator for DocLines<'_> {} + #[derive(Default)] struct StringLinesVisitor { string_lines: Vec, diff --git a/crates/ruff/src/fs.rs b/crates/ruff/src/fs.rs index 3c747cfc13..7750a124a7 100644 --- a/crates/ruff/src/fs.rs +++ b/crates/ruff/src/fs.rs @@ -1,5 +1,3 @@ -use std::fs::File; -use std::io::{BufReader, Read}; use std::ops::Deref; use std::path::{Path, PathBuf}; @@ -88,12 +86,3 @@ pub fn relativize_path(path: impl AsRef) -> String { } format!("{}", path.display()) } - -/// Read a file's contents from disk. -pub fn read_file>(path: P) -> Result { - let file = File::open(path)?; - let mut buf_reader = BufReader::new(file); - let mut contents = String::new(); - buf_reader.read_to_string(&mut contents)?; - Ok(contents) -} diff --git a/crates/ruff/src/linter.rs b/crates/ruff/src/linter.rs index 5efde1fa1a..db6e361757 100644 --- a/crates/ruff/src/linter.rs +++ b/crates/ruff/src/linter.rs @@ -223,7 +223,7 @@ const MAX_ITERATIONS: usize = 100; /// Add any missing `# noqa` pragmas to the source code at the given `Path`. pub fn add_noqa_to_path(path: &Path, package: Option<&Path>, settings: &Settings) -> Result { // Read the file from disk. - let contents = fs::read_file(path)?; + let contents = std::fs::read_to_string(path)?; // Tokenize once. let tokens: Vec = rustpython_helpers::tokenize(&contents); diff --git a/crates/ruff/src/settings/pyproject.rs b/crates/ruff/src/settings/pyproject.rs index 7a6a88d7d2..e3207ba52b 100644 --- a/crates/ruff/src/settings/pyproject.rs +++ b/crates/ruff/src/settings/pyproject.rs @@ -5,7 +5,6 @@ use std::path::{Path, PathBuf}; use anyhow::{anyhow, Result}; use serde::{Deserialize, Serialize}; -use crate::fs; use crate::settings::options::Options; #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -30,13 +29,13 @@ impl Pyproject { /// Parse a `ruff.toml` file. fn parse_ruff_toml>(path: P) -> Result { - let contents = fs::read_file(path)?; + let contents = std::fs::read_to_string(path)?; toml::from_str(&contents).map_err(Into::into) } /// Parse a `pyproject.toml` file. fn parse_pyproject_toml>(path: P) -> Result { - let contents = fs::read_file(path)?; + let contents = std::fs::read_to_string(path)?; toml::from_str(&contents).map_err(Into::into) } diff --git a/crates/ruff/src/test.rs b/crates/ruff/src/test.rs index 542eb168ec..c1bc2fe445 100644 --- a/crates/ruff/src/test.rs +++ b/crates/ruff/src/test.rs @@ -12,7 +12,7 @@ use crate::packaging::detect_package_root; use crate::registry::Diagnostic; use crate::settings::{flags, Settings}; use crate::source_code::{Indexer, Locator, Stylist}; -use crate::{directives, fs, rustpython_helpers}; +use crate::{directives, rustpython_helpers}; pub fn test_resource_path(path: impl AsRef) -> std::path::PathBuf { Path::new("./resources/test/").join(path) @@ -22,7 +22,7 @@ pub fn test_resource_path(path: impl AsRef) -> std::path::PathBuf { /// asserts that autofixes converge after 10 iterations. pub fn test_path(path: &Path, settings: &Settings) -> Result> { let path = test_resource_path("fixtures").join(path); - let contents = fs::read_file(&path)?; + let contents = std::fs::read_to_string(&path)?; let tokens: Vec = rustpython_helpers::tokenize(&contents); let locator = Locator::new(&contents); let stylist = Stylist::from_contents(&contents, &locator); diff --git a/crates/ruff_cli/src/diagnostics.rs b/crates/ruff_cli/src/diagnostics.rs index e831de093a..9238acae7e 100644 --- a/crates/ruff_cli/src/diagnostics.rs +++ b/crates/ruff_cli/src/diagnostics.rs @@ -81,7 +81,7 @@ pub fn lint_path( }; // Read the file from disk. - let contents = fs::read_file(path)?; + let contents = std::fs::read_to_string(path)?; // Lint the file. let (