mirror of
https://github.com/astral-sh/ruff.git
synced 2025-09-27 20:42:10 +00:00
Implement an iterator for universal newlines (#3454)
# Summary We need to support CR line endings (as opposed to LF and CRLF line endings, which are already supported). They're rare, but they do appear in Python code, and we tend to panic on any file that uses them. Our `Locator` abstraction now supports CR line endings. However, Rust's `str#lines` implementation does _not_. This PR adds a `UniversalNewlineIterator` implementation that respects all of CR, LF, and CRLF line endings, and plugs it into most of the `.lines()` call sites. As an alternative design, it could be nice if we could leverage `Locator` for this. We've already computed all of the line endings, so we could probably iterate much more efficiently? # Test Plan Largely relying on automated testing, however, also ran over some known failure cases, like #3404.
This commit is contained in:
parent
2a4d6ab3b2
commit
c2750a59ab
35 changed files with 325 additions and 126 deletions
|
@ -1,5 +1,3 @@
|
|||
use std::str::Lines;
|
||||
|
||||
use rustpython_parser::ast::{Located, Location};
|
||||
|
||||
use crate::source_code::Locator;
|
||||
|
@ -39,38 +37,3 @@ pub fn clean(indentation: &str) -> String {
|
|||
.map(|char| if char.is_whitespace() { char } else { ' ' })
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Like `str#lines`, but includes a trailing newline as an empty line.
|
||||
pub struct LinesWithTrailingNewline<'a> {
|
||||
trailing: Option<&'a str>,
|
||||
underlying: Lines<'a>,
|
||||
}
|
||||
|
||||
impl<'a> LinesWithTrailingNewline<'a> {
|
||||
pub fn from(input: &'a str) -> LinesWithTrailingNewline<'a> {
|
||||
LinesWithTrailingNewline {
|
||||
underlying: input.lines(),
|
||||
trailing: if input.ends_with('\n') {
|
||||
Some("")
|
||||
} else {
|
||||
None
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Iterator for LinesWithTrailingNewline<'a> {
|
||||
type Item = &'a str;
|
||||
|
||||
#[inline]
|
||||
fn next(&mut self) -> Option<&'a str> {
|
||||
let mut next = self.underlying.next();
|
||||
if next.is_none() {
|
||||
if self.trailing.is_some() {
|
||||
next = self.trailing;
|
||||
self.trailing = None;
|
||||
}
|
||||
}
|
||||
next
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue