mirrors/LibCST
1
0
Fork 0
mirror of https://github.com/Instagram/LibCST.git synced 2025-12-23 10:35:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Make sure dedents are emitted for inputs without trailing newlines (#573)

Browse source
This commit is contained in:
Zsolt Dollenstein 2022-01-04 20:04:21 +00:00 • committed by GitHub
parent cff47b767b
commit 86431eea89
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 57 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

15
native/libcst/src/tokenizer/core/mod.rs
View file

@ -342,7 +342,20 @@ impl<'t> TokState<'t> {
self.missing_nl_before_eof = false;
Ok(TokType::Newline)
} else {
Ok(TokType::EndMarker)
let hanging_indents = self.indent_stack.len() as i32;
if self.pending_indents == 0 && hanging_indents != 0 {
// We've reached EOF but there are still pending indents not
// accounted for. Flush them out.
self.pending_indents = -hanging_indents;
self.indent_stack.clear();
self.alt_indent_stack.clear();
self.missing_nl_before_eof = false;
}
if let Some(t) = self.process_pending_indents() {
Ok(t)
} else {
Ok(TokType::EndMarker)
}
}
}

40
native/libcst/src/tokenizer/tests.rs
View file

@ -687,3 +687,43 @@ fn test_add_dedents_after_fake_newline() {
])
);
}
#[test]
fn test_add_dedents_for_dangling_indent() {
assert_eq!(
tokenize_with_end_marker("if 1:\n if 2:\n ", &default_config()),
Ok(vec![
(TokType::Name, "if"),
(TokType::Number, "1"),
(TokType::Op, ":"),
(TokType::Newline, "\n"),
(TokType::Indent, ""),
(TokType::Name, "if"),
(TokType::Number, "2"),
(TokType::Op, ":"),
(TokType::Newline, "\n"),
(TokType::Dedent, ""),
(TokType::EndMarker, "")
])
);
}
#[test]
fn test_add_dedents_for_dangling_indent_with_comment() {
assert_eq!(
tokenize_with_end_marker("if 1:\n if 2:\n # foo", &default_config()),
Ok(vec![
(TokType::Name, "if"),
(TokType::Number, "1"),
(TokType::Op, ":"),
(TokType::Newline, "\n"),
(TokType::Indent, ""),
(TokType::Name, "if"),
(TokType::Number, "2"),
(TokType::Op, ":"),
(TokType::Newline, "\n"),
(TokType::Dedent, ""),
(TokType::EndMarker, "")
])
);
}

3
native/libcst/tests/fixtures/dangling_indent.py vendored Normal file
View file

@ -0,0 +1,3 @@
if 1:
pass