mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-07-08 05:35:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix full-lexer feature (#38)

Browse source
This commit is contained in:
Jeong, YunWon 2023-05-16 15:45:03 +09:00 • committed by GitHub
parent 02f13abf50
commit 735c06d5f4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 6 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

6
parser/src/lexer.rs
View file

@ -641,9 +641,12 @@ where
}
Some('\n' | '\r') => {
// Empty line!
#[cfg(feature = "full-lexer")]
let tok_start = self.get_pos();
self.next_char();
#[cfg(feature = "full-lexer")]
let tok_end = self.get_pos();
#[cfg(feature = "full-lexer")]
self.emit((Tok::NonLogicalNewline, TextRange::new(tok_start, tok_end)));
spaces = 0;
tabs = 0;
@ -1506,6 +1509,7 @@ mod tests {
($($name:ident: $eol:expr,)*) => {
$(
#[test]
#[cfg(feature = "full-lexer")]
fn $name() {
let source = format!("def foo():{} return 99{}{}", $eol, $eol, $eol);
let tokens = lex_source(&source);
@ -1543,6 +1547,7 @@ mod tests {
($($name:ident: $eol:expr,)*) => {
$(
#[test]
#[cfg(feature = "full-lexer")]
fn $name() {
let source = format!("def foo():{} if x:{}{} return 99{}{}", $eol, $eol, $eol, $eol, $eol);
let tokens = lex_source(&source);
@ -1583,6 +1588,7 @@ mod tests {
($($name:ident: $eol:expr,)*) => {
$(
#[test]
#[cfg(feature = "full-lexer")]
fn $name() {
let source = format!("def foo():{}\tif x:{}{}\t return 99{}{}", $eol, $eol, $eol, $eol, $eol);
let tokens = lex_source(&source);