mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-08-25 12:55:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix full-lexer feature (#38)

Browse source
This commit is contained in:
Jeong, YunWon 2023-05-16 15:45:03 +09:00 • committed by GitHub
parent 02f13abf50
commit 735c06d5f4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 6 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

6
parser/src/lexer.rs
View file

@ -641,9 +641,12 @@ where
} }
Some('\n' | '\r') => { Some('\n' | '\r') => {
// Empty line! // Empty line!
#[cfg(feature = "full-lexer")]
let tok_start = self.get_pos(); let tok_start = self.get_pos();
self.next_char(); self.next_char();
#[cfg(feature = "full-lexer")]
let tok_end = self.get_pos(); let tok_end = self.get_pos();
#[cfg(feature = "full-lexer")]
self.emit((Tok::NonLogicalNewline, TextRange::new(tok_start, tok_end))); self.emit((Tok::NonLogicalNewline, TextRange::new(tok_start, tok_end)));
spaces = 0; spaces = 0;
tabs = 0; tabs = 0;
@ -1506,6 +1509,7 @@ mod tests {
($($name:ident: $eol:expr,)*) => { ($($name:ident: $eol:expr,)*) => {
$( $(
#[test] #[test]
#[cfg(feature = "full-lexer")]
fn $name() { fn $name() {
let source = format!("def foo():{} return 99{}{}", $eol, $eol, $eol); let source = format!("def foo():{} return 99{}{}", $eol, $eol, $eol);
let tokens = lex_source(&source); let tokens = lex_source(&source);
@ -1543,6 +1547,7 @@ mod tests {
($($name:ident: $eol:expr,)*) => { ($($name:ident: $eol:expr,)*) => {
$( $(
#[test] #[test]
#[cfg(feature = "full-lexer")]
fn $name() { fn $name() {
let source = format!("def foo():{} if x:{}{} return 99{}{}", $eol, $eol, $eol, $eol, $eol); let source = format!("def foo():{} if x:{}{} return 99{}{}", $eol, $eol, $eol, $eol, $eol);
let tokens = lex_source(&source); let tokens = lex_source(&source);
@ -1583,6 +1588,7 @@ mod tests {
($($name:ident: $eol:expr,)*) => { ($($name:ident: $eol:expr,)*) => {
$( $(
#[test] #[test]
#[cfg(feature = "full-lexer")]
fn $name() { fn $name() {
let source = format!("def foo():{}\tif x:{}{}\t return 99{}{}", $eol, $eol, $eol, $eol, $eol); let source = format!("def foo():{}\tif x:{}{}\t return 99{}{}", $eol, $eol, $eol, $eol, $eol);
let tokens = lex_source(&source); let tokens = lex_source(&source);