diff --git a/compiler/parse/tests/peg_grammar.rs b/compiler/parse/tests/peg_grammar.rs index 91db6d7c6d..1e1d389782 100644 --- a/compiler/parse/tests/peg_grammar.rs +++ b/compiler/parse/tests/peg_grammar.rs @@ -280,6 +280,7 @@ mod test_peg_grammar { skip } + // also skips lines that contain only whitespace fn skip_newlines(bytes: &[u8]) -> (usize, usize) { let mut skip = 0; let mut indent = 0; @@ -293,8 +294,13 @@ mod test_peg_grammar { } else { 0 }; - - indent = spaces; + + if bytes.len() > (skip + spaces) && bytes[skip + spaces] == b'\n' { + indent = 0; + skip += spaces; + } else { + indent = spaces; + } } (skip, indent) @@ -477,6 +483,28 @@ fn test_indent_tokenization_2() { ); } +#[test] +fn test_tokenization_line_with_only_spaces() { + let tokens = test_tokenize(r#"\key -> + when dict is + Empty -> + 4 + + Node -> + 5"#); + + assert_eq!( + tokens, + [T::LambdaStart, T::LowercaseIdent, T::Arrow, + T::OpenIndent, T::KeywordWhen, T::LowercaseIdent, T::KeywordIs, + T::OpenIndent, T::UppercaseIdent, T::Arrow, + T::OpenIndent, T::Number, + T::CloseIndent, + T::UppercaseIdent, T::Arrow, + T::OpenIndent, T::Number] + ); +} + // Inspired by https://ziglang.org/documentation/0.7.1/#Grammar // license information can be found in the LEGAL_DETAILS file in // the root directory of this distribution.