ra_syntax: refactored the lexer design as per @matklad and @kiljacken PR review

This commit is contained in:
Veetaha 2020-01-28 07:09:13 +02:00
parent bf60661aa3
commit 9e7eaa959f
9 changed files with 202 additions and 181 deletions

View file

@ -11,7 +11,7 @@ use crate::{fuzz, SourceFile};
fn lexer_tests() {
dir_tests(&test_data_dir(), &["lexer"], |text, _| {
// FIXME: add tests for errors (their format is up to discussion)
let tokens = crate::tokenize(text).tokens;
let (tokens, _errors) = crate::tokenize(text);
dump_tokens(&tokens, text)
})
}