Reimplemented lexer with vectors instead of iterators

This commit is contained in:
Veetaha 2020-01-26 20:44:49 +02:00
parent ad24976da3
commit ac37a11f04
10 changed files with 254 additions and 200 deletions

View file

@ -10,7 +10,8 @@ use crate::{fuzz, SourceFile};
#[test]
fn lexer_tests() {
dir_tests(&test_data_dir(), &["lexer"], |text, _| {
let tokens = crate::tokenize(text);
// FIXME: add tests for errors (their format is up to discussion)
let tokens = crate::tokenize(text).tokens;
dump_tokens(&tokens, text)
})
}