Reimplemented lexer with vectors instead of iterators

This commit is contained in:
Veetaha 2020-01-26 20:44:49 +02:00
parent ad24976da3
commit ac37a11f04
10 changed files with 254 additions and 200 deletions

View file

@ -41,7 +41,7 @@ use crate::syntax_node::GreenNode;
pub use crate::{
algo::InsertPosition,
ast::{AstNode, AstToken},
parsing::{classify_literal, tokenize, Token},
parsing::{first_token, single_token, tokenize, tokenize_append, Token, TokenizeError},
ptr::{AstPtr, SyntaxNodePtr},
syntax_error::{Location, SyntaxError, SyntaxErrorKind},
syntax_node::{