parser tests work

crates/parser/src/lib.rs

@@ -24,40 +24,11 @@ mod tokens;
 
 pub(crate) use token_set::TokenSet;
 
-pub use syntax_kind::SyntaxKind;
-
-use crate::tokens::Tokens;
+pub use crate::{syntax_kind::SyntaxKind, tokens::Tokens};
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct ParseError(pub Box<String>);
 
-/// `TokenSource` abstracts the source of the tokens parser operates on.
-///
-/// Hopefully this will allow us to treat text and token trees in the same way!
-pub trait TokenSource {
-    fn current(&self) -> Token;
-
-    /// Lookahead n token
-    fn lookahead_nth(&self, n: usize) -> Token;
-
-    /// bump cursor to next token
-    fn bump(&mut self);
-
-    /// Is the current token a specified keyword?
-    fn is_keyword(&self, kw: &str) -> bool;
-}
-
-/// `Token` abstracts the cursor of `TokenSource` operates on.
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
-pub struct Token {
-    /// What is the current token?
-    pub kind: SyntaxKind,
-
-    /// Is the current token joined to the next one (`> >` vs `>>`).
-    pub is_jointed_to_next: bool,
-    pub contextual_kw: SyntaxKind,
-}
-
 /// `TreeSink` abstracts details of a particular syntax tree implementation.
 pub trait TreeSink {
     /// Adds new token to the current branch.

crates/parser/src/syntax_kind/generated.rs

@@ -334,6 +334,18 @@ impl SyntaxKind {
         };
         Some(kw)
     }
+    pub fn from_contextual_keyword(ident: &str) -> Option<SyntaxKind> {
+        let kw = match ident {
+            "auto" => AUTO_KW,
+            "default" => DEFAULT_KW,
+            "existential" => EXISTENTIAL_KW,
+            "union" => UNION_KW,
+            "raw" => RAW_KW,
+            "macro_rules" => MACRO_RULES_KW,
+            _ => return None,
+        };
+        Some(kw)
+    }
     pub fn from_char(c: char) -> Option<SyntaxKind> {
         let tok = match c {
             ';' => SEMICOLON,

crates/parser/src/tokens.rs

@@ -1,8 +1,19 @@
-use crate::{SyntaxKind, Token};
+use crate::SyntaxKind;
 
 #[allow(non_camel_case_types)]
 type bits = u64;
 
+/// `Token` abstracts the cursor of `TokenSource` operates on.
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+pub(crate) struct Token {
+    /// What is the current token?
+    pub(crate) kind: SyntaxKind,
+
+    /// Is the current token joined to the next one (`> >` vs `>>`).
+    pub(crate) is_jointed_to_next: bool,
+    pub(crate) contextual_kw: SyntaxKind,
+}
+
 /// Main input to the parser.
 ///
 /// A sequence of tokens represented internally as a struct of arrays.
@@ -49,13 +60,14 @@ impl Tokens {
         self.kind.len()
     }
     pub(crate) fn get(&self, idx: usize) -> Token {
-        if idx > self.len() {
-            return self.eof();
+        if idx < self.len() {
+            let kind = self.kind[idx];
+            let is_jointed_to_next = self.get_joint(idx);
+            let contextual_kw = self.contextual_kw[idx];
+            Token { kind, is_jointed_to_next, contextual_kw }
+        } else {
+            self.eof()
         }
-        let kind = self.kind[idx];
-        let is_jointed_to_next = self.get_joint(idx);
-        let contextual_kw = self.contextual_kw[idx];
-        Token { kind, is_jointed_to_next, contextual_kw }
     }
 
     #[cold]