Expose a method to parse AST from tokens directly

parser/src/parser.rs

@@ -5,9 +5,11 @@
 //! parse a whole program, a single statement, or a single
 //! expression.
 
+use std::iter;
+
+use crate::lexer::LexResult;
 pub use crate::mode::Mode;
 use crate::{ast, error::ParseError, lexer, python};
-use std::iter;
 
 /*
  * Parse python code.
@@ -23,6 +25,17 @@ pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, Pars
     })
 }
 
+/// Parse the token stream for a full python program.
+pub fn parse_program_tokens(
+    lxr: impl IntoIterator<Item=LexResult>,
+    source_path: &str,
+) -> Result<ast::Suite, ParseError> {
+    parse_tokens(lxr, Mode::Module, source_path).map(|top| match top {
+        ast::Mod::Module { body, .. } => body,
+        _ => unreachable!(),
+    })
+}
+
 /// Parses a python expression
 ///
 /// # Example
@@ -80,6 +93,20 @@ pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, Pa
         .map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
 }
 
+// Parse a given token iterator.
+pub fn parse_tokens(
+    lxr: impl IntoIterator<Item=LexResult>,
+    mode: Mode,
+    source_path: &str,
+) -> Result<ast::Mod, ParseError> {
+    let marker_token = (Default::default(), mode.to_marker(), Default::default());
+    let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
+
+    python::TopParser::new()
+        .parse(tokenizer)
+        .map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;