From 1ce5ec49c583b74828b841faa109262025490fc3 Mon Sep 17 00:00:00 2001 From: Tad Hardesty Date: Wed, 1 Nov 2017 21:49:33 -0700 Subject: [PATCH] Expand the parser's documentation --- dreammaker/ast.rs | 80 +++++++++++++++++++++++++++++++---------- dreammaker/constants.rs | 33 ++++++++++------- dreammaker/lexer.rs | 9 ++++- dreammaker/lib.rs | 31 ++++++++++++++-- dreammaker/parser.rs | 14 ++++++-- 5 files changed, 132 insertions(+), 35 deletions(-) diff --git a/dreammaker/ast.rs b/dreammaker/ast.rs index 7f381352..327d16fb 100644 --- a/dreammaker/ast.rs +++ b/dreammaker/ast.rs @@ -1,7 +1,11 @@ +//! The DM abstract syntax tree. +//! +//! Most AST types can be pretty-printed using the `Display` trait. use std::fmt; use linked_hash_map::LinkedHashMap; +/// The unary operators, both prefix and postfix. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum UnaryOp { Neg, @@ -13,10 +17,16 @@ pub enum UnaryOp { PostDecr, } +/// The DM path operators. +/// +/// Which path operator is used typically only matters at the start of a path. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum PathOp { + /// `/` for absolute pathing. Slash, + /// `.` for checked relative pathing. Dot, + /// `:` for unchecked relative pathing. Colon, } @@ -30,35 +40,34 @@ impl fmt::Display for PathOp { } } +/// A series of identifiers separated by path operators. +pub type TypePath = Vec<(PathOp, String)>; + +/// The binary operators. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum BinaryOp { - Pow, Add, Sub, Mul, Div, + Pow, Mod, + Eq, + NotEq, Less, Greater, LessEq, GreaterEq, - LShift, - RShift, - Eq, - NotEq, BitAnd, BitXor, BitOr, + LShift, + RShift, And, Or, } -impl BinaryOp { - pub fn assignop(self) -> Option { - None // TODO - } -} - +/// The assignment operators, including augmented assignment. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum AssignOp { Assign, @@ -73,14 +82,42 @@ pub enum AssignOp { RShiftAssign, } -impl AssignOp { - pub fn binop(self) -> Option { - None // TODO +macro_rules! augmented { + ($($bin:ident = $aug:ident;)*) => { + impl BinaryOp { + /// Get the corresponding augmented assignment operator, if available. + pub fn assign_op(self) -> Option { + match self { + $(BinaryOp::$bin => Some(AssignOp::$aug),)* + _ => None, + } + } + } + + impl AssignOp { + /// Get the corresponding binary operator, if available. + pub fn binary_op(self) -> Option { + match self { + $(AssignOp::$aug => Some(BinaryOp::$bin),)* + _ => None, + } + } + } } } +augmented! { + Add = AddAssign; + Sub = SubAssign; + Mul = MulAssign; + Div = DivAssign; + BitAnd = BitAndAssign; + BitOr = BitOrAssign; + BitXor = BitXorAssign; + LShift = LShiftAssign; + RShift = RShiftAssign; +} -pub type TypePath = Vec<(PathOp, String)>; - +/// A path optionally followed by a set of variables. #[derive(Clone, PartialEq, Debug)] pub struct Prefab { pub path: TypePath, @@ -108,10 +145,14 @@ impl fmt::Display for Prefab { } } +/// The different forms of the `new` command. #[derive(Clone, PartialEq, Debug)] pub enum NewType { + /// Implicit type, taken from context. Implicit, + /// The name of a variable in which to find the prefab to instantiate. Ident(String), + /// A prefab to be instantiated. Prefab(Prefab), } @@ -125,6 +166,7 @@ impl fmt::Display for NewType { } } +/// The structure of an expression, a tree of terms and operators. #[derive(Clone, PartialEq, Debug)] pub enum Expression { /// An expression containing a term directly. The term is evaluated first, @@ -174,10 +216,12 @@ pub enum Term { Null, /// A `new` call. New { + /// The type to be instantiated. type_: NewType, + /// The list of arguments to pass to the `New()` proc. args: Option>, }, - /// A `list` call. + /// A `list` call. Elements have optional associations. List(Vec<(Expression, Option)>), /// An unscoped function call. Call(String, Vec), @@ -213,7 +257,7 @@ impl From for Term { } } -/// A "follow", an expression part which is applied to a term or another follow. +/// An expression part which is applied to a term or another follow. #[derive(Debug, Clone, PartialEq)] pub enum Follow { /// Access a field of the value. diff --git a/dreammaker/constants.rs b/dreammaker/constants.rs index a1fd318e..74291d82 100644 --- a/dreammaker/constants.rs +++ b/dreammaker/constants.rs @@ -7,6 +7,8 @@ use super::{DMError, Location, HasLocation}; use super::objtree::*; use super::ast::*; +/// Evaluate all the type-level variables in an object tree into constants. +#[doc(hidden)] pub fn evaluate_all(tree: &mut ObjectTree) -> Result<(), DMError> { for ty in tree.graph.node_indices() { let keys: Vec = tree.graph.node_weight(ty).unwrap().vars.keys().cloned().collect(); @@ -23,6 +25,7 @@ pub fn evaluate_all(tree: &mut ObjectTree) -> Result<(), DMError> { Ok(()) } +/// Evaluate an expression in the absence of any surrounding context. pub fn simple_evaluate(expr: Expression) -> Result { ConstantFolder { tree: None, location: Location::default(), ty: NodeIndex::new(0) }.expr(expr, None) } @@ -313,10 +316,14 @@ impl<'a> ConstantFolder<'a> { } } +/// The constant functions which are represented as-is. #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum ConstFn { + /// The `icon()` type constructor. Icon, + /// The `matrix()` type constructor. Matrix, + /// The `newlist()` function, which combines `new` mapped over a `list`. Newlist, } @@ -330,20 +337,34 @@ impl fmt::Display for ConstFn { } } +/// A DM constant, usually a literal or simple combination of other constants. +/// +/// This is intended to represent the degree to which constants are evaluated +/// before being displayed in DreamMaker. #[derive(Debug, Clone, PartialEq)] pub enum Constant { /// The literal `null`. Null(Option), + /// A `new` call. New { + /// The type to be instantiated. type_: NewType, + /// The list of arugments to pass to the `New()` proc. args: Option>, }, + /// A `list` literal. Elements have optional associations. List(Vec<(Constant, Option)>), + /// A call to a constant type constructor. Call(ConstFn, Vec), + /// A prefab literal. Prefab(Prefab), + /// A string literal. String(String), + /// A resource literal. Resource(String), + /// An integer literal. Int(i32), + /// A floating-point literal. Float(f32), } @@ -482,15 +503,3 @@ impl fmt::Display for Constant { } } } - -#[derive(Debug, Clone, PartialEq)] -pub struct TypedConstant { - pub type_hint: Option, - pub constant: Constant, -} - -impl fmt::Display for TypedConstant { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - self.constant.fmt(f) - } -} diff --git a/dreammaker/lexer.rs b/dreammaker/lexer.rs index 0d0e1f77..bd1b41f4 100644 --- a/dreammaker/lexer.rs +++ b/dreammaker/lexer.rs @@ -31,6 +31,9 @@ macro_rules! table { // (paren) {brace} [bracket] table! { /// A punctuation token recognized by the language. + /// + /// Not all punctuation types will actually appear in the lexer's output; + /// some (such as comments) are handled internally. table PUNCT_TABLE: &'static [u8] => Punctuation; // Order is significant; see read_punct below. b"\t", Tab; @@ -92,6 +95,7 @@ table! { b"~", BitNot; } +/// A single DM token. #[derive(Clone, Debug, PartialEq)] pub enum Token { /// An end-of-file. Exists as a convenience and is not emitted by the lexer. @@ -102,9 +106,11 @@ pub enum Token { Ident(String, bool), /// A string literal with no interpolation. String(String), - /// Interpolation markers. Strings and expressions in-between are combined. + /// The opening portion of an interpolated string. Followed by an expression. InterpStringBegin(String), + /// An internal portion of an interpolated string. Preceded and followed by an expression. InterpStringPart(String), + /// The closing portion of an interpolated string. Preceded by an expression. InterpStringEnd(String), /// A resource literal, referring to a filename. Resource(String), @@ -190,6 +196,7 @@ impl fmt::Display for Token { } } +/// A token with a location attached. #[derive(Clone, Debug, PartialEq)] pub struct LocatedToken { pub location: Location, diff --git a/dreammaker/lib.rs b/dreammaker/lib.rs index badc279e..0450ad4e 100644 --- a/dreammaker/lib.rs +++ b/dreammaker/lib.rs @@ -1,4 +1,4 @@ -//! DreamMaker code parsing suite +//! Parsing suite for DreamMaker, the language of the BYOND game engine. #[cfg(feature="xml-rs")] extern crate xml; extern crate petgraph; extern crate linked_hash_map; @@ -22,9 +22,12 @@ pub mod indents; pub mod parser; pub mod ast; pub mod objtree; -pub mod builtins; +mod builtins; pub mod constants; +/// Run the parsing suite on a given `.dme` file, producing an object tree. +/// +/// Errors are automatically pretty-printed to stdout before they are returned. pub fn parse_environment(dme: &Path) -> Result { let mut preprocessor = preprocessor::Preprocessor::new(dme.to_owned()).unwrap(); parser::parse(indents::IndentProcessor::new(&mut preprocessor)).map_err(|e| { @@ -36,6 +39,7 @@ pub fn parse_environment(dme: &Path) -> Result { // ---------------------------------------------------------------------------- // Error handling +/// An error produced during DM parsing, with location information. #[derive(Debug)] pub struct DMError { location: Location, @@ -44,6 +48,7 @@ pub struct DMError { #[allow(unused_variables)] impl DMError { + #[doc(hidden)] pub fn new>(location: Location, desc: S) -> DMError { DMError { location, @@ -56,6 +61,16 @@ impl DMError { { Self::new(location, desc) // TODO } + + /// Get the location in the code at which this error was observed. + pub fn location(&self) -> Location { + self.location + } + + /// Get the description associated with this error. + pub fn description(&self) -> &str { + &self.desc + } } impl From for DMError { @@ -67,17 +82,24 @@ impl From for DMError { // ---------------------------------------------------------------------------- // Location handling +/// File, line, and column information for an error. #[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Default)] pub struct Location { + /// The index into the file table. pub file: u32, + /// The line number, starting at 1. pub line: u32, + /// The column number, starting at 1. pub column: u32, } +/// A trait for types which may yield location information. pub trait HasLocation { + /// Get the current location of this parsing stage. fn location(&self) -> Location; #[inline] + #[doc(hidden)] fn error>(&self, message: S) -> DMError { DMError::new(self.location(), message) } @@ -94,6 +116,10 @@ impl<'a, T: HasLocation> HasLocation for &'a mut T { // ---------------------------------------------------------------------------- // Pretty printing +/// Pretty-print a series of tokens to the given output. +/// +/// If `show_ws` is true, braces and semicolons are included directly in the +/// output rather than only being implied by the indentation. pub fn pretty_print(w: &mut W, input: I, show_ws: bool) -> Result<(), DMError> where W: io::Write, I: IntoIterator> @@ -144,6 +170,7 @@ pub fn pretty_print(w: &mut W, input: I, show_ws: bool) -> Result<(), DMEr Ok(()) } +/// Pretty-print a `DMError` to the given output. pub fn pretty_print_error(w: &mut W, pp: &preprocessor::Preprocessor, error: &DMError) -> io::Result<()> { writeln!(w, "\n{}, line {}, column {}:", pp.file_path(error.location.file).display(), diff --git a/dreammaker/parser.rs b/dreammaker/parser.rs index e4a6a92a..d0740a1d 100644 --- a/dreammaker/parser.rs +++ b/dreammaker/parser.rs @@ -7,9 +7,10 @@ use super::lexer::{LocatedToken, Token, Punctuation}; use super::objtree::ObjectTree; use super::ast::*; +/// Parse a token stream, in the form emitted by the indent processor, into +/// an object tree. pub fn parse(iter: I) -> Result where - I: IntoIterator>, - I::IntoIter: HasLocation + I: IntoIterator> { let mut parser = Parser::new(iter.into_iter()); let mut tree = match parser.root()? { @@ -186,6 +187,10 @@ oper_table! { BINARY_OPS; // ---------------------------------------------------------------------------- // The parser +/// A single-lookahead, recursive-descent DM parser. +/// +/// Results are accumulated into an inner `ObjectTree`. To parse an entire +/// environment, use the `parse` or `parse_environment` functions. pub struct Parser { tree: ObjectTree, @@ -205,6 +210,7 @@ impl HasLocation for Parser { impl Parser where I: Iterator> { + /// Construct a new parser using the given input stream. pub fn new(input: I) -> Parser { Parser { tree: ObjectTree::with_builtins(), @@ -465,6 +471,10 @@ impl Parser where success(Prefab { path: parts, vars }) } + /// Parse an expression at the current position. + /// + /// If `disallow_assign` is set, assignment operators are not considered. + /// This is useful when parsing the left-hand side of a list association. pub fn expression(&mut self, disallow_assign: bool) -> Status { let mut expr = leading!(self.group()); loop {