Expand the parser's documentation

dreammaker/ast.rs

@@ -1,7 +1,11 @@
+//! The DM abstract syntax tree.
+//!
+//! Most AST types can be pretty-printed using the `Display` trait.
 use std::fmt;
 
 use linked_hash_map::LinkedHashMap;
 
+/// The unary operators, both prefix and postfix.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum UnaryOp {
     Neg,
@@ -13,10 +17,16 @@ pub enum UnaryOp {
     PostDecr,
 }
 
+/// The DM path operators.
+///
+/// Which path operator is used typically only matters at the start of a path.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum PathOp {
+    /// `/` for absolute pathing.
     Slash,
+    /// `.` for checked relative pathing.
     Dot,
+    /// `:` for unchecked relative pathing.
     Colon,
 }
 
@@ -30,35 +40,34 @@ impl fmt::Display for PathOp {
     }
 }
 
+/// A series of identifiers separated by path operators.
+pub type TypePath = Vec<(PathOp, String)>;
+
+/// The binary operators.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum BinaryOp {
-    Pow,
     Add,
     Sub,
     Mul,
     Div,
+    Pow,
     Mod,
+    Eq,
+    NotEq,
     Less,
     Greater,
     LessEq,
     GreaterEq,
-    LShift,
-    RShift,
-    Eq,
-    NotEq,
     BitAnd,
     BitXor,
     BitOr,
+    LShift,
+    RShift,
     And,
     Or,
 }
 
-impl BinaryOp {
-    pub fn assignop(self) -> Option<AssignOp> {
-        None  // TODO
-    }
-}
-
+/// The assignment operators, including augmented assignment.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub enum AssignOp {
     Assign,
@@ -73,14 +82,42 @@ pub enum AssignOp {
     RShiftAssign,
 }
 
-impl AssignOp {
-    pub fn binop(self) -> Option<BinaryOp> {
-        None  // TODO
+macro_rules! augmented {
+    ($($bin:ident = $aug:ident;)*) => {
+        impl BinaryOp {
+            /// Get the corresponding augmented assignment operator, if available.
+            pub fn assign_op(self) -> Option<AssignOp> {
+                match self {
+                    $(BinaryOp::$bin => Some(AssignOp::$aug),)*
+                    _ => None,
+                }
+            }
+        }
+
+        impl AssignOp {
+            /// Get the corresponding binary operator, if available.
+            pub fn binary_op(self) -> Option<BinaryOp> {
+                match self {
+                    $(AssignOp::$aug => Some(BinaryOp::$bin),)*
+                    _ => None,
+                }
+            }
+        }
     }
 }
+augmented! {
+    Add = AddAssign;
+    Sub = SubAssign;
+    Mul = MulAssign;
+    Div = DivAssign;
+    BitAnd = BitAndAssign;
+    BitOr = BitOrAssign;
+    BitXor = BitXorAssign;
+    LShift = LShiftAssign;
+    RShift = RShiftAssign;
+}
 
-pub type TypePath = Vec<(PathOp, String)>;
-
+/// A path optionally followed by a set of variables.
 #[derive(Clone, PartialEq, Debug)]
 pub struct Prefab<E=Expression> {
     pub path: TypePath,
@@ -108,10 +145,14 @@ impl<E: fmt::Display> fmt::Display for Prefab<E> {
     }
 }
 
+/// The different forms of the `new` command.
 #[derive(Clone, PartialEq, Debug)]
 pub enum NewType<E=Expression> {
+    /// Implicit type, taken from context.
     Implicit,
+    /// The name of a variable in which to find the prefab to instantiate.
     Ident(String),
+    /// A prefab to be instantiated.
     Prefab(Prefab<E>),
 }
 
@@ -125,6 +166,7 @@ impl<E: fmt::Display> fmt::Display for NewType<E> {
     }
 }
 
+/// The structure of an expression, a tree of terms and operators.
 #[derive(Clone, PartialEq, Debug)]
 pub enum Expression {
     /// An expression containing a term directly. The term is evaluated first,
@@ -174,10 +216,12 @@ pub enum Term {
     Null,
     /// A `new` call.
     New {
+        /// The type to be instantiated.
         type_: NewType,
+        /// The list of arguments to pass to the `New()` proc.
         args: Option<Vec<Expression>>,
     },
-    /// A `list` call.
+    /// A `list` call. Elements have optional associations.
     List(Vec<(Expression, Option<Expression>)>),
     /// An unscoped function call.
     Call(String, Vec<Expression>),
@@ -213,7 +257,7 @@ impl From<Expression> for Term {
     }
 }
 
-/// A "follow", an expression part which is applied to a term or another follow.
+/// An expression part which is applied to a term or another follow.
 #[derive(Debug, Clone, PartialEq)]
 pub enum Follow {
     /// Access a field of the value.

dreammaker/constants.rs

@@ -7,6 +7,8 @@ use super::{DMError, Location, HasLocation};
 use super::objtree::*;
 use super::ast::*;
 
+/// Evaluate all the type-level variables in an object tree into constants.
+#[doc(hidden)]
 pub fn evaluate_all(tree: &mut ObjectTree) -> Result<(), DMError> {
     for ty in tree.graph.node_indices() {
         let keys: Vec<String> = tree.graph.node_weight(ty).unwrap().vars.keys().cloned().collect();
@@ -23,6 +25,7 @@ pub fn evaluate_all(tree: &mut ObjectTree) -> Result<(), DMError> {
     Ok(())
 }
 
+/// Evaluate an expression in the absence of any surrounding context.
 pub fn simple_evaluate(expr: Expression) -> Result<Constant, DMError> {
     ConstantFolder { tree: None, location: Location::default(), ty: NodeIndex::new(0) }.expr(expr, None)
 }
@@ -313,10 +316,14 @@ impl<'a> ConstantFolder<'a> {
     }
 }
 
+/// The constant functions which are represented as-is.
 #[derive(Debug, Clone, Copy, Eq, PartialEq)]
 pub enum ConstFn {
+    /// The `icon()` type constructor.
     Icon,
+    /// The `matrix()` type constructor.
     Matrix,
+    /// The `newlist()` function, which combines `new` mapped over a `list`.
     Newlist,
 }
 
@@ -330,20 +337,34 @@ impl fmt::Display for ConstFn {
     }
 }
 
+/// A DM constant, usually a literal or simple combination of other constants.
+///
+/// This is intended to represent the degree to which constants are evaluated
+/// before being displayed in DreamMaker.
 #[derive(Debug, Clone, PartialEq)]
 pub enum Constant {
     /// The literal `null`.
     Null(Option<TypePath>),
+    /// A `new` call.
     New {
+        /// The type to be instantiated.
         type_: NewType<Constant>,
+        /// The list of arugments to pass to the `New()` proc.
         args: Option<Vec<Constant>>,
     },
+    /// A `list` literal. Elements have optional associations.
     List(Vec<(Constant, Option<Constant>)>),
+    /// A call to a constant type constructor.
     Call(ConstFn, Vec<Constant>),
+    /// A prefab literal.
     Prefab(Prefab<Constant>),
+    /// A string literal.
     String(String),
+    /// A resource literal.
     Resource(String),
+    /// An integer literal.
     Int(i32),
+    /// A floating-point literal.
     Float(f32),
 }
 
@@ -482,15 +503,3 @@ impl fmt::Display for Constant {
         }
     }
 }
-
-#[derive(Debug, Clone, PartialEq)]
-pub struct TypedConstant {
-    pub type_hint: Option<TypePath>,
-    pub constant: Constant,
-}
-
-impl fmt::Display for TypedConstant {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        self.constant.fmt(f)
-    }
-}

dreammaker/lexer.rs

@@ -31,6 +31,9 @@ macro_rules! table {
 // (paren) {brace} [bracket]
 table! {
     /// A punctuation token recognized by the language.
+    ///
+    /// Not all punctuation types will actually appear in the lexer's output;
+    /// some (such as comments) are handled internally.
     table PUNCT_TABLE: &'static [u8] => Punctuation;
     // Order is significant; see read_punct below.
     b"\t",  Tab;
@@ -92,6 +95,7 @@ table! {
     b"~",	BitNot;
 }
 
+/// A single DM token.
 #[derive(Clone, Debug, PartialEq)]
 pub enum Token {
     /// An end-of-file. Exists as a convenience and is not emitted by the lexer.
@@ -102,9 +106,11 @@ pub enum Token {
     Ident(String, bool),
     /// A string literal with no interpolation.
     String(String),
-    /// Interpolation markers. Strings and expressions in-between are combined.
+    /// The opening portion of an interpolated string. Followed by an expression.
     InterpStringBegin(String),
+    /// An internal portion of an interpolated string. Preceded and followed by an expression.
     InterpStringPart(String),
+    /// The closing portion of an interpolated string. Preceded by an expression.
     InterpStringEnd(String),
     /// A resource literal, referring to a filename.
     Resource(String),
@@ -190,6 +196,7 @@ impl fmt::Display for Token {
     }
 }
 
+/// A token with a location attached.
 #[derive(Clone, Debug, PartialEq)]
 pub struct LocatedToken {
     pub location: Location,

dreammaker/lib.rs

@@ -1,4 +1,4 @@
-//! DreamMaker code parsing suite
+//! Parsing suite for DreamMaker, the language of the BYOND game engine.
 #[cfg(feature="xml-rs")] extern crate xml;
 extern crate petgraph;
 extern crate linked_hash_map;
@@ -22,9 +22,12 @@ pub mod indents;
 pub mod parser;
 pub mod ast;
 pub mod objtree;
-pub mod builtins;
+mod builtins;
 pub mod constants;
 
+/// Run the parsing suite on a given `.dme` file, producing an object tree.
+///
+/// Errors are automatically pretty-printed to stdout before they are returned.
 pub fn parse_environment(dme: &Path) -> Result<objtree::ObjectTree, DMError> {
     let mut preprocessor = preprocessor::Preprocessor::new(dme.to_owned()).unwrap();
     parser::parse(indents::IndentProcessor::new(&mut preprocessor)).map_err(|e| {
@@ -36,6 +39,7 @@ pub fn parse_environment(dme: &Path) -> Result<objtree::ObjectTree, DMError> {
 // ----------------------------------------------------------------------------
 // Error handling
 
+/// An error produced during DM parsing, with location information.
 #[derive(Debug)]
 pub struct DMError {
     location: Location,
@@ -44,6 +48,7 @@ pub struct DMError {
 
 #[allow(unused_variables)]
 impl DMError {
+    #[doc(hidden)]
     pub fn new<S: Into<String>>(location: Location, desc: S) -> DMError {
         DMError {
             location,
@@ -56,6 +61,16 @@ impl DMError {
     {
         Self::new(location, desc) // TODO
     }
+
+    /// Get the location in the code at which this error was observed.
+    pub fn location(&self) -> Location {
+        self.location
+    }
+
+    /// Get the description associated with this error.
+    pub fn description(&self) -> &str {
+        &self.desc
+    }
 }
 
 impl From<io::Error> for DMError {
@@ -67,17 +82,24 @@ impl From<io::Error> for DMError {
 // ----------------------------------------------------------------------------
 // Location handling
 
+/// File, line, and column information for an error.
 #[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Default)]
 pub struct Location {
+    /// The index into the file table.
     pub file: u32,
+    /// The line number, starting at 1.
     pub line: u32,
+    /// The column number, starting at 1.
     pub column: u32,
 }
 
+/// A trait for types which may yield location information.
 pub trait HasLocation {
+    /// Get the current location of this parsing stage.
     fn location(&self) -> Location;
 
     #[inline]
+    #[doc(hidden)]
     fn error<S: Into<String>>(&self, message: S) -> DMError {
         DMError::new(self.location(), message)
     }
@@ -94,6 +116,10 @@ impl<'a, T: HasLocation> HasLocation for &'a mut T {
 // ----------------------------------------------------------------------------
 // Pretty printing
 
+/// Pretty-print a series of tokens to the given output.
+///
+/// If `show_ws` is true, braces and semicolons are included directly in the
+/// output rather than only being implied by the indentation.
 pub fn pretty_print<W, I>(w: &mut W, input: I, show_ws: bool) -> Result<(), DMError> where
     W: io::Write,
     I: IntoIterator<Item=Result<lexer::Token, DMError>>
@@ -144,6 +170,7 @@ pub fn pretty_print<W, I>(w: &mut W, input: I, show_ws: bool) -> Result<(), DMEr
     Ok(())
 }
 
+/// Pretty-print a `DMError` to the given output.
 pub fn pretty_print_error<W: io::Write>(w: &mut W, pp: &preprocessor::Preprocessor, error: &DMError) -> io::Result<()> {
     writeln!(w, "\n{}, line {}, column {}:",
         pp.file_path(error.location.file).display(),

dreammaker/parser.rs

@@ -7,9 +7,10 @@ use super::lexer::{LocatedToken, Token, Punctuation};
 use super::objtree::ObjectTree;
 use super::ast::*;
 
+/// Parse a token stream, in the form emitted by the indent processor, into
+/// an object tree.
 pub fn parse<I>(iter: I) -> Result<ObjectTree, DMError> where
-    I: IntoIterator<Item=Result<LocatedToken, DMError>>,
-    I::IntoIter: HasLocation
+    I: IntoIterator<Item=Result<LocatedToken, DMError>>
 {
     let mut parser = Parser::new(iter.into_iter());
     let mut tree = match parser.root()? {
@@ -186,6 +187,10 @@ oper_table! { BINARY_OPS;
 // ----------------------------------------------------------------------------
 // The parser
 
+/// A single-lookahead, recursive-descent DM parser.
+///
+/// Results are accumulated into an inner `ObjectTree`. To parse an entire
+/// environment, use the `parse` or `parse_environment` functions.
 pub struct Parser<I> {
     tree: ObjectTree,
 
@@ -205,6 +210,7 @@ impl<I> HasLocation for Parser<I> {
 impl<I> Parser<I> where
     I: Iterator<Item=Result<LocatedToken, DMError>>
 {
+    /// Construct a new parser using the given input stream.
     pub fn new(input: I) -> Parser<I> {
         Parser {
             tree: ObjectTree::with_builtins(),
@@ -465,6 +471,10 @@ impl<I> Parser<I> where
         success(Prefab { path: parts, vars })
     }
 
+    /// Parse an expression at the current position.
+    ///
+    /// If `disallow_assign` is set, assignment operators are not considered.
+    /// This is useful when parsing the left-hand side of a list association.
     pub fn expression(&mut self, disallow_assign: bool) -> Status<Expression> {
         let mut expr = leading!(self.group());
         loop {