Add some tests for documentation gen

Cargo.lock

@@ -617,6 +617,8 @@ dependencies = [
  "bumpalo",
  "fs_extra",
  "handlebars",
+ "maplit",
+ "pretty_assertions",
  "pulldown-cmark",
  "roc_builtins",
  "roc_collections",

compiler/load/src/docs.rs

@@ -1,5 +1,3 @@
-// This file was copied from file.rs and modified to expose information
-// required to auto-generate documentation
 use inlinable_string::InlinableString;
 use roc_module::ident::ModuleName;
 use roc_module::symbol::IdentIds;
@@ -71,7 +69,7 @@ fn generate_module_doc<'a>(
                 // If there are comments before, attach to this definition
                 generate_module_doc(exposed_ident_ids, acc, before_comments_or_new_lines, sub_def);
 
-            // Comments after a definition are attached to the next defition
+            // Comments after a definition are attached to the next definition
             (new_acc, Some(comments_or_new_lines))
         }
 
@@ -98,7 +96,11 @@ fn generate_module_doc<'a>(
             name: _,
             vars: _,
             ann: _,
-        } => (acc, None),
+        } =>
+        // TODO
+        {
+            (acc, None)
+        }
 
         Body(_, _) | Nested(_) => (acc, None),
     }
@@ -113,9 +115,16 @@ fn comments_or_new_lines_to_docs<'a>(
 
     for comment_or_new_line in comments_or_new_lines.iter() {
         match comment_or_new_line {
-            Newline => {}
+            DocComment(doc_str) => {
-            LineComment(_) => {}
+                docs.push_str(doc_str);
-            DocComment(doc_str) => docs.push_str(doc_str),
+                docs.push_str("\n");
+            }
+            // TODO: Lines with only `##` are not being parsed as a
+            // DocComment, but as a LineComment("#\r"). This pattern should cover this.
+            // The problem is that this is only valid if it is at the start
+            // of a line. False positive example: `x = 2 ##`.
+            LineComment("#\r") => docs.push_str("\n"),
+            Newline | LineComment(_) => {}
         }
     }
     if docs.is_empty() {

docs/Cargo.toml

@@ -17,3 +17,7 @@ roc_load = { path = "../compiler/load" }
 roc_builtins = { path = "../compiler/builtins" }
 roc_collections = { path = "../compiler/collections" }
 bumpalo = { version = "3.2", features = ["collections"] }
+
+[dev-dependencies]
+pretty_assertions = "0.5.1"
+maplit = "1.0.1"

docs/src/main.rs

@@ -5,7 +5,10 @@ extern crate serde;
 extern crate serde_derive;
 extern crate pulldown_cmark;
 extern crate serde_json;
-use std::error::Error;
+use roc_builtins::std::StdLib;
+use roc_load::docs::Documentation;
+use roc_load::docs::ModuleDocumentation;
+
 use std::fs;
 extern crate roc_load;
 use bumpalo::Bump;
@@ -22,7 +25,7 @@ pub struct Template {
     module_links: Vec<TemplateLink>,
 }
 
-#[derive(Serialize, Clone)]
+#[derive(Serialize, Clone, Debug, PartialEq)]
 pub struct ModuleEntry {
     name: String,
     docs: String,
@@ -41,113 +44,63 @@ pub struct TemplateLinkEntry {
     name: String,
 }
 
-fn main() -> Result<(), Box<dyn Error>> {
+fn main() {
-    let std_lib = roc_builtins::std::standard_stdlib();
+    generate(
-    let subs_by_module = MutMap::default();
+        vec![
-    let arena = Bump::new();
+            PathBuf::from(r"../compiler/builtins/docs/Bool.roc"),
+            PathBuf::from(r"../compiler/builtins/docs/Map.roc"),
+            // Not working
+            // PathBuf::from(r"../compiler/builtins/docs/List.roc"),
+            // Not working
+            // PathBuf::from(r"../compiler/builtins/docs/Num.roc"),
+            PathBuf::from(r"../compiler/builtins/docs/Set.roc"),
+            PathBuf::from(r"../compiler/builtins/docs/Str.roc"),
+        ],
+        roc_builtins::std::standard_stdlib(),
+        Path::new("../compiler/builtins/docs"),
+        Path::new("./build"),
+    )
+}
 
-    let src_dir = Path::new("../compiler/builtins/docs");
+pub fn generate(filenames: Vec<PathBuf>, std_lib: StdLib, src_dir: &Path, build_dir: &Path) {
-    let files = vec![
+    let files_docs = files_to_documentations(filenames, std_lib, src_dir);
-        PathBuf::from(r"../compiler/builtins/docs/Bool.roc"),
-        PathBuf::from(r"../compiler/builtins/docs/Map.roc"),
-        // Not working
-        // PathBuf::from(r"../compiler/builtins/docs/List.roc"),
-        // Not working
-        // PathBuf::from(r"../compiler/builtins/docs/Num.roc"),
-        PathBuf::from(r"../compiler/builtins/docs/Set.roc"),
-        PathBuf::from(r"../compiler/builtins/docs/Str.roc"),
-    ];
-
-    let mut modules_docs = vec![];
-
-    // Load each file is files vector
-    for filename in files {
-        let mut loaded = roc_load::file::load_and_typecheck(
-            &arena,
-            filename,
-            std_lib.clone(),
-            src_dir,
-            subs_by_module.clone(),
-        )
-        .expect("TODO gracefully handle load failing");
-        modules_docs.extend(loaded.documentation.drain().map(|x| x.1));
-    }
 
+    // TODO: get info from a file like "elm.json"
     let package = roc_load::docs::Documentation {
         name: "roc/builtins".to_string(),
         version: "1.0.0".to_string(),
         docs: "Package introduction or README.".to_string(),
-        modules: modules_docs,
+        modules: files_docs,
     };
 
-    // Remove old build folder
+    // Remove old build folder, if exists
-    fs::remove_dir_all("./build")?;
+    let _ = fs::remove_dir_all(build_dir);
+
+    let version_folder = build_dir
+        .join(package.name.clone())
+        .join(package.version.clone());
 
     // Make sure the output directories exists
-    fs::create_dir_all(format!("./build/{}/{}", package.name, package.version))?;
+    fs::create_dir_all(&version_folder)
+        .expect("TODO gracefully handle creating directories failing");
 
     // Register handlebars template
     let mut handlebars = handlebars::Handlebars::new();
-    assert!(handlebars
+    handlebars
         .register_template_file("page", "./src/templates/page.hbs")
-        .is_ok());
+        .expect("TODO gracefully handle registering template failing");
 
-    let markdown_options = pulldown_cmark::Options::all();
+    // Write each package's module docs html file
-
-    // Write each package's module docs
     for module in &package.modules {
-        // Convert module docs from markdown to html
+        let template = documentation_to_template_data(&package, module);
-        let docs_parser = pulldown_cmark::Parser::new_ext(&module.docs, markdown_options);
-        let mut docs_html: String = String::with_capacity(module.docs.len() * 3 / 2);
-        pulldown_cmark::html::push_html(&mut docs_html, docs_parser);
-
-        let template = Template {
-            package_name: package.name.clone(),
-            package_version: package.version.clone(),
-            module_name: module.name.clone(),
-            module_docs: docs_html,
-            module_entries: module
-                .entries
-                .clone()
-                .into_iter()
-                .map(|entry| {
-                    // Convert entry docs from markdown to html
-                    let mut entry_docs_html: String = String::new();
-                    if let Some(docs) = entry.docs {
-                        let entry_docs_parser =
-                            pulldown_cmark::Parser::new_ext(&docs, markdown_options);
-                        pulldown_cmark::html::push_html(&mut entry_docs_html, entry_docs_parser);
-                    }
-
-                    ModuleEntry {
-                        name: entry.name.clone(),
-                        docs: entry_docs_html,
-                    }
-                })
-                .collect(),
-            module_links: package
-                .modules
-                .clone()
-                .into_iter()
-                .map(|module_link| TemplateLink {
-                    name: module_link.name.clone(),
-                    href: format!("./{}.html", module_link.name),
-                    classes: "".to_string(),
-                    entries: module_link
-                        .entries
-                        .into_iter()
-                        .map(|entry| TemplateLinkEntry { name: entry.name })
-                        .collect(),
-                })
-                .collect(),
-        };
 
         let handlebars_data = handlebars::to_json(&template);
-        let mut output_file = fs::File::create(format!(
+        let filepath = version_folder.join(format!("{}.html", module.name));
-            "./build/{}/{}/{}.html",
+        let mut output_file =
-            package.name, package.version, module.name
+            fs::File::create(filepath).expect("TODO gracefully handle creating file failing");
-        ))?;
+        handlebars
-        handlebars.render_to_write("page", &handlebars_data, &mut output_file)?;
+            .render_to_write("page", &handlebars_data, &mut output_file)
+            .expect("TODO gracefully handle writing file failing");
     }
 
     // Copy /static folder content to /build
@@ -159,7 +112,181 @@ fn main() -> Result<(), Box<dyn Error>> {
         content_only: true,
         depth: 0,
     };
-    fs_extra::dir::copy("./src/static/", "./build", &copy_options)?;
+    fs_extra::dir::copy("./src/static/", &build_dir, &copy_options)
-    println!("Docs generated at /build");
+        .expect("TODO gracefully handle copying static content failing");
-    Ok(())
+    println!("Docs generated at {}", build_dir.display());
+}
+
+fn files_to_documentations(
+    filenames: Vec<PathBuf>,
+    std_lib: StdLib,
+    src_dir: &Path,
+) -> Vec<ModuleDocumentation> {
+    let arena = Bump::new();
+    let mut files_docs = vec![];
+
+    for filename in filenames {
+        let mut loaded = roc_load::file::load_and_typecheck(
+            &arena,
+            filename,
+            std_lib.clone(),
+            src_dir,
+            MutMap::default(),
+        )
+        .expect("TODO gracefully handle load failing");
+        files_docs.extend(loaded.documentation.drain().map(|x| x.1));
+    }
+    files_docs
+}
+
+fn documentation_to_template_data(doc: &Documentation, module: &ModuleDocumentation) -> Template {
+    Template {
+        package_name: doc.name.clone(),
+        package_version: doc.version.clone(),
+        module_name: module.name.clone(),
+        module_docs: markdown_to_html(module.docs.clone()),
+        module_entries: module
+            .entries
+            .clone()
+            .into_iter()
+            .map(|entry| ModuleEntry {
+                name: entry.name.clone(),
+                docs: match entry.docs {
+                    Some(docs) => markdown_to_html(docs),
+                    None => String::new(),
+                },
+            })
+            .collect(),
+        module_links: doc
+            .modules
+            .clone()
+            .into_iter()
+            .map(|module_link| TemplateLink {
+                name: module_link.name.clone(),
+                href: format!("./{}.html", module_link.name),
+                classes: "".to_string(),
+                entries: module_link
+                    .entries
+                    .into_iter()
+                    .map(|entry| TemplateLinkEntry { name: entry.name })
+                    .collect(),
+            })
+            .collect(),
+    }
+}
+
+fn markdown_to_html(markdown: String) -> String {
+    use pulldown_cmark::CodeBlockKind::*;
+    use pulldown_cmark::Event::*;
+    use pulldown_cmark::Tag::*;
+
+    let markdown_options = pulldown_cmark::Options::all();
+    // let docs_parser =
+    //     pulldown_cmark::Parser::new_ext(&markdown, markdown_options).map(|event| match event {
+    //         Event::Start(Tag::BlockQuote) => {println!("{:?}", event);event},
+    //         Event::End(Tag::BlockQuote) => {println!("{:?}", event);event},
+    //         _ => {println!("{:?}", event);event},
+    //     });
+    let mut docs_parser = vec![];
+    let (_, _) = pulldown_cmark::Parser::new_ext(&markdown, markdown_options).fold(
+        (0, 0),
+        |(start_quote_count, end_quote_count), event| match event {
+            // Replace this pattern (`>>>` syntax):
+            //     Start(BlockQuote)
+            //     Start(BlockQuote)
+            //     Start(BlockQuote)
+            //     Start(Paragraph)
+            // For `Start(CodeBlock(Indented))`
+            Start(BlockQuote) => {
+                docs_parser.push(event);
+                (start_quote_count + 1, 0)
+            }
+            Start(Paragraph) => {
+                if start_quote_count == 3 {
+                    docs_parser.pop();
+                    docs_parser.pop();
+                    docs_parser.pop();
+                    docs_parser.push(Start(CodeBlock(Indented)));
+                } else {
+                    docs_parser.push(event);
+                }
+                (0, 0)
+            }
+            // Replace this pattern (`>>>` syntax):
+            //     End(Paragraph)
+            //     End(BlockQuote)
+            //     End(BlockQuote)
+            //     End(BlockQuote)
+            // For `End(CodeBlock(Indented))`
+            End(Paragraph) => {
+                docs_parser.push(event);
+                (0, 1)
+            }
+            End(BlockQuote) => {
+                if end_quote_count == 3 {
+                    docs_parser.pop();
+                    docs_parser.pop();
+                    docs_parser.pop();
+                    docs_parser.push(End(CodeBlock(Indented)));
+                    (0, 0)
+                } else {
+                    docs_parser.push(event);
+                    (0, end_quote_count + 1)
+                }
+            }
+            _ => {
+                docs_parser.push(event);
+                (0, 0)
+            }
+        },
+    );
+    let mut docs_html = String::new();
+    pulldown_cmark::html::push_html(&mut docs_html, docs_parser.into_iter());
+    docs_html
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn internal() {
+        let files_docs = files_to_documentations(
+            vec![PathBuf::from(r"tests/fixtures/Interface.roc")],
+            roc_builtins::std::standard_stdlib(),
+            Path::new("tests/fixtures"),
+        );
+
+        let package = roc_load::docs::Documentation {
+            name: "roc/builtins".to_string(),
+            version: "1.0.0".to_string(),
+            docs: "Package introduction or README.".to_string(),
+            modules: files_docs,
+        };
+
+        let expected_entries = vec![ModuleEntry {
+                name: "singleline".to_string(),
+                docs: "<p>Single line documentation.</p>\n".to_string(),
+            }, ModuleEntry {
+                name: "multiline".to_string(),
+                docs: "<p>Multiline documentation.\nWithout any complex syntax yet!</p>\n".to_string(),
+            }, ModuleEntry {
+                name: "multiparagraph".to_string(),
+                docs: "<p>Multiparagraph documentation.</p>\n<p>Without any complex syntax yet!</p>\n".to_string(),
+            }, ModuleEntry {
+                name: "codeblock".to_string(),
+                docs: "<p>Turns &gt;&gt;&gt; into code block for now.</p>\n<pre><code>codeblock</code></pre>\n".to_string(),
+            },
+        ];
+
+        for module in &package.modules {
+            let template = documentation_to_template_data(&package, module);
+            assert_eq!(template.module_name, "Test");
+            template
+                .module_entries
+                .iter()
+                .zip(expected_entries.iter())
+                .for_each(|(x, y)| assert_eq!(x, y));
+        }
+    }
 }

docs/tests/fixtures/Interface.roc

@@ -0,0 +1,23 @@
+interface Test
+    exposes [ singleline, multiline, multiparagraph, codeblock ]
+    imports []
+
+## Single line documentation.
+singleline : Bool -> Bool
+
+## Multiline documentation.
+## Without any complex syntax yet!
+multiline : Bool -> Bool
+
+## Multiparagraph documentation.
+##
+## Without any complex syntax yet!
+multiparagraph : Bool -> Bool
+
+## No documentation for not exposed function.
+notExposed : Bool -> Bool
+
+## Turns >>> into code block for now.
+##
+## >>> codeblock
+codeblock : Bool -> Bool