roc/docs/src/lib.rs
2021-10-04 19:26:31 +02:00

1012 lines
32 KiB
Rust

extern crate pulldown_cmark;
extern crate roc_load;
use bumpalo::{collections::String as BumpString, collections::Vec as BumpVec, Bump};
use def::defs_to_html;
use expr::{expr_to_html};
use roc_builtins::std::StdLib;
use roc_can::builtins::builtin_defs_map;
use roc_can::scope::Scope;
use roc_collections::all::MutMap;
use roc_load::docs::DocEntry::DocDef;
use roc_load::docs::{DocEntry, TypeAnnotation};
use roc_load::docs::{ModuleDocumentation, RecordField};
use roc_load::file::{LoadedModule, LoadingProblem};
use roc_module::symbol::{IdentIds, Interns, ModuleId, ModuleIds};
use roc_parse::ident::{parse_ident, Ident};
use roc_parse::parser::{State, SyntaxError};
use roc_region::all::Region;
use std::fs;
use std::path::{Path, PathBuf};
mod comment_or_new_line;
mod def;
mod expr;
mod html;
mod pattern;
pub fn generate_docs_html(filenames: Vec<PathBuf>, std_lib: StdLib, build_dir: &Path) {
let loaded_modules = load_modules_for_files(filenames, std_lib);
let mut arena = Bump::new();
//
// TODO: get info from a file like "elm.json"
let mut package = roc_load::docs::Documentation {
name: "roc/builtins".to_string(),
version: "1.0.0".to_string(),
docs: "Package introduction or README.".to_string(),
modules: loaded_modules,
};
if !build_dir.exists() {
fs::create_dir_all(build_dir).expect("TODO gracefully handle unable to create build dir");
}
// Copy over the assets
fs::write(
build_dir.join("search.js"),
include_str!("./static/search.js"),
)
.expect("TODO gracefully handle failing to make the search javascript");
fs::write(
build_dir.join("styles.css"),
include_str!("./static/styles.css"),
)
.expect("TODO gracefully handle failing to make the stylesheet");
fs::write(
build_dir.join("favicon.svg"),
include_str!("./static/favicon.svg"),
)
.expect("TODO gracefully handle failing to make the favicon");
let template_html = include_str!("./static/index.html")
.replace("<!-- search.js -->", &format!("{}search.js", base_url()))
.replace("<!-- styles.css -->", &format!("{}styles.css", base_url()))
.replace(
"<!-- favicon.svg -->",
&format!("{}favicon.svg", base_url()),
)
.replace(
"<!-- Module links -->",
render_sidebar(package.modules.iter().flat_map(|loaded_module| {
loaded_module.documentation.values().map(move |d| {
let exposed_values = loaded_module
.exposed_values
.iter()
.map(|symbol| symbol.ident_str(&loaded_module.interns).to_string())
.collect::<Vec<String>>();
(exposed_values, d)
})
}))
.as_str(),
);
// Write each package's module docs html file
for loaded_module in package.modules.iter_mut() {
arena.reset();
let mut exports: BumpVec<&str> =
BumpVec::with_capacity_in(loaded_module.exposed_values.len(), &arena);
// TODO should this also include exposed_aliases?
for symbol in loaded_module.exposed_values.iter() {
exports.push(symbol.ident_str(&loaded_module.interns));
}
let exports = exports.into_bump_slice();
for module_docs in loaded_module.documentation.values() {
let module_dir = build_dir.join(module_docs.name.replace(".", "/").as_str());
fs::create_dir_all(&module_dir)
.expect("TODO gracefully handle not being able to create the module dir");
let rendered_module = template_html
.replace(
"<!-- Package Name and Version -->",
render_name_and_version(package.name.as_str(), package.version.as_str())
.as_str(),
)
.replace(
"<!-- Module Docs -->",
render_module_documentation(
exports,
module_docs,
&loaded_module
)
.as_str(),
);
fs::write(module_dir.join("index.html"), rendered_module)
.expect("TODO gracefully handle failing to write html");
}
}
println!("🎉 Docs generated in {}", build_dir.display());
}
// converts plain-text code to highlighted html
pub fn syntax_highlight_expr<'a>(
arena: &'a Bump,
buf: &mut BumpString<'a>,
code_str: &'a str,
env_module_id: ModuleId,
env_module_ids: &'a ModuleIds,
interns: &Interns
) -> Result<String, SyntaxError<'a>> {
let trimmed_code_str = code_str.trim_end().trim();
let state = State::new(trimmed_code_str.as_bytes());
match roc_parse::expr::test_parse_expr(0, arena, state) {
Ok(loc_expr) => {
expr_to_html(buf, loc_expr.value, env_module_id, env_module_ids, interns);
Ok(buf.to_string())
}
Err(fail) => Err(SyntaxError::Expr(fail)),
}
}
// converts plain-text code to highlighted html
pub fn syntax_highlight_top_level_defs<'a>(
arena: &'a Bump,
buf: &mut BumpString<'a>,
code_str: &'a str,
env_module_id: ModuleId,
env_module_ids: &'a ModuleIds,
interns: &Interns
) -> Result<String, SyntaxError<'a>> {
let trimmed_code_str = code_str.trim_end().trim();
match roc_parse::test_helpers::parse_defs_with(arena, trimmed_code_str) {
Ok(vec_loc_def) => {
let vec_def =
vec_loc_def.iter().map(|loc| loc.value).collect();
defs_to_html(buf, vec_def, env_module_id, env_module_ids, interns);
Ok(buf.to_string())
}
Err(err) => Err(err),
}
}
fn render_module_documentation(
exposed_values: &[&str],
module: &ModuleDocumentation,
loaded_module: &LoadedModule,
) -> String {
let mut buf = String::new();
buf.push_str(
html_to_string(
"h2",
vec![("class", "module-name")],
html_to_string("a", vec![("href", "/#")], module.name.as_str()).as_str(),
)
.as_str(),
);
for entry in &module.entries {
let mut should_render_entry = true;
if let DocDef(def) = entry {
// We dont want to render entries that arent exposed
should_render_entry = exposed_values.contains(&def.name.as_str());
}
if should_render_entry {
match entry {
DocEntry::DocDef(doc_def) => {
let mut href = String::new();
href.push('#');
href.push_str(doc_def.name.as_str());
let name = doc_def.name.as_str();
let mut content = String::new();
content.push_str(html_to_string("a", vec![("href", href.as_str())], name).as_str());
for type_var in &doc_def.type_vars {
content.push(' ');
content.push_str(type_var.as_str());
}
let type_ann = &doc_def.type_annotation;
match type_ann {
TypeAnnotation::NoTypeAnn => {}
_ => {
content.push_str(" : ");
}
}
type_annotation_to_html(0, &mut content, type_ann);
buf.push_str(
html_to_string(
"h3",
vec![("id", name), ("class", "entry-name")],
content.as_str(),
)
.as_str(),
);
if let Some(docs) = &doc_def.docs {
buf.push_str(
markdown_to_html(
exposed_values,
&module.scope,
docs.to_string(),
loaded_module,
)
.as_str(),
);
}
}
DocEntry::DetachedDoc(docs) => {
let markdown = markdown_to_html(
exposed_values,
&module.scope,
docs.to_string(),
loaded_module,
);
buf.push_str(markdown.as_str());
}
};
}
}
buf
}
fn html_to_string(tag_name: &str, attrs: Vec<(&str, &str)>, content: &str) -> String {
let mut buf = String::new();
buf.push('<');
buf.push_str(tag_name);
for (key, value) in &attrs {
buf.push(' ');
buf.push_str(key);
buf.push_str("=\"");
buf.push_str(value);
buf.push('"');
}
if !&attrs.is_empty() {
buf.push(' ');
}
buf.push('>');
buf.push_str(content);
buf.push_str("</");
buf.push_str(tag_name);
buf.push('>');
buf
}
fn base_url() -> String {
// e.g. "builtins/" in "https://roc-lang.org/builtins/Str"
//
// TODO make this a CLI flag to the `docs` subcommand instead of an env var
match std::env::var("ROC_DOCS_URL_ROOT") {
Ok(root_builtins_path) => {
let mut url_str = String::with_capacity(root_builtins_path.len() + 64);
if !root_builtins_path.starts_with('/') {
url_str.push('/');
}
url_str.push_str(&root_builtins_path);
if !root_builtins_path.ends_with('/') {
url_str.push('/');
}
url_str
}
_ => {
let mut url_str = String::with_capacity(64);
url_str.push('/');
url_str
}
}
}
fn render_name_and_version(name: &str, version: &str) -> String {
let mut buf = String::new();
let mut url_str = base_url();
url_str.push_str(name);
buf.push_str(
html_to_string(
"h1",
vec![("class", "pkg-full-name")],
html_to_string("a", vec![("href", url_str.as_str())], name).as_str(),
)
.as_str(),
);
let mut versions_url_str = base_url();
versions_url_str.push('/');
versions_url_str.push_str(name);
versions_url_str.push('/');
versions_url_str.push_str(version);
buf.push_str(
html_to_string(
"a",
vec![("class", "version"), ("href", versions_url_str.as_str())],
version,
)
.as_str(),
);
buf
}
fn render_sidebar<'a, I: Iterator<Item = (Vec<String>, &'a ModuleDocumentation)>>(
modules: I,
) -> String {
let mut buf = String::new();
for (exposed_values, module) in modules {
let mut sidebar_entry_content = String::new();
let name = module.name.as_str();
let href = {
let mut href_buf = base_url();
href_buf.push_str(name);
href_buf
};
sidebar_entry_content.push_str(
html_to_string(
"a",
vec![("class", "sidebar-module-link"), ("href", href.as_str())],
name,
)
.as_str(),
);
let entries = {
let mut entries_buf = String::new();
for entry in &module.entries {
if let DocEntry::DocDef(doc_def) = entry {
if exposed_values.contains(&doc_def.name) {
let mut entry_href = String::new();
entry_href.push_str(href.as_str());
entry_href.push('#');
entry_href.push_str(doc_def.name.as_str());
entries_buf.push_str(
html_to_string(
"a",
vec![("href", entry_href.as_str())],
doc_def.name.as_str(),
)
.as_str(),
);
}
}
}
entries_buf
};
sidebar_entry_content.push_str(
html_to_string(
"div",
vec![("class", "sidebar-sub-entries")],
entries.as_str(),
)
.as_str(),
);
buf.push_str(
html_to_string(
"div",
vec![("class", "sidebar-entry")],
sidebar_entry_content.as_str(),
)
.as_str(),
);
}
buf
}
pub fn load_modules_for_files(filenames: Vec<PathBuf>, std_lib: StdLib) -> Vec<LoadedModule> {
let arena = Bump::new();
let mut modules = vec![];
for filename in filenames {
let mut src_dir = filename.clone();
src_dir.pop();
match roc_load::file::load_and_typecheck(
&arena,
filename,
&std_lib,
src_dir.as_path(),
MutMap::default(),
std::mem::size_of::<usize>() as u32, // This is just type-checking for docs, so "target" doesn't matter
builtin_defs_map,
) {
Ok(loaded) => modules.push(loaded),
Err(LoadingProblem::FormattedReport(report)) => {
println!("{}", report);
panic!();
}
Err(e) => panic!("{:?}", e),
}
}
modules
}
const INDENT: &str = " ";
fn indent(buf: &mut String, times: usize) {
for _ in 0..times {
buf.push_str(INDENT);
}
}
fn new_line(buf: &mut String) {
buf.push('\n');
}
// html is written to buf
fn type_annotation_to_html(indent_level: usize, buf: &mut String, type_ann: &TypeAnnotation) {
let is_multiline = should_be_multiline(type_ann);
match type_ann {
TypeAnnotation::TagUnion { tags, extension } => {
let tags_len = tags.len();
let tag_union_indent = indent_level + 1;
if is_multiline {
new_line(buf);
indent(buf, tag_union_indent);
}
buf.push('[');
if is_multiline {
new_line(buf);
}
let next_indent_level = tag_union_indent + 1;
for (index, tag) in tags.iter().enumerate() {
if is_multiline {
indent(buf, next_indent_level);
} else {
buf.push(' ');
}
buf.push_str(tag.name.as_str());
for type_value in &tag.values {
buf.push(' ');
type_annotation_to_html(next_indent_level, buf, type_value);
}
if is_multiline {
if index < (tags_len - 1) {
buf.push(',');
}
new_line(buf);
}
}
if is_multiline {
indent(buf, tag_union_indent);
} else {
buf.push(' ');
}
buf.push(']');
type_annotation_to_html(indent_level, buf, extension);
}
TypeAnnotation::BoundVariable(var_name) => {
buf.push_str(var_name);
}
TypeAnnotation::Apply { name, parts } => {
if parts.is_empty() {
buf.push_str(name);
} else {
buf.push('(');
buf.push_str(name);
for part in parts {
buf.push(' ');
type_annotation_to_html(indent_level, buf, part);
}
buf.push(')');
}
}
TypeAnnotation::Record { fields, extension } => {
let fields_len = fields.len();
let record_indent = indent_level + 1;
if is_multiline {
new_line(buf);
indent(buf, record_indent);
}
buf.push('{');
if is_multiline {
new_line(buf);
}
let next_indent_level = record_indent + 1;
for (index, field) in fields.iter().enumerate() {
if is_multiline {
indent(buf, next_indent_level);
} else {
buf.push(' ');
}
let fields_name = match field {
RecordField::RecordField { name, .. } => name,
RecordField::OptionalField { name, .. } => name,
RecordField::LabelOnly { name } => name,
};
buf.push_str(fields_name.as_str());
match field {
RecordField::RecordField {
type_annotation, ..
} => {
buf.push_str(" : ");
type_annotation_to_html(next_indent_level, buf, type_annotation);
}
RecordField::OptionalField {
type_annotation, ..
} => {
buf.push_str(" ? ");
type_annotation_to_html(next_indent_level, buf, type_annotation);
}
RecordField::LabelOnly { .. } => {}
}
if is_multiline {
if index < (fields_len - 1) {
buf.push(',');
}
new_line(buf);
}
}
if is_multiline {
indent(buf, record_indent);
} else {
buf.push(' ');
}
buf.push('}');
type_annotation_to_html(indent_level, buf, extension);
}
TypeAnnotation::Function { args, output } => {
let mut peekable_args = args.iter().peekable();
while let Some(arg) = peekable_args.next() {
if is_multiline {
if !should_be_multiline(arg) {
new_line(buf);
}
indent(buf, indent_level + 1);
}
type_annotation_to_html(indent_level, buf, arg);
if peekable_args.peek().is_some() {
buf.push_str(", ");
}
}
if is_multiline {
new_line(buf);
indent(buf, indent_level + 1);
}
buf.push_str(" -> ");
let mut next_indent_level = indent_level;
if should_be_multiline(output) {
next_indent_level += 1;
}
type_annotation_to_html(next_indent_level, buf, output);
}
TypeAnnotation::ObscuredTagUnion => {
buf.push_str("[ @.. ]");
}
TypeAnnotation::ObscuredRecord => {
buf.push_str("{ @.. }");
}
TypeAnnotation::NoTypeAnn => {}
TypeAnnotation::Wildcard => buf.push('*'),
}
}
fn should_be_multiline(type_ann: &TypeAnnotation) -> bool {
match type_ann {
TypeAnnotation::TagUnion { tags, extension } => {
let mut is_multiline = should_be_multiline(extension) || tags.len() > 1;
for tag in tags {
for value in &tag.values {
if is_multiline {
break;
}
is_multiline = should_be_multiline(value);
}
}
is_multiline
}
TypeAnnotation::Function { args, output } => {
let mut is_multiline = should_be_multiline(output) || args.len() > 2;
for arg in args {
if is_multiline {
break;
}
is_multiline = should_be_multiline(arg);
}
is_multiline
}
TypeAnnotation::ObscuredTagUnion => false,
TypeAnnotation::ObscuredRecord => false,
TypeAnnotation::BoundVariable(_) => false,
TypeAnnotation::Apply { parts, .. } => {
let mut is_multiline = false;
for part in parts {
is_multiline = should_be_multiline(part);
if is_multiline {
break;
}
}
is_multiline
}
TypeAnnotation::Record { fields, extension } => {
let mut is_multiline = should_be_multiline(extension) || fields.len() > 1;
for field in fields {
if is_multiline {
break;
}
match field {
RecordField::RecordField {
type_annotation, ..
} => is_multiline = should_be_multiline(type_annotation),
RecordField::OptionalField {
type_annotation, ..
} => is_multiline = should_be_multiline(type_annotation),
RecordField::LabelOnly { .. } => {}
}
}
is_multiline
}
TypeAnnotation::Wildcard => false,
TypeAnnotation::NoTypeAnn => false,
}
}
struct DocUrl {
url: String,
title: String,
}
fn doc_url<'a>(
home: ModuleId,
exposed_values: &[&str],
dep_idents: &MutMap<ModuleId, IdentIds>,
scope: &Scope,
interns: &'a Interns,
mut module_name: &'a str,
ident: &str,
) -> DocUrl {
if module_name.is_empty() {
// This is an unqualified lookup, so look for the ident
// in scope!
match scope.lookup(&ident.into(), Region::zero()) {
Ok(symbol) => {
// Get the exact module_name from scope. It could be the
// current module's name, but it also could be a different
// module - for example, if this is in scope from an
// unqualified import.
module_name = symbol.module_string(interns);
}
Err(_) => {
// TODO return Err here
panic!(
"Tried to generate an automatic link in docs for symbol `{}`, but that symbol was not in scope in this module. Scope was: {:?}",
ident, scope
);
}
}
} else {
match interns.module_ids.get_id(&module_name.into()) {
Some(&module_id) => {
// You can do qualified lookups on your own module, e.g.
// if I'm in the Foo module, I can do a `Foo.bar` lookup.
if module_id == home {
// Check to see if the value is exposed in this module.
// If it's not exposed, then we can't link to it!
if !exposed_values.contains(&ident) {
// TODO return Err here
panic!(
"Tried to generate an automatic link in docs for `{}.{}`, but `{}` does not expose `{}`.",
module_name, ident, module_name, ident);
}
} else {
// This is not the home module
match dep_idents
.get(&module_id)
.and_then(|exposed_ids| exposed_ids.get_id(&ident.into()))
{
Some(_) => {
// This is a valid symbol for this dependency,
// so proceed using the current module's name.
//
// TODO: In the future, this is where we'll
// incorporate the package name into the link.
}
_ => {
// TODO return Err here
panic!(
"Tried to generate an automatic link in docs for `{}.{}`, but `{}` is not exposed in `{}`.",
module_name, ident, ident, module_name);
}
}
}
}
None => {
// TODO return Err here
panic!("Tried to generate a doc link for `{}.{}` but the `{}` module was not imported!", module_name, ident, module_name);
}
}
}
let mut url = base_url();
// Example:
//
// module_name: "Str", ident: "join" => "/Str#join"
url.push_str(module_name);
url.push('#');
url.push_str(ident);
DocUrl {
url,
title: format!("Docs for {}.{}", module_name, ident),
}
}
fn markdown_to_html(
exposed_values: &[&str],
scope: &Scope,
markdown: String,
loaded_module: &LoadedModule,
) -> String {
use pulldown_cmark::{BrokenLink, CodeBlockKind, CowStr, Event, LinkType, Tag::*};
let mut arena = Bump::new();
let mut broken_link_callback = |link: BrokenLink| {
// A shortcut link - see https://spec.commonmark.org/0.30/#shortcut-reference-link -
// is something like `[foo]` in markdown. If you have a shortcut link
// without a corresponding `[foo]: https://foo.com` entry
// at the end of the document, we resolve it as an identifier based on
// what's currently in scope, so you write things like [Str.join] or
// [myFunction] and have them resolve to the docs for what you wrote.
match link.link_type {
LinkType::Shortcut => {
let state = State::new(link.reference.as_bytes());
// Reset the bump arena so we aren't constantly reallocating
// more memory as we iterate through these.
arena.reset();
match parse_ident(&arena, state) {
Ok((_, Ident::Access { module_name, parts }, _)) => {
let mut iter = parts.iter();
match iter.next() {
Some(symbol_name) if iter.next().is_none() => {
let DocUrl { url, title } = doc_url(
loaded_module.module_id,
exposed_values,
&loaded_module.dep_idents,
scope,
&loaded_module.interns,
module_name,
symbol_name,
);
Some((url.into(), title.into()))
}
_ => {
// This had record field access,
// e.g. [foo.bar] - which we
// can't create a doc link to!
None
}
}
}
Ok((_, Ident::GlobalTag(type_name), _)) => {
// This looks like a global tag name, but it could
// be a type alias that's in scope, e.g. [I64]
let DocUrl { url, title } = doc_url(
loaded_module.module_id,
exposed_values,
&loaded_module.dep_idents,
scope,
&loaded_module.interns,
"",
type_name,
);
Some((url.into(), title.into()))
}
_ => None,
}
}
_ => None,
}
};
let markdown_options = pulldown_cmark::Options::empty();
let mut expecting_code_block = false;
let mut docs_parser = vec![];
let (_, _) = pulldown_cmark::Parser::new_with_broken_link_callback(
&markdown,
markdown_options,
Some(&mut broken_link_callback),
)
.fold((0, 0), |(start_quote_count, end_quote_count), event| {
match &event {
// Replace this sequence (`>>>` syntax):
// Start(BlockQuote)
// Start(BlockQuote)
// Start(BlockQuote)
// Start(Paragraph)
// For `Start(CodeBlock(Fenced(Borrowed("roc"))))`
Event::Start(BlockQuote) => {
docs_parser.push(event);
(start_quote_count + 1, 0)
}
Event::Start(Paragraph) => {
if start_quote_count == 3 {
docs_parser.pop();
docs_parser.pop();
docs_parser.pop();
docs_parser.push(Event::Start(CodeBlock(CodeBlockKind::Fenced(
CowStr::Borrowed("roc"),
))));
} else {
docs_parser.push(event);
}
(0, 0)
}
// Replace this sequence (`>>>` syntax):
// End(Paragraph)
// End(BlockQuote)
// End(BlockQuote)
// End(BlockQuote)
// For `End(CodeBlock(Fenced(Borrowed("roc"))))`
Event::End(Paragraph) => {
docs_parser.push(event);
(0, 1)
}
Event::End(BlockQuote) => {
if end_quote_count == 3 {
docs_parser.pop();
docs_parser.pop();
docs_parser.pop();
docs_parser.push(Event::End(CodeBlock(CodeBlockKind::Fenced(
CowStr::Borrowed("roc"),
))));
(0, 0)
} else {
docs_parser.push(event);
(0, end_quote_count + 1)
}
}
Event::End(Link(LinkType::ShortcutUnknown, _url, _title)) => {
// Replace the preceding Text node with a Code node, so it
// renders as the equivalent of [`List.len`] instead of [List.len]
match docs_parser.pop() {
Some(Event::Text(string)) => {
docs_parser.push(Event::Code(string));
}
Some(first) => {
docs_parser.push(first);
}
None => {}
}
docs_parser.push(event);
(start_quote_count, end_quote_count)
}
Event::Start(CodeBlock(CodeBlockKind::Fenced(_))) => {
expecting_code_block = true;
docs_parser.push(event);
(0, 0)
}
Event::End(CodeBlock(_)) => {
expecting_code_block = false;
docs_parser.push(event);
(0, 0)
}
Event::Text(CowStr::Borrowed(code_str)) if expecting_code_block => {
let code_block_arena = Bump::new();
let mut code_block_buf = BumpString::new_in(&code_block_arena);
match syntax_highlight_expr(
&code_block_arena,
&mut code_block_buf,
code_str,
loaded_module.module_id,
&loaded_module.interns.module_ids,
&loaded_module.interns
)
{
Ok(highlighted_code_str) => {
docs_parser.push(Event::Html(CowStr::from(highlighted_code_str)));
}
Err(syntax_error) => {
panic!("Unexpected parse failure when parsing this for rendering in docs:\n\n{}\n\nParse error was:\n\n{:?}\n\n", code_str, syntax_error)
}
};
(0, 0)
}
_ => {
docs_parser.push(event);
(0, 0)
}
}
});
let mut docs_html = String::new();
pulldown_cmark::html::push_html(&mut docs_html, docs_parser.into_iter());
docs_html
}