language-servers/roc
1
0
Fork 0
mirror of https://github.com/roc-lang/roc.git synced 2025-09-28 14:24:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Merge branch 'trunk' of github.com:rtfeldman/roc into docs-parse-code-blocks

Browse source
This commit is contained in:
Chadtech 2021-08-10 00:31:06 -04:00
commit c72400c619
144 changed files with 4265 additions and 2174 deletions
Download patch file Download diff file Expand all files Collapse all files

533
docs/src/lib.rs
View file

@ -1,23 +1,25 @@
extern crate pulldown_cmark;
use roc_builtins::std::StdLib;
use roc_can::builtins::builtin_defs_map;
use roc_load::docs::{DocEntry, TypeAnnotation};
use roc_load::docs::{ModuleDocumentation, RecordField};
use roc_load::file::{LoadedModule, LoadingProblem};
use roc_module::symbol::Interns;
use std::fs;
extern crate roc_load;
use bumpalo::Bump;
use roc_builtins::std::StdLib;
use roc_can::builtins::builtin_defs_map;
use roc_can::scope::Scope;
use roc_collections::all::MutMap;
use roc_load::docs::DocEntry::DocDef;
use roc_load::docs::{DocEntry, TypeAnnotation};
use roc_load::docs::{ModuleDocumentation, RecordField};
use roc_load::file::{LoadedModule, LoadingProblem};
use roc_module::symbol::{IdentIds, Interns, ModuleId};
use roc_parse::ident::{parse_ident, Ident};
use roc_parse::parser::State;
use roc_region::all::Region;
use std::fs;
use std::path::{Path, PathBuf};
use std::str::Split;
pub fn generate(filenames: Vec<PathBuf>, std_lib: StdLib, build_dir: &Path) {
let files_docs = files_to_documentations(filenames, std_lib);
let mut arena = Bump::new();
//
// TODO: get info from a file like "elm.json"
@ -51,24 +53,42 @@ pub fn generate(filenames: Vec<PathBuf>, std_lib: StdLib, build_dir: &Path) {
)
.expect("TODO gracefully handle failing to make the favicon");
let template_html = include_str!("./static/index.html").replace(
"<!-- Module links -->",
render_sidebar(
package
.modules
.iter()
.flat_map(|loaded_module| loaded_module.documentation.values()),
let template_html = include_str!("./static/index.html")
.replace("<!-- search.js -->", &format!("{}search.js", base_href()))
.replace("<!-- styles.css -->", &format!("{}styles.css", base_href()))
.replace(
"<!-- favicon.svg -->",
&format!("{}favicon.svg", base_href()),
)
.as_str(),
);
.replace(
"<!-- Module links -->",
render_sidebar(package.modules.iter().flat_map(|loaded_module| {
loaded_module.documentation.values().map(move |d| {
let exposed_values = loaded_module
.exposed_values
.iter()
.map(|symbol| symbol.ident_str(&loaded_module.interns).to_string())
.collect::<Vec<String>>();
(exposed_values, d)
})
}))
.as_str(),
);
// Write each package's module docs html file
for loaded_module in package.modules.iter_mut() {
let exports = loaded_module
.exposed_values
.iter()
.map(|symbol| symbol.ident_string(&loaded_module.interns).to_string())
.collect::<Vec<String>>();
arena.reset();
let mut exports: bumpalo::collections::Vec<&str> =
bumpalo::collections::Vec::with_capacity_in(loaded_module.exposed_values.len(), &arena);
// TODO should this also include exposed_aliases?
for symbol in loaded_module.exposed_values.iter() {
exports.push(symbol.ident_str(&loaded_module.interns));
}
let exports = exports.into_bump_slice();
for module in loaded_module.documentation.values_mut() {
let module_dir = build_dir.join(module.name.replace(".", "/").as_str());
@ -84,7 +104,14 @@ pub fn generate(filenames: Vec<PathBuf>, std_lib: StdLib, build_dir: &Path) {
)
.replace(
"<!-- Module Docs -->",
render_main_content(&loaded_module.interns, &exports, module).as_str(),
render_main_content(
loaded_module.module_id,
exports,
&loaded_module.dep_idents,
&loaded_module.interns,
module,
)
.as_str(),
);
fs::write(
@ -138,8 +165,10 @@ fn drop_letters(s: &mut String, pos: usize) {
}
fn render_main_content(
home: ModuleId,
exposed_values: &[&str],
dep_idents: &MutMap<ModuleId, IdentIds>,
interns: &Interns,
exposed_values: &[String],
module: &mut ModuleDocumentation,
) -> String {
let mut buf = String::new();
@ -158,7 +187,7 @@ fn render_main_content(
if let DocDef(def) = entry {
// We dont want to render entries that arent exposed
should_render_entry = exposed_values.contains(&def.name);
should_render_entry = exposed_values.contains(&def.name.as_str());
}
if should_render_entry {
@ -201,12 +230,27 @@ fn render_main_content(
if let Some(docs) = &doc_def.docs {
buf.push_str(
markdown_to_html(&mut module.scope, interns, docs.to_string()).as_str(),
markdown_to_html(
home,
exposed_values,
dep_idents,
&mut module.scope,
interns,
docs.to_string(),
)
.as_str(),
);
}
}
DocEntry::DetachedDoc(docs) => {
let markdown = markdown_to_html(&mut module.scope, interns, docs.to_string());
let markdown = markdown_to_html(
home,
exposed_values,
dep_idents,
&mut module.scope,
interns,
docs.to_string(),
);
buf.push_str(markdown.as_str());
}
};
@ -245,11 +289,40 @@ fn html_node(tag_name: &str, attrs: Vec<(&str, &str)>, content: &str) -> String
buf
}
fn base_href() -> String {
// e.g. "builtins/" in "https://roc-lang.org/builtins/Str"
//
// TODO make this a CLI flag to the `docs` subcommand instead of an env var
match std::env::var("ROC_DOCS_URL_ROOT") {
Ok(root_builtins_path) => {
let mut href = String::with_capacity(root_builtins_path.len() + 64);
if !root_builtins_path.starts_with('/') {
href.push('/');
}
href.push_str(&root_builtins_path);
if !root_builtins_path.ends_with('/') {
href.push('/');
}
href
}
_ => {
let mut href = String::with_capacity(64);
href.push('/');
href
}
}
}
fn render_name_and_version(name: &str, version: &str) -> String {
let mut buf = String::new();
let mut href = base_href();
let mut href = String::new();
href.push('/');
href.push_str(name);
buf.push_str(
@ -261,7 +334,7 @@ fn render_name_and_version(name: &str, version: &str) -> String {
.as_str(),
);
let mut versions_href = String::new();
let mut versions_href = base_href();
versions_href.push('/');
versions_href.push_str(name);
@ -280,17 +353,18 @@ fn render_name_and_version(name: &str, version: &str) -> String {
buf
}
fn render_sidebar<'a, I: Iterator<Item = &'a ModuleDocumentation>>(modules: I) -> String {
fn render_sidebar<'a, I: Iterator<Item = (Vec<String>, &'a ModuleDocumentation)>>(
modules: I,
) -> String {
let mut buf = String::new();
for module in modules {
for (exposed_values, module) in modules {
let mut sidebar_entry_content = String::new();
let name = module.name.as_str();
let href = {
let mut href_buf = String::new();
href_buf.push('/');
let mut href_buf = base_href();
href_buf.push_str(name);
href_buf
};
@ -309,20 +383,22 @@ fn render_sidebar<'a, I: Iterator<Item = &'a ModuleDocumentation>>(modules: I) -
for entry in &module.entries {
if let DocEntry::DocDef(doc_def) = entry {
let mut entry_href = String::new();
if exposed_values.contains(&doc_def.name) {
let mut entry_href = String::new();
entry_href.push_str(href.as_str());
entry_href.push('#');
entry_href.push_str(doc_def.name.as_str());
entry_href.push_str(href.as_str());
entry_href.push('#');
entry_href.push_str(doc_def.name.as_str());
entries_buf.push_str(
html_node(
"a",
vec![("href", entry_href.as_str())],
doc_def.name.as_str(),
)
.as_str(),
);
entries_buf.push_str(
html_node(
"a",
vec![("href", entry_href.as_str())],
doc_def.name.as_str(),
)
.as_str(),
);
}
}
}
@ -644,161 +720,250 @@ fn should_be_multiline(type_ann: &TypeAnnotation) -> bool {
}
}
pub fn insert_doc_links(scope: &mut Scope, interns: &Interns, markdown: String) -> String {
let buf = &markdown;
let mut result = String::new();
let mut chomping_from: Option<usize> = None;
let mut chars = buf.chars().enumerate().peekable();
while let Some((index, char)) = chars.next() {
match chomping_from {
None => {
let next_is_alphabetic = match chars.peek() {
None => false,
Some((_, next_char)) => next_char.is_alphabetic(),
};
if char == '#' && next_is_alphabetic {
chomping_from = Some(index);
}
}
Some(from) => {
if !(char.is_alphabetic() || char == '.') {
let after_link = buf.chars().skip(from + buf.len());
result = buf.chars().take(from).collect();
let doc_link = make_doc_link(
scope,
interns,
&buf.chars()
.skip(from + 1)
.take(index - from - 1)
.collect::<String>(),
);
result.insert_str(from, doc_link.as_str());
let remainder = insert_doc_links(scope, interns, after_link.collect());
result.push_str(remainder.as_str());
break;
}
}
}
}
if chomping_from == None {
markdown
} else {
result
}
struct DocUrl {
url: String,
title: String,
}
fn make_doc_link(scope: &mut Scope, interns: &Interns, doc_item: &str) -> String {
match scope.lookup(&doc_item.into(), Region::zero()) {
Ok(symbol) => {
let module_str = symbol.module_string(interns);
let ident_str = symbol.ident_string(interns);
let mut link = String::new();
link.push('/');
link.push_str(module_str);
link.push('#');
link.push_str(ident_str);
let mut buf = String::new();
buf.push('[');
buf.push_str(doc_item);
buf.push_str("](");
buf.push_str(link.as_str());
buf.push(')');
buf
}
Err(_) => {
panic!(
fn doc_url<'a>(
home: ModuleId,
exposed_values: &[&str],
dep_idents: &MutMap<ModuleId, IdentIds>,
scope: &mut Scope,
interns: &'a Interns,
mut module_name: &'a str,
ident: &str,
) -> DocUrl {
if module_name.is_empty() {
// This is an unqualified lookup, so look for the ident
// in scope!
match scope.lookup(&ident.into(), Region::zero()) {
Ok(symbol) => {
// Get the exact module_name from scope. It could be the
// current module's name, but it also could be a different
// module - for example, if this is in scope from an
// unqualified import.
module_name = symbol.module_string(interns);
}
Err(_) => {
// TODO return Err here
panic!(
"Tried to generate an automatic link in docs for symbol `{}`, but that symbol was not in scope in this module. Scope was: {:?}",
doc_item, scope
)
ident, scope
);
}
}
} else {
match interns.module_ids.get_id(&module_name.into()) {
Some(&module_id) => {
// You can do qualified lookups on your own module, e.g.
// if I'm in the Foo module, I can do a `Foo.bar` lookup.
if module_id == home {
// Check to see if the value is exposed in this module.
// If it's not exposed, then we can't link to it!
if !exposed_values.contains(&ident) {
// TODO return Err here
panic!(
"Tried to generate an automatic link in docs for `{}.{}`, but `{}` does not expose `{}`.",
module_name, ident, module_name, ident);
}
} else {
// This is not the home module
match dep_idents
.get(&module_id)
.and_then(|exposed_ids| exposed_ids.get_id(&ident.into()))
{
Some(_) => {
// This is a valid symbol for this dependency,
// so proceed using the current module's name.
//
// TODO: In the future, this is where we'll
// incorporate the package name into the link.
}
_ => {
// TODO return Err here
panic!(
"Tried to generate an automatic link in docs for `{}.{}`, but `{}` is not exposed in `{}`.",
module_name, ident, ident, module_name);
}
}
}
}
None => {
// TODO return Err here
panic!("Tried to generate a doc link for `{}.{}` but the `{}` module was not imported!", module_name, ident, module_name);
}
}
}
let mut url = base_href();
// Example:
//
// module_name: "Str", ident: "join" => "/Str#join"
url.push_str(module_name);
url.push('#');
url.push_str(ident);
DocUrl {
url,
title: format!("Docs for {}.{}", module_name, ident),
}
}
fn markdown_to_html(scope: &mut Scope, interns: &Interns, markdown: String) -> String {
use pulldown_cmark::CodeBlockKind;
use pulldown_cmark::CowStr;
use pulldown_cmark::Event;
use pulldown_cmark::Tag::*;
fn markdown_to_html(
home: ModuleId,
exposed_values: &[&str],
dep_idents: &MutMap<ModuleId, IdentIds>,
scope: &mut Scope,
interns: &Interns,
markdown: String,
) -> String {
use pulldown_cmark::{BrokenLink, CodeBlockKind, CowStr, Event, LinkType, Tag::*};
let markdown_with_links = insert_doc_links(scope, interns, markdown);
let mut arena = Bump::new();
let mut broken_link_callback = |link: BrokenLink| {
// A shortcut link - see https://spec.commonmark.org/0.30/#shortcut-reference-link -
// is something like `[foo]` in markdown. If you have a shortcut link
// without a corresponding `[foo]: https://foo.com` entry
// at the end of the document, we resolve it as an identifier based on
// what's currently in scope, so you write things like [Str.join] or
// [myFunction] and have them resolve to the docs for what you wrote.
match link.link_type {
LinkType::Shortcut => {
let state = State::new(link.reference.as_bytes());
// Reset the bump arena so we aren't constantly reallocating
// more memory as we iterate through these.
arena.reset();
match parse_ident(&arena, state) {
Ok((_, Ident::Access { module_name, parts }, _)) => {
let mut iter = parts.iter();
match iter.next() {
Some(symbol_name) if iter.next().is_none() => {
let DocUrl { url, title } = doc_url(
home,
exposed_values,
dep_idents,
scope,
interns,
module_name,
symbol_name,
);
Some((url.into(), title.into()))
}
_ => {
// This had record field access,
// e.g. [foo.bar] - which we
// can't create a doc link to!
None
}
}
}
Ok((_, Ident::GlobalTag(type_name), _)) => {
// This looks like a global tag name, but it could
// be a type alias that's in scope, e.g. [I64]
let DocUrl { url, title } = doc_url(
home,
exposed_values,
dep_idents,
scope,
interns,
"",
type_name,
);
Some((url.into(), title.into()))
}
_ => None,
}
}
_ => None,
}
};
let markdown_options = pulldown_cmark::Options::empty();
let mut docs_parser = vec![];
let (_, _) = pulldown_cmark::Parser::new_ext(&markdown_with_links, markdown_options).fold(
(0, 0),
|(start_quote_count, end_quote_count), event| {
match event {
// Replace this sequence (`>>>` syntax):
// Start(BlockQuote)
// Start(BlockQuote)
// Start(BlockQuote)
// Start(Paragraph)
// For `Start(CodeBlock(Fenced(Borrowed("roc"))))`
Event::Start(BlockQuote) => {
let (_, _) = pulldown_cmark::Parser::new_with_broken_link_callback(
&markdown,
markdown_options,
Some(&mut broken_link_callback),
)
.fold((0, 0), |(start_quote_count, end_quote_count), event| {
match &event {
// Replace this sequence (`>>>` syntax):
// Start(BlockQuote)
// Start(BlockQuote)
// Start(BlockQuote)
// Start(Paragraph)
// For `Start(CodeBlock(Fenced(Borrowed("roc"))))`
Event::Start(BlockQuote) => {
docs_parser.push(event);
(start_quote_count + 1, 0)
}
Event::Start(Paragraph) => {
if start_quote_count == 3 {
docs_parser.pop();
docs_parser.pop();
docs_parser.pop();
docs_parser.push(Event::Start(CodeBlock(CodeBlockKind::Fenced(
CowStr::Borrowed("roc"),
))));
} else {
docs_parser.push(event);
(start_quote_count + 1, 0)
}
Event::Start(Paragraph) => {
if start_quote_count == 3 {
docs_parser.pop();
docs_parser.pop();
docs_parser.pop();
docs_parser.push(Event::Start(CodeBlock(CodeBlockKind::Fenced(
CowStr::Borrowed("roc"),
))));
} else {
docs_parser.push(event);
}
(0, 0)
}
// Replace this sequence (`>>>` syntax):
// End(Paragraph)
// End(BlockQuote)
// End(BlockQuote)
// End(BlockQuote)
// For `End(CodeBlock(Fenced(Borrowed("roc"))))`
Event::End(Paragraph) => {
docs_parser.push(event);
(0, 1)
}
Event::End(BlockQuote) => {
if end_quote_count == 3 {
docs_parser.pop();
docs_parser.pop();
docs_parser.pop();
docs_parser.push(Event::End(CodeBlock(CodeBlockKind::Fenced(
CowStr::Borrowed("roc"),
))));
(0, 0)
}
// Replace this sequence (`>>>` syntax):
// End(Paragraph)
// End(BlockQuote)
// End(BlockQuote)
// End(BlockQuote)
// For `End(CodeBlock(Fenced(Borrowed("roc"))))`
Event::End(Paragraph) => {
} else {
docs_parser.push(event);
(0, 1)
}
Event::End(BlockQuote) => {
if end_quote_count == 3 {
docs_parser.pop();
docs_parser.pop();
docs_parser.pop();
docs_parser.push(Event::End(CodeBlock(CodeBlockKind::Fenced(
CowStr::Borrowed("roc"),
))));
(0, 0)
} else {
docs_parser.push(event);
(0, end_quote_count + 1)
}
}
_ => {
docs_parser.push(event);
(0, 0)
(0, end_quote_count + 1)
}
}
},
);
Event::End(Link(LinkType::ShortcutUnknown, _url, _title)) => {
// Replace the preceding Text node with a Code node, so it
// renders as the equivalent of [`List.len`] instead of [List.len]
match docs_parser.pop() {
Some(Event::Text(string)) => {
docs_parser.push(Event::Code(string));
}
Some(first) => {
docs_parser.push(first);
}
None => {}
}
docs_parser.push(event);
(start_quote_count, end_quote_count)
}
_ => {
docs_parser.push(event);
(0, 0)
}
}
});
let mut docs_html = String::new();