docs flow working with mocked markup to html

This commit is contained in:
Anton-4 2021-10-01 18:22:27 +02:00
parent 28cf11a1d9
commit f63c276a5f
9 changed files with 178 additions and 135 deletions

View file

@ -214,7 +214,7 @@ pub fn build_app<'a>() -> App<'a> {
}
pub fn docs(files: Vec<PathBuf>) {
roc_docs::generate(
roc_docs::generate_docs_html(
files,
roc_builtins::std::standard_stdlib(),
Path::new("./generated-docs"),

View file

@ -13,7 +13,6 @@ use super::{
};
use crate::markup_error::{ExpectedTextNode, NestedNodeMissingChild, NestedNodeRequired};
use bumpalo::Bump;
use roc_ast::{
ast_error::ASTResult,
lang::{
@ -306,7 +305,6 @@ fn new_markup_node(
}
pub fn def2_to_markup<'a, 'b>(
arena: &'a Bump,
env: &mut Env<'b>,
def2: &Def2,
def2_node_id: DefId,
@ -321,7 +319,6 @@ pub fn def2_to_markup<'a, 'b>(
expr_id,
} => {
let expr_mn_id = expr2_to_markup(
arena,
env,
env.pool.get(*expr_id),
*expr_id,
@ -348,7 +345,6 @@ pub fn def2_to_markup<'a, 'b>(
// make Markup Nodes: generate String representation, assign Highlighting Style
pub fn expr2_to_markup<'a, 'b>(
arena: &'a Bump,
env: &mut Env<'b>,
expr2: &Expr2,
expr2_node_id: ExprId,
@ -380,7 +376,7 @@ pub fn expr2_to_markup<'a, 'b>(
),
Expr2::Call { expr: expr_id, .. } => {
let expr = env.pool.get(*expr_id);
expr2_to_markup(arena, env, expr, *expr_id, mark_node_pool, interns)?
expr2_to_markup(env, expr, *expr_id, mark_node_pool, interns)?
}
Expr2::Var(symbol) => {
//TODO make bump_format with arena
@ -398,7 +394,6 @@ pub fn expr2_to_markup<'a, 'b>(
let sub_expr2 = env.pool.get(*node_id);
children_ids.push(expr2_to_markup(
arena,
env,
sub_expr2,
*node_id,
@ -460,7 +455,6 @@ pub fn expr2_to_markup<'a, 'b>(
let sub_expr2 = env.pool.get(*sub_expr2_node_id);
children_ids.push(expr2_to_markup(
arena,
env,
sub_expr2,
*sub_expr2_node_id,
@ -520,7 +514,6 @@ pub fn expr2_to_markup<'a, 'b>(
expr_var: _,
} => {
let body_mn_id = expr2_to_markup(
arena,
env,
env.pool.get(*expr_id),
*expr_id,
@ -798,7 +791,6 @@ fn add_header_mn_list(
}
pub fn ast_to_mark_nodes<'a, 'b>(
arena: &'a Bump,
env: &mut Env<'b>,
ast: &AST,
mark_node_pool: &mut SlowPool,
@ -809,7 +801,7 @@ pub fn ast_to_mark_nodes<'a, 'b>(
for &def_id in ast.def_ids.iter() {
let def2 = env.pool.get(def_id);
let def2_markup_id = def2_to_markup(arena, env, def2, def_id, mark_node_pool, interns)?;
let def2_markup_id = def2_to_markup(env, def2, def_id, mark_node_pool, interns)?;
set_parent_for_all(def2_markup_id, mark_node_pool);

View file

@ -83,7 +83,7 @@ impl Scope {
self.idents.len()
}
pub fn lookup(&mut self, ident: &Ident, region: Region) -> Result<Symbol, RuntimeError> {
pub fn lookup(&self, ident: &Ident, region: Region) -> Result<Symbol, RuntimeError> {
match self.idents.get(ident) {
Some((symbol, _)) => Ok(*symbol),
None => Err(RuntimeError::LookupNotInScope(

View file

@ -1,14 +1,4 @@
use crate::html::ToHtml;
use roc_code_markup::{markup::nodes::{MarkupNode}};
impl<'a> ToHtml<'a> for MarkupNode {
fn css_class(&self) -> Option<&'a str> {
Some("operator")
}
fn html_body(&self, buf: &mut bumpalo::collections::String<'a>) {
buf.push_str("MarkupNode")
}
}
struct EqualSign;

View file

@ -1,10 +1,19 @@
use crate::html::ToHtml;
use bumpalo::Bump;
use roc_ast::{ast_error::ASTResult, lang::{core::expr::expr_to_expr2::expr_to_expr2, env::Env, scope::Scope}, mem_pool::pool::Pool};
use roc_ast::{ast_error::ASTResult, lang::{self, core::expr::expr_to_expr2::expr_to_expr2}, mem_pool::pool::Pool};
use roc_code_markup::{markup::nodes::expr2_to_markup, slow_pool::SlowPool};
use roc_module::symbol::Interns;
use roc_parse::ast::{Expr, StrLiteral};
use roc_region::all::{Located, Region};
use roc_region::all::{Region};
use roc_code_markup::{markup::nodes::{MarkupNode}};
impl<'a> ToHtml<'a> for MarkupNode {
fn css_class(&self) -> Option<&'a str> {
Some("operator")
}
fn html_body(&self, buf: &mut bumpalo::collections::String<'a>) {
buf.push_str("MarkupNode")
}
}
impl<'a> ToHtml<'a> for Expr<'a> {
fn css_class(&self) -> Option<&'a str> {
@ -95,7 +104,8 @@ impl<'a> ToHtml<'a> for Expr<'a> {
}
Expr::Defs(defs, sub_expr) => {
for def_loc in defs.iter() {
def_loc.html(buf);
unimplemented!();
//def_loc.html(buf);
}
sub_expr.html(buf)
}
@ -160,14 +170,13 @@ impl<'a> ToHtml<'a> for ParamComma {
}
}
fn write_expr_to_bump_str_html<'a>(
arena: &mut Bump,
env: &mut Env<'a>,
scope: &mut Scope,
pub fn write_expr_to_bump_str_html<'a, 'b>(
env: &mut lang::env::Env<'a>,
scope: &mut lang::scope::Scope,
region: Region,
expr: &'a Expr,
interns: &Interns,
bump_str: &mut bumpalo::collections::String<'a>
bump_str: &mut bumpalo::collections::String<'b>
) -> ASTResult<()> {
let (expr2, _) = expr_to_expr2(env, scope, expr, region);
@ -177,9 +186,8 @@ fn write_expr_to_bump_str_html<'a>(
let mut mark_node_pool = SlowPool::default();
let expr2_markup_id = expr2_to_markup(
arena,
env,
&expr2,
&expr2_pool.get(expr2_id),
expr2_id,
&mut mark_node_pool,
interns,

View file

@ -2,7 +2,9 @@ use roc_region::all::Located;
pub trait ToHtml<'a> {
fn css_class(&self) -> Option<&'a str>;
fn html_body(&self, buf: &mut bumpalo::collections::String<'a>);
fn html(&self, buf: &mut bumpalo::collections::String<'a>) {
let maybe_css_class = self.css_class();

View file

@ -1,7 +1,9 @@
extern crate pulldown_cmark;
extern crate roc_load;
use crate::html::ToHtml;
use bumpalo::Bump;
use bumpalo::{collections::String as BumpString, collections::Vec as BumpVec, Bump};
use expr::write_expr_to_bump_str_html;
use roc_ast::lang;
use roc_ast::mem_pool::pool::Pool;
use roc_builtins::std::StdLib;
use roc_can::builtins::builtin_defs_map;
use roc_can::scope::Scope;
@ -10,10 +12,12 @@ use roc_load::docs::DocEntry::DocDef;
use roc_load::docs::{DocEntry, TypeAnnotation};
use roc_load::docs::{ModuleDocumentation, RecordField};
use roc_load::file::{LoadedModule, LoadingProblem};
use roc_module::symbol::{IdentIds, Interns, ModuleId};
use roc_module::symbol::{IdentIds, Interns, ModuleId, ModuleIds};
use roc_parse::ast::Expr;
use roc_parse::ident::{parse_ident, Ident};
use roc_parse::parser::{State, SyntaxError};
use roc_region::all::Region;
use roc_types::subs::VarStore;
use std::fs;
use std::path::{Path, PathBuf};
@ -23,8 +27,8 @@ mod expr;
mod html;
mod pattern;
pub fn generate(filenames: Vec<PathBuf>, std_lib: StdLib, build_dir: &Path) {
let files_docs = files_to_documentations(filenames, std_lib);
pub fn generate_docs_html(filenames: Vec<PathBuf>, std_lib: StdLib, build_dir: &Path) {
let files_docs = load_modules_for_files(filenames, std_lib);
let mut arena = Bump::new();
//
@ -60,11 +64,11 @@ pub fn generate(filenames: Vec<PathBuf>, std_lib: StdLib, build_dir: &Path) {
.expect("TODO gracefully handle failing to make the favicon");
let template_html = include_str!("./static/index.html")
.replace("<!-- search.js -->", &format!("{}search.js", base_href()))
.replace("<!-- styles.css -->", &format!("{}styles.css", base_href()))
.replace("<!-- search.js -->", &format!("{}search.js", base_url()))
.replace("<!-- styles.css -->", &format!("{}styles.css", base_url()))
.replace(
"<!-- favicon.svg -->",
&format!("{}favicon.svg", base_href()),
&format!("{}favicon.svg", base_url()),
)
.replace(
"<!-- Module links -->",
@ -86,8 +90,8 @@ pub fn generate(filenames: Vec<PathBuf>, std_lib: StdLib, build_dir: &Path) {
for loaded_module in package.modules.iter_mut() {
arena.reset();
let mut exports: bumpalo::collections::Vec<&str> =
bumpalo::collections::Vec::with_capacity_in(loaded_module.exposed_values.len(), &arena);
let mut exports: BumpVec<&str> =
BumpVec::with_capacity_in(loaded_module.exposed_values.len(), &arena);
// TODO should this also include exposed_aliases?
for symbol in loaded_module.exposed_values.iter() {
@ -96,8 +100,8 @@ pub fn generate(filenames: Vec<PathBuf>, std_lib: StdLib, build_dir: &Path) {
let exports = exports.into_bump_slice();
for module in loaded_module.documentation.values_mut() {
let module_dir = build_dir.join(module.name.replace(".", "/").as_str());
for module_docs in loaded_module.documentation.values() {
let module_dir = build_dir.join(module_docs.name.replace(".", "/").as_str());
fs::create_dir_all(&module_dir)
.expect("TODO gracefully handle not being able to create the module dir");
@ -111,11 +115,9 @@ pub fn generate(filenames: Vec<PathBuf>, std_lib: StdLib, build_dir: &Path) {
.replace(
"<!-- Module Docs -->",
render_main_content(
loaded_module.module_id,
exports,
&loaded_module.dep_idents,
&loaded_module.interns,
module,
module_docs,
&loaded_module
)
.as_str(),
);
@ -128,17 +130,56 @@ pub fn generate(filenames: Vec<PathBuf>, std_lib: StdLib, build_dir: &Path) {
println!("🎉 Docs generated in {}", build_dir.display());
}
// html is written to buf
fn expr_to_html<'a>(buf: &mut BumpString<'a>, expr: Expr<'a>, env_module_id: ModuleId, env_module_ids: &'a ModuleIds, interns: &Interns) {
let mut env_pool = Pool::with_capacity(1024);
let mut env_arena = Bump::new();
let mut var_store = VarStore::default();
let dep_idents = IdentIds::exposed_builtins(8);
let exposed_ident_ids = IdentIds::default();
let mut env = lang::env::Env::new(
env_module_id,
&mut env_arena,
&mut env_pool,
&mut var_store,
dep_idents,
env_module_ids,
exposed_ident_ids,
);
let mut scope = lang::scope::Scope::new(env.home, env.pool, env.var_store);
let region = Region::new(0, 0, 0, 0);
// TODO remove unwrap
write_expr_to_bump_str_html(
&mut env,
&mut scope,
region,
&expr,
interns,
buf
).unwrap();
}
// converts plain text code to highlighted html
pub fn syntax_highlight_code<'a>(
arena: &'a Bump,
buf: &mut bumpalo::collections::String<'a>,
buf: &mut BumpString<'a>,
code_str: &'a str,
env_module_id: ModuleId,
env_module_ids: &'a ModuleIds,
interns: &Interns
) -> Result<String, SyntaxError<'a>> {
let trimmed_code_str = code_str.trim_end().trim();
let state = State::new(trimmed_code_str.as_bytes());
match roc_parse::expr::test_parse_expr(0, arena, state) {
Ok(loc_expr) => {
loc_expr.value.html(buf);
expr_to_html(buf, loc_expr.value, env_module_id, env_module_ids, interns);
Ok(buf.to_string())
}
Err(fail) => Err(SyntaxError::Expr(fail)),
@ -150,20 +191,19 @@ pub fn syntax_highlight_code<'a>(
// })
}
// TODO improve name, what is main content?
fn render_main_content(
home: ModuleId,
exposed_values: &[&str],
dep_idents: &MutMap<ModuleId, IdentIds>,
interns: &Interns,
module: &mut ModuleDocumentation,
module: &ModuleDocumentation,
loaded_module: &LoadedModule,
) -> String {
let mut buf = String::new();
buf.push_str(
html_node(
html_to_string(
"h2",
vec![("class", "module-name")],
html_node("a", vec![("href", "/#")], module.name.as_str()).as_str(),
html_to_string("a", vec![("href", "/#")], module.name.as_str()).as_str(),
)
.as_str(),
);
@ -187,7 +227,7 @@ fn render_main_content(
let mut content = String::new();
content.push_str(html_node("a", vec![("href", href.as_str())], name).as_str());
content.push_str(html_to_string("a", vec![("href", href.as_str())], name).as_str());
for type_var in &doc_def.type_vars {
content.push(' ');
@ -206,7 +246,7 @@ fn render_main_content(
type_annotation_to_html(0, &mut content, type_ann);
buf.push_str(
html_node(
html_to_string(
"h3",
vec![("id", name), ("class", "entry-name")],
content.as_str(),
@ -217,12 +257,10 @@ fn render_main_content(
if let Some(docs) = &doc_def.docs {
buf.push_str(
markdown_to_html(
home,
exposed_values,
dep_idents,
&mut module.scope,
interns,
&module.scope,
docs.to_string(),
loaded_module,
)
.as_str(),
);
@ -230,12 +268,10 @@ fn render_main_content(
}
DocEntry::DetachedDoc(docs) => {
let markdown = markdown_to_html(
home,
exposed_values,
dep_idents,
&mut module.scope,
interns,
&module.scope,
docs.to_string(),
loaded_module,
);
buf.push_str(markdown.as_str());
}
@ -246,7 +282,7 @@ fn render_main_content(
buf
}
fn html_node(tag_name: &str, attrs: Vec<(&str, &str)>, content: &str) -> String {
fn html_to_string(tag_name: &str, attrs: Vec<(&str, &str)>, content: &str) -> String {
let mut buf = String::new();
buf.push('<');
@ -275,62 +311,62 @@ fn html_node(tag_name: &str, attrs: Vec<(&str, &str)>, content: &str) -> String
buf
}
fn base_href() -> String {
fn base_url() -> String {
// e.g. "builtins/" in "https://roc-lang.org/builtins/Str"
//
// TODO make this a CLI flag to the `docs` subcommand instead of an env var
match std::env::var("ROC_DOCS_URL_ROOT") {
Ok(root_builtins_path) => {
let mut href = String::with_capacity(root_builtins_path.len() + 64);
let mut url_str = String::with_capacity(root_builtins_path.len() + 64);
if !root_builtins_path.starts_with('/') {
href.push('/');
url_str.push('/');
}
href.push_str(&root_builtins_path);
url_str.push_str(&root_builtins_path);
if !root_builtins_path.ends_with('/') {
href.push('/');
url_str.push('/');
}
href
url_str
}
_ => {
let mut href = String::with_capacity(64);
let mut url_str = String::with_capacity(64);
href.push('/');
url_str.push('/');
href
url_str
}
}
}
fn render_name_and_version(name: &str, version: &str) -> String {
let mut buf = String::new();
let mut href = base_href();
let mut url_str = base_url();
href.push_str(name);
url_str.push_str(name);
buf.push_str(
html_node(
html_to_string(
"h1",
vec![("class", "pkg-full-name")],
html_node("a", vec![("href", href.as_str())], name).as_str(),
html_to_string("a", vec![("href", url_str.as_str())], name).as_str(),
)
.as_str(),
);
let mut versions_href = base_href();
let mut versions_url_str = base_url();
versions_href.push('/');
versions_href.push_str(name);
versions_href.push('/');
versions_href.push_str(version);
versions_url_str.push('/');
versions_url_str.push_str(name);
versions_url_str.push('/');
versions_url_str.push_str(version);
buf.push_str(
html_node(
html_to_string(
"a",
vec![("class", "version"), ("href", versions_href.as_str())],
vec![("class", "version"), ("href", versions_url_str.as_str())],
version,
)
.as_str(),
@ -350,13 +386,13 @@ fn render_sidebar<'a, I: Iterator<Item = (Vec<String>, &'a ModuleDocumentation)>
let name = module.name.as_str();
let href = {
let mut href_buf = base_href();
let mut href_buf = base_url();
href_buf.push_str(name);
href_buf
};
sidebar_entry_content.push_str(
html_node(
html_to_string(
"a",
vec![("class", "sidebar-module-link"), ("href", href.as_str())],
name,
@ -377,7 +413,7 @@ fn render_sidebar<'a, I: Iterator<Item = (Vec<String>, &'a ModuleDocumentation)>
entry_href.push_str(doc_def.name.as_str());
entries_buf.push_str(
html_node(
html_to_string(
"a",
vec![("href", entry_href.as_str())],
doc_def.name.as_str(),
@ -392,7 +428,7 @@ fn render_sidebar<'a, I: Iterator<Item = (Vec<String>, &'a ModuleDocumentation)>
};
sidebar_entry_content.push_str(
html_node(
html_to_string(
"div",
vec![("class", "sidebar-sub-entries")],
entries.as_str(),
@ -401,7 +437,7 @@ fn render_sidebar<'a, I: Iterator<Item = (Vec<String>, &'a ModuleDocumentation)>
);
buf.push_str(
html_node(
html_to_string(
"div",
vec![("class", "sidebar-entry")],
sidebar_entry_content.as_str(),
@ -413,9 +449,9 @@ fn render_sidebar<'a, I: Iterator<Item = (Vec<String>, &'a ModuleDocumentation)>
buf
}
pub fn files_to_documentations(filenames: Vec<PathBuf>, std_lib: StdLib) -> Vec<LoadedModule> {
pub fn load_modules_for_files(filenames: Vec<PathBuf>, std_lib: StdLib) -> Vec<LoadedModule> {
let arena = Bump::new();
let mut files_docs = vec![];
let mut modules = vec![];
for filename in filenames {
let mut src_dir = filename.clone();
@ -430,7 +466,7 @@ pub fn files_to_documentations(filenames: Vec<PathBuf>, std_lib: StdLib) -> Vec<
std::mem::size_of::<usize>() as u32, // This is just type-checking for docs, so "target" doesn't matter
builtin_defs_map,
) {
Ok(loaded) => files_docs.push(loaded),
Ok(loaded) => modules.push(loaded),
Err(LoadingProblem::FormattedReport(report)) => {
println!("{}", report);
panic!();
@ -439,7 +475,7 @@ pub fn files_to_documentations(filenames: Vec<PathBuf>, std_lib: StdLib) -> Vec<
}
}
files_docs
modules
}
const INDENT: &str = " ";
@ -454,6 +490,7 @@ fn new_line(buf: &mut String) {
buf.push('\n');
}
// html is written to buf
fn type_annotation_to_html(indent_level: usize, buf: &mut String, type_ann: &TypeAnnotation) {
let is_multiline = should_be_multiline(type_ann);
match type_ann {
@ -715,7 +752,7 @@ fn doc_url<'a>(
home: ModuleId,
exposed_values: &[&str],
dep_idents: &MutMap<ModuleId, IdentIds>,
scope: &mut Scope,
scope: &Scope,
interns: &'a Interns,
mut module_name: &'a str,
ident: &str,
@ -782,7 +819,7 @@ fn doc_url<'a>(
}
}
let mut url = base_href();
let mut url = base_url();
// Example:
//
@ -798,12 +835,10 @@ fn doc_url<'a>(
}
fn markdown_to_html(
home: ModuleId,
exposed_values: &[&str],
dep_idents: &MutMap<ModuleId, IdentIds>,
scope: &mut Scope,
interns: &Interns,
scope: &Scope,
markdown: String,
loaded_module: &LoadedModule,
) -> String {
use pulldown_cmark::{BrokenLink, CodeBlockKind, CowStr, Event, LinkType, Tag::*};
@ -830,11 +865,11 @@ fn markdown_to_html(
match iter.next() {
Some(symbol_name) if iter.next().is_none() => {
let DocUrl { url, title } = doc_url(
home,
loaded_module.module_id,
exposed_values,
dep_idents,
&loaded_module.dep_idents,
scope,
interns,
&loaded_module.interns,
module_name,
symbol_name,
);
@ -853,11 +888,11 @@ fn markdown_to_html(
// This looks like a global tag name, but it could
// be a type alias that's in scope, e.g. [I64]
let DocUrl { url, title } = doc_url(
home,
loaded_module.module_id,
exposed_values,
dep_idents,
&loaded_module.dep_idents,
scope,
interns,
&loaded_module.interns,
"",
type_name,
);
@ -961,8 +996,16 @@ fn markdown_to_html(
Event::Text(CowStr::Borrowed(code_str)) if expecting_code_block => {
let code_block_arena = Bump::new();
let mut code_block_buf = bumpalo::collections::String::new_in(&code_block_arena);
match syntax_highlight_code(&code_block_arena, &mut code_block_buf, code_str) {
let mut code_block_buf = BumpString::new_in(&code_block_arena);
match syntax_highlight_code(
&code_block_arena,
&mut code_block_buf,
code_str,
loaded_module.module_id,
&loaded_module.interns.module_ids,
&loaded_module.interns
)
{
Ok(highlighted_code_str) => {
docs_parser.push(Event::Html(CowStr::from(highlighted_code_str)));
}

View file

@ -3,7 +3,7 @@ extern crate pretty_assertions;
#[cfg(test)]
mod insert_doc_syntax_highlighting {
use bumpalo::Bump;
use bumpalo::{collections::String as BumpString, Bump};
use roc_can::env::Env;
use roc_can::scope::Scope;
use roc_collections::all::MutMap;
@ -14,8 +14,24 @@ mod insert_doc_syntax_highlighting {
fn expect_html(code_str: &str, want: &str) {
let code_block_arena = Bump::new();
let mut code_block_buf = bumpalo::collections::String::new_in(&code_block_arena);
match syntax_highlight_code(&code_block_arena, &mut code_block_buf, code_str) {
let mut module_ids = ModuleIds::default();
let mod_id = module_ids.get_or_insert(&"ModId123".into());
let interns = Interns {
module_ids: module_ids.clone(),
all_ident_ids: IdentIds::exposed_builtins(8),
};
let mut code_block_buf = BumpString::new_in(&code_block_arena);
match syntax_highlight_code(
&code_block_arena,
&mut code_block_buf,
code_str,
mod_id,
&module_ids,
&interns
) {
Ok(highlighted_code_str) => {
assert_eq!(highlighted_code_str, want);
}
@ -25,7 +41,7 @@ mod insert_doc_syntax_highlighting {
};
}
#[test]
/*#[test]
fn simple_code_block() {
expect_html(
r#"
@ -43,24 +59,17 @@ mod insert_doc_syntax_highlighting {
<div class="syntax-number">2</div>
"#,
);
}*/
#[test]
fn simple_code_block() {
expect_html(
r#"
2
"#,
r#"
<div class="syntax-number">2</div>
"#,
);
}
// #[test]
// fn simple_code_block() {
// expect_html(
// r#"
// x : Nat
// x =
// 4
//
// 2
// "#,
// r#"
// <div class="syntax-var">x</div> <div class="syntax-operator">:</div> <div class="syntax-type">Nat</div>
// <div class="syntax-var">x</div> <div class="syntax-operator">=</div>
// <div class="syntax-number">4</div>
//
// <div class="syntax-number">2</div>
// "#,
// );
// }
}

View file

@ -63,7 +63,6 @@ pub fn init_model<'a>(
EmptyCodeString {}.fail()
} else {
Ok(ast_to_mark_nodes(
code_arena,
&mut module.env,
&module.ast,
&mut mark_node_pool,