From 432dbee422ddabaf17e5003dca7219bebdc112ab Mon Sep 17 00:00:00 2001 From: Folkert Date: Wed, 21 Oct 2020 15:43:56 +0200 Subject: [PATCH 1/2] switch docs to use the standard file load mechanism --- Cargo.lock | 1 + compiler/load/src/docs.rs | 2 +- compiler/load/src/file.rs | 34 ++++++++++++++++++++++++++++++++++ docs/Cargo.toml | 1 + docs/src/main.rs | 15 ++++++++++++--- 5 files changed, 49 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 22175bc283..c952c253be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -614,6 +614,7 @@ dependencies = [ name = "docs" version = "0.1.0" dependencies = [ + "bumpalo", "fs_extra", "handlebars", "pulldown-cmark", diff --git a/compiler/load/src/docs.rs b/compiler/load/src/docs.rs index 3d034f5306..28ad23926c 100644 --- a/compiler/load/src/docs.rs +++ b/compiler/load/src/docs.rs @@ -1574,7 +1574,7 @@ pub struct DocEntry { pub docs: Option, } -fn generate_module_docs<'a>( +pub fn generate_module_docs<'a>( module_name: ModuleName, exposed_ident_ids: &'a IdentIds, parsed_defs: &'a bumpalo::collections::Vec<'a, Located>>, diff --git a/compiler/load/src/file.rs b/compiler/load/src/file.rs index 5318e5296b..7d6f5d9692 100644 --- a/compiler/load/src/file.rs +++ b/compiler/load/src/file.rs @@ -1,3 +1,4 @@ +use crate::docs::ModuleDocumentation; use bumpalo::Bump; use crossbeam::channel::{bounded, Sender}; use crossbeam::deque::{Injector, Stealer, Worker}; @@ -202,6 +203,7 @@ struct ModuleCache<'a> { typechecked: MutMap>, found_specializations: MutMap>, external_specializations_requested: MutMap, + documentation: MutMap, } fn start_phase<'a>(module_id: ModuleId, phase: Phase, state: &mut State<'a>) -> BuildTask<'a> { @@ -378,6 +380,7 @@ pub struct LoadedModule { pub exposed_to_host: MutMap, pub src: Box, pub timings: MutMap, + pub documentation: MutMap, } #[derive(Debug)] @@ -458,6 +461,7 @@ enum Msg<'a> { problems: Vec, var_store: VarStore, module_timing: ModuleTiming, + module_docs: ModuleDocumentation, }, SolvedTypes { src: &'a str, @@ -471,6 +475,7 @@ enum Msg<'a> { FinishedAllTypeChecking { solved_subs: Solved, exposed_vars_by_symbol: Vec<(Symbol, Variable)>, + documentation: MutMap, src: &'a str, }, FoundSpecializations { @@ -1096,6 +1101,7 @@ where Msg::FinishedAllTypeChecking { solved_subs, exposed_vars_by_symbol, + documentation, src, } => { // We're done! There should be no more messages pending. @@ -1112,6 +1118,7 @@ where state, solved_subs, exposed_vars_by_symbol, + documentation, src, ))); } @@ -1228,11 +1235,17 @@ fn update<'a>( problems, var_store, module_timing, + module_docs, } => { log!("generated constraints for {:?}", module.module_id); let module_id = module.module_id; state.can_problems.extend(problems); + state + .module_cache + .documentation + .insert(module_id, module_docs); + let constrained_module = ConstrainedModule { module, constraint, @@ -1243,6 +1256,7 @@ fn update<'a>( var_store, imported_modules, }; + state .module_cache .constrained @@ -1288,10 +1302,18 @@ fn update<'a>( state.timings.insert(module_id, module_timing); + let documentation = { + let mut empty = MutMap::default(); + std::mem::swap(&mut empty, &mut state.module_cache.documentation); + + empty + }; + msg_tx .send(Msg::FinishedAllTypeChecking { solved_subs, exposed_vars_by_symbol: solved_module.exposed_vars_by_symbol, + documentation, src, }) .map_err(|_| LoadingProblem::MsgChannelDied)?; @@ -1514,6 +1536,7 @@ fn finish<'a>( state: State<'a>, solved: Solved, exposed_vars_by_symbol: Vec<(Symbol, Variable)>, + documentation: MutMap, src: &'a str, ) -> LoadedModule { let module_ids = Arc::try_unwrap(state.arc_modules) @@ -1535,6 +1558,7 @@ fn finish<'a>( exposed_to_host: exposed_vars_by_symbol.into_iter().collect(), src: src.into(), timings: state.timings, + documentation, } } @@ -1994,6 +2018,15 @@ fn parse_and_constrain<'a>( // immediately afterward (for the beginning of canonicalization). let parse_end = SystemTime::now(); let module_id = header.module_id; + + // Generate documentation information + // TODO: store timing information? + let module_docs = crate::docs::generate_module_docs( + header.module_name, + &header.exposed_ident_ids, + &parsed_defs, + ); + let mut var_store = VarStore::default(); let canonicalized = canonicalize_module_defs( &arena, @@ -2070,6 +2103,7 @@ fn parse_and_constrain<'a>( problems, var_store, module_timing, + module_docs, }) } diff --git a/docs/Cargo.toml b/docs/Cargo.toml index 22014d6153..7deb6bd0ec 100644 --- a/docs/Cargo.toml +++ b/docs/Cargo.toml @@ -16,3 +16,4 @@ pulldown-cmark = { version = "0.8", default-features = false } roc_load = { path = "../compiler/load" } roc_builtins = { path = "../compiler/builtins" } roc_collections = { path = "../compiler/collections" } +bumpalo = { version = "3.2", features = ["collections"] } diff --git a/docs/src/main.rs b/docs/src/main.rs index c825e4f8aa..101c28c6f5 100644 --- a/docs/src/main.rs +++ b/docs/src/main.rs @@ -8,6 +8,7 @@ extern crate serde_json; use std::error::Error; use std::fs; extern crate roc_load; +use bumpalo::Bump; use roc_collections::all::MutMap; use std::path::{Path, PathBuf}; @@ -43,6 +44,8 @@ pub struct TemplateLinkEntry { fn main() -> Result<(), Box> { let std_lib = roc_builtins::std::standard_stdlib(); let subs_by_module = MutMap::default(); + let arena = Bump::new(); + let src_dir = Path::new("../compiler/builtins/docs"); let files = vec![ PathBuf::from(r"../compiler/builtins/docs/Bool.roc"), @@ -59,9 +62,15 @@ fn main() -> Result<(), Box> { // Load each file is files vector for filename in files { - let loaded = roc_load::docs::load(filename, &std_lib, src_dir, subs_by_module.clone()) - .expect("TODO gracefully handle load failing"); - modules_docs.push(loaded.module_docs); + let mut loaded = roc_load::file::load_and_typecheck( + &arena, + filename, + std_lib.clone(), + src_dir, + subs_by_module.clone(), + ) + .expect("TODO gracefully handle load failing"); + modules_docs.extend(loaded.documentation.drain().map(|x| x.1)); } let package = roc_load::docs::Documentation { From 7bff0b50c60f43387432d157919984bb594c99b0 Mon Sep 17 00:00:00 2001 From: Folkert Date: Wed, 21 Oct 2020 15:49:20 +0200 Subject: [PATCH 2/2] remove now unneeded code --- compiler/load/src/docs.rs | 1553 +------------------------------------ 1 file changed, 4 insertions(+), 1549 deletions(-) diff --git a/compiler/load/src/docs.rs b/compiler/load/src/docs.rs index 28ad23926c..5284790c8c 100644 --- a/compiler/load/src/docs.rs +++ b/compiler/load/src/docs.rs @@ -1,1555 +1,10 @@ // This file was copied from file.rs and modified to expose information // required to auto-generate documentation -use bumpalo::Bump; -use crossbeam::channel::{bounded, Sender}; -use crossbeam::deque::{Injector, Stealer, Worker}; -use crossbeam::thread; use inlinable_string::InlinableString; -use parking_lot::Mutex; -use roc_builtins::std::{Mode, StdLib}; -use roc_can::constraint::Constraint; -use roc_can::def::Declaration; -use roc_can::module::{canonicalize_module_defs, Module}; -use roc_collections::all::{default_hasher, MutMap, MutSet}; -use roc_constrain::module::{ - constrain_imports, load_builtin_aliases, pre_constrain_imports, ConstrainableImports, Import, -}; -use roc_constrain::module::{constrain_module, ExposedModuleTypes, SubsByModule}; -use roc_module::ident::{Ident, ModuleName}; -use roc_module::symbol::{IdentIds, Interns, ModuleId, ModuleIds, Symbol}; +use roc_module::ident::ModuleName; +use roc_module::symbol::IdentIds; use roc_parse::ast::Def; -use roc_parse::ast::{self, Attempting, ExposesEntry, ImportsEntry}; -use roc_parse::module::module_defs; -use roc_parse::parser::{self, Fail, Parser}; -use roc_region::all::{Located, Region}; -use roc_solve::module::SolvedModule; -use roc_solve::solve; -use roc_types::solved_types::Solved; -use roc_types::subs::{Subs, VarStore, Variable}; -use roc_types::types::Alias; -use std::collections::{HashMap, HashSet}; -use std::fs; -use std::io; -use std::iter; -use std::path::{Path, PathBuf}; -use std::str::from_utf8_unchecked; -use std::sync::Arc; -use std::time::{Duration, SystemTime}; - -/// Filename extension for normal Roc modules -const ROC_FILE_EXTENSION: &str = "roc"; - -/// The . in between module names like Foo.Bar.Baz -const MODULE_SEPARATOR: char = '.'; - -#[derive(Debug)] -pub struct LoadedModule { - pub module_id: ModuleId, - pub interns: Interns, - pub solved: Solved, - pub can_problems: Vec, - pub type_problems: Vec, - pub declarations_by_id: MutMap>, - pub exposed_vars_by_symbol: Vec<(Symbol, Variable)>, - pub src: Box, - pub timings: MutMap, - pub module_docs: ModuleDocumentation, -} - -#[derive(Debug)] -pub struct LoadedModuleForDocumentation { - pub module_id: ModuleId, - pub interns: Interns, - pub solved: Solved, - pub can_problems: Vec, - pub type_problems: Vec, - pub declarations_by_id: MutMap>, - pub exposed_vars_by_symbol: Vec<(Symbol, Variable)>, - pub src: Box, - pub timings: MutMap, -} - -#[derive(Debug)] -pub enum BuildProblem<'a> { - FileNotFound(&'a Path), -} - -#[derive(Debug)] -struct ModuleHeader<'a> { - module_id: ModuleId, - module_name: ModuleName, - exposed_ident_ids: IdentIds, - deps_by_name: MutMap, - imported_modules: MutSet, - exposes: Vec, - exposed_imports: MutMap, - src: &'a [u8], - module_timing: ModuleTiming, -} - -#[derive(Debug)] -enum Msg<'a> { - Header(ModuleHeader<'a>), - Constrained { - module: Module, - declarations: Vec, - imported_modules: MutSet, - src: &'a str, - constraint: Constraint, - ident_ids: IdentIds, - problems: Vec, - var_store: VarStore, - module_timing: ModuleTiming, - module_docs: ModuleDocumentation, - }, - Solved { - src: &'a str, - module_id: ModuleId, - solved_module: SolvedModule, - solved_subs: Solved, - module_timing: ModuleTiming, - module_docs: ModuleDocumentation, - }, - Finished { - solved_subs: Solved, - problems: Vec, - exposed_vars_by_symbol: Vec<(Symbol, Variable)>, - src: &'a str, - module_docs: ModuleDocumentation, - }, -} - -#[derive(Debug)] -struct State<'a> { - pub root_id: ModuleId, - pub exposed_types: SubsByModule, - - pub can_problems: Vec, - pub headers_parsed: MutSet, - pub type_problems: Vec, - - /// This is the "final" list of IdentIds, after canonicalization and constraint gen - /// have completed for a given module. - pub constrained_ident_ids: MutMap, - - /// From now on, these will be used by multiple threads; time to make an Arc>! - pub arc_modules: Arc>, - - pub ident_ids_by_module: Arc>, - - /// All the dependent modules we've already begun loading - - /// meaning we should never kick off another load_module on them! - pub loading_started: MutSet, - - pub declarations_by_id: MutMap>, - - pub exposed_symbols_by_module: MutMap>, - - /// Modules which are waiting for certain headers to be parsed - pub waiting_for_headers: MutMap>, - - // When the key ModuleId gets solved, iterate through each of the given modules - // a,d remove that ModuleId from the appropriate waiting_for_headers entry. - // If the relevant module's waiting_for_headers entry is now empty, canonicalize the module. - pub header_listeners: MutMap>, - - pub unparsed_modules: MutMap>, - - // Modules which are waiting for certain deps to be solved - pub waiting_for_solve: MutMap>, - - // When the key ModuleId gets solved, iterate through each of the given modules - // and remove that ModuleId from the appropriate waiting_for_solve entry. - // If the relevant module's waiting_for_solve entry is now empty, solve the module. - pub solve_listeners: MutMap>, - - pub unsolved_modules: MutMap>, - - pub timings: MutMap, -} - -#[derive(Debug)] -struct UnsolvedModule<'a> { - module: Module, - src: &'a str, - imported_modules: MutSet, - constraint: Constraint, - var_store: VarStore, - module_timing: ModuleTiming, -} - -#[derive(Debug)] -pub struct ModuleTiming { - pub read_roc_file: Duration, - pub parse_header: Duration, - pub parse_body: Duration, - pub canonicalize: Duration, - pub constrain: Duration, - pub solve: Duration, - // TODO pub monomorphize: Duration, - /// Total duration will always be more than the sum of the other fields, due - /// to things like state lookups in between phases, waiting on other threads, etc. - start_time: SystemTime, - end_time: SystemTime, -} - -impl ModuleTiming { - pub fn total(&self) -> Duration { - self.end_time.duration_since(self.start_time).unwrap() - } - - /// Subtract all the other fields from total_start_to_finish - pub fn other(&self) -> Duration { - let Self { - read_roc_file, - parse_header, - parse_body, - canonicalize, - constrain, - solve, - start_time, - end_time, - } = self; - - end_time - .duration_since(*start_time) - .ok() - .and_then(|t| { - t.checked_sub(*solve).and_then(|t| { - t.checked_sub(*constrain).and_then(|t| { - t.checked_sub(*canonicalize).and_then(|t| { - t.checked_sub(*parse_body).and_then(|t| { - t.checked_sub(*parse_header) - .and_then(|t| t.checked_sub(*read_roc_file)) - }) - }) - }) - }) - }) - .unwrap_or_else(Duration::default) - } -} - -#[derive(Debug)] -enum BuildTask<'a> { - LoadModule { - module_name: ModuleName, - module_ids: Arc>, - ident_ids_by_module: Arc>, - }, - ParseAndConstrain { - header: ModuleHeader<'a>, - mode: Mode, - module_ids: ModuleIds, - dep_idents: IdentIdsByModule, - exposed_symbols: MutSet, - }, - - Solve { - module: Module, - imported_symbols: Vec, - imported_aliases: MutMap, - module_timing: ModuleTiming, - constraint: Constraint, - var_store: VarStore, - src: &'a str, - module_docs: ModuleDocumentation, - }, -} - -enum WorkerMsg { - Shutdown, - TaskAdded, -} - -#[derive(Debug)] -pub enum LoadingProblem { - FileProblem { - filename: PathBuf, - error: io::ErrorKind, - }, - ParsingFailed { - filename: PathBuf, - fail: Fail, - }, - MsgChannelDied, - ErrJoiningWorkerThreads, - TriedToImportAppModule, -} - -type IdentIdsByModule = MutMap; -type MsgSender<'a> = Sender>; - -/// Add a task to the queue, and notify all the listeners. -fn enqueue_task<'a>( - injector: &Injector>, - listeners: &[Sender], - task: BuildTask<'a>, -) -> Result<(), LoadingProblem> { - injector.push(task); - - for listener in listeners { - listener - .send(WorkerMsg::TaskAdded) - .map_err(|_| LoadingProblem::MsgChannelDied)?; - } - - Ok(()) -} - -/// The loading process works like this, starting from the given filename (e.g. "main.roc"): -/// -/// 1. Open the file. -/// 2. Parse the module's header. -/// 3. For each of its imports, send a message on the channel to the coordinator thread, which -/// will repeat this process to load that module - starting with step 1. -/// 4. Add everything we were able to import unqualified to the module's default scope. -/// 5. Parse the module's defs. -/// 6. Canonicalize the module. -/// 7. Before type checking, block on waiting for type checking to complete on all imports. -/// (Since Roc doesn't allow cyclic dependencies, this ctypeot deadlock.) -/// 8. Type check the module and create type annotations for its top-level declarations. -/// 9. Report the completed type annotation to the coordinator thread, so other modules -/// that are waiting in step 7 can unblock. -/// -/// The loaded_modules argument specifies which modules have already been loaded. -/// It typically contains *at least* the standard modules, but is empty when loading -/// the standard modules themselves. -/// -/// If we're just type-checking everything (e.g. running `roc check` at the command line), -/// we can stop there. However, if we're generating code, then there are additional steps. -/// -/// 10. After reporting the completed type annotation, we have all the information necessary -/// to monomorphize. However, since we want to monomorphize in parallel without -/// duplicating work, we do monomorphization in two steps. First, we go through and -/// determine all the specializations this module *wants*. We compute the hashes -/// and report them to the coordinator thread, along with the mono::expr::Expr values of -/// the current function's body. At this point, we have not yet begun to assemble Procs; -/// all we've done is send a list of requetsted specializations to the coordinator. -/// 11. The coordinator works through the specialization requests in parallel, adding them -/// to a global map once they're finished. Performing one specialization may result -/// in requests for others; these are added to the queue and worked through as normal. -/// This process continues until *both* all modules have reported that they've finished -/// adding specialization requests to the queue, *and* the queue is empty (including -/// of any requestss that were added in the course of completing other requests). Now -/// we have a map of specializations, and everything was assembled in parallel with -/// no unique specialization ever getting assembled twice (meanaing no wasted effort). -/// 12. Now that we have our final map of specializations, we can proceed to code gen! -/// As long as the specializations are stored in a per-ModuleId map, we can also -/// parallelize this code gen. (e.g. in dev builds, building separate LLVM modules -/// and then linking them together, and possibly caching them by the hash of their -/// specializations, so if none of their specializations changed, we don't even need -/// to rebuild the module and can link in the cached one directly.) -// #[allow(clippy::cognitive_complexity)] -pub fn load( - filename: PathBuf, - stdlib: &StdLib, - src_dir: &Path, - exposed_types: SubsByModule, -) -> Result { - let arena = Bump::new(); - - // Reserve one CPU for the main thread, and let all the others be eligible - // to spawn workers. We use .max(2) to enforce that we always - // end up with at least 1 worker - since (.max(2) - 1) will - // always return a number that's at least 1. Using - // .max(2) on the initial number of CPUs instead of - // doing .max(1) on the entire expression guards against - // num_cpus returning 0, while also avoiding wrapping - // unsigned subtraction overflow. - let num_workers = num_cpus::get().max(2) - 1; - - let mut worker_arenas = bumpalo::collections::Vec::with_capacity_in(num_workers, &arena); - - for _ in 0..num_workers { - worker_arenas.push(Bump::new()); - } - - let (msg_tx, msg_rx) = bounded(1024); - let arc_modules = Arc::new(Mutex::new(ModuleIds::default())); - let root_exposed_ident_ids = IdentIds::exposed_builtins(0); - let ident_ids_by_module = Arc::new(Mutex::new(root_exposed_ident_ids)); - - // Load the root module synchronously; we can't proceed until we have its id. - let (root_id, root_msg) = { - let root_start_time = SystemTime::now(); - - load_filename( - &arena, - filename, - Arc::clone(&arc_modules), - Arc::clone(&ident_ids_by_module), - root_start_time, - )? - }; - - msg_tx - .send(root_msg) - .map_err(|_| LoadingProblem::MsgChannelDied)?; - - // We'll add tasks to this, and then worker threads will take tasks from it. - let injector = Injector::new(); - - // We need to allocate worker *queues* on the main thread and then move them - // into the worker threads, because those workers' stealers need to be - // shared bet,een all threads, and this coordination work is much easier - // on the main thread. - let mut worker_queues = bumpalo::collections::Vec::with_capacity_in(num_workers, &arena); - let mut stealers = bumpalo::collections::Vec::with_capacity_in(num_workers, &arena); - - thread::scope(|thread_scope| { - for _ in 0..num_workers { - let worker = Worker::new_lifo(); - - stealers.push(worker.stealer()); - worker_queues.push(worker); - } - - // Get a reference to the completed stealers, so we can send that - // reference to each worker. (Slices are Sync, but bumpalo Vecs are not.) - let stealers = stealers.into_bump_slice(); - - let mut headers_parsed = MutSet::default(); - - // We've already parsed the root's header. (But only its header, so far.) - headers_parsed.insert(root_id); - - let mut loading_started = MutSet::default(); - - // If the root module we're still processing happens to be an interface, - // it's possible that something else will import it. That will - // necessarily cause a cyclic import error, but in the meantime - // we still shouldn't load it. - loading_started.insert(root_id); - - let mut state = State { - root_id, - exposed_types, - headers_parsed, - loading_started, - can_problems: Vec::new(), - type_problems: Vec::new(), - arc_modules, - constrained_ident_ids: IdentIds::exposed_builtins(0), - ident_ids_by_module, - declarations_by_id: MutMap::default(), - exposed_symbols_by_module: MutMap::default(), - waiting_for_headers: MutMap::default(), - header_listeners: MutMap::default(), - unparsed_modules: MutMap::default(), - waiting_for_solve: MutMap::default(), - solve_listeners: MutMap::default(), - unsolved_modules: MutMap::default(), - timings: MutMap::default(), - }; - - let mut worker_listeners = bumpalo::collections::Vec::with_capacity_in(num_workers, &arena); - - for worker_arena in worker_arenas.iter_mut() { - let msg_tx = msg_tx.clone(); - let worker = worker_queues.pop().unwrap(); - let (worker_msg_tx, worker_msg_rx) = bounded(1024); - - worker_listeners.push(worker_msg_tx); - - // We only want to move a *reference* to the main task queue's - // injector in the thread, not the injector itself - // (since other threads need to reference it too). - let injector = &injector; - - // Record this thread's handle so the main thread can join it later. - thread_scope.spawn(move |_| { - // Keep listening until we receive a Shutdown msg - for msg in worker_msg_rx.iter() { - match msg { - WorkerMsg::Shutdown => { - // We've finished all our work. It's time to - // shut down the thread, so when the main thread - // blocks on joining with all the worker threads, - // it can finally exit too! - return; - } - WorkerMsg::TaskAdded => { - // Find a task - either from this thread's queue, - // or from the main queue, or from another worker's - // queue - and run it. - // - // There might be no tasks to work on! That could - // happen if another thread is working on a task - // which will later result in more tasks being - // added. In that case, do nothing, and keep waiting - // until we receive a Shutdown message. - if let Some(task) = find_task(&worker, injector, stealers) { - run_task(task, worker_arena, src_dir, msg_tx.clone(), stdlib) - .expect("Msg channel closed unexpectedly."); - } - } - } - } - - // Needed to prevent a borrow checker error about this closure - // outliving its enclosing function. - drop(worker_msg_rx); - }); - } - - // We've now distributed one worker queue to each thread. - // There should be no queues left to distribute! - debug_assert!(worker_queues.is_empty()); - drop(worker_queues); - - // Grab a reference to these Senders outside the loop, so we can share - // it across each iteration of the loop. - let worker_listeners = worker_listeners.into_bump_slice(); - let msg_tx = msg_tx.clone(); - - // The root module will have already queued up messages to process, - // and processing those messages will in turn queue up more messages. - for msg in msg_rx.iter() { - match msg { - Msg::Finished { - solved_subs, - problems, - exposed_vars_by_symbol, - src, - module_docs, - } => { - // We're done! There should be no more messages pending. - debug_assert!(msg_rx.is_empty()); - - // Shut down all the worker threads. - for listener in worker_listeners { - listener - .send(WorkerMsg::Shutdown) - .map_err(|_| LoadingProblem::MsgChannelDied)?; - } - - return Ok(finish( - state, - solved_subs, - problems, - exposed_vars_by_symbol, - src, - module_docs, - )); - } - msg => { - // This is where most of the main thread's work gets done. - // Everything up to this point has been setting up the threading - // system which lets this logic work efficiently. - state = update( - state, - msg, - stdlib, - msg_tx.clone(), - &injector, - worker_listeners, - )?; - } - } - } - - // The msg_rx receiver closed unexpectedly before we finished solving everything - Err(LoadingProblem::MsgChannelDied) - }) - .unwrap() -} - -fn update<'a>( - mut state: State<'a>, - msg: Msg<'a>, - stdlib: &StdLib, - msg_tx: MsgSender<'a>, - injector: &Injector>, - worker_listeners: &'a [Sender], -) -> Result, LoadingProblem> { - use self::Msg::*; - - match msg { - Header(header) => { - let home = header.module_id; - let deps_by_name = &header.deps_by_name; - let mut headers_needed = - HashSet::with_capacity_and_hasher(deps_by_name.len(), default_hasher()); - - state.headers_parsed.insert(home); - - for dep_id in deps_by_name.values() { - if !state.headers_parsed.contains(&dep_id) { - headers_needed.insert(*dep_id); - } - } - - // This was a dependency. Write it down and keep processing messaages. - let mut exposed_symbols: MutSet = - HashSet::with_capacity_and_hasher(header.exposes.len(), default_hasher()); - - // TODO can we avoid this loop by storing them as a Set in Header to begin with? - for symbol in header.exposes.iter() { - exposed_symbols.insert(*symbol); - } - - debug_assert!(!state.exposed_symbols_by_module.contains_key(&home)); - state - .exposed_symbols_by_module - .insert(home, exposed_symbols); - - // Notify all the listeners that headers are now available for this module. - if let Some(listeners) = state.header_listeners.remove(&home) { - for listener_id in listeners { - // This listener is longer waiting for this module, - // because this module's headers are now available! - let waiting_for = state - .waiting_for_headers - .get_mut(&listener_id) - .expect("Unable to find module ID in waiting_for_headers"); - - waiting_for.remove(&home); - - // If it's no longer waiting for anything else, solve it. - if waiting_for.is_empty() { - let header = state - .unparsed_modules - .remove(&listener_id) - .expect("Could not find listener ID in unparsed_modules"); - - let exposed_symbols = state - .exposed_symbols_by_module - .remove(&listener_id) - .expect("Could not find listener ID in exposed_symbols_by_module"); - - enqueue_task( - injector, - worker_listeners, - BuildTask::parse_and_constrain( - header, - stdlib.mode, - Arc::clone(&state.arc_modules), - Arc::clone(&state.ident_ids_by_module), - &state.exposed_types, - exposed_symbols.clone(), - &mut state.waiting_for_solve, - ), - )?; - } - } - } - - // If any of our deps weren't loaded before, start loading them. - for (dep_name, dep_id) in deps_by_name.iter() { - if !state.loading_started.contains(&dep_id) { - // Record that we've started loading the module *before* - // we actually start loading it. - state.loading_started.insert(*dep_id); - - // Start loading this module in the background. - enqueue_task( - injector, - worker_listeners, - BuildTask::LoadModule { - module_name: dep_name.clone(), - // Provide mutexes of ModuleIds and IdentIds by module, - // so other modules can populate them as they load. - module_ids: Arc::clone(&state.arc_modules), - ident_ids_by_module: Arc::clone(&state.ident_ids_by_module), - }, - )?; - } - } - - if headers_needed.is_empty() { - let exposed_symbols = state - .exposed_symbols_by_module - .remove(&home) - .expect("Could not find listener ID in exposed_symbols_by_module"); - - enqueue_task( - injector, - worker_listeners, - BuildTask::parse_and_constrain( - header, - stdlib.mode, - Arc::clone(&state.arc_modules), - Arc::clone(&state.ident_ids_by_module), - &state.exposed_types, - exposed_symbols, - &mut state.waiting_for_solve, - ), - )?; - } else { - // We will have to wait for our deps' headers to be parsed, - // so we can access their IdentId, which we need for canonicalization. - debug_assert!(!state.unparsed_modules.contains_key(&home)); - state.unparsed_modules.insert(home, header); - - // Register a listener with each of these. - for dep_id in headers_needed.iter() { - let listeners = state - .header_listeners - .entry(*dep_id) - .or_insert_with(|| Vec::with_capacity(1)); - - (*listeners).push(home); - } - - debug_assert!(!state.waiting_for_headers.contains_key(&home)); - state.waiting_for_headers.insert(home, headers_needed); - } - - Ok(state) - } - Constrained { - module, - declarations, - src, - ident_ids, - imported_modules, - constraint, - problems, - var_store, - module_timing, - module_docs, - } => { - state.can_problems.extend(problems); - - let module_id = module.module_id; - let State { - waiting_for_solve, - exposed_types, - constrained_ident_ids, - declarations_by_id, - unsolved_modules, - solve_listeners, - .. - } = &mut state; - let waiting_for = waiting_for_solve.get_mut(&module_id).unwrap_or_else(|| { - panic!( - "Could not find module ID {:?} in waiting_for_solve", - module_id - ) - }); - - // Record the final IdentIds - debug_assert!(!constrained_ident_ids.contains_key(&module_id)); - constrained_ident_ids.insert(module_id, ident_ids); - - // It's possible that some modules have been solved since - // we began waiting for them. Remove those from waiting_for, - // because we no longer need to wait for them! - waiting_for.retain(|id| !exposed_types.contains_key(id)); - - declarations_by_id.insert(module_id, declarations); - - if waiting_for.is_empty() { - // All of our dependencies have already been solved. Great! - // That means we can proceed directly to solving. - enqueue_task( - injector, - worker_listeners, - BuildTask::solve_module( - module, - module_timing, - src, - constraint, - var_store, - imported_modules, - &mut state.exposed_types, - stdlib, - module_docs, - ), - )?; - } else { - // We will have to wait for our dependencies to be solved. - debug_assert!(!unsolved_modules.contains_key(&module_id)); - unsolved_modules.insert( - module_id, - UnsolvedModule { - module, - src, - imported_modules, - constraint, - var_store, - module_timing, - }, - ); - - // Register a listener with each of these. - for dep_id in waiting_for.iter() { - let listeners = solve_listeners - .entry(*dep_id) - .or_insert_with(|| Vec::with_capacity(1)); - - (*listeners).push(module_id); - } - } - - Ok(state) - } - Solved { - src, - module_id, - solved_module, - solved_subs, - mut module_timing, - module_docs, - } => { - module_timing.end_time = SystemTime::now(); - - // We've finished recording all the timings for this module, - // add them to state.timings - state.timings.insert(module_id, module_timing); - - if module_id == state.root_id { - msg_tx - .send(Msg::Finished { - solved_subs, - problems: solved_module.problems, - exposed_vars_by_symbol: solved_module.exposed_vars_by_symbol, - src, - module_docs, - }) - .map_err(|_| LoadingProblem::MsgChannelDied)?; - } else { - state.type_problems.extend(solved_module.problems); - - // This was a dependency. Write it down and keep processing messages. - debug_assert!(!state.exposed_types.contains_key(&module_id)); - state.exposed_types.insert( - module_id, - ExposedModuleTypes::Valid(solved_module.solved_types, solved_module.aliases), - ); - - // Notify all the listeners that this solved. - if let Some(listeners) = state.solve_listeners.remove(&module_id) { - for listener_id in listeners { - // This listener is longer waiting for this module, - // because this module has now been solved! - let waiting_for = state - .waiting_for_solve - .get_mut(&listener_id) - .expect("Unable to find module ID in waiting_for_solve"); - - waiting_for.remove(&module_id); - - // If it's no longer waiting for anything else, solve it. - if waiting_for.is_empty() { - let UnsolvedModule { - module, - src, - imported_modules, - constraint, - var_store, - module_timing, - } = state - .unsolved_modules - .remove(&listener_id) - .expect("Could not find listener ID in unsolved_modules"); - - enqueue_task( - injector, - worker_listeners, - BuildTask::solve_module( - module, - module_timing, - src, - constraint, - var_store, - imported_modules, - &mut state.exposed_types, - stdlib, - module_docs.clone(), - ), - )?; - } - } - } - } - - Ok(state) - } - Msg::Finished { .. } => { - unreachable!(); - } - } -} - -fn finish<'a>( - mut state: State<'a>, - solved: Solved, - problems: Vec, - exposed_vars_by_symbol: Vec<(Symbol, Variable)>, - src: &'a str, - module_docs: ModuleDocumentation, -) -> LoadedModule { - state.type_problems.extend(problems); - - let module_ids = Arc::try_unwrap(state.arc_modules) - .unwrap_or_else(|_| panic!("There were still outstanding Arc references to module_ids")) - .into_inner(); - - let interns = Interns { - module_ids, - all_ident_ids: state.constrained_ident_ids, - }; - - // let module_docs_unreferenced = ModuleDocumentation { .. module_docs }; - - LoadedModule { - module_id: state.root_id, - interns, - solved, - can_problems: state.can_problems, - type_problems: state.type_problems, - declarations_by_id: state.declarations_by_id, - exposed_vars_by_symbol, - src: src.into(), - timings: state.timings, - module_docs, - } -} - -/// Load a module by its module name, rather than by its filename -fn load_module<'a>( - arena: &'a Bump, - src_dir: &Path, - module_name: ModuleName, - module_ids: Arc>, - ident_ids_by_module: Arc>, -) -> Result<(ModuleId, Msg<'a>), LoadingProblem> { - let module_start_time = SystemTime::now(); - let mut filename = PathBuf::new(); - - filename.push(src_dir); - - // Convert dots in module name to directories - for part in module_name.as_str().split(MODULE_SEPARATOR) { - filename.push(part); - } - - // End with .roc - filename.set_extension(ROC_FILE_EXTENSION); - - load_filename( - arena, - filename, - module_ids, - ident_ids_by_module, - module_start_time, - ) -} - -/// Find a task according to the following algorithm: -/// -/// 1. Look in a local Worker queue. If it has a task, pop it off the queue and return it. -/// 2. If that queue was empty, ask the global queue for a task. -/// 3. If the global queue is also empty, iterate through each Stealer (each Worker queue has a -/// corresponding Stealer, which can steal from it. Stealers can be shared across threads.) -/// -/// Based on https://docs.rs/crossbeam/0.7.3/crossbeam/deque/index.html#examples -fn find_task(local: &Worker, global: &Injector, stealers: &[Stealer]) -> Option { - // Pop a task from the local queue, if not empty. - local.pop().or_else(|| { - // Otherwise, we need to look for a task elsewhere. - iter::repeat_with(|| { - // Try stealing a task from the global queue. - global - .steal() - // Or try stealing a task from one of the other threads. - .or_else(|| stealers.iter().map(|s| s.steal()).collect()) - }) - // Loop while no task was stolen and any steal operation needs to be retried. - .find(|s| !s.is_retry()) - // Extract the stolen task, if there is one. - .and_then(|s| s.success()) - }) -} - -fn parse_header<'a>( - arena: &'a Bump, - read_file_duration: Duration, - filename: PathBuf, - module_ids: Arc>, - ident_ids_by_module: Arc>, - src_bytes: &'a [u8], - start_time: SystemTime, -) -> Result<(ModuleId, Msg<'a>), LoadingProblem> { - let parse_start = SystemTime::now(); - let parse_state = parser::State::new(src_bytes, Attempting::Module); - let parsed = roc_parse::module::header().parse(&arena, parse_state); - let parse_header_duration = parse_start.elapsed().unwrap(); - - // Insert the first entries for this module's timings - let mut module_timing = ModuleTiming { - read_roc_file: Duration::default(), - parse_header: Duration::default(), - parse_body: Duration::default(), - canonicalize: Duration::default(), - constrain: Duration::default(), - solve: Duration::default(), - start_time, - end_time: start_time, // just for now; we'll overwrite this at the end - }; - - module_timing.read_roc_file = read_file_duration; - module_timing.parse_header = parse_header_duration; - - match parsed { - Ok((ast::Module::Interface { header }, parse_state)) => Ok(send_header( - header.name, - header.exposes.into_bump_slice(), - header.imports.into_bump_slice(), - parse_state, - module_ids, - ident_ids_by_module, - module_timing, - )), - Ok((ast::Module::App { header }, parse_state)) => Ok(send_header( - header.name, - header.provides.into_bump_slice(), - header.imports.into_bump_slice(), - parse_state, - module_ids, - ident_ids_by_module, - module_timing, - )), - Ok((ast::Module::Platform { header }, parse_state)) => { - todo!( - "TODO implement docs for platform with {:?} and {:?}", - header, - parse_state - ); - } - Err((fail, _)) => Err(LoadingProblem::ParsingFailed { filename, fail }), - } -} - -/// Load a module by its filename -fn load_filename<'a>( - arena: &'a Bump, - filename: PathBuf, - module_ids: Arc>, - ident_ids_by_module: Arc>, - module_start_time: SystemTime, -) -> Result<(ModuleId, Msg<'a>), LoadingProblem> { - let file_io_start = SystemTime::now(); - let file = fs::read(&filename); - let file_io_duration = file_io_start.elapsed().unwrap(); - - match file { - Ok(bytes) => parse_header( - arena, - file_io_duration, - filename, - module_ids, - ident_ids_by_module, - arena.alloc(bytes), - module_start_time, - ), - Err(err) => Err(LoadingProblem::FileProblem { - filename, - error: err.kind(), - }), - } -} - -#[allow(clippy::too_many_arguments)] -fn send_header<'a>( - name: Located>, - exposes: &'a [Located>], - imports: &'a [Located>], - parse_state: parser::State<'a>, - module_ids: Arc>, - ident_ids_by_module: Arc>, - module_timing: ModuleTiming, -) -> (ModuleId, Msg<'a>) { - let declared_name: ModuleName = name.value.as_str().into(); - - // TODO check to see if declared_name is consistent with filename. - // If it isn't, report a problem! - - let mut imported: Vec<(ModuleName, Vec, Region)> = Vec::with_capacity(imports.len()); - let mut imported_modules: MutSet = MutSet::default(); - let mut scope_size = 0; - - for loc_entry in imports { - let (module_name, exposed) = exposed_from_import(&loc_entry.value); - - scope_size += exposed.len(); - - imported.push((module_name, exposed, loc_entry.region)); - } - - let num_exposes = exposes.len(); - let mut deps_by_name: MutMap = - HashMap::with_capacity_and_hasher(num_exposes, default_hasher()); - let mut exposed: Vec = Vec::with_capacity(num_exposes); - - // Make sure the module_ids has ModuleIds for all our deps, - // then record those ModuleIds in can_module_ids for later. - let mut scope: MutMap = - HashMap::with_capacity_and_hasher(scope_size, default_hasher()); - let home: ModuleId; - - let ident_ids = { - // Lock just long enough to perform the minimal operations necessary. - let mut module_ids = (*module_ids).lock(); - let mut ident_ids_by_module = (*ident_ids_by_module).lock(); - - home = module_ids.get_or_insert(&declared_name.as_inline_str()); - - // Ensure this module has an entry in the exposed_ident_ids map. - ident_ids_by_module - .entry(home) - .or_insert_with(IdentIds::default); - - // For each of our imports, add an entry to deps_by_name - // - // e.g. for `imports [ Foo.{ bar } ]`, add `Foo` to deps_by_name - // - // Also build a list of imported_values_to_expose (like `bar` above.) - for (module_name, exposed_idents, region) in imported.into_iter() { - let cloned_module_name = module_name.clone(); - let module_id = module_ids.get_or_insert(&module_name.into()); - - deps_by_name.insert(cloned_module_name, module_id); - - imported_modules.insert(module_id); - - // Add the new exposed idents to the dep module's IdentIds, so - // once that module later gets loaded, its lookups will resolve - // to the same symbols as the ones we're using here. - let ident_ids = ident_ids_by_module - .entry(module_id) - .or_insert_with(IdentIds::default); - - for ident in exposed_idents { - let ident_id = ident_ids.get_or_insert(ident.as_inline_str()); - let symbol = Symbol::new(module_id, ident_id); - - // Since this value is exposed, add it to our module's default scope. - debug_assert!(!scope.contains_key(&ident.clone())); - - scope.insert(ident, (symbol, region)); - } - } - - let ident_ids = ident_ids_by_module.get_mut(&home).unwrap(); - - // Generate IdentIds entries for all values this module exposes. - // This way, when we encounter them in Defs later, they already - // have an IdentIds entry. - // - // We must *not* add them to scope yet, or else the Defs will - // incorrectly think they're shadowing them! - for loc_exposed in exposes.iter() { - // Use get_or_insert here because the ident_ids may already - // created an IdentId for this, when it was imported exposed - // in a dependent module. - // - // For example, if module A has [ B.{ foo } ], then - // when we get here for B, `foo` will already have - // an IdentId. We must reuse that! - let ident_id = ident_ids.get_or_insert(&loc_exposed.value.as_str().into()); - let symbol = Symbol::new(home, ident_id); - - exposed.push(symbol); - } - - if cfg!(debug_assertions) { - home.register_debug_idents(&ident_ids); - } - - ident_ids.clone() - }; - - // Send the deps to the coordinator thread for processing, - // then continue on to parsing and canonicalizing defs. - // - // We always need to send these, even if deps is empty, - // because the coordinator thread needs to receive this message - // to decrement its "pending" count. - - // Send the header the main thread for processing, - ( - home, - Msg::Header(ModuleHeader { - module_id: home, - exposed_ident_ids: ident_ids, - module_name: declared_name, - imported_modules, - deps_by_name, - exposes: exposed, - src: parse_state.bytes, - exposed_imports: scope, - module_timing, - }), - ) -} - -impl<'a> BuildTask<'a> { - // TODO trim down these arguments - possibly by moving Constraint into Module - #[allow(clippy::too_many_arguments)] - pub fn solve_module( - module: Module, - module_timing: ModuleTiming, - src: &'a str, - constraint: Constraint, - var_store: VarStore, - imported_modules: MutSet, - exposed_types: &mut SubsByModule, - stdlib: &StdLib, - module_docs: ModuleDocumentation, - ) -> Self { - let home = module.module_id; - - // Get the constraints for this module's imports. We do this on the main thread - // to avoid having to lock the map of exposed types, or to clone it - // (which would be more expensive for the main thread). - let ConstrainableImports { - imported_symbols, - imported_aliases, - unused_imports, - } = pre_constrain_imports( - home, - &module.references, - imported_modules, - exposed_types, - stdlib, - ); - - for unused_import in unused_imports { - todo!( - "TODO gracefully handle unused import {:?} from module {:?}", - unused_import, - home - ); - } - - // Next, solve this module in the background. - Self::Solve { - module, - imported_symbols, - imported_aliases, - constraint, - var_store, - src, - module_timing, - module_docs, - } - } - - #[allow(clippy::too_many_arguments)] - pub fn parse_and_constrain( - header: ModuleHeader<'a>, - mode: Mode, - module_ids: Arc>, - ident_ids_by_module: Arc>, - exposed_types: &SubsByModule, - exposed_symbols: MutSet, - waiting_for_solve: &mut MutMap>, - ) -> Self { - let module_id = header.module_id; - let deps_by_name = &header.deps_by_name; - let num_deps = deps_by_name.len(); - let mut dep_idents: IdentIdsByModule = IdentIds::exposed_builtins(num_deps); - - { - let ident_ids_by_module = (*ident_ids_by_module).lock(); - - // Populate dep_idents with each of their IdentIds, - // which we'll need during canonicalization to translate - // identifier strings into IdentIds, which we need to build Symbols. - // We only include the modules we care about (the ones we import). - // - // At the end of this loop, dep_idents contains all the information to - // resolve a symbol from another module: if it's in here, that means - // we have both imported the module and the ident was exported by that mdoule. - for dep_id in header.deps_by_name.values() { - // We already verified that these are all present, - // so unwrapping should always succeed here. - let idents = ident_ids_by_module.get(&dep_id).unwrap(); - - dep_idents.insert(*dep_id, idents.clone()); - } - } - - // Once this step has completed, the next thing we'll need - // is solving. Register the modules we'll need to have been - // solved before we can solve. - let mut solve_needed = HashSet::with_capacity_and_hasher(num_deps, default_hasher()); - - for dep_id in deps_by_name.values() { - if !exposed_types.contains_key(dep_id) { - solve_needed.insert(*dep_id); - } - } - - waiting_for_solve.insert(module_id, solve_needed); - - // Clone the module_ids we'll need for canonicalization. - // This should be small, and cloning it should be quick. - // We release the lock as soon as we're done cloning, so we don't have - // to lock the global module_ids while canonicalizing any given module. - let module_ids = { (*module_ids).lock().clone() }; - - // Now that we have waiting_for_solve populated, continue parsing, - // canonicalizing, and constraining the module. - Self::ParseAndConstrain { - header, - mode, - module_ids, - dep_idents, - exposed_symbols, - } - } -} - -#[allow(clippy::too_many_arguments)] -fn run_solve<'a>( - module: Module, - mut module_timing: ModuleTiming, - stdlib: &StdLib, - imported_symbols: Vec, - imported_aliases: MutMap, - constraint: Constraint, - mut var_store: VarStore, - src: &'a str, - module_docs: ModuleDocumentation, -) -> Msg<'a> { - // Rebuild the aliases in this thread, so we don't have to clone all of - // stdlib.aliases on the main thread. - let aliases = match stdlib.mode { - Mode::Standard => roc_builtins::std::aliases(), - Mode::Uniqueness => roc_builtins::unique::aliases(), - }; - - // We have more constraining work to do now, so we'll add it to our timings. - let constrain_start = SystemTime::now(); - - // Finish constraining the module by wrapping the existing Constraint - // in the ones we just computed. We can do this off the main thread. - let constraint = constrain_imports( - imported_symbols, - imported_aliases, - constraint, - &mut var_store, - ); - let mut constraint = load_builtin_aliases(aliases, constraint, &mut var_store); - - // Turn Apply into Alias - constraint.instantiate_aliases(&mut var_store); - - let constrain_end = SystemTime::now(); - - let module_id = module.module_id; - let (solved_subs, solved_module) = - roc_solve::module::solve_module(module, constraint, var_store); - - // Record the final timings - let solve_end = SystemTime::now(); - let constrain_elapsed = constrain_end.duration_since(constrain_start).unwrap(); - - module_timing.constrain += constrain_elapsed; - module_timing.solve = solve_end.duration_since(constrain_end).unwrap(); - - // Send the subs to the main thread for processing, - Msg::Solved { - src, - module_id, - solved_subs, - solved_module, - module_timing, - module_docs, - } -} - -/// Parse the module, canonicalize it, and generate constraints for it. -fn parse_and_constrain<'a>( - header: ModuleHeader<'a>, - mode: Mode, - module_ids: ModuleIds, - dep_idents: IdentIdsByModule, - exposed_symbols: MutSet, -) -> Result, LoadingProblem> { - let mut module_timing = header.module_timing; - let parse_start = SystemTime::now(); - let arena = Bump::new(); - let parse_state = parser::State::new(&header.src, Attempting::Module); - let (parsed_defs, _) = module_defs() - .parse(&arena, parse_state) - .expect("TODO gracefully handle parse error on module defs. IMPORTANT: Bail out entirely if there are any BadUtf8 problems! That means the whole source file is not valid UTF-8 and any other errors we report may get mis-reported. We rely on this for safety in an `unsafe` block later on in this function."); - - // Record the parse end time once, to avoid checking the time a second time - // immediately afterward (for the beginning of canonicalization). - let parse_end = SystemTime::now(); - - // Generate documentation information - // TODO: store timing information? - let module_docs = - generate_module_docs(header.module_name, &header.exposed_ident_ids, &parsed_defs); - - let module_id = header.module_id; - let mut var_store = VarStore::default(); - let canonicalized = canonicalize_module_defs( - &arena, - parsed_defs, - module_id, - &module_ids, - header.exposed_ident_ids, - dep_idents, - header.exposed_imports, - exposed_symbols, - &mut var_store, - ); - let canonicalize_end = SystemTime::now(); - let (module, declarations, ident_ids, constraint, problems) = match canonicalized { - Ok(mut module_output) => { - // Add builtin defs (e.g. List.get) to the module's defs - let builtin_defs = roc_can::builtins::builtin_defs(&mut var_store); - let references = &module_output.references; - - for (symbol, def) in builtin_defs { - if references.contains(&symbol) { - module_output.declarations.push(Declaration::Builtin(def)); - } - } - - let constraint = constrain_module(&module_output, module_id, mode, &mut var_store); - - // Now that we're done with parsing, canonicalization, and constraint gen, - // add the timings for those to module_timing - module_timing.constrain = canonicalize_end.elapsed().unwrap(); - module_timing.parse_body = parse_end.duration_since(parse_start).unwrap(); - module_timing.canonicalize = canonicalize_end.duration_since(parse_start).unwrap(); - - let module = Module { - module_id, - exposed_imports: module_output.exposed_imports, - exposed_vars_by_symbol: module_output.exposed_vars_by_symbol, - references: module_output.references, - aliases: module_output.aliases, - rigid_variables: module_output.rigid_variables, - }; - - ( - module, - module_output.declarations, - module_output.ident_ids, - constraint, - module_output.problems, - ) - } - Err(runtime_error) => { - panic!( - "TODO gracefully handle module canonicalization error {:?}", - runtime_error - ); - } - }; - - let imported_modules = header.imported_modules; - - // SAFETY: By this point we've already incrementally verified that there - // are no UTF-8 errors in these bytes. If there had been any UTF-8 errors, - // we'd have bailed out before now. - let src = unsafe { from_utf8_unchecked(header.src) }; - - // Send the constraint to the main thread for processing. - Ok(Msg::Constrained { - module, - src, - declarations, - imported_modules, - ident_ids, - constraint, - problems, - var_store, - module_timing, - module_docs, - }) -} - -fn exposed_from_import(entry: &ImportsEntry<'_>) -> (ModuleName, Vec) { - use roc_parse::ast::ImportsEntry::*; - - match entry { - Module(module_name, exposes) => { - let mut exposed = Vec::with_capacity(exposes.len()); - - for loc_entry in exposes { - exposed.push(ident_from_exposed(&loc_entry.value)); - } - - (module_name.as_str().into(), exposed) - } - - SpaceBefore(sub_entry, _) | SpaceAfter(sub_entry, _) => { - // Ignore spaces. - exposed_from_import(*sub_entry) - } - } -} - -fn ident_from_exposed(entry: &ExposesEntry<'_>) -> Ident { - use roc_parse::ast::ExposesEntry::*; - - match entry { - Ident(ident) => (*ident).into(), - SpaceBefore(sub_entry, _) | SpaceAfter(sub_entry, _) => ident_from_exposed(sub_entry), - } -} - -fn run_task<'a>( - task: BuildTask<'a>, - arena: &'a Bump, - src_dir: &Path, - msg_tx: MsgSender<'a>, - stdlib: &StdLib, -) -> Result<(), LoadingProblem> { - use BuildTask::*; - - let msg = match task { - LoadModule { - module_name, - module_ids, - ident_ids_by_module, - } => load_module(arena, src_dir, module_name, module_ids, ident_ids_by_module) - .map(|(_, msg)| msg), - ParseAndConstrain { - header, - mode, - module_ids, - dep_idents, - exposed_symbols, - } => parse_and_constrain(header, mode, module_ids, dep_idents, exposed_symbols), - Solve { - module, - module_timing, - imported_symbols, - imported_aliases, - constraint, - var_store, - src, - module_docs, - } => Ok(run_solve( - module, - module_timing, - stdlib, - imported_symbols, - imported_aliases, - constraint, - var_store, - src, - module_docs, - )), - }?; - - msg_tx - .send(msg) - .map_err(|_| LoadingProblem::MsgChannelDied)?; - - Ok(()) -} +use roc_region::all::Located; // Documentation generation requirements @@ -1597,7 +52,7 @@ fn generate_module_doc<'a>( exposed_ident_ids: &'a IdentIds, mut acc: Vec, before_comments_or_new_lines: Option<&'a [roc_parse::ast::CommentOrNewline<'a>]>, - def: &'a ast::Def<'a>, + def: &'a Def<'a>, ) -> ( Vec, Option<&'a [roc_parse::ast::CommentOrNewline<'a>]>,