diff --git a/compiler/build/src/program.rs b/compiler/build/src/program.rs index e5d08f831c..bffd546449 100644 --- a/compiler/build/src/program.rs +++ b/compiler/build/src/program.rs @@ -350,13 +350,6 @@ pub fn gen_from_mono_module( let ptr_bytes = target.pointer_width().unwrap().bytes() as u32; - let mut exposed_to_host = - HashSet::with_capacity_and_hasher(loaded.exposed_vars_by_symbol.len(), default_hasher()); - - for (symbol, _) in loaded.exposed_vars_by_symbol { - exposed_to_host.insert(symbol); - } - // Compile and add all the Procs before adding main let env = roc_gen::llvm::build::Env { arena: &arena, @@ -366,7 +359,7 @@ pub fn gen_from_mono_module( module, ptr_bytes, leak: false, - exposed_to_host, + exposed_to_host: loaded.exposed_to_host, }; // Populate Procs further and get the low-level Expr from the canonical Expr diff --git a/compiler/constrain/src/module.rs b/compiler/constrain/src/module.rs index c51100c9fe..509f3bd7b6 100644 --- a/compiler/constrain/src/module.rs +++ b/compiler/constrain/src/module.rs @@ -174,7 +174,11 @@ pub struct FreeVars { pub wildcards: Vec, } -fn to_type(solved_type: &SolvedType, free_vars: &mut FreeVars, var_store: &mut VarStore) -> Type { +pub fn to_type( + solved_type: &SolvedType, + free_vars: &mut FreeVars, + var_store: &mut VarStore, +) -> Type { use roc_types::solved_types::SolvedType::*; match solved_type { diff --git a/compiler/load/src/file.rs b/compiler/load/src/file.rs index 07307acdf2..5d4ccd0f27 100644 --- a/compiler/load/src/file.rs +++ b/compiler/load/src/file.rs @@ -22,8 +22,10 @@ use roc_region::all::{Located, Region}; use roc_solve::module::SolvedModule; use roc_solve::solve; use roc_types::solved_types::Solved; +use roc_types::solved_types::SolvedType; use roc_types::subs::{Subs, VarStore, Variable}; use roc_types::types::Alias; +use std::collections::hash_map::Entry::{Occupied, Vacant}; use std::collections::{HashMap, HashSet}; use std::fs; use std::io; @@ -182,6 +184,7 @@ struct ModuleCache<'a> { constrained: MutMap>, typechecked: MutMap>, found_specializations: MutMap>, + external_specializations_requested: MutMap>, } fn start_phase<'a>(module_id: ModuleId, phase: Phase, state: &mut State<'a>) -> BuildTask<'a> { @@ -303,6 +306,7 @@ fn start_phase<'a>(module_id: ModuleId, phase: Phase, state: &mut State<'a>) -> decls, finished_info, ident_ids, + pending_specializations: state.all_pending_specializations.clone(), } } Phase::MakeSpecializations => { @@ -312,6 +316,12 @@ fn start_phase<'a>(module_id: ModuleId, phase: Phase, state: &mut State<'a>) -> .remove(&module_id) .unwrap(); + let specializations_we_must_make = state + .module_cache + .external_specializations_requested + .remove(&module_id) + .unwrap_or(MutMap::default()); + let FoundSpecializationsModule { module_id, ident_ids, @@ -327,6 +337,7 @@ fn start_phase<'a>(module_id: ModuleId, phase: Phase, state: &mut State<'a>) -> subs, procs, layout_cache, + specializations_we_must_make, finished_info, } } @@ -406,7 +417,7 @@ pub struct MonomorphizedModule<'a> { pub type_problems: Vec, pub mono_problems: Vec, pub procedures: MutMap<(Symbol, Layout<'a>), Proc<'a>>, - pub exposed_vars_by_symbol: Vec<(Symbol, Variable)>, + pub exposed_to_host: MutSet, pub src: Box, pub timings: MutMap, } @@ -453,6 +464,7 @@ enum Msg<'a> { module_id: ModuleId, ident_ids: IdentIds, layout_cache: LayoutCache<'a>, + external_specializations_requested: MutMap>, procedures: MutMap<(Symbol, Layout<'a>), Proc<'a>>, problems: Vec, subs: Subs, @@ -464,7 +476,7 @@ enum Msg<'a> { FinishedAllSpecialization { subs: Subs, problems: Vec, - exposed_vars_by_symbol: Vec<(Symbol, Variable)>, + exposed_to_host: MutSet, src: &'a str, }, } @@ -491,6 +503,7 @@ struct State<'a> { pub module_cache: ModuleCache<'a>, pub dependencies: Dependencies, pub procedures: MutMap<(Symbol, Layout<'a>), Proc<'a>>, + pub exposed_to_host: MutSet, /// This is the "final" list of IdentIds, after canonicalization and constraint gen /// have completed for a given module. @@ -518,7 +531,7 @@ struct State<'a> { /// pending specializations in the same thread. pub needs_specialization: MutSet, - pub all_pending_specializations: MutMap<(Symbol, Layout<'a>), PendingSpecialization<'a>>, + pub all_pending_specializations: MutMap, PendingSpecialization<'a>>>, pub specializations_in_flight: u32, @@ -632,6 +645,8 @@ enum BuildTask<'a> { ident_ids: IdentIds, decls: Vec, finished_info: FinishedInfo<'a>, + // TODO remove? + pending_specializations: MutMap, PendingSpecialization<'a>>>, }, MakeSpecializations { module_id: ModuleId, @@ -640,6 +655,7 @@ enum BuildTask<'a> { procs: Procs<'a>, layout_cache: LayoutCache<'a>, finished_info: FinishedInfo<'a>, + specializations_we_must_make: MutMap, }, } @@ -932,6 +948,7 @@ where module_cache: ModuleCache::default(), dependencies: Dependencies::default(), procedures: MutMap::default(), + exposed_to_host: MutSet::default(), exposed_types, headers_parsed, loading_started, @@ -993,7 +1010,7 @@ where Msg::FinishedAllSpecialization { subs, problems, - exposed_vars_by_symbol, + exposed_to_host, src, } => { // We're done! There should be no more messages pending. @@ -1010,7 +1027,7 @@ where state, subs, problems, - exposed_vars_by_symbol, + exposed_to_host, src, ))); } @@ -1140,6 +1157,12 @@ fn update<'a>( let work = state.dependencies.notify(module_id, Phase::SolveTypes); + if module_id == state.root_id { + state + .exposed_to_host + .extend(solved_module.exposed_vars_by_symbol.iter().map(|x| x.0)); + } + if module_id == state.root_id && state.goal_phase == Phase::SolveTypes { debug_assert!(work.is_empty()); debug_assert!(state.dependencies.solved_all()); @@ -1220,6 +1243,19 @@ fn update<'a>( } => { let subs = solved_subs.into_inner(); + if let Some(pending) = &procs.pending_specializations { + for (symbol, specs) in pending { + let mut existing = match state.all_pending_specializations.entry(*symbol) { + Vacant(entry) => entry.insert(MutMap::default()), + Occupied(entry) => entry.into_mut(), + }; + + for (layout, pend) in specs { + existing.insert(layout.clone(), pend.clone()); + } + } + } + let found_specializations_module = FoundSpecializationsModule { layout_cache, module_id, @@ -1251,17 +1287,32 @@ fn update<'a>( subs, finished_info, procedures, + external_specializations_requested, .. } => { println!("done specializing {:?}", module_id); + for (module_id, requested) in external_specializations_requested { + let existing = match state + .module_cache + .external_specializations_requested + .entry(module_id) + { + Vacant(entry) => entry.insert(MutMap::default()), + Occupied(entry) => entry.into_mut(), + }; + + existing.extend(requested); + } + state.procedures.extend(procedures); - dbg!(&state.procedures); let work = state .dependencies .notify(module_id, Phase::MakeSpecializations); + state.constrained_ident_ids.insert(module_id, ident_ids); + if work.is_empty() && state.dependencies.solved_all() && state.goal_phase == Phase::MakeSpecializations @@ -1273,14 +1324,11 @@ fn update<'a>( subs, // TODO thread through mono problems problems: vec![], - exposed_vars_by_symbol: finished_info.exposed_vars_by_symbol, + exposed_to_host: state.exposed_to_host.clone(), src: finished_info.src, }) .map_err(|_| LoadingProblem::MsgChannelDied)?; - // bookkeeping - state.constrained_ident_ids.insert(module_id, ident_ids); - // As far as type-checking goes, once we've solved // the originally requested module, we're all done! return Ok(state); @@ -1307,7 +1355,7 @@ fn finish_specialization<'a>( mut state: State<'a>, subs: Subs, problems: Vec, - exposed_vars_by_symbol: Vec<(Symbol, Variable)>, + exposed_to_host: MutSet, src: &'a str, ) -> MonomorphizedModule<'a> { state.mono_problems.extend(problems); @@ -1334,7 +1382,7 @@ fn finish_specialization<'a>( can_problems, mono_problems, type_problems, - exposed_vars_by_symbol, + exposed_to_host, module_id: state.root_id, subs, interns, @@ -1902,6 +1950,7 @@ fn make_specializations<'a>( mut subs: Subs, mut procs: Procs<'a>, mut layout_cache: LayoutCache<'a>, + specializations_we_must_make: MutMap, finished_info: FinishedInfo<'a>, ) -> Msg<'a> { let mut mono_problems = Vec::new(); @@ -1914,7 +1963,10 @@ fn make_specializations<'a>( ident_ids: &mut ident_ids, }; - dbg!(&procs); + procs + .externals_others_need + .extend(specializations_we_must_make); + // TODO: for now this final specialization pass is sequential, // with no parallelization at all. We should try to parallelize // this, but doing so will require a redesign of Procs. @@ -1925,8 +1977,8 @@ fn make_specializations<'a>( // &finished_info.vars_by_symbol, ); + let external_specializations_requested = procs.externals_we_need.clone(); let (procedures, _param_map) = procs.get_specialized_procs_help(mono_env.arena); - dbg!(&procedures); Msg::MadeSpecializations { module_id: home, @@ -1936,6 +1988,7 @@ fn make_specializations<'a>( problems: mono_problems, subs, finished_info, + external_specializations_requested, } } @@ -1949,9 +2002,12 @@ fn build_pending_specializations<'a>( // TODO use this? _module_timing: ModuleTiming, mut layout_cache: LayoutCache<'a>, + // TODO remove + _pending_specializations: MutMap, PendingSpecialization<'a>>>, finished_info: FinishedInfo<'a>, ) -> Msg<'a> { let mut procs = Procs::default(); + let mut mono_problems = std::vec::Vec::new(); let mut subs = solved_subs.into_inner(); let mut mono_env = roc_mono::ir::Env { @@ -1971,6 +2027,12 @@ fn build_pending_specializations<'a>( match decl { Declare(def) | Builtin(def) => match def.loc_pattern.value { Identifier(symbol) => { + let is_exposed = finished_info + .exposed_vars_by_symbol + .iter() + .find(|(k, _)| *k == symbol) + .is_some(); + match def.loc_expr.value { Closure { function_type: annotation, @@ -1981,6 +2043,32 @@ fn build_pending_specializations<'a>( } => { // this is a non-recursive declaration let is_tail_recursive = false; + // If this is an exposed symbol, we need to + // register it as such. Otherwise, since it + // never gets called by Roc code, it will never + // get specialized! + if is_exposed { + let mut pattern_vars = bumpalo::collections::Vec::with_capacity_in( + loc_args.len(), + arena, + ); + + for (var, _) in loc_args.iter() { + pattern_vars.push(*var); + } + + let layout = layout_cache.from_var(mono_env.arena, annotation, mono_env.subs).unwrap_or_else(|err| + todo!("TODO gracefully handle the situation where we expose a function to the host which doesn't have a valid layout (e.g. maybe the function wasn't monomorphic): {:?}", err) + ); + + procs.insert_exposed( + symbol, + layout, + pattern_vars.into_bump_slice(), + annotation, + ret_var, + ); + } procs.insert_named( &mut mono_env, @@ -1994,6 +2082,20 @@ fn build_pending_specializations<'a>( ); } body => { + // If this is an exposed symbol, we need to + // register it as such. Otherwise, since it + // never gets called by Roc code, it will never + // get specialized! + if is_exposed { + let annotation = def.expr_var; + let ret_var = def.expr_var; + let layout = layout_cache.from_var(mono_env.arena, annotation, mono_env.subs).unwrap_or_else(|err| + todo!("TODO gracefully handle the situation where we expose a function to the host which doesn't have a valid layout (e.g. maybe the function wasn't monomorphic): {:?}", err) + ); + + procs.insert_exposed(symbol, layout, &[], annotation, ret_var); + } + let proc = PartialProc { annotation: def.expr_var, // This is a 0-arity thunk, so it has no arguments. @@ -2087,6 +2189,7 @@ fn run_task<'a>( layout_cache, solved_subs, finished_info, + pending_specializations, } => Ok(build_pending_specializations( arena, solved_subs, @@ -2095,6 +2198,7 @@ fn run_task<'a>( decls, module_timing, layout_cache, + pending_specializations, finished_info, )), MakeSpecializations { @@ -2103,6 +2207,7 @@ fn run_task<'a>( subs, procs, layout_cache, + specializations_we_must_make, finished_info, } => Ok(make_specializations( arena, @@ -2111,6 +2216,7 @@ fn run_task<'a>( subs, procs, layout_cache, + specializations_we_must_make, finished_info, )), }?; diff --git a/compiler/mono/src/ir.rs b/compiler/mono/src/ir.rs index cdd183940d..3ec87e91ab 100644 --- a/compiler/mono/src/ir.rs +++ b/compiler/mono/src/ir.rs @@ -9,6 +9,7 @@ use roc_module::low_level::LowLevel; use roc_module::symbol::{IdentIds, ModuleId, Symbol}; use roc_problem::can::RuntimeError; use roc_region::all::{Located, Region}; +use roc_types::solved_types::SolvedType; use roc_types::subs::{Content, FlatType, Subs, Variable}; use std::collections::HashMap; use ven_pretty::{BoxAllocator, DocAllocator, DocBuilder}; @@ -89,6 +90,8 @@ pub struct Procs<'a> { Option, PendingSpecialization<'a>>>>, pub specialized: MutMap<(Symbol, Layout<'a>), InProgressProc<'a>>, pub runtime_errors: MutMap, + pub externals_others_need: MutMap, + pub externals_we_need: MutMap>, } impl<'a> Default for Procs<'a> { @@ -99,6 +102,8 @@ impl<'a> Default for Procs<'a> { pending_specializations: Some(MutMap::default()), specialized: MutMap::default(), runtime_errors: MutMap::default(), + externals_we_need: MutMap::default(), + externals_others_need: MutMap::default(), } } } @@ -1205,6 +1210,41 @@ pub fn specialize_all<'a>( ) -> Procs<'a> { let mut pending_specializations = procs.pending_specializations.unwrap_or_default(); + // add the specializations that other modules require of us + use roc_constrain::module::{to_type, FreeVars}; + use roc_solve::solve::insert_type_into_subs; + for (name, solved_type) in procs.externals_others_need.drain() { + let mut free_vars = FreeVars::default(); + let mut var_store = (); + let normal_type = to_type(solved_type, &mut free_vars, &mut var_store); + let fn_var = insert_type_into_subs(env.subs, &normal_type); + + let layout = layout_cache + .from_var(&env.arena, fn_var, env.subs) + .unwrap_or_else(|err| panic!("TODO handle invalid function {:?}", err)); + + let partial_proc = match procs.partial_procs.get(&name) { + Some(v) => v.clone(), + None => { + unreachable!("now this is an error"); + } + }; + + match specialize_external(env, &mut procs, name, layout_cache, fn_var, partial_proc) { + Ok(proc) => { + procs.specialized.insert((name, layout), Done(proc)); + } + Err(error) => { + let error_msg = env.arena.alloc(format!( + "TODO generate a RuntimeError message for {:?}", + error + )); + + procs.runtime_errors.insert(name, error_msg); + } + } + } + // When calling from_can, pending_specializations should be unavailable. // This must be a single pass, and we must not add any more entries to it! procs.pending_specializations = None; @@ -1219,11 +1259,14 @@ pub fn specialize_all<'a>( #[allow(clippy::map_entry)] if !procs.specialized.contains_key(&(name, layout.clone())) { // TODO should pending_procs hold a Rc? - let partial_proc = procs - .partial_procs - .get(&name) - .unwrap_or_else(|| panic!("Could not find partial_proc for {:?}", name)) - .clone(); + let partial_proc = match procs.partial_procs.get(&name) { + Some(v) => v.clone(), + None => { + // TODO this assumes the specialization is done by another module + // make sure this does not become a problem down the road! + continue; + } + }; // Mark this proc as in-progress, so if we're dealing with // mutually recursive functions, we don't loop forever. @@ -1250,6 +1293,110 @@ pub fn specialize_all<'a>( procs } +fn specialize_external<'a>( + env: &mut Env<'a, '_>, + procs: &mut Procs<'a>, + proc_name: Symbol, + layout_cache: &mut LayoutCache<'a>, + fn_var: Variable, + partial_proc: PartialProc<'a>, +) -> Result, LayoutProblem> { + let PartialProc { + annotation, + pattern_symbols, + body, + is_self_recursive, + } = partial_proc; + + // unify the called function with the specialized signature, then specialize the function body + let snapshot = env.subs.snapshot(); + let unified = roc_unify::unify::unify(env.subs, annotation, fn_var); + + debug_assert!(matches!(unified, roc_unify::unify::Unified::Success(_))); + + let specialized_body = from_can(env, body, procs, layout_cache); + + let (proc_args, ret_layout) = + build_specialized_proc_from_var(env, layout_cache, pattern_symbols, fn_var)?; + + // reset subs, so we don't get type errors when specializing for a different signature + env.subs.rollback_to(snapshot); + + // TODO WRONG + let closes_over_layout = Layout::Struct(&[]); + + let recursivity = if is_self_recursive { + SelfRecursive::SelfRecursive(JoinPointId(env.unique_symbol())) + } else { + SelfRecursive::NotSelfRecursive + }; + + let proc = Proc { + name: proc_name, + args: proc_args, + body: specialized_body, + closes_over: closes_over_layout, + ret_layout, + is_self_recursive: recursivity, + }; + + Ok(proc) +} + +fn build_specialized_proc_from_var<'a>( + env: &mut Env<'a, '_>, + layout_cache: &mut LayoutCache<'a>, + pattern_symbols: &[Symbol], + fn_var: Variable, +) -> Result<(&'a [(Layout<'a>, Symbol)], Layout<'a>), LayoutProblem> { + match env.subs.get_without_compacting(fn_var).content { + Content::Structure(FlatType::Func(pattern_vars, _closure_var, ret_var)) => { + build_specialized_proc(env, layout_cache, pattern_symbols, &pattern_vars, ret_var) + } + Content::Structure(FlatType::Apply(Symbol::ATTR_ATTR, args)) => { + build_specialized_proc_from_var(env, layout_cache, pattern_symbols, args[1]) + } + Content::Alias(_, _, actual) => { + build_specialized_proc_from_var(env, layout_cache, pattern_symbols, actual) + } + _ => { + // a top-level constant 0-argument thunk + + build_specialized_proc(env, layout_cache, pattern_symbols, &[], fn_var) + } + } +} + +fn build_specialized_proc<'a>( + env: &mut Env<'a, '_>, + layout_cache: &mut LayoutCache<'a>, + pattern_symbols: &[Symbol], + pattern_vars: &[Variable], + ret_var: Variable, +) -> Result<(&'a [(Layout<'a>, Symbol)], Layout<'a>), LayoutProblem> { + let mut proc_args = Vec::with_capacity_in(pattern_vars.len(), &env.arena); + + debug_assert_eq!( + &pattern_vars.len(), + &pattern_symbols.len(), + "Tried to zip two vecs with different lengths!" + ); + + for (arg_var, arg_name) in pattern_vars.iter().zip(pattern_symbols.iter()) { + let layout = layout_cache.from_var(&env.arena, *arg_var, env.subs)?; + + proc_args.push((layout, *arg_name)); + } + + let proc_args = proc_args.into_bump_slice(); + + let ret_layout = layout_cache + .from_var(&env.arena, ret_var, env.subs) + .unwrap_or_else(|err| panic!("TODO handle invalid function {:?}", err)); + + Ok((proc_args, ret_layout)) +} + fn specialize<'a>( env: &mut Env<'a, '_>, procs: &mut Procs<'a>, @@ -2202,21 +2349,23 @@ pub fn with_hole<'a>( // So we check the function name against the list of partial procedures, // the procedures that we have lifted to the top-level and can call by name // if it's in there, it's a call by name, otherwise it's a call by pointer - let known_functions = &procs.partial_procs; + let is_known = |key| { + // a proc in this module, or an imported symbol + procs.partial_procs.contains_key(key) || key.module_id() != assigned.module_id() + }; + match loc_expr.value { - roc_can::expr::Expr::Var(proc_name) if known_functions.contains_key(&proc_name) => { - call_by_name( - env, - procs, - fn_var, - ret_var, - proc_name, - loc_args, - layout_cache, - assigned, - hole, - ) - } + roc_can::expr::Expr::Var(proc_name) if is_known(&proc_name) => call_by_name( + env, + procs, + fn_var, + ret_var, + proc_name, + loc_args, + layout_cache, + assigned, + hole, + ), _ => { // Call by pointer - the closure was anonymous, e.g. // @@ -3546,6 +3695,36 @@ fn call_by_name<'a>( } } + None if assigned.module_id() != proc_name.module_id() => { + // call of a function that is not not in this module + use std::collections::hash_map::Entry::{Occupied, Vacant}; + + let existing = + match procs.externals_we_need.entry(proc_name.module_id()) { + Vacant(entry) => entry.insert(MutMap::default()), + Occupied(entry) => entry.into_mut(), + }; + + existing.insert( + proc_name, + SolvedType::from_var(env.subs, pending.fn_var), + ); + + let call = Expr::FunctionCall { + call_type: CallType::ByName(proc_name), + ret_layout: ret_layout.clone(), + full_layout: full_layout.clone(), + arg_layouts, + args: field_symbols, + }; + + let iter = + loc_args.into_iter().rev().zip(field_symbols.iter().rev()); + + let result = Stmt::Let(assigned, call, ret_layout.clone(), hole); + assign_to_symbols(env, procs, layout_cache, iter, result) + } + None => { // This must have been a runtime error. match procs.runtime_errors.get(&proc_name) { diff --git a/compiler/solve/src/solve.rs b/compiler/solve/src/solve.rs index d7d06350d9..084aa7fc1b 100644 --- a/compiler/solve/src/solve.rs +++ b/compiler/solve/src/solve.rs @@ -567,6 +567,16 @@ fn type_to_var( type_to_variable(subs, rank, pools, cached, typ) } +/// Abusing existing functions for our purposes +/// this is to put a solved type back into subs +pub fn insert_type_into_subs(subs: &mut Subs, typ: &Type) -> Variable { + let rank = Rank::NONE; + let mut pools = Pools::default(); + let mut cached = MutMap::default(); + + type_to_variable(subs, rank, &mut pools, &mut cached, typ) +} + fn type_to_variable( subs: &mut Subs, rank: Rank, diff --git a/compiler/types/src/solved_types.rs b/compiler/types/src/solved_types.rs index 9b8ed9f61d..e5c61434b1 100644 --- a/compiler/types/src/solved_types.rs +++ b/compiler/types/src/solved_types.rs @@ -197,7 +197,7 @@ impl SolvedType { } } - fn from_var(subs: &Subs, var: Variable) -> Self { + pub fn from_var(subs: &Subs, var: Variable) -> Self { use crate::subs::Content::*; match subs.get_without_compacting(var).content {