language-servers/roc
1
0
Fork 0
mirror of https://github.com/roc-lang/roc.git synced 2025-09-29 06:44:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

cross module specialization WIP

Browse source
This commit is contained in:
Folkert 2020-10-11 01:23:52 +02:00
parent 5c558a9a87
commit 79d3b0ac01
6 changed files with 335 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

9
compiler/build/src/program.rs
View file

@ -350,13 +350,6 @@ pub fn gen_from_mono_module(
let ptr_bytes = target.pointer_width().unwrap().bytes() as u32;
let mut exposed_to_host =
HashSet::with_capacity_and_hasher(loaded.exposed_vars_by_symbol.len(), default_hasher());
for (symbol, _) in loaded.exposed_vars_by_symbol {
exposed_to_host.insert(symbol);
}
// Compile and add all the Procs before adding main
let env = roc_gen::llvm::build::Env {
arena: &arena,
@ -366,7 +359,7 @@ pub fn gen_from_mono_module(
module,
ptr_bytes,
leak: false,
exposed_to_host,
exposed_to_host: loaded.exposed_to_host,
};
// Populate Procs further and get the low-level Expr from the canonical Expr

6
compiler/constrain/src/module.rs
View file

@ -174,7 +174,11 @@ pub struct FreeVars {
pub wildcards: Vec<Variable>,
}
fn to_type(solved_type: &SolvedType, free_vars: &mut FreeVars, var_store: &mut VarStore) -> Type {
pub fn to_type(
solved_type: &SolvedType,
free_vars: &mut FreeVars,
var_store: &mut VarStore,
) -> Type {
use roc_types::solved_types::SolvedType::*;
match solved_type {

134
compiler/load/src/file.rs
View file

@ -22,8 +22,10 @@ use roc_region::all::{Located, Region};
use roc_solve::module::SolvedModule;
use roc_solve::solve;
use roc_types::solved_types::Solved;
use roc_types::solved_types::SolvedType;
use roc_types::subs::{Subs, VarStore, Variable};
use roc_types::types::Alias;
use std::collections::hash_map::Entry::{Occupied, Vacant};
use std::collections::{HashMap, HashSet};
use std::fs;
use std::io;
@ -182,6 +184,7 @@ struct ModuleCache<'a> {
constrained: MutMap<ModuleId, ConstrainedModule<'a>>,
typechecked: MutMap<ModuleId, TypeCheckedModule<'a>>,
found_specializations: MutMap<ModuleId, FoundSpecializationsModule<'a>>,
external_specializations_requested: MutMap<ModuleId, MutMap<Symbol, SolvedType>>,
}
fn start_phase<'a>(module_id: ModuleId, phase: Phase, state: &mut State<'a>) -> BuildTask<'a> {
@ -303,6 +306,7 @@ fn start_phase<'a>(module_id: ModuleId, phase: Phase, state: &mut State<'a>) ->
decls,
finished_info,
ident_ids,
pending_specializations: state.all_pending_specializations.clone(),
}
}
Phase::MakeSpecializations => {
@ -312,6 +316,12 @@ fn start_phase<'a>(module_id: ModuleId, phase: Phase, state: &mut State<'a>) ->
.remove(&module_id)
.unwrap();
let specializations_we_must_make = state
.module_cache
.external_specializations_requested
.remove(&module_id)
.unwrap_or(MutMap::default());
let FoundSpecializationsModule {
module_id,
ident_ids,
@ -327,6 +337,7 @@ fn start_phase<'a>(module_id: ModuleId, phase: Phase, state: &mut State<'a>) ->
subs,
procs,
layout_cache,
specializations_we_must_make,
finished_info,
}
}
@ -406,7 +417,7 @@ pub struct MonomorphizedModule<'a> {
pub type_problems: Vec<solve::TypeError>,
pub mono_problems: Vec<roc_mono::ir::MonoProblem>,
pub procedures: MutMap<(Symbol, Layout<'a>), Proc<'a>>,
pub exposed_vars_by_symbol: Vec<(Symbol, Variable)>,
pub exposed_to_host: MutSet<Symbol>,
pub src: Box<str>,
pub timings: MutMap<ModuleId, ModuleTiming>,
}
@ -453,6 +464,7 @@ enum Msg<'a> {
module_id: ModuleId,
ident_ids: IdentIds,
layout_cache: LayoutCache<'a>,
external_specializations_requested: MutMap<ModuleId, MutMap<Symbol, SolvedType>>,
procedures: MutMap<(Symbol, Layout<'a>), Proc<'a>>,
problems: Vec<roc_mono::ir::MonoProblem>,
subs: Subs,
@ -464,7 +476,7 @@ enum Msg<'a> {
FinishedAllSpecialization {
subs: Subs,
problems: Vec<MonoProblem>,
exposed_vars_by_symbol: Vec<(Symbol, Variable)>,
exposed_to_host: MutSet<Symbol>,
src: &'a str,
},
}
@ -491,6 +503,7 @@ struct State<'a> {
pub module_cache: ModuleCache<'a>,
pub dependencies: Dependencies,
pub procedures: MutMap<(Symbol, Layout<'a>), Proc<'a>>,
pub exposed_to_host: MutSet<Symbol>,
/// This is the "final" list of IdentIds, after canonicalization and constraint gen
/// have completed for a given module.
@ -518,7 +531,7 @@ struct State<'a> {
/// pending specializations in the same thread.
pub needs_specialization: MutSet<ModuleId>,
pub all_pending_specializations: MutMap<(Symbol, Layout<'a>), PendingSpecialization<'a>>,
pub all_pending_specializations: MutMap<Symbol, MutMap<Layout<'a>, PendingSpecialization<'a>>>,
pub specializations_in_flight: u32,
@ -632,6 +645,8 @@ enum BuildTask<'a> {
ident_ids: IdentIds,
decls: Vec<Declaration>,
finished_info: FinishedInfo<'a>,
// TODO remove?
pending_specializations: MutMap<Symbol, MutMap<Layout<'a>, PendingSpecialization<'a>>>,
},
MakeSpecializations {
module_id: ModuleId,
@ -640,6 +655,7 @@ enum BuildTask<'a> {
procs: Procs<'a>,
layout_cache: LayoutCache<'a>,
finished_info: FinishedInfo<'a>,
specializations_we_must_make: MutMap<Symbol, SolvedType>,
},
}
@ -932,6 +948,7 @@ where
module_cache: ModuleCache::default(),
dependencies: Dependencies::default(),
procedures: MutMap::default(),
exposed_to_host: MutSet::default(),
exposed_types,
headers_parsed,
loading_started,
@ -993,7 +1010,7 @@ where
Msg::FinishedAllSpecialization {
subs,
problems,
exposed_vars_by_symbol,
exposed_to_host,
src,
} => {
// We're done! There should be no more messages pending.
@ -1010,7 +1027,7 @@ where
state,
subs,
problems,
exposed_vars_by_symbol,
exposed_to_host,
src,
)));
}
@ -1140,6 +1157,12 @@ fn update<'a>(
let work = state.dependencies.notify(module_id, Phase::SolveTypes);
if module_id == state.root_id {
state
.exposed_to_host
.extend(solved_module.exposed_vars_by_symbol.iter().map(|x| x.0));
}
if module_id == state.root_id && state.goal_phase == Phase::SolveTypes {
debug_assert!(work.is_empty());
debug_assert!(state.dependencies.solved_all());
@ -1220,6 +1243,19 @@ fn update<'a>(
} => {
let subs = solved_subs.into_inner();
if let Some(pending) = &procs.pending_specializations {
for (symbol, specs) in pending {
let mut existing = match state.all_pending_specializations.entry(*symbol) {
Vacant(entry) => entry.insert(MutMap::default()),
Occupied(entry) => entry.into_mut(),
};
for (layout, pend) in specs {
existing.insert(layout.clone(), pend.clone());
}
}
}
let found_specializations_module = FoundSpecializationsModule {
layout_cache,
module_id,
@ -1251,17 +1287,32 @@ fn update<'a>(
subs,
finished_info,
procedures,
external_specializations_requested,
..
} => {
println!("done specializing {:?}", module_id);
for (module_id, requested) in external_specializations_requested {
let existing = match state
.module_cache
.external_specializations_requested
.entry(module_id)
{
Vacant(entry) => entry.insert(MutMap::default()),
Occupied(entry) => entry.into_mut(),
};
existing.extend(requested);
}
state.procedures.extend(procedures);
dbg!(&state.procedures);
let work = state
.dependencies
.notify(module_id, Phase::MakeSpecializations);
state.constrained_ident_ids.insert(module_id, ident_ids);
if work.is_empty()
&& state.dependencies.solved_all()
&& state.goal_phase == Phase::MakeSpecializations
@ -1273,14 +1324,11 @@ fn update<'a>(
subs,
// TODO thread through mono problems
problems: vec![],
exposed_vars_by_symbol: finished_info.exposed_vars_by_symbol,
exposed_to_host: state.exposed_to_host.clone(),
src: finished_info.src,
})
.map_err(|_| LoadingProblem::MsgChannelDied)?;
// bookkeeping
state.constrained_ident_ids.insert(module_id, ident_ids);
// As far as type-checking goes, once we've solved
// the originally requested module, we're all done!
return Ok(state);
@ -1307,7 +1355,7 @@ fn finish_specialization<'a>(
mut state: State<'a>,
subs: Subs,
problems: Vec<MonoProblem>,
exposed_vars_by_symbol: Vec<(Symbol, Variable)>,
exposed_to_host: MutSet<Symbol>,
src: &'a str,
) -> MonomorphizedModule<'a> {
state.mono_problems.extend(problems);
@ -1334,7 +1382,7 @@ fn finish_specialization<'a>(
can_problems,
mono_problems,
type_problems,
exposed_vars_by_symbol,
exposed_to_host,
module_id: state.root_id,
subs,
interns,
@ -1902,6 +1950,7 @@ fn make_specializations<'a>(
mut subs: Subs,
mut procs: Procs<'a>,
mut layout_cache: LayoutCache<'a>,
specializations_we_must_make: MutMap<Symbol, SolvedType>,
finished_info: FinishedInfo<'a>,
) -> Msg<'a> {
let mut mono_problems = Vec::new();
@ -1914,7 +1963,10 @@ fn make_specializations<'a>(
ident_ids: &mut ident_ids,
};
dbg!(&procs);
procs
.externals_others_need
.extend(specializations_we_must_make);
// TODO: for now this final specialization pass is sequential,
// with no parallelization at all. We should try to parallelize
// this, but doing so will require a redesign of Procs.
@ -1925,8 +1977,8 @@ fn make_specializations<'a>(
// &finished_info.vars_by_symbol,
);
let external_specializations_requested = procs.externals_we_need.clone();
let (procedures, _param_map) = procs.get_specialized_procs_help(mono_env.arena);
dbg!(&procedures);
Msg::MadeSpecializations {
module_id: home,
@ -1936,6 +1988,7 @@ fn make_specializations<'a>(
problems: mono_problems,
subs,
finished_info,
external_specializations_requested,
}
}
@ -1949,9 +2002,12 @@ fn build_pending_specializations<'a>(
// TODO use this?
_module_timing: ModuleTiming,
mut layout_cache: LayoutCache<'a>,
// TODO remove
_pending_specializations: MutMap<Symbol, MutMap<Layout<'a>, PendingSpecialization<'a>>>,
finished_info: FinishedInfo<'a>,
) -> Msg<'a> {
let mut procs = Procs::default();
let mut mono_problems = std::vec::Vec::new();
let mut subs = solved_subs.into_inner();
let mut mono_env = roc_mono::ir::Env {
@ -1971,6 +2027,12 @@ fn build_pending_specializations<'a>(
match decl {
Declare(def) | Builtin(def) => match def.loc_pattern.value {
Identifier(symbol) => {
let is_exposed = finished_info
.exposed_vars_by_symbol
.iter()
.find(|(k, _)| *k == symbol)
.is_some();
match def.loc_expr.value {
Closure {
function_type: annotation,
@ -1981,6 +2043,32 @@ fn build_pending_specializations<'a>(
} => {
// this is a non-recursive declaration
let is_tail_recursive = false;
// If this is an exposed symbol, we need to
// register it as such. Otherwise, since it
// never gets called by Roc code, it will never
// get specialized!
if is_exposed {
let mut pattern_vars = bumpalo::collections::Vec::with_capacity_in(
loc_args.len(),
arena,
);
for (var, _) in loc_args.iter() {
pattern_vars.push(*var);
}
let layout = layout_cache.from_var(mono_env.arena, annotation, mono_env.subs).unwrap_or_else(|err|
todo!("TODO gracefully handle the situation where we expose a function to the host which doesn't have a valid layout (e.g. maybe the function wasn't monomorphic): {:?}", err)
);
procs.insert_exposed(
symbol,
layout,
pattern_vars.into_bump_slice(),
annotation,
ret_var,
);
}
procs.insert_named(
&mut mono_env,
@ -1994,6 +2082,20 @@ fn build_pending_specializations<'a>(
);
}
body => {
// If this is an exposed symbol, we need to
// register it as such. Otherwise, since it
// never gets called by Roc code, it will never
// get specialized!
if is_exposed {
let annotation = def.expr_var;
let ret_var = def.expr_var;
let layout = layout_cache.from_var(mono_env.arena, annotation, mono_env.subs).unwrap_or_else(|err|
todo!("TODO gracefully handle the situation where we expose a function to the host which doesn't have a valid layout (e.g. maybe the function wasn't monomorphic): {:?}", err)
);
procs.insert_exposed(symbol, layout, &[], annotation, ret_var);
}
let proc = PartialProc {
annotation: def.expr_var,
// This is a 0-arity thunk, so it has no arguments.
@ -2087,6 +2189,7 @@ fn run_task<'a>(
layout_cache,
solved_subs,
finished_info,
pending_specializations,
} => Ok(build_pending_specializations(
arena,
solved_subs,
@ -2095,6 +2198,7 @@ fn run_task<'a>(
decls,
module_timing,
layout_cache,
pending_specializations,
finished_info,
)),
MakeSpecializations {
@ -2103,6 +2207,7 @@ fn run_task<'a>(
subs,
procs,
layout_cache,
specializations_we_must_make,
finished_info,
} => Ok(make_specializations(
arena,
@ -2111,6 +2216,7 @@ fn run_task<'a>(
subs,
procs,
layout_cache,
specializations_we_must_make,
finished_info,
)),
}?;

199
compiler/mono/src/ir.rs
View file

@ -9,6 +9,7 @@ use roc_module::low_level::LowLevel;
use roc_module::symbol::{IdentIds, ModuleId, Symbol};
use roc_problem::can::RuntimeError;
use roc_region::all::{Located, Region};
use roc_types::solved_types::SolvedType;
use roc_types::subs::{Content, FlatType, Subs, Variable};
use std::collections::HashMap;
use ven_pretty::{BoxAllocator, DocAllocator, DocBuilder};
@ -89,6 +90,8 @@ pub struct Procs<'a> {
Option<MutMap<Symbol, MutMap<Layout<'a>, PendingSpecialization<'a>>>>,
pub specialized: MutMap<(Symbol, Layout<'a>), InProgressProc<'a>>,
pub runtime_errors: MutMap<Symbol, &'a str>,
pub externals_others_need: MutMap<Symbol, SolvedType>,
pub externals_we_need: MutMap<ModuleId, MutMap<Symbol, SolvedType>>,
}
impl<'a> Default for Procs<'a> {
@ -99,6 +102,8 @@ impl<'a> Default for Procs<'a> {
pending_specializations: Some(MutMap::default()),
specialized: MutMap::default(),
runtime_errors: MutMap::default(),
externals_we_need: MutMap::default(),
externals_others_need: MutMap::default(),
}
}
}
@ -1205,6 +1210,41 @@ pub fn specialize_all<'a>(
) -> Procs<'a> {
let mut pending_specializations = procs.pending_specializations.unwrap_or_default();
// add the specializations that other modules require of us
use roc_constrain::module::{to_type, FreeVars};
use roc_solve::solve::insert_type_into_subs;
for (name, solved_type) in procs.externals_others_need.drain() {
let mut free_vars = FreeVars::default();
let mut var_store = ();
let normal_type = to_type(solved_type, &mut free_vars, &mut var_store);
let fn_var = insert_type_into_subs(env.subs, &normal_type);
let layout = layout_cache
.from_var(&env.arena, fn_var, env.subs)
.unwrap_or_else(|err| panic!("TODO handle invalid function {:?}", err));
let partial_proc = match procs.partial_procs.get(&name) {
Some(v) => v.clone(),
None => {
unreachable!("now this is an error");
}
};
match specialize_external(env, &mut procs, name, layout_cache, fn_var, partial_proc) {
Ok(proc) => {
procs.specialized.insert((name, layout), Done(proc));
}
Err(error) => {
let error_msg = env.arena.alloc(format!(
"TODO generate a RuntimeError message for {:?}",
error
));
procs.runtime_errors.insert(name, error_msg);
}
}
}
// When calling from_can, pending_specializations should be unavailable.
// This must be a single pass, and we must not add any more entries to it!
procs.pending_specializations = None;
@ -1219,11 +1259,14 @@ pub fn specialize_all<'a>(
#[allow(clippy::map_entry)]
if !procs.specialized.contains_key(&(name, layout.clone())) {
// TODO should pending_procs hold a Rc<Proc>?
let partial_proc = procs
.partial_procs
.get(&name)
.unwrap_or_else(|| panic!("Could not find partial_proc for {:?}", name))
.clone();
let partial_proc = match procs.partial_procs.get(&name) {
Some(v) => v.clone(),
None => {
// TODO this assumes the specialization is done by another module
// make sure this does not become a problem down the road!
continue;
}
};
// Mark this proc as in-progress, so if we're dealing with
// mutually recursive functions, we don't loop forever.
@ -1250,6 +1293,110 @@ pub fn specialize_all<'a>(
procs
}
fn specialize_external<'a>(
env: &mut Env<'a, '_>,
procs: &mut Procs<'a>,
proc_name: Symbol,
layout_cache: &mut LayoutCache<'a>,
fn_var: Variable,
partial_proc: PartialProc<'a>,
) -> Result<Proc<'a>, LayoutProblem> {
let PartialProc {
annotation,
pattern_symbols,
body,
is_self_recursive,
} = partial_proc;
// unify the called function with the specialized signature, then specialize the function body
let snapshot = env.subs.snapshot();
let unified = roc_unify::unify::unify(env.subs, annotation, fn_var);
debug_assert!(matches!(unified, roc_unify::unify::Unified::Success(_)));
let specialized_body = from_can(env, body, procs, layout_cache);
let (proc_args, ret_layout) =
build_specialized_proc_from_var(env, layout_cache, pattern_symbols, fn_var)?;
// reset subs, so we don't get type errors when specializing for a different signature
env.subs.rollback_to(snapshot);
// TODO WRONG
let closes_over_layout = Layout::Struct(&[]);
let recursivity = if is_self_recursive {
SelfRecursive::SelfRecursive(JoinPointId(env.unique_symbol()))
} else {
SelfRecursive::NotSelfRecursive
};
let proc = Proc {
name: proc_name,
args: proc_args,
body: specialized_body,
closes_over: closes_over_layout,
ret_layout,
is_self_recursive: recursivity,
};
Ok(proc)
}
fn build_specialized_proc_from_var<'a>(
env: &mut Env<'a, '_>,
layout_cache: &mut LayoutCache<'a>,
pattern_symbols: &[Symbol],
fn_var: Variable,
) -> Result<(&'a [(Layout<'a>, Symbol)], Layout<'a>), LayoutProblem> {
match env.subs.get_without_compacting(fn_var).content {
Content::Structure(FlatType::Func(pattern_vars, _closure_var, ret_var)) => {
build_specialized_proc(env, layout_cache, pattern_symbols, &pattern_vars, ret_var)
}
Content::Structure(FlatType::Apply(Symbol::ATTR_ATTR, args)) => {
build_specialized_proc_from_var(env, layout_cache, pattern_symbols, args[1])
}
Content::Alias(_, _, actual) => {
build_specialized_proc_from_var(env, layout_cache, pattern_symbols, actual)
}
_ => {
// a top-level constant 0-argument thunk
build_specialized_proc(env, layout_cache, pattern_symbols, &[], fn_var)
}
}
}
fn build_specialized_proc<'a>(
env: &mut Env<'a, '_>,
layout_cache: &mut LayoutCache<'a>,
pattern_symbols: &[Symbol],
pattern_vars: &[Variable],
ret_var: Variable,
) -> Result<(&'a [(Layout<'a>, Symbol)], Layout<'a>), LayoutProblem> {
let mut proc_args = Vec::with_capacity_in(pattern_vars.len(), &env.arena);
debug_assert_eq!(
&pattern_vars.len(),
&pattern_symbols.len(),
"Tried to zip two vecs with different lengths!"
);
for (arg_var, arg_name) in pattern_vars.iter().zip(pattern_symbols.iter()) {
let layout = layout_cache.from_var(&env.arena, *arg_var, env.subs)?;
proc_args.push((layout, *arg_name));
}
let proc_args = proc_args.into_bump_slice();
let ret_layout = layout_cache
.from_var(&env.arena, ret_var, env.subs)
.unwrap_or_else(|err| panic!("TODO handle invalid function {:?}", err));
Ok((proc_args, ret_layout))
}
fn specialize<'a>(
env: &mut Env<'a, '_>,
procs: &mut Procs<'a>,
@ -2202,10 +2349,13 @@ pub fn with_hole<'a>(
// So we check the function name against the list of partial procedures,
// the procedures that we have lifted to the top-level and can call by name
// if it's in there, it's a call by name, otherwise it's a call by pointer
let known_functions = &procs.partial_procs;
let is_known = |key| {
// a proc in this module, or an imported symbol
procs.partial_procs.contains_key(key) || key.module_id() != assigned.module_id()
};
match loc_expr.value {
roc_can::expr::Expr::Var(proc_name) if known_functions.contains_key(&proc_name) => {
call_by_name(
roc_can::expr::Expr::Var(proc_name) if is_known(&proc_name) => call_by_name(
env,
procs,
fn_var,
@ -2215,8 +2365,7 @@ pub fn with_hole<'a>(
layout_cache,
assigned,
hole,
)
}
),
_ => {
// Call by pointer - the closure was anonymous, e.g.
//
@ -3546,6 +3695,36 @@ fn call_by_name<'a>(
}
}
None if assigned.module_id() != proc_name.module_id() => {
// call of a function that is not not in this module
use std::collections::hash_map::Entry::{Occupied, Vacant};
let existing =
match procs.externals_we_need.entry(proc_name.module_id()) {
Vacant(entry) => entry.insert(MutMap::default()),
Occupied(entry) => entry.into_mut(),
};
existing.insert(
proc_name,
SolvedType::from_var(env.subs, pending.fn_var),
);
let call = Expr::FunctionCall {
call_type: CallType::ByName(proc_name),
ret_layout: ret_layout.clone(),
full_layout: full_layout.clone(),
arg_layouts,
args: field_symbols,
};
let iter =
loc_args.into_iter().rev().zip(field_symbols.iter().rev());
let result = Stmt::Let(assigned, call, ret_layout.clone(), hole);
assign_to_symbols(env, procs, layout_cache, iter, result)
}
None => {
// This must have been a runtime error.
match procs.runtime_errors.get(&proc_name) {

10
compiler/solve/src/solve.rs
View file

@ -567,6 +567,16 @@ fn type_to_var(
type_to_variable(subs, rank, pools, cached, typ)
}
/// Abusing existing functions for our purposes
/// this is to put a solved type back into subs
pub fn insert_type_into_subs(subs: &mut Subs, typ: &Type) -> Variable {
let rank = Rank::NONE;
let mut pools = Pools::default();
let mut cached = MutMap::default();
type_to_variable(subs, rank, &mut pools, &mut cached, typ)
}
fn type_to_variable(
subs: &mut Subs,
rank: Rank,

2
compiler/types/src/solved_types.rs
View file

@ -197,7 +197,7 @@ impl SolvedType {
}
}
fn from_var(subs: &Subs, var: Variable) -> Self {
pub fn from_var(subs: &Subs, var: Variable) -> Self {
use crate::subs::Content::*;
match subs.get_without_compacting(var).content {