Merge branch 'trunk' into optional-fields-in-exprs

BUILDING_FROM_SOURCE.md

@@ -1,13 +1,13 @@
 # Building the Roc compiler from source
 
 
-## Installing LLVM, Python 2.7, Zig, valgrind, libunwind, and libc++-dev
+## Installing LLVM, Python, Zig, valgrind, libunwind, and libc++-dev
 
 To build the compiler, you need these installed:
 
 * `libunwind` (macOS should already have this one installed)
 * `libc++-dev`
-* Python 2.7
+* Python 2.7 (Windows only), `python-is-python3` (Ubuntu)
 * a particular version of Zig (see below)
 * a particular version of LLVM (see below)
 
@@ -34,7 +34,7 @@ We use a specific version of Zig, a build off the the commit `0088efc4b`. The la
   tar xvf zig-linux-x86_64-0.6.0+0088efc4b.tar
   # move the files into /opt:
   sudo mkdir -p /opt/zig
-  sudo mv tar xvf zig-linux-x86_64-0.6.0+0088efc4b.tar/* /opt/zig/
+  sudo mv zig-linux-x86_64-0.6.0+0088efc4b/* /opt/zig/
   ```
   Then add `/opt/zig/` to your `PATH` (e.g. in `~/.bashrc`).
   

Cargo.lock

@@ -661,9 +661,9 @@ checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650"
 
 [[package]]
 name = "either"
-version = "1.6.0"
+version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f"
+checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
 
 [[package]]
 name = "encode_unicode"
@@ -2463,6 +2463,7 @@ name = "roc_gen"
 version = "0.1.0"
 dependencies = [
  "bumpalo",
+ "either",
  "im",
  "im-rc",
  "indoc",

compiler/build/src/link.rs

@@ -6,7 +6,7 @@ use libloading::{Error, Library};
 use roc_gen::llvm::build::OptLevel;
 use std::io;
 use std::path::{Path, PathBuf};
-use std::process::{Child, Command};
+use std::process::{Child, Command, Output};
 use target_lexicon::{Architecture, OperatingSystem, Triple};
 use tempfile::tempdir;
 
@@ -47,7 +47,7 @@ pub fn rebuild_host(host_input_path: &Path) {
     let host_dest = host_input_path.with_file_name("host.o");
 
     // Compile host.c
-    Command::new("clang")
+    let output = Command::new("clang")
         .env_clear()
         .args(&[
             "-c",
@@ -58,18 +58,22 @@ pub fn rebuild_host(host_input_path: &Path) {
         .output()
         .unwrap();
 
+    validate_output("host.c", "clang", output);
+
     if cargo_host_src.exists() {
         // Compile and link Cargo.toml, if it exists
         let cargo_dir = host_input_path.parent().unwrap();
         let libhost_dir = cargo_dir.join("target").join("release");
 
-        Command::new("cargo")
+        let output = Command::new("cargo")
             .args(&["build", "--release"])
             .current_dir(cargo_dir)
             .output()
             .unwrap();
 
-        Command::new("ld")
+        validate_output("host.rs", "cargo build --release", output);
+
+        let output = Command::new("ld")
             .env_clear()
             .args(&[
                 "-r",
@@ -82,9 +86,11 @@ pub fn rebuild_host(host_input_path: &Path) {
             ])
             .output()
             .unwrap();
+
+        validate_output("c_host.o", "ld", output);
     } else if rust_host_src.exists() {
         // Compile and link host.rs, if it exists
-        Command::new("rustc")
+        let output = Command::new("rustc")
             .args(&[
                 rust_host_src.to_str().unwrap(),
                 "-o",
@@ -93,7 +99,9 @@ pub fn rebuild_host(host_input_path: &Path) {
             .output()
             .unwrap();
 
-        Command::new("ld")
+        validate_output("host.rs", "rustc", output);
+
+        let output = Command::new("ld")
             .env_clear()
             .args(&[
                 "-r",
@@ -105,8 +113,10 @@ pub fn rebuild_host(host_input_path: &Path) {
             .output()
             .unwrap();
 
+        validate_output("rust_host.o", "ld", output);
+
         // Clean up rust_host.o
-        Command::new("rm")
+        let output = Command::new("rm")
             .env_clear()
             .args(&[
                 "-f",
@@ -115,13 +125,17 @@ pub fn rebuild_host(host_input_path: &Path) {
             ])
             .output()
             .unwrap();
+
+        validate_output("rust_host.o", "rm", output);
     } else {
         // Clean up rust_host.o
-        Command::new("mv")
+        let output = Command::new("mv")
             .env_clear()
             .args(&[c_host_dest, host_dest])
             .output()
             .unwrap();
+
+        validate_output("rust_host.o", "mv", output);
     }
 }
 
@@ -303,3 +317,18 @@ pub fn module_to_dylib(
 
     Library::new(path)
 }
+
+fn validate_output(file_name: &str, cmd_name: &str, output: Output) {
+    if !output.status.success() {
+        match std::str::from_utf8(&output.stderr) {
+            Ok(stderr) => panic!(
+                "Failed to rebuild {} - stderr of the `{}` command was:\n{}",
+                file_name, cmd_name, stderr
+            ),
+            Err(utf8_err) => panic!(
+                "Failed to rebuild {} - stderr of the `{}` command was invalid utf8 ({:?})",
+                file_name, cmd_name, utf8_err
+            ),
+        }
+    }
+}

compiler/can/src/annotation.rs

@@ -1,6 +1,6 @@
 use crate::env::Env;
 use crate::scope::Scope;
-use roc_collections::all::{ImMap, MutSet, SendMap};
+use roc_collections::all::{ImMap, MutMap, MutSet, SendMap};
 use roc_module::ident::{Ident, Lowercase, TagName};
 use roc_module::symbol::Symbol;
 use roc_parse::ast::{AssignedField, Tag, TypeAnnotation};
@@ -31,6 +31,7 @@ pub struct IntroducedVariables {
     pub wildcards: Vec<Variable>,
     pub var_by_name: SendMap<Lowercase, Variable>,
     pub name_by_var: SendMap<Variable, Lowercase>,
+    pub host_exposed_aliases: MutMap<Symbol, Variable>,
 }
 
 impl IntroducedVariables {
@@ -43,10 +44,16 @@ impl IntroducedVariables {
         self.wildcards.push(var);
     }
 
+    pub fn insert_host_exposed_alias(&mut self, symbol: Symbol, var: Variable) {
+        self.host_exposed_aliases.insert(symbol, var);
+    }
+
     pub fn union(&mut self, other: &Self) {
         self.wildcards.extend(other.wildcards.iter().cloned());
         self.var_by_name.extend(other.var_by_name.clone());
         self.name_by_var.extend(other.name_by_var.clone());
+        self.host_exposed_aliases
+            .extend(other.host_exposed_aliases.clone());
     }
 
     pub fn var_by_name(&self, name: &Lowercase) -> Option<&Variable> {
@@ -220,7 +227,15 @@ fn can_annotation_help(
                     // instantiate variables
                     actual.substitute(&substitutions);
 
-                    Type::Alias(symbol, vars, Box::new(actual))
+                    // Type::Alias(symbol, vars, Box::new(actual))
+                    let actual_var = var_store.fresh();
+                    introduced_variables.insert_host_exposed_alias(symbol, actual_var);
+                    Type::HostExposedAlias {
+                        name: symbol,
+                        arguments: vars,
+                        actual: Box::new(actual),
+                        actual_var,
+                    }
                 }
                 None => {
                     let mut args = Vec::new();
@@ -352,7 +367,16 @@ fn can_annotation_help(
                 let alias = scope.lookup_alias(symbol).unwrap();
                 local_aliases.insert(symbol, alias.clone());
 
-                Type::Alias(symbol, vars, Box::new(alias.typ.clone()))
+                // Type::Alias(symbol, vars, Box::new(alias.typ.clone()))
+
+                let actual_var = var_store.fresh();
+                introduced_variables.insert_host_exposed_alias(symbol, actual_var);
+                Type::HostExposedAlias {
+                    name: symbol,
+                    arguments: vars,
+                    actual: Box::new(alias.typ.clone()),
+                    actual_var,
+                }
             }
             _ => {
                 // This is a syntactically invalid type alias.

compiler/constrain/src/uniq.rs

@@ -2105,6 +2105,46 @@ fn annotation_to_attr_type(
 
                 let alias = Type::Alias(*symbol, new_fields, Box::new(actual_type));
 
+                (
+                    actual_vars,
+                    crate::builtins::builtin_type(Symbol::ATTR_ATTR, vec![uniq_type, alias]),
+                )
+            } else {
+                panic!("lifted type is not Attr")
+            }
+        }
+        HostExposedAlias {
+            name: symbol,
+            arguments: fields,
+            actual_var,
+            actual,
+        } => {
+            let (mut actual_vars, lifted_actual) =
+                annotation_to_attr_type(var_store, actual, rigids, change_var_kind);
+
+            if let Type::Apply(attr_symbol, args) = lifted_actual {
+                debug_assert!(attr_symbol == Symbol::ATTR_ATTR);
+
+                let uniq_type = args[0].clone();
+                let actual_type = args[1].clone();
+
+                let mut new_fields = Vec::with_capacity(fields.len());
+                for (name, tipe) in fields {
+                    let (lifted_vars, lifted) =
+                        annotation_to_attr_type(var_store, tipe, rigids, change_var_kind);
+
+                    actual_vars.extend(lifted_vars);
+
+                    new_fields.push((name.clone(), lifted));
+                }
+
+                let alias = Type::HostExposedAlias {
+                    name: *symbol,
+                    arguments: new_fields,
+                    actual_var: *actual_var,
+                    actual: Box::new(actual_type),
+                };
+
                 (
                     actual_vars,
                     crate::builtins::builtin_type(Symbol::ATTR_ATTR, vec![uniq_type, alias]),

compiler/gen/Cargo.toml

@@ -21,6 +21,7 @@ im    = "14" # im and im-rc should always have the same version!
 im-rc = "14" # im and im-rc should always have the same version!
 bumpalo = { version = "3.2", features = ["collections"] }
 inlinable_string = "0.1"
+either = "1.6.1"
 # NOTE: rtfeldman/inkwell is a fork of TheDan64/inkwell which does not change anything.
 #
 # The reason for this fork is that the way Inkwell is designed, you have to use

compiler/gen/src/llvm/build.rs

@@ -34,7 +34,7 @@ use roc_collections::all::{ImMap, MutSet};
 use roc_module::low_level::LowLevel;
 use roc_module::symbol::{Interns, ModuleId, Symbol};
 use roc_mono::ir::{JoinPointId, Wrapped};
-use roc_mono::layout::{Builtin, Layout, MemoryMode};
+use roc_mono::layout::{Builtin, ClosureLayout, Layout, MemoryMode};
 use target_lexicon::CallingConvention;
 
 /// This is for Inkwell's FunctionValue::verify - we want to know the verification
@@ -1915,55 +1915,37 @@ fn expose_function_to_host<'a, 'ctx, 'env>(
     builder.build_return(Some(&size));
 }
 
-fn make_exception_catching_wrapper<'a, 'ctx, 'env>(
+fn invoke_and_catch<'a, 'ctx, 'env, F, T>(
     env: &Env<'a, 'ctx, 'env>,
-    roc_function: FunctionValue<'ctx>,
+    parent: FunctionValue<'ctx>,
-) -> FunctionValue<'ctx> {
+    function: F,
-    // build the C calling convention wrapper
+    arguments: &[BasicValueEnum<'ctx>],
-
+    return_type: T,
+) -> BasicValueEnum<'ctx>
+where
+    F: Into<either::Either<FunctionValue<'ctx>, PointerValue<'ctx>>>,
+    T: inkwell::types::BasicType<'ctx>,
+{
     let context = env.context;
     let builder = env.builder;
 
     let u8_ptr = env.context.i8_type().ptr_type(AddressSpace::Generic);
 
-    let roc_function_type = roc_function.get_type();
+    let call_result_type = context.struct_type(
-    let argument_types = roc_function_type.get_param_types();
+        &[context.i64_type().into(), return_type.as_basic_type_enum()],
-
-    let wrapper_function_name = format!("{}_catcher", roc_function.get_name().to_str().unwrap());
-
-    let wrapper_return_type = context.struct_type(
-        &[
-            context.i64_type().into(),
-            roc_function_type.get_return_type().unwrap(),
-        ],
         false,
     );
 
-    let wrapper_function_type = wrapper_return_type.fn_type(&argument_types, false);
+    let then_block = context.append_basic_block(parent, "then_block");
+    let catch_block = context.append_basic_block(parent, "catch_block");
+    let cont_block = context.append_basic_block(parent, "cont_block");
 
-    // Add main to the module.
+    let result_alloca = builder.build_alloca(call_result_type, "result");
-    let wrapper_function =
-        env.module
-            .add_function(&wrapper_function_name, wrapper_function_type, None);
-
-    // our exposed main function adheres to the C calling convention
-    wrapper_function.set_call_conventions(FAST_CALL_CONV);
-
-    // Add main's body
-    let basic_block = context.append_basic_block(wrapper_function, "entry");
-    let then_block = context.append_basic_block(wrapper_function, "then_block");
-    let catch_block = context.append_basic_block(wrapper_function, "catch_block");
-    let cont_block = context.append_basic_block(wrapper_function, "cont_block");
-
-    builder.position_at_end(basic_block);
-
-    let result_alloca = builder.build_alloca(wrapper_return_type, "result");
 
     // invoke instead of call, so that we can catch any exeptions thrown in Roc code
-    let arguments = wrapper_function.get_params();
     let call_result = {
         let call = builder.build_invoke(
-            roc_function,
+            function,
             &arguments,
             then_block,
             catch_block,
@@ -2050,7 +2032,7 @@ fn make_exception_catching_wrapper<'a, 'ctx, 'env>(
         builder.position_at_end(then_block);
 
         let return_value = {
-            let v1 = wrapper_return_type.const_zero();
+            let v1 = call_result_type.const_zero();
 
             let v2 = builder
                 .build_insert_value(v1, context.i64_type().const_zero(), 0, "set_no_error")
@@ -2064,7 +2046,7 @@ fn make_exception_catching_wrapper<'a, 'ctx, 'env>(
 
         let ptr = builder.build_bitcast(
             result_alloca,
-            wrapper_return_type.ptr_type(AddressSpace::Generic),
+            call_result_type.ptr_type(AddressSpace::Generic),
             "name",
         );
         builder.build_store(ptr.into_pointer_value(), return_value);
@@ -2072,13 +2054,65 @@ fn make_exception_catching_wrapper<'a, 'ctx, 'env>(
         builder.build_unconditional_branch(cont_block);
     }
 
-    {
+    builder.position_at_end(cont_block);
-        builder.position_at_end(cont_block);
 
-        let result = builder.build_load(result_alloca, "result");
+    let result = builder.build_load(result_alloca, "result");
 
-        builder.build_return(Some(&result));
+    // MUST set the personality at the very end;
-    }
+    // doing it earlier can cause the personality to be ignored
+    let personality_func = get_gxx_personality_v0(env);
+    parent.set_personality_function(personality_func);
+
+    result
+}
+
+fn make_exception_catching_wrapper<'a, 'ctx, 'env>(
+    env: &Env<'a, 'ctx, 'env>,
+    roc_function: FunctionValue<'ctx>,
+) -> FunctionValue<'ctx> {
+    // build the C calling convention wrapper
+
+    let context = env.context;
+    let builder = env.builder;
+
+    let roc_function_type = roc_function.get_type();
+    let argument_types = roc_function_type.get_param_types();
+
+    let wrapper_function_name = format!("{}_catcher", roc_function.get_name().to_str().unwrap());
+
+    let wrapper_return_type = context.struct_type(
+        &[
+            context.i64_type().into(),
+            roc_function_type.get_return_type().unwrap(),
+        ],
+        false,
+    );
+
+    let wrapper_function_type = wrapper_return_type.fn_type(&argument_types, false);
+
+    // Add main to the module.
+    let wrapper_function =
+        env.module
+            .add_function(&wrapper_function_name, wrapper_function_type, None);
+
+    // our exposed main function adheres to the C calling convention
+    wrapper_function.set_call_conventions(FAST_CALL_CONV);
+
+    // invoke instead of call, so that we can catch any exeptions thrown in Roc code
+    let arguments = wrapper_function.get_params();
+
+    let basic_block = context.append_basic_block(wrapper_function, "entry");
+    builder.position_at_end(basic_block);
+
+    let result = invoke_and_catch(
+        env,
+        wrapper_function,
+        roc_function,
+        &arguments,
+        roc_function_type.get_return_type().unwrap(),
+    );
+
+    builder.build_return(Some(&result));
 
     // MUST set the personality at the very end;
     // doing it earlier can cause the personality to be ignored
@@ -2099,6 +2133,32 @@ pub fn build_proc_header<'a, 'ctx, 'env>(
     let arena = env.arena;
     let context = &env.context;
 
+    let fn_name = layout_ids
+        .get(symbol, layout)
+        .to_symbol_string(symbol, &env.interns);
+
+    use roc_mono::ir::HostExposedLayouts;
+    match &proc.host_exposed_layouts {
+        HostExposedLayouts::NotHostExposed => {}
+        HostExposedLayouts::HostExposed { rigids: _, aliases } => {
+            for (name, layout) in aliases {
+                match layout {
+                    Layout::Closure(arguments, closure, result) => {
+                        build_closure_caller(env, &fn_name, *name, arguments, closure, result)
+                    }
+                    Layout::FunctionPointer(_arguments, _result) => {
+                        // TODO should this be considered a closure of size 0?
+                        // or do we let the host call it directly?
+                        // then we have no RocCallResult wrapping though
+                    }
+                    _ => {
+                        // TODO
+                    }
+                }
+            }
+        }
+    }
+
     let ret_type = basic_type_from_layout(arena, context, &proc.ret_layout, env.ptr_bytes);
     let mut arg_basic_types = Vec::with_capacity_in(args.len(), arena);
 
@@ -2110,9 +2170,6 @@ pub fn build_proc_header<'a, 'ctx, 'env>(
 
     let fn_type = get_fn_type(&ret_type, &arg_basic_types);
 
-    let fn_name = layout_ids
-        .get(symbol, layout)
-        .to_symbol_string(symbol, &env.interns);
     let fn_val = env
         .module
         .add_function(fn_name.as_str(), fn_type, Some(Linkage::Private));
@@ -2126,8 +2183,124 @@ pub fn build_proc_header<'a, 'ctx, 'env>(
     fn_val
 }
 
-#[allow(dead_code)]
 pub fn build_closure_caller<'a, 'ctx, 'env>(
+    env: &'a Env<'a, 'ctx, 'env>,
+    def_name: &str,
+    alias_symbol: Symbol,
+    arguments: &[Layout<'a>],
+    closure: &ClosureLayout<'a>,
+    result: &Layout<'a>,
+) {
+    use inkwell::types::BasicType;
+
+    let arena = env.arena;
+    let context = &env.context;
+    let builder = env.builder;
+
+    // STEP 1: build function header
+
+    let function_name = format!(
+        "{}_{}_caller",
+        def_name,
+        alias_symbol.ident_string(&env.interns)
+    );
+
+    let mut argument_types = Vec::with_capacity_in(arguments.len() + 3, env.arena);
+
+    for layout in arguments {
+        argument_types.push(basic_type_from_layout(
+            arena,
+            context,
+            layout,
+            env.ptr_bytes,
+        ));
+    }
+
+    let function_pointer_type = {
+        let function_layout =
+            ClosureLayout::extend_function_layout(arena, arguments, closure.clone(), result);
+
+        // this is already a (function) pointer type
+        basic_type_from_layout(arena, context, &function_layout, env.ptr_bytes)
+    };
+    argument_types.push(function_pointer_type);
+
+    let closure_argument_type = {
+        let basic_type = basic_type_from_layout(
+            arena,
+            context,
+            &closure.as_block_of_memory_layout(),
+            env.ptr_bytes,
+        );
+
+        basic_type.ptr_type(AddressSpace::Generic)
+    };
+    argument_types.push(closure_argument_type.into());
+
+    let result_type = basic_type_from_layout(arena, context, result, env.ptr_bytes);
+
+    let roc_call_result_type =
+        context.struct_type(&[context.i64_type().into(), result_type], false);
+
+    let output_type = { roc_call_result_type.ptr_type(AddressSpace::Generic) };
+    argument_types.push(output_type.into());
+
+    let function_type = context.void_type().fn_type(&argument_types, false);
+
+    let function_value = env.module.add_function(
+        function_name.as_str(),
+        function_type,
+        Some(Linkage::External),
+    );
+
+    function_value.set_call_conventions(C_CALL_CONV);
+
+    // STEP 2: build function body
+
+    let entry = context.append_basic_block(function_value, "entry");
+
+    builder.position_at_end(entry);
+
+    let mut parameters = function_value.get_params();
+    let output = parameters.pop().unwrap().into_pointer_value();
+    let closure_data_ptr = parameters.pop().unwrap().into_pointer_value();
+    let function_ptr = parameters.pop().unwrap().into_pointer_value();
+
+    let closure_data = builder.build_load(closure_data_ptr, "load_closure_data");
+
+    let mut arguments = parameters;
+    arguments.push(closure_data);
+
+    let result = invoke_and_catch(env, function_value, function_ptr, &arguments, result_type);
+
+    builder.build_store(output, result);
+
+    builder.build_return(None);
+
+    // STEP 3: build a {} -> u64 function that gives the size of the return type
+    let size_function_type = env.context.i64_type().fn_type(&[], false);
+    let size_function_name: String = format!(
+        "{}_{}_size",
+        def_name,
+        alias_symbol.ident_string(&env.interns)
+    );
+
+    let size_function = env.module.add_function(
+        size_function_name.as_str(),
+        size_function_type,
+        Some(Linkage::External),
+    );
+
+    let entry = context.append_basic_block(size_function, "entry");
+
+    builder.position_at_end(entry);
+
+    let size: BasicValueEnum = roc_call_result_type.size_of().unwrap().into();
+    builder.build_return(Some(&size));
+}
+
+#[allow(dead_code)]
+pub fn build_closure_caller_old<'a, 'ctx, 'env>(
     env: &'a Env<'a, 'ctx, 'env>,
     closure_function: FunctionValue<'ctx>,
 ) {

compiler/load/src/file.rs

@@ -13,7 +13,7 @@ use roc_constrain::module::{
     constrain_imports, pre_constrain_imports, ConstrainableImports, Import,
 };
 use roc_constrain::module::{constrain_module, ExposedModuleTypes, SubsByModule};
-use roc_module::ident::{Ident, ModuleName};
+use roc_module::ident::{Ident, Lowercase, ModuleName};
 use roc_module::symbol::{IdentIds, Interns, ModuleId, ModuleIds, Symbol};
 use roc_mono::ir::{
     CapturedSymbols, ExternalSpecializations, PartialProc, PendingSpecialization, Proc, Procs,
@@ -473,6 +473,12 @@ pub struct MonomorphizedModule<'a> {
     pub timings: MutMap<ModuleId, ModuleTiming>,
 }
 
+#[derive(Debug, Default)]
+pub struct VariablySizedLayouts<'a> {
+    rigids: MutMap<Lowercase, Layout<'a>>,
+    aliases: MutMap<Symbol, Layout<'a>>,
+}
+
 #[derive(Debug)]
 struct ParsedModule<'a> {
     module_id: ModuleId,
@@ -1556,7 +1562,6 @@ fn finish_specialization<'a>(
         subs,
         interns,
         procedures,
-        // src: src.into(),
         sources,
         timings: state.timings,
     }
@@ -2355,7 +2360,13 @@ fn add_def_to_module<'a>(
                             }
                         };
 
-                        procs.insert_exposed(symbol, layout, mono_env.subs, annotation);
+                        procs.insert_exposed(
+                            symbol,
+                            layout,
+                            mono_env.subs,
+                            def.annotation,
+                            annotation,
+                        );
                     }
 
                     procs.insert_named(
@@ -2381,7 +2392,13 @@ fn add_def_to_module<'a>(
                                         todo!("TODO gracefully handle the situation where we expose a function to the host which doesn't have a valid layout (e.g. maybe the function wasn't monomorphic): {:?}", err)
                                     );
 
-                        procs.insert_exposed(symbol, layout, mono_env.subs, annotation);
+                        procs.insert_exposed(
+                            symbol,
+                            layout,
+                            mono_env.subs,
+                            def.annotation,
+                            annotation,
+                        );
                     }
 
                     let proc = PartialProc {

compiler/mono/src/ir.rs

@@ -28,6 +28,12 @@ pub struct PartialProc<'a> {
     pub is_self_recursive: bool,
 }
 
+#[derive(Clone, Debug, PartialEq, Default)]
+pub struct HostExposedVariables {
+    rigids: MutMap<Lowercase, Variable>,
+    aliases: MutMap<Symbol, Variable>,
+}
+
 #[derive(Clone, Debug, PartialEq)]
 pub enum CapturedSymbols<'a> {
     None,
@@ -46,12 +52,34 @@ impl<'a> CapturedSymbols<'a> {
 #[derive(Clone, Debug, PartialEq)]
 pub struct PendingSpecialization {
     solved_type: SolvedType,
+    host_exposed_aliases: MutMap<Symbol, SolvedType>,
 }
 
 impl PendingSpecialization {
     pub fn from_var(subs: &Subs, var: Variable) -> Self {
         let solved_type = SolvedType::from_var(subs, var);
-        PendingSpecialization { solved_type }
+        PendingSpecialization {
+            solved_type,
+            host_exposed_aliases: MutMap::default(),
+        }
+    }
+
+    pub fn from_var_host_exposed(
+        subs: &Subs,
+        var: Variable,
+        host_exposed_aliases: &MutMap<Symbol, Variable>,
+    ) -> Self {
+        let solved_type = SolvedType::from_var(subs, var);
+
+        let host_exposed_aliases = host_exposed_aliases
+            .iter()
+            .map(|(symbol, variable)| (*symbol, SolvedType::from_var(subs, *variable)))
+            .collect();
+
+        PendingSpecialization {
+            solved_type,
+            host_exposed_aliases,
+        }
     }
 }
 
@@ -63,6 +91,16 @@ pub struct Proc<'a> {
     pub closure_data_layout: Option<Layout<'a>>,
     pub ret_layout: Layout<'a>,
     pub is_self_recursive: SelfRecursive,
+    pub host_exposed_layouts: HostExposedLayouts<'a>,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum HostExposedLayouts<'a> {
+    NotHostExposed,
+    HostExposed {
+        rigids: MutMap<Lowercase, Layout<'a>>,
+        aliases: MutMap<Symbol, Layout<'a>>,
+    },
 }
 
 #[derive(Clone, Debug, PartialEq)]
@@ -398,8 +436,7 @@ impl<'a> Procs<'a> {
                 // Changing it to use .entry() would necessarily make it incorrect.
                 #[allow(clippy::map_entry)]
                 if !already_specialized {
-                    let solved_type = SolvedType::from_var(env.subs, annotation);
+                    let pending = PendingSpecialization::from_var(env.subs, annotation);
-                    let pending = PendingSpecialization { solved_type };
 
                     let pattern_symbols = pattern_symbols.into_bump_slice();
                     match &mut self.pending_specializations {
@@ -478,6 +515,7 @@ impl<'a> Procs<'a> {
         name: Symbol,
         layout: Layout<'a>,
         subs: &Subs,
+        opt_annotation: Option<roc_can::def::Annotation>,
         fn_var: Variable,
     ) {
         let tuple = (name, layout);
@@ -489,7 +527,14 @@ impl<'a> Procs<'a> {
 
         // We're done with that tuple, so move layout back out to avoid cloning it.
         let (name, layout) = tuple;
-        let pending = PendingSpecialization::from_var(subs, fn_var);
+        let pending = match opt_annotation {
+            None => PendingSpecialization::from_var(subs, fn_var),
+            Some(annotation) => PendingSpecialization::from_var_host_exposed(
+                subs,
+                fn_var,
+                &annotation.introduced_variables.host_exposed_aliases,
+            ),
+        };
 
         // This should only be called when pending_specializations is Some.
         // Otherwise, it's being called in the wrong pass!
@@ -1302,6 +1347,7 @@ pub fn specialize_all<'a>(
             name,
             layout_cache,
             solved_type,
+            MutMap::default(),
             partial_proc,
         ) {
             Ok((proc, layout)) => {
@@ -1387,6 +1433,7 @@ fn specialize_external<'a>(
     proc_name: Symbol,
     layout_cache: &mut LayoutCache<'a>,
     fn_var: Variable,
+    host_exposed_variables: &[(Symbol, Variable)],
     partial_proc: PartialProc<'a>,
 ) -> Result<Proc<'a>, LayoutProblem> {
     let PartialProc {
@@ -1451,6 +1498,25 @@ fn specialize_external<'a>(
         }
     }
 
+    // determine the layout of aliases/rigids exposed to the host
+    let host_exposed_layouts = if host_exposed_variables.is_empty() {
+        HostExposedLayouts::NotHostExposed
+    } else {
+        let mut aliases = MutMap::default();
+
+        for (symbol, variable) in host_exposed_variables {
+            let layout = layout_cache
+                .from_var(env.arena, *variable, env.subs)
+                .unwrap();
+            aliases.insert(*symbol, layout);
+        }
+
+        HostExposedLayouts::HostExposed {
+            rigids: MutMap::default(),
+            aliases,
+        }
+    };
+
     // reset subs, so we don't get type errors when specializing for a different signature
     layout_cache.rollback_to(cache_snapshot);
     env.subs.rollback_to(snapshot);
@@ -1473,6 +1539,7 @@ fn specialize_external<'a>(
         closure_data_layout,
         ret_layout,
         is_self_recursive: recursivity,
+        host_exposed_layouts,
     };
 
     Ok(proc)
@@ -1693,51 +1760,80 @@ fn specialize<'a>(
     pending: PendingSpecialization,
     partial_proc: PartialProc<'a>,
 ) -> Result<(Proc<'a>, Layout<'a>), LayoutProblem> {
-    let PendingSpecialization { solved_type } = pending;
+    let PendingSpecialization {
+        solved_type,
+        host_exposed_aliases,
+    } = pending;
+
     specialize_solved_type(
         env,
         procs,
         proc_name,
         layout_cache,
         solved_type,
+        host_exposed_aliases,
         partial_proc,
     )
 }
 
+fn introduce_solved_type_to_subs<'a>(env: &mut Env<'a, '_>, solved_type: &SolvedType) -> Variable {
+    use roc_solve::solve::insert_type_into_subs;
+    use roc_types::solved_types::{to_type, FreeVars};
+    use roc_types::subs::VarStore;
+    let mut free_vars = FreeVars::default();
+    let mut var_store = VarStore::new_from_subs(env.subs);
+
+    let before = var_store.peek();
+
+    let normal_type = to_type(solved_type, &mut free_vars, &mut var_store);
+
+    let after = var_store.peek();
+    let variables_introduced = after - before;
+
+    env.subs.extend_by(variables_introduced as usize);
+
+    insert_type_into_subs(env.subs, &normal_type)
+}
+
 fn specialize_solved_type<'a>(
     env: &mut Env<'a, '_>,
     procs: &mut Procs<'a>,
     proc_name: Symbol,
     layout_cache: &mut LayoutCache<'a>,
     solved_type: SolvedType,
+    host_exposed_aliases: MutMap<Symbol, SolvedType>,
     partial_proc: PartialProc<'a>,
 ) -> Result<(Proc<'a>, Layout<'a>), LayoutProblem> {
     // add the specializations that other modules require of us
-    use roc_solve::solve::{insert_type_into_subs, instantiate_rigids};
+    use roc_solve::solve::instantiate_rigids;
-    use roc_types::solved_types::{to_type, FreeVars};
-    use roc_types::subs::VarStore;
 
     let snapshot = env.subs.snapshot();
     let cache_snapshot = layout_cache.snapshot();
 
-    let mut free_vars = FreeVars::default();
+    let fn_var = introduce_solved_type_to_subs(env, &solved_type);
-    let mut var_store = VarStore::new_from_subs(env.subs);
-
-    let before = var_store.peek();
-
-    let normal_type = to_type(&solved_type, &mut free_vars, &mut var_store);
-
-    let after = var_store.peek();
-    let variables_introduced = after - before;
-
-    env.subs.extend_by(variables_introduced as usize);
-
-    let fn_var = insert_type_into_subs(env.subs, &normal_type);
 
     // make sure rigid variables in the annotation are converted to flex variables
     instantiate_rigids(env.subs, partial_proc.annotation);
 
-    match specialize_external(env, procs, proc_name, layout_cache, fn_var, partial_proc) {
+    let mut host_exposed_variables = Vec::with_capacity_in(host_exposed_aliases.len(), env.arena);
+
+    for (symbol, solved_type) in host_exposed_aliases {
+        let alias_var = introduce_solved_type_to_subs(env, &solved_type);
+
+        host_exposed_variables.push((symbol, alias_var));
+    }
+
+    let specialized = specialize_external(
+        env,
+        procs,
+        proc_name,
+        layout_cache,
+        fn_var,
+        &host_exposed_variables,
+        partial_proc,
+    );
+
+    match specialized {
         Ok(proc) => {
             let layout = layout_cache
                 .from_var(&env.arena, fn_var, env.subs)

compiler/solve/src/solve.rs

@@ -813,6 +813,45 @@ fn type_to_variable(
 
             result
         }
+        HostExposedAlias {
+            name: symbol,
+            arguments: args,
+            actual: alias_type,
+            actual_var,
+            ..
+        } => {
+            let mut arg_vars = Vec::with_capacity(args.len());
+            let mut new_aliases = ImMap::default();
+
+            for (arg, arg_type) in args {
+                let arg_var = type_to_variable(subs, rank, pools, cached, arg_type);
+
+                arg_vars.push((arg.clone(), arg_var));
+                new_aliases.insert(arg.clone(), arg_var);
+            }
+
+            let alias_var = type_to_variable(subs, rank, pools, cached, alias_type);
+
+            // unify the actual_var with the result var
+            // this can be used to access the type of the actual_var
+            // to determine its layout later
+            // subs.set_content(*actual_var, descriptor.content);
+
+            //subs.set(*actual_var, descriptor.clone());
+            let content = Content::Alias(*symbol, arg_vars, alias_var);
+
+            let result = register(subs, rank, pools, content);
+
+            // We only want to unify the actual_var with the alias once
+            // if it's already redirected (and therefore, redundant)
+            // don't do it again
+            if !subs.redundant(*actual_var) {
+                let descriptor = subs.get(result);
+                subs.union(result, *actual_var, descriptor);
+            }
+
+            result
+        }
         Erroneous(problem) => {
             let content = Content::Structure(FlatType::Erroneous(problem.clone()));
 

compiler/types/src/solved_types.rs

@@ -53,6 +53,13 @@ pub enum SolvedType {
     /// A type alias
     Alias(Symbol, Vec<(Lowercase, SolvedType)>, Box<SolvedType>),
 
+    HostExposedAlias {
+        name: Symbol,
+        arguments: Vec<(Lowercase, SolvedType)>,
+        actual_var: VarId,
+        actual: Box<SolvedType>,
+    },
+
     /// a boolean algebra Bool
     Boolean(SolvedBool),
 
@@ -194,6 +201,26 @@ impl SolvedType {
 
                 SolvedType::Alias(*symbol, solved_args, Box::new(solved_type))
             }
+            HostExposedAlias {
+                name,
+                arguments,
+                actual_var,
+                actual,
+            } => {
+                let solved_type = Self::from_type(solved_subs, actual);
+                let mut solved_args = Vec::with_capacity(arguments.len());
+
+                for (name, var) in arguments {
+                    solved_args.push((name.clone(), Self::from_type(solved_subs, var)));
+                }
+
+                SolvedType::HostExposedAlias {
+                    name: *name,
+                    arguments: solved_args,
+                    actual_var: VarId::from_var(*actual_var, solved_subs.inner()),
+                    actual: Box::new(solved_type),
+                }
+            }
             Boolean(val) => SolvedType::Boolean(SolvedBool::from_bool(&val, solved_subs.inner())),
             Variable(var) => Self::from_var(solved_subs.inner(), *var),
         }
@@ -486,6 +513,27 @@ pub fn to_type(
 
             Type::Alias(*symbol, type_variables, Box::new(actual))
         }
+        HostExposedAlias {
+            name,
+            arguments: solved_type_variables,
+            actual_var,
+            actual: solved_actual,
+        } => {
+            let mut type_variables = Vec::with_capacity(solved_type_variables.len());
+
+            for (lowercase, solved_arg) in solved_type_variables {
+                type_variables.push((lowercase.clone(), to_type(solved_arg, free_vars, var_store)));
+            }
+
+            let actual = to_type(solved_actual, free_vars, var_store);
+
+            Type::HostExposedAlias {
+                name: *name,
+                arguments: type_variables,
+                actual_var: var_id_to_flex_var(*actual_var, free_vars, var_store),
+                actual: Box::new(actual),
+            }
+        }
         Error => Type::Erroneous(Problem::SolvedTypeError),
         Erroneous(problem) => Type::Erroneous(problem.clone()),
     }

compiler/types/src/types.rs

@@ -143,6 +143,12 @@ pub enum Type {
     Record(SendMap<Lowercase, RecordField<Type>>, Box<Type>),
     TagUnion(Vec<(TagName, Vec<Type>)>, Box<Type>),
     Alias(Symbol, Vec<(Lowercase, Type)>, Box<Type>),
+    HostExposedAlias {
+        name: Symbol,
+        arguments: Vec<(Lowercase, Type)>,
+        actual_var: Variable,
+        actual: Box<Type>,
+    },
     RecursiveTagUnion(Variable, Vec<(TagName, Vec<Type>)>, Box<Type>),
     /// Applying a type to some arguments (e.g. Map.Map String Int)
     Apply(Symbol, Vec<Type>),
@@ -206,6 +212,20 @@ impl fmt::Debug for Type {
 
                 Ok(())
             }
+            Type::HostExposedAlias {
+                name, arguments, ..
+            } => {
+                write!(f, "HostExposedAlias {:?}", name)?;
+
+                for (_, arg) in arguments {
+                    write!(f, " {:?}", arg)?;
+                }
+
+                // Sometimes it's useful to see the expansion of the alias
+                // write!(f, "[ but actually {:?} ]", _actual)?;
+
+                Ok(())
+            }
             Type::Record(fields, ext) => {
                 write!(f, "{{")?;
 
@@ -405,6 +425,16 @@ impl Type {
                 }
                 actual_type.substitute(substitutions);
             }
+            HostExposedAlias {
+                arguments,
+                actual: actual_type,
+                ..
+            } => {
+                for (_, value) in arguments.iter_mut() {
+                    value.substitute(substitutions);
+                }
+                actual_type.substitute(substitutions);
+            }
             Apply(_, args) => {
                 for arg in args {
                     arg.substitute(substitutions);
@@ -453,6 +483,12 @@ impl Type {
             Alias(_, _, actual_type) => {
                 actual_type.substitute_alias(rep_symbol, actual);
             }
+            HostExposedAlias {
+                actual: actual_type,
+                ..
+            } => {
+                actual_type.substitute_alias(rep_symbol, actual);
+            }
             Apply(symbol, _) if *symbol == rep_symbol => {
                 *self = actual.clone();
 
@@ -496,6 +532,9 @@ impl Type {
             Alias(alias_symbol, _, actual_type) => {
                 alias_symbol == &rep_symbol || actual_type.contains_symbol(rep_symbol)
             }
+            HostExposedAlias { name, actual, .. } => {
+                name == &rep_symbol || actual.contains_symbol(rep_symbol)
+            }
             Apply(symbol, _) if *symbol == rep_symbol => true,
             Apply(_, args) => args.iter().any(|arg| arg.contains_symbol(rep_symbol)),
             EmptyRec | EmptyTagUnion | Erroneous(_) | Variable(_) | Boolean(_) => false,
@@ -528,6 +567,7 @@ impl Type {
                         .any(|arg| arg.contains_variable(rep_variable))
             }
             Alias(_, _, actual_type) => actual_type.contains_variable(rep_variable),
+            HostExposedAlias { actual, .. } => actual.contains_variable(rep_variable),
             Apply(_, args) => args.iter().any(|arg| arg.contains_variable(rep_variable)),
             EmptyRec | EmptyTagUnion | Erroneous(_) | Boolean(_) => false,
         }
@@ -579,7 +619,12 @@ impl Type {
                 }
                 ext.instantiate_aliases(region, aliases, var_store, introduced);
             }
-            Alias(_, type_args, actual_type) => {
+            HostExposedAlias {
+                arguments: type_args,
+                actual: actual_type,
+                ..
+            }
+            | Alias(_, type_args, actual_type) => {
                 for arg in type_args {
                     arg.1
                         .instantiate_aliases(region, aliases, var_store, introduced);
@@ -761,6 +806,10 @@ fn symbols_help(tipe: &Type, accum: &mut ImSet<Symbol>) {
             accum.insert(*alias_symbol);
             symbols_help(&actual_type, accum);
         }
+        HostExposedAlias { name, actual, .. } => {
+            accum.insert(*name);
+            symbols_help(&actual, accum);
+        }
         Apply(symbol, args) => {
             accum.insert(*symbol);
             args.iter().for_each(|arg| symbols_help(arg, accum));
@@ -830,6 +879,14 @@ fn variables_help(tipe: &Type, accum: &mut ImSet<Variable>) {
             }
             variables_help(actual, accum);
         }
+        HostExposedAlias {
+            arguments, actual, ..
+        } => {
+            for (_, arg) in arguments {
+                variables_help(arg, accum);
+            }
+            variables_help(actual, accum);
+        }
         Apply(_, args) => {
             for x in args {
                 variables_help(x, accum);

examples/.gitignore

@@ -1,4 +1,5 @@
 app
 host.o
 c_host.o
+roc_app.o
 app.dSYM

examples/closure/Closure.roc

@@ -1,8 +1,9 @@
-app Closure provides [ closure ] imports []
+app Closure provides [ makeClosure ] imports []
 
-closure : {} -> Int
+makeClosure : ({} -> Int) as MyClosure
-closure = 
+makeClosure = 
     x = 42
+    y = 42
 
-    \{} -> x
+    \{} -> x + y
 

examples/closure/platform/src/lib.rs

@@ -1,19 +1,42 @@
+use roc_std::alloca;
+use roc_std::RocCallResult;
 use std::alloc::Layout;
-use std::ffi::CString;
-use std::mem::MaybeUninit;
-use std::os::raw::c_char;
 use std::time::SystemTime;
-use RocCallResult::*;
 
 extern "C" {
-    #[link_name = "closure_1_exposed"]
+    #[link_name = "makeClosure_1_exposed"]
     fn make_closure(output: *mut u8) -> ();
 
-    // #[link_name = "0_1_caller"]
+    #[link_name = "makeClosure_1_size"]
-    // fn call_closure_0(unit: (), closure_data: *const u8, output: *mut u8) -> ();
-
-    #[link_name = "closure_1_size"]
     fn closure_size() -> i64;
+
+    #[link_name = "makeClosure_1_MyClosure_caller"]
+    fn call_MyClosure(function_pointer: *const u8, closure_data: *const u8, output: *mut u8) -> ();
+
+    #[link_name = "makeClosure_1_MyClosure_size"]
+    fn size_MyClosure() -> i64;
+}
+
+unsafe fn call_the_closure(function_pointer: *const u8, closure_data_ptr: *const u8) -> i64 {
+    let size = size_MyClosure() as usize;
+
+    alloca::with_stack_bytes(size, |buffer| {
+        let buffer: *mut std::ffi::c_void = buffer;
+        let buffer: *mut u8 = buffer as *mut u8;
+
+        call_MyClosure(
+            function_pointer,
+            closure_data_ptr as *const u8,
+            buffer as *mut u8,
+        );
+
+        let output = &*(buffer as *mut RocCallResult<i64>);
+
+        match output.into() {
+            Ok(v) => v,
+            Err(e) => panic!("failed with {}", e),
+        }
+    })
 }
 
 #[no_mangle]
@@ -23,19 +46,26 @@ pub fn rust_main() -> isize {
 
     let size = unsafe { closure_size() } as usize;
     let layout = Layout::array::<u8>(size).unwrap();
-    let roc_closure = unsafe {
+    let answer = unsafe {
         let buffer = std::alloc::alloc(layout);
 
         make_closure(buffer);
 
-        type CLOSURE_DATA = i64;
+        let output = &*(buffer as *mut RocCallResult<()>);
-        let output = &*(buffer as *mut RocCallResult<(fn(CLOSURE_DATA) -> i64, CLOSURE_DATA)>);
 
         match output.into() {
-            Ok((function_pointer, closure_data)) => {
+            Ok(()) => {
-                std::alloc::dealloc(buffer, layout);
+                let function_pointer = {
+                    // this is a pointer to the location where the function pointer is stored
+                    // we pass just the function pointer
+                    let temp = buffer.offset(8) as *const i64;
 
-                move || function_pointer(closure_data)
+                    (*temp) as *const u8
+                };
+
+                let closure_data_ptr = buffer.offset(16);
+
+                call_the_closure(function_pointer as *const u8, closure_data_ptr as *const u8)
             }
             Err(msg) => {
                 std::alloc::dealloc(buffer, layout);
@@ -44,7 +74,6 @@ pub fn rust_main() -> isize {
             }
         }
     };
-    let answer = roc_closure();
     let end_time = SystemTime::now();
     let duration = end_time.duration_since(start_time).unwrap();
 
@@ -58,45 +87,3 @@ pub fn rust_main() -> isize {
     // Exit code
     0
 }
-
-#[repr(u64)]
-pub enum RocCallResult<T> {
-    Success(T),
-    Failure(*mut c_char),
-}
-
-impl<T: Sized> Into<Result<T, String>> for RocCallResult<T> {
-    fn into(self) -> Result<T, String> {
-        match self {
-            Success(value) => Ok(value),
-            Failure(failure) => Err({
-                let raw = unsafe { CString::from_raw(failure) };
-
-                let result = format!("{:?}", raw);
-
-                // make sure rust does not try to free the Roc string
-                std::mem::forget(raw);
-
-                result
-            }),
-        }
-    }
-}
-
-impl<T: Sized + Copy> Into<Result<T, String>> for &RocCallResult<T> {
-    fn into(self) -> Result<T, String> {
-        match self {
-            Success(value) => Ok(*value),
-            Failure(failure) => Err({
-                let raw = unsafe { CString::from_raw(*failure) };
-
-                let result = format!("{:?}", raw);
-
-                // make sure rust does not try to free the Roc string
-                std::mem::forget(raw);
-
-                result
-            }),
-        }
-    }
-}

roc_std/src/alloca.rs

@@ -63,7 +63,8 @@ unsafe fn malloc_or_alloca(bytes: usize) -> *mut c_void {
 #[cfg(not(debug_assertions))]
 #[inline(always)]
 unsafe fn malloc_or_alloca(bytes: usize) -> *mut c_void {
-    c_alloca(bytes)
+    // c_alloca(bytes)
+    libc::malloc(bytes)
 }
 
 #[cfg(debug_assertions)]

roc_std/src/lib.rs

@@ -464,3 +464,30 @@ impl<T: Sized> Into<Result<T, &'static str>> for RocCallResult<T> {
         }
     }
 }
+
+impl<'a, T: Sized + Copy> Into<Result<T, &'a str>> for &'a RocCallResult<T> {
+    fn into(self) -> Result<T, &'a str> {
+        use RocCallResult::*;
+
+        match self {
+            Success(value) => Ok(*value),
+            Failure(failure) => Err({
+                let msg = unsafe {
+                    let mut null_byte_index = 0;
+                    loop {
+                        if *failure.offset(null_byte_index) == 0 {
+                            break;
+                        }
+                        null_byte_index += 1;
+                    }
+
+                    let bytes = core::slice::from_raw_parts(*failure, null_byte_index as usize);
+
+                    core::str::from_utf8_unchecked(bytes)
+                };
+
+                msg
+            }),
+        }
+    }
+}

shell.nix

@@ -29,18 +29,25 @@ let
       [ ];
   llvm = pkgs.llvm_10;
   lld = pkgs.lld_10; # this should match llvm's version
+  clang = pkgs.clang_10; # this should match llvm's version
   zig = import ./nix/zig.nix { inherit pkgs isMacOS; };
   inputs =
     [
       # build libraries
-      pkgs.rustup
+      pkgs.rustc
       pkgs.cargo
+      pkgs.cmake
+      pkgs.git
+      pkgs.python3
       llvm
+      clang
       pkgs.valgrind
+      pkgs.pkg-config
       zig
       # llb deps
       pkgs.libffi
       pkgs.libxml2
+      pkgs.xorg.libX11
       pkgs.zlib
       # faster builds - see https://github.com/rtfeldman/roc/blob/trunk/BUILDING_FROM_SOURCE.md#use-lld-for-the-linker
       lld