Merge remote-tracking branch 'origin/trunk' into layout-builtin-numbers-refactor

.github/workflows/spellcheck.yml

@@ -8,7 +8,7 @@ env:
 jobs:
   spell-check:
     name: spell check
-    runs-on: [self-hosted]
+    runs-on: [self-hosted, linux]
     timeout-minutes: 10
     env:
       FORCE_COLOR: 1

.github/workflows/www.yml

@@ -9,7 +9,7 @@ on:
 jobs:
   deploy:
     name: 'Deploy to Netlify'
-    runs-on: [self-hosted]
+    runs-on: [self-hosted, linux]
     steps:
       - uses: jsmrcaga/action-netlify-deploy@v1.6.0
         with:

BUILDING_FROM_SOURCE.md

@@ -94,13 +94,7 @@ Install nix:
 
 `curl -L https://nixos.org/nix/install | sh`
 
-If you're on MacOS and using a OS version >= 10.15:
+You will need to start a fresh terminal session to use nix.
-
-`sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume`
-
-You may prefer to setup up the volume manually by following nix documentation.
-
-> You may need to restart your terminal
 
 ### Usage
 

Cargo.lock

@@ -3555,6 +3555,7 @@ version = "0.1.0"
 name = "roc_types"
 version = "0.1.0"
 dependencies = [
+ "bumpalo",
  "roc_collections",
  "roc_module",
  "roc_region",

Earthfile

@@ -79,17 +79,17 @@ test-rust:
     # not pre-compiling the host can cause race conditions
     RUN echo "4" | cargo run --release examples/benchmarks/NQueens.roc
     RUN --mount=type=cache,target=$SCCACHE_DIR \
-        cargo test --release --features with_sound --workspace && sccache --show-stats
+        cargo test --locked --release --features with_sound --workspace && sccache --show-stats
     # test the dev and wasm backend: they require an explicit feature flag.
     RUN --mount=type=cache,target=$SCCACHE_DIR \
-        cargo test --release --package test_gen --no-default-features --features gen-dev && sccache --show-stats
+        cargo test --locked --release --package test_gen --no-default-features --features gen-dev && sccache --show-stats
     # gen-wasm has some multithreading problems to do with the wasmer runtime. Run it single-threaded as a separate job
     RUN --mount=type=cache,target=$SCCACHE_DIR \
-        cargo test --release --package test_gen --no-default-features --features gen-wasm -- --test-threads=1 && sccache --show-stats
+        cargo test --locked --release --package test_gen --no-default-features --features gen-wasm -- --test-threads=1 && sccache --show-stats
     # run i386 (32-bit linux) cli tests
-    RUN echo "4" | cargo run --release --features="target-x86" -- --backend=x86_32 examples/benchmarks/NQueens.roc
+    RUN echo "4" | cargo run --locked --release --features="target-x86" -- --backend=x86_32 examples/benchmarks/NQueens.roc
     RUN --mount=type=cache,target=$SCCACHE_DIR \
-        cargo test --release --features with_sound --test cli_run i386 --features="i386-cli-run" && sccache --show-stats
+        cargo test --locked --release --features with_sound --test cli_run i386 --features="i386-cli-run" && sccache --show-stats
 
 verify-no-git-changes:
     FROM +test-rust

compiler/gen_dev/src/generic64/aarch64.rs

@@ -500,6 +500,16 @@ impl Assembler<AArch64GeneralReg, AArch64FloatReg> for AArch64Assembler {
         unimplemented!("registers non-equality not implemented yet for AArch64");
     }
 
+    #[inline(always)]
+    fn lt_reg64_reg64_reg64(
+        _buf: &mut Vec<'_, u8>,
+        _dst: AArch64GeneralReg,
+        _src1: AArch64GeneralReg,
+        _src2: AArch64GeneralReg,
+    ) {
+        unimplemented!("registers less than not implemented yet for AArch64");
+    }
+
     #[inline(always)]
     fn ret(buf: &mut Vec<'_, u8>) {
         ret_reg64(buf, AArch64GeneralReg::LR)

compiler/gen_dev/src/generic64/mod.rs

@@ -180,6 +180,13 @@ pub trait Assembler<GeneralReg: RegTrait, FloatReg: RegTrait> {
         src2: GeneralReg,
     );
 
+    fn lt_reg64_reg64_reg64(
+        buf: &mut Vec<'_, u8>,
+        dst: GeneralReg,
+        src1: GeneralReg,
+        src2: GeneralReg,
+    );
+
     fn ret(buf: &mut Vec<'_, u8>);
 }
 
@@ -869,6 +876,25 @@ impl<
         }
     }
 
+    fn build_num_lt(
+        &mut self,
+        dst: &Symbol,
+        src1: &Symbol,
+        src2: &Symbol,
+        arg_layout: &Layout<'a>,
+    ) -> Result<(), String> {
+        match arg_layout {
+            Layout::Builtin(Builtin::Int64) => {
+                let dst_reg = self.claim_general_reg(dst)?;
+                let src1_reg = self.load_to_general_reg(src1)?;
+                let src2_reg = self.load_to_general_reg(src2)?;
+                ASM::lt_reg64_reg64_reg64(&mut self.buf, dst_reg, src1_reg, src2_reg);
+                Ok(())
+            }
+            x => Err(format!("NumLt: layout, {:?}, not implemented yet", x)),
+        }
+    }
+
     fn create_struct(
         &mut self,
         sym: &Symbol,

compiler/gen_dev/src/generic64/x86_64.rs

@@ -1108,6 +1108,17 @@ impl Assembler<X86_64GeneralReg, X86_64FloatReg> for X86_64Assembler {
         setne_reg64(buf, dst);
     }
 
+    #[inline(always)]
+    fn lt_reg64_reg64_reg64(
+        buf: &mut Vec<'_, u8>,
+        dst: X86_64GeneralReg,
+        src1: X86_64GeneralReg,
+        src2: X86_64GeneralReg,
+    ) {
+        cmp_reg64_reg64(buf, src1, src2);
+        setl_reg64(buf, dst);
+    }
+
     #[inline(always)]
     fn ret(buf: &mut Vec<'_, u8>) {
         ret(buf);
@@ -1499,6 +1510,12 @@ fn setne_reg64(buf: &mut Vec<'_, u8>, reg: X86_64GeneralReg) {
     set_reg64_help(buf, reg, 0x95);
 }
 
+/// `SETL r/m64` -> Set byte if less (SF≠ OF).
+#[inline(always)]
+fn setl_reg64(buf: &mut Vec<'_, u8>, reg: X86_64GeneralReg) {
+    set_reg64_help(buf, reg, 0x9c);
+}
+
 /// `RET` -> Near return to calling procedure.
 #[inline(always)]
 fn ret(buf: &mut Vec<'_, u8>) {
@@ -2083,7 +2100,7 @@ mod tests {
     }
 
     #[test]
-    fn test_sete_reg64() {
+    fn test_set_reg64_help() {
         let arena = bumpalo::Bump::new();
         let mut buf = bumpalo::vec![in &arena];
 
@@ -2096,7 +2113,7 @@ mod tests {
             ],
         );
         buf.clear();
-        sete_reg64(&mut buf, reg);
+        set_reg64_help(&mut buf, reg, 0x94); // sete_reg64
         assert_eq!(expected, &buf[..]);
 
         // tests for 8 bytes in the output buffer
@@ -2121,47 +2138,7 @@ mod tests {
             ),
         ] {
             buf.clear();
-            sete_reg64(&mut buf, *reg);
+            set_reg64_help(&mut buf, *reg, 0x94); // sete_reg64
-            assert_eq!(expected, &buf[..]);
-        }
-    }
-
-    #[test]
-    // follow test_sete_reg64
-    // refer it
-    fn test_setne_reg64() {
-        let arena = bumpalo::Bump::new();
-        let mut buf = bumpalo::vec![in &arena];
-
-        let (reg, expected) = (
-            X86_64GeneralReg::RAX,
-            [
-                0x0F, 0x95, 0xC0, // SETNE al ;
-                0x48, 0x83, 0xE0, 0x01,
-            ],
-        );
-        buf.clear();
-        setne_reg64(&mut buf, reg);
-        assert_eq!(expected, &buf[..]);
-
-        for (reg, expected) in &[
-            (
-                X86_64GeneralReg::RSP,
-                [
-                    // SETNE spl;
-                    0x40, 0x0F, 0x95, 0xC4, 0x48, 0x83, 0xE4, 0x01,
-                ],
-            ),
-            (
-                X86_64GeneralReg::R15,
-                [
-                    // SETNE r15b;
-                    0x41, 0x0F, 0x95, 0xC7, 0x49, 0x83, 0xE7, 0x01,
-                ],
-            ),
-        ] {
-            buf.clear();
-            setne_reg64(&mut buf, *reg);
             assert_eq!(expected, &buf[..]);
         }
     }

compiler/gen_dev/src/lib.rs

@@ -433,6 +433,23 @@ where
                 );
                 self.build_neq(sym, &args[0], &args[1], &arg_layouts[0])
             }
+            LowLevel::NumLt => {
+                debug_assert_eq!(
+                    2,
+                    args.len(),
+                    "NumLt: expected to have exactly two argument"
+                );
+                debug_assert_eq!(
+                    arg_layouts[0], arg_layouts[1],
+                    "NumLt: expected all arguments of to have the same layout"
+                );
+                debug_assert_eq!(
+                    Layout::Builtin(Builtin::Int1),
+                    *ret_layout,
+                    "NumLt: expected to have return layout of type I1"
+                );
+                self.build_num_lt(sym, &args[0], &args[1], &arg_layouts[0])
+            }
             LowLevel::NumRound => self.build_fn_call(
                 sym,
                 bitcode::NUM_ROUND[FloatWidth::F64].to_string(),
@@ -523,6 +540,15 @@ where
         arg_layout: &Layout<'a>,
     ) -> Result<(), String>;
 
+    /// build_num_lt stores the result of `src1 < src2` into dst.
+    fn build_num_lt(
+        &mut self,
+        dst: &Symbol,
+        src1: &Symbol,
+        src2: &Symbol,
+        arg_layout: &Layout<'a>,
+    ) -> Result<(), String>;
+
     /// literal_map gets the map from symbol to literal, used for lazy loading and literal folding.
     fn literal_map(&mut self) -> &mut MutMap<Symbol, Literal<'a>>;
 

compiler/load/src/file.rs

@@ -19,8 +19,7 @@ use roc_module::symbol::{
     Symbol,
 };
 use roc_mono::ir::{
-    CapturedSymbols, EntryPoint, ExternalSpecializations, PartialProc, PendingSpecialization, Proc,
+    CapturedSymbols, EntryPoint, ExternalSpecializations, PartialProc, Proc, ProcLayout, Procs,
-    ProcLayout, Procs,
 };
 use roc_mono::layout::{Layout, LayoutCache, LayoutProblem};
 use roc_parse::ast::{self, StrLiteral, TypeAnnotation};
@@ -356,7 +355,7 @@ struct ModuleCache<'a> {
     constrained: MutMap<ModuleId, ConstrainedModule>,
     typechecked: MutMap<ModuleId, TypeCheckedModule<'a>>,
     found_specializations: MutMap<ModuleId, FoundSpecializationsModule<'a>>,
-    external_specializations_requested: MutMap<ModuleId, Vec<ExternalSpecializations<'a>>>,
+    external_specializations_requested: MutMap<ModuleId, Vec<ExternalSpecializations>>,
 
     /// Various information
     imports: MutMap<ModuleId, MutSet<ModuleId>>,
@@ -831,7 +830,7 @@ enum Msg<'a> {
         module_id: ModuleId,
         ident_ids: IdentIds,
         layout_cache: LayoutCache<'a>,
-        external_specializations_requested: BumpMap<ModuleId, ExternalSpecializations<'a>>,
+        external_specializations_requested: BumpMap<ModuleId, ExternalSpecializations>,
         procedures: MutMap<(Symbol, ProcLayout<'a>), Proc<'a>>,
         problems: Vec<roc_mono::ir::MonoProblem>,
         module_timing: ModuleTiming,
@@ -911,9 +910,6 @@ struct State<'a> {
     /// pending specializations in the same thread.
     pub needs_specialization: MutSet<ModuleId>,
 
-    pub all_pending_specializations:
-        MutMap<Symbol, MutMap<ProcLayout<'a>, PendingSpecialization<'a>>>,
-
     pub specializations_in_flight: u32,
 
     pub timings: MutMap<ModuleId, ModuleTiming>,
@@ -1054,7 +1050,7 @@ enum BuildTask<'a> {
         subs: Subs,
         procs_base: ProcsBase<'a>,
         layout_cache: LayoutCache<'a>,
-        specializations_we_must_make: Vec<ExternalSpecializations<'a>>,
+        specializations_we_must_make: Vec<ExternalSpecializations>,
         module_timing: ModuleTiming,
     },
 }
@@ -1538,7 +1534,6 @@ where
                 unsolved_modules: MutMap::default(),
                 timings: MutMap::default(),
                 needs_specialization: MutSet::default(),
-                all_pending_specializations: MutMap::default(),
                 specializations_in_flight: 0,
                 layout_caches: std::vec::Vec::with_capacity(num_cpus::get()),
                 procs: Procs::new_in(arena),
@@ -2067,17 +2062,6 @@ fn update<'a>(
             log!("found specializations for {:?}", module_id);
             let subs = solved_subs.into_inner();
 
-            for (symbol, specs) in &procs_base.specializations_for_host {
-                let existing = match state.all_pending_specializations.entry(*symbol) {
-                    Vacant(entry) => entry.insert(MutMap::default()),
-                    Occupied(entry) => entry.into_mut(),
-                };
-
-                for (layout, pend) in specs {
-                    existing.insert(*layout, pend.clone());
-                }
-            }
-
             state
                 .module_cache
                 .top_level_thunks
@@ -3936,7 +3920,7 @@ fn make_specializations<'a>(
     mut subs: Subs,
     procs_base: ProcsBase<'a>,
     mut layout_cache: LayoutCache<'a>,
-    specializations_we_must_make: Vec<ExternalSpecializations<'a>>,
+    specializations_we_must_make: Vec<ExternalSpecializations>,
     mut module_timing: ModuleTiming,
     ptr_bytes: u32,
 ) -> Msg<'a> {
@@ -3972,7 +3956,7 @@ fn make_specializations<'a>(
         &mut mono_env,
         procs,
         specializations_we_must_make,
-        procs_base.specializations_for_host,
+        procs_base.host_specializations,
         &mut layout_cache,
     );
 
@@ -4004,27 +3988,11 @@ struct ProcsBase<'a> {
     partial_procs: BumpMap<Symbol, PartialProc<'a>>,
     module_thunks: &'a [Symbol],
     /// A host-exposed function must be specialized; it's a seed for subsequent specializations
-    specializations_for_host: BumpMap<Symbol, MutMap<ProcLayout<'a>, PendingSpecialization<'a>>>,
+    host_specializations: roc_mono::ir::HostSpecializations,
     runtime_errors: BumpMap<Symbol, &'a str>,
     imported_module_thunks: &'a [Symbol],
 }
 
-impl<'a> ProcsBase<'a> {
-    fn add_specialization_for_host(
-        &mut self,
-        symbol: Symbol,
-        layout: ProcLayout<'a>,
-        pending: PendingSpecialization<'a>,
-    ) {
-        let all_pending = self
-            .specializations_for_host
-            .entry(symbol)
-            .or_insert_with(|| HashMap::with_capacity_and_hasher(1, default_hasher()));
-
-        all_pending.insert(layout, pending);
-    }
-}
-
 #[allow(clippy::too_many_arguments)]
 fn build_pending_specializations<'a>(
     arena: &'a Bump,
@@ -4046,7 +4014,7 @@ fn build_pending_specializations<'a>(
     let mut procs_base = ProcsBase {
         partial_procs: BumpMap::default(),
         module_thunks: &[],
-        specializations_for_host: BumpMap::default(),
+        host_specializations: roc_mono::ir::HostSpecializations::new(),
         runtime_errors: BumpMap::default(),
         imported_module_thunks,
     };
@@ -4134,7 +4102,7 @@ fn add_def_to_module<'a>(
 
     match def.loc_pattern.value {
         Identifier(symbol) => {
-            let is_exposed = exposed_to_host.contains_key(&symbol);
+            let is_host_exposed = exposed_to_host.contains_key(&symbol);
 
             match def.loc_expr.value {
                 Closure(ClosureData {
@@ -4152,19 +4120,19 @@ fn add_def_to_module<'a>(
                     // register it as such. Otherwise, since it
                     // never gets called by Roc code, it will never
                     // get specialized!
-                    if is_exposed {
+                    if is_host_exposed {
-                        let layout = match layout_cache.raw_from_var(
+                        let layout_result =
-                            mono_env.arena,
+                            layout_cache.raw_from_var(mono_env.arena, annotation, mono_env.subs);
-                            annotation,
+
-                            mono_env.subs,
+                        // cannot specialize when e.g. main's type contains type variables
-                        ) {
+                        if let Err(e) = layout_result {
-                            Ok(l) => l,
+                            match e {
-                            Err(LayoutProblem::Erroneous) => {
+                                LayoutProblem::Erroneous => {
                                     let message = "top level function has erroneous type";
                                     procs.runtime_errors.insert(symbol, message);
                                     return;
                                 }
-                            Err(LayoutProblem::UnresolvedTypeVar(v)) => {
+                                LayoutProblem::UnresolvedTypeVar(v) => {
                                     let message = format!(
                                         "top level function has unresolved type variable {:?}",
                                         v
@@ -4174,20 +4142,15 @@ fn add_def_to_module<'a>(
                                         .insert(symbol, mono_env.arena.alloc(message));
                                     return;
                                 }
-                        };
+                            }
+                        }
 
-                        let pending = PendingSpecialization::from_exposed_function(
+                        procs.host_specializations.insert_host_exposed(
-                            mono_env.arena,
                             mono_env.subs,
+                            symbol,
                             def.annotation,
                             annotation,
                         );
-
-                        procs.add_specialization_for_host(
-                            symbol,
-                            ProcLayout::from_raw(mono_env.arena, layout),
-                            pending,
-                        );
                     }
 
                     let partial_proc = PartialProc::from_named_function(
@@ -4207,28 +4170,25 @@ fn add_def_to_module<'a>(
                     // mark this symbols as a top-level thunk before any other work on the procs
                     module_thunks.push(symbol);
 
+                    let annotation = def.expr_var;
+
                     // If this is an exposed symbol, we need to
                     // register it as such. Otherwise, since it
                     // never gets called by Roc code, it will never
                     // get specialized!
-                    if is_exposed {
+                    if is_host_exposed {
-                        let annotation = def.expr_var;
+                        let layout_result =
+                            layout_cache.raw_from_var(mono_env.arena, annotation, mono_env.subs);
 
-                        let top_level = match layout_cache.from_var(
+                        // cannot specialize when e.g. main's type contains type variables
-                            mono_env.arena,
+                        if let Err(e) = layout_result {
-                            annotation,
+                            match e {
-                            mono_env.subs,
+                                LayoutProblem::Erroneous => {
-                        ) {
-                            Ok(l) => {
-                                // remember, this is a 0-argument thunk
-                                ProcLayout::new(mono_env.arena, &[], l)
-                            }
-                            Err(LayoutProblem::Erroneous) => {
                                     let message = "top level function has erroneous type";
                                     procs.runtime_errors.insert(symbol, message);
                                     return;
                                 }
-                            Err(LayoutProblem::UnresolvedTypeVar(v)) => {
+                                LayoutProblem::UnresolvedTypeVar(v) => {
                                     let message = format!(
                                         "top level function has unresolved type variable {:?}",
                                         v
@@ -4238,20 +4198,19 @@ fn add_def_to_module<'a>(
                                         .insert(symbol, mono_env.arena.alloc(message));
                                     return;
                                 }
-                        };
+                            }
+                        }
 
-                        let pending = PendingSpecialization::from_exposed_function(
+                        procs.host_specializations.insert_host_exposed(
-                            mono_env.arena,
                             mono_env.subs,
+                            symbol,
                             def.annotation,
                             annotation,
                         );
-
-                        procs.add_specialization_for_host(symbol, top_level, pending);
                     }
 
                     let proc = PartialProc {
-                        annotation: def.expr_var,
+                        annotation,
                         // This is a 0-arity thunk, so it has no arguments.
                         pattern_symbols: &[],
                         // This is a top-level definition, so it cannot capture anything

compiler/mono/src/ir.rs

@@ -16,7 +16,6 @@ use roc_module::symbol::{IdentIds, ModuleId, Symbol};
 use roc_problem::can::RuntimeError;
 use roc_region::all::{Located, Region};
 use roc_std::RocDec;
-use roc_types::solved_types::SolvedType;
 use roc_types::subs::{Content, FlatType, StorageSubs, Subs, Variable, VariableSubsSlice};
 use std::collections::HashMap;
 use ven_pretty::{BoxAllocator, DocAllocator, DocBuilder};
@@ -223,65 +222,6 @@ impl<'a> Default for CapturedSymbols<'a> {
     }
 }
 
-#[derive(Clone, Debug)]
-pub struct PendingSpecialization<'a> {
-    solved_type: SolvedType,
-    host_exposed_aliases: BumpMap<Symbol, SolvedType>,
-    _lifetime: std::marker::PhantomData<&'a u8>,
-}
-
-impl<'a> PendingSpecialization<'a> {
-    pub fn from_var(arena: &'a Bump, subs: &Subs, var: Variable) -> Self {
-        let solved_type = SolvedType::from_var(subs, var);
-        PendingSpecialization {
-            solved_type,
-            host_exposed_aliases: BumpMap::new_in(arena),
-            _lifetime: std::marker::PhantomData,
-        }
-    }
-
-    pub fn from_var_host_exposed(
-        arena: &'a Bump,
-        subs: &Subs,
-        var: Variable,
-        exposed: &MutMap<Symbol, Variable>,
-    ) -> Self {
-        let solved_type = SolvedType::from_var(subs, var);
-
-        let mut host_exposed_aliases = BumpMap::with_capacity_in(exposed.len(), arena);
-
-        host_exposed_aliases.extend(
-            exposed
-                .iter()
-                .map(|(symbol, variable)| (*symbol, SolvedType::from_var(subs, *variable))),
-        );
-
-        PendingSpecialization {
-            solved_type,
-            host_exposed_aliases,
-            _lifetime: std::marker::PhantomData,
-        }
-    }
-
-    /// Add a named function that will be publicly exposed to the host
-    pub fn from_exposed_function(
-        arena: &'a Bump,
-        subs: &Subs,
-        opt_annotation: Option<roc_can::def::Annotation>,
-        fn_var: Variable,
-    ) -> Self {
-        match opt_annotation {
-            None => PendingSpecialization::from_var(arena, subs, fn_var),
-            Some(annotation) => PendingSpecialization::from_var_host_exposed(
-                arena,
-                subs,
-                fn_var,
-                &annotation.introduced_variables.host_exposed_aliases,
-            ),
-        }
-    }
-}
-
 #[derive(Clone, Debug, PartialEq)]
 pub struct Proc<'a> {
     pub name: Symbol,
@@ -415,24 +355,108 @@ impl<'a> Proc<'a> {
     }
 }
 
+/// A host-exposed function must be specialized; it's a seed for subsequent specializations
 #[derive(Clone, Debug)]
-pub struct ExternalSpecializations<'a> {
+pub struct HostSpecializations {
+    /// Not a bumpalo vec because bumpalo is not thread safe
+    /// Separate array so we can search for membership quickly
+    symbols: std::vec::Vec<Symbol>,
+    storage_subs: StorageSubs,
+    /// For each symbol, what types to specialize it for, points into the storage_subs
+    types_to_specialize: std::vec::Vec<Variable>,
+    /// Variables for an exposed alias
+    exposed_aliases: std::vec::Vec<std::vec::Vec<(Symbol, Variable)>>,
+}
+
+impl Default for HostSpecializations {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl HostSpecializations {
+    pub fn new() -> Self {
+        Self {
+            symbols: std::vec::Vec::new(),
+            storage_subs: StorageSubs::new(Subs::default()),
+            types_to_specialize: std::vec::Vec::new(),
+            exposed_aliases: std::vec::Vec::new(),
+        }
+    }
+
+    pub fn insert_host_exposed(
+        &mut self,
+        env_subs: &mut Subs,
+        symbol: Symbol,
+        opt_annotation: Option<roc_can::def::Annotation>,
+        variable: Variable,
+    ) {
+        let variable = self.storage_subs.extend_with_variable(env_subs, variable);
+
+        let mut host_exposed_aliases = std::vec::Vec::new();
+
+        if let Some(annotation) = opt_annotation {
+            host_exposed_aliases.extend(annotation.introduced_variables.host_exposed_aliases);
+        }
+
+        match self.symbols.iter().position(|s| *s == symbol) {
+            None => {
+                self.symbols.push(symbol);
+                self.types_to_specialize.push(variable);
+                self.exposed_aliases.push(host_exposed_aliases);
+            }
+            Some(_) => {
+                // we assume that only one specialization of a function is directly exposed to the
+                // host. Other host-exposed symbols may (transitively) specialize this symbol,
+                // but then the existing specialization mechanism will find those specializations
+                panic!("A host-exposed symbol can only be exposed once");
+            }
+        }
+
+        debug_assert_eq!(self.types_to_specialize.len(), self.exposed_aliases.len());
+    }
+
+    fn decompose(
+        self,
+    ) -> (
+        StorageSubs,
+        impl Iterator<Item = (Symbol, Variable, std::vec::Vec<(Symbol, Variable)>)>,
+    ) {
+        let it1 = self.symbols.into_iter();
+
+        let it2 = self.types_to_specialize.into_iter();
+        let it3 = self.exposed_aliases.into_iter();
+
+        (
+            self.storage_subs,
+            it1.zip(it2).zip(it3).map(|((a, b), c)| (a, b, c)),
+        )
+    }
+}
+
+/// Specializations of this module's symbols that other modules need
+#[derive(Clone, Debug)]
+pub struct ExternalSpecializations {
     /// Not a bumpalo vec because bumpalo is not thread safe
     /// Separate array so we can search for membership quickly
     symbols: std::vec::Vec<Symbol>,
     storage_subs: StorageSubs,
     /// For each symbol, what types to specialize it for, points into the storage_subs
     types_to_specialize: std::vec::Vec<std::vec::Vec<Variable>>,
-    _lifetime: std::marker::PhantomData<&'a u8>,
 }
 
-impl<'a> ExternalSpecializations<'a> {
+impl Default for ExternalSpecializations {
-    pub fn new_in(_arena: &'a Bump) -> Self {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl ExternalSpecializations {
+    pub fn new() -> Self {
         Self {
             symbols: std::vec::Vec::new(),
             storage_subs: StorageSubs::new(Subs::default()),
             types_to_specialize: std::vec::Vec::new(),
-            _lifetime: std::marker::PhantomData,
         }
     }
 
@@ -637,7 +661,7 @@ pub struct Procs<'a> {
     pending_specializations: PendingSpecializations<'a>,
     specialized: Specialized<'a>,
     pub runtime_errors: BumpMap<Symbol, &'a str>,
-    pub externals_we_need: BumpMap<ModuleId, ExternalSpecializations<'a>>,
+    pub externals_we_need: BumpMap<ModuleId, ExternalSpecializations>,
 }
 
 impl<'a> Procs<'a> {
@@ -803,7 +827,7 @@ impl<'a> Procs<'a> {
                                 symbol,
                                 layout_cache,
                                 annotation,
-                                BumpMap::new_in(env.arena),
+                                &[],
                                 partial_proc_id,
                             ) {
                                 Ok((proc, layout)) => {
@@ -1890,25 +1914,12 @@ fn pattern_to_when<'a>(
     }
 }
 
-pub fn specialize_all<'a>(
+fn specialize_suspended<'a>(
     env: &mut Env<'a, '_>,
-    mut procs: Procs<'a>,
+    procs: &mut Procs<'a>,
-    externals_others_need: std::vec::Vec<ExternalSpecializations<'a>>,
-    specializations_for_host: BumpMap<Symbol, MutMap<ProcLayout<'a>, PendingSpecialization<'a>>>,
     layout_cache: &mut LayoutCache<'a>,
-) -> Procs<'a> {
+    suspended: Suspended<'a>,
-    specialize_externals_others_need(env, &mut procs, externals_others_need, layout_cache);
+) {
-
-    // When calling from_can, pending_specializations should be unavailable.
-    // This must be a single pass, and we must not add any more entries to it!
-    let pending_specializations = std::mem::replace(
-        &mut procs.pending_specializations,
-        PendingSpecializations::Making,
-    );
-
-    match pending_specializations {
-        PendingSpecializations::Making => {}
-        PendingSpecializations::Finding(suspended) => {
     let offset_variable = StorageSubs::merge_into(suspended.store, env.subs);
 
     for (i, (symbol, var)) in suspended
@@ -1944,15 +1955,7 @@ pub fn specialize_all<'a>(
             }
         };
 
-                match specialize_variable(
+        match specialize_variable(env, procs, name, layout_cache, var, &[], partial_proc) {
-                    env,
-                    &mut procs,
-                    name,
-                    layout_cache,
-                    var,
-                    BumpMap::new_in(env.arena),
-                    partial_proc,
-                ) {
             Ok((proc, layout)) => {
                 // TODO thiscode is duplicated elsewhere
                 let top_level = ProcLayout::from_raw(env.arena, layout);
@@ -1980,91 +1983,97 @@ pub fn specialize_all<'a>(
             }
         }
     }
-        }
+}
+
+pub fn specialize_all<'a>(
+    env: &mut Env<'a, '_>,
+    mut procs: Procs<'a>,
+    externals_others_need: std::vec::Vec<ExternalSpecializations>,
+    specializations_for_host: HostSpecializations,
+    layout_cache: &mut LayoutCache<'a>,
+) -> Procs<'a> {
+    for externals in externals_others_need {
+        specialize_external_specializations(env, &mut procs, layout_cache, externals);
     }
 
-    let it = specializations_for_host.into_iter();
+    // When calling from_can, pending_specializations should be unavailable.
-
+    // This must be a single pass, and we must not add any more entries to it!
-    for (name, by_layout) in it {
+    let pending_specializations = std::mem::replace(
-        for (outside_layout, pending) in by_layout.into_iter() {
+        &mut procs.pending_specializations,
-            // If we've already seen this (Symbol, Layout) combination before,
+        PendingSpecializations::Making,
-            // don't try to specialize it again. If we do, we'll loop forever!
-
-            let partial_proc = if procs.specialized.is_specialized(name, &outside_layout) {
-                // already specialized, just continue
-                continue;
-            } else {
-                match procs.partial_procs.symbol_to_id(name) {
-                    Some(v) => {
-                        // Mark this proc as in-progress, so if we're dealing with
-                        // mutually recursive functions, we don't loop forever.
-                        // (We had a bug around this before this system existed!)
-                        procs.specialized.mark_in_progress(name, outside_layout);
-
-                        v
-                    }
-                    None => {
-                        // TODO this assumes the specialization is done by another module
-                        // make sure this does not become a problem down the road!
-                        continue;
-                    }
-                }
-            };
-
-            match specialize(env, &mut procs, name, layout_cache, pending, partial_proc) {
-                Ok((proc, layout)) => {
-                    // TODO thiscode is duplicated elsewhere
-                    let top_level = ProcLayout::from_raw(env.arena, layout);
-
-                    if procs.is_module_thunk(proc.name) {
-                        debug_assert!(
-                            top_level.arguments.is_empty(),
-                            "{:?} from {:?}",
-                            name,
-                            layout
     );
+
+    match pending_specializations {
+        PendingSpecializations::Making => {}
+        PendingSpecializations::Finding(suspended) => {
+            specialize_suspended(env, &mut procs, layout_cache, suspended)
+        }
     }
 
-                    debug_assert_eq!(outside_layout, top_level, " in {:?}", name);
+    specialize_host_specializations(env, &mut procs, layout_cache, specializations_for_host);
-                    procs.specialized.insert_specialized(name, top_level, proc);
-                }
-                Err(SpecializeFailure {
-                    attempted_layout, ..
-                }) => {
-                    let proc = generate_runtime_error_function(env, name, attempted_layout);
-
-                    let top_level = ProcLayout::from_raw(env.arena, attempted_layout);
-
-                    procs.specialized.insert_specialized(name, top_level, proc);
-                }
-            }
-        }
-    }
 
     procs
 }
 
-fn specialize_externals_others_need<'a>(
+fn specialize_host_specializations<'a>(
     env: &mut Env<'a, '_>,
     procs: &mut Procs<'a>,
-    all_externals_others_need: std::vec::Vec<ExternalSpecializations<'a>>,
     layout_cache: &mut LayoutCache<'a>,
+    host_specializations: HostSpecializations,
+) {
+    let (store, it) = host_specializations.decompose();
+
+    let offset_variable = StorageSubs::merge_into(store, env.subs);
+
+    for (symbol, variable, host_exposed_aliases) in it {
+        specialize_external_help(
+            env,
+            procs,
+            layout_cache,
+            symbol,
+            offset_variable(variable),
+            &host_exposed_aliases,
+        )
+    }
+}
+
+fn specialize_external_specializations<'a>(
+    env: &mut Env<'a, '_>,
+    procs: &mut Procs<'a>,
+    layout_cache: &mut LayoutCache<'a>,
+    externals_others_need: ExternalSpecializations,
 ) {
-    for externals_others_need in all_externals_others_need {
     let (store, it) = externals_others_need.decompose();
 
     let offset_variable = StorageSubs::merge_into(store, env.subs);
 
     for (symbol, solved_types) in it {
         for store_variable in solved_types {
-                let variable = offset_variable(store_variable);
             // historical note: we used to deduplicate with a hash here,
             // but the cost of that hash is very high. So for now we make
             // duplicate specializations, and the insertion into a hash map
             // below will deduplicate them.
 
-                let name = symbol;
+            specialize_external_help(
+                env,
+                procs,
+                layout_cache,
+                symbol,
+                offset_variable(store_variable),
+                &[],
+            )
+        }
+    }
+}
 
+fn specialize_external_help<'a>(
+    env: &mut Env<'a, '_>,
+    procs: &mut Procs<'a>,
+    layout_cache: &mut LayoutCache<'a>,
+    name: Symbol,
+    variable: Variable,
+    host_exposed_aliases: &[(Symbol, Variable)],
+) {
     let partial_proc_id = match procs.partial_procs.symbol_to_id(name) {
         Some(v) => v,
         None => {
@@ -2072,16 +2081,17 @@ fn specialize_externals_others_need<'a>(
         }
     };
 
-                // TODO I believe this is also duplicated
+    let specialization_result = specialize_variable(
-                match specialize_variable(
         env,
         procs,
         name,
         layout_cache,
         variable,
-                    BumpMap::new_in(env.arena),
+        host_exposed_aliases,
         partial_proc_id,
-                ) {
+    );
+
+    match specialization_result {
         Ok((proc, layout)) => {
             let top_level = ProcLayout::from_raw(env.arena, layout);
 
@@ -2102,9 +2112,6 @@ fn specialize_externals_others_need<'a>(
             procs.specialized.insert_specialized(name, top_level, proc);
         }
     }
-            }
-        }
-    }
 }
 
 fn generate_runtime_error_function<'a>(
@@ -2672,77 +2679,13 @@ struct SpecializeFailure<'a> {
 
 type SpecializeSuccess<'a> = (Proc<'a>, RawFunctionLayout<'a>);
 
-fn specialize<'a, 'b>(
-    env: &mut Env<'a, '_>,
-    procs: &'b mut Procs<'a>,
-    proc_name: Symbol,
-    layout_cache: &mut LayoutCache<'a>,
-    pending: PendingSpecialization,
-    partial_proc_id: PartialProcId,
-) -> Result<SpecializeSuccess<'a>, SpecializeFailure<'a>> {
-    let PendingSpecialization {
-        solved_type,
-        host_exposed_aliases,
-        ..
-    } = pending;
-
-    specialize_solved_type(
-        env,
-        procs,
-        proc_name,
-        layout_cache,
-        &solved_type,
-        host_exposed_aliases,
-        partial_proc_id,
-    )
-}
-
-fn introduce_solved_type_to_subs<'a>(env: &mut Env<'a, '_>, solved_type: &SolvedType) -> Variable {
-    use roc_solve::solve::insert_type_into_subs;
-    use roc_types::solved_types::{to_type, FreeVars};
-    use roc_types::subs::VarStore;
-    let mut free_vars = FreeVars::default();
-    let mut var_store = VarStore::new_from_subs(env.subs);
-
-    let before = var_store.peek();
-
-    let normal_type = to_type(solved_type, &mut free_vars, &mut var_store);
-
-    let after = var_store.peek();
-    let variables_introduced = after - before;
-
-    env.subs.extend_by(variables_introduced as usize);
-
-    insert_type_into_subs(env.subs, &normal_type)
-}
-
-fn specialize_solved_type<'a>(
-    env: &mut Env<'a, '_>,
-    procs: &mut Procs<'a>,
-    proc_name: Symbol,
-    layout_cache: &mut LayoutCache<'a>,
-    solved_type: &SolvedType,
-    host_exposed_aliases: BumpMap<Symbol, SolvedType>,
-    partial_proc_id: PartialProcId,
-) -> Result<SpecializeSuccess<'a>, SpecializeFailure<'a>> {
-    specialize_variable_help(
-        env,
-        procs,
-        proc_name,
-        layout_cache,
-        |env| introduce_solved_type_to_subs(env, solved_type),
-        host_exposed_aliases,
-        partial_proc_id,
-    )
-}
-
 fn specialize_variable<'a>(
     env: &mut Env<'a, '_>,
     procs: &mut Procs<'a>,
     proc_name: Symbol,
     layout_cache: &mut LayoutCache<'a>,
     fn_var: Variable,
-    host_exposed_aliases: BumpMap<Symbol, SolvedType>,
+    host_exposed_aliases: &[(Symbol, Variable)],
     partial_proc_id: PartialProcId,
 ) -> Result<SpecializeSuccess<'a>, SpecializeFailure<'a>> {
     specialize_variable_help(
@@ -2762,7 +2705,7 @@ fn specialize_variable_help<'a, F>(
     proc_name: Symbol,
     layout_cache: &mut LayoutCache<'a>,
     fn_var_thunk: F,
-    host_exposed_aliases: BumpMap<Symbol, SolvedType>,
+    host_exposed_variables: &[(Symbol, Variable)],
     partial_proc_id: PartialProcId,
 ) -> Result<SpecializeSuccess<'a>, SpecializeFailure<'a>>
 where
@@ -2797,21 +2740,13 @@ where
     let annotation_var = procs.partial_procs.get_id(partial_proc_id).annotation;
     instantiate_rigids(env.subs, annotation_var);
 
-    let mut host_exposed_variables = Vec::with_capacity_in(host_exposed_aliases.len(), env.arena);
-
-    for (symbol, solved_type) in host_exposed_aliases {
-        let alias_var = introduce_solved_type_to_subs(env, &solved_type);
-
-        host_exposed_variables.push((symbol, alias_var));
-    }
-
     let specialized = specialize_external(
         env,
         procs,
         proc_name,
         layout_cache,
         fn_var,
-        &host_exposed_variables,
+        host_exposed_variables,
         partial_proc_id,
     );
 
@@ -6552,7 +6487,7 @@ fn add_needed_external<'a>(
     use hashbrown::hash_map::Entry::{Occupied, Vacant};
 
     let existing = match procs.externals_we_need.entry(name.module_id()) {
-        Vacant(entry) => entry.insert(ExternalSpecializations::new_in(env.arena)),
+        Vacant(entry) => entry.insert(ExternalSpecializations::new()),
         Occupied(entry) => entry.into_mut(),
     };
 
@@ -6915,7 +6850,7 @@ fn call_by_name_help<'a>(
                             proc_name,
                             layout_cache,
                             fn_var,
-                            BumpMap::new_in(env.arena),
+                            &[],
                             partial_proc,
                         ) {
                             Ok((proc, layout)) => {
@@ -7037,7 +6972,7 @@ fn call_by_name_module_thunk<'a>(
                             proc_name,
                             layout_cache,
                             fn_var,
-                            BumpMap::new_in(env.arena),
+                            &[],
                             partial_proc,
                         ) {
                             Ok((proc, raw_layout)) => {

compiler/solve/Cargo.toml

@@ -12,6 +12,7 @@ roc_module = { path = "../module" }
 roc_types = { path = "../types" }
 roc_can = { path = "../can" }
 roc_unify = { path = "../unify" }
+bumpalo = { version = "3.8.0", features = ["collections"] }
 
 [dev-dependencies]
 roc_load = { path = "../load" }

compiler/solve/src/solve.rs

@@ -619,10 +619,13 @@ fn type_to_var(
     subs: &mut Subs,
     rank: Rank,
     pools: &mut Pools,
-    cached: &mut MutMap<Symbol, Variable>,
+    _: &mut MutMap<Symbol, Variable>,
     typ: &Type,
 ) -> Variable {
-    type_to_variable(subs, rank, pools, cached, typ)
+    // capacity based on the false hello world program
+    let arena = bumpalo::Bump::with_capacity(4 * 1024);
+
+    type_to_variable(subs, rank, pools, &arena, typ)
 }
 
 /// Abusing existing functions for our purposes
@@ -630,25 +633,29 @@ fn type_to_var(
 pub fn insert_type_into_subs(subs: &mut Subs, typ: &Type) -> Variable {
     let rank = Rank::NONE;
     let mut pools = Pools::default();
-    let mut cached = MutMap::default();
 
-    type_to_variable(subs, rank, &mut pools, &mut cached, typ)
+    // capacity based on the false hello world program
+    let arena = bumpalo::Bump::with_capacity(4 * 1024);
+
+    type_to_variable(subs, rank, &mut pools, &arena, typ)
 }
 
-fn type_to_variable(
+fn type_to_variable<'a>(
     subs: &mut Subs,
     rank: Rank,
     pools: &mut Pools,
-    cached: &mut MutMap<Symbol, Variable>,
+    arena: &'a bumpalo::Bump,
     typ: &Type,
 ) -> Variable {
+    use bumpalo::collections::Vec;
+
     match typ {
         Variable(var) => *var,
         Apply(symbol, args) => {
-            let mut new_arg_vars = Vec::with_capacity(args.len());
+            let mut new_arg_vars = Vec::with_capacity_in(args.len(), arena);
 
             for arg in args {
-                let var = type_to_variable(subs, rank, pools, cached, arg);
+                let var = type_to_variable(subs, rank, pools, arena, arg);
                 new_arg_vars.push(var);
             }
 
@@ -663,32 +670,32 @@ fn type_to_variable(
 
         // This case is important for the rank of boolean variables
         Function(arg_vars, closure_type, ret_type) => {
-            let mut new_arg_vars = Vec::with_capacity(arg_vars.len());
+            let mut new_arg_vars = Vec::with_capacity_in(arg_vars.len(), arena);
 
             for arg in arg_vars {
-                let var = type_to_variable(subs, rank, pools, cached, arg);
+                let var = type_to_variable(subs, rank, pools, arena, arg);
                 new_arg_vars.push(var);
             }
 
             let arg_vars = VariableSubsSlice::insert_into_subs(subs, new_arg_vars);
 
-            let ret_var = type_to_variable(subs, rank, pools, cached, ret_type);
+            let ret_var = type_to_variable(subs, rank, pools, arena, ret_type);
-            let closure_var = type_to_variable(subs, rank, pools, cached, closure_type);
+            let closure_var = type_to_variable(subs, rank, pools, arena, closure_type);
             let content = Content::Structure(FlatType::Func(arg_vars, closure_var, ret_var));
 
             register(subs, rank, pools, content)
         }
         Record(fields, ext) => {
-            let mut field_vars = Vec::with_capacity(fields.len());
+            let mut field_vars = Vec::with_capacity_in(fields.len(), arena);
 
             for (field, field_type) in fields {
                 let field_var =
-                    field_type.map(|typ| type_to_variable(subs, rank, pools, cached, typ));
+                    field_type.map(|typ| type_to_variable(subs, rank, pools, arena, typ));
 
                 field_vars.push((field.clone(), field_var));
             }
 
-            let temp_ext_var = type_to_variable(subs, rank, pools, cached, ext);
+            let temp_ext_var = type_to_variable(subs, rank, pools, arena, ext);
 
             let (it, new_ext_var) =
                 gather_fields_unsorted_iter(subs, RecordFields::empty(), temp_ext_var);
@@ -707,14 +714,14 @@ fn type_to_variable(
             register(subs, rank, pools, content)
         }
         TagUnion(tags, ext) => {
-            let (union_tags, ext) = type_to_union_tags(subs, rank, pools, cached, tags, ext);
+            let (union_tags, ext) = type_to_union_tags(subs, rank, pools, arena, tags, ext);
             let content = Content::Structure(FlatType::TagUnion(union_tags, ext));
 
             register(subs, rank, pools, content)
         }
         FunctionOrTagUnion(tag_name, symbol, ext) => {
-            let temp_ext_var = type_to_variable(subs, rank, pools, cached, ext);
+            let temp_ext_var = type_to_variable(subs, rank, pools, arena, ext);
-            let mut ext_tag_vec = Vec::new();
+            let mut ext_tag_vec = std::vec::Vec::new();
             let new_ext_var = match roc_types::pretty_print::chase_ext_tag_union(
                 subs,
                 temp_ext_var,
@@ -735,7 +742,7 @@ fn type_to_variable(
             register(subs, rank, pools, content)
         }
         RecursiveTagUnion(rec_var, tags, ext) => {
-            let (union_tags, ext) = type_to_union_tags(subs, rank, pools, cached, tags, ext);
+            let (union_tags, ext) = type_to_union_tags(subs, rank, pools, arena, tags, ext);
             let content =
                 Content::Structure(FlatType::RecursiveTagUnion(*rec_var, union_tags, ext));
 
@@ -780,22 +787,22 @@ fn type_to_variable(
                 }
             }
 
-            let mut arg_vars = Vec::with_capacity(args.len());
+            let mut arg_vars = Vec::with_capacity_in(args.len(), arena);
 
             for (arg, arg_type) in args {
-                let arg_var = type_to_variable(subs, rank, pools, cached, arg_type);
+                let arg_var = type_to_variable(subs, rank, pools, arena, arg_type);
 
                 arg_vars.push((arg.clone(), arg_var));
             }
 
-            let lambda_set_variables: Vec<_> = lambda_set_variables
+            let lambda_set_variables_it = lambda_set_variables
                 .iter()
-                .map(|ls| type_to_variable(subs, rank, pools, cached, &ls.0))
+                .map(|ls| type_to_variable(subs, rank, pools, arena, &ls.0));
-                .collect();
+            let lambda_set_variables = Vec::from_iter_in(lambda_set_variables_it, arena);
 
             let arg_vars = AliasVariables::insert_into_subs(subs, arg_vars, lambda_set_variables);
 
-            let alias_var = type_to_variable(subs, rank, pools, cached, alias_type);
+            let alias_var = type_to_variable(subs, rank, pools, arena, alias_type);
             let content = Content::Alias(*symbol, arg_vars, alias_var);
 
             register(subs, rank, pools, content)
@@ -808,22 +815,22 @@ fn type_to_variable(
             lambda_set_variables,
             ..
         } => {
-            let mut arg_vars = Vec::with_capacity(args.len());
+            let mut arg_vars = Vec::with_capacity_in(args.len(), arena);
 
             for (arg, arg_type) in args {
-                let arg_var = type_to_variable(subs, rank, pools, cached, arg_type);
+                let arg_var = type_to_variable(subs, rank, pools, arena, arg_type);
 
                 arg_vars.push((arg.clone(), arg_var));
             }
 
-            let lambda_set_variables: Vec<_> = lambda_set_variables
+            let lambda_set_variables_it = lambda_set_variables
                 .iter()
-                .map(|ls| type_to_variable(subs, rank, pools, cached, &ls.0))
+                .map(|ls| type_to_variable(subs, rank, pools, arena, &ls.0));
-                .collect();
+            let lambda_set_variables = Vec::from_iter_in(lambda_set_variables_it, arena);
 
             let arg_vars = AliasVariables::insert_into_subs(subs, arg_vars, lambda_set_variables);
 
-            let alias_var = type_to_variable(subs, rank, pools, cached, alias_type);
+            let alias_var = type_to_variable(subs, rank, pools, arena, alias_type);
 
             // unify the actual_var with the result var
             // this can be used to access the type of the actual_var
@@ -853,20 +860,22 @@ fn type_to_variable(
     }
 }
 
-fn type_to_union_tags(
+fn type_to_union_tags<'a>(
     subs: &mut Subs,
     rank: Rank,
     pools: &mut Pools,
-    cached: &mut MutMap<Symbol, Variable>,
+    arena: &'a bumpalo::Bump,
     tags: &[(TagName, Vec<Type>)],
     ext: &Type,
 ) -> (UnionTags, Variable) {
-    let mut tag_vars = Vec::with_capacity(tags.len());
+    use bumpalo::collections::Vec;
 
-    let mut tag_argument_vars = Vec::new();
+    let mut tag_vars = Vec::with_capacity_in(tags.len(), arena);
+
+    let mut tag_argument_vars = Vec::with_capacity_in(tags.len(), arena);
     for (tag, tag_argument_types) in tags {
         for arg_type in tag_argument_types {
-            let new_var = type_to_variable(subs, rank, pools, cached, arg_type);
+            let new_var = type_to_variable(subs, rank, pools, arena, arg_type);
             tag_argument_vars.push(new_var);
         }
 
@@ -875,7 +884,7 @@ fn type_to_union_tags(
         tag_vars.push((tag.clone(), new_slice));
     }
 
-    let temp_ext_var = type_to_variable(subs, rank, pools, cached, ext);
+    let temp_ext_var = type_to_variable(subs, rank, pools, arena, ext);
 
     let ext = {
         let (it, ext) =
@@ -1031,8 +1040,7 @@ fn pool_to_rank_table(
 
     // Sort the variables into buckets by rank.
     for &var in young_vars.iter() {
-        let rank = subs.get_rank(var);
+        let rank = subs.get_rank_set_mark(var, young_mark);
-        subs.set_mark(var, young_mark);
 
         debug_assert!(rank.into_usize() < young_rank.into_usize() + 1);
         pools.get_mut(rank).push(var);
@@ -1248,125 +1256,131 @@ pub fn instantiate_rigids(subs: &mut Subs, var: Variable) {
 
     instantiate_rigids_help(subs, rank, var);
 
-    subs.restore(var);
+    // NOTE subs.restore(var) is done at the end of instantiate_rigids_help
 }
 
-fn instantiate_rigids_help(subs: &mut Subs, max_rank: Rank, var: Variable) {
+fn instantiate_rigids_help(subs: &mut Subs, max_rank: Rank, initial: Variable) {
-    use roc_types::subs::Content::*;
+    let mut visited = vec![];
-    use roc_types::subs::FlatType::*;
+    let mut stack = vec![initial];
 
-    let desc = subs.get_without_compacting(var);
+    macro_rules! var_slice {
+        ($variable_subs_slice:expr) => {{
+            let slice = $variable_subs_slice;
+            &subs.variables[slice.slice.start as usize..][..slice.slice.length as usize]
+        }};
+    }
 
+    while let Some(var) = stack.pop() {
+        visited.push(var);
+
+        let desc = subs.get_ref_mut(var);
         if desc.copy.is_some() {
-        return;
+            continue;
         }
 
-    // Link the original variable to the new variable. This lets us
+        desc.rank = Rank::NONE;
-    // avoid making multiple copies of the variable we are instantiating.
+        desc.mark = Mark::NONE;
-    //
+        desc.copy = OptVariable::from(var);
-    // Need to do this before recursively copying to avoid looping.
-    subs.set(
-        var,
-        Descriptor {
-            content: desc.content.clone(),
-            rank: desc.rank,
-            mark: Mark::NONE,
-            copy: var.into(),
-        },
-    );
 
-    // Now we recursively copy the content of the variable.
+        use Content::*;
-    // We have already marked the variable as copied, so we
+        use FlatType::*;
-    // will not repeat this work or crawl this variable again.
+
-    match desc.content {
+        match &desc.content {
-        Structure(flat_type) => {
+            RigidVar(name) => {
-            match flat_type {
+                // what it's all about: convert the rigid var into a flex var
+                let name = name.clone();
+
+                // NOTE: we must write to the mutually borrowed `desc` value here
+                // using `subs.set` does not work (unclear why, really)
+                // but get_ref_mut approach saves a lookup, so the weirdness is worth it
+                desc.content = FlexVar(Some(name));
+                desc.rank = max_rank;
+                desc.mark = Mark::NONE;
+                desc.copy = OptVariable::NONE;
+            }
+            FlexVar(_) | Error => (),
+
+            RecursionVar { structure, .. } => {
+                stack.push(*structure);
+            }
+
+            Structure(flat_type) => match flat_type {
                 Apply(_, args) => {
-                    for var_index in args.into_iter() {
+                    stack.extend(var_slice!(*args));
-                        let var = subs[var_index];
-                        instantiate_rigids_help(subs, max_rank, var);
-                    }
                 }
 
                 Func(arg_vars, closure_var, ret_var) => {
-                    instantiate_rigids_help(subs, max_rank, ret_var);
+                    let arg_vars = *arg_vars;
-                    instantiate_rigids_help(subs, max_rank, closure_var);
+                    let ret_var = *ret_var;
+                    let closure_var = *closure_var;
 
-                    for index in arg_vars.into_iter() {
+                    stack.extend(var_slice!(arg_vars));
-                        let var = subs[index];
+
-                        instantiate_rigids_help(subs, max_rank, var);
+                    stack.push(ret_var);
-                    }
+                    stack.push(closure_var);
                 }
 
-                EmptyRecord | EmptyTagUnion | Erroneous(_) => {}
+                EmptyRecord => (),
+                EmptyTagUnion => (),
 
                 Record(fields, ext_var) => {
-                    for index in fields.iter_variables() {
+                    let fields = *fields;
-                        let var = subs[index];
+                    let ext_var = *ext_var;
-                        instantiate_rigids_help(subs, max_rank, var);
+                    stack.extend(var_slice!(fields.variables()));
-                    }
 
-                    instantiate_rigids_help(subs, max_rank, ext_var);
+                    stack.push(ext_var);
                 }
-
                 TagUnion(tags, ext_var) => {
-                    for (_, index) in tags.iter_all() {
+                    let tags = *tags;
-                        let slice = subs[index];
+                    let ext_var = *ext_var;
-                        for var_index in slice {
+
-                            let var = subs[var_index];
+                    for slice_index in tags.variables() {
-                            instantiate_rigids_help(subs, max_rank, var);
+                        let slice = subs.variable_slices[slice_index.start as usize];
-                        }
+                        stack.extend(var_slice!(slice));
                     }
 
-                    instantiate_rigids_help(subs, max_rank, ext_var);
+                    stack.push(ext_var);
                 }
-
+                FunctionOrTagUnion(_, _, ext_var) => {
-                FunctionOrTagUnion(_tag_name, _symbol, ext_var) => {
+                    stack.push(*ext_var);
-                    instantiate_rigids_help(subs, max_rank, ext_var);
                 }
 
                 RecursiveTagUnion(rec_var, tags, ext_var) => {
-                    instantiate_rigids_help(subs, max_rank, rec_var);
+                    let tags = *tags;
+                    let ext_var = *ext_var;
+                    let rec_var = *rec_var;
 
-                    for (_, index) in tags.iter_all() {
+                    for slice_index in tags.variables() {
-                        let slice = subs[index];
+                        let slice = subs.variable_slices[slice_index.start as usize];
-                        for var_index in slice {
+                        stack.extend(var_slice!(slice));
-                            let var = subs[var_index];
-                            instantiate_rigids_help(subs, max_rank, var);
-                        }
                     }
 
-                    instantiate_rigids_help(subs, max_rank, ext_var);
+                    stack.push(ext_var);
-                }
+                    stack.push(rec_var);
-            };
                 }
 
-        FlexVar(_) | Error => {}
+                Erroneous(_) => (),
-
-        RecursionVar { structure, .. } => {
-            instantiate_rigids_help(subs, max_rank, structure);
-        }
-
-        RigidVar(name) => {
-            // what it's all about: convert the rigid var into a flex var
-            subs.set(
-                var,
-                Descriptor {
-                    content: FlexVar(Some(name)),
-                    rank: max_rank,
-                    mark: Mark::NONE,
-                    copy: OptVariable::NONE,
             },
-            );
+            Alias(_, args, var) => {
+                let var = *var;
+                let args = *args;
+
+                stack.extend(var_slice!(args.variables()));
+
+                stack.push(var);
+            }
+        }
     }
 
-        Alias(_symbol, args, real_type_var) => {
+    // we have tracked all visited variables, and can now traverse them
-            for var_index in args.variables().into_iter() {
+    // in one go (without looking at the UnificationTable) and clear the copy field
-                let var = subs[var_index];
+    for var in visited {
-                instantiate_rigids_help(subs, max_rank, var);
+        let descriptor = subs.get_ref_mut(var);
-            }
 
-            instantiate_rigids_help(subs, max_rank, real_type_var);
+        if descriptor.copy.is_some() {
+            descriptor.rank = Rank::NONE;
+            descriptor.mark = Mark::NONE;
+            descriptor.copy = OptVariable::NONE;
         }
     }
 }

compiler/test_gen/src/gen_num.rs

@@ -726,6 +726,20 @@ fn gen_int_neq() {
     );
 }
 
+#[test]
+#[cfg(any(feature = "gen-llvm", feature = "gen-wasm", feature = "gen-dev"))]
+fn gen_int_less_than() {
+    assert_evals_to!(
+        indoc!(
+            r#"
+                    4 < 5
+                "#
+        ),
+        true,
+        bool
+    );
+}
+
 #[test]
 #[cfg(any(feature = "gen-llvm"))]
 fn gen_dec_eq() {

compiler/types/Cargo.toml

@@ -10,4 +10,5 @@ roc_collections = { path = "../collections" }
 roc_region = { path = "../region" }
 roc_module = { path = "../module" }
 ven_ena = { path = "../../vendor/ena" }
+bumpalo = { version = "3.8.0", features = ["collections"] }
 static_assertions = "1.1.0"

compiler/types/src/subs.rs

@@ -79,7 +79,7 @@ pub struct SubsSlice<T> {
 
 /// An index into the Vec<T> of subs
 pub struct SubsIndex<T> {
-    start: u32,
+    pub start: u32,
     _marker: std::marker::PhantomData<T>,
 }
 
@@ -1164,6 +1164,26 @@ impl Subs {
         });
     }
 
+    pub fn modify<F>(&mut self, key: Variable, mapper: F)
+    where
+        F: Fn(&mut Descriptor),
+    {
+        mapper(self.get_ref_mut(key));
+    }
+
+    pub fn get_rank_set_mark(&mut self, key: Variable, mark: Mark) -> Rank {
+        let l_key = self.utable.get_root_key(key);
+
+        let mut rank = Rank::NONE;
+
+        self.utable.update_value(l_key, |node| {
+            node.value.mark = mark;
+            rank = node.value.rank;
+        });
+
+        rank
+    }
+
     pub fn equivalent(&mut self, left: Variable, right: Variable) -> bool {
         self.utable.unioned(left, right)
     }
@@ -1501,7 +1521,7 @@ pub enum Builtin {
 
 #[derive(Clone, Copy, Debug, Default)]
 pub struct VariableSubsSlice {
-    slice: SubsSlice<Variable>,
+    pub slice: SubsSlice<Variable>,
 }
 
 impl VariableSubsSlice {
@@ -1825,7 +1845,7 @@ impl RecordFields {
         }
     }
 
-    fn variables(&self) -> VariableSubsSlice {
+    pub const fn variables(&self) -> VariableSubsSlice {
         let slice = SubsSlice::new(self.variables_start, self.length);
 
         VariableSubsSlice { slice }
@@ -2837,7 +2857,7 @@ impl StorageSubs {
         };
 
         let offsets = StorageSubsOffsets {
-            utable: target.utable.len() as u32,
+            utable: (target.utable.len() - Variable::NUM_RESERVED_VARS) as u32,
             variables: target.variables.len() as u32,
             tag_names: target.tag_names.len() as u32,
             field_names: target.field_names.len() as u32,
@@ -2845,7 +2865,9 @@ impl StorageSubs {
             variable_slices: target.variable_slices.len() as u32,
         };
 
-        let range = 0..self.subs.utable.len();
+        // The first Variable::NUM_RESERVED_VARS are the same in every subs,
+        // so we can skip copying them!
+        let range = Variable::NUM_RESERVED_VARS..self.subs.utable.len();
 
         // fill new slots with empty values
         target.extend_by(range.len());
@@ -2860,8 +2882,6 @@ impl StorageSubs {
             let new_descriptor = Descriptor {
                 rank: descriptor.rank,
                 mark: descriptor.mark,
-                // rank: Rank::NONE,
-                // mark: Mark::NONE,
                 copy: OptVariable::NONE,
                 content: new_content,
             };
@@ -2998,9 +3018,13 @@ impl StorageSubs {
     }
 
     fn offset_variable(offsets: &StorageSubsOffsets, variable: Variable) -> Variable {
+        if variable.index() < Variable::FIRST_USER_SPACE_VAR.index() {
+            variable
+        } else {
             let new_index = variable.0 + offsets.utable;
             Variable(new_index)
         }
+    }
 
     fn offset_variable_slice(
         offsets: &StorageSubsOffsets,
@@ -3019,20 +3043,37 @@ pub fn deep_copy_var_to(
 ) -> Variable {
     let rank = Rank::toplevel();
 
-    let copy = deep_copy_var_to_help(source, target, rank, var);
+    // capacity based on the false hello world program
+    let arena = bumpalo::Bump::with_capacity(4 * 1024);
 
-    source.restore(var);
+    let mut visited = bumpalo::collections::Vec::with_capacity_in(256, &arena);
+
+    let copy = deep_copy_var_to_help(&arena, &mut visited, source, target, rank, var);
+
+    // we have tracked all visited variables, and can now traverse them
+    // in one go (without looking at the UnificationTable) and clear the copy field
+    for var in visited {
+        let descriptor = source.get_ref_mut(var);
+
+        if descriptor.copy.is_some() {
+            descriptor.rank = Rank::NONE;
+            descriptor.mark = Mark::NONE;
+            descriptor.copy = OptVariable::NONE;
+        }
+    }
 
     copy
 }
 
-fn deep_copy_var_to_help(
+fn deep_copy_var_to_help<'a>(
-    // source: &mut Subs, // mut to set the copy
+    arena: &'a bumpalo::Bump,
+    visited: &mut bumpalo::collections::Vec<'a, Variable>,
     source: &mut Subs,
     target: &mut Subs,
     max_rank: Rank,
     var: Variable,
 ) -> Variable {
+    use bumpalo::collections::Vec;
     use Content::*;
     use FlatType::*;
 
@@ -3042,7 +3083,7 @@ fn deep_copy_var_to_help(
         debug_assert!(target.contains(copy));
         return copy;
     } else if desc.rank != Rank::NONE {
-        // DO NOTHING
+        // DO NOTHING, Fall through
         //
         // The original deep_copy_var can do
         // return var;
@@ -3051,6 +3092,8 @@ fn deep_copy_var_to_help(
         // should only return variables in the target
     }
 
+    visited.push(var);
+
     let make_descriptor = |content| Descriptor {
         content,
         rank: max_rank,
@@ -3058,58 +3101,57 @@ fn deep_copy_var_to_help(
         copy: OptVariable::NONE,
     };
 
-    let content = desc.content;
-    // let copy = target.fresh(make_descriptor(content.clone()));
     let copy = target.fresh_unnamed_flex_var();
 
-    // pools.get_mut(max_rank).push(copy);
-
     // Link the original variable to the new variable. This lets us
     // avoid making multiple copies of the variable we are instantiating.
     //
     // Need to do this before recursively copying to avoid looping.
-
+    source.modify(var, |descriptor| {
-    source.set(
+        descriptor.mark = Mark::NONE;
-        var,
+        descriptor.copy = copy.into();
-        Descriptor {
+    });
-            content: content.clone(),
-            rank: desc.rank,
-            mark: Mark::NONE,
-            copy: copy.into(),
-        },
-    );
 
     // Now we recursively copy the content of the variable.
     // We have already marked the variable as copied, so we
     // will not repeat this work or crawl this variable again.
-    match content {
+    match desc.content {
         Structure(flat_type) => {
             let new_flat_type = match flat_type {
                 Apply(symbol, args) => {
-                    let mut new_arg_vars = Vec::with_capacity(args.len());
+                    let mut new_args = Vec::with_capacity_in(args.len(), arena);
 
                     for index in args.into_iter() {
                         let var = source[index];
-                        let copy_var = deep_copy_var_to_help(source, target, max_rank, var);
+                        new_args.push(deep_copy_var_to_help(
-                        new_arg_vars.push(copy_var);
+                            arena, visited, source, target, max_rank, var,
+                        ));
                     }
 
-                    let arg_vars = VariableSubsSlice::insert_into_subs(target, new_arg_vars);
+                    let arg_vars = VariableSubsSlice::insert_into_subs(target, new_args);
 
                     Apply(symbol, arg_vars)
                 }
 
                 Func(arg_vars, closure_var, ret_var) => {
-                    let new_ret_var = deep_copy_var_to_help(source, target, max_rank, ret_var);
+                    let new_ret_var =
+                        deep_copy_var_to_help(arena, visited, source, target, max_rank, ret_var);
 
-                    let new_closure_var =
+                    let new_closure_var = deep_copy_var_to_help(
-                        deep_copy_var_to_help(source, target, max_rank, closure_var);
+                        arena,
+                        visited,
+                        source,
+                        target,
+                        max_rank,
+                        closure_var,
+                    );
 
-                    let mut new_arg_vars = Vec::with_capacity(arg_vars.len());
+                    let mut new_arg_vars = Vec::with_capacity_in(arg_vars.len(), arena);
 
                     for index in arg_vars.into_iter() {
                         let var = source[index];
-                        let copy_var = deep_copy_var_to_help(source, target, max_rank, var);
+                        let copy_var =
+                            deep_copy_var_to_help(arena, visited, source, target, max_rank, var);
                         new_arg_vars.push(copy_var);
                     }
 
@@ -3122,11 +3164,13 @@ fn deep_copy_var_to_help(
 
                 Record(fields, ext_var) => {
                     let record_fields = {
-                        let mut new_vars = Vec::with_capacity(fields.len());
+                        let mut new_vars = Vec::with_capacity_in(fields.len(), arena);
 
                         for index in fields.iter_variables() {
                             let var = source[index];
-                            let copy_var = deep_copy_var_to_help(source, target, max_rank, var);
+                            let copy_var = deep_copy_var_to_help(
+                                arena, visited, source, target, max_rank, var,
+                            );
 
                             new_vars.push(copy_var);
                         }
@@ -3157,21 +3201,24 @@ fn deep_copy_var_to_help(
 
                     Record(
                         record_fields,
-                        deep_copy_var_to_help(source, target, max_rank, ext_var),
+                        deep_copy_var_to_help(arena, visited, source, target, max_rank, ext_var),
                     )
                 }
 
                 TagUnion(tags, ext_var) => {
-                    let new_ext = deep_copy_var_to_help(source, target, max_rank, ext_var);
+                    let new_ext =
+                        deep_copy_var_to_help(arena, visited, source, target, max_rank, ext_var);
 
-                    let mut new_variable_slices = Vec::with_capacity(tags.len());
+                    let mut new_variable_slices = Vec::with_capacity_in(tags.len(), arena);
 
-                    let mut new_variables = Vec::new();
+                    let mut new_variables = Vec::with_capacity_in(tags.len(), arena);
                     for index in tags.variables() {
                         let slice = source[index];
                         for var_index in slice {
                             let var = source[var_index];
-                            let new_var = deep_copy_var_to_help(source, target, max_rank, var);
+                            let new_var = deep_copy_var_to_help(
+                                arena, visited, source, target, max_rank, var,
+                            );
                             new_variables.push(new_var);
                         }
 
@@ -3215,19 +3262,21 @@ fn deep_copy_var_to_help(
                     FunctionOrTagUnion(
                         new_tag_name,
                         symbol,
-                        deep_copy_var_to_help(source, target, max_rank, ext_var),
+                        deep_copy_var_to_help(arena, visited, source, target, max_rank, ext_var),
                     )
                 }
 
                 RecursiveTagUnion(rec_var, tags, ext_var) => {
-                    let mut new_variable_slices = Vec::with_capacity(tags.len());
+                    let mut new_variable_slices = Vec::with_capacity_in(tags.len(), arena);
 
-                    let mut new_variables = Vec::new();
+                    let mut new_variables = Vec::with_capacity_in(tags.len(), arena);
                     for index in tags.variables() {
                         let slice = source[index];
                         for var_index in slice {
                             let var = source[var_index];
-                            let new_var = deep_copy_var_to_help(source, target, max_rank, var);
+                            let new_var = deep_copy_var_to_help(
+                                arena, visited, source, target, max_rank, var,
+                            );
                             new_variables.push(new_var);
                         }
 
@@ -3260,8 +3309,10 @@ fn deep_copy_var_to_help(
 
                     let union_tags = UnionTags::from_slices(new_tag_names, new_variables);
 
-                    let new_ext = deep_copy_var_to_help(source, target, max_rank, ext_var);
+                    let new_ext =
-                    let new_rec_var = deep_copy_var_to_help(source, target, max_rank, rec_var);
+                        deep_copy_var_to_help(arena, visited, source, target, max_rank, ext_var);
+                    let new_rec_var =
+                        deep_copy_var_to_help(arena, visited, source, target, max_rank, rec_var);
 
                     RecursiveTagUnion(new_rec_var, union_tags, new_ext)
                 }
@@ -3278,7 +3329,8 @@ fn deep_copy_var_to_help(
             opt_name,
             structure,
         } => {
-            let new_structure = deep_copy_var_to_help(source, target, max_rank, structure);
+            let new_structure =
+                deep_copy_var_to_help(arena, visited, source, target, max_rank, structure);
 
             debug_assert!((new_structure.index() as usize) < target.len());
 
@@ -3300,11 +3352,11 @@ fn deep_copy_var_to_help(
         }
 
         Alias(symbol, mut args, real_type_var) => {
-            let mut new_vars = Vec::with_capacity(args.variables().len());
+            let mut new_vars = Vec::with_capacity_in(args.variables().len(), arena);
 
             for var_index in args.variables() {
                 let var = source[var_index];
-                let new_var = deep_copy_var_to_help(source, target, max_rank, var);
+                let new_var = deep_copy_var_to_help(arena, visited, source, target, max_rank, var);
 
                 new_vars.push(new_var);
             }
@@ -3317,7 +3369,8 @@ fn deep_copy_var_to_help(
             args.lowercases_start = target.field_names.len() as u32;
             target.field_names.extend(lowercases.iter().cloned());
 
-            let new_real_type_var = deep_copy_var_to_help(source, target, max_rank, real_type_var);
+            let new_real_type_var =
+                deep_copy_var_to_help(arena, visited, source, target, max_rank, real_type_var);
             let new_content = Alias(symbol, args, new_real_type_var);
 
             target.set(copy, make_descriptor(new_content));

reporting/src/lib.rs

@@ -4,3 +4,35 @@
 
 pub mod error;
 pub mod report;
+
+/// `internal_error!` should be used whenever a compiler invariant is broken.
+/// It is a wrapper around panic that tells the user to file a bug.
+/// This should only be used in cases where there would be a compiler bug and the user can't fix it.
+/// If there is simply an unimplemented feature, please use `unimplemented!`
+/// If there is a user error, please use roc_reporting to print a nice error message.
+#[macro_export]
+macro_rules! internal_error {
+    ($($arg:tt)*) => ({
+        eprintln!("An internal compiler expectation was broken.");
+        eprintln!("This is definitely a compiler bug.");
+        // TODO: update this to the new bug template.
+        eprintln!("Please file an issue here: https://github.com/rtfeldman/roc/issues/new/choose");
+        #[allow(clippy::panic)] {
+            panic!($($arg)*);
+        }
+    })
+}
+
+/// `user_error!` should only ever be used temporarily.
+/// It is a way to document locations where we do not yet have nice error reporting.
+/// All cases of `user_error!` should eventually be replaced with pretty error printing using roc_reporting.
+#[macro_export]
+macro_rules! user_error {
+    ($($arg:tt)*) => ({
+        eprintln!("We ran into an issue while compiling your code.");
+        eprintln!("Sadly, we don't havs a pretty error message for this case yet.");
+        eprintln!("If you can't figure out the problem from the context below, please reach out at: https://roc.zulipchat.com/");
+        eprintln!($($arg)*);
+        std::process::exit(1);
+    })
+}