Merge remote-tracking branch 'origin/trunk' into refcount

Cargo.lock

@@ -69,6 +69,13 @@ version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034"
 
+[[package]]
+name = "arena-pool"
+version = "0.1.0"
+dependencies = [
+ "pretty_assertions",
+]
+
 [[package]]
 name = "arrayvec"
 version = "0.5.1"
@@ -386,6 +393,20 @@ dependencies = [
  "itertools",
 ]
 
+[[package]]
+name = "crossbeam"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69323bff1fb41c635347b8ead484a5ca6c3f11914d784170b158d8449ab07f8e"
+dependencies = [
+ "cfg-if",
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-epoch",
+ "crossbeam-queue",
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-channel"
 version = "0.4.3"
@@ -527,6 +548,12 @@ version = "1.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
 
+[[package]]
+name = "encode_unicode"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
+
 [[package]]
 name = "env_logger"
 version = "0.6.2"
@@ -2167,9 +2194,11 @@ name = "roc_load"
 version = "0.1.0"
 dependencies = [
  "bumpalo",
+ "crossbeam",
  "indoc",
  "inlinable_string",
  "maplit",
+ "num_cpus",
  "pretty_assertions",
  "quickcheck",
  "quickcheck_macros",
@@ -2184,7 +2213,6 @@ dependencies = [
  "roc_solve",
  "roc_types",
  "roc_unify",
- "tokio",
 ]
 
 [[package]]
@@ -2230,6 +2258,7 @@ name = "roc_parse"
 version = "0.1.0"
 dependencies = [
  "bumpalo",
+ "encode_unicode",
  "indoc",
  "inlinable_string",
  "pretty_assertions",

Cargo.toml

@@ -20,6 +20,7 @@ members = [
     "compiler/load",
     "compiler/gen",
     "compiler/build",
+    "compiler/arena_pool",
     "vendor/ena",
     "vendor/pathfinding",
     "vendor/pretty",

cli/src/main.rs

@@ -2,19 +2,17 @@
 extern crate clap;
 
 use bumpalo::Bump;
+use clap::{App, Arg, ArgMatches};
 use roc_build::program::gen;
 use roc_collections::all::MutMap;
 use roc_gen::llvm::build::OptLevel;
 use roc_load::file::LoadingProblem;
-use std::time::SystemTime;
-
-use clap::{App, Arg, ArgMatches};
 use std::io::{self, ErrorKind};
 use std::path::{Path, PathBuf};
 use std::process;
+use std::process::Command;
+use std::time::{Duration, SystemTime};
 use target_lexicon::Triple;
-use tokio::process::Command;
-use tokio::runtime::Builder;
 
 pub mod repl;
 
@@ -105,14 +103,6 @@ pub fn build(matches: &ArgMatches, run_after_build: bool) -> io::Result<()> {
     let path = Path::new(filename).canonicalize().unwrap();
     let src_dir = path.parent().unwrap().canonicalize().unwrap();
 
-    // Create the runtime
-    let mut rt = Builder::new()
-        .thread_name("roc")
-        .threaded_scheduler()
-        .enable_io()
-        .build()
-        .expect("Error spawning initial compiler thread."); // TODO make this error nicer.
-
     // Spawn the root task
     let path = path.canonicalize().unwrap_or_else(|err| {
         use ErrorKind::*;
@@ -131,28 +121,31 @@ pub fn build(matches: &ArgMatches, run_after_build: bool) -> io::Result<()> {
             }
         }
     });
-    let binary_path = rt
+
-        .block_on(build_file(src_dir, path, opt_level))
+    let binary_path =
-        .expect("TODO gracefully handle block_on failing");
+        build_file(src_dir, path, opt_level).expect("TODO gracefully handle build_file failing");
 
     if run_after_build {
         // Run the compiled app
-        rt.block_on(async {
         Command::new(binary_path)
             .spawn()
             .unwrap_or_else(|err| panic!("Failed to run app after building it: {:?}", err))
-                .await
+            .wait()
-                .map_err(|_| {
-                    todo!("gracefully handle error after `app` spawned");
-                })
-        })
             .expect("TODO gracefully handle block_on failing");
     }
 
     Ok(())
 }
 
-async fn build_file(
+fn report_timing(buf: &mut String, label: &str, duration: Duration) {
+    buf.push_str(&format!(
+        "        {:.3} ms   {}\n",
+        duration.as_secs_f64() * 1000.0,
+        label,
+    ));
+}
+
+fn build_file(
     src_dir: PathBuf,
     filename: PathBuf,
     opt_level: OptLevel,
@@ -168,9 +161,35 @@ async fn build_file(
         OptLevel::Normal => roc_builtins::std::standard_stdlib(),
         OptLevel::Optimize => roc_builtins::unique::uniq_stdlib(),
     };
-    let loaded = roc_load::file::load(&stdlib, src_dir, filename.clone(), subs_by_module).await?;
+    let loaded =
+        roc_load::file::load(filename.clone(), &stdlib, src_dir.as_path(), subs_by_module)?;
     let dest_filename = filename.with_extension("o");
 
+    let buf = &mut String::with_capacity(1024);
+
+    for (module_id, module_timing) in loaded.timings.iter() {
+        let module_name = loaded.interns.module_name(*module_id);
+
+        buf.push_str("    ");
+        buf.push_str(module_name);
+        buf.push_str("\n");
+
+        report_timing(buf, "Read .roc file from disk", module_timing.read_roc_file);
+        report_timing(buf, "Parse header", module_timing.parse_header);
+        report_timing(buf, "Parse body", module_timing.parse_body);
+        report_timing(buf, "Canonicalize", module_timing.canonicalize);
+        report_timing(buf, "Constrain", module_timing.constrain);
+        report_timing(buf, "Solve", module_timing.solve);
+        report_timing(buf, "Other", module_timing.other());
+        buf.push('\n');
+        report_timing(buf, "Total", module_timing.total());
+    }
+
+    println!(
+        "\n\nCompilation finished! Here's how long each module took to compile:\n\n{}",
+        buf
+    );
+
     gen(
         &arena,
         loaded,
@@ -201,7 +220,7 @@ async fn build_file(
         .map_err(|_| {
             todo!("gracefully handle `ar` failing to spawn.");
         })?
-        .await
+        .wait()
         .map_err(|_| {
             todo!("gracefully handle error after `ar` spawned");
         })?;
@@ -224,7 +243,7 @@ async fn build_file(
         .map_err(|_| {
             todo!("gracefully handle `rustc` failing to spawn.");
         })?
-        .await
+        .wait()
         .map_err(|_| {
             todo!("gracefully handle error after `rustc` spawned");
         })?;

cli/src/repl.rs

@@ -33,6 +33,7 @@ use roc_types::types::Type;
 use std::hash::Hash;
 use std::io::{self, Write};
 use std::path::PathBuf;
+use std::str::from_utf8_unchecked;
 use target_lexicon::Triple;
 
 pub fn main() -> io::Result<()> {
@@ -145,7 +146,7 @@ fn report_parse_error(fail: Fail) {
 }
 
 fn print_output(src: &str) -> Result<String, Fail> {
-    gen(src, Triple::host(), OptLevel::Normal).map(|(answer, answer_type)| {
+    gen(src.as_bytes(), Triple::host(), OptLevel::Normal).map(|(answer, answer_type)| {
         format!("\n{} \u{001b}[35m:\u{001b}[0m {}", answer, answer_type)
     })
 }
@@ -154,7 +155,7 @@ pub fn repl_home() -> ModuleId {
     ModuleIds::default().get_or_insert(&"REPL".into())
 }
 
-pub fn gen(src: &str, target: Triple, opt_level: OptLevel) -> Result<(String, String), Fail> {
+pub fn gen(src: &[u8], target: Triple, opt_level: OptLevel) -> Result<(String, String), Fail> {
     use roc_reporting::report::{can_problem, type_problem, RocDocAllocator, DEFAULT_PALETTE};
 
     // Look up the types and expressions of the `provided` values
@@ -169,13 +170,16 @@ pub fn gen(src: &str, target: Triple, opt_level: OptLevel) -> Result<(String, St
         interns,
         problems: can_problems,
         ..
-    } = can_expr(src)?;
+    } = can_expr(src)?; // IMPORTANT: we must bail out here if there were UTF-8 errors!
+
     let subs = Subs::new(var_store.into());
     let mut type_problems = Vec::new();
     let (content, mut subs) = infer_expr(subs, &mut type_problems, &constraint, var);
 
+    // SAFETY: we've already verified that this is valid UTF-8 during parsing.
+    let src_lines: Vec<&str> = unsafe { from_utf8_unchecked(src).split('\n').collect() };
+
     // Report problems
-    let src_lines: Vec<&str> = src.split('\n').collect();
     let palette = DEFAULT_PALETTE;
 
     // Report parsing and canonicalization problems
@@ -219,7 +223,7 @@ pub fn gen(src: &str, target: Triple, opt_level: OptLevel) -> Result<(String, St
     let expr_type_str = content_to_string(content.clone(), &subs, home, &interns);
 
     // Compute main_fn_type before moving subs to Env
-    let layout = Layout::new(&arena, content, &subs, ptr_bytes).unwrap_or_else(|err| {
+    let layout = Layout::new(&arena, content, &subs).unwrap_or_else(|err| {
         panic!(
             "Code gen error in test: could not convert to layout. Err was {:?}",
             err
@@ -255,7 +259,6 @@ pub fn gen(src: &str, target: Triple, opt_level: OptLevel) -> Result<(String, St
         problems: &mut mono_problems,
         home,
         ident_ids: &mut ident_ids,
-        pointer_size: ptr_bytes,
         jump_counter: arena.alloc(0),
     };
 
@@ -391,8 +394,11 @@ pub fn infer_expr(
     (content, solved.into_inner())
 }
 
-pub fn parse_loc_with<'a>(arena: &'a Bump, input: &'a str) -> Result<Located<ast::Expr<'a>>, Fail> {
+pub fn parse_loc_with<'a>(
-    let state = State::new(&input, Attempting::Module);
+    arena: &'a Bump,
+    bytes: &'a [u8],
+) -> Result<Located<ast::Expr<'a>>, Fail> {
+    let state = State::new(&bytes, Attempting::Module);
     let parser = space0_before(loc(roc_parse::expr::expr(0)), 0);
     let answer = parser.parse(&arena, state);
 
@@ -401,14 +407,14 @@ pub fn parse_loc_with<'a>(arena: &'a Bump, input: &'a str) -> Result<Located<ast
         .map_err(|(fail, _)| fail)
 }
 
-pub fn can_expr(expr_str: &str) -> Result<CanExprOut, Fail> {
+pub fn can_expr(expr_bytes: &[u8]) -> Result<CanExprOut, Fail> {
-    can_expr_with(&Bump::new(), repl_home(), expr_str)
+    can_expr_with(&Bump::new(), repl_home(), expr_bytes)
 }
 
 // TODO make this return a named struct instead of a big tuple
 #[allow(clippy::type_complexity)]
 pub fn uniq_expr(
-    expr_str: &str,
+    expr_bytes: &[u8],
 ) -> Result<
     (
         Located<roc_can::expr::Expr>,
@@ -424,14 +430,14 @@ pub fn uniq_expr(
 > {
     let declared_idents: &ImMap<Ident, (Symbol, Region)> = &ImMap::default();
 
-    uniq_expr_with(&Bump::new(), expr_str, declared_idents)
+    uniq_expr_with(&Bump::new(), expr_bytes, declared_idents)
 }
 
 // TODO make this return a named struct instead of a big tuple
 #[allow(clippy::type_complexity)]
 pub fn uniq_expr_with(
     arena: &Bump,
-    expr_str: &str,
+    expr_bytes: &[u8],
     declared_idents: &ImMap<Ident, (Symbol, Region)>,
 ) -> Result<
     (
@@ -455,7 +461,7 @@ pub fn uniq_expr_with(
         var,
         interns,
         ..
-    } = can_expr_with(arena, home, expr_str)?;
+    } = can_expr_with(arena, home, expr_bytes)?;
 
     // double check
     let mut var_store = VarStore::new(old_var_store.fresh());
@@ -510,8 +516,8 @@ pub struct CanExprOut {
     pub constraint: Constraint,
 }
 
-pub fn can_expr_with(arena: &Bump, home: ModuleId, expr_str: &str) -> Result<CanExprOut, Fail> {
+pub fn can_expr_with(arena: &Bump, home: ModuleId, expr_bytes: &[u8]) -> Result<CanExprOut, Fail> {
-    let loc_expr = parse_loc_with(&arena, expr_str)?;
+    let loc_expr = parse_loc_with(&arena, expr_bytes)?;
     let mut var_store = VarStore::default();
     let var = var_store.fresh();
     let expected = Expected::NoExpectation(Type::Variable(var));

compiler/arena_pool/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "arena-pool"
+version = "0.1.0"
+authors = ["Richard Feldman <oss@rtfeldman.com>"]
+repository = "https://github.com/rtfeldman/roc"
+edition = "2018"
+description = "A CLI for Roc"
+license = "Apache-2.0"
+
+[dev-dependencies]
+pretty_assertions = "0.5.1"

compiler/arena_pool/src/lib.rs

@@ -0,0 +1 @@
+pub mod pool;

compiler/arena_pool/src/pool.rs

@@ -0,0 +1,396 @@
+use std::marker::PhantomPinned;
+use std::ptr::{copy_nonoverlapping, NonNull};
+
+pub struct ArenaRef<T> {
+    ptr: NonNull<T>,
+    _pin: PhantomPinned,
+}
+
+impl<T> ArenaRef<T> {
+    pub fn get<'a, A: AsArena<T>>(&'a self, arena: &A) -> &'a T {
+        arena.verify_ownership(self.ptr);
+
+        // SAFETY: we know this pointer is safe to follow because it will only
+        // get deallocated once the pool where it was created gets deallocated
+        // (along with all of the Arenas it detached), and we just verified that
+        // this ArenaRef's ID matches a pool which has not yet been deallocated.
+        unsafe { self.ptr.as_ref() }
+    }
+
+    pub fn get_mut<'a, A: AsArena<T>>(&'a mut self, arena: &A) -> &'a mut T {
+        arena.verify_ownership(self.ptr);
+
+        // SAFETY: we know this pointer is safe to follow because it will only
+        // get deallocated once the pool where it was created gets deallocated
+        // (along with all of the Arenas it detached), and we just verified that
+        // this ArenaRef's ID matches a pool which has not yet been deallocated.
+        unsafe { self.ptr.as_mut() }
+    }
+}
+
+/// Like a Vec, except the capacity you give it initially is its maximum
+/// capacity forever. If you ever exceed it, it'll panic!
+pub struct ArenaVec<T> {
+    buffer_ptr: NonNull<T>,
+    len: usize,
+    capacity: usize,
+    _pin: PhantomPinned,
+}
+
+impl<T> ArenaVec<T> {
+    pub fn new_in(arena: &mut Arena<T>) -> Self {
+        // We can't start with a NonNull::dangling pointer because when we go
+        // to push elements into this, they'll try to verify the dangling
+        // pointer resides in the arena it was given, which will likely panic.
+        //
+        // Instead, we'll take a pointer inside the array but never use it
+        // other than for verification, because our capacity is 0.
+        Self::with_capacity_in(0, arena)
+    }
+
+    pub fn with_capacity_in(capacity: usize, arena: &mut Arena<T>) -> Self {
+        let ptr = arena.alloc_vec(capacity);
+
+        Self {
+            buffer_ptr: unsafe { NonNull::new_unchecked(ptr) },
+            capacity,
+            len: 0,
+            _pin: PhantomPinned,
+        }
+    }
+
+    pub fn push<'a>(&'a mut self, val: T, arena: &mut Arena<T>) {
+        // Verify that this is the arena where we originally got our buffer,
+        // and is therefore safe to read and to write to. (If we have sufficient
+        // capacity, we'll write to it, and otherwise we'll read from it when
+        // copying our buffer over to the new reserved block.)
+        arena.verify_ownership(self.buffer_ptr);
+
+        if self.len <= self.capacity {
+            // We're all set!
+            //
+            // This empty branch is just here for branch prediction,
+            // since this should be the most common case in practice.
+        } else {
+            // Double our capacity and reserve a new block.
+            self.capacity *= 2;
+
+            let ptr = arena.alloc_vec(self.capacity);
+
+            // SAFETY: the existing buffer must have at least self.len elements,
+            // as must the new one, so copying that many between them is safe.
+            unsafe {
+                // Copy all elements from the current buffer into the new one
+                copy_nonoverlapping(self.buffer_ptr.as_ptr(), ptr, self.len);
+            }
+
+            self.buffer_ptr = unsafe { NonNull::new_unchecked(ptr) };
+        }
+
+        // Store the element in the appropriate memory address.
+        let elem_ptr = unsafe { &mut *self.buffer_ptr.as_ptr().add(self.len) };
+
+        *elem_ptr = val;
+
+        self.len += 1;
+    }
+
+    pub fn get<'a>(&'a self, index: usize, arena: &Arena<T>) -> Option<&'a T> {
+        arena.verify_ownership(self.buffer_ptr);
+
+        if index < self.len {
+            // SAFETY: we know this pointer is safe to follow because we've
+            // done a bounds check, and because we know it will only get
+            // deallocated once the pool where it was created gets deallocated
+            // (along with all of the Arenas it detached), and we just verified that
+            // this ArenaRef's ID matches a pool which has not yet been deallocated.
+            Some(unsafe { &*self.buffer_ptr.as_ptr().add(index) })
+        } else {
+            None
+        }
+    }
+
+    pub fn get_mut<'a>(&'a mut self, index: usize, arena: &Arena<T>) -> Option<&'a mut T> {
+        arena.verify_ownership(self.buffer_ptr);
+
+        if index < self.len {
+            // SAFETY: we know this pointer is safe to follow because we've
+            // done a bounds check, and because we know it will only get
+            // deallocated once the pool where it was created gets deallocated
+            // (along with all of the Arenas it detached), and we just verified that
+            // this ArenaRef's ID matches a pool which has not yet been deallocated.
+            Some(unsafe { &mut *self.buffer_ptr.as_ptr().add(index) })
+        } else {
+            None
+        }
+    }
+}
+
+#[derive(PartialEq, Eq)]
+pub struct ArenaPool<T> {
+    first_chunk: Vec<T>,
+    extra_chunks: Vec<Vec<T>>,
+    num_leased: usize,
+    default_chunk_capacity: usize,
+}
+
+impl<T> ArenaPool<T> {
+    const DEFAULT_CHUNK_SIZE: usize = 1024;
+
+    /// Be careful! Both of these arguments are of type usize.
+    ///
+    /// The first is the number of elements that will be in each arena.
+    /// The second is the number of arenas.
+    ///
+    /// This returns a new Pool, and also an iterator of Arenas. These Arenas can
+    /// be given to different threads, where they can be used to allocate
+    /// ArenaRef and ArenaVec values which can then be dereferenced by the Arena
+    /// that created them, or by this pool once those Arenas have been
+    /// reabsorbed back into it.
+    ///
+    /// (A word of warning: if you try to use this pool to dereference ArenaRec
+    /// and ArenaVec values which were allocated by arenas that have *not* yet
+    /// been reabsorbed, it may work some of the time and panic other times,
+    /// depending on whether the arena needed to allocate extra chunks beyond
+    /// its initial chunk. tl;dr - doing that may panic, so don't try it!)
+    ///
+    /// Before this pool gets dropped, you must call reabsorb() on every
+    /// arena that has been leased - otherwise, you'll get a panic when this
+    /// gets dropped! The memory safety of the system depends on all arenas
+    /// having been reabsorbed before the pool gets deallocated, which is why
+    /// the pool's Drop implementation enforces it.
+    pub fn new(num_arenas: usize, elems_per_arena: usize) -> (ArenaPool<T>, ArenaIter<T>) {
+        Self::with_chunk_size(num_arenas, elems_per_arena, Self::DEFAULT_CHUNK_SIZE)
+    }
+
+    /// Like `new`, except you can also specify the chunk size that each
+    /// arena will use to allocate its extra chunks if it runs out of space
+    /// in its main buffer.
+    ///
+    /// Things will run fastest if that main buffer never runs out, though!
+    pub fn with_chunk_size(
+        num_arenas: usize,
+        elems_per_arena: usize,
+        chunk_size: usize,
+    ) -> (ArenaPool<T>, ArenaIter<T>) {
+        let mut first_chunk = Vec::with_capacity(elems_per_arena * num_arenas);
+        let iter = ArenaIter {
+            ptr: first_chunk.as_mut_ptr(),
+            quantity_remaining: num_arenas,
+            first_chunk_capacity: elems_per_arena,
+        };
+        let pool = Self {
+            first_chunk,
+            extra_chunks: Vec::new(),
+            num_leased: num_arenas,
+            default_chunk_capacity: chunk_size,
+        };
+
+        (pool, iter)
+    }
+
+    /// Return an arena to the pool. (This would have been called "return" but
+    /// that's a reserved keyword.)
+    pub fn reabsorb(&mut self, arena: Arena<T>) {
+        // Ensure we're reabsorbing an arena that was
+        // actually leased by this pool in the first place!
+        verify_ownership(
+            self.first_chunk.as_ptr(),
+            self.first_chunk.capacity(),
+            &self.extra_chunks,
+            arena.first_chunk_ptr,
+        );
+
+        // Add the arena's extra chunks to our own, so their memory remains live
+        // after the arena gets dropped. This is important, because at this
+        // point their pointers can still potentially be dereferenced!
+        self.extra_chunks.extend(arena.extra_chunks.into_iter());
+
+        self.num_leased -= 1;
+    }
+}
+
+impl<T> Drop for ArenaPool<T> {
+    fn drop(&mut self) {
+        // When an ArenaPool gets dropped, it must not have any leased
+        // arenas remaining. If it does, there will be outstanding IDs which
+        // could be used with those non-reabsorbed Arenas to read freed memory!
+        // This would be a use-after-free; we panic rather than permit that.
+        assert_eq!(self.num_leased, 0);
+    }
+}
+
+pub struct ArenaIter<T> {
+    ptr: *mut T,
+    quantity_remaining: usize,
+    first_chunk_capacity: usize,
+}
+
+// Implement `Iterator` for `Fibonacci`.
+// The `Iterator` trait only requires a method to be defined for the `next` element.
+impl<T> Iterator for ArenaIter<T> {
+    type Item = Arena<T>;
+
+    // Here, we define the sequence using `.curr` and `.next`.
+    // The return type is `Option<T>`:
+    //     * When the `Iterator` is finished, `None` is returned.
+    //     * Otherwise, the next value is wrapped in `Some` and returned.
+    fn next(&mut self) -> Option<Arena<T>> {
+        if self.quantity_remaining != 0 {
+            let first_chunk_ptr = self.ptr;
+
+            self.ptr = unsafe { self.ptr.add(self.first_chunk_capacity) };
+            self.quantity_remaining -= 1;
+
+            Some(Arena {
+                first_chunk_ptr,
+                first_chunk_len: 0,
+                first_chunk_cap: self.first_chunk_capacity,
+                extra_chunks: Vec::new(),
+            })
+        } else {
+            None
+        }
+    }
+}
+
+#[derive(PartialEq, Eq)]
+pub struct Arena<T> {
+    first_chunk_ptr: *mut T,
+    first_chunk_len: usize,
+    first_chunk_cap: usize,
+    extra_chunks: Vec<Vec<T>>,
+}
+
+impl<T> Arena<T> {
+    pub fn alloc(&mut self, val: T) -> ArenaRef<T> {
+        let ptr: *mut T = if self.first_chunk_len < self.first_chunk_cap {
+            // We have enough room in the first chunk for 1 allocation.
+            self.first_chunk_len += 1;
+
+            // Return a pointer to the next available slot.
+            unsafe { self.first_chunk_ptr.add(self.first_chunk_len) }
+        } else {
+            // We ran out of space in the first chunk, so we turn to extra chunks.
+            // First, ensure that we have an extra chunk with enough space in it.
+            match self.extra_chunks.last() {
+                Some(chunk) => {
+                    if chunk.len() >= chunk.capacity() {
+                        // We've run out of space in our last chunk. Create a new one!
+                        self.extra_chunks
+                            .push(Vec::with_capacity(self.first_chunk_cap));
+                    }
+                }
+                None => {
+                    // We've never had extra chunks until now. Create the first one!
+                    self.extra_chunks
+                        .push(Vec::with_capacity(self.first_chunk_cap));
+                }
+            }
+
+            let chunk = self.extra_chunks.last_mut().unwrap();
+            let index = chunk.len();
+
+            chunk.push(val);
+
+            // Get a pointer to a memory address within our particular chunk.
+            &mut chunk[index]
+        };
+
+        ArenaRef {
+            ptr: unsafe { NonNull::new_unchecked(ptr) },
+            _pin: PhantomPinned,
+        }
+    }
+
+    fn alloc_vec(&mut self, num_elems: usize) -> *mut T {
+        if self.first_chunk_len + num_elems <= self.first_chunk_cap {
+            // We have enough room in the first chunk for this vec.
+            self.first_chunk_len += num_elems;
+
+            // Return a pointer to the next available element.
+            unsafe { self.first_chunk_ptr.add(self.first_chunk_len) }
+        } else {
+            let new_chunk_cap = self.first_chunk_cap.max(num_elems);
+
+            // We ran out of space in the first chunk, so we turn to extra chunks.
+            // First, ensure that we have an extra chunk with enough space in it.
+            match self.extra_chunks.last() {
+                Some(chunk) => {
+                    if chunk.len() + num_elems >= chunk.capacity() {
+                        // We don't have enough space in our last chunk.
+                        // Create a new one!
+                        self.extra_chunks.push(Vec::with_capacity(new_chunk_cap));
+                    }
+                }
+                None => {
+                    // We've never had extra chunks until now. Create the first one!
+                    self.extra_chunks.push(Vec::with_capacity(new_chunk_cap));
+                }
+            }
+
+            let chunk = self.extra_chunks.last_mut().unwrap();
+            let index = chunk.len();
+
+            // Get a pointer to a memory address within our particular chunk.
+            &mut chunk[index]
+        }
+    }
+}
+
+pub trait AsArena<T> {
+    fn verify_ownership(&self, ptr: NonNull<T>);
+}
+
+impl<T> AsArena<T> for ArenaPool<T> {
+    fn verify_ownership(&self, ptr: NonNull<T>) {
+        verify_ownership(
+            self.first_chunk.as_ptr(),
+            self.first_chunk.capacity(),
+            &self.extra_chunks,
+            ptr.as_ptr(),
+        );
+    }
+}
+
+impl<T> AsArena<T> for Arena<T> {
+    fn verify_ownership(&self, ptr: NonNull<T>) {
+        verify_ownership(
+            self.first_chunk_ptr,
+            self.first_chunk_cap,
+            &self.extra_chunks,
+            ptr.as_ptr(),
+        );
+    }
+}
+
+fn verify_ownership<T>(
+    first_chunk_ptr: *const T,
+    first_chunk_cap: usize,
+    extra_chunks: &[Vec<T>],
+    ptr: *const T,
+) {
+    let addr = ptr as usize;
+    let start_addr = first_chunk_ptr as usize;
+    let end_addr = start_addr + first_chunk_cap;
+
+    if start_addr <= addr && addr < end_addr {
+        // This is within our first chunk's address space, so it's verified!
+    } else {
+        // This wasn't within our first chunk's address space, so we need
+        // to see if we can find it in one of our extra_chunks.
+        for chunk in extra_chunks {
+            let start_addr = chunk.as_ptr() as usize;
+            let end_addr = start_addr + chunk.capacity();
+
+            if start_addr <= addr && addr < end_addr {
+                // Found it! No need to loop anymore; verification passed.
+                return;
+            }
+        }
+
+        // The address wasn't within any of our chunks' bounds.
+        // Panic to avoid use-after-free errors!
+        panic!("Pointer ownership verification failed.");
+    }
+}

compiler/arena_pool/tests/test_arena_pool.rs

@@ -0,0 +1,17 @@
+// #[macro_use]
+// extern crate pretty_assertions;
+
+extern crate arena_pool;
+
+#[cfg(test)]
+mod test_arena_pool {
+    use arena_pool::pool::{ArenaIter, ArenaPool};
+
+    #[test]
+    fn empty_pool() {
+        // Neither of these does anything, but they
+        // at least shouldn't panic or anything.
+        let _: (ArenaPool<()>, ArenaIter<()>) = ArenaPool::new(0, 0);
+        let _: (ArenaPool<()>, ArenaIter<()>) = ArenaPool::with_chunk_size(0, 0, 0);
+    }
+}

compiler/build/src/program.rs

@@ -136,14 +136,14 @@ pub fn gen(
     fpm.initialize();
 
     // Compute main_fn_type before moving subs to Env
-    let ptr_bytes = target.pointer_width().unwrap().bytes() as u32;
+    let layout = Layout::new(&arena, content, &subs).unwrap_or_else(|err| {
-    let layout = Layout::new(&arena, content, &subs, ptr_bytes).unwrap_or_else(|err| {
         panic!(
             "Code gen error in Program: could not convert to layout. Err was {:?}",
             err
         )
     });
 
+    let ptr_bytes = target.pointer_width().unwrap().bytes() as u32;
     let main_fn_type =
         basic_type_from_layout(&arena, &context, &layout, ptr_bytes).fn_type(&[], false);
     let main_fn_name = "$main";
@@ -169,7 +169,6 @@ pub fn gen(
         problems: &mut mono_problems,
         home,
         ident_ids: &mut ident_ids,
-        pointer_size: ptr_bytes,
         jump_counter: arena.alloc(0),
     };
 

compiler/builtins/docs/Bool.roc

@@ -19,45 +19,24 @@ not : [True, False] -> [True, False]
 ##
 ## ## Performance Notes
 ##
-## In dev builds, this works exactly as described. In release builds, as a
+## In some languages, `&&` and `||` are special-cased in the compiler to skip
-## performance optimization, the compiler translates calls to #Bool.and
+## evaluating the expression after the operator under certain circumstances.
-## (and #Bool.or) from function calls into conditionals. If its first
+## For example, in some languages, `enablePets && likesDogs user` would compile
-## argument evalutes to #False, then any function calls in the second argument
+## to the equivalent of:
-## get skipped, and the entire expression immediately evaluates to #False.
 ##
-## For example:
+##     if enablePets then
+##         likesDogs user
+##     else
+##         False
 ##
-##     List.isEmpty list && Str.isEmpty str
+## In Roc, however, `&&` and `||` are not special. They work the same way as
+## other functions. Conditionals like `if` and `when` have a performance cost,
+## and sometimes calling a function like `likesDogs user` can be faster across
+## the board than doing an `if` to decide whether to skip calling it.
 ##
-## In a release build, `Str.isEmpty` will only be called if `List.isEmpty list`
+## (Naturally, if you expect the `if` to improve performance, you can always add
-## returns #True. If `List.isEmpty list` returns #False, the entire expression
+## one explicitly!)
-## will immediately evaluate to #False.
+and : Bool, Bool -> Bool
-##
-## Since all Roc expressions are pure, this will always give the same answer
-## as if both #Bool arguments had been fully evaluated (as they are in
-## dev builds), but it can potentially avoid costly function calls in release builds.
-##
-## Because this optimization only skips function calls, you can opt out of it
-## by calling the function up front, and giving its result a name. For example:
-##
-##     emptyStr = Str.isEmpty str
-##
-##     List.isEmpty list && emptyStr
-##
-## Here, `Str.isEmpty` will always be called no matter what, and the `&&` will
-## not get compiled to a conditional because there are no function calls
-## involved in its second argument.
-##
-## If you know the functions involved in the second argument are trivial
-## (for example, they are other #&&, #||, and #Bool.not operations), then
-## this can potentially be a (likely extremely minor) performance optimization
-## because a logical `AND` instruction typically executes faster than a
-## [branch misprediction](https://danluu.com/branch-prediction).
-##
-## That said, in practice the `&& Str.isEmpty str` approach will typically run
-## faster than the `&& emptyStr` approach - both for `Str.isEmpty` in particular
-## as well as for most functions in general.
-and : [True, False], [True, False] -> [True, False]
 
 
 ## Returns #True when given #True for either argument, and #False only when given #False and #False.
@@ -74,18 +53,19 @@ and : [True, False], [True, False] -> [True, False]
 ##
 ## ## Performance Notes
 ##
-## #Bool.or does the same "compile to a conditional in release mode" optimization
+## In some languages, `&&` and `||` are special-cased in the compiler to skip
-## that #Bool.and does, except it short-circuits when the first argument is
+## evaluating the expression after the operator under certain circumstances.
-## #True (causing it to immediately returns #True).
 ##
-## See the performance notes for #Bool.and for details.
+## In Roc, this is not the case. See the performance notes for #Bool.and for details.
-or : [True, False], [True, False] -> [True, False]
+or : Bool, Bool -> Bool
 
 ## Exclusive or
-xor : [True, False], [True, False] -> [True, False]
+xor : Bool, Bool -> Bool
 
 ## Returns #True if the two values are *structurally equal*, and #False otherwise.
 ##
+## `a == b` is shorthand for `Bool.isEq a b`
+##
 ## Structural equality works as follows:
 ##
 ## 1. #Int and #Float values are equal if their numbers are equal.
@@ -93,18 +73,15 @@ xor : [True, False], [True, False] -> [True, False]
 ## 3. Global tags are equal if they are the same tag, and also their contents (if any) are equal.
 ## 4. Private tags are equal if they are the same tag, in the same module, and also their contents (if any) are equal.
 ## 5. Collections (#String, #List, #Map, #Set, and #Bytes) are equal if they are the same length, and also all their corresponding elements are equal.
-## 6. All functions are considered equal. (So `Bool.not == Bool.not` will return #True, as you might expect, but also `Num.abs == Num.negate` will return #True, as you might not. This design is because function equality has been formally proven to be undecidable in the general case, and returning #True in all cases turns out to be mostly harmless - especially compared to alternative designs like crashing, making #equal inconvenient to use, and so on.)
 ##
-## This function always crashes when given two functions, or an erroneous
+## Note that `isEq` takes `'val` instead of `val`, which means `isEq` does not
-## #Float value (see #Float.isErroneous)
+## accept arguments whose types contain functions.
-##
+isEq : 'val, 'val -> Bool
-## This is the same as the #== operator.
-isEq : val, val -> [True, False]
 
 ## Calls #eq on the given values, then calls #not on the result.
 ##
-## This is the same as the #!= operator.
+## `a != b` is shorthand for `Bool.isNotEq a b`
-isNe : val, val -> [True, False]
+##
-isNe = \left, right ->
+## Note that `isNotEq` takes `'val` instead of `val`, which means `isNotEq` does not
-    not (equal left right)
+## accept arguments whose types contain functions.
-
+isNotEq : 'val, 'val -> Bool

compiler/builtins/docs/Set.roc

@@ -2,17 +2,27 @@ interface Set
     exposes [ Set, map, isEmpty ]
     imports []
 
+
+## An empty set.
+empty : Set *
+
 ## Check
 
-# isEmpty : List * -> Bool
+isEmpty : Set * -> Bool
 
-## Convert each element in the list to something new, by calling a conversion
+len : Set * -> Len
-## function on each of them. Then return a new list of the converted values.
+
+add : Set 'elem, 'elem -> Set 'elem
+
+rem : Set 'elem, 'elem -> Set 'elem
+
+## Convert each element in the set to something new, by calling a conversion
+## function on each of them. Then return a new set of the converted values.
 ##
-## >>> Set.map {[ -1, 1, 3 ]} Num.negate
+## >>> Set.map {: -1, 1, 3 :} Num.negate
 ##
-## >>> Set.map {[ "", "a", "bc" ]} Str.isEmpty
+## >>> Set.map {: "", "a", "bc" :} Str.isEmpty
 ##
 ## `map` functions like this are common in Roc, and they all work similarly.
 ## See for example #Result.map, #List.map, and #Map.map.
-map : List before, (before -> after) -> List after
+map : Set 'elem, ('before -> 'after) -> Set 'after

compiler/builtins/src/std.rs

@@ -510,9 +510,18 @@ pub fn types() -> MutMap<Symbol, (SolvedType, Region)> {
         ),
     );
 
-    // push : List elem -> elem -> List elem
+    // append : List elem, elem -> List elem
     add_type(
-        Symbol::LIST_PUSH,
+        Symbol::LIST_APPEND,
+        SolvedType::Func(
+            vec![list_type(flex(TVAR1)), flex(TVAR1)],
+            Box::new(list_type(flex(TVAR1))),
+        ),
+    );
+
+    // prepend : List elem, elem -> List elem
+    add_type(
+        Symbol::LIST_PREPEND,
         SolvedType::Func(
             vec![list_type(flex(TVAR1)), flex(TVAR1)],
             Box::new(list_type(flex(TVAR1))),
@@ -543,15 +552,6 @@ pub fn types() -> MutMap<Symbol, (SolvedType, Region)> {
         ),
     );
 
-    // append : List elem, List elem -> List elem
-    add_type(
-        Symbol::LIST_APPEND,
-        SolvedType::Func(
-            vec![list_type(flex(TVAR1)), list_type(flex(TVAR1))],
-            Box::new(list_type(flex(TVAR1))),
-        ),
-    );
-
     // len : List * -> Int
     add_type(
         Symbol::LIST_LEN,

compiler/builtins/src/unique.rs

@@ -638,8 +638,8 @@ pub fn types() -> MutMap<Symbol, (SolvedType, Region)> {
         )
     });
 
-    // append : Attr * (List (Attr * a)), Attr * (List (Attr * a)) -> Attr * (List (Attr * a))
+    // concat : Attr * (List (Attr * a)), Attr * (List (Attr * a)) -> Attr * (List (Attr * a))
-    add_type(Symbol::LIST_APPEND, {
+    add_type(Symbol::LIST_CONCAT, {
         let_tvars! { a, star1, star2, star3 };
 
         unique_function(
@@ -669,13 +669,43 @@ pub fn types() -> MutMap<Symbol, (SolvedType, Region)> {
         )
     });
 
-    // push : Attr * (List a)
+    // append : Attr * (List a)
     //      , a
     //     -> Attr * (List a)
     //
     // NOTE: we demand the new item to have the same uniqueness as the other list items.
     // It could be allowed to add unique items to shared lists, but that requires special code gen
-    add_type(Symbol::LIST_PUSH, {
+    add_type(Symbol::LIST_APPEND, {
+        let_tvars! { a, star1, star2 };
+
+        unique_function(
+            vec![
+                SolvedType::Apply(
+                    Symbol::ATTR_ATTR,
+                    vec![
+                        flex(star1),
+                        SolvedType::Apply(Symbol::LIST_LIST, vec![flex(a)]),
+                    ],
+                ),
+                flex(a),
+            ],
+            SolvedType::Apply(
+                Symbol::ATTR_ATTR,
+                vec![
+                    boolean(star2),
+                    SolvedType::Apply(Symbol::LIST_LIST, vec![flex(a)]),
+                ],
+            ),
+        )
+    });
+
+    // prepend : Attr * (List a)
+    //      , a
+    //     -> Attr * (List a)
+    //
+    // NOTE: we demand the new item to have the same uniqueness as the other list items.
+    // It could be allowed to add unique items to shared lists, but that requires special code gen
+    add_type(Symbol::LIST_PREPEND, {
         let_tvars! { a, star1, star2 };
 
         unique_function(

compiler/can/src/builtins.rs

@@ -53,13 +53,14 @@ pub fn builtin_defs(var_store: &mut VarStore) -> MutMap<Symbol, Def> {
         Symbol::LIST_LEN => list_len,
         Symbol::LIST_GET => list_get,
         Symbol::LIST_SET => list_set,
-        Symbol::LIST_PUSH => list_push,
+        Symbol::LIST_APPEND => list_append,
         Symbol::LIST_FIRST => list_first,
         Symbol::LIST_IS_EMPTY => list_is_empty,
         Symbol::LIST_SINGLE => list_single,
         Symbol::LIST_REPEAT => list_repeat,
         Symbol::LIST_REVERSE => list_reverse,
-        Symbol::LIST_APPEND => list_append,
+        Symbol::LIST_CONCAT => list_concat,
+        Symbol::LIST_PREPEND => list_prepend,
         Symbol::NUM_ADD => num_add,
         Symbol::NUM_SUB => num_sub,
         Symbol::NUM_MUL => num_mul,
@@ -617,12 +618,12 @@ fn list_reverse(symbol: Symbol, var_store: &mut VarStore) -> Def {
     )
 }
 
-/// List.append : List elem, List elem -> List elem
+/// List.concat : List elem, List elem -> List elem
-fn list_append(symbol: Symbol, var_store: &mut VarStore) -> Def {
+fn list_concat(symbol: Symbol, var_store: &mut VarStore) -> Def {
     let list_var = var_store.fresh();
 
     let body = RunLowLevel {
-        op: LowLevel::ListAppend,
+        op: LowLevel::ListConcat,
         args: vec![
             (list_var, Var(Symbol::ARG_1)),
             (list_var, Var(Symbol::ARG_2)),
@@ -856,13 +857,36 @@ fn list_set(symbol: Symbol, var_store: &mut VarStore) -> Def {
     )
 }
 
-/// List.push : List elem, elem -> List elem
+/// List.append : List elem, elem -> List elem
-fn list_push(symbol: Symbol, var_store: &mut VarStore) -> Def {
+fn list_append(symbol: Symbol, var_store: &mut VarStore) -> Def {
     let list_var = var_store.fresh();
     let elem_var = var_store.fresh();
 
     let body = RunLowLevel {
-        op: LowLevel::ListPush,
+        op: LowLevel::ListAppend,
+        args: vec![
+            (list_var, Var(Symbol::ARG_1)),
+            (elem_var, Var(Symbol::ARG_2)),
+        ],
+        ret_var: list_var,
+    };
+
+    defn(
+        symbol,
+        vec![(list_var, Symbol::ARG_1), (elem_var, Symbol::ARG_2)],
+        var_store,
+        body,
+        list_var,
+    )
+}
+
+/// List.prepend : List elem, elem -> List elem
+fn list_prepend(symbol: Symbol, var_store: &mut VarStore) -> Def {
+    let list_var = var_store.fresh();
+    let elem_var = var_store.fresh();
+
+    let body = RunLowLevel {
+        op: LowLevel::ListPrepend,
         args: vec![
             (list_var, Var(Symbol::ARG_1)),
             (elem_var, Var(Symbol::ARG_2)),

compiler/can/tests/helpers/mod.rs

@@ -27,7 +27,7 @@ pub fn parse_with<'a>(arena: &'a Bump, input: &'a str) -> Result<ast::Expr<'a>,
 
 #[allow(dead_code)]
 pub fn parse_loc_with<'a>(arena: &'a Bump, input: &'a str) -> Result<Located<ast::Expr<'a>>, Fail> {
-    let state = State::new(&input, Attempting::Module);
+    let state = State::new(input.as_bytes(), Attempting::Module);
     let parser = space0_before(loc(roc_parse::expr::expr(0)), 0);
     let answer = parser.parse(&arena, state);
 

compiler/fmt/tests/test_fmt.rs

@@ -20,7 +20,7 @@ mod test_fmt {
     use roc_parse::parser::{Fail, Parser, State};
 
     fn parse_with<'a>(arena: &'a Bump, input: &'a str) -> Result<Expr<'a>, Fail> {
-        let state = State::new(&input, Attempting::Module);
+        let state = State::new(input.as_bytes(), Attempting::Module);
         let parser = space0_before(loc!(roc_parse::expr::expr(0)), 0);
         let answer = parser.parse(&arena, state);
 
@@ -55,7 +55,7 @@ mod test_fmt {
         let src = src.trim_end();
         let expected = expected.trim_end();
 
-        match module::header().parse(&arena, State::new(&src, Attempting::Module)) {
+        match module::header().parse(&arena, State::new(src.as_bytes(), Attempting::Module)) {
             Ok((actual, state)) => {
                 let mut buf = String::new_in(&arena);
 

compiler/gen/src/llvm/build.rs

@@ -1828,7 +1828,7 @@ fn list_push<'a, 'ctx, 'env>(
     let elem_type = basic_type_from_layout(env.arena, ctx, elem_layout, env.ptr_bytes);
     let ptr_type = get_ptr_type(&elem_type, AddressSpace::Generic);
 
-    let elems_ptr = load_list_ptr(builder, original_wrapper, ptr_type);
+    let list_ptr = load_list_ptr(builder, original_wrapper, ptr_type);
 
     // The output list length, which is the old list length + 1
     let new_list_len = env.builder.build_int_add(
@@ -1837,7 +1837,6 @@ fn list_push<'a, 'ctx, 'env>(
         "new_list_length",
     );
 
-    let ctx = env.context;
     let ptr_bytes = env.ptr_bytes;
 
     // Calculate the number of bytes we'll need to allocate.
@@ -1863,7 +1862,7 @@ fn list_push<'a, 'ctx, 'env>(
         // one we just malloc'd.
         //
         // TODO how do we decide when to do the small memcpy vs the normal one?
-        builder.build_memcpy(clone_ptr, ptr_bytes, elems_ptr, ptr_bytes, list_size);
+        builder.build_memcpy(clone_ptr, ptr_bytes, list_ptr, ptr_bytes, list_size);
     } else {
         panic!("TODO Cranelift currently only knows how to clone list elements that are Copy.");
     }
@@ -1887,17 +1886,105 @@ fn list_push<'a, 'ctx, 'env>(
         .build_insert_value(struct_val, new_list_len, Builtin::WRAPPER_LEN, "insert_len")
         .unwrap();
 
-    let answer = builder.build_bitcast(
-        struct_val.into_struct_value(),
-        collection(ctx, ptr_bytes),
-        "cast_collection",
-    );
-
     let elem_ptr = unsafe { builder.build_in_bounds_gep(clone_ptr, &[list_len], "load_index") };
 
     builder.build_store(elem_ptr, elem);
 
-    answer
+    builder.build_bitcast(
+        struct_val.into_struct_value(),
+        collection(ctx, ptr_bytes),
+        "cast_collection",
+    )
+}
+
+/// List.prepend List elem, elem -> List elem
+fn list_prepend<'a, 'ctx, 'env>(
+    env: &Env<'a, 'ctx, 'env>,
+    original_wrapper: StructValue<'ctx>,
+    elem: BasicValueEnum<'ctx>,
+    elem_layout: &Layout<'a>,
+) -> BasicValueEnum<'ctx> {
+    let builder = env.builder;
+    let ctx = env.context;
+
+    // Load the usize length from the wrapper.
+    let list_len = load_list_len(builder, original_wrapper);
+    let elem_type = basic_type_from_layout(env.arena, ctx, elem_layout, env.ptr_bytes);
+    let ptr_type = get_ptr_type(&elem_type, AddressSpace::Generic);
+    let list_ptr = load_list_ptr(builder, original_wrapper, ptr_type);
+
+    // The output list length, which is the old list length + 1
+    let new_list_len = env.builder.build_int_add(
+        ctx.i64_type().const_int(1 as u64, false),
+        list_len,
+        "new_list_length",
+    );
+
+    let ptr_bytes = env.ptr_bytes;
+
+    // Allocate space for the new array that we'll copy into.
+    let elem_type = basic_type_from_layout(env.arena, ctx, elem_layout, env.ptr_bytes);
+    let clone_ptr = builder
+        .build_array_malloc(elem_type, new_list_len, "list_ptr")
+        .unwrap();
+    let int_type = ptr_int(ctx, ptr_bytes);
+    let ptr_as_int = builder.build_ptr_to_int(clone_ptr, int_type, "list_cast_ptr");
+
+    builder.build_store(clone_ptr, elem);
+
+    let index_1_ptr = unsafe {
+        builder.build_in_bounds_gep(
+            clone_ptr,
+            &[ctx.i64_type().const_int(1 as u64, false)],
+            "load_index",
+        )
+    };
+
+    // Calculate the number of bytes we'll need to allocate.
+    let elem_bytes = env
+        .ptr_int()
+        .const_int(elem_layout.stack_size(env.ptr_bytes) as u64, false);
+
+    // This is the size of the list coming in, before we have added an element
+    // to the beginning.
+    let list_size = env
+        .builder
+        .build_int_mul(elem_bytes, list_len, "mul_old_len_by_elem_bytes");
+
+    if elem_layout.safe_to_memcpy() {
+        // Copy the bytes from the original array into the new
+        // one we just malloc'd.
+        //
+        // TODO how do we decide when to do the small memcpy vs the normal one?
+        builder.build_memcpy(index_1_ptr, ptr_bytes, list_ptr, ptr_bytes, list_size);
+    } else {
+        panic!("TODO Cranelift currently only knows how to clone list elements that are Copy.");
+    }
+
+    // Create a fresh wrapper struct for the newly populated array
+    let struct_type = collection(ctx, env.ptr_bytes);
+    let mut struct_val;
+
+    // Store the pointer
+    struct_val = builder
+        .build_insert_value(
+            struct_type.get_undef(),
+            ptr_as_int,
+            Builtin::WRAPPER_PTR,
+            "insert_ptr",
+        )
+        .unwrap();
+
+    // Store the length
+    struct_val = builder
+        .build_insert_value(struct_val, new_list_len, Builtin::WRAPPER_LEN, "insert_len")
+        .unwrap();
+
+    builder.build_bitcast(
+        struct_val.into_struct_value(),
+        collection(ctx, ptr_bytes),
+        "cast_collection",
+    )
 }
 
 fn list_set<'a, 'ctx, 'env>(
@@ -2177,8 +2264,8 @@ fn run_low_level<'a, 'ctx, 'env>(
                 }
             }
         }
-        ListAppend => list_append(env, scope, parent, args),
+        ListConcat => list_concat(env, scope, parent, args),
-        ListPush => {
+        ListAppend => {
             // List.push List elem, elem -> List elem
             debug_assert_eq!(args.len(), 2);
 
@@ -2187,6 +2274,15 @@ fn run_low_level<'a, 'ctx, 'env>(
 
             list_push(env, original_wrapper, elem, elem_layout)
         }
+        ListPrepend => {
+            // List.prepend List elem, elem -> List elem
+            debug_assert_eq!(args.len(), 2);
+
+            let original_wrapper = load_symbol(env, scope, &args[0]).into_struct_value();
+            let (elem, elem_layout) = load_symbol_and_layout(env, scope, &args[1]);
+
+            list_prepend(env, original_wrapper, elem, elem_layout)
+        }
         NumAbs | NumNeg | NumRound | NumSqrtUnchecked | NumSin | NumCos | NumToFloat => {
             debug_assert_eq!(args.len(), 1);
 
@@ -2395,13 +2491,13 @@ fn build_int_binop<'a, 'ctx, 'env>(
     }
 }
 
-fn list_append<'a, 'ctx, 'env>(
+fn list_concat<'a, 'ctx, 'env>(
     env: &Env<'a, 'ctx, 'env>,
     scope: &Scope<'a, 'ctx>,
     parent: FunctionValue<'ctx>,
     args: &[Symbol],
 ) -> BasicValueEnum<'ctx> {
-    // List.append : List elem, List elem -> List elem
+    // List.concat : List elem, List elem -> List elem
     debug_assert_eq!(args.len(), 2);
 
     // This implementation is quite long, let me explain what is complicating it. Here are our
@@ -2489,7 +2585,7 @@ fn list_append<'a, 'ctx, 'env>(
                 }
                 _ => {
                     unreachable!(
-                        "Invalid List layout for second input list of List.append: {:?}",
+                        "Invalid List layout for second input list of List.concat: {:?}",
                         second_list_layout
                     );
                 }
@@ -2557,7 +2653,7 @@ fn list_append<'a, 'ctx, 'env>(
                             // FIRST LOOP
                             {
                                 let first_loop_bb =
-                                    ctx.append_basic_block(parent, "first_list_append_loop");
+                                    ctx.append_basic_block(parent, "first_list_concat_loop");
 
                                 builder.build_unconditional_branch(first_loop_bb);
                                 builder.position_at_end(first_loop_bb);
@@ -2628,7 +2724,7 @@ fn list_append<'a, 'ctx, 'env>(
                             // SECOND LOOP
                             {
                                 let second_loop_bb =
-                                    ctx.append_basic_block(parent, "second_list_append_loop");
+                                    ctx.append_basic_block(parent, "second_list_concat_loop");
 
                                 builder.build_unconditional_branch(second_loop_bb);
                                 builder.position_at_end(second_loop_bb);
@@ -2754,7 +2850,7 @@ fn list_append<'a, 'ctx, 'env>(
                     }
                     _ => {
                         unreachable!(
-                            "Invalid List layout for second input list of List.append: {:?}",
+                            "Invalid List layout for second input list of List.concat: {:?}",
                             second_list_layout
                         );
                     }
@@ -2799,7 +2895,7 @@ fn list_append<'a, 'ctx, 'env>(
                     }
                     _ => {
                         unreachable!(
-                            "Invalid List layout for second input list of List.append: {:?}",
+                            "Invalid List layout for second input list of List.concat: {:?}",
                             second_list_layout
                         );
                     }
@@ -2817,7 +2913,7 @@ fn list_append<'a, 'ctx, 'env>(
         }
         _ => {
             unreachable!(
-                "Invalid List layout for first list in List.append : {:?}",
+                "Invalid List layout for first list in List.concat : {:?}",
                 first_list_layout
             );
         }

compiler/gen/tests/gen_list.rs

@@ -31,10 +31,10 @@ mod gen_list {
     }
 
     #[test]
-    fn list_push() {
+    fn list_append() {
-        assert_evals_to!("List.push [1] 2", &[1, 2], &'static [i64]);
+        assert_evals_to!("List.append [1] 2", &[1, 2], &'static [i64]);
-        assert_evals_to!("List.push [1, 1] 2", &[1, 1, 2], &'static [i64]);
+        assert_evals_to!("List.append [1, 1] 2", &[1, 1, 2], &'static [i64]);
-        assert_evals_to!("List.push [] 3", &[3], &'static [i64]);
+        assert_evals_to!("List.append [] 3", &[3], &'static [i64]);
         assert_evals_to!(
             indoc!(
                 r#"
@@ -42,24 +42,55 @@ mod gen_list {
                     initThrees =
                         []
 
-                    List.push (List.push initThrees 3) 3
+                    List.append (List.append initThrees 3) 3
                 "#
             ),
             &[3, 3],
             &'static [i64]
         );
         assert_evals_to!(
-            "List.push [ True, False ] True",
+            "List.append [ True, False ] True",
             &[true, false, true],
             &'static [bool]
         );
         assert_evals_to!(
-            "List.push [ 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22 ] 23",
+            "List.append [ 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22 ] 23",
             &[11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23],
             &'static [i64]
         );
     }
 
+    #[test]
+    fn list_prepend() {
+        assert_evals_to!("List.prepend [] 1", &[1], &'static [i64]);
+        assert_evals_to!("List.prepend [2] 1", &[1, 2], &'static [i64]);
+
+        assert_evals_to!(
+            indoc!(
+                r#"
+                    init : List Int
+                    init =
+                        []
+
+                    List.prepend (List.prepend init 4) 6
+                "#
+            ),
+            &[6, 4],
+            &'static [i64]
+        );
+
+        assert_evals_to!(
+            "List.prepend [ True, False ] True",
+            &[true, true, false],
+            &'static [bool]
+        );
+        assert_evals_to!(
+            "List.prepend [ 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 100, 100, 100, 100 ] 9",
+            &[9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 100, 100, 100, 100],
+            &'static [i64]
+        );
+    }
+
     #[test]
     fn list_single() {
         assert_evals_to!("List.single 1", &[1], &'static [i64]);
@@ -119,8 +150,8 @@ mod gen_list {
     }
 
     #[test]
-    fn list_append() {
+    fn list_concat() {
-        assert_evals_to!("List.append [] []", &[], &'static [i64]);
+        assert_evals_to!("List.concat [] []", &[], &'static [i64]);
 
         assert_evals_to!(
             indoc!(
@@ -133,30 +164,30 @@ mod gen_list {
                     secondList =
                         []
 
-                    List.append firstList secondList
+                    List.concat firstList secondList
                 "#
             ),
             &[],
             &'static [i64]
         );
 
-        assert_evals_to!("List.append [ 12, 13 ] []", &[12, 13], &'static [i64]);
+        assert_evals_to!("List.concat [ 12, 13 ] []", &[12, 13], &'static [i64]);
         assert_evals_to!(
-            "List.append [ 34, 43 ] [ 64, 55, 66 ]",
+            "List.concat [ 34, 43 ] [ 64, 55, 66 ]",
             &[34, 43, 64, 55, 66],
             &'static [i64]
         );
 
-        assert_evals_to!("List.append [] [ 23, 24 ]", &[23, 24], &'static [i64]);
+        assert_evals_to!("List.concat [] [ 23, 24 ]", &[23, 24], &'static [i64]);
 
         assert_evals_to!(
-            "List.append [ 1, 2 ] [ 3, 4 ]",
+            "List.concat [ 1, 2 ] [ 3, 4 ]",
             &[1, 2, 3, 4],
             &'static [i64]
         );
     }
 
-    fn assert_append_worked(num_elems1: i64, num_elems2: i64) {
+    fn assert_concat_worked(num_elems1: i64, num_elems2: i64) {
         let vec1: Vec<i64> = (0..num_elems1)
             .map(|i| 12345 % (i + num_elems1 + num_elems2 + 1))
             .collect();
@@ -172,51 +203,51 @@ mod gen_list {
         let expected_slice: &[i64] = expected.as_ref();
 
         assert_evals_to!(
-            &format!("List.append {} {}", slice_str1, slice_str2),
+            &format!("List.concat {} {}", slice_str1, slice_str2),
             expected_slice,
             &'static [i64]
         );
     }
 
     #[test]
-    fn list_append_empty_list() {
+    fn list_concat_empty_list() {
-        assert_append_worked(0, 0);
+        assert_concat_worked(0, 0);
-        assert_append_worked(1, 0);
+        assert_concat_worked(1, 0);
-        assert_append_worked(2, 0);
+        assert_concat_worked(2, 0);
-        assert_append_worked(3, 0);
+        assert_concat_worked(3, 0);
-        assert_append_worked(4, 0);
+        assert_concat_worked(4, 0);
-        assert_append_worked(7, 0);
+        assert_concat_worked(7, 0);
-        assert_append_worked(8, 0);
+        assert_concat_worked(8, 0);
-        assert_append_worked(9, 0);
+        assert_concat_worked(9, 0);
-        assert_append_worked(25, 0);
+        assert_concat_worked(25, 0);
-        assert_append_worked(150, 0);
+        assert_concat_worked(150, 0);
-        assert_append_worked(0, 1);
+        assert_concat_worked(0, 1);
-        assert_append_worked(0, 2);
+        assert_concat_worked(0, 2);
-        assert_append_worked(0, 3);
+        assert_concat_worked(0, 3);
-        assert_append_worked(0, 4);
+        assert_concat_worked(0, 4);
-        assert_append_worked(0, 7);
+        assert_concat_worked(0, 7);
-        assert_append_worked(0, 8);
+        assert_concat_worked(0, 8);
-        assert_append_worked(0, 9);
+        assert_concat_worked(0, 9);
-        assert_append_worked(0, 25);
+        assert_concat_worked(0, 25);
-        assert_append_worked(0, 150);
+        assert_concat_worked(0, 150);
     }
 
     #[test]
-    fn list_append_nonempty_lists() {
+    fn list_concat_nonempty_lists() {
-        assert_append_worked(1, 1);
+        assert_concat_worked(1, 1);
-        assert_append_worked(1, 2);
+        assert_concat_worked(1, 2);
-        assert_append_worked(1, 3);
+        assert_concat_worked(1, 3);
-        assert_append_worked(2, 3);
+        assert_concat_worked(2, 3);
-        assert_append_worked(2, 1);
+        assert_concat_worked(2, 1);
-        assert_append_worked(2, 2);
+        assert_concat_worked(2, 2);
-        assert_append_worked(3, 1);
+        assert_concat_worked(3, 1);
-        assert_append_worked(3, 2);
+        assert_concat_worked(3, 2);
-        assert_append_worked(2, 3);
+        assert_concat_worked(2, 3);
-        assert_append_worked(3, 3);
+        assert_concat_worked(3, 3);
-        assert_append_worked(4, 4);
+        assert_concat_worked(4, 4);
-        assert_append_worked(150, 150);
+        assert_concat_worked(150, 150);
-        assert_append_worked(129, 350);
+        assert_concat_worked(129, 350);
-        assert_append_worked(350, 129);
+        assert_concat_worked(350, 129);
     }
 
     #[test]

compiler/gen/tests/helpers/eval.rs

@@ -67,7 +67,7 @@ pub fn helper_without_uniqueness<'a>(
     fpm.initialize();
 
     // Compute main_fn_type before moving subs to Env
-    let layout = Layout::new(&arena, content, &subs, ptr_bytes).unwrap_or_else(|err| {
+    let layout = Layout::new(&arena, content, &subs).unwrap_or_else(|err| {
         panic!(
             "Code gen error in NON-OPTIMIZED test: could not convert to layout. Err was {:?}",
             err
@@ -103,7 +103,6 @@ pub fn helper_without_uniqueness<'a>(
         problems: &mut mono_problems,
         home,
         ident_ids: &mut ident_ids,
-        pointer_size: ptr_bytes,
         jump_counter: arena.alloc(0),
     };
 
@@ -258,7 +257,7 @@ pub fn helper_with_uniqueness<'a>(
     fpm.initialize();
 
     // Compute main_fn_type before moving subs to Env
-    let layout = Layout::new(&arena, content, &subs, ptr_bytes).unwrap_or_else(|err| {
+    let layout = Layout::new(&arena, content, &subs).unwrap_or_else(|err| {
         panic!(
             "Code gen error in OPTIMIZED test: could not convert to layout. Err was {:?}",
             err
@@ -296,7 +295,6 @@ pub fn helper_with_uniqueness<'a>(
         problems: &mut mono_problems,
         home,
         ident_ids: &mut ident_ids,
-        pointer_size: ptr_bytes,
         jump_counter: arena.alloc(0),
     };
 

compiler/gen/tests/helpers/mod.rs

@@ -87,7 +87,7 @@ pub fn infer_expr(
 }
 
 pub fn parse_loc_with<'a>(arena: &'a Bump, input: &'a str) -> Result<Located<ast::Expr<'a>>, Fail> {
-    let state = State::new(&input, Attempting::Module);
+    let state = State::new(input.as_bytes(), Attempting::Module);
     let parser = space0_before(loc(roc_parse::expr::expr(0)), 0);
     let answer = parser.parse(&arena, state);
 

compiler/load/Cargo.toml

@@ -19,7 +19,8 @@ roc_parse = { path = "../parse" }
 roc_solve = { path = "../solve" }
 bumpalo = { version = "3.2", features = ["collections"] }
 inlinable_string = "0.1"
-tokio = { version = "0.2", features = ["blocking", "fs", "sync", "rt-threaded"] }
+crossbeam = "0.7"
+num_cpus = "1"
 
 [dev-dependencies]
 pretty_assertions = "0.5.1"

compiler/load/tests/fixtures/build/app_with_deps/AStar.roc

@@ -54,7 +54,7 @@ reconstructPath = \cameFrom, goal ->
             []
 
         Ok next ->
-            List.push (reconstructPath cameFrom next) goal
+            List.append (reconstructPath cameFrom next) goal
 
 updateCost : position, position, Model position -> Model position
 updateCost = \current, neighbour, model ->

compiler/load/tests/fixtures/build/interface_with_deps/AStar.roc

@@ -54,7 +54,7 @@ reconstructPath = \cameFrom, goal ->
             []
 
         Ok next ->
-            List.push (reconstructPath cameFrom next) goal
+            List.append (reconstructPath cameFrom next) goal
 
 updateCost : position, position, Model position -> Model position
 updateCost = \current, neighbour, model ->

compiler/load/tests/helpers/mod.rs

@@ -29,29 +29,6 @@ pub fn test_home() -> ModuleId {
     ModuleIds::default().get_or_insert(&"Test".into())
 }
 
-/// Without a larger-than-default stack size, some tests
-/// run out of stack space in debug builds (but don't in --release builds)
-#[allow(dead_code)]
-const THREAD_STACK_SIZE: usize = 4 * 1024 * 1024;
-
-pub fn test_async<F: std::future::Future>(future: F) -> F::Output {
-    use tokio::runtime::Builder;
-
-    // Create the runtime
-    let mut rt = Builder::new()
-        .thread_name("tokio-thread-for-tests")
-        .thread_stack_size(THREAD_STACK_SIZE)
-        // DEBUG: Replace this with .basic_scheduler() to make tests run single-threaded on the main thread.
-        // Doing this makes assertion failures easier to read, but means
-        // the tests can't reveal concurrency bugs, so leave this off by default!
-        .threaded_scheduler()
-        .build()
-        .expect("Error initializing Tokio runtime.");
-
-    // Spawn the root task
-    rt.block_on(future)
-}
-
 #[allow(dead_code)]
 pub fn infer_expr(
     subs: Subs,
@@ -92,7 +69,7 @@ pub fn parse_with<'a>(arena: &'a Bump, input: &'a str) -> Result<ast::Expr<'a>,
 
 #[allow(dead_code)]
 pub fn parse_loc_with<'a>(arena: &'a Bump, input: &'a str) -> Result<Located<ast::Expr<'a>>, Fail> {
-    let state = State::new(&input, Attempting::Module);
+    let state = State::new(input.as_bytes(), Attempting::Module);
     let parser = space0_before(loc(roc_parse::expr::expr(0)), 0);
     let answer = parser.parse(&arena, state);
 

compiler/load/tests/test_load.rs

@@ -13,7 +13,7 @@ mod helpers;
 
 #[cfg(test)]
 mod test_load {
-    use crate::helpers::{fixtures_dir, test_async};
+    use crate::helpers::fixtures_dir;
     use inlinable_string::InlinableString;
     use roc_can::def::Declaration::*;
     use roc_can::def::Def;
@@ -27,7 +27,7 @@ mod test_load {
 
     // HELPERS
 
-    async fn load_fixture(
+    fn load_fixture(
         dir_name: &str,
         module_name: &str,
         subs_by_module: SubsByModule,
@@ -35,12 +35,11 @@ mod test_load {
         let src_dir = fixtures_dir().join(dir_name);
         let filename = src_dir.join(format!("{}.roc", module_name));
         let loaded = load(
-            &roc_builtins::std::standard_stdlib(),
-            src_dir,
             filename,
+            &roc_builtins::std::standard_stdlib(),
+            src_dir.as_path(),
             subs_by_module,
-        )
+        );
-        .await;
         let loaded_module = loaded.expect("Test module failed to load");
 
         assert_eq!(loaded_module.can_problems, Vec::new());
@@ -129,15 +128,12 @@ mod test_load {
         let subs_by_module = MutMap::default();
         let src_dir = fixtures_dir().join("interface_with_deps");
         let filename = src_dir.join("Primary.roc");
-
-        test_async(async {
         let loaded = load(
-                &roc_builtins::std::standard_stdlib(),
-                src_dir,
             filename,
+            &roc_builtins::std::standard_stdlib(),
+            src_dir.as_path(),
             subs_by_module,
-            )
+        );
-            .await;
 
         let mut loaded_module = loaded.expect("Test module failed to load");
 
@@ -160,14 +156,12 @@ mod test_load {
 
         assert_eq!(expected_name, &InlinableString::from("Primary"));
         assert_eq!(def_count, 10);
-        });
     }
 
     #[test]
     fn load_unit() {
-        test_async(async {
         let subs_by_module = MutMap::default();
-            let loaded_module = load_fixture("no_deps", "Unit", subs_by_module).await;
+        let loaded_module = load_fixture("no_deps", "Unit", subs_by_module);
 
         expect_types(
             loaded_module,
@@ -175,15 +169,12 @@ mod test_load {
                 "unit" => "Unit",
             },
         );
-        });
     }
 
     #[test]
     fn import_alias() {
-        test_async(async {
         let subs_by_module = MutMap::default();
-            let loaded_module =
+        let loaded_module = load_fixture("interface_with_deps", "ImportAlias", subs_by_module);
-                load_fixture("interface_with_deps", "ImportAlias", subs_by_module).await;
 
         expect_types(
             loaded_module,
@@ -191,15 +182,12 @@ mod test_load {
                 "unit" => "Dep1.Unit",
             },
         );
-        });
     }
 
     #[test]
     fn load_and_typecheck() {
-        test_async(async {
         let subs_by_module = MutMap::default();
-            let loaded_module =
+        let loaded_module = load_fixture("interface_with_deps", "WithBuiltins", subs_by_module);
-                load_fixture("interface_with_deps", "WithBuiltins", subs_by_module).await;
 
         expect_types(
             loaded_module,
@@ -214,15 +202,12 @@ mod test_load {
                 "fromDep2" => "Float",
             },
         );
-        });
     }
 
     #[test]
     fn iface_quicksort() {
-        test_async(async {
         let subs_by_module = MutMap::default();
-            let loaded_module =
+        let loaded_module = load_fixture("interface_with_deps", "Quicksort", subs_by_module);
-                load_fixture("interface_with_deps", "Quicksort", subs_by_module).await;
 
         expect_types(
             loaded_module,
@@ -232,14 +217,12 @@ mod test_load {
                 "quicksort" => "List (Num a), Int, Int -> List (Num a)",
             },
         );
-        });
     }
 
     #[test]
     fn app_quicksort() {
-        test_async(async {
         let subs_by_module = MutMap::default();
-            let loaded_module = load_fixture("app_with_deps", "Quicksort", subs_by_module).await;
+        let loaded_module = load_fixture("app_with_deps", "Quicksort", subs_by_module);
 
         expect_types(
             loaded_module,
@@ -249,14 +232,12 @@ mod test_load {
                 "quicksort" => "List (Num a), Int, Int -> List (Num a)",
             },
         );
-        });
     }
 
     #[test]
     fn load_astar() {
-        test_async(async {
         let subs_by_module = MutMap::default();
-            let loaded_module = load_fixture("interface_with_deps", "AStar", subs_by_module).await;
+        let loaded_module = load_fixture("interface_with_deps", "AStar", subs_by_module);
 
         expect_types(
             loaded_module,
@@ -269,14 +250,12 @@ mod test_load {
                 "astar" => "(position, position -> Float), (position -> Set position), position, Model position -> [ Err [ KeyNotFound ]*, Ok (List position) ]*",
             },
         );
-        });
     }
 
     #[test]
     fn load_principal_types() {
-        test_async(async {
         let subs_by_module = MutMap::default();
-            let loaded_module = load_fixture("no_deps", "Principal", subs_by_module).await;
+        let loaded_module = load_fixture("no_deps", "Principal", subs_by_module);
 
         expect_types(
             loaded_module,
@@ -285,15 +264,12 @@ mod test_load {
                 "identity" => "a -> a",
             },
         );
-        });
     }
 
     #[test]
     fn iface_dep_types() {
-        test_async(async {
         let subs_by_module = MutMap::default();
-            let loaded_module =
+        let loaded_module = load_fixture("interface_with_deps", "Primary", subs_by_module);
-                load_fixture("interface_with_deps", "Primary", subs_by_module).await;
 
         expect_types(
             loaded_module,
@@ -310,14 +286,12 @@ mod test_load {
                 "withDefault" => "Res.Res a *, a -> a",
             },
         );
-        });
     }
 
     #[test]
     fn app_dep_types() {
-        test_async(async {
         let subs_by_module = MutMap::default();
-            let loaded_module = load_fixture("app_with_deps", "Primary", subs_by_module).await;
+        let loaded_module = load_fixture("app_with_deps", "Primary", subs_by_module);
 
         expect_types(
             loaded_module,
@@ -334,14 +308,12 @@ mod test_load {
                 "withDefault" => "Res.Res a *, a -> a",
             },
         );
-        });
     }
 
     #[test]
     fn imported_dep_regression() {
-        test_async(async {
         let subs_by_module = MutMap::default();
-            let loaded_module = load_fixture("interface_with_deps", "OneDep", subs_by_module).await;
+        let loaded_module = load_fixture("interface_with_deps", "OneDep", subs_by_module);
 
         expect_types(
             loaded_module,
@@ -349,17 +321,15 @@ mod test_load {
                 "str" => "Str",
             },
         );
-        });
     }
 
     // #[test]
     // fn load_records() {
-    //     test_async(async {
     //     use roc::types::{ErrorType, Mismatch, Problem, TypeExt};
 
     //     let subs_by_module = MutMap::default();
     //     let loaded_module =
-    //             load_fixture("interface_with_deps", "Records", subs_by_module).await;
+    //         load_fixture("interface_with_deps", "Records", subs_by_module);
 
     //     // NOTE: `a` here is unconstrained, so unifies with <type error>
     //     let expected_types = hashmap! {
@@ -409,6 +379,5 @@ mod test_load {
     //             assert_eq!((&symbol, expected_type), (&symbol, &actual_str.as_str()));
     //         }
     //     }
-    //     });
     // }
 }

compiler/load/tests/test_uniq_load.rs

@@ -13,7 +13,7 @@ mod helpers;
 
 #[cfg(test)]
 mod test_uniq_load {
-    use crate::helpers::{fixtures_dir, test_async};
+    use crate::helpers::fixtures_dir;
     use inlinable_string::InlinableString;
     use roc_builtins::unique;
     use roc_can::def::Declaration::*;
@@ -28,14 +28,19 @@ mod test_uniq_load {
 
     // HELPERS
 
-    async fn load_fixture(
+    fn load_fixture(
         dir_name: &str,
         module_name: &str,
         subs_by_module: SubsByModule,
     ) -> LoadedModule {
         let src_dir = fixtures_dir().join(dir_name);
         let filename = src_dir.join(format!("{}.roc", module_name));
-        let loaded = load(&unique::uniq_stdlib(), src_dir, filename, subs_by_module).await;
+        let loaded = load(
+            filename,
+            &unique::uniq_stdlib(),
+            src_dir.as_path(),
+            subs_by_module,
+        );
         let loaded_module = loaded.expect("Test module failed to load");
 
         assert_eq!(loaded_module.can_problems, Vec::new());
@@ -124,15 +129,12 @@ mod test_uniq_load {
         let subs_by_module = MutMap::default();
         let src_dir = fixtures_dir().join("interface_with_deps");
         let filename = src_dir.join("Primary.roc");
-
-        test_async(async {
         let loaded = load(
-                &roc_builtins::std::standard_stdlib(),
-                src_dir,
             filename,
+            &roc_builtins::std::standard_stdlib(),
+            src_dir.as_path(),
             subs_by_module,
-            )
+        );
-            .await;
 
         let mut loaded_module = loaded.expect("Test module failed to load");
 
@@ -155,14 +157,12 @@ mod test_uniq_load {
 
         assert_eq!(expected_name, &InlinableString::from("Primary"));
         assert_eq!(def_count, 10);
-        });
     }
 
     #[test]
     fn load_unit() {
-        test_async(async {
         let subs_by_module = MutMap::default();
-            let loaded_module = load_fixture("no_deps", "Unit", subs_by_module).await;
+        let loaded_module = load_fixture("no_deps", "Unit", subs_by_module);
 
         expect_types(
             loaded_module,
@@ -170,15 +170,12 @@ mod test_uniq_load {
                 "unit" => "Attr * Unit",
             },
         );
-        });
     }
 
     #[test]
     fn import_alias() {
-        test_async(async {
         let subs_by_module = MutMap::default();
-            let loaded_module =
+        let loaded_module = load_fixture("interface_with_deps", "ImportAlias", subs_by_module);
-                load_fixture("interface_with_deps", "ImportAlias", subs_by_module).await;
 
         expect_types(
             loaded_module,
@@ -186,15 +183,12 @@ mod test_uniq_load {
                 "unit" => "Attr * Dep1.Unit",
             },
         );
-        });
     }
 
     #[test]
     fn load_and_typecheck() {
-        test_async(async {
         let subs_by_module = MutMap::default();
-            let loaded_module =
+        let loaded_module = load_fixture("interface_with_deps", "WithBuiltins", subs_by_module);
-                load_fixture("interface_with_deps", "WithBuiltins", subs_by_module).await;
 
         expect_types(
             loaded_module,
@@ -209,14 +203,12 @@ mod test_uniq_load {
                 "fromDep2" => "Attr * Float",
             },
         );
-        });
     }
 
     #[test]
     fn load_astar() {
-        test_async(async {
         let subs_by_module = MutMap::default();
-            let loaded_module = load_fixture("interface_with_deps", "AStar", subs_by_module).await;
+        let loaded_module = load_fixture("interface_with_deps", "AStar", subs_by_module);
 
         expect_types(
             loaded_module,
@@ -229,15 +221,12 @@ mod test_uniq_load {
                 "astar" => "Attr Shared (Attr Shared (Attr Shared position, Attr Shared position -> Attr * Float), Attr Shared (Attr Shared position -> Attr * (Set (Attr * position))), Attr Shared position, Attr Shared (Model (Attr Shared position)) -> Attr * [ Err (Attr * [ KeyNotFound ]*), Ok (Attr * (List (Attr Shared position))) ]*)",
             },
         );
-        });
     }
 
     #[test]
     fn load_and_typecheck_quicksort() {
-        test_async(async {
         let subs_by_module = MutMap::default();
-            let loaded_module =
+        let loaded_module = load_fixture("interface_with_deps", "Quicksort", subs_by_module);
-                load_fixture("interface_with_deps", "Quicksort", subs_by_module).await;
 
         expect_types(
             loaded_module,
@@ -247,14 +236,12 @@ mod test_uniq_load {
                 "quicksort" => "Attr Shared (Attr b (List (Attr Shared (Num (Attr Shared a)))), Attr Shared Int, Attr Shared Int -> Attr b (List (Attr Shared (Num (Attr Shared a)))))",
             },
         );
-        });
     }
 
     #[test]
     fn load_principal_types() {
-        test_async(async {
         let subs_by_module = MutMap::default();
-            let loaded_module = load_fixture("no_deps", "Principal", subs_by_module).await;
+        let loaded_module = load_fixture("no_deps", "Principal", subs_by_module);
 
         expect_types(
             loaded_module,
@@ -263,15 +250,12 @@ mod test_uniq_load {
                 "identity" => "Attr * (a -> a)",
             },
         );
-        });
     }
 
     #[test]
     fn load_dep_types() {
-        test_async(async {
         let subs_by_module = MutMap::default();
-            let loaded_module =
+        let loaded_module = load_fixture("interface_with_deps", "Primary", subs_by_module);
-                load_fixture("interface_with_deps", "Primary", subs_by_module).await;
 
         // the inferred signature for withDefault is wrong, part of the alias in alias issue.
         // "withDefault" => "Attr * (Attr * (Res.Res (Attr a b) (Attr * *)), Attr a b -> Attr a b)",
@@ -290,14 +274,12 @@ mod test_uniq_load {
                 "withDefault" => "Attr * (Attr (* | b | c) (Res.Res (Attr b a) (Attr c *)), Attr b a -> Attr b a)",
             },
         );
-        });
     }
 
     #[test]
     fn load_custom_res() {
-        test_async(async {
         let subs_by_module = MutMap::default();
-            let loaded_module = load_fixture("interface_with_deps", "Res", subs_by_module).await;
+        let loaded_module = load_fixture("interface_with_deps", "Res", subs_by_module);
 
         expect_types(
             loaded_module,
@@ -307,14 +289,12 @@ mod test_uniq_load {
                 "andThen" => "Attr * (Attr (* | c | d) (Res (Attr c a) (Attr d err)), Attr * (Attr c a -> Attr f (Res (Attr e b) (Attr d err))) -> Attr f (Res (Attr e b) (Attr d err)))",
             },
         );
-        });
     }
 
     #[test]
     fn imported_dep_regression() {
-        test_async(async {
         let subs_by_module = MutMap::default();
-            let loaded_module = load_fixture("interface_with_deps", "OneDep", subs_by_module).await;
+        let loaded_module = load_fixture("interface_with_deps", "OneDep", subs_by_module);
 
         expect_types(
             loaded_module,
@@ -322,7 +302,6 @@ mod test_uniq_load {
                 "str" => "Attr * Str",
             },
         );
-        });
     }
 
     // #[test]
@@ -332,7 +311,7 @@ mod test_uniq_load {
 
     //         let subs_by_module = MutMap::default();
     //         let loaded_module =
-    //             load_fixture("interface_with_deps", "Records", subs_by_module).await;
+    //             load_fixture("interface_with_deps", "Records", subs_by_module);
 
     //         // NOTE: `a` here is unconstrained, so unifies with <type error>
     //         let expected_types = hashmap! {

compiler/module/src/low_level.rs

@@ -10,8 +10,9 @@ pub enum LowLevel {
     ListSingle,
     ListRepeat,
     ListReverse,
+    ListConcat,
     ListAppend,
-    ListPush,
+    ListPrepend,
     NumAdd,
     NumSub,
     NumMul,

compiler/module/src/symbol.rs

@@ -659,7 +659,7 @@ define_builtins! {
         2 LIST_IS_EMPTY: "isEmpty"
         3 LIST_GET: "get"
         4 LIST_SET: "set"
-        5 LIST_PUSH: "push"
+        5 LIST_APPEND: "append"
         6 LIST_MAP: "map"
         7 LIST_LEN: "len"
         8 LIST_FOLDL: "foldl"
@@ -669,7 +669,7 @@ define_builtins! {
         12 LIST_SINGLE: "single"
         13 LIST_REPEAT: "repeat"
         14 LIST_REVERSE: "reverse"
-        15 LIST_APPEND: "append"
+        15 LIST_PREPEND: "prepend"
     }
     5 RESULT: "Result" => {
         0 RESULT_RESULT: "Result" imported // the Result.Result type alias

compiler/mono/src/decision_tree.rs

@@ -154,9 +154,8 @@ fn to_decision_tree(raw_branches: Vec<Branch>) -> DecisionTree {
     match check_for_match(&branches) {
         Some(goal) => DecisionTree::Match(goal),
         None => {
-            // TODO remove clone
+            // must clone here to release the borrow on `branches`
-            let path = pick_path(branches.clone());
+            let path = pick_path(&branches).clone();
-
             let (edges, fallback) = gather_edges(branches, &path);
 
             let mut decision_edges: Vec<_> = edges
@@ -218,15 +217,16 @@ fn flatten<'a>(
     path_pattern: (Path, Guard<'a>, Pattern<'a>),
     path_patterns: &mut Vec<(Path, Guard<'a>, Pattern<'a>)>,
 ) {
-    match &path_pattern.2 {
+    match path_pattern.2 {
         Pattern::AppliedTag {
             union,
             arguments,
             tag_id,
-            ..
+            tag_name,
-        } => {
+            layout,
-            // TODO do we need to check that guard.is_none() here?
+        } if union.alternatives.len() == 1 => {
-            if union.alternatives.len() == 1 {
+            // TODO ^ do we need to check that guard.is_none() here?
+
             let path = path_pattern.0;
             // Theory: unbox doesn't have any value for us, because one-element tag unions
             // don't store the tag anyway.
@@ -234,7 +234,13 @@ fn flatten<'a>(
                 path_patterns.push((
                     Path::Unbox(Box::new(path)),
                     path_pattern.1.clone(),
-                        path_pattern.2.clone(),
+                    Pattern::AppliedTag {
+                        union,
+                        arguments,
+                        tag_id,
+                        tag_name,
+                        layout,
+                    },
                 ));
             } else {
                 for (index, (arg_pattern, _)) in arguments.iter().enumerate() {
@@ -242,7 +248,7 @@ fn flatten<'a>(
                         (
                             Path::Index {
                                 index: index as u64,
-                                    tag_id: *tag_id,
+                                tag_id,
                                 path: Box::new(path.clone()),
                             },
                             // same guard here?
@@ -253,9 +259,6 @@ fn flatten<'a>(
                     );
                 }
             }
-            } else {
-                path_patterns.push(path_pattern);
-            }
         }
 
         _ => {
@@ -289,8 +292,7 @@ fn gather_edges<'a>(
     branches: Vec<Branch<'a>>,
     path: &Path,
 ) -> (Vec<(Test<'a>, Vec<Branch<'a>>)>, Vec<Branch<'a>>) {
-    // TODO remove clone
+    let relevant_tests = tests_at_path(path, &branches);
-    let relevant_tests = tests_at_path(path, branches.clone());
 
     let check = is_complete(&relevant_tests);
 
@@ -314,12 +316,12 @@ fn gather_edges<'a>(
 
 /// FIND RELEVANT TESTS
 
-fn tests_at_path<'a>(selected_path: &Path, branches: Vec<Branch<'a>>) -> Vec<Test<'a>> {
+fn tests_at_path<'a>(selected_path: &Path, branches: &[Branch<'a>]) -> Vec<Test<'a>> {
     // NOTE the ordering of the result is important!
 
     let mut all_tests = Vec::new();
 
-    for branch in branches.into_iter() {
+    for branch in branches {
         test_at_path(selected_path, branch, &mut all_tests);
     }
 
@@ -348,7 +350,7 @@ fn tests_at_path<'a>(selected_path: &Path, branches: Vec<Branch<'a>>) -> Vec<Tes
     unique
 }
 
-fn test_at_path<'a>(selected_path: &Path, branch: Branch<'a>, all_tests: &mut Vec<Test<'a>>) {
+fn test_at_path<'a>(selected_path: &Path, branch: &Branch<'a>, all_tests: &mut Vec<Test<'a>>) {
     use Pattern::*;
     use Test::*;
 
@@ -466,7 +468,7 @@ fn edges_for<'a>(
 ) -> (Test<'a>, Vec<Branch<'a>>) {
     let mut new_branches = Vec::new();
 
-    for branch in branches.into_iter() {
+    for branch in branches.iter() {
         to_relevant_branch(&test, path, branch, &mut new_branches);
     }
 
@@ -476,13 +478,13 @@ fn edges_for<'a>(
 fn to_relevant_branch<'a>(
     test: &Test<'a>,
     path: &Path,
-    branch: Branch<'a>,
+    branch: &Branch<'a>,
     new_branches: &mut Vec<Branch<'a>>,
 ) {
     // TODO remove clone
     match extract(path, branch.patterns.clone()) {
         Extract::NotFound => {
-            new_branches.push(branch);
+            new_branches.push(branch.clone());
         }
         Extract::Found {
             start,
@@ -518,7 +520,7 @@ fn to_relevant_branch_help<'a>(
     path: &Path,
     mut start: Vec<(Path, Guard<'a>, Pattern<'a>)>,
     end: Vec<(Path, Guard<'a>, Pattern<'a>)>,
-    branch: Branch<'a>,
+    branch: &Branch<'a>,
     guard: Guard<'a>,
     pattern: Pattern<'a>,
 ) -> Option<Branch<'a>> {
@@ -526,7 +528,7 @@ fn to_relevant_branch_help<'a>(
     use Test::*;
 
     match pattern {
-        Identifier(_) | Underscore | Shadowed(_, _) | UnsupportedPattern(_) => Some(branch),
+        Identifier(_) | Underscore | Shadowed(_, _) | UnsupportedPattern(_) => Some(branch.clone()),
 
         RecordDestructure(destructs, _) => match test {
             IsCtor {
@@ -689,19 +691,14 @@ fn extract<'a>(
 ) -> Extract<'a> {
     let mut start = Vec::new();
 
-    // TODO remove this clone
-    let mut copy = path_patterns.clone();
-
     // TODO potential ordering problem
-    for (index, current) in path_patterns.into_iter().enumerate() {
+    let mut it = path_patterns.into_iter();
+    while let Some(current) = it.next() {
         if &current.0 == selected_path {
             return Extract::Found {
                 start,
                 found_pattern: (current.1, current.2),
-                end: {
+                end: it.collect::<Vec<_>>(),
-                    copy.drain(0..=index);
-                    copy
-                },
             };
         } else {
             start.push(current);
@@ -742,22 +739,27 @@ fn needs_tests<'a>(pattern: &Pattern<'a>) -> bool {
 
 /// PICK A PATH
 
-fn pick_path(branches: Vec<Branch>) -> Path {
+fn pick_path<'a>(branches: &'a [Branch]) -> &'a Path {
-    // TODO remove this clone
+    let mut all_paths = Vec::with_capacity(branches.len());
-    let all_paths = branches
-        .clone()
-        .into_iter()
-        .map(|v| v.patterns)
-        .flatten()
-        .filter_map(is_choice_path);
 
-    let mut by_small_defaults = bests_by_small_defaults(&branches, all_paths);
+    // is choice path
+    for branch in branches {
+        for (path, guard, pattern) in &branch.patterns {
+            if !guard.is_none() || needs_tests(&pattern) {
+                all_paths.push(path);
+            } else {
+                // do nothing
+            }
+        }
+    }
+
+    let mut by_small_defaults = bests_by_small_defaults(branches, all_paths.into_iter());
 
     if by_small_defaults.len() == 1 {
         by_small_defaults.remove(0)
     } else {
         debug_assert!(!by_small_defaults.is_empty());
-        let mut result = bests_by_small_branching_factor(&branches, by_small_defaults.into_iter());
+        let mut result = bests_by_small_branching_factor(branches, by_small_defaults.into_iter());
 
         match result.pop() {
             None => unreachable!("bests_by will always return at least one value in the vec"),
@@ -766,33 +768,23 @@ fn pick_path(branches: Vec<Branch>) -> Path {
     }
 }
 
-fn is_choice_path<'a>(path_and_pattern: (Path, Guard<'a>, Pattern<'a>)) -> Option<Path> {
+fn bests_by_small_branching_factor<'a, I>(branches: &[Branch], mut all_paths: I) -> Vec<&'a Path>
-    let (path, guard, pattern) = path_and_pattern;
-
-    if !guard.is_none() || needs_tests(&pattern) {
-        Some(path)
-    } else {
-        None
-    }
-}
-
-fn bests_by_small_branching_factor<I>(branches: &Vec<Branch>, mut all_paths: I) -> Vec<Path>
 where
-    I: Iterator<Item = Path>,
+    I: Iterator<Item = &'a Path>,
 {
     match all_paths.next() {
         None => panic!("Cannot choose the best of zero paths. This should never happen."),
         Some(first_path) => {
-            let mut min_weight = small_branching_factor(branches, &first_path);
+            let mut min_weight = small_branching_factor(branches, first_path);
             let mut min_paths = vec![first_path];
 
             for path in all_paths {
-                let weight = small_branching_factor(branches, &path);
+                let weight = small_branching_factor(branches, path);
 
                 use std::cmp::Ordering;
                 match weight.cmp(&min_weight) {
                     Ordering::Equal => {
-                        min_paths.push(path.clone());
+                        min_paths.push(path);
                     }
                     Ordering::Less => {
                         min_weight = weight;
@@ -808,14 +800,14 @@ where
     }
 }
 
-fn bests_by_small_defaults<I>(branches: &Vec<Branch>, mut all_paths: I) -> Vec<Path>
+fn bests_by_small_defaults<'a, I>(branches: &[Branch], mut all_paths: I) -> Vec<&'a Path>
 where
-    I: Iterator<Item = Path>,
+    I: Iterator<Item = &'a Path>,
 {
     match all_paths.next() {
         None => panic!("Cannot choose the best of zero paths. This should never happen."),
         Some(first_path) => {
-            let mut min_weight = small_defaults(branches, &first_path);
+            let mut min_weight = small_defaults(branches, first_path);
             let mut min_paths = vec![first_path];
 
             for path in all_paths {
@@ -824,7 +816,7 @@ where
                 use std::cmp::Ordering;
                 match weight.cmp(&min_weight) {
                     Ordering::Equal => {
-                        min_paths.push(path.clone());
+                        min_paths.push(path);
                     }
                     Ordering::Less => {
                         min_weight = weight;
@@ -842,7 +834,7 @@ where
 
 /// PATH PICKING HEURISTICS
 
-fn small_defaults(branches: &Vec<Branch>, path: &Path) -> usize {
+fn small_defaults(branches: &[Branch], path: &Path) -> usize {
     branches
         .iter()
         .filter(|b| is_irrelevant_to(path, b))
@@ -850,7 +842,7 @@ fn small_defaults(branches: &Vec<Branch>, path: &Path) -> usize {
         .sum()
 }
 
-fn small_branching_factor(branches: &Vec<Branch>, path: &Path) -> usize {
+fn small_branching_factor(branches: &[Branch], path: &Path) -> usize {
     // TODO remove clone
     let (edges, fallback) = gather_edges(branches.to_vec(), path);
 

compiler/mono/src/exhaustive.rs

@@ -53,7 +53,6 @@ fn simplify<'a>(pattern: &crate::ir::Pattern<'a>) -> Pattern {
         StrLiteral(v) => Literal(Literal::Str(v.clone())),
 
         // To make sure these are exhaustive, we have to "fake" a union here
-        // TODO: use the hash or some other integer to discriminate between constructors
         BitLiteral { value, union, .. } => Ctor(union.clone(), TagId(*value as u8), vec![]),
         EnumLiteral { tag_id, union, .. } => Ctor(union.clone(), TagId(*tag_id), vec![]),
 
@@ -217,7 +216,7 @@ fn is_exhaustive(matrix: &PatternMatrix, n: usize) -> PatternMatrix {
 
                 let last: _ = alt_list
                     .iter()
-                    .filter_map(|r| is_missing(alts.clone(), ctors.clone(), r));
+                    .filter_map(|r| is_missing(alts.clone(), &ctors, r));
 
                 let mut result = Vec::new();
 
@@ -257,7 +256,7 @@ fn is_exhaustive(matrix: &PatternMatrix, n: usize) -> PatternMatrix {
     }
 }
 
-fn is_missing<T>(union: Union, ctors: MutMap<TagId, T>, ctor: &Ctor) -> Option<Pattern> {
+fn is_missing<T>(union: Union, ctors: &MutMap<TagId, T>, ctor: &Ctor) -> Option<Pattern> {
     let Ctor { arity, tag_id, .. } = ctor;
 
     if ctors.contains_key(tag_id) {
@@ -336,7 +335,7 @@ fn to_nonredundant_rows<'a>(
             vec![simplify(&loc_pat.value)]
         };
 
-        if is_useful(&checked_rows, &next_row) {
+        if is_useful(checked_rows.clone(), next_row.clone()) {
             checked_rows.push(next_row);
         } else {
             return Err(Error::Redundant {
@@ -351,80 +350,139 @@ fn to_nonredundant_rows<'a>(
 }
 
 /// Check if a new row "vector" is useful given previous rows "matrix"
-fn is_useful(matrix: &PatternMatrix, vector: &Row) -> bool {
+fn is_useful(mut old_matrix: PatternMatrix, mut vector: Row) -> bool {
-    if matrix.is_empty() {
+    let mut matrix = Vec::with_capacity(old_matrix.len());
+
+    // this loop ping-pongs the rows between old_matrix and matrix
+    'outer: loop {
+        match vector.pop() {
+            _ if old_matrix.is_empty() => {
                 // No rows are the same as the new vector! The vector is useful!
-        true
+                break true;
-    } else if vector.is_empty() {
+            }
+            None => {
                 // There is nothing left in the new vector, but we still have
                 // rows that match the same things. This is not a useful vector!
-        false
+                break false;
-    } else {
+            }
+            Some(first_pattern) => {
                 // NOTE: if there are bugs in this code, look at the ordering of the row/matrix
-        let mut vector = vector.clone();
-        let first_pattern = vector.remove(0);
-        let patterns = vector;
 
                 match first_pattern {
                     // keep checking rows that start with this Ctor or Anything
                     Ctor(_, id, args) => {
-                let new_matrix: Vec<_> = matrix
+                        specialize_row_by_ctor2(id, args.len(), &mut old_matrix, &mut matrix);
-                    .iter()
-                    .filter_map(|r| specialize_row_by_ctor(id, args.len(), r))
-                    .collect();
 
-                let mut new_row = Vec::new();
+                        std::mem::swap(&mut old_matrix, &mut matrix);
-                new_row.extend(patterns);
-                new_row.extend(args);
 
-                is_useful(&new_matrix, &new_row)
+                        vector.extend(args);
                     }
 
                     Anything => {
-                // check if all alts appear in matrix
+                        // check if all alternatives appear in matrix
-                match is_complete(matrix) {
+                        match is_complete(&old_matrix) {
                             Complete::No => {
                                 // This Anything is useful because some Ctors are missing.
                                 // But what if a previous row has an Anything?
                                 // If so, this one is not useful.
-                        let new_matrix: Vec<_> = matrix
+                                for mut row in old_matrix.drain(..) {
-                            .iter()
+                                    if let Some(Anything) = row.pop() {
-                            .filter_map(|r| specialize_row_by_anything(r))
+                                        matrix.push(row);
-                            .collect();
-
-                        is_useful(&new_matrix, &patterns)
                                     }
-                    Complete::Yes(alts) => {
+                                }
+
+                                std::mem::swap(&mut old_matrix, &mut matrix);
+                            }
+                            Complete::Yes(alternatives) => {
                                 // All Ctors are covered, so this Anything is not needed for any
                                 // of those. But what if some of those Ctors have subpatterns
                                 // that make them less general? If so, this actually is useful!
-                        let is_useful_alt = |Ctor { arity, tag_id, .. }| {
+                                for alternative in alternatives {
-                            let new_matrix = matrix
+                                    let Ctor { arity, tag_id, .. } = alternative;
-                                .iter()
-                                .filter_map(|r| specialize_row_by_ctor(tag_id, arity, r))
-                                .collect();
-                            let mut new_row: Vec<Pattern> =
-                                std::iter::repeat(Anything).take(arity).collect::<Vec<_>>();
 
-                            new_row.extend(patterns.clone());
+                                    let mut old_matrix = old_matrix.clone();
+                                    let mut matrix = vec![];
+                                    specialize_row_by_ctor2(
+                                        tag_id,
+                                        arity,
+                                        &mut old_matrix,
+                                        &mut matrix,
+                                    );
 
-                            is_useful(&new_matrix, &new_row)
+                                    let mut vector = vector.clone();
-                        };
+                                    vector.extend(std::iter::repeat(Anything).take(arity));
 
-                        alts.iter().cloned().any(is_useful_alt)
+                                    if is_useful(matrix, vector) {
+                                        break 'outer true;
+                                    }
+                                }
+
+                                break false;
                             }
                         }
                     }
 
                     Literal(literal) => {
                         // keep checking rows that start with this Literal or Anything
-                let new_matrix = matrix
+
-                    .iter()
+                        for mut row in old_matrix.drain(..) {
-                    .filter_map(|r| specialize_row_by_literal(&literal, r))
+                            let head = row.pop();
-                    .collect();
+                            let patterns = row;
-                is_useful(&new_matrix, &patterns)
+
+                            match head {
+                                Some(Literal(lit)) => {
+                                    if lit == literal {
+                                        matrix.push(patterns);
+                                    } else {
+                                        // do nothing
                                     }
                                 }
+                                Some(Anything) => matrix.push(patterns),
+
+                                Some(Ctor(_, _, _)) => panic!(
+                                    r#"Compiler bug! After type checking, constructors and literals should never align in pattern match exhaustiveness checks."#
+                                ),
+
+                                None => panic!(
+                                    "Compiler error! Empty matrices should not get specialized."
+                                ),
+                            }
+                        }
+                        std::mem::swap(&mut old_matrix, &mut matrix);
+                    }
+                }
+            }
+        }
+    }
+}
+
+/// INVARIANT: (length row == N) ==> (length result == arity + N - 1)
+fn specialize_row_by_ctor2(
+    tag_id: TagId,
+    arity: usize,
+    old_matrix: &mut PatternMatrix,
+    matrix: &mut PatternMatrix,
+) {
+    for mut row in old_matrix.drain(..) {
+        let head = row.pop();
+        let mut patterns = row;
+
+        match head {
+        Some(Ctor(_, id, args)) =>
+            if id == tag_id {
+                patterns.extend(args);
+                matrix.push(patterns);
+            } else {
+                // do nothing 
+            }
+        Some(Anything) => {
+            // TODO order!
+            patterns.extend(std::iter::repeat(Anything).take(arity));
+            matrix.push(patterns);
+            }
+        Some(Literal(_)) => panic!( "Compiler bug! After type checking, constructors and literal should never align in pattern match exhaustiveness checks."),
+        None => panic!("Compiler error! Empty matrices should not get specialized."),
+    }
     }
 }
 
@@ -436,7 +494,7 @@ fn specialize_row_by_ctor(tag_id: TagId, arity: usize, row: &Row) -> Option<Row>
     let patterns = row;
 
     match head {
-        Some(Ctor(_, id, args)) =>
+        Some(Ctor(_, id, args)) => {
             if id == tag_id {
                 // TODO order!
                 let mut new_patterns = Vec::new();
@@ -446,38 +504,18 @@ fn specialize_row_by_ctor(tag_id: TagId, arity: usize, row: &Row) -> Option<Row>
             } else {
                 None
             }
+        }
         Some(Anything) => {
             // TODO order!
-            let new_patterns =
+            let new_patterns = std::iter::repeat(Anything)
-                    std::iter::repeat(Anything).take(arity).chain(patterns).collect();
+                .take(arity)
+                .chain(patterns)
+                .collect();
             Some(new_patterns)
         }
-        Some(Literal(_)) => panic!( "Compiler bug! After type checking, constructors and literal should never align in pattern match exhaustiveness checks."),
+        Some(Literal(_)) => unreachable!(
-        None => panic!("Compiler error! Empty matrices should not get specialized."),
+            r#"Compiler bug! After type checking, a constructor can never align with a literal: that should be a type error!"#
-    }
-}
-
-/// INVARIANT: (length row == N) ==> (length result == N-1)
-fn specialize_row_by_literal(literal: &Literal, row: &Row) -> Option<Row> {
-    let mut row = row.clone();
-
-    let head = row.pop();
-    let patterns = row;
-
-    match head {
-        Some(Literal(lit)) => {
-            if &lit == literal {
-                Some(patterns)
-            } else {
-                None
-            }
-        }
-        Some(Anything) => Some(patterns),
-
-        Some(Ctor(_, _, _)) => panic!(
-            r#"Compiler bug! After type checking, constructors and literals should never align in pattern match exhaustiveness checks."#
         ),
-
         None => panic!("Compiler error! Empty matrices should not get specialized."),
     }
 }
@@ -501,14 +539,14 @@ pub enum Complete {
 
 fn is_complete(matrix: &PatternMatrix) -> Complete {
     let ctors = collect_ctors(matrix);
-
+    let length = ctors.len();
-    let mut it = ctors.values();
+    let mut it = ctors.into_iter();
 
     match it.next() {
         None => Complete::No,
-        Some(Union { alternatives, .. }) => {
+        Some((_, Union { alternatives, .. })) => {
-            if ctors.len() == alternatives.len() {
+            if length == alternatives.len() {
-                Complete::Yes(alternatives.to_vec())
+                Complete::Yes(alternatives)
             } else {
                 Complete::No
             }

compiler/mono/src/ir.rs

@@ -168,7 +168,7 @@ impl<'a> Procs<'a> {
                 // by the surrounding context, so we can add pending specializations
                 // for them immediately.
                 let layout = layout_cache
-                    .from_var(env.arena, annotation, env.subs, env.pointer_size)
+                    .from_var(env.arena, annotation, env.subs)
                     .unwrap_or_else(|err| panic!("TODO turn fn_var into a RuntimeError {:?}", err));
 
                 // if we've already specialized this one, no further work is needed.
@@ -306,7 +306,6 @@ pub struct Env<'a, 'i> {
     pub problems: &'i mut std::vec::Vec<MonoProblem>,
     pub home: ModuleId,
     pub ident_ids: &'i mut IdentIds,
-    pub pointer_size: u32,
     pub jump_counter: &'a mut u64,
 }
 
@@ -976,13 +975,13 @@ fn specialize<'a>(
     );
 
     for (arg_var, arg_name) in pattern_vars.iter().zip(pattern_symbols.iter()) {
-        let layout = layout_cache.from_var(&env.arena, *arg_var, env.subs, env.pointer_size)?;
+        let layout = layout_cache.from_var(&env.arena, *arg_var, env.subs)?;
 
         proc_args.push((layout, *arg_name));
     }
 
     let ret_layout = layout_cache
-        .from_var(&env.arena, ret_var, env.subs, env.pointer_size)
+        .from_var(&env.arena, ret_var, env.subs)
         .unwrap_or_else(|err| panic!("TODO handle invalid function {:?}", err));
 
     // TODO WRONG
@@ -1107,12 +1106,7 @@ pub fn with_hole<'a>(
             use crate::layout::UnionVariant::*;
             let arena = env.arena;
 
-            let variant = crate::layout::union_sorted_tags(
+            let variant = crate::layout::union_sorted_tags(env.arena, variant_var, env.subs);
-                env.arena,
-                variant_var,
-                env.subs,
-                env.pointer_size,
-            );
 
             match variant {
                 Never => unreachable!("The `[]` type has no constructors"),
@@ -1150,7 +1144,7 @@ pub fn with_hole<'a>(
 
                     // Layout will unpack this unwrapped tack if it only has one (non-zero-sized) field
                     let layout = layout_cache
-                        .from_var(env.arena, variant_var, env.subs, env.pointer_size)
+                        .from_var(env.arena, variant_var, env.subs)
                         .unwrap_or_else(|err| {
                             panic!("TODO turn fn_var into a RuntimeError {:?}", err)
                         });
@@ -1255,12 +1249,7 @@ pub fn with_hole<'a>(
             mut fields,
             ..
         } => {
-            let sorted_fields = crate::layout::sort_record_fields(
+            let sorted_fields = crate::layout::sort_record_fields(env.arena, record_var, env.subs);
-                env.arena,
-                record_var,
-                env.subs,
-                env.pointer_size,
-            );
 
             let mut field_symbols = Vec::with_capacity_in(fields.len(), env.arena);
             let mut field_layouts = Vec::with_capacity_in(fields.len(), env.arena);
@@ -1281,7 +1270,7 @@ pub fn with_hole<'a>(
 
             // creating a record from the var will unpack it if it's just a single field.
             let layout = layout_cache
-                .from_var(env.arena, record_var, env.subs, env.pointer_size)
+                .from_var(env.arena, record_var, env.subs)
                 .unwrap_or_else(|err| panic!("TODO turn fn_var into a RuntimeError {:?}", err));
 
             let field_symbols = field_symbols.into_bump_slice();
@@ -1313,10 +1302,10 @@ pub fn with_hole<'a>(
             final_else,
         } => {
             let ret_layout = layout_cache
-                .from_var(env.arena, branch_var, env.subs, env.pointer_size)
+                .from_var(env.arena, branch_var, env.subs)
                 .expect("invalid ret_layout");
             let cond_layout = layout_cache
-                .from_var(env.arena, cond_var, env.subs, env.pointer_size)
+                .from_var(env.arena, cond_var, env.subs)
                 .expect("invalid cond_layout");
 
             let assigned_in_jump = env.unique_symbol();
@@ -1367,7 +1356,7 @@ pub fn with_hole<'a>(
             }
 
             let layout = layout_cache
-                .from_var(env.arena, branch_var, env.subs, env.pointer_size)
+                .from_var(env.arena, branch_var, env.subs)
                 .unwrap_or_else(|err| panic!("TODO turn fn_var into a RuntimeError {:?}", err));
 
             let param = Param {
@@ -1426,7 +1415,7 @@ pub fn with_hole<'a>(
             };
 
             let layout = layout_cache
-                .from_var(env.arena, expr_var, env.subs, env.pointer_size)
+                .from_var(env.arena, expr_var, env.subs)
                 .unwrap_or_else(|err| panic!("TODO turn fn_var into a RuntimeError {:?}", err));
 
             let param = Param {
@@ -1464,7 +1453,7 @@ pub fn with_hole<'a>(
             let arg_symbols = arg_symbols.into_bump_slice();
 
             let elem_layout = layout_cache
-                .from_var(env.arena, elem_var, env.subs, env.pointer_size)
+                .from_var(env.arena, elem_var, env.subs)
                 .unwrap_or_else(|err| panic!("TODO turn fn_var into a RuntimeError {:?}", err));
 
             let expr = Expr::Array {
@@ -1508,12 +1497,7 @@ pub fn with_hole<'a>(
             loc_expr,
             ..
         } => {
-            let sorted_fields = crate::layout::sort_record_fields(
+            let sorted_fields = crate::layout::sort_record_fields(env.arena, record_var, env.subs);
-                env.arena,
-                record_var,
-                env.subs,
-                env.pointer_size,
-            );
 
             let mut index = None;
             let mut field_layouts = Vec::with_capacity_in(sorted_fields.len(), env.arena);
@@ -1540,7 +1524,7 @@ pub fn with_hole<'a>(
             };
 
             let layout = layout_cache
-                .from_var(env.arena, field_var, env.subs, env.pointer_size)
+                .from_var(env.arena, field_var, env.subs)
                 .unwrap_or_else(|err| panic!("TODO turn fn_var into a RuntimeError {:?}", err));
 
             let mut stmt = Stmt::Let(assigned, expr, layout, hole);
@@ -1646,7 +1630,7 @@ pub fn with_hole<'a>(
                     }
 
                     let layout = layout_cache
-                        .from_var(env.arena, fn_var, env.subs, env.pointer_size)
+                        .from_var(env.arena, fn_var, env.subs)
                         .unwrap_or_else(|err| {
                             panic!("TODO turn fn_var into a RuntimeError {:?}", err)
                         });
@@ -1657,7 +1641,7 @@ pub fn with_hole<'a>(
                     };
 
                     let ret_layout = layout_cache
-                        .from_var(env.arena, ret_var, env.subs, env.pointer_size)
+                        .from_var(env.arena, ret_var, env.subs)
                         .unwrap_or_else(|err| {
                             panic!("TODO turn fn_var into a RuntimeError {:?}", err)
                         });
@@ -1720,7 +1704,7 @@ pub fn with_hole<'a>(
 
             // layout of the return type
             let layout = layout_cache
-                .from_var(env.arena, ret_var, env.subs, env.pointer_size)
+                .from_var(env.arena, ret_var, env.subs)
                 .unwrap_or_else(|err| todo!("TODO turn fn_var into a RuntimeError {:?}", err));
 
             let mut result = Stmt::Let(assigned, Expr::RunLowLevel(op, arg_symbols), layout, hole);
@@ -1857,7 +1841,7 @@ pub fn from_can<'a>(
                 }
 
                 let layout = layout_cache
-                    .from_var(env.arena, def.expr_var, env.subs, env.pointer_size)
+                    .from_var(env.arena, def.expr_var, env.subs)
                     .expect("invalid layout");
 
                 // convert the continuation
@@ -1998,11 +1982,11 @@ fn from_can_when<'a>(
     let opt_branches = to_opt_branches(env, region, branches, layout_cache);
 
     let cond_layout = layout_cache
-        .from_var(env.arena, cond_var, env.subs, env.pointer_size)
+        .from_var(env.arena, cond_var, env.subs)
         .unwrap_or_else(|err| panic!("TODO turn this into a RuntimeError {:?}", err));
 
     let ret_layout = layout_cache
-        .from_var(env.arena, expr_var, env.subs, env.pointer_size)
+        .from_var(env.arena, expr_var, env.subs)
         .unwrap_or_else(|err| panic!("TODO turn this into a RuntimeError {:?}", err));
 
     let arena = env.arena;
@@ -2291,7 +2275,7 @@ fn call_by_name<'a>(
     hole: &'a Stmt<'a>,
 ) -> Stmt<'a> {
     // Register a pending_specialization for this function
-    match layout_cache.from_var(env.arena, fn_var, env.subs, env.pointer_size) {
+    match layout_cache.from_var(env.arena, fn_var, env.subs) {
         Ok(layout) => {
             // Build the CallByName node
             let arena = env.arena;
@@ -2309,7 +2293,7 @@ fn call_by_name<'a>(
             let field_symbols = field_symbols.into_bump_slice();
 
             for (var, _) in &loc_args {
-                match layout_cache.from_var(&env.arena, *var, &env.subs, env.pointer_size) {
+                match layout_cache.from_var(&env.arena, *var, &env.subs) {
                     Ok(_) => {
                         pattern_vars.push(*var);
                     }
@@ -2591,8 +2575,7 @@ pub fn from_can_pattern<'a>(
             use crate::exhaustive::Union;
             use crate::layout::UnionVariant::*;
 
-            let variant =
+            let variant = crate::layout::union_sorted_tags(env.arena, *whole_var, env.subs);
-                crate::layout::union_sorted_tags(env.arena, *whole_var, env.subs, env.pointer_size);
 
             match variant {
                 Never => unreachable!("there is no pattern of type `[]`"),
@@ -2745,12 +2728,7 @@ pub fn from_can_pattern<'a>(
             let mut it = destructs.iter();
             let mut opt_destruct = it.next();
 
-            let sorted_fields = crate::layout::sort_record_fields(
+            let sorted_fields = crate::layout::sort_record_fields(env.arena, *whole_var, env.subs);
-                env.arena,
-                *whole_var,
-                env.subs,
-                env.pointer_size,
-            );
 
             let mut field_layouts = Vec::with_capacity_in(sorted_fields.len(), env.arena);
 

compiler/mono/src/layout.rs

@@ -64,17 +64,12 @@ pub enum Builtin<'a> {
 }
 
 impl<'a> Layout<'a> {
-    pub fn new(
+    pub fn new(arena: &'a Bump, content: Content, subs: &Subs) -> Result<Self, LayoutProblem> {
-        arena: &'a Bump,
-        content: Content,
-        subs: &Subs,
-        pointer_size: u32,
-    ) -> Result<Self, LayoutProblem> {
         use roc_types::subs::Content::*;
 
         match content {
             FlexVar(_) | RigidVar(_) => Err(LayoutProblem::UnresolvedTypeVar),
-            Structure(flat_type) => layout_from_flat_type(arena, flat_type, subs, pointer_size),
+            Structure(flat_type) => layout_from_flat_type(arena, flat_type, subs),
 
             Alias(Symbol::NUM_INT, args, _) => {
                 debug_assert!(args.is_empty());
@@ -84,12 +79,7 @@ impl<'a> Layout<'a> {
                 debug_assert!(args.is_empty());
                 Ok(Layout::Builtin(Builtin::Float64))
             }
-            Alias(_, _, var) => Self::new(
+            Alias(_, _, var) => Self::new(arena, subs.get_without_compacting(var).content, subs),
-                arena,
-                subs.get_without_compacting(var).content,
-                subs,
-                pointer_size,
-            ),
             Error => Err(LayoutProblem::Erroneous),
         }
     }
@@ -97,15 +87,10 @@ impl<'a> Layout<'a> {
     /// Returns Err(()) if given an error, or Ok(Layout) if given a non-erroneous Structure.
     /// Panics if given a FlexVar or RigidVar, since those should have been
     /// monomorphized away already!
-    fn from_var(
+    fn from_var(arena: &'a Bump, var: Variable, subs: &Subs) -> Result<Self, LayoutProblem> {
-        arena: &'a Bump,
-        var: Variable,
-        subs: &Subs,
-        pointer_size: u32,
-    ) -> Result<Self, LayoutProblem> {
         let content = subs.get_without_compacting(var).content;
 
-        Self::new(arena, content, subs, pointer_size)
+        Self::new(arena, content, subs)
     }
 
     pub fn safe_to_memcpy(&self) -> bool {
@@ -130,6 +115,13 @@ impl<'a> Layout<'a> {
         }
     }
 
+    pub fn is_zero_sized(&self) -> bool {
+        // For this calculation, we don't need an accurate
+        // stack size, we just need to know whether it's zero,
+        // so it's fine to use a pointer size of 1.
+        self.stack_size(1) == 0
+    }
+
     pub fn stack_size(&self, pointer_size: u32) -> u32 {
         use Layout::*;
 
@@ -175,7 +167,6 @@ impl<'a> LayoutCache<'a> {
         arena: &'a Bump,
         var: Variable,
         subs: &Subs,
-        pointer_size: u32,
     ) -> Result<Layout<'a>, LayoutProblem> {
         // Store things according to the root Variable, to avoid duplicate work.
         let var = subs.get_root_key_without_compacting(var);
@@ -185,7 +176,7 @@ impl<'a> LayoutCache<'a> {
             .or_insert_with(|| {
                 let content = subs.get_without_compacting(var).content;
 
-                Layout::new(arena, content, subs, pointer_size)
+                Layout::new(arena, content, subs)
             })
             .clone()
     }
@@ -252,7 +243,6 @@ fn layout_from_flat_type<'a>(
     arena: &'a Bump,
     flat_type: FlatType,
     subs: &Subs,
-    pointer_size: u32,
 ) -> Result<Layout<'a>, LayoutProblem> {
     use roc_types::subs::FlatType::*;
 
@@ -277,7 +267,7 @@ fn layout_from_flat_type<'a>(
                     layout_from_num_content(content)
                 }
                 Symbol::STR_STR => Ok(Layout::Builtin(Builtin::Str)),
-                Symbol::LIST_LIST => list_layout_from_elem(arena, subs, args[0], pointer_size),
+                Symbol::LIST_LIST => list_layout_from_elem(arena, subs, args[0]),
                 Symbol::ATTR_ATTR => {
                     debug_assert_eq!(args.len(), 2);
 
@@ -288,7 +278,7 @@ fn layout_from_flat_type<'a>(
                     // For now, layout is unaffected by uniqueness.
                     // (Incorporating refcounting may change this.)
                     // Unwrap and continue
-                    Layout::from_var(arena, wrapped_var, subs, pointer_size)
+                    Layout::from_var(arena, wrapped_var, subs)
                 }
                 _ => {
                     panic!("TODO layout_from_flat_type for {:?}", Apply(symbol, args));
@@ -301,11 +291,11 @@ fn layout_from_flat_type<'a>(
             for arg_var in args {
                 let arg_content = subs.get_without_compacting(arg_var).content;
 
-                fn_args.push(Layout::new(arena, arg_content, subs, pointer_size)?);
+                fn_args.push(Layout::new(arena, arg_content, subs)?);
             }
 
             let ret_content = subs.get_without_compacting(ret_var).content;
-            let ret = Layout::new(arena, ret_content, subs, pointer_size)?;
+            let ret = Layout::new(arena, ret_content, subs)?;
 
             Ok(Layout::FunctionPointer(
                 fn_args.into_bump_slice(),
@@ -333,10 +323,10 @@ fn layout_from_flat_type<'a>(
                 let field_var = field.into_inner();
                 let field_content = subs.get_without_compacting(field_var).content;
 
-                match Layout::new(arena, field_content, subs, pointer_size) {
+                match Layout::new(arena, field_content, subs) {
                     Ok(layout) => {
-                        // Drop any zero-sized fields like {}
+                        // Drop any zero-sized fields like {}.
-                        if layout.stack_size(pointer_size) != 0 {
+                        if !layout.is_zero_sized() {
                             layouts.push(layout);
                         }
                     }
@@ -358,7 +348,7 @@ fn layout_from_flat_type<'a>(
         TagUnion(tags, ext_var) => {
             debug_assert!(ext_var_is_empty_tag_union(subs, ext_var));
 
-            Ok(layout_from_tag_union(arena, tags, subs, pointer_size))
+            Ok(layout_from_tag_union(arena, tags, subs))
         }
         RecursiveTagUnion(_rec_var, _tags, _ext_var) => {
             panic!("TODO make Layout for empty RecursiveTagUnion");
@@ -378,7 +368,6 @@ pub fn sort_record_fields<'a>(
     arena: &'a Bump,
     var: Variable,
     subs: &Subs,
-    pointer_size: u32,
 ) -> Vec<'a, (Lowercase, Layout<'a>)> {
     let mut fields_map = MutMap::default();
 
@@ -389,11 +378,10 @@ pub fn sort_record_fields<'a>(
 
             for (label, field) in fields_map {
                 let var = field.into_inner();
-                let layout = Layout::from_var(arena, var, subs, pointer_size)
+                let layout = Layout::from_var(arena, var, subs).expect("invalid layout from var");
-                    .expect("invalid layout from var");
 
                 // Drop any zero-sized fields like {}
-                if layout.stack_size(pointer_size) != 0 {
+                if !layout.is_zero_sized() {
                     sorted_fields.push((label, layout));
                 }
             }
@@ -416,17 +404,10 @@ pub enum UnionVariant<'a> {
     Wrapped(Vec<'a, (TagName, &'a [Layout<'a>])>),
 }
 
-pub fn union_sorted_tags<'a>(
+pub fn union_sorted_tags<'a>(arena: &'a Bump, var: Variable, subs: &Subs) -> UnionVariant<'a> {
-    arena: &'a Bump,
-    var: Variable,
-    subs: &Subs,
-    pointer_size: u32,
-) -> UnionVariant<'a> {
     let mut tags_vec = std::vec::Vec::new();
     match roc_types::pretty_print::chase_ext_tag_union(subs, var, &mut tags_vec) {
-        Ok(()) | Err((_, Content::FlexVar(_))) => {
+        Ok(()) | Err((_, Content::FlexVar(_))) => union_sorted_tags_help(arena, tags_vec, subs),
-            union_sorted_tags_help(arena, tags_vec, subs, pointer_size)
-        }
         Err(other) => panic!("invalid content in tag union variable: {:?}", other),
     }
 }
@@ -435,7 +416,6 @@ fn union_sorted_tags_help<'a>(
     arena: &'a Bump,
     mut tags_vec: std::vec::Vec<(TagName, std::vec::Vec<Variable>)>,
     subs: &Subs,
-    pointer_size: u32,
 ) -> UnionVariant<'a> {
     // sort up front; make sure the ordering stays intact!
     tags_vec.sort();
@@ -458,10 +438,10 @@ fn union_sorted_tags_help<'a>(
                 }
                 _ => {
                     for var in arguments {
-                        match Layout::from_var(arena, var, subs, pointer_size) {
+                        match Layout::from_var(arena, var, subs) {
                             Ok(layout) => {
                                 // Drop any zero-sized arguments like {}
-                                if layout.stack_size(pointer_size) != 0 {
+                                if !layout.is_zero_sized() {
                                     layouts.push(layout);
                                 }
                             }
@@ -497,10 +477,10 @@ fn union_sorted_tags_help<'a>(
                 arg_layouts.push(Layout::Builtin(Builtin::Int64));
 
                 for var in arguments {
-                    match Layout::from_var(arena, var, subs, pointer_size) {
+                    match Layout::from_var(arena, var, subs) {
                         Ok(layout) => {
                             // Drop any zero-sized arguments like {}
-                            if layout.stack_size(pointer_size) != 0 {
+                            if !layout.is_zero_sized() {
                                 has_any_arguments = true;
 
                                 arg_layouts.push(layout);
@@ -551,14 +531,13 @@ pub fn layout_from_tag_union<'a>(
     arena: &'a Bump,
     tags: MutMap<TagName, std::vec::Vec<Variable>>,
     subs: &Subs,
-    pointer_size: u32,
 ) -> Layout<'a> {
     use UnionVariant::*;
 
     let tags_vec: std::vec::Vec<_> = tags.into_iter().collect();
 
     if tags_vec[0].0 != TagName::Private(Symbol::NUM_AT_NUM) {
-        let variant = union_sorted_tags_help(arena, tags_vec, subs, pointer_size);
+        let variant = union_sorted_tags_help(arena, tags_vec, subs);
 
         match variant {
             Never => panic!("TODO gracefully handle trying to instantiate Never"),
@@ -692,7 +671,6 @@ pub fn list_layout_from_elem<'a>(
     arena: &'a Bump,
     subs: &Subs,
     var: Variable,
-    pointer_size: u32,
 ) -> Result<Layout<'a>, LayoutProblem> {
     match subs.get_without_compacting(var).content {
         Content::Structure(FlatType::Apply(Symbol::ATTR_ATTR, args)) => {
@@ -700,14 +678,14 @@ pub fn list_layout_from_elem<'a>(
 
             let arg_var = args.get(1).unwrap();
 
-            list_layout_from_elem(arena, subs, *arg_var, pointer_size)
+            list_layout_from_elem(arena, subs, *arg_var)
         }
         Content::FlexVar(_) | Content::RigidVar(_) => {
             // If this was still a (List *) then it must have been an empty list
             Ok(Layout::Builtin(Builtin::EmptyList))
         }
         content => {
-            let elem_layout = Layout::new(arena, content, subs, pointer_size)?;
+            let elem_layout = Layout::new(arena, content, subs)?;
 
             // This is a normal list.
             Ok(Layout::Builtin(Builtin::List(

compiler/mono/tests/helpers/mod.rs

@@ -53,7 +53,7 @@ pub fn parse_with<'a>(arena: &'a Bump, input: &'a str) -> Result<ast::Expr<'a>,
 
 #[allow(dead_code)]
 pub fn parse_loc_with<'a>(arena: &'a Bump, input: &'a str) -> Result<Located<ast::Expr<'a>>, Fail> {
-    let state = State::new(&input, Attempting::Module);
+    let state = State::new(input.as_bytes(), Attempting::Module);
     let parser = space0_before(loc(roc_parse::expr::expr(0)), 0);
     let answer = parser.parse(&arena, state);
 

compiler/mono/tests/test_mono.rs

@@ -14,7 +14,6 @@ mod helpers;
 mod test_mono {
     use crate::helpers::{can_expr, infer_expr, CanExprOut};
     use bumpalo::Bump;
-    use roc_collections::all::MutMap;
     use roc_mono::layout::LayoutCache;
     use roc_types::subs::Subs;
 
@@ -38,8 +37,8 @@ mod test_mono {
         let mut procs = roc_mono::ir::Procs::default();
         let mut ident_ids = interns.all_ident_ids.remove(&home).unwrap();
 
-        // assume 64-bit pointers
+        // Put this module's ident_ids back in the interns
-        let pointer_size = std::mem::size_of::<u64>() as u32;
+        interns.all_ident_ids.insert(home, ident_ids.clone());
 
         // Populate Procs and Subs, and get the low-level Expr from the canonical Expr
         let mut mono_problems = Vec::new();
@@ -49,7 +48,6 @@ mod test_mono {
             problems: &mut mono_problems,
             home,
             ident_ids: &mut ident_ids,
-            pointer_size,
             jump_counter: arena.alloc(0),
         };
 
@@ -69,9 +67,6 @@ mod test_mono {
             roc_collections::all::MutMap::default()
         );
 
-        // Put this module's ident_ids back in the interns
-        interns.all_ident_ids.insert(home, ident_ids);
-
         let mut procs_string = procs
             .get_specialized_procs(mono_env.arena)
             .values()

compiler/parse/Cargo.toml

@@ -11,6 +11,7 @@ roc_region = { path = "../region" }
 roc_module = { path = "../module" }
 bumpalo = { version = "3.2", features = ["collections"] }
 inlinable_string = "0.1"
+encode_unicode = "0.3"
 
 [dev-dependencies]
 pretty_assertions = "0.5.1"

compiler/parse/src/blankspace.rs

@@ -1,6 +1,8 @@
 use crate::ast::CommentOrNewline::{self, *};
 use crate::ast::Spaceable;
-use crate::parser::{self, and, unexpected, unexpected_eof, Parser, State};
+use crate::parser::{
+    self, and, peek_utf8_char, unexpected, unexpected_eof, FailReason, Parser, State,
+};
 use bumpalo::collections::string::String;
 use bumpalo::collections::vec::Vec;
 use bumpalo::Bump;
@@ -216,16 +218,17 @@ fn spaces<'a>(
 ) -> impl Parser<'a, &'a [CommentOrNewline<'a>]> {
     move |arena: &'a Bump, state: State<'a>| {
         let original_state = state.clone();
-        let chars = state.input.chars().peekable();
         let mut space_list = Vec::new_in(arena);
-        let mut chars_parsed = 0;
+        let mut bytes_parsed = 0;
         let mut comment_line_buf = String::new_in(arena);
         let mut line_state = LineState::Normal;
         let mut state = state;
         let mut any_newlines = false;
 
-        for ch in chars {
+        while !state.bytes.is_empty() {
-            chars_parsed += 1;
+            match peek_utf8_char(&state) {
+                Ok((ch, utf8_len)) => {
+                    bytes_parsed += utf8_len;
 
                     match line_state {
                         LineState::Normal => {
@@ -259,11 +262,11 @@ fn spaces<'a>(
                                     // We're now parsing a line comment!
                                     line_state = LineState::Comment;
                                 }
-                        nonblank => {
+                                _ => {
-                            return if require_at_least_one && chars_parsed <= 1 {
+                                    return if require_at_least_one && bytes_parsed <= 1 {
                                         // We've parsed 1 char and it was not a space,
                                         // but we require parsing at least one space!
-                                Err(unexpected(nonblank, 0, state.clone(), state.attempting))
+                                        Err(unexpected(0, state.clone(), state.attempting))
                                     } else {
                                         // First make sure we were indented enough!
                                         //
@@ -346,8 +349,7 @@ fn spaces<'a>(
                                     line_state = LineState::Normal;
                                 }
                                 nonblank => {
-                            // Chars can have btye lengths of more than 1!
+                                    state = state.advance_without_indenting(utf8_len)?;
-                            state = state.advance_without_indenting(nonblank.len_utf8())?;
 
                                     comment_line_buf.push(nonblank);
                                 }
@@ -355,8 +357,40 @@ fn spaces<'a>(
                         }
                     }
                 }
+                Err(FailReason::BadUtf8) => {
+                    // If we hit an invalid UTF-8 character, bail out immediately.
+                    return state.fail(FailReason::BadUtf8);
+                }
+                Err(_) => {
+                    if require_at_least_one && bytes_parsed == 0 {
+                        return Err(unexpected_eof(0, state.attempting, state));
+                    } else {
+                        let space_slice = space_list.into_bump_slice();
 
-        if require_at_least_one && chars_parsed == 0 {
+                        // First make sure we were indented enough!
+                        //
+                        // (We only do this if we've encountered any newlines.
+                        // Otherwise, we assume indentation is already correct.
+                        // It's actively important for correctness that we skip
+                        // this check if there are no newlines, because otherwise
+                        // we would have false positives for single-line defs.)
+                        if any_newlines {
+                            return Ok((
+                                space_slice,
+                                state
+                                    .check_indent(min_indent)
+                                    .map_err(|(fail, _)| (fail, original_state))?,
+                            ));
+                        }
+
+                        return Ok((space_slice, state));
+                    }
+                }
+            };
+        }
+
+        // If we didn't parse anything, return unexpected EOF
+        if require_at_least_one && original_state.bytes.len() == state.bytes.len() {
             Err(unexpected_eof(0, state.attempting, state))
         } else {
             // First make sure we were indented enough!

compiler/parse/src/expr.rs

@@ -8,8 +8,8 @@ use crate::ident::{global_tag_or_ident, ident, lowercase_ident, Ident};
 use crate::keyword;
 use crate::number_literal::number_literal;
 use crate::parser::{
-    self, allocated, char, fail, not, not_followed_by, optional, sep_by1, string, then, unexpected,
+    self, allocated, ascii_char, ascii_string, fail, not, not_followed_by, optional, sep_by1, then,
-    unexpected_eof, Either, Fail, FailReason, ParseResult, Parser, State,
+    unexpected, unexpected_eof, Either, Fail, FailReason, ParseResult, Parser, State,
 };
 use crate::type_annotation;
 use bumpalo::collections::string::String;
@@ -22,7 +22,7 @@ pub fn expr<'a>(min_indent: u16) -> impl Parser<'a, Expr<'a>> {
     // Recursive parsers must not directly invoke functions which return (impl Parser),
     // as this causes rustc to stack overflow. Thus, parse_expr must be a
     // separate function which recurses by calling itself directly.
-    move |arena, state| parse_expr(min_indent, arena, state)
+    move |arena, state: State<'a>| parse_expr(min_indent, arena, state)
 }
 
 macro_rules! loc_parenthetical_expr {
@@ -30,7 +30,7 @@ macro_rules! loc_parenthetical_expr {
     then(
         loc!(and!(
             between!(
-                char('('),
+                ascii_char('(' ),
                 map_with_arena!(
                     space0_around(
                         loc!(move |arena, state| parse_expr($min_indent, arena, state)),
@@ -43,7 +43,7 @@ macro_rules! loc_parenthetical_expr {
                         }
                     }
                 ),
-                char(')')
+                ascii_char(')' )
             ),
             optional(either!(
                 // There may optionally be function args after the ')'
@@ -59,7 +59,7 @@ macro_rules! loc_parenthetical_expr {
                 // as if there were any args they'd have consumed it anyway
                 // e.g. in `((foo bar) baz.blah)` the `.blah` will be consumed by the `baz` parser
                 either!(
-                    one_or_more!(skip_first!(char('.'), lowercase_ident())),
+                    one_or_more!(skip_first!(ascii_char('.' ), lowercase_ident())),
                     and!(space0($min_indent), equals_with_indent())
                 )
             ))
@@ -170,7 +170,7 @@ pub fn unary_op<'a>(min_indent: u16) -> impl Parser<'a, Expr<'a>> {
     one_of!(
         map_with_arena!(
             and!(
-                loc!(char('!')),
+                loc!(ascii_char('!')),
                 loc!(move |arena, state| parse_expr(min_indent, arena, state))
             ),
             |arena: &'a Bump, (loc_op, loc_expr): (Located<()>, Located<Expr<'a>>)| {
@@ -179,7 +179,7 @@ pub fn unary_op<'a>(min_indent: u16) -> impl Parser<'a, Expr<'a>> {
         ),
         map_with_arena!(
             and!(
-                loc!(char('-')),
+                loc!(ascii_char('-')),
                 loc!(move |arena, state| parse_expr(min_indent, arena, state))
             ),
             |arena: &'a Bump, (loc_op, loc_expr): (Located<()>, Located<Expr<'a>>)| {
@@ -450,9 +450,9 @@ pub fn loc_parenthetical_def<'a>(min_indent: u16) -> impl Parser<'a, Located<Exp
         let (loc_tuple, state) = loc!(and!(
             space0_after(
                 between!(
-                    char('('),
+                    ascii_char('('),
                     space0_around(loc_pattern(min_indent), min_indent),
-                    char(')')
+                    ascii_char(')')
                 ),
                 min_indent,
             ),
@@ -482,7 +482,7 @@ pub fn loc_parenthetical_def<'a>(min_indent: u16) -> impl Parser<'a, Located<Exp
 /// The '=' used in a def can't be followed by another '=' (or else it's actually
 /// an "==") and also it can't be followed by '>' (or else it's actually an "=>")
 fn equals_for_def<'a>() -> impl Parser<'a, ()> {
-    not_followed_by(char('='), one_of!(char('='), char('>')))
+    not_followed_by(ascii_char('='), one_of!(ascii_char('='), ascii_char('>')))
 }
 
 /// A definition, consisting of one of these:
@@ -513,7 +513,7 @@ pub fn def<'a>(min_indent: u16) -> impl Parser<'a, Def<'a>> {
                 ),
                 // Annotation
                 skip_first!(
-                    char(':'),
+                    ascii_char(':'),
                     // Spaces after the ':' (at a normal indentation level) and then the type.
                     // The type itself must be indented more than the pattern and ':'
                     space0_before(type_annotation::located(indented_more), indented_more)
@@ -811,12 +811,12 @@ fn loc_parse_function_arg<'a>(
 
 fn reserved_keyword<'a>() -> impl Parser<'a, ()> {
     one_of!(
-        string(keyword::IF),
+        ascii_string(keyword::IF),
-        string(keyword::THEN),
+        ascii_string(keyword::THEN),
-        string(keyword::ELSE),
+        ascii_string(keyword::ELSE),
-        string(keyword::WHEN),
+        ascii_string(keyword::WHEN),
-        string(keyword::IS),
+        ascii_string(keyword::IS),
-        string(keyword::AS)
+        ascii_string(keyword::AS)
     )
 }
 
@@ -824,7 +824,7 @@ fn closure<'a>(min_indent: u16) -> impl Parser<'a, Expr<'a>> {
     map_with_arena!(
         skip_first!(
             // All closures start with a '\' - e.g. (\x -> x + 1)
-            char('\\'),
+            ascii_char('\\'),
             // Once we see the '\', we're committed to parsing this as a closure.
             // It may turn out to be malformed, but it is definitely a closure.
             optional(and!(
@@ -833,13 +833,13 @@ fn closure<'a>(min_indent: u16) -> impl Parser<'a, Expr<'a>> {
                     Attempting::ClosureParams,
                     // Params are comma-separated
                     sep_by1(
-                        char(','),
+                        ascii_char(','),
                         space0_around(loc_closure_param(min_indent), min_indent)
                     )
                 ),
                 skip_first!(
                     // Parse the -> which separates params from body
-                    string("->"),
+                    ascii_string("->"),
                     // Parse the body
                     attempt!(
                         Attempting::ClosureBody,
@@ -877,9 +877,9 @@ fn parse_closure_param<'a>(
         // If you wrap it in parens, you can match any arbitrary pattern at all.
         // e.g. \User.UserId userId -> ...
         between!(
-            char('('),
+            ascii_char('('),
             space0_around(loc_pattern(min_indent), min_indent),
-            char(')')
+            ascii_char(')')
         )
     )
     .parse(arena, state)
@@ -903,9 +903,9 @@ fn loc_pattern<'a>(min_indent: u16) -> impl Parser<'a, Located<Pattern<'a>>> {
 
 fn loc_parenthetical_pattern<'a>(min_indent: u16) -> impl Parser<'a, Located<Pattern<'a>>> {
     between!(
-        char('('),
+        ascii_char('('),
         move |arena, state| loc_pattern(min_indent).parse(arena, state),
-        char(')')
+        ascii_char(')')
     )
 }
 
@@ -923,13 +923,13 @@ fn string_pattern<'a>() -> impl Parser<'a, Pattern<'a>> {
 }
 
 fn underscore_pattern<'a>() -> impl Parser<'a, Pattern<'a>> {
-    map!(char('_'), |_| Pattern::Underscore)
+    map!(ascii_char('_'), |_| Pattern::Underscore)
 }
 
 fn record_destructure<'a>(min_indent: u16) -> impl Parser<'a, Pattern<'a>> {
     then(
         collection!(
-            char('{'),
+            ascii_char('{'),
             move |arena: &'a bumpalo::Bump,
                   state: crate::parser::State<'a>|
                   -> crate::parser::ParseResult<'a, Located<crate::ast::Pattern<'a>>> {
@@ -947,10 +947,13 @@ fn record_destructure<'a>(min_indent: u16) -> impl Parser<'a, Pattern<'a>> {
                 // (This is true in both literals and types.)
                 let (opt_loc_val, state) = crate::parser::optional(either!(
                     skip_first!(
-                        char(':'),
+                        ascii_char(':'),
                         space0_before(loc_pattern(min_indent), min_indent)
                     ),
-                    skip_first!(char('?'), space0_before(loc!(expr(min_indent)), min_indent))
+                    skip_first!(
+                        ascii_char('?'),
+                        space0_before(loc!(expr(min_indent)), min_indent)
+                    )
                 ))
                 .parse(arena, state)?;
 
@@ -987,8 +990,8 @@ fn record_destructure<'a>(min_indent: u16) -> impl Parser<'a, Pattern<'a>> {
 
                 Ok((answer, state))
             },
-            char(','),
+            ascii_char(','),
-            char('}'),
+            ascii_char('}'),
             min_indent
         ),
         move |_arena, state, loc_patterns| {
@@ -1109,7 +1112,7 @@ mod when {
                             loc!(move |arena, state| parse_expr(min_indent, arena, state)),
                             min_indent,
                         ),
-                        string(keyword::IS)
+                        ascii_string(keyword::IS)
                     )
                 )
             ),
@@ -1132,7 +1135,7 @@ mod when {
     /// Parsing when with indentation.
     fn when_with_indent<'a>() -> impl Parser<'a, u16> {
         move |arena, state: State<'a>| {
-            string(keyword::WHEN)
+            ascii_string(keyword::WHEN)
                 .parse(arena, state)
                 .map(|((), state)| (state.indent_col, state))
         }
@@ -1185,7 +1188,7 @@ mod when {
                 }
             );
 
-            loop {
+            while !state.bytes.is_empty() {
                 match branch_parser.parse(arena, state) {
                     Ok((next_output, next_state)) => {
                         state = next_state;
@@ -1210,11 +1213,11 @@ mod when {
     ) -> impl Parser<'a, (Vec<'a, Located<Pattern<'a>>>, Option<Located<Expr<'a>>>)> {
         and!(
             sep_by1(
-                char('|'),
+                ascii_char('|'),
                 space0_around(loc_pattern(min_indent), min_indent),
             ),
             optional(skip_first!(
-                string(keyword::IF),
+                ascii_string(keyword::IF),
                 // TODO we should require space before the expression but not after
                 space1_around(
                     loc!(move |arena, state| parse_expr(min_indent, arena, state)),
@@ -1240,7 +1243,7 @@ mod when {
     /// Parsing the righthandside of a branch in a when conditional.
     fn branch_result<'a>(indent: u16) -> impl Parser<'a, Located<Expr<'a>>> {
         skip_first!(
-            string("->"),
+            ascii_string("->"),
             space0_before(
                 loc!(move |arena, state| parse_expr(indent, arena, state)),
                 indent,
@@ -1253,7 +1256,7 @@ pub fn if_expr<'a>(min_indent: u16) -> impl Parser<'a, Expr<'a>> {
     map_with_arena!(
         and!(
             skip_first!(
-                string(keyword::IF),
+                ascii_string(keyword::IF),
                 space1_around(
                     loc!(move |arena, state| parse_expr(min_indent, arena, state)),
                     min_indent,
@@ -1261,14 +1264,14 @@ pub fn if_expr<'a>(min_indent: u16) -> impl Parser<'a, Expr<'a>> {
             ),
             and!(
                 skip_first!(
-                    string(keyword::THEN),
+                    ascii_string(keyword::THEN),
                     space1_around(
                         loc!(move |arena, state| parse_expr(min_indent, arena, state)),
                         min_indent,
                     )
                 ),
                 skip_first!(
-                    string(keyword::ELSE),
+                    ascii_string(keyword::ELSE),
                     space1_before(
                         loc!(move |arena, state| parse_expr(min_indent, arena, state)),
                         min_indent,
@@ -1310,10 +1313,15 @@ fn unary_negate_function_arg<'a>(min_indent: u16) -> impl Parser<'a, Located<Exp
                     // Try to parse a number literal *before* trying to parse unary negate,
                     // because otherwise (foo -1) will parse as (foo (Num.neg 1))
                     loc!(number_literal()),
-                    loc!(char('-'))
+                    loc!(ascii_char('-'))
                 )
             ),
-            one_of!(char(' '), char('#'), char('\n'), char('>')),
+            one_of!(
+                ascii_char(' '),
+                ascii_char('#'),
+                ascii_char('\n'),
+                ascii_char('>')
+            ),
         ),
         move |arena, state, (spaces, num_or_minus_char)| {
             match num_or_minus_char {
@@ -1530,17 +1538,15 @@ pub fn ident_without_apply<'a>() -> impl Parser<'a, Expr<'a>> {
 /// Like equals_for_def(), except it produces the indent_col of the state rather than ()
 pub fn equals_with_indent<'a>() -> impl Parser<'a, u16> {
     move |_arena, state: State<'a>| {
-        let mut iter = state.input.chars();
+        match state.bytes.first() {
-
+            Some(&byte) if byte == b'=' => {
-        match iter.next() {
+                match state.bytes.get(1) {
-            Some(ch) if ch == '=' => {
-                match iter.peekable().peek() {
                     // The '=' must not be followed by another `=` or `>`
                     // (See equals_for_def() for explanation)
-                    Some(next_ch) if next_ch != &'=' && next_ch != &'>' => {
+                    Some(&next_byte) if next_byte != b'=' && next_byte != b'>' => {
                         Ok((state.indent_col, state.advance_without_indenting(1)?))
                     }
-                    Some(next_ch) => Err(unexpected(*next_ch, 0, state, Attempting::Def)),
+                    Some(_) => Err(unexpected(0, state, Attempting::Def)),
                     None => Err(unexpected_eof(
                         1,
                         Attempting::Def,
@@ -1548,23 +1554,19 @@ pub fn equals_with_indent<'a>() -> impl Parser<'a, u16> {
                     )),
                 }
             }
-            Some(ch) => Err(unexpected(ch, 0, state, Attempting::Def)),
+            Some(_) => Err(unexpected(0, state, Attempting::Def)),
             None => Err(unexpected_eof(0, Attempting::Def, state)),
         }
     }
 }
 
 pub fn colon_with_indent<'a>() -> impl Parser<'a, u16> {
-    move |_arena, state: State<'a>| {
+    move |_arena, state: State<'a>| match state.bytes.first() {
-        let mut iter = state.input.chars();
+        Some(&byte) if byte == b':' => Ok((state.indent_col, state.advance_without_indenting(1)?)),
-
+        Some(_) => Err(unexpected(0, state, Attempting::Def)),
-        match iter.next() {
-            Some(ch) if ch == ':' => Ok((state.indent_col, state.advance_without_indenting(1)?)),
-            Some(ch) => Err(unexpected(ch, 0, state, Attempting::Def)),
         None => Err(unexpected_eof(0, Attempting::Def, state)),
     }
 }
-}
 
 pub fn ident_to_expr<'a>(arena: &'a Bump, src: Ident<'a>) -> Expr<'a> {
     match src {
@@ -1606,32 +1608,32 @@ fn binop<'a>() -> impl Parser<'a, BinOp> {
         // with other valid operators (e.g. "<=" begins with "<") must
         // come before the shorter ones; otherwise, they will never
         // be reached because the shorter one will pass and consume!
-        map!(string("|>"), |_| BinOp::Pizza),
+        map!(ascii_string("|>"), |_| BinOp::Pizza),
-        map!(string("=="), |_| BinOp::Equals),
+        map!(ascii_string("=="), |_| BinOp::Equals),
-        map!(string("!="), |_| BinOp::NotEquals),
+        map!(ascii_string("!="), |_| BinOp::NotEquals),
-        map!(string("&&"), |_| BinOp::And),
+        map!(ascii_string("&&"), |_| BinOp::And),
-        map!(string("||"), |_| BinOp::Or),
+        map!(ascii_string("||"), |_| BinOp::Or),
-        map!(char('+'), |_| BinOp::Plus),
+        map!(ascii_char('+'), |_| BinOp::Plus),
-        map!(char('*'), |_| BinOp::Star),
+        map!(ascii_char('*'), |_| BinOp::Star),
-        map!(char('-'), |_| BinOp::Minus),
+        map!(ascii_char('-'), |_| BinOp::Minus),
-        map!(string("//"), |_| BinOp::DoubleSlash),
+        map!(ascii_string("//"), |_| BinOp::DoubleSlash),
-        map!(char('/'), |_| BinOp::Slash),
+        map!(ascii_char('/'), |_| BinOp::Slash),
-        map!(string("<="), |_| BinOp::LessThanOrEq),
+        map!(ascii_string("<="), |_| BinOp::LessThanOrEq),
-        map!(char('<'), |_| BinOp::LessThan),
+        map!(ascii_char('<'), |_| BinOp::LessThan),
-        map!(string(">="), |_| BinOp::GreaterThanOrEq),
+        map!(ascii_string(">="), |_| BinOp::GreaterThanOrEq),
-        map!(char('>'), |_| BinOp::GreaterThan),
+        map!(ascii_char('>'), |_| BinOp::GreaterThan),
-        map!(char('^'), |_| BinOp::Caret),
+        map!(ascii_char('^'), |_| BinOp::Caret),
-        map!(string("%%"), |_| BinOp::DoublePercent),
+        map!(ascii_string("%%"), |_| BinOp::DoublePercent),
-        map!(char('%'), |_| BinOp::Percent)
+        map!(ascii_char('%'), |_| BinOp::Percent)
     )
 }
 
 pub fn list_literal<'a>(min_indent: u16) -> impl Parser<'a, Expr<'a>> {
     let elems = collection!(
-        char('['),
+        ascii_char('['),
         loc!(expr(min_indent)),
-        char(','),
+        ascii_char(','),
-        char(']'),
+        ascii_char(']'),
         min_indent
     );
 
@@ -1673,8 +1675,10 @@ pub fn record_literal<'a>(min_indent: u16) -> impl Parser<'a, Expr<'a>> {
                     };
 
                     // there can be field access, e.g. `{ x : 4 }.x`
-                    let (accesses, state) =
+                    let (accesses, state) = optional(one_or_more!(skip_first!(
-                        optional(one_or_more!(skip_first!(char('.'), lowercase_ident())))
+                        ascii_char('.'),
+                        lowercase_ident()
+                    )))
                     .parse(arena, state)?;
 
                     if let Some(fields) = accesses {
@@ -1768,7 +1772,7 @@ pub fn record_literal<'a>(min_indent: u16) -> impl Parser<'a, Expr<'a>> {
 /// This is mainly for matching tags in closure params, e.g. \@Foo -> ...
 pub fn private_tag<'a>() -> impl Parser<'a, &'a str> {
     map_with_arena!(
-        skip_first!(char('@'), global_tag()),
+        skip_first!(ascii_char('@'), global_tag()),
         |arena: &'a Bump, name: &'a str| {
             let mut buf = String::with_capacity_in(1 + name.len(), arena);
 

compiler/parse/src/ident.rs

@@ -1,6 +1,6 @@
 use crate::ast::Attempting;
 use crate::keyword;
-use crate::parser::{unexpected, unexpected_eof, Fail, FailReason, ParseResult, Parser, State};
+use crate::parser::{peek_utf8_char, unexpected, Fail, FailReason, ParseResult, Parser, State};
 use bumpalo::collections::string::String;
 use bumpalo::collections::vec::Vec;
 use bumpalo::Bump;
@@ -67,70 +67,68 @@ impl<'a> Ident<'a> {
 /// Sometimes we may want to check for those later in the process, and give
 /// more contextually-aware error messages than "unexpected `if`" or the like.
 #[inline(always)]
-pub fn parse_ident<'a, I>(
+pub fn parse_ident<'a>(
     arena: &'a Bump,
-    chars: &mut I,
+    mut state: State<'a>,
-    state: State<'a>,
+) -> ParseResult<'a, (Ident<'a>, Option<char>)> {
-) -> ParseResult<'a, (Ident<'a>, Option<char>)>
-where
-    I: Iterator<Item = char>,
-{
     let mut part_buf = String::new_in(arena); // The current "part" (parts are dot-separated.)
     let mut capitalized_parts: Vec<&'a str> = Vec::new_in(arena);
     let mut noncapitalized_parts: Vec<&'a str> = Vec::new_in(arena);
     let mut is_capitalized;
     let is_accessor_fn;
     let mut is_private_tag = false;
-    let mut chars_parsed;
 
     // Identifiers and accessor functions must start with either a letter or a dot.
     // If this starts with neither, it must be something else!
-    match chars.next() {
+    match peek_utf8_char(&state) {
-        Some(ch) => {
+        Ok((first_ch, bytes_parsed)) => {
-            if ch == '@' {
+            if first_ch.is_alphabetic() {
+                part_buf.push(first_ch);
+
+                is_capitalized = first_ch.is_uppercase();
+                is_accessor_fn = false;
+
+                state = state.advance_without_indenting(bytes_parsed)?;
+            } else if first_ch == '.' {
+                is_capitalized = false;
+                is_accessor_fn = true;
+
+                state = state.advance_without_indenting(bytes_parsed)?;
+            } else if first_ch == '@' {
+                state = state.advance_without_indenting(bytes_parsed)?;
+
                 // '@' must always be followed by a capital letter!
-                match chars.next() {
+                match peek_utf8_char(&state) {
-                    Some(ch) if ch.is_uppercase() => {
+                    Ok((next_ch, next_bytes_parsed)) => {
+                        if next_ch.is_uppercase() {
+                            state = state.advance_without_indenting(next_bytes_parsed)?;
+
                             part_buf.push('@');
-                        part_buf.push(ch);
+                            part_buf.push(next_ch);
 
                             is_private_tag = true;
                             is_capitalized = true;
                             is_accessor_fn = false;
-
-                        chars_parsed = 2;
-                    }
-                    Some(ch) => {
-                        return Err(unexpected(ch, 0, state, Attempting::Identifier));
-                    }
-                    None => {
-                        return Err(unexpected_eof(0, Attempting::Identifier, state));
-                    }
-                }
-            } else if ch.is_alphabetic() {
-                part_buf.push(ch);
-
-                is_capitalized = ch.is_uppercase();
-                is_accessor_fn = false;
-
-                chars_parsed = 1;
-            } else if ch == '.' {
-                is_capitalized = false;
-                is_accessor_fn = true;
-
-                chars_parsed = 1;
                         } else {
-                return Err(unexpected(ch, 0, state, Attempting::Identifier));
+                            return Err(unexpected(
+                                bytes_parsed + next_bytes_parsed,
+                                state,
+                                Attempting::Identifier,
+                            ));
                         }
                     }
-        None => {
+                    Err(reason) => return state.fail(reason),
-            return Err(unexpected_eof(0, Attempting::Identifier, state));
+                }
+            } else {
+                return Err(unexpected(0, state, Attempting::Identifier));
+            }
+        }
+        Err(reason) => return state.fail(reason),
     }
-    };
 
-    let mut next_char = None;
+    while !state.bytes.is_empty() {
-
+        match peek_utf8_char(&state) {
-    while let Some(ch) = chars.next() {
+            Ok((ch, bytes_parsed)) => {
                 // After the first character, only these are allowed:
                 //
                 // * Unicode alphabetic chars - you might name a variable `鹏` if that's clear to your readers
@@ -150,7 +148,6 @@ where
                             Some(ch),
                             arena,
                             state,
-                    chars,
                             capitalized_parts,
                             noncapitalized_parts,
                         );
@@ -167,7 +164,6 @@ where
                             Some(ch),
                             arena,
                             state,
-                    chars,
                             capitalized_parts,
                             noncapitalized_parts,
                         );
@@ -184,12 +180,13 @@ where
                 } else {
                     // This must be the end of the identifier. We're done!
 
-            next_char = Some(ch);
-
                     break;
                 }
 
-        chars_parsed += 1;
+                state = state.advance_without_indenting(bytes_parsed)?;
+            }
+            Err(reason) => return state.fail(reason),
+        }
     }
 
     if part_buf.is_empty() {
@@ -200,10 +197,9 @@ where
         // If we made it this far and don't have a next_char, then necessarily
         // we have consumed a '.' char previously.
         return malformed(
-            next_char.or_else(|| Some('.')),
+            Some('.'),
             arena,
             state,
-            chars,
             capitalized_parts,
             noncapitalized_parts,
         );
@@ -224,14 +220,7 @@ where
 
             Ident::AccessorFunction(value)
         } else {
-            return malformed(
+            return malformed(None, arena, state, capitalized_parts, noncapitalized_parts);
-                None,
-                arena,
-                state,
-                chars,
-                capitalized_parts,
-                noncapitalized_parts,
-            );
         }
     } else if noncapitalized_parts.is_empty() {
         // We have capitalized parts only, so this must be a tag.
@@ -245,33 +234,19 @@ where
                     }
                 } else {
                     // This is a qualified tag, which is not allowed!
-                    return malformed(
+                    return malformed(None, arena, state, capitalized_parts, noncapitalized_parts);
-                        None,
-                        arena,
-                        state,
-                        chars,
-                        capitalized_parts,
-                        noncapitalized_parts,
-                    );
                 }
             }
             None => {
                 // We had neither capitalized nor noncapitalized parts,
                 // yet we made it this far. The only explanation is that this was
                 // a stray '.' drifting through the cosmos.
-                return Err(unexpected('.', 1, state, Attempting::Identifier));
+                return Err(unexpected(1, state, Attempting::Identifier));
             }
         }
     } else if is_private_tag {
         // This is qualified field access with an '@' in front, which does not make sense!
-        return malformed(
+        return malformed(None, arena, state, capitalized_parts, noncapitalized_parts);
-            None,
-            arena,
-            state,
-            chars,
-            capitalized_parts,
-            noncapitalized_parts,
-        );
     } else {
         // We have multiple noncapitalized parts, so this must be field access.
         Ident::Access {
@@ -280,22 +255,16 @@ where
         }
     };
 
-    let state = state.advance_without_indenting(chars_parsed)?;
+    Ok(((answer, None), state))
-
-    Ok(((answer, next_char), state))
 }
 
-fn malformed<'a, I>(
+fn malformed<'a>(
     opt_bad_char: Option<char>,
     arena: &'a Bump,
-    state: State<'a>,
+    mut state: State<'a>,
-    chars: &mut I,
     capitalized_parts: Vec<&'a str>,
     noncapitalized_parts: Vec<&'a str>,
-) -> ParseResult<'a, (Ident<'a>, Option<char>)>
+) -> ParseResult<'a, (Ident<'a>, Option<char>)> {
-where
-    I: Iterator<Item = char>,
-{
     // Reconstruct the original string that we've been parsing.
     let mut full_string = String::new_in(arena);
 
@@ -311,7 +280,9 @@ where
     // Consume the remaining chars in the identifier.
     let mut next_char = None;
 
-    for ch in chars {
+    while !state.bytes.is_empty() {
+        match peek_utf8_char(&state) {
+            Ok((ch, bytes_parsed)) => {
                 // We can't use ch.is_alphanumeric() here because that passes for
                 // things that are "numeric" but not ASCII digits, like `¾`
                 if ch == '.' || ch.is_alphabetic() || ch.is_ascii_digit() {
@@ -321,20 +292,23 @@ where
 
                     break;
                 }
-    }
 
-    let chars_parsed = full_string.len();
+                state = state.advance_without_indenting(bytes_parsed)?;
+            }
+            Err(reason) => return state.fail(reason),
+        }
+    }
 
     Ok((
         (Ident::Malformed(full_string.into_bump_str()), next_char),
-        state.advance_without_indenting(chars_parsed)?,
+        state,
     ))
 }
 
 pub fn ident<'a>() -> impl Parser<'a, Ident<'a>> {
     move |arena: &'a Bump, state: State<'a>| {
         // Discard next_char; we don't need it.
-        let ((string, _), state) = parse_ident(arena, &mut state.input.chars(), state)?;
+        let ((string, _), state) = parse_ident(arena, state)?;
 
         Ok((string, state))
     }
@@ -344,33 +318,28 @@ pub fn global_tag_or_ident<'a, F>(pred: F) -> impl Parser<'a, &'a str>
 where
     F: Fn(char) -> bool,
 {
-    move |arena, state: State<'a>| {
+    move |arena, mut state: State<'a>| {
-        let mut chars = state.input.chars();
-
         // pred will determine if this is a tag or ident (based on capitalization)
-        let first_letter = match chars.next() {
+        let (first_letter, bytes_parsed) = match peek_utf8_char(&state) {
-            Some(first_char) => {
+            Ok((first_letter, bytes_parsed)) => {
-                if pred(first_char) {
+                if !pred(first_letter) {
-                    first_char
+                    return Err(unexpected(0, state, Attempting::RecordFieldLabel));
-                } else {
-                    return Err(unexpected(
-                        first_char,
-                        0,
-                        state,
-                        Attempting::RecordFieldLabel,
-                    ));
                 }
+
+                (first_letter, bytes_parsed)
             }
-            None => {
+            Err(reason) => return state.fail(reason),
-                return Err(unexpected_eof(0, Attempting::RecordFieldLabel, state));
-            }
         };
 
         let mut buf = String::with_capacity_in(1, arena);
 
         buf.push(first_letter);
 
-        for ch in chars {
+        state = state.advance_without_indenting(bytes_parsed)?;
+
+        while !state.bytes.is_empty() {
+            match peek_utf8_char(&state) {
+                Ok((ch, bytes_parsed)) => {
                     // After the first character, only these are allowed:
                     //
                     // * Unicode alphabetic chars - you might include `鹏` if that's clear to your readers
@@ -378,18 +347,18 @@ where
                     // * A ':' indicating the end of the field
                     if ch.is_alphabetic() || ch.is_ascii_digit() {
                         buf.push(ch);
+
+                        state = state.advance_without_indenting(bytes_parsed)?;
                     } else {
                         // This is the end of the field. We're done!
                         break;
                     }
                 }
+                Err(reason) => return state.fail(reason),
+            };
+        }
 
-        let chars_parsed = buf.len();
+        Ok((buf.into_bump_str(), state))
-
-        Ok((
-            buf.into_bump_str(),
-            state.advance_without_indenting(chars_parsed)?,
-        ))
     }
 }
 

compiler/parse/src/module.rs

@@ -6,7 +6,10 @@ use crate::blankspace::{space0_around, space1};
 use crate::expr::def;
 use crate::header::ModuleName;
 use crate::ident::unqualified_ident;
-use crate::parser::{self, char, loc, optional, string, unexpected, unexpected_eof, Parser, State};
+use crate::parser::{
+    self, ascii_char, ascii_string, loc, optional, peek_utf8_char, peek_utf8_char_at, unexpected,
+    Parser, State,
+};
 use bumpalo::collections::{String, Vec};
 use roc_region::all::Located;
 
@@ -30,7 +33,10 @@ pub fn app_module<'a>() -> impl Parser<'a, Module<'a>> {
 pub fn interface_header<'a>() -> impl Parser<'a, InterfaceHeader<'a>> {
     parser::map(
         and!(
-            skip_first!(string("interface"), and!(space1(1), loc!(module_name()))),
+            skip_first!(
+                ascii_string("interface"),
+                and!(space1(1), loc!(module_name()))
+            ),
             and!(exposes(), imports())
         ),
         |(
@@ -56,72 +62,68 @@ pub fn interface_header<'a>() -> impl Parser<'a, InterfaceHeader<'a>> {
 
 #[inline(always)]
 pub fn module_name<'a>() -> impl Parser<'a, ModuleName<'a>> {
-    move |arena, state: State<'a>| {
+    move |arena, mut state: State<'a>| {
-        let mut chars = state.input.chars();
+        match peek_utf8_char(&state) {
-
+            Ok((first_letter, bytes_parsed)) => {
-        let first_letter = match chars.next() {
+                if !first_letter.is_uppercase() {
-            Some(first_char) => {
+                    return Err(unexpected(0, state, Attempting::Module));
-                // Module names must all be uppercase
-                if first_char.is_uppercase() {
-                    first_char
-                } else {
-                    return Err(unexpected(
-                        first_char,
-                        0,
-                        state,
-                        Attempting::RecordFieldLabel,
-                    ));
-                }
-            }
-            None => {
-                return Err(unexpected_eof(0, Attempting::Identifier, state));
-            }
                 };
 
-        let mut buf = String::with_capacity_in(1, arena);
+                let mut buf = String::with_capacity_in(4, arena);
 
                 buf.push(first_letter);
 
-        while let Some(ch) = chars.next() {
+                state = state.advance_without_indenting(bytes_parsed)?;
+
+                while !state.bytes.is_empty() {
+                    match peek_utf8_char(&state) {
+                        Ok((ch, bytes_parsed)) => {
                             // After the first character, only these are allowed:
                             //
                             // * Unicode alphabetic chars - you might include `鹏` if that's clear to your readers
                             // * ASCII digits - e.g. `1` but not `¾`, both of which pass .is_numeric()
                             // * A '.' separating module parts
                             if ch.is_alphabetic() || ch.is_ascii_digit() {
+                                state = state.advance_without_indenting(bytes_parsed)?;
+
                                 buf.push(ch);
                             } else if ch == '.' {
-                match chars.next() {
+                                match peek_utf8_char_at(&state, 1) {
-                    Some(next) => {
+                                    Ok((next, next_bytes_parsed)) => {
                                         if next.is_uppercase() {
                                             // If we hit another uppercase letter, keep going!
                                             buf.push('.');
                                             buf.push(next);
+
+                                            state = state.advance_without_indenting(
+                                                bytes_parsed + next_bytes_parsed,
+                                            )?;
                                         } else {
                                             // We have finished parsing the module name.
                                             //
                                             // There may be an identifier after this '.',
                                             // e.g. "baz" in `Foo.Bar.baz`
-                            break;
+                                            return Ok((
+                                                ModuleName::new(buf.into_bump_str()),
+                                                state,
+                                            ));
                                         }
                                     }
-                    None => {
+                                    Err(reason) => return state.fail(reason),
-                        // A module name can't end with a '.'
-                        return Err(unexpected_eof(0, Attempting::Identifier, state));
-                    }
                                 }
                             } else {
                                 // This is the end of the module name. We're done!
                                 break;
                             }
                         }
+                        Err(reason) => return state.fail(reason),
+                    }
+                }
 
-        let chars_parsed = buf.len();
+                Ok((ModuleName::new(buf.into_bump_str()), state))
-
+            }
-        Ok((
+            Err(reason) => state.fail(reason),
-            ModuleName::new(buf.into_bump_str()),
+        }
-            state.advance_without_indenting(chars_parsed)?,
-        ))
     }
 }
 
@@ -129,7 +131,7 @@ pub fn module_name<'a>() -> impl Parser<'a, ModuleName<'a>> {
 fn app_header<'a>() -> impl Parser<'a, AppHeader<'a>> {
     parser::map(
         and!(
-            skip_first!(string("app"), and!(space1(1), loc!(module_name()))),
+            skip_first!(ascii_string("app"), and!(space1(1), loc!(module_name()))),
             and!(provides(), imports())
         ),
         |(
@@ -167,8 +169,14 @@ fn provides<'a>() -> impl Parser<
     ),
 > {
     and!(
-        and!(skip_second!(space1(1), string("provides")), space1(1)),
+        and!(skip_second!(space1(1), ascii_string("provides")), space1(1)),
-        collection!(char('['), loc!(exposes_entry()), char(','), char(']'), 1)
+        collection!(
+            ascii_char('['),
+            loc!(exposes_entry()),
+            ascii_char(','),
+            ascii_char(']'),
+            1
+        )
     )
 }
 
@@ -181,8 +189,14 @@ fn exposes<'a>() -> impl Parser<
     ),
 > {
     and!(
-        and!(skip_second!(space1(1), string("exposes")), space1(1)),
+        and!(skip_second!(space1(1), ascii_string("exposes")), space1(1)),
-        collection!(char('['), loc!(exposes_entry()), char(','), char(']'), 1)
+        collection!(
+            ascii_char('['),
+            loc!(exposes_entry()),
+            ascii_char(','),
+            ascii_char(']'),
+            1
+        )
     )
 }
 
@@ -195,8 +209,14 @@ fn imports<'a>() -> impl Parser<
     ),
 > {
     and!(
-        and!(skip_second!(space1(1), string("imports")), space1(1)),
+        and!(skip_second!(space1(1), ascii_string("imports")), space1(1)),
-        collection!(char('['), loc!(imports_entry()), char(','), char(']'), 1)
+        collection!(
+            ascii_char('['),
+            loc!(imports_entry()),
+            ascii_char(','),
+            ascii_char(']'),
+            1
+        )
     )
 }
 
@@ -213,8 +233,14 @@ fn imports_entry<'a>() -> impl Parser<'a, ImportsEntry<'a>> {
             module_name(),
             // e.g. `.{ Task, after}`
             optional(skip_first!(
-                char('.'),
+                ascii_char('.'),
-                collection!(char('{'), loc!(exposes_entry()), char(','), char('}'), 1)
+                collection!(
+                    ascii_char('{'),
+                    loc!(exposes_entry()),
+                    ascii_char(','),
+                    ascii_char('}'),
+                    1
+                )
             ))
         ),
         |arena,

compiler/parse/src/number_literal.rs

@@ -1,23 +1,19 @@
 use crate::ast::{Attempting, Base, Expr};
-use crate::parser::{unexpected, unexpected_eof, ParseResult, Parser, State};
+use crate::parser::{parse_utf8, unexpected, unexpected_eof, ParseResult, Parser, State};
 use std::char;
+use std::str::from_utf8_unchecked;
 
 pub fn number_literal<'a>() -> impl Parser<'a, Expr<'a>> {
     move |_arena, state: State<'a>| {
-        let mut chars = state.input.chars();
+        let bytes = &mut state.bytes.iter();
 
-        match chars.next() {
+        match bytes.next() {
-            Some(first_ch) => {
+            Some(&first_byte) => {
                 // Number literals must start with either an '-' or a digit.
-                if first_ch == '-' || first_ch.is_ascii_digit() {
+                if first_byte == b'-' || (first_byte as char).is_ascii_digit() {
-                    parse_number_literal(first_ch, &mut chars, state)
+                    parse_number_literal(first_byte as char, bytes, state)
                 } else {
-                    Err(unexpected(
+                    Err(unexpected(1, state, Attempting::NumberLiteral))
-                        first_ch,
-                        first_ch.len_utf8(),
-                        state,
-                        Attempting::NumberLiteral,
-                    ))
                 }
             }
             None => Err(unexpected_eof(0, state.attempting, state)),
@@ -28,11 +24,11 @@ pub fn number_literal<'a>() -> impl Parser<'a, Expr<'a>> {
 #[inline(always)]
 fn parse_number_literal<'a, I>(
     first_ch: char,
-    chars: &mut I,
+    bytes: &mut I,
     state: State<'a>,
 ) -> ParseResult<'a, Expr<'a>>
 where
-    I: Iterator<Item = char>,
+    I: Iterator<Item = &'a u8>,
 {
     use self::LiteralType::*;
 
@@ -40,13 +36,12 @@ where
 
     // We already parsed 1 character (which may have been a minus sign).
     let mut bytes_parsed = 1;
-    let mut prev_ch = first_ch;
+    let mut prev_byte = first_ch as u8;
     let mut has_parsed_digits = first_ch.is_ascii_digit();
 
-    for next_ch in chars {
+    for &next_byte in bytes {
         let err_unexpected = || {
             Err(unexpected(
-                next_ch,
                 bytes_parsed,
                 state.clone(),
                 Attempting::NumberLiteral,
@@ -55,23 +50,26 @@ where
 
         let is_potentially_non_base10 = || {
             (bytes_parsed == 1 && first_ch == '0')
-                || (bytes_parsed == 2 && first_ch == '-' && prev_ch == '0')
+                || (bytes_parsed == 2 && first_ch == '-' && prev_byte == b'0')
         };
 
-        if next_ch == '.' {
+        match next_byte as char {
+            '.' => {
                 if typ == Float {
                     // You only get one decimal point!
                     return err_unexpected();
                 } else {
                     typ = Float;
                 }
-        } else if next_ch == 'x' {
+            }
+            'x' => {
                 if is_potentially_non_base10() {
                     typ = Hex;
                 } else {
                     return err_unexpected();
                 }
-        } else if next_ch == 'b' && typ == Num {
+            }
+            'b' if typ == Num => {
                 // We have to check for typ == Num because otherwise we get a false
                 // positive here when parsing a hex literal that happens to have
                 // a 'b' in it, e.g. 0xbbbb
@@ -80,20 +78,24 @@ where
                 } else {
                     return err_unexpected();
                 }
-        } else if next_ch == 'o' {
+            }
+            'o' => {
                 if is_potentially_non_base10() {
                     typ = Octal;
                 } else {
                     return err_unexpected();
                 }
-        } else if next_ch.is_ascii_digit() {
+            }
+            next_ch if next_ch.is_ascii_digit() => {
                 has_parsed_digits = true;
-        } else if next_ch != '_' &&
+            }
+            next_ch
+                if next_ch != '_' &&
             // ASCII alphabetic chars (like 'a' and 'f') are allowed in Hex int literals.
             // We parse them in any int literal, so we can give a more helpful error
             // in canonicalization (e.g. "the character 'f' is not allowed in Octal literals"
             // or "the character 'g' is outside the range of valid Hex literals")
-            !next_ch.is_ascii_alphabetic()
+            !next_ch.is_ascii_alphabetic() =>
             {
                 if has_parsed_digits {
                     // We hit an invalid number literal character; we're done!
@@ -103,43 +105,36 @@ where
                     return err_unexpected();
                 }
             }
+            _ => {}
+        }
 
         // Since we only consume characters in the ASCII range for number literals,
         // this will always be exactly 1. There's no need to call next_ch.utf8_len().
         bytes_parsed += 1;
-        prev_ch = next_ch;
+        prev_byte = next_byte;
     }
 
-    let from_base = |base| {
-        let is_negative = first_ch == '-';
-        let string = if is_negative {
-            &state.input[3..bytes_parsed]
-        } else {
-            &state.input[2..bytes_parsed]
-        };
-
-        Expr::NonBase10Int {
-            is_negative,
-            string,
-            base,
-        }
-    };
-
     // At this point we have a number, and will definitely succeed.
     // If the number is malformed (outside the supported range),
     // we'll succeed with an appropriate Expr which records that.
-    let expr = match typ {
+    match typ {
-        Num => Expr::Num(&state.input[0..bytes_parsed]),
+        Num => Ok((
-        Float => Expr::Float(&state.input[0..bytes_parsed]),
+            // SAFETY: it's safe to use from_utf8_unchecked here, because we've
+            // already validated that this range contains only ASCII digits
+            Expr::Num(unsafe { from_utf8_unchecked(&state.bytes[0..bytes_parsed]) }),
+            state.advance_without_indenting(bytes_parsed)?,
+        )),
+        Float => Ok((
+            // SAFETY: it's safe to use from_utf8_unchecked here, because we've
+            // already validated that this range contains only ASCII digits
+            Expr::Float(unsafe { from_utf8_unchecked(&state.bytes[0..bytes_parsed]) }),
+            state.advance_without_indenting(bytes_parsed)?,
+        )),
         // For these we trim off the 0x/0o/0b part
-        Hex => from_base(Base::Hex),
+        Hex => from_base(Base::Hex, first_ch, bytes_parsed, state),
-        Octal => from_base(Base::Octal),
+        Octal => from_base(Base::Octal, first_ch, bytes_parsed, state),
-        Binary => from_base(Base::Binary),
+        Binary => from_base(Base::Binary, first_ch, bytes_parsed, state),
-    };
+    }
-
-    let next_state = state.advance_without_indenting(bytes_parsed)?;
-
-    Ok((expr, next_state))
 }
 
 #[derive(Debug, PartialEq, Eq)]
@@ -150,3 +145,29 @@ enum LiteralType {
     Octal,
     Binary,
 }
+
+fn from_base(
+    base: Base,
+    first_ch: char,
+    bytes_parsed: usize,
+    state: State<'_>,
+) -> ParseResult<'_, Expr<'_>> {
+    let is_negative = first_ch == '-';
+    let bytes = if is_negative {
+        &state.bytes[3..bytes_parsed]
+    } else {
+        &state.bytes[2..bytes_parsed]
+    };
+
+    match parse_utf8(bytes) {
+        Ok(string) => Ok((
+            Expr::NonBase10Int {
+                is_negative,
+                string,
+                base,
+            },
+            state.advance_without_indenting(bytes_parsed)?,
+        )),
+        Err(reason) => state.fail(reason),
+    }
+}

compiler/parse/src/parser.rs

@@ -1,14 +1,17 @@
 use crate::ast::Attempting;
 use bumpalo::collections::vec::Vec;
 use bumpalo::Bump;
+use encode_unicode::CharExt;
 use roc_region::all::{Located, Region};
+use std::fmt;
+use std::str::from_utf8;
 use std::{char, u16};
 
 /// A position in a source file.
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Clone, PartialEq, Eq)]
 pub struct State<'a> {
-    /// The raw input string.
+    /// The raw input bytes from the file.
-    pub input: &'a str,
+    pub bytes: &'a [u8],
 
     /// Current line of the input
     pub line: u32,
@@ -39,15 +42,15 @@ pub enum Either<First, Second> {
 }
 
 impl<'a> State<'a> {
-    pub fn new(input: &'a str, attempting: Attempting) -> State<'a> {
+    pub fn new(bytes: &'a [u8], attempting: Attempting) -> State<'a> {
         State {
-            input,
+            bytes,
             line: 0,
             column: 0,
             indent_col: 0,
             is_indenting: true,
             attempting,
-            original_len: input.len(),
+            original_len: bytes.len(),
         }
     }
 
@@ -69,7 +72,7 @@ impl<'a> State<'a> {
     ///
     /// So if the parser has consumed 8 bytes, this function will return 8.
     pub fn bytes_consumed(&self) -> usize {
-        self.original_len - self.input.len()
+        self.original_len - self.bytes.len()
     }
 
     /// Increments the line, then resets column, indent_col, and is_indenting.
@@ -77,7 +80,7 @@ impl<'a> State<'a> {
     pub fn newline(&self) -> Result<Self, (Fail, Self)> {
         match self.line.checked_add(1) {
             Some(line) => Ok(State {
-                input: &self.input[1..],
+                bytes: &self.bytes[1..],
                 line,
                 column: 0,
                 indent_col: 0,
@@ -99,11 +102,11 @@ impl<'a> State<'a> {
     /// This assumes we are *not* advancing with spaces, or at least that
     /// any spaces on the line were preceded by non-spaces - which would mean
     /// they weren't eligible to indent anyway.
-    pub fn advance_without_indenting(&self, quantity: usize) -> Result<Self, (Fail, Self)> {
+    pub fn advance_without_indenting(self, quantity: usize) -> Result<Self, (Fail, Self)> {
         match (self.column as usize).checked_add(quantity) {
             Some(column_usize) if column_usize <= u16::MAX as usize => {
                 Ok(State {
-                    input: &self.input[quantity..],
+                    bytes: &self.bytes[quantity..],
                     line: self.line,
                     column: column_usize as u16,
                     indent_col: self.indent_col,
@@ -141,7 +144,7 @@ impl<'a> State<'a> {
                 };
 
                 Ok(State {
-                    input: &self.input[spaces..],
+                    bytes: &self.bytes[spaces..],
                     line: self.line,
                     column: column_usize as u16,
                     indent_col,
@@ -169,6 +172,35 @@ impl<'a> State<'a> {
             end_line: self.line,
         }
     }
+
+    /// Return a failing ParseResult for the given FailReason
+    pub fn fail<T>(self, reason: FailReason) -> Result<(T, Self), (Fail, Self)> {
+        Err((
+            Fail {
+                reason,
+                attempting: self.attempting,
+            },
+            self,
+        ))
+    }
+}
+
+impl<'a> fmt::Debug for State<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "State {{")?;
+
+        match from_utf8(self.bytes) {
+            Ok(string) => write!(f, "\n\tbytes: [utf8] {:?}", string)?,
+            Err(_) => write!(f, "\n\tbytes: [invalid utf8] {:?}", self.bytes)?,
+        }
+
+        write!(f, "\n\t(line, col): ({}, {}),", self.line, self.column)?;
+        write!(f, "\n\tindent_col: {}", self.indent_col)?;
+        write!(f, "\n\tis_indenting: {:?}", self.is_indenting)?;
+        write!(f, "\n\tattempting: {:?}", self.attempting)?;
+        write!(f, "\n\toriginal_len: {}", self.original_len)?;
+        write!(f, "\n}}")
+    }
 }
 
 #[test]
@@ -182,13 +214,14 @@ pub type ParseResult<'a, Output> = Result<(Output, State<'a>), (Fail, State<'a>)
 
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum FailReason {
-    Unexpected(char, Region),
+    Unexpected(Region),
     OutdentedTooFar,
     ConditionFailed,
     LineTooLong(u32 /* which line was too long */),
     TooManyLines,
     Eof(Region),
     InvalidPattern,
+    BadUtf8,
     ReservedKeyword(Region),
     ArgumentsBeforeEquals(Region),
 }
@@ -332,13 +365,12 @@ pub fn unexpected_eof(
 }
 
 pub fn unexpected(
-    ch: char,
     chars_consumed: usize,
     state: State<'_>,
     attempting: Attempting,
 ) -> (Fail, State<'_>) {
     checked_unexpected(chars_consumed, state, |region| Fail {
-        reason: FailReason::Unexpected(ch, region),
+        reason: FailReason::Unexpected(region),
         attempting,
     })
 }
@@ -385,9 +417,9 @@ fn line_too_long(attempting: Attempting, state: State<'_>) -> (Fail, State<'_>)
     // (for example) the LineTooLong initially occurs in the middle of
     // a one_of chain, which would otherwise prevent it from propagating.
     let column = u16::MAX;
-    let input = state.input.get(0..state.input.len()).unwrap();
+    let bytes = state.bytes.get(0..state.bytes.len()).unwrap();
     let state = State {
-        input,
+        bytes,
         line: state.line,
         indent_col: state.indent_col,
         is_indenting: state.is_indenting,
@@ -399,29 +431,75 @@ fn line_too_long(attempting: Attempting, state: State<'_>) -> (Fail, State<'_>)
     (fail, state)
 }
 
-/// A single char.
+/// A single ASCII char.
-pub fn char<'a>(expected: char) -> impl Parser<'a, ()> {
+pub fn ascii_char<'a>(expected: char) -> impl Parser<'a, ()> {
-    move |_arena, state: State<'a>| match state.input.chars().next() {
+    // Make sure this really is an ASCII char!
-        Some(actual) if expected == actual => Ok(((), state.advance_without_indenting(1)?)),
+    debug_assert!(expected.len_utf8() == 1);
-        Some(other_ch) => Err(unexpected(other_ch, 0, state, Attempting::Keyword)),
+
+    move |_arena, state: State<'a>| match state.bytes.first() {
+        Some(&actual) if expected == actual as char => {
+            Ok(((), state.advance_without_indenting(1)?))
+        }
+        Some(_) => Err(unexpected(0, state, Attempting::Keyword)),
         _ => Err(unexpected_eof(0, Attempting::Keyword, state)),
     }
 }
 
-/// A hardcoded keyword string with no newlines in it.
+/// A single UTF-8-encoded char. This will both parse *and* validate that the
-pub fn string<'a>(keyword: &'static str) -> impl Parser<'a, ()> {
+/// char is valid UTF-8, but it will *not* advance the state.
-    // We can't have newlines because we don't attempt to advance the row
+pub fn peek_utf8_char<'a>(state: &State<'a>) -> Result<(char, usize), FailReason> {
-    // in the state, only the column.
+    if !state.bytes.is_empty() {
-    debug_assert!(!keyword.contains('\n'));
+        match char::from_utf8_slice_start(state.bytes) {
+            Ok((ch, len_utf8)) => Ok((ch, len_utf8)),
+            Err(_) => Err(FailReason::BadUtf8),
+        }
+    } else {
+        Err(FailReason::Eof(
+            Region::zero(), /* TODO get a better region */
+        ))
+    }
+}
+
+/// A single UTF-8-encoded char, with an offset. This will both parse *and*
+/// validate that the char is valid UTF-8, but it will *not* advance the state.
+pub fn peek_utf8_char_at<'a>(
+    state: &State<'a>,
+    offset: usize,
+) -> Result<(char, usize), FailReason> {
+    if state.bytes.len() > offset {
+        let bytes = &state.bytes[offset..];
+
+        match char::from_utf8_slice_start(bytes) {
+            Ok((ch, len_utf8)) => Ok((ch, len_utf8)),
+            Err(_) => Err(FailReason::BadUtf8),
+        }
+    } else {
+        Err(FailReason::Eof(
+            Region::zero(), /* TODO get a better region */
+        ))
+    }
+}
+
+/// A hardcoded string with no newlines, consisting only of ASCII characters
+pub fn ascii_string<'a>(keyword: &'static str) -> impl Parser<'a, ()> {
+    // Verify that this really is exclusively ASCII characters.
+    // The `unsafe` block in this function relies upon this assumption!
+    //
+    // Also, this can't have newlines because we don't attempt to advance
+    // the row in the state, only the column.
+    debug_assert!(keyword.chars().all(|ch| ch.len_utf8() == 1 && ch != '\n'));
 
     move |_arena, state: State<'a>| {
-        let input = state.input;
         let len = keyword.len();
 
         // TODO do this comparison in one SIMD instruction (on supported systems)
-        match input.get(0..len) {
+        match state.bytes.get(0..len) {
-            Some(next_str) if next_str == keyword => {
+            Some(next_str) => {
+                if next_str == keyword.as_bytes() {
                     Ok(((), state.advance_without_indenting(len)?))
+                } else {
+                    Err(unexpected(len, state, Attempting::Keyword))
+                }
             }
             _ => Err(unexpected_eof(0, Attempting::Keyword, state)),
         }
@@ -686,7 +764,7 @@ macro_rules! collection {
                 // We could change the AST to add extra storage specifically to
                 // support empty literals containing newlines or comments, but this
                 // does not seem worth even the tiniest regression in compiler performance.
-                zero_or_more!($crate::parser::char(' ')),
+                zero_or_more!($crate::parser::ascii_char(' ')),
                 skip_second!(
                     $crate::parser::sep_by0(
                         $delimiter,
@@ -912,6 +990,7 @@ macro_rules! record_field {
             use $crate::ast::AssignedField::*;
             use $crate::blankspace::{space0, space0_before};
             use $crate::ident::lowercase_ident;
+            use $crate::parser::ascii_char;
             use $crate::parser::Either::*;
 
             // You must have a field name, e.g. "email"
@@ -922,8 +1001,8 @@ macro_rules! record_field {
             // Having a value is optional; both `{ email }` and `{ email: blah }` work.
             // (This is true in both literals and types.)
             let (opt_loc_val, state) = $crate::parser::optional(either!(
-                skip_first!(char(':'), space0_before($val_parser, $min_indent)),
+                skip_first!(ascii_char(':'), space0_before($val_parser, $min_indent)),
-                skip_first!(char('?'), space0_before($val_parser, $min_indent))
+                skip_first!(ascii_char('?'), space0_before($val_parser, $min_indent))
             ))
             .parse(arena, state)?;
 
@@ -952,10 +1031,10 @@ macro_rules! record_field {
 macro_rules! record_without_update {
     ($val_parser:expr, $min_indent:expr) => {
         collection!(
-            char('{'),
+            ascii_char('{'),
             loc!(record_field!($val_parser, $min_indent)),
-            char(','),
+            ascii_char(','),
-            char('}'),
+            ascii_char('}'),
             $min_indent
         )
     };
@@ -965,7 +1044,7 @@ macro_rules! record_without_update {
 macro_rules! record {
     ($val_parser:expr, $min_indent:expr) => {
         skip_first!(
-            $crate::parser::char('{'),
+            $crate::parser::ascii_char('{'),
             and!(
                 // You can optionally have an identifier followed by an '&' to
                 // make this a record update, e.g. { Foo.user & username: "blah" }.
@@ -981,7 +1060,7 @@ macro_rules! record {
                         )),
                         $min_indent
                     ),
-                    $crate::parser::char('&')
+                    $crate::parser::ascii_char('&')
                 )),
                 loc!(skip_first!(
                     // We specifically allow space characters inside here, so that
@@ -995,16 +1074,16 @@ macro_rules! record {
                     // We could change the AST to add extra storage specifically to
                     // support empty literals containing newlines or comments, but this
                     // does not seem worth even the tiniest regression in compiler performance.
-                    zero_or_more!($crate::parser::char(' ')),
+                    zero_or_more!($crate::parser::ascii_char(' ')),
                     skip_second!(
                         $crate::parser::sep_by0(
-                            $crate::parser::char(','),
+                            $crate::parser::ascii_char(','),
                             $crate::blankspace::space0_around(
                                 loc!(record_field!($val_parser, $min_indent)),
                                 $min_indent
                             )
                         ),
-                        $crate::parser::char('}')
+                        $crate::parser::ascii_char('}')
                     )
                 ))
             )
@@ -1067,3 +1146,10 @@ where
 {
     attempt!(attempting, parser)
 }
+
+pub fn parse_utf8(bytes: &[u8]) -> Result<&str, FailReason> {
+    match from_utf8(bytes) {
+        Ok(string) => Ok(string),
+        Err(_) => Err(FailReason::BadUtf8),
+    }
+}

compiler/parse/src/string_literal.rs

@@ -1,8 +1,7 @@
 use crate::ast::Attempting;
-use crate::parser::{unexpected, unexpected_eof, ParseResult, Parser, State};
+use crate::parser::{parse_utf8, unexpected, unexpected_eof, ParseResult, Parser, State};
 use bumpalo::collections::vec::Vec;
 use bumpalo::Bump;
-use std::char;
 
 pub enum StringLiteral<'a> {
     Line(&'a str),
@@ -11,14 +10,15 @@ pub enum StringLiteral<'a> {
 
 pub fn parse<'a>() -> impl Parser<'a, StringLiteral<'a>> {
     move |arena: &'a Bump, state: State<'a>| {
-        let mut chars = state.input.chars();
+        let mut bytes = state.bytes.iter();
 
         // String literals must start with a quote.
         // If this doesn't, it must not be a string literal!
-        match chars.next() {
+        match bytes.next() {
-            Some('"') => (),
+            Some(&byte) => {
-            Some(other_char) => {
+                if byte != b'"' {
-                return Err(unexpected(other_char, 0, state, Attempting::StringLiteral));
+                    return Err(unexpected(0, state, Attempting::StringLiteral));
+                }
             }
             None => {
                 return Err(unexpected_eof(0, Attempting::StringLiteral, state));
@@ -35,44 +35,49 @@ pub fn parse<'a>() -> impl Parser<'a, StringLiteral<'a>> {
         // Since we're keeping the entire raw string, all we need to track is
         // how many characters we've parsed. So far, that's 1 (the opening `"`).
         let mut parsed_chars = 1;
-        let mut prev_ch = '"';
+        let mut prev_byte = b'"';
 
-        while let Some(ch) = chars.next() {
+        while let Some(&byte) = bytes.next() {
             parsed_chars += 1;
 
             // Potentially end the string (unless this is an escaped `"`!)
-            if ch == '"' && prev_ch != '\\' {
+            if byte == b'"' && prev_byte != b'\\' {
-                let string = if parsed_chars == 2 {
+                let (string, state) = if parsed_chars == 2 {
-                    if let Some('"') = chars.next() {
+                    match bytes.next() {
+                        Some(byte) if *byte == b'"' => {
                             // If the first three chars were all `"`, then this
                             // literal begins with `"""` and is a block string.
-                        return parse_block_string(arena, state, &mut chars);
+                            return parse_block_string(arena, state, &mut bytes);
-                    } else {
+                        }
-                        ""
+                        _ => ("", state.advance_without_indenting(2)?),
                     }
                 } else {
                     // Start at 1 so we omit the opening `"`.
                     // Subtract 1 from parsed_chars so we omit the closing `"`.
-                    &state.input[1..(parsed_chars - 1)]
+                    let string_bytes = &state.bytes[1..(parsed_chars - 1)];
+
+                    match parse_utf8(string_bytes) {
+                        Ok(string) => (string, state.advance_without_indenting(parsed_chars)?),
+                        Err(reason) => {
+                            return state.fail(reason);
+                        }
+                    }
                 };
 
-                let next_state = state.advance_without_indenting(parsed_chars)?;
+                return Ok((StringLiteral::Line(string), state));
-
+            } else if byte == b'\n' {
-                return Ok((StringLiteral::Line(string), next_state));
-            } else if ch == '\n' {
                 // This is a single-line string, which cannot have newlines!
                 // Treat this as an unclosed string literal, and consume
                 // all remaining chars. This will mask all other errors, but
                 // it should make it easiest to debug; the file will be a giant
                 // error starting from where the open quote appeared.
                 return Err(unexpected(
-                    '\n',
+                    state.bytes.len() - 1,
-                    state.input.len() - 1,
                     state,
                     Attempting::StringLiteral,
                 ));
             } else {
-                prev_ch = ch;
+                prev_byte = byte;
             }
         }
 
@@ -88,48 +93,64 @@ pub fn parse<'a>() -> impl Parser<'a, StringLiteral<'a>> {
 fn parse_block_string<'a, I>(
     arena: &'a Bump,
     state: State<'a>,
-    chars: &mut I,
+    bytes: &mut I,
 ) -> ParseResult<'a, StringLiteral<'a>>
 where
-    I: Iterator<Item = char>,
+    I: Iterator<Item = &'a u8>,
 {
     // So far we have consumed the `"""` and that's it.
     let mut parsed_chars = 3;
-    let mut prev_ch = '"';
+    let mut prev_byte = b'"';
     let mut quotes_seen = 0;
 
     // start at 3 to omit the opening `"`.
     let mut line_start = 3;
 
-    let mut lines = Vec::new_in(arena);
+    let mut lines: Vec<'a, &'a str> = Vec::new_in(arena);
 
-    for ch in chars {
+    for byte in bytes {
         parsed_chars += 1;
 
         // Potentially end the string (unless this is an escaped `"`!)
-        if ch == '"' && prev_ch != '\\' {
+        if *byte == b'"' && prev_byte != b'\\' {
             if quotes_seen == 2 {
                 // three consecutive qoutes, end string
 
                 // Subtract 3 from parsed_chars so we omit the closing `"`.
-                let string = &state.input[line_start..(parsed_chars - 3)];
+                let line_bytes = &state.bytes[line_start..(parsed_chars - 3)];
-                lines.push(string);
 
-                let next_state = state.advance_without_indenting(parsed_chars)?;
+                return match parse_utf8(line_bytes) {
+                    Ok(line) => {
+                        let state = state.advance_without_indenting(parsed_chars)?;
 
-                return Ok((StringLiteral::Block(arena.alloc(lines)), next_state));
+                        lines.push(line);
+
+                        Ok((StringLiteral::Block(arena.alloc(lines)), state))
+                    }
+                    Err(reason) => state.fail(reason),
+                };
             }
             quotes_seen += 1;
-        } else if ch == '\n' {
+        } else if *byte == b'\n' {
             // note this includes the newline
-            let string = &state.input[line_start..parsed_chars];
+            let line_bytes = &state.bytes[line_start..parsed_chars];
-            lines.push(string);
+
+            match parse_utf8(line_bytes) {
+                Ok(line) => {
+                    lines.push(line);
+
                     quotes_seen = 0;
                     line_start = parsed_chars;
+                }
+                Err(reason) => {
+                    return state.fail(reason);
+                }
+            }
         } else {
             quotes_seen = 0;
         }
-        prev_ch = ch;
+
+        prev_byte = *byte;
     }
 
     // We ran out of characters before finding 3 closing quotes
@@ -137,6 +158,6 @@ where
         parsed_chars,
         // TODO custom BlockStringLiteral?
         Attempting::StringLiteral,
-        state.clone(),
+        state,
     ))
 }

compiler/parse/src/type_annotation.rs

@@ -4,8 +4,8 @@ use crate::expr::{global_tag, private_tag};
 use crate::ident::join_module_parts;
 use crate::keyword;
 use crate::parser::{
-    allocated, char, not, optional, string, unexpected, unexpected_eof, Either, ParseResult,
+    allocated, ascii_char, ascii_string, not, optional, peek_utf8_char, unexpected, Either,
-    Parser, State,
+    ParseResult, Parser, State,
 };
 use bumpalo::collections::string::String;
 use bumpalo::collections::vec::Vec;
@@ -22,10 +22,10 @@ macro_rules! tag_union {
         map!(
             and!(
                 collection!(
-                    char('['),
+                    ascii_char('['),
                     loc!(tag_type($min_indent)),
-                    char(','),
+                    ascii_char(','),
-                    char(']'),
+                    ascii_char(']'),
                     $min_indent
                 ),
                 optional(
@@ -61,7 +61,7 @@ pub fn term<'a>(min_indent: u16) -> impl Parser<'a, Located<TypeAnnotation<'a>>>
                 and!(
                     space1(min_indent),
                     skip_first!(
-                        string(keyword::AS),
+                        ascii_string(keyword::AS),
                         space1_before(term(min_indent), min_indent)
                     )
                 )
@@ -89,7 +89,7 @@ pub fn term<'a>(min_indent: u16) -> impl Parser<'a, Located<TypeAnnotation<'a>>>
 
 /// The `*` type variable, e.g. in (List *) Wildcard,
 fn loc_wildcard<'a>() -> impl Parser<'a, Located<TypeAnnotation<'a>>> {
-    map!(loc!(char('*')), |loc_val: Located<()>| {
+    map!(loc!(ascii_char('*')), |loc_val: Located<()>| {
         loc_val.map(|_| TypeAnnotation::Wildcard)
     })
 }
@@ -97,7 +97,7 @@ fn loc_wildcard<'a>() -> impl Parser<'a, Located<TypeAnnotation<'a>>> {
 pub fn loc_applied_arg<'a>(min_indent: u16) -> impl Parser<'a, Located<TypeAnnotation<'a>>> {
     skip_first!(
         // Once we hit an "as", stop parsing args
-        not(string(keyword::AS)),
+        not(ascii_string(keyword::AS)),
         one_of!(
             loc_wildcard(),
             loc_parenthetical_type(min_indent),
@@ -112,12 +112,12 @@ pub fn loc_applied_arg<'a>(min_indent: u16) -> impl Parser<'a, Located<TypeAnnot
 #[inline(always)]
 fn loc_parenthetical_type<'a>(min_indent: u16) -> impl Parser<'a, Located<TypeAnnotation<'a>>> {
     between!(
-        char('('),
+        ascii_char('('),
         space0_around(
             move |arena, state| expression(min_indent).parse(arena, state),
             min_indent,
         ),
-        char(')')
+        ascii_char(')')
     )
 }
 
@@ -208,7 +208,7 @@ fn expression<'a>(min_indent: u16) -> impl Parser<'a, Located<TypeAnnotation<'a>
     move |arena, state: State<'a>| {
         let (first, state) = space0_before(term(min_indent), min_indent).parse(arena, state)?;
         let (rest, state) = zero_or_more!(skip_first!(
-            char(','),
+            ascii_char(','),
             space0_around(term(min_indent), min_indent)
         ))
         .parse(arena, state)?;
@@ -216,7 +216,7 @@ fn expression<'a>(min_indent: u16) -> impl Parser<'a, Located<TypeAnnotation<'a>
         // TODO this space0 is dropped, so newlines just before the function arrow when there
         // is only one argument are not seen by the formatter. Can we do better?
         let (is_function, state) =
-            optional(skip_first!(space0(min_indent), string("->"))).parse(arena, state)?;
+            optional(skip_first!(space0(min_indent), ascii_string("->"))).parse(arena, state)?;
 
         if is_function.is_some() {
             let (return_type, state) =
@@ -263,30 +263,30 @@ fn expression<'a>(min_indent: u16) -> impl Parser<'a, Located<TypeAnnotation<'a>
 
 fn parse_concrete_type<'a>(
     arena: &'a Bump,
-    state: State<'a>,
+    mut state: State<'a>,
 ) -> ParseResult<'a, TypeAnnotation<'a>> {
-    let mut chars = state.input.chars();
     let mut part_buf = String::new_in(arena); // The current "part" (parts are dot-separated.)
     let mut parts: Vec<&'a str> = Vec::new_in(arena);
 
     // Qualified types must start with a capitalized letter.
-    match chars.next() {
+    match peek_utf8_char(&state) {
-        Some(ch) => {
+        Ok((first_letter, bytes_parsed)) => {
-            if ch.is_alphabetic() && ch.is_uppercase() {
+            if first_letter.is_alphabetic() && first_letter.is_uppercase() {
-                part_buf.push(ch);
+                part_buf.push(first_letter);
             } else {
-                return Err(unexpected(ch, 0, state, Attempting::ConcreteType));
+                return Err(unexpected(0, state, Attempting::ConcreteType));
+            }
+
+            state = state.advance_without_indenting(bytes_parsed)?;
+        }
+        Err(reason) => return state.fail(reason),
     }
-        }
-        None => {
-            return Err(unexpected_eof(0, Attempting::ConcreteType, state));
-        }
-    };
 
-    let mut chars_parsed = 1;
     let mut next_char = None;
 
-    while let Some(ch) = chars.next() {
+    while !state.bytes.is_empty() {
+        match peek_utf8_char(&state) {
+            Ok((ch, bytes_parsed)) => {
                 // After the first character, only these are allowed:
                 //
                 // * Unicode alphabetic chars - you might name a variable `鹏` if that's clear to your readers
@@ -295,21 +295,21 @@ fn parse_concrete_type<'a>(
                 if ch.is_alphabetic() {
                     if part_buf.is_empty() && !ch.is_uppercase() {
                         // Each part must begin with a capital letter.
-                return malformed(Some(ch), arena, state, &mut chars, parts);
+                        return malformed(Some(ch), arena, state, parts);
                     }
 
                     part_buf.push(ch);
                 } else if ch.is_ascii_digit() {
                     // Parts may not start with numbers!
                     if part_buf.is_empty() {
-                return malformed(Some(ch), arena, state, &mut chars, parts);
+                        return malformed(Some(ch), arena, state, parts);
                     }
 
                     part_buf.push(ch);
                 } else if ch == '.' {
                     // Having two consecutive dots is an error.
                     if part_buf.is_empty() {
-                return malformed(Some(ch), arena, state, &mut chars, parts);
+                        return malformed(Some(ch), arena, state, parts);
                     }
 
                     parts.push(part_buf.into_bump_str());
@@ -323,7 +323,10 @@ fn parse_concrete_type<'a>(
                     break;
                 }
 
-        chars_parsed += 1;
+                state = state.advance_without_indenting(bytes_parsed)?;
+            }
+            Err(reason) => return state.fail(reason),
+        }
     }
 
     if part_buf.is_empty() {
@@ -333,23 +336,16 @@ fn parse_concrete_type<'a>(
         //
         // If we made it this far and don't have a next_char, then necessarily
         // we have consumed a '.' char previously.
-        return malformed(
+        return malformed(next_char.or_else(|| Some('.')), arena, state, parts);
-            next_char.or_else(|| Some('.')),
-            arena,
-            state,
-            &mut chars,
-            parts,
-        );
     }
 
     if part_buf.is_empty() {
         // We had neither capitalized nor noncapitalized parts,
         // yet we made it this far. The only explanation is that this was
         // a stray '.' drifting through the cosmos.
-        return Err(unexpected('.', 1, state, Attempting::Identifier));
+        return Err(unexpected(1, state, Attempting::Identifier));
     }
 
-    let state = state.advance_without_indenting(chars_parsed)?;
     let answer = TypeAnnotation::Apply(
         join_module_parts(arena, parts.into_bump_slice()),
         part_buf.into_bump_str(),
@@ -361,28 +357,27 @@ fn parse_concrete_type<'a>(
 
 fn parse_type_variable<'a>(
     arena: &'a Bump,
-    state: State<'a>,
+    mut state: State<'a>,
 ) -> ParseResult<'a, TypeAnnotation<'a>> {
-    let mut chars = state.input.chars();
     let mut buf = String::new_in(arena);
 
+    match peek_utf8_char(&state) {
+        Ok((first_letter, bytes_parsed)) => {
             // Type variables must start with a lowercase letter.
-    match chars.next() {
+            if first_letter.is_alphabetic() && first_letter.is_lowercase() {
-        Some(ch) => {
+                buf.push(first_letter);
-            if ch.is_alphabetic() && ch.is_lowercase() {
-                buf.push(ch);
             } else {
-                return Err(unexpected(ch, 0, state, Attempting::TypeVariable));
+                return Err(unexpected(0, state, Attempting::TypeVariable));
             }
-        }
-        None => {
-            return Err(unexpected_eof(0, Attempting::TypeVariable, state));
-        }
-    };
 
-    let mut chars_parsed = 1;
+            state = state.advance_without_indenting(bytes_parsed)?;
+        }
+        Err(reason) => return state.fail(reason),
+    }
 
-    for ch in chars {
+    while !state.bytes.is_empty() {
+        match peek_utf8_char(&state) {
+            Ok((ch, bytes_parsed)) => {
                 // After the first character, only these are allowed:
                 //
                 // * Unicode alphabetic chars - you might name a variable `鹏` if that's clear to your readers
@@ -394,25 +389,23 @@ fn parse_type_variable<'a>(
                     break;
                 }
 
-        chars_parsed += 1;
+                state = state.advance_without_indenting(bytes_parsed)?;
+            }
+            Err(reason) => return state.fail(reason),
+        }
     }
 
-    let state = state.advance_without_indenting(chars_parsed)?;
     let answer = TypeAnnotation::BoundVariable(buf.into_bump_str());
 
     Ok((answer, state))
 }
 
-fn malformed<'a, I>(
+fn malformed<'a>(
     opt_bad_char: Option<char>,
     arena: &'a Bump,
-    state: State<'a>,
+    mut state: State<'a>,
-    chars: &mut I,
     parts: Vec<&'a str>,
-) -> ParseResult<'a, TypeAnnotation<'a>>
+) -> ParseResult<'a, TypeAnnotation<'a>> {
-where
-    I: Iterator<Item = char>,
-{
     // Reconstruct the original string that we've been parsing.
     let mut full_string = String::new_in(arena);
 
@@ -423,7 +416,9 @@ where
     }
 
     // Consume the remaining chars in the identifier.
-    for ch in chars {
+    while !state.bytes.is_empty() {
+        match peek_utf8_char(&state) {
+            Ok((ch, bytes_parsed)) => {
                 // We can't use ch.is_alphanumeric() here because that passes for
                 // things that are "numeric" but not ASCII digits, like `¾`
                 if ch == '.' || ch.is_alphabetic() || ch.is_ascii_digit() {
@@ -431,12 +426,15 @@ where
                 } else {
                     break;
                 }
-    }
 
-    let chars_parsed = full_string.len();
+                state = state.advance_without_indenting(bytes_parsed)?;
+            }
+            Err(reason) => return state.fail(reason),
+        }
+    }
 
     Ok((
         TypeAnnotation::Malformed(full_string.into_bump_str()),
-        state.advance_without_indenting(chars_parsed)?,
+        state,
     ))
 }

compiler/parse/tests/helpers/mod.rs

@@ -13,7 +13,7 @@ pub fn parse_with<'a>(arena: &'a Bump, input: &'a str) -> Result<ast::Expr<'a>,
 
 #[allow(dead_code)]
 pub fn parse_loc_with<'a>(arena: &'a Bump, input: &'a str) -> Result<Located<ast::Expr<'a>>, Fail> {
-    let state = State::new(&input, Attempting::Module);
+    let state = State::new(input.as_bytes(), Attempting::Module);
     let parser = space0_before(loc(roc_parse::expr::expr(0)), 0);
     let answer = parser.parse(&arena, state);
 

compiler/parse/tests/test_parse.rs

@@ -918,17 +918,12 @@ mod test_parse {
         let arena = Bump::new();
         let arg = arena.alloc(Located::new(0, 0, 5, 6, Num("1")));
         let args = bumpalo::vec![in &arena; &*arg];
-        let expected = Expr::Apply(
+        let expr = Var {
-            arena.alloc(Located::new(
-                0,
-                0,
-                0,
-                4,
-                Var {
             module_name: "",
             ident: "whee",
-                },
+        };
-            )),
+        let expected = Expr::Apply(
+            arena.alloc(Located::new(0, 0, 0, 4, expr)),
             args,
             CalledVia::Space,
         );
@@ -1040,16 +1035,11 @@ mod test_parse {
     fn unary_negation() {
         let arena = Bump::new();
         let loc_op = Located::new(0, 0, 0, 1, UnaryOp::Negate);
-        let loc_arg1_expr = Located::new(
+        let arg1_expr = Var {
-            0,
-            0,
-            1,
-            4,
-            Var {
             module_name: "",
             ident: "foo",
-            },
+        };
-        );
+        let loc_arg1_expr = Located::new(0, 0, 1, 4, arg1_expr);
         let expected = UnaryOp(arena.alloc(loc_arg1_expr), loc_op);
         let actual = parse_with(&arena, "-foo");
 
@@ -1060,16 +1050,11 @@ mod test_parse {
     fn unary_not() {
         let arena = Bump::new();
         let loc_op = Located::new(0, 0, 0, 1, UnaryOp::Not);
-        let loc_arg1_expr = Located::new(
+        let arg1_expr = Var {
-            0,
-            0,
-            1,
-            5,
-            Var {
             module_name: "",
             ident: "blah",
-            },
+        };
-        );
+        let loc_arg1_expr = Located::new(0, 0, 1, 5, arg1_expr);
         let expected = UnaryOp(arena.alloc(loc_arg1_expr), loc_op);
         let actual = parse_with(&arena, "!blah");
 
@@ -2092,7 +2077,7 @@ mod test_parse {
             "#
         );
         let actual = interface_header()
-            .parse(&arena, State::new(&src, Attempting::Module))
+            .parse(&arena, State::new(src.as_bytes(), Attempting::Module))
             .map(|tuple| tuple.0);
 
         assert_eq!(Ok(expected), actual);
@@ -2121,7 +2106,7 @@ mod test_parse {
             "#
         );
         let actual = interface_header()
-            .parse(&arena, State::new(&src, Attempting::Module))
+            .parse(&arena, State::new(src.as_bytes(), Attempting::Module))
             .map(|tuple| tuple.0);
 
         assert_eq!(Ok(expected), actual);
@@ -2174,7 +2159,7 @@ mod test_parse {
             "#
         );
         let actual = module_defs()
-            .parse(&arena, State::new(&src, Attempting::Module))
+            .parse(&arena, State::new(src.as_bytes(), Attempting::Module))
             .map(|tuple| tuple.0);
 
         assert_eq!(Ok(expected), actual);

compiler/reporting/tests/helpers/mod.rs

@@ -91,7 +91,7 @@ pub fn parse_with<'a>(arena: &'a Bump, input: &'a str) -> Result<ast::Expr<'a>,
 
 #[allow(dead_code)]
 pub fn parse_loc_with<'a>(arena: &'a Bump, input: &'a str) -> Result<Located<ast::Expr<'a>>, Fail> {
-    let state = State::new(&input, Attempting::Module);
+    let state = State::new(input.as_bytes(), Attempting::Module);
     let parser = space0_before(loc(roc_parse::expr::expr(0)), 0);
     let answer = parser.parse(&arena, state);
 

compiler/reporting/tests/test_reporting.rs

@@ -86,9 +86,6 @@ mod test_reporting {
             let mut procs = Procs::default();
             let mut ident_ids = interns.all_ident_ids.remove(&home).unwrap();
 
-            // assume 64-bit pointers
-            let pointer_size = std::mem::size_of::<u64>() as u32;
-
             // Populate Procs and Subs, and get the low-level Expr from the canonical Expr
             let mut mono_env = roc_mono::ir::Env {
                 arena: &arena,
@@ -96,7 +93,6 @@ mod test_reporting {
                 problems: &mut mono_problems,
                 home,
                 ident_ids: &mut ident_ids,
-                pointer_size,
                 jump_counter: arena.alloc(0),
             };
             let _mono_expr = Stmt::new(&mut mono_env, loc_expr.value, &mut procs);
@@ -3771,4 +3767,48 @@ mod test_reporting {
             ),
         )
     }
+
+    #[test]
+    fn first_wildcard_is_required() {
+        report_problem_as(
+            indoc!(
+                r#"
+                when Foo 1 2 3 is
+                    Foo _ 1 _ -> 1
+                    _ -> 2
+                "#
+            ),
+            "",
+        )
+    }
+
+    #[test]
+    fn second_wildcard_is_redundant() {
+        report_problem_as(
+            indoc!(
+                r#"
+                when Foo 1 2 3 is
+                    Foo _ 1 _ -> 1
+                    _ -> 2
+                    _ -> 3
+                "#
+            ),
+            indoc!(
+                r#"
+            -- REDUNDANT PATTERN -----------------------------------------------------------
+
+            The 3rd pattern is redundant:
+
+            1 ┆  when Foo 1 2 3 is
+            2 ┆      Foo _ 1 _ -> 1
+            3 ┆      _ -> 2
+            4 ┆      _ -> 3
+              ┆      ^
+
+            Any value of this shape will be handled by a previous pattern, so this
+            one should be removed.
+            "#
+            ),
+        )
+    }
 }

compiler/solve/tests/helpers/mod.rs

@@ -93,7 +93,7 @@ pub fn parse_with<'a>(arena: &'a Bump, input: &'a str) -> Result<ast::Expr<'a>,
 
 #[allow(dead_code)]
 pub fn parse_loc_with<'a>(arena: &'a Bump, input: &'a str) -> Result<Located<ast::Expr<'a>>, Fail> {
-    let state = State::new(&input, Attempting::Module);
+    let state = State::new(input.as_bytes(), Attempting::Module);
     let parser = space0_before(loc(roc_parse::expr::expr(0)), 0);
     let answer = parser.parse(&arena, state);
 

compiler/solve/tests/solve_expr.rs

@@ -2362,7 +2362,7 @@ mod solve_expr {
                             []
 
                         Ok next ->
-                            List.push (reconstructPath cameFrom next) goal
+                            List.append (reconstructPath cameFrom next) goal
 
                 reconstructPath
                 "#
@@ -2534,7 +2534,7 @@ mod solve_expr {
                     x = []
 
                     when List.get input 0 is
-                        Ok val -> List.push x val
+                        Ok val -> List.append x val
                         Err _ -> f input
                 f
                 "#

compiler/solve/tests/solve_uniq_expr.rs

@@ -2275,9 +2275,9 @@ mod solve_uniq_expr {
     }
 
     #[test]
-    fn list_push() {
+    fn list_append() {
         infer_eq(
-            "List.push",
+            "List.append",
             "Attr * (Attr * (List a), a -> Attr * (List a))",
         );
     }
@@ -2303,7 +2303,7 @@ mod solve_uniq_expr {
         infer_eq(
             indoc!(
                 r#"
-                    singleton = \x -> List.push [] x
+                    singleton = \x -> List.append [] x
 
                     singleton
                 "#
@@ -2317,7 +2317,7 @@ mod solve_uniq_expr {
         infer_eq(
             indoc!(
                 r#"
-                    reverse = \list -> List.foldr list (\e, l -> List.push l e) []
+                    reverse = \list -> List.foldr list (\e, l -> List.append l e) []
 
                     reverse
                 "#
@@ -2742,7 +2742,7 @@ mod solve_uniq_expr {
                             []
 
                         Ok next ->
-                            List.push (reconstructPath cameFrom next) goal
+                            List.append (reconstructPath cameFrom next) goal
 
                 reconstructPath
                 "#
@@ -2812,7 +2812,7 @@ mod solve_uniq_expr {
                             []
 
                         Ok next ->
-                            List.push (reconstructPath cameFrom next) goal
+                            List.append (reconstructPath cameFrom next) goal
 
                 updateCost : position, position, Model position -> Model position
                 updateCost = \current, neighbour, model ->
@@ -2897,7 +2897,7 @@ mod solve_uniq_expr {
                                 []
 
                             Ok next ->
-                                List.push (reconstructPath cameFrom next) goal
+                                List.append (reconstructPath cameFrom next) goal
 
 
                     updateCost : position, position, Model position -> Model position

compiler/uniq/tests/helpers/mod.rs

@@ -93,7 +93,7 @@ pub fn parse_with<'a>(arena: &'a Bump, input: &'a str) -> Result<ast::Expr<'a>,
 
 #[allow(dead_code)]
 pub fn parse_loc_with<'a>(arena: &'a Bump, input: &'a str) -> Result<Located<ast::Expr<'a>>, Fail> {
-    let state = State::new(&input, Attempting::Module);
+    let state = State::new(input.as_bytes(), Attempting::Module);
     let parser = space0_before(loc(roc_parse::expr::expr(0)), 0);
     let answer = parser.parse(&arena, state);
 

roc-for-elm-programmers.md

@@ -718,6 +718,64 @@ However, it cannot involve record field access. So this would *not* compile:
 { Foo.defaults.config & timeZone: utc }
 ```
 
+## Optional Record Fields
+
+There's a pattern in Elm where you pass a function a record of configuration
+values, some of which you don't really care about and want to leave as defaults.
+To incorporate the default config options, you call the function like so:
+
+```elm
+table { defaultConfig | height = 800, width = 600 }
+```
+
+This way, as the caller I'm specifying only the `height` and `width` fields,
+and leaving the others to whatever is inside `defaultConfig`. Perhaps it also
+has the fields `x` and `y`.
+
+In Roc, you can do this like so:
+
+```elm
+table { height = 800, width = 600 }
+```
+
+...and the `table` function will fill in its default values for `x` and `y`.
+There is no need to use a `defaultConfig` record.
+
+Here's how `table` would be defined in Roc:
+
+```
+table = \{ height, width, x ? 0.0, y ? 0.0 } ->
+```
+
+This is using *optional field destructuring* to destructure a record while
+also providing default values for any fields that might be missing.
+Here's the type of `table`:
+
+```
+table : { height : Float, width : Float, x ? Float, y ? Float } -> Table
+table = \{ height, width, x ? 0.0, y ? 0.0 } ->
+```
+
+This says that `table` takes a record with two *required* fields (`height` and
+`width` and two *optional* fields (`x` and `y`). It also says that all of those
+fields have the type `Float` This means you can choose to omit `x`, `y`, or both,
+when calling the function...but if you provide them, they must be numbers.
+
+This is also the type that would have been inferred for `table` if no annotation
+had been written. Roc's compiler can tell from the destructuring syntax
+`x ? 0.0` that `x` is an optional field, and that it has the type `Float`. These
+default values can reference other expressions in the record destructure; if you
+wanted, you could write `{ height, width, x ? 0.0, y ? x + 1 }`.
+
+Destructuring is the only way to implement a record with optional fields.
+(For example, if you write the expression `config.x` and `x` is an optional field,
+you'll get a compile error.)
+
+This means it's never possible to end up with an "optional value" that exists
+outside a record field. Optionality is a concept that exists only in record fields,
+and it's intended for the use case of config records like this. The ergonomics
+of destructuring mean this wouldn't be a good fit for data modeling.
+
 ## Standard Data Structures
 
 Elm has `List`, `Array`, `Set`, and `Dict` in the standard library.