Merge remote-tracking branch 'origin/trunk' into update_zig_09

cli/src/build.rs

@@ -350,6 +350,7 @@ pub fn check_file(
     src_dir: PathBuf,
     roc_file_path: PathBuf,
     emit_timings: bool,
+    threading: Threading,
 ) -> Result<(program::Problems, Duration), LoadingProblem> {
     let compilation_start = SystemTime::now();
 
@@ -368,7 +369,7 @@ pub fn check_file(
         target_info,
         // TODO: expose this from CLI?
         RenderTarget::ColorTerminal,
-        Threading::Multi,
+        threading,
     )?;
 
     let buf = &mut String::with_capacity(1024);

cli/src/lib.rs

@@ -34,6 +34,7 @@ pub const CMD_FORMAT: &str = "format";
 pub const FLAG_DEBUG: &str = "debug";
 pub const FLAG_DEV: &str = "dev";
 pub const FLAG_OPTIMIZE: &str = "optimize";
+pub const FLAG_MAX_THREADS: &str = "max-threads";
 pub const FLAG_OPT_SIZE: &str = "opt-size";
 pub const FLAG_LIB: &str = "lib";
 pub const FLAG_NO_LINK: &str = "no-link";
@@ -57,6 +58,14 @@ pub fn build_app<'a>() -> Command<'a> {
         .requires(ROC_FILE)
         .required(false);
 
+    let flag_max_threads = Arg::new(FLAG_MAX_THREADS)
+        .long(FLAG_MAX_THREADS)
+        .help("Limit the number of threads (and hence cores) used during compilation.")
+        .requires(ROC_FILE)
+        .takes_value(true)
+        .validator(|s| s.parse::<usize>())
+        .required(false);
+
     let flag_opt_size = Arg::new(FLAG_OPT_SIZE)
         .long(FLAG_OPT_SIZE)
         .help("Optimize the compiled program to have a small binary size. (Optimization takes time to complete.)")
@@ -111,6 +120,7 @@ pub fn build_app<'a>() -> Command<'a> {
         .subcommand(Command::new(CMD_BUILD)
             .about("Build a binary from the given .roc file, but don't run it")
             .arg(flag_optimize.clone())
+            .arg(flag_max_threads.clone())
             .arg(flag_opt_size.clone())
             .arg(flag_dev.clone())
             .arg(flag_debug.clone())
@@ -151,6 +161,7 @@ pub fn build_app<'a>() -> Command<'a> {
         .subcommand(Command::new(CMD_RUN)
             .about("Run a .roc file even if it has build errors")
             .arg(flag_optimize.clone())
+            .arg(flag_max_threads.clone())
             .arg(flag_opt_size.clone())
             .arg(flag_dev.clone())
             .arg(flag_debug.clone())
@@ -181,6 +192,7 @@ pub fn build_app<'a>() -> Command<'a> {
         .subcommand(Command::new(CMD_CHECK)
             .about("Check the code for problems, but doesn’t build or run it")
             .arg(flag_time.clone())
+            .arg(flag_max_threads.clone())
             .arg(
                 Arg::new(ROC_FILE)
                     .help("The .roc file of an app to check")
@@ -201,6 +213,7 @@ pub fn build_app<'a>() -> Command<'a> {
         )
         .trailing_var_arg(true)
         .arg(flag_optimize)
+            .arg(flag_max_threads.clone())
         .arg(flag_opt_size)
         .arg(flag_dev)
         .arg(flag_debug)
@@ -271,6 +284,16 @@ pub fn build(
     let emit_debug_info = matches.is_present(FLAG_DEBUG);
     let emit_timings = matches.is_present(FLAG_TIME);
 
+    let threading = match matches
+        .value_of(FLAG_MAX_THREADS)
+        .and_then(|s| s.parse::<usize>().ok())
+    {
+        None => Threading::AllAvailable,
+        Some(0) => user_error!("cannot build with at most 0 threads"),
+        Some(1) => Threading::Single,
+        Some(n) => Threading::AtMost(n),
+    };
+
     // Use surgical linking when supported, or when explicitly requested with --linker surgical
     let surgically_link = if matches.is_present(FLAG_LINKER) {
         matches.value_of(FLAG_LINKER) == Some("surgical")
@@ -320,7 +343,7 @@ pub fn build(
         surgically_link,
         precompiled,
         target_valgrind,
-        Threading::Multi,
+        threading,
     );
 
     match res_binary_path {

cli/src/main.rs

@@ -6,7 +6,7 @@ use roc_cli::{
     FLAG_NO_LINK, FLAG_TARGET, FLAG_TIME, ROC_FILE,
 };
 use roc_error_macros::user_error;
-use roc_load::LoadingProblem;
+use roc_load::{LoadingProblem, Threading};
 use std::fs::{self, FileType};
 use std::io;
 use std::path::{Path, PathBuf};
@@ -82,7 +82,17 @@ fn main() -> io::Result<()> {
             let roc_file_path = PathBuf::from(filename);
             let src_dir = roc_file_path.parent().unwrap().to_owned();
 
-            match check_file(&arena, src_dir, roc_file_path, emit_timings) {
+            let threading = match matches
+                .value_of(roc_cli::FLAG_MAX_THREADS)
+                .and_then(|s| s.parse::<usize>().ok())
+            {
+                None => Threading::AllAvailable,
+                Some(0) => user_error!("cannot build with at most 0 threads"),
+                Some(1) => Threading::Single,
+                Some(n) => Threading::AtMost(n),
+            };
+
+            match check_file(&arena, src_dir, roc_file_path, emit_timings, threading) {
                 Ok((problems, total_time)) => {
                     println!(
                         "\x1B[{}m{}\x1B[39m {} and \x1B[{}m{}\x1B[39m {} found in {} ms.",

compiler/load/build.rs

@@ -38,7 +38,7 @@ fn write_subs_for_module(module_id: ModuleId, filename: &str) {
         Default::default(),
         target_info,
         roc_reporting::report::RenderTarget::ColorTerminal,
-        Threading::Multi,
+        Threading::AllAvailable,
     );
 
     let module = res_module.unwrap();

compiler/load_internal/Cargo.toml

@@ -26,7 +26,6 @@ ven_pretty = { path = "../../vendor/pretty" }
 bumpalo = { version = "3.8.0", features = ["collections"] }
 parking_lot = "0.12"
 crossbeam = "0.8.1"
-num_cpus = "1.13.0"
 
 [dev-dependencies]
 pretty_assertions = "1.0.0"

compiler/load_internal/src/file.rs

@@ -747,6 +747,7 @@ impl<'a> State<'a> {
         ident_ids_by_module: SharedIdentIdsByModule,
         cached_subs: MutMap<ModuleId, (Subs, Vec<(Symbol, Variable)>)>,
         render: RenderTarget,
+        number_of_workers: usize,
     ) -> Self {
         let arc_shorthands = Arc::new(Mutex::new(MutMap::default()));
 
@@ -770,7 +771,7 @@ impl<'a> State<'a> {
             declarations_by_id: MutMap::default(),
             exposed_symbols_by_module: MutMap::default(),
             timings: MutMap::default(),
-            layout_caches: std::vec::Vec::with_capacity(num_cpus::get()),
+            layout_caches: std::vec::Vec::with_capacity(number_of_workers),
             cached_subs: Arc::new(Mutex::new(cached_subs)),
             render,
         }
@@ -1099,7 +1100,8 @@ pub enum LoadResult<'a> {
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub enum Threading {
     Single,
-    Multi,
+    AllAvailable,
+    AtMost(usize),
 }
 
 /// The loading process works like this, starting from the given filename (e.g. "main.roc"):
@@ -1157,21 +1159,32 @@ pub fn load<'a>(
     render: RenderTarget,
     threading: Threading,
 ) -> Result<LoadResult<'a>, LoadingProblem<'a>> {
+    enum Threads {
+        Single,
+        Many(usize),
+    }
+
+    let threads = {
+        if cfg!(target_family = "wasm") {
             // When compiling to wasm, we cannot spawn extra threads
             // so we have a single-threaded implementation
-    if threading == Threading::Single || cfg!(target_family = "wasm") {
+            Threads::Single
-        load_single_threaded(
-            arena,
-            load_start,
-            src_dir,
-            exposed_types,
-            goal_phase,
-            target_info,
-            cached_subs,
-            render,
-        )
         } else {
-        load_multi_threaded(
+            match std::thread::available_parallelism().map(|v| v.get()) {
+                Err(_) => Threads::Single,
+                Ok(0) => unreachable!("NonZeroUsize"),
+                Ok(1) => Threads::Single,
+                Ok(reported) => match threading {
+                    Threading::Single => Threads::Single,
+                    Threading::AllAvailable => Threads::Many(reported),
+                    Threading::AtMost(at_most) => Threads::Many(Ord::min(reported, at_most)),
+                },
+            }
+        }
+    };
+
+    match threads {
+        Threads::Single => load_single_threaded(
             arena,
             load_start,
             src_dir,
@@ -1180,7 +1193,18 @@ pub fn load<'a>(
             target_info,
             cached_subs,
             render,
-        )
+        ),
+        Threads::Many(threads) => load_multi_threaded(
+            arena,
+            load_start,
+            src_dir,
+            exposed_types,
+            goal_phase,
+            target_info,
+            cached_subs,
+            render,
+            threads,
+        ),
     }
 }
 
@@ -1210,6 +1234,7 @@ pub fn load_single_threaded<'a>(
         .send(root_msg)
         .map_err(|_| LoadingProblem::MsgChannelDied)?;
 
+    let number_of_workers = 1;
     let mut state = State::new(
         root_id,
         target_info,
@@ -1219,6 +1244,7 @@ pub fn load_single_threaded<'a>(
         ident_ids_by_module,
         cached_subs,
         render,
+        number_of_workers,
     );
 
     // We'll add tasks to this, and then worker threads will take tasks from it.
@@ -1390,6 +1416,7 @@ fn load_multi_threaded<'a>(
     target_info: TargetInfo,
     cached_subs: MutMap<ModuleId, (Subs, Vec<(Symbol, Variable)>)>,
     render: RenderTarget,
+    available_threads: usize,
 ) -> Result<LoadResult<'a>, LoadingProblem<'a>> {
     let LoadStart {
         arc_modules,
@@ -1399,6 +1426,28 @@ fn load_multi_threaded<'a>(
         ..
     } = load_start;
 
+    let (msg_tx, msg_rx) = bounded(1024);
+    msg_tx
+        .send(root_msg)
+        .map_err(|_| LoadingProblem::MsgChannelDied)?;
+
+    // Reserve one CPU for the main thread, and let all the others be eligible
+    // to spawn workers.
+    let available_workers = available_threads - 1;
+
+    let num_workers = match env::var("ROC_NUM_WORKERS") {
+        Ok(env_str) => env_str
+            .parse::<usize>()
+            .unwrap_or(available_workers)
+            .min(available_workers),
+        Err(_) => available_workers,
+    };
+
+    assert!(
+        num_workers >= 1,
+        "`load_multi_threaded` needs at least one worker"
+    );
+
     let mut state = State::new(
         root_id,
         target_info,
@@ -1408,28 +1457,9 @@ fn load_multi_threaded<'a>(
         ident_ids_by_module,
         cached_subs,
         render,
+        num_workers,
     );
 
-    let (msg_tx, msg_rx) = bounded(1024);
-    msg_tx
-        .send(root_msg)
-        .map_err(|_| LoadingProblem::MsgChannelDied)?;
-
-    // Reserve one CPU for the main thread, and let all the others be eligible
-    // to spawn workers. We use .max(2) to enforce that we always
-    // end up with at least 1 worker - since (.max(2) - 1) will
-    // always return a number that's at least 1. Using
-    // .max(2) on the initial number of CPUs instead of
-    // doing .max(1) on the entire expression guards against
-    // num_cpus returning 0, while also avoiding wrapping
-    // unsigned subtraction overflow.
-    let default_num_workers = num_cpus::get().max(2) - 1;
-
-    let num_workers = match env::var("ROC_NUM_WORKERS") {
-        Ok(env_str) => env_str.parse::<usize>().unwrap_or(default_num_workers),
-        Err(_) => default_num_workers,
-    };
-
     // an arena for every worker, stored in an arena-allocated bumpalo vec to make the lifetimes work
     let arenas = std::iter::repeat_with(Bump::new).take(num_workers);
     let worker_arenas = arena.alloc(bumpalo::collections::Vec::from_iter_in(arenas, arena));

compiler/mono/src/ir.rs

@@ -15,6 +15,7 @@ use roc_debug_flags::{
     dbg_do, ROC_PRINT_IR_AFTER_REFCOUNT, ROC_PRINT_IR_AFTER_RESET_REUSE,
     ROC_PRINT_IR_AFTER_SPECIALIZATION,
 };
+use roc_error_macros::internal_error;
 use roc_exhaustive::{Ctor, CtorName, Guard, RenderAs, TagId};
 use roc_module::ident::{ForeignSymbol, Lowercase, TagName};
 use roc_module::low_level::LowLevel;
@@ -5866,27 +5867,10 @@ pub fn from_can<'a>(
                     Err(_) => todo!(),
                 };
 
-            if let Pattern::Identifier(symbol) = mono_pattern {
+            if let Pattern::Identifier(_symbol) = mono_pattern {
-                let mut hole =
+                internal_error!("Identifier patterns should be handled in a higher code pass!")
-                    env.arena
-                        .alloc(from_can(env, variable, cont.value, procs, layout_cache));
-
-                for (symbol, variable, expr) in assignments {
-                    let stmt = with_hole(env, expr, variable, procs, layout_cache, symbol, hole);
-
-                    hole = env.arena.alloc(stmt);
             }
 
-                with_hole(
-                    env,
-                    def.loc_expr.value,
-                    def.expr_var,
-                    procs,
-                    layout_cache,
-                    symbol,
-                    hole,
-                )
-            } else {
             // convert the continuation
             let mut stmt = from_can(env, variable, cont.value, procs, layout_cache);
 
@@ -5915,8 +5899,7 @@ pub fn from_can<'a>(
                 store_pattern(env, procs, layout_cache, &mono_pattern, outer_symbol, stmt)
             } else {
                 let outer_symbol = env.unique_symbol();
-                    stmt =
+                stmt = store_pattern(env, procs, layout_cache, &mono_pattern, outer_symbol, stmt);
-                        store_pattern(env, procs, layout_cache, &mono_pattern, outer_symbol, stmt);
 
                 // convert the def body, store in outer_symbol
                 with_hole(
@@ -5930,7 +5913,6 @@ pub fn from_can<'a>(
                 )
             }
         }
-        }
 
         _ => {
             let symbol = env.unique_symbol();

compiler/solve/src/solve.rs

@@ -476,10 +476,13 @@ impl Pools {
         self.0.iter()
     }
 
-    pub fn split_last(&self) -> (&Vec<Variable>, &[Vec<Variable>]) {
+    pub fn split_last(mut self) -> (Vec<Variable>, Vec<Vec<Variable>>) {
-        self.0
+        let last = self
-            .split_last()
+            .0
-            .unwrap_or_else(|| panic!("Attempted to split_last() on non-empty Pools"))
+            .pop()
+            .unwrap_or_else(|| panic!("Attempted to split_last() on non-empty Pools"));
+
+        (last, self.0)
     }
 
     pub fn extend_to(&mut self, n: usize) {
@@ -737,8 +740,7 @@ fn solve(
 
                 // pop pool
                 generalize(subs, young_mark, visit_mark, next_rank, pools);
-
+                debug_assert!(pools.get(next_rank).is_empty());
-                pools.get_mut(next_rank).clear();
 
                 // check that things went well
                 dbg_do!(ROC_VERIFY_RIGID_LET_GENERALIZED, {
@@ -2426,7 +2428,7 @@ fn generalize(
     young_rank: Rank,
     pools: &mut Pools,
 ) {
-    let young_vars = pools.get(young_rank);
+    let young_vars = std::mem::take(pools.get_mut(young_rank));
     let rank_table = pool_to_rank_table(subs, young_mark, young_rank, young_vars);
 
     // Get the ranks right for each entry.
@@ -2437,12 +2439,12 @@ fn generalize(
         }
     }
 
-    let (last_pool, all_but_last_pool) = rank_table.split_last();
+    let (mut last_pool, all_but_last_pool) = rank_table.split_last();
 
     // For variables that have rank lowerer than young_rank, register them in
     // the appropriate old pool if they are not redundant.
     for vars in all_but_last_pool {
-        for &var in vars {
+        for var in vars {
             if !subs.redundant(var) {
                 let rank = subs.get_rank(var);
 
@@ -2453,7 +2455,7 @@ fn generalize(
 
     // For variables with rank young_rank, if rank < young_rank: register in old pool,
     // otherwise generalize
-    for &var in last_pool {
+    for var in last_pool.drain(..) {
         if !subs.redundant(var) {
             let desc_rank = subs.get_rank(var);
 
@@ -2464,32 +2466,38 @@ fn generalize(
             }
         }
     }
+
+    // re-use the last_vector (which likely has a good capacity for future runs
+    *pools.get_mut(young_rank) = last_pool;
 }
 
 /// Sort the variables into buckets by rank.
+#[inline]
 fn pool_to_rank_table(
     subs: &mut Subs,
     young_mark: Mark,
     young_rank: Rank,
-    young_vars: &[Variable],
+    mut young_vars: Vec<Variable>,
 ) -> Pools {
     let mut pools = Pools::new(young_rank.into_usize() + 1);
 
     // the vast majority of young variables have young_rank
-    // using `retain` here prevents many `pools.get_mut(young_rank)` lookups
+    let mut i = 0;
-    let mut young_vars = young_vars.to_vec();
+    while i < young_vars.len() {
-    young_vars.retain(|var| {
+        let var = young_vars[i];
-        let rank = subs.get_rank_set_mark(*var, young_mark);
+        let rank = subs.get_rank_set_mark(var, young_mark);
 
         if rank != young_rank {
             debug_assert!(rank.into_usize() < young_rank.into_usize() + 1);
 
-            pools.get_mut(rank).push(*var);
+            pools.get_mut(rank).push(var);
-            false
+
+            // swap an element in; don't increment i
+            young_vars.swap_remove(i);
         } else {
-            true
+            i += 1;
+        }
     }
-    });
 
     std::mem::swap(pools.get_mut(young_rank), &mut young_vars);
 

compiler/test_gen/src/helpers/llvm.rs

@@ -60,7 +60,7 @@ fn create_llvm_module<'a>(
         Default::default(),
         target_info,
         RenderTarget::ColorTerminal,
-        Threading::Multi,
+        Threading::AllAvailable,
     );
 
     let mut loaded = match loaded {

docs/src/lib.rs

@@ -435,7 +435,7 @@ pub fn load_modules_for_files(filenames: Vec<PathBuf>) -> Vec<LoadedModule> {
             Default::default(),
             roc_target::TargetInfo::default_x86_64(), // This is just type-checking for docs, so "target" doesn't matter
             roc_reporting::report::RenderTarget::ColorTerminal,
-            Threading::Multi,
+            Threading::AllAvailable,
         ) {
             Ok(loaded) => modules.push(loaded),
             Err(LoadingProblem::FormattedReport(report)) => {

editor/src/editor/main.rs

@@ -129,7 +129,7 @@ fn run_event_loop(project_dir_path_opt: Option<&Path>) -> Result<(), Box<dyn Err
 
     let file_path = Path::new(&file_path_str);
 
-    let loaded_module = load_module(file_path, Threading::Multi);
+    let loaded_module = load_module(file_path, Threading::AllAvailable);
 
     let mut var_store = VarStore::default();
     let dep_idents = IdentIds::exposed_builtins(8);

editor/src/editor/mvc/ed_model.rs

@@ -330,7 +330,7 @@ pub mod test_ed_model {
         writeln!(file, "{}", clean_code_str)
             .unwrap_or_else(|_| panic!("Failed to write {:?} to file: {:?}", clean_code_str, file));
 
-        let loaded_module = load_module(&temp_file_full_path, Threading::Multi);
+        let loaded_module = load_module(&temp_file_full_path, Threading::AllAvailable);
 
         let mut ed_model = init_dummy_model(
             clean_code_str,