Merge remote-tracking branch 'origin/trunk' into improve-dependency-solving

This commit is contained in:
Folkert 2020-10-12 16:10:25 +02:00
commit 27cdcffec0
5 changed files with 197 additions and 47 deletions

156
Cargo.lock generated
View file

@ -16,6 +16,21 @@ version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2692800d602527d2b8fea50036119c37df74ab565b10e285706a3dcec0ec3e16"
[[package]]
name = "addr2line"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b6a2d3371669ab3ca9797670853d61402b03d0b4b9ebf33d677dfa720203072"
dependencies = [
"gimli",
]
[[package]]
name = "adler"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
[[package]]
name = "aho-corasick"
version = "0.7.13"
@ -120,6 +135,20 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "backtrace"
version = "0.3.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f813291114c186a042350e787af10c26534601062603d888be110f59f85ef8fa"
dependencies = [
"addr2line",
"cfg-if",
"libc",
"miniz_oxide",
"object",
"rustc-demangle",
]
[[package]]
name = "bitflags"
version = "1.2.1"
@ -279,6 +308,15 @@ dependencies = [
"bitflags",
]
[[package]]
name = "cloudabi"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467"
dependencies = [
"bitflags",
]
[[package]]
name = "cocoa"
version = "0.20.2"
@ -634,6 +672,12 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
[[package]]
name = "fixedbitset"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d"
[[package]]
name = "fnv"
version = "1.0.7"
@ -829,7 +873,7 @@ dependencies = [
"gfx-hal",
"libloading",
"log",
"parking_lot",
"parking_lot 0.10.2",
"range-alloc",
"raw-window-handle",
"smallvec",
@ -885,7 +929,7 @@ dependencies = [
"log",
"metal",
"objc",
"parking_lot",
"parking_lot 0.10.2",
"range-alloc",
"raw-window-handle",
"smallvec",
@ -947,6 +991,12 @@ dependencies = [
"slab",
]
[[package]]
name = "gimli"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724"
[[package]]
name = "glyph_brush"
version = "0.7.0"
@ -1119,7 +1169,7 @@ dependencies = [
"libc",
"llvm-sys",
"once_cell",
"parking_lot",
"parking_lot 0.10.2",
"regex",
]
@ -1259,6 +1309,15 @@ dependencies = [
"scopeguard",
]
[[package]]
name = "lock_api"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c"
dependencies = [
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.11"
@ -1328,6 +1387,16 @@ dependencies = [
"objc",
]
[[package]]
name = "miniz_oxide"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d"
dependencies = [
"adler",
"autocfg 1.0.1",
]
[[package]]
name = "mio"
version = "0.6.22"
@ -1519,6 +1588,12 @@ dependencies = [
"cc",
]
[[package]]
name = "object"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5"
[[package]]
name = "once_cell"
version = "1.4.1"
@ -1561,8 +1636,19 @@ version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e"
dependencies = [
"lock_api",
"parking_lot_core",
"lock_api 0.3.4",
"parking_lot_core 0.7.2",
]
[[package]]
name = "parking_lot"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4893845fa2ca272e647da5d0e46660a314ead9c2fdd9a883aabc32e481a8733"
dependencies = [
"instant",
"lock_api 0.4.1",
"parking_lot_core 0.8.0",
]
[[package]]
@ -1572,13 +1658,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3"
dependencies = [
"cfg-if",
"cloudabi",
"cloudabi 0.0.3",
"libc",
"redox_syscall",
"smallvec",
"winapi 0.3.9",
]
[[package]]
name = "parking_lot_core"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b"
dependencies = [
"backtrace",
"cfg-if",
"cloudabi 0.1.0",
"instant",
"libc",
"petgraph",
"redox_syscall",
"smallvec",
"thread-id",
"winapi 0.3.9",
]
[[package]]
name = "peek-poke"
version = "0.2.0"
@ -1650,6 +1754,16 @@ dependencies = [
"sha-1",
]
[[package]]
name = "petgraph"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7"
dependencies = [
"fixedbitset",
"indexmap",
]
[[package]]
name = "pin-project"
version = "0.4.23"
@ -1965,7 +2079,7 @@ version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071"
dependencies = [
"cloudabi",
"cloudabi 0.0.3",
"fuchsia-cprng",
"libc",
"rand_core 0.4.2",
@ -2343,6 +2457,7 @@ dependencies = [
"inlinable_string",
"maplit",
"num_cpus",
"parking_lot 0.11.0",
"pretty_assertions",
"quickcheck",
"quickcheck_macros",
@ -2549,6 +2664,12 @@ dependencies = [
"roc_types",
]
[[package]]
name = "rustc-demangle"
version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2610b7f643d18c87dff3b489950269617e6601a51f1f05aa5daefee36f64f0b"
[[package]]
name = "rustc-hash"
version = "1.1.0"
@ -2756,7 +2877,7 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd0a4829a5c591dc24a944a736d6b1e4053e51339a79fd5d4702c4c999a9c45e"
dependencies = [
"lock_api",
"lock_api 0.3.4",
]
[[package]]
@ -2857,6 +2978,17 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "thread-id"
version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7fbf4c9d56b320106cd64fd024dadfa0be7cb4706725fc44a7d7ce952d820c1"
dependencies = [
"libc",
"redox_syscall",
"winapi 0.3.9",
]
[[package]]
name = "thread_local"
version = "1.0.1"
@ -3184,7 +3316,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5dece29f3cd403aabf4056595eabe4b9af56b8bfae12445f097cf8666a41829"
dependencies = [
"arrayvec",
"parking_lot",
"parking_lot 0.10.2",
"raw-window-handle",
"smallvec",
"wgpu-core",
@ -3211,7 +3343,7 @@ dependencies = [
"gfx-hal",
"gfx-memory",
"log",
"parking_lot",
"parking_lot 0.10.2",
"peek-poke",
"smallvec",
"vec_map",
@ -3228,7 +3360,7 @@ dependencies = [
"lazy_static",
"libc",
"objc",
"parking_lot",
"parking_lot 0.10.2",
"raw-window-handle",
"wgpu-core",
"wgpu-types",
@ -3321,7 +3453,7 @@ dependencies = [
"ndk-glue",
"ndk-sys",
"objc",
"parking_lot",
"parking_lot 0.10.2",
"percent-encoding",
"raw-window-handle",
"smithay-client-toolkit",

View file

@ -20,6 +20,7 @@ roc_solve = { path = "../solve" }
roc_mono = { path = "../mono" }
bumpalo = { version = "3.2", features = ["collections"] }
inlinable_string = "0.1"
parking_lot = { version = "0.11", features = ["deadlock_detection"] }
crossbeam = "0.7"
num_cpus = "1"

View file

@ -5,6 +5,7 @@ use crossbeam::channel::{bounded, Sender};
use crossbeam::deque::{Injector, Stealer, Worker};
use crossbeam::thread;
use inlinable_string::InlinableString;
use parking_lot::Mutex;
use roc_builtins::std::{Mode, StdLib};
use roc_can::constraint::Constraint;
use roc_can::def::Declaration;
@ -32,7 +33,7 @@ use std::io;
use std::iter;
use std::path::{Path, PathBuf};
use std::str::from_utf8_unchecked;
use std::sync::{Arc, Mutex};
use std::sync::Arc;
use std::time::{Duration, SystemTime};
/// Filename extension for normal Roc modules
@ -878,8 +879,7 @@ fn finish<'a>(
let module_ids = Arc::try_unwrap(state.arc_modules)
.unwrap_or_else(|_| panic!("There were still outstanding Arc references to module_ids"))
.into_inner()
.expect("Unwrapping mutex for module_ids");
.into_inner();
let interns = Interns {
module_ids,
@ -1079,10 +1079,8 @@ fn send_header<'a>(
let ident_ids = {
// Lock just long enough to perform the minimal operations necessary.
let mut module_ids = (*module_ids).lock().expect("Failed to acquire lock for interning module IDs, presumably because a thread panicked.");
let mut ident_ids_by_module = (*ident_ids_by_module).lock().expect(
"Failed to acquire lock for interning ident IDs, presumably because a thread panicked.",
);
let mut module_ids = (*module_ids).lock();
let mut ident_ids_by_module = (*ident_ids_by_module).lock();
home = module_ids.get_or_insert(&declared_name.as_inline_str());
@ -1243,9 +1241,7 @@ impl<'a> BuildTask<'a> {
let mut dep_idents: IdentIdsByModule = IdentIds::exposed_builtins(num_deps);
{
let ident_ids_by_module = (*ident_ids_by_module).lock().expect(
"Failed to acquire lock for interning ident IDs, presumably because a thread panicked.",
);
let ident_ids_by_module = (*ident_ids_by_module).lock();
// Populate dep_idents with each of their IdentIds,
// which we'll need during canonicalization to translate
@ -1277,9 +1273,11 @@ impl<'a> BuildTask<'a> {
waiting_for_solve.insert(module_id, solve_needed);
let module_ids = {
(*module_ids).lock().expect("Failed to acquire lock for obtaining module IDs, presumably because a thread panicked.").clone()
};
// Clone the module_ids we'll need for canonicalization.
// This should be small, and cloning it should be quick.
// We release the lock as soon as we're done cloning, so we don't have
// to lock the global module_ids while canonicalizing any given module.
let module_ids = { (*module_ids).lock().clone() };
// Now that we have waiting_for_solve populated, continue parsing,
// canonicalizing, and constraining the module.

View file

@ -2,6 +2,7 @@ use bumpalo::Bump;
use crossbeam::channel::{bounded, Sender};
use crossbeam::deque::{Injector, Stealer, Worker};
use crossbeam::thread;
use parking_lot::Mutex;
use roc_builtins::std::{Mode, StdLib};
use roc_can::constraint::Constraint;
use roc_can::def::Declaration;
@ -33,7 +34,7 @@ use std::io;
use std::iter;
use std::path::{Path, PathBuf};
use std::str::from_utf8_unchecked;
use std::sync::{Arc, Mutex};
use std::sync::Arc;
use std::time::{Duration, SystemTime};
/// Filename extension for normal Roc modules
@ -42,6 +43,13 @@ const ROC_FILE_EXTENSION: &str = "roc";
/// The . in between module names like Foo.Bar.Baz
const MODULE_SEPARATOR: char = '.';
const SHOW_MESSAGE_LOG: bool = true;
macro_rules! log {
() => (if SHOW_MESSAGE_LOG { println!()} else {});
($($arg:tt)*) => (if SHOW_MESSAGE_LOG { println!($($arg)*); } else {})
}
/// NOTE the order of definition of the phases is used by the ord instance
/// make sure they are ordered from first to last!
#[derive(PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy, Debug)]
@ -220,8 +228,7 @@ fn start_phase<'a>(module_id: ModuleId, phase: Phase, state: &mut State<'a>) ->
} = &state;
{
let msg = "Failed to acquire lock for interning ident IDs, presumably because a thread panicked.";
let ident_ids_by_module = (*ident_ids_by_module).lock().expect(msg);
let ident_ids_by_module = (*ident_ids_by_module).lock();
// Populate dep_idents with each of their IdentIds,
// which we'll need during canonicalization to translate
@ -240,10 +247,17 @@ fn start_phase<'a>(module_id: ModuleId, phase: Phase, state: &mut State<'a>) ->
}
}
// Clone the module_ids we'll need for canonicalization.
// This should be small, and cloning it should be quick.
// We release the lock as soon as we're done cloning, so we don't have
// to lock the global module_ids while canonicalizing any given module.
let module_ids = Arc::clone(&state.arc_modules);
let module_ids = {
(*module_ids).lock().expect("Failed to acquire lock for obtaining module IDs, presumably because a thread panicked.").clone()
};
let module_ids = { (*module_ids).lock().clone() };
debug_assert!(header
.imported_modules
.iter()
.all(|id| module_ids.get_name(*id).is_some()));
let exposed_symbols = state
.exposed_symbols_by_module
@ -810,7 +824,7 @@ where
{
// Reserve one CPU for the main thread, and let all the others be eligible
// to spawn workers.
let num_workers = num_cpus::get() - 1;
let num_workers = 1; // num_cpus::get() - 1;
let worker_arenas = arena.alloc(bumpalo::collections::Vec::with_capacity_in(
num_workers,
arena,
@ -1064,6 +1078,7 @@ fn update<'a>(
match msg {
Header(header) => {
log!("loaded header for {:?}", header.module_id);
let home = header.module_id;
// store an ID to name mapping, so we know the file to read when fetching dependencies' headers
@ -1112,6 +1127,7 @@ fn update<'a>(
var_store,
module_timing,
} => {
log!("generated constraints for {:?}", module.module_id);
let module_id = module.module_id;
state.can_problems.extend(problems);
@ -1151,6 +1167,7 @@ fn update<'a>(
decls,
mut module_timing,
} => {
log!("solved types for {:?}", module_id);
module_timing.end_time = SystemTime::now();
let work = state.dependencies.notify(module_id, Phase::SolveTypes);
@ -1239,6 +1256,7 @@ fn update<'a>(
layout_cache,
problems: _,
} => {
log!("found specializations for {:?}", module_id);
let subs = solved_subs.into_inner();
if let Some(pending) = &procs.pending_specializations {
@ -1288,6 +1306,7 @@ fn update<'a>(
external_specializations_requested,
..
} => {
log!("made specializations for {:?}", module_id);
for (module_id, requested) in external_specializations_requested {
let existing = match state
.module_cache
@ -1360,8 +1379,7 @@ fn finish_specialization<'a>(
let module_ids = Arc::try_unwrap(state.arc_modules)
.unwrap_or_else(|_| panic!("There were still outstanding Arc references to module_ids"))
.into_inner()
.expect("Unwrapping mutex for module_ids");
.into_inner();
let interns = Interns {
module_ids,
@ -1401,8 +1419,7 @@ fn finish<'a>(
let module_ids = Arc::try_unwrap(state.arc_modules)
.unwrap_or_else(|_| panic!("There were still outstanding Arc references to module_ids"))
.into_inner()
.expect("Unwrapping mutex for module_ids");
.into_inner();
let interns = Interns {
module_ids,
@ -1599,10 +1616,8 @@ fn send_header<'a>(
let ident_ids = {
// Lock just long enough to perform the minimal operations necessary.
let mut module_ids = (*module_ids).lock().expect("Failed to acquire lock for interning module IDs, presumably because a thread panicked.");
let mut ident_ids_by_module = (*ident_ids_by_module).lock().expect(
"Failed to acquire lock for interning ident IDs, presumably because a thread panicked.",
);
let mut module_ids = (*module_ids).lock();
let mut ident_ids_by_module = (*ident_ids_by_module).lock();
home = module_ids.get_or_insert(&declared_name.as_inline_str());
@ -1727,11 +1742,13 @@ impl<'a> BuildTask<'a> {
stdlib,
);
for unused_import in unused_imports {
if !unused_imports.is_empty() {
todo!(
"TODO gracefully handle unused import {:?} from module {:?}",
unused_import,
home
"TODO gracefully handle unused import {:?} from module {:?} {:#?} {}",
&unused_imports,
home,
&module,
src
);
}
@ -1815,7 +1832,7 @@ fn run_solve<'a>(
fn parse_and_constrain<'a>(
header: ModuleHeader<'a>,
mode: Mode,
module_ids: ModuleIds,
module_ids: &ModuleIds,
dep_idents: IdentIdsByModule,
exposed_symbols: MutSet<Symbol>,
) -> Result<Msg<'a>, LoadingProblem> {
@ -1836,7 +1853,7 @@ fn parse_and_constrain<'a>(
&arena,
parsed_defs,
module_id,
&module_ids,
module_ids,
header.exposed_ident_ids,
dep_idents,
header.exposed_imports,
@ -2153,7 +2170,7 @@ fn run_task<'a>(
module_ids,
dep_idents,
exposed_symbols,
} => parse_and_constrain(header, mode, module_ids, dep_idents, exposed_symbols),
} => parse_and_constrain(header, mode, &module_ids, dep_idents, exposed_symbols),
Solve {
module,
module_timing,

View file

@ -47,6 +47,8 @@ These are potentially inspirational resources for the editor's design.
* [Unreal Engine 4](https://www.unrealengine.com/en-US/)
* [Blueprints](https://docs.unrealengine.com/en-US/Engine/Blueprints/index.html) visual scripting (not suggesting visual scripting for Roc)
* [Live Programing](https://www.microsoft.com/en-us/research/project/live-programming/?from=http%3A%2F%2Fresearch.microsoft.com%2Fen-us%2Fprojects%2Fliveprogramming%2Ftypography.aspx#!publications) by [Microsoft Research] it contains many interesting research papers.
### Non-Code Related Inspiration
* [Scrivner](https://www.literatureandlatte.com/scrivener/overview) writing app for novelists, screenwriters, and more