From 37cfb0f650834281901481c75719fcd91f538432 Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Wed, 27 Nov 2019 22:12:51 -0500 Subject: [PATCH 01/20] Add tokio --- Cargo.lock | 137 +++++++++++++++++++++++++++++++---------------------- Cargo.toml | 1 + 2 files changed, 82 insertions(+), 56 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 79178d4c94..755455da8c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -33,7 +33,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -43,7 +43,7 @@ version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -87,6 +87,11 @@ dependencies = [ "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "bytes" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "c2-chacha" version = "0.2.3" @@ -155,7 +160,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -276,7 +281,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", "synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -292,7 +297,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "miniz_oxide 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -353,7 +358,7 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -379,7 +384,7 @@ name = "hermit-abi" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -428,12 +433,12 @@ dependencies = [ "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-buf 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-reactor 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-tcp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-timer 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-timer 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", "want 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -507,7 +512,7 @@ dependencies = [ "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", "unindent 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -518,7 +523,7 @@ source = "git+https://github.com/TheDan64/inkwell?branch=llvm8-0#c74bb768c8f2bd8 dependencies = [ "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", "inkwell_internals 0.1.0 (git+https://github.com/TheDan64/inkwell?branch=llvm8-0)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "llvm-sys 80.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "once_cell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -542,7 +547,7 @@ name = "iovec" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -566,7 +571,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "libc" -version = "0.2.65" +version = "0.2.66" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -576,7 +581,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -649,14 +654,15 @@ dependencies = [ [[package]] name = "mio" -version = "0.6.19" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", @@ -681,7 +687,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "openssl 0.10.26 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -698,7 +704,7 @@ version = "0.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -708,7 +714,7 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -725,7 +731,7 @@ dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "openssl-sys 0.9.53 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -741,7 +747,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", "vcpkg 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -768,7 +774,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", @@ -794,6 +800,11 @@ dependencies = [ "ordermap 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "pin-project-lite" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "pkg-config" version = "0.3.17" @@ -820,7 +831,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -894,7 +905,7 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -912,7 +923,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -986,7 +997,7 @@ name = "rand_jitter" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -998,7 +1009,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1090,10 +1101,10 @@ dependencies = [ "serde_urlencoded 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-timer 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-timer 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "uuid 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", "winreg 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1113,6 +1124,7 @@ dependencies = [ "pretty_assertions 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "quickcheck 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", "quickcheck_macros 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "wyhash 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1155,7 +1167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "core-foundation 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "security-framework-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1195,7 +1207,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1266,7 +1278,7 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1281,7 +1293,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1291,7 +1303,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", "remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1311,7 +1323,7 @@ name = "time" version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1323,15 +1335,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", - "mio 0.6.19 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-current-thread 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-reactor 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-tcp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-timer 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-timer 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-project-lite 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1350,12 +1372,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "tokio-executor" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1374,18 +1396,18 @@ dependencies = [ [[package]] name = "tokio-reactor" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "mio 0.6.19 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-sync 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1407,9 +1429,9 @@ dependencies = [ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "mio 0.6.19 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-reactor 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-reactor 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1425,18 +1447,18 @@ dependencies = [ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "tokio-timer" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1629,6 +1651,7 @@ dependencies = [ "checksum bumpalo 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ad807f2fc2bf185eeb98ff3a901bd46dc5ad58163d0fa4577ba0d25674d71708" "checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" "checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +"checksum bytes 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1c85319f157e4e26c703678e68e26ab71a46c0199286fa670b21cc9fec13d895" "checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb" "checksum cargo_toml 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e7877b00aaf997d7ed66a81281d3a8b9f9da5361df05b72785b985349979a0f3" "checksum cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)" = "aa87058dce70a3ff5621797f1506cb837edd02ac4c0ae642b4542dce802908b8" @@ -1682,7 +1705,7 @@ dependencies = [ "checksum itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -"checksum libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)" = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8" +"checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" "checksum llvm-sys 80.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2110cd4daf9cd8e39dd3b933b1a2a2ac7315e91f7c92b3a20beab526c63b5978" "checksum lock_api 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e57b3997725d2b60dbec1297f6c2e2957cc383db1cebd6be812163f969c7d586" "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" @@ -1694,7 +1717,7 @@ dependencies = [ "checksum mime 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "dd1d63acd1b78403cc0c325605908475dd9b9a3acbf65ed8bcab97e27014afcf" "checksum mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1a0ed03949aef72dbdf3116a383d7b38b4768e6f960528cd6a6044aa9ed68599" "checksum miniz_oxide 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6f3f74f726ae935c3f514300cc6773a0c9492abc5e972d42ba0c0ebb88757625" -"checksum mio 0.6.19 (registry+https://github.com/rust-lang/crates.io-index)" = "83f51996a3ed004ef184e16818edc51fadffe8e7ca68be67f9dee67d84d0ff23" +"checksum mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f" "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" "checksum native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4b2df1a4c22fd44a62147fd8f13dd0f95c9d8ca7b2610299b2a2f9cf8964274e" "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" @@ -1709,6 +1732,7 @@ dependencies = [ "checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" "checksum percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" "checksum petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f" +"checksum pin-project-lite 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f0af6cbca0e6e3ce8692ee19fb8d734b641899e07b68eb73e9bbbd32f1703991" "checksum pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" "checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" "checksum pretty_assertions 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3a029430f0d744bc3d15dd474d591bed2402b645d024583082b9f63bb936dac6" @@ -1760,21 +1784,22 @@ dependencies = [ "checksum smallvec 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ecf3b85f68e8abaa7555aa5abdb1153079387e60b718283d732f03897fcfc86" "checksum string 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d24114bfcceb867ca7f71a0d3fe45d45619ec47a6fbfa98cb14e14250bfa5d6d" "checksum syn 0.15.44 (registry+https://github.com/rust-lang/crates.io-index)" = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" -"checksum syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "661641ea2aa15845cddeb97dad000d22070bb5c1fb456b96c1cba883ec691e92" +"checksum syn 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "f89693ae015201f8de93fd96bde2d065f8bfc3f97ce006d5bc9f900b97c0c7c0" "checksum synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" "checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" "checksum tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)" = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" +"checksum tokio 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2e765bf9f550bd9b8a970633ca3b56b8120c4b6c5dcbe26a93744cb02fee4b17" "checksum tokio-buf 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46" "checksum tokio-current-thread 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "d16217cad7f1b840c5a97dfb3c43b0c871fef423a6e8d2118c604e843662a443" -"checksum tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "0f27ee0e6db01c5f0b2973824547ce7e637b2ed79b891a9677b0de9bd532b6ac" +"checksum tokio-executor 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "ca6df436c42b0c3330a82d855d2ef017cd793090ad550a6bc2184f4b933532ab" "checksum tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "5090db468dad16e1a7a54c8c67280c5e4b544f3d3e018f0b913b400261f85926" -"checksum tokio-reactor 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "c56391be9805bc80163151c0b9e5164ee64f4b0200962c346fea12773158f22d" +"checksum tokio-reactor 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "6732fe6b53c8d11178dcb77ac6d9682af27fc6d4cb87789449152e5377377146" "checksum tokio-sync 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "d06554cce1ae4a50f42fba8023918afa931413aded705b560e29600ccf7c6d76" "checksum tokio-tcp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1d14b10654be682ac43efee27401d792507e30fd8d26389e1da3b185de2e4119" "checksum tokio-threadpool 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "2bd2c6a3885302581f4401c82af70d792bb9df1700e7437b0aeb4ada94d5388c" -"checksum tokio-timer 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "f2106812d500ed25a4f38235b9cae8f78a09edf43203e16e59c3b769a342a60e" +"checksum tokio-timer 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "1739638e364e558128461fc1ad84d997702c8e31c2e6b18fb99842268199e827" "checksum toml 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "01d1404644c8b12b16bfcffa4322403a91a451584daaaa7c28d3152e6cbc98cf" "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" "checksum try_from 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "283d3b89e1368717881a9d51dad843cc435380d8109c9e47d38780a324698d8b" diff --git a/Cargo.toml b/Cargo.toml index ee288f98a8..55021a08b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ log = "0.4.8" petgraph = { version = "0.4.5", optional = true } im-rc = "14.0.0" wyhash = "0.3.0" +tokio = { version = "0.2", features = ["fs", "rt-threaded"] } bumpalo = "2.6.0" # NOTE: Breaking API changes get pushed directly to this Inkwell branch, so be # very careful when running `cargo update` to get a new revision into Cargo.lock. From 29acb7a2b448c031c20574f9dbf6446204d2474c Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Wed, 27 Nov 2019 22:43:28 -0500 Subject: [PATCH 02/20] First pass at async stuff --- src/load/mod.rs | 162 ++++++++++++++++++++++++++++-------------------- 1 file changed, 94 insertions(+), 68 deletions(-) diff --git a/src/load/mod.rs b/src/load/mod.rs index 6f1df69a2a..4c3b8c51b9 100644 --- a/src/load/mod.rs +++ b/src/load/mod.rs @@ -8,22 +8,21 @@ use crate::parse::parser::{Fail, Parser, State}; use crate::region::{Located, Region}; use bumpalo::collections::Vec; use bumpalo::Bump; -use std::fs::read_to_string; +use std::future::Future; use std::io; use std::path::{Path, PathBuf}; +use std::pin::Pin; +use tokio::fs::read_to_string; pub struct Loaded<'a> { pub requested_header: LoadedHeader<'a>, - pub dependent_headers: MutMap, LoadedHeader<'a>>, + pub dependent_headers: ImMap, LoadedHeader<'a>>, pub defs: MutMap, Result>>, Fail>>, - pub problems: Vec<'a, BuildProblem<'a>>, } -struct Env<'a, 'p> { +struct Env<'a> { pub arena: &'a Bump, - pub src_dir: &'p Path, - pub problems: Vec<'a, BuildProblem<'a>>, - pub loaded_headers: MutMap, LoadedHeader<'a>>, + pub src_dir: &'a Path, pub queue: Queue<'a>, } @@ -34,7 +33,7 @@ pub enum BuildProblem<'a> { FileNotFound(&'a Path), } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone)] pub enum LoadedHeader<'a> { Valid { scope: ImMap, (Symbol, Region)>, @@ -43,32 +42,38 @@ pub enum LoadedHeader<'a> { ParsingFailed(Fail), } -pub fn load<'a>(arena: &'a Bump, src_dir: &Path, filename: &Path) -> Loaded<'a> { - let mut env = Env { +pub async fn load<'a>(arena: &'a Bump, src_dir: &'a Path, filename: &Path) -> Loaded<'a> { + let env = Env { arena, src_dir, - problems: Vec::new_in(&arena), - loaded_headers: MutMap::default(), queue: MutMap::default(), }; - let requested_header = load_filename(&mut env, filename); + + /// TODO proof of concept: + /// + /// set up a job queue, and load *all* modules using that. + /// after each one loads, maintain a cache of "we've already started loading this" + /// so subsequent ones don't need to enqueue redundantly - + /// but also check again before running a fresh load! + /// Also, use a similar (maybe even the same?) queue for parsing defs in parallel + + let (requested_header, dependent_headers) = load_filename(&env, filename).await; let mut defs = MutMap::default(); - for (module_name, state) in env.queue { - let loaded_defs = match module::module_defs().parse(arena, state) { - Ok((defs, _)) => Ok(defs), - Err((fail, _)) => Err(fail), - }; + // for (module_name, state) in env.queue { + // let loaded_defs = match module::module_defs().parse(arena, state) { + // Ok((defs, _)) => Ok(defs), + // Err((fail, _)) => Err(fail), + // }; - defs.insert(module_name, loaded_defs); - } + // defs.insert(module_name, loaded_defs); + // } Loaded { requested_header, - dependent_headers: env.loaded_headers, + dependent_headers, defs, - problems: env.problems, } } @@ -117,7 +122,9 @@ pub fn load<'a>(arena: &'a Bump, src_dir: &Path, filename: &Path) -> Loaded<'a> /// module's canonicalization. /// /// If a given import has not been loaded yet, load it too. -fn load_module<'a, 'p>(env: &mut Env<'a, 'p>, module_name: &ModuleName<'a>) -> LoadedHeader<'a> { +async fn load_module<'a>(env: &'a Env<'a>, + loaded_headers: ImMap, LoadedHeader<'a>>, + module_name: &ModuleName<'a>) -> (LoadedHeader<'a>, ImMap, LoadedHeader<'a>>) { // 1. Convert module_name to filename, using src_dir. // 2. Open that file for reading. (If there's a problem, record it and bail.) // 3. Read the whole file into a string. (In the future, we can read just the header.) @@ -138,11 +145,14 @@ fn load_module<'a, 'p>(env: &mut Env<'a, 'p>, module_name: &ModuleName<'a>) -> L // End with .roc filename.set_extension("roc"); - load_filename(env, &filename) + load_filename(env, loaded_headers,&filename).await } -fn load_filename<'a, 'p>(env: &mut Env<'a, 'p>, filename: &Path) -> LoadedHeader<'a> { - match read_to_string(filename) { +async fn load_filename<'a, 'p>(env: &'a Env<'a>, + + loaded_headers: ImMap, LoadedHeader<'a>>, + filename: &Path) -> (LoadedHeader<'a>, ImMap, LoadedHeader<'a>>) { + let imports = match read_to_string(filename).await { Ok(src) => { // TODO instead of env.arena.alloc(src), we should create a new buffer // in the arena as a Vec<'a, u8> and call .as_mut_slice() on it to @@ -154,64 +164,80 @@ fn load_filename<'a, 'p>(env: &mut Env<'a, 'p>, filename: &Path) -> LoadedHeader match module::module().parse(env.arena, state) { Ok((Module::Interface { header }, state)) => { - let mut scope = ImMap::default(); - // Enqueue the defs parsing job for background processing. - env.queue.insert(header.name.value, state); - - for loc_entry in header.imports { - load_import(env, loc_entry.region, &loc_entry.value, &mut scope); - } - - LoadedHeader::Valid { scope } + // env.queue.insert(header.name.value, state); + + header.imports } Ok((Module::App { header }, state)) => { - let mut scope = ImMap::default(); - // Enqueue the defs parsing job for background processing. // The app module has a module name of "" - env.queue.insert(ModuleName::new(""), state); - - for loc_entry in header.imports { - load_import(env, loc_entry.region, &loc_entry.value, &mut scope); - } - - LoadedHeader::Valid { scope } + // env.queue.insert(ModuleName::new(""), state); + + header.imports + } + Err((fail, _)) => { + return LoadedHeader::ParsingFailed(fail); } - Err((fail, _)) => LoadedHeader::ParsingFailed(fail), } } - Err(err) => LoadedHeader::FileProblem(err.kind()), - } -} + Err(err) => return LoadedHeader::FileProblem(err.kind()), + }; -fn load_import<'a, 'p>( - env: &mut Env<'a, 'p>, + let mut scope = ImMap::default(); + let mut headers = ImMap::default(); + + for loc_entry in imports { + let (new_scope, opt_header) = + load_import(env, loc_entry.region, loaded_headers, env.arena.alloc(loc_entry.value)).await; + + scope = scope.union(new_scope); + + if let Some((module_name, loaded_header)) = opt_header { + headers.insert(module_name, loaded_header); + } + } + + (LoadedHeader::Valid { scope }, headers) +} + +type Scope<'a>= ImMap, (Symbol, Region)>; + +fn load_import<'a>( + env: &'a Env<'a>, region: Region, - entry: &ImportsEntry<'a>, - scope: &mut ImMap, (Symbol, Region)>, -) { - use crate::parse::ast::ImportsEntry::*; + loaded_headers: ImMap, LoadedHeader<'a>>, + entry: &'a ImportsEntry<'a>, +) -> Pin, Option<(ModuleName<'a>, LoadedHeader<'a>)>)> + 'a>> { + Box::pin(async move { + use crate::parse::ast::ImportsEntry::*; - match entry { - Module(module_name, exposes) => { - // If we haven't already loaded the module, load it! - if !env.loaded_headers.contains_key(&module_name) { - let loaded = load_module(env, module_name); + match entry { + Module(module_name, exposes) => { + // If we haven't already loaded the module, load it! + let new_header = if !loaded_headers.contains_key(&module_name) { + let loaded = load_module(env, loaded_headers, module_name).await; - env.loaded_headers.insert(*module_name, loaded); + Some((*module_name, loaded)) + } else { + None + }; + + let mut scope = ImMap::default(); + + for loc_entry in exposes { + expose(*module_name, &loc_entry.value, loc_entry.region, &mut scope) + } + + (scope, new_header) } - for loc_entry in exposes { - expose(*module_name, &loc_entry.value, loc_entry.region, scope) + SpaceBefore(sub_entry, _) | SpaceAfter(sub_entry, _) => { + // Ignore spaces. + load_import(env, region, *sub_entry).await } } - - SpaceBefore(sub_entry, _) | SpaceAfter(sub_entry, _) => { - // Ignore spaces. - load_import(env, region, *sub_entry, scope) - } - } + }) } fn expose<'a>( From 700fd6d2f20059e5f21e60cda666bef663f9f71c Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Sat, 30 Nov 2019 08:48:07 -0500 Subject: [PATCH 03/20] Revert "First pass at async stuff" This reverts commit 43440629ea81978f379c79576d8d9cada1be85fa. --- src/load/mod.rs | 166 ++++++++++++++++++++---------------------------- 1 file changed, 70 insertions(+), 96 deletions(-) diff --git a/src/load/mod.rs b/src/load/mod.rs index 4c3b8c51b9..6f1df69a2a 100644 --- a/src/load/mod.rs +++ b/src/load/mod.rs @@ -8,21 +8,22 @@ use crate::parse::parser::{Fail, Parser, State}; use crate::region::{Located, Region}; use bumpalo::collections::Vec; use bumpalo::Bump; -use std::future::Future; +use std::fs::read_to_string; use std::io; use std::path::{Path, PathBuf}; -use std::pin::Pin; -use tokio::fs::read_to_string; pub struct Loaded<'a> { pub requested_header: LoadedHeader<'a>, - pub dependent_headers: ImMap, LoadedHeader<'a>>, + pub dependent_headers: MutMap, LoadedHeader<'a>>, pub defs: MutMap, Result>>, Fail>>, + pub problems: Vec<'a, BuildProblem<'a>>, } -struct Env<'a> { +struct Env<'a, 'p> { pub arena: &'a Bump, - pub src_dir: &'a Path, + pub src_dir: &'p Path, + pub problems: Vec<'a, BuildProblem<'a>>, + pub loaded_headers: MutMap, LoadedHeader<'a>>, pub queue: Queue<'a>, } @@ -33,7 +34,7 @@ pub enum BuildProblem<'a> { FileNotFound(&'a Path), } -#[derive(Debug, PartialEq, Eq, Clone)] +#[derive(Debug, PartialEq, Eq)] pub enum LoadedHeader<'a> { Valid { scope: ImMap, (Symbol, Region)>, @@ -42,38 +43,32 @@ pub enum LoadedHeader<'a> { ParsingFailed(Fail), } -pub async fn load<'a>(arena: &'a Bump, src_dir: &'a Path, filename: &Path) -> Loaded<'a> { - let env = Env { +pub fn load<'a>(arena: &'a Bump, src_dir: &Path, filename: &Path) -> Loaded<'a> { + let mut env = Env { arena, src_dir, + problems: Vec::new_in(&arena), + loaded_headers: MutMap::default(), queue: MutMap::default(), }; - - /// TODO proof of concept: - /// - /// set up a job queue, and load *all* modules using that. - /// after each one loads, maintain a cache of "we've already started loading this" - /// so subsequent ones don't need to enqueue redundantly - - /// but also check again before running a fresh load! - /// Also, use a similar (maybe even the same?) queue for parsing defs in parallel - - let (requested_header, dependent_headers) = load_filename(&env, filename).await; + let requested_header = load_filename(&mut env, filename); let mut defs = MutMap::default(); - // for (module_name, state) in env.queue { - // let loaded_defs = match module::module_defs().parse(arena, state) { - // Ok((defs, _)) => Ok(defs), - // Err((fail, _)) => Err(fail), - // }; + for (module_name, state) in env.queue { + let loaded_defs = match module::module_defs().parse(arena, state) { + Ok((defs, _)) => Ok(defs), + Err((fail, _)) => Err(fail), + }; - // defs.insert(module_name, loaded_defs); - // } + defs.insert(module_name, loaded_defs); + } Loaded { requested_header, - dependent_headers, + dependent_headers: env.loaded_headers, defs, + problems: env.problems, } } @@ -122,9 +117,7 @@ pub async fn load<'a>(arena: &'a Bump, src_dir: &'a Path, filename: &Path) -> Lo /// module's canonicalization. /// /// If a given import has not been loaded yet, load it too. -async fn load_module<'a>(env: &'a Env<'a>, - loaded_headers: ImMap, LoadedHeader<'a>>, - module_name: &ModuleName<'a>) -> (LoadedHeader<'a>, ImMap, LoadedHeader<'a>>) { +fn load_module<'a, 'p>(env: &mut Env<'a, 'p>, module_name: &ModuleName<'a>) -> LoadedHeader<'a> { // 1. Convert module_name to filename, using src_dir. // 2. Open that file for reading. (If there's a problem, record it and bail.) // 3. Read the whole file into a string. (In the future, we can read just the header.) @@ -145,14 +138,11 @@ async fn load_module<'a>(env: &'a Env<'a>, // End with .roc filename.set_extension("roc"); - load_filename(env, loaded_headers,&filename).await + load_filename(env, &filename) } -async fn load_filename<'a, 'p>(env: &'a Env<'a>, - - loaded_headers: ImMap, LoadedHeader<'a>>, - filename: &Path) -> (LoadedHeader<'a>, ImMap, LoadedHeader<'a>>) { - let imports = match read_to_string(filename).await { +fn load_filename<'a, 'p>(env: &mut Env<'a, 'p>, filename: &Path) -> LoadedHeader<'a> { + match read_to_string(filename) { Ok(src) => { // TODO instead of env.arena.alloc(src), we should create a new buffer // in the arena as a Vec<'a, u8> and call .as_mut_slice() on it to @@ -164,80 +154,64 @@ async fn load_filename<'a, 'p>(env: &'a Env<'a>, match module::module().parse(env.arena, state) { Ok((Module::Interface { header }, state)) => { + let mut scope = ImMap::default(); + // Enqueue the defs parsing job for background processing. - // env.queue.insert(header.name.value, state); - - header.imports + env.queue.insert(header.name.value, state); + + for loc_entry in header.imports { + load_import(env, loc_entry.region, &loc_entry.value, &mut scope); + } + + LoadedHeader::Valid { scope } } Ok((Module::App { header }, state)) => { + let mut scope = ImMap::default(); + // Enqueue the defs parsing job for background processing. // The app module has a module name of "" - // env.queue.insert(ModuleName::new(""), state); - - header.imports - } - Err((fail, _)) => { - return LoadedHeader::ParsingFailed(fail); + env.queue.insert(ModuleName::new(""), state); + + for loc_entry in header.imports { + load_import(env, loc_entry.region, &loc_entry.value, &mut scope); + } + + LoadedHeader::Valid { scope } } + Err((fail, _)) => LoadedHeader::ParsingFailed(fail), } } - Err(err) => return LoadedHeader::FileProblem(err.kind()), - }; + Err(err) => LoadedHeader::FileProblem(err.kind()), + } +} - let mut scope = ImMap::default(); - let mut headers = ImMap::default(); +fn load_import<'a, 'p>( + env: &mut Env<'a, 'p>, + region: Region, + entry: &ImportsEntry<'a>, + scope: &mut ImMap, (Symbol, Region)>, +) { + use crate::parse::ast::ImportsEntry::*; - for loc_entry in imports { - let (new_scope, opt_header) = - load_import(env, loc_entry.region, loaded_headers, env.arena.alloc(loc_entry.value)).await; + match entry { + Module(module_name, exposes) => { + // If we haven't already loaded the module, load it! + if !env.loaded_headers.contains_key(&module_name) { + let loaded = load_module(env, module_name); - scope = scope.union(new_scope); + env.loaded_headers.insert(*module_name, loaded); + } - if let Some((module_name, loaded_header)) = opt_header { - headers.insert(module_name, loaded_header); + for loc_entry in exposes { + expose(*module_name, &loc_entry.value, loc_entry.region, scope) + } + } + + SpaceBefore(sub_entry, _) | SpaceAfter(sub_entry, _) => { + // Ignore spaces. + load_import(env, region, *sub_entry, scope) } } - - (LoadedHeader::Valid { scope }, headers) -} - -type Scope<'a>= ImMap, (Symbol, Region)>; - -fn load_import<'a>( - env: &'a Env<'a>, - region: Region, - loaded_headers: ImMap, LoadedHeader<'a>>, - entry: &'a ImportsEntry<'a>, -) -> Pin, Option<(ModuleName<'a>, LoadedHeader<'a>)>)> + 'a>> { - Box::pin(async move { - use crate::parse::ast::ImportsEntry::*; - - match entry { - Module(module_name, exposes) => { - // If we haven't already loaded the module, load it! - let new_header = if !loaded_headers.contains_key(&module_name) { - let loaded = load_module(env, loaded_headers, module_name).await; - - Some((*module_name, loaded)) - } else { - None - }; - - let mut scope = ImMap::default(); - - for loc_entry in exposes { - expose(*module_name, &loc_entry.value, loc_entry.region, &mut scope) - } - - (scope, new_header) - } - - SpaceBefore(sub_entry, _) | SpaceAfter(sub_entry, _) => { - // Ignore spaces. - load_import(env, region, *sub_entry).await - } - } - }) } fn expose<'a>( From 0477b68dc3af1c952a1ade56e07faaabb6585af0 Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Sat, 30 Nov 2019 10:49:37 -0500 Subject: [PATCH 04/20] Add SendMap and SendSet --- Cargo.lock | 15 +++++++++++++++ Cargo.toml | 3 ++- src/collections.rs | 4 ++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 755455da8c..e1e5b56edb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -474,6 +474,19 @@ dependencies = [ "unicode-normalization 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "im" +version = "14.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitmaps 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_xoshiro 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sized-chunks 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "typenum 1.11.2 (registry+https://github.com/rust-lang/crates.io-index)", + "version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "im-rc" version = "14.0.0" @@ -1115,6 +1128,7 @@ name = "roc" version = "0.1.0" dependencies = [ "bumpalo 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "im 14.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "im-rc 14.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "indoc 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "inkwell 0.1.0 (git+https://github.com/TheDan64/inkwell?branch=llvm8-0)", @@ -1695,6 +1709,7 @@ dependencies = [ "checksum hyper-tls 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f" "checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" "checksum idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +"checksum im 14.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a1f9b6540e530defef7f2df4ed330d45b739b10450548c74a9913f63ea1acc6b" "checksum im-rc 14.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e9ad726dce25993be6352b0bff048e4d2647440c0a673d32257c4fac49356d18" "checksum indexmap 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712d7b3ea5827fcb9d4fda14bf4da5f136f0db2ae9c8f4bd4e2d1c6fde4e6db2" "checksum indoc 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "3f9553c1e16c114b8b77ebeb329e5f2876eed62a8d51178c8bc6bff0d65f98f8" diff --git a/Cargo.toml b/Cargo.toml index 55021a08b2..8fcbf47fb6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,8 @@ edition = "2018" [dependencies] log = "0.4.8" petgraph = { version = "0.4.5", optional = true } -im-rc = "14.0.0" +im = "14.0.0" # im and im-rc should always have the same version! +im-rc = "14.0.0" # im and im-rc should always have the same version! wyhash = "0.3.0" tokio = { version = "0.2", features = ["fs", "rt-threaded"] } bumpalo = "2.6.0" diff --git a/src/collections.rs b/src/collections.rs index 3ad344c6e4..39ce1cb339 100644 --- a/src/collections.rs +++ b/src/collections.rs @@ -24,6 +24,10 @@ pub type ImMap = im_rc::hashmap::HashMap; pub type ImSet = im_rc::hashset::HashSet; +pub type SendMap = im::hashmap::HashMap; + +pub type SendSet = im::hashset::HashSet; + pub fn arena_join<'a, I>(arena: &'a Bump, strings: &mut I, join_str: &str) -> String<'a> where I: Iterator, From 7c815b276a6d9a2739204790bf8872957bb78da2 Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Sat, 30 Nov 2019 10:49:46 -0500 Subject: [PATCH 05/20] Fix UnqualifiedIdent docs --- src/ident.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/ident.rs b/src/ident.rs index 0064d116fb..a07e058d3d 100644 --- a/src/ident.rs +++ b/src/ident.rs @@ -1,8 +1,6 @@ use std::fmt::{self, Display, Formatter}; -/// An identifier, possibly fully-qualified with a module name -/// e.g. (Http.Request from http) -/// Parameterized on a phantom marker for whether it has been canonicalized +/// An unqualified identifier, possibly capitalized. #[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct UnqualifiedIdent<'a>(&'a str); From d3f6d74fd762cb67f57728eff3b923a035b12220 Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Sat, 30 Nov 2019 10:50:06 -0500 Subject: [PATCH 06/20] Make State derive Eq --- src/parse/parser.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parse/parser.rs b/src/parse/parser.rs index ee2c17ecb0..456408e2ec 100644 --- a/src/parse/parser.rs +++ b/src/parse/parser.rs @@ -5,7 +5,7 @@ use bumpalo::Bump; use std::{char, u16}; /// A position in a source file. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct State<'a> { /// The raw input string. pub input: &'a str, From 4cf040a07604fb974418afb0065d2313754b4b38 Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Sat, 30 Nov 2019 10:50:20 -0500 Subject: [PATCH 07/20] Add State::bytes_consumed() --- src/parse/parser.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/parse/parser.rs b/src/parse/parser.rs index 456408e2ec..659045e078 100644 --- a/src/parse/parser.rs +++ b/src/parse/parser.rs @@ -24,6 +24,12 @@ pub struct State<'a> { pub is_indenting: bool, pub attempting: Attempting, + + /// The original length of the string, before any bytes were consumed. + /// This is used internally by the State::bytes_consumed() function. + /// + /// TODO make this private, in a way that doesn't break macros! + pub original_len: usize, } #[derive(Debug, PartialEq, Eq)] @@ -41,6 +47,7 @@ impl<'a> State<'a> { indent_col: 0, is_indenting: true, attempting, + original_len: input.len(), } } @@ -58,6 +65,13 @@ impl<'a> State<'a> { } } + /// Returns the total number of bytes consumed since the parser began parsing. + /// + /// So if the parser has consumed 8 bytes, this function will return 8. + pub fn bytes_consumed(&self) -> usize { + self.original_len - self.input.len() + } + /// Increments the line, then resets column, indent_col, and is_indenting. /// Advances the input by 1, to consume the newline character. pub fn newline(&self) -> Result { @@ -69,6 +83,7 @@ impl<'a> State<'a> { indent_col: 0, is_indenting: true, attempting: self.attempting, + original_len: self.original_len, }), None => Err(( Fail { @@ -95,6 +110,7 @@ impl<'a> State<'a> { // Once we hit a nonspace character, we are no longer indenting. is_indenting: false, attempting: self.attempting, + original_len: self.original_len, }) } _ => Err(line_too_long(self.attempting, self.clone())), @@ -131,6 +147,7 @@ impl<'a> State<'a> { indent_col, is_indenting, attempting: self.attempting, + original_len: self.original_len, }) } _ => Err(line_too_long(self.attempting, self.clone())), @@ -360,6 +377,7 @@ fn line_too_long(attempting: Attempting, state: State<'_>) -> (Fail, State<'_>) is_indenting: state.is_indenting, column, attempting, + original_len: state.original_len, }; (fail, state) From fd116296dff9f0ccd156585fd84698935453f402 Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Sat, 30 Nov 2019 09:12:25 -0500 Subject: [PATCH 08/20] Second pass at parallel loading --- src/load/mod.rs | 225 ++++++++++++++++++++++++++++-------------------- 1 file changed, 131 insertions(+), 94 deletions(-) diff --git a/src/load/mod.rs b/src/load/mod.rs index 6f1df69a2a..b44bf60024 100644 --- a/src/load/mod.rs +++ b/src/load/mod.rs @@ -1,75 +1,82 @@ use crate::can::symbol::Symbol; -use crate::collections::{ImMap, MutMap}; +use crate::collections::{SendMap, SendSet}; use crate::ident::UnqualifiedIdent; use crate::module::ModuleName; use crate::parse::ast::{Attempting, Def, ExposesEntry, ImportsEntry, Module}; use crate::parse::module; use crate::parse::parser::{Fail, Parser, State}; use crate::region::{Located, Region}; -use bumpalo::collections::Vec; +use im::Vector; use bumpalo::Bump; -use std::fs::read_to_string; +use tokio::fs::read_to_string; use std::io; use std::path::{Path, PathBuf}; +use tokio::prelude::*; + pub struct Loaded<'a> { - pub requested_header: LoadedHeader<'a>, - pub dependent_headers: MutMap, LoadedHeader<'a>>, - pub defs: MutMap, Result>>, Fail>>, - pub problems: Vec<'a, BuildProblem<'a>>, + pub requested_header: LoadedHeader, + pub dependent_headers: SendMap, LoadedHeader>, + pub defs: SendMap, Result>>, Fail>>, } -struct Env<'a, 'p> { - pub arena: &'a Bump, - pub src_dir: &'p Path, - pub problems: Vec<'a, BuildProblem<'a>>, - pub loaded_headers: MutMap, LoadedHeader<'a>>, - pub queue: Queue<'a>, +struct Env { + pub src_dir: PathBuf } -type Queue<'a> = MutMap, State<'a>>; - #[derive(Debug)] pub enum BuildProblem<'a> { FileNotFound(&'a Path), } #[derive(Debug, PartialEq, Eq)] -pub enum LoadedHeader<'a> { +pub enum LoadedHeader { Valid { - scope: ImMap, (Symbol, Region)>, + declared_name: Option>, + deps: SendSet>, + scope: SendMap, (Symbol, Region)>, + bytes_parsed: usize }, FileProblem(io::ErrorKind), ParsingFailed(Fail), } -pub fn load<'a>(arena: &'a Bump, src_dir: &Path, filename: &Path) -> Loaded<'a> { - let mut env = Env { - arena, - src_dir, - problems: Vec::new_in(&arena), - loaded_headers: MutMap::default(), - queue: MutMap::default(), - }; - - let requested_header = load_filename(&mut env, filename); - let mut defs = MutMap::default(); - - for (module_name, state) in env.queue { - let loaded_defs = match module::module_defs().parse(arena, state) { - Ok((defs, _)) => Ok(defs), - Err((fail, _)) => Err(fail), +pub async fn load<'a>(src_dir: PathBuf, filename: PathBuf) -> Loaded<'a> { + let handle = tokio::spawn(async move { + let mut env = Env { + src_dir }; - defs.insert(module_name, loaded_defs); - } + load_filename(&mut env, &filename).await + }); - Loaded { - requested_header, - dependent_headers: env.loaded_headers, - defs, - problems: env.problems, - } + let requested_header = handle.await; + + panic!("TODO"); + + +// // TODO parse defs on a different thread, and parse them +// // directly into a Vector rather than into a Vec, so +// // we don't have to reallocate here. +// let defs = match module::module_defs().parse(&arena, state) { +// Ok((defs, _)) => { +// let mut send_vec = Vector::new(); + +// for def in defs { +// send_vec.push_back(def); +// } + +// Ok(send_vec) +// }, +// Err((fail, _)) => Err(fail), +// }; + + // Loaded { + // requested_header, + // dependent_headers: env.loaded_headers, + // defs, + // problems: env.problems, + // } } /// The long-term plan is for the loading process to work like this, starting from main.roc: @@ -117,94 +124,98 @@ pub fn load<'a>(arena: &'a Bump, src_dir: &Path, filename: &Path) -> Loaded<'a> /// module's canonicalization. /// /// If a given import has not been loaded yet, load it too. -fn load_module<'a, 'p>(env: &mut Env<'a, 'p>, module_name: &ModuleName<'a>) -> LoadedHeader<'a> { - // 1. Convert module_name to filename, using src_dir. - // 2. Open that file for reading. (If there's a problem, record it and bail.) - // 3. Read the whole file into a string. (In the future, we can read just the header.) - // 4. Parse the header. - // 5. Use the parsed header to load more modules as necessary. - // 6. Now that all the headers have been parsed, parse the bodies too. - // 7. Once all the bodies have been parsed, canonicalize beginning with the leaves. +// fn load_module<'a, 'p>(env: &mut Env<'a, 'p>, module_name: &ModuleName<'a>) -> LoadedHeader<'a> { +// // 1. Convert module_name to filename, using src_dir. +// // 2. Open that file for reading. (If there's a problem, record it and bail.) +// // 3. Read the whole file into a string. (In the future, we can read just the header.) +// // 4. Parse the header. +// // 5. Use the parsed header to load more modules as necessary. +// // 6. Now that all the headers have been parsed, parse the bodies too. +// // 7. Once all the bodies have been parsed, canonicalize beginning with the leaves. - let mut filename = PathBuf::new(); +// let mut filename = PathBuf::new(); - filename.push(env.src_dir); +// filename.push(env.src_dir); - // Convert dots in module name to directories - for part in module_name.as_str().split('.') { - filename.push(part); - } +// // Convert dots in module name to directories +// for part in module_name.as_str().split('.') { +// filename.push(part); +// } - // End with .roc - filename.set_extension("roc"); +// // End with .roc +// filename.set_extension("roc"); - load_filename(env, &filename) -} +// load_filename(env, &filename) +// } -fn load_filename<'a, 'p>(env: &mut Env<'a, 'p>, filename: &Path) -> LoadedHeader<'a> { - match read_to_string(filename) { +async fn load_filename(env: &mut Env, filename: &Path) -> LoadedHeader { + match read_to_string(filename).await { Ok(src) => { + let arena = Bump::new(); // TODO instead of env.arena.alloc(src), we should create a new buffer // in the arena as a Vec<'a, u8> and call .as_mut_slice() on it to // get a (&mut [u8]) which can be passed to io::Read::read directly // instead of using read_to_string. This way, we avoid both heap-allocating // the String (which read_to_string does) and also re-allocating it // in the arena after read_to_string completes. - let state = State::new(env.arena.alloc(src), Attempting::Module); + let state = State::new(&src, Attempting::Module); - match module::module().parse(env.arena, state) { + let answer = match module::module().parse(&arena, state) { Ok((Module::Interface { header }, state)) => { - let mut scope = ImMap::default(); + let declared_name = Some(header.name.value.as_str().into()); - // Enqueue the defs parsing job for background processing. - env.queue.insert(header.name.value, state); + let mut scope = SendMap::default(); + let mut deps = SendSet::default(); for loc_entry in header.imports { - load_import(env, loc_entry.region, &loc_entry.value, &mut scope); + deps.insert(load_import(env, loc_entry.region, &loc_entry.value, &mut scope)); } - LoadedHeader::Valid { scope } + let bytes_parsed = state.bytes_consumed(); + + LoadedHeader::Valid { scope, declared_name, deps, bytes_parsed } } Ok((Module::App { header }, state)) => { - let mut scope = ImMap::default(); + // The app module has no declared name. + let declared_name = None; - // Enqueue the defs parsing job for background processing. - // The app module has a module name of "" - env.queue.insert(ModuleName::new(""), state); + let mut scope = SendMap::default(); + let mut deps = SendSet::default(); for loc_entry in header.imports { - load_import(env, loc_entry.region, &loc_entry.value, &mut scope); + deps.insert(load_import(env, loc_entry.region, &loc_entry.value, &mut scope)); } - LoadedHeader::Valid { scope } + let bytes_parsed = state.bytes_consumed(); + + LoadedHeader::Valid { scope, declared_name, deps, bytes_parsed } } Err((fail, _)) => LoadedHeader::ParsingFailed(fail), - } + }; + + answer } Err(err) => LoadedHeader::FileProblem(err.kind()), } } -fn load_import<'a, 'p>( - env: &mut Env<'a, 'p>, +fn load_import<'a, 'p, 'out>( + env: &mut Env, region: Region, - entry: &ImportsEntry<'a>, - scope: &mut ImMap, (Symbol, Region)>, -) { + entry: &ImportsEntry<'_>, + scope: &mut SendMap, (Symbol, Region)>, +) -> Box { use crate::parse::ast::ImportsEntry::*; match entry { Module(module_name, exposes) => { - // If we haven't already loaded the module, load it! - if !env.loaded_headers.contains_key(&module_name) { - let loaded = load_module(env, module_name); - - env.loaded_headers.insert(*module_name, loaded); - } - for loc_entry in exposes { - expose(*module_name, &loc_entry.value, loc_entry.region, scope) + let (key, value) = expose(*module_name, &loc_entry.value, loc_entry.region); + + scope.insert(key, value); } + + module_name.as_str().into() } SpaceBefore(sub_entry, _) | SpaceAfter(sub_entry, _) => { @@ -214,12 +225,11 @@ fn load_import<'a, 'p>( } } -fn expose<'a>( +fn expose<'out>( module_name: ModuleName<'_>, - entry: &ExposesEntry<'a>, + entry: &ExposesEntry<'_>, region: Region, - scope: &mut ImMap, (Symbol, Region)>, -) { +)-> (Box, (Symbol, Region)){ use crate::parse::ast::ExposesEntry::*; match entry { @@ -227,11 +237,38 @@ fn expose<'a>( // Since this value is exposed, add it to our module's default scope. let symbol = Symbol::from_module(&module_name, &ident); - scope.insert(ident.clone(), (symbol, region)); + (ident.as_str().into(), (symbol, region)) } SpaceBefore(sub_entry, _) | SpaceAfter(sub_entry, _) => { // Ignore spaces. - expose(module_name, *sub_entry, region, scope) + expose(module_name, *sub_entry, region) } } } + +#[test] +fn test_tokio() { + test_async(async { + let handle = tokio::spawn(async { + println!("doing some work, asynchronously"); + + // Return a value for the example + "result of the computation" + }); + + // Wait for the spawned task to finish + let res = handle.await; + + println!("got {:?}", res); + }) +} + +fn test_async(future: F) -> F::Output { + use tokio::runtime::Runtime; + + // Create the runtime + let mut rt = Runtime::new().expect("Error initializing Tokio runtime."); + + // Spawn the root task + rt.block_on(future) +} From 064ac5ce84a603dd0d6f5566b59ade45b007a1d5 Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Sat, 30 Nov 2019 17:31:58 -0500 Subject: [PATCH 09/20] Add tokio sync and futures crate --- Cargo.lock | 105 +++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 3 +- 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index e1e5b56edb..31b2d1f7b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -343,6 +343,34 @@ name = "futures" version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "futures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-channel 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-executor 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-io 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-channel" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "futures-cpupool" version = "0.1.8" @@ -352,6 +380,60 @@ dependencies = [ "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "futures-executor" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-io" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-macro" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-sink" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-task" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-channel 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-io 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-macro 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-nested 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "getrandom" version = "0.1.13" @@ -818,6 +900,11 @@ name = "pin-project-lite" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "pin-utils" +version = "0.1.0-alpha.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "pkg-config" version = "0.3.17" @@ -847,6 +934,11 @@ dependencies = [ "syn 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "proc-macro-nested" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "proc-macro2" version = "0.4.30" @@ -1128,6 +1220,7 @@ name = "roc" version = "0.1.0" dependencies = [ "bumpalo 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "im 14.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "im-rc 14.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "indoc 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1366,6 +1459,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)", "pin-project-lite 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1698,7 +1792,16 @@ dependencies = [ "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" "checksum futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)" = "1b980f2816d6ee8673b6517b52cb0e808a180efc92e5c19d02cdda79066703ef" +"checksum futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b6f16056ecbb57525ff698bb955162d0cd03bee84e6241c27ff75c08d8ca5987" +"checksum futures-channel 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fcae98ca17d102fd8a3603727b9259fcf7fa4239b603d2142926189bc8999b86" +"checksum futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "79564c427afefab1dfb3298535b21eda083ef7935b4f0ecbfcb121f0aec10866" "checksum futures-cpupool 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" +"checksum futures-executor 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1e274736563f686a837a0568b478bdabfeaec2dca794b5649b04e2fe1627c231" +"checksum futures-io 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e676577d229e70952ab25f3945795ba5b16d63ca794ca9d2c860e5595d20b5ff" +"checksum futures-macro 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "52e7c56c15537adb4f76d0b7a76ad131cb4d2f4f32d3b0bcabcbe1c7c5e87764" +"checksum futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "171be33efae63c2d59e6dbba34186fe0d6394fb378069a76dfd80fdcffd43c16" +"checksum futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bae52d6b29cf440e298856fec3965ee6fa71b06aa7495178615953fd669e5f9" +"checksum futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c0d66274fb76985d3c62c886d1da7ac4c0903a8c9f754e8fe0f35a6a6cc39e76" "checksum getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407" "checksum h2 0.1.26 (registry+https://github.com/rust-lang/crates.io-index)" = "a5b34c246847f938a410a03c5458c7fee2274436675e76d8b903c08efc29c462" "checksum hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "307c3c9f937f38e3534b1d6447ecf090cafcc9744e4a6360e8b037b2cf5af120" @@ -1748,10 +1851,12 @@ dependencies = [ "checksum percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" "checksum petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f" "checksum pin-project-lite 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f0af6cbca0e6e3ce8692ee19fb8d734b641899e07b68eb73e9bbbd32f1703991" +"checksum pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587" "checksum pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" "checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" "checksum pretty_assertions 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3a029430f0d744bc3d15dd474d591bed2402b645d024583082b9f63bb936dac6" "checksum proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)" = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" +"checksum proc-macro-nested 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e" "checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" "checksum proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9c9e470a8dc4aeae2dee2f335e8f533e2d4b347e1434e5671afc49b054592f27" "checksum publicsuffix 1.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "3bbaa49075179162b49acac1c6aa45fb4dafb5f13cf6794276d77bc7fd95757b" diff --git a/Cargo.toml b/Cargo.toml index 8fcbf47fb6..906eb071ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ petgraph = { version = "0.4.5", optional = true } im = "14.0.0" # im and im-rc should always have the same version! im-rc = "14.0.0" # im and im-rc should always have the same version! wyhash = "0.3.0" -tokio = { version = "0.2", features = ["fs", "rt-threaded"] } +tokio = { version = "0.2", features = ["fs", "sync", "rt-threaded"] } bumpalo = "2.6.0" # NOTE: Breaking API changes get pushed directly to this Inkwell branch, so be # very careful when running `cargo update` to get a new revision into Cargo.lock. @@ -19,6 +19,7 @@ bumpalo = "2.6.0" # `rev` works locally, it causes an error on GitHub Actions. (It's unclear why, # but after several hours of trying unsuccessfully to fix it, `branch` is it.) inkwell = { git = "https://github.com/TheDan64/inkwell", branch = "llvm8-0" } +futures = "0.3.1" [dev-dependencies] pretty_assertions = "0.5.1" From 997b6ec4ad8df9dc8edc0db7ce8020cb3608d646 Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Sat, 30 Nov 2019 17:32:29 -0500 Subject: [PATCH 10/20] Add PatternType::TopLevelDef --- src/can/pattern.rs | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/can/pattern.rs b/src/can/pattern.rs index b1282f25b1..c17d59b48b 100644 --- a/src/can/pattern.rs +++ b/src/can/pattern.rs @@ -40,6 +40,7 @@ pub enum Pattern { /// arg patterns and in case branch patterns, but not in assignments. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum PatternType { + TopLevelDef, Assignment, FunctionArg, CaseBranch, @@ -188,12 +189,14 @@ pub fn canonicalize_pattern<'a>( Pattern::FloatLiteral(float) } - ptype @ Assignment | ptype @ FunctionArg => unsupported_pattern(env, ptype, region), + ptype @ Assignment | ptype @ TopLevelDef | ptype @ FunctionArg => { + unsupported_pattern(env, ptype, region) + } }, &Underscore => match pattern_type { CaseBranch | FunctionArg => Pattern::Underscore(subs.mk_flex_var()), - Assignment => unsupported_pattern(env, Assignment, region), + ptype @ Assignment | ptype @ TopLevelDef => unsupported_pattern(env, ptype, region), }, &IntLiteral(string) => match pattern_type { @@ -203,7 +206,9 @@ pub fn canonicalize_pattern<'a>( Pattern::IntLiteral(int) } - ptype @ Assignment | ptype @ FunctionArg => unsupported_pattern(env, ptype, region), + ptype @ Assignment | ptype @ TopLevelDef | ptype @ FunctionArg => { + unsupported_pattern(env, ptype, region) + } }, &HexIntLiteral(string) => match pattern_type { @@ -213,7 +218,9 @@ pub fn canonicalize_pattern<'a>( Pattern::IntLiteral(int) } - ptype @ Assignment | ptype @ FunctionArg => unsupported_pattern(env, ptype, region), + ptype @ Assignment | ptype @ TopLevelDef | ptype @ FunctionArg => { + unsupported_pattern(env, ptype, region) + } }, &OctalIntLiteral(string) => match pattern_type { @@ -223,7 +230,9 @@ pub fn canonicalize_pattern<'a>( Pattern::IntLiteral(int) } - ptype @ Assignment | ptype @ FunctionArg => unsupported_pattern(env, ptype, region), + ptype @ Assignment | ptype @ TopLevelDef | ptype @ FunctionArg => { + unsupported_pattern(env, ptype, region) + } }, &BinaryIntLiteral(string) => match pattern_type { @@ -233,7 +242,9 @@ pub fn canonicalize_pattern<'a>( Pattern::IntLiteral(int) } - ptype @ Assignment | ptype @ FunctionArg => unsupported_pattern(env, ptype, region), + ptype @ Assignment | ptype @ TopLevelDef | ptype @ FunctionArg => { + unsupported_pattern(env, ptype, region) + } }, &StrLiteral(_string) => match pattern_type { @@ -241,7 +252,9 @@ pub fn canonicalize_pattern<'a>( panic!("TODO check whether string pattern is malformed."); // Pattern::ExactString((*string).into()) } - ptype @ Assignment | ptype @ FunctionArg => unsupported_pattern(env, ptype, region), + ptype @ Assignment | ptype @ TopLevelDef | ptype @ FunctionArg => { + unsupported_pattern(env, ptype, region) + } }, // &EmptyRecordLiteral => Pattern::EmptyRecordLiteral, From 787d76b36aa4ef638f78d8cf049ff25a9c12406d Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Sat, 30 Nov 2019 17:33:09 -0500 Subject: [PATCH 11/20] Get parallel loading working --- src/can/mod.rs | 114 +++++++++++++++++++++++ src/load/mod.rs | 227 +++++++++++++++++++++++---------------------- tests/test_load.rs | 143 ++++++++++++++-------------- 3 files changed, 307 insertions(+), 177 deletions(-) diff --git a/src/can/mod.rs b/src/can/mod.rs index e4f408ded8..ea5bbf081e 100644 --- a/src/can/mod.rs +++ b/src/can/mod.rs @@ -13,6 +13,7 @@ use self::problem::RuntimeError::*; use self::procedure::References; use self::scope::Scope; use self::symbol::Symbol; +use crate::can::pattern::PatternType; use crate::collections::{ImMap, ImSet, MutMap, MutSet}; use crate::constrain::{self, exists}; use crate::graph::{strongly_connected_component, topological_sort}; @@ -26,10 +27,12 @@ use crate::types::Expected::{self, *}; use crate::types::Type::{self, *}; use crate::types::{LetConstraint, PExpected, PReason, Reason}; use bumpalo::Bump; +use im::Vector; use std::fmt::Debug; pub mod env; pub mod expr; +pub mod module; pub mod num; pub mod operator; pub mod pattern; @@ -44,6 +47,117 @@ pub mod symbol; /// map so that expressions within that annotation can share these vars. type Rigids = ImMap, Type>; +pub fn canonicalize_module_defs<'a>( + arena: &Bump, + loc_defs: bumpalo::collections::Vec<'a, Located>>, + home: Box, + scope: &mut ImMap, (Symbol, Region)>, +) -> Vector<(Located, Located)> { + // TODO FIXME need to remove Subs from this - distribute Variables but don't make Subs yet! + let mut subs = Subs::new(); + let mut buf = Vector::new(); + + for loc_def in loc_defs { + buf.push_back(canonicalize_def( + arena, + loc_def.value, + loc_def.region, + home.clone(), + scope, + &mut subs, + )); + } + + buf +} + +fn canonicalize_def<'a>( + arena: &Bump, + def: Def<'a>, + region: Region, + home: Box, + scope: &mut ImMap, (Symbol, Region)>, + // TODO FIXME need to remove Subs from this - distribute Variables but don't make Subs yet! + subs: &mut Subs, +) -> (Located, Located) { + match def { + Def::Annotation(_loc_pattern, _loc_ann) => { + panic!("TODO canonicalize top-level annotations"); + } + Def::Body(loc_pattern, loc_expr) => { + let variable = subs.mk_flex_var(); + let expected = Expected::NoExpectation(Type::Variable(variable)); + let declared_idents = ImMap::default(); // TODO FIXME infer this from scope arg + let declared_variants = ImMap::default(); // TODO get rid of this + let name: Box = "TODOfixme".into(); + + // Desugar operators (convert them to Apply calls, taking into account + // operator precedence and associativity rules), before doing other canonicalization. + // + // If we did this *during* canonicalization, then each time we + // visited a BinOp node we'd recursively try to apply this to each of its nested + // operators, and then again on *their* nested operators, ultimately applying the + // rules multiple times unnecessarily. + let loc_expr = operator::desugar(arena, &loc_expr); + + // If we're canonicalizing the declaration `foo = ...` inside the `Main` module, + // scope_prefix will be "Main.foo$" and its first closure will be named "Main.foo$0" + let scope_prefix = format!("{}.{}$", home, name).into(); + let mut scope = Scope::new(scope_prefix, declared_idents.clone()); + let mut env = Env::new(home, declared_variants.clone()); + let (loc_expr, _) = canonicalize_expr( + &ImMap::default(), + &mut env, + subs, + &mut scope, + region, + &loc_expr.value, + expected, + ); + + // Exclude the current ident from shadowable_idents; you can't shadow yourself! + // (However, still include it in scope, because you *can* recursively refer to yourself.) + let mut shadowable_idents = scope.idents.clone(); + remove_idents(&loc_pattern.value, &mut shadowable_idents); + + let pattern_var = subs.mk_flex_var(); + let pattern_type = Type::Variable(pattern_var); + let pattern_expected = PExpected::NoExpectation(pattern_type.clone()); + + let mut pattern_state = PatternState { + headers: ImMap::default(), + vars: Vec::with_capacity(1), + constraints: Vec::with_capacity(1), + }; + let loc_pattern = canonicalize_pattern( + &mut env, + &mut pattern_state, + subs, + &mut scope, + PatternType::TopLevelDef, + &loc_pattern.value, + loc_pattern.region, + &mut shadowable_idents, + pattern_expected, + ); + + (loc_pattern, loc_expr) + } + Def::CustomType(_, _) => { + panic!("TODO remove CustomType syntax"); + } + Def::TypeAlias(_, _) => { + panic!("TODO remove TypeAlias syntax"); + } + + // Ignore spaces + Def::SpaceBefore(def, _) | Def::SpaceAfter(def, _) => { + // TODO FIXME performance disaster!!! + canonicalize_def(arena, def.clone(), region, home, scope, subs) + } + } +} + // TODO trim down these arguments #[allow(clippy::too_many_arguments)] pub fn canonicalize_declaration<'a>( diff --git a/src/load/mod.rs b/src/load/mod.rs index b44bf60024..a46c6bf29d 100644 --- a/src/load/mod.rs +++ b/src/load/mod.rs @@ -1,25 +1,28 @@ use crate::can::symbol::Symbol; -use crate::collections::{SendMap, SendSet}; -use crate::ident::UnqualifiedIdent; +use crate::can::expr::Expr; +use crate::can::pattern::Pattern; +use crate::can::canonicalize_module_defs; +use crate::collections::{SendSet, ImMap}; use crate::module::ModuleName; -use crate::parse::ast::{Attempting, Def, ExposesEntry, ImportsEntry, Module}; -use crate::parse::module; +use crate::parse::ast::{self, Attempting, ExposesEntry, ImportsEntry}; +use crate::parse::module::{self, module_defs}; use crate::parse::parser::{Fail, Parser, State}; use crate::region::{Located, Region}; use im::Vector; use bumpalo::Bump; use tokio::fs::read_to_string; +use tokio::sync::mpsc::{self, Sender, Receiver}; use std::io; use std::path::{Path, PathBuf}; -use tokio::prelude::*; +use crate::can::module::Module; +use futures::future::join_all; - -pub struct Loaded<'a> { - pub requested_header: LoadedHeader, - pub dependent_headers: SendMap, LoadedHeader>, - pub defs: SendMap, Result>>, Fail>>, +pub struct Loaded { + pub requested_module: LoadedModule, + pub deps: Deps, } +#[derive(Debug, Clone)] struct Env { pub src_dir: PathBuf } @@ -29,54 +32,62 @@ pub enum BuildProblem<'a> { FileNotFound(&'a Path), } -#[derive(Debug, PartialEq, Eq)] -pub enum LoadedHeader { - Valid { - declared_name: Option>, - deps: SendSet>, - scope: SendMap, (Symbol, Region)>, - bytes_parsed: usize - }, +type Deps = SendSet>; + +#[derive(Debug, PartialEq)] +pub enum LoadedModule { + Valid(Module), FileProblem(io::ErrorKind), ParsingFailed(Fail), } -pub async fn load<'a>(src_dir: PathBuf, filename: PathBuf) -> Loaded<'a> { - let handle = tokio::spawn(async move { - let mut env = Env { - src_dir - }; +pub async fn load<'a>(src_dir: PathBuf, filename: PathBuf) -> Loaded { + let env = Env { src_dir: src_dir.clone() }; + let (tx, mut rx): (Sender, Receiver) = mpsc::channel(1024); - load_filename(&mut env, &filename).await + let main_tx = tx.clone(); + let handle = tokio::spawn(async move { + load_filename(&env, &filename, main_tx).await }); - let requested_header = handle.await; + let requested_module = handle.await.expect("Unable to load requested module."); + let mut other_modules = Vec::new(); + let mut all_deps = SendSet::default(); - panic!("TODO"); + // Get a fresh env, since the previous one has been consumed + let env = Env { src_dir }; + // At first, 1 module is pending (namely the `filename` one). + let mut pending = 1; + while let Some(module_deps) = rx.recv().await { + let deps_to_load = module_deps.relative_complement(all_deps.clone()); -// // TODO parse defs on a different thread, and parse them -// // directly into a Vector rather than into a Vec, so -// // we don't have to reallocate here. -// let defs = match module::module_defs().parse(&arena, state) { -// Ok((defs, _)) => { -// let mut send_vec = Vector::new(); + // We just loaded 1 module, and gained deps_to_load more + pending = pending + deps_to_load.len() - 1; -// for def in defs { -// send_vec.push_back(def); -// } + // Record that these are loaded *before* spawning threads to load them. + all_deps = all_deps.union(deps_to_load.clone()); -// Ok(send_vec) -// }, -// Err((fail, _)) => Err(fail), -// }; + let loaded_modules = join_all(deps_to_load.into_iter().map(|dep|{ + let env = env.clone(); + let tx = tx.clone(); - // Loaded { - // requested_header, - // dependent_headers: env.loaded_headers, - // defs, - // problems: env.problems, - // } + tokio::spawn(async move { + load_module(&env, dep, tx).await + }) + })).await; + + for module in loaded_modules { + other_modules.push(module.expect("Unable to load dependent module")); + } + + // Once we've run out of pending modules to process, we're done! + if pending == 0 { + break; + } + } + + Loaded { requested_module, deps: all_deps } } /// The long-term plan is for the loading process to work like this, starting from main.roc: @@ -124,31 +135,31 @@ pub async fn load<'a>(src_dir: PathBuf, filename: PathBuf) -> Loaded<'a> { /// module's canonicalization. /// /// If a given import has not been loaded yet, load it too. -// fn load_module<'a, 'p>(env: &mut Env<'a, 'p>, module_name: &ModuleName<'a>) -> LoadedHeader<'a> { -// // 1. Convert module_name to filename, using src_dir. -// // 2. Open that file for reading. (If there's a problem, record it and bail.) -// // 3. Read the whole file into a string. (In the future, we can read just the header.) -// // 4. Parse the header. -// // 5. Use the parsed header to load more modules as necessary. -// // 6. Now that all the headers have been parsed, parse the bodies too. -// // 7. Once all the bodies have been parsed, canonicalize beginning with the leaves. +async fn load_module(env: &Env, module_name: Box, tx: Sender) -> LoadedModule { + // 1. Convert module_name to filename, using src_dir. + // 2. Open that file for reading. (If there's a problem, record it and bail.) + // 3. Read the whole file into a string. (In the future, we can read just the header.) + // 4. Parse the header. + // 5. Use the parsed header to load more modules as necessary. + // 6. Now that all the headers have been parsed, parse the bodies too. + // 7. Once all the bodies have been parsed, canonicalize beginning with the leaves. -// let mut filename = PathBuf::new(); + let mut filename = PathBuf::new(); -// filename.push(env.src_dir); + filename.push(env.src_dir.clone()); -// // Convert dots in module name to directories -// for part in module_name.as_str().split('.') { -// filename.push(part); -// } + // Convert dots in module name to directories + for part in module_name.split('.') { + filename.push(part); + } -// // End with .roc -// filename.set_extension("roc"); + // End with .roc + filename.set_extension("roc"); -// load_filename(env, &filename) -// } + load_filename(env, &filename, tx).await +} -async fn load_filename(env: &mut Env, filename: &Path) -> LoadedHeader { +async fn load_filename(env: &Env, filename: &Path, tx: Sender) -> LoadedModule { match read_to_string(filename).await { Ok(src) => { let arena = Bump::new(); @@ -161,49 +172,74 @@ async fn load_filename(env: &mut Env, filename: &Path) -> LoadedHeader { let state = State::new(&src, Attempting::Module); let answer = match module::module().parse(&arena, state) { - Ok((Module::Interface { header }, state)) => { - let declared_name = Some(header.name.value.as_str().into()); + Ok((ast::Module::Interface { header }, state)) => { + let declared_name: Box = header.name.value.as_str().into(); - let mut scope = SendMap::default(); + // TODO check to see if declared_name is consistent with filename. + // If it isn't, report a problem! + + let mut scope = ImMap::default(); let mut deps = SendSet::default(); for loc_entry in header.imports { deps.insert(load_import(env, loc_entry.region, &loc_entry.value, &mut scope)); } - let bytes_parsed = state.bytes_consumed(); + tokio::spawn(async move { + let mut tx = tx; - LoadedHeader::Valid { scope, declared_name, deps, bytes_parsed } + // Send the deps to the main thread for processing, + // then continue on to parsing and canonicalizing defs. + tx.send(deps).await.unwrap(); + }); + + let defs = parse_and_canonicalize_defs(&arena, state, declared_name.clone(), &mut scope); + let module = Module { name: Some(declared_name), defs }; + + LoadedModule::Valid(module) } - Ok((Module::App { header }, state)) => { - // The app module has no declared name. - let declared_name = None; - - let mut scope = SendMap::default(); + Ok((ast::Module::App { header }, state)) => { + let mut scope = ImMap::default(); let mut deps = SendSet::default(); for loc_entry in header.imports { deps.insert(load_import(env, loc_entry.region, &loc_entry.value, &mut scope)); } - let bytes_parsed = state.bytes_consumed(); + tokio::spawn(async move { + let mut tx = tx; - LoadedHeader::Valid { scope, declared_name, deps, bytes_parsed } + // Send the deps to the main thread for processing, + // then continue on to parsing and canonicalizing defs. + tx.send(deps).await.unwrap(); + }); + + // The app module has no declared name. Pass it as "". + let defs = parse_and_canonicalize_defs(&arena, state, "".into(), &mut scope); + let module = Module { name: None, defs }; + + LoadedModule::Valid(module) } - Err((fail, _)) => LoadedHeader::ParsingFailed(fail), + Err((fail, _)) => LoadedModule::ParsingFailed(fail), }; answer } - Err(err) => LoadedHeader::FileProblem(err.kind()), + Err(err) => LoadedModule::FileProblem(err.kind()), } } -fn load_import<'a, 'p, 'out>( - env: &mut Env, +fn parse_and_canonicalize_defs(arena: &Bump, state: State<'_>, home: Box, scope: &mut ImMap, (Symbol, Region)>) -> Vector<(Located, Located)> { + let (parsed_defs, _) = module_defs().parse(arena, state).expect("TODO gracefully handle parse error on module defs"); + + canonicalize_module_defs(arena, parsed_defs, home, scope) +} + +fn load_import( + env: &Env, region: Region, entry: &ImportsEntry<'_>, - scope: &mut SendMap, (Symbol, Region)>, + scope: &mut ImMap, (Symbol, Region)>, ) -> Box { use crate::parse::ast::ImportsEntry::*; @@ -225,7 +261,7 @@ fn load_import<'a, 'p, 'out>( } } -fn expose<'out>( +fn expose( module_name: ModuleName<'_>, entry: &ExposesEntry<'_>, region: Region, @@ -245,30 +281,3 @@ fn expose<'out>( } } } - -#[test] -fn test_tokio() { - test_async(async { - let handle = tokio::spawn(async { - println!("doing some work, asynchronously"); - - // Return a value for the example - "result of the computation" - }); - - // Wait for the spawned task to finish - let res = handle.await; - - println!("got {:?}", res); - }) -} - -fn test_async(future: F) -> F::Output { - use tokio::runtime::Runtime; - - // Create the runtime - let mut rt = Runtime::new().expect("Error initializing Tokio runtime."); - - // Spawn the root task - rt.block_on(future) -} diff --git a/tests/test_load.rs b/tests/test_load.rs index a1617fdceb..4131ea8d7e 100644 --- a/tests/test_load.rs +++ b/tests/test_load.rs @@ -12,86 +12,93 @@ mod helpers; #[cfg(test)] mod test_load { use crate::helpers::{fixtures_dir, im_map_from_pairs, mut_map_from_pairs}; - use bumpalo::Bump; - use roc::can::symbol::Symbol; - use roc::ident::UnqualifiedIdent; - use roc::load::LoadedHeader::*; - use roc::load::{load, LoadedHeader}; - use roc::module::ModuleName; - use roc::region::Region; + use roc::load::load; + // use roc::region::Region; + + fn test_async(future: F) -> F::Output { + use tokio::runtime::Runtime; + + // Create the runtime + let mut rt = Runtime::new().expect("Error initializing Tokio runtime."); + + // Spawn the root task + rt.block_on(future) + } #[test] fn interface_with_deps() { let src_dir = fixtures_dir().join("interface_with_deps"); let filename = src_dir.join("Primary.roc"); - let arena = Bump::new(); - let loaded = load(&arena, &src_dir, &filename); - assert!(loaded.problems.is_empty()); + test_async(async { + let loaded = load(src_dir, filename).await; + }); - let dep1_scope = im_map_from_pairs(vec![( - UnqualifiedIdent::new("foo"), - (Symbol::new("Dep3.Blah.", "foo"), Region::new(2, 2, 26, 29)), - )]); - let dep2_scope = im_map_from_pairs(vec![ - ( - UnqualifiedIdent::new("bar"), - (Symbol::new("Dep3.Blah.", "bar"), Region::new(2, 2, 31, 34)), - ), - ( - UnqualifiedIdent::new("foo"), - (Symbol::new("Dep3.Blah.", "foo"), Region::new(2, 2, 26, 29)), - ), - ]); - let dep3_scope = im_map_from_pairs(vec![]); + // assert!(loaded.problems.is_empty()); - assert_eq!( - loaded.dependent_headers, - mut_map_from_pairs(vec![ - (ModuleName::new("Dep1"), Valid { scope: dep1_scope }), - (ModuleName::new("Dep3.Blah"), Valid { scope: dep3_scope }), - (ModuleName::new("Dep2"), Valid { scope: dep2_scope }), - ]) - ); + // let dep1_scope = im_map_from_pairs(vec![( + // UnqualifiedIdent::new("foo"), + // (Symbol::new("Dep3.Blah.", "foo"), Region::new(2, 2, 26, 29)), + // )]); + // let dep2_scope = im_map_from_pairs(vec![ + // ( + // UnqualifiedIdent::new("bar"), + // (Symbol::new("Dep3.Blah.", "bar"), Region::new(2, 2, 31, 34)), + // ), + // ( + // UnqualifiedIdent::new("foo"), + // (Symbol::new("Dep3.Blah.", "foo"), Region::new(2, 2, 26, 29)), + // ), + // ]); + // let dep3_scope = im_map_from_pairs(vec![]); - assert_eq!(loaded.defs.len(), 4); + // assert_eq!( + // loaded.dependent_headers, + // mut_map_from_pairs(vec![ + // (ModuleName::new("Dep1"), Valid { scope: dep1_scope }), + // (ModuleName::new("Dep3.Blah"), Valid { scope: dep3_scope }), + // (ModuleName::new("Dep2"), Valid { scope: dep2_scope }), + // ]) + // ); - let defs = loaded - .defs - .get(&ModuleName::new("Primary")) - .expect("No defs found for `Primary` module") - .clone() - .expect("Defs failed to parse for `Primary` module"); + // assert_eq!(loaded.defs.len(), 4); - assert_eq!( - dbg!(/* problem: module_defs() only parses 1 module - TODO add parsing unit test for it!*/ defs) - .len(), - 6 - ); + // let defs = loaded + // .defs + // .get(&ModuleName::new("Primary")) + // .expect("No defs found for `Primary` module") + // .clone() + // .expect("Defs failed to parse for `Primary` module"); - match loaded.requested_header { - LoadedHeader::Valid { scope } => assert_eq!( - scope, - im_map_from_pairs(vec![ - ( - UnqualifiedIdent::new("bar"), - (Symbol::new("Dep3.Blah.", "bar"), Region::new(2, 2, 51, 54)), - ), - ( - UnqualifiedIdent::new("foo"), - (Symbol::new("Dep2.", "foo"), Region::new(2, 2, 32, 35)), - ), - ( - UnqualifiedIdent::new("two"), - (Symbol::new("Dep2.", "two"), Region::new(2, 2, 27, 30)), - ), - ]) - ), + // assert_eq!( + // dbg!(/* problem: module_defs() only parses 1 module - TODO add parsing unit test for it!*/ defs) + // .len(), + // 6 + // ); - other => panic!( - "app_header should have been Valid, but instead was: {:?}", - other - ), - }; + // match loaded.requested_header { + // LoadedHeader::Valid { scope } => assert_eq!( + // scope, + // im_map_from_pairs(vec![ + // ( + // UnqualifiedIdent::new("bar"), + // (Symbol::new("Dep3.Blah.", "bar"), Region::new(2, 2, 51, 54)), + // ), + // ( + // UnqualifiedIdent::new("foo"), + // (Symbol::new("Dep2.", "foo"), Region::new(2, 2, 32, 35)), + // ), + // ( + // UnqualifiedIdent::new("two"), + // (Symbol::new("Dep2.", "two"), Region::new(2, 2, 27, 30)), + // ), + // ]) + // ), + + // other => panic!( + // "app_header should have been Valid, but instead was: {:?}", + // other + // ), + // }; } } From 773d304f6b2938c17f803c235fb84ebeb7df52b5 Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Sat, 30 Nov 2019 20:30:21 -0500 Subject: [PATCH 12/20] Update some doc comments --- src/load/mod.rs | 62 +++++++------------------------------------------ 1 file changed, 9 insertions(+), 53 deletions(-) diff --git a/src/load/mod.rs b/src/load/mod.rs index a46c6bf29d..a54ded3ace 100644 --- a/src/load/mod.rs +++ b/src/load/mod.rs @@ -41,6 +41,15 @@ pub enum LoadedModule { ParsingFailed(Fail), } +/// The loading process works like this, starting from the given filename (e.g. "main.roc"): +/// +/// 1. Open the file. +/// 2. Parse the module's header. +/// 3. For each of its imports, send a message on the channel to the coordinator thread, which +/// will repeat this process to load that module - starting with step 1. +/// 4. Add everything we were able to import unqualified to the module's default scope. +/// 5. Parse the module's defs. +/// 6. Canonicalize the module. pub async fn load<'a>(src_dir: PathBuf, filename: PathBuf) -> Loaded { let env = Env { src_dir: src_dir.clone() }; let (tx, mut rx): (Sender, Receiver) = mpsc::channel(1024); @@ -90,60 +99,7 @@ pub async fn load<'a>(src_dir: PathBuf, filename: PathBuf) -> Loaded { Loaded { requested_module, deps: all_deps } } -/// The long-term plan is for the loading process to work like this, starting from main.roc: -/// -/// 1. Open the file. -/// 2. Parse its header. -/// 3. For each of its imports, repeat this process starting with step 1. -/// 4. Once a given import is finished parsing, we can process that import. -/// 5. Processing an import entails checking what we want to import against what it exposes. -/// 6. If anything we want to import unqualified is not exposed, record a problem. -/// 7. Add everything we were able to import unqualified to the module's default scope. -/// 8. Once all imports have been processed for this module, canonicalize it. -/// -/// This would ideally be done using a parallel work-stealing scheduler like tokio_threadpool. -/// However, a prerequisite of this is that we are able to canonicalize in parallel! -/// -/// To canonicalize in parallel, we want to be able to generate Variables in parallel, -/// which currently would require a Mutex on Subs. We can avoid that Mutex in one of two ways. -/// -/// One way would be to give each thread in a thread pool a "starting id" - -/// distributed into (usize::MAX / n) ranges. For example, if there are 2 threads, -/// the first thread gets to start at id 0, and the second thread starts at -/// id (usize::MAX / 2). That way both of them can increment in parallel without colliding. -/// (If we have 1024 threads running at once, on a 64-bit system, we still have -/// over 1 quadrillion Variables per thread. Seems like enough.) -/// However, to support that, we need to change Subs to be able to look up arbitrary IDs, -/// instead of being backed by a flat Vec where each Variable is a direct array index. -/// -/// A strategy I like better, which should be slightly slower for canonicalization -/// (which is likely I/O bound anyway since it'll be happening concurrently with file reads), -/// but *much* faster for unification, is to give each thread a shared AtomicUsize which -/// they each call .fetch_add(1) on to get a fresh ID. Atomic increment is a bit slower than -/// regular increment, but it means afterwards unification (which I'm not yet sure how to -/// parallelize) no longer needs to use a hashing function to get the contents of each ID; -/// the IDs will already correspond directly to array indices like they do in the status quo. -/// -/// Separately, if we use that strategy, there's probably another optimization opportunity: -/// instead of instantiating fresh structs with mk_fresh_var(), ensure that the default of -/// each struct will be all 0s in memory. That way, after we've distributed all the IDs, -/// we can do one single Vec resize (to zeroed memory) and they're all instantly ready to go. -/// -/// Anyway, that'll all take awhile; for now, we'll do this in a synchronous, blocking way. - -/// Resolve a module's list of imports, creating a Scope map for use in the -/// module's canonicalization. -/// -/// If a given import has not been loaded yet, load it too. async fn load_module(env: &Env, module_name: Box, tx: Sender) -> LoadedModule { - // 1. Convert module_name to filename, using src_dir. - // 2. Open that file for reading. (If there's a problem, record it and bail.) - // 3. Read the whole file into a string. (In the future, we can read just the header.) - // 4. Parse the header. - // 5. Use the parsed header to load more modules as necessary. - // 6. Now that all the headers have been parsed, parse the bodies too. - // 7. Once all the bodies have been parsed, canonicalize beginning with the leaves. - let mut filename = PathBuf::new(); filename.push(env.src_dir.clone()); From 0a9abed1d4f587f838ad040938fb5af61a7017a0 Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Sat, 30 Nov 2019 20:31:29 -0500 Subject: [PATCH 13/20] Update Inkwell --- Cargo.lock | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 31b2d1f7b7..71f720a06d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -281,7 +281,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", "synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -403,7 +403,7 @@ dependencies = [ "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -607,7 +607,7 @@ dependencies = [ "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", "unindent 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -931,7 +931,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1314,7 +1314,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1385,7 +1385,7 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1400,7 +1400,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1904,7 +1904,7 @@ dependencies = [ "checksum smallvec 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ecf3b85f68e8abaa7555aa5abdb1153079387e60b718283d732f03897fcfc86" "checksum string 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d24114bfcceb867ca7f71a0d3fe45d45619ec47a6fbfa98cb14e14250bfa5d6d" "checksum syn 0.15.44 (registry+https://github.com/rust-lang/crates.io-index)" = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" -"checksum syn 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "f89693ae015201f8de93fd96bde2d065f8bfc3f97ce006d5bc9f900b97c0c7c0" +"checksum syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "dff0acdb207ae2fe6d5976617f887eb1e35a2ba52c13c7234c790960cdad9238" "checksum synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" "checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" From 3a151b6c249d2fe2779ec3f5912907d64f1b06e3 Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Sat, 30 Nov 2019 20:36:35 -0500 Subject: [PATCH 14/20] wip --- src/can/module.rs | 10 ++++++++++ src/load/mod.rs | 5 +++++ tests/test_load.rs | 4 ++-- 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 src/can/module.rs diff --git a/src/can/module.rs b/src/can/module.rs new file mode 100644 index 0000000000..4efd9234dd --- /dev/null +++ b/src/can/module.rs @@ -0,0 +1,10 @@ +use crate::can::expr::Expr; +use crate::can::pattern::Pattern; +use crate::region::Located; +use im::Vector; + +#[derive(Clone, Debug, PartialEq)] +pub struct Module { + pub name: Option>, + pub defs: Vector<(Located, Located)>, +} diff --git a/src/load/mod.rs b/src/load/mod.rs index a54ded3ace..d9ddeb1909 100644 --- a/src/load/mod.rs +++ b/src/load/mod.rs @@ -75,6 +75,7 @@ pub async fn load<'a>(src_dir: PathBuf, filename: PathBuf) -> Loaded { pending = pending + deps_to_load.len() - 1; // Record that these are loaded *before* spawning threads to load them. + // We don't want to accidentally process them more than once! all_deps = all_deps.union(deps_to_load.clone()); let loaded_modules = join_all(deps_to_load.into_iter().map(|dep|{ @@ -96,6 +97,10 @@ pub async fn load<'a>(src_dir: PathBuf, filename: PathBuf) -> Loaded { } } + println!("= = = = = = = = = = = = = = = = = = = = = = = = finished!"); + println!("\n\nmain module: {:?}", requested_module); + println!("\n\nother modules: {:?}", other_modules); + Loaded { requested_module, deps: all_deps } } diff --git a/tests/test_load.rs b/tests/test_load.rs index 4131ea8d7e..a0a4deefbe 100644 --- a/tests/test_load.rs +++ b/tests/test_load.rs @@ -1,5 +1,5 @@ -#[macro_use] -extern crate pretty_assertions; +// #[macro_use] +// extern crate pretty_assertions; // #[macro_use] // extern crate indoc; From 4de8f43ae15b1a70ba687f914f42c32e8f649efb Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Sat, 30 Nov 2019 21:44:36 -0500 Subject: [PATCH 15/20] Add Debug and Default to VarStore --- src/subs.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/subs.rs b/src/subs.rs index f7b513f864..638dd9a9be 100644 --- a/src/subs.rs +++ b/src/subs.rs @@ -9,6 +9,7 @@ pub struct Subs { utable: UnificationTable>, } +#[derive(Debug, Default)] pub struct VarStore { next: AtomicUsize, } From 0693d2ff5e151c97e0bd7a89748e4cbb3fcad227 Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Sat, 30 Nov 2019 21:44:48 -0500 Subject: [PATCH 16/20] Use VarStore over Subs in loading --- src/can/mod.rs | 18 ++++--- src/load/mod.rs | 123 ++++++++++++++++++++++++++++++++++-------------- 2 files changed, 96 insertions(+), 45 deletions(-) diff --git a/src/can/mod.rs b/src/can/mod.rs index db23da0ad4..ff0f315559 100644 --- a/src/can/mod.rs +++ b/src/can/mod.rs @@ -52,9 +52,8 @@ pub fn canonicalize_module_defs<'a>( loc_defs: bumpalo::collections::Vec<'a, Located>>, home: Box, scope: &mut ImMap, (Symbol, Region)>, + var_store: &VarStore, ) -> Vector<(Located, Located)> { - // TODO FIXME need to remove Subs from this - distribute Variables but don't make Subs yet! - let mut subs = Subs::new(); let mut buf = Vector::new(); for loc_def in loc_defs { @@ -64,7 +63,7 @@ pub fn canonicalize_module_defs<'a>( loc_def.region, home.clone(), scope, - &mut subs, + var_store, )); } @@ -77,15 +76,14 @@ fn canonicalize_def<'a>( region: Region, home: Box, scope: &mut ImMap, (Symbol, Region)>, - // TODO FIXME need to remove Subs from this - distribute Variables but don't make Subs yet! - subs: &mut Subs, + var_store: &VarStore, ) -> (Located, Located) { match def { Def::Annotation(_loc_pattern, _loc_ann) => { panic!("TODO canonicalize top-level annotations"); } Def::Body(loc_pattern, loc_expr) => { - let variable = subs.mk_flex_var(); + let variable = var_store.fresh(); let expected = Expected::NoExpectation(Type::Variable(variable)); let declared_idents = ImMap::default(); // TODO FIXME infer this from scope arg let declared_variants = ImMap::default(); // TODO get rid of this @@ -108,7 +106,7 @@ fn canonicalize_def<'a>( let (loc_expr, _) = canonicalize_expr( &ImMap::default(), &mut env, - subs, + var_store, &mut scope, region, &loc_expr.value, @@ -120,7 +118,7 @@ fn canonicalize_def<'a>( let mut shadowable_idents = scope.idents.clone(); remove_idents(&loc_pattern.value, &mut shadowable_idents); - let pattern_var = subs.mk_flex_var(); + let pattern_var = var_store.fresh(); let pattern_type = Type::Variable(pattern_var); let pattern_expected = PExpected::NoExpectation(pattern_type.clone()); @@ -132,7 +130,7 @@ fn canonicalize_def<'a>( let loc_pattern = canonicalize_pattern( &mut env, &mut pattern_state, - subs, + var_store, &mut scope, PatternType::TopLevelDef, &loc_pattern.value, @@ -153,7 +151,7 @@ fn canonicalize_def<'a>( // Ignore spaces Def::SpaceBefore(def, _) | Def::SpaceAfter(def, _) => { // TODO FIXME performance disaster!!! - canonicalize_def(arena, def.clone(), region, home, scope, subs) + canonicalize_def(arena, def.clone(), region, home, scope, var_store) } } } diff --git a/src/load/mod.rs b/src/load/mod.rs index d9ddeb1909..ead7bb0aaf 100644 --- a/src/load/mod.rs +++ b/src/load/mod.rs @@ -1,30 +1,33 @@ -use crate::can::symbol::Symbol; -use crate::can::expr::Expr; -use crate::can::pattern::Pattern; use crate::can::canonicalize_module_defs; -use crate::collections::{SendSet, ImMap}; +use crate::can::expr::Expr; +use crate::can::module::Module; +use crate::can::pattern::Pattern; +use crate::can::symbol::Symbol; +use crate::collections::{ImMap, SendSet}; use crate::module::ModuleName; use crate::parse::ast::{self, Attempting, ExposesEntry, ImportsEntry}; use crate::parse::module::{self, module_defs}; use crate::parse::parser::{Fail, Parser, State}; use crate::region::{Located, Region}; -use im::Vector; +use crate::subs::VarStore; use bumpalo::Bump; -use tokio::fs::read_to_string; -use tokio::sync::mpsc::{self, Sender, Receiver}; +use futures::future::join_all; +use im::Vector; use std::io; use std::path::{Path, PathBuf}; -use crate::can::module::Module; -use futures::future::join_all; +use std::sync::Arc; +use tokio::fs::read_to_string; +use tokio::sync::mpsc::{self, Receiver, Sender}; pub struct Loaded { pub requested_module: LoadedModule, + pub vars_created: usize, pub deps: Deps, } #[derive(Debug, Clone)] struct Env { - pub src_dir: PathBuf + pub src_dir: PathBuf, } #[derive(Debug)] @@ -51,13 +54,16 @@ pub enum LoadedModule { /// 5. Parse the module's defs. /// 6. Canonicalize the module. pub async fn load<'a>(src_dir: PathBuf, filename: PathBuf) -> Loaded { - let env = Env { src_dir: src_dir.clone() }; + let env = Env { + src_dir: src_dir.clone(), + }; let (tx, mut rx): (Sender, Receiver) = mpsc::channel(1024); + let arc_var_store = Arc::new(VarStore::new()); let main_tx = tx.clone(); - let handle = tokio::spawn(async move { - load_filename(&env, &filename, main_tx).await - }); + let var_store = Arc::clone(&arc_var_store); + let handle = + tokio::spawn(async move { load_filename(&env, &filename, main_tx, &var_store).await }); let requested_module = handle.await.expect("Unable to load requested module."); let mut other_modules = Vec::new(); @@ -78,14 +84,14 @@ pub async fn load<'a>(src_dir: PathBuf, filename: PathBuf) -> Loaded { // We don't want to accidentally process them more than once! all_deps = all_deps.union(deps_to_load.clone()); - let loaded_modules = join_all(deps_to_load.into_iter().map(|dep|{ + let loaded_modules = join_all(deps_to_load.into_iter().map(|dep| { let env = env.clone(); let tx = tx.clone(); + let var_store = Arc::clone(&arc_var_store); - tokio::spawn(async move { - load_module(&env, dep, tx).await - }) - })).await; + tokio::spawn(async move { load_module(&env, dep, tx, &var_store).await }) + })) + .await; for module in loaded_modules { other_modules.push(module.expect("Unable to load dependent module")); @@ -97,14 +103,23 @@ pub async fn load<'a>(src_dir: PathBuf, filename: PathBuf) -> Loaded { } } - println!("= = = = = = = = = = = = = = = = = = = = = = = = finished!"); - println!("\n\nmain module: {:?}", requested_module); - println!("\n\nother modules: {:?}", other_modules); + let vars_created: usize = Arc::try_unwrap(arc_var_store) + .expect("TODO better error for Arc being unable to unwrap") + .into(); - Loaded { requested_module, deps: all_deps } + Loaded { + requested_module, + deps: all_deps, + vars_created, + } } -async fn load_module(env: &Env, module_name: Box, tx: Sender) -> LoadedModule { +async fn load_module( + env: &Env, + module_name: Box, + tx: Sender, + var_store: &VarStore, +) -> LoadedModule { let mut filename = PathBuf::new(); filename.push(env.src_dir.clone()); @@ -117,10 +132,15 @@ async fn load_module(env: &Env, module_name: Box, tx: Sender) -> Load // End with .roc filename.set_extension("roc"); - load_filename(env, &filename, tx).await + load_filename(env, &filename, tx, var_store).await } -async fn load_filename(env: &Env, filename: &Path, tx: Sender) -> LoadedModule { +async fn load_filename( + env: &Env, + filename: &Path, + tx: Sender, + var_store: &VarStore, +) -> LoadedModule { match read_to_string(filename).await { Ok(src) => { let arena = Bump::new(); @@ -143,7 +163,12 @@ async fn load_filename(env: &Env, filename: &Path, tx: Sender) -> LoadedMo let mut deps = SendSet::default(); for loc_entry in header.imports { - deps.insert(load_import(env, loc_entry.region, &loc_entry.value, &mut scope)); + deps.insert(load_import( + env, + loc_entry.region, + &loc_entry.value, + &mut scope, + )); } tokio::spawn(async move { @@ -154,8 +179,17 @@ async fn load_filename(env: &Env, filename: &Path, tx: Sender) -> LoadedMo tx.send(deps).await.unwrap(); }); - let defs = parse_and_canonicalize_defs(&arena, state, declared_name.clone(), &mut scope); - let module = Module { name: Some(declared_name), defs }; + let defs = parse_and_canonicalize_defs( + &arena, + state, + declared_name.clone(), + &mut scope, + var_store, + ); + let module = Module { + name: Some(declared_name), + defs, + }; LoadedModule::Valid(module) } @@ -164,7 +198,12 @@ async fn load_filename(env: &Env, filename: &Path, tx: Sender) -> LoadedMo let mut deps = SendSet::default(); for loc_entry in header.imports { - deps.insert(load_import(env, loc_entry.region, &loc_entry.value, &mut scope)); + deps.insert(load_import( + env, + loc_entry.region, + &loc_entry.value, + &mut scope, + )); } tokio::spawn(async move { @@ -176,7 +215,13 @@ async fn load_filename(env: &Env, filename: &Path, tx: Sender) -> LoadedMo }); // The app module has no declared name. Pass it as "". - let defs = parse_and_canonicalize_defs(&arena, state, "".into(), &mut scope); + let defs = parse_and_canonicalize_defs( + &arena, + state, + "".into(), + &mut scope, + var_store, + ); let module = Module { name: None, defs }; LoadedModule::Valid(module) @@ -190,10 +235,18 @@ async fn load_filename(env: &Env, filename: &Path, tx: Sender) -> LoadedMo } } -fn parse_and_canonicalize_defs(arena: &Bump, state: State<'_>, home: Box, scope: &mut ImMap, (Symbol, Region)>) -> Vector<(Located, Located)> { - let (parsed_defs, _) = module_defs().parse(arena, state).expect("TODO gracefully handle parse error on module defs"); +fn parse_and_canonicalize_defs( + arena: &Bump, + state: State<'_>, + home: Box, + scope: &mut ImMap, (Symbol, Region)>, + var_store: &VarStore, +) -> Vector<(Located, Located)> { + let (parsed_defs, _) = module_defs() + .parse(arena, state) + .expect("TODO gracefully handle parse error on module defs"); - canonicalize_module_defs(arena, parsed_defs, home, scope) + canonicalize_module_defs(arena, parsed_defs, home, scope, var_store) } fn load_import( @@ -226,7 +279,7 @@ fn expose( module_name: ModuleName<'_>, entry: &ExposesEntry<'_>, region: Region, -)-> (Box, (Symbol, Region)){ +) -> (Box, (Symbol, Region)) { use crate::parse::ast::ExposesEntry::*; match entry { From 4e6fcd0eb0e14332fd3f3eba48b3d64054405ac1 Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Sat, 30 Nov 2019 22:14:23 -0500 Subject: [PATCH 17/20] Add Debug to Loaded --- src/load/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/load/mod.rs b/src/load/mod.rs index ead7bb0aaf..37a7f09cca 100644 --- a/src/load/mod.rs +++ b/src/load/mod.rs @@ -19,6 +19,7 @@ use std::sync::Arc; use tokio::fs::read_to_string; use tokio::sync::mpsc::{self, Receiver, Sender}; +#[derive(Debug)] pub struct Loaded { pub requested_module: LoadedModule, pub vars_created: usize, From fde17405f526efa6fb69651e2b85098c149bed6c Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Sat, 30 Nov 2019 22:14:31 -0500 Subject: [PATCH 18/20] Make send_set_from helper --- tests/helpers/mod.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs index 707bbdbc6a..9e7dafd341 100644 --- a/tests/helpers/mod.rs +++ b/tests/helpers/mod.rs @@ -6,7 +6,7 @@ use roc::can::expr::Expr; use roc::can::problem::Problem; use roc::can::symbol::Symbol; use roc::can::Output; -use roc::collections::{ImMap, MutMap}; +use roc::collections::{ImMap, MutMap, SendSet}; use roc::ident::Ident; use roc::parse; use roc::parse::ast::{self, Attempting}; @@ -110,6 +110,21 @@ where answer } +#[allow(dead_code)] +pub fn send_set_from(elems: I) -> SendSet +where + I: IntoIterator, + V: Hash + Eq + Clone, +{ + let mut answer = SendSet::default(); + + for elem in elems { + answer.insert(elem); + } + + answer +} + #[allow(dead_code)] pub fn fixtures_dir<'a>() -> PathBuf { Path::new("tests").join("fixtures").join("build") From 9a0ae83e0d2807c29b2bdfe3e60d6d12fff2c341 Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Sat, 30 Nov 2019 22:14:50 -0500 Subject: [PATCH 19/20] Update interface_with_deps test --- tests/test_load.rs | 92 +++++++++++----------------------------------- 1 file changed, 22 insertions(+), 70 deletions(-) diff --git a/tests/test_load.rs b/tests/test_load.rs index a0a4deefbe..0014aeb0ef 100644 --- a/tests/test_load.rs +++ b/tests/test_load.rs @@ -11,9 +11,8 @@ mod helpers; #[cfg(test)] mod test_load { - use crate::helpers::{fixtures_dir, im_map_from_pairs, mut_map_from_pairs}; - use roc::load::load; - // use roc::region::Region; + use crate::helpers::{fixtures_dir, send_set_from}; + use roc::load::{load, LoadedModule}; fn test_async(future: F) -> F::Output { use tokio::runtime::Runtime; @@ -32,73 +31,26 @@ mod test_load { test_async(async { let loaded = load(src_dir, filename).await; + + let module = match loaded.requested_module { + LoadedModule::Valid(module) => module, + LoadedModule::FileProblem(err) => panic!( + "requested_module failed to load with FileProblem: {:?}", + err + ), + LoadedModule::ParsingFailed(fail) => panic!( + "requested_module failed to load with ParsingFailed: {:?}", + fail + ), + }; + + assert_eq!(module.name, Some("Primary".into())); + assert_eq!(module.defs.len(), 6); + + assert_eq!( + loaded.deps, + send_set_from(vec!["Dep1".into(), "Dep2".into(), "Dep3.Blah".into()]) + ); }); - - // assert!(loaded.problems.is_empty()); - - // let dep1_scope = im_map_from_pairs(vec![( - // UnqualifiedIdent::new("foo"), - // (Symbol::new("Dep3.Blah.", "foo"), Region::new(2, 2, 26, 29)), - // )]); - // let dep2_scope = im_map_from_pairs(vec![ - // ( - // UnqualifiedIdent::new("bar"), - // (Symbol::new("Dep3.Blah.", "bar"), Region::new(2, 2, 31, 34)), - // ), - // ( - // UnqualifiedIdent::new("foo"), - // (Symbol::new("Dep3.Blah.", "foo"), Region::new(2, 2, 26, 29)), - // ), - // ]); - // let dep3_scope = im_map_from_pairs(vec![]); - - // assert_eq!( - // loaded.dependent_headers, - // mut_map_from_pairs(vec![ - // (ModuleName::new("Dep1"), Valid { scope: dep1_scope }), - // (ModuleName::new("Dep3.Blah"), Valid { scope: dep3_scope }), - // (ModuleName::new("Dep2"), Valid { scope: dep2_scope }), - // ]) - // ); - - // assert_eq!(loaded.defs.len(), 4); - - // let defs = loaded - // .defs - // .get(&ModuleName::new("Primary")) - // .expect("No defs found for `Primary` module") - // .clone() - // .expect("Defs failed to parse for `Primary` module"); - - // assert_eq!( - // dbg!(/* problem: module_defs() only parses 1 module - TODO add parsing unit test for it!*/ defs) - // .len(), - // 6 - // ); - - // match loaded.requested_header { - // LoadedHeader::Valid { scope } => assert_eq!( - // scope, - // im_map_from_pairs(vec![ - // ( - // UnqualifiedIdent::new("bar"), - // (Symbol::new("Dep3.Blah.", "bar"), Region::new(2, 2, 51, 54)), - // ), - // ( - // UnqualifiedIdent::new("foo"), - // (Symbol::new("Dep2.", "foo"), Region::new(2, 2, 32, 35)), - // ), - // ( - // UnqualifiedIdent::new("two"), - // (Symbol::new("Dep2.", "two"), Region::new(2, 2, 27, 30)), - // ), - // ]) - // ), - - // other => panic!( - // "app_header should have been Valid, but instead was: {:?}", - // other - // ), - // }; } } From 2ad1520eb17c476dc2d0b929aea8f59901d3e169 Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Sat, 30 Nov 2019 22:30:55 -0500 Subject: [PATCH 20/20] Satisfy clippy --- src/load/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/load/mod.rs b/src/load/mod.rs index 37a7f09cca..d842173d3a 100644 --- a/src/load/mod.rs +++ b/src/load/mod.rs @@ -153,6 +153,10 @@ async fn load_filename( // in the arena after read_to_string completes. let state = State::new(&src, Attempting::Module); + // TODO figure out if there's a way to address this clippy error + // without introducing a borrow error. ("let and return" is literally + // what the borrow checker suggested using here to fix the problem, so...) + #[allow(clippy::let_and_return)] let answer = match module::module().parse(&arena, state) { Ok((ast::Module::Interface { header }, state)) => { let declared_name: Box = header.name.value.as_str().into();