Use memmap for reading files

This commit is contained in:
Richard Feldman 2020-07-25 21:19:47 -04:00
parent baa0a5030e
commit 15f087c93e
3 changed files with 72 additions and 44 deletions

1
Cargo.lock generated
View file

@ -2170,6 +2170,7 @@ dependencies = [
"indoc", "indoc",
"inlinable_string", "inlinable_string",
"maplit", "maplit",
"memmap",
"pretty_assertions", "pretty_assertions",
"quickcheck", "quickcheck",
"quickcheck_macros", "quickcheck_macros",

View file

@ -19,6 +19,7 @@ roc_parse = { path = "../parse" }
roc_solve = { path = "../solve" } roc_solve = { path = "../solve" }
bumpalo = { version = "3.2", features = ["collections"] } bumpalo = { version = "3.2", features = ["collections"] }
inlinable_string = "0.1" inlinable_string = "0.1"
memmap = "0.7"
tokio = { version = "0.2", features = ["blocking", "fs", "sync", "rt-threaded"] } tokio = { version = "0.2", features = ["blocking", "fs", "sync", "rt-threaded"] }
[dev-dependencies] [dev-dependencies]

View file

@ -1,4 +1,5 @@
use bumpalo::Bump; use bumpalo::Bump;
use memmap::MmapOptions;
use roc_builtins::std::{Mode, StdLib}; use roc_builtins::std::{Mode, StdLib};
use roc_can::constraint::Constraint; use roc_can::constraint::Constraint;
use roc_can::def::Declaration; use roc_can::def::Declaration;
@ -19,9 +20,10 @@ use roc_solve::solve;
use roc_types::solved_types::Solved; use roc_types::solved_types::Solved;
use roc_types::subs::{Subs, VarStore, Variable}; use roc_types::subs::{Subs, VarStore, Variable};
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet};
use std::fs::read_to_string; use std::fs::File;
use std::io; use std::io;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::str::from_utf8_unchecked;
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use tokio::sync::mpsc; use tokio::sync::mpsc;
use tokio::task::spawn_blocking; use tokio::task::spawn_blocking;
@ -526,16 +528,14 @@ fn load_module(
load_filename(filename, msg_tx, module_ids) load_filename(filename, msg_tx, module_ids)
} }
/// Load a module by its filename fn parse_src(
fn load_filename(
filename: PathBuf, filename: PathBuf,
msg_tx: MsgSender, msg_tx: MsgSender,
module_ids: SharedModules<'_, '_>, module_ids: SharedModules<'_, '_>,
src: &str
) -> Result<ModuleId, LoadingProblem> { ) -> Result<ModuleId, LoadingProblem> {
match read_to_string(&filename) { let state = State::new(src, Attempting::Module);
Ok(src) => {
let arena = Bump::new(); let arena = Bump::new();
let state = State::new(&src, Attempting::Module);
// TODO figure out if there's a way to address this clippy error // TODO figure out if there's a way to address this clippy error
// without introducing a borrow error. ("let and return" is literally // without introducing a borrow error. ("let and return" is literally
@ -577,6 +577,32 @@ fn load_filename(
}; };
answer answer
}
/// Load a module by its filename
///
/// This has two unsafe calls:
///
/// * memory map the filename instead of doing a buffered read
/// * assume the contents of the file are valid utf8
fn load_filename(
filename: PathBuf,
msg_tx: MsgSender,
module_ids: SharedModules<'_, '_>,
) -> Result<ModuleId, LoadingProblem> {
match File::open(&filename) {
Ok(file) => {
match unsafe { MmapOptions::new().map(&file) } {
Ok(mmap) => {
let src = unsafe { from_utf8_unchecked(mmap.as_ref()) };
parse_src(filename, msg_tx, module_ids, src)
}
Err(err) => Err(LoadingProblem::FileProblem {
filename,
error: err.kind(),
}),
}
} }
Err(err) => Err(LoadingProblem::FileProblem { Err(err) => Err(LoadingProblem::FileProblem {
filename, filename,