diff --git a/src/dreammaker/constants.rs b/src/dreammaker/constants.rs index 13739469..245325ac 100644 --- a/src/dreammaker/constants.rs +++ b/src/dreammaker/constants.rs @@ -394,11 +394,12 @@ impl fmt::Display for ConstFn { pub fn evaluate_str(location: Location, input: &[u8]) -> Result { use super::lexer::{Lexer, from_utf8_or_latin1_borrowed}; - let mut bytes = input.iter().map(|&x| Ok(x)); let ctx = Context::default(); - let expr = crate::parser::parse_expression(&ctx, location, Lexer::new(&ctx, location.file, &mut bytes))?; - if bytes.next().is_some() { - return Err(DMError::new(location, format!("leftover: {:?} {}", from_utf8_or_latin1_borrowed(&input), bytes.len()))); + let mut lexer = Lexer::new(&ctx, location.file, input); + let expr = crate::parser::parse_expression(&ctx, location, &mut lexer)?; + let leftover = lexer.remaining(); + if !leftover.is_empty() { + return Err(DMError::new(location, format!("leftover: {:?} {}", from_utf8_or_latin1_borrowed(&input), leftover.len()))); } expr.simple_evaluate(location) } diff --git a/src/dreammaker/lexer.rs b/src/dreammaker/lexer.rs index c28cff52..62f30820 100644 --- a/src/dreammaker/lexer.rs +++ b/src/dreammaker/lexer.rs @@ -1,5 +1,5 @@ //! The lexer/tokenizer. -use std::io; +use std::io::Read; use std::str::FromStr; use std::fmt; use std::borrow::Cow; @@ -469,25 +469,61 @@ enum Directive { Stringy, } +fn buffer_read(file: FileId, mut read: R) -> Result, DMError> { + let mut buffer = Vec::new(); + + if let Err(error) = read.read_to_end(&mut buffer) { + let mut tracker = LocationTracker::new(file, buffer.as_slice().into()); + tracker.by_ref().count(); + return Err(DMError::new(tracker.location(), "i/o error reading file").with_cause(error)); + } + + Ok(buffer) +} + +/// Attempt to read an entire file into memory, returning a line and column if +/// an I/O error occurs. +pub fn buffer_file(file: FileId, path: &std::path::Path) -> Result, DMError> { + let mut buffer = match std::fs::metadata(path) { + Ok(metadata) => Vec::with_capacity(metadata.len() as usize), + Err(_) => Vec::new(), + }; + + let mut read = match std::fs::File::open(path) { + Ok(read) => read, + Err(error) => return Err(DMError::new(Location { file, line: 1, column: 1 }, "i/o error opening file").with_cause(error)), + }; + + if let Err(error) = read.read_to_end(&mut buffer) { + let mut tracker = LocationTracker::new(file, buffer.as_slice().into()); + tracker.by_ref().count(); + return Err(DMError::new(tracker.location(), "i/o error reading file").with_cause(error)); + } + + Ok(buffer) +} + /// A wrapper for an input stream which tracks line and column numbers. /// /// All characters, including tabs, are considered to occupy one column /// regardless of position. /// /// `io::Error`s are converted to `DMError`s which include the location. -pub struct LocationTracker { - inner: I, +pub struct LocationTracker<'a> { + inner: Cow<'a, [u8]>, + offset: usize, /// The location of the last character returned by `next()`. location: Location, at_line_end: bool, } -impl LocationTracker { - pub fn new(file_number: FileId, inner: I) -> LocationTracker { +impl<'a> LocationTracker<'a> { + pub fn new(file: FileId, inner: Cow<'a, [u8]>) -> LocationTracker<'a> { LocationTracker { inner, + offset: 0, location: Location { - file: file_number, + file, line: 0, column: 0, }, @@ -498,22 +534,27 @@ impl LocationTracker { pub fn location(&self) -> Location { self.location } + + pub fn remaining(&self) -> &[u8] { + &self.inner[self.offset..] + } } -impl fmt::Debug for LocationTracker { +impl<'a> fmt::Debug for LocationTracker<'a> { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { fmt.debug_struct("LocationTracker") // inner omitted + .field("offset", &self.offset) .field("location", &self.location) .field("at_line_end", &self.at_line_end) .finish() } } -impl>> Iterator for LocationTracker { - type Item = Result; +impl<'a> Iterator for LocationTracker<'a> { + type Item = u8; - fn next(&mut self) -> Option> { + fn next(&mut self) -> Option { if self.at_line_end { self.at_line_end = false; match self.location.line.checked_add(1) { @@ -523,27 +564,27 @@ impl>> Iterator for LocationTracker { self.location.column = 0; } - match self.inner.next() { - None => None, - Some(Ok(ch)) => { - if ch == b'\n' { - self.at_line_end = true; - } - match self.location.column.checked_add(1) { - Some(new) => self.location.column = new, - None => panic!("per-line column limit of {} exceeded", self.location.column), - } - Some(Ok(ch)) - } - Some(Err(e)) => Some(Err(DMError::new(self.location, "i/o error").with_cause(e))), + let ch = match self.inner.get(self.offset) { + Some(&ch) => ch, + None => return None, + }; + self.offset += 1; + + if ch == b'\n' { + self.at_line_end = true; } + match self.location.column.checked_add(1) { + Some(new) => self.location.column = new, + None => panic!("per-line column limit of {} exceeded", self.location.column), + } + Some(ch) } } /// The lexer, which serves as a source of tokens through iteration. -pub struct Lexer<'ctx, I> { +pub struct Lexer<'ctx> { context: &'ctx Context, - input: LocationTracker, + input: LocationTracker<'ctx>, next: Option, final_newline: bool, at_line_head: bool, @@ -552,7 +593,7 @@ pub struct Lexer<'ctx, I> { interp_stack: Vec, } -impl<'ctx, I> fmt::Debug for Lexer<'ctx, I> { +impl<'ctx> fmt::Debug for Lexer<'ctx> { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { fmt.debug_struct("Lexer") .field("context", self.context) @@ -566,26 +607,19 @@ impl<'ctx, I> fmt::Debug for Lexer<'ctx, I> { } } -impl<'ctx, I: Iterator>> HasLocation for Lexer<'ctx, I> { +impl<'ctx> HasLocation for Lexer<'ctx> { #[inline] fn location(&self) -> Location { self.input.location } } -impl<'ctx, R: io::Read> Lexer<'ctx, io::Bytes> { - /// Create a new lexer from a reader. - pub fn from_read(context: &'ctx Context, file_number: FileId, source: R) -> Lexer> { - Lexer::new(context, file_number, source.bytes()) - } -} - -impl<'ctx, I: Iterator>> Lexer<'ctx, I> { +impl<'ctx> Lexer<'ctx> { /// Create a new lexer from a byte stream. - pub fn new(context: &'ctx Context, file_number: FileId, input: I) -> Lexer { + pub fn new>>(context: &'ctx Context, file_number: FileId, input: I) -> Self { Lexer { context, - input: LocationTracker::new(file_number, input), + input: LocationTracker::new(file_number, input.into()), next: None, final_newline: false, at_line_head: true, @@ -595,6 +629,20 @@ impl<'ctx, I: Iterator>> Lexer<'ctx, I> { } } + /// Create a new lexer from a reader. + pub fn from_read(context: &'ctx Context, file: FileId, read: R) -> Result { + Ok(Lexer::new(context, file, buffer_read(file, read)?)) + } + + /// Create a new lexer from a reader. + pub fn from_file(context: &'ctx Context, file: FileId, path: &std::path::Path) -> Result { + Ok(Lexer::new(context, file, buffer_file(file, path)?)) + } + + pub fn remaining(&self) -> &[u8] { + self.input.remaining() + } + fn next(&mut self) -> Option { if let Some(next) = self.next.take() { return Some(next); @@ -608,17 +656,12 @@ impl<'ctx, I: Iterator>> Lexer<'ctx, I> { } match result { None => None, - Some(Ok(ch)) => { + Some(ch) => { if ch != b'\t' && ch != b' ' { self.at_line_head = false; } Some(ch) } - Some(Err(err)) => { - // I/O error is effectively EOF. - self.context.register_error(err); - None - } } } @@ -1049,7 +1092,7 @@ impl<'ctx, I: Iterator>> Lexer<'ctx, I> { } } -impl<'ctx, I: Iterator>> Iterator for Lexer<'ctx, I> { +impl<'ctx> Iterator for Lexer<'ctx> { type Item = LocatedToken; fn next(&mut self) -> Option { diff --git a/src/dreammaker/lib.rs b/src/dreammaker/lib.rs index 37043ca3..99286bf1 100644 --- a/src/dreammaker/lib.rs +++ b/src/dreammaker/lib.rs @@ -50,7 +50,7 @@ impl Context { /// Will only return failure on an `io::Error`. Compilation failures will /// return a best-effort parse. Call `print_all_errors` to pretty-print /// errors to standard error. - pub fn parse_environment(&self, dme: &Path) -> io::Result { + pub fn parse_environment(&self, dme: &Path) -> Result { Ok(parser::parse(self, indents::IndentProcessor::new(self, preprocessor::Preprocessor::new(self, dme.to_owned())? diff --git a/src/dreammaker/preprocessor.rs b/src/dreammaker/preprocessor.rs index 48f79451..5907d57b 100644 --- a/src/dreammaker/preprocessor.rs +++ b/src/dreammaker/preprocessor.rs @@ -246,7 +246,7 @@ enum Include<'ctx> { File { path: PathBuf, file: FileId, - lexer: Lexer<'ctx, io::Bytes>>, + lexer: Lexer<'ctx>, }, Expansion { name: String, @@ -256,11 +256,20 @@ enum Include<'ctx> { } impl<'ctx> Include<'ctx> { - fn from_read(context: &'ctx Context, path: PathBuf, read: Box) -> Include { + fn from_path(context: &'ctx Context, path: PathBuf) -> Result, DMError> { + let idx = context.register_file(&path); + Ok(Include::File { + file: idx, + lexer: Lexer::from_file(context, idx, &path)?, + path, + }) + } + + fn from_buffer(context: &'ctx Context, path: PathBuf, buffer: Cow<'ctx, [u8]>) -> Include<'ctx> { let idx = context.register_file(&path); Include::File { file: idx, - lexer: Lexer::from_read(context, idx, read), + lexer: Lexer::new(context, idx, buffer), path, } } @@ -387,26 +396,12 @@ impl<'ctx> HasLocation for Preprocessor<'ctx> { } } -fn buffer_file(path: &Path) -> io::Result> { - use std::io::Read; - - let mut buffer = if let Ok(metadata) = std::fs::metadata(path) { - Vec::with_capacity(metadata.len() as usize) - } else { - Vec::new() - }; - let mut file = File::open(path)?; - file.read_to_end(&mut buffer)?; - Ok(buffer) -} - impl<'ctx> Preprocessor<'ctx> { /// Create a new preprocessor from the given Context and environment file. - pub fn new(context: &'ctx Context, env_file: PathBuf) -> io::Result { + pub fn new(context: &'ctx Context, env_file: PathBuf) -> Result { // Buffer the entire environment file. Large environments take a while // to load and locking it for the whole time is somewhat inconvenient. - let buffer = buffer_file(&env_file)?; - let include = Include::from_read(context, env_file.clone(), Box::new(io::Cursor::new(buffer))); + let include = Include::from_path(context, env_file.clone())?; Ok(Preprocessor { context, @@ -436,7 +431,7 @@ impl<'ctx> Preprocessor<'ctx> { Cow::Borrowed(s) => Cow::Borrowed(s.as_bytes()), Cow::Owned(s) => Cow::Owned(s.into_bytes()), }; - let include = Include::from_read(context, env_file.clone(), Box::new(io::Cursor::new(cow_u8))); + let include = Include::from_buffer(context, env_file.clone(), cow_u8); Preprocessor { context, env_file, @@ -499,14 +494,14 @@ impl<'ctx> Preprocessor<'ctx> { */ /// Push a DM file to the top of this preprocessor's stack. - pub fn push_file(&mut self, path: PathBuf, read: R) -> FileId { + pub fn push_file(&mut self, path: PathBuf, read: R) -> Result { let idx = self.context.register_file(&path); self.include_stack.stack.push(Include::File { - lexer: Lexer::from_read(self.context, idx, Box::new(read)), + lexer: Lexer::from_read(self.context, idx, read)?, file: idx, path, }); - idx + Ok(idx) } /// Enable source file annotations. @@ -642,7 +637,7 @@ impl<'ctx> Preprocessor<'ctx> { Ok(Include::File { path, file: file_id, - lexer: Lexer::from_read(&self.context, file_id, Box::new(read)), + lexer: Lexer::from_read(&self.context, file_id, read)?, }) } } diff --git a/src/dreammaker/tests/annotation_tests.rs b/src/dreammaker/tests/annotation_tests.rs index 12a08bbc..f26a1ddf 100644 --- a/src/dreammaker/tests/annotation_tests.rs +++ b/src/dreammaker/tests/annotation_tests.rs @@ -22,7 +22,7 @@ fn annotation_basic() { "##.trim(); let context = Default::default(); - let lexer = Lexer::new(&context, Default::default(), code.bytes().map(Ok)); + let lexer = Lexer::new(&context, Default::default(), code.as_bytes()); let indent = IndentProcessor::new(&context, lexer); let mut annotations = AnnotationTree::default(); Parser::new(&context, indent).parse_annotations_only(&mut annotations); diff --git a/src/dreammaker/tests/expression_tests.rs b/src/dreammaker/tests/expression_tests.rs index e6001058..644f079c 100644 --- a/src/dreammaker/tests/expression_tests.rs +++ b/src/dreammaker/tests/expression_tests.rs @@ -6,7 +6,7 @@ use dm::ast::*; fn parse_expr(f: &str) -> Expression { let context = Default::default(); - let lexer = Lexer::new(&context, Default::default(), f.bytes().map(Ok)); + let lexer = Lexer::new(&context, Default::default(), f.as_bytes()); let result = parse_expression(&context, Default::default(), lexer).expect("failed to parse expression"); context.assert_success(); result diff --git a/src/dreammaker/tests/lexer_tests.rs b/src/dreammaker/tests/lexer_tests.rs index 7937381b..59232298 100644 --- a/src/dreammaker/tests/lexer_tests.rs +++ b/src/dreammaker/tests/lexer_tests.rs @@ -6,7 +6,7 @@ use dm::lexer::Punctuation::*; fn lex(f: &str) -> Vec { let context = Default::default(); - let result = Lexer::new(&context, Default::default(), f.bytes().map(Ok)) + let result = Lexer::new(&context, Default::default(), f.as_bytes()) .map(|t| t.token) .collect(); context.assert_success(); diff --git a/src/dreammaker/tests/location_tests.rs b/src/dreammaker/tests/location_tests.rs index abcf6533..912da1b6 100644 --- a/src/dreammaker/tests/location_tests.rs +++ b/src/dreammaker/tests/location_tests.rs @@ -18,7 +18,7 @@ fn simple_location_test() { "##.trim(); let context = Default::default(); - let located_tokens: Vec<_> = Lexer::new(&context, Default::default(), code.bytes().map(Ok)).collect(); + let located_tokens: Vec<_> = Lexer::new(&context, Default::default(), code.as_bytes()).collect(); context.assert_success(); assert_eq!(located_tokens[0].location.line, 1); diff --git a/src/langserver/main.rs b/src/langserver/main.rs index fd0d7f87..69ca6bee 100644 --- a/src/langserver/main.rs +++ b/src/langserver/main.rs @@ -471,7 +471,7 @@ impl<'a> Engine<'a> { None => (FileId::default(), defines.branch_at_end(&self.context)), }; let contents = self.docs.read(url).map_err(invalid_request)?; - let file_id = preprocessor.push_file(stripped.to_owned(), contents); + let file_id = preprocessor.push_file(stripped.to_owned(), contents).map_err(invalid_request)?; preprocessor.enable_annotations(); let mut annotations = AnnotationTree::default(); { diff --git a/src/tools/dmm.rs b/src/tools/dmm.rs index 18ea6693..cb965421 100644 --- a/src/tools/dmm.rs +++ b/src/tools/dmm.rs @@ -7,7 +7,7 @@ use std::fmt; use ndarray::{self, Array3, Axis}; use linked_hash_map::LinkedHashMap; -use dm::{DMError, Location}; +use dm::DMError; use dm::constants::Constant; use crate::dmi::Dir; @@ -158,9 +158,7 @@ impl Map { dictionary: Default::default(), grid: Array3::default((1, 1, 1)), }; - read::parse_map(&mut map, File::open(path).map_err(|e| { - DMError::new(Location::default(), "i/o error").with_cause(e) - })?)?; + read::parse_map(&mut map, path)?; Ok(map) } diff --git a/src/tools/dmm/read.rs b/src/tools/dmm/read.rs index 928b23b1..771c7c51 100644 --- a/src/tools/dmm/read.rs +++ b/src/tools/dmm/read.rs @@ -1,7 +1,5 @@ //! Map parser, supporting standard DMM or TGM-format files. use std::collections::BTreeMap; -use std::fs::File; -use std::io::{Read, BufReader}; use std::cmp::max; use ndarray::Array3; @@ -16,8 +14,9 @@ fn take(t: &mut T) -> T { std::mem::replace(t, T::default()) } -pub fn parse_map(map: &mut Map, f: File) -> Result<(), DMError> { - let mut chars = LocationTracker::new(Default::default(), BufReader::new(f).bytes()); +pub fn parse_map(map: &mut Map, path: &std::path::Path) -> Result<(), DMError> { + let file_id = Default::default(); + let mut chars = LocationTracker::new(file_id, dm::lexer::buffer_file(file_id, path)?.into()); let mut in_comment_line = false; let mut comment_trigger = false; @@ -39,7 +38,6 @@ pub fn parse_map(map: &mut Map, f: File) -> Result<(), DMError> { let mut skip_whitespace = false; while let Some(ch) = chars.next() { - let ch = ch?; if ch == b'\n' || ch == b'\r' { in_comment_line = false; comment_trigger = false; @@ -178,7 +176,6 @@ pub fn parse_map(map: &mut Map, f: File) -> Result<(), DMError> { let mut adjust_y = true; while let Some(ch) = chars.next() { - let ch = ch?; if in_coord_block { if ch == b',' { if reading_coord == Coord::X {