language-servers/roc
1
0
Fork 0
mirror of https://github.com/roc-lang/roc.git synced 2025-09-29 06:44:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

WIP

Browse source
This commit is contained in:
Folkert 2021-02-03 20:21:12 +01:00
parent 7b4378219e
commit d211ca7e61
30 changed files with 679 additions and 614 deletions
Download patch file Download diff file Expand all files Collapse all files

1
compiler/parse/src/ast.rs
View file

@ -611,6 +611,7 @@ pub enum Attempting {
TypeVariable,
WhenCondition,
WhenBranch,
TODO,
}
impl<'a> Expr<'a> {

66
compiler/parse/src/blankspace.rs
View file

@ -227,9 +227,9 @@ enum LineState {
pub fn line_comment<'a>() -> impl Parser<'a, &'a str> {
then(
and!(ascii_char(b'#'), optional(ascii_string("# "))),
|_arena: &'a Bump, state: State<'a>, _, (_, opt_doc)| {
|arena: &'a Bump, state: State<'a>, _, (_, opt_doc)| {
if opt_doc != None {
return Err(unexpected(3, state, Attempting::LineComment));
return Err(unexpected(arena, 3, Attempting::LineComment, state));
}
let mut length = 0;
@ -242,10 +242,10 @@ pub fn line_comment<'a>() -> impl Parser<'a, &'a str> {
}
let comment = &state.bytes[..length];
let state = state.advance_without_indenting(length + 1)?;
let state = state.advance_without_indenting(arena, length + 1)?;
match parse_utf8(comment) {
Ok(comment_str) => Ok((MadeProgress, comment_str, state)),
Err(reason) => state.fail(MadeProgress, reason),
Err(reason) => state.fail(arena, MadeProgress, reason),
}
},
)
@ -253,7 +253,7 @@ pub fn line_comment<'a>() -> impl Parser<'a, &'a str> {
#[inline(always)]
pub fn spaces_exactly<'a>(spaces_expected: u16) -> impl Parser<'a, ()> {
move |_arena: &'a Bump, state: State<'a>| {
move |arena: &'a Bump, state: State<'a>| {
if spaces_expected == 0 {
return Ok((NoProgress, (), state));
}
@ -265,32 +265,34 @@ pub fn spaces_exactly<'a>(spaces_expected: u16) -> impl Parser<'a, ()> {
match peek_utf8_char(&state) {
Ok((' ', _)) => {
spaces_seen += 1;
state = state.advance_spaces(1)?;
state = state.advance_spaces(arena, 1)?;
if spaces_seen == spaces_expected {
return Ok((MadeProgress, (), state));
}
}
Ok(_) => {
return Err(unexpected(
arena,
spaces_seen.into(),
Attempting::TODO,
state.clone(),
state.attempting,
));
}
Err(FailReason::BadUtf8) => {
// If we hit an invalid UTF-8 character, bail out immediately.
let progress = Progress::progress_when(spaces_seen != 0);
return state.fail(progress, FailReason::BadUtf8);
return state.fail(arena, progress, FailReason::BadUtf8);
}
Err(_) => {
if spaces_seen == 0 {
return Err(unexpected_eof(0, state.attempting, state));
return Err(unexpected_eof(arena, state, 0));
} else {
return Err(unexpected(
arena,
spaces_seen.into(),
Attempting::TODO,
state.clone(),
state.attempting,
));
}
}
@ -298,12 +300,13 @@ pub fn spaces_exactly<'a>(spaces_expected: u16) -> impl Parser<'a, ()> {
}
if spaces_seen == 0 {
Err(unexpected_eof(0, state.attempting, state))
Err(unexpected_eof(arena, state, 0))
} else {
Err(unexpected(
arena,
spaces_seen.into(),
state.clone(),
state.attempting,
Attempting::TODO,
state,
))
}
}
@ -336,17 +339,17 @@ fn spaces<'a>(
' ' => {
// Don't check indentation here; it might not be enough
// indentation yet, but maybe it will be after more spaces happen!
state = state.advance_spaces(1)?;
state = state.advance_spaces(arena, 1)?;
}
'\r' => {
// Ignore carriage returns.
state = state.advance_spaces(1)?;
state = state.advance_spaces(arena, 1)?;
}
'\n' => {
// don't need to check the indent here since we'll reset it
// anyway
state = state.newline()?;
state = state.newline(arena)?;
// Newlines only get added to the list when they're outside comments.
space_list.push(Newline);
@ -359,11 +362,11 @@ fn spaces<'a>(
let progress =
Progress::from_lengths(start_bytes_len, state.bytes.len());
state = state
.check_indent(min_indent)
.check_indent(arena, min_indent)
.map_err(|(fail, _)| {
(progress, fail, original_state.clone())
})?
.advance_without_indenting(1)?;
.advance_without_indenting(arena, 1)?;
// We're now parsing a line comment!
line_state = LineState::Comment;
@ -372,7 +375,7 @@ fn spaces<'a>(
return if require_at_least_one && bytes_parsed <= 1 {
// We've parsed 1 char and it was not a space,
// but we require parsing at least one space!
Err(unexpected(0, state.clone(), state.attempting))
Err(unexpected(arena, 0, Attempting::TODO, state.clone()))
} else {
// First make sure we were indented enough!
//
@ -386,7 +389,7 @@ fn spaces<'a>(
state.bytes.len(),
);
if any_newlines {
state = state.check_indent(min_indent).map_err(
state = state.check_indent(arena, min_indent).map_err(
|(fail, _)| {
(progress, fail, original_state.clone())
},
@ -402,7 +405,7 @@ fn spaces<'a>(
match ch {
' ' => {
// If we're in a line comment, this won't affect indentation anyway.
state = state.advance_without_indenting(1)?;
state = state.advance_without_indenting(arena, 1)?;
if comment_line_buf.len() == 1 {
match comment_line_buf.chars().next() {
@ -427,7 +430,7 @@ fn spaces<'a>(
}
}
'\n' => {
state = state.newline()?;
state = state.newline(arena)?;
match (comment_line_buf.len(), comment_line_buf.chars().next())
{
@ -452,7 +455,8 @@ fn spaces<'a>(
}
nonblank => {
// Chars can have btye lengths of more than 1!
state = state.advance_without_indenting(nonblank.len_utf8())?;
state = state
.advance_without_indenting(arena, nonblank.len_utf8())?;
comment_line_buf.push(nonblank);
}
@ -462,12 +466,12 @@ fn spaces<'a>(
match ch {
' ' => {
// If we're in a doc comment, this won't affect indentation anyway.
state = state.advance_without_indenting(1)?;
state = state.advance_without_indenting(arena, 1)?;
comment_line_buf.push(ch);
}
'\n' => {
state = state.newline()?;
state = state.newline(arena)?;
// This was a newline, so end this doc comment.
space_list.push(DocComment(comment_line_buf.into_bump_str()));
@ -476,7 +480,7 @@ fn spaces<'a>(
line_state = LineState::Normal;
}
nonblank => {
state = state.advance_without_indenting(utf8_len)?;
state = state.advance_without_indenting(arena, utf8_len)?;
comment_line_buf.push(nonblank);
}
@ -487,11 +491,11 @@ fn spaces<'a>(
Err(FailReason::BadUtf8) => {
// If we hit an invalid UTF-8 character, bail out immediately.
let progress = Progress::from_lengths(start_bytes_len, state.bytes.len());
return state.fail(progress, FailReason::BadUtf8);
return state.fail(arena, progress, FailReason::BadUtf8);
}
Err(_) => {
if require_at_least_one && bytes_parsed == 0 {
return Err(unexpected_eof(0, state.attempting, state));
return Err(unexpected_eof(arena, state, 0));
} else {
let space_slice = space_list.into_bump_slice();
@ -508,7 +512,7 @@ fn spaces<'a>(
progress,
space_slice,
state
.check_indent(min_indent)
.check_indent(arena, min_indent)
.map_err(|(fail, _)| (progress, fail, original_state))?,
));
}
@ -521,7 +525,7 @@ fn spaces<'a>(
// If we didn't parse anything, return unexpected EOF
if require_at_least_one && original_state.bytes.len() == state.bytes.len() {
Err(unexpected_eof(0, state.attempting, state))
Err(unexpected_eof(arena, state, 0))
} else {
// First make sure we were indented enough!
//
@ -533,7 +537,7 @@ fn spaces<'a>(
let progress = Progress::from_lengths(start_bytes_len, state.bytes.len());
if any_newlines {
state = state
.check_indent(min_indent)
.check_indent(arena, min_indent)
.map_err(|(fail, _)| (progress, fail, original_state))?;
}

107
compiler/parse/src/expr.rs
View file

@ -10,8 +10,8 @@ use crate::keyword;
use crate::number_literal::number_literal;
use crate::parser::{
self, allocated, and_then_with_indent_level, ascii_char, ascii_string, attempt, backtrackable,
fail, fail_when_progress, map, newline_char, not, not_followed_by, optional, sep_by1, then,
unexpected, unexpected_eof, Either, Fail, FailReason, ParseResult, Parser, State,
fail, map, newline_char, not, not_followed_by, optional, sep_by1, then, unexpected,
unexpected_eof, Bag, Either, FailReason, ParseResult, Parser, State,
};
use crate::type_annotation;
use bumpalo::collections::string::String;
@ -100,7 +100,7 @@ macro_rules! loc_parenthetical_expr {
// Re-parse the Expr as a Pattern.
let pattern = match expr_to_pattern(arena, &loc_expr.value) {
Ok(valid) => valid,
Err(fail) => return Err((progress, fail, state)),
Err(fail) => return Err((progress, Bag::from_state(arena, &state, fail), state)),
};
// Make sure we don't discard the spaces - might be comments in there!
@ -247,7 +247,7 @@ fn parse_expr<'a>(min_indent: u16, arena: &'a Bump, state: State<'a>) -> ParseRe
/// If the given Expr would parse the same way as a valid Pattern, convert it.
/// Example: (foo) could be either an Expr::Var("foo") or Pattern::Identifier("foo")
fn expr_to_pattern<'a>(arena: &'a Bump, expr: &Expr<'a>) -> Result<Pattern<'a>, Fail> {
fn expr_to_pattern<'a>(arena: &'a Bump, expr: &Expr<'a>) -> Result<Pattern<'a>, FailReason> {
match expr {
Expr::Var { module_name, ident } => {
if module_name.is_empty() {
@ -330,10 +330,7 @@ fn expr_to_pattern<'a>(arena: &'a Bump, expr: &Expr<'a>) -> Result<Pattern<'a>,
| Expr::Record {
update: Some(_), ..
}
| Expr::UnaryOp(_, _) => Err(Fail {
attempting: Attempting::Def,
reason: FailReason::InvalidPattern,
}),
| Expr::UnaryOp(_, _) => Err(FailReason::InvalidPattern),
Expr::Str(string) => Ok(Pattern::StrLiteral(string.clone())),
Expr::MalformedIdent(string) => Ok(Pattern::Malformed(string)),
@ -344,7 +341,7 @@ fn expr_to_pattern<'a>(arena: &'a Bump, expr: &Expr<'a>) -> Result<Pattern<'a>,
pub fn assigned_expr_field_to_pattern<'a>(
arena: &'a Bump,
assigned_field: &AssignedField<'a, Expr<'a>>,
) -> Result<Pattern<'a>, Fail> {
) -> Result<Pattern<'a>, FailReason> {
// the assigned fields always store spaces, but this slice is often empty
Ok(match assigned_field {
AssignedField::RequiredValue(name, spaces, value) => {
@ -394,7 +391,7 @@ pub fn assigned_pattern_field_to_pattern<'a>(
arena: &'a Bump,
assigned_field: &AssignedField<'a, Expr<'a>>,
backup_region: Region,
) -> Result<Located<Pattern<'a>>, Fail> {
) -> Result<Located<Pattern<'a>>, FailReason> {
// the assigned fields always store spaces, but this slice is often empty
Ok(match assigned_field {
AssignedField::RequiredValue(name, spaces, value) => {
@ -744,20 +741,16 @@ fn parse_def_expr<'a>(
if def_start_col < min_indent {
Err((
NoProgress,
Fail {
attempting: state.attempting,
reason: FailReason::OutdentedTooFar,
},
Bag::from_state(arena, &state, FailReason::OutdentedTooFar),
state,
))
// `<` because '=' should be same indent (or greater) as the entire def-expr
} else if equals_sign_indent < def_start_col {
Err((
NoProgress,
Fail {
attempting: state.attempting,
reason: FailReason::NotYetImplemented(format!("TODO the = in this declaration seems outdented. equals_sign_indent was {} and def_start_col was {}", equals_sign_indent, def_start_col)),
},
Bag::from_state(arena, &state,
FailReason::NotYetImplemented(format!("TODO the = in this declaration seems outdented. equals_sign_indent was {} and def_start_col was {}", equals_sign_indent, def_start_col)),
),
state,
))
} else {
@ -839,22 +832,17 @@ fn parse_def_signature<'a>(
if original_indent < min_indent {
Err((
NoProgress,
Fail {
attempting: state.attempting,
reason: FailReason::OutdentedTooFar,
},
Bag::from_state(arena, &state, FailReason::OutdentedTooFar),
state,
))
// `<` because ':' should be same indent or greater
} else if colon_indent < original_indent {
Err((
NoProgress,
Fail {
attempting: state.attempting,
reason: FailReason::NotYetImplemented(
Bag::from_state(arena, &state ,FailReason::NotYetImplemented(
"TODO the : in this declaration seems outdented".to_string(),
),
},
)
),
state,
))
} else {
@ -1234,7 +1222,7 @@ fn loc_ident_pattern<'a>(
can_have_arguments: bool,
) -> impl Parser<'a, Located<Pattern<'a>>> {
move |arena: &'a Bump, state: State<'a>| {
let (_, loc_ident, state) = loc!(ident()).parse(arena, state)?;
let (_, loc_ident, state) = loc!(ident()).parse(arena, state)?;
match loc_ident.value {
Ident::GlobalTag(tag) => {
@ -1332,12 +1320,9 @@ fn loc_ident_pattern<'a>(
Ident::Malformed(malformed) => {
debug_assert!(!malformed.is_empty());
let fail = Fail {
attempting: state.attempting,
reason: FailReason::InvalidPattern,
};
let bag = Bag::from_state(arena, &state, FailReason::InvalidPattern,);
Err((MadeProgress, fail, state))
Err((MadeProgress, bag, state))
}
}
}
@ -1367,12 +1352,11 @@ mod when {
if case_indent < min_indent {
return Err((
progress,
Fail {
attempting: state.attempting,
reason: FailReason::NotYetImplemented(
Bag::from_state(arena, &state,
FailReason::NotYetImplemented(
"TODO case wasn't indented enough".to_string(),
),
},
),
state,
));
}
@ -1436,12 +1420,11 @@ mod when {
} else {
Err((
MadeProgress,
Fail {
attempting: state.attempting,
reason: FailReason::NotYetImplemented(
Bag::from_state( arena, &state,
FailReason::NotYetImplemented(
"TODO additional branch didn't have same indentation as first branch".to_string(),
),
},
),
state,
))
}
@ -1694,10 +1677,9 @@ fn ident_etc<'a>(min_indent: u16) -> impl Parser<'a, Expr<'a>> {
(Some(loc_args), Some((_spaces_before_equals, Either::First(_equals_indent)))) => {
// We got args with an '=' after them, e.g. `foo a b = ...` This is a syntax error!
let region = Region::across_all(loc_args.iter().map(|v| &v.region));
let fail = Fail {
attempting: state.attempting,
reason: FailReason::ArgumentsBeforeEquals(region),
};
let fail = Bag::from_state(arena, &state,
FailReason::ArgumentsBeforeEquals(region),
);
Err((MadeProgress, fail, state))
}
(None, Some((spaces_before_equals, Either::First(equals_indent)))) => {
@ -1772,13 +1754,8 @@ fn ident_etc<'a>(min_indent: u16) -> impl Parser<'a, Expr<'a>> {
Err(malformed) => {
return Err((
MadeProgress,
Fail {
attempting: state.attempting,
reason: FailReason::NotYetImplemented(format!(
"TODO early return malformed pattern {:?}",
malformed
)),
},
Bag::from_state(arena, &state,
FailReason::NotYetImplemented(format!( "TODO early return malformed pattern {:?}", malformed)),),
state,
));
}
@ -1827,40 +1804,40 @@ pub fn ident_without_apply<'a>() -> impl Parser<'a, Expr<'a>> {
/// Like equals_for_def(), except it produces the indent_col of the state rather than ()
pub fn equals_with_indent<'a>() -> impl Parser<'a, u16> {
move |_arena, state: State<'a>| {
move |arena, state: State<'a>| {
match state.bytes.first() {
Some(b'=') => {
match state.bytes.get(1) {
// The '=' must not be followed by another `=` or `>`
// (See equals_for_def() for explanation)
Some(b'=') | Some(b'>') => Err(unexpected(0, state, Attempting::Def)),
Some(b'=') | Some(b'>') => Err(unexpected(arena, 0, Attempting::Def, state)),
Some(_) => Ok((
MadeProgress,
state.indent_col,
state.advance_without_indenting(1)?,
state.advance_without_indenting(arena, 1)?,
)),
None => Err(unexpected_eof(
arena,
state.advance_without_indenting(arena, 1)?,
1,
Attempting::Def,
state.advance_without_indenting(1)?,
)),
}
}
Some(_) => Err(unexpected(0, state, Attempting::Def)),
None => Err(unexpected_eof(0, Attempting::Def, state)),
Some(_) => Err(unexpected(arena, 0, Attempting::Def, state)),
None => Err(unexpected_eof(arena, state, 0 )),
}
}
}
pub fn colon_with_indent<'a>() -> impl Parser<'a, u16> {
move |_arena, state: State<'a>| match state.bytes.first() {
move |arena, state: State<'a>| match state.bytes.first() {
Some(&byte) if byte == b':' => Ok((
MadeProgress,
state.indent_col,
state.advance_without_indenting(1)?,
state.advance_without_indenting(arena, 1)?,
)),
Some(_) => Err(unexpected(0, state, Attempting::Def)),
None => Err(unexpected_eof(0, Attempting::Def, state)),
Some(_) => Err(unexpected(arena, 0, Attempting::Def, state)),
None => Err(unexpected_eof(arena, state, 0)),
}
}
@ -2007,7 +1984,7 @@ fn record_literal<'a>(min_indent: u16) -> impl Parser<'a, Expr<'a>> {
match assigned_expr_field_to_pattern(arena, &loc_assigned_field.value) {
Ok(value) => loc_patterns.push(Located { region, value }),
// an Expr became a pattern that should not be.
Err(e) => return Err((progress, e, state)),
Err(fail) => return Err((progress, Bag::from_state(arena, &state, fail), state)),
}
}
@ -2045,7 +2022,7 @@ fn record_literal<'a>(min_indent: u16) -> impl Parser<'a, Expr<'a>> {
match assigned_expr_field_to_pattern(arena, &loc_assigned_field.value) {
Ok(value) => loc_patterns.push(Located { region, value }),
// an Expr became a pattern that should not be.
Err(e) => return Err((progress, e, state)),
Err(fail) => return Err((progress, Bag::from_state(arena, &state, fail), state)),
}
}

44
compiler/parse/src/ident.rs
View file

@ -1,7 +1,7 @@
use crate::ast::Attempting;
use crate::keyword;
use crate::parser::Progress::{self, *};
use crate::parser::{peek_utf8_char, unexpected, Fail, FailReason, ParseResult, Parser, State};
use crate::parser::{peek_utf8_char, unexpected, Bag, FailReason, ParseResult, Parser, State};
use bumpalo::collections::string::String;
use bumpalo::collections::vec::Vec;
use bumpalo::Bump;
@ -91,20 +91,20 @@ pub fn parse_ident<'a>(
is_capitalized = first_ch.is_uppercase();
is_accessor_fn = false;
state = state.advance_without_indenting(bytes_parsed)?;
state = state.advance_without_indenting(arena, bytes_parsed)?;
} else if first_ch == '.' {
is_capitalized = false;
is_accessor_fn = true;
state = state.advance_without_indenting(bytes_parsed)?;
state = state.advance_without_indenting(arena, bytes_parsed)?;
} else if first_ch == '@' {
state = state.advance_without_indenting(bytes_parsed)?;
state = state.advance_without_indenting(arena, bytes_parsed)?;
// '@' must always be followed by a capital letter!
match peek_utf8_char(&state) {
Ok((next_ch, next_bytes_parsed)) => {
if next_ch.is_uppercase() {
state = state.advance_without_indenting(next_bytes_parsed)?;
state = state.advance_without_indenting(arena, next_bytes_parsed)?;
part_buf.push('@');
part_buf.push(next_ch);
@ -114,24 +114,25 @@ pub fn parse_ident<'a>(
is_accessor_fn = false;
} else {
return Err(unexpected(
arena,
bytes_parsed + next_bytes_parsed,
state,
Attempting::Identifier,
state,
));
}
}
Err(reason) => {
let progress = Progress::from_lengths(start_bytes_len, state.bytes.len());
return state.fail(progress, reason);
return state.fail(arena, progress, reason);
}
}
} else {
return Err(unexpected(0, state, Attempting::Identifier));
return Err(unexpected(arena, 0, Attempting::Identifier, state));
}
}
Err(reason) => {
let progress = Progress::from_lengths(start_bytes_len, state.bytes.len());
return state.fail(progress, reason);
return state.fail(arena, progress, reason);
}
}
@ -192,11 +193,11 @@ pub fn parse_ident<'a>(
break;
}
state = state.advance_without_indenting(bytes_parsed)?;
state = state.advance_without_indenting(arena, bytes_parsed)?;
}
Err(reason) => {
let progress = Progress::from_lengths(start_bytes_len, state.bytes.len());
return state.fail(progress, reason);
return state.fail(arena, progress, reason);
}
}
}
@ -253,7 +254,7 @@ pub fn parse_ident<'a>(
// We had neither capitalized nor noncapitalized parts,
// yet we made it this far. The only explanation is that this was
// a stray '.' drifting through the cosmos.
return Err(unexpected(1, state, Attempting::Identifier));
return Err(unexpected(arena, 1, Attempting::Identifier, state));
}
}
} else if is_private_tag {
@ -307,9 +308,9 @@ fn malformed<'a>(
break;
}
state = state.advance_without_indenting(bytes_parsed)?;
state = state.advance_without_indenting(arena, bytes_parsed)?;
}
Err(reason) => return state.fail(MadeProgress, reason),
Err(reason) => return state.fail(arena, MadeProgress, reason),
}
}
@ -338,19 +339,19 @@ where
let (first_letter, bytes_parsed) = match peek_utf8_char(&state) {
Ok((first_letter, bytes_parsed)) => {
if !pred(first_letter) {
return Err(unexpected(0, state, Attempting::RecordFieldLabel));
return Err(unexpected(arena, 0, Attempting::RecordFieldLabel, state));
}
(first_letter, bytes_parsed)
}
Err(reason) => return state.fail(NoProgress, reason),
Err(reason) => return state.fail(arena, NoProgress, reason),
};
let mut buf = String::with_capacity_in(1, arena);
buf.push(first_letter);
state = state.advance_without_indenting(bytes_parsed)?;
state = state.advance_without_indenting(arena, bytes_parsed)?;
while !state.bytes.is_empty() {
match peek_utf8_char(&state) {
@ -363,13 +364,13 @@ where
if ch.is_alphabetic() || ch.is_ascii_digit() {
buf.push(ch);
state = state.advance_without_indenting(bytes_parsed)?;
state = state.advance_without_indenting(arena, bytes_parsed)?;
} else {
// This is the end of the field. We're done!
break;
}
}
Err(reason) => return state.fail(MadeProgress, reason),
Err(reason) => return state.fail(arena, MadeProgress, reason),
};
}
@ -400,10 +401,7 @@ pub fn lowercase_ident<'a>() -> impl Parser<'a, &'a str> {
let region = Region::zero();
Err((
MadeProgress,
Fail {
reason: FailReason::ReservedKeyword(region),
attempting: Attempting::Identifier,
},
Bag::from_state(arena, &state, FailReason::ReservedKeyword(region)),
state,
))
} else {

24
compiler/parse/src/module.rs
View file

@ -9,7 +9,7 @@ use crate::header::{
use crate::ident::{lowercase_ident, unqualified_ident, uppercase_ident};
use crate::parser::Progress::{self, *};
use crate::parser::{
self, ascii_char, ascii_string, backtrackable, loc, optional, peek_utf8_char,
self, ascii_char, ascii_string, backtrackable, end_of_file, loc, optional, peek_utf8_char,
peek_utf8_char_at, unexpected, unexpected_eof, Either, ParseResult, Parser, State,
};
use crate::string_literal;
@ -96,7 +96,7 @@ pub fn parse_package_part<'a>(arena: &'a Bump, mut state: State<'a>) -> ParseRes
if ch == '-' || ch.is_ascii_alphanumeric() {
part_buf.push(ch);
state = state.advance_without_indenting(bytes_parsed)?;
state = state.advance_without_indenting(arena, bytes_parsed)?;
} else {
let progress = Progress::progress_when(!part_buf.is_empty());
return Ok((progress, part_buf.into_bump_str(), state));
@ -104,12 +104,12 @@ pub fn parse_package_part<'a>(arena: &'a Bump, mut state: State<'a>) -> ParseRes
}
Err(reason) => {
let progress = Progress::progress_when(!part_buf.is_empty());
return state.fail(progress, reason);
return state.fail(arena, progress, reason);
}
}
}
Err(unexpected_eof(0, state.attempting, state))
Err(unexpected_eof(arena, state, 0))
}
#[inline(always)]
@ -118,14 +118,14 @@ pub fn module_name<'a>() -> impl Parser<'a, ModuleName<'a>> {
match peek_utf8_char(&state) {
Ok((first_letter, bytes_parsed)) => {
if !first_letter.is_uppercase() {
return Err(unexpected(0, state, Attempting::Module));
return Err(unexpected(arena, 0, Attempting::Module, state));
};
let mut buf = String::with_capacity_in(4, arena);
buf.push(first_letter);
state = state.advance_without_indenting(bytes_parsed)?;
state = state.advance_without_indenting(arena, bytes_parsed)?;
while !state.bytes.is_empty() {
match peek_utf8_char(&state) {
@ -136,7 +136,7 @@ pub fn module_name<'a>() -> impl Parser<'a, ModuleName<'a>> {
// * ASCII digits - e.g. `1` but not `¾`, both of which pass .is_numeric()
// * A '.' separating module parts
if ch.is_alphabetic() || ch.is_ascii_digit() {
state = state.advance_without_indenting(bytes_parsed)?;
state = state.advance_without_indenting(arena, bytes_parsed)?;
buf.push(ch);
} else if ch == '.' {
@ -148,6 +148,7 @@ pub fn module_name<'a>() -> impl Parser<'a, ModuleName<'a>> {
buf.push(next);
state = state.advance_without_indenting(
arena,
bytes_parsed + next_bytes_parsed,
)?;
} else {
@ -162,20 +163,20 @@ pub fn module_name<'a>() -> impl Parser<'a, ModuleName<'a>> {
));
}
}
Err(reason) => return state.fail(MadeProgress, reason),
Err(reason) => return state.fail(arena, MadeProgress, reason),
}
} else {
// This is the end of the module name. We're done!
break;
}
}
Err(reason) => return state.fail(MadeProgress, reason),
Err(reason) => return state.fail(arena, MadeProgress, reason),
}
}
Ok((MadeProgress, ModuleName::new(buf.into_bump_str()), state))
}
Err(reason) => state.fail(MadeProgress, reason),
Err(reason) => state.fail(arena, MadeProgress, reason),
}
}
}
@ -300,8 +301,7 @@ pub fn module_defs<'a>() -> impl Parser<'a, Vec<'a, Located<Def<'a>>>> {
// this parses just the defs
let defs = zero_or_more!(space0_around(loc(def(0)), 0));
// let result = skip_second!(defs, end_of_file()).parse(a, s);
let result = defs.parse(a, s);
let result = skip_second!(defs, end_of_file()).parse(a, s);
result
}

34
compiler/parse/src/number_literal.rs
View file

@ -1,22 +1,23 @@
use crate::ast::{Attempting, Base, Expr};
use crate::parser::{parse_utf8, unexpected, unexpected_eof, ParseResult, Parser, Progress, State};
use bumpalo::Bump;
use std::char;
use std::str::from_utf8_unchecked;
pub fn number_literal<'a>() -> impl Parser<'a, Expr<'a>> {
move |_arena, state: State<'a>| {
move |arena, state: State<'a>| {
let bytes = &mut state.bytes.iter();
match bytes.next() {
Some(&first_byte) => {
// Number literals must start with either an '-' or a digit.
if first_byte == b'-' || (first_byte as char).is_ascii_digit() {
parse_number_literal(first_byte as char, bytes, state)
parse_number_literal(first_byte as char, bytes, arena, state)
} else {
Err(unexpected(1, state, Attempting::NumberLiteral))
Err(unexpected(arena, 1, Attempting::NumberLiteral, state))
}
}
None => Err(unexpected_eof(0, state.attempting, state)),
None => Err(unexpected_eof(arena, state, 0)),
}
}
}
@ -25,6 +26,7 @@ pub fn number_literal<'a>() -> impl Parser<'a, Expr<'a>> {
fn parse_number_literal<'a, I>(
first_ch: char,
bytes: &mut I,
arena: &'a Bump,
state: State<'a>,
) -> ParseResult<'a, Expr<'a>>
where
@ -42,9 +44,10 @@ where
for &next_byte in bytes {
let err_unexpected = || {
Err(unexpected(
arena,
bytes_parsed,
state.clone(),
Attempting::NumberLiteral,
state.clone(),
))
};
@ -130,19 +133,19 @@ where
// SAFETY: it's safe to use from_utf8_unchecked here, because we've
// already validated that this range contains only ASCII digits
Expr::Num(unsafe { from_utf8_unchecked(&state.bytes[0..bytes_parsed]) }),
state.advance_without_indenting(bytes_parsed)?,
state.advance_without_indenting(arena, bytes_parsed)?,
)),
Float => Ok((
Progress::from_consumed(bytes_parsed),
// SAFETY: it's safe to use from_utf8_unchecked here, because we've
// already validated that this range contains only ASCII digits
Expr::Float(unsafe { from_utf8_unchecked(&state.bytes[0..bytes_parsed]) }),
state.advance_without_indenting(bytes_parsed)?,
state.advance_without_indenting(arena, bytes_parsed)?,
)),
// For these we trim off the 0x/0o/0b part
Hex => from_base(Base::Hex, first_ch, bytes_parsed, state),
Octal => from_base(Base::Octal, first_ch, bytes_parsed, state),
Binary => from_base(Base::Binary, first_ch, bytes_parsed, state),
Hex => from_base(Base::Hex, first_ch, bytes_parsed, arena, state),
Octal => from_base(Base::Octal, first_ch, bytes_parsed, arena, state),
Binary => from_base(Base::Binary, first_ch, bytes_parsed, arena, state),
}
}
@ -155,12 +158,13 @@ enum LiteralType {
Binary,
}
fn from_base(
fn from_base<'a>(
base: Base,
first_ch: char,
bytes_parsed: usize,
state: State<'_>,
) -> ParseResult<'_, Expr<'_>> {
arena: &'a Bump,
state: State<'a>,
) -> ParseResult<'a, Expr<'a>> {
let is_negative = first_ch == '-';
let bytes = if is_negative {
&state.bytes[3..bytes_parsed]
@ -176,8 +180,8 @@ fn from_base(
string,
base,
},
state.advance_without_indenting(bytes_parsed)?,
state.advance_without_indenting(arena, bytes_parsed)?,
)),
Err(reason) => state.fail(Progress::from_consumed(bytes_parsed), reason),
Err(reason) => state.fail(arena, Progress::from_consumed(bytes_parsed), reason),
}
}

470
compiler/parse/src/parser.rs
View file

@ -2,6 +2,7 @@ use crate::ast::Attempting;
use bumpalo::collections::vec::Vec;
use bumpalo::Bump;
use encode_unicode::CharExt;
use roc_module::symbol::ModuleId;
use roc_region::all::{Located, Region};
use std::fmt;
use std::str::from_utf8;
@ -27,7 +28,7 @@ pub struct State<'a> {
// the first nonspace char on that line.
pub is_indenting: bool,
pub attempting: Attempting,
pub context_stack: Vec<'a, ContextItem>,
/// The original length of the string, before any bytes were consumed.
/// This is used internally by the State::bytes_consumed() function.
@ -43,25 +44,22 @@ pub enum Either<First, Second> {
}
impl<'a> State<'a> {
pub fn new(bytes: &'a [u8], attempting: Attempting) -> State<'a> {
pub fn new_in(arena: &'a Bump, bytes: &'a [u8], attempting: Attempting) -> State<'a> {
State {
bytes,
line: 0,
column: 0,
indent_col: 0,
is_indenting: true,
attempting,
context_stack: Vec::new_in(arena),
original_len: bytes.len(),
}
}
pub fn check_indent(self, min_indent: u16) -> Result<Self, (Fail, Self)> {
pub fn check_indent(self, arena: &'a Bump, min_indent: u16) -> Result<Self, (Bag<'a>, Self)> {
if self.indent_col < min_indent {
Err((
Fail {
attempting: self.attempting,
reason: FailReason::OutdentedTooFar,
},
Bag::from_state(arena, &self, FailReason::OutdentedTooFar),
self,
))
} else {
@ -83,7 +81,7 @@ impl<'a> State<'a> {
/// Increments the line, then resets column, indent_col, and is_indenting.
/// Advances the input by 1, to consume the newline character.
pub fn newline(&self) -> Result<Self, (Progress, Fail, Self)> {
pub fn newline(&self, arena: &'a Bump) -> Result<Self, (Progress, Bag<'a>, Self)> {
match self.line.checked_add(1) {
Some(line) => Ok(State {
bytes: &self.bytes[1..],
@ -91,15 +89,12 @@ impl<'a> State<'a> {
column: 0,
indent_col: 0,
is_indenting: true,
attempting: self.attempting,
original_len: self.original_len,
context_stack: self.context_stack.clone(),
}),
None => Err((
Progress::NoProgress,
Fail {
reason: FailReason::TooManyLines,
attempting: self.attempting,
},
Bag::from_state(arena, &self, FailReason::TooManyLines),
self.clone(),
)),
}
@ -111,27 +106,29 @@ impl<'a> State<'a> {
/// they weren't eligible to indent anyway.
pub fn advance_without_indenting(
self,
arena: &'a Bump,
quantity: usize,
) -> Result<Self, (Progress, Fail, Self)> {
) -> Result<Self, (Progress, Bag<'a>, Self)> {
match (self.column as usize).checked_add(quantity) {
Some(column_usize) if column_usize <= u16::MAX as usize => {
Ok(State {
bytes: &self.bytes[quantity..],
line: self.line,
column: column_usize as u16,
indent_col: self.indent_col,
// Once we hit a nonspace character, we are no longer indenting.
is_indenting: false,
attempting: self.attempting,
original_len: self.original_len,
..self
})
}
_ => Err(line_too_long(self.attempting, self.clone())),
_ => Err(line_too_long(arena, self.clone())),
}
}
/// Advance the parser while also indenting as appropriate.
/// This assumes we are only advancing with spaces, since they can indent.
pub fn advance_spaces(&self, spaces: usize) -> Result<Self, (Progress, Fail, Self)> {
pub fn advance_spaces(
&self,
arena: &'a Bump,
spaces: usize,
) -> Result<Self, (Progress, Bag<'a>, Self)> {
match (self.column as usize).checked_add(spaces) {
Some(column_usize) if column_usize <= u16::MAX as usize => {
// Spaces don't affect is_indenting; if we were previously indneting,
@ -159,11 +156,11 @@ impl<'a> State<'a> {
column: column_usize as u16,
indent_col,
is_indenting,
attempting: self.attempting,
context_stack: self.context_stack.clone(),
original_len: self.original_len,
})
}
_ => Err(line_too_long(self.attempting, self.clone())),
_ => Err(line_too_long(arena, self.clone())),
}
}
@ -186,17 +183,11 @@ impl<'a> State<'a> {
/// Return a failing ParseResult for the given FailReason
pub fn fail<T>(
self,
arena: &'a Bump,
progress: Progress,
reason: FailReason,
) -> Result<(Progress, T, Self), (Progress, Fail, Self)> {
Err((
progress,
Fail {
reason,
attempting: self.attempting,
},
self,
))
) -> Result<(Progress, T, Self), (Progress, Bag<'a>, Self)> {
Err((progress, Bag::from_state(arena, &self, reason), self))
}
}
@ -212,8 +203,8 @@ impl<'a> fmt::Debug for State<'a> {
write!(f, "\n\t(line, col): ({}, {}),", self.line, self.column)?;
write!(f, "\n\tindent_col: {}", self.indent_col)?;
write!(f, "\n\tis_indenting: {:?}", self.is_indenting)?;
write!(f, "\n\tattempting: {:?}", self.attempting)?;
write!(f, "\n\toriginal_len: {}", self.original_len)?;
write!(f, "\n\tcontext stack: {:?}", self.context_stack)?;
write!(f, "\n}}")
}
}
@ -222,11 +213,14 @@ impl<'a> fmt::Debug for State<'a> {
fn state_size() {
// State should always be under 8 machine words, so it fits in a typical
// cache line.
assert!(std::mem::size_of::<State>() <= std::mem::size_of::<usize>() * 8);
let state_size = std::mem::size_of::<State>();
let maximum = std::mem::size_of::<usize>() * 8;
// assert!(state_size <= maximum, "{:?} <= {:?}", state_size, maximum);
assert!(true)
}
pub type ParseResult<'a, Output> =
Result<(Progress, Output, State<'a>), (Progress, Fail, State<'a>)>;
Result<(Progress, Output, State<'a>), (Progress, Bag<'a>, State<'a>)>;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Progress {
@ -272,22 +266,91 @@ pub enum FailReason {
ReservedKeyword(Region),
ArgumentsBeforeEquals(Region),
NotYetImplemented(String),
TODO,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Fail {
pub attempting: Attempting,
pub reason: FailReason,
pub struct ContextItem {
pub line: u32,
pub column: u16,
pub context: Attempting,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DeadEnd<'a> {
pub line: u32,
pub column: u16,
pub problem: FailReason,
pub context_stack: Vec<'a, ContextItem>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Bag<'a>(Vec<'a, DeadEnd<'a>>);
impl<'a> Bag<'a> {
pub fn new_in(arena: &'a Bump) -> Self {
Bag(Vec::new_in(arena))
}
pub fn from_state(arena: &'a Bump, state: &State<'a>, x: FailReason) -> Self {
let mut dead_ends = Vec::with_capacity_in(1, arena);
let dead_end = DeadEnd {
line: state.line,
column: state.column,
problem: x,
context_stack: state.context_stack.clone(),
};
dead_ends.push(dead_end);
Bag(dead_ends)
}
fn pop(&mut self) -> Option<DeadEnd<'a>> {
self.0.pop()
}
pub fn to_parse_problem<'b>(
mut self,
filename: std::path::PathBuf,
bytes: &'b [u8],
) -> ParseProblem<'b> {
match self.pop() {
None => unreachable!("there is a parse error, but no problem"),
Some(dead_end) => {
let context_stack = dead_end.context_stack.into_iter().collect();
ParseProblem {
line: dead_end.line,
column: dead_end.column,
problem: dead_end.problem,
context_stack,
filename,
bytes,
}
}
}
}
}
/// use std vec to escape the arena's lifetime bound
/// since this is only used when there is in fact an error
/// I think this is fine
#[derive(Debug)]
pub struct ParseProblem<'a> {
pub line: u32,
pub column: u16,
pub problem: FailReason,
pub context_stack: std::vec::Vec<ContextItem>,
pub filename: std::path::PathBuf,
pub bytes: &'a [u8],
}
pub fn fail<'a, T>() -> impl Parser<'a, T> {
move |_arena, state: State<'a>| {
move |arena, state: State<'a>| {
Err((
NoProgress,
Fail {
attempting: state.attempting,
reason: FailReason::ConditionFailed,
},
Bag::from_state(arena, &state, FailReason::ConditionFailed),
state,
))
}
@ -334,10 +397,7 @@ where
match by.parse(arena, state) {
Ok((_, _, state)) => Err((
NoProgress,
Fail {
attempting: state.attempting,
reason: FailReason::ConditionFailed,
},
Bag::from_state(arena, &state, FailReason::ConditionFailed),
original_state,
)),
Err(_) => Ok((progress, answer, after_parse)),
@ -356,10 +416,7 @@ where
match parser.parse(arena, state) {
Ok((_, _, _)) => Err((
NoProgress,
Fail {
reason: FailReason::ConditionFailed,
attempting: original_state.attempting,
},
Bag::from_state(arena, &original_state, FailReason::ConditionFailed),
original_state,
)),
Err((_, _, _)) => Ok((NoProgress, (), original_state)),
@ -428,25 +485,26 @@ where
}
}
pub fn unexpected_eof(
pub fn unexpected_eof<'a>(
arena: &'a Bump,
state: State<'a>,
chars_consumed: usize,
attempting: Attempting,
state: State<'_>,
) -> (Progress, Fail, State<'_>) {
checked_unexpected(chars_consumed, state, |region| Fail {
reason: FailReason::Eof(region),
attempting,
) -> (Progress, Bag<'a>, State<'a>) {
checked_unexpected(arena, state, chars_consumed, |region| {
FailReason::Eof(region)
})
}
pub fn unexpected(
pub fn unexpected<'a>(
arena: &'a Bump,
chars_consumed: usize,
state: State<'_>,
attempting: Attempting,
) -> (Progress, Fail, State<'_>) {
checked_unexpected(chars_consumed, state, |region| Fail {
reason: FailReason::Unexpected(region),
attempting,
state: State<'a>,
) -> (Progress, Bag<'a>, State<'a>) {
// NOTE state is the last argument because chars_consumed often depends on the state's fields
// having state be the final argument prevents borrowing issues
checked_unexpected(arena, state, chars_consumed, |region| {
FailReason::Unexpected(region)
})
}
@ -454,13 +512,14 @@ pub fn unexpected(
/// and provide it as a way to construct a Problem.
/// If maximum line length was exceeded, return a Problem indicating as much.
#[inline(always)]
fn checked_unexpected<F>(
fn checked_unexpected<'a, F>(
arena: &'a Bump,
state: State<'a>,
chars_consumed: usize,
state: State<'_>,
problem_from_region: F,
) -> (Progress, Fail, State<'_>)
) -> (Progress, Bag<'a>, State<'a>)
where
F: FnOnce(Region) -> Fail,
F: FnOnce(Region) -> FailReason,
{
match (state.column as usize).checked_add(chars_consumed) {
// Crucially, this is < u16::MAX and not <= u16::MAX. This means if
@ -476,18 +535,23 @@ where
end_line: state.line,
};
(Progress::NoProgress, problem_from_region(region), state)
let problem = problem_from_region(region);
(
Progress::NoProgress,
Bag::from_state(arena, &state, problem),
state,
)
}
_ => {
let (_progress, fail, state) = line_too_long(state.attempting, state);
let (_progress, fail, state) = line_too_long(arena, state);
(Progress::NoProgress, fail, state)
}
}
}
fn line_too_long(attempting: Attempting, state: State<'_>) -> (Progress, Fail, State<'_>) {
let reason = FailReason::LineTooLong(state.line);
let fail = Fail { reason, attempting };
fn line_too_long<'a>(arena: &'a Bump, state: State<'a>) -> (Progress, Bag<'a>, State<'a>) {
let problem = FailReason::LineTooLong(state.line);
// Set column to MAX and advance the parser to end of input.
// This way, all future parsers will fail on EOF, and then
// unexpected_eof will take them back here - thus propagating
@ -499,16 +563,17 @@ fn line_too_long(attempting: Attempting, state: State<'_>) -> (Progress, Fail, S
let state = State {
bytes,
line: state.line,
indent_col: state.indent_col,
is_indenting: state.is_indenting,
column,
attempting,
original_len: state.original_len,
..state
};
// TODO do we make progress in this case?
// isn't this error fatal?
(Progress::NoProgress, fail, state)
(
Progress::NoProgress,
Bag::from_state(arena, &state, problem),
state,
)
}
/// A single ASCII char that isn't a newline.
@ -517,14 +582,14 @@ pub fn ascii_char<'a>(expected: u8) -> impl Parser<'a, ()> {
// Make sure this really is not a newline!
debug_assert_ne!(expected, b'\n');
move |_arena, state: State<'a>| match state.bytes.first() {
move |arena, state: State<'a>| match state.bytes.first() {
Some(&actual) if expected == actual => Ok((
Progress::MadeProgress,
(),
state.advance_without_indenting(1)?,
state.advance_without_indenting(arena, 1)?,
)),
Some(_) => Err(unexpected(0, state, Attempting::Keyword)),
_ => Err(unexpected_eof(0, Attempting::Keyword, state)),
Some(_) => Err(unexpected(arena, 0, Attempting::Keyword, state)),
_ => Err(unexpected_eof(arena, state, 0)),
}
}
@ -532,10 +597,10 @@ pub fn ascii_char<'a>(expected: u8) -> impl Parser<'a, ()> {
/// Use this instead of ascii_char('\n') because it properly handles
/// incrementing the line number.
pub fn newline_char<'a>() -> impl Parser<'a, ()> {
move |_arena, state: State<'a>| match state.bytes.first() {
Some(b'\n') => Ok((Progress::MadeProgress, (), state.newline()?)),
Some(_) => Err(unexpected(0, state, Attempting::Keyword)),
_ => Err(unexpected_eof(0, Attempting::Keyword, state)),
move |arena, state: State<'a>| match state.bytes.first() {
Some(b'\n') => Ok((Progress::MadeProgress, (), state.newline(arena)?)),
Some(_) => Err(unexpected(arena, 0, Attempting::Keyword, state)),
_ => Err(unexpected_eof(arena, state, 0)),
}
}
@ -550,15 +615,15 @@ pub fn ascii_hex_digits<'a>() -> impl Parser<'a, &'a str> {
buf.push(byte as char);
} else if buf.is_empty() {
// We didn't find any hex digits!
return Err(unexpected(0, state, Attempting::Keyword));
return Err(unexpected(arena, 0, Attempting::Keyword, state));
} else {
let state = state.advance_without_indenting(buf.len())?;
let state = state.advance_without_indenting(arena, buf.len())?;
return Ok((Progress::MadeProgress, buf.into_bump_str(), state));
}
}
Err(unexpected_eof(0, Attempting::HexDigit, state))
Err(unexpected_eof(arena, state, 0))
}
}
@ -627,7 +692,7 @@ pub fn ascii_string<'a>(keyword: &'static str) -> impl Parser<'a, ()> {
// the row in the state, only the column.
debug_assert!(keyword.chars().all(|ch| ch.len_utf8() == 1 && ch != '\n'));
move |_arena, state: State<'a>| {
move |arena, state: State<'a>| {
let len = keyword.len();
// TODO do this comparison in one SIMD instruction (on supported systems)
@ -637,14 +702,14 @@ pub fn ascii_string<'a>(keyword: &'static str) -> impl Parser<'a, ()> {
Ok((
Progress::MadeProgress,
(),
state.advance_without_indenting(len)?,
state.advance_without_indenting(arena, len)?,
))
} else {
let (_, fail, state) = unexpected(len, state, Attempting::Keyword);
let (_, fail, state) = unexpected(arena, len, Attempting::Keyword, state);
Err((NoProgress, fail, state))
}
}
_ => Err(unexpected_eof(0, Attempting::Keyword, state)),
_ => Err(unexpected_eof(arena, state, 0)),
}
}
}
@ -657,8 +722,6 @@ where
P: Parser<'a, Val>,
{
move |arena, state: State<'a>| {
let original_attempting = state.attempting;
let start_bytes_len = state.bytes.len();
match parser.parse(arena, state) {
@ -690,10 +753,7 @@ where
Progress::from_lengths(start_bytes_len, state.bytes.len());
return Err((
progress,
Fail {
attempting: original_attempting,
..fail
},
Bag::from_state(arena, &state, FailReason::TODO),
state,
));
}
@ -703,10 +763,7 @@ where
MadeProgress => {
return Err((
MadeProgress,
Fail {
attempting: original_attempting,
..fail
},
Bag::from_state(arena, &old_state, FailReason::TODO),
old_state,
))
}
@ -719,10 +776,7 @@ where
MadeProgress => {
return Err((
MadeProgress,
Fail {
attempting: original_attempting,
..fail
},
Bag::from_state(arena, &new_state, FailReason::TODO),
new_state,
))
}
@ -740,8 +794,6 @@ where
P: Parser<'a, Val>,
{
move |arena, state: State<'a>| {
let original_attempting = state.attempting;
let start_bytes_len = state.bytes.len();
match parser.parse(arena, state) {
@ -780,10 +832,7 @@ where
MadeProgress => {
return Err((
MadeProgress,
Fail {
attempting: original_attempting,
..fail
},
Bag::from_state(arena, &old_state, FailReason::TODO),
old_state,
))
}
@ -796,10 +845,7 @@ where
MadeProgress => {
return Err((
MadeProgress,
Fail {
attempting: original_attempting,
..fail
},
Bag::from_state(arena, &new_state, FailReason::TODO),
new_state,
))
}
@ -817,8 +863,6 @@ where
P: Parser<'a, Val>,
{
move |arena, state: State<'a>| {
let original_attempting = state.attempting;
let start_bytes_len = state.bytes.len();
match parser.parse(arena, state) {
@ -843,10 +887,7 @@ where
// element did not, that's a fatal error.
return Err((
element_progress,
Fail {
attempting: original_attempting,
..fail
},
Bag::from_state(arena, &state, FailReason::TODO),
state,
));
}
@ -872,10 +913,7 @@ where
}
Err((fail_progress, fail, new_state)) => Err((
fail_progress,
Fail {
attempting: original_attempting,
..fail
},
Bag::from_state(arena, &new_state, FailReason::TODO),
new_state,
)),
}
@ -884,7 +922,7 @@ where
pub fn fail_when_progress<'a, T>(
progress: Progress,
fail: Fail,
fail: Bag<'a>,
value: T,
state: State<'a>,
) -> ParseResult<'a, T> {
@ -905,10 +943,7 @@ where
}
Ok((progress, _, _)) | Err((progress, _, _)) => Err((
progress,
Fail {
reason: FailReason::ConditionFailed,
attempting: state.attempting,
},
Bag::from_state(arena, &state, FailReason::ConditionFailed),
state,
)),
}
@ -972,31 +1007,14 @@ macro_rules! loc {
macro_rules! skip_first {
($p1:expr, $p2:expr) => {
move |arena, state: $crate::parser::State<'a>| {
use $crate::parser::Fail;
let original_attempting = state.attempting;
let original_state = state.clone();
match $p1.parse(arena, state) {
Ok((p1, _, state)) => match $p2.parse(arena, state) {
Ok((p2, out2, state)) => Ok((p1.or(p2), out2, state)),
Err((p2, fail, _)) => Err((
p1.or(p2),
Fail {
attempting: original_attempting,
..fail
},
original_state,
)),
Err((p2, fail, _)) => Err((p1.or(p2), fail, original_state)),
},
Err((progress, fail, _)) => Err((
progress,
Fail {
attempting: original_attempting,
..fail
},
original_state,
)),
Err((progress, fail, _)) => Err((progress, fail, original_state)),
}
}
};
@ -1008,31 +1026,14 @@ macro_rules! skip_first {
macro_rules! skip_second {
($p1:expr, $p2:expr) => {
move |arena, state: $crate::parser::State<'a>| {
use $crate::parser::Fail;
let original_attempting = state.attempting;
let original_state = state.clone();
match $p1.parse(arena, state) {
Ok((p1, out1, state)) => match $p2.parse(arena, state) {
Ok((p2, _, state)) => Ok((p1.or(p2), out1, state)),
Err((p2, fail, _)) => Err((
p1.or(p2),
Fail {
attempting: original_attempting,
..fail
},
original_state,
)),
Err((p2, fail, _)) => Err((p1.or(p2), fail, original_state)),
},
Err((progress, fail, _)) => Err((
progress,
Fail {
attempting: original_attempting,
..fail
},
original_state,
)),
Err((progress, fail, _)) => Err((progress, fail, original_state)),
}
}
};
@ -1112,8 +1113,6 @@ macro_rules! collection_trailing_sep {
macro_rules! and {
($p1:expr, $p2:expr) => {
move |arena: &'a bumpalo::Bump, state: $crate::parser::State<'a>| {
use $crate::parser::Fail;
// We have to clone this because if the first parser passes and then
// the second one fails, we need to revert back to the original state.
let original_state = state.clone();
@ -1121,23 +1120,9 @@ macro_rules! and {
match $p1.parse(arena, state) {
Ok((p1, out1, state)) => match $p2.parse(arena, state) {
Ok((p2, out2, state)) => Ok((p1.or(p2), (out1, out2), state)),
Err((p2, fail, _)) => Err((
p1.or(p2),
Fail {
attempting: original_state.attempting,
..fail
},
original_state,
)),
Err((p2, fail, _)) => Err((p1.or(p2), fail, original_state)),
},
Err((progress, fail, state)) => Err((
progress,
Fail {
attempting: original_state.attempting,
..fail
},
state,
)),
Err((progress, fail, state)) => Err((progress, fail, state)),
}
}
};
@ -1147,20 +1132,11 @@ macro_rules! and {
macro_rules! one_of {
($p1:expr, $p2:expr) => {
move |arena: &'a bumpalo::Bump, state: $crate::parser::State<'a>| {
let original_attempting = state.attempting;
match $p1.parse(arena, state) {
valid @ Ok(_) => valid,
Err((MadeProgress, fail, state)) => Err((MadeProgress, fail, state)),
Err((NoProgress, _, state)) => $p2.parse(
arena,
State {
// Try again, using the original `attempting` value.
// We don't care what the failed first attempt was trying to do.
attempting: original_attempting,
..state
},
),
Err((NoProgress, _, state)) => $p2.parse( arena, state),
}
}
};
@ -1270,18 +1246,16 @@ macro_rules! one_or_more {
buf.push(next_output);
}
Err((progress, fail, old_state)) => {
return fail_when_progress(progress, fail, buf, old_state)
return $crate::parser::fail_when_progress(
progress, fail, buf, old_state,
)
}
}
}
}
Err((progress, _, new_state)) => {
debug_assert_eq!(progress, NoProgress, "{:?}", &new_state);
Err($crate::parser::unexpected_eof(
0,
new_state.attempting,
new_state,
))
Err($crate::parser::unexpected_eof(arena, new_state, 0))
}
}
}
@ -1298,30 +1272,23 @@ macro_rules! debug {
#[macro_export]
macro_rules! attempt {
($attempting:expr, $parser:expr) => {
move |arena, state: $crate::parser::State<'a>| {
use crate::parser::State;
let original_attempting = state.attempting;
move |arena, mut state: $crate::parser::State<'a>| {
let item = $crate::parser::ContextItem {
context: $attempting,
line: state.line,
column: state.column,
};
state.context_stack.push(item);
$parser
.parse(
arena,
State {
attempting: $attempting,
..state
},
)
.map(|(progress, answer, state)| {
.parse(arena, state)
.map(|(progress, answer, mut state)| {
// If the parser suceeded, go back to what we were originally attempting.
// (If it failed, that's exactly where we care what we were attempting!)
(
progress,
answer,
State {
attempting: original_attempting,
..state
},
)
// debug_assert_eq!(!state.context_stack.is_empty());
state.context_stack.pop().unwrap();
(progress, answer, state)
})
}
};
@ -1330,37 +1297,19 @@ macro_rules! attempt {
#[macro_export]
macro_rules! either {
($p1:expr, $p2:expr) => {
move |arena: &'a bumpalo::Bump, state: $crate::parser::State<'a>| {
use $crate::parser::Fail;
let original_attempting = state.attempting;
match $p1.parse(arena, state) {
Ok((progress, output, state)) => {
Ok((progress, $crate::parser::Either::First(output), state))
}
Err((NoProgress, _, state)) => match $p2.parse(arena, state) {
Ok((progress, output, state)) => {
Ok((progress, $crate::parser::Either::Second(output), state))
}
Err((progress, fail, state)) => Err((
progress,
Fail {
attempting: original_attempting,
..fail
},
state,
)),
},
Err((MadeProgress, fail, state)) => Err((
MadeProgress,
Fail {
attempting: original_attempting,
..fail
},
state,
)),
move |arena: &'a bumpalo::Bump, state: $crate::parser::State<'a>| match $p1
.parse(arena, state)
{
Ok((progress, output, state)) => {
Ok((progress, $crate::parser::Either::First(output), state))
}
Err((NoProgress, _, state)) => match $p2.parse(arena, state) {
Ok((progress, output, state)) => {
Ok((progress, $crate::parser::Either::Second(output), state))
}
Err((progress, fail, state)) => Err((progress, fail, state)),
},
Err((MadeProgress, fail, state)) => Err((MadeProgress, fail, state)),
}
};
}
@ -1544,16 +1493,15 @@ pub fn parse_utf8(bytes: &[u8]) -> Result<&str, FailReason> {
}
pub fn end_of_file<'a>() -> impl Parser<'a, ()> {
|_arena: &'a Bump, state: State<'a>| {
|arena: &'a Bump, state: State<'a>| {
if state.has_reached_end() {
Ok((NoProgress, (), state))
} else {
let fail = Fail {
attempting: state.attempting,
reason: FailReason::ConditionFailed,
};
Err((NoProgress, fail, state))
Err((
NoProgress,
Bag::from_state(arena, &state, FailReason::ConditionFailed),
state,
))
}
}
}

58
compiler/parse/src/string_literal.rs
View file

@ -2,7 +2,7 @@ use crate::ast::{Attempting, EscapedChar, StrLiteral, StrSegment};
use crate::expr;
use crate::parser::Progress::*;
use crate::parser::{
allocated, ascii_char, ascii_hex_digits, loc, parse_utf8, unexpected, unexpected_eof, Fail,
allocated, ascii_char, ascii_hex_digits, loc, parse_utf8, unexpected, unexpected_eof, Bag,
FailReason, ParseResult, Parser, State,
};
use bumpalo::collections::vec::Vec;
@ -18,16 +18,16 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>> {
match bytes.next() {
Some(&byte) => {
if byte != b'"' {
return Err(unexpected(0, state, Attempting::StrLiteral));
return Err(unexpected(arena, 0, Attempting::StrLiteral, state));
}
}
None => {
return Err(unexpected_eof(0, Attempting::StrLiteral, state));
return Err(unexpected_eof(arena, state, 0));
}
}
// Advance past the opening quotation mark.
state = state.advance_without_indenting(1)?;
state = state.advance_without_indenting(arena, 1)?;
// At the parsing stage we keep the entire raw string, because the formatter
// needs the raw string. (For example, so it can "remember" whether you
@ -44,7 +44,7 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>> {
segments.push(StrSegment::EscapedChar($ch));
// Advance past the segment we just added
state = state.advance_without_indenting(segment_parsed_bytes)?;
state = state.advance_without_indenting(arena, segment_parsed_bytes)?;
// Reset the segment
segment_parsed_bytes = 0;
@ -63,12 +63,12 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>> {
match parse_utf8(string_bytes) {
Ok(string) => {
state = state.advance_without_indenting(string.len())?;
state = state.advance_without_indenting(arena, string.len())?;
segments.push($transform(string));
}
Err(reason) => {
return state.fail(MadeProgress, reason);
return state.fail(arena, MadeProgress, reason);
}
}
}
@ -105,7 +105,7 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>> {
return Ok((
MadeProgress,
PlainLine(""),
state.advance_without_indenting(1)?,
state.advance_without_indenting(arena, 1)?,
));
}
}
@ -128,7 +128,11 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>> {
};
// Advance the state 1 to account for the closing `"`
return Ok((MadeProgress, expr, state.advance_without_indenting(1)?));
return Ok((
MadeProgress,
expr,
state.advance_without_indenting(arena, 1)?,
));
};
}
b'\n' => {
@ -138,9 +142,10 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>> {
// it should make it easiest to debug; the file will be a giant
// error starting from where the open quote appeared.
return Err(unexpected(
arena,
state.bytes.len() - 1,
state,
Attempting::StrLiteral,
state,
));
}
b'\\' => {
@ -158,7 +163,7 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>> {
match bytes.next() {
Some(b'(') => {
// Advance past the `\(` before using the expr parser
state = state.advance_without_indenting(2)?;
state = state.advance_without_indenting(arena, 2)?;
let original_byte_count = state.bytes.len();
@ -183,7 +188,7 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>> {
}
Some(b'u') => {
// Advance past the `\u` before using the expr parser
state = state.advance_without_indenting(2)?;
state = state.advance_without_indenting(arena, 2)?;
let original_byte_count = state.bytes.len();
@ -228,9 +233,10 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>> {
// by either an open paren or else one of the
// escapable characters (\n, \t, \", \\, etc)
return Err(unexpected(
arena,
state.bytes.len() - 1,
state,
Attempting::StrLiteral,
state,
));
}
}
@ -242,11 +248,7 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>> {
}
// We ran out of characters before finding a closed quote
Err(unexpected_eof(
state.bytes.len(),
Attempting::StrLiteral,
state.clone(),
))
Err(unexpected_eof(arena, state.clone(), state.bytes.len()))
}
}
@ -289,17 +291,18 @@ where
// Ok((StrLiteral::Block(lines.into_bump_slice()), state))
Err((
MadeProgress,
Fail {
attempting: state.attempting,
reason: FailReason::NotYetImplemented(format!(
Bag::from_state(
arena,
&state,
FailReason::NotYetImplemented(format!(
"TODO parse this line in a block string: {:?}",
line
)),
},
),
state,
))
}
Err(reason) => state.fail(MadeProgress, reason),
Err(reason) => state.fail(arena, MadeProgress, reason),
};
}
quotes_seen += 1;
@ -316,7 +319,7 @@ where
line_start = parsed_chars;
}
Err(reason) => {
return state.fail(MadeProgress, reason);
return state.fail(arena, MadeProgress, reason);
}
}
}
@ -329,10 +332,5 @@ where
}
// We ran out of characters before finding 3 closing quotes
Err(unexpected_eof(
parsed_chars,
// TODO custom BlockStrLiteral?
Attempting::StrLiteral,
state,
))
Err(unexpected_eof(arena, state, parsed_chars))
}

19
compiler/parse/src/test_helpers.rs
View file

@ -2,18 +2,18 @@ use crate::ast::{self, Attempting};
use crate::blankspace::space0_before;
use crate::expr::expr;
use crate::module::{header, module_defs};
use crate::parser::{loc, Fail, Parser, State};
use crate::parser::{loc, Bag, Parser, State};
use bumpalo::collections::Vec;
use bumpalo::Bump;
use roc_region::all::Located;
#[allow(dead_code)]
pub fn parse_expr_with<'a>(arena: &'a Bump, input: &'a str) -> Result<ast::Expr<'a>, Fail> {
pub fn parse_expr_with<'a>(arena: &'a Bump, input: &'a str) -> Result<ast::Expr<'a>, Bag<'a>> {
parse_loc_with(arena, input).map(|loc_expr| loc_expr.value)
}
pub fn parse_header_with<'a>(arena: &'a Bump, input: &'a str) -> Result<ast::Module<'a>, Fail> {
let state = State::new(input.trim().as_bytes(), Attempting::Module);
pub fn parse_header_with<'a>(arena: &'a Bump, input: &'a str) -> Result<ast::Module<'a>, Bag<'a>> {
let state = State::new_in(arena, input.trim().as_bytes(), Attempting::Module);
let answer = header().parse(arena, state);
answer
@ -25,8 +25,8 @@ pub fn parse_header_with<'a>(arena: &'a Bump, input: &'a str) -> Result<ast::Mod
pub fn parse_defs_with<'a>(
arena: &'a Bump,
input: &'a str,
) -> Result<Vec<'a, Located<ast::Def<'a>>>, Fail> {
let state = State::new(input.trim().as_bytes(), Attempting::Module);
) -> Result<Vec<'a, Located<ast::Def<'a>>>, Bag<'a>> {
let state = State::new_in(arena, input.trim().as_bytes(), Attempting::Module);
let answer = module_defs().parse(arena, state);
answer
.map(|(_, loc_expr, _)| loc_expr)
@ -34,8 +34,11 @@ pub fn parse_defs_with<'a>(
}
#[allow(dead_code)]
pub fn parse_loc_with<'a>(arena: &'a Bump, input: &'a str) -> Result<Located<ast::Expr<'a>>, Fail> {
let state = State::new(input.trim().as_bytes(), Attempting::Module);
pub fn parse_loc_with<'a>(
arena: &'a Bump,
input: &'a str,
) -> Result<Located<ast::Expr<'a>>, Bag<'a>> {
let state = State::new_in(arena, input.trim().as_bytes(), Attempting::Module);
let parser = space0_before(loc(expr(0)), 0);
let answer = parser.parse(&arena, state);

38
compiler/parse/src/type_annotation.rs
View file

@ -4,10 +4,10 @@ use crate::expr::{global_tag, private_tag};
use crate::ident::join_module_parts;
use crate::keyword;
use crate::parser::{
allocated, ascii_char, ascii_string, not, optional, peek_utf8_char, unexpected, Either, Fail,
allocated, ascii_char, ascii_string, not, optional, peek_utf8_char, unexpected, Either,
FailReason, ParseResult, Parser,
Progress::{self, *},
State,
State, Bag,
};
use bumpalo::collections::string::String;
use bumpalo::collections::vec::Vec;
@ -258,10 +258,10 @@ fn expression<'a>(min_indent: u16) -> impl Parser<'a, Located<TypeAnnotation<'a>
// e.g. `Int,Int` without an arrow and return type
Err((
progress,
Fail {
attempting: state.attempting,
reason: FailReason::NotYetImplemented("TODO: Decide the correct error to return for 'Invalid function signature'".to_string()),
},
Bag::from_state(arena, &state,
FailReason::NotYetImplemented("TODO: Decide the correct error to return for 'Invalid function signature'".to_string()),
),
state,
))
}
@ -300,12 +300,12 @@ fn parse_concrete_type<'a>(
if first_letter.is_alphabetic() && first_letter.is_uppercase() {
part_buf.push(first_letter);
} else {
return Err(unexpected(0, state, Attempting::ConcreteType));
return Err(unexpected(arena, 0, Attempting::ConcreteType, state));
}
state = state.advance_without_indenting(bytes_parsed)?;
state = state.advance_without_indenting(arena, bytes_parsed)?;
}
Err(reason) => return state.fail(NoProgress, reason),
Err(reason) => return state.fail(arena, NoProgress, reason),
}
let mut next_char = None;
@ -349,12 +349,12 @@ fn parse_concrete_type<'a>(
break;
}
state = state.advance_without_indenting(bytes_parsed)?;
state = state.advance_without_indenting(arena, bytes_parsed)?;
}
Err(reason) => {
let progress = Progress::from_lengths(start_bytes_len, state.bytes.len());
return state.fail(progress, reason);
return state.fail(arena, progress, reason);
}
}
}
@ -373,7 +373,7 @@ fn parse_concrete_type<'a>(
// We had neither capitalized nor noncapitalized parts,
// yet we made it this far. The only explanation is that this was
// a stray '.' drifting through the cosmos.
return Err(unexpected(1, state, Attempting::Identifier));
return Err(unexpected(arena, 1, Attempting::Identifier, state));
}
let answer = TypeAnnotation::Apply(
@ -400,14 +400,14 @@ fn parse_type_variable<'a>(
if first_letter.is_alphabetic() && first_letter.is_lowercase() {
buf.push(first_letter);
} else {
return Err(unexpected(0, state, Attempting::TypeVariable));
return Err(unexpected(arena, 0, Attempting::TypeVariable, state));
}
state = state.advance_without_indenting(bytes_parsed)?;
state = state.advance_without_indenting(arena, bytes_parsed)?;
}
Err(reason) => {
let progress = Progress::from_lengths(start_bytes_len, state.bytes.len());
return state.fail(progress, reason);
return state.fail(arena, progress, reason);
}
}
@ -425,11 +425,11 @@ fn parse_type_variable<'a>(
break;
}
state = state.advance_without_indenting(bytes_parsed)?;
state = state.advance_without_indenting(arena, bytes_parsed)?;
}
Err(reason) => {
let progress = Progress::from_lengths(start_bytes_len, state.bytes.len());
return state.fail(progress, reason);
return state.fail(arena, progress, reason);
}
}
}
@ -469,9 +469,9 @@ fn malformed<'a>(
break;
}
state = state.advance_without_indenting(bytes_parsed)?;
state = state.advance_without_indenting(arena, bytes_parsed)?;
}
Err(reason) => return state.fail(MadeProgress, reason),
Err(reason) => return state.fail(arena, MadeProgress, reason),
}
}