language-servers/roc
1
0
Fork 0
mirror of https://github.com/roc-lang/roc.git synced 2025-09-30 23:31:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Merge remote-tracking branch 'origin/trunk' into single-quote-literal

Browse source
This commit is contained in:
Brendan Hansknecht 2022-02-24 09:35:01 -08:00
commit f7c0e2ef19
634 changed files with 47014 additions and 21117 deletions
Download patch file Download diff file Expand all files Collapse all files

109
compiler/parse/src/string_literal.rs
View file

@ -1,7 +1,8 @@
use crate::ast::{EscapedChar, StrLiteral, StrSegment};
use crate::expr;
use crate::parser::Progress::*;
use crate::parser::{allocated, loc, specialize_ref, word1, BadInputError, EString, Parser, State};
use crate::parser::{allocated, loc, specialize_ref, word1, BadInputError, EString, Parser};
use crate::state::State;
use bumpalo::collections::vec::Vec;
use bumpalo::Bump;
@ -11,47 +12,35 @@ fn ascii_hex_digits<'a>() -> impl Parser<'a, &'a str, EString<'a>> {
move |arena, state: State<'a>| {
let mut buf = bumpalo::collections::String::new_in(arena);
for &byte in state.bytes.iter() {
for &byte in state.bytes().iter() {
if (byte as char).is_ascii_hexdigit() {
buf.push(byte as char);
} else if buf.is_empty() {
// We didn't find any hex digits!
return Err((
NoProgress,
EString::CodePtEnd(state.line, state.column),
state,
));
return Err((NoProgress, EString::CodePtEnd(state.pos()), state));
} else {
let state = state.advance_without_indenting_ee(buf.len(), |r, c| {
EString::Space(BadInputError::LineTooLong, r, c)
})?;
let state = state.advance(buf.len());
return Ok((MadeProgress, buf.into_bump_str(), state));
}
}
Err((
NoProgress,
EString::CodePtEnd(state.line, state.column),
state,
))
Err((NoProgress, EString::CodePtEnd(state.pos()), state))
}
}
macro_rules! advance_state {
($state:expr, $n:expr) => {
$state.advance_without_indenting_ee($n, |r, c| {
EString::Space(BadInputError::LineTooLong, r, c)
})
Ok($state.advance($n))
};
}
pub fn parse_single_quote<'a>() -> impl Parser<'a, &'a str, EString<'a>> {
move |arena: &'a Bump, mut state: State<'a>| {
if state.bytes.starts_with(b"\'") {
if state.bytes().starts_with(b"\'") {
// we will be parsing a single-quote-string
} else {
return Err((NoProgress, EString::Open(state.line, state.column), state));
return Err((NoProgress, EString::Open(state.pos()), state));
}
// early return did not hit, just advance one byte
@ -60,14 +49,14 @@ pub fn parse_single_quote<'a>() -> impl Parser<'a, &'a str, EString<'a>> {
// Handle back slaches in byte literal
// - starts with a backslash and used as an escape character. ex: '\n', '\t'
// - single quote floating (un closed single quote) should be an error
match state.bytes.first() {
match state.bytes().first() {
Some(b'\\') => {
state = advance_state!(state, 1)?;
match state.bytes.first() {
match state.bytes().first() {
Some(&ch) => {
state = advance_state!(state, 1)?;
if (ch == b'n' || ch == b'r' || ch == b't' || ch == b'\'' || ch == b'\\')
&& (state.bytes.first() == Some(&b'\''))
&& (state.bytes().first() == Some(&b'\''))
{
state = advance_state!(state, 1)?;
// since we checked the current char between the single quotes we
@ -77,35 +66,21 @@ pub fn parse_single_quote<'a>() -> impl Parser<'a, &'a str, EString<'a>> {
return Ok((MadeProgress, &*arena.alloc_str(&test.to_string()), state));
}
// invalid error, backslah escaping something we do not recognize
return Err((
NoProgress,
EString::CodePtEnd(state.line, state.column),
state,
));
return Err((NoProgress, EString::CodePtEnd(state.pos()), state));
}
None => {
// no close quote found
return Err((
NoProgress,
EString::CodePtEnd(state.line, state.column),
state,
));
return Err((NoProgress, EString::CodePtEnd(state.pos()), state));
}
}
}
Some(_) => {
// do nothing for other characters, handled below
}
None => {
return Err((
NoProgress,
EString::CodePtEnd(state.line, state.column),
state,
))
}
None => return Err((NoProgress, EString::CodePtEnd(state.pos()), state)),
}
let mut bytes = state.bytes.iter();
let mut bytes = state.bytes().iter();
let mut end_index = 1;
// Copy paste problem in mono
@ -117,7 +92,7 @@ pub fn parse_single_quote<'a>() -> impl Parser<'a, &'a str, EString<'a>> {
}
Some(_) => end_index += 1,
None => {
return Err((NoProgress, EString::Open(state.line, state.column), state));
return Err((NoProgress, EString::Open(state.pos()), state));
}
}
}
@ -126,28 +101,24 @@ pub fn parse_single_quote<'a>() -> impl Parser<'a, &'a str, EString<'a>> {
// no progress was made
// this case is a double single quote, ex: ''
// not supporting empty single quotes
return Err((NoProgress, EString::Open(state.line, state.column), state));
return Err((NoProgress, EString::Open(state.pos()), state));
}
if end_index > (std::mem::size_of::<u32>() + 1) {
// bad case: too big to fit into u32
return Err((NoProgress, EString::Open(state.line, state.column), state));
return Err((NoProgress, EString::Open(state.pos()), state));
}
// happy case -> we have some bytes that will fit into a u32
// ending up w/ a slice of bytes that we want to convert into an integer
let raw_bytes = &state.bytes[0..end_index - 1];
let raw_bytes = &state.bytes()[0..end_index - 1];
state = advance_state!(state, end_index)?;
match std::str::from_utf8(raw_bytes) {
Ok(string) => Ok((MadeProgress, string, state)),
Err(_) => {
// invalid UTF-8
return Err((
NoProgress,
EString::CodePtEnd(state.line, state.column),
state,
));
return Err((NoProgress, EString::CodePtEnd(state.pos()), state));
}
}
}
@ -160,18 +131,18 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>, EString<'a>> {
let is_multiline;
let mut bytes;
if state.bytes.starts_with(b"\"\"\"") {
if state.bytes().starts_with(b"\"\"\"") {
// we will be parsing a multi-string
is_multiline = true;
bytes = state.bytes[3..].iter();
bytes = state.bytes()[3..].iter();
state = advance_state!(state, 3)?;
} else if state.bytes.starts_with(b"\"") {
} else if state.bytes().starts_with(b"\"") {
// we will be parsing a single-string
is_multiline = false;
bytes = state.bytes[1..].iter();
bytes = state.bytes()[1..].iter();
state = advance_state!(state, 1)?;
} else {
return Err((NoProgress, EString::Open(state.line, state.column), state));
return Err((NoProgress, EString::Open(state.pos()), state));
}
// At the parsing stage we keep the entire raw string, because the formatter
@ -204,7 +175,7 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>, EString<'a>> {
// something which signalled that we should end the
// current segment - so use segment_parsed_bytes - 1 here,
// to exclude that char we just parsed.
let string_bytes = &state.bytes[0..(segment_parsed_bytes - 1)];
let string_bytes = &state.bytes()[0..(segment_parsed_bytes - 1)];
match std::str::from_utf8(string_bytes) {
Ok(string) => {
@ -215,7 +186,7 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>, EString<'a>> {
Err(_) => {
return Err((
MadeProgress,
EString::Space(BadInputError::BadUtf8, state.line, state.column),
EString::Space(BadInputError::BadUtf8, state.pos()),
state,
));
}
@ -307,11 +278,7 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>, EString<'a>> {
// all remaining chars. This will mask all other errors, but
// it should make it easiest to debug; the file will be a giant
// error starting from where the open quote appeared.
return Err((
MadeProgress,
EString::EndlessSingle(state.line, state.column),
state,
));
return Err((MadeProgress, EString::EndlessSingle(state.pos()), state));
}
}
b'\\' => {
@ -331,7 +298,7 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>, EString<'a>> {
// Advance past the `\(` before using the expr parser
state = advance_state!(state, 2)?;
let original_byte_count = state.bytes.len();
let original_byte_count = state.bytes().len();
// This is an interpolated variable.
// Parse an arbitrary expression, then give a
@ -344,7 +311,7 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>, EString<'a>> {
.parse(arena, state)?;
// Advance the iterator past the expr we just parsed.
for _ in 0..(original_byte_count - new_state.bytes.len()) {
for _ in 0..(original_byte_count - new_state.bytes().len()) {
bytes.next();
}
@ -358,7 +325,7 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>, EString<'a>> {
// Advance past the `\u` before using the expr parser
state = advance_state!(state, 2)?;
let original_byte_count = state.bytes.len();
let original_byte_count = state.bytes().len();
// Parse the hex digits, surrounded by parens, then
// give a canonicalization error if the digits form
@ -371,7 +338,7 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>, EString<'a>> {
.parse(arena, state)?;
// Advance the iterator past the expr we just parsed.
for _ in 0..(original_byte_count - new_state.bytes.len()) {
for _ in 0..(original_byte_count - new_state.bytes().len()) {
bytes.next();
}
@ -400,11 +367,7 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>, EString<'a>> {
// Invalid escape! A backslash must be followed
// by either an open paren or else one of the
// escapable characters (\n, \t, \", \\, etc)
return Err((
MadeProgress,
EString::UnknownEscape(state.line, state.column),
state,
));
return Err((MadeProgress, EString::UnknownEscape(state.pos()), state));
}
}
}
@ -418,9 +381,9 @@ pub fn parse<'a>() -> impl Parser<'a, StrLiteral<'a>, EString<'a>> {
Err((
MadeProgress,
if is_multiline {
EString::EndlessMulti(state.line, state.column)
EString::EndlessMulti(state.pos())
} else {
EString::EndlessSingle(state.line, state.column)
EString::EndlessSingle(state.pos())
},
state,
))