Move Full from fmt to parse and reorganize confusingly-named Module ast type

This commit is contained in:
Joshua Warner 2024-08-06 19:03:08 -07:00
parent 698bbc3cf1
commit d25c048d48
No known key found for this signature in database
GPG key ID: 89AD497003F93FDD
47 changed files with 1759 additions and 1776 deletions

View file

@ -5,11 +5,12 @@ use std::path::{Path, PathBuf};
use bumpalo::Bump;
use roc_error_macros::{internal_error, user_error};
use roc_fmt::def::fmt_defs;
use roc_fmt::module::fmt_module;
use roc_fmt::{Ast, Buf};
use roc_parse::module::parse_module_defs;
use roc_fmt::header::fmt_header;
use roc_fmt::Buf;
use roc_parse::ast::{Full, SpacesBefore};
use roc_parse::header::parse_module_defs;
use roc_parse::remove_spaces::RemoveSpaces;
use roc_parse::{module, parser::SyntaxError, state::State};
use roc_parse::{header, parser::SyntaxError, state::State};
#[derive(Copy, Clone, Debug)]
pub enum FormatMode {
@ -230,19 +231,25 @@ pub fn format_src(arena: &Bump, src: &str) -> Result<String, FormatProblem> {
Ok(buf.as_str().to_string())
}
fn parse_all<'a>(arena: &'a Bump, src: &'a str) -> Result<Ast<'a>, SyntaxError<'a>> {
let (module, state) = module::parse_header(arena, State::new(src.as_bytes()))
fn parse_all<'a>(arena: &'a Bump, src: &'a str) -> Result<Full<'a>, SyntaxError<'a>> {
let (header, state) = header::parse_header(arena, State::new(src.as_bytes()))
.map_err(|e| SyntaxError::Header(e.problem))?;
let (module, defs) = module.upgrade_header_imports(arena);
let (h, defs) = header.item.upgrade_header_imports(arena);
let defs = parse_module_defs(arena, state, defs)?;
Ok(Ast { module, defs })
Ok(Full {
header: SpacesBefore {
before: header.before,
item: h,
},
defs,
})
}
fn fmt_all<'a>(buf: &mut Buf<'a>, ast: &'a Ast) {
fmt_module(buf, &ast.module);
fn fmt_all<'a>(buf: &mut Buf<'a>, ast: &'a Full) {
fmt_header(buf, &ast.header);
fmt_defs(buf, &ast.defs, 0);

View file

@ -5,7 +5,7 @@ use crate::collection::{fmt_collection, Braces};
use crate::expr::fmt_str_literal;
use crate::spaces::{fmt_comments_only, fmt_default_spaces, fmt_spaces, NewlineAt, INDENT};
use crate::Buf;
use roc_parse::ast::{Collection, CommentOrNewline, Header, Module, Spaced, Spaces};
use roc_parse::ast::{Collection, CommentOrNewline, Header, Spaced, Spaces, SpacesBefore};
use roc_parse::header::{
AppHeader, ExposedName, ExposesKeyword, GeneratesKeyword, HostedHeader, ImportsEntry,
ImportsKeyword, Keyword, KeywordItem, ModuleHeader, ModuleName, PackageEntry, PackageHeader,
@ -16,9 +16,9 @@ use roc_parse::header::{
use roc_parse::ident::UppercaseIdent;
use roc_region::all::Loc;
pub fn fmt_module<'a>(buf: &mut Buf<'_>, module: &'a Module<'a>) {
fmt_comments_only(buf, module.comments.iter(), NewlineAt::Bottom, 0);
match &module.header {
pub fn fmt_header<'a>(buf: &mut Buf<'_>, header: &'a SpacesBefore<'a, Header<'a>>) {
fmt_comments_only(buf, header.before.iter(), NewlineAt::Bottom, 0);
match &header.item {
Header::Module(header) => {
fmt_module_header(buf, header);
}

View file

@ -6,18 +6,11 @@ pub mod annotation;
pub mod collection;
pub mod def;
pub mod expr;
pub mod module;
pub mod header;
pub mod pattern;
pub mod spaces;
use bumpalo::{collections::String, Bump};
use roc_parse::ast::Module;
#[derive(Debug)]
pub struct Ast<'a> {
pub module: Module<'a>,
pub defs: roc_parse::ast::Defs<'a>,
}
#[derive(Debug)]
pub struct Buf<'a> {

View file

@ -1,7 +1,6 @@
use bumpalo::Bump;
use roc_parse::{ast::CommentOrNewline, remove_spaces::RemoveSpaces};
use roc_parse::ast::CommentOrNewline;
use crate::{Ast, Buf};
use crate::Buf;
/// The number of spaces to indent.
pub const INDENT: u16 = 4;
@ -192,12 +191,3 @@ fn fmt_docs(buf: &mut Buf, docs: &str) {
}
buf.push_str(docs.trim_end());
}
impl<'a> RemoveSpaces<'a> for Ast<'a> {
fn remove_spaces(&self, arena: &'a Bump) -> Self {
Ast {
module: self.module.remove_spaces(arena),
defs: self.defs.remove_spaces(arena),
}
}
}

View file

@ -16,7 +16,7 @@ mod test_reporting {
use roc_load::{self, ExecutionMode, LoadConfig, LoadedModule, LoadingProblem, Threading};
use roc_module::symbol::{Interns, ModuleId};
use roc_packaging::cache::RocCacheDir;
use roc_parse::module::parse_header;
use roc_parse::header::parse_header;
use roc_parse::state::State;
use roc_parse::test_helpers::parse_expr_with;
use roc_problem::Severity;
@ -359,7 +359,7 @@ mod test_reporting {
let src_lines: Vec<&str> = src.split('\n').collect();
let lines = LineInfo::new(src);
match roc_parse::module::parse_header(arena, state) {
match roc_parse::header::parse_header(arena, state) {
Err(fail) => {
let interns = Interns::default();
let home = crate::helpers::test_home();

View file

@ -48,11 +48,11 @@ use roc_mono::reset_reuse;
use roc_mono::{drop_specialization, inc_dec};
use roc_packaging::cache::RocCacheDir;
use roc_parse::ast::{self, CommentOrNewline, ExtractSpaces, Spaced, ValueDef};
use roc_parse::header::parse_module_defs;
use roc_parse::header::{
self, AppHeader, ExposedName, HeaderType, ImportsKeywordItem, PackageEntry, PackageHeader,
PlatformHeader, To, TypedIdent,
};
use roc_parse::module::parse_module_defs;
use roc_parse::parser::{FileError, SourceError, SyntaxError};
use roc_problem::Severity;
use roc_region::all::{LineInfo, Loc, Region};
@ -1327,8 +1327,8 @@ fn load_packages_from_main<'a>(
let parse_state = roc_parse::state::State::new(arena.alloc(src_bytes));
let (parsed_module, _) =
roc_parse::module::parse_header(arena, parse_state.clone()).map_err(|fail| {
let (parsed_header, _) =
roc_parse::header::parse_header(arena, parse_state.clone()).map_err(|fail| {
LoadingProblem::ParsingFailed(
fail.map_problem(SyntaxError::Header)
.into_file_error(filename.clone()),
@ -1337,7 +1337,7 @@ fn load_packages_from_main<'a>(
use ast::Header::*;
let packages = match parsed_module.header {
let packages = match parsed_header.item {
App(AppHeader { packages, .. }) | Package(PackageHeader { packages, .. }) => {
unspace(arena, packages.value.items)
}
@ -3349,7 +3349,7 @@ fn load_package_from_disk<'a>(
let parse_start = Instant::now();
let bytes = arena.alloc(bytes_vec);
let parse_state = roc_parse::state::State::new(bytes);
let parsed = roc_parse::module::parse_header(arena, parse_state.clone());
let parsed = roc_parse::header::parse_header(arena, parse_state.clone());
let parse_header_duration = parse_start.elapsed();
// Insert the first entries for this module's timings
@ -3360,8 +3360,8 @@ fn load_package_from_disk<'a>(
match parsed {
Ok((
ast::Module {
header: ast::Header::Module(header),
ast::SpacesBefore {
item: ast::Header::Module(header),
..
},
_parse_state,
@ -3369,8 +3369,8 @@ fn load_package_from_disk<'a>(
"expected platform/package module, got Module with header\n{header:?}"
))),
Ok((
ast::Module {
header: ast::Header::Hosted(header),
ast::SpacesBefore {
item: ast::Header::Hosted(header),
..
},
_parse_state,
@ -3378,8 +3378,8 @@ fn load_package_from_disk<'a>(
"expected platform/package module, got Hosted module with header\n{header:?}"
))),
Ok((
ast::Module {
header: ast::Header::App(header),
ast::SpacesBefore {
item: ast::Header::App(header),
..
},
_parse_state,
@ -3387,9 +3387,9 @@ fn load_package_from_disk<'a>(
"expected platform/package module, got App with header\n{header:?}"
))),
Ok((
ast::Module {
header: ast::Header::Package(header),
comments,
ast::SpacesBefore {
item: ast::Header::Package(header),
before: comments,
},
parser_state,
)) => {
@ -3430,9 +3430,9 @@ fn load_package_from_disk<'a>(
Ok(Msg::Many(messages))
}
Ok((
ast::Module {
header: ast::Header::Platform(header),
comments,
ast::SpacesBefore {
item: ast::Header::Platform(header),
before: comments,
},
parser_state,
)) => {
@ -3530,13 +3530,13 @@ fn load_builtin_module_help<'a>(
let opt_shorthand = None;
let filename = PathBuf::from(filename);
let parse_state = roc_parse::state::State::new(src_bytes.as_bytes());
let parsed = roc_parse::module::parse_header(arena, parse_state.clone());
let parsed = roc_parse::header::parse_header(arena, parse_state.clone());
match parsed {
Ok((
ast::Module {
header: ast::Header::Module(header),
comments,
ast::SpacesBefore {
item: ast::Header::Module(header),
before: comments,
},
parse_state,
)) => {
@ -3786,7 +3786,7 @@ fn parse_header<'a>(
) -> Result<HeaderOutput<'a>, LoadingProblem<'a>> {
let parse_start = Instant::now();
let parse_state = roc_parse::state::State::new(src_bytes);
let parsed = roc_parse::module::parse_header(arena, parse_state.clone());
let parsed = roc_parse::header::parse_header(arena, parse_state.clone());
let parse_header_duration = parse_start.elapsed();
if let Err(problem) = ensure_roc_file(&filename, src_bytes) {
@ -3815,9 +3815,9 @@ fn parse_header<'a>(
match parsed {
Ok((
ast::Module {
header: ast::Header::Module(header),
comments,
ast::SpacesBefore {
item: ast::Header::Module(header),
before: comments,
},
parse_state,
)) => {
@ -3852,9 +3852,9 @@ fn parse_header<'a>(
})
}
Ok((
ast::Module {
header: ast::Header::Hosted(header),
comments,
ast::SpacesBefore {
item: ast::Header::Hosted(header),
before: comments,
},
parse_state,
)) => {
@ -3883,9 +3883,9 @@ fn parse_header<'a>(
})
}
Ok((
ast::Module {
header: ast::Header::App(header),
comments,
ast::SpacesBefore {
item: ast::Header::App(header),
before: comments,
},
parse_state,
)) => {
@ -3988,9 +3988,9 @@ fn parse_header<'a>(
})
}
Ok((
ast::Module {
header: ast::Header::Package(header),
comments,
ast::SpacesBefore {
item: ast::Header::Package(header),
before: comments,
},
parse_state,
)) => {
@ -4015,9 +4015,9 @@ fn parse_header<'a>(
}
Ok((
ast::Module {
header: ast::Header::Platform(header),
comments,
ast::SpacesBefore {
item: ast::Header::Platform(header),
before: comments,
},
parse_state,
)) => {
@ -5153,7 +5153,7 @@ fn parse<'a>(
let parse_state = header.parse_state;
let header_import_defs =
roc_parse::ast::Module::header_imports_to_defs(arena, header.header_imports);
roc_parse::ast::Header::header_imports_to_defs(arena, header.header_imports);
let parsed_defs = match parse_module_defs(arena, parse_state.clone(), header_import_defs) {
Ok(success) => success,

View file

@ -2,7 +2,7 @@ use bumpalo::Bump;
use criterion::{black_box, criterion_group, criterion_main, Criterion};
use roc_parse::{
ast::Defs,
module::{self, parse_module_defs},
header::{self, parse_module_defs},
state::State,
};
use std::path::PathBuf;
@ -20,7 +20,7 @@ pub fn parse_benchmark(c: &mut Criterion) {
let arena = Bump::new();
let (_actual, state) =
module::parse_header(&arena, State::new(src.as_bytes())).unwrap();
header::parse_header(&arena, State::new(src.as_bytes())).unwrap();
let res = parse_module_defs(&arena, state, Defs::default()).unwrap();
@ -41,7 +41,7 @@ pub fn parse_benchmark(c: &mut Criterion) {
let arena = Bump::new();
let (_actual, state) =
module::parse_header(&arena, State::new(src.as_bytes())).unwrap();
header::parse_header(&arena, State::new(src.as_bytes())).unwrap();
let res = parse_module_defs(&arena, state, Defs::default()).unwrap();

View file

@ -14,6 +14,12 @@ use roc_module::called_via::{BinOp, CalledVia, UnaryOp};
use roc_module::ident::QualifiedModuleName;
use roc_region::all::{Loc, Position, Region};
#[derive(Debug, Clone)]
pub struct Full<'a> {
pub header: SpacesBefore<'a, Header<'a>>,
pub defs: Defs<'a>,
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub struct Spaces<'a, T> {
pub before: &'a [CommentOrNewline<'a>],
@ -111,15 +117,9 @@ impl<'a, T: ExtractSpaces<'a>> ExtractSpaces<'a> for Loc<T> {
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct Module<'a> {
pub comments: &'a [CommentOrNewline<'a>],
pub header: Header<'a>,
}
impl<'a> Module<'a> {
impl<'a> Header<'a> {
pub fn upgrade_header_imports(self, arena: &'a Bump) -> (Self, Defs<'a>) {
let (header, defs) = match self.header {
let (header, defs) = match self {
Header::Module(header) => (
Header::Module(ModuleHeader {
interface_imports: None,
@ -134,12 +134,10 @@ impl<'a> Module<'a> {
}),
Self::header_imports_to_defs(arena, header.old_imports),
),
Header::Package(_) | Header::Platform(_) | Header::Hosted(_) => {
(self.header, Defs::default())
}
Header::Package(_) | Header::Platform(_) | Header::Hosted(_) => (self, Defs::default()),
};
(Module { header, ..self }, defs)
(header, defs)
}
pub fn header_imports_to_defs(
@ -2438,9 +2436,9 @@ pub trait Malformed {
fn is_malformed(&self) -> bool;
}
impl<'a> Malformed for Module<'a> {
impl<'a> Malformed for Full<'a> {
fn is_malformed(&self) -> bool {
self.header.is_malformed()
self.header.item.is_malformed() || self.defs.is_malformed()
}
}
@ -2462,6 +2460,12 @@ impl<'a, T: Malformed> Malformed for Spaces<'a, T> {
}
}
impl<'a, T: Malformed> Malformed for SpacesBefore<'a, T> {
fn is_malformed(&self) -> bool {
self.item.is_malformed()
}
}
impl<'a> Malformed for Expr<'a> {
fn is_malformed(&self) -> bool {
use Expr::*;

View file

@ -9,10 +9,10 @@ use crate::blankspace::{
loc_space0_e, require_newline_or_eof, space0_after_e, space0_around_ee, space0_before_e,
space0_before_optional_after, space0_e, spaces, spaces_around, spaces_before,
};
use crate::header::module_name_help;
use crate::ident::{
integer_ident, lowercase_ident, parse_ident, unqualified_ident, Accessor, Ident, Suffix,
};
use crate::module::module_name_help;
use crate::parser::{
self, and, backtrackable, between, byte, byte_indent, collection_inner,
collection_trailing_sep_e, either, increment_min_indent, indented_seq_skip_first, loc, map,
@ -24,8 +24,8 @@ use crate::parser::{
use crate::pattern::closure_param;
use crate::state::State;
use crate::string_literal::{self, StrLikeLiteral};
use crate::type_annotation;
use crate::{header, keyword};
use crate::{module, type_annotation};
use bumpalo::collections::Vec;
use bumpalo::Bump;
use roc_collections::soa::Slice;
@ -948,7 +948,7 @@ fn imported_module_name<'a>() -> impl Parser<'a, ImportedModuleName<'a>, EImport
fn import_as<'a>(
) -> impl Parser<'a, header::KeywordItem<'a, ImportAsKeyword, Loc<ImportAlias<'a>>>, EImport<'a>> {
record!(header::KeywordItem {
keyword: module::spaces_around_keyword(
keyword: header::spaces_around_keyword(
ImportAsKeyword,
EImport::As,
EImport::IndentAs,
@ -982,7 +982,7 @@ fn import_exposing<'a>() -> impl Parser<
EImport<'a>,
> {
record!(header::KeywordItem {
keyword: module::spaces_around_keyword(
keyword: header::spaces_around_keyword(
ImportExposingKeyword,
EImport::Exposing,
EImport::IndentExposing,
@ -1027,7 +1027,7 @@ fn import_ingested_file_body<'a>() -> impl Parser<'a, ValueDef<'a>, EImport<'a>>
fn import_ingested_file_as<'a>(
) -> impl Parser<'a, header::KeywordItem<'a, ImportAsKeyword, Loc<&'a str>>, EImport<'a>> {
record!(header::KeywordItem {
keyword: module::spaces_around_keyword(
keyword: header::spaces_around_keyword(
ImportAsKeyword,
EImport::As,
EImport::IndentAs,

View file

@ -1,19 +1,949 @@
use crate::ast::{
Collection, CommentOrNewline, Malformed, Pattern, Spaced, Spaces, StrLiteral, TypeAnnotation,
};
use crate::blankspace::space0_e;
use crate::expr::merge_spaces;
use crate::ident::{lowercase_ident, UppercaseIdent};
use crate::parser::{
and, byte, loc, map_with_arena, skip_first, skip_second, specialize_err, EPackageEntry,
EPackageName, Parser,
};
use crate::parser::{optional, then};
use crate::string_literal;
use roc_module::symbol::{ModuleId, Symbol};
use roc_region::all::Loc;
use std::fmt::Debug;
use crate::ast::{
Collection, CommentOrNewline, Defs, Header, Malformed, Pattern, Spaced, Spaces, SpacesBefore,
StrLiteral, TypeAnnotation,
};
use crate::blankspace::{space0_around_ee, space0_before_e, space0_e};
use crate::expr::merge_spaces;
use crate::ident::{self, lowercase_ident, unqualified_ident, uppercase, UppercaseIdent};
use crate::parser::Progress::{self, *};
use crate::parser::{
and, backtrackable, byte, collection_trailing_sep_e, increment_min_indent, loc, map,
map_with_arena, optional, reset_min_indent, skip_first, skip_second, specialize_err, succeed,
then, two_bytes, zero_or_more, EExposes, EGenerates, EGeneratesWith, EHeader, EImports,
EPackageEntry, EPackageName, EPackages, EParams, EProvides, ERequires, ETypedIdent, Parser,
SourceError, SpaceProblem, SyntaxError,
};
use crate::pattern::record_pattern_fields;
use crate::state::State;
use crate::string_literal::{self, parse_str_literal};
use crate::type_annotation;
use roc_module::symbol::{ModuleId, Symbol};
use roc_region::all::{Loc, Position, Region};
fn end_of_file<'a>() -> impl Parser<'a, (), SyntaxError<'a>> {
|_arena, state: State<'a>, _min_indent: u32| {
if state.has_reached_end() {
Ok((NoProgress, (), state))
} else {
Err((NoProgress, SyntaxError::NotEndOfFile(state.pos())))
}
}
}
pub fn parse_module_defs<'a>(
arena: &'a bumpalo::Bump,
state: State<'a>,
defs: Defs<'a>,
) -> Result<Defs<'a>, SyntaxError<'a>> {
let min_indent = 0;
match crate::expr::parse_top_level_defs(arena, state.clone(), defs) {
Ok((_, defs, state)) => match end_of_file().parse(arena, state, min_indent) {
Ok(_) => Ok(defs),
Err((_, fail)) => Err(fail),
},
Err((_, fail)) => Err(SyntaxError::Expr(fail, state.pos())),
}
}
pub fn parse_header<'a>(
arena: &'a bumpalo::Bump,
state: State<'a>,
) -> Result<(SpacesBefore<'a, Header<'a>>, State<'a>), SourceError<'a, EHeader<'a>>> {
let min_indent = 0;
match header().parse(arena, state.clone(), min_indent) {
Ok((_, module, state)) => Ok((module, state)),
Err((_, fail)) => Err(SourceError::new(fail, &state)),
}
}
pub fn header<'a>() -> impl Parser<'a, SpacesBefore<'a, Header<'a>>, EHeader<'a>> {
use crate::parser::keyword;
record!(SpacesBefore {
before: space0_e(EHeader::IndentStart),
item: one_of![
map(
skip_first(
keyword("module", EHeader::Start),
increment_min_indent(module_header())
),
Header::Module
),
map(
skip_first(
keyword("interface", EHeader::Start),
increment_min_indent(interface_header())
),
Header::Module
),
map(
skip_first(
keyword("app", EHeader::Start),
increment_min_indent(one_of![app_header(), old_app_header()])
),
Header::App
),
map(
skip_first(
keyword("package", EHeader::Start),
increment_min_indent(one_of![package_header(), old_package_header()])
),
Header::Package
),
map(
skip_first(
keyword("platform", EHeader::Start),
increment_min_indent(platform_header())
),
Header::Platform
),
map(
skip_first(
keyword("hosted", EHeader::Start),
increment_min_indent(hosted_header())
),
Header::Hosted
),
]
})
}
#[inline(always)]
fn module_header<'a>() -> impl Parser<'a, ModuleHeader<'a>, EHeader<'a>> {
record!(ModuleHeader {
after_keyword: space0_e(EHeader::IndentStart),
params: optional(specialize_err(EHeader::Params, module_params())),
exposes: specialize_err(EHeader::Exposes, exposes_list()),
interface_imports: succeed(None)
})
.trace("module_header")
}
fn module_params<'a>() -> impl Parser<'a, ModuleParams<'a>, EParams<'a>> {
record!(ModuleParams {
params: specialize_err(EParams::Pattern, record_pattern_fields()),
before_arrow: skip_second(
space0_e(EParams::BeforeArrow),
loc(two_bytes(b'-', b'>', EParams::Arrow))
),
after_arrow: space0_e(EParams::AfterArrow),
})
}
// TODO does this need to be a macro?
macro_rules! merge_n_spaces {
($arena:expr, $($slice:expr),*) => {
{
let mut merged = bumpalo::collections::Vec::with_capacity_in(0 $(+ $slice.len())*, $arena);
$(merged.extend_from_slice($slice);)*
merged.into_bump_slice()
}
};
}
/// Parse old interface headers so we can format them into module headers
#[inline(always)]
fn interface_header<'a>() -> impl Parser<'a, ModuleHeader<'a>, EHeader<'a>> {
let after_keyword = map_with_arena(
and(
skip_second(
space0_e(EHeader::IndentStart),
loc(module_name_help(EHeader::ModuleName)),
),
specialize_err(EHeader::Exposes, exposes_kw()),
),
|arena: &'a bumpalo::Bump,
(before_name, kw): (&'a [CommentOrNewline<'a>], Spaces<'a, ExposesKeyword>)| {
merge_n_spaces!(arena, before_name, kw.before, kw.after)
},
);
record!(ModuleHeader {
after_keyword: after_keyword,
params: succeed(None),
exposes: specialize_err(EHeader::Exposes, exposes_list()).trace("exposes_list"),
interface_imports: map(
specialize_err(EHeader::Imports, imports()),
imports_none_if_empty
)
.trace("imports"),
})
.trace("interface_header")
}
fn imports_none_if_empty(value: ImportsKeywordItem<'_>) -> Option<ImportsKeywordItem<'_>> {
if value.item.is_empty() {
None
} else {
Some(value)
}
}
#[inline(always)]
fn hosted_header<'a>() -> impl Parser<'a, HostedHeader<'a>, EHeader<'a>> {
record!(HostedHeader {
before_name: space0_e(EHeader::IndentStart),
name: loc(module_name_help(EHeader::ModuleName)),
exposes: specialize_err(EHeader::Exposes, exposes_values_kw()),
imports: specialize_err(EHeader::Imports, imports()),
generates: specialize_err(EHeader::Generates, generates()),
generates_with: specialize_err(EHeader::GeneratesWith, generates_with()),
})
.trace("hosted_header")
}
fn chomp_module_name(buffer: &[u8]) -> Result<&str, Progress> {
use encode_unicode::CharExt;
let mut chomped = 0;
if let Ok((first_letter, width)) = char::from_utf8_slice_start(&buffer[chomped..]) {
if first_letter.is_uppercase() {
chomped += width;
} else {
return Err(Progress::NoProgress);
}
}
while let Ok((ch, width)) = char::from_utf8_slice_start(&buffer[chomped..]) {
// After the first character, only these are allowed:
//
// * Unicode alphabetic chars - you might include `鹏` if that's clear to your readers
// * ASCII digits - e.g. `1` but not `¾`, both of which pass .is_numeric()
// * A '.' separating module parts
if ch.is_alphabetic() || ch.is_ascii_digit() {
chomped += width;
} else if ch == '.' {
chomped += width;
if let Ok((first_letter, width)) = char::from_utf8_slice_start(&buffer[chomped..]) {
if first_letter.is_uppercase() {
chomped += width;
} else if first_letter == '{' {
// the .{ starting a `Foo.{ bar, baz }` importing clauses
chomped -= width;
break;
} else {
return Err(Progress::MadeProgress);
}
}
} else {
// we're done
break;
}
}
let name = unsafe { std::str::from_utf8_unchecked(&buffer[..chomped]) };
Ok(name)
}
#[inline(always)]
fn module_name<'a>() -> impl Parser<'a, ModuleName<'a>, ()> {
|_, mut state: State<'a>, _min_indent: u32| match chomp_module_name(state.bytes()) {
Ok(name) => {
let width = name.len();
state = state.advance(width);
Ok((MadeProgress, ModuleName::new(name), state))
}
Err(progress) => Err((progress, ())),
}
}
#[inline(always)]
fn app_header<'a>() -> impl Parser<'a, AppHeader<'a>, EHeader<'a>> {
record!(AppHeader {
before_provides: space0_e(EHeader::IndentStart),
provides: specialize_err(EHeader::Exposes, exposes_list()),
before_packages: space0_e(EHeader::IndentStart),
packages: specialize_err(EHeader::Packages, loc(packages_collection())),
old_imports: succeed(None),
old_provides_to_new_package: succeed(None),
})
.trace("app_header")
}
struct OldAppHeader<'a> {
pub before_name: &'a [CommentOrNewline<'a>],
pub packages: Option<Loc<OldAppPackages<'a>>>,
pub imports: Option<KeywordItem<'a, ImportsKeyword, ImportsCollection<'a>>>,
pub provides: ProvidesTo<'a>,
}
type OldAppPackages<'a> =
KeywordItem<'a, PackagesKeyword, Collection<'a, Loc<Spaced<'a, PackageEntry<'a>>>>>;
#[inline(always)]
fn old_app_header<'a>() -> impl Parser<'a, AppHeader<'a>, EHeader<'a>> {
let old = record!(OldAppHeader {
before_name: skip_second(
space0_e(EHeader::IndentStart),
loc(crate::parser::specialize_err(
EHeader::AppName,
string_literal::parse_str_literal()
))
),
packages: optional(specialize_err(EHeader::Packages, loc(packages()))),
imports: optional(specialize_err(EHeader::Imports, imports())),
provides: specialize_err(EHeader::Provides, provides_to()),
});
map_with_arena(old, |arena: &'a bumpalo::Bump, old: OldAppHeader<'a>| {
let mut before_packages: &'a [CommentOrNewline] = &[];
let packages = match old.packages {
Some(packages) => {
before_packages = merge_spaces(
arena,
packages.value.keyword.before,
packages.value.keyword.after,
);
if let To::ExistingPackage(platform_shorthand) = old.provides.to.value {
packages.map(|coll| {
coll.item.map_items(arena, |loc_spaced_pkg| {
if loc_spaced_pkg.value.item().shorthand == platform_shorthand {
loc_spaced_pkg.map(|spaced_pkg| {
spaced_pkg.map(arena, |pkg| {
let mut new_pkg = *pkg;
new_pkg.platform_marker = Some(merge_spaces(
arena,
old.provides.to_keyword.before,
old.provides.to_keyword.after,
));
new_pkg
})
})
} else {
*loc_spaced_pkg
}
})
})
} else {
packages.map(|kw| kw.item)
}
}
None => Loc {
region: Region::zero(),
value: Collection::empty(),
},
};
let provides = match old.provides.types {
Some(types) => {
let mut combined_items = bumpalo::collections::Vec::with_capacity_in(
old.provides.entries.items.len() + types.items.len(),
arena,
);
combined_items.extend_from_slice(old.provides.entries.items);
for loc_spaced_type_ident in types.items {
combined_items.push(loc_spaced_type_ident.map(|spaced_type_ident| {
spaced_type_ident.map(arena, |type_ident| {
ExposedName::new(From::from(*type_ident))
})
}));
}
let value_comments = old.provides.entries.final_comments();
let type_comments = types.final_comments();
let mut combined_comments = bumpalo::collections::Vec::with_capacity_in(
value_comments.len() + type_comments.len(),
arena,
);
combined_comments.extend_from_slice(value_comments);
combined_comments.extend_from_slice(type_comments);
Collection::with_items_and_comments(
arena,
combined_items.into_bump_slice(),
combined_comments.into_bump_slice(),
)
}
None => old.provides.entries,
};
AppHeader {
before_provides: merge_spaces(
arena,
old.before_name,
old.provides.provides_keyword.before,
),
provides,
before_packages: merge_spaces(
arena,
before_packages,
old.provides.provides_keyword.after,
),
packages,
old_imports: old.imports.and_then(imports_none_if_empty),
old_provides_to_new_package: match old.provides.to.value {
To::NewPackage(new_pkg) => Some(new_pkg),
To::ExistingPackage(_) => None,
},
}
})
}
#[inline(always)]
fn package_header<'a>() -> impl Parser<'a, PackageHeader<'a>, EHeader<'a>> {
record!(PackageHeader {
before_exposes: space0_e(EHeader::IndentStart),
exposes: specialize_err(EHeader::Exposes, exposes_module_collection()),
before_packages: space0_e(EHeader::IndentStart),
packages: specialize_err(EHeader::Packages, loc(packages_collection())),
})
.trace("package_header")
}
#[derive(Debug, Clone, PartialEq)]
struct OldPackageHeader<'a> {
before_name: &'a [CommentOrNewline<'a>],
exposes: KeywordItem<'a, ExposesKeyword, Collection<'a, Loc<Spaced<'a, ModuleName<'a>>>>>,
packages:
Loc<KeywordItem<'a, PackagesKeyword, Collection<'a, Loc<Spaced<'a, PackageEntry<'a>>>>>>,
}
#[inline(always)]
fn old_package_header<'a>() -> impl Parser<'a, PackageHeader<'a>, EHeader<'a>> {
map_with_arena(
record!(OldPackageHeader {
before_name: skip_second(
space0_e(EHeader::IndentStart),
specialize_err(EHeader::PackageName, package_name())
),
exposes: specialize_err(EHeader::Exposes, exposes_modules()),
packages: specialize_err(EHeader::Packages, loc(packages())),
}),
|arena: &'a bumpalo::Bump, old: OldPackageHeader<'a>| {
let before_exposes = merge_n_spaces!(
arena,
old.before_name,
old.exposes.keyword.before,
old.exposes.keyword.after
);
let before_packages = merge_spaces(
arena,
old.packages.value.keyword.before,
old.packages.value.keyword.after,
);
PackageHeader {
before_exposes,
exposes: old.exposes.item,
before_packages,
packages: old.packages.map(|kw| kw.item),
}
},
)
.trace("old_package_header")
}
#[inline(always)]
fn platform_header<'a>() -> impl Parser<'a, PlatformHeader<'a>, EHeader<'a>> {
record!(PlatformHeader {
before_name: space0_e(EHeader::IndentStart),
name: loc(specialize_err(EHeader::PlatformName, package_name())),
requires: specialize_err(EHeader::Requires, requires()),
exposes: specialize_err(EHeader::Exposes, exposes_modules()),
packages: specialize_err(EHeader::Packages, packages()),
imports: specialize_err(EHeader::Imports, imports()),
provides: specialize_err(EHeader::Provides, provides_exposed()),
})
.trace("platform_header")
}
fn provides_to_package<'a>() -> impl Parser<'a, To<'a>, EProvides<'a>> {
one_of![
specialize_err(
|_, pos| EProvides::Identifier(pos),
map(lowercase_ident(), To::ExistingPackage)
),
specialize_err(EProvides::Package, map(package_name(), To::NewPackage))
]
}
#[inline(always)]
fn provides_to<'a>() -> impl Parser<'a, ProvidesTo<'a>, EProvides<'a>> {
record!(ProvidesTo {
provides_keyword: spaces_around_keyword(
ProvidesKeyword,
EProvides::Provides,
EProvides::IndentProvides,
EProvides::IndentListStart
),
entries: collection_trailing_sep_e(
byte(b'[', EProvides::ListStart),
exposes_entry(EProvides::Identifier),
byte(b',', EProvides::ListEnd),
byte(b']', EProvides::ListEnd),
Spaced::SpaceBefore
),
types: optional(backtrackable(provides_types())),
to_keyword: spaces_around_keyword(
ToKeyword,
EProvides::To,
EProvides::IndentTo,
EProvides::IndentListStart
),
to: loc(provides_to_package()),
})
.trace("provides_to")
}
fn provides_exposed<'a>() -> impl Parser<
'a,
KeywordItem<'a, ProvidesKeyword, Collection<'a, Loc<Spaced<'a, ExposedName<'a>>>>>,
EProvides<'a>,
> {
record!(KeywordItem {
keyword: spaces_around_keyword(
ProvidesKeyword,
EProvides::Provides,
EProvides::IndentProvides,
EProvides::IndentListStart
),
item: collection_trailing_sep_e(
byte(b'[', EProvides::ListStart),
exposes_entry(EProvides::Identifier),
byte(b',', EProvides::ListEnd),
byte(b']', EProvides::ListEnd),
Spaced::SpaceBefore
),
})
}
#[inline(always)]
fn provides_types<'a>(
) -> impl Parser<'a, Collection<'a, Loc<Spaced<'a, UppercaseIdent<'a>>>>, EProvides<'a>> {
skip_first(
// We only support spaces here, not newlines, because this is not intended
// to be the design forever. Someday it will hopefully work like Elm,
// where platform authors can provide functions like Browser.sandbox which
// present an API based on ordinary-looking type variables.
zero_or_more(byte(
b' ',
// HACK: If this errors, EProvides::Provides is not an accurate reflection
// of what went wrong. However, this is both skipped and zero_or_more,
// so this error should never be visible to anyone in practice!
EProvides::Provides,
)),
collection_trailing_sep_e(
byte(b'{', EProvides::ListStart),
provides_type_entry(EProvides::Identifier),
byte(b',', EProvides::ListEnd),
byte(b'}', EProvides::ListEnd),
Spaced::SpaceBefore,
),
)
}
fn provides_type_entry<'a, F, E>(
to_expectation: F,
) -> impl Parser<'a, Loc<Spaced<'a, UppercaseIdent<'a>>>, E>
where
F: Fn(Position) -> E,
F: Copy,
E: 'a,
{
loc(map(
specialize_err(move |_, pos| to_expectation(pos), ident::uppercase()),
Spaced::Item,
))
}
fn exposes_entry<'a, F, E>(
to_expectation: F,
) -> impl Parser<'a, Loc<Spaced<'a, ExposedName<'a>>>, E>
where
F: Fn(Position) -> E,
F: Copy,
E: 'a,
{
loc(map(
specialize_err(move |_, pos| to_expectation(pos), unqualified_ident()),
|n| Spaced::Item(ExposedName::new(n)),
))
}
#[inline(always)]
fn requires<'a>(
) -> impl Parser<'a, KeywordItem<'a, RequiresKeyword, PlatformRequires<'a>>, ERequires<'a>> {
record!(KeywordItem {
keyword: spaces_around_keyword(
RequiresKeyword,
ERequires::Requires,
ERequires::IndentRequires,
ERequires::IndentListStart
),
item: platform_requires(),
})
}
#[inline(always)]
fn platform_requires<'a>() -> impl Parser<'a, PlatformRequires<'a>, ERequires<'a>> {
record!(PlatformRequires {
rigids: skip_second(requires_rigids(), space0_e(ERequires::ListStart)),
signature: requires_typed_ident()
})
}
#[inline(always)]
fn requires_rigids<'a>(
) -> impl Parser<'a, Collection<'a, Loc<Spaced<'a, UppercaseIdent<'a>>>>, ERequires<'a>> {
collection_trailing_sep_e(
byte(b'{', ERequires::ListStart),
specialize_err(
|_, pos| ERequires::Rigid(pos),
loc(map(ident::uppercase(), Spaced::Item)),
),
byte(b',', ERequires::ListEnd),
byte(b'}', ERequires::ListEnd),
Spaced::SpaceBefore,
)
}
#[inline(always)]
fn requires_typed_ident<'a>() -> impl Parser<'a, Loc<Spaced<'a, TypedIdent<'a>>>, ERequires<'a>> {
skip_first(
byte(b'{', ERequires::ListStart),
skip_second(
reset_min_indent(space0_around_ee(
specialize_err(ERequires::TypedIdent, loc(typed_ident())),
ERequires::ListStart,
ERequires::ListEnd,
)),
byte(b'}', ERequires::ListStart),
),
)
}
#[inline(always)]
fn exposes_values_kw<'a>() -> impl Parser<
'a,
KeywordItem<'a, ExposesKeyword, Collection<'a, Loc<Spaced<'a, ExposedName<'a>>>>>,
EExposes,
> {
record!(KeywordItem {
keyword: exposes_kw(),
item: exposes_list()
})
}
#[inline(always)]
fn exposes_kw<'a>() -> impl Parser<'a, Spaces<'a, ExposesKeyword>, EExposes> {
spaces_around_keyword(
ExposesKeyword,
EExposes::Exposes,
EExposes::IndentExposes,
EExposes::IndentListStart,
)
}
#[inline(always)]
fn exposes_list<'a>() -> impl Parser<'a, Collection<'a, Loc<Spaced<'a, ExposedName<'a>>>>, EExposes>
{
collection_trailing_sep_e(
byte(b'[', EExposes::ListStart),
exposes_entry(EExposes::Identifier),
byte(b',', EExposes::ListEnd),
byte(b']', EExposes::ListEnd),
Spaced::SpaceBefore,
)
}
pub fn spaces_around_keyword<'a, K: Keyword, E>(
keyword_item: K,
expectation: fn(Position) -> E,
indent_problem1: fn(Position) -> E,
indent_problem2: fn(Position) -> E,
) -> impl Parser<'a, Spaces<'a, K>, E>
where
E: 'a + SpaceProblem,
{
map(
and(
skip_second(
// parse any leading space before the keyword
backtrackable(space0_e(indent_problem1)),
// parse the keyword
crate::parser::keyword(K::KEYWORD, expectation),
),
// parse the trailing space
space0_e(indent_problem2),
),
move |(before, after)| Spaces {
before,
item: keyword_item,
after,
},
)
}
#[inline(always)]
fn exposes_modules<'a>() -> impl Parser<
'a,
KeywordItem<'a, ExposesKeyword, Collection<'a, Loc<Spaced<'a, ModuleName<'a>>>>>,
EExposes,
> {
record!(KeywordItem {
keyword: spaces_around_keyword(
ExposesKeyword,
EExposes::Exposes,
EExposes::IndentExposes,
EExposes::IndentListStart
),
item: exposes_module_collection(),
})
}
fn exposes_module_collection<'a>(
) -> impl Parser<'a, Collection<'a, Loc<Spaced<'a, ModuleName<'a>>>>, EExposes> {
collection_trailing_sep_e(
byte(b'[', EExposes::ListStart),
exposes_module(EExposes::Identifier),
byte(b',', EExposes::ListEnd),
byte(b']', EExposes::ListEnd),
Spaced::SpaceBefore,
)
}
fn exposes_module<'a, F, E>(
to_expectation: F,
) -> impl Parser<'a, Loc<Spaced<'a, ModuleName<'a>>>, E>
where
F: Fn(Position) -> E,
F: Copy,
E: 'a,
{
loc(map(
specialize_err(move |_, pos| to_expectation(pos), module_name()),
Spaced::Item,
))
}
#[inline(always)]
fn packages<'a>() -> impl Parser<
'a,
KeywordItem<'a, PackagesKeyword, Collection<'a, Loc<Spaced<'a, PackageEntry<'a>>>>>,
EPackages<'a>,
> {
record!(KeywordItem {
keyword: packages_kw(),
item: packages_collection()
})
}
#[inline(always)]
fn packages_kw<'a>() -> impl Parser<'a, Spaces<'a, PackagesKeyword>, EPackages<'a>> {
spaces_around_keyword(
PackagesKeyword,
EPackages::Packages,
EPackages::IndentPackages,
EPackages::IndentListStart,
)
}
#[inline(always)]
fn packages_collection<'a>(
) -> impl Parser<'a, Collection<'a, Loc<Spaced<'a, PackageEntry<'a>>>>, EPackages<'a>> {
collection_trailing_sep_e(
byte(b'{', EPackages::ListStart),
specialize_err(EPackages::PackageEntry, loc(package_entry())),
byte(b',', EPackages::ListEnd),
byte(b'}', EPackages::ListEnd),
Spaced::SpaceBefore,
)
}
#[inline(always)]
fn generates<'a>(
) -> impl Parser<'a, KeywordItem<'a, GeneratesKeyword, UppercaseIdent<'a>>, EGenerates> {
record!(KeywordItem {
keyword: spaces_around_keyword(
GeneratesKeyword,
EGenerates::Generates,
EGenerates::IndentGenerates,
EGenerates::IndentTypeStart
),
item: specialize_err(|(), pos| EGenerates::Identifier(pos), uppercase())
})
}
#[inline(always)]
fn generates_with<'a>() -> impl Parser<
'a,
KeywordItem<'a, WithKeyword, Collection<'a, Loc<Spaced<'a, ExposedName<'a>>>>>,
EGeneratesWith,
> {
record!(KeywordItem {
keyword: spaces_around_keyword(
WithKeyword,
EGeneratesWith::With,
EGeneratesWith::IndentWith,
EGeneratesWith::IndentListStart
),
item: collection_trailing_sep_e(
byte(b'[', EGeneratesWith::ListStart),
exposes_entry(EGeneratesWith::Identifier),
byte(b',', EGeneratesWith::ListEnd),
byte(b']', EGeneratesWith::ListEnd),
Spaced::SpaceBefore
)
})
}
#[inline(always)]
fn imports<'a>() -> impl Parser<
'a,
KeywordItem<'a, ImportsKeyword, Collection<'a, Loc<Spaced<'a, ImportsEntry<'a>>>>>,
EImports,
> {
record!(KeywordItem {
keyword: spaces_around_keyword(
ImportsKeyword,
EImports::Imports,
EImports::IndentImports,
EImports::IndentListStart
),
item: collection_trailing_sep_e(
byte(b'[', EImports::ListStart),
loc(imports_entry()),
byte(b',', EImports::ListEnd),
byte(b']', EImports::ListEnd),
Spaced::SpaceBefore
)
})
.trace("imports")
}
#[inline(always)]
pub fn typed_ident<'a>() -> impl Parser<'a, Spaced<'a, TypedIdent<'a>>, ETypedIdent<'a>> {
// e.g.
//
// printLine : Str -> Effect {}
map(
and(
and(
loc(specialize_err(
|_, pos| ETypedIdent::Identifier(pos),
lowercase_ident(),
)),
space0_e(ETypedIdent::IndentHasType),
),
skip_first(
byte(b':', ETypedIdent::HasType),
space0_before_e(
specialize_err(
ETypedIdent::Type,
reset_min_indent(type_annotation::located(true)),
),
ETypedIdent::IndentType,
),
),
),
|((ident, spaces_before_colon), ann)| {
Spaced::Item(TypedIdent {
ident,
spaces_before_colon,
ann,
})
},
)
}
fn shortname<'a>() -> impl Parser<'a, &'a str, EImports> {
specialize_err(|_, pos| EImports::Shorthand(pos), lowercase_ident())
}
pub fn module_name_help<'a, F, E>(to_expectation: F) -> impl Parser<'a, ModuleName<'a>, E>
where
F: Fn(Position) -> E,
E: 'a,
F: 'a,
{
specialize_err(move |_, pos| to_expectation(pos), module_name())
}
#[inline(always)]
fn imports_entry<'a>() -> impl Parser<'a, Spaced<'a, ImportsEntry<'a>>, EImports> {
type Temp<'a> = (
(Option<&'a str>, ModuleName<'a>),
Option<Collection<'a, Loc<Spaced<'a, ExposedName<'a>>>>>,
);
let spaced_import = |((opt_shortname, module_name), opt_values): Temp<'a>| {
let exposed_values = opt_values.unwrap_or_else(Collection::empty);
let entry = match opt_shortname {
Some(shortname) => ImportsEntry::Package(shortname, module_name, exposed_values),
None => ImportsEntry::Module(module_name, exposed_values),
};
Spaced::Item(entry)
};
one_of!(
map(
and(
and(
// e.g. `pf.`
optional(backtrackable(skip_second(
shortname(),
byte(b'.', EImports::ShorthandDot)
))),
// e.g. `Task`
module_name_help(EImports::ModuleName)
),
// e.g. `.{ Task, after}`
optional(skip_first(
byte(b'.', EImports::ExposingDot),
collection_trailing_sep_e(
byte(b'{', EImports::SetStart),
exposes_entry(EImports::Identifier),
byte(b',', EImports::SetEnd),
byte(b'}', EImports::SetEnd),
Spaced::SpaceBefore
)
))
),
spaced_import
)
.trace("normal_import"),
map(
and(
and(
// e.g. "filename"
// TODO: str literal allows for multiline strings. We probably don't want that for file names.
specialize_err(|_, pos| EImports::StrLiteral(pos), parse_str_literal()),
// e.g. as
and(
and(
space0_e(EImports::AsKeyword),
two_bytes(b'a', b's', EImports::AsKeyword)
),
space0_e(EImports::AsKeyword)
)
),
// e.g. file : Str
specialize_err(|_, pos| EImports::TypedIdent(pos), typed_ident())
),
|((file_name, _), typed_ident)| {
// TODO: look at blacking block strings during parsing.
Spaced::Item(ImportsEntry::IngestedFile(file_name, typed_ident))
}
)
.trace("ingest_file_import")
)
.trace("imports_entry")
}
impl<'a> HeaderType<'a> {
pub fn exposed_or_provided_values(&'a self) -> &'a [Loc<ExposedName<'a>>] {
match self {
@ -178,7 +1108,7 @@ impl<'a> ModuleName<'a> {
ModuleName(name)
}
pub const fn as_str(&'a self) -> &'a str {
pub const fn as_str(&self) -> &'a str {
self.0
}

View file

@ -74,7 +74,7 @@ pub fn highlight(text: &str) -> Vec<Loc<Token>> {
let header_keywords = HEADER_KEYWORDS.iter().copied().collect::<HashSet<_>>();
let body_keywords = KEYWORDS.iter().copied().collect::<HashSet<_>>();
if let Ok((_prog, _, new_state)) = crate::module::header().parse(&arena, state.clone(), 0) {
if let Ok((_prog, _, new_state)) = crate::header::header().parse(&arena, state.clone(), 0) {
let inner_state =
State::new(text[..state.bytes().len() - new_state.bytes().len()].as_bytes());
highlight_inner(&arena, inner_state, &mut tokens, &header_keywords);

View file

@ -13,7 +13,6 @@ pub mod header;
pub mod highlight;
pub mod ident;
pub mod keyword;
pub mod module;
pub mod number_literal;
pub mod pattern;
pub mod problems;

View file

@ -1,945 +0,0 @@
use crate::ast::{Collection, CommentOrNewline, Defs, Header, Module, Spaced, Spaces};
use crate::blankspace::{space0_around_ee, space0_before_e, space0_e};
use crate::expr::merge_spaces;
use crate::header::{
package_entry, package_name, AppHeader, ExposedName, ExposesKeyword, GeneratesKeyword,
HostedHeader, ImportsCollection, ImportsEntry, ImportsKeyword, ImportsKeywordItem, Keyword,
KeywordItem, ModuleHeader, ModuleName, ModuleParams, PackageEntry, PackageHeader,
PackagesKeyword, PlatformHeader, PlatformRequires, ProvidesKeyword, ProvidesTo,
RequiresKeyword, To, ToKeyword, TypedIdent, WithKeyword,
};
use crate::ident::{self, lowercase_ident, unqualified_ident, uppercase, UppercaseIdent};
use crate::parser::Progress::{self, *};
use crate::parser::{
and, backtrackable, byte, collection_trailing_sep_e, increment_min_indent, loc, map,
map_with_arena, optional, reset_min_indent, skip_first, skip_second, specialize_err, succeed,
two_bytes, zero_or_more, EExposes, EGenerates, EGeneratesWith, EHeader, EImports, EPackages,
EParams, EProvides, ERequires, ETypedIdent, Parser, SourceError, SpaceProblem, SyntaxError,
};
use crate::pattern::record_pattern_fields;
use crate::state::State;
use crate::string_literal::{self, parse_str_literal};
use crate::type_annotation;
use roc_region::all::{Loc, Position, Region};
fn end_of_file<'a>() -> impl Parser<'a, (), SyntaxError<'a>> {
|_arena, state: State<'a>, _min_indent: u32| {
if state.has_reached_end() {
Ok((NoProgress, (), state))
} else {
Err((NoProgress, SyntaxError::NotEndOfFile(state.pos())))
}
}
}
pub fn parse_module_defs<'a>(
arena: &'a bumpalo::Bump,
state: State<'a>,
defs: Defs<'a>,
) -> Result<Defs<'a>, SyntaxError<'a>> {
let min_indent = 0;
match crate::expr::parse_top_level_defs(arena, state.clone(), defs) {
Ok((_, defs, state)) => match end_of_file().parse(arena, state, min_indent) {
Ok(_) => Ok(defs),
Err((_, fail)) => Err(fail),
},
Err((_, fail)) => Err(SyntaxError::Expr(fail, state.pos())),
}
}
pub fn parse_header<'a>(
arena: &'a bumpalo::Bump,
state: State<'a>,
) -> Result<(Module<'a>, State<'a>), SourceError<'a, EHeader<'a>>> {
let min_indent = 0;
match header().parse(arena, state.clone(), min_indent) {
Ok((_, module, state)) => Ok((module, state)),
Err((_, fail)) => Err(SourceError::new(fail, &state)),
}
}
pub fn header<'a>() -> impl Parser<'a, Module<'a>, EHeader<'a>> {
use crate::parser::keyword;
record!(Module {
comments: space0_e(EHeader::IndentStart),
header: one_of![
map(
skip_first(
keyword("module", EHeader::Start),
increment_min_indent(module_header())
),
Header::Module
),
map(
skip_first(
keyword("interface", EHeader::Start),
increment_min_indent(interface_header())
),
Header::Module
),
map(
skip_first(
keyword("app", EHeader::Start),
increment_min_indent(one_of![app_header(), old_app_header()])
),
Header::App
),
map(
skip_first(
keyword("package", EHeader::Start),
increment_min_indent(one_of![package_header(), old_package_header()])
),
Header::Package
),
map(
skip_first(
keyword("platform", EHeader::Start),
increment_min_indent(platform_header())
),
Header::Platform
),
map(
skip_first(
keyword("hosted", EHeader::Start),
increment_min_indent(hosted_header())
),
Header::Hosted
),
]
})
}
#[inline(always)]
fn module_header<'a>() -> impl Parser<'a, ModuleHeader<'a>, EHeader<'a>> {
record!(ModuleHeader {
after_keyword: space0_e(EHeader::IndentStart),
params: optional(specialize_err(EHeader::Params, module_params())),
exposes: specialize_err(EHeader::Exposes, exposes_list()),
interface_imports: succeed(None)
})
.trace("module_header")
}
fn module_params<'a>() -> impl Parser<'a, ModuleParams<'a>, EParams<'a>> {
record!(ModuleParams {
params: specialize_err(EParams::Pattern, record_pattern_fields()),
before_arrow: skip_second(
space0_e(EParams::BeforeArrow),
loc(two_bytes(b'-', b'>', EParams::Arrow))
),
after_arrow: space0_e(EParams::AfterArrow),
})
}
// TODO does this need to be a macro?
macro_rules! merge_n_spaces {
($arena:expr, $($slice:expr),*) => {
{
let mut merged = bumpalo::collections::Vec::with_capacity_in(0 $(+ $slice.len())*, $arena);
$(merged.extend_from_slice($slice);)*
merged.into_bump_slice()
}
};
}
/// Parse old interface headers so we can format them into module headers
#[inline(always)]
fn interface_header<'a>() -> impl Parser<'a, ModuleHeader<'a>, EHeader<'a>> {
let after_keyword = map_with_arena(
and(
skip_second(
space0_e(EHeader::IndentStart),
loc(module_name_help(EHeader::ModuleName)),
),
specialize_err(EHeader::Exposes, exposes_kw()),
),
|arena: &'a bumpalo::Bump,
(before_name, kw): (&'a [CommentOrNewline<'a>], Spaces<'a, ExposesKeyword>)| {
merge_n_spaces!(arena, before_name, kw.before, kw.after)
},
);
record!(ModuleHeader {
after_keyword: after_keyword,
params: succeed(None),
exposes: specialize_err(EHeader::Exposes, exposes_list()).trace("exposes_list"),
interface_imports: map(
specialize_err(EHeader::Imports, imports()),
imports_none_if_empty
)
.trace("imports"),
})
.trace("interface_header")
}
fn imports_none_if_empty(value: ImportsKeywordItem<'_>) -> Option<ImportsKeywordItem<'_>> {
if value.item.is_empty() {
None
} else {
Some(value)
}
}
#[inline(always)]
fn hosted_header<'a>() -> impl Parser<'a, HostedHeader<'a>, EHeader<'a>> {
record!(HostedHeader {
before_name: space0_e(EHeader::IndentStart),
name: loc(module_name_help(EHeader::ModuleName)),
exposes: specialize_err(EHeader::Exposes, exposes_values_kw()),
imports: specialize_err(EHeader::Imports, imports()),
generates: specialize_err(EHeader::Generates, generates()),
generates_with: specialize_err(EHeader::GeneratesWith, generates_with()),
})
.trace("hosted_header")
}
fn chomp_module_name(buffer: &[u8]) -> Result<&str, Progress> {
use encode_unicode::CharExt;
let mut chomped = 0;
if let Ok((first_letter, width)) = char::from_utf8_slice_start(&buffer[chomped..]) {
if first_letter.is_uppercase() {
chomped += width;
} else {
return Err(Progress::NoProgress);
}
}
while let Ok((ch, width)) = char::from_utf8_slice_start(&buffer[chomped..]) {
// After the first character, only these are allowed:
//
// * Unicode alphabetic chars - you might include `鹏` if that's clear to your readers
// * ASCII digits - e.g. `1` but not `¾`, both of which pass .is_numeric()
// * A '.' separating module parts
if ch.is_alphabetic() || ch.is_ascii_digit() {
chomped += width;
} else if ch == '.' {
chomped += width;
if let Ok((first_letter, width)) = char::from_utf8_slice_start(&buffer[chomped..]) {
if first_letter.is_uppercase() {
chomped += width;
} else if first_letter == '{' {
// the .{ starting a `Foo.{ bar, baz }` importing clauses
chomped -= width;
break;
} else {
return Err(Progress::MadeProgress);
}
}
} else {
// we're done
break;
}
}
let name = unsafe { std::str::from_utf8_unchecked(&buffer[..chomped]) };
Ok(name)
}
#[inline(always)]
fn module_name<'a>() -> impl Parser<'a, ModuleName<'a>, ()> {
|_, mut state: State<'a>, _min_indent: u32| match chomp_module_name(state.bytes()) {
Ok(name) => {
let width = name.len();
state = state.advance(width);
Ok((MadeProgress, ModuleName::new(name), state))
}
Err(progress) => Err((progress, ())),
}
}
#[inline(always)]
fn app_header<'a>() -> impl Parser<'a, AppHeader<'a>, EHeader<'a>> {
record!(AppHeader {
before_provides: space0_e(EHeader::IndentStart),
provides: specialize_err(EHeader::Exposes, exposes_list()),
before_packages: space0_e(EHeader::IndentStart),
packages: specialize_err(EHeader::Packages, loc(packages_collection())),
old_imports: succeed(None),
old_provides_to_new_package: succeed(None),
})
.trace("app_header")
}
struct OldAppHeader<'a> {
pub before_name: &'a [CommentOrNewline<'a>],
pub packages: Option<Loc<OldAppPackages<'a>>>,
pub imports: Option<KeywordItem<'a, ImportsKeyword, ImportsCollection<'a>>>,
pub provides: ProvidesTo<'a>,
}
type OldAppPackages<'a> =
KeywordItem<'a, PackagesKeyword, Collection<'a, Loc<Spaced<'a, PackageEntry<'a>>>>>;
#[inline(always)]
fn old_app_header<'a>() -> impl Parser<'a, AppHeader<'a>, EHeader<'a>> {
let old = record!(OldAppHeader {
before_name: skip_second(
space0_e(EHeader::IndentStart),
loc(crate::parser::specialize_err(
EHeader::AppName,
string_literal::parse_str_literal()
))
),
packages: optional(specialize_err(EHeader::Packages, loc(packages()))),
imports: optional(specialize_err(EHeader::Imports, imports())),
provides: specialize_err(EHeader::Provides, provides_to()),
});
map_with_arena(old, |arena: &'a bumpalo::Bump, old: OldAppHeader<'a>| {
let mut before_packages: &'a [CommentOrNewline] = &[];
let packages = match old.packages {
Some(packages) => {
before_packages = merge_spaces(
arena,
packages.value.keyword.before,
packages.value.keyword.after,
);
if let To::ExistingPackage(platform_shorthand) = old.provides.to.value {
packages.map(|coll| {
coll.item.map_items(arena, |loc_spaced_pkg| {
if loc_spaced_pkg.value.item().shorthand == platform_shorthand {
loc_spaced_pkg.map(|spaced_pkg| {
spaced_pkg.map(arena, |pkg| {
let mut new_pkg = *pkg;
new_pkg.platform_marker = Some(merge_spaces(
arena,
old.provides.to_keyword.before,
old.provides.to_keyword.after,
));
new_pkg
})
})
} else {
*loc_spaced_pkg
}
})
})
} else {
packages.map(|kw| kw.item)
}
}
None => Loc {
region: Region::zero(),
value: Collection::empty(),
},
};
let provides = match old.provides.types {
Some(types) => {
let mut combined_items = bumpalo::collections::Vec::with_capacity_in(
old.provides.entries.items.len() + types.items.len(),
arena,
);
combined_items.extend_from_slice(old.provides.entries.items);
for loc_spaced_type_ident in types.items {
combined_items.push(loc_spaced_type_ident.map(|spaced_type_ident| {
spaced_type_ident.map(arena, |type_ident| {
ExposedName::new(From::from(*type_ident))
})
}));
}
let value_comments = old.provides.entries.final_comments();
let type_comments = types.final_comments();
let mut combined_comments = bumpalo::collections::Vec::with_capacity_in(
value_comments.len() + type_comments.len(),
arena,
);
combined_comments.extend_from_slice(value_comments);
combined_comments.extend_from_slice(type_comments);
Collection::with_items_and_comments(
arena,
combined_items.into_bump_slice(),
combined_comments.into_bump_slice(),
)
}
None => old.provides.entries,
};
AppHeader {
before_provides: merge_spaces(
arena,
old.before_name,
old.provides.provides_keyword.before,
),
provides,
before_packages: merge_spaces(
arena,
before_packages,
old.provides.provides_keyword.after,
),
packages,
old_imports: old.imports.and_then(imports_none_if_empty),
old_provides_to_new_package: match old.provides.to.value {
To::NewPackage(new_pkg) => Some(new_pkg),
To::ExistingPackage(_) => None,
},
}
})
}
#[inline(always)]
fn package_header<'a>() -> impl Parser<'a, PackageHeader<'a>, EHeader<'a>> {
record!(PackageHeader {
before_exposes: space0_e(EHeader::IndentStart),
exposes: specialize_err(EHeader::Exposes, exposes_module_collection()),
before_packages: space0_e(EHeader::IndentStart),
packages: specialize_err(EHeader::Packages, loc(packages_collection())),
})
.trace("package_header")
}
#[derive(Debug, Clone, PartialEq)]
struct OldPackageHeader<'a> {
before_name: &'a [CommentOrNewline<'a>],
exposes: KeywordItem<'a, ExposesKeyword, Collection<'a, Loc<Spaced<'a, ModuleName<'a>>>>>,
packages:
Loc<KeywordItem<'a, PackagesKeyword, Collection<'a, Loc<Spaced<'a, PackageEntry<'a>>>>>>,
}
#[inline(always)]
fn old_package_header<'a>() -> impl Parser<'a, PackageHeader<'a>, EHeader<'a>> {
map_with_arena(
record!(OldPackageHeader {
before_name: skip_second(
space0_e(EHeader::IndentStart),
specialize_err(EHeader::PackageName, package_name())
),
exposes: specialize_err(EHeader::Exposes, exposes_modules()),
packages: specialize_err(EHeader::Packages, loc(packages())),
}),
|arena: &'a bumpalo::Bump, old: OldPackageHeader<'a>| {
let before_exposes = merge_n_spaces!(
arena,
old.before_name,
old.exposes.keyword.before,
old.exposes.keyword.after
);
let before_packages = merge_spaces(
arena,
old.packages.value.keyword.before,
old.packages.value.keyword.after,
);
PackageHeader {
before_exposes,
exposes: old.exposes.item,
before_packages,
packages: old.packages.map(|kw| kw.item),
}
},
)
.trace("old_package_header")
}
#[inline(always)]
fn platform_header<'a>() -> impl Parser<'a, PlatformHeader<'a>, EHeader<'a>> {
record!(PlatformHeader {
before_name: space0_e(EHeader::IndentStart),
name: loc(specialize_err(EHeader::PlatformName, package_name())),
requires: specialize_err(EHeader::Requires, requires()),
exposes: specialize_err(EHeader::Exposes, exposes_modules()),
packages: specialize_err(EHeader::Packages, packages()),
imports: specialize_err(EHeader::Imports, imports()),
provides: specialize_err(EHeader::Provides, provides_exposed()),
})
.trace("platform_header")
}
fn provides_to_package<'a>() -> impl Parser<'a, To<'a>, EProvides<'a>> {
one_of![
specialize_err(
|_, pos| EProvides::Identifier(pos),
map(lowercase_ident(), To::ExistingPackage)
),
specialize_err(EProvides::Package, map(package_name(), To::NewPackage))
]
}
#[inline(always)]
fn provides_to<'a>() -> impl Parser<'a, ProvidesTo<'a>, EProvides<'a>> {
record!(ProvidesTo {
provides_keyword: spaces_around_keyword(
ProvidesKeyword,
EProvides::Provides,
EProvides::IndentProvides,
EProvides::IndentListStart
),
entries: collection_trailing_sep_e(
byte(b'[', EProvides::ListStart),
exposes_entry(EProvides::Identifier),
byte(b',', EProvides::ListEnd),
byte(b']', EProvides::ListEnd),
Spaced::SpaceBefore
),
types: optional(backtrackable(provides_types())),
to_keyword: spaces_around_keyword(
ToKeyword,
EProvides::To,
EProvides::IndentTo,
EProvides::IndentListStart
),
to: loc(provides_to_package()),
})
.trace("provides_to")
}
fn provides_exposed<'a>() -> impl Parser<
'a,
KeywordItem<'a, ProvidesKeyword, Collection<'a, Loc<Spaced<'a, ExposedName<'a>>>>>,
EProvides<'a>,
> {
record!(KeywordItem {
keyword: spaces_around_keyword(
ProvidesKeyword,
EProvides::Provides,
EProvides::IndentProvides,
EProvides::IndentListStart
),
item: collection_trailing_sep_e(
byte(b'[', EProvides::ListStart),
exposes_entry(EProvides::Identifier),
byte(b',', EProvides::ListEnd),
byte(b']', EProvides::ListEnd),
Spaced::SpaceBefore
),
})
}
#[inline(always)]
fn provides_types<'a>(
) -> impl Parser<'a, Collection<'a, Loc<Spaced<'a, UppercaseIdent<'a>>>>, EProvides<'a>> {
skip_first(
// We only support spaces here, not newlines, because this is not intended
// to be the design forever. Someday it will hopefully work like Elm,
// where platform authors can provide functions like Browser.sandbox which
// present an API based on ordinary-looking type variables.
zero_or_more(byte(
b' ',
// HACK: If this errors, EProvides::Provides is not an accurate reflection
// of what went wrong. However, this is both skipped and zero_or_more,
// so this error should never be visible to anyone in practice!
EProvides::Provides,
)),
collection_trailing_sep_e(
byte(b'{', EProvides::ListStart),
provides_type_entry(EProvides::Identifier),
byte(b',', EProvides::ListEnd),
byte(b'}', EProvides::ListEnd),
Spaced::SpaceBefore,
),
)
}
fn provides_type_entry<'a, F, E>(
to_expectation: F,
) -> impl Parser<'a, Loc<Spaced<'a, UppercaseIdent<'a>>>, E>
where
F: Fn(Position) -> E,
F: Copy,
E: 'a,
{
loc(map(
specialize_err(move |_, pos| to_expectation(pos), ident::uppercase()),
Spaced::Item,
))
}
fn exposes_entry<'a, F, E>(
to_expectation: F,
) -> impl Parser<'a, Loc<Spaced<'a, ExposedName<'a>>>, E>
where
F: Fn(Position) -> E,
F: Copy,
E: 'a,
{
loc(map(
specialize_err(move |_, pos| to_expectation(pos), unqualified_ident()),
|n| Spaced::Item(ExposedName::new(n)),
))
}
#[inline(always)]
fn requires<'a>(
) -> impl Parser<'a, KeywordItem<'a, RequiresKeyword, PlatformRequires<'a>>, ERequires<'a>> {
record!(KeywordItem {
keyword: spaces_around_keyword(
RequiresKeyword,
ERequires::Requires,
ERequires::IndentRequires,
ERequires::IndentListStart
),
item: platform_requires(),
})
}
#[inline(always)]
fn platform_requires<'a>() -> impl Parser<'a, PlatformRequires<'a>, ERequires<'a>> {
record!(PlatformRequires {
rigids: skip_second(requires_rigids(), space0_e(ERequires::ListStart)),
signature: requires_typed_ident()
})
}
#[inline(always)]
fn requires_rigids<'a>(
) -> impl Parser<'a, Collection<'a, Loc<Spaced<'a, UppercaseIdent<'a>>>>, ERequires<'a>> {
collection_trailing_sep_e(
byte(b'{', ERequires::ListStart),
specialize_err(
|_, pos| ERequires::Rigid(pos),
loc(map(ident::uppercase(), Spaced::Item)),
),
byte(b',', ERequires::ListEnd),
byte(b'}', ERequires::ListEnd),
Spaced::SpaceBefore,
)
}
#[inline(always)]
fn requires_typed_ident<'a>() -> impl Parser<'a, Loc<Spaced<'a, TypedIdent<'a>>>, ERequires<'a>> {
skip_first(
byte(b'{', ERequires::ListStart),
skip_second(
reset_min_indent(space0_around_ee(
specialize_err(ERequires::TypedIdent, loc(typed_ident())),
ERequires::ListStart,
ERequires::ListEnd,
)),
byte(b'}', ERequires::ListStart),
),
)
}
#[inline(always)]
fn exposes_values_kw<'a>() -> impl Parser<
'a,
KeywordItem<'a, ExposesKeyword, Collection<'a, Loc<Spaced<'a, ExposedName<'a>>>>>,
EExposes,
> {
record!(KeywordItem {
keyword: exposes_kw(),
item: exposes_list()
})
}
#[inline(always)]
fn exposes_kw<'a>() -> impl Parser<'a, Spaces<'a, ExposesKeyword>, EExposes> {
spaces_around_keyword(
ExposesKeyword,
EExposes::Exposes,
EExposes::IndentExposes,
EExposes::IndentListStart,
)
}
#[inline(always)]
fn exposes_list<'a>() -> impl Parser<'a, Collection<'a, Loc<Spaced<'a, ExposedName<'a>>>>, EExposes>
{
collection_trailing_sep_e(
byte(b'[', EExposes::ListStart),
exposes_entry(EExposes::Identifier),
byte(b',', EExposes::ListEnd),
byte(b']', EExposes::ListEnd),
Spaced::SpaceBefore,
)
}
pub fn spaces_around_keyword<'a, K: Keyword, E>(
keyword_item: K,
expectation: fn(Position) -> E,
indent_problem1: fn(Position) -> E,
indent_problem2: fn(Position) -> E,
) -> impl Parser<'a, Spaces<'a, K>, E>
where
E: 'a + SpaceProblem,
{
map(
and(
skip_second(
// parse any leading space before the keyword
backtrackable(space0_e(indent_problem1)),
// parse the keyword
crate::parser::keyword(K::KEYWORD, expectation),
),
// parse the trailing space
space0_e(indent_problem2),
),
move |(before, after)| Spaces {
before,
item: keyword_item,
after,
},
)
}
#[inline(always)]
fn exposes_modules<'a>() -> impl Parser<
'a,
KeywordItem<'a, ExposesKeyword, Collection<'a, Loc<Spaced<'a, ModuleName<'a>>>>>,
EExposes,
> {
record!(KeywordItem {
keyword: spaces_around_keyword(
ExposesKeyword,
EExposes::Exposes,
EExposes::IndentExposes,
EExposes::IndentListStart
),
item: exposes_module_collection(),
})
}
fn exposes_module_collection<'a>(
) -> impl Parser<'a, Collection<'a, Loc<Spaced<'a, ModuleName<'a>>>>, EExposes> {
collection_trailing_sep_e(
byte(b'[', EExposes::ListStart),
exposes_module(EExposes::Identifier),
byte(b',', EExposes::ListEnd),
byte(b']', EExposes::ListEnd),
Spaced::SpaceBefore,
)
}
fn exposes_module<'a, F, E>(
to_expectation: F,
) -> impl Parser<'a, Loc<Spaced<'a, ModuleName<'a>>>, E>
where
F: Fn(Position) -> E,
F: Copy,
E: 'a,
{
loc(map(
specialize_err(move |_, pos| to_expectation(pos), module_name()),
Spaced::Item,
))
}
#[inline(always)]
fn packages<'a>() -> impl Parser<
'a,
KeywordItem<'a, PackagesKeyword, Collection<'a, Loc<Spaced<'a, PackageEntry<'a>>>>>,
EPackages<'a>,
> {
record!(KeywordItem {
keyword: packages_kw(),
item: packages_collection()
})
}
#[inline(always)]
fn packages_kw<'a>() -> impl Parser<'a, Spaces<'a, PackagesKeyword>, EPackages<'a>> {
spaces_around_keyword(
PackagesKeyword,
EPackages::Packages,
EPackages::IndentPackages,
EPackages::IndentListStart,
)
}
#[inline(always)]
fn packages_collection<'a>(
) -> impl Parser<'a, Collection<'a, Loc<Spaced<'a, PackageEntry<'a>>>>, EPackages<'a>> {
collection_trailing_sep_e(
byte(b'{', EPackages::ListStart),
specialize_err(EPackages::PackageEntry, loc(package_entry())),
byte(b',', EPackages::ListEnd),
byte(b'}', EPackages::ListEnd),
Spaced::SpaceBefore,
)
}
#[inline(always)]
fn generates<'a>(
) -> impl Parser<'a, KeywordItem<'a, GeneratesKeyword, UppercaseIdent<'a>>, EGenerates> {
record!(KeywordItem {
keyword: spaces_around_keyword(
GeneratesKeyword,
EGenerates::Generates,
EGenerates::IndentGenerates,
EGenerates::IndentTypeStart
),
item: specialize_err(|(), pos| EGenerates::Identifier(pos), uppercase())
})
}
#[inline(always)]
fn generates_with<'a>() -> impl Parser<
'a,
KeywordItem<'a, WithKeyword, Collection<'a, Loc<Spaced<'a, ExposedName<'a>>>>>,
EGeneratesWith,
> {
record!(KeywordItem {
keyword: spaces_around_keyword(
WithKeyword,
EGeneratesWith::With,
EGeneratesWith::IndentWith,
EGeneratesWith::IndentListStart
),
item: collection_trailing_sep_e(
byte(b'[', EGeneratesWith::ListStart),
exposes_entry(EGeneratesWith::Identifier),
byte(b',', EGeneratesWith::ListEnd),
byte(b']', EGeneratesWith::ListEnd),
Spaced::SpaceBefore
)
})
}
#[inline(always)]
fn imports<'a>() -> impl Parser<
'a,
KeywordItem<'a, ImportsKeyword, Collection<'a, Loc<Spaced<'a, ImportsEntry<'a>>>>>,
EImports,
> {
record!(KeywordItem {
keyword: spaces_around_keyword(
ImportsKeyword,
EImports::Imports,
EImports::IndentImports,
EImports::IndentListStart
),
item: collection_trailing_sep_e(
byte(b'[', EImports::ListStart),
loc(imports_entry()),
byte(b',', EImports::ListEnd),
byte(b']', EImports::ListEnd),
Spaced::SpaceBefore
)
})
.trace("imports")
}
#[inline(always)]
pub fn typed_ident<'a>() -> impl Parser<'a, Spaced<'a, TypedIdent<'a>>, ETypedIdent<'a>> {
// e.g.
//
// printLine : Str -> Effect {}
map(
and(
and(
loc(specialize_err(
|_, pos| ETypedIdent::Identifier(pos),
lowercase_ident(),
)),
space0_e(ETypedIdent::IndentHasType),
),
skip_first(
byte(b':', ETypedIdent::HasType),
space0_before_e(
specialize_err(
ETypedIdent::Type,
reset_min_indent(type_annotation::located(true)),
),
ETypedIdent::IndentType,
),
),
),
|((ident, spaces_before_colon), ann)| {
Spaced::Item(TypedIdent {
ident,
spaces_before_colon,
ann,
})
},
)
}
fn shortname<'a>() -> impl Parser<'a, &'a str, EImports> {
specialize_err(|_, pos| EImports::Shorthand(pos), lowercase_ident())
}
pub fn module_name_help<'a, F, E>(to_expectation: F) -> impl Parser<'a, ModuleName<'a>, E>
where
F: Fn(Position) -> E,
E: 'a,
F: 'a,
{
specialize_err(move |_, pos| to_expectation(pos), module_name())
}
#[inline(always)]
fn imports_entry<'a>() -> impl Parser<'a, Spaced<'a, ImportsEntry<'a>>, EImports> {
type Temp<'a> = (
(Option<&'a str>, ModuleName<'a>),
Option<Collection<'a, Loc<Spaced<'a, ExposedName<'a>>>>>,
);
let spaced_import = |((opt_shortname, module_name), opt_values): Temp<'a>| {
let exposed_values = opt_values.unwrap_or_else(Collection::empty);
let entry = match opt_shortname {
Some(shortname) => ImportsEntry::Package(shortname, module_name, exposed_values),
None => ImportsEntry::Module(module_name, exposed_values),
};
Spaced::Item(entry)
};
one_of!(
map(
and(
and(
// e.g. `pf.`
optional(backtrackable(skip_second(
shortname(),
byte(b'.', EImports::ShorthandDot)
))),
// e.g. `Task`
module_name_help(EImports::ModuleName)
),
// e.g. `.{ Task, after}`
optional(skip_first(
byte(b'.', EImports::ExposingDot),
collection_trailing_sep_e(
byte(b'{', EImports::SetStart),
exposes_entry(EImports::Identifier),
byte(b',', EImports::SetEnd),
byte(b'}', EImports::SetEnd),
Spaced::SpaceBefore
)
))
),
spaced_import
)
.trace("normal_import"),
map(
and(
and(
// e.g. "filename"
// TODO: str literal allows for multiline strings. We probably don't want that for file names.
specialize_err(|_, pos| EImports::StrLiteral(pos), parse_str_literal()),
// e.g. as
and(
and(
space0_e(EImports::AsKeyword),
two_bytes(b'a', b's', EImports::AsKeyword)
),
space0_e(EImports::AsKeyword)
)
),
// e.g. file : Str
specialize_err(|_, pos| EImports::TypedIdent(pos), typed_ident())
),
|((file_name, _), typed_ident)| {
// TODO: look at blacking block strings during parsing.
Spaced::Item(ImportsEntry::IngestedFile(file_name, typed_ident))
}
)
.trace("ingest_file_import")
)
.trace("imports_entry")
}

View file

@ -5,12 +5,12 @@ use roc_region::all::{Loc, Position, Region};
use crate::{
ast::{
AbilityImpls, AbilityMember, AssignedField, Collection, Defs, Expr, Header, Implements,
ImplementsAbilities, ImplementsAbility, ImplementsClause, ImportAlias, ImportAsKeyword,
ImportExposingKeyword, ImportedModuleName, IngestedFileAnnotation, IngestedFileImport,
Module, ModuleImport, ModuleImportParams, OldRecordBuilderField, Pattern, PatternAs,
Spaced, Spaces, StrLiteral, StrSegment, Tag, TypeAnnotation, TypeDef, TypeHeader, ValueDef,
WhenBranch,
AbilityImpls, AbilityMember, AssignedField, Collection, Defs, Expr, Full, Header,
Implements, ImplementsAbilities, ImplementsAbility, ImplementsClause, ImportAlias,
ImportAsKeyword, ImportExposingKeyword, ImportedModuleName, IngestedFileAnnotation,
IngestedFileImport, ModuleImport, ModuleImportParams, OldRecordBuilderField, Pattern,
PatternAs, Spaced, Spaces, SpacesBefore, StrLiteral, StrSegment, Tag, TypeAnnotation,
TypeDef, TypeHeader, ValueDef, WhenBranch,
},
header::{
AppHeader, ExposedName, ExposesKeyword, GeneratesKeyword, HostedHeader, ImportsEntry,
@ -102,6 +102,24 @@ impl<'a, V: RemoveSpaces<'a>> RemoveSpaces<'a> for Spaces<'a, V> {
}
}
impl<'a, V: RemoveSpaces<'a>> RemoveSpaces<'a> for SpacesBefore<'a, V> {
fn remove_spaces(&self, arena: &'a Bump) -> Self {
SpacesBefore {
before: &[],
item: self.item.remove_spaces(arena),
}
}
}
impl<'a> RemoveSpaces<'a> for Full<'a> {
fn remove_spaces(&self, arena: &'a Bump) -> Self {
Full {
header: self.header.remove_spaces(arena),
defs: self.defs.remove_spaces(arena),
}
}
}
impl<'a, K: RemoveSpaces<'a>, V: RemoveSpaces<'a>> RemoveSpaces<'a> for KeywordItem<'a, K, V> {
fn remove_spaces(&self, arena: &'a Bump) -> Self {
KeywordItem {
@ -123,9 +141,9 @@ impl<'a> RemoveSpaces<'a> for ProvidesTo<'a> {
}
}
impl<'a> RemoveSpaces<'a> for Module<'a> {
impl<'a> RemoveSpaces<'a> for Header<'a> {
fn remove_spaces(&self, arena: &'a Bump) -> Self {
let header = match &self.header {
match self {
Header::Module(header) => Header::Module(ModuleHeader {
after_keyword: &[],
params: header.params.remove_spaces(arena),
@ -165,10 +183,6 @@ impl<'a> RemoveSpaces<'a> for Module<'a> {
generates: header.generates.remove_spaces(arena),
generates_with: header.generates_with.remove_spaces(arena),
}),
};
Module {
comments: &[],
header,
}
}
}

View file

@ -1,6 +1,8 @@
use crate::ast;
use crate::ast::Defs;
use crate::module::parse_module_defs;
use crate::ast::Header;
use crate::ast::SpacesBefore;
use crate::header::parse_module_defs;
use crate::parser::SourceError;
use crate::parser::SyntaxError;
use crate::state::State;
@ -39,10 +41,10 @@ pub fn parse_defs_with<'a>(arena: &'a Bump, input: &'a str) -> Result<Defs<'a>,
pub fn parse_header_with<'a>(
arena: &'a Bump,
input: &'a str,
) -> Result<ast::Module<'a>, SyntaxError<'a>> {
) -> Result<SpacesBefore<'a, Header<'a>>, SyntaxError<'a>> {
let state = State::new(input.as_bytes());
match crate::module::parse_header(arena, state.clone()) {
match crate::header::parse_header(arena, state.clone()) {
Ok((header, _)) => Ok(header),
Err(fail) => Err(SyntaxError::Header(fail.problem)),
}

View file

@ -22,7 +22,7 @@ mod test_parse {
use roc_parse::ast::StrSegment::*;
use roc_parse::ast::{self, EscapedChar};
use roc_parse::ast::{CommentOrNewline, StrLiteral::*};
use roc_parse::module::parse_module_defs;
use roc_parse::header::parse_module_defs;
use roc_parse::parser::SyntaxError;
use roc_parse::state::State;
use roc_parse::test_helpers::parse_expr_with;

View file

@ -1,8 +1,8 @@
use bumpalo::Bump;
use roc_fmt::{annotation::Formattable, module::fmt_module};
use roc_fmt::{annotation::Formattable, header::fmt_header};
use roc_parse::{
ast::{Defs, Expr, Malformed, Module},
module::parse_module_defs,
ast::{Defs, Expr, Full, Header, Malformed, SpacesBefore},
header::parse_module_defs,
parser::{Parser, SyntaxError},
remove_spaces::RemoveSpaces,
state::State,
@ -70,16 +70,13 @@ impl InputOwned {
/// Output AST of a successful parse
#[derive(Debug, Clone)]
pub enum Output<'a> {
Header(Module<'a>),
Header(SpacesBefore<'a, Header<'a>>),
ModuleDefs(Defs<'a>),
Expr(Expr<'a>),
Full {
header: Module<'a>,
module_defs: Defs<'a>,
},
Full(Full<'a>),
}
impl<'a> Output<'a> {
@ -88,7 +85,7 @@ impl<'a> Output<'a> {
let mut buf = Buf::new_in(&arena);
match self {
Output::Header(header) => {
fmt_module(&mut buf, header);
fmt_header(&mut buf, header);
buf.fmt_end_of_file();
InputOwned::Header(buf.as_str().to_string())
}
@ -101,12 +98,9 @@ impl<'a> Output<'a> {
expr.format(&mut buf, 0);
InputOwned::Expr(buf.as_str().to_string())
}
Output::Full {
header,
module_defs,
} => {
fmt_module(&mut buf, header);
module_defs.format(&mut buf, 0);
Output::Full(full) => {
fmt_header(&mut buf, &full.header);
full.defs.format(&mut buf, 0);
buf.fmt_end_of_file();
InputOwned::Full(buf.as_str().to_string())
}
@ -129,10 +123,7 @@ impl<'a> Malformed for Output<'a> {
Output::Header(header) => header.is_malformed(),
Output::ModuleDefs(defs) => defs.is_malformed(),
Output::Expr(expr) => expr.is_malformed(),
Output::Full {
header,
module_defs,
} => header.is_malformed() || module_defs.is_malformed(),
Output::Full(full) => full.is_malformed(),
}
}
}
@ -143,13 +134,7 @@ impl<'a> RemoveSpaces<'a> for Output<'a> {
Output::Header(header) => Output::Header(header.remove_spaces(arena)),
Output::ModuleDefs(defs) => Output::ModuleDefs(defs.remove_spaces(arena)),
Output::Expr(expr) => Output::Expr(expr.remove_spaces(arena)),
Output::Full {
header,
module_defs,
} => Output::Full {
header: header.remove_spaces(arena),
module_defs: module_defs.remove_spaces(arena),
},
Output::Full(full) => Output::Full(full.remove_spaces(arena)),
}
}
}
@ -185,18 +170,19 @@ impl<'a> Input<'a> {
let state = State::new(input.as_bytes());
let min_indent = 0;
let (_, header, state) = roc_parse::module::header()
let (_, header, state) = roc_parse::header::header()
.parse(arena, state.clone(), min_indent)
.map_err(|(_, fail)| SyntaxError::Header(fail))?;
let (header, defs) = header.upgrade_header_imports(arena);
let (new_header, defs) = header.item.upgrade_header_imports(arena);
let header = SpacesBefore {
before: header.before,
item: new_header,
};
let module_defs = parse_module_defs(arena, state, defs)?;
let defs = parse_module_defs(arena, state, defs)?;
Ok(Output::Full {
header,
module_defs,
})
Ok(Output::Full(Full { header, defs }))
}
}
}

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: App(
SpacesBefore {
before: [],
item: App(
AppHeader {
before_provides: [],
provides: [],

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: Hosted(
SpacesBefore {
before: [],
item: Hosted(
HostedHeader {
before_name: [],
name: @7-10 ModuleName(

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: Module(
SpacesBefore {
before: [],
item: Module(
ModuleHeader {
after_keyword: [],
params: None,

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: Package(
SpacesBefore {
before: [],
item: Package(
PackageHeader {
before_exposes: [],
exposes: [],

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: Platform(
SpacesBefore {
before: [],
item: Platform(
PlatformHeader {
before_name: [],
name: @9-25 PackageName(

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: App(
SpacesBefore {
before: [],
item: App(
AppHeader {
before_provides: [],
provides: [

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: App(
SpacesBefore {
before: [],
item: App(
AppHeader {
before_provides: [],
provides: [

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: Platform(
SpacesBefore {
before: [],
item: Platform(
PlatformHeader {
before_name: [],
name: @9-14 PackageName(

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: App(
SpacesBefore {
before: [],
item: App(
AppHeader {
before_provides: [],
provides: [],

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: Module(
SpacesBefore {
before: [],
item: Module(
ModuleHeader {
after_keyword: [],
params: None,

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: Module(
SpacesBefore {
before: [],
item: Module(
ModuleHeader {
after_keyword: [],
params: Some(

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: Module(
SpacesBefore {
before: [],
item: Module(
ModuleHeader {
after_keyword: [],
params: None,

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: Module(
SpacesBefore {
before: [],
item: Module(
ModuleHeader {
after_keyword: [],
params: Some(

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: Module(
SpacesBefore {
before: [],
item: Module(
ModuleHeader {
after_keyword: [],
params: Some(

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: Module(
SpacesBefore {
before: [],
item: Module(
ModuleHeader {
after_keyword: [],
params: Some(

View file

@ -1,7 +1,8 @@
Full {
header: Module {
comments: [],
header: App(
Full(
Full {
header: SpacesBefore {
before: [],
item: App(
AppHeader {
before_provides: [],
provides: [
@ -32,7 +33,7 @@ Full {
},
),
},
module_defs: Defs {
defs: Defs {
tags: [
Index(2147483648),
],
@ -78,4 +79,5 @@ Full {
),
],
},
}
},
)

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: Hosted(
SpacesBefore {
before: [],
item: Hosted(
HostedHeader {
before_name: [],
name: @7-10 ModuleName(

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: Package(
SpacesBefore {
before: [],
item: Package(
PackageHeader {
before_exposes: [],
exposes: [

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: Platform(
SpacesBefore {
before: [],
item: Platform(
PlatformHeader {
before_name: [],
name: @9-21 PackageName(

View file

@ -1,7 +1,8 @@
Full {
header: Module {
comments: [],
header: App(
Full(
Full {
header: SpacesBefore {
before: [],
item: App(
AppHeader {
before_provides: [
Newline,
@ -41,7 +42,7 @@ Full {
},
),
},
module_defs: Defs {
defs: Defs {
tags: [
Index(2147483648),
Index(2147483649),
@ -114,4 +115,5 @@ Full {
),
],
},
}
},
)

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: Module(
SpacesBefore {
before: [],
item: Module(
ModuleHeader {
after_keyword: [],
params: None,

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: App(
SpacesBefore {
before: [],
item: App(
AppHeader {
before_provides: [],
provides: [

View file

@ -1,6 +1,6 @@
Module {
comments: [],
header: Platform(
SpacesBefore {
before: [],
item: Platform(
PlatformHeader {
before_name: [],
name: @9-21 PackageName(

View file

@ -1,7 +1,8 @@
Full {
header: Module {
comments: [],
header: App(
Full(
Full {
header: SpacesBefore {
before: [],
item: App(
AppHeader {
before_provides: [],
provides: [
@ -27,7 +28,7 @@ Full {
},
),
},
module_defs: Defs {
defs: Defs {
tags: [
Index(2147483648),
],
@ -68,4 +69,5 @@ Full {
),
],
},
}
},
)

View file

@ -1,7 +1,8 @@
Full {
header: Module {
comments: [],
header: App(
Full(
Full {
header: SpacesBefore {
before: [],
item: App(
AppHeader {
before_provides: [],
provides: [
@ -35,7 +36,7 @@ Full {
},
),
},
module_defs: Defs {
defs: Defs {
tags: [
Index(2147483648),
Index(2147483649),
@ -206,4 +207,5 @@ Full {
),
],
},
}
},
)

View file

@ -1,7 +1,8 @@
Full {
header: Module {
comments: [],
header: App(
Full(
Full {
header: SpacesBefore {
before: [],
item: App(
AppHeader {
before_provides: [
Newline,
@ -41,7 +42,7 @@ Full {
},
),
},
module_defs: Defs {
defs: Defs {
tags: [
Index(2147483648),
],
@ -128,4 +129,5 @@ Full {
),
],
},
}
},
)

View file

@ -5,10 +5,10 @@ extern crate indoc;
mod test_fmt {
use bumpalo::Bump;
use roc_fmt::def::fmt_defs;
use roc_fmt::module::fmt_module;
use roc_fmt::header::fmt_header;
use roc_fmt::Buf;
use roc_parse::ast::{Defs, Module};
use roc_parse::module::{self, parse_module_defs};
use roc_parse::ast::{Defs, Header, SpacesBefore};
use roc_parse::header::{self, parse_module_defs};
use roc_parse::state::State;
use roc_test_utils::assert_multiline_str_eq;
use roc_test_utils_dir::workspace_root;
@ -32,11 +32,11 @@ mod test_fmt {
fn fmt_module_and_defs<'a>(
arena: &Bump,
src: &str,
module: &Module<'a>,
header: &SpacesBefore<'a, Header<'a>>,
state: State<'a>,
buf: &mut Buf<'_>,
) {
fmt_module(buf, module);
fmt_header(buf, header);
match parse_module_defs(arena, state, Defs::default()) {
Ok(loc_defs) => {
@ -61,7 +61,7 @@ mod test_fmt {
let src = src.trim();
let expected = expected.trim();
match module::parse_header(&arena, State::new(src.as_bytes())) {
match header::parse_header(&arena, State::new(src.as_bytes())) {
Ok((actual, state)) => {
use roc_parse::remove_spaces::RemoveSpaces;
@ -71,7 +71,7 @@ mod test_fmt {
let output = buf.as_str().trim();
let (reparsed_ast, state) = module::parse_header(&arena, State::new(output.as_bytes())).unwrap_or_else(|err| {
let (reparsed_ast, state) = header::parse_header(&arena, State::new(output.as_bytes())).unwrap_or_else(|err| {
panic!(
"After formatting, the source code no longer parsed!\n\nParse error was: {err:?}\n\nThe code that failed to parse:\n\n{output}\n\n"
);

View file

@ -1,8 +1,8 @@
use bumpalo::Bump;
use roc_fmt::Buf;
use roc_parse::{
ast::{Defs, Module},
module::parse_module_defs,
ast::{Defs, Header, SpacesBefore},
header::parse_module_defs,
parser::SyntaxError,
};
use roc_region::all::Loc;
@ -15,23 +15,26 @@ mod format;
pub struct Ast<'a> {
arena: &'a Bump,
module: Module<'a>,
module: SpacesBefore<'a, Header<'a>>,
defs: Defs<'a>,
}
impl<'a> Ast<'a> {
pub fn parse(arena: &'a Bump, src: &'a str) -> Result<Ast<'a>, SyntaxError<'a>> {
use roc_parse::{module::parse_header, state::State};
use roc_parse::{header::parse_header, state::State};
let (module, state) = parse_header(arena, State::new(src.as_bytes()))
.map_err(|e| SyntaxError::Header(e.problem))?;
let (module, defs) = module.upgrade_header_imports(arena);
let (header, defs) = module.item.upgrade_header_imports(arena);
let defs = parse_module_defs(arena, state, defs)?;
Ok(Ast {
module,
module: SpacesBefore {
before: module.before,
item: header,
},
defs,
arena,
})
@ -40,7 +43,7 @@ impl<'a> Ast<'a> {
pub fn fmt(&self) -> FormattedAst<'a> {
let mut buf = Buf::new_in(self.arena);
roc_fmt::module::fmt_module(&mut buf, &self.module);
roc_fmt::header::fmt_header(&mut buf, &self.module);
roc_fmt::def::fmt_defs(&mut buf, &self.defs, 0);
@ -50,7 +53,7 @@ impl<'a> Ast<'a> {
}
pub fn semantic_tokens(&self) -> impl IntoIterator<Item = Loc<Token>> + '_ {
let header_tokens = self.module.iter_tokens(self.arena);
let header_tokens = self.module.item.iter_tokens(self.arena);
let body_tokens = self.defs.iter_tokens(self.arena);
header_tokens.into_iter().chain(body_tokens)

View file

@ -6,8 +6,8 @@ use roc_module::called_via::{BinOp, UnaryOp};
use roc_parse::{
ast::{
AbilityImpls, AbilityMember, AssignedField, Collection, Defs, Expr, Header, Implements,
ImplementsAbilities, ImplementsAbility, ImplementsClause, Module, OldRecordBuilderField,
Pattern, PatternAs, Spaced, StrLiteral, Tag, TypeAnnotation, TypeDef, TypeHeader, ValueDef,
ImplementsAbilities, ImplementsAbility, ImplementsClause, OldRecordBuilderField, Pattern,
PatternAs, Spaced, StrLiteral, Tag, TypeAnnotation, TypeDef, TypeHeader, ValueDef,
WhenBranch,
},
header::{
@ -189,16 +189,6 @@ impl<T: IterTokens, U: IterTokens> IterTokens for (T, U) {
}
}
impl IterTokens for Module<'_> {
fn iter_tokens<'a>(&self, arena: &'a Bump) -> BumpVec<'a, Loc<Token>> {
let Self {
comments: _,
header,
} = self;
header.iter_tokens(arena)
}
}
impl IterTokens for Header<'_> {
fn iter_tokens<'a>(&self, arena: &'a Bump) -> BumpVec<'a, Loc<Token>> {
match self {

View file

@ -2,10 +2,10 @@ use brotli::enc::BrotliEncoderParams;
use bumpalo::Bump;
use flate2::write::GzEncoder;
use roc_parse::ast::{
Header, IngestedFileImport, Module, RecursiveValueDefIter, StrLiteral, ValueDef,
Header, IngestedFileImport, RecursiveValueDefIter, SpacesBefore, StrLiteral, ValueDef,
};
use roc_parse::header::PlatformHeader;
use roc_parse::module::{parse_header, parse_module_defs};
use roc_parse::header::{parse_header, parse_module_defs};
use roc_parse::state::State;
use std::ffi::OsStr;
use std::fs::File;
@ -129,7 +129,7 @@ fn write_archive<W: Write>(path: &Path, writer: W) -> io::Result<()> {
// TODO use this when finding .roc files by discovering them from the root module.
// let other_modules: &[Module<'_>] =
match read_header(&arena, &mut buf, path)?.0.header {
match read_header(&arena, &mut buf, path)?.0.item {
Header::Module(_) => {
todo!();
// TODO report error
@ -261,7 +261,7 @@ fn read_header<'a>(
arena: &'a Bump,
buf: &'a mut Vec<u8>,
path: &'a Path,
) -> io::Result<(Module<'a>, State<'a>)> {
) -> io::Result<(SpacesBefore<'a, Header<'a>>, State<'a>)> {
// Read all the bytes into the buffer.
{
let mut file = File::open(path)?;
@ -287,8 +287,8 @@ fn add_ingested_files<W: Write>(
builder: &mut tar::Builder<W>,
) -> io::Result<()> {
let mut buf = Vec::new();
let (module, state) = read_header(arena, &mut buf, dot_roc_path)?;
let (_, defs) = module.upgrade_header_imports(arena);
let (header, state) = read_header(arena, &mut buf, dot_roc_path)?;
let (_, defs) = header.item.upgrade_header_imports(arena);
let defs = parse_module_defs(arena, state, defs).unwrap_or_else(|err| {
panic!("{} failed to parse: {:?}", dot_roc_path.display(), err);