Fix a bunch of parser/formatter bugs found in fuzzing

Notably:
* Unified how parens are formatted between (1) when we have a ParensAround, and (2) when we've decided an Apply needs to have parens
* Made unary minus require the be indented to the same level as any other expression continuation. (it used to accidentally have rules meant for binary operators applied)
* Don't apply extra indent to the backpassing continuation in the case that the call does itself require indentation
* Make `try@foo` correctly parse as `try @foo`, so that formatting doesn't change the tree when it adds that space
* Detect more cases where we need to outdent trailing e.g. {} blocks in applies
* Approximately a bagillion other things, 90% of which I added tests for, and none of which affected the formatting of examples or builtins
This commit is contained in:
Joshua Warner 2024-11-17 19:34:47 -08:00
parent 335a8eb258
commit ed62bcc15a
No known key found for this signature in database
GPG key ID: 89AD497003F93FDD
347 changed files with 8219 additions and 1162 deletions

View file

@ -1,11 +1,14 @@
use crate::{
collection::{fmt_collection, Braces},
expr::merge_spaces_conservative,
spaces::{fmt_comments_only, fmt_spaces, NewlineAt, INDENT},
Buf,
};
use bumpalo::{collections::Vec, Bump};
use roc_parse::ast::{
AbilityImpls, AssignedField, Collection, Expr, ExtractSpaces, FunctionArrow,
ImplementsAbilities, ImplementsAbility, ImplementsClause, Tag, TypeAnnotation, TypeHeader,
AbilityImpls, AssignedField, Collection, CommentOrNewline, Expr, ExtractSpaces, FunctionArrow,
ImplementsAbilities, ImplementsAbility, ImplementsClause, Spaceable, Spaces, SpacesAfter,
SpacesBefore, Tag, TypeAnnotation, TypeHeader,
};
use roc_parse::ident::UppercaseIdent;
use roc_region::all::Loc;
@ -35,6 +38,7 @@ use roc_region::all::Loc;
#[derive(PartialEq, Eq, Clone, Copy, Debug)]
pub enum Parens {
NotNeeded,
InCollection,
InFunctionType,
InApply,
InOperator,
@ -148,7 +152,10 @@ impl<'a> Formattable for TypeAnnotation<'a> {
true
}
Wildcard | Inferred | BoundVariable(_) | Malformed(_) => false,
TypeAnnotation::Wildcard
| TypeAnnotation::Inferred
| BoundVariable(_)
| Malformed(_) => false,
Function(args, _arrow, result) => {
result.value.is_multiline()
|| args.iter().any(|loc_arg| loc_arg.value.is_multiline())
@ -165,8 +172,7 @@ impl<'a> Formattable for TypeAnnotation<'a> {
Some(ann) if ann.value.is_multiline() => return true,
_ => {}
}
fields.items.iter().any(|field| field.value.is_multiline())
is_collection_multiline(fields)
}
Record { fields, ext } => {
@ -175,7 +181,7 @@ impl<'a> Formattable for TypeAnnotation<'a> {
_ => {}
}
fields.items.iter().any(|field| field.value.is_multiline())
is_collection_multiline(fields)
}
TagUnion { tags, ext } => {
@ -184,221 +190,333 @@ impl<'a> Formattable for TypeAnnotation<'a> {
_ => {}
}
tags.iter().any(|tag| tag.value.is_multiline())
!tags.final_comments().is_empty() || tags.iter().any(|tag| tag.value.is_multiline())
}
}
}
fn format_with_options(&self, buf: &mut Buf, parens: Parens, newlines: Newlines, indent: u16) {
use roc_parse::ast::TypeAnnotation::*;
fmt_ty_ann(self, buf, indent, parens, newlines, false);
}
}
let self_is_multiline = self.is_multiline();
fn fmt_ty_ann(
me: &TypeAnnotation<'_>,
buf: &mut Buf<'_>,
indent: u16,
parens: Parens,
newlines: Newlines,
newline_at_top: bool,
) {
let me = ann_lift_spaces(buf.text.bump(), me);
match self {
Function(args, arrow, ret) => {
let needs_parens = parens != Parens::NotNeeded;
let self_is_multiline = me.item.is_multiline();
buf.indent(indent);
if !me.before.is_empty() {
buf.ensure_ends_with_newline();
fmt_comments_only(buf, me.before.iter(), NewlineAt::Bottom, indent);
}
if newline_at_top {
buf.ensure_ends_with_newline();
}
if needs_parens {
buf.push('(')
}
match &me.item {
TypeAnnotation::SpaceBefore(_ann, _spaces) | TypeAnnotation::SpaceAfter(_ann, _spaces) => {
unreachable!()
}
TypeAnnotation::Function(args, arrow, ret) => {
let needs_parens = parens != Parens::NotNeeded;
let mut it = args.iter().enumerate().peekable();
buf.indent(indent);
while let Some((index, argument)) = it.next() {
let is_first = index == 0;
let is_multiline = &argument.value.is_multiline();
if !is_first && !is_multiline && self_is_multiline {
buf.newline();
}
argument.value.format_with_options(
buf,
Parens::InFunctionType,
Newlines::Yes,
indent,
);
if it.peek().is_some() {
buf.push_str(",");
if !self_is_multiline {
buf.spaces(1);
}
}
}
if self_is_multiline {
buf.newline();
buf.indent(indent);
} else {
buf.spaces(1);
}
match arrow {
FunctionArrow::Pure => buf.push_str("->"),
FunctionArrow::Effectful => buf.push_str("=>"),
}
buf.spaces(1);
ret.value
.format_with_options(buf, Parens::InFunctionType, Newlines::No, indent);
if needs_parens {
buf.push(')')
}
if needs_parens {
buf.push('(')
}
Apply(pkg, name, arguments) => {
buf.indent(indent);
let write_parens = parens == Parens::InApply && !arguments.is_empty();
if write_parens {
buf.push('(')
}
for (index, argument) in args.iter().enumerate() {
let is_first = index == 0;
if !pkg.is_empty() {
buf.push_str(pkg);
buf.push('.');
}
buf.push_str(name);
let needs_indent = except_last(arguments).any(|a| a.is_multiline())
|| arguments
.last()
.map(|a| {
a.is_multiline()
&& (!a.extract_spaces().before.is_empty()
|| !is_outdentable(&a.value))
})
.unwrap_or_default();
let arg_indent = if needs_indent {
indent + INDENT
} else {
indent
};
for arg in arguments.iter() {
if needs_indent {
let arg = arg.extract_spaces();
fmt_spaces(buf, arg.before.iter(), arg_indent);
buf.ensure_ends_with_newline();
arg.item.format_with_options(
buf,
Parens::InApply,
Newlines::Yes,
arg_indent,
);
fmt_spaces(buf, arg.after.iter(), arg_indent);
} else {
if !is_first {
buf.push_str(",");
if !self_is_multiline {
buf.spaces(1);
arg.format_with_options(buf, Parens::InApply, Newlines::No, arg_indent);
}
}
if write_parens {
buf.push(')')
}
let newline_at_top = !is_first && self_is_multiline;
fmt_ty_ann(
&argument.value,
buf,
indent,
Parens::InFunctionType,
Newlines::Yes,
newline_at_top,
);
}
BoundVariable(v) => {
if self_is_multiline {
buf.newline();
buf.indent(indent);
buf.push_str(v)
}
Wildcard => {
buf.indent(indent);
buf.push('*')
}
Inferred => {
buf.indent(indent);
buf.push('_')
}
TagUnion { tags, ext } => {
fmt_collection(buf, indent, Braces::Square, *tags, newlines);
if let Some(loc_ext_ann) = *ext {
loc_ext_ann.value.format(buf, indent);
}
}
Tuple { elems: fields, ext } => {
fmt_collection(buf, indent, Braces::Round, *fields, newlines);
if let Some(loc_ext_ann) = *ext {
loc_ext_ann.value.format(buf, indent);
}
}
Record { fields, ext } => {
fmt_collection(buf, indent, Braces::Curly, *fields, newlines);
if let Some(loc_ext_ann) = *ext {
loc_ext_ann.value.format(buf, indent);
}
}
As(lhs, _spaces, TypeHeader { name, vars }) => {
// TODO use _spaces?
lhs.value
.format_with_options(buf, Parens::InFunctionType, Newlines::No, indent);
} else {
buf.spaces(1);
buf.push_str("as");
buf.spaces(1);
buf.push_str(name.value);
for var in *vars {
buf.spaces(1);
var.value
.format_with_options(buf, Parens::NotNeeded, Newlines::No, indent);
}
}
Where(annot, implements_clauses) => {
annot.format_with_options(buf, parens, newlines, indent);
if implements_clauses
.iter()
.any(|implements| implements.is_multiline())
{
buf.newline();
buf.indent(indent);
match arrow {
FunctionArrow::Pure => buf.push_str("->"),
FunctionArrow::Effectful => buf.push_str("=>"),
}
buf.spaces(1);
ret.value
.format_with_options(buf, Parens::InFunctionType, Newlines::No, indent);
if needs_parens {
buf.push(')')
}
}
TypeAnnotation::Apply(pkg, name, arguments) => {
buf.indent(indent);
let write_parens = parens == Parens::InApply && !arguments.is_empty();
if write_parens {
buf.push('(')
}
if !pkg.is_empty() {
buf.push_str(pkg);
buf.push('.');
}
buf.push_str(name);
let needs_indent = except_last(arguments).any(|a| a.is_multiline())
|| arguments
.last()
.map(|a| {
a.is_multiline()
&& (!a.extract_spaces().before.is_empty()
|| !ty_is_outdentable(&a.value))
})
.unwrap_or_default();
let arg_indent = if needs_indent {
indent + INDENT
} else {
indent
};
for arg in arguments.iter() {
if needs_indent {
let arg = arg.extract_spaces();
fmt_spaces(buf, arg.before.iter(), arg_indent);
buf.ensure_ends_with_newline();
arg.item
.format_with_options(buf, Parens::InApply, Newlines::Yes, arg_indent);
fmt_spaces(buf, arg.after.iter(), arg_indent);
} else {
buf.spaces(1);
}
for (i, has) in implements_clauses.iter().enumerate() {
buf.push_str(if i == 0 {
roc_parse::keyword::WHERE
} else {
","
});
buf.spaces(1);
has.format_with_options(buf, parens, newlines, indent);
arg.format_with_options(buf, Parens::InApply, Newlines::No, arg_indent);
}
}
SpaceBefore(ann, spaces) => {
buf.ensure_ends_with_newline();
fmt_comments_only(buf, spaces.iter(), NewlineAt::Bottom, indent);
ann.format_with_options(buf, parens, newlines, indent)
if write_parens {
buf.push(')')
}
SpaceAfter(ann, spaces) => {
ann.format_with_options(buf, parens, newlines, indent);
fmt_comments_only(buf, spaces.iter(), NewlineAt::Bottom, indent);
}
TypeAnnotation::BoundVariable(v) => {
buf.indent(indent);
buf.push_str(v)
}
TypeAnnotation::Wildcard => {
buf.indent(indent);
buf.push('*')
}
TypeAnnotation::Inferred => {
buf.indent(indent);
buf.push('_')
}
TypeAnnotation::TagUnion { tags, ext } => {
fmt_collection(buf, indent, Braces::Square, *tags, newlines);
fmt_ext(ext, buf, indent);
}
TypeAnnotation::Tuple { elems: fields, ext } => {
fmt_ty_collection(buf, indent, Braces::Round, *fields, newlines);
fmt_ext(ext, buf, indent);
}
TypeAnnotation::Record { fields, ext } => {
fmt_collection(buf, indent, Braces::Curly, *fields, newlines);
fmt_ext(ext, buf, indent);
}
TypeAnnotation::As(lhs, _spaces, TypeHeader { name, vars }) => {
// TODO use _spaces?
lhs.value
.format_with_options(buf, Parens::InFunctionType, Newlines::No, indent);
buf.spaces(1);
buf.push_str("as");
buf.spaces(1);
buf.push_str(name.value);
for var in *vars {
buf.spaces(1);
var.value
.format_with_options(buf, Parens::NotNeeded, Newlines::No, indent);
}
Malformed(raw) => {
}
TypeAnnotation::Where(annot, implements_clauses) => {
annot.format_with_options(buf, parens, newlines, indent);
if implements_clauses
.iter()
.any(|implements| implements.is_multiline())
{
buf.newline();
buf.indent(indent);
buf.push_str(raw)
} else {
buf.spaces(1);
}
for (i, has) in implements_clauses.iter().enumerate() {
buf.push_str(if i == 0 {
roc_parse::keyword::WHERE
} else {
","
});
buf.spaces(1);
has.format_with_options(buf, parens, newlines, indent);
}
}
TypeAnnotation::Malformed(raw) => {
buf.indent(indent);
buf.push_str(raw)
}
}
if !me.after.is_empty() {
fmt_comments_only(buf, me.after.iter(), NewlineAt::Bottom, indent);
}
}
fn lower<'a, 'b: 'a>(
arena: &'b Bump,
lifted: Spaces<'b, TypeAnnotation<'b>>,
) -> TypeAnnotation<'b> {
if lifted.before.is_empty() && lifted.after.is_empty() {
return lifted.item;
}
if lifted.before.is_empty() {
return TypeAnnotation::SpaceAfter(arena.alloc(lifted.item), lifted.after);
}
if lifted.after.is_empty() {
return TypeAnnotation::SpaceBefore(arena.alloc(lifted.item), lifted.before);
}
TypeAnnotation::SpaceBefore(
arena.alloc(TypeAnnotation::SpaceAfter(
arena.alloc(lifted.item),
lifted.after,
)),
lifted.before,
)
}
fn fmt_ty_collection(
buf: &mut Buf<'_>,
indent: u16,
braces: Braces,
items: Collection<'_, Loc<TypeAnnotation<'_>>>,
newlines: Newlines,
) {
let arena = buf.text.bump();
let mut new_items: Vec<'_, NodeSpaces<'_, Node<'_>>> =
Vec::with_capacity_in(items.len(), arena);
let mut last_after: &[CommentOrNewline<'_>] = &[];
for (i, item) in items.items.iter().enumerate() {
let func_ty_risky = i > 0;
let lifted = ann_lift_to_node(
if func_ty_risky {
Parens::InCollection
} else {
Parens::NotNeeded
},
arena,
&item.value,
);
let before = merge_spaces_conservative(arena, last_after, lifted.before);
last_after = lifted.after;
new_items.push(NodeSpaces {
before,
item: lifted.item,
after: &[],
});
}
let final_comments = merge_spaces_conservative(arena, last_after, items.final_comments());
let new_items =
Collection::with_items_and_comments(arena, new_items.into_bump_slice(), final_comments);
fmt_collection(buf, indent, braces, new_items, newlines)
}
fn fmt_ext(ext: &Option<&Loc<TypeAnnotation<'_>>>, buf: &mut Buf<'_>, indent: u16) {
if let Some(loc_ext_ann) = *ext {
let me = ann_lift_spaces(buf.text.bump(), &loc_ext_ann.value);
let parens_needed = !me.before.is_empty() || ext_needs_parens(me.item);
if parens_needed {
// We need to make sure to not have whitespace before the ext of a type,
// since that would make it parse as something else.
buf.push('(');
loc_ext_ann.value.format(buf, indent + INDENT);
buf.indent(indent);
buf.push(')');
} else {
loc_ext_ann.value.format(buf, indent + INDENT);
}
}
}
fn is_outdentable(ann: &TypeAnnotation) -> bool {
matches!(
ann.extract_spaces().item,
TypeAnnotation::Tuple { .. } | TypeAnnotation::Record { .. }
)
fn ext_needs_parens(item: TypeAnnotation<'_>) -> bool {
match item {
TypeAnnotation::Record { .. }
| TypeAnnotation::TagUnion { .. }
| TypeAnnotation::Tuple { .. }
| TypeAnnotation::BoundVariable(..)
| TypeAnnotation::Wildcard
| TypeAnnotation::Inferred => false,
TypeAnnotation::Apply(_module, _func, args) => !args.is_empty(),
_ => true,
}
}
pub fn ty_is_outdentable(mut rhs: &TypeAnnotation) -> bool {
loop {
match rhs {
TypeAnnotation::SpaceBefore(sub_def, spaces) => {
let is_only_newlines = spaces.iter().all(|s| s.is_newline());
if !is_only_newlines || !sub_def.is_multiline() {
return false;
}
rhs = sub_def;
}
TypeAnnotation::SpaceAfter(sub_def, _) => {
rhs = sub_def;
}
TypeAnnotation::Where(ann, _clauses) => {
if !ann.is_multiline() {
return false;
}
rhs = &ann.value;
}
TypeAnnotation::Record { .. }
| TypeAnnotation::TagUnion { .. }
| TypeAnnotation::Tuple { .. } => return rhs.is_multiline(),
_ => return false,
}
}
}
/// Fields are subtly different on the type and term level:
@ -466,6 +584,7 @@ fn format_assigned_field_help<T>(
}
buf.spaces(separator_spaces);
buf.indent(indent);
buf.push(':');
buf.spaces(1);
ann.value.format(buf, indent);
@ -483,6 +602,7 @@ fn format_assigned_field_help<T>(
}
buf.spaces(separator_spaces);
buf.indent(indent);
buf.push('?');
buf.spaces(1);
ann.value.format(buf, indent);
@ -704,3 +824,279 @@ pub fn except_last<T>(items: &[T]) -> impl Iterator<Item = &T> {
items[..items.len() - 1].iter()
}
}
pub fn ann_lift_spaces<'a, 'b: 'a>(
arena: &'a Bump,
ann: &TypeAnnotation<'b>,
) -> Spaces<'a, TypeAnnotation<'a>> {
match ann {
TypeAnnotation::Apply(module, func, args) => {
if args.is_empty() {
return Spaces {
item: *ann,
before: &[],
after: &[],
};
}
let mut new_args = Vec::with_capacity_in(args.len(), arena);
if !args.is_empty() {
for arg in args.iter().take(args.len() - 1) {
let lifted = ann_lift_spaces(arena, &arg.value);
new_args.push(Loc::at(arg.region, lower(arena, lifted)));
}
}
let after = if let Some(last) = args.last() {
let lifted = ann_lift_spaces(arena, &last.value);
if lifted.before.is_empty() {
new_args.push(Loc::at(last.region, lifted.item));
} else {
new_args.push(Loc::at(
last.region,
TypeAnnotation::SpaceBefore(arena.alloc(lifted.item), lifted.before),
));
}
lifted.after
} else {
&[]
};
Spaces {
before: &[],
item: TypeAnnotation::Apply(module, func, new_args.into_bump_slice()),
after,
}
}
TypeAnnotation::SpaceBefore(expr, spaces) => {
let mut inner = ann_lift_spaces(arena, expr);
inner.before = merge_spaces_conservative(arena, spaces, inner.before);
inner
}
TypeAnnotation::SpaceAfter(expr, spaces) => {
let mut inner = ann_lift_spaces(arena, expr);
inner.after = merge_spaces_conservative(arena, inner.after, spaces);
inner
}
_ => Spaces {
before: &[],
item: *ann,
after: &[],
},
}
}
pub fn ann_lift_spaces_before<'a, 'b: 'a>(
arena: &'a Bump,
ann: &TypeAnnotation<'b>,
) -> SpacesBefore<'a, TypeAnnotation<'a>> {
let lifted = ann_lift_spaces(arena, ann);
SpacesBefore {
before: lifted.before,
item: lifted.item.maybe_after(arena, lifted.after),
}
}
pub fn ann_lift_spaces_after<'a, 'b: 'a>(
arena: &'a Bump,
ann: &TypeAnnotation<'b>,
) -> SpacesAfter<'a, TypeAnnotation<'a>> {
let lifted = ann_lift_spaces(arena, ann);
SpacesAfter {
item: lifted.item.maybe_before(arena, lifted.before),
after: lifted.after,
}
}
type Sp<'a> = &'a [CommentOrNewline<'a>];
#[derive(Copy, Clone, Debug)]
enum Node<'a> {
DelimitedSequence(Braces, &'a [(Sp<'a>, Node<'a>)], Sp<'a>),
TypeAnnotation(TypeAnnotation<'a>),
}
impl<'a> Formattable for Node<'a> {
fn is_multiline(&self) -> bool {
match self {
Node::DelimitedSequence(_braces, lefts, right) => {
if !right.is_empty() {
return true;
}
for (sp, l) in *lefts {
if l.is_multiline() || !sp.is_empty() {
return true;
}
}
false
}
Node::TypeAnnotation(type_annotation) => type_annotation.is_multiline(),
}
}
fn format_with_options(&self, buf: &mut Buf, parens: Parens, newlines: Newlines, indent: u16) {
match self {
Node::DelimitedSequence(braces, lefts, right) => {
buf.indent(indent);
buf.push(braces.start());
for (sp, l) in *lefts {
if !sp.is_empty() {
fmt_spaces(buf, sp.iter(), indent);
}
l.format_with_options(buf, parens, newlines, indent);
}
if !right.is_empty() {
fmt_spaces(buf, right.iter(), indent);
}
buf.indent(indent);
buf.push(braces.end());
}
Node::TypeAnnotation(type_annotation) => {
type_annotation.format_with_options(buf, parens, newlines, indent);
}
}
}
}
fn ann_lift_to_node<'a, 'b: 'a>(
parens: Parens,
arena: &'a Bump,
ann: &TypeAnnotation<'b>,
) -> Spaces<'a, Node<'a>> {
match ann {
TypeAnnotation::Apply(module, func, args) => {
if args.is_empty() {
return Spaces {
item: Node::TypeAnnotation(*ann),
before: &[],
after: &[],
};
}
let mut new_args = Vec::with_capacity_in(args.len(), arena);
if !args.is_empty() {
for arg in args.iter().take(args.len() - 1) {
let lifted = ann_lift_spaces(arena, &arg.value);
new_args.push(Loc::at(arg.region, lower(arena, lifted)));
}
}
let after = if let Some(last) = args.last() {
let lifted = ann_lift_spaces(arena, &last.value);
if lifted.before.is_empty() {
new_args.push(Loc::at(last.region, lifted.item));
} else {
new_args.push(Loc::at(
last.region,
TypeAnnotation::SpaceBefore(arena.alloc(lifted.item), lifted.before),
));
}
lifted.after
} else {
&[]
};
Spaces {
before: &[],
item: Node::TypeAnnotation(TypeAnnotation::Apply(
module,
func,
new_args.into_bump_slice(),
)),
after,
}
}
TypeAnnotation::SpaceBefore(expr, spaces) => {
let mut inner = ann_lift_to_node(parens, arena, expr);
inner.before = merge_spaces_conservative(arena, spaces, inner.before);
inner
}
TypeAnnotation::SpaceAfter(expr, spaces) => {
let mut inner = ann_lift_to_node(parens, arena, expr);
inner.after = merge_spaces_conservative(arena, inner.after, spaces);
inner
}
TypeAnnotation::Function(args, _purity, res) => {
let new_args = arena.alloc_slice_copy(args);
let before = if let Some(first) = new_args.first_mut() {
let lifted = ann_lift_spaces_before(arena, &first.value);
first.value = lifted.item;
lifted.before
} else {
&[]
};
let new_res = ann_lift_spaces_after(arena, &res.value);
let new_ann = TypeAnnotation::Function(
new_args,
FunctionArrow::Pure,
arena.alloc(Loc::at_zero(new_res.item)),
);
let inner = Spaces {
before,
item: Node::TypeAnnotation(new_ann),
after: new_res.after,
};
if parens == Parens::InCollection {
let node = Node::DelimitedSequence(
Braces::Round,
arena.alloc_slice_copy(&[(inner.before, inner.item)]),
inner.after,
);
Spaces {
before: &[],
item: node,
after: &[],
}
} else {
inner
}
}
_ => Spaces {
before: &[],
item: Node::TypeAnnotation(*ann),
after: &[],
},
}
}
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub struct NodeSpaces<'a, T> {
pub before: &'a [CommentOrNewline<'a>],
pub item: T,
pub after: &'a [CommentOrNewline<'a>],
}
impl<'a, T: Copy> ExtractSpaces<'a> for NodeSpaces<'a, T> {
type Item = T;
fn extract_spaces(&self) -> Spaces<'a, T> {
Spaces {
before: self.before,
item: self.item,
after: self.after,
}
}
}
impl<'a, V: Formattable> Formattable for NodeSpaces<'a, V> {
fn is_multiline(&self) -> bool {
!self.before.is_empty() || !self.after.is_empty() || self.item.is_multiline()
}
fn format_with_options(
&self,
buf: &mut Buf,
parens: crate::annotation::Parens,
newlines: Newlines,
indent: u16,
) {
fmt_spaces(buf, self.before.iter(), indent);
self.item.format_with_options(buf, parens, newlines, indent);
fmt_spaces(buf, self.after.iter(), indent);
}
}