Merge branch 'trunk' of ssh://github.com/rtfeldman/roc into text_highlighting

cli/src/repl/eval.rs

@@ -93,7 +93,10 @@ fn jit_to_ast_help<'a>(
         ),
         Layout::Builtin(Builtin::EmptyList) => {
             Ok(run_jit_function!(lib, main_fn_name, &'static str, |_| {
-                Expr::List(&[])
+                Expr::List {
+                    items: &[],
+                    final_comments: &[],
+                }
             }))
         }
         Layout::Builtin(Builtin::List(_, elem_layout)) => Ok(run_jit_function!(
@@ -251,7 +254,10 @@ fn ptr_to_ast<'a>(
 
             num_to_ast(env, f64_to_ast(env.arena, num), content)
         }
-        Layout::Builtin(Builtin::EmptyList) => Expr::List(&[]),
+        Layout::Builtin(Builtin::EmptyList) => Expr::List {
+            items: &[],
+            final_comments: &[],
+        },
         Layout::Builtin(Builtin::List(_, elem_layout)) => {
             // Turn the (ptr, len) wrapper struct into actual ptr and len values.
             let len = unsafe { *(ptr.offset(env.ptr_bytes as isize) as *const usize) };
@@ -331,7 +337,10 @@ fn list_to_ast<'a>(
 
     let output = output.into_bump_slice();
 
-    Expr::List(output)
+    Expr::List {
+        items: output,
+        final_comments: &[],
+    }
 }
 
 fn single_tag_union_to_ast<'a>(

compiler/can/src/expr.rs

@@ -285,7 +285,9 @@ pub fn canonicalize_expr<'a>(
             }
         }
         ast::Expr::Str(literal) => flatten_str_literal(env, var_store, scope, literal),
-        ast::Expr::List(loc_elems) => {
+        ast::Expr::List {
+            items: loc_elems, ..
+        } => {
             if loc_elems.is_empty() {
                 (
                     List {

compiler/can/src/operator.rs

@@ -109,14 +109,24 @@ pub fn desugar_expr<'a>(arena: &'a Bump, loc_expr: &'a Located<Expr<'a>>) -> &'a
 
             arena.alloc(Located { region, value })
         }
-        List(elems) | Nested(List(elems)) => {
+        List {
-            let mut new_elems = Vec::with_capacity_in(elems.len(), arena);
+            items,
+            final_comments,
+        }
+        | Nested(List {
+            items,
+            final_comments,
+        }) => {
+            let mut new_items = Vec::with_capacity_in(items.len(), arena);
 
-            for elem in elems.iter() {
+            for item in items.iter() {
-                new_elems.push(desugar_expr(arena, elem));
+                new_items.push(desugar_expr(arena, item));
             }
-            let new_elems = new_elems.into_bump_slice();
+            let new_items = new_items.into_bump_slice();
-            let value: Expr<'a> = List(new_elems);
+            let value: Expr<'a> = List {
+                items: new_items,
+                final_comments,
+            };
 
             arena.alloc(Located {
                 region: loc_expr.region,

compiler/fmt/src/def.rs

@@ -122,7 +122,7 @@ pub fn fmt_body<'a>(
             Expr::SpaceBefore(_, _) => {
                 body.format_with_options(buf, Parens::NotNeeded, Newlines::Yes, indent + INDENT);
             }
-            Expr::Record { .. } | Expr::List(_) => {
+            Expr::Record { .. } | Expr::List { .. } => {
                 newline(buf, indent + INDENT);
                 body.format_with_options(buf, Parens::NotNeeded, Newlines::Yes, indent + INDENT);
             }

compiler/fmt/src/expr.rs

@@ -38,7 +38,7 @@ impl<'a> Formattable<'a> for Expr<'a> {
             // These expressions always have newlines
             Defs(_, _) | When(_, _) => true,
 
-            List(elems) => elems.iter().any(|loc_expr| loc_expr.is_multiline()),
+            List { items, .. } => items.iter().any(|loc_expr| loc_expr.is_multiline()),
 
             Str(literal) => {
                 use roc_parse::ast::StrLiteral::*;
@@ -261,8 +261,11 @@ impl<'a> Formattable<'a> for Expr<'a> {
                 fmt_if(buf, loc_condition, loc_then, loc_else, indent);
             }
             When(loc_condition, branches) => fmt_when(buf, loc_condition, branches, indent),
-            List(loc_items) => {
+            List {
-                fmt_list(buf, &loc_items, indent);
+                items,
+                final_comments,
+            } => {
+                fmt_list(buf, &items, final_comments, indent);
             }
             BinOp((loc_left_side, bin_op, loc_right_side)) => fmt_bin_op(
                 buf,
@@ -396,92 +399,88 @@ fn fmt_bin_op<'a>(
     }
 }
 
-pub fn fmt_list<'a>(buf: &mut String<'a>, loc_items: &[&Located<Expr<'a>>], indent: u16) {
+pub fn fmt_list<'a>(
-    buf.push('[');
+    buf: &mut String<'a>,
-
+    loc_items: &[&Located<Expr<'a>>],
-    let mut iter = loc_items.iter().peekable();
+    final_comments: &'a [CommentOrNewline<'a>],
-
+    indent: u16,
-    let is_multiline = loc_items.iter().any(|item| (&item.value).is_multiline());
+) {
-
+    if loc_items.is_empty() && final_comments.iter().all(|c| c.is_newline()) {
-    let item_indent = if is_multiline {
+        buf.push_str("[]");
-        indent + INDENT
     } else {
-        indent
+        buf.push('[');
-    };
+        let is_multiline = loc_items.iter().any(|item| (&item.value).is_multiline());
-
-    while let Some(item) = iter.next() {
         if is_multiline {
-            match &item.value {
+            let item_indent = indent + INDENT;
-                Expr::SpaceBefore(expr_below, spaces_above_expr) => {
+            for item in loc_items.iter() {
-                    newline(buf, item_indent);
+                match &item.value {
-                    fmt_comments_only(
+                    // TODO?? These SpaceAfter/SpaceBefore litany seems overcomplicated
-                        buf,
+                    // Can we simplify this? Or at least move this in a separate function.
-                        spaces_above_expr.iter(),
+                    Expr::SpaceBefore(expr_below, spaces_above_expr) => {
-                        NewlineAt::Bottom,
+                        newline(buf, item_indent);
-                        item_indent,
+                        fmt_comments_only(
-                    );
+                            buf,
+                            spaces_above_expr.iter(),
+                            NewlineAt::Bottom,
+                            item_indent,
+                        );
 
-                    match &expr_below {
+                        match &expr_below {
-                        Expr::SpaceAfter(expr_above, spaces_below_expr) => {
+                            Expr::SpaceAfter(expr_above, spaces_below_expr) => {
-                            expr_above.format(buf, item_indent);
+                                expr_above.format(buf, item_indent);
-
-                            if iter.peek().is_some() {
                                 buf.push(',');
-                            }
 
-                            fmt_comments_only(
+                                fmt_comments_only(
-                                buf,
+                                    buf,
-                                spaces_below_expr.iter(),
+                                    spaces_below_expr.iter(),
-                                NewlineAt::Top,
+                                    NewlineAt::Top,
-                                item_indent,
+                                    item_indent,
-                            );
+                                );
-                        }
+                            }
-                        _ => {
+                            _ => {
-                            expr_below.format(buf, item_indent);
+                                expr_below.format(buf, item_indent);
-                            if iter.peek().is_some() {
                                 buf.push(',');
                             }
                         }
                     }
-                }
 
-                Expr::SpaceAfter(sub_expr, spaces) => {
+                    Expr::SpaceAfter(sub_expr, spaces) => {
-                    newline(buf, item_indent);
+                        newline(buf, item_indent);
 
-                    sub_expr.format(buf, item_indent);
+                        sub_expr.format(buf, item_indent);
-
-                    if iter.peek().is_some() {
                         buf.push(',');
+
+                        fmt_comments_only(buf, spaces.iter(), NewlineAt::Top, item_indent);
                     }
 
-                    fmt_comments_only(buf, spaces.iter(), NewlineAt::Top, item_indent);
+                    _ => {
-                }
+                        newline(buf, item_indent);
-
+                        item.format_with_options(
-                _ => {
+                            buf,
-                    newline(buf, item_indent);
+                            Parens::NotNeeded,
-                    item.format_with_options(buf, Parens::NotNeeded, Newlines::Yes, item_indent);
+                            Newlines::Yes,
-                    if iter.peek().is_some() {
+                            item_indent,
+                        );
                         buf.push(',');
                     }
                 }
             }
+            fmt_comments_only(buf, final_comments.iter(), NewlineAt::Top, item_indent);
+            newline(buf, indent);
+            buf.push(']');
         } else {
-            buf.push(' ');
+            // is_multiline == false
-            item.format_with_options(buf, Parens::NotNeeded, Newlines::Yes, item_indent);
+            let mut iter = loc_items.iter().peekable();
-            if iter.peek().is_some() {
+            while let Some(item) = iter.next() {
-                buf.push(',');
+                buf.push(' ');
+                item.format_with_options(buf, Parens::NotNeeded, Newlines::Yes, indent);
+                if iter.peek().is_some() {
+                    buf.push(',');
+                }
             }
+            buf.push_str(" ]");
         }
     }
-
-    if is_multiline {
-        newline(buf, indent);
-    }
-
-    if !loc_items.is_empty() && !is_multiline {
-        buf.push(' ');
-    }
-    buf.push(']');
 }
 
 pub fn empty_line_before_expr<'a>(expr: &'a Expr<'a>) -> bool {
@@ -809,7 +808,7 @@ pub fn fmt_record<'a>(
     final_comments: &'a [CommentOrNewline<'a>],
     indent: u16,
 ) {
-    if loc_fields.is_empty() {
+    if loc_fields.is_empty() && final_comments.iter().all(|c| c.is_newline()) {
         buf.push_str("{}");
     } else {
         buf.push('{');
@@ -846,7 +845,7 @@ pub fn fmt_record<'a>(
 
             newline(buf, indent);
         } else {
-            // is_multiline == false */
+            // is_multiline == false
             buf.push(' ');
             let field_indent = indent;
             let mut iter = loc_fields.iter().peekable();

compiler/fmt/tests/test_fmt.rs

@@ -1022,7 +1022,7 @@ mod test_fmt {
             [
                 7,
                 8,
-                9
+                9,
             ]
             "#
         ));
@@ -1041,7 +1041,7 @@ mod test_fmt {
                 [
                     17,
                     18,
-                    19
+                    19,
                 ]
                 "#
             ),
@@ -1064,7 +1064,7 @@ mod test_fmt {
                 [
                     27,
                     28,
-                    29
+                    29,
                 ]
                 "#
             ),
@@ -1084,7 +1084,7 @@ mod test_fmt {
                 [
                     157,
                     158,
-                    159
+                    159,
                 ]
                 "#
             ),
@@ -1105,7 +1105,7 @@ mod test_fmt {
                     557,
                     648,
                     759,
-                    837
+                    837,
                 ]
                 "#
             ),
@@ -1127,7 +1127,7 @@ mod test_fmt {
                     257,
                     358,
                     # Hey!
-                    459
+                    459,
                 ]
                 "#
             ),
@@ -1155,7 +1155,7 @@ mod test_fmt {
                     37,
                     # Thirty Eight
                     38,
-                    39
+                    39,
                 ]
                 "#
             ),
@@ -1189,7 +1189,32 @@ mod test_fmt {
                     48,
                     # Bottom 48
                     # Top 49
-                    49
+                    49,
+                    # Bottom 49
+                    # 49!
+                ]
+                "#
+            ),
+        );
+    }
+    #[test]
+    fn ending_comments_in_list() {
+        expr_formats_to(
+            indoc!(
+                r#"
+                [ # Top 49
+                 49
+                # Bottom 49
+                ,
+                # 49!
+                ]
+                "#
+            ),
+            indoc!(
+                r#"
+                [
+                    # Top 49
+                    49,
                     # Bottom 49
                     # 49!
                 ]
@@ -1197,7 +1222,6 @@ mod test_fmt {
             ),
         );
     }
-
     #[test]
     fn multi_line_list_def() {
         // expr_formats_same(indoc!(
@@ -1228,7 +1252,7 @@ mod test_fmt {
                 results =
                     [
                         Ok 4,
-                        Ok 5
+                        Ok 5,
                     ]
 
                 allOks results
@@ -1271,6 +1295,27 @@ mod test_fmt {
         expr_formats_same("{}");
     }
 
+    #[test]
+    fn empty_record_with_comment() {
+        expr_formats_same(indoc!(
+            r#"
+            {
+                # comment
+            }"#
+        ));
+    }
+
+    #[test]
+    fn empty_record_with_newline() {
+        expr_formats_to(
+            indoc!(
+                r#"
+            {    
+            }"#
+            ),
+            "{}",
+        );
+    }
     #[test]
     fn one_field() {
         expr_formats_same("{ x: 4 }");
@@ -2407,22 +2452,54 @@ mod test_fmt {
         ));
     }
 
-    // TODO This raises a parse error:
+    #[test]
-    // NotYetImplemented("TODO the : in this declaration seems outdented")
+    fn multiline_tag_union_annotation() {
-    // #[test]
+        expr_formats_same(indoc!(
-    // fn multiline_tag_union_annotation() {
+            r#"
-    //     expr_formats_same(indoc!(
+            b :
-    //         r#"
+                [
-    //         b :
+                    True,
-    //             [
+                    False,
-    //                 True,
+                ]
-    //                 False,
-    //             ]
 
-    //         b
+            b
-    //         "#
+            "#
-    //     ));
+        ));
-    // }
+    }
+
+    #[test]
+    fn multiline_tag_union_annotation_with_final_comment() {
+        expr_formats_to(
+            indoc!(
+                r#"
+            b :
+                [
+                    True,
+                    # comment 1
+                    False # comment 2
+                    ,
+                    # comment 3
+                ]
+
+            b
+            "#
+            ),
+            indoc!(
+                r#"
+                b :
+                    [
+                        True,
+                        # comment 1
+                        False,
+                        # comment 2
+                        # comment 3
+                    ]
+    
+                b
+                "#
+            ),
+        );
+    }
 
     #[test]
     fn tag_union() {
@@ -2436,6 +2513,28 @@ mod test_fmt {
         ));
     }
 
+    // TODO: the current formatting seems a bit odd for multiline function annotations
+    // (beside weird indentation, note the trailing space after the "->")
+    // #[test]
+    // fn multiline_tag_union_function_annotation() {
+    //     expr_formats_same(indoc!(
+    //         r#"
+    //         f :
+    //             [
+    //                 True,
+    //                 False,
+    //             ] ->
+    //             [
+    //                 True,
+    //                 False,
+    //             ]
+    //         f = \x -> x
+
+    //         a
+    //         "#
+    //     ));
+    // }
+
     #[test]
     fn recursive_tag_union() {
         expr_formats_same(indoc!(

compiler/gen/src/run_roc.rs

@@ -29,7 +29,7 @@ impl<T: Sized> Into<Result<T, String>> for RocCallResult<T> {
 #[macro_export]
 macro_rules! run_jit_function {
     ($lib: expr, $main_fn_name: expr, $ty:ty, $transform:expr) => {{
-        let v: std::vec::Vec<roc_problem::can::Problem> = std::vec::Vec::new();
+        let v: String = String::new();
         run_jit_function!($lib, $main_fn_name, $ty, $transform, v)
     }};
 
@@ -52,12 +52,7 @@ macro_rules! run_jit_function {
             match result.assume_init().into() {
                 Ok(success) => {
                     // only if there are no exceptions thrown, check for errors
-                    assert_eq!(
+                    assert!($errors.is_empty(), "Encountered errors:\n{}", $errors);
-                        $errors,
-                        std::vec::Vec::new(),
-                        "Encountered errors: {:?}",
-                        $errors
-                    );
 
                     $transform(success)
                 }
@@ -73,7 +68,7 @@ macro_rules! run_jit_function {
 #[macro_export]
 macro_rules! run_jit_function_dynamic_type {
     ($lib: expr, $main_fn_name: expr, $bytes:expr, $transform:expr) => {{
-        let v: std::vec::Vec<roc_problem::can::Problem> = std::vec::Vec::new();
+        let v: String = String::new();
         run_jit_function_dynamic_type!($lib, $main_fn_name, $bytes, $transform, v)
     }};
 

compiler/gen/tests/gen_primitives.rs

@@ -1748,4 +1748,59 @@ mod gen_primitives {
             u8
         );
     }
+
+    #[test]
+    #[should_panic(
+        expected = "Roc failed with message: \"Shadowing { original_region: |L 3-3, C 4-5|, shadow: |L 6-6, C 8-9| Ident(\\\"x\\\") }\""
+    )]
+    fn pattern_shadowing() {
+        assert_evals_to!(
+            indoc!(
+                r#"
+                x = 4
+
+                when 4 is
+                    x -> x
+                "#
+            ),
+            0,
+            i64
+        );
+    }
+
+    #[test]
+    #[should_panic(expected = "TODO non-exhaustive pattern")]
+    fn non_exhaustive_pattern_let() {
+        assert_evals_to!(
+            indoc!(
+                r#"
+                x : Result I64 F64
+                x = Ok 4
+
+                (Ok y) = x
+
+                y
+                "#
+            ),
+            0,
+            i64
+        );
+    }
+
+    #[test]
+    #[ignore]
+    #[should_panic(expected = "")]
+    fn unsupported_pattern_str_interp() {
+        assert_evals_to!(
+            indoc!(
+                r#"
+                { x: 4 } = { x : 4 }
+
+                x
+                "#
+            ),
+            0,
+            i64
+        );
+    }
 }

compiler/gen/tests/gen_records.rs

@@ -561,7 +561,9 @@ mod gen_records {
     }
 
     #[test]
+    #[ignore]
     fn optional_field_function_no_use_default() {
+        // blocked on https://github.com/rtfeldman/roc/issues/786
         assert_evals_to!(
             indoc!(
                 r#"
@@ -579,7 +581,9 @@ mod gen_records {
     }
 
     #[test]
+    #[ignore]
     fn optional_field_function_no_use_default_nested() {
+        // blocked on https://github.com/rtfeldman/roc/issues/786
         assert_evals_to!(
             indoc!(
                 r#"

compiler/gen/tests/helpers/eval.rs

@@ -22,7 +22,7 @@ pub fn helper<'a>(
     stdlib: roc_builtins::std::StdLib,
     leak: bool,
     context: &'a inkwell::context::Context,
-) -> (&'static str, Vec<roc_problem::can::Problem>, Library) {
+) -> (&'static str, String, Library) {
     use roc_gen::llvm::build::{build_proc, build_proc_header, Scope};
     use std::path::{Path, PathBuf};
 
@@ -64,11 +64,14 @@ pub fn helper<'a>(
     debug_assert_eq!(exposed_to_host.len(), 1);
     let main_fn_symbol = exposed_to_host.keys().copied().nth(0).unwrap();
 
-    let (_, main_fn_layout) = procedures
+    let (_, main_fn_layout) = match procedures.keys().find(|(s, _)| *s == main_fn_symbol) {
-        .keys()
+        Some(found) => found.clone(),
-        .find(|(s, _)| *s == main_fn_symbol)
+        None => panic!(
-        .unwrap()
+            "The main function symbol {:?} does not have a procedure in {:?}",
-        .clone();
+            main_fn_symbol,
+            &procedures.keys()
+        ),
+    };
 
     let target = target_lexicon::Triple::host();
     let ptr_bytes = target.pointer_width().unwrap().bytes() as u32;
@@ -106,14 +109,14 @@ pub fn helper<'a>(
                 | UnusedArgument(_, _, _)
                 | UnusedImport(_, _)
                 | RuntimeError(_)
+                | UnsupportedPattern(_, _)
                 | ExposedButNotDefined(_) => {
-                    delayed_errors.push(problem.clone());
-
                     let report = can_problem(&alloc, module_path.clone(), problem);
                     let mut buf = String::new();
 
                     report.render_color_terminal(&mut buf, &alloc, &palette);
 
+                    delayed_errors.push(buf.clone());
                     lines.push(buf);
                 }
                 _ => {
@@ -142,6 +145,7 @@ pub fn helper<'a>(
 
             report.render_color_terminal(&mut buf, &alloc, &palette);
 
+            delayed_errors.push(buf.clone());
             lines.push(buf);
         }
     }
@@ -283,7 +287,7 @@ pub fn helper<'a>(
     let lib = module_to_dylib(&env.module, &target, opt_level)
         .expect("Error loading compiled dylib for test");
 
-    (main_fn_name, delayed_errors, lib)
+    (main_fn_name, delayed_errors.join("\n"), lib)
 }
 
 // TODO this is almost all code duplication with assert_llvm_evals_to

compiler/mono/src/decision_tree.rs

@@ -379,7 +379,7 @@ fn test_at_path<'a>(selected_path: &Path, branch: &Branch<'a>, all_tests: &mut V
 
             match pattern {
                 // TODO use guard!
-                Identifier(_) | Underscore | Shadowed(_, _) | UnsupportedPattern(_) => {
+                Identifier(_) | Underscore => {
                     if let Guard::Guard { symbol, id, stmt } = guard {
                         all_tests.push(Guarded {
                             opt_test: None,
@@ -408,12 +408,9 @@ fn test_at_path<'a>(selected_path: &Path, branch: &Branch<'a>, all_tests: &mut V
                             DestructType::Guard(guard) => {
                                 arguments.push((guard.clone(), destruct.layout.clone()));
                             }
-                            DestructType::Required => {
+                            DestructType::Required(_) => {
                                 arguments.push((Pattern::Underscore, destruct.layout.clone()));
                             }
-                            DestructType::Optional(_expr) => {
-                                // do nothing
-                            }
                         }
                     }
 
@@ -531,7 +528,7 @@ fn to_relevant_branch_help<'a>(
     use Test::*;
 
     match pattern {
-        Identifier(_) | Underscore | Shadowed(_, _) | UnsupportedPattern(_) => Some(branch.clone()),
+        Identifier(_) | Underscore => Some(branch.clone()),
 
         RecordDestructure(destructs, _) => match test {
             IsCtor {
@@ -540,27 +537,22 @@ fn to_relevant_branch_help<'a>(
                 ..
             } => {
                 debug_assert!(test_name == &TagName::Global(RECORD_TAG_NAME.into()));
-                let sub_positions = destructs
+                let sub_positions = destructs.into_iter().enumerate().map(|(index, destruct)| {
-                    .into_iter()
+                    let pattern = match destruct.typ {
-                    .filter(|destruct| !matches!(destruct.typ, DestructType::Optional(_)))
+                        DestructType::Guard(guard) => guard.clone(),
-                    .enumerate()
+                        DestructType::Required(_) => Pattern::Underscore,
-                    .map(|(index, destruct)| {
+                    };
-                        let pattern = match destruct.typ {
-                            DestructType::Guard(guard) => guard.clone(),
-                            DestructType::Required => Pattern::Underscore,
-                            DestructType::Optional(_expr) => unreachable!("because of the filter"),
-                        };
 
-                        (
+                    (
-                            Path::Index {
+                        Path::Index {
-                                index: index as u64,
+                            index: index as u64,
-                                tag_id: *tag_id,
+                            tag_id: *tag_id,
-                                path: Box::new(path.clone()),
+                            path: Box::new(path.clone()),
-                            },
+                        },
-                            Guard::NoGuard,
+                        Guard::NoGuard,
-                            pattern,
+                        pattern,
-                        )
+                    )
-                    });
+                });
                 start.extend(sub_positions);
                 start.extend(end);
 
@@ -742,7 +734,7 @@ fn needs_tests<'a>(pattern: &Pattern<'a>) -> bool {
     use Pattern::*;
 
     match pattern {
-        Identifier(_) | Underscore | Shadowed(_, _) | UnsupportedPattern(_) => false,
+        Identifier(_) | Underscore => false,
 
         RecordDestructure(_, _)
         | AppliedTag { .. }

compiler/mono/src/exhaustive.rs

@@ -67,7 +67,7 @@ fn simplify<'a>(pattern: &crate::ir::Pattern<'a>) -> Pattern {
                 field_names.push(destruct.label.clone());
 
                 match &destruct.typ {
-                    DestructType::Required | DestructType::Optional(_) => patterns.push(Anything),
+                    DestructType::Required(_) => patterns.push(Anything),
                     DestructType::Guard(guard) => patterns.push(simplify(guard)),
                 }
             }
@@ -84,17 +84,6 @@ fn simplify<'a>(pattern: &crate::ir::Pattern<'a>) -> Pattern {
             Ctor(union, tag_id, patterns)
         }
 
-        Shadowed(_region, _ident) => {
-            // Treat as an Anything
-            // code-gen will make a runtime error out of the branch
-            Anything
-        }
-        UnsupportedPattern(_region) => {
-            // Treat as an Anything
-            // code-gen will make a runtime error out of the branch
-            Anything
-        }
-
         AppliedTag {
             tag_id,
             arguments,

compiler/mono/src/ir.rs

@@ -4,7 +4,7 @@ use crate::layout::{Builtin, ClosureLayout, Layout, LayoutCache, LayoutProblem};
 use bumpalo::collections::Vec;
 use bumpalo::Bump;
 use roc_collections::all::{default_hasher, MutMap, MutSet};
-use roc_module::ident::{ForeignSymbol, Ident, Lowercase, TagName};
+use roc_module::ident::{ForeignSymbol, Lowercase, TagName};
 use roc_module::low_level::LowLevel;
 use roc_module::symbol::{IdentIds, ModuleId, Symbol};
 use roc_problem::can::RuntimeError;
@@ -1264,7 +1264,50 @@ fn patterns_to_when<'a>(
     // this must be fixed when moving exhaustiveness checking to the new canonical AST
     for (pattern_var, pattern) in patterns.into_iter() {
         let context = crate::exhaustive::Context::BadArg;
-        let mono_pattern = from_can_pattern(env, layout_cache, &pattern.value);
+        let mono_pattern = match from_can_pattern(env, layout_cache, &pattern.value) {
+            Ok((pat, assignments)) => {
+                for (symbol, variable, expr) in assignments.into_iter().rev() {
+                    if let Ok(old_body) = body {
+                        let def = roc_can::def::Def {
+                            annotation: None,
+                            expr_var: variable,
+                            loc_expr: Located::at(pattern.region, expr),
+                            loc_pattern: Located::at(
+                                pattern.region,
+                                roc_can::pattern::Pattern::Identifier(symbol),
+                            ),
+                            pattern_vars: std::iter::once((symbol, variable)).collect(),
+                        };
+                        let new_expr = roc_can::expr::Expr::LetNonRec(
+                            Box::new(def),
+                            Box::new(old_body),
+                            variable,
+                        );
+                        let new_body = Located {
+                            region: pattern.region,
+                            value: new_expr,
+                        };
+
+                        body = Ok(new_body);
+                    }
+                }
+
+                pat
+            }
+            Err(runtime_error) => {
+                // Even if the body was Ok, replace it with this Err.
+                // If it was already an Err, leave it at that Err, so the first
+                // RuntimeError we encountered remains the first.
+                body = body.and({
+                    Err(Located {
+                        region: pattern.region,
+                        value: runtime_error,
+                    })
+                });
+
+                continue;
+            }
+        };
 
         match crate::exhaustive::check(
             pattern.region,
@@ -2392,7 +2435,14 @@ pub fn with_hole<'a>(
                 }
             } else {
                 // this may be a destructure pattern
-                let mono_pattern = from_can_pattern(env, layout_cache, &def.loc_pattern.value);
+                let (mono_pattern, assignments) =
+                    match from_can_pattern(env, layout_cache, &def.loc_pattern.value) {
+                        Ok(v) => v,
+                        Err(_runtime_error) => {
+                            // todo
+                            panic!();
+                        }
+                    };
 
                 let context = crate::exhaustive::Context::BadDestruct;
                 match crate::exhaustive::check(
@@ -2412,6 +2462,14 @@ pub fn with_hole<'a>(
                       // return Stmt::RuntimeError("TODO non-exhaustive pattern");
                 }
 
+                let mut hole = hole;
+
+                for (symbol, variable, expr) in assignments {
+                    let stmt = with_hole(env, expr, variable, procs, layout_cache, symbol, hole);
+
+                    hole = env.arena.alloc(stmt);
+                }
+
                 // convert the continuation
                 let mut stmt = with_hole(
                     env,
@@ -3923,13 +3981,23 @@ pub fn from_can<'a>(
             }
 
             // this may be a destructure pattern
-            let mono_pattern = from_can_pattern(env, layout_cache, &def.loc_pattern.value);
+            let (mono_pattern, assignments) =
+                match from_can_pattern(env, layout_cache, &def.loc_pattern.value) {
+                    Ok(v) => v,
+                    Err(_) => todo!(),
+                };
 
             if let Pattern::Identifier(symbol) = mono_pattern {
-                let hole =
+                let mut hole =
                     env.arena
                         .alloc(from_can(env, variable, cont.value, procs, layout_cache));
 
+                for (symbol, variable, expr) in assignments {
+                    let stmt = with_hole(env, expr, variable, procs, layout_cache, symbol, hole);
+
+                    hole = env.arena.alloc(stmt);
+                }
+
                 with_hole(
                     env,
                     def.loc_expr.value,
@@ -3954,13 +4022,20 @@ pub fn from_can<'a>(
                         for error in errors {
                             env.problems.push(MonoProblem::PatternProblem(error))
                         }
-                    } // TODO make all variables bound in the pattern evaluate to a runtime error
+
-                      // return Stmt::RuntimeError("TODO non-exhaustive pattern");
+                        return Stmt::RuntimeError("TODO non-exhaustive pattern");
+                    }
                 }
 
                 // convert the continuation
                 let mut stmt = from_can(env, variable, cont.value, procs, layout_cache);
 
+                // layer on any default record fields
+                for (symbol, variable, expr) in assignments {
+                    let hole = env.arena.alloc(stmt);
+                    stmt = with_hole(env, expr, variable, procs, layout_cache, symbol, hole);
+                }
+
                 if let roc_can::expr::Expr::Var(outer_symbol) = def.loc_expr.value {
                     store_pattern(env, procs, layout_cache, &mono_pattern, outer_symbol, stmt)
                         .unwrap()
@@ -4015,19 +4090,51 @@ fn to_opt_branches<'a>(
         };
 
         for loc_pattern in when_branch.patterns {
-            let mono_pattern = from_can_pattern(env, layout_cache, &loc_pattern.value);
+            match from_can_pattern(env, layout_cache, &loc_pattern.value) {
+                Ok((mono_pattern, assignments)) => {
+                    loc_branches.push((
+                        Located::at(loc_pattern.region, mono_pattern.clone()),
+                        exhaustive_guard.clone(),
+                    ));
 
-            loc_branches.push((
+                    let mut loc_expr = when_branch.value.clone();
-                Located::at(loc_pattern.region, mono_pattern.clone()),
+                    let region = loc_pattern.region;
-                exhaustive_guard.clone(),
+                    for (symbol, variable, expr) in assignments.into_iter().rev() {
-            ));
+                        let def = roc_can::def::Def {
+                            annotation: None,
+                            expr_var: variable,
+                            loc_expr: Located::at(region, expr),
+                            loc_pattern: Located::at(
+                                region,
+                                roc_can::pattern::Pattern::Identifier(symbol),
+                            ),
+                            pattern_vars: std::iter::once((symbol, variable)).collect(),
+                        };
+                        let new_expr = roc_can::expr::Expr::LetNonRec(
+                            Box::new(def),
+                            Box::new(loc_expr),
+                            variable,
+                        );
+                        loc_expr = Located::at(region, new_expr);
+                    }
 
-            // TODO remove clone?
+                    // TODO remove clone?
-            opt_branches.push((
+                    opt_branches.push((mono_pattern, when_branch.guard.clone(), loc_expr.value));
-                mono_pattern,
+                }
-                when_branch.guard.clone(),
+                Err(runtime_error) => {
-                when_branch.value.value.clone(),
+                    loc_branches.push((
-            ));
+                        Located::at(loc_pattern.region, Pattern::Underscore),
+                        exhaustive_guard.clone(),
+                    ));
+
+                    // TODO remove clone?
+                    opt_branches.push((
+                        Pattern::Underscore,
+                        when_branch.guard.clone(),
+                        roc_can::expr::Expr::RuntimeError(runtime_error),
+                    ));
+                }
+            }
         }
     }
 
@@ -4658,14 +4765,6 @@ fn store_pattern<'a>(
         RecordDestructure(_, _) => {
             unreachable!("a record destructure must always occur on a struct layout");
         }
-
-        Shadowed(_region, _ident) => {
-            return Err(&"shadowed");
-        }
-
-        UnsupportedPattern(_region) => {
-            return Err(&"unsupported pattern");
-        }
     }
 
     Ok(stmt)
@@ -4695,25 +4794,14 @@ fn store_record_destruct<'a>(
     };
 
     match &destruct.typ {
-        DestructType::Required => {
+        DestructType::Required(symbol) => {
             stmt = Stmt::Let(
-                destruct.symbol,
+                *symbol,
                 load,
                 destruct.layout.clone(),
                 env.arena.alloc(stmt),
             );
         }
-        DestructType::Optional(expr) => {
-            stmt = with_hole(
-                env,
-                expr.clone(),
-                destruct.variable,
-                procs,
-                layout_cache,
-                destruct.symbol,
-                env.arena.alloc(stmt),
-            );
-        }
         DestructType::Guard(guard_pattern) => match &guard_pattern {
             Identifier(symbol) => {
                 stmt = Stmt::Let(
@@ -5429,11 +5517,6 @@ pub enum Pattern<'a> {
         layout: Layout<'a>,
         union: crate::exhaustive::Union,
     },
-
-    // Runtime Exceptions
-    Shadowed(Region, Located<Ident>),
-    // Example: (5 = 1 + 2) is an unsupported pattern in an assignment; Int patterns aren't allowed in assignments!
-    UnsupportedPattern(Region),
 }
 
 #[derive(Clone, Debug, PartialEq)]
@@ -5441,14 +5524,12 @@ pub struct RecordDestruct<'a> {
     pub label: Lowercase,
     pub variable: Variable,
     pub layout: Layout<'a>,
-    pub symbol: Symbol,
     pub typ: DestructType<'a>,
 }
 
 #[derive(Clone, Debug, PartialEq)]
 pub enum DestructType<'a> {
-    Required,
+    Required(Symbol),
-    Optional(roc_can::expr::Expr),
     Guard(Pattern<'a>),
 }
 
@@ -5459,27 +5540,51 @@ pub struct WhenBranch<'a> {
     pub guard: Option<Stmt<'a>>,
 }
 
-pub fn from_can_pattern<'a>(
+#[allow(clippy::type_complexity)]
+fn from_can_pattern<'a>(
     env: &mut Env<'a, '_>,
     layout_cache: &mut LayoutCache<'a>,
     can_pattern: &roc_can::pattern::Pattern,
-) -> Pattern<'a> {
+) -> Result<
+    (
+        Pattern<'a>,
+        Vec<'a, (Symbol, Variable, roc_can::expr::Expr)>,
+    ),
+    RuntimeError,
+> {
+    let mut assignments = Vec::new_in(env.arena);
+    let pattern = from_can_pattern_help(env, layout_cache, can_pattern, &mut assignments)?;
+
+    Ok((pattern, assignments))
+}
+
+fn from_can_pattern_help<'a>(
+    env: &mut Env<'a, '_>,
+    layout_cache: &mut LayoutCache<'a>,
+    can_pattern: &roc_can::pattern::Pattern,
+    assignments: &mut Vec<'a, (Symbol, Variable, roc_can::expr::Expr)>,
+) -> Result<Pattern<'a>, RuntimeError> {
     use roc_can::pattern::Pattern::*;
+
     match can_pattern {
-        Underscore => Pattern::Underscore,
+        Underscore => Ok(Pattern::Underscore),
-        Identifier(symbol) => Pattern::Identifier(*symbol),
+        Identifier(symbol) => Ok(Pattern::Identifier(*symbol)),
-        IntLiteral(v) => Pattern::IntLiteral(*v),
+        IntLiteral(v) => Ok(Pattern::IntLiteral(*v)),
-        FloatLiteral(v) => Pattern::FloatLiteral(f64::to_bits(*v)),
+        FloatLiteral(v) => Ok(Pattern::FloatLiteral(f64::to_bits(*v))),
-        StrLiteral(v) => Pattern::StrLiteral(v.clone()),
+        StrLiteral(v) => Ok(Pattern::StrLiteral(v.clone())),
-        Shadowed(region, ident) => Pattern::Shadowed(*region, ident.clone()),
+        Shadowed(region, ident) => Err(RuntimeError::Shadowing {
-        UnsupportedPattern(region) => Pattern::UnsupportedPattern(*region),
+            original_region: *region,
+            shadow: ident.clone(),
+        }),
+        UnsupportedPattern(region) => Err(RuntimeError::UnsupportedPattern(*region)),
+
         MalformedPattern(_problem, region) => {
             // TODO preserve malformed problem information here?
-            Pattern::UnsupportedPattern(*region)
+            Err(RuntimeError::UnsupportedPattern(*region))
         }
         NumLiteral(var, num) => match num_argument_to_int_or_float(env.subs, *var) {
-            IntOrFloat::IntType => Pattern::IntLiteral(*num),
+            IntOrFloat::IntType => Ok(Pattern::IntLiteral(*num)),
-            IntOrFloat::FloatType => Pattern::FloatLiteral(*num as u64),
+            IntOrFloat::FloatType => Ok(Pattern::FloatLiteral(*num as u64)),
         },
 
         AppliedTag {
@@ -5493,7 +5598,7 @@ pub fn from_can_pattern<'a>(
 
             let variant = crate::layout::union_sorted_tags(env.arena, *whole_var, env.subs);
 
-            match variant {
+            let result = match variant {
                 Never => unreachable!(
                     "there is no pattern of type `[]`, union var {:?}",
                     *whole_var
@@ -5582,7 +5687,7 @@ pub fn from_can_pattern<'a>(
                     let mut mono_args = Vec::with_capacity_in(arguments.len(), env.arena);
                     for ((_, loc_pat), layout) in arguments.iter().zip(field_layouts.iter()) {
                         mono_args.push((
-                            from_can_pattern(env, layout_cache, &loc_pat.value),
+                            from_can_pattern_help(env, layout_cache, &loc_pat.value, assignments)?,
                             layout.clone(),
                         ));
                     }
@@ -5642,7 +5747,7 @@ pub fn from_can_pattern<'a>(
                     let it = argument_layouts[1..].iter();
                     for ((_, loc_pat), layout) in arguments.iter().zip(it) {
                         mono_args.push((
-                            from_can_pattern(env, layout_cache, &loc_pat.value),
+                            from_can_pattern_help(env, layout_cache, &loc_pat.value, assignments)?,
                             layout.clone(),
                         ));
                     }
@@ -5664,7 +5769,9 @@ pub fn from_can_pattern<'a>(
                         layout,
                     }
                 }
-            }
+            };
+
+            Ok(result)
         }
 
         RecordDestructure {
@@ -5704,14 +5811,14 @@ pub fn from_can_pattern<'a>(
                                     layout_cache,
                                     &destruct.value,
                                     field_layout.clone(),
-                                ));
+                                    assignments,
+                                )?);
                             }
                             None => {
                                 // this field is not destructured by the pattern
                                 // put in an underscore
                                 mono_destructs.push(RecordDestruct {
                                     label: label.clone(),
-                                    symbol: env.unique_symbol(),
                                     variable,
                                     layout: field_layout.clone(),
                                     typ: DestructType::Guard(Pattern::Underscore),
@@ -5727,29 +5834,26 @@ pub fn from_can_pattern<'a>(
                         match destructs_by_label.remove(&label) {
                             Some(destruct) => {
                                 // this field is destructured by the pattern
-                                mono_destructs.push(RecordDestruct {
+                                match &destruct.value.typ {
-                                    label: destruct.value.label.clone(),
+                                    roc_can::pattern::DestructType::Optional(_, loc_expr) => {
-                                    symbol: destruct.value.symbol,
+                                        // if we reach this stage, the optional field is not present
-                                    layout: field_layout,
+                                        // so we push the default assignment into the branch
-                                    variable,
+                                        assignments.push((
-                                    typ: match &destruct.value.typ {
+                                            destruct.value.symbol,
-                                        roc_can::pattern::DestructType::Optional(_, loc_expr) => {
+                                            variable,
-                                            // if we reach this stage, the optional field is not present
+                                            loc_expr.value.clone(),
-                                            // so use the default
+                                        ));
-                                            DestructType::Optional(loc_expr.value.clone())
+                                    }
-                                        }
+                                    _ => unreachable!(
-                                        _ => unreachable!(
+                                        "only optional destructs can be optional fields"
-                                            "only optional destructs can be optional fields"
+                                    ),
-                                        ),
+                                };
-                                    },
-                                });
                             }
                             None => {
                                 // this field is not destructured by the pattern
                                 // put in an underscore
                                 mono_destructs.push(RecordDestruct {
                                     label: label.clone(),
-                                    symbol: env.unique_symbol(),
                                     variable,
                                     layout: field_layout.clone(),
                                     typ: DestructType::Guard(Pattern::Underscore),
@@ -5765,28 +5869,30 @@ pub fn from_can_pattern<'a>(
                 // it must be an optional field, and we will use the default
                 match &destruct.value.typ {
                     roc_can::pattern::DestructType::Optional(field_var, loc_expr) => {
-                        let field_layout = layout_cache
+                        // TODO these don't match up in the uniqueness inference; when we remove
-                            .from_var(env.arena, *field_var, env.subs)
+                        // that, reinstate this assert!
-                            .unwrap_or_else(|err| {
+                        //
-                                panic!("TODO turn fn_var into a RuntimeError {:?}", err)
+                        // dbg!(&env.subs.get_without_compacting(*field_var).content);
-                            });
+                        // dbg!(&env.subs.get_without_compacting(destruct.value.var).content);
-
+                        // debug_assert_eq!(
-                        mono_destructs.push(RecordDestruct {
+                        //     env.subs.get_root_key_without_compacting(*field_var),
-                            label: destruct.value.label.clone(),
+                        //     env.subs.get_root_key_without_compacting(destruct.value.var)
-                            symbol: destruct.value.symbol,
+                        // );
-                            variable: destruct.value.var,
+                        assignments.push((
-                            layout: field_layout,
+                            destruct.value.symbol,
-                            typ: DestructType::Optional(loc_expr.value.clone()),
+                            // destruct.value.var,
-                        })
+                            *field_var,
+                            loc_expr.value.clone(),
+                        ));
                     }
                     _ => unreachable!("only optional destructs can be optional fields"),
                 }
             }
 
-            Pattern::RecordDestructure(
+            Ok(Pattern::RecordDestructure(
                 mono_destructs,
                 Layout::Struct(field_layouts.into_bump_slice()),
-            )
+            ))
         }
     }
 }
@@ -5796,24 +5902,23 @@ fn from_can_record_destruct<'a>(
     layout_cache: &mut LayoutCache<'a>,
     can_rd: &roc_can::pattern::RecordDestruct,
     field_layout: Layout<'a>,
-) -> RecordDestruct<'a> {
+    assignments: &mut Vec<'a, (Symbol, Variable, roc_can::expr::Expr)>,
-    RecordDestruct {
+) -> Result<RecordDestruct<'a>, RuntimeError> {
+    Ok(RecordDestruct {
         label: can_rd.label.clone(),
-        symbol: can_rd.symbol,
         variable: can_rd.var,
         layout: field_layout,
         typ: match &can_rd.typ {
-            roc_can::pattern::DestructType::Required => DestructType::Required,
+            roc_can::pattern::DestructType::Required => DestructType::Required(can_rd.symbol),
             roc_can::pattern::DestructType::Optional(_, _) => {
                 // if we reach this stage, the optional field is present
-                // DestructType::Optional(loc_expr.value.clone())
+                DestructType::Required(can_rd.symbol)
-                DestructType::Required
-            }
-            roc_can::pattern::DestructType::Guard(_, loc_pattern) => {
-                DestructType::Guard(from_can_pattern(env, layout_cache, &loc_pattern.value))
             }
+            roc_can::pattern::DestructType::Guard(_, loc_pattern) => DestructType::Guard(
+                from_can_pattern_help(env, layout_cache, &loc_pattern.value, assignments)?,
+            ),
         },
-    }
+    })
 }
 
 /// Potentially translate LowLevel operations into more efficient ones based on

compiler/parse/src/ast.rs

@@ -93,7 +93,10 @@ pub enum Expr<'a> {
     AccessorFunction(&'a str),
 
     // Collection Literals
-    List(&'a [&'a Loc<Expr<'a>>]),
+    List {
+        items: &'a [&'a Loc<Expr<'a>>],
+        final_comments: &'a [CommentOrNewline<'a>],
+    },
 
     Record {
         update: Option<&'a Loc<Expr<'a>>>,
@@ -300,6 +303,15 @@ impl<'a> CommentOrNewline<'a> {
             DocComment(_) => true,
         }
     }
+
+    pub fn is_newline(&self) -> bool {
+        use CommentOrNewline::*;
+        match self {
+            Newline => true,
+            LineComment(_) => false,
+            DocComment(_) => false,
+        }
+    }
 }
 
 #[derive(Clone, Debug, PartialEq)]

compiler/parse/src/expr.rs

@@ -306,7 +306,7 @@ fn expr_to_pattern<'a>(arena: &'a Bump, expr: &Expr<'a>) -> Result<Pattern<'a>,
         // These would not have parsed as patterns
         Expr::AccessorFunction(_)
         | Expr::Access(_, _)
-        | Expr::List(_)
+        | Expr::List { .. }
         | Expr::Closure(_, _)
         | Expr::BinOp(_)
         | Expr::Defs(_, _)
@@ -1848,7 +1848,7 @@ fn binop<'a>() -> impl Parser<'a, BinOp> {
 }
 
 pub fn list_literal<'a>(min_indent: u16) -> impl Parser<'a, Expr<'a>> {
-    let elems = collection!(
+    let elems = collection_trailing_sep!(
         ascii_char(b'['),
         loc!(expr(min_indent)),
         ascii_char(b','),
@@ -1858,14 +1858,21 @@ pub fn list_literal<'a>(min_indent: u16) -> impl Parser<'a, Expr<'a>> {
 
     parser::attempt(
         Attempting::List,
-        map_with_arena!(elems, |arena, parsed_elems: Vec<'a, Located<Expr<'a>>>| {
+        map_with_arena!(elems, |arena,
+                                (parsed_elems, final_comments): (
+            Vec<'a, Located<Expr<'a>>>,
+            &'a [CommentOrNewline<'a>]
+        )| {
             let mut allocated = Vec::with_capacity_in(parsed_elems.len(), arena);
 
             for parsed_elem in parsed_elems {
                 allocated.push(&*arena.alloc(parsed_elem));
             }
 
-            Expr::List(allocated.into_bump_slice())
+            Expr::List {
+                items: allocated.into_bump_slice(),
+                final_comments,
+            }
         }),
     )
 }

compiler/parse/tests/test_parse.rs

@@ -1017,8 +1017,10 @@ mod test_parse {
     #[test]
     fn empty_list() {
         let arena = Bump::new();
-        let elems = &[];
+        let expected = List {
-        let expected = List(elems);
+            items: &[],
+            final_comments: &[],
+        };
         let actual = parse_expr_with(&arena, "[]");
 
         assert_eq!(Ok(expected), actual);
@@ -1028,8 +1030,10 @@ mod test_parse {
     fn spaces_inside_empty_list() {
         // This is a regression test!
         let arena = Bump::new();
-        let elems = &[];
+        let expected = List {
-        let expected = List(elems);
+            items: &[],
+            final_comments: &[],
+        };
         let actual = parse_expr_with(&arena, "[  ]");
 
         assert_eq!(Ok(expected), actual);
@@ -1038,8 +1042,11 @@ mod test_parse {
     #[test]
     fn packed_singleton_list() {
         let arena = Bump::new();
-        let elems = &[&*arena.alloc(Located::new(0, 0, 1, 2, Num("1")))];
+        let items = &[&*arena.alloc(Located::new(0, 0, 1, 2, Num("1")))];
-        let expected = List(elems);
+        let expected = List {
+            items,
+            final_comments: &[],
+        };
         let actual = parse_expr_with(&arena, "[1]");
 
         assert_eq!(Ok(expected), actual);
@@ -1048,8 +1055,11 @@ mod test_parse {
     #[test]
     fn spaced_singleton_list() {
         let arena = Bump::new();
-        let elems = &[&*arena.alloc(Located::new(0, 0, 2, 3, Num("1")))];
+        let items = &[&*arena.alloc(Located::new(0, 0, 2, 3, Num("1")))];
-        let expected = List(elems);
+        let expected = List {
+            items,
+            final_comments: &[],
+        };
         let actual = parse_expr_with(&arena, "[ 1 ]");
 
         assert_eq!(Ok(expected), actual);

editor/src/def.rs

@@ -43,6 +43,28 @@ pub enum Def {
     Function(FunctionDef),
 }
 
+impl Def {
+    pub fn symbols(&self, pool: &Pool) -> MutSet<Symbol> {
+        let mut output = MutSet::default();
+
+        match self {
+            Def::AnnotationOnly { .. } => todo!("lost pattern information here ... "),
+            Def::Value(ValueDef { pattern, .. }) => {
+                let pattern2 = &pool[*pattern];
+                output.extend(symbols_from_pattern(pool, pattern2));
+            }
+            Def::Function(function_def) => match function_def {
+                FunctionDef::NoAnnotation { name, .. }
+                | FunctionDef::WithAnnotation { name, .. } => {
+                    output.insert(*name);
+                }
+            },
+        }
+
+        output
+    }
+}
+
 impl ShallowClone for Def {
     fn shallow_clone(&self) -> Self {
         match self {
@@ -787,7 +809,6 @@ pub struct CanDefs {
 pub fn canonicalize_defs<'a>(
     env: &mut Env<'a>,
     mut output: Output,
-    var_store: &mut VarStore,
     original_scope: &Scope,
     loc_defs: &'a [&'a Located<ast::Def<'a>>],
     pattern_type: PatternType,

editor/src/expr.rs

@@ -43,7 +43,7 @@ impl Output {
 
 pub struct Env<'a> {
     pub home: ModuleId,
-    pub var_store: VarStore,
+    pub var_store: &'a mut VarStore,
     pub pool: &'a mut Pool,
     pub arena: &'a Bump,
 
@@ -63,6 +63,32 @@ pub struct Env<'a> {
 }
 
 impl<'a> Env<'a> {
+    pub fn new(
+        home: ModuleId,
+        arena: &'a Bump,
+        pool: &'a mut Pool,
+        var_store: &'a mut VarStore,
+        dep_idents: MutMap<ModuleId, IdentIds>,
+        module_ids: &'a ModuleIds,
+        exposed_ident_ids: IdentIds,
+    ) -> Env<'a> {
+        Env {
+            home,
+            arena,
+            pool,
+            var_store,
+            dep_idents,
+            module_ids,
+            ident_ids: exposed_ident_ids.clone(), // we start with these, but will add more later
+            exposed_ident_ids,
+            closures: MutMap::default(),
+            qualified_lookups: MutSet::default(),
+            tailcallable_symbol: None,
+            closure_name_symbol: None,
+            top_level_symbols: MutSet::default(),
+        }
+    }
+
     pub fn add<T>(&mut self, item: T, region: Region) -> NodeId<T> {
         let id = self.pool.add(item);
         self.set_region(id, region);
@@ -285,14 +311,14 @@ pub fn to_expr2<'a>(
 
         Str(literal) => flatten_str_literal(env, scope, &literal),
 
-        List(elements) => {
+        List { items, .. } => {
             let mut output = Output::default();
             let output_ref = &mut output;
 
-            let elems = PoolVec::with_capacity(elements.len() as u32, env.pool);
+            let elems = PoolVec::with_capacity(items.len() as u32, env.pool);
 
-            for (node_id, element) in elems.iter_node_ids().zip(elements.iter()) {
+            for (node_id, item) in elems.iter_node_ids().zip(items.iter()) {
-                let (expr, sub_output) = to_expr2(env, scope, &element.value, element.region);
+                let (expr, sub_output) = to_expr2(env, scope, &item.value, item.region);
 
                 output_ref.union(sub_output);
 

editor/src/lib.rs

@@ -34,6 +34,7 @@ pub mod error;
 pub mod expr;
 pub mod file;
 mod keyboard_input;
+mod module;
 mod ortho;
 mod pattern;
 pub mod pool;

editor/src/module.rs

@@ -0,0 +1,317 @@
+#![allow(clippy::all)]
+#![allow(dead_code)]
+#![allow(unused_imports)]
+#![allow(unused_variables)]
+use crate::ast::{FunctionDef, ValueDef};
+use crate::def::{canonicalize_defs, sort_can_defs, Declaration, Def};
+use crate::expr::Env;
+use crate::expr::Output;
+use crate::pattern::Pattern2;
+use crate::pool::{NodeId, Pool, PoolStr, PoolVec, ShallowClone};
+use crate::scope::Scope;
+use crate::types::Alias;
+use bumpalo::Bump;
+use roc_can::operator::desugar_def;
+use roc_collections::all::{default_hasher, ImMap, ImSet, MutMap, MutSet, SendMap};
+use roc_module::ident::Ident;
+use roc_module::ident::Lowercase;
+use roc_module::symbol::{IdentIds, ModuleId, ModuleIds, Symbol};
+use roc_parse::ast;
+use roc_parse::pattern::PatternType;
+use roc_problem::can::{Problem, RuntimeError};
+use roc_region::all::{Located, Region};
+use roc_types::subs::{VarStore, Variable};
+
+pub struct ModuleOutput {
+    pub aliases: MutMap<Symbol, NodeId<Alias>>,
+    pub rigid_variables: MutMap<Variable, Lowercase>,
+    pub declarations: Vec<Declaration>,
+    pub exposed_imports: MutMap<Symbol, Variable>,
+    pub lookups: Vec<(Symbol, Variable, Region)>,
+    pub problems: Vec<Problem>,
+    pub ident_ids: IdentIds,
+    pub references: MutSet<Symbol>,
+}
+
+// TODO trim these down
+#[allow(clippy::too_many_arguments)]
+pub fn canonicalize_module_defs<'a>(
+    arena: &Bump,
+    loc_defs: &'a [Located<ast::Def<'a>>],
+    home: ModuleId,
+    module_ids: &ModuleIds,
+    exposed_ident_ids: IdentIds,
+    dep_idents: MutMap<ModuleId, IdentIds>,
+    aliases: MutMap<Symbol, Alias>,
+    exposed_imports: MutMap<Ident, (Symbol, Region)>,
+    mut exposed_symbols: MutSet<Symbol>,
+    var_store: &mut VarStore,
+) -> Result<ModuleOutput, RuntimeError> {
+    let mut pool = Pool::with_capacity(1 << 10);
+    let mut can_exposed_imports = MutMap::default();
+    let mut scope = Scope::new(home, &mut pool, var_store);
+    let num_deps = dep_idents.len();
+
+    for (name, alias) in aliases.into_iter() {
+        let vars = PoolVec::with_capacity(alias.targs.len() as u32, &mut pool);
+
+        for (node_id, targ_id) in vars.iter_node_ids().zip(alias.targs.iter_node_ids()) {
+            let (poolstr, var) = &pool[targ_id];
+            pool[node_id] = (poolstr.shallow_clone(), *var);
+        }
+        scope.add_alias(&mut pool, name, vars, alias.actual);
+    }
+
+    // Desugar operators (convert them to Apply calls, taking into account
+    // operator precedence and associativity rules), before doing other canonicalization.
+    //
+    // If we did this *during* canonicalization, then each time we
+    // visited a BinOp node we'd recursively try to apply this to each of its nested
+    // operators, and then again on *their* nested operators, ultimately applying the
+    // rules multiple times unnecessarily.
+    let mut desugared =
+        bumpalo::collections::Vec::with_capacity_in(loc_defs.len() + num_deps, arena);
+
+    for loc_def in loc_defs.iter() {
+        desugared.push(&*arena.alloc(Located {
+            value: desugar_def(arena, &loc_def.value),
+            region: loc_def.region,
+        }));
+    }
+
+    let mut env = Env::new(
+        home,
+        arena,
+        &mut pool,
+        var_store,
+        dep_idents,
+        module_ids,
+        exposed_ident_ids,
+    );
+    let mut lookups = Vec::with_capacity(num_deps);
+    let rigid_variables = MutMap::default();
+
+    // Exposed values are treated like defs that appear before any others, e.g.
+    //
+    // imports [ Foo.{ bar, baz } ]
+    //
+    // ...is basically the same as if we'd added these extra defs at the start of the module:
+    //
+    // bar = Foo.bar
+    // baz = Foo.baz
+    //
+    // Here we essentially add those "defs" to "the beginning of the module"
+    // by canonicalizing them right before we canonicalize the actual ast::Def nodes.
+    for (ident, (symbol, region)) in exposed_imports {
+        let first_char = ident.as_inline_str().chars().next().unwrap();
+
+        if first_char.is_lowercase() {
+            // this is a value definition
+            let expr_var = env.var_store.fresh();
+
+            match scope.import(ident, symbol, region) {
+                Ok(()) => {
+                    // Add an entry to exposed_imports using the current module's name
+                    // as the key; e.g. if this is the Foo module and we have
+                    // exposes [ Bar.{ baz } ] then insert Foo.baz as the key, so when
+                    // anything references `baz` in this Foo module, it will resolve to Bar.baz.
+                    can_exposed_imports.insert(symbol, expr_var);
+
+                    // This will be used during constraint generation,
+                    // to add the usual Lookup constraint as if this were a normal def.
+                    lookups.push((symbol, expr_var, region));
+                }
+                Err((_shadowed_symbol, _region)) => {
+                    panic!("TODO gracefully handle shadowing in imports.")
+                }
+            }
+        } else {
+            // This is a type alias
+
+            // the should already be added to the scope when this module is canonicalized
+            debug_assert!(scope.contains_alias(symbol));
+        }
+    }
+
+    let (defs, _scope, output, symbols_introduced) = canonicalize_defs(
+        &mut env,
+        Output::default(),
+        &scope,
+        &desugared,
+        PatternType::TopLevelDef,
+    );
+
+    // See if any of the new idents we defined went unused.
+    // If any were unused and also not exposed, report it.
+    for (symbol, region) in symbols_introduced {
+        if !output.references.has_lookup(symbol) && !exposed_symbols.contains(&symbol) {
+            env.problem(Problem::UnusedDef(symbol, region));
+        }
+    }
+
+    // TODO register rigids
+    //    for (var, lowercase) in output.introduced_variables.name_by_var.clone() {
+    //        rigid_variables.insert(var, lowercase);
+    //    }
+
+    let mut references = MutSet::default();
+
+    // Gather up all the symbols that were referenced across all the defs' lookups.
+    for symbol in output.references.lookups.iter() {
+        references.insert(*symbol);
+    }
+
+    // Gather up all the symbols that were referenced across all the defs' calls.
+    for symbol in output.references.calls.iter() {
+        references.insert(*symbol);
+    }
+
+    // Gather up all the symbols that were referenced from other modules.
+    for symbol in env.qualified_lookups.iter() {
+        references.insert(*symbol);
+    }
+
+    // NOTE previously we inserted builtin defs into the list of defs here
+    // this is now done later, in file.rs.
+
+    match sort_can_defs(&mut env, defs, Output::default()) {
+        (Ok(mut declarations), output) => {
+            use crate::def::Declaration::*;
+
+            for decl in declarations.iter() {
+                match decl {
+                    Declare(def) => {
+                        for symbol in def.symbols(env.pool) {
+                            if exposed_symbols.contains(&symbol) {
+                                // Remove this from exposed_symbols,
+                                // so that at the end of the process,
+                                // we can see if there were any
+                                // exposed symbols which did not have
+                                // corresponding defs.
+                                exposed_symbols.remove(&symbol);
+                            }
+                        }
+                    }
+                    DeclareRec(defs) => {
+                        for def in defs {
+                            for symbol in def.symbols(env.pool) {
+                                if exposed_symbols.contains(&symbol) {
+                                    // Remove this from exposed_symbols,
+                                    // so that at the end of the process,
+                                    // we can see if there were any
+                                    // exposed symbols which did not have
+                                    // corresponding defs.
+                                    exposed_symbols.remove(&symbol);
+                                }
+                            }
+                        }
+                    }
+
+                    InvalidCycle(identifiers, _) => {
+                        panic!("TODO gracefully handle potentially attempting to expose invalid cyclic defs {:?}" , identifiers);
+                    }
+                    Builtin(def) => {
+                        // Builtins cannot be exposed in module declarations.
+                        // This should never happen!
+                        debug_assert!(def
+                            .symbols(env.pool)
+                            .iter()
+                            .all(|symbol| !exposed_symbols.contains(symbol)));
+                    }
+                }
+            }
+
+            let mut aliases = MutMap::default();
+
+            for (symbol, alias) in output.aliases {
+                // Remove this from exposed_symbols,
+                // so that at the end of the process,
+                // we can see if there were any
+                // exposed symbols which did not have
+                // corresponding defs.
+                exposed_symbols.remove(&symbol);
+
+                aliases.insert(symbol, alias);
+            }
+
+            // By this point, all exposed symbols should have been removed from
+            // exposed_symbols and added to exposed_vars_by_symbol. If any were
+            // not, that means they were declared as exposed but there was
+            // no actual declaration with that name!
+            for symbol in exposed_symbols {
+                env.problem(Problem::ExposedButNotDefined(symbol));
+
+                // In case this exposed value is referenced by other modules,
+                // create a decl for it whose implementation is a runtime error.
+                let mut pattern_vars = SendMap::default();
+                pattern_vars.insert(symbol, env.var_store.fresh());
+
+                let runtime_error = RuntimeError::ExposedButNotDefined(symbol);
+
+                let value_def = {
+                    let pattern = env.pool.add(Pattern2::Identifier(symbol));
+                    ValueDef {
+                        pattern,
+                        expr_type: None,
+                        expr_var: env.var_store.fresh(),
+                    }
+                };
+
+                let def = Def::Value(value_def);
+
+                declarations.push(Declaration::Declare(def));
+            }
+
+            // Incorporate any remaining output.lookups entries into references.
+            for symbol in output.references.lookups {
+                references.insert(symbol);
+            }
+
+            // Incorporate any remaining output.calls entries into references.
+            for symbol in output.references.calls {
+                references.insert(symbol);
+            }
+
+            // Gather up all the symbols that were referenced from other modules.
+            for symbol in env.qualified_lookups.iter() {
+                references.insert(*symbol);
+            }
+
+            // TODO find captured variables
+            // for declaration in declarations.iter_mut() {
+            //     match declaration {
+            //         Declare(def) => fix_values_captured_in_closure_def(def, &mut MutSet::default()),
+            //         DeclareRec(defs) => {
+            //             fix_values_captured_in_closure_defs(defs, &mut MutSet::default())
+            //         }
+            //         InvalidCycle(_, _) | Builtin(_) => {}
+            //     }
+            // }
+
+            // TODO this loops over all symbols in the module, we can speed it up by having an
+            // iterator over all builtin symbols
+
+            // TODO move over the builtins
+            // for symbol in references.iter() {
+            //     if symbol.is_builtin() {
+            //         // this can fail when the symbol is for builtin types, or has no implementation yet
+            //         if let Some(def) = builtins::builtin_defs_map(*symbol, var_store) {
+            //             declarations.push(Declaration::Builtin(def));
+            //         }
+            //     }
+            // }
+
+            Ok(ModuleOutput {
+                aliases,
+                rigid_variables,
+                declarations,
+                references,
+                exposed_imports: can_exposed_imports,
+                problems: vec![], // TODO env.problems,
+                lookups,
+                ident_ids: env.ident_ids,
+            })
+        }
+        (Err(runtime_error), _) => Err(runtime_error),
+    }
+}