Make nested datatypes into errors

I was hoping to add nested datatypes into the language, but it turns out
doing so is quite tricky and not all that useful with Roc's current
compilation model. Basically every implementation strategy I could think
of ended up requiring a uniform representation for the data layout
(or some ugly workaround). Furhermore it increased the complexity of the
checker/mono IR generator a little bit - basically, we must always pass
around the alias definitions of nested datatypes and instantiate them
at usage sites, rather than being able to unroll aliases as we currently
do during canonicalization.

So, especially because we don't support polymorphic recursion anyway, I
think it may be better to simply disallow any kind of nested datatypes
in the language. In any case, Stephanie Weirich [seems to think nested
datatypes are not needed](https://www.cis.upenn.edu/~plclub/blog/2020-12-04-nested-datatypes/).

Closes #2293

compiler/can/src/def.rs

@@ -277,7 +277,7 @@ pub fn canonicalize_defs<'a>(
         let mut can_vars: Vec<Loc<(Lowercase, Variable)>> = Vec::with_capacity(vars.len());
         let mut is_phantom = false;
 
-        for loc_lowercase in vars {
+        for loc_lowercase in vars.iter() {
             if let Some(var) = can_ann
                 .introduced_variables
                 .var_by_name(&loc_lowercase.value)
@@ -303,10 +303,18 @@ pub fn canonicalize_defs<'a>(
             continue;
         }
 
+        let mut is_nested_datatype = false;
         if can_ann.typ.contains_symbol(symbol) {
-            make_tag_union_recursive(
+            let alias_args = can_vars
+                .iter()
+                .map(|l| (l.value.0.clone(), Type::Variable(l.value.1)))
+                .collect::<Vec<_>>();
+            let alias_region =
+                Region::across_all([name.region].iter().chain(vars.iter().map(|l| &l.region)));
+
+            let made_recursive = make_tag_union_recursive(
                 env,
-                symbol,
+                Loc::at(alias_region, (symbol, &alias_args)),
                 name.region,
                 vec![],
                 &mut can_ann.typ,
@@ -315,6 +323,13 @@ pub fn canonicalize_defs<'a>(
                 // recursion errors after the sorted introductions are complete.
                 &mut false,
             );
+
+            is_nested_datatype = made_recursive.is_err();
+        }
+
+        if is_nested_datatype {
+            // Bail out
+            continue;
         }
 
         scope.add_alias(symbol, name.region, can_vars.clone(), can_ann.typ.clone());
@@ -1624,9 +1639,16 @@ fn correct_mutual_recursive_type_alias<'a>(
                             var_store,
                             &mut ImSet::default(),
                         );
-                        make_tag_union_recursive(
+
+                        let alias_args = &alias
+                            .type_variables
+                            .iter()
+                            .map(|l| (l.value.0.clone(), Type::Variable(l.value.1)))
+                            .collect::<Vec<_>>();
+
+                        let _made_recursive = make_tag_union_recursive(
                             env,
-                            *rec,
+                            Loc::at(alias.header_region(), (*rec, &alias_args)),
                             alias.region,
                             others,
                             &mut alias.typ,
@@ -1640,25 +1662,71 @@ fn correct_mutual_recursive_type_alias<'a>(
     }
 }
 
+/// Attempt to make a tag union recursive at the position of `recursive_alias`; for example,
+///
+/// ```roc
+/// [ Cons a (ConsList a), Nil ] as ConsList a
+/// ```
+///
+/// can be made recursive at the position "ConsList a" with a fresh recursive variable, say r1:
+///
+/// ```roc
+/// [ Cons a r1, Nil ] as r1
+/// ```
+///
+/// Returns `Err` if the tag union is recursive, but there is no structure-preserving recursion
+/// variable for it. This can happen when the type is a nested datatype, for example in either of
+///
+/// ```roc
+/// Nested a : [ Chain a (Nested (List a)), Term ]
+/// DuoList a b : [ Cons a (DuoList b a), Nil ]
+/// ```
+///
+/// When `Err` is returned, a problem will be added to `env`.
 fn make_tag_union_recursive<'a>(
     env: &mut Env<'a>,
-    symbol: Symbol,
+    recursive_alias: Loc<(Symbol, &[(Lowercase, Type)])>,
     region: Region,
     others: Vec<Symbol>,
     typ: &mut Type,
     var_store: &mut VarStore,
     can_report_error: &mut bool,
-) {
+) -> Result<(), ()> {
+    let Loc {
+        value: (symbol, args),
+        region: alias_region,
+    } = recursive_alias;
+    let vars = args.iter().map(|(_, t)| t.clone()).collect::<Vec<_>>();
     match typ {
         Type::TagUnion(tags, ext) => {
             let rec_var = var_store.fresh();
-            *typ = Type::RecursiveTagUnion(rec_var, tags.to_vec(), ext.clone());
-            typ.substitute_alias(symbol, &Type::Variable(rec_var));
+            let mut pending_typ = Type::RecursiveTagUnion(rec_var, tags.to_vec(), ext.clone());
+            let substitution_result =
+                pending_typ.substitute_alias(symbol, &vars, &Type::Variable(rec_var));
+            match substitution_result {
+                Ok(()) => {
+                    // We can substitute the alias presence for the variable exactly.
+                    *typ = pending_typ;
+                    Ok(())
+                }
+                Err(differing_recursion_region) => {
+                    env.problems.push(Problem::NestedDatatype {
+                        alias: symbol,
+                        def_region: alias_region,
+                        differing_recursion_region,
+                    });
+                    Err(())
+                }
+            }
         }
-        Type::RecursiveTagUnion(_, _, _) => {}
-        Type::Alias { actual, .. } => make_tag_union_recursive(
+        Type::RecursiveTagUnion(_, _, _) => Ok(()),
+        Type::Alias {
+            actual,
+            type_arguments,
+            ..
+        } => make_tag_union_recursive(
             env,
-            symbol,
+            Loc::at_zero((symbol, &type_arguments)),
             region,
             others,
             actual,
@@ -1676,6 +1744,7 @@ fn make_tag_union_recursive<'a>(
                 let problem = Problem::CyclicAlias(symbol, region, others);
                 env.problems.push(problem);
             }
+            Ok(())
         }
     }
 }