Check for invalid cycles after type solving recursive defs

Disallow cycles that pass through a non-function value. Since we
evaluate eagerly, having one such cycle means there is at least one path
in the program that (likely) has unbounded recursion. Of course we can't
be certain (halting problem), but it's very likely, and avoids stuff
like #1926. Also, mono (as it's done today) won't work if things in a
cycle aren't functions.

Closes #1926

compiler/can/src/def.rs

@@ -28,6 +28,7 @@ use roc_parse::pattern::PatternType;
 use roc_problem::can::ShadowKind;
 use roc_problem::can::{CycleEntry, Problem, RuntimeError};
 use roc_region::all::{Loc, Region};
+use roc_types::subs::IllegalCycleMark;
 use roc_types::subs::{VarStore, Variable};
 use roc_types::types::AliasKind;
 use roc_types::types::AliasVar;
@@ -158,8 +159,10 @@ impl PendingTypeDef<'_> {
 #[allow(clippy::large_enum_variant)]
 pub enum Declaration {
     Declare(Def),
-    DeclareRec(Vec<Def>),
+    DeclareRec(Vec<Def>, IllegalCycleMark),
     Builtin(Def),
+    /// If we know a cycle is illegal during canonicalization.
+    /// Otherwise we will try to detect this during solving; see [`IllegalCycleMark`].
     InvalidCycle(Vec<CycleEntry>),
 }
 
@@ -168,7 +171,7 @@ impl Declaration {
         use Declaration::*;
         match self {
             Declare(_) => 1,
-            DeclareRec(defs) => defs.len(),
+            DeclareRec(defs, _) => defs.len(),
             InvalidCycle { .. } => 0,
             Builtin(_) => 0,
         }
@@ -748,6 +751,7 @@ impl DefOrdering {
 #[inline(always)]
 pub(crate) fn sort_can_defs(
     env: &mut Env<'_>,
+    var_store: &mut VarStore,
     defs: CanDefs,
     mut output: Output,
 ) -> (Vec<Declaration>, Output) {
@@ -810,7 +814,10 @@ pub(crate) fn sort_can_defs(
                 debug_assert!(!is_specialization, "Self-recursive specializations can only be determined during solving - but it was determined for {:?} now, that's a bug!", def);
 
                 // this function calls itself, and must be typechecked as a recursive def
-                Declaration::DeclareRec(vec![mark_def_recursive(def)])
+                Declaration::DeclareRec(
+                    vec![mark_def_recursive(def)],
+                    IllegalCycleMark::new(var_store),
+                )
             } else {
                 Declaration::Declare(def)
             };
@@ -827,7 +834,9 @@ pub(crate) fn sort_can_defs(
             //
             // boom = \{} -> boom {}
             //
-            // In general we cannot spot faulty recursion (halting problem) so this is our best attempt
+            // In general we cannot spot faulty recursion (halting problem), so this is our
+            // purely-syntactic heuristic. We'll have a second attempt once we know the types in
+            // the cycle.
             let direct_sccs = def_ordering
                 .direct_references
                 .strongly_connected_components_subset(group);
@@ -857,7 +866,7 @@ pub(crate) fn sort_can_defs(
                     .map(|index| mark_def_recursive(take_def!(index)))
                     .collect();
 
-                Declaration::DeclareRec(rec_defs)
+                Declaration::DeclareRec(rec_defs, IllegalCycleMark::new(var_store))
             };
 
             declarations.push(declaration);
@@ -1288,7 +1297,7 @@ pub fn can_defs_with_return<'a>(
         }
     }
 
-    let (declarations, output) = sort_can_defs(env, unsorted, output);
+    let (declarations, output) = sort_can_defs(env, var_store, unsorted, output);
 
     let mut loc_expr: Loc<Expr> = ret_expr;
 
@@ -1305,7 +1314,9 @@ pub fn can_defs_with_return<'a>(
 fn decl_to_let(decl: Declaration, loc_ret: Loc<Expr>) -> Expr {
     match decl {
         Declaration::Declare(def) => Expr::LetNonRec(Box::new(def), Box::new(loc_ret)),
-        Declaration::DeclareRec(defs) => Expr::LetRec(defs, Box::new(loc_ret)),
+        Declaration::DeclareRec(defs, cycle_mark) => {
+            Expr::LetRec(defs, Box::new(loc_ret), cycle_mark)
+        }
         Declaration::InvalidCycle(entries) => {
             Expr::RuntimeError(RuntimeError::CircularDef(entries))
         }