From 5250e930aa77f59ef3ed7a37f10a78fd5a84908d Mon Sep 17 00:00:00 2001
From: Folkert <folkert@folkertdev.nl>
Date: Sat, 3 Jul 2021 13:43:04 +0200
Subject: [PATCH] don't unroll recursive layouts

---
 compiler/mono/src/layout.rs      | 20 +++++++++++++++++---
 compiler/mono/src/reset_reuse.rs | 22 +++++++++++++---------
 2 files changed, 30 insertions(+), 12 deletions(-)

diff --git a/compiler/mono/src/layout.rs b/compiler/mono/src/layout.rs
index 89ea62d0b2..9b7e6de623 100644
--- a/compiler/mono/src/layout.rs
+++ b/compiler/mono/src/layout.rs
@@ -1508,6 +1508,18 @@ fn get_recursion_var(subs: &Subs, var: Variable) -> Option<Variable> {
     }
 }
 
+fn is_recursive_tag_union(layout: &Layout) -> bool {
+    match layout {
+        Layout::Union(
+            UnionLayout::NullableUnwrapped { .. }
+            | UnionLayout::Recursive(_)
+            | UnionLayout::NullableWrapped { .. }
+            | UnionLayout::NonNullableUnwrapped { .. },
+        ) => true,
+        _ => false,
+    }
+}
+
 pub fn union_sorted_tags_help<'a>(
     arena: &'a Bump,
     mut tags_vec: std::vec::Vec<(TagName, std::vec::Vec<Variable>)>,
@@ -1623,10 +1635,12 @@ pub fn union_sorted_tags_help<'a>(
                 for var in arguments {
                     match Layout::from_var(&mut env, var) {
                         Ok(layout) => {
-                            // Drop any zero-sized arguments like {}
-                            if !layout.is_dropped_because_empty() {
-                                has_any_arguments = true;
+                            has_any_arguments = true;
 
+                            // make sure to not unroll recursive types!
+                            if opt_rec_var.is_some() && is_recursive_tag_union(&layout) {
+                                arg_layouts.push(Layout::RecursivePointer);
+                            } else {
                                 arg_layouts.push(layout);
                             }
                         }
diff --git a/compiler/mono/src/reset_reuse.rs b/compiler/mono/src/reset_reuse.rs
index 39464fadfc..5ec9fa8244 100644
--- a/compiler/mono/src/reset_reuse.rs
+++ b/compiler/mono/src/reset_reuse.rs
@@ -32,20 +32,24 @@ struct CtorInfo<'a> {
 }
 
 fn may_reuse(tag_layout: UnionLayout, tag_id: u8, other: &CtorInfo) -> bool {
-    //    if tag_layout != other.layout {
-    //        return false;
-    //    }
+    if tag_layout != other.layout {
+        return false;
+    }
 
-    // if the tag id is represented as NULL, there is no memory to re-use
     match tag_layout {
         UnionLayout::NonRecursive(_)
         | UnionLayout::Recursive(_)
         | UnionLayout::NonNullableUnwrapped(_) => true,
-        UnionLayout::NullableWrapped { nullable_id, .. } => tag_id as i64 != nullable_id,
-        UnionLayout::NullableUnwrapped {
-            nullable_id,
-            other_fields,
-        } => (tag_id != 0) != nullable_id,
+        UnionLayout::NullableWrapped { nullable_id, .. } => {
+            // if the source tag id is represented as NULL, there is no memory to re-use
+            // if the current tag id is represented as NULL, then we don't need to re-use the
+            // memory here and can use it somewhere else
+            other.id as i64 != nullable_id && tag_id as i64 != nullable_id
+        }
+        UnionLayout::NullableUnwrapped { nullable_id, .. } => {
+            // idem
+            (other.id != 0) != nullable_id && (tag_id != 0) != nullable_id
+        }
     }
 }