consider tags that are represented as NULL

compiler/mono/src/layout.rs

@@ -196,6 +196,16 @@ impl<'a> UnionLayout<'a> {
             UnionLayout::NonNullableUnwrapped(_) | UnionLayout::NullableUnwrapped { .. } => false,
         }
     }
+
+    pub fn tag_is_null(&self, tag_id: u8) -> bool {
+        match self {
+            UnionLayout::NonRecursive(_)
+            | UnionLayout::NonNullableUnwrapped(_)
+            | UnionLayout::Recursive(_) => false,
+            UnionLayout::NullableWrapped { nullable_id, .. } => *nullable_id == tag_id as i64,
+            UnionLayout::NullableUnwrapped { nullable_id, .. } => *nullable_id == (tag_id != 0),
+        }
+    }
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]

compiler/mono/src/reset_reuse.rs

@@ -36,21 +36,11 @@ fn may_reuse(tag_layout: UnionLayout, tag_id: u8, other: &CtorInfo) -> bool {
         return false;
     }
 
-    match tag_layout {
+    // we should not get here if the tag we matched on is represented as NULL
-        UnionLayout::NonRecursive(_)
+    debug_assert!(!tag_layout.tag_is_null(other.id));
-        | UnionLayout::Recursive(_)
+
-        | UnionLayout::NonNullableUnwrapped(_) => true,
+    // furthermore, we can only use the memory if the tag we're creating is non-NULL
-        UnionLayout::NullableWrapped { nullable_id, .. } => {
+    !tag_layout.tag_is_null(tag_id)
-            // if the source tag id is represented as NULL, there is no memory to re-use
-            // if the current tag id is represented as NULL, then we don't need to re-use the
-            // memory here and can use it somewhere else
-            other.id as i64 != nullable_id && tag_id as i64 != nullable_id
-        }
-        UnionLayout::NullableUnwrapped { nullable_id, .. } => {
-            // idem
-            (other.id != 0) != nullable_id && (tag_id != 0) != nullable_id
-        }
-    }
 }
 
 #[derive(Debug)]
@@ -421,7 +411,7 @@ fn function_r<'a, 'i>(env: &mut Env<'a, 'i>, stmt: &'a Stmt<'a>) -> &'a Stmt<'a>
                         layout,
                         tag_id,
                     } => match layout {
-                        Layout::Union(union_layout) => {
+                        Layout::Union(union_layout) if !union_layout.tag_is_null(*tag_id) => {
                             let ctor_info = CtorInfo {
                                 layout: *union_layout,
                                 id: *tag_id,
@@ -446,7 +436,7 @@ fn function_r<'a, 'i>(env: &mut Env<'a, 'i>, stmt: &'a Stmt<'a>) -> &'a Stmt<'a>
                         layout,
                         tag_id,
                     } => match layout {
-                        Layout::Union(union_layout) => {
+                        Layout::Union(union_layout) if !union_layout.tag_is_null(*tag_id) => {
                             let ctor_info = CtorInfo {
                                 layout: *union_layout,
                                 id: *tag_id,