When possible, prefer NonNullableUnwrapped over NullableWrapped

e.g. in the scenario of `[ A Str, B, C, D ]`, rather than having one tag be nullable, we want
to store the tag id in the pointer. This generally saves 8 bytes for
every allocation, and prevents an allocations for all tags except A.

crates/compiler/mono/src/layout.rs

@@ -3883,6 +3883,36 @@ where
     union_sorted_tags_help(env, tags_vec, opt_rec_var, DropUninhabitedVariants(true)).value()
 }
 
+fn find_nullable_tag<'a, L, I>(tags: I) -> Option<(TagIdIntType, L)>
+where
+    I: Iterator<Item = (&'a L, &'a [Variable])>,
+    L: Into<TagOrClosure> + Ord + Clone + 'a,
+{
+    let mut length = 0;
+    let mut has_payload = 0;
+    let mut nullable = None;
+
+    for (index, (name, variables)) in tags.enumerate() {
+        length += 1;
+
+        if variables.is_empty() {
+            nullable = nullable.or_else(|| Some((index as TagIdIntType, name.clone())));
+        } else {
+            has_payload += 1;
+        }
+    }
+
+    let has_no_payload = length - has_payload;
+
+    // in the scenario of `[ A Str, B, C, D ]`, rather than having one tag be nullable, we want
+    // to store the tag id in the pointer. (we want NonNullableUnwrapped, not NullableWrapped)
+    if (has_payload > 1 && has_no_payload > 0) || has_no_payload == 1 {
+        nullable
+    } else {
+        None
+    }
+}
+
 fn union_sorted_tags_help<'a, L>(
     env: &mut Env<'a, '_>,
     mut tags_vec: std::vec::Vec<(L, std::vec::Vec<Variable>)>,
@@ -3965,12 +3995,7 @@ where
             // only recursive tag unions can be nullable
             let is_recursive = opt_rec_var.is_some();
             if is_recursive && GENERATE_NULLABLE {
-                for (index, (name, variables)) in tags_vec.iter().enumerate() {
-                    if variables.is_empty() {
-                        nullable = Some((index as TagIdIntType, name.clone()));
-                        break;
-                    }
-                }
+                nullable = find_nullable_tag(tags_vec.iter().map(|(a, b)| (a, b.as_slice())));
             }
 
             for (index, (tag_name, arguments)) in tags_vec.into_iter().enumerate() {
@@ -4260,17 +4285,12 @@ where
     let mut nullable = None;
 
     if GENERATE_NULLABLE {
-        for (index, (_name, variables)) in tags_vec.iter().enumerate() {
-            if variables.is_empty() {
-                nullable = Some(index as TagIdIntType);
-                break;
-            }
-        }
+        nullable = find_nullable_tag(tags_vec.iter().map(|(a, b)| (*a, *b)));
     }
 
     env.insert_seen(rec_var);
     for (index, &(_name, variables)) in tags_vec.iter().enumerate() {
-        if matches!(nullable, Some(i) if i == index as TagIdIntType) {
+        if matches!(nullable, Some((i, _)) if i == index as TagIdIntType) {
             // don't add the nullable case
             continue;
         }
@@ -4303,7 +4323,7 @@ where
     }
     env.remove_seen(rec_var);
 
-    let union_layout = if let Some(tag_id) = nullable {
+    let union_layout = if let Some((tag_id, _)) = nullable {
         match tag_layouts.into_bump_slice() {
             [one] => {
                 let nullable_id = tag_id != 0;

crates/compiler/test_mono/generated/non_nullable_unwrapped_instead_of_nullable_wrapped.txt

@@ -0,0 +1,17 @@
+procedure Test.0 ():
+    let Test.4 : [<r>C , C , C Str *self] = TagId(0) ;
+    let Test.9 : U8 = GetTagId Test.4;
+    dec Test.4;
+    switch Test.9:
+        case 0:
+            let Test.6 : Str = "A";
+            ret Test.6;
+    
+        case 1:
+            let Test.7 : Str = "B";
+            ret Test.7;
+    
+        default:
+            let Test.8 : Str = "C";
+            ret Test.8;
+    

crates/compiler/test_mono/generated/nullable_wrapped_with_non_nullable_singleton_tags.txt

@@ -1,34 +0,0 @@
-procedure Str.3 (#Attr.2, #Attr.3):
-    let Str.299 : Str = lowlevel StrConcat #Attr.2 #Attr.3;
-    ret Str.299;
-
-procedure Test.2 (Test.4):
-    let Test.16 : U8 = GetTagId Test.4;
-    dec Test.4;
-    switch Test.16:
-        case 0:
-            let Test.13 : Str = "A";
-            ret Test.13;
-    
-        case 1:
-            let Test.14 : Str = "B";
-            ret Test.14;
-    
-        default:
-            let Test.15 : Str = "C";
-            ret Test.15;
-    
-
-procedure Test.0 ():
-    let Test.21 : [<rnw>C *self, <null>, C ] = TagId(1) ;
-    let Test.20 : [<rnw>C *self, <null>, C ] = TagId(0) Test.21;
-    let Test.17 : Str = CallByName Test.2 Test.20;
-    let Test.19 : [<rnw>C *self, <null>, C ] = TagId(1) ;
-    let Test.18 : Str = CallByName Test.2 Test.19;
-    let Test.10 : Str = CallByName Str.3 Test.17 Test.18;
-    dec Test.18;
-    let Test.12 : [<rnw>C *self, <null>, C ] = TagId(2) ;
-    let Test.11 : Str = CallByName Test.2 Test.12;
-    let Test.9 : Str = CallByName Str.3 Test.10 Test.11;
-    dec Test.11;
-    ret Test.9;

crates/compiler/test_mono/src/tests.rs

@@ -2327,32 +2327,6 @@ fn issue_4557() {
     )
 }
 
-#[mono_test]
-fn nullable_wrapped_with_non_nullable_singleton_tags() {
-    indoc!(
-        r###"
-        app "test" provides [main] to "./platform"
-
-        F : [
-            A F,
-            B,
-            C,
-        ]
-
-        g : F -> Str
-        g = \f -> when f is
-                A _ -> "A"
-                B -> "B"
-                C -> "C"
-
-        main =
-            g (A (B))
-            |> Str.concat (g B)
-            |> Str.concat (g C)
-        "###
-    )
-}
-
 #[mono_test]
 fn nullable_wrapped_with_nullable_not_last_index() {
     indoc!(
@@ -3260,3 +3234,24 @@ fn capture_void_layout_task() {
         "#
     )
 }
+
+#[mono_test]
+fn non_nullable_unwrapped_instead_of_nullable_wrapped() {
+    indoc!(
+        r#"
+        app "test" provides [main] to "./platform"
+
+        Ast : [ A, B, C Str Ast ]
+
+        main : Str
+        main =
+            x : Ast
+            x = A
+
+            when x is
+                A -> "A"
+                B -> "B"
+                C _ _ -> "C"
+        "#
+    )
+}