use the WhenRecursive trick in refcounting

compiler/gen/src/llvm/refcounting.rs

@@ -285,6 +285,7 @@ fn modify_refcount_struct<'a, 'ctx, 'env>(
     value: BasicValueEnum<'ctx>,
     layouts: &[Layout<'a>],
     mode: Mode,
+    when_recursive: &WhenRecursive<'a>,
 ) {
     let wrapper_struct = value.into_struct_value();
 
@@ -295,7 +296,15 @@ fn modify_refcount_struct<'a, 'ctx, 'env>(
                 .build_extract_value(wrapper_struct, i as u32, "decrement_struct_field")
                 .unwrap();
 
-            modify_refcount_layout(env, parent, layout_ids, mode, field_ptr, field_layout);
+            modify_refcount_layout_help(
+                env,
+                parent,
+                layout_ids,
+                mode,
+                when_recursive,
+                field_ptr,
+                field_layout,
+            );
         }
     }
 }
@@ -332,6 +341,7 @@ fn modify_refcount_builtin<'a, 'ctx, 'env>(
     env: &Env<'a, 'ctx, 'env>,
     layout_ids: &mut LayoutIds<'a>,
     mode: Mode,
+    when_recursive: &WhenRecursive<'a>,
     value: BasicValueEnum<'ctx>,
     layout: &Layout<'a>,
     builtin: &Builtin<'a>,
@@ -347,6 +357,7 @@ fn modify_refcount_builtin<'a, 'ctx, 'env>(
                     env,
                     layout_ids,
                     mode,
+                    when_recursive,
                     layout,
                     element_layout,
                     wrapper_struct,
@@ -365,6 +376,7 @@ fn modify_refcount_builtin<'a, 'ctx, 'env>(
                 env,
                 layout_ids,
                 mode,
+                when_recursive,
                 layout,
                 key_layout,
                 value_layout,
@@ -389,11 +401,45 @@ fn modify_refcount_layout<'a, 'ctx, 'env>(
     mode: Mode,
     value: BasicValueEnum<'ctx>,
     layout: &Layout<'a>,
+) {
+    modify_refcount_layout_help(
+        env,
+        parent,
+        layout_ids,
+        mode,
+        &WhenRecursive::Unreachable,
+        value,
+        layout,
+    );
+}
+
+#[derive(Clone, Debug, PartialEq)]
+enum WhenRecursive<'a> {
+    Unreachable,
+    Loop(UnionLayout<'a>),
+}
+
+fn modify_refcount_layout_help<'a, 'ctx, 'env>(
+    env: &Env<'a, 'ctx, 'env>,
+    parent: FunctionValue<'ctx>,
+    layout_ids: &mut LayoutIds<'a>,
+    mode: Mode,
+    when_recursive: &WhenRecursive<'a>,
+    value: BasicValueEnum<'ctx>,
+    layout: &Layout<'a>,
 ) {
     use Layout::*;
 
     match layout {
-        Builtin(builtin) => modify_refcount_builtin(env, layout_ids, mode, value, layout, builtin),
+        Builtin(builtin) => modify_refcount_builtin(
+            env,
+            layout_ids,
+            mode,
+            when_recursive,
+            value,
+            layout,
+            builtin,
+        ),
 
         Union(variant) => {
             use UnionLayout::*;
@@ -408,6 +454,7 @@ fn modify_refcount_layout<'a, 'ctx, 'env>(
                         env,
                         layout_ids,
                         mode,
+                        &WhenRecursive::Loop(variant.clone()),
                         tags,
                         value.into_pointer_value(),
                         true,
@@ -423,6 +470,7 @@ fn modify_refcount_layout<'a, 'ctx, 'env>(
                         env,
                         layout_ids,
                         mode,
+                        &WhenRecursive::Loop(variant.clone()),
                         &*env.arena.alloc([other_fields]),
                         value.into_pointer_value(),
                         true,
@@ -436,6 +484,7 @@ fn modify_refcount_layout<'a, 'ctx, 'env>(
                         env,
                         layout_ids,
                         mode,
+                        &WhenRecursive::Loop(variant.clone()),
                         &*env.arena.alloc([*fields]),
                         value.into_pointer_value(),
                         true,
@@ -448,13 +497,16 @@ fn modify_refcount_layout<'a, 'ctx, 'env>(
                         env,
                         layout_ids,
                         mode,
+                        &WhenRecursive::Loop(variant.clone()),
                         tags,
                         value.into_pointer_value(),
                         false,
                     );
                 }
 
-                NonRecursive(tags) => modify_refcount_union(env, layout_ids, mode, tags, value),
+                NonRecursive(tags) => {
+                    modify_refcount_union(env, layout_ids, mode, when_recursive, tags, value)
+                }
             }
         }
         Closure(_, closure_layout, _) => {
@@ -466,11 +518,12 @@ fn modify_refcount_layout<'a, 'ctx, 'env>(
                     .build_extract_value(wrapper_struct, 1, "modify_rc_closure_data")
                     .unwrap();
 
-                modify_refcount_layout(
+                modify_refcount_layout_help(
                     env,
                     parent,
                     layout_ids,
                     mode,
+                    when_recursive,
                     field_ptr,
                     &closure_layout.as_block_of_memory_layout(),
                 )
@@ -478,12 +531,45 @@ fn modify_refcount_layout<'a, 'ctx, 'env>(
         }
 
         Struct(layouts) => {
-            modify_refcount_struct(env, parent, layout_ids, value, layouts, mode);
+            modify_refcount_struct(
+                env,
+                parent,
+                layout_ids,
+                value,
+                layouts,
+                mode,
+                when_recursive,
+            );
         }
 
         PhantomEmptyStruct => {}
 
-        RecursivePointer => todo!("TODO implement decrement layout of recursive tag union"),
+        Layout::RecursivePointer => match when_recursive {
+            WhenRecursive::Unreachable => {
+                unreachable!("recursion pointers should never be hashed directly")
+            }
+            WhenRecursive::Loop(union_layout) => {
+                let layout = Layout::Union(union_layout.clone());
+
+                let bt = basic_type_from_layout(env.arena, env.context, &layout, env.ptr_bytes);
+
+                // cast the i64 pointer to a pointer to block of memory
+                let field_cast = env
+                    .builder
+                    .build_bitcast(value, bt, "i64_to_opaque")
+                    .into_pointer_value();
+
+                modify_refcount_layout_help(
+                    env,
+                    parent,
+                    layout_ids,
+                    mode,
+                    when_recursive,
+                    field_cast.into(),
+                    &layout,
+                )
+            }
+        },
 
         FunctionPointer(_, _) | Pointer(_) => {}
     }
@@ -493,6 +579,7 @@ fn modify_refcount_list<'a, 'ctx, 'env>(
     env: &Env<'a, 'ctx, 'env>,
     layout_ids: &mut LayoutIds<'a>,
     mode: Mode,
+    when_recursive: &WhenRecursive<'a>,
     layout: &Layout<'a>,
     element_layout: &Layout<'a>,
     original_wrapper: StructValue<'ctx>,
@@ -519,6 +606,7 @@ fn modify_refcount_list<'a, 'ctx, 'env>(
                 env,
                 layout_ids,
                 mode,
+                when_recursive,
                 layout,
                 element_layout,
                 function_value,
@@ -546,6 +634,7 @@ fn modify_refcount_list_help<'a, 'ctx, 'env>(
     env: &Env<'a, 'ctx, 'env>,
     layout_ids: &mut LayoutIds<'a>,
     mode: Mode,
+    when_recursive: &WhenRecursive<'a>,
     layout: &Layout<'a>,
     element_layout: &Layout<'a>,
     fn_val: FunctionValue<'ctx>,
@@ -594,7 +683,15 @@ fn modify_refcount_list_help<'a, 'ctx, 'env>(
         let (len, ptr) = load_list(env.builder, original_wrapper, ptr_type);
 
         let loop_fn = |_index, element| {
-            modify_refcount_layout(env, parent, layout_ids, mode, element, element_layout);
+            modify_refcount_layout_help(
+                env,
+                parent,
+                layout_ids,
+                mode,
+                when_recursive,
+                element,
+                element_layout,
+            );
         };
 
         incrementing_elem_loop(
@@ -720,6 +817,7 @@ fn modify_refcount_dict<'a, 'ctx, 'env>(
     env: &Env<'a, 'ctx, 'env>,
     layout_ids: &mut LayoutIds<'a>,
     mode: Mode,
+    when_recursive: &WhenRecursive<'a>,
     layout: &Layout<'a>,
     key_layout: &Layout<'a>,
     value_layout: &Layout<'a>,
@@ -747,6 +845,7 @@ fn modify_refcount_dict<'a, 'ctx, 'env>(
                 env,
                 layout_ids,
                 mode,
+                when_recursive,
                 layout,
                 key_layout,
                 value_layout,
@@ -768,11 +867,18 @@ fn modify_refcount_dict_help<'a, 'ctx, 'env>(
     env: &Env<'a, 'ctx, 'env>,
     layout_ids: &mut LayoutIds<'a>,
     mode: Mode,
+    when_recursive: &WhenRecursive<'a>,
     layout: &Layout<'a>,
     key_layout: &Layout<'a>,
     value_layout: &Layout<'a>,
     fn_val: FunctionValue<'ctx>,
 ) {
+    debug_assert_eq!(
+        when_recursive,
+        &WhenRecursive::Unreachable,
+        "TODO pipe when_recursive through the dict key/value inc/dec"
+    );
+
     let builder = env.builder;
     let ctx = env.context;
 
@@ -909,6 +1015,7 @@ fn build_rec_union<'a, 'ctx, 'env>(
     env: &Env<'a, 'ctx, 'env>,
     layout_ids: &mut LayoutIds<'a>,
     mode: Mode,
+    when_recursive: &WhenRecursive<'a>,
     fields: &'a [&'a [Layout<'a>]],
     value: PointerValue<'ctx>,
     is_nullable: bool,
@@ -935,7 +1042,15 @@ fn build_rec_union<'a, 'ctx, 'env>(
                 .into();
             let function_value = build_header(env, basic_type, mode, &fn_name);
 
-            build_rec_union_help(env, layout_ids, mode, fields, function_value, is_nullable);
+            build_rec_union_help(
+                env,
+                layout_ids,
+                mode,
+                when_recursive,
+                fields,
+                function_value,
+                is_nullable,
+            );
 
             env.builder.position_at_end(block);
             env.builder
@@ -952,6 +1067,7 @@ fn build_rec_union_help<'a, 'ctx, 'env>(
     env: &Env<'a, 'ctx, 'env>,
     layout_ids: &mut LayoutIds<'a>,
     mode: Mode,
+    when_recursive: &WhenRecursive<'a>,
     tags: &[&[Layout<'a>]],
     fn_val: FunctionValue<'ctx>,
     is_nullable: bool,
@@ -1108,7 +1224,15 @@ fn build_rec_union_help<'a, 'ctx, 'env>(
         refcount_ptr.modify(call_mode, &layout, env);
 
         for (field, field_layout) in deferred_nonrec {
-            modify_refcount_layout(env, parent, layout_ids, mode, field, field_layout);
+            modify_refcount_layout_help(
+                env,
+                parent,
+                layout_ids,
+                mode,
+                when_recursive,
+                field,
+                field_layout,
+            );
         }
 
         let call_name = pick("recursive_tag_increment", "recursive_tag_decrement");
@@ -1223,6 +1347,7 @@ fn modify_refcount_union<'a, 'ctx, 'env>(
     env: &Env<'a, 'ctx, 'env>,
     layout_ids: &mut LayoutIds<'a>,
     mode: Mode,
+    when_recursive: &WhenRecursive<'a>,
     fields: &'a [&'a [Layout<'a>]],
     value: BasicValueEnum<'ctx>,
 ) {
@@ -1246,7 +1371,14 @@ fn modify_refcount_union<'a, 'ctx, 'env>(
             let basic_type = block_of_memory(env.context, &layout, env.ptr_bytes);
             let function_value = build_header(env, basic_type, mode, &fn_name);
 
-            modify_refcount_union_help(env, layout_ids, mode, fields, function_value);
+            modify_refcount_union_help(
+                env,
+                layout_ids,
+                mode,
+                when_recursive,
+                fields,
+                function_value,
+            );
 
             function_value
         }
@@ -1263,6 +1395,7 @@ fn modify_refcount_union_help<'a, 'ctx, 'env>(
     env: &Env<'a, 'ctx, 'env>,
     layout_ids: &mut LayoutIds<'a>,
     mode: Mode,
+    when_recursive: &WhenRecursive<'a>,
     tags: &[&[Layout<'a>]],
     fn_val: FunctionValue<'ctx>,
 ) {
@@ -1347,7 +1480,15 @@ fn modify_refcount_union_help<'a, 'ctx, 'env>(
                     .build_extract_value(wrapper_struct, i as u32, "modify_tag_field")
                     .unwrap();
 
-                modify_refcount_layout(env, parent, layout_ids, mode, field_ptr, field_layout);
+                modify_refcount_layout_help(
+                    env,
+                    parent,
+                    layout_ids,
+                    mode,
+                    when_recursive,
+                    field_ptr,
+                    field_layout,
+                );
             }
         }