#![allow(clippy::too_many_arguments)] use bumpalo::collections::vec::Vec; use bumpalo::collections::CollectIn; use roc_error_macros::todo_lambda_erasure; use roc_module::low_level::{LowLevel, LowLevel::*}; use roc_module::symbol::{IdentIds, Symbol}; use roc_target::PtrWidth; use crate::code_gen_help::let_lowlevel; use crate::ir::{ BranchInfo, Call, CallType, Expr, JoinPointId, Literal, ModifyRc, Param, Stmt, UpdateModeId, }; use crate::layout::{ Builtin, InLayout, Layout, LayoutInterner, LayoutRepr, STLayoutInterner, TagIdIntType, UnionLayout, }; use super::{CodeGenHelp, Context, HelperOp}; const LAYOUT_BOOL: InLayout = Layout::BOOL; const LAYOUT_UNIT: InLayout = Layout::UNIT; const LAYOUT_U32: InLayout = Layout::U32; pub fn refcount_stmt<'a>( root: &mut CodeGenHelp<'a>, ident_ids: &mut IdentIds, ctx: &mut Context<'a>, layout_interner: &mut STLayoutInterner<'a>, layout: InLayout<'a>, modify: &ModifyRc, following: &'a Stmt<'a>, ) -> &'a Stmt<'a> { let arena = root.arena; match modify { ModifyRc::Inc(structure, amount) => { let layout_isize = root.layout_isize; // Define a constant for the amount to increment let amount_sym = root.create_symbol(ident_ids, "amount"); let amount_expr = Expr::Literal(Literal::Int((*amount as i128).to_ne_bytes())); let amount_stmt = |next| Stmt::Let(amount_sym, amount_expr, layout_isize, next); // Call helper proc, passing the Roc structure and constant amount let call_result_empty = root.create_symbol(ident_ids, "call_result_empty"); let call_expr = root .call_specialized_op( ident_ids, ctx, layout_interner, layout, arena.alloc([*structure, amount_sym]), ) .unwrap(); let call_stmt = Stmt::Let(call_result_empty, call_expr, LAYOUT_UNIT, following); arena.alloc(amount_stmt(arena.alloc(call_stmt))) } ModifyRc::Dec(structure) => { // Call helper proc, passing the Roc structure let call_result_empty = root.create_symbol(ident_ids, "call_result_empty"); let call_expr = root .call_specialized_op( ident_ids, ctx, layout_interner, layout, arena.alloc([*structure]), ) .unwrap(); let call_stmt = Stmt::Let(call_result_empty, call_expr, LAYOUT_UNIT, following); arena.alloc(call_stmt) } ModifyRc::DecRef(structure) => { match layout_interner.get_repr(layout) { // Str has no children, so Dec is the same as DecRef. LayoutRepr::Builtin(Builtin::Str) => { ctx.op = HelperOp::Dec; refcount_stmt( root, ident_ids, ctx, layout_interner, layout, modify, following, ) } // Struct and non-recursive Unions are stack-only, so DecRef is a no-op LayoutRepr::Struct { .. } => following, LayoutRepr::Union(UnionLayout::NonRecursive(_)) => following, // Inline the refcounting code instead of making a function. Don't iterate fields, // and replace any return statements with jumps to the `following` statement. _ => match ctx.op { HelperOp::DecRef(jp_decref) => { let rc_stmt = refcount_generic( root, ident_ids, ctx, layout_interner, layout, *structure, ); let join = Stmt::Join { id: jp_decref, parameters: &[], body: following, remainder: arena.alloc(rc_stmt), }; arena.alloc(join) } _ => unreachable!(), }, } } ModifyRc::Free(_) => { unreachable!("free should be handled by the backend directly") } } } pub fn refcount_indirect<'a>( root: &mut CodeGenHelp<'a>, ident_ids: &mut IdentIds, ctx: &mut Context<'a>, layout_interner: &mut STLayoutInterner<'a>, element_layout: InLayout<'a>, structure: Symbol, ) -> Stmt<'a> { let arena = root.arena; let unit = root.create_symbol(ident_ids, "unit"); let loaded = root.create_symbol(ident_ids, "loaded"); let indirect_op = ctx.op; let direct_op = match ctx.op { HelperOp::IndirectInc => HelperOp::Inc, HelperOp::IndirectDec => HelperOp::Dec, _ => unreachable!(), }; // we've done the indirection, the inner value shoud be inc- or decremented directly ctx.op = direct_op; let mod_args = refcount_args(root, ctx, loaded); let opt_mod_expr = root.call_specialized_op(ident_ids, ctx, layout_interner, element_layout, mod_args); // set the op back to indirect ; this is important for correct layout generation ctx.op = indirect_op; if let Some(mod_expr) = opt_mod_expr { Stmt::Let( loaded, Expr::ptr_load(arena.alloc(structure)), element_layout, arena.alloc( // Stmt::Let( unit, mod_expr, Layout::UNIT, arena.alloc( // Stmt::Ret(unit), ), ), ), ) } else { rc_return_stmt(root, ident_ids, ctx) } } pub fn refcount_generic<'a>( root: &mut CodeGenHelp<'a>, ident_ids: &mut IdentIds, ctx: &mut Context<'a>, layout_interner: &mut STLayoutInterner<'a>, layout: InLayout<'a>, structure: Symbol, ) -> Stmt<'a> { match layout_interner.get_repr(layout) { LayoutRepr::Builtin( Builtin::Int(_) | Builtin::Float(_) | Builtin::Bool | Builtin::Decimal, ) | LayoutRepr::FunctionPointer(_) => { // Generate a dummy function that immediately returns Unit // Some higher-order Zig builtins *always* call an RC function on List elements. rc_return_stmt(root, ident_ids, ctx) } LayoutRepr::Builtin(Builtin::Str) => refcount_str(root, ident_ids, ctx), LayoutRepr::Builtin(Builtin::List(elem_layout)) => refcount_list( root, ident_ids, ctx, layout_interner, elem_layout, structure, ), LayoutRepr::Struct(field_layouts) => refcount_struct( root, ident_ids, ctx, layout_interner, field_layouts, structure, ), LayoutRepr::Union(union_layout) => refcount_union( root, ident_ids, ctx, layout_interner, layout, union_layout, structure, ), LayoutRepr::LambdaSet(lambda_set) => { let runtime_layout = lambda_set.representation; refcount_generic( root, ident_ids, ctx, layout_interner, runtime_layout, structure, ) } LayoutRepr::Erased(_) => { todo_lambda_erasure!() } LayoutRepr::RecursivePointer(_) => unreachable!( "We should never call a refcounting helper on a RecursivePointer layout directly" ), LayoutRepr::Ptr(_) => { unreachable!("We should never call a refcounting helper on a Ptr layout directly") } } } pub fn refcount_reset_proc_body<'a>( root: &mut CodeGenHelp<'a>, ident_ids: &mut IdentIds, ctx: &mut Context<'a>, layout_interner: &mut STLayoutInterner<'a>, layout: InLayout<'a>, structure: Symbol, ) -> Stmt<'a> { let rc_ptr = root.create_symbol(ident_ids, "rc_ptr"); let rc = root.create_symbol(ident_ids, "rc"); let refcount_1 = root.create_symbol(ident_ids, "refcount_1"); let is_unique = root.create_symbol(ident_ids, "is_unique"); let addr = root.create_symbol(ident_ids, "addr"); let union_layout = match layout_interner.get_repr(layout) { LayoutRepr::Union(u) => u, _ => unimplemented!("Reset is only implemented for UnionLayout"), }; // Whenever we recurse into a child layout we will want to Decrement ctx.op = HelperOp::Dec; ctx.recursive_union = Some(union_layout); let recursion_ptr = layout_interner.insert_direct_no_semantic(LayoutRepr::RecursivePointer(layout)); // Reset structure is unique. Decrement its children and return a pointer to the allocation. let then_stmt = { use UnionLayout::*; let tag_layouts; let mut null_id = None; match union_layout { NonRecursive(tags) => { tag_layouts = tags; } Recursive(tags) => { tag_layouts = tags; } NonNullableUnwrapped(field_layouts) => { tag_layouts = root.arena.alloc([field_layouts]); } NullableWrapped { other_tags: tags, nullable_id, } => { null_id = Some(nullable_id); tag_layouts = tags; } NullableUnwrapped { other_fields, nullable_id, } => { null_id = Some(nullable_id as TagIdIntType); tag_layouts = root.arena.alloc([other_fields]); } }; let tag_id_layout = union_layout.tag_id_layout(); let tag_id_sym = root.create_symbol(ident_ids, "tag_id"); let tag_id_stmt = |next| { Stmt::Let( tag_id_sym, Expr::GetTagId { structure, union_layout, }, tag_id_layout, next, ) }; let rc_contents_stmt = refcount_union_contents( root, ident_ids, ctx, layout_interner, union_layout, tag_layouts, null_id, structure, tag_id_sym, tag_id_layout, Stmt::Ret(addr), ); tag_id_stmt(root.arena.alloc( // rc_contents_stmt, )) }; // Reset structure is not unique. Decrement it and return a NULL pointer. let else_stmt = { let decrement_unit = root.create_symbol(ident_ids, "decrement_unit"); let decrement_expr = root .call_specialized_op( ident_ids, ctx, layout_interner, layout, root.arena.alloc([structure]), ) .unwrap(); let decrement_stmt = |next| Stmt::Let(decrement_unit, decrement_expr, LAYOUT_UNIT, next); // Null pointer with union layout let null = root.create_symbol(ident_ids, "null"); let null_stmt = |next| Stmt::Let(null, Expr::NullPointer, layout, next); decrement_stmt(root.arena.alloc( // null_stmt(root.arena.alloc( // Stmt::Ret(null), )), )) }; let if_stmt = Stmt::Switch { cond_symbol: is_unique, cond_layout: LAYOUT_BOOL, branches: root.arena.alloc([(1, BranchInfo::None, then_stmt)]), default_branch: (BranchInfo::None, root.arena.alloc(else_stmt)), ret_layout: layout, }; // Uniqueness test let is_unique_stmt = { let_lowlevel( root.arena, LAYOUT_BOOL, is_unique, Eq, &[rc, refcount_1], root.arena.alloc(if_stmt), ) }; // Constant for unique refcount let refcount_1_encoded = match root.target_info.ptr_width() { PtrWidth::Bytes4 => i32::MIN as i128, PtrWidth::Bytes8 => i64::MIN as i128, } .to_ne_bytes(); let refcount_1_expr = Expr::Literal(Literal::Int(refcount_1_encoded)); let refcount_1_stmt = Stmt::Let( refcount_1, refcount_1_expr, root.layout_isize, root.arena.alloc(is_unique_stmt), ); // Refcount value let rc_expr = Expr::ptr_load(root.arena.alloc(rc_ptr)); let rc_stmt = Stmt::Let( rc, rc_expr, root.layout_isize, root.arena.alloc(refcount_1_stmt), ); let mask_lower_bits = match layout_interner.get_repr(layout) { LayoutRepr::Union(ul) => ul.stores_tag_id_in_pointer(root.target_info), _ => false, }; // Refcount pointer let rc_ptr_stmt = { rc_ptr_from_data_ptr_help( root, ident_ids, structure, rc_ptr, mask_lower_bits, root.arena.alloc(rc_stmt), addr, recursion_ptr, ) }; rc_ptr_stmt } pub fn refcount_resetref_proc_body<'a>( root: &mut CodeGenHelp<'a>, ident_ids: &mut IdentIds, ctx: &mut Context<'a>, layout_interner: &mut STLayoutInterner<'a>, layout: InLayout<'a>, structure: Symbol, ) -> Stmt<'a> { let rc_ptr = root.create_symbol(ident_ids, "rc_ptr"); let rc = root.create_symbol(ident_ids, "rc"); let refcount_1 = root.create_symbol(ident_ids, "refcount_1"); let is_unique = root.create_symbol(ident_ids, "is_unique"); let addr = root.create_symbol(ident_ids, "addr"); let union_layout = match layout_interner.get_repr(layout) { LayoutRepr::Union(u) => u, _ => unimplemented!("Resetref is only implemented for UnionLayout"), }; // Whenever we recurse into a child layout we will want to Decrement ctx.op = HelperOp::Dec; ctx.recursive_union = Some(union_layout); let recursion_ptr = layout_interner.insert_direct_no_semantic(LayoutRepr::RecursivePointer(layout)); // Reset structure is unique. Return a pointer to the allocation. let then_stmt = Stmt::Ret(addr); // Reset structure is not unique. Decrement it and return a NULL pointer. let else_stmt = { // Set up the context for a decref. let jp_decref = JoinPointId(root.create_symbol(ident_ids, "jp_decref")); ctx.op = HelperOp::DecRef(jp_decref); // Generate the decref code. let rc_stmt = refcount_generic(root, ident_ids, ctx, layout_interner, layout, structure); // Null pointer with union layout let null = root.create_symbol(ident_ids, "null"); let null_stmt = |next| Stmt::Let(null, Expr::NullPointer, layout, next); // Inline the refcounting code instead of making a function. Don't iterate fields, // and replace any return statements with jumps to the `following` statement. let join = Stmt::Join { id: jp_decref, parameters: &[], body: root.arena.alloc(root.arena.alloc( // null_stmt(root.arena.alloc( // Stmt::Ret(null), )), )), remainder: root.arena.alloc(rc_stmt), }; root.arena.alloc(join) }; let if_stmt = Stmt::if_then_else( root.arena, is_unique, layout, then_stmt, root.arena.alloc(else_stmt), ); // Uniqueness test let is_unique_stmt = { let_lowlevel( root.arena, LAYOUT_BOOL, is_unique, Eq, &[rc, refcount_1], root.arena.alloc(if_stmt), ) }; // Constant for unique refcount let refcount_1_encoded = match root.target_info.ptr_width() { PtrWidth::Bytes4 => i32::MIN as i128, PtrWidth::Bytes8 => i64::MIN as i128, } .to_ne_bytes(); let refcount_1_expr = Expr::Literal(Literal::Int(refcount_1_encoded)); let refcount_1_stmt = Stmt::Let( refcount_1, refcount_1_expr, root.layout_isize, root.arena.alloc(is_unique_stmt), ); // Refcount value let rc_expr = Expr::ptr_load(root.arena.alloc(rc_ptr)); let rc_stmt = Stmt::Let( rc, rc_expr, root.layout_isize, root.arena.alloc(refcount_1_stmt), ); let mask_lower_bits = match layout_interner.get_repr(layout) { LayoutRepr::Union(ul) => ul.stores_tag_id_in_pointer(root.target_info), _ => false, }; // Refcount pointer let rc_ptr_stmt = { rc_ptr_from_data_ptr_help( root, ident_ids, structure, rc_ptr, mask_lower_bits, root.arena.alloc(rc_stmt), addr, recursion_ptr, ) }; rc_ptr_stmt } fn rc_return_stmt<'a>( root: &CodeGenHelp<'a>, ident_ids: &mut IdentIds, ctx: &mut Context<'a>, ) -> Stmt<'a> { if let HelperOp::DecRef(jp_decref) = ctx.op { Stmt::Jump(jp_decref, &[]) } else { let unit = root.create_symbol(ident_ids, "unit"); let ret_stmt = root.arena.alloc(Stmt::Ret(unit)); Stmt::Let(unit, Expr::Struct(&[]), LAYOUT_UNIT, ret_stmt) } } fn refcount_args<'a>(root: &CodeGenHelp<'a>, ctx: &Context<'a>, structure: Symbol) -> &'a [Symbol] { if ctx.op == HelperOp::Inc { // second argument is always `amount`, passed down through the call stack root.arena.alloc([structure, Symbol::ARG_2]) } else { root.arena.alloc([structure]) } } fn rc_ptr_from_data_ptr_help<'a>( root: &CodeGenHelp<'a>, ident_ids: &mut IdentIds, structure: Symbol, rc_ptr_sym: Symbol, mask_lower_bits: bool, following: &'a Stmt<'a>, addr_sym: Symbol, recursion_ptr: InLayout<'a>, ) -> Stmt<'a> { // symbol of a pointer with any tag id bits cleared let cleared_sym = if mask_lower_bits { root.create_symbol(ident_ids, "cleared") } else { structure }; let clear_tag_id_expr = Expr::Call(Call { call_type: CallType::LowLevel { op: LowLevel::PtrClearTagId, update_mode: UpdateModeId::BACKEND_DUMMY, }, arguments: root.arena.alloc([structure]), }); let clear_tag_id_stmt = |next| Stmt::Let(cleared_sym, clear_tag_id_expr, root.layout_isize, next); // Typecast the structure pointer to an integer // Backends expect a number Layout to choose the right "subtract" instruction let as_int_expr = Expr::Call(Call { call_type: CallType::LowLevel { op: LowLevel::PtrCast, update_mode: UpdateModeId::BACKEND_DUMMY, }, arguments: root.arena.alloc([cleared_sym]), }); let as_int_stmt = |next| Stmt::Let(addr_sym, as_int_expr, root.layout_isize, next); // Pointer size constant let ptr_size_sym = root.create_symbol(ident_ids, "ptr_size"); let ptr_size_expr = Expr::Literal(Literal::Int( (root.target_info.ptr_width() as i128).to_ne_bytes(), )); let ptr_size_stmt = |next| Stmt::Let(ptr_size_sym, ptr_size_expr, root.layout_isize, next); // Refcount address let rc_addr_sym = root.create_symbol(ident_ids, "rc_addr"); let sub_expr = Expr::Call(Call { call_type: CallType::LowLevel { op: LowLevel::NumSubSaturated, update_mode: UpdateModeId::BACKEND_DUMMY, }, arguments: root.arena.alloc([addr_sym, ptr_size_sym]), }); let sub_stmt = |next| Stmt::Let(rc_addr_sym, sub_expr, Layout::usize(root.target_info), next); // Typecast the refcount address from integer to pointer let cast_expr = Expr::Call(Call { call_type: CallType::LowLevel { op: LowLevel::PtrCast, update_mode: UpdateModeId::BACKEND_DUMMY, }, arguments: root.arena.alloc([rc_addr_sym]), }); let cast_stmt = |next| Stmt::Let(rc_ptr_sym, cast_expr, recursion_ptr, next); let body = as_int_stmt(root.arena.alloc( // ptr_size_stmt(root.arena.alloc( // sub_stmt(root.arena.alloc( // cast_stmt(root.arena.alloc( // following, )), )), )), )); if mask_lower_bits { clear_tag_id_stmt(root.arena.alloc(body)) } else { body } } enum Pointer { ToData(Symbol), #[allow(unused)] ToRefcount(Symbol), } fn modify_refcount<'a>( root: &CodeGenHelp<'a>, ident_ids: &mut IdentIds, ctx: &mut Context<'a>, ptr: Pointer, alignment: u32, following: &'a Stmt<'a>, ) -> Stmt<'a> { // Call the relevant Zig lowlevel to actually modify the refcount let zig_call_result = root.create_symbol(ident_ids, "zig_call_result"); match ctx.op { HelperOp::Inc => { let (op, ptr) = match ptr { Pointer::ToData(s) => (LowLevel::RefCountIncDataPtr, s), Pointer::ToRefcount(s) => (LowLevel::RefCountIncRcPtr, s), }; let zig_call_expr = Expr::Call(Call { call_type: CallType::LowLevel { op, update_mode: UpdateModeId::BACKEND_DUMMY, }, arguments: root.arena.alloc([ptr, Symbol::ARG_2]), }); Stmt::Let(zig_call_result, zig_call_expr, LAYOUT_UNIT, following) } HelperOp::Dec | HelperOp::DecRef(_) => { debug_assert!(alignment >= root.target_info.ptr_width() as u32); let (op, ptr) = match ptr { Pointer::ToData(s) => (LowLevel::RefCountDecDataPtr, s), Pointer::ToRefcount(s) => (LowLevel::RefCountDecRcPtr, s), }; let alignment_sym = root.create_symbol(ident_ids, "alignment"); let alignment_expr = Expr::Literal(Literal::Int((alignment as i128).to_ne_bytes())); let alignment_stmt = |next| Stmt::Let(alignment_sym, alignment_expr, LAYOUT_U32, next); let zig_call_expr = Expr::Call(Call { call_type: CallType::LowLevel { op, update_mode: UpdateModeId::BACKEND_DUMMY, }, arguments: root.arena.alloc([ptr, alignment_sym]), }); let zig_call_stmt = Stmt::Let(zig_call_result, zig_call_expr, LAYOUT_UNIT, following); alignment_stmt(root.arena.alloc( // zig_call_stmt, )) } _ => unreachable!(), } } /// Generate a procedure to modify the reference count of a Str fn refcount_str<'a>( root: &CodeGenHelp<'a>, ident_ids: &mut IdentIds, ctx: &mut Context<'a>, ) -> Stmt<'a> { let arena = root.arena; let string = Symbol::ARG_1; let layout_isize = root.layout_isize; let field_layouts = arena.alloc([Layout::OPAQUE_PTR, layout_isize, layout_isize]); // Get the last word as a signed int let last_word = root.create_symbol(ident_ids, "last_word"); let last_word_expr = Expr::StructAtIndex { index: 2, field_layouts, structure: string, }; let last_word_stmt = |next| Stmt::Let(last_word, last_word_expr, layout_isize, next); // Zero let zero = root.create_symbol(ident_ids, "zero"); let zero_expr = Expr::Literal(Literal::Int(0i128.to_ne_bytes())); let zero_stmt = |next| Stmt::Let(zero, zero_expr, layout_isize, next); // is_big_str = (last_word >= 0); // Treat last word as isize so that the small string flag is the same as the sign bit // (assuming a little-endian target, where the sign bit is in the last byte of the word) let is_big_str = root.create_symbol(ident_ids, "is_big_str"); let is_big_str_stmt = |next| { let_lowlevel( arena, LAYOUT_BOOL, is_big_str, NumGte, &[last_word, zero], next, ) }; // // Check for seamless slice // // Get the length field as a signed int let length = root.create_symbol(ident_ids, "length"); let length_expr = Expr::StructAtIndex { index: 1, field_layouts, structure: string, }; let length_stmt = |next| Stmt::Let(length, length_expr, layout_isize, next); let alignment = root.target_info.ptr_width() as u32; // let is_slice = lowlevel NumLt length zero let is_slice = root.create_symbol(ident_ids, "is_slice"); let is_slice_stmt = |next| let_lowlevel(arena, LAYOUT_BOOL, is_slice, NumLt, &[length, zero], next); // // Branch on seamless slice vs "real" string // let return_unit = arena.alloc(rc_return_stmt(root, ident_ids, ctx)); let one = root.create_symbol(ident_ids, "one"); let one_expr = Expr::Literal(Literal::Int(1i128.to_ne_bytes())); let one_stmt = |next| Stmt::Let(one, one_expr, layout_isize, next); let data_ptr_int = root.create_symbol(ident_ids, "data_ptr_int"); let data_ptr_int_stmt = |next| { let_lowlevel( arena, layout_isize, data_ptr_int, PtrCast, &[last_word], next, ) }; let data_ptr = root.create_symbol(ident_ids, "data_ptr"); let data_ptr_stmt = |next| { let_lowlevel( arena, layout_isize, data_ptr, NumShiftLeftBy, &[data_ptr_int, one], next, ) }; // when the string is a slice, the capacity field is a pointer to the refcount let slice_branch = one_stmt(arena.alloc( // data_ptr_int_stmt(arena.alloc( // data_ptr_stmt(arena.alloc( // modify_refcount( root, ident_ids, ctx, Pointer::ToData(data_ptr), alignment, return_unit, ), )), )), )); // Characters pointer for a real string let string_chars = root.create_symbol(ident_ids, "string_chars"); let string_chars_expr = Expr::StructAtIndex { index: 0, field_layouts, structure: string, }; let string_chars_stmt = |next| Stmt::Let(string_chars, string_chars_expr, layout_isize, next); let modify_refcount_stmt = modify_refcount( root, ident_ids, ctx, Pointer::ToData(string_chars), alignment, return_unit, ); let string_branch = arena.alloc( // string_chars_stmt(arena.alloc( // modify_refcount_stmt, )), ); let if_slice = Stmt::if_then_else( root.arena, is_slice, Layout::UNIT, slice_branch, string_branch, ); // // JoinPoint for slice vs list // let modify_stmt = length_stmt(arena.alloc( // is_slice_stmt(arena.alloc( // if_slice, )), )); let if_big_stmt = Stmt::if_then_else( root.arena, is_big_str, Layout::UNIT, modify_stmt, root.arena.alloc(rc_return_stmt(root, ident_ids, ctx)), ); // Combine the statements in sequence last_word_stmt(arena.alloc( // zero_stmt(arena.alloc( // is_big_str_stmt(arena.alloc( // if_big_stmt, )), )), )) } fn refcount_list<'a>( root: &mut CodeGenHelp<'a>, ident_ids: &mut IdentIds, ctx: &mut Context<'a>, layout_interner: &mut STLayoutInterner<'a>, elem_layout: InLayout<'a>, structure: Symbol, ) -> Stmt<'a> { let layout_isize = root.layout_isize; let arena = root.arena; // A "Ptr" layout (heap pointer to a single list element) let ptr_layout = layout_interner.insert_direct_no_semantic(LayoutRepr::Ptr(elem_layout)); // // Check if the list is empty // let len = root.create_symbol(ident_ids, "len"); let len_stmt = |next| let_lowlevel(arena, layout_isize, len, ListLenUsize, &[structure], next); // let zero = 0 let zero = root.create_symbol(ident_ids, "zero"); let zero_expr = Expr::Literal(Literal::Int(0i128.to_ne_bytes())); let zero_stmt = |next| Stmt::Let(zero, zero_expr, layout_isize, next); // let is_empty = lowlevel Eq len zero let is_empty = root.create_symbol(ident_ids, "is_empty"); let is_empty_stmt = |next| let_lowlevel(arena, LAYOUT_BOOL, is_empty, Eq, &[len, zero], next); // // Check for seamless slice // // let capacity = StructAtIndex 2 structure let capacity = root.create_symbol(ident_ids, "capacity"); let list_field_layouts = arena.alloc([ptr_layout, layout_isize, layout_isize]); let capacity_expr = Expr::StructAtIndex { index: 2, field_layouts: list_field_layouts, structure, }; let capacity_stmt = |next| Stmt::Let(capacity, capacity_expr, layout_isize, next); // let is_slice = lowlevel NumLt capacity zero let is_slice = root.create_symbol(ident_ids, "is_slice"); let is_slice_stmt = |next| let_lowlevel(arena, LAYOUT_BOOL, is_slice, NumLt, &[capacity, zero], next); // // Branch on slice vs list // // let first_element = StructAtIndex 0 structure let first_element = root.create_symbol(ident_ids, "first_element"); let first_element_expr = Expr::StructAtIndex { index: 0, field_layouts: list_field_layouts, structure, }; let first_element_stmt = |next| Stmt::Let(first_element, first_element_expr, ptr_layout, next); let jp_elements = JoinPointId(root.create_symbol(ident_ids, "jp_elements")); let data_pointer = root.create_symbol(ident_ids, "data_pointer"); let param_data_pointer = Param { symbol: data_pointer, layout: Layout::OPAQUE_PTR, }; let first_element_pointer = root.create_symbol(ident_ids, "first_element_pointer"); let param_first_element_pointer = Param { symbol: first_element_pointer, layout: Layout::OPAQUE_PTR, }; // one = 1 let one = root.create_symbol(ident_ids, "one"); let one_expr = Expr::Literal(Literal::Int(1i128.to_ne_bytes())); let one_stmt = |next| Stmt::Let(one, one_expr, layout_isize, next); let slice_data_pointer = root.create_symbol(ident_ids, "slice_data_pointer"); let slice_data_pointer_stmt = move |next| { one_stmt(arena.alloc( // let_lowlevel( arena, layout_isize, slice_data_pointer, LowLevel::NumShiftLeftBy, &[capacity, one], arena.alloc(next), ), )) }; let slice_branch = slice_data_pointer_stmt( // Stmt::Jump( jp_elements, arena.alloc([slice_data_pointer, first_element]), ), ); let list_branch = arena.alloc( // Stmt::Jump(jp_elements, arena.alloc([first_element, first_element])), ); let switch_slice_list = arena.alloc(first_element_stmt(arena.alloc( // Stmt::if_then_else( root.arena, is_slice, Layout::UNIT, slice_branch, arena.alloc(list_branch), ), ))); // // modify refcount of the list and its elements // (elements first, to avoid use-after-free for when decrementing) // let alignment = Ord::max( root.target_info.ptr_width() as u32, layout_interner.alignment_bytes(elem_layout), ); let ret_stmt = arena.alloc(rc_return_stmt(root, ident_ids, ctx)); let mut modify_refcount_stmt = |ptr| modify_refcount(root, ident_ids, ctx, ptr, alignment, ret_stmt); let modify_list = modify_refcount_stmt(Pointer::ToData(data_pointer)); let is_relevant_op = ctx.op.is_dec() || ctx.op.is_inc(); let modify_elems_and_list = if is_relevant_op && layout_interner.is_refcounted(elem_layout) { refcount_list_elems( root, ident_ids, ctx, layout_interner, elem_layout, LAYOUT_UNIT, ptr_layout, len, first_element_pointer, modify_list, ) } else { modify_list }; // // JoinPoint for slice vs list // let joinpoint_elems = Stmt::Join { id: jp_elements, parameters: arena.alloc([param_data_pointer, param_first_element_pointer]), body: arena.alloc(modify_elems_and_list), remainder: arena.alloc(switch_slice_list), }; // // Do nothing if the list is empty // let non_empty_branch = arena.alloc( // capacity_stmt(arena.alloc( // is_slice_stmt(arena.alloc( // joinpoint_elems, )), )), ); let if_empty_stmt = Stmt::if_then_else( root.arena, is_empty, Layout::UNIT, rc_return_stmt(root, ident_ids, ctx), non_empty_branch, ); len_stmt(arena.alloc( // zero_stmt(arena.alloc( // is_empty_stmt(arena.alloc( // if_empty_stmt, )), )), )) } fn refcount_list_elems<'a>( root: &mut CodeGenHelp<'a>, ident_ids: &mut IdentIds, ctx: &mut Context<'a>, layout_interner: &mut STLayoutInterner<'a>, elem_layout: InLayout<'a>, ret_layout: InLayout<'a>, ptr_layout: InLayout<'a>, length: Symbol, elements: Symbol, following: Stmt<'a>, ) -> Stmt<'a> { use LowLevel::*; let layout_isize = root.layout_isize; let arena = root.arena; // Cast to integer let start = root.create_symbol(ident_ids, "start"); let start_stmt = |next| let_lowlevel(arena, layout_isize, start, PtrCast, &[elements], next); // // Loop initialisation // // let size = literal int let elem_size = root.create_symbol(ident_ids, "elem_size"); let elem_size_expr = Expr::Literal(Literal::Int( (layout_interner.stack_size(elem_layout) as i128).to_ne_bytes(), )); let elem_size_stmt = |next| Stmt::Let(elem_size, elem_size_expr, layout_isize, next); // let list_size = len * size let list_size = root.create_symbol(ident_ids, "list_size"); let list_size_stmt = |next| { let_lowlevel( arena, layout_isize, list_size, NumMul, &[length, elem_size], next, ) }; // let end = start + list_size let end = root.create_symbol(ident_ids, "end"); let end_stmt = |next| let_lowlevel(arena, layout_isize, end, NumAdd, &[start, list_size], next); // // Loop name & parameter // let elems_loop = JoinPointId(root.create_symbol(ident_ids, "elems_loop")); let addr = root.create_symbol(ident_ids, "addr"); let param_addr = Param { symbol: addr, layout: layout_isize, }; // // if we haven't reached the end yet... // // Cast integer to pointer let ptr_symbol = root.create_symbol(ident_ids, "ptr"); let ptr_stmt = |next| let_lowlevel(arena, ptr_layout, ptr_symbol, PtrCast, &[addr], next); // Dereference the pointer to get the current element let elem = root.create_symbol(ident_ids, "elem"); let elem_expr = Expr::ptr_load(arena.alloc(ptr_symbol)); let elem_stmt = |next| Stmt::Let(elem, elem_expr, elem_layout, next); // // Modify element refcount // let mod_elem_unit = root.create_symbol(ident_ids, "mod_elem_unit"); let mod_elem_args = refcount_args(root, ctx, elem); let mod_elem_expr = root .call_specialized_op(ident_ids, ctx, layout_interner, elem_layout, mod_elem_args) .unwrap(); let mod_elem_stmt = |next| Stmt::Let(mod_elem_unit, mod_elem_expr, LAYOUT_UNIT, next); // // Next loop iteration // let next_addr = root.create_symbol(ident_ids, "next_addr"); let next_addr_stmt = |next| { // let_lowlevel( arena, layout_isize, next_addr, NumAddSaturated, &[addr, elem_size], next, ) }; // // Control flow // let is_end = root.create_symbol(ident_ids, "is_end"); let is_end_stmt = |next| let_lowlevel(arena, LAYOUT_BOOL, is_end, NumGte, &[addr, end], next); let if_end_of_list = Stmt::if_then_else( arena, is_end, ret_layout, following, arena.alloc(ptr_stmt(arena.alloc( // elem_stmt(arena.alloc( // mod_elem_stmt(arena.alloc( // next_addr_stmt(arena.alloc( // Stmt::Jump(elems_loop, arena.alloc([next_addr])), )), )), )), ))), ); let joinpoint_loop = Stmt::Join { id: elems_loop, parameters: arena.alloc([param_addr]), body: arena.alloc( // is_end_stmt( // arena.alloc(if_end_of_list), ), ), remainder: root .arena .alloc(Stmt::Jump(elems_loop, arena.alloc([start, end]))), }; start_stmt(arena.alloc( // elem_size_stmt(arena.alloc( // list_size_stmt(arena.alloc( // end_stmt(arena.alloc( // joinpoint_loop, )), )), )), )) } fn refcount_struct<'a>( root: &mut CodeGenHelp<'a>, ident_ids: &mut IdentIds, ctx: &mut Context<'a>, layout_interner: &mut STLayoutInterner<'a>, field_layouts: &'a [InLayout<'a>], structure: Symbol, ) -> Stmt<'a> { let mut stmt = rc_return_stmt(root, ident_ids, ctx); for (i, field_layout) in field_layouts.iter().enumerate().rev() { if layout_interner.contains_refcounted(*field_layout) { let field_val = root.create_symbol(ident_ids, &format!("field_val_{i}")); let field_val_expr = Expr::StructAtIndex { index: i as u64, field_layouts, structure, }; let field_val_stmt = |next| Stmt::Let(field_val, field_val_expr, *field_layout, next); let mod_unit = root.create_symbol(ident_ids, &format!("mod_field_{i}")); let mod_args = refcount_args(root, ctx, field_val); let mod_expr = root .call_specialized_op(ident_ids, ctx, layout_interner, *field_layout, mod_args) .unwrap(); let mod_stmt = |next| Stmt::Let(mod_unit, mod_expr, LAYOUT_UNIT, next); stmt = field_val_stmt(root.arena.alloc( // mod_stmt(root.arena.alloc( // stmt, )), )) } } stmt } fn refcount_union<'a>( root: &mut CodeGenHelp<'a>, ident_ids: &mut IdentIds, ctx: &mut Context<'a>, layout_interner: &mut STLayoutInterner<'a>, union_in_layout: InLayout<'a>, union: UnionLayout<'a>, structure: Symbol, ) -> Stmt<'a> { use UnionLayout::*; let parent_rec_ptr_layout = ctx.recursive_union; if !matches!(union, NonRecursive(_)) { ctx.recursive_union = Some(union); } let body = match union { NonRecursive(tags) => refcount_union_nonrec( root, ident_ids, ctx, layout_interner, union, tags, structure, ), Recursive(tags) => { let tailrec_idx = root.union_tail_recursion_fields(union_in_layout, union); if let (Some(tail_idx), true) = (tailrec_idx, ctx.op.is_dec()) { refcount_union_tailrec( root, ident_ids, ctx, layout_interner, union, tags, None, tail_idx, structure, ) } else { refcount_union_rec( root, ident_ids, ctx, layout_interner, union, tags, None, structure, ) } } NonNullableUnwrapped(field_layouts) => { // We don't do tail recursion on NonNullableUnwrapped. // Its RecursionPointer is always nested inside a List, Option, or other sub-layout, since // a direct RecursionPointer is only possible if there's at least one non-recursive variant. // This nesting makes it harder to do tail recursion, so we just don't. let tags = root.arena.alloc([field_layouts]); refcount_union_rec( root, ident_ids, ctx, layout_interner, union, tags, None, structure, ) } NullableWrapped { other_tags: tags, nullable_id, } => { let null_id = Some(nullable_id); let tailrec_idx = root.union_tail_recursion_fields(union_in_layout, union); if let (Some(tail_idx), true) = (tailrec_idx, ctx.op.is_dec()) { refcount_union_tailrec( root, ident_ids, ctx, layout_interner, union, tags, null_id, tail_idx, structure, ) } else { refcount_union_rec( root, ident_ids, ctx, layout_interner, union, tags, null_id, structure, ) } } NullableUnwrapped { other_fields, nullable_id, } => { let null_id = Some(nullable_id as TagIdIntType); let tags = root.arena.alloc([other_fields]); let tailrec_idx = root.union_tail_recursion_fields(union_in_layout, union); if let (Some(tail_idx), true) = (tailrec_idx, ctx.op.is_dec()) { refcount_union_tailrec( root, ident_ids, ctx, layout_interner, union, tags, null_id, tail_idx, structure, ) } else { refcount_union_rec( root, ident_ids, ctx, layout_interner, union, tags, null_id, structure, ) } } }; ctx.recursive_union = parent_rec_ptr_layout; body } fn refcount_union_nonrec<'a>( root: &mut CodeGenHelp<'a>, ident_ids: &mut IdentIds, ctx: &mut Context<'a>, layout_interner: &mut STLayoutInterner<'a>, union_layout: UnionLayout<'a>, tag_layouts: &'a [&'a [InLayout<'a>]], structure: Symbol, ) -> Stmt<'a> { let tag_id_layout = union_layout.tag_id_layout(); let tag_id_sym = root.create_symbol(ident_ids, "tag_id"); let tag_id_stmt = |next| { Stmt::Let( tag_id_sym, Expr::GetTagId { structure, union_layout, }, tag_id_layout, next, ) }; let continuation = rc_return_stmt(root, ident_ids, ctx); if tag_layouts.is_empty() { continuation } else { let switch_stmt = refcount_union_contents( root, ident_ids, ctx, layout_interner, union_layout, tag_layouts, None, structure, tag_id_sym, tag_id_layout, continuation, ); tag_id_stmt(root.arena.alloc( // switch_stmt, )) } } fn refcount_union_contents<'a>( root: &mut CodeGenHelp<'a>, ident_ids: &mut IdentIds, ctx: &mut Context<'a>, layout_interner: &mut STLayoutInterner<'a>, union_layout: UnionLayout<'a>, tag_layouts: &'a [&'a [InLayout<'a>]], null_id: Option, structure: Symbol, tag_id_sym: Symbol, tag_id_layout: InLayout<'a>, next_stmt: Stmt<'a>, ) -> Stmt<'a> { let jp_contents_modified = JoinPointId(root.create_symbol(ident_ids, "jp_contents_modified")); let mut tag_branches = Vec::with_capacity_in(tag_layouts.len() + 1, root.arena); if let Some(id) = null_id { let ret = rc_return_stmt(root, ident_ids, ctx); tag_branches.push((id as u64, BranchInfo::None, ret)); }; for (field_layouts, tag_id) in tag_layouts .iter() .zip((0..).filter(|tag_id| !matches!(null_id, Some(id) if tag_id == &id))) { // After refcounting the fields, jump to modify the union itself // (Order is important, to avoid use-after-free for Dec) let following = Stmt::Jump(jp_contents_modified, &[]); let field_layouts = field_layouts .iter() .copied() .enumerate() .collect_in::>(root.arena) .into_bump_slice(); let fields_stmt = refcount_tag_fields( root, ident_ids, ctx, layout_interner, union_layout, field_layouts, structure, tag_id, following, ); tag_branches.push((tag_id as u64, BranchInfo::None, fields_stmt)); } let default_stmt: Stmt<'a> = tag_branches.pop().unwrap().2; let tag_id_switch = Stmt::Switch { cond_symbol: tag_id_sym, cond_layout: tag_id_layout, branches: tag_branches.into_bump_slice(), default_branch: (BranchInfo::None, root.arena.alloc(default_stmt)), ret_layout: LAYOUT_UNIT, }; if let UnionLayout::NonRecursive(_) = union_layout { Stmt::Join { id: jp_contents_modified, parameters: &[], body: root.arena.alloc(next_stmt), remainder: root.arena.alloc(tag_id_switch), } } else { let is_unique = root.create_symbol(ident_ids, "is_unique"); let switch_with_unique_check = Stmt::if_then_else( root.arena, is_unique, Layout::UNIT, tag_id_switch, root.arena.alloc(Stmt::Jump(jp_contents_modified, &[])), ); let switch_with_unique_check_and_let = let_lowlevel( root.arena, Layout::BOOL, is_unique, LowLevel::RefCountIsUnique, &[structure], root.arena.alloc(switch_with_unique_check), ); Stmt::Join { id: jp_contents_modified, parameters: &[], body: root.arena.alloc(next_stmt), remainder: root.arena.alloc(switch_with_unique_check_and_let), } } } fn refcount_union_rec<'a>( root: &mut CodeGenHelp<'a>, ident_ids: &mut IdentIds, ctx: &mut Context<'a>, layout_interner: &mut STLayoutInterner<'a>, union_layout: UnionLayout<'a>, tag_layouts: &'a [&'a [InLayout<'a>]], null_id: Option, structure: Symbol, ) -> Stmt<'a> { let tag_id_layout = union_layout.tag_id_layout(); let tag_id_sym = root.create_symbol(ident_ids, "tag_id"); let tag_id_stmt = |next| { Stmt::Let( tag_id_sym, Expr::GetTagId { structure, union_layout, }, tag_id_layout, next, ) }; let rc_structure_stmt = { let alignment = LayoutRepr::Union(union_layout).allocation_alignment_bytes(layout_interner); let ret_stmt = rc_return_stmt(root, ident_ids, ctx); modify_refcount( root, ident_ids, ctx, Pointer::ToData(structure), alignment, root.arena.alloc(ret_stmt), ) }; let rc_contents_then_structure = if ctx.op.is_dec() { refcount_union_contents( root, ident_ids, ctx, layout_interner, union_layout, tag_layouts, null_id, structure, tag_id_sym, tag_id_layout, rc_structure_stmt, ) } else { rc_structure_stmt }; if ctx.op.is_decref() && null_id.is_none() { rc_contents_then_structure } else { tag_id_stmt(root.arena.alloc( // rc_contents_then_structure, )) } } // Refcount a recursive union using tail-call elimination to limit stack growth fn refcount_union_tailrec<'a>( root: &mut CodeGenHelp<'a>, ident_ids: &mut IdentIds, ctx: &mut Context<'a>, layout_interner: &mut STLayoutInterner<'a>, union_layout: UnionLayout<'a>, tag_layouts: &'a [&'a [InLayout<'a>]], null_id: Option, tailrec_indices: Vec<'a, Option>, initial_structure: Symbol, ) -> Stmt<'a> { let tailrec_loop = JoinPointId(root.create_symbol(ident_ids, "tailrec_loop")); let current = root.create_symbol(ident_ids, "current"); let next_ptr = root.create_symbol(ident_ids, "next_ptr"); let layout = layout_interner.insert_direct_no_semantic(LayoutRepr::Union(union_layout)); let tag_id_layout = union_layout.tag_id_layout(); let tag_id_sym = root.create_symbol(ident_ids, "tag_id"); let tag_id_stmt = |next| { Stmt::Let( tag_id_sym, Expr::GetTagId { structure: current, union_layout, }, tag_id_layout, next, ) }; // Do refcounting on the structure itself // In the control flow, this comes *after* refcounting the fields // It receives a `next` parameter to pass through to the outer joinpoint let rc_structure_stmt = { let next_addr = root.create_symbol(ident_ids, "next_addr"); let exit_stmt = rc_return_stmt(root, ident_ids, ctx); let jump_to_loop = Stmt::Jump(tailrec_loop, root.arena.alloc([next_ptr])); let loop_or_exit = Stmt::Switch { cond_symbol: next_addr, cond_layout: root.layout_isize, branches: root.arena.alloc([(0, BranchInfo::None, exit_stmt)]), default_branch: (BranchInfo::None, root.arena.alloc(jump_to_loop)), ret_layout: LAYOUT_UNIT, }; let loop_or_exit_based_on_next_addr = { let_lowlevel( root.arena, root.layout_isize, next_addr, PtrCast, &[next_ptr], root.arena.alloc(loop_or_exit), ) }; let alignment = layout_interner.allocation_alignment_bytes(layout); modify_refcount( root, ident_ids, ctx, Pointer::ToData(current), alignment, root.arena.alloc(loop_or_exit_based_on_next_addr), ) }; let rc_contents_then_structure = { let jp_modify_union = JoinPointId(root.create_symbol(ident_ids, "jp_modify_union")); let mut tag_branches = Vec::with_capacity_in(tag_layouts.len() + 1, root.arena); // If this is null, there is no refcount, no `next`, no fields. Just return. if let Some(id) = null_id { let ret = rc_return_stmt(root, ident_ids, ctx); tag_branches.push((id as u64, BranchInfo::None, ret)); } for ((field_layouts, opt_tailrec_index), tag_id) in tag_layouts .iter() .zip(tailrec_indices) .zip((0..).filter(|tag_id| !matches!(null_id, Some(id) if tag_id == &id))) { // After refcounting the fields, jump to modify the union itself. // The loop param is a pointer to the next union. It gets passed through two jumps. let (non_tailrec_fields, jump_to_modify_union) = if let Some(tailrec_index) = opt_tailrec_index { let mut filtered = Vec::with_capacity_in(field_layouts.len() - 1, root.arena); let mut tail_stmt = None; for (i, field) in field_layouts.iter().enumerate() { if i != tailrec_index { filtered.push((i, *field)); } else { let field_val = root.create_symbol(ident_ids, &format!("field_{tag_id}_{i}")); let field_val_expr = Expr::UnionAtIndex { union_layout, tag_id, index: i as u64, structure: current, }; let jump_params = root.arena.alloc([field_val]); let jump = root.arena.alloc(Stmt::Jump(jp_modify_union, jump_params)); tail_stmt = Some(Stmt::Let(field_val, field_val_expr, *field, jump)); } } (filtered.into_bump_slice(), tail_stmt.unwrap()) } else { let null = root.create_symbol(ident_ids, "null"); let null_stmt = |next| Stmt::Let(null, Expr::NullPointer, layout, next); let tail_stmt = null_stmt(root.arena.alloc( // Stmt::Jump(jp_modify_union, root.arena.alloc([null])), )); let field_layouts = field_layouts .iter() .copied() .enumerate() .collect_in::>(root.arena) .into_bump_slice(); (field_layouts, tail_stmt) }; let fields_stmt = refcount_tag_fields( root, ident_ids, ctx, layout_interner, union_layout, non_tailrec_fields, current, tag_id, jump_to_modify_union, ); tag_branches.push((tag_id as u64, BranchInfo::None, fields_stmt)); } let default_stmt: Stmt<'a> = tag_branches.pop().unwrap().2; let tag_id_switch = Stmt::Switch { cond_symbol: tag_id_sym, cond_layout: tag_id_layout, branches: tag_branches.into_bump_slice(), default_branch: (BranchInfo::None, root.arena.alloc(default_stmt)), ret_layout: LAYOUT_UNIT, }; let is_unique = root.create_symbol(ident_ids, "is_unique"); let null_pointer = root.create_symbol(ident_ids, "null_pointer"); let jump_with_null_ptr = Stmt::Let( null_pointer, Expr::NullPointer, layout_interner.insert_direct_no_semantic(LayoutRepr::Union(union_layout)), root.arena.alloc(Stmt::Jump( jp_modify_union, root.arena.alloc([null_pointer]), )), ); let switch_with_unique_check = Stmt::if_then_else( root.arena, is_unique, Layout::UNIT, tag_id_switch, root.arena.alloc(jump_with_null_ptr), ); let switch_with_unique_check_and_let = let_lowlevel( root.arena, Layout::BOOL, is_unique, LowLevel::RefCountIsUnique, &[current], root.arena.alloc(switch_with_unique_check), ); let jp_param = Param { symbol: next_ptr, layout, }; Stmt::Join { id: jp_modify_union, parameters: root.arena.alloc([jp_param]), body: root.arena.alloc(rc_structure_stmt), remainder: root.arena.alloc(switch_with_unique_check_and_let), } }; let loop_body = tag_id_stmt(root.arena.alloc( // rc_contents_then_structure, )); let loop_init = Stmt::Jump(tailrec_loop, root.arena.alloc([initial_structure])); let union_layout = layout_interner.insert_direct_no_semantic(LayoutRepr::Union(union_layout)); let loop_param = Param { symbol: current, layout: union_layout, }; Stmt::Join { id: tailrec_loop, parameters: root.arena.alloc([loop_param]), body: root.arena.alloc(loop_body), remainder: root.arena.alloc(loop_init), } } fn refcount_tag_fields<'a>( root: &mut CodeGenHelp<'a>, ident_ids: &mut IdentIds, ctx: &mut Context<'a>, layout_interner: &mut STLayoutInterner<'a>, union_layout: UnionLayout<'a>, field_layouts: &'a [(usize, InLayout<'a>)], structure: Symbol, tag_id: TagIdIntType, following: Stmt<'a>, ) -> Stmt<'a> { let mut stmt = following; for (i, field_layout) in field_layouts.iter().rev() { if layout_interner.contains_refcounted(*field_layout) { let field_val = root.create_symbol(ident_ids, &format!("field_{tag_id}_{i}")); let field_val_expr = Expr::UnionAtIndex { union_layout, tag_id, index: *i as u64, structure, }; let field_val_stmt = |next| Stmt::Let(field_val, field_val_expr, *field_layout, next); let mod_unit = root.create_symbol(ident_ids, &format!("mod_field_{tag_id}_{i}")); let mod_args = refcount_args(root, ctx, field_val); let mod_expr = root .call_specialized_op(ident_ids, ctx, layout_interner, *field_layout, mod_args) .unwrap(); let mod_stmt = |next| Stmt::Let(mod_unit, mod_expr, LAYOUT_UNIT, next); stmt = field_val_stmt(root.arena.alloc( // mod_stmt(root.arena.alloc( // stmt, )), )) } } stmt }