diff --git a/compiler/gen_llvm/src/llvm/build.rs b/compiler/gen_llvm/src/llvm/build.rs index 76e2ed8b70..a6707a4884 100644 --- a/compiler/gen_llvm/src/llvm/build.rs +++ b/compiler/gen_llvm/src/llvm/build.rs @@ -893,7 +893,7 @@ pub fn build_exp_call<'a, 'ctx, 'env>( } } -pub const TAG_ID_INDEX: u32 = 1; +const TAG_ID_INDEX: u32 = 1; pub const TAG_DATA_INDEX: u32 = 0; pub fn struct_from_fields<'a, 'ctx, 'env, I>( @@ -919,6 +919,34 @@ where struct_value.into_struct_value() } +fn struct_pointer_from_fields<'a, 'ctx, 'env, I>( + env: &Env<'a, 'ctx, 'env>, + struct_type: StructType<'ctx>, + input_pointer: PointerValue<'ctx>, + values: I, +) where + I: Iterator)>, +{ + let struct_ptr = env + .builder + .build_bitcast( + input_pointer, + struct_type.ptr_type(AddressSpace::Generic), + "struct_ptr", + ) + .into_pointer_value(); + + // Insert field exprs into struct_val + for (index, field_val) in values { + let field_ptr = env + .builder + .build_struct_gep(struct_ptr, index as u32, "field_struct_gep") + .unwrap(); + + env.builder.build_store(field_ptr, field_val); + } +} + pub fn build_exp_expr<'a, 'ctx, 'env>( env: &Env<'a, 'ctx, 'env>, layout_ids: &mut LayoutIds<'a>, @@ -1010,7 +1038,8 @@ pub fn build_exp_expr<'a, 'ctx, 'env>( let else_block = ctx.append_basic_block(parent, "else_decref"); let cont_block = ctx.append_basic_block(parent, "cont"); - let refcount_ptr = PointerToRefcount::from_ptr_to_data(env, tag_ptr); + let refcount_ptr = + PointerToRefcount::from_ptr_to_data(env, tag_pointer_clear_tag_id(env, tag_ptr)); let is_unique = refcount_ptr.is_1(env); env.builder @@ -1154,13 +1183,15 @@ pub fn build_exp_expr<'a, 'ctx, 'env>( let tag_id_type = basic_type_from_layout(env, &union_layout.tag_id_layout()).into_int_type(); + let ptr = tag_pointer_clear_tag_id(env, argument.into_pointer_value()); + lookup_at_index_ptr2( env, union_layout, tag_id_type, field_layouts, *index as usize, - argument.into_pointer_value(), + ptr, ) } UnionLayout::NonNullableUnwrapped(field_layouts) => { @@ -1195,13 +1226,14 @@ pub fn build_exp_expr<'a, 'ctx, 'env>( let tag_id_type = basic_type_from_layout(env, &union_layout.tag_id_layout()).into_int_type(); + let ptr = tag_pointer_clear_tag_id(env, argument.into_pointer_value()); lookup_at_index_ptr2( env, union_layout, tag_id_type, field_layouts, *index as usize, - argument.into_pointer_value(), + ptr, ) } UnionLayout::NullableUnwrapped { @@ -1241,6 +1273,92 @@ pub fn build_exp_expr<'a, 'ctx, 'env>( } } +#[allow(clippy::too_many_arguments)] +fn build_wrapped_tag<'a, 'ctx, 'env>( + env: &Env<'a, 'ctx, 'env>, + scope: &Scope<'a, 'ctx>, + union_layout: &UnionLayout<'a>, + tag_id: u8, + arguments: &[Symbol], + tag_field_layouts: &[Layout<'a>], + tags: &[&[Layout<'a>]], + reuse_allocation: Option>, + parent: FunctionValue<'ctx>, +) -> BasicValueEnum<'ctx> { + let ctx = env.context; + let builder = env.builder; + + let tag_id_layout = union_layout.tag_id_layout(); + + // Determine types + let num_fields = arguments.len() + 1; + let mut field_types = Vec::with_capacity_in(num_fields, env.arena); + let mut field_vals = Vec::with_capacity_in(num_fields, env.arena); + + for (field_symbol, tag_field_layout) in arguments.iter().zip(tag_field_layouts.iter()) { + let (val, _val_layout) = load_symbol_and_layout(scope, field_symbol); + + let field_type = basic_type_from_layout(env, tag_field_layout); + + field_types.push(field_type); + + if let Layout::RecursivePointer = tag_field_layout { + debug_assert!(val.is_pointer_value()); + + // we store recursive pointers as `i64*` + let ptr = env.builder.build_bitcast( + val, + ctx.i64_type().ptr_type(AddressSpace::Generic), + "cast_recursive_pointer", + ); + + field_vals.push(ptr); + } else { + // this check fails for recursive tag unions, but can be helpful while debugging + // debug_assert_eq!(tag_field_layout, val_layout); + + field_vals.push(val); + } + } + + // Create the struct_type + let raw_data_ptr = allocate_tag(env, parent, reuse_allocation, union_layout, tags); + let struct_type = env.context.struct_type(&field_types, false); + + if union_layout.stores_tag_id_as_data(env.ptr_bytes) { + let tag_id_ptr = builder + .build_struct_gep(raw_data_ptr, TAG_ID_INDEX, "tag_id_index") + .unwrap(); + + let tag_id_type = basic_type_from_layout(env, &tag_id_layout).into_int_type(); + + env.builder + .build_store(tag_id_ptr, tag_id_type.const_int(tag_id as u64, false)); + + let opaque_struct_ptr = builder + .build_struct_gep(raw_data_ptr, TAG_DATA_INDEX, "tag_data_index") + .unwrap(); + + struct_pointer_from_fields( + env, + struct_type, + opaque_struct_ptr, + field_vals.into_iter().enumerate(), + ); + + raw_data_ptr.into() + } else { + struct_pointer_from_fields( + env, + struct_type, + raw_data_ptr, + field_vals.into_iter().enumerate(), + ); + + tag_pointer_set_tag_id(env, tag_id, raw_data_ptr).into() + } +} + pub fn build_tag<'a, 'ctx, 'env>( env: &Env<'a, 'ctx, 'env>, scope: &Scope<'a, 'ctx>, @@ -1251,7 +1369,6 @@ pub fn build_tag<'a, 'ctx, 'env>( parent: FunctionValue<'ctx>, ) -> BasicValueEnum<'ctx> { let tag_id_layout = union_layout.tag_id_layout(); - let union_size = union_layout.number_of_tags(); match union_layout { @@ -1342,78 +1459,51 @@ pub fn build_tag<'a, 'ctx, 'env>( UnionLayout::Recursive(tags) => { debug_assert!(union_size > 1); - let ctx = env.context; - let builder = env.builder; - - // Determine types - let num_fields = arguments.len() + 1; - let mut field_types = Vec::with_capacity_in(num_fields, env.arena); - let mut field_vals = Vec::with_capacity_in(num_fields, env.arena); - let tag_field_layouts = &tags[tag_id as usize]; - for (field_symbol, tag_field_layout) in arguments.iter().zip(tag_field_layouts.iter()) { - let (val, _val_layout) = load_symbol_and_layout(scope, field_symbol); + build_wrapped_tag( + env, + scope, + union_layout, + tag_id, + arguments, + &tag_field_layouts, + tags, + reuse_allocation, + parent, + ) + } + UnionLayout::NullableWrapped { + nullable_id, + other_tags: tags, + } => { + let tag_field_layouts = { + use std::cmp::Ordering::*; + match tag_id.cmp(&(*nullable_id as u8)) { + Equal => { + let layout = Layout::Union(*union_layout); - let field_type = basic_type_from_layout(env, tag_field_layout); - - field_types.push(field_type); - - if let Layout::RecursivePointer = tag_field_layout { - debug_assert!(val.is_pointer_value()); - - // we store recursive pointers as `i64*` - let ptr = env.builder.build_bitcast( - val, - ctx.i64_type().ptr_type(AddressSpace::Generic), - "cast_recursive_pointer", - ); - - field_vals.push(ptr); - } else { - // this check fails for recursive tag unions, but can be helpful while debugging - // debug_assert_eq!(tag_field_layout, val_layout); - - field_vals.push(val); + return basic_type_from_layout(env, &layout) + .into_pointer_type() + .const_null() + .into(); + } + Less => &tags[tag_id as usize], + Greater => &tags[tag_id as usize - 1], } - } + }; - // Create the struct_type - let raw_data_ptr = allocate_tag(env, parent, reuse_allocation, union_layout, tags); - - let tag_id_ptr = builder - .build_struct_gep(raw_data_ptr, TAG_ID_INDEX, "tag_id_index") - .unwrap(); - - let tag_id_type = basic_type_from_layout(env, &tag_id_layout).into_int_type(); - - env.builder - .build_store(tag_id_ptr, tag_id_type.const_int(tag_id as u64, false)); - - let opaque_struct_ptr = builder - .build_struct_gep(raw_data_ptr, TAG_DATA_INDEX, "tag_data_index") - .unwrap(); - - let struct_type = env.context.struct_type(&field_types, false); - let struct_ptr = env - .builder - .build_bitcast( - opaque_struct_ptr, - struct_type.ptr_type(AddressSpace::Generic), - "struct_ptr", - ) - .into_pointer_value(); - - // Insert field exprs into struct_val - for (index, field_val) in field_vals.into_iter().enumerate() { - let field_ptr = builder - .build_struct_gep(struct_ptr, index as u32, "field_struct_gep") - .unwrap(); - - builder.build_store(field_ptr, field_val); - } - - raw_data_ptr.into() + build_wrapped_tag( + env, + scope, + union_layout, + tag_id, + arguments, + &tag_field_layouts, + tags, + reuse_allocation, + parent, + ) } UnionLayout::NonNullableUnwrapped(fields) => { debug_assert_eq!(union_size, 1); @@ -1421,7 +1511,6 @@ pub fn build_tag<'a, 'ctx, 'env>( debug_assert_eq!(arguments.len(), fields.len()); let ctx = env.context; - let builder = env.builder; // Determine types let num_fields = arguments.len() + 1; @@ -1458,125 +1547,16 @@ pub fn build_tag<'a, 'ctx, 'env>( reserve_with_refcount_union_as_block_of_memory(env, *union_layout, &[fields]); let struct_type = ctx.struct_type(field_types.into_bump_slice(), false); - let struct_ptr = env - .builder - .build_bitcast( - data_ptr, - struct_type.ptr_type(AddressSpace::Generic), - "block_of_memory_to_tag", - ) - .into_pointer_value(); - // Insert field exprs into struct_val - for (index, field_val) in field_vals.into_iter().enumerate() { - let field_ptr = builder - .build_struct_gep(struct_ptr, index as u32, "struct_gep") - .unwrap(); - - builder.build_store(field_ptr, field_val); - } + struct_pointer_from_fields( + env, + struct_type, + data_ptr, + field_vals.into_iter().enumerate(), + ); data_ptr.into() } - UnionLayout::NullableWrapped { - nullable_id, - other_tags: tags, - } => { - if tag_id == *nullable_id as u8 { - let layout = Layout::Union(*union_layout); - - return basic_type_from_layout(env, &layout) - .into_pointer_type() - .const_null() - .into(); - } - - debug_assert!(union_size > 1); - - let ctx = env.context; - let builder = env.builder; - - // Determine types - let num_fields = arguments.len() + 1; - let mut field_types = Vec::with_capacity_in(num_fields, env.arena); - let mut field_vals = Vec::with_capacity_in(num_fields, env.arena); - - let tag_field_layouts = { - use std::cmp::Ordering::*; - match tag_id.cmp(&(*nullable_id as u8)) { - Equal => unreachable!("early return above"), - Less => &tags[tag_id as usize], - Greater => &tags[tag_id as usize - 1], - } - }; - - for (field_symbol, tag_field_layout) in arguments.iter().zip(tag_field_layouts.iter()) { - let val = load_symbol(scope, field_symbol); - - // Zero-sized fields have no runtime representation. - // The layout of the struct expects them to be dropped! - if !tag_field_layout.is_dropped_because_empty() { - let field_type = basic_type_from_layout(env, tag_field_layout); - - field_types.push(field_type); - - if let Layout::RecursivePointer = tag_field_layout { - debug_assert!(val.is_pointer_value()); - - // we store recursive pointers as `i64*` - let ptr = env.builder.build_bitcast( - val, - ctx.i64_type().ptr_type(AddressSpace::Generic), - "cast_recursive_pointer", - ); - - field_vals.push(ptr); - } else { - // this check fails for recursive tag unions, but can be helpful while debugging - // debug_assert_eq!(tag_field_layout, val_layout); - - field_vals.push(val); - } - } - } - - // Create the struct_type - let raw_data_ptr = allocate_tag(env, parent, reuse_allocation, union_layout, tags); - - let tag_id_ptr = builder - .build_struct_gep(raw_data_ptr, TAG_ID_INDEX, "tag_id_index") - .unwrap(); - - let tag_id_type = basic_type_from_layout(env, &tag_id_layout).into_int_type(); - - env.builder - .build_store(tag_id_ptr, tag_id_type.const_int(tag_id as u64, false)); - - let opaque_struct_ptr = builder - .build_struct_gep(raw_data_ptr, TAG_DATA_INDEX, "tag_data_index") - .unwrap(); - - let struct_type = env.context.struct_type(&field_types, false); - let struct_ptr = env - .builder - .build_bitcast( - opaque_struct_ptr, - struct_type.ptr_type(AddressSpace::Generic), - "struct_ptr", - ) - .into_pointer_value(); - - // Insert field exprs into struct_val - for (index, field_val) in field_vals.into_iter().enumerate() { - let field_ptr = builder - .build_struct_gep(struct_ptr, index as u32, "field_struct_gep") - .unwrap(); - - builder.build_store(field_ptr, field_val); - } - - raw_data_ptr.into() - } UnionLayout::NullableUnwrapped { nullable_id, other_fields, @@ -1597,7 +1577,6 @@ pub fn build_tag<'a, 'ctx, 'env>( debug_assert!(union_size == 2); let ctx = env.context; - let builder = env.builder; // Determine types let num_fields = arguments.len() + 1; @@ -1641,29 +1620,75 @@ pub fn build_tag<'a, 'ctx, 'env>( allocate_tag(env, parent, reuse_allocation, union_layout, &[other_fields]); let struct_type = ctx.struct_type(field_types.into_bump_slice(), false); - let struct_ptr = env - .builder - .build_bitcast( - data_ptr, - struct_type.ptr_type(AddressSpace::Generic), - "block_of_memory_to_tag", - ) - .into_pointer_value(); - // Insert field exprs into struct_val - for (index, field_val) in field_vals.into_iter().enumerate() { - let field_ptr = builder - .build_struct_gep(struct_ptr, index as u32, "struct_gep") - .unwrap(); - - builder.build_store(field_ptr, field_val); - } + struct_pointer_from_fields( + env, + struct_type, + data_ptr, + field_vals.into_iter().enumerate(), + ); data_ptr.into() } } } +fn tag_pointer_set_tag_id<'a, 'ctx, 'env>( + env: &Env<'a, 'ctx, 'env>, + tag_id: u8, + pointer: PointerValue<'ctx>, +) -> PointerValue<'ctx> { + // we only have 3 bits, so can encode only 0..7 + debug_assert!(tag_id < 8); + + let ptr_int = env.ptr_int(); + + let as_int = env.builder.build_ptr_to_int(pointer, ptr_int, "to_int"); + + let tag_id_intval = ptr_int.const_int(tag_id as u64, false); + let combined = env.builder.build_or(as_int, tag_id_intval, "store_tag_id"); + + env.builder + .build_int_to_ptr(combined, pointer.get_type(), "to_ptr") +} + +pub fn tag_pointer_read_tag_id<'a, 'ctx, 'env>( + env: &Env<'a, 'ctx, 'env>, + pointer: PointerValue<'ctx>, +) -> IntValue<'ctx> { + let mask: u64 = 0b0000_0111; + + let ptr_int = env.ptr_int(); + + let as_int = env.builder.build_ptr_to_int(pointer, ptr_int, "to_int"); + let mask_intval = env.ptr_int().const_int(mask, false); + + let masked = env.builder.build_and(as_int, mask_intval, "mask"); + + env.builder + .build_int_cast(masked, env.context.i8_type(), "to_u8") +} + +pub fn tag_pointer_clear_tag_id<'a, 'ctx, 'env>( + env: &Env<'a, 'ctx, 'env>, + pointer: PointerValue<'ctx>, +) -> PointerValue<'ctx> { + let ptr_int = env.ptr_int(); + + let as_int = env.builder.build_ptr_to_int(pointer, ptr_int, "to_int"); + + let mask = { + let a = env.ptr_int().const_all_ones(); + let tag_id_bits = env.ptr_int().const_int(3, false); + env.builder.build_left_shift(a, tag_id_bits, "make_mask") + }; + + let masked = env.builder.build_and(as_int, mask, "masked"); + + env.builder + .build_int_to_ptr(masked, pointer.get_type(), "to_ptr") +} + fn allocate_tag<'a, 'ctx, 'env>( env: &Env<'a, 'ctx, 'env>, parent: FunctionValue<'ctx>, @@ -1694,9 +1719,11 @@ fn allocate_tag<'a, 'ctx, 'env>( let reuse_ptr = { env.builder.position_at_end(else_block); + let cleared = tag_pointer_clear_tag_id(env, ptr); + env.builder.build_unconditional_branch(cont_block); - ptr + cleared }; { @@ -1729,7 +1756,15 @@ pub fn get_tag_id<'a, 'ctx, 'env>( get_tag_id_non_recursive(env, tag) } - UnionLayout::Recursive(_) => get_tag_id_wrapped(env, argument.into_pointer_value()), + UnionLayout::Recursive(_) => { + let argument_ptr = argument.into_pointer_value(); + + if union_layout.stores_tag_id_as_data(env.ptr_bytes) { + get_tag_id_wrapped(env, argument_ptr) + } else { + tag_pointer_read_tag_id(env, argument_ptr) + } + } UnionLayout::NonNullableUnwrapped(_) => tag_id_int_type.const_zero(), UnionLayout::NullableWrapped { nullable_id, .. } => { let argument_ptr = argument.into_pointer_value(); @@ -1754,7 +1789,12 @@ pub fn get_tag_id<'a, 'ctx, 'env>( { env.builder.position_at_end(else_block); - let tag_id = get_tag_id_wrapped(env, argument_ptr); + + let tag_id = if union_layout.stores_tag_id_as_data(env.ptr_bytes) { + get_tag_id_wrapped(env, argument_ptr) + } else { + tag_pointer_read_tag_id(env, argument_ptr) + }; env.builder.build_store(result, tag_id); env.builder.build_unconditional_branch(cont_block); } @@ -1890,9 +1930,11 @@ fn reserve_with_refcount_union_as_block_of_memory<'a, 'ctx, 'env>( union_layout: UnionLayout<'a>, fields: &[&[Layout<'a>]], ) -> PointerValue<'ctx> { + let ptr_bytes = env.ptr_bytes; + let block_type = block_of_memory_slices(env.context, fields, env.ptr_bytes); - let basic_type = if union_layout.stores_tag_id() { + let basic_type = if union_layout.stores_tag_id_as_data(ptr_bytes) { let tag_id_type = basic_type_from_layout(env, &union_layout.tag_id_layout()); env.context @@ -1908,7 +1950,7 @@ fn reserve_with_refcount_union_as_block_of_memory<'a, 'ctx, 'env>( .max() .unwrap_or_default(); - if union_layout.stores_tag_id() { + if union_layout.stores_tag_id_as_data(ptr_bytes) { stack_size += union_layout.tag_id_layout().stack_size(env.ptr_bytes); } diff --git a/compiler/gen_llvm/src/llvm/build_hash.rs b/compiler/gen_llvm/src/llvm/build_hash.rs index 61a74dd232..99af6b9e11 100644 --- a/compiler/gen_llvm/src/llvm/build_hash.rs +++ b/compiler/gen_llvm/src/llvm/build_hash.rs @@ -1,5 +1,6 @@ use crate::debug_info_init; use crate::llvm::bitcode::call_bitcode_fn; +use crate::llvm::build::tag_pointer_clear_tag_id; use crate::llvm::build::Env; use crate::llvm::build::{cast_block_of_memory_to_tag, get_tag_id, FAST_CALL_CONV, TAG_DATA_INDEX}; use crate::llvm::build_str; @@ -493,14 +494,9 @@ fn hash_tag<'a, 'ctx, 'env>( ); // hash the tag data - let answer = hash_ptr_to_struct( - env, - layout_ids, - union_layout, - field_layouts, - seed, - tag.into_pointer_value(), - ); + let tag = tag_pointer_clear_tag_id(env, tag.into_pointer_value()); + let answer = + hash_ptr_to_struct(env, layout_ids, union_layout, field_layouts, seed, tag); merge_phi.add_incoming(&[(&answer, block)]); env.builder.build_unconditional_branch(merge_block); @@ -598,6 +594,7 @@ fn hash_tag<'a, 'ctx, 'env>( ); // hash tag data + let tag = tag_pointer_clear_tag_id(env, tag); let answer = hash_ptr_to_struct( env, layout_ids, diff --git a/compiler/gen_llvm/src/llvm/compare.rs b/compiler/gen_llvm/src/llvm/compare.rs index 2a9d1d3fa3..ffc0b04e99 100644 --- a/compiler/gen_llvm/src/llvm/compare.rs +++ b/compiler/gen_llvm/src/llvm/compare.rs @@ -1,5 +1,5 @@ -use crate::llvm::build::Env; use crate::llvm::build::{cast_block_of_memory_to_tag, get_tag_id, FAST_CALL_CONV}; +use crate::llvm::build::{tag_pointer_clear_tag_id, Env}; use crate::llvm::build_list::{list_len, load_list_ptr}; use crate::llvm::build_str::str_equal; use crate::llvm::convert::basic_type_from_layout; @@ -925,6 +925,10 @@ fn build_tag_eq_help<'a, 'ctx, 'env>( let id1 = get_tag_id(env, parent, union_layout, tag1); let id2 = get_tag_id(env, parent, union_layout, tag2); + // clear the tag_id so we get a pointer to the actual data + let tag1 = tag_pointer_clear_tag_id(env, tag1.into_pointer_value()); + let tag2 = tag_pointer_clear_tag_id(env, tag2.into_pointer_value()); + let compare_tag_fields = ctx.append_basic_block(parent, "compare_tag_fields"); let same_tag = @@ -944,14 +948,8 @@ fn build_tag_eq_help<'a, 'ctx, 'env>( let block = env.context.append_basic_block(parent, "tag_id_modify"); env.builder.position_at_end(block); - let answer = eq_ptr_to_struct( - env, - layout_ids, - union_layout, - field_layouts, - tag1.into_pointer_value(), - tag2.into_pointer_value(), - ); + let answer = + eq_ptr_to_struct(env, layout_ids, union_layout, field_layouts, tag1, tag2); env.builder.build_return(Some(&answer)); @@ -1073,6 +1071,10 @@ fn build_tag_eq_help<'a, 'ctx, 'env>( let id1 = get_tag_id(env, parent, union_layout, tag1); let id2 = get_tag_id(env, parent, union_layout, tag2); + // clear the tag_id so we get a pointer to the actual data + let tag1 = tag_pointer_clear_tag_id(env, tag1.into_pointer_value()); + let tag2 = tag_pointer_clear_tag_id(env, tag2.into_pointer_value()); + let compare_tag_fields = ctx.append_basic_block(parent, "compare_tag_fields"); let same_tag = @@ -1093,14 +1095,8 @@ fn build_tag_eq_help<'a, 'ctx, 'env>( let block = env.context.append_basic_block(parent, "tag_id_modify"); env.builder.position_at_end(block); - let answer = eq_ptr_to_struct( - env, - layout_ids, - union_layout, - field_layouts, - tag1.into_pointer_value(), - tag2.into_pointer_value(), - ); + let answer = + eq_ptr_to_struct(env, layout_ids, union_layout, field_layouts, tag1, tag2); env.builder.build_return(Some(&answer)); diff --git a/compiler/gen_llvm/src/llvm/convert.rs b/compiler/gen_llvm/src/llvm/convert.rs index 1cb299c0ae..729c118626 100644 --- a/compiler/gen_llvm/src/llvm/convert.rs +++ b/compiler/gen_llvm/src/llvm/convert.rs @@ -31,21 +31,31 @@ pub fn basic_type_from_layout<'a, 'ctx, 'env>( basic_type_from_layout(env, &closure_data_layout) } Struct(sorted_fields) => basic_type_from_record(env, sorted_fields), - Union(variant) => { + Union(union_layout) => { use UnionLayout::*; - let tag_id_type = basic_type_from_layout(env, &variant.tag_id_layout()); + let tag_id_type = basic_type_from_layout(env, &union_layout.tag_id_layout()); - match variant { - NullableWrapped { + match union_layout { + NonRecursive(tags) => { + let data = block_of_memory_slices(env.context, tags, env.ptr_bytes); + + env.context.struct_type(&[data, tag_id_type], false).into() + } + Recursive(tags) + | NullableWrapped { other_tags: tags, .. } => { let data = block_of_memory_slices(env.context, tags, env.ptr_bytes); - env.context - .struct_type(&[data, tag_id_type], false) - .ptr_type(AddressSpace::Generic) - .into() + if union_layout.stores_tag_id_as_data(env.ptr_bytes) { + env.context + .struct_type(&[data, tag_id_type], false) + .ptr_type(AddressSpace::Generic) + .into() + } else { + data.ptr_type(AddressSpace::Generic).into() + } } NullableUnwrapped { other_fields, .. } => { let block = @@ -56,19 +66,6 @@ pub fn basic_type_from_layout<'a, 'ctx, 'env>( let block = block_of_memory_slices(env.context, &[fields], env.ptr_bytes); block.ptr_type(AddressSpace::Generic).into() } - Recursive(tags) => { - let data = block_of_memory_slices(env.context, tags, env.ptr_bytes); - - env.context - .struct_type(&[data, tag_id_type], false) - .ptr_type(AddressSpace::Generic) - .into() - } - NonRecursive(tags) => { - let data = block_of_memory_slices(env.context, tags, env.ptr_bytes); - - env.context.struct_type(&[data, tag_id_type], false).into() - } } } RecursivePointer => { @@ -145,16 +142,6 @@ pub fn union_data_is_struct_type<'ctx>( context.struct_type(&[struct_type.into(), tag_id_type.into()], false) } -pub fn union_data_block_of_memory<'ctx>( - context: &'ctx Context, - tag_id_int_type: IntType<'ctx>, - layouts: &[&[Layout<'_>]], - ptr_bytes: u32, -) -> StructType<'ctx> { - let data_type = block_of_memory_slices(context, layouts, ptr_bytes); - context.struct_type(&[data_type, tag_id_int_type.into()], false) -} - pub fn block_of_memory<'ctx>( context: &'ctx Context, layout: &Layout<'_>, diff --git a/compiler/gen_llvm/src/llvm/refcounting.rs b/compiler/gen_llvm/src/llvm/refcounting.rs index 870e2218fa..9c09469a97 100644 --- a/compiler/gen_llvm/src/llvm/refcounting.rs +++ b/compiler/gen_llvm/src/llvm/refcounting.rs @@ -1,12 +1,10 @@ use crate::debug_info_init; use crate::llvm::build::{ add_func, cast_basic_basic, cast_block_of_memory_to_tag, get_tag_id, get_tag_id_non_recursive, - Env, FAST_CALL_CONV, LLVM_SADD_WITH_OVERFLOW_I64, TAG_DATA_INDEX, + tag_pointer_clear_tag_id, Env, FAST_CALL_CONV, LLVM_SADD_WITH_OVERFLOW_I64, TAG_DATA_INDEX, }; use crate::llvm::build_list::{incrementing_elem_loop, list_len, load_list}; -use crate::llvm::convert::{ - basic_type_from_layout, block_of_memory_slices, ptr_int, union_data_block_of_memory, -}; +use crate::llvm::convert::{basic_type_from_layout, ptr_int}; use bumpalo::collections::Vec; use inkwell::basic_block::BasicBlock; use inkwell::context::Context; @@ -1233,7 +1231,8 @@ fn build_rec_union_help<'a, 'ctx, 'env>( let parent = fn_val; debug_assert!(arg_val.is_pointer_value()); - let value_ptr = arg_val.into_pointer_value(); + let current_tag_id = get_tag_id(env, fn_val, &union_layout, arg_val); + let value_ptr = tag_pointer_clear_tag_id(env, arg_val.into_pointer_value()); // to increment/decrement the cons-cell itself let refcount_ptr = PointerToRefcount::from_ptr_to_data(env, value_ptr); @@ -1298,6 +1297,7 @@ fn build_rec_union_help<'a, 'ctx, 'env>( union_layout, tags, value_ptr, + current_tag_id, refcount_ptr, do_recurse_block, DecOrReuse::Dec, @@ -1322,6 +1322,7 @@ fn build_rec_union_recursive_decrement<'a, 'ctx, 'env>( union_layout: UnionLayout<'a>, tags: &[&[Layout<'a>]], value_ptr: PointerValue<'ctx>, + current_tag_id: IntValue<'ctx>, refcount_ptr: PointerToRefcount<'ctx>, match_block: BasicBlock<'ctx>, decrement_or_reuse: DecOrReuse, @@ -1396,28 +1397,8 @@ fn build_rec_union_recursive_decrement<'a, 'ctx, 'env>( debug_assert!(ptr_as_i64_ptr.is_pointer_value()); // therefore we must cast it to our desired type - let union_type = match union_layout { - UnionLayout::NonRecursive(_) => unreachable!(), - UnionLayout::Recursive(_) | UnionLayout::NullableWrapped { .. } => { - union_data_block_of_memory( - env.context, - tag_id_int_type, - tags, - env.ptr_bytes, - ) - .into() - } - UnionLayout::NonNullableUnwrapped { .. } - | UnionLayout::NullableUnwrapped { .. } => { - block_of_memory_slices(env.context, tags, env.ptr_bytes) - } - }; - - let recursive_field_ptr = cast_basic_basic( - env.builder, - ptr_as_i64_ptr, - union_type.ptr_type(AddressSpace::Generic).into(), - ); + let union_type = basic_type_from_layout(env, &Layout::Union(union_layout)); + let recursive_field_ptr = cast_basic_basic(env.builder, ptr_as_i64_ptr, union_type); deferred_rec.push(recursive_field_ptr); } else if field_layout.contains_refcounted() { @@ -1484,9 +1465,6 @@ fn build_rec_union_recursive_decrement<'a, 'ctx, 'env>( let (_, only_branch) = cases.pop().unwrap(); env.builder.build_unconditional_branch(only_branch); } else { - // read the tag_id - let current_tag_id = get_tag_id(env, parent, &union_layout, value_ptr.into()); - let default_block = env.context.append_basic_block(parent, "switch_default"); // switch on it @@ -1601,7 +1579,8 @@ fn build_reuse_rec_union_help<'a, 'ctx, 'env>( let parent = reset_function; debug_assert!(arg_val.is_pointer_value()); - let value_ptr = arg_val.into_pointer_value(); + let current_tag_id = get_tag_id(env, reset_function, &union_layout, arg_val); + let value_ptr = tag_pointer_clear_tag_id(env, arg_val.into_pointer_value()); // to increment/decrement the cons-cell itself let refcount_ptr = PointerToRefcount::from_ptr_to_data(env, value_ptr); @@ -1654,6 +1633,7 @@ fn build_reuse_rec_union_help<'a, 'ctx, 'env>( union_layout, tags, value_ptr, + current_tag_id, refcount_ptr, do_recurse_block, DecOrReuse::Reuse, diff --git a/compiler/mono/src/layout.rs b/compiler/mono/src/layout.rs index 4bd851184a..48584c947d 100644 --- a/compiler/mono/src/layout.rs +++ b/compiler/mono/src/layout.rs @@ -188,11 +188,29 @@ impl<'a> UnionLayout<'a> { Layout::Builtin(self.tag_id_builtin()) } - pub fn stores_tag_id(&self) -> bool { + fn stores_tag_id_in_pointer_bits(tags: &[&[Layout<'a>]], ptr_bytes: u32) -> bool { + tags.len() <= ptr_bytes as usize + } + + // i.e. it is not implicit and not stored in the pointer bits + pub fn stores_tag_id_as_data(&self, ptr_bytes: u32) -> bool { match self { - UnionLayout::NonRecursive(_) - | UnionLayout::Recursive(_) - | UnionLayout::NullableWrapped { .. } => true, + UnionLayout::NonRecursive(_) => true, + UnionLayout::Recursive(tags) + | UnionLayout::NullableWrapped { + other_tags: tags, .. + } => !Self::stores_tag_id_in_pointer_bits(tags, ptr_bytes), + UnionLayout::NonNullableUnwrapped(_) | UnionLayout::NullableUnwrapped { .. } => false, + } + } + + pub fn stores_tag_id_in_pointer(&self, ptr_bytes: u32) -> bool { + match self { + UnionLayout::NonRecursive(_) => false, + UnionLayout::Recursive(tags) + | UnionLayout::NullableWrapped { + other_tags: tags, .. + } => Self::stores_tag_id_in_pointer_bits(tags, ptr_bytes), UnionLayout::NonNullableUnwrapped(_) | UnionLayout::NullableUnwrapped { .. } => false, } } diff --git a/compiler/test_gen/src/gen_primitives.rs b/compiler/test_gen/src/gen_primitives.rs index 3c932db913..77b4f7cb24 100644 --- a/compiler/test_gen/src/gen_primitives.rs +++ b/compiler/test_gen/src/gen_primitives.rs @@ -1555,9 +1555,9 @@ fn rbtree_balance_full() { balance Red 0 0 Empty Empty "# ), - false, - *const i64, - |x: *const i64| x.is_null() + true, + usize, + |x| x != 0 ); } diff --git a/compiler/test_gen/src/gen_tags.rs b/compiler/test_gen/src/gen_tags.rs index 38c28b9fff..247bebde87 100644 --- a/compiler/test_gen/src/gen_tags.rs +++ b/compiler/test_gen/src/gen_tags.rs @@ -1079,8 +1079,8 @@ fn nested_recursive_literal() { #" ), 0, - &(i64, i64, u8), - |x: &(i64, i64, u8)| x.2 + usize, + |_| 0 ); }