use std::alloc::{dealloc, handle_alloc_error, Layout}; use std::fmt; use std::hash::{Hash, Hasher}; use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; use std::ptr::{addr_of_mut, slice_from_raw_parts_mut, NonNull}; /// A type that is functionally equivalent to `(Header, Box<[Item]>)`, /// but all data is stored in one heap allocation and the pointer is thin, /// so the whole thing's size is like a pointer. pub struct ThinVecWithHeader { /// INVARIANT: Points to a valid heap allocation that contains `ThinVecInner
`, /// followed by (suitably aligned) `len` `Item`s. ptr: NonNull>, _marker: PhantomData<(Header, Box<[Item]>)>, } // SAFETY: We essentially own both the header and the items. unsafe impl Send for ThinVecWithHeader {} unsafe impl Sync for ThinVecWithHeader {} #[derive(Clone)] struct ThinVecInner
{ header: Header, len: usize, } impl ThinVecWithHeader { /// # Safety /// /// The iterator must produce `len` elements. #[inline] unsafe fn from_trusted_len_iter( header: Header, len: usize, items: impl Iterator, ) -> Self { let (ptr, layout, items_offset) = Self::allocate(len); struct DeallocGuard(*mut u8, Layout); impl Drop for DeallocGuard { fn drop(&mut self) { // SAFETY: We allocated this above. unsafe { dealloc(self.0, self.1); } } } let _dealloc_guard = DeallocGuard(ptr.as_ptr().cast::(), layout); // INVARIANT: Between `0..1` there are only initialized items. struct ItemsGuard(*mut Item, *mut Item); impl Drop for ItemsGuard { fn drop(&mut self) { // SAFETY: Our invariant. unsafe { slice_from_raw_parts_mut(self.0, self.1.offset_from(self.0) as usize) .drop_in_place(); } } } // SAFETY: We allocated enough space. let mut items_ptr = unsafe { ptr.as_ptr().byte_add(items_offset).cast::() }; // INVARIANT: There are zero elements in this range. let mut items_guard = ItemsGuard(items_ptr, items_ptr); items.for_each(|item| { // SAFETY: Our precondition guarantee we won't get more than `len` items, and we allocated // enough space for `len` items. unsafe { items_ptr.write(item); items_ptr = items_ptr.add(1); } // INVARIANT: We just initialized this item. items_guard.1 = items_ptr; }); // SAFETY: We allocated enough space. unsafe { ptr.write(ThinVecInner { header, len }); } std::mem::forget(items_guard); std::mem::forget(_dealloc_guard); // INVARIANT: We allocated and initialized all fields correctly. Self { ptr, _marker: PhantomData } } #[inline] fn allocate(len: usize) -> (NonNull>, Layout, usize) { let (layout, items_offset) = Self::layout(len); // SAFETY: We always have `len`, so our allocation cannot be zero-sized. let ptr = unsafe { std::alloc::alloc(layout).cast::>() }; let Some(ptr) = NonNull::>::new(ptr) else { handle_alloc_error(layout); }; (ptr, layout, items_offset) } #[inline] #[allow(clippy::should_implement_trait)] pub fn from_iter(header: Header, items: I) -> Self where I: IntoIterator, I::IntoIter: TrustedLen, { let items = items.into_iter(); // SAFETY: `TrustedLen` guarantees the iterator length is exact. unsafe { Self::from_trusted_len_iter(header, items.len(), items) } } #[inline] fn items_offset(&self) -> usize { // SAFETY: We `pad_to_align()` in `layout()`, so at most where accessing past the end of the allocation, // which is allowed. unsafe { Layout::new::>().extend(Layout::new::()).unwrap_unchecked().1 } } #[inline] fn header_and_len(&self) -> &ThinVecInner
{ // SAFETY: By `ptr`'s invariant, it is correctly allocated and initialized. unsafe { &*self.ptr.as_ptr() } } #[inline] fn items_ptr(&self) -> *mut [Item] { let len = self.header_and_len().len; // SAFETY: `items_offset()` returns the correct offset of the items, where they are allocated. let ptr = unsafe { self.ptr.as_ptr().byte_add(self.items_offset()).cast::() }; slice_from_raw_parts_mut(ptr, len) } #[inline] pub fn header(&self) -> &Header { &self.header_and_len().header } #[inline] pub fn header_mut(&mut self) -> &mut Header { // SAFETY: By `ptr`'s invariant, it is correctly allocated and initialized. unsafe { &mut *addr_of_mut!((*self.ptr.as_ptr()).header) } } #[inline] pub fn items(&self) -> &[Item] { // SAFETY: `items_ptr()` gives a valid pointer. unsafe { &*self.items_ptr() } } #[inline] pub fn items_mut(&mut self) -> &mut [Item] { // SAFETY: `items_ptr()` gives a valid pointer. unsafe { &mut *self.items_ptr() } } #[inline] pub fn len(&self) -> usize { self.header_and_len().len } #[inline] fn layout(len: usize) -> (Layout, usize) { let (layout, items_offset) = Layout::new::>() .extend(Layout::array::(len).expect("too big `ThinVec` requested")) .expect("too big `ThinVec` requested"); let layout = layout.pad_to_align(); (layout, items_offset) } } /// # Safety /// /// The length reported must be exactly the number of items yielded. pub unsafe trait TrustedLen: ExactSizeIterator {} unsafe impl TrustedLen for std::vec::IntoIter {} unsafe impl TrustedLen for std::slice::Iter<'_, T> {} unsafe impl<'a, T: Clone + 'a, I: TrustedLen> TrustedLen for std::iter::Cloned {} unsafe impl T> TrustedLen for std::iter::Map {} unsafe impl TrustedLen for std::vec::Drain<'_, T> {} unsafe impl TrustedLen for std::array::IntoIter {} impl Clone for ThinVecWithHeader { #[inline] fn clone(&self) -> Self { Self::from_iter(self.header().clone(), self.items().iter().cloned()) } } impl Drop for ThinVecWithHeader { #[inline] fn drop(&mut self) { // This must come before we drop `header`, because after that we cannot make a reference to it in `len()`. let len = self.len(); // SAFETY: The contents are allocated and initialized. unsafe { addr_of_mut!((*self.ptr.as_ptr()).header).drop_in_place(); self.items_ptr().drop_in_place(); } let (layout, _) = Self::layout(len); // SAFETY: This was allocated in `new()` with the same layout calculation. unsafe { dealloc(self.ptr.as_ptr().cast::(), layout); } } } impl fmt::Debug for ThinVecWithHeader { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("ThinVecWithHeader") .field("header", self.header()) .field("items", &self.items()) .finish() } } impl PartialEq for ThinVecWithHeader { #[inline] fn eq(&self, other: &Self) -> bool { self.header() == other.header() && self.items() == other.items() } } impl Eq for ThinVecWithHeader {} impl Hash for ThinVecWithHeader { #[inline] fn hash(&self, state: &mut H) { self.header().hash(state); self.items().hash(state); } } #[derive(Clone, PartialEq, Eq, Hash)] pub struct ThinVec(ThinVecWithHeader<(), T>); impl ThinVec { #[inline] #[allow(clippy::should_implement_trait)] pub fn from_iter(values: I) -> Self where I: IntoIterator, I::IntoIter: TrustedLen, { Self(ThinVecWithHeader::from_iter((), values)) } #[inline] pub fn len(&self) -> usize { self.0.len() } #[inline] pub fn iter(&self) -> std::slice::Iter<'_, T> { (**self).iter() } #[inline] pub fn iter_mut(&mut self) -> std::slice::IterMut<'_, T> { (**self).iter_mut() } } impl Deref for ThinVec { type Target = [T]; #[inline] fn deref(&self) -> &Self::Target { self.0.items() } } impl DerefMut for ThinVec { #[inline] fn deref_mut(&mut self) -> &mut Self::Target { self.0.items_mut() } } impl<'a, T> IntoIterator for &'a ThinVec { type IntoIter = std::slice::Iter<'a, T>; type Item = &'a T; #[inline] fn into_iter(self) -> Self::IntoIter { self.iter() } } impl<'a, T> IntoIterator for &'a mut ThinVec { type IntoIter = std::slice::IterMut<'a, T>; type Item = &'a mut T; #[inline] fn into_iter(self) -> Self::IntoIter { self.iter_mut() } } impl fmt::Debug for ThinVec { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_list().entries(&**self).finish() } } /// A [`ThinVec`] that requires no allocation for the empty case. #[derive(Clone, PartialEq, Eq, Hash)] pub struct EmptyOptimizedThinVec(Option>); impl EmptyOptimizedThinVec { #[inline] #[allow(clippy::should_implement_trait)] pub fn from_iter(values: I) -> Self where I: IntoIterator, I::IntoIter: TrustedLen, { let values = values.into_iter(); if values.len() == 0 { Self::empty() } else { Self(Some(ThinVec::from_iter(values))) } } #[inline] pub fn empty() -> Self { Self(None) } #[inline] pub fn len(&self) -> usize { self.0.as_ref().map_or(0, ThinVec::len) } #[inline] pub fn iter(&self) -> std::slice::Iter<'_, T> { (**self).iter() } #[inline] pub fn iter_mut(&mut self) -> std::slice::IterMut<'_, T> { (**self).iter_mut() } } impl Default for EmptyOptimizedThinVec { #[inline] fn default() -> Self { Self::empty() } } impl Deref for EmptyOptimizedThinVec { type Target = [T]; #[inline] fn deref(&self) -> &Self::Target { self.0.as_deref().unwrap_or_default() } } impl DerefMut for EmptyOptimizedThinVec { #[inline] fn deref_mut(&mut self) -> &mut Self::Target { self.0.as_deref_mut().unwrap_or_default() } } impl<'a, T> IntoIterator for &'a EmptyOptimizedThinVec { type IntoIter = std::slice::Iter<'a, T>; type Item = &'a T; #[inline] fn into_iter(self) -> Self::IntoIter { self.iter() } } impl<'a, T> IntoIterator for &'a mut EmptyOptimizedThinVec { type IntoIter = std::slice::IterMut<'a, T>; type Item = &'a mut T; #[inline] fn into_iter(self) -> Self::IntoIter { self.iter_mut() } } impl fmt::Debug for EmptyOptimizedThinVec { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_list().entries(&**self).finish() } } /// Syntax: /// /// ```ignore /// thin_vec_with_header_struct! { /// pub new(pub(crate)) struct MyCoolStruct, MyCoolStructHeader { /// pub(crate) variable_length: [Ty], /// pub field1: CopyTy, /// pub field2: NonCopyTy; ref, /// } /// } /// ``` #[doc(hidden)] #[macro_export] macro_rules! thin_vec_with_header_struct_ { (@maybe_ref (ref) $($t:tt)*) => { &$($t)* }; (@maybe_ref () $($t:tt)*) => { $($t)* }; ( $vis:vis new($new_vis:vis) struct $struct:ident, $header:ident { $items_vis:vis $items:ident : [$items_ty:ty], $( $header_var_vis:vis $header_var:ident : $header_var_ty:ty $(; $ref:ident)?, )+ } ) => { #[derive(Debug, Clone, Eq, PartialEq, Hash)] struct $header { $( $header_var : $header_var_ty, )+ } #[derive(Clone, Eq, PartialEq, Hash)] $vis struct $struct($crate::thin_vec::ThinVecWithHeader<$header, $items_ty>); impl $struct { #[inline] #[allow(unused)] $new_vis fn new( $( $header_var: $header_var_ty, )+ $items: I, ) -> Self where I: ::std::iter::IntoIterator, I::IntoIter: $crate::thin_vec::TrustedLen, { Self($crate::thin_vec::ThinVecWithHeader::from_iter( $header { $( $header_var, )+ }, $items, )) } #[inline] $items_vis fn $items(&self) -> &[$items_ty] { self.0.items() } $( #[inline] $header_var_vis fn $header_var(&self) -> $crate::thin_vec_with_header_struct_!(@maybe_ref ($($ref)?) $header_var_ty) { $crate::thin_vec_with_header_struct_!(@maybe_ref ($($ref)?) self.0.header().$header_var) } )+ } impl ::std::fmt::Debug for $struct { fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { f.debug_struct(stringify!($struct)) $( .field(stringify!($header_var), &self.$header_var()) )* .field(stringify!($items), &self.$items()) .finish() } } }; } pub use crate::thin_vec_with_header_struct_ as thin_vec_with_header_struct;