diff --git a/library/alloc/src/raw_vec/mod.rs b/library/alloc/src/raw_vec/mod.rs index 5d4ad3ac4bf98..81d6bac46ae83 100644 --- a/library/alloc/src/raw_vec/mod.rs +++ b/library/alloc/src/raw_vec/mod.rs @@ -9,6 +9,20 @@ use core::mem::{Alignment, ManuallyDrop, MaybeUninit, SizedTypeProperties}; use core::ptr::{self, NonNull, Unique}; use core::{cmp, hint}; +// Unlike the public declaration in `core`, this accepts a conditionally-const +// callback. That lets a conditionally-const allocator stay generic during +// const evaluation while the runtime callback erases its concrete type. +#[rustc_const_unstable(feature = "const_heap", issue = "79597")] +#[rustc_intrinsic] +const fn const_eval_select( + _arg: ARG, + _called_in_const: F, + _called_at_rt: G, +) -> RET +where + G: FnOnce, + F: [const] FnOnce; + #[cfg(not(no_global_oom_handling))] use crate::alloc::handle_alloc_error; use crate::alloc::{Allocator, Global, Layout}; @@ -71,18 +85,21 @@ const unsafe fn new_cap(cap: usize) -> Cap { /// `Box<[T]>`, since `capacity()` won't yield the length. #[allow(missing_debug_implementations)] pub(crate) struct RawVec { - inner: RawVecInner, + inner: RawVecInner, + alloc: A, _marker: PhantomData, } -/// Like a `RawVec`, but only generic over the allocator, not the type. +/// Like a `RawVec`, but not generic over the allocator. /// -/// As such, all the methods need the layout passed-in as a parameter. +/// As such, all the methods need the layout passed-in as a parameter, +/// and methods that need an allocator receive it separately. /// /// Having this separation reduces the amount of code we need to monomorphize, /// as most operations don't need the actual type, just its layout. #[allow(missing_debug_implementations)] -struct RawVecInner { +#[derive(Clone, Copy)] +struct RawVecInner { ptr: Unique, /// Never used for ZSTs; it's `capacity()`'s responsibility to return usize::MAX in that case. /// @@ -90,7 +107,6 @@ struct RawVecInner { /// /// `cap` must be in the `0..=isize::MAX` range. cap: Cap, - alloc: A, } impl RawVec { @@ -123,7 +139,7 @@ impl RawVec { #[must_use] #[inline] pub(crate) fn with_capacity(capacity: usize) -> Self { - Self { inner: RawVecInner::with_capacity(capacity, T::LAYOUT), _marker: PhantomData } + Self::with_capacity_in(capacity, Global) } /// Like `with_capacity`, but guarantees the buffer is zeroed. @@ -131,22 +147,7 @@ impl RawVec { #[must_use] #[inline] pub(crate) fn with_capacity_zeroed(capacity: usize) -> Self { - Self { - inner: RawVecInner::with_capacity_zeroed_in(capacity, Global, T::LAYOUT), - _marker: PhantomData, - } - } -} - -impl RawVecInner { - #[cfg(not(any(no_global_oom_handling, test)))] - #[must_use] - #[inline] - fn with_capacity(capacity: usize, elem_layout: Layout) -> Self { - match Self::try_allocate_in(capacity, AllocInit::Uninitialized, Global, elem_layout) { - Ok(res) => res, - Err(err) => handle_error(err), - } + Self::with_capacity_zeroed_in(capacity, Global) } } @@ -174,7 +175,8 @@ const impl RawVec { #[inline] pub(crate) fn with_capacity_in(capacity: usize, alloc: A) -> Self { Self { - inner: RawVecInner::with_capacity_in(capacity, alloc, T::LAYOUT), + inner: RawVecInner::with_capacity_in(capacity, &alloc, T::LAYOUT), + alloc, _marker: PhantomData, } } @@ -182,10 +184,66 @@ const impl RawVec { /// A specialized version of `self.reserve(len, 1)` which requires the /// caller to ensure `len == self.capacity()`. #[cfg(not(no_global_oom_handling))] - #[inline(never)] + #[inline(always)] pub(crate) fn grow_one(&mut self) { - // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout - unsafe { self.inner.grow_one(T::LAYOUT) } + // Move the allocator to a local to prevent the address of `self` from + // escaping through the allocator call. The guard restores it while + // unwinding as well as on the normal return path. + let local_alloc = CaptureLocally::new(&mut self.alloc); + let alloc = local_alloc.get(); + + // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout. + self.inner = const_eval_select( + (self.inner, alloc, T::LAYOUT), + RawVecInner::grow_one_const_select::, + RawVecInner::grow_one_runtime::, + ); + local_alloc.restore(); + } +} + +struct CaptureLocally<'a, T> { + value: ManuallyDrop, + old: *mut T, + _marker: PhantomData<&'a mut T>, +} + +#[rustc_const_unstable(feature = "const_heap", issue = "79597")] +const impl<'a, T: [const] Destruct> Drop for CaptureLocally<'a, T> { + fn drop(&mut self) { + // SAFETY: We have sole ownership of `self.old` during the lifetime of + // this struct, so writing back to it is safe. + unsafe { + let value = ptr::read((&raw const self.value).cast::()); + ptr::write(self.old, value); + } + } +} + +impl<'a, T> CaptureLocally<'a, T> { + const fn new(old: &'a mut T) -> Self { + // SAFETY: We are taking ownership of the value at `old`, given that we + // store the mut reference, we have sole ownership of it for the duration of + // this struct's lifetime. The Drop impl will write it back. + Self { value: ManuallyDrop::new(unsafe { ptr::read(old) }), old, _marker: PhantomData } + } + + const fn get(&self) -> &T { + // SAFETY: `ManuallyDrop` has the same layout and validity as `T`, + // and the guard owns the value until it is restored. + unsafe { &*((&raw const self.value).cast::()) } + } + + const fn restore(self) { + // Suppress `Drop` on the normal path so codegen can see the restoration + // directly. If the allocator operation unwinds before this call, the + // guard's `Drop` implementation performs the same restoration. + let this = ManuallyDrop::new(self); + let this = (&raw const this).cast::(); + unsafe { + let value = ptr::read((&raw const (*this).value).cast::()); + ptr::write((*this).old, value); + } } } @@ -199,15 +257,15 @@ impl RawVec { pub(crate) const fn new_in(alloc: A) -> Self { // Check assumption made in `current_memory` const { assert!(T::LAYOUT.size() % T::LAYOUT.align() == 0) }; - Self { inner: RawVecInner::new_in(alloc, Alignment::of::()), _marker: PhantomData } + Self { inner: RawVecInner::new(Alignment::of::()), alloc, _marker: PhantomData } } /// Like `try_with_capacity`, but parameterized over the choice of /// allocator for the returned `RawVec`. #[inline] pub(crate) fn try_with_capacity_in(capacity: usize, alloc: A) -> Result { - match RawVecInner::try_with_capacity_in(capacity, alloc, T::LAYOUT) { - Ok(inner) => Ok(Self { inner, _marker: PhantomData }), + match RawVecInner::try_allocate_in(capacity, AllocInit::Uninitialized, &alloc, T::LAYOUT) { + Ok(inner) => Ok(Self { inner, alloc, _marker: PhantomData }), Err(e) => Err(e), } } @@ -217,9 +275,9 @@ impl RawVec { #[cfg(not(no_global_oom_handling))] #[inline] pub(crate) fn with_capacity_zeroed_in(capacity: usize, alloc: A) -> Self { - Self { - inner: RawVecInner::with_capacity_zeroed_in(capacity, alloc, T::LAYOUT), - _marker: PhantomData, + match RawVecInner::try_allocate_in(capacity, AllocInit::Zeroed, &alloc, T::LAYOUT) { + Ok(inner) => Self { inner, alloc, _marker: PhantomData }, + Err(err) => handle_error(err), } } @@ -245,7 +303,7 @@ impl RawVec { let me = ManuallyDrop::new(self); unsafe { let slice = me.ptr().cast::>().cast_slice(len); - Box::from_raw_in(slice, ptr::read(&me.inner.alloc)) + Box::from_raw_in(slice, ptr::read(&me.alloc)) } } @@ -265,10 +323,7 @@ impl RawVec { unsafe { let ptr = ptr.cast(); let capacity = new_cap::(capacity); - Self { - inner: RawVecInner::from_raw_parts_in(ptr, capacity, alloc), - _marker: PhantomData, - } + Self { inner: RawVecInner::from_raw_parts(ptr, capacity), alloc, _marker: PhantomData } } } @@ -284,7 +339,7 @@ impl RawVec { unsafe { let ptr = ptr.cast(); let capacity = new_cap::(capacity); - Self { inner: RawVecInner::from_nonnull_in(ptr, capacity, alloc), _marker: PhantomData } + Self { inner: RawVecInner::from_nonnull(ptr, capacity), alloc, _marker: PhantomData } } } @@ -312,7 +367,7 @@ impl RawVec { /// Returns a shared reference to the allocator backing this `RawVec`. #[inline] pub(crate) const fn allocator(&self) -> &A { - self.inner.allocator() + &self.alloc } /// Ensures that the buffer contains at least enough space to hold `len + @@ -338,17 +393,18 @@ impl RawVec { #[inline] pub(crate) fn reserve(&mut self, len: usize, additional: usize) { // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout - unsafe { self.inner.reserve(len, additional, T::LAYOUT) } + unsafe { self.inner.reserve(len, additional, T::LAYOUT, &self.alloc) } } /// The same as `reserve`, but returns on errors instead of panicking or aborting. + #[inline] pub(crate) fn try_reserve( &mut self, len: usize, additional: usize, ) -> Result<(), TryReserveError> { // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout - unsafe { self.inner.try_reserve(len, additional, T::LAYOUT) } + unsafe { self.inner.try_reserve(len, additional, T::LAYOUT, &self.alloc) } } /// Ensures that the buffer contains at least enough space to hold `len + @@ -369,19 +425,21 @@ impl RawVec { /// /// Aborts on OOM. #[cfg(not(no_global_oom_handling))] + #[inline] pub(crate) fn reserve_exact(&mut self, len: usize, additional: usize) { // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout - unsafe { self.inner.reserve_exact(len, additional, T::LAYOUT) } + unsafe { self.inner.reserve_exact(len, additional, T::LAYOUT, &self.alloc) } } /// The same as `reserve_exact`, but returns on errors instead of panicking or aborting. + #[inline] pub(crate) fn try_reserve_exact( &mut self, len: usize, additional: usize, ) -> Result<(), TryReserveError> { // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout - unsafe { self.inner.try_reserve_exact(len, additional, T::LAYOUT) } + unsafe { self.inner.try_reserve_exact(len, additional, T::LAYOUT, &self.alloc) } } /// Shrinks the buffer down to the specified capacity. If the given amount @@ -398,7 +456,7 @@ impl RawVec { #[inline] pub(crate) fn shrink_to_fit(&mut self, cap: usize) { // SAFETY: All calls on self.inner pass T::LAYOUT as the elem_layout - unsafe { self.inner.shrink_to_fit(cap, T::LAYOUT) } + unsafe { self.inner.shrink_to_fit(cap, T::LAYOUT, &self.alloc) } } /// Shrinks the buffer down to the specified capacity. If the given amount @@ -413,41 +471,45 @@ impl RawVec { /// Panics if the given amount is *larger* than the current capacity. #[inline] pub(crate) fn try_shrink_to_fit(&mut self, cap: usize) -> Result<(), TryReserveError> { - unsafe { self.inner.try_shrink_to_fit(cap, T::LAYOUT) } + unsafe { self.inner.try_shrink_to_fit(cap, T::LAYOUT, &self.alloc) } } } #[rustc_const_unstable(feature = "const_heap", issue = "79597")] const unsafe impl<#[may_dangle] T, A: [const] Allocator + [const] Destruct> Drop for RawVec { /// Frees the memory owned by the `RawVec` *without* trying to drop its contents. + #[inline(always)] fn drop(&mut self) { - // SAFETY: We are in a Drop impl, self.inner will not be used again. - unsafe { self.inner.deallocate(T::LAYOUT) } + // SAFETY: We are in a Drop impl, self.inner and self.alloc will not be used again. + // Copy to temporaries to prevent &self from escaping. + let inner = self.inner; + unsafe { inner.deallocate(T::LAYOUT, &self.alloc) } } } #[rustc_const_unstable(feature = "const_heap", issue = "79597")] #[rustfmt::skip] // FIXME(fee1-dead): temporary measure before rustfmt is bumped -const impl RawVecInner { +const impl RawVecInner { #[cfg(not(no_global_oom_handling))] #[inline] - fn with_capacity_in(capacity: usize, alloc: A, elem_layout: Layout) -> Self { - match Self::try_allocate_in(capacity, AllocInit::Uninitialized, alloc, elem_layout) { - Ok(this) => { + pub(crate) fn with_capacity_in(capacity: usize, alloc: &A, elem_layout: Layout) -> Self { + match RawVecInner::try_allocate_in(capacity, AllocInit::Uninitialized, alloc, elem_layout) { + Ok(inner) => { unsafe { // Make it more obvious that a subsequent Vec::reserve(capacity) will not allocate. - hint::assert_unchecked(!this.needs_to_grow(0, capacity, elem_layout)); + hint::assert_unchecked(!inner.needs_to_grow(0, capacity, elem_layout)); } - this + inner } Err(err) => handle_error(err), } } - fn try_allocate_in( + #[inline] + fn try_allocate_in( capacity: usize, init: AllocInit, - alloc: A, + alloc: &A, elem_layout: Layout, ) -> Result { // We avoid `unwrap_or_else` here because it bloats the amount of @@ -459,7 +521,7 @@ const impl RawVecInner { // Don't allocate here because `Drop` will not deallocate when `capacity` is 0. if layout.size() == 0 { - return Ok(Self::new_in(alloc, elem_layout.alignment())); + return Ok(Self::new(elem_layout.alignment())); } let result = match init { @@ -478,22 +540,129 @@ const impl RawVecInner { Ok(Self { ptr: Unique::from(ptr.cast()), cap: unsafe { Cap::new_unchecked(capacity) }, - alloc, }) } + /// Const-evaluable growth path used by `Vec::push`. `RawVec` moves the + /// allocator to a local before calling this so the vector fields can stay + /// local to the caller at runtime. + #[cfg(not(no_global_oom_handling))] + fn grow_one_const_select( + inner: Self, + alloc: &A, + elem_layout: Layout, + ) -> Self { + // SAFETY: The selector is only called by `RawVec::grow_one`, which + // always passes the element layout belonging to the allocation. + unsafe { inner.grow_one_const(alloc, elem_layout) } + } + + #[cfg(not(no_global_oom_handling))] + unsafe fn grow_one_const( + mut self, + alloc: &A, + elem_layout: Layout, + ) -> Self { + // SAFETY: Precondition passed to caller. + if let Err(err) = + unsafe { self.grow_amortized_const(self.cap.as_inner(), 1, elem_layout, alloc) } + { + handle_error(err); + } + self + } + /// # Safety - /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to - /// initially construct `self` + /// - `elem_layout` must be valid for `self` /// - `elem_layout`'s size must be a multiple of its alignment + /// - `len + additional` must be greater than the current capacity #[cfg(not(no_global_oom_handling))] + unsafe fn grow_amortized_const( + &mut self, + len: usize, + additional: usize, + elem_layout: Layout, + alloc: &A, + ) -> Result<(), TryReserveError> { + debug_assert!(additional > 0); + + if elem_layout.size() == 0 { + return Err(CapacityOverflow.into()); + } + + let required_cap = len.checked_add(additional).ok_or(CapacityOverflow)?; + let cap = cmp::max(self.cap.as_inner() * 2, required_cap); + let cap = cmp::max(min_non_zero_cap(elem_layout.size()), cap); + + // SAFETY: `cap` is greater than the current capacity and the other + // preconditions were passed to this function. + let ptr = unsafe { self.finish_grow_const(cap, elem_layout, alloc)? }; + // SAFETY: `finish_grow_const` rejects capacities above `isize::MAX`. + unsafe { self.set_ptr_and_cap(ptr, cap) }; + Ok(()) + } + + /// # Safety + /// - `elem_layout` must be valid for `self` + /// - `elem_layout`'s size must be a multiple of its alignment + /// - `cap` must be greater than the current capacity + #[cfg(not(no_global_oom_handling))] + #[cold] + unsafe fn finish_grow_const( + &self, + cap: usize, + elem_layout: Layout, + alloc: &A, + ) -> Result, TryReserveError> { + let new_layout = layout_array(cap, elem_layout)?; + + let memory = if let Some((ptr, old_layout)) = unsafe { self.current_memory(elem_layout) } { + debug_assert!(old_layout.align() == new_layout.align()); + unsafe { + hint::assert_unchecked(old_layout.align() == new_layout.align()); + alloc.grow(ptr, old_layout, new_layout) + } + } else { + alloc.allocate(new_layout) + }; + + match memory { + Ok(memory) => Ok(memory), + Err(_) => Err(AllocError { layout: new_layout, non_exhaustive: () }.into()), + } + } + + /// # Safety + /// + /// This should only be called once for a given `RawVecInner`. After this function any copies + /// of this `RawVecInner` are invalidated. #[inline] - unsafe fn grow_one(&mut self, elem_layout: Layout) { - // SAFETY: Precondition passed to caller - if let Err(err) = unsafe { self.grow_amortized(self.cap.as_inner(), 1, elem_layout) } { - handle_error(err); + unsafe fn deallocate(self, elem_layout: Layout, alloc: &A) { + // SAFETY: Precondition passed to caller. + if let Some((ptr, layout)) = unsafe { self.current_memory(elem_layout) } { + unsafe { alloc.deallocate(ptr, layout) }; } } +} + +impl RawVecInner { + #[cfg(not(no_global_oom_handling))] + #[inline(always)] + fn grow_one_runtime(inner: Self, alloc: &A, elem_layout: Layout) -> Self { + // Coerce the concrete allocator to a trait object in this small + // adapter; the substantial growth implementation is emitted once. + unsafe { inner.grow_one_outlined(alloc, elem_layout) } + } + + /// # Safety + /// `elem_layout` must be the layout used to create this allocation. + #[cfg(not(no_global_oom_handling))] + #[inline(never)] + unsafe fn grow_one_outlined(self, alloc: &dyn Allocator, elem_layout: Layout) -> Self { + // Calling a conditionally-const function at runtime only requires the + // ordinary `Allocator` implementation, which `dyn Allocator` has. + unsafe { self.grow_one_const(alloc, elem_layout) } + } /// # Safety /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to @@ -505,6 +674,7 @@ const impl RawVecInner { len: usize, additional: usize, elem_layout: Layout, + alloc: &dyn Allocator, ) -> Result<(), TryReserveError> { // This is ensured by the calling contexts. debug_assert!(additional > 0); @@ -526,7 +696,7 @@ const impl RawVecInner { // SAFETY: // - cap >= len + additional // - other preconditions passed to caller - let ptr = unsafe { self.finish_grow(cap, elem_layout)? }; + let ptr = unsafe { self.finish_grow(cap, elem_layout, alloc)? }; // SAFETY: `finish_grow` would have failed if `cap > isize::MAX` unsafe { self.set_ptr_and_cap(ptr, cap) }; @@ -545,6 +715,7 @@ const impl RawVecInner { &self, cap: usize, elem_layout: Layout, + alloc: &dyn Allocator, ) -> Result, TryReserveError> { let new_layout = layout_array(cap, elem_layout)?; @@ -554,51 +725,31 @@ const impl RawVecInner { unsafe { // The allocator checks for alignment equality hint::assert_unchecked(old_layout.align() == new_layout.align()); - self.alloc.grow(ptr, old_layout, new_layout) + alloc.grow(ptr, old_layout, new_layout) } } else { - self.alloc.allocate(new_layout) + alloc.allocate(new_layout) }; - memory.map_err(const |_| AllocError { layout: new_layout, non_exhaustive: () }.into()) + memory.map_err(|_| AllocError { layout: new_layout, non_exhaustive: () }.into()) } -} -impl RawVecInner { #[inline] - const fn new_in(alloc: A, align: Alignment) -> Self { + const fn new(align: Alignment) -> Self { let ptr = Unique::from_non_null(NonNull::without_provenance(align.as_nonzero_usize())); // `cap: 0` means "unallocated". zero-sized types are ignored. - Self { ptr, cap: ZERO_CAP, alloc } - } - - #[inline] - fn try_with_capacity_in( - capacity: usize, - alloc: A, - elem_layout: Layout, - ) -> Result { - Self::try_allocate_in(capacity, AllocInit::Uninitialized, alloc, elem_layout) - } - - #[cfg(not(no_global_oom_handling))] - #[inline] - fn with_capacity_zeroed_in(capacity: usize, alloc: A, elem_layout: Layout) -> Self { - match Self::try_allocate_in(capacity, AllocInit::Zeroed, alloc, elem_layout) { - Ok(res) => res, - Err(err) => handle_error(err), - } + Self { ptr, cap: ZERO_CAP } } #[inline] - const unsafe fn from_raw_parts_in(ptr: *mut u8, cap: Cap, alloc: A) -> Self { - Self { ptr: unsafe { Unique::new_unchecked(ptr) }, cap, alloc } + const unsafe fn from_raw_parts(ptr: *mut u8, cap: Cap) -> Self { + Self { ptr: unsafe { Unique::new_unchecked(ptr) }, cap } } #[inline] #[rustc_const_unstable(feature = "const_heap", issue = "79597")] - const unsafe fn from_nonnull_in(ptr: NonNull, cap: Cap, alloc: A) -> Self { - Self { ptr: Unique::from(ptr), cap, alloc } + const unsafe fn from_nonnull(ptr: NonNull, cap: Cap) -> Self { + Self { ptr: Unique::from(ptr), cap } } #[inline] @@ -616,11 +767,6 @@ impl RawVecInner { if elem_size == 0 { usize::MAX } else { self.cap.as_inner() } } - #[inline] - const fn allocator(&self) -> &A { - &self.alloc - } - /// # Safety /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to /// initially construct `self` @@ -649,27 +795,34 @@ impl RawVecInner { /// - `elem_layout`'s size must be a multiple of its alignment #[cfg(not(no_global_oom_handling))] #[inline] - unsafe fn reserve(&mut self, len: usize, additional: usize, elem_layout: Layout) { + unsafe fn reserve( + &mut self, + len: usize, + additional: usize, + elem_layout: Layout, + alloc: &A, + ) { // Callers expect this function to be very cheap when there is already sufficient capacity. // Therefore, we move all the resizing and error-handling logic from grow_amortized and // handle_reserve behind a call, while making sure that this function is likely to be // inlined as just a comparison and a call if the comparison fails. #[cold] - unsafe fn do_reserve_and_handle( - slf: &mut RawVecInner, + unsafe fn do_reserve_and_handle( + slf: &mut RawVecInner, len: usize, additional: usize, elem_layout: Layout, + alloc: &dyn Allocator, ) { // SAFETY: Precondition passed to caller - if let Err(err) = unsafe { slf.grow_amortized(len, additional, elem_layout) } { + if let Err(err) = unsafe { slf.grow_amortized(len, additional, elem_layout, alloc) } { handle_error(err); } } if self.needs_to_grow(len, additional, elem_layout) { unsafe { - do_reserve_and_handle(self, len, additional, elem_layout); + do_reserve_and_handle(self, len, additional, elem_layout, alloc); } } } @@ -678,20 +831,22 @@ impl RawVecInner { /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to /// initially construct `self` /// - `elem_layout`'s size must be a multiple of its alignment - unsafe fn try_reserve( + #[inline] + unsafe fn try_reserve( &mut self, len: usize, additional: usize, elem_layout: Layout, + alloc: &A, ) -> Result<(), TryReserveError> { if self.needs_to_grow(len, additional, elem_layout) { // SAFETY: Precondition passed to caller unsafe { - self.grow_amortized(len, additional, elem_layout)?; + self.grow_amortized(len, additional, elem_layout, alloc)?; } } unsafe { - // Inform the optimizer that the reservation has succeeded or wasn't needed + // Inform the optimizer that the reservation has succeeded or wasn't needed. hint::assert_unchecked(!self.needs_to_grow(len, additional, elem_layout)); } Ok(()) @@ -702,9 +857,15 @@ impl RawVecInner { /// initially construct `self` /// - `elem_layout`'s size must be a multiple of its alignment #[cfg(not(no_global_oom_handling))] - unsafe fn reserve_exact(&mut self, len: usize, additional: usize, elem_layout: Layout) { + unsafe fn reserve_exact( + &mut self, + len: usize, + additional: usize, + elem_layout: Layout, + alloc: &A, + ) { // SAFETY: Precondition passed to caller - if let Err(err) = unsafe { self.try_reserve_exact(len, additional, elem_layout) } { + if let Err(err) = unsafe { self.try_reserve_exact(len, additional, elem_layout, alloc) } { handle_error(err); } } @@ -713,20 +874,22 @@ impl RawVecInner { /// - `elem_layout` must be valid for `self`, i.e. it must be the same `elem_layout` used to /// initially construct `self` /// - `elem_layout`'s size must be a multiple of its alignment - unsafe fn try_reserve_exact( + #[inline] + unsafe fn try_reserve_exact( &mut self, len: usize, additional: usize, elem_layout: Layout, + alloc: &A, ) -> Result<(), TryReserveError> { if self.needs_to_grow(len, additional, elem_layout) { // SAFETY: Precondition passed to caller unsafe { - self.grow_exact(len, additional, elem_layout)?; + self.grow_exact(len, additional, elem_layout, alloc)?; } } unsafe { - // Inform the optimizer that the reservation has succeeded or wasn't needed + // Inform the optimizer that the reservation has succeeded or wasn't needed. hint::assert_unchecked(!self.needs_to_grow(len, additional, elem_layout)); } Ok(()) @@ -739,8 +902,8 @@ impl RawVecInner { /// - `cap` must be less than or equal to `self.capacity(elem_layout.size())` #[cfg(not(no_global_oom_handling))] #[inline] - unsafe fn shrink_to_fit(&mut self, cap: usize, elem_layout: Layout) { - if let Err(err) = unsafe { self.shrink(cap, elem_layout) } { + unsafe fn shrink_to_fit(&mut self, cap: usize, elem_layout: Layout, alloc: &A) { + if let Err(err) = unsafe { self.shrink(cap, elem_layout, alloc) } { handle_error(err); } } @@ -751,12 +914,13 @@ impl RawVecInner { /// initially construct `self` /// - `elem_layout`'s size must be a multiple of its alignment /// - `cap` must be less than or equal to `self.capacity(elem_layout.size())` - unsafe fn try_shrink_to_fit( + unsafe fn try_shrink_to_fit( &mut self, cap: usize, elem_layout: Layout, + alloc: &A, ) -> Result<(), TryReserveError> { - unsafe { self.shrink(cap, elem_layout) } + unsafe { self.shrink(cap, elem_layout, alloc) } } #[inline] @@ -784,6 +948,7 @@ impl RawVecInner { len: usize, additional: usize, elem_layout: Layout, + alloc: &dyn Allocator, ) -> Result<(), TryReserveError> { if elem_layout.size() == 0 { // Since we return a capacity of `usize::MAX` when the type size is @@ -794,7 +959,7 @@ impl RawVecInner { let cap = len.checked_add(additional).ok_or(CapacityOverflow)?; // SAFETY: preconditions passed to caller - let ptr = unsafe { self.finish_grow(cap, elem_layout)? }; + let ptr = unsafe { self.finish_grow(cap, elem_layout, alloc)? }; // SAFETY: `finish_grow` would have failed if `cap > isize::MAX` unsafe { self.set_ptr_and_cap(ptr, cap) }; @@ -806,11 +971,17 @@ impl RawVecInner { /// initially construct `self` /// - `elem_layout`'s size must be a multiple of its alignment /// - `cap` must be less than or equal to `self.capacity(elem_layout.size())` + #[cfg(not(no_global_oom_handling))] #[inline] - unsafe fn shrink(&mut self, cap: usize, elem_layout: Layout) -> Result<(), TryReserveError> { + unsafe fn shrink( + &mut self, + cap: usize, + elem_layout: Layout, + alloc: &A, + ) -> Result<(), TryReserveError> { assert!(cap <= self.capacity(elem_layout.size()), "Tried to shrink to a larger capacity"); // SAFETY: Just checked this isn't trying to grow - unsafe { self.shrink_unchecked(cap, elem_layout) } + unsafe { self.shrink_unchecked(cap, elem_layout, alloc) } } /// `shrink`, but without the capacity check. @@ -827,6 +998,7 @@ impl RawVecInner { &mut self, cap: usize, elem_layout: Layout, + alloc: &dyn Allocator, ) -> Result<(), TryReserveError> { // SAFETY: Precondition passed to caller let Some((ptr, layout)) = (unsafe { self.current_memory(elem_layout) }) else { @@ -837,7 +1009,7 @@ impl RawVecInner { // for the T::IS_ZST case since current_memory() will have returned // None. if cap == 0 { - unsafe { self.alloc.deallocate(ptr, layout) }; + unsafe { alloc.deallocate(ptr, layout) }; self.ptr = unsafe { Unique::new_unchecked(ptr::without_provenance_mut(elem_layout.align())) }; self.cap = ZERO_CAP; @@ -847,7 +1019,7 @@ impl RawVecInner { // overflowed earlier when capacity was larger. let new_size = elem_layout.size().unchecked_mul(cap); let new_layout = Layout::from_size_align_unchecked(new_size, layout.align()); - self.alloc + alloc .shrink(ptr, layout, new_layout) .map_err(|_| AllocError { layout: new_layout, non_exhaustive: () })? }; @@ -860,25 +1032,6 @@ impl RawVecInner { } } -#[rustc_const_unstable(feature = "const_heap", issue = "79597")] -const impl RawVecInner { - /// # Safety - /// - /// This function deallocates the owned allocation, but does not update `ptr` or `cap` to - /// prevent double-free or use-after-free. Essentially, do not do anything with the caller - /// after this function returns. - /// Ideally this function would take `self` by move, but it cannot because it exists to be - /// called from a `Drop` impl. - unsafe fn deallocate(&mut self, elem_layout: Layout) { - // SAFETY: Precondition passed to caller - if let Some((ptr, layout)) = unsafe { self.current_memory(elem_layout) } { - unsafe { - self.alloc.deallocate(ptr, layout); - } - } - } -} - // Central function for reserve error handling. #[cfg(not(no_global_oom_handling))] #[cold] diff --git a/library/alloc/src/raw_vec/tests.rs b/library/alloc/src/raw_vec/tests.rs index 15f48c03dc54c..d80d66c832fa3 100644 --- a/library/alloc/src/raw_vec/tests.rs +++ b/library/alloc/src/raw_vec/tests.rs @@ -42,9 +42,9 @@ fn allocator_param() { let a = BoundedAlloc { fuel: Cell::new(500) }; let mut v: RawVec = RawVec::with_capacity_in(50, a); - assert_eq!(v.inner.alloc.fuel.get(), 450); + assert_eq!(v.allocator().fuel.get(), 450); v.reserve(50, 150); // (causes a realloc, thus using 50 + 150 = 200 units of fuel) - assert_eq!(v.inner.alloc.fuel.get(), 250); + assert_eq!(v.allocator().fuel.get(), 250); } #[test] @@ -126,12 +126,18 @@ fn zst() { assert_eq!(v.try_reserve_exact(101, usize::MAX - 100), cap_err); zst_sanity(&v); - assert_eq!(unsafe { v.inner.grow_amortized(100, usize::MAX - 100, ZST::LAYOUT) }, cap_err); - assert_eq!(unsafe { v.inner.grow_amortized(101, usize::MAX - 100, ZST::LAYOUT) }, cap_err); + assert_eq!( + unsafe { v.inner.grow_amortized(100, usize::MAX - 100, ZST::LAYOUT, &Global) }, + cap_err + ); + assert_eq!( + unsafe { v.inner.grow_amortized(101, usize::MAX - 100, ZST::LAYOUT, &Global) }, + cap_err + ); zst_sanity(&v); - assert_eq!(unsafe { v.inner.grow_exact(100, usize::MAX - 100, ZST::LAYOUT) }, cap_err); - assert_eq!(unsafe { v.inner.grow_exact(101, usize::MAX - 100, ZST::LAYOUT) }, cap_err); + assert_eq!(unsafe { v.inner.grow_exact(100, usize::MAX - 100, ZST::LAYOUT, &Global) }, cap_err); + assert_eq!(unsafe { v.inner.grow_exact(101, usize::MAX - 100, ZST::LAYOUT, &Global) }, cap_err); zst_sanity(&v); } diff --git a/library/alloc/src/vec/mod.rs b/library/alloc/src/vec/mod.rs index 7965a10c98289..c7c06c6e0037a 100644 --- a/library/alloc/src/vec/mod.rs +++ b/library/alloc/src/vec/mod.rs @@ -1528,7 +1528,14 @@ impl Vec { /// ``` #[stable(feature = "try_reserve", since = "1.57.0")] pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> { - self.buf.try_reserve(self.len, additional) + let len = self.len; + let result = self.buf.try_reserve(len, additional); + unsafe { + // The buffer and its allocator cannot change the vector's length. + // Keep that fact visible when allocator operations are type-erased. + hint::assert_unchecked(self.len == len); + } + result } /// Tries to reserve the minimum capacity for at least `additional` @@ -1571,7 +1578,12 @@ impl Vec { /// ``` #[stable(feature = "try_reserve", since = "1.57.0")] pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> { - self.buf.try_reserve_exact(self.len, additional) + let len = self.len; + let result = self.buf.try_reserve_exact(len, additional); + unsafe { + hint::assert_unchecked(self.len == len); + } + result } /// Shrinks the capacity of the vector as much as possible. diff --git a/library/alloctests/benches/vec.rs b/library/alloctests/benches/vec.rs index 656164da72084..0876329cf017d 100644 --- a/library/alloctests/benches/vec.rs +++ b/library/alloctests/benches/vec.rs @@ -3,6 +3,59 @@ use std::iter::repeat; use rand::RngCore; use test::{Bencher, black_box}; +fn do_bench_push(b: &mut Bencher, n: usize) { + b.iter(|| { + let mut v = Vec::new(); + for i in 0..n { + v.push(i); + } + v + }); +} + +#[bench] +fn bench_push_0100(b: &mut Bencher) { + do_bench_push(b, 100); +} + +#[bench] +fn bench_push_1000(b: &mut Bencher) { + do_bench_push(b, 1000); +} + +#[bench] +fn bench_push_10000(b: &mut Bencher) { + do_bench_push(b, 10000); +} + +#[inline(never)] +fn push_preallocated(n: usize) -> Vec { + let mut v = Vec::with_capacity(n); + for i in 0..n { + v.push(i); + } + v +} + +fn do_bench_push_preallocated(b: &mut Bencher, n: usize) { + b.iter(|| push_preallocated(n)); +} + +#[bench] +fn bench_push_preallocated_0100(b: &mut Bencher) { + do_bench_push_preallocated(b, 100); +} + +#[bench] +fn bench_push_preallocated_1000(b: &mut Bencher) { + do_bench_push_preallocated(b, 1000); +} + +#[bench] +fn bench_push_preallocated_10000(b: &mut Bencher) { + do_bench_push_preallocated(b, 10000); +} + #[bench] fn bench_new(b: &mut Bencher) { b.iter(|| Vec::::new()) diff --git a/library/alloctests/lib.rs b/library/alloctests/lib.rs index eca8444812521..1c2022ad51bed 100644 --- a/library/alloctests/lib.rs +++ b/library/alloctests/lib.rs @@ -52,6 +52,7 @@ #![feature(trusted_random_access)] #![feature(try_reserve_kind)] #![feature(try_trait_v2)] +#![feature(tuple_trait)] #![feature(unwrap_infallible)] #![feature(wtf8_internals)] // tidy-alphabetical-end @@ -61,12 +62,14 @@ #![feature(const_closures)] #![feature(const_trait_impl)] #![feature(dropck_eyepatch)] +#![feature(intrinsics)] #![feature(min_specialization)] #![feature(optimize_attribute)] #![feature(prelude_import)] #![feature(rustc_attrs)] #![feature(staged_api)] #![feature(test)] +#![feature(unboxed_closures)] #![rustc_preserve_ub_checks] // tidy-alphabetical-end