xref: /linux-6.15/rust/kernel/sync/arc.rs (revision 08d3f549)
1 // SPDX-License-Identifier: GPL-2.0
2 
3 //! A reference-counted pointer.
4 //!
5 //! This module implements a way for users to create reference-counted objects and pointers to
6 //! them. Such a pointer automatically increments and decrements the count, and drops the
7 //! underlying object when it reaches zero. It is also safe to use concurrently from multiple
8 //! threads.
9 //!
10 //! It is different from the standard library's [`Arc`] in a few ways:
11 //! 1. It is backed by the kernel's `refcount_t` type.
12 //! 2. It does not support weak references, which allows it to be half the size.
13 //! 3. It saturates the reference count instead of aborting when it goes over a threshold.
14 //! 4. It does not provide a `get_mut` method, so the ref counted object is pinned.
15 //!
16 //! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html
17 
18 use crate::{
19     alloc::{box_ext::BoxExt, flags::*},
20     bindings,
21     error::{self, Error},
22     init::{self, InPlaceInit, Init, PinInit},
23     try_init,
24     types::{ForeignOwnable, Opaque},
25 };
26 use alloc::boxed::Box;
27 use core::{
28     alloc::{AllocError, Layout},
29     fmt,
30     marker::{PhantomData, Unsize},
31     mem::{ManuallyDrop, MaybeUninit},
32     ops::{Deref, DerefMut},
33     pin::Pin,
34     ptr::NonNull,
35 };
36 use macros::pin_data;
37 
38 mod std_vendor;
39 
40 /// A reference-counted pointer to an instance of `T`.
41 ///
42 /// The reference count is incremented when new instances of [`Arc`] are created, and decremented
43 /// when they are dropped. When the count reaches zero, the underlying `T` is also dropped.
44 ///
45 /// # Invariants
46 ///
47 /// The reference count on an instance of [`Arc`] is always non-zero.
48 /// The object pointed to by [`Arc`] is always pinned.
49 ///
50 /// # Examples
51 ///
52 /// ```
53 /// use kernel::sync::Arc;
54 ///
55 /// struct Example {
56 ///     a: u32,
57 ///     b: u32,
58 /// }
59 ///
60 /// // Create a refcounted instance of `Example`.
61 /// let obj = Arc::try_new(Example { a: 10, b: 20 })?;
62 ///
63 /// // Get a new pointer to `obj` and increment the refcount.
64 /// let cloned = obj.clone();
65 ///
66 /// // Assert that both `obj` and `cloned` point to the same underlying object.
67 /// assert!(core::ptr::eq(&*obj, &*cloned));
68 ///
69 /// // Destroy `obj` and decrement its refcount.
70 /// drop(obj);
71 ///
72 /// // Check that the values are still accessible through `cloned`.
73 /// assert_eq!(cloned.a, 10);
74 /// assert_eq!(cloned.b, 20);
75 ///
76 /// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed.
77 /// # Ok::<(), Error>(())
78 /// ```
79 ///
80 /// Using `Arc<T>` as the type of `self`:
81 ///
82 /// ```
83 /// use kernel::sync::Arc;
84 ///
85 /// struct Example {
86 ///     a: u32,
87 ///     b: u32,
88 /// }
89 ///
90 /// impl Example {
91 ///     fn take_over(self: Arc<Self>) {
92 ///         // ...
93 ///     }
94 ///
95 ///     fn use_reference(self: &Arc<Self>) {
96 ///         // ...
97 ///     }
98 /// }
99 ///
100 /// let obj = Arc::try_new(Example { a: 10, b: 20 })?;
101 /// obj.use_reference();
102 /// obj.take_over();
103 /// # Ok::<(), Error>(())
104 /// ```
105 ///
106 /// Coercion from `Arc<Example>` to `Arc<dyn MyTrait>`:
107 ///
108 /// ```
109 /// use kernel::sync::{Arc, ArcBorrow};
110 ///
111 /// trait MyTrait {
112 ///     // Trait has a function whose `self` type is `Arc<Self>`.
113 ///     fn example1(self: Arc<Self>) {}
114 ///
115 ///     // Trait has a function whose `self` type is `ArcBorrow<'_, Self>`.
116 ///     fn example2(self: ArcBorrow<'_, Self>) {}
117 /// }
118 ///
119 /// struct Example;
120 /// impl MyTrait for Example {}
121 ///
122 /// // `obj` has type `Arc<Example>`.
123 /// let obj: Arc<Example> = Arc::try_new(Example)?;
124 ///
125 /// // `coerced` has type `Arc<dyn MyTrait>`.
126 /// let coerced: Arc<dyn MyTrait> = obj;
127 /// # Ok::<(), Error>(())
128 /// ```
129 pub struct Arc<T: ?Sized> {
130     ptr: NonNull<ArcInner<T>>,
131     _p: PhantomData<ArcInner<T>>,
132 }
133 
134 #[pin_data]
135 #[repr(C)]
136 struct ArcInner<T: ?Sized> {
137     refcount: Opaque<bindings::refcount_t>,
138     data: T,
139 }
140 
141 // This is to allow [`Arc`] (and variants) to be used as the type of `self`.
142 impl<T: ?Sized> core::ops::Receiver for Arc<T> {}
143 
144 // This is to allow coercion from `Arc<T>` to `Arc<U>` if `T` can be converted to the
145 // dynamically-sized type (DST) `U`.
146 impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {}
147 
148 // This is to allow `Arc<U>` to be dispatched on when `Arc<T>` can be coerced into `Arc<U>`.
149 impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<Arc<U>> for Arc<T> {}
150 
151 // SAFETY: It is safe to send `Arc<T>` to another thread when the underlying `T` is `Sync` because
152 // it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs
153 // `T` to be `Send` because any thread that has an `Arc<T>` may ultimately access `T` using a
154 // mutable reference when the reference count reaches zero and `T` is dropped.
155 unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {}
156 
157 // SAFETY: It is safe to send `&Arc<T>` to another thread when the underlying `T` is `Sync`
158 // because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally,
159 // it needs `T` to be `Send` because any thread that has a `&Arc<T>` may clone it and get an
160 // `Arc<T>` on that thread, so the thread may ultimately access `T` using a mutable reference when
161 // the reference count reaches zero and `T` is dropped.
162 unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {}
163 
164 impl<T> Arc<T> {
165     /// Constructs a new reference counted instance of `T`.
166     pub fn try_new(contents: T) -> Result<Self, AllocError> {
167         // INVARIANT: The refcount is initialised to a non-zero value.
168         let value = ArcInner {
169             // SAFETY: There are no safety requirements for this FFI call.
170             refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }),
171             data: contents,
172         };
173 
174         let inner = <Box<_> as BoxExt<_>>::new(value, GFP_KERNEL)?;
175 
176         // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new
177         // `Arc` object.
178         Ok(unsafe { Self::from_inner(Box::leak(inner).into()) })
179     }
180 
181     /// Use the given initializer to in-place initialize a `T`.
182     ///
183     /// If `T: !Unpin` it will not be able to move afterwards.
184     #[inline]
185     pub fn pin_init<E>(init: impl PinInit<T, E>) -> error::Result<Self>
186     where
187         Error: From<E>,
188     {
189         UniqueArc::pin_init(init).map(|u| u.into())
190     }
191 
192     /// Use the given initializer to in-place initialize a `T`.
193     ///
194     /// This is equivalent to [`Arc<T>::pin_init`], since an [`Arc`] is always pinned.
195     #[inline]
196     pub fn init<E>(init: impl Init<T, E>) -> error::Result<Self>
197     where
198         Error: From<E>,
199     {
200         UniqueArc::init(init).map(|u| u.into())
201     }
202 }
203 
204 impl<T: ?Sized> Arc<T> {
205     /// Constructs a new [`Arc`] from an existing [`ArcInner`].
206     ///
207     /// # Safety
208     ///
209     /// The caller must ensure that `inner` points to a valid location and has a non-zero reference
210     /// count, one of which will be owned by the new [`Arc`] instance.
211     unsafe fn from_inner(inner: NonNull<ArcInner<T>>) -> Self {
212         // INVARIANT: By the safety requirements, the invariants hold.
213         Arc {
214             ptr: inner,
215             _p: PhantomData,
216         }
217     }
218 
219     /// Convert the [`Arc`] into a raw pointer.
220     ///
221     /// The raw pointer has ownership of the refcount that this Arc object owned.
222     pub fn into_raw(self) -> *const T {
223         let ptr = self.ptr.as_ptr();
224         core::mem::forget(self);
225         // SAFETY: The pointer is valid.
226         unsafe { core::ptr::addr_of!((*ptr).data) }
227     }
228 
229     /// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`].
230     ///
231     /// # Safety
232     ///
233     /// `ptr` must have been returned by a previous call to [`Arc::into_raw`]. Additionally, it
234     /// must not be called more than once for each previous call to [`Arc::into_raw`].
235     pub unsafe fn from_raw(ptr: *const T) -> Self {
236         let refcount_layout = Layout::new::<bindings::refcount_t>();
237         // SAFETY: The caller guarantees that the pointer is valid.
238         let val_layout = Layout::for_value(unsafe { &*ptr });
239         // SAFETY: We're computing the layout of a real struct that existed when compiling this
240         // binary, so its layout is not so large that it can trigger arithmetic overflow.
241         let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 };
242 
243         // Pointer casts leave the metadata unchanged. This is okay because the metadata of `T` and
244         // `ArcInner<T>` is the same since `ArcInner` is a struct with `T` as its last field.
245         //
246         // This is documented at:
247         // <https://doc.rust-lang.org/std/ptr/trait.Pointee.html>.
248         let ptr = ptr as *const ArcInner<T>;
249 
250         // SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the
251         // pointer, since it originates from a previous call to `Arc::into_raw` and is still valid.
252         let ptr = unsafe { ptr.byte_sub(val_offset) };
253 
254         // SAFETY: By the safety requirements we know that `ptr` came from `Arc::into_raw`, so the
255         // reference count held then will be owned by the new `Arc` object.
256         unsafe { Self::from_inner(NonNull::new_unchecked(ptr.cast_mut())) }
257     }
258 
259     /// Returns an [`ArcBorrow`] from the given [`Arc`].
260     ///
261     /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method
262     /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised.
263     #[inline]
264     pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> {
265         // SAFETY: The constraint that the lifetime of the shared reference must outlive that of
266         // the returned `ArcBorrow` ensures that the object remains alive and that no mutable
267         // reference can be created.
268         unsafe { ArcBorrow::new(self.ptr) }
269     }
270 
271     /// Compare whether two [`Arc`] pointers reference the same underlying object.
272     pub fn ptr_eq(this: &Self, other: &Self) -> bool {
273         core::ptr::eq(this.ptr.as_ptr(), other.ptr.as_ptr())
274     }
275 }
276 
277 impl<T: 'static> ForeignOwnable for Arc<T> {
278     type Borrowed<'a> = ArcBorrow<'a, T>;
279 
280     fn into_foreign(self) -> *const core::ffi::c_void {
281         ManuallyDrop::new(self).ptr.as_ptr() as _
282     }
283 
284     unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> ArcBorrow<'a, T> {
285         // SAFETY: By the safety requirement of this function, we know that `ptr` came from
286         // a previous call to `Arc::into_foreign`.
287         let inner = NonNull::new(ptr as *mut ArcInner<T>).unwrap();
288 
289         // SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive
290         // for the lifetime of the returned value.
291         unsafe { ArcBorrow::new(inner) }
292     }
293 
294     unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self {
295         // SAFETY: By the safety requirement of this function, we know that `ptr` came from
296         // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and
297         // holds a reference count increment that is transferrable to us.
298         unsafe { Self::from_inner(NonNull::new(ptr as _).unwrap()) }
299     }
300 }
301 
302 impl<T: ?Sized> Deref for Arc<T> {
303     type Target = T;
304 
305     fn deref(&self) -> &Self::Target {
306         // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
307         // safe to dereference it.
308         unsafe { &self.ptr.as_ref().data }
309     }
310 }
311 
312 impl<T: ?Sized> AsRef<T> for Arc<T> {
313     fn as_ref(&self) -> &T {
314         self.deref()
315     }
316 }
317 
318 impl<T: ?Sized> Clone for Arc<T> {
319     fn clone(&self) -> Self {
320         // INVARIANT: C `refcount_inc` saturates the refcount, so it cannot overflow to zero.
321         // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
322         // safe to increment the refcount.
323         unsafe { bindings::refcount_inc(self.ptr.as_ref().refcount.get()) };
324 
325         // SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`.
326         unsafe { Self::from_inner(self.ptr) }
327     }
328 }
329 
330 impl<T: ?Sized> Drop for Arc<T> {
331     fn drop(&mut self) {
332         // SAFETY: By the type invariant, there is necessarily a reference to the object. We cannot
333         // touch `refcount` after it's decremented to a non-zero value because another thread/CPU
334         // may concurrently decrement it to zero and free it. It is ok to have a raw pointer to
335         // freed/invalid memory as long as it is never dereferenced.
336         let refcount = unsafe { self.ptr.as_ref() }.refcount.get();
337 
338         // INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and
339         // this instance is being dropped, so the broken invariant is not observable.
340         // SAFETY: Also by the type invariant, we are allowed to decrement the refcount.
341         let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) };
342         if is_zero {
343             // The count reached zero, we must free the memory.
344             //
345             // SAFETY: The pointer was initialised from the result of `Box::leak`.
346             unsafe { drop(Box::from_raw(self.ptr.as_ptr())) };
347         }
348     }
349 }
350 
351 impl<T: ?Sized> From<UniqueArc<T>> for Arc<T> {
352     fn from(item: UniqueArc<T>) -> Self {
353         item.inner
354     }
355 }
356 
357 impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> {
358     fn from(item: Pin<UniqueArc<T>>) -> Self {
359         // SAFETY: The type invariants of `Arc` guarantee that the data is pinned.
360         unsafe { Pin::into_inner_unchecked(item).inner }
361     }
362 }
363 
364 /// A borrowed reference to an [`Arc`] instance.
365 ///
366 /// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler
367 /// to use just `&T`, which we can trivially get from an [`Arc<T>`] instance.
368 ///
369 /// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow<T>`
370 /// over `&Arc<T>` because the latter results in a double-indirection: a pointer (shared reference)
371 /// to a pointer ([`Arc<T>`]) to the object (`T`). An [`ArcBorrow`] eliminates this double
372 /// indirection while still allowing one to increment the refcount and getting an [`Arc<T>`] when/if
373 /// needed.
374 ///
375 /// # Invariants
376 ///
377 /// There are no mutable references to the underlying [`Arc`], and it remains valid for the
378 /// lifetime of the [`ArcBorrow`] instance.
379 ///
380 /// # Example
381 ///
382 /// ```
383 /// use kernel::sync::{Arc, ArcBorrow};
384 ///
385 /// struct Example;
386 ///
387 /// fn do_something(e: ArcBorrow<'_, Example>) -> Arc<Example> {
388 ///     e.into()
389 /// }
390 ///
391 /// let obj = Arc::try_new(Example)?;
392 /// let cloned = do_something(obj.as_arc_borrow());
393 ///
394 /// // Assert that both `obj` and `cloned` point to the same underlying object.
395 /// assert!(core::ptr::eq(&*obj, &*cloned));
396 /// # Ok::<(), Error>(())
397 /// ```
398 ///
399 /// Using `ArcBorrow<T>` as the type of `self`:
400 ///
401 /// ```
402 /// use kernel::sync::{Arc, ArcBorrow};
403 ///
404 /// struct Example {
405 ///     a: u32,
406 ///     b: u32,
407 /// }
408 ///
409 /// impl Example {
410 ///     fn use_reference(self: ArcBorrow<'_, Self>) {
411 ///         // ...
412 ///     }
413 /// }
414 ///
415 /// let obj = Arc::try_new(Example { a: 10, b: 20 })?;
416 /// obj.as_arc_borrow().use_reference();
417 /// # Ok::<(), Error>(())
418 /// ```
419 pub struct ArcBorrow<'a, T: ?Sized + 'a> {
420     inner: NonNull<ArcInner<T>>,
421     _p: PhantomData<&'a ()>,
422 }
423 
424 // This is to allow [`ArcBorrow`] (and variants) to be used as the type of `self`.
425 impl<T: ?Sized> core::ops::Receiver for ArcBorrow<'_, T> {}
426 
427 // This is to allow `ArcBorrow<U>` to be dispatched on when `ArcBorrow<T>` can be coerced into
428 // `ArcBorrow<U>`.
429 impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<ArcBorrow<'_, U>>
430     for ArcBorrow<'_, T>
431 {
432 }
433 
434 impl<T: ?Sized> Clone for ArcBorrow<'_, T> {
435     fn clone(&self) -> Self {
436         *self
437     }
438 }
439 
440 impl<T: ?Sized> Copy for ArcBorrow<'_, T> {}
441 
442 impl<T: ?Sized> ArcBorrow<'_, T> {
443     /// Creates a new [`ArcBorrow`] instance.
444     ///
445     /// # Safety
446     ///
447     /// Callers must ensure the following for the lifetime of the returned [`ArcBorrow`] instance:
448     /// 1. That `inner` remains valid;
449     /// 2. That no mutable references to `inner` are created.
450     unsafe fn new(inner: NonNull<ArcInner<T>>) -> Self {
451         // INVARIANT: The safety requirements guarantee the invariants.
452         Self {
453             inner,
454             _p: PhantomData,
455         }
456     }
457 }
458 
459 impl<T: ?Sized> From<ArcBorrow<'_, T>> for Arc<T> {
460     fn from(b: ArcBorrow<'_, T>) -> Self {
461         // SAFETY: The existence of `b` guarantees that the refcount is non-zero. `ManuallyDrop`
462         // guarantees that `drop` isn't called, so it's ok that the temporary `Arc` doesn't own the
463         // increment.
464         ManuallyDrop::new(unsafe { Arc::from_inner(b.inner) })
465             .deref()
466             .clone()
467     }
468 }
469 
470 impl<T: ?Sized> Deref for ArcBorrow<'_, T> {
471     type Target = T;
472 
473     fn deref(&self) -> &Self::Target {
474         // SAFETY: By the type invariant, the underlying object is still alive with no mutable
475         // references to it, so it is safe to create a shared reference.
476         unsafe { &self.inner.as_ref().data }
477     }
478 }
479 
480 /// A refcounted object that is known to have a refcount of 1.
481 ///
482 /// It is mutable and can be converted to an [`Arc`] so that it can be shared.
483 ///
484 /// # Invariants
485 ///
486 /// `inner` always has a reference count of 1.
487 ///
488 /// # Examples
489 ///
490 /// In the following example, we make changes to the inner object before turning it into an
491 /// `Arc<Test>` object (after which point, it cannot be mutated directly). Note that `x.into()`
492 /// cannot fail.
493 ///
494 /// ```
495 /// use kernel::sync::{Arc, UniqueArc};
496 ///
497 /// struct Example {
498 ///     a: u32,
499 ///     b: u32,
500 /// }
501 ///
502 /// fn test() -> Result<Arc<Example>> {
503 ///     let mut x = UniqueArc::try_new(Example { a: 10, b: 20 })?;
504 ///     x.a += 1;
505 ///     x.b += 1;
506 ///     Ok(x.into())
507 /// }
508 ///
509 /// # test().unwrap();
510 /// ```
511 ///
512 /// In the following example we first allocate memory for a refcounted `Example` but we don't
513 /// initialise it on allocation. We do initialise it later with a call to [`UniqueArc::write`],
514 /// followed by a conversion to `Arc<Example>`. This is particularly useful when allocation happens
515 /// in one context (e.g., sleepable) and initialisation in another (e.g., atomic):
516 ///
517 /// ```
518 /// use kernel::sync::{Arc, UniqueArc};
519 ///
520 /// struct Example {
521 ///     a: u32,
522 ///     b: u32,
523 /// }
524 ///
525 /// fn test() -> Result<Arc<Example>> {
526 ///     let x = UniqueArc::try_new_uninit()?;
527 ///     Ok(x.write(Example { a: 10, b: 20 }).into())
528 /// }
529 ///
530 /// # test().unwrap();
531 /// ```
532 ///
533 /// In the last example below, the caller gets a pinned instance of `Example` while converting to
534 /// `Arc<Example>`; this is useful in scenarios where one needs a pinned reference during
535 /// initialisation, for example, when initialising fields that are wrapped in locks.
536 ///
537 /// ```
538 /// use kernel::sync::{Arc, UniqueArc};
539 ///
540 /// struct Example {
541 ///     a: u32,
542 ///     b: u32,
543 /// }
544 ///
545 /// fn test() -> Result<Arc<Example>> {
546 ///     let mut pinned = Pin::from(UniqueArc::try_new(Example { a: 10, b: 20 })?);
547 ///     // We can modify `pinned` because it is `Unpin`.
548 ///     pinned.as_mut().a += 1;
549 ///     Ok(pinned.into())
550 /// }
551 ///
552 /// # test().unwrap();
553 /// ```
554 pub struct UniqueArc<T: ?Sized> {
555     inner: Arc<T>,
556 }
557 
558 impl<T> UniqueArc<T> {
559     /// Tries to allocate a new [`UniqueArc`] instance.
560     pub fn try_new(value: T) -> Result<Self, AllocError> {
561         Ok(Self {
562             // INVARIANT: The newly-created object has a refcount of 1.
563             inner: Arc::try_new(value)?,
564         })
565     }
566 
567     /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet.
568     pub fn try_new_uninit() -> Result<UniqueArc<MaybeUninit<T>>, AllocError> {
569         // INVARIANT: The refcount is initialised to a non-zero value.
570         let inner = Box::try_init::<AllocError>(try_init!(ArcInner {
571             // SAFETY: There are no safety requirements for this FFI call.
572             refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }),
573             data <- init::uninit::<T, AllocError>(),
574         }? AllocError))?;
575         Ok(UniqueArc {
576             // INVARIANT: The newly-created object has a refcount of 1.
577             // SAFETY: The pointer from the `Box` is valid.
578             inner: unsafe { Arc::from_inner(Box::leak(inner).into()) },
579         })
580     }
581 }
582 
583 impl<T> UniqueArc<MaybeUninit<T>> {
584     /// Converts a `UniqueArc<MaybeUninit<T>>` into a `UniqueArc<T>` by writing a value into it.
585     pub fn write(mut self, value: T) -> UniqueArc<T> {
586         self.deref_mut().write(value);
587         // SAFETY: We just wrote the value to be initialized.
588         unsafe { self.assume_init() }
589     }
590 
591     /// Unsafely assume that `self` is initialized.
592     ///
593     /// # Safety
594     ///
595     /// The caller guarantees that the value behind this pointer has been initialized. It is
596     /// *immediate* UB to call this when the value is not initialized.
597     pub unsafe fn assume_init(self) -> UniqueArc<T> {
598         let inner = ManuallyDrop::new(self).inner.ptr;
599         UniqueArc {
600             // SAFETY: The new `Arc` is taking over `ptr` from `self.inner` (which won't be
601             // dropped). The types are compatible because `MaybeUninit<T>` is compatible with `T`.
602             inner: unsafe { Arc::from_inner(inner.cast()) },
603         }
604     }
605 
606     /// Initialize `self` using the given initializer.
607     pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> {
608         // SAFETY: The supplied pointer is valid for initialization.
609         match unsafe { init.__init(self.as_mut_ptr()) } {
610             // SAFETY: Initialization completed successfully.
611             Ok(()) => Ok(unsafe { self.assume_init() }),
612             Err(err) => Err(err),
613         }
614     }
615 
616     /// Pin-initialize `self` using the given pin-initializer.
617     pub fn pin_init_with<E>(
618         mut self,
619         init: impl PinInit<T, E>,
620     ) -> core::result::Result<Pin<UniqueArc<T>>, E> {
621         // SAFETY: The supplied pointer is valid for initialization and we will later pin the value
622         // to ensure it does not move.
623         match unsafe { init.__pinned_init(self.as_mut_ptr()) } {
624             // SAFETY: Initialization completed successfully.
625             Ok(()) => Ok(unsafe { self.assume_init() }.into()),
626             Err(err) => Err(err),
627         }
628     }
629 }
630 
631 impl<T: ?Sized> From<UniqueArc<T>> for Pin<UniqueArc<T>> {
632     fn from(obj: UniqueArc<T>) -> Self {
633         // SAFETY: It is not possible to move/replace `T` inside a `Pin<UniqueArc<T>>` (unless `T`
634         // is `Unpin`), so it is ok to convert it to `Pin<UniqueArc<T>>`.
635         unsafe { Pin::new_unchecked(obj) }
636     }
637 }
638 
639 impl<T: ?Sized> Deref for UniqueArc<T> {
640     type Target = T;
641 
642     fn deref(&self) -> &Self::Target {
643         self.inner.deref()
644     }
645 }
646 
647 impl<T: ?Sized> DerefMut for UniqueArc<T> {
648     fn deref_mut(&mut self) -> &mut Self::Target {
649         // SAFETY: By the `Arc` type invariant, there is necessarily a reference to the object, so
650         // it is safe to dereference it. Additionally, we know there is only one reference when
651         // it's inside a `UniqueArc`, so it is safe to get a mutable reference.
652         unsafe { &mut self.inner.ptr.as_mut().data }
653     }
654 }
655 
656 impl<T: fmt::Display + ?Sized> fmt::Display for UniqueArc<T> {
657     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
658         fmt::Display::fmt(self.deref(), f)
659     }
660 }
661 
662 impl<T: fmt::Display + ?Sized> fmt::Display for Arc<T> {
663     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
664         fmt::Display::fmt(self.deref(), f)
665     }
666 }
667 
668 impl<T: fmt::Debug + ?Sized> fmt::Debug for UniqueArc<T> {
669     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
670         fmt::Debug::fmt(self.deref(), f)
671     }
672 }
673 
674 impl<T: fmt::Debug + ?Sized> fmt::Debug for Arc<T> {
675     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
676         fmt::Debug::fmt(self.deref(), f)
677     }
678 }
679