| // SPDX-License-Identifier: GPL-2.0 |
| |
| //! Kernel types. |
| |
| use core::{ |
| cell::UnsafeCell, |
| marker::{PhantomData, PhantomPinned}, |
| mem::{ManuallyDrop, MaybeUninit}, |
| ops::{Deref, DerefMut}, |
| ptr::NonNull, |
| }; |
| use pin_init::{PinInit, Zeroable}; |
| |
| /// Used to transfer ownership to and from foreign (non-Rust) languages. |
| /// |
| /// Ownership is transferred from Rust to a foreign language by calling [`Self::into_foreign`] and |
| /// later may be transferred back to Rust by calling [`Self::from_foreign`]. |
| /// |
| /// This trait is meant to be used in cases when Rust objects are stored in C objects and |
| /// eventually "freed" back to Rust. |
| pub trait ForeignOwnable: Sized { |
| /// Type used to immutably borrow a value that is currently foreign-owned. |
| type Borrowed<'a>; |
| |
| /// Type used to mutably borrow a value that is currently foreign-owned. |
| type BorrowedMut<'a>; |
| |
| /// Converts a Rust-owned object to a foreign-owned one. |
| /// |
| /// The foreign representation is a pointer to void. There are no guarantees for this pointer. |
| /// For example, it might be invalid, dangling or pointing to uninitialized memory. Using it in |
| /// any way except for [`from_foreign`], [`try_from_foreign`], [`borrow`], or [`borrow_mut`] can |
| /// result in undefined behavior. |
| /// |
| /// [`from_foreign`]: Self::from_foreign |
| /// [`try_from_foreign`]: Self::try_from_foreign |
| /// [`borrow`]: Self::borrow |
| /// [`borrow_mut`]: Self::borrow_mut |
| fn into_foreign(self) -> *mut crate::ffi::c_void; |
| |
| /// Converts a foreign-owned object back to a Rust-owned one. |
| /// |
| /// # Safety |
| /// |
| /// The provided pointer must have been returned by a previous call to [`into_foreign`], and it |
| /// must not be passed to `from_foreign` more than once. |
| /// |
| /// [`into_foreign`]: Self::into_foreign |
| unsafe fn from_foreign(ptr: *mut crate::ffi::c_void) -> Self; |
| |
| /// Tries to convert a foreign-owned object back to a Rust-owned one. |
| /// |
| /// A convenience wrapper over [`ForeignOwnable::from_foreign`] that returns [`None`] if `ptr` |
| /// is null. |
| /// |
| /// # Safety |
| /// |
| /// `ptr` must either be null or satisfy the safety requirements for [`from_foreign`]. |
| /// |
| /// [`from_foreign`]: Self::from_foreign |
| unsafe fn try_from_foreign(ptr: *mut crate::ffi::c_void) -> Option<Self> { |
| if ptr.is_null() { |
| None |
| } else { |
| // SAFETY: Since `ptr` is not null here, then `ptr` satisfies the safety requirements |
| // of `from_foreign` given the safety requirements of this function. |
| unsafe { Some(Self::from_foreign(ptr)) } |
| } |
| } |
| |
| /// Borrows a foreign-owned object immutably. |
| /// |
| /// This method provides a way to access a foreign-owned value from Rust immutably. It provides |
| /// you with exactly the same abilities as an `&Self` when the value is Rust-owned. |
| /// |
| /// # Safety |
| /// |
| /// The provided pointer must have been returned by a previous call to [`into_foreign`], and if |
| /// the pointer is ever passed to [`from_foreign`], then that call must happen after the end of |
| /// the lifetime `'a`. |
| /// |
| /// [`into_foreign`]: Self::into_foreign |
| /// [`from_foreign`]: Self::from_foreign |
| unsafe fn borrow<'a>(ptr: *mut crate::ffi::c_void) -> Self::Borrowed<'a>; |
| |
| /// Borrows a foreign-owned object mutably. |
| /// |
| /// This method provides a way to access a foreign-owned value from Rust mutably. It provides |
| /// you with exactly the same abilities as an `&mut Self` when the value is Rust-owned, except |
| /// that the address of the object must not be changed. |
| /// |
| /// Note that for types like [`Arc`], an `&mut Arc<T>` only gives you immutable access to the |
| /// inner value, so this method also only provides immutable access in that case. |
| /// |
| /// In the case of `Box<T>`, this method gives you the ability to modify the inner `T`, but it |
| /// does not let you change the box itself. That is, you cannot change which allocation the box |
| /// points at. |
| /// |
| /// # Safety |
| /// |
| /// The provided pointer must have been returned by a previous call to [`into_foreign`], and if |
| /// the pointer is ever passed to [`from_foreign`], then that call must happen after the end of |
| /// the lifetime `'a`. |
| /// |
| /// The lifetime `'a` must not overlap with the lifetime of any other call to [`borrow`] or |
| /// `borrow_mut` on the same object. |
| /// |
| /// [`into_foreign`]: Self::into_foreign |
| /// [`from_foreign`]: Self::from_foreign |
| /// [`borrow`]: Self::borrow |
| /// [`Arc`]: crate::sync::Arc |
| unsafe fn borrow_mut<'a>(ptr: *mut crate::ffi::c_void) -> Self::BorrowedMut<'a>; |
| } |
| |
| impl ForeignOwnable for () { |
| type Borrowed<'a> = (); |
| type BorrowedMut<'a> = (); |
| |
| fn into_foreign(self) -> *mut crate::ffi::c_void { |
| core::ptr::NonNull::dangling().as_ptr() |
| } |
| |
| unsafe fn from_foreign(_: *mut crate::ffi::c_void) -> Self {} |
| |
| unsafe fn borrow<'a>(_: *mut crate::ffi::c_void) -> Self::Borrowed<'a> {} |
| unsafe fn borrow_mut<'a>(_: *mut crate::ffi::c_void) -> Self::BorrowedMut<'a> {} |
| } |
| |
| /// Runs a cleanup function/closure when dropped. |
| /// |
| /// The [`ScopeGuard::dismiss`] function prevents the cleanup function from running. |
| /// |
| /// # Examples |
| /// |
| /// In the example below, we have multiple exit paths and we want to log regardless of which one is |
| /// taken: |
| /// |
| /// ``` |
| /// # use kernel::types::ScopeGuard; |
| /// fn example1(arg: bool) { |
| /// let _log = ScopeGuard::new(|| pr_info!("example1 completed\n")); |
| /// |
| /// if arg { |
| /// return; |
| /// } |
| /// |
| /// pr_info!("Do something...\n"); |
| /// } |
| /// |
| /// # example1(false); |
| /// # example1(true); |
| /// ``` |
| /// |
| /// In the example below, we want to log the same message on all early exits but a different one on |
| /// the main exit path: |
| /// |
| /// ``` |
| /// # use kernel::types::ScopeGuard; |
| /// fn example2(arg: bool) { |
| /// let log = ScopeGuard::new(|| pr_info!("example2 returned early\n")); |
| /// |
| /// if arg { |
| /// return; |
| /// } |
| /// |
| /// // (Other early returns...) |
| /// |
| /// log.dismiss(); |
| /// pr_info!("example2 no early return\n"); |
| /// } |
| /// |
| /// # example2(false); |
| /// # example2(true); |
| /// ``` |
| /// |
| /// In the example below, we need a mutable object (the vector) to be accessible within the log |
| /// function, so we wrap it in the [`ScopeGuard`]: |
| /// |
| /// ``` |
| /// # use kernel::types::ScopeGuard; |
| /// fn example3(arg: bool) -> Result { |
| /// let mut vec = |
| /// ScopeGuard::new_with_data(KVec::new(), |v| pr_info!("vec had {} elements\n", v.len())); |
| /// |
| /// vec.push(10u8, GFP_KERNEL)?; |
| /// if arg { |
| /// return Ok(()); |
| /// } |
| /// vec.push(20u8, GFP_KERNEL)?; |
| /// Ok(()) |
| /// } |
| /// |
| /// # assert_eq!(example3(false), Ok(())); |
| /// # assert_eq!(example3(true), Ok(())); |
| /// ``` |
| /// |
| /// # Invariants |
| /// |
| /// The value stored in the struct is nearly always `Some(_)`, except between |
| /// [`ScopeGuard::dismiss`] and [`ScopeGuard::drop`]: in this case, it will be `None` as the value |
| /// will have been returned to the caller. Since [`ScopeGuard::dismiss`] consumes the guard, |
| /// callers won't be able to use it anymore. |
| pub struct ScopeGuard<T, F: FnOnce(T)>(Option<(T, F)>); |
| |
| impl<T, F: FnOnce(T)> ScopeGuard<T, F> { |
| /// Creates a new guarded object wrapping the given data and with the given cleanup function. |
| pub fn new_with_data(data: T, cleanup_func: F) -> Self { |
| // INVARIANT: The struct is being initialised with `Some(_)`. |
| Self(Some((data, cleanup_func))) |
| } |
| |
| /// Prevents the cleanup function from running and returns the guarded data. |
| pub fn dismiss(mut self) -> T { |
| // INVARIANT: This is the exception case in the invariant; it is not visible to callers |
| // because this function consumes `self`. |
| self.0.take().unwrap().0 |
| } |
| } |
| |
| impl ScopeGuard<(), fn(())> { |
| /// Creates a new guarded object with the given cleanup function. |
| pub fn new(cleanup: impl FnOnce()) -> ScopeGuard<(), impl FnOnce(())> { |
| ScopeGuard::new_with_data((), move |()| cleanup()) |
| } |
| } |
| |
| impl<T, F: FnOnce(T)> Deref for ScopeGuard<T, F> { |
| type Target = T; |
| |
| fn deref(&self) -> &T { |
| // The type invariants guarantee that `unwrap` will succeed. |
| &self.0.as_ref().unwrap().0 |
| } |
| } |
| |
| impl<T, F: FnOnce(T)> DerefMut for ScopeGuard<T, F> { |
| fn deref_mut(&mut self) -> &mut T { |
| // The type invariants guarantee that `unwrap` will succeed. |
| &mut self.0.as_mut().unwrap().0 |
| } |
| } |
| |
| impl<T, F: FnOnce(T)> Drop for ScopeGuard<T, F> { |
| fn drop(&mut self) { |
| // Run the cleanup function if one is still present. |
| if let Some((data, cleanup)) = self.0.take() { |
| cleanup(data) |
| } |
| } |
| } |
| |
| /// Stores an opaque value. |
| /// |
| /// [`Opaque<T>`] is meant to be used with FFI objects that are never interpreted by Rust code. |
| /// |
| /// It is used to wrap structs from the C side, like for example `Opaque<bindings::mutex>`. |
| /// It gets rid of all the usual assumptions that Rust has for a value: |
| /// |
| /// * The value is allowed to be uninitialized (for example have invalid bit patterns: `3` for a |
| /// [`bool`]). |
| /// * The value is allowed to be mutated, when a `&Opaque<T>` exists on the Rust side. |
| /// * No uniqueness for mutable references: it is fine to have multiple `&mut Opaque<T>` point to |
| /// the same value. |
| /// * The value is not allowed to be shared with other threads (i.e. it is `!Sync`). |
| /// |
| /// This has to be used for all values that the C side has access to, because it can't be ensured |
| /// that the C side is adhering to the usual constraints that Rust needs. |
| /// |
| /// Using [`Opaque<T>`] allows to continue to use references on the Rust side even for values shared |
| /// with C. |
| /// |
| /// # Examples |
| /// |
| /// ``` |
| /// # #![expect(unreachable_pub, clippy::disallowed_names)] |
| /// use kernel::types::Opaque; |
| /// # // Emulate a C struct binding which is from C, maybe uninitialized or not, only the C side |
| /// # // knows. |
| /// # mod bindings { |
| /// # pub struct Foo { |
| /// # pub val: u8, |
| /// # } |
| /// # } |
| /// |
| /// // `foo.val` is assumed to be handled on the C side, so we use `Opaque` to wrap it. |
| /// pub struct Foo { |
| /// foo: Opaque<bindings::Foo>, |
| /// } |
| /// |
| /// impl Foo { |
| /// pub fn get_val(&self) -> u8 { |
| /// let ptr = Opaque::get(&self.foo); |
| /// |
| /// // SAFETY: `Self` is valid from C side. |
| /// unsafe { (*ptr).val } |
| /// } |
| /// } |
| /// |
| /// // Create an instance of `Foo` with the `Opaque` wrapper. |
| /// let foo = Foo { |
| /// foo: Opaque::new(bindings::Foo { val: 0xdb }), |
| /// }; |
| /// |
| /// assert_eq!(foo.get_val(), 0xdb); |
| /// ``` |
| #[repr(transparent)] |
| pub struct Opaque<T> { |
| value: UnsafeCell<MaybeUninit<T>>, |
| _pin: PhantomPinned, |
| } |
| |
| // SAFETY: `Opaque<T>` allows the inner value to be any bit pattern, including all zeros. |
| unsafe impl<T> Zeroable for Opaque<T> {} |
| |
| impl<T> Opaque<T> { |
| /// Creates a new opaque value. |
| pub const fn new(value: T) -> Self { |
| Self { |
| value: UnsafeCell::new(MaybeUninit::new(value)), |
| _pin: PhantomPinned, |
| } |
| } |
| |
| /// Creates an uninitialised value. |
| pub const fn uninit() -> Self { |
| Self { |
| value: UnsafeCell::new(MaybeUninit::uninit()), |
| _pin: PhantomPinned, |
| } |
| } |
| |
| /// Create an opaque pin-initializer from the given pin-initializer. |
| pub fn pin_init(slot: impl PinInit<T>) -> impl PinInit<Self> { |
| Self::ffi_init(|ptr: *mut T| { |
| // SAFETY: |
| // - `ptr` is a valid pointer to uninitialized memory, |
| // - `slot` is not accessed on error; the call is infallible, |
| // - `slot` is pinned in memory. |
| let _ = unsafe { PinInit::<T>::__pinned_init(slot, ptr) }; |
| }) |
| } |
| |
| /// Creates a pin-initializer from the given initializer closure. |
| /// |
| /// The returned initializer calls the given closure with the pointer to the inner `T` of this |
| /// `Opaque`. Since this memory is uninitialized, the closure is not allowed to read from it. |
| /// |
| /// This function is safe, because the `T` inside of an `Opaque` is allowed to be |
| /// uninitialized. Additionally, access to the inner `T` requires `unsafe`, so the caller needs |
| /// to verify at that point that the inner value is valid. |
| pub fn ffi_init(init_func: impl FnOnce(*mut T)) -> impl PinInit<Self> { |
| // SAFETY: We contain a `MaybeUninit`, so it is OK for the `init_func` to not fully |
| // initialize the `T`. |
| unsafe { |
| pin_init::pin_init_from_closure::<_, ::core::convert::Infallible>(move |slot| { |
| init_func(Self::raw_get(slot)); |
| Ok(()) |
| }) |
| } |
| } |
| |
| /// Creates a fallible pin-initializer from the given initializer closure. |
| /// |
| /// The returned initializer calls the given closure with the pointer to the inner `T` of this |
| /// `Opaque`. Since this memory is uninitialized, the closure is not allowed to read from it. |
| /// |
| /// This function is safe, because the `T` inside of an `Opaque` is allowed to be |
| /// uninitialized. Additionally, access to the inner `T` requires `unsafe`, so the caller needs |
| /// to verify at that point that the inner value is valid. |
| pub fn try_ffi_init<E>( |
| init_func: impl FnOnce(*mut T) -> Result<(), E>, |
| ) -> impl PinInit<Self, E> { |
| // SAFETY: We contain a `MaybeUninit`, so it is OK for the `init_func` to not fully |
| // initialize the `T`. |
| unsafe { |
| pin_init::pin_init_from_closure::<_, E>(move |slot| init_func(Self::raw_get(slot))) |
| } |
| } |
| |
| /// Returns a raw pointer to the opaque data. |
| pub const fn get(&self) -> *mut T { |
| UnsafeCell::get(&self.value).cast::<T>() |
| } |
| |
| /// Gets the value behind `this`. |
| /// |
| /// This function is useful to get access to the value without creating intermediate |
| /// references. |
| pub const fn raw_get(this: *const Self) -> *mut T { |
| UnsafeCell::raw_get(this.cast::<UnsafeCell<MaybeUninit<T>>>()).cast::<T>() |
| } |
| } |
| |
| /// Types that are _always_ reference counted. |
| /// |
| /// It allows such types to define their own custom ref increment and decrement functions. |
| /// Additionally, it allows users to convert from a shared reference `&T` to an owned reference |
| /// [`ARef<T>`]. |
| /// |
| /// This is usually implemented by wrappers to existing structures on the C side of the code. For |
| /// Rust code, the recommendation is to use [`Arc`](crate::sync::Arc) to create reference-counted |
| /// instances of a type. |
| /// |
| /// # Safety |
| /// |
| /// Implementers must ensure that increments to the reference count keep the object alive in memory |
| /// at least until matching decrements are performed. |
| /// |
| /// Implementers must also ensure that all instances are reference-counted. (Otherwise they |
| /// won't be able to honour the requirement that [`AlwaysRefCounted::inc_ref`] keep the object |
| /// alive.) |
| pub unsafe trait AlwaysRefCounted { |
| /// Increments the reference count on the object. |
| fn inc_ref(&self); |
| |
| /// Decrements the reference count on the object. |
| /// |
| /// Frees the object when the count reaches zero. |
| /// |
| /// # Safety |
| /// |
| /// Callers must ensure that there was a previous matching increment to the reference count, |
| /// and that the object is no longer used after its reference count is decremented (as it may |
| /// result in the object being freed), unless the caller owns another increment on the refcount |
| /// (e.g., it calls [`AlwaysRefCounted::inc_ref`] twice, then calls |
| /// [`AlwaysRefCounted::dec_ref`] once). |
| unsafe fn dec_ref(obj: NonNull<Self>); |
| } |
| |
| /// An owned reference to an always-reference-counted object. |
| /// |
| /// The object's reference count is automatically decremented when an instance of [`ARef`] is |
| /// dropped. It is also automatically incremented when a new instance is created via |
| /// [`ARef::clone`]. |
| /// |
| /// # Invariants |
| /// |
| /// The pointer stored in `ptr` is non-null and valid for the lifetime of the [`ARef`] instance. In |
| /// particular, the [`ARef`] instance owns an increment on the underlying object's reference count. |
| pub struct ARef<T: AlwaysRefCounted> { |
| ptr: NonNull<T>, |
| _p: PhantomData<T>, |
| } |
| |
| // SAFETY: It is safe to send `ARef<T>` to another thread when the underlying `T` is `Sync` because |
| // it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs |
| // `T` to be `Send` because any thread that has an `ARef<T>` may ultimately access `T` using a |
| // mutable reference, for example, when the reference count reaches zero and `T` is dropped. |
| unsafe impl<T: AlwaysRefCounted + Sync + Send> Send for ARef<T> {} |
| |
| // SAFETY: It is safe to send `&ARef<T>` to another thread when the underlying `T` is `Sync` |
| // because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, |
| // it needs `T` to be `Send` because any thread that has a `&ARef<T>` may clone it and get an |
| // `ARef<T>` on that thread, so the thread may ultimately access `T` using a mutable reference, for |
| // example, when the reference count reaches zero and `T` is dropped. |
| unsafe impl<T: AlwaysRefCounted + Sync + Send> Sync for ARef<T> {} |
| |
| impl<T: AlwaysRefCounted> ARef<T> { |
| /// Creates a new instance of [`ARef`]. |
| /// |
| /// It takes over an increment of the reference count on the underlying object. |
| /// |
| /// # Safety |
| /// |
| /// Callers must ensure that the reference count was incremented at least once, and that they |
| /// are properly relinquishing one increment. That is, if there is only one increment, callers |
| /// must not use the underlying object anymore -- it is only safe to do so via the newly |
| /// created [`ARef`]. |
| pub unsafe fn from_raw(ptr: NonNull<T>) -> Self { |
| // INVARIANT: The safety requirements guarantee that the new instance now owns the |
| // increment on the refcount. |
| Self { |
| ptr, |
| _p: PhantomData, |
| } |
| } |
| |
| /// Consumes the `ARef`, returning a raw pointer. |
| /// |
| /// This function does not change the refcount. After calling this function, the caller is |
| /// responsible for the refcount previously managed by the `ARef`. |
| /// |
| /// # Examples |
| /// |
| /// ``` |
| /// use core::ptr::NonNull; |
| /// use kernel::types::{ARef, AlwaysRefCounted}; |
| /// |
| /// struct Empty {} |
| /// |
| /// # // SAFETY: TODO. |
| /// unsafe impl AlwaysRefCounted for Empty { |
| /// fn inc_ref(&self) {} |
| /// unsafe fn dec_ref(_obj: NonNull<Self>) {} |
| /// } |
| /// |
| /// let mut data = Empty {}; |
| /// let ptr = NonNull::<Empty>::new(&mut data).unwrap(); |
| /// # // SAFETY: TODO. |
| /// let data_ref: ARef<Empty> = unsafe { ARef::from_raw(ptr) }; |
| /// let raw_ptr: NonNull<Empty> = ARef::into_raw(data_ref); |
| /// |
| /// assert_eq!(ptr, raw_ptr); |
| /// ``` |
| pub fn into_raw(me: Self) -> NonNull<T> { |
| ManuallyDrop::new(me).ptr |
| } |
| } |
| |
| impl<T: AlwaysRefCounted> Clone for ARef<T> { |
| fn clone(&self) -> Self { |
| self.inc_ref(); |
| // SAFETY: We just incremented the refcount above. |
| unsafe { Self::from_raw(self.ptr) } |
| } |
| } |
| |
| impl<T: AlwaysRefCounted> Deref for ARef<T> { |
| type Target = T; |
| |
| fn deref(&self) -> &Self::Target { |
| // SAFETY: The type invariants guarantee that the object is valid. |
| unsafe { self.ptr.as_ref() } |
| } |
| } |
| |
| impl<T: AlwaysRefCounted> From<&T> for ARef<T> { |
| fn from(b: &T) -> Self { |
| b.inc_ref(); |
| // SAFETY: We just incremented the refcount above. |
| unsafe { Self::from_raw(NonNull::from(b)) } |
| } |
| } |
| |
| impl<T: AlwaysRefCounted> Drop for ARef<T> { |
| fn drop(&mut self) { |
| // SAFETY: The type invariants guarantee that the `ARef` owns the reference we're about to |
| // decrement. |
| unsafe { T::dec_ref(self.ptr) }; |
| } |
| } |
| |
| /// A sum type that always holds either a value of type `L` or `R`. |
| /// |
| /// # Examples |
| /// |
| /// ``` |
| /// use kernel::types::Either; |
| /// |
| /// let left_value: Either<i32, &str> = Either::Left(7); |
| /// let right_value: Either<i32, &str> = Either::Right("right value"); |
| /// ``` |
| pub enum Either<L, R> { |
| /// Constructs an instance of [`Either`] containing a value of type `L`. |
| Left(L), |
| |
| /// Constructs an instance of [`Either`] containing a value of type `R`. |
| Right(R), |
| } |
| |
| /// Zero-sized type to mark types not [`Send`]. |
| /// |
| /// Add this type as a field to your struct if your type should not be sent to a different task. |
| /// Since [`Send`] is an auto trait, adding a single field that is `!Send` will ensure that the |
| /// whole type is `!Send`. |
| /// |
| /// If a type is `!Send` it is impossible to give control over an instance of the type to another |
| /// task. This is useful to include in types that store or reference task-local information. A file |
| /// descriptor is an example of such task-local information. |
| /// |
| /// This type also makes the type `!Sync`, which prevents immutable access to the value from |
| /// several threads in parallel. |
| pub type NotThreadSafe = PhantomData<*mut ()>; |
| |
| /// Used to construct instances of type [`NotThreadSafe`] similar to how `PhantomData` is |
| /// constructed. |
| /// |
| /// [`NotThreadSafe`]: type@NotThreadSafe |
| #[allow(non_upper_case_globals)] |
| pub const NotThreadSafe: NotThreadSafe = PhantomData; |