| blob | 347049df556b143c01fc6e74e4355a75828faaf5 |
1 // SPDX-License-Identifier: Apache-2.0 OR MIT
3 //! API to safely and fallibly initialize pinned `struct`s using in-place constructors.
4 //!
5 //! It also allows in-place initialization of big `struct`s that would otherwise produce a stack
6 //! overflow.
7 //!
8 //! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential
9 //! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move.
10 //!
11 //! # Overview
12 //!
13 //! To initialize a `struct` with an in-place constructor you will need two things:
14 //! - an in-place constructor,
15 //! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`],
16 //! [`UniqueArc<T>`], [`KBox<T>`] or any other smart pointer that implements [`InPlaceInit`]).
17 //!
18 //! To get an in-place constructor there are generally three options:
19 //! - directly creating an in-place constructor using the [`pin_init!`] macro,
20 //! - a custom function/macro returning an in-place constructor provided by someone else,
21 //! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer.
22 //!
23 //! Aside from pinned initialization, this API also supports in-place construction without pinning,
24 //! the macros/types/functions are generally named like the pinned variants without the `pin`
25 //! prefix.
26 //!
27 //! # Examples
28 //!
29 //! ## Using the [`pin_init!`] macro
30 //!
31 //! If you want to use [`PinInit`], then you will have to annotate your `struct` with
32 //! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for
33 //! [structurally pinned fields]. After doing this, you can then create an in-place constructor via
34 //! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is
35 //! that you need to write `<-` instead of `:` for fields that you want to initialize in-place.
36 //!
37 //! ```rust
38 //! # #![expect(clippy::disallowed_names)]
39 //! use kernel::sync::{new_mutex, Mutex};
40 //! # use core::pin::Pin;
41 //! #[pin_data]
42 //! struct Foo {
43 //! #[pin]
44 //! a: Mutex<usize>,
45 //! b: u32,
46 //! }
47 //!
48 //! let foo = pin_init!(Foo {
49 //! a <- new_mutex!(42, "Foo::a"),
50 //! b: 24,
51 //! });
52 //! ```
53 //!
54 //! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like
55 //! (or just the stack) to actually initialize a `Foo`:
56 //!
57 //! ```rust
58 //! # #![expect(clippy::disallowed_names)]
59 //! # use kernel::sync::{new_mutex, Mutex};
60 //! # use core::pin::Pin;
61 //! # #[pin_data]
62 //! # struct Foo {
63 //! # #[pin]
64 //! # a: Mutex<usize>,
65 //! # b: u32,
66 //! # }
67 //! # let foo = pin_init!(Foo {
68 //! # a <- new_mutex!(42, "Foo::a"),
69 //! # b: 24,
70 //! # });
71 //! let foo: Result<Pin<KBox<Foo>>> = KBox::pin_init(foo, GFP_KERNEL);
72 //! ```
73 //!
74 //! For more information see the [`pin_init!`] macro.
75 //!
76 //! ## Using a custom function/macro that returns an initializer
77 //!
78 //! Many types from the kernel supply a function/macro that returns an initializer, because the
79 //! above method only works for types where you can access the fields.
80 //!
81 //! ```rust
82 //! # use kernel::sync::{new_mutex, Arc, Mutex};
83 //! let mtx: Result<Arc<Mutex<usize>>> =
84 //! Arc::pin_init(new_mutex!(42, "example::mtx"), GFP_KERNEL);
85 //! ```
86 //!
87 //! To declare an init macro/function you just return an [`impl PinInit<T, E>`]:
88 //!
89 //! ```rust
90 //! # use kernel::{sync::Mutex, new_mutex, init::PinInit, try_pin_init};
91 //! #[pin_data]
92 //! struct DriverData {
93 //! #[pin]
94 //! status: Mutex<i32>,
95 //! buffer: KBox<[u8; 1_000_000]>,
96 //! }
97 //!
98 //! impl DriverData {
99 //! fn new() -> impl PinInit<Self, Error> {
100 //! try_pin_init!(Self {
101 //! status <- new_mutex!(0, "DriverData::status"),
102 //! buffer: KBox::init(kernel::init::zeroed(), GFP_KERNEL)?,
103 //! })
104 //! }
105 //! }
106 //! ```
107 //!
108 //! ## Manual creation of an initializer
109 //!
110 //! Often when working with primitives the previous approaches are not sufficient. That is where
111 //! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a
112 //! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure
113 //! actually does the initialization in the correct way. Here are the things to look out for
114 //! (we are calling the parameter to the closure `slot`):
115 //! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so
116 //! `slot` now contains a valid bit pattern for the type `T`,
117 //! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so
118 //! you need to take care to clean up anything if your initialization fails mid-way,
119 //! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of
120 //! `slot` gets called.
121 //!
122 //! ```rust
123 //! # #![expect(unreachable_pub, clippy::disallowed_names)]
124 //! use kernel::{init, types::Opaque};
125 //! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin};
126 //! # mod bindings {
127 //! # #![expect(non_camel_case_types)]
128 //! # #![expect(clippy::missing_safety_doc)]
129 //! # pub struct foo;
130 //! # pub unsafe fn init_foo(_ptr: *mut foo) {}
131 //! # pub unsafe fn destroy_foo(_ptr: *mut foo) {}
132 //! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 }
133 //! # }
134 //! # // `Error::from_errno` is `pub(crate)` in the `kernel` crate, thus provide a workaround.
135 //! # trait FromErrno {
136 //! # fn from_errno(errno: kernel::ffi::c_int) -> Error {
137 //! # // Dummy error that can be constructed outside the `kernel` crate.
138 //! # Error::from(core::fmt::Error)
139 //! # }
140 //! # }
141 //! # impl FromErrno for Error {}
142 //! /// # Invariants
143 //! ///
144 //! /// `foo` is always initialized
145 //! #[pin_data(PinnedDrop)]
146 //! pub struct RawFoo {
147 //! #[pin]
148 //! foo: Opaque<bindings::foo>,
149 //! #[pin]
150 //! _p: PhantomPinned,
151 //! }
152 //!
153 //! impl RawFoo {
154 //! pub fn new(flags: u32) -> impl PinInit<Self, Error> {
155 //! // SAFETY:
156 //! // - when the closure returns `Ok(())`, then it has successfully initialized and
157 //! // enabled `foo`,
158 //! // - when it returns `Err(e)`, then it has cleaned up before
159 //! unsafe {
160 //! init::pin_init_from_closure(move |slot: *mut Self| {
161 //! // `slot` contains uninit memory, avoid creating a reference.
162 //! let foo = addr_of_mut!((*slot).foo);
163 //!
164 //! // Initialize the `foo`
165 //! bindings::init_foo(Opaque::raw_get(foo));
166 //!
167 //! // Try to enable it.
168 //! let err = bindings::enable_foo(Opaque::raw_get(foo), flags);
169 //! if err != 0 {
170 //! // Enabling has failed, first clean up the foo and then return the error.
171 //! bindings::destroy_foo(Opaque::raw_get(foo));
172 //! return Err(Error::from_errno(err));
173 //! }
174 //!
175 //! // All fields of `RawFoo` have been initialized, since `_p` is a ZST.
176 //! Ok(())
177 //! })
178 //! }
179 //! }
180 //! }
181 //!
182 //! #[pinned_drop]
183 //! impl PinnedDrop for RawFoo {
184 //! fn drop(self: Pin<&mut Self>) {
185 //! // SAFETY: Since `foo` is initialized, destroying is safe.
186 //! unsafe { bindings::destroy_foo(self.foo.get()) };
187 //! }
188 //! }
189 //! ```
190 //!
191 //! For the special case where initializing a field is a single FFI-function call that cannot fail,
192 //! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single
193 //! [`Opaque`] field by just delegating to the supplied closure. You can use these in combination
194 //! with [`pin_init!`].
195 //!
196 //! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside
197 //! the `kernel` crate. The [`sync`] module is a good starting point.
198 //!
199 //! [`sync`]: kernel::sync
200 //! [pinning]: https://doc.rust-lang.org/std/pin/index.html
201 //! [structurally pinned fields]:
202 //! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field
203 //! [stack]: crate::stack_pin_init
204 //! [`Arc<T>`]: crate::sync::Arc
205 //! [`impl PinInit<Foo>`]: PinInit
206 //! [`impl PinInit<T, E>`]: PinInit
207 //! [`impl Init<T, E>`]: Init
208 //! [`Opaque`]: kernel::types::Opaque
209 //! [`Opaque::ffi_init`]: kernel::types::Opaque::ffi_init
210 //! [`pin_data`]: ::macros::pin_data
211 //! [`pin_init!`]: crate::pin_init!
213 use crate::{
214 alloc::{AllocError, Flags, KBox},
215 error::{self, Error},
216 sync::Arc,
217 sync::UniqueArc,
218 types::{Opaque, ScopeGuard},
219 };
220 use core::{
221 cell::UnsafeCell,
222 convert::Infallible,
223 marker::PhantomData,
224 mem::MaybeUninit,
225 num::*,
226 pin::Pin,
227 ptr::{self, NonNull},
228 };
230 #[doc(hidden)]
231 pub mod __internal;
232 #[doc(hidden)]
233 pub mod macros;
235 /// Initialize and pin a type directly on the stack.
236 ///
237 /// # Examples
238 ///
239 /// ```rust
240 /// # #![expect(clippy::disallowed_names)]
241 /// # use kernel::{init, macros::pin_data, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex};
242 /// # use core::pin::Pin;
243 /// #[pin_data]
244 /// struct Foo {
245 /// #[pin]
246 /// a: Mutex<usize>,
247 /// b: Bar,
248 /// }
249 ///
250 /// #[pin_data]
251 /// struct Bar {
252 /// x: u32,
253 /// }
254 ///
255 /// stack_pin_init!(let foo = pin_init!(Foo {
256 /// a <- new_mutex!(42),
257 /// b: Bar {
258 /// x: 64,
259 /// },
260 /// }));
261 /// let foo: Pin<&mut Foo> = foo;
262 /// pr_info!("a: {}", &*foo.a.lock());
263 /// ```
264 ///
265 /// # Syntax
266 ///
267 /// A normal `let` binding with optional type annotation. The expression is expected to implement
268 /// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error
269 /// type, then use [`stack_try_pin_init!`].
270 ///
271 /// [`stack_try_pin_init!`]: crate::stack_try_pin_init!
272 #[macro_export]
273 macro_rules! stack_pin_init {
274 (let $var:ident $(: $t:ty)? = $val:expr) => {
275 let val = $val;
276 let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
277 let mut $var = match $crate::init::__internal::StackInit::init($var, val) {
278 Ok(res) => res,
279 Err(x) => {
280 let x: ::core::convert::Infallible = x;
281 match x {}
282 }
283 };
284 };
285 }
287 /// Initialize and pin a type directly on the stack.
288 ///
289 /// # Examples
290 ///
291 /// ```rust,ignore
292 /// # #![expect(clippy::disallowed_names)]
293 /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex};
294 /// # use macros::pin_data;
295 /// # use core::{alloc::AllocError, pin::Pin};
296 /// #[pin_data]
297 /// struct Foo {
298 /// #[pin]
299 /// a: Mutex<usize>,
300 /// b: KBox<Bar>,
301 /// }
302 ///
303 /// struct Bar {
304 /// x: u32,
305 /// }
306 ///
307 /// stack_try_pin_init!(let foo: Result<Pin<&mut Foo>, AllocError> = pin_init!(Foo {
308 /// a <- new_mutex!(42),
309 /// b: KBox::new(Bar {
310 /// x: 64,
311 /// }, GFP_KERNEL)?,
312 /// }));
313 /// let foo = foo.unwrap();
314 /// pr_info!("a: {}", &*foo.a.lock());
315 /// ```
316 ///
317 /// ```rust,ignore
318 /// # #![expect(clippy::disallowed_names)]
319 /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex};
320 /// # use macros::pin_data;
321 /// # use core::{alloc::AllocError, pin::Pin};
322 /// #[pin_data]
323 /// struct Foo {
324 /// #[pin]
325 /// a: Mutex<usize>,
326 /// b: KBox<Bar>,
327 /// }
328 ///
329 /// struct Bar {
330 /// x: u32,
331 /// }
332 ///
333 /// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo {
334 /// a <- new_mutex!(42),
335 /// b: KBox::new(Bar {
336 /// x: 64,
337 /// }, GFP_KERNEL)?,
338 /// }));
339 /// pr_info!("a: {}", &*foo.a.lock());
340 /// # Ok::<_, AllocError>(())
341 /// ```
342 ///
343 /// # Syntax
344 ///
345 /// A normal `let` binding with optional type annotation. The expression is expected to implement
346 /// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the
347 /// `=` will propagate this error.
348 #[macro_export]
349 macro_rules! stack_try_pin_init {
350 (let $var:ident $(: $t:ty)? = $val:expr) => {
351 let val = $val;
352 let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
353 let mut $var = $crate::init::__internal::StackInit::init($var, val);
354 };
355 (let $var:ident $(: $t:ty)? =? $val:expr) => {
356 let val = $val;
357 let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
358 let mut $var = $crate::init::__internal::StackInit::init($var, val)?;
359 };
360 }
362 /// Construct an in-place, pinned initializer for `struct`s.
363 ///
364 /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
365 /// [`try_pin_init!`].
366 ///
367 /// The syntax is almost identical to that of a normal `struct` initializer:
368 ///
369 /// ```rust
370 /// # use kernel::{init, pin_init, macros::pin_data, init::*};
371 /// # use core::pin::Pin;
372 /// #[pin_data]
373 /// struct Foo {
374 /// a: usize,
375 /// b: Bar,
376 /// }
377 ///
378 /// #[pin_data]
379 /// struct Bar {
380 /// x: u32,
381 /// }
382 ///
383 /// # fn demo() -> impl PinInit<Foo> {
384 /// let a = 42;
385 ///
386 /// let initializer = pin_init!(Foo {
387 /// a,
388 /// b: Bar {
389 /// x: 64,
390 /// },
391 /// });
392 /// # initializer }
393 /// # KBox::pin_init(demo(), GFP_KERNEL).unwrap();
394 /// ```
395 ///
396 /// Arbitrary Rust expressions can be used to set the value of a variable.
397 ///
398 /// The fields are initialized in the order that they appear in the initializer. So it is possible
399 /// to read already initialized fields using raw pointers.
400 ///
401 /// IMPORTANT: You are not allowed to create references to fields of the struct inside of the
402 /// initializer.
403 ///
404 /// # Init-functions
405 ///
406 /// When working with this API it is often desired to let others construct your types without
407 /// giving access to all fields. This is where you would normally write a plain function `new`
408 /// that would return a new instance of your type. With this API that is also possible.
409 /// However, there are a few extra things to keep in mind.
410 ///
411 /// To create an initializer function, simply declare it like this:
412 ///
413 /// ```rust
414 /// # use kernel::{init, pin_init, init::*};
415 /// # use core::pin::Pin;
416 /// # #[pin_data]
417 /// # struct Foo {
418 /// # a: usize,
419 /// # b: Bar,
420 /// # }
421 /// # #[pin_data]
422 /// # struct Bar {
423 /// # x: u32,
424 /// # }
425 /// impl Foo {
426 /// fn new() -> impl PinInit<Self> {
427 /// pin_init!(Self {
428 /// a: 42,
429 /// b: Bar {
430 /// x: 64,
431 /// },
432 /// })
433 /// }
434 /// }
435 /// ```
436 ///
437 /// Users of `Foo` can now create it like this:
438 ///
439 /// ```rust
440 /// # #![expect(clippy::disallowed_names)]
441 /// # use kernel::{init, pin_init, macros::pin_data, init::*};
442 /// # use core::pin::Pin;
443 /// # #[pin_data]
444 /// # struct Foo {
445 /// # a: usize,
446 /// # b: Bar,
447 /// # }
448 /// # #[pin_data]
449 /// # struct Bar {
450 /// # x: u32,
451 /// # }
452 /// # impl Foo {
453 /// # fn new() -> impl PinInit<Self> {
454 /// # pin_init!(Self {
455 /// # a: 42,
456 /// # b: Bar {
457 /// # x: 64,
458 /// # },
459 /// # })
460 /// # }
461 /// # }
462 /// let foo = KBox::pin_init(Foo::new(), GFP_KERNEL);
463 /// ```
464 ///
465 /// They can also easily embed it into their own `struct`s:
466 ///
467 /// ```rust
468 /// # use kernel::{init, pin_init, macros::pin_data, init::*};
469 /// # use core::pin::Pin;
470 /// # #[pin_data]
471 /// # struct Foo {
472 /// # a: usize,
473 /// # b: Bar,
474 /// # }
475 /// # #[pin_data]
476 /// # struct Bar {
477 /// # x: u32,
478 /// # }
479 /// # impl Foo {
480 /// # fn new() -> impl PinInit<Self> {
481 /// # pin_init!(Self {
482 /// # a: 42,
483 /// # b: Bar {
484 /// # x: 64,
485 /// # },
486 /// # })
487 /// # }
488 /// # }
489 /// #[pin_data]
490 /// struct FooContainer {
491 /// #[pin]
492 /// foo1: Foo,
493 /// #[pin]
494 /// foo2: Foo,
495 /// other: u32,
496 /// }
497 ///
498 /// impl FooContainer {
499 /// fn new(other: u32) -> impl PinInit<Self> {
500 /// pin_init!(Self {
501 /// foo1 <- Foo::new(),
502 /// foo2 <- Foo::new(),
503 /// other,
504 /// })
505 /// }
506 /// }
507 /// ```
508 ///
509 /// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`.
510 /// This signifies that the given field is initialized in-place. As with `struct` initializers, just
511 /// writing the field (in this case `other`) without `:` or `<-` means `other: other,`.
512 ///
513 /// # Syntax
514 ///
515 /// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with
516 /// the following modifications is expected:
517 /// - Fields that you want to initialize in-place have to use `<-` instead of `:`.
518 /// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`]
519 /// pointer named `this` inside of the initializer.
520 /// - Using struct update syntax one can place `..Zeroable::zeroed()` at the very end of the
521 /// struct, this initializes every field with 0 and then runs all initializers specified in the
522 /// body. This can only be done if [`Zeroable`] is implemented for the struct.
523 ///
524 /// For instance:
525 ///
526 /// ```rust
527 /// # use kernel::{macros::{Zeroable, pin_data}, pin_init};
528 /// # use core::{ptr::addr_of_mut, marker::PhantomPinned};
529 /// #[pin_data]
530 /// #[derive(Zeroable)]
531 /// struct Buf {
532 /// // `ptr` points into `buf`.
533 /// ptr: *mut u8,
534 /// buf: [u8; 64],
535 /// #[pin]
536 /// pin: PhantomPinned,
537 /// }
538 /// pin_init!(&this in Buf {
539 /// buf: [0; 64],
540 /// // SAFETY: TODO.
541 /// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() },
542 /// pin: PhantomPinned,
543 /// });
544 /// pin_init!(Buf {
545 /// buf: [1; 64],
546 /// ..Zeroable::zeroed()
547 /// });
548 /// ```
549 ///
550 /// [`try_pin_init!`]: kernel::try_pin_init
551 /// [`NonNull<Self>`]: core::ptr::NonNull
552 // For a detailed example of how this macro works, see the module documentation of the hidden
553 // module `__internal` inside of `init/__internal.rs`.
554 #[macro_export]
555 macro_rules! pin_init {
556 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
557 $($fields:tt)*
558 }) => {
559 $crate::__init_internal!(
560 @this($($this)?),
561 @typ($t $(::<$($generics),*>)?),
562 @fields($($fields)*),
563 @error(::core::convert::Infallible),
564 @data(PinData, use_data),
565 @has_data(HasPinData, __pin_data),
566 @construct_closure(pin_init_from_closure),
567 @munch_fields($($fields)*),
568 )
569 };
570 }
572 /// Construct an in-place, fallible pinned initializer for `struct`s.
573 ///
574 /// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`].
575 ///
576 /// You can use the `?` operator or use `return Err(err)` inside the initializer to stop
577 /// initialization and return the error.
578 ///
579 /// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when
580 /// initialization fails, the memory can be safely deallocated without any further modifications.
581 ///
582 /// This macro defaults the error to [`Error`].
583 ///
584 /// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type`
585 /// after the `struct` initializer to specify the error type you want to use.
586 ///
587 /// # Examples
588 ///
589 /// ```rust
590 /// use kernel::{init::{self, PinInit}, error::Error};
591 /// #[pin_data]
592 /// struct BigBuf {
593 /// big: KBox<[u8; 1024 * 1024 * 1024]>,
594 /// small: [u8; 1024 * 1024],
595 /// ptr: *mut u8,
596 /// }
597 ///
598 /// impl BigBuf {
599 /// fn new() -> impl PinInit<Self, Error> {
600 /// try_pin_init!(Self {
601 /// big: KBox::init(init::zeroed(), GFP_KERNEL)?,
602 /// small: [0; 1024 * 1024],
603 /// ptr: core::ptr::null_mut(),
604 /// }? Error)
605 /// }
606 /// }
607 /// ```
608 // For a detailed example of how this macro works, see the module documentation of the hidden
609 // module `__internal` inside of `init/__internal.rs`.
610 #[macro_export]
611 macro_rules! try_pin_init {
612 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
613 $($fields:tt)*
614 }) => {
615 $crate::__init_internal!(
616 @this($($this)?),
617 @typ($t $(::<$($generics),*>)? ),
618 @fields($($fields)*),
619 @error($crate::error::Error),
620 @data(PinData, use_data),
621 @has_data(HasPinData, __pin_data),
622 @construct_closure(pin_init_from_closure),
623 @munch_fields($($fields)*),
624 )
625 };
626 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
627 $($fields:tt)*
628 }? $err:ty) => {
629 $crate::__init_internal!(
630 @this($($this)?),
631 @typ($t $(::<$($generics),*>)? ),
632 @fields($($fields)*),
633 @error($err),
634 @data(PinData, use_data),
635 @has_data(HasPinData, __pin_data),
636 @construct_closure(pin_init_from_closure),
637 @munch_fields($($fields)*),
638 )
639 };
640 }
642 /// Construct an in-place initializer for `struct`s.
643 ///
644 /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
645 /// [`try_init!`].
646 ///
647 /// The syntax is identical to [`pin_init!`] and its safety caveats also apply:
648 /// - `unsafe` code must guarantee either full initialization or return an error and allow
649 /// deallocation of the memory.
650 /// - the fields are initialized in the order given in the initializer.
651 /// - no references to fields are allowed to be created inside of the initializer.
652 ///
653 /// This initializer is for initializing data in-place that might later be moved. If you want to
654 /// pin-initialize, use [`pin_init!`].
655 ///
656 /// [`try_init!`]: crate::try_init!
657 // For a detailed example of how this macro works, see the module documentation of the hidden
658 // module `__internal` inside of `init/__internal.rs`.
659 #[macro_export]
660 macro_rules! init {
661 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
662 $($fields:tt)*
663 }) => {
664 $crate::__init_internal!(
665 @this($($this)?),
666 @typ($t $(::<$($generics),*>)?),
667 @fields($($fields)*),
668 @error(::core::convert::Infallible),
669 @data(InitData, /*no use_data*/),
670 @has_data(HasInitData, __init_data),
671 @construct_closure(init_from_closure),
672 @munch_fields($($fields)*),
673 )
674 }
675 }
677 /// Construct an in-place fallible initializer for `struct`s.
678 ///
679 /// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use
680 /// [`init!`].
681 ///
682 /// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error,
683 /// append `? $type` after the `struct` initializer.
684 /// The safety caveats from [`try_pin_init!`] also apply:
685 /// - `unsafe` code must guarantee either full initialization or return an error and allow
686 /// deallocation of the memory.
687 /// - the fields are initialized in the order given in the initializer.
688 /// - no references to fields are allowed to be created inside of the initializer.
689 ///
690 /// # Examples
691 ///
692 /// ```rust
693 /// use kernel::{alloc::KBox, init::{PinInit, zeroed}, error::Error};
694 /// struct BigBuf {
695 /// big: KBox<[u8; 1024 * 1024 * 1024]>,
696 /// small: [u8; 1024 * 1024],
697 /// }
698 ///
699 /// impl BigBuf {
700 /// fn new() -> impl Init<Self, Error> {
701 /// try_init!(Self {
702 /// big: KBox::init(zeroed(), GFP_KERNEL)?,
703 /// small: [0; 1024 * 1024],
704 /// }? Error)
705 /// }
706 /// }
707 /// ```
708 // For a detailed example of how this macro works, see the module documentation of the hidden
709 // module `__internal` inside of `init/__internal.rs`.
710 #[macro_export]
711 macro_rules! try_init {
712 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
713 $($fields:tt)*
714 }) => {
715 $crate::__init_internal!(
716 @this($($this)?),
717 @typ($t $(::<$($generics),*>)?),
718 @fields($($fields)*),
719 @error($crate::error::Error),
720 @data(InitData, /*no use_data*/),
721 @has_data(HasInitData, __init_data),
722 @construct_closure(init_from_closure),
723 @munch_fields($($fields)*),
724 )
725 };
726 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
727 $($fields:tt)*
728 }? $err:ty) => {
729 $crate::__init_internal!(
730 @this($($this)?),
731 @typ($t $(::<$($generics),*>)?),
732 @fields($($fields)*),
733 @error($err),
734 @data(InitData, /*no use_data*/),
735 @has_data(HasInitData, __init_data),
736 @construct_closure(init_from_closure),
737 @munch_fields($($fields)*),
738 )
739 };
740 }
742 /// Asserts that a field on a struct using `#[pin_data]` is marked with `#[pin]` ie. that it is
743 /// structurally pinned.
744 ///
745 /// # Example
746 ///
747 /// This will succeed:
748 /// ```
749 /// use kernel::assert_pinned;
750 /// #[pin_data]
751 /// struct MyStruct {
752 /// #[pin]
753 /// some_field: u64,
754 /// }
755 ///
756 /// assert_pinned!(MyStruct, some_field, u64);
757 /// ```
758 ///
759 /// This will fail:
760 // TODO: replace with `compile_fail` when supported.
761 /// ```ignore
762 /// use kernel::assert_pinned;
763 /// #[pin_data]
764 /// struct MyStruct {
765 /// some_field: u64,
766 /// }
767 ///
768 /// assert_pinned!(MyStruct, some_field, u64);
769 /// ```
770 ///
771 /// Some uses of the macro may trigger the `can't use generic parameters from outer item` error. To
772 /// work around this, you may pass the `inline` parameter to the macro. The `inline` parameter can
773 /// only be used when the macro is invoked from a function body.
774 /// ```
775 /// use kernel::assert_pinned;
776 /// #[pin_data]
777 /// struct Foo<T> {
778 /// #[pin]
779 /// elem: T,
780 /// }
781 ///
782 /// impl<T> Foo<T> {
783 /// fn project(self: Pin<&mut Self>) -> Pin<&mut T> {
784 /// assert_pinned!(Foo<T>, elem, T, inline);
785 ///
786 /// // SAFETY: The field is structurally pinned.
787 /// unsafe { self.map_unchecked_mut(|me| &mut me.elem) }
788 /// }
789 /// }
790 /// ```
791 #[macro_export]
792 macro_rules! assert_pinned {
793 ($ty:ty, $field:ident, $field_ty:ty, inline) => {
794 let _ = move |ptr: *mut $field_ty| {
795 // SAFETY: This code is unreachable.
796 let data = unsafe { <$ty as $crate::init::__internal::HasPinData>::__pin_data() };
797 let init = $crate::init::__internal::AlwaysFail::<$field_ty>::new();
798 // SAFETY: This code is unreachable.
799 unsafe { data.$field(ptr, init) }.ok();
800 };
801 };
803 ($ty:ty, $field:ident, $field_ty:ty) => {
804 const _: () = {
805 $crate::assert_pinned!($ty, $field, $field_ty, inline);
806 };
807 };
808 }
810 /// A pin-initializer for the type `T`.
811 ///
812 /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
813 /// be [`KBox<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use
814 /// the [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc<T>`] on this.
815 ///
816 /// Also see the [module description](self).
817 ///
818 /// # Safety
819 ///
820 /// When implementing this trait you will need to take great care. Also there are probably very few
821 /// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible.
822 ///
823 /// The [`PinInit::__pinned_init`] function:
824 /// - returns `Ok(())` if it initialized every field of `slot`,
825 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
826 /// - `slot` can be deallocated without UB occurring,
827 /// - `slot` does not need to be dropped,
828 /// - `slot` is not partially initialized.
829 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
830 ///
831 /// [`Arc<T>`]: crate::sync::Arc
832 /// [`Arc::pin_init`]: crate::sync::Arc::pin_init
833 #[must_use = "An initializer must be used in order to create its value."]
834 pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
835 /// Initializes `slot`.
836 ///
837 /// # Safety
838 ///
839 /// - `slot` is a valid pointer to uninitialized memory.
840 /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
841 /// deallocate.
842 /// - `slot` will not move until it is dropped, i.e. it will be pinned.
843 unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>;
845 /// First initializes the value using `self` then calls the function `f` with the initialized
846 /// value.
847 ///
848 /// If `f` returns an error the value is dropped and the initializer will forward the error.
849 ///
850 /// # Examples
851 ///
852 /// ```rust
853 /// # #![expect(clippy::disallowed_names)]
854 /// use kernel::{types::Opaque, init::pin_init_from_closure};
855 /// #[repr(C)]
856 /// struct RawFoo([u8; 16]);
857 /// extern {
858 /// fn init_foo(_: *mut RawFoo);
859 /// }
860 ///
861 /// #[pin_data]
862 /// struct Foo {
863 /// #[pin]
864 /// raw: Opaque<RawFoo>,
865 /// }
866 ///
867 /// impl Foo {
868 /// fn setup(self: Pin<&mut Self>) {
869 /// pr_info!("Setting up foo");
870 /// }
871 /// }
872 ///
873 /// let foo = pin_init!(Foo {
874 /// // SAFETY: TODO.
875 /// raw <- unsafe {
876 /// Opaque::ffi_init(|s| {
877 /// init_foo(s);
878 /// })
879 /// },
880 /// }).pin_chain(|foo| {
881 /// foo.setup();
882 /// Ok(())
883 /// });
884 /// ```
885 fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E>
886 where
887 F: FnOnce(Pin<&mut T>) -> Result<(), E>,
888 {
889 ChainPinInit(self, f, PhantomData)
890 }
891 }
893 /// An initializer returned by [`PinInit::pin_chain`].
894 pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, KBox<T>)>);
896 // SAFETY: The `__pinned_init` function is implemented such that it
897 // - returns `Ok(())` on successful initialization,
898 // - returns `Err(err)` on error and in this case `slot` will be dropped.
899 // - considers `slot` pinned.
900 unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainPinInit<I, F, T, E>
901 where
902 I: PinInit<T, E>,
903 F: FnOnce(Pin<&mut T>) -> Result<(), E>,
904 {
905 unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
906 // SAFETY: All requirements fulfilled since this function is `__pinned_init`.
907 unsafe { self.0.__pinned_init(slot)? };
908 // SAFETY: The above call initialized `slot` and we still have unique access.
909 let val = unsafe { &mut *slot };
910 // SAFETY: `slot` is considered pinned.
911 let val = unsafe { Pin::new_unchecked(val) };
912 // SAFETY: `slot` was initialized above.
913 (self.1)(val).inspect_err(|_| unsafe { core::ptr::drop_in_place(slot) })
914 }
915 }
917 /// An initializer for `T`.
918 ///
919 /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
920 /// be [`KBox<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use
921 /// the [`InPlaceInit::init`] function of a smart pointer like [`Arc<T>`] on this. Because
922 /// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well.
923 ///
924 /// Also see the [module description](self).
925 ///
926 /// # Safety
927 ///
928 /// When implementing this trait you will need to take great care. Also there are probably very few
929 /// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible.
930 ///
931 /// The [`Init::__init`] function:
932 /// - returns `Ok(())` if it initialized every field of `slot`,
933 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
934 /// - `slot` can be deallocated without UB occurring,
935 /// - `slot` does not need to be dropped,
936 /// - `slot` is not partially initialized.
937 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
938 ///
939 /// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same
940 /// code as `__init`.
941 ///
942 /// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to
943 /// move the pointee after initialization.
944 ///
945 /// [`Arc<T>`]: crate::sync::Arc
946 #[must_use = "An initializer must be used in order to create its value."]
947 pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
948 /// Initializes `slot`.
949 ///
950 /// # Safety
951 ///
952 /// - `slot` is a valid pointer to uninitialized memory.
953 /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
954 /// deallocate.
955 unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
957 /// First initializes the value using `self` then calls the function `f` with the initialized
958 /// value.
959 ///
960 /// If `f` returns an error the value is dropped and the initializer will forward the error.
961 ///
962 /// # Examples
963 ///
964 /// ```rust
965 /// # #![expect(clippy::disallowed_names)]
966 /// use kernel::{types::Opaque, init::{self, init_from_closure}};
967 /// struct Foo {
968 /// buf: [u8; 1_000_000],
969 /// }
970 ///
971 /// impl Foo {
972 /// fn setup(&mut self) {
973 /// pr_info!("Setting up foo");
974 /// }
975 /// }
976 ///
977 /// let foo = init!(Foo {
978 /// buf <- init::zeroed()
979 /// }).chain(|foo| {
980 /// foo.setup();
981 /// Ok(())
982 /// });
983 /// ```
984 fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E>
985 where
986 F: FnOnce(&mut T) -> Result<(), E>,
987 {
988 ChainInit(self, f, PhantomData)
989 }
990 }
992 /// An initializer returned by [`Init::chain`].
993 pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, KBox<T>)>);
995 // SAFETY: The `__init` function is implemented such that it
996 // - returns `Ok(())` on successful initialization,
997 // - returns `Err(err)` on error and in this case `slot` will be dropped.
998 unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E>
999 where
1000 I: Init<T, E>,
1001 F: FnOnce(&mut T) -> Result<(), E>,
1002 {
1003 unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
1004 // SAFETY: All requirements fulfilled since this function is `__init`.
1005 unsafe { self.0.__pinned_init(slot)? };
1006 // SAFETY: The above call initialized `slot` and we still have unique access.
1007 (self.1)(unsafe { &mut *slot }).inspect_err(|_|
1008 // SAFETY: `slot` was initialized above.
1009 unsafe { core::ptr::drop_in_place(slot) })
1010 }
1011 }
1013 // SAFETY: `__pinned_init` behaves exactly the same as `__init`.
1014 unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E>
1015 where
1016 I: Init<T, E>,
1017 F: FnOnce(&mut T) -> Result<(), E>,
1018 {
1019 unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
1020 // SAFETY: `__init` has less strict requirements compared to `__pinned_init`.
1021 unsafe { self.__init(slot) }
1022 }
1023 }
1025 /// Creates a new [`PinInit<T, E>`] from the given closure.
1026 ///
1027 /// # Safety
1028 ///
1029 /// The closure:
1030 /// - returns `Ok(())` if it initialized every field of `slot`,
1031 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1032 /// - `slot` can be deallocated without UB occurring,
1033 /// - `slot` does not need to be dropped,
1034 /// - `slot` is not partially initialized.
1035 /// - may assume that the `slot` does not move if `T: !Unpin`,
1036 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1037 #[inline]
1038 pub const unsafe fn pin_init_from_closure<T: ?Sized, E>(
1039 f: impl FnOnce(*mut T) -> Result<(), E>,
1040 ) -> impl PinInit<T, E> {
1041 __internal::InitClosure(f, PhantomData)
1042 }
1044 /// Creates a new [`Init<T, E>`] from the given closure.
1045 ///
1046 /// # Safety
1047 ///
1048 /// The closure:
1049 /// - returns `Ok(())` if it initialized every field of `slot`,
1050 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1051 /// - `slot` can be deallocated without UB occurring,
1052 /// - `slot` does not need to be dropped,
1053 /// - `slot` is not partially initialized.
1054 /// - the `slot` may move after initialization.
1055 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1056 #[inline]
1057 pub const unsafe fn init_from_closure<T: ?Sized, E>(
1058 f: impl FnOnce(*mut T) -> Result<(), E>,
1059 ) -> impl Init<T, E> {
1060 __internal::InitClosure(f, PhantomData)
1061 }
1063 /// An initializer that leaves the memory uninitialized.
1064 ///
1065 /// The initializer is a no-op. The `slot` memory is not changed.
1066 #[inline]
1067 pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
1068 // SAFETY: The memory is allowed to be uninitialized.
1069 unsafe { init_from_closure(|_| Ok(())) }
1070 }
1072 /// Initializes an array by initializing each element via the provided initializer.
1073 ///
1074 /// # Examples
1075 ///
1076 /// ```rust
1077 /// use kernel::{alloc::KBox, error::Error, init::init_array_from_fn};
1078 /// let array: KBox<[usize; 1_000]> =
1079 /// KBox::init::<Error>(init_array_from_fn(|i| i), GFP_KERNEL).unwrap();
1080 /// assert_eq!(array.len(), 1_000);
1081 /// ```
1082 pub fn init_array_from_fn<I, const N: usize, T, E>(
1083 mut make_init: impl FnMut(usize) -> I,
1084 ) -> impl Init<[T; N], E>
1085 where
1086 I: Init<T, E>,
1087 {
1088 let init = move |slot: *mut [T; N]| {
1089 let slot = slot.cast::<T>();
1090 // Counts the number of initialized elements and when dropped drops that many elements from
1091 // `slot`.
1092 let mut init_count = ScopeGuard::new_with_data(0, |i| {
1093 // We now free every element that has been initialized before.
1094 // SAFETY: The loop initialized exactly the values from 0..i and since we
1095 // return `Err` below, the caller will consider the memory at `slot` as
1096 // uninitialized.
1097 unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
1098 });
1099 for i in 0..N {
1100 let init = make_init(i);
1101 // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
1102 let ptr = unsafe { slot.add(i) };
1103 // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
1104 // requirements.
1105 unsafe { init.__init(ptr) }?;
1106 *init_count += 1;
1107 }
1108 init_count.dismiss();
1109 Ok(())
1110 };
1111 // SAFETY: The initializer above initializes every element of the array. On failure it drops
1112 // any initialized elements and returns `Err`.
1113 unsafe { init_from_closure(init) }
1114 }
1116 /// Initializes an array by initializing each element via the provided initializer.
1117 ///
1118 /// # Examples
1119 ///
1120 /// ```rust
1121 /// use kernel::{sync::{Arc, Mutex}, init::pin_init_array_from_fn, new_mutex};
1122 /// let array: Arc<[Mutex<usize>; 1_000]> =
1123 /// Arc::pin_init(pin_init_array_from_fn(|i| new_mutex!(i)), GFP_KERNEL).unwrap();
1124 /// assert_eq!(array.len(), 1_000);
1125 /// ```
1126 pub fn pin_init_array_from_fn<I, const N: usize, T, E>(
1127 mut make_init: impl FnMut(usize) -> I,
1128 ) -> impl PinInit<[T; N], E>
1129 where
1130 I: PinInit<T, E>,
1131 {
1132 let init = move |slot: *mut [T; N]| {
1133 let slot = slot.cast::<T>();
1134 // Counts the number of initialized elements and when dropped drops that many elements from
1135 // `slot`.
1136 let mut init_count = ScopeGuard::new_with_data(0, |i| {
1137 // We now free every element that has been initialized before.
1138 // SAFETY: The loop initialized exactly the values from 0..i and since we
1139 // return `Err` below, the caller will consider the memory at `slot` as
1140 // uninitialized.
1141 unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
1142 });
1143 for i in 0..N {
1144 let init = make_init(i);
1145 // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
1146 let ptr = unsafe { slot.add(i) };
1147 // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
1148 // requirements.
1149 unsafe { init.__pinned_init(ptr) }?;
1150 *init_count += 1;
1151 }
1152 init_count.dismiss();
1153 Ok(())
1154 };
1155 // SAFETY: The initializer above initializes every element of the array. On failure it drops
1156 // any initialized elements and returns `Err`.
1157 unsafe { pin_init_from_closure(init) }
1158 }
1160 // SAFETY: Every type can be initialized by-value.
1161 unsafe impl<T, E> Init<T, E> for T {
1162 unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
1163 // SAFETY: TODO.
1164 unsafe { slot.write(self) };
1165 Ok(())
1166 }
1167 }
1169 // SAFETY: Every type can be initialized by-value. `__pinned_init` calls `__init`.
1170 unsafe impl<T, E> PinInit<T, E> for T {
1171 unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
1172 // SAFETY: TODO.
1173 unsafe { self.__init(slot) }
1174 }
1175 }
1177 /// Smart pointer that can initialize memory in-place.
1178 pub trait InPlaceInit<T>: Sized {
1179 /// Pinned version of `Self`.
1180 ///
1181 /// If a type already implicitly pins its pointee, `Pin<Self>` is unnecessary. In this case use
1182 /// `Self`, otherwise just use `Pin<Self>`.
1183 type PinnedSelf;
1185 /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
1186 /// type.
1187 ///
1188 /// If `T: !Unpin` it will not be able to move afterwards.
1189 fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
1190 where
1191 E: From<AllocError>;
1193 /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
1194 /// type.
1195 ///
1196 /// If `T: !Unpin` it will not be able to move afterwards.
1197 fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Self::PinnedSelf>
1198 where
1199 Error: From<E>,
1200 {
1201 // SAFETY: We delegate to `init` and only change the error type.
1202 let init = unsafe {
1203 pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
1204 };
1205 Self::try_pin_init(init, flags)
1206 }
1208 /// Use the given initializer to in-place initialize a `T`.
1209 fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
1210 where
1211 E: From<AllocError>;
1213 /// Use the given initializer to in-place initialize a `T`.
1214 fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self>
1215 where
1216 Error: From<E>,
1217 {
1218 // SAFETY: We delegate to `init` and only change the error type.
1219 let init = unsafe {
1220 init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
1221 };
1222 Self::try_init(init, flags)
1223 }
1224 }
1226 impl<T> InPlaceInit<T> for Arc<T> {
1227 type PinnedSelf = Self;
1229 #[inline]
1230 fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
1231 where
1232 E: From<AllocError>,
1233 {
1234 UniqueArc::try_pin_init(init, flags).map(|u| u.into())
1235 }
1237 #[inline]
1238 fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
1239 where
1240 E: From<AllocError>,
1241 {
1242 UniqueArc::try_init(init, flags).map(|u| u.into())
1243 }
1244 }
1246 impl<T> InPlaceInit<T> for UniqueArc<T> {
1247 type PinnedSelf = Pin<Self>;
1249 #[inline]
1250 fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
1251 where
1252 E: From<AllocError>,
1253 {
1254 UniqueArc::new_uninit(flags)?.write_pin_init(init)
1255 }
1257 #[inline]
1258 fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
1259 where
1260 E: From<AllocError>,
1261 {
1262 UniqueArc::new_uninit(flags)?.write_init(init)
1263 }
1264 }
1266 /// Smart pointer containing uninitialized memory and that can write a value.
1267 pub trait InPlaceWrite<T> {
1268 /// The type `Self` turns into when the contents are initialized.
1269 type Initialized;
1271 /// Use the given initializer to write a value into `self`.
1272 ///
1273 /// Does not drop the current value and considers it as uninitialized memory.
1274 fn write_init<E>(self, init: impl Init<T, E>) -> Result<Self::Initialized, E>;
1276 /// Use the given pin-initializer to write a value into `self`.
1277 ///
1278 /// Does not drop the current value and considers it as uninitialized memory.
1279 fn write_pin_init<E>(self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E>;
1280 }
1282 impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> {
1283 type Initialized = UniqueArc<T>;
1285 fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
1286 let slot = self.as_mut_ptr();
1287 // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1288 // slot is valid.
1289 unsafe { init.__init(slot)? };
1290 // SAFETY: All fields have been initialized.
1291 Ok(unsafe { self.assume_init() })
1292 }
1294 fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
1295 let slot = self.as_mut_ptr();
1296 // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1297 // slot is valid and will not be moved, because we pin it later.
1298 unsafe { init.__pinned_init(slot)? };
1299 // SAFETY: All fields have been initialized.
1300 Ok(unsafe { self.assume_init() }.into())
1301 }
1302 }
1304 /// Trait facilitating pinned destruction.
1305 ///
1306 /// Use [`pinned_drop`] to implement this trait safely:
1307 ///
1308 /// ```rust
1309 /// # use kernel::sync::Mutex;
1310 /// use kernel::macros::pinned_drop;
1311 /// use core::pin::Pin;
1312 /// #[pin_data(PinnedDrop)]
1313 /// struct Foo {
1314 /// #[pin]
1315 /// mtx: Mutex<usize>,
1316 /// }
1317 ///
1318 /// #[pinned_drop]
1319 /// impl PinnedDrop for Foo {
1320 /// fn drop(self: Pin<&mut Self>) {
1321 /// pr_info!("Foo is being dropped!");
1322 /// }
1323 /// }
1324 /// ```
1325 ///
1326 /// # Safety
1327 ///
1328 /// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl.
1329 ///
1330 /// [`pinned_drop`]: kernel::macros::pinned_drop
1331 pub unsafe trait PinnedDrop: __internal::HasPinData {
1332 /// Executes the pinned destructor of this type.
1333 ///
1334 /// While this function is marked safe, it is actually unsafe to call it manually. For this
1335 /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code
1336 /// and thus prevents this function from being called where it should not.
1337 ///
1338 /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute
1339 /// automatically.
1340 fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop);
1341 }
1343 /// Marker trait for types that can be initialized by writing just zeroes.
1344 ///
1345 /// # Safety
1346 ///
1347 /// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words,
1348 /// this is not UB:
1349 ///
1350 /// ```rust,ignore
1351 /// let val: Self = unsafe { core::mem::zeroed() };
1352 /// ```
1353 pub unsafe trait Zeroable {}
1355 /// Create a new zeroed T.
1356 ///
1357 /// The returned initializer will write `0x00` to every byte of the given `slot`.
1358 #[inline]
1359 pub fn zeroed<T: Zeroable>() -> impl Init<T> {
1360 // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T`
1361 // and because we write all zeroes, the memory is initialized.
1362 unsafe {
1363 init_from_closure(|slot: *mut T| {
1364 slot.write_bytes(0, 1);
1365 Ok(())
1366 })
1367 }
1368 }
1370 macro_rules! impl_zeroable {
1371 ($($({$($generics:tt)*})? $t:ty, )*) => {
1372 // SAFETY: Safety comments written in the macro invocation.
1373 $(unsafe impl$($($generics)*)? Zeroable for $t {})*
1374 };
1375 }
1377 impl_zeroable! {
1378 // SAFETY: All primitives that are allowed to be zero.
1379 bool,
1380 char,
1381 u8, u16, u32, u64, u128, usize,
1382 i8, i16, i32, i64, i128, isize,
1383 f32, f64,
1385 // Note: do not add uninhabited types (such as `!` or `core::convert::Infallible`) to this list;
1386 // creating an instance of an uninhabited type is immediate undefined behavior. For more on
1387 // uninhabited/empty types, consult The Rustonomicon:
1388 // <https://doc.rust-lang.org/stable/nomicon/exotic-sizes.html#empty-types>. The Rust Reference
1389 // also has information on undefined behavior:
1390 // <https://doc.rust-lang.org/stable/reference/behavior-considered-undefined.html>.
1391 //
1392 // SAFETY: These are inhabited ZSTs; there is nothing to zero and a valid value exists.
1393 {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, (),
1395 // SAFETY: Type is allowed to take any value, including all zeros.
1396 {<T>} MaybeUninit<T>,
1397 // SAFETY: Type is allowed to take any value, including all zeros.
1398 {<T>} Opaque<T>,
1400 // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`.
1401 {<T: ?Sized + Zeroable>} UnsafeCell<T>,
1403 // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
1404 Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>,
1405 Option<NonZeroU128>, Option<NonZeroUsize>,
1406 Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>,
1407 Option<NonZeroI128>, Option<NonZeroIsize>,
1409 // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
1410 //
1411 // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant.
1412 {<T: ?Sized>} Option<NonNull<T>>,
1413 {<T: ?Sized>} Option<KBox<T>>,
1415 // SAFETY: `null` pointer is valid.
1416 //
1417 // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be
1418 // null.
1419 //
1420 // When `Pointee` gets stabilized, we could use
1421 // `T: ?Sized where <T as Pointee>::Metadata: Zeroable`
1422 {<T>} *mut T, {<T>} *const T,
1424 // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be
1425 // zero.
1426 {<T>} *mut [T], {<T>} *const [T], *mut str, *const str,
1428 // SAFETY: `T` is `Zeroable`.
1429 {<const N: usize, T: Zeroable>} [T; N], {<T: Zeroable>} Wrapping<T>,
1430 }
1432 macro_rules! impl_tuple_zeroable {
1433 ($(,)?) => {};
1434 ($first:ident, $($t:ident),* $(,)?) => {
1435 // SAFETY: All elements are zeroable and padding can be zero.
1436 unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {}
1437 impl_tuple_zeroable!($($t),* ,);
1438 }
1439 }
1441 impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J);