rust/kernel/init/macros.rs

   1 // SPDX-License-Identifier: Apache-2.0 OR MIT
   2
   3 //! This module provides the macros that actually implement the proc-macros `pin_data` and
   4 //! `pinned_drop`. It also contains `__init_internal` the implementation of the `{try_}{pin_}init!`
   5 //! macros.
   6 //!
   7 //! These macros should never be called directly, since they expect their input to be
   8 //! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in
   9 //! safe code! Use the public facing macros instead.
  10 //!
  11 //! This architecture has been chosen because the kernel does not yet have access to `syn` which
  12 //! would make matters a lot easier for implementing these as proc-macros.
  13 //!
  14 //! # Macro expansion example
  15 //!
  16 //! This section is intended for readers trying to understand the macros in this module and the
  17 //! `pin_init!` macros from `init.rs`.
  18 //!
  19 //! We will look at the following example:
  20 //!
  21 //! ```rust,ignore
  22 //! # use kernel::init::*;
  23 //! # use core::pin::Pin;
  24 //! #[pin_data]
  25 //! #[repr(C)]
  26 //! struct Bar<T> {
  27 //!     #[pin]
  28 //!     t: T,
  29 //!     pub x: usize,
  30 //! }
  31 //!
  32 //! impl<T> Bar<T> {
  33 //!     fn new(t: T) -> impl PinInit<Self> {
  34 //!         pin_init!(Self { t, x: 0 })
  35 //!     }
  36 //! }
  37 //!
  38 //! #[pin_data(PinnedDrop)]
  39 //! struct Foo {
  40 //!     a: usize,
  41 //!     #[pin]
  42 //!     b: Bar<u32>,
  43 //! }
  44 //!
  45 //! #[pinned_drop]
  46 //! impl PinnedDrop for Foo {
  47 //!     fn drop(self: Pin<&mut Self>) {
  48 //!         pr_info!("{self:p} is getting dropped.");
  49 //!     }
  50 //! }
  51 //!
  52 //! let a = 42;
  53 //! let initializer = pin_init!(Foo {
  54 //!     a,
  55 //!     b <- Bar::new(36),
  56 //! });
  57 //! ```
  58 //!
  59 //! This example includes the most common and important features of the pin-init API.
  60 //!
  61 //! Below you can find individual section about the different macro invocations. Here are some
  62 //! general things we need to take into account when designing macros:
  63 //! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()`
  64 //!   this ensures that the correct item is used, since users could define their own `mod core {}`
  65 //!   and then their own `panic!` inside to execute arbitrary code inside of our macro.
  66 //! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied
  67 //!   expressions inside of an `unsafe` block in the macro, because this would allow users to do
  68 //!   `unsafe` operations without an associated `unsafe` block.
  69 //!
  70 //! ## `#[pin_data]` on `Bar`
  71 //!
  72 //! This macro is used to specify which fields are structurally pinned and which fields are not. It
  73 //! is placed on the struct definition and allows `#[pin]` to be placed on the fields.
  74 //!
  75 //! Here is the definition of `Bar` from our example:
  76 //!
  77 //! ```rust,ignore
  78 //! # use kernel::init::*;
  79 //! #[pin_data]
  80 //! #[repr(C)]
  81 //! struct Bar<T> {
  82 //!     #[pin]
  83 //!     t: T,
  84 //!     pub x: usize,
  85 //! }
  86 //! ```
  87 //!
  88 //! This expands to the following code:
  89 //!
  90 //! ```rust,ignore
  91 //! // Firstly the normal definition of the struct, attributes are preserved:
  92 //! #[repr(C)]
  93 //! struct Bar<T> {
  94 //!     t: T,
  95 //!     pub x: usize,
  96 //! }
  97 //! // Then an anonymous constant is defined, this is because we do not want any code to access the
  98 //! // types that we define inside:
  99 //! const _: () = {
 100 //!     // We define the pin-data carrying struct, it is a ZST and needs to have the same generics,
 101 //!     // since we need to implement access functions for each field and thus need to know its
 102 //!     // type.
 103 //!     struct __ThePinData<T> {
 104 //!         __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
 105 //!     }
 106 //!     // We implement `Copy` for the pin-data struct, since all functions it defines will take
 107 //!     // `self` by value.
 108 //!     impl<T> ::core::clone::Clone for __ThePinData<T> {
 109 //!         fn clone(&self) -> Self {
 110 //!             *self
 111 //!         }
 112 //!     }
 113 //!     impl<T> ::core::marker::Copy for __ThePinData<T> {}
 114 //!     // For every field of `Bar`, the pin-data struct will define a function with the same name
 115 //!     // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the
 116 //!     // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field
 117 //!     // (if pinning is structural for the field, then `PinInit` otherwise `Init`).
 118 //!     #[allow(dead_code)]
 119 //!     impl<T> __ThePinData<T> {
 120 //!         unsafe fn t<E>(
 121 //!             self,
 122 //!             slot: *mut T,
 123 //!             // Since `t` is `#[pin]`, this is `PinInit`.
 124 //!             init: impl ::kernel::init::PinInit<T, E>,
 125 //!         ) -> ::core::result::Result<(), E> {
 126 //!             unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
 127 //!         }
 128 //!         pub unsafe fn x<E>(
 129 //!             self,
 130 //!             slot: *mut usize,
 131 //!             // Since `x` is not `#[pin]`, this is `Init`.
 132 //!             init: impl ::kernel::init::Init<usize, E>,
 133 //!         ) -> ::core::result::Result<(), E> {
 134 //!             unsafe { ::kernel::init::Init::__init(init, slot) }
 135 //!         }
 136 //!     }
 137 //!     // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct
 138 //!     // that we constructed above.
 139 //!     unsafe impl<T> ::kernel::init::__internal::HasPinData for Bar<T> {
 140 //!         type PinData = __ThePinData<T>;
 141 //!         unsafe fn __pin_data() -> Self::PinData {
 142 //!             __ThePinData {
 143 //!                 __phantom: ::core::marker::PhantomData,
 144 //!             }
 145 //!         }
 146 //!     }
 147 //!     // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data
 148 //!     // struct. This is important to ensure that no user can implement a rogue `__pin_data`
 149 //!     // function without using `unsafe`.
 150 //!     unsafe impl<T> ::kernel::init::__internal::PinData for __ThePinData<T> {
 151 //!         type Datee = Bar<T>;
 152 //!     }
 153 //!     // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is
 154 //!     // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned
 155 //!     // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our
 156 //!     // case no such fields exist, hence this is almost empty. The two phantomdata fields exist
 157 //!     // for two reasons:
 158 //!     // - `__phantom`: every generic must be used, since we cannot really know which generics
 159 //!     //   are used, we declare all and then use everything here once.
 160 //!     // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant
 161 //!     //   over it. The lifetime is needed to work around the limitation that trait bounds must
 162 //!     //   not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is
 163 //!     //   unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler
 164 //!     //   into accepting these bounds regardless.
 165 //!     #[allow(dead_code)]
 166 //!     struct __Unpin<'__pin, T> {
 167 //!         __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
 168 //!         __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
 169 //!         // Our only `#[pin]` field is `t`.
 170 //!         t: T,
 171 //!     }
 172 //!     #[doc(hidden)]
 173 //!     impl<'__pin, T> ::core::marker::Unpin for Bar<T>
 174 //!     where
 175 //!         __Unpin<'__pin, T>: ::core::marker::Unpin,
 176 //!     {}
 177 //!     // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users
 178 //!     // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to
 179 //!     // UB with only safe code, so we disallow this by giving a trait implementation error using
 180 //!     // a direct impl and a blanket implementation.
 181 //!     trait MustNotImplDrop {}
 182 //!     // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do
 183 //!     // (normally people want to know if a type has any kind of drop glue at all, here we want
 184 //!     // to know if it has any kind of custom drop glue, which is exactly what this bound does).
 185 //!     #[expect(drop_bounds)]
 186 //!     impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
 187 //!     impl<T> MustNotImplDrop for Bar<T> {}
 188 //!     // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to
 189 //!     // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed
 190 //!     // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`.
 191 //!     #[expect(non_camel_case_types)]
 192 //!     trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
 193 //!     impl<
 194 //!         T: ::kernel::init::PinnedDrop,
 195 //!     > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
 196 //!     impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {}
 197 //! };
 198 //! ```
 199 //!
 200 //! ## `pin_init!` in `impl Bar`
 201 //!
 202 //! This macro creates an pin-initializer for the given struct. It requires that the struct is
 203 //! annotated by `#[pin_data]`.
 204 //!
 205 //! Here is the impl on `Bar` defining the new function:
 206 //!
 207 //! ```rust,ignore
 208 //! impl<T> Bar<T> {
 209 //!     fn new(t: T) -> impl PinInit<Self> {
 210 //!         pin_init!(Self { t, x: 0 })
 211 //!     }
 212 //! }
 213 //! ```
 214 //!
 215 //! This expands to the following code:
 216 //!
 217 //! ```rust,ignore
 218 //! impl<T> Bar<T> {
 219 //!     fn new(t: T) -> impl PinInit<Self> {
 220 //!         {
 221 //!             // We do not want to allow arbitrary returns, so we declare this type as the `Ok`
 222 //!             // return type and shadow it later when we insert the arbitrary user code. That way
 223 //!             // there will be no possibility of returning without `unsafe`.
 224 //!             struct __InitOk;
 225 //!             // Get the data about fields from the supplied type.
 226 //!             // - the function is unsafe, hence the unsafe block
 227 //!             // - we `use` the `HasPinData` trait in the block, it is only available in that
 228 //!             //   scope.
 229 //!             let data = unsafe {
 230 //!                 use ::kernel::init::__internal::HasPinData;
 231 //!                 Self::__pin_data()
 232 //!             };
 233 //!             // Ensure that `data` really is of type `PinData` and help with type inference:
 234 //!             let init = ::kernel::init::__internal::PinData::make_closure::<
 235 //!                 _,
 236 //!                 __InitOk,
 237 //!                 ::core::convert::Infallible,
 238 //!             >(data, move |slot| {
 239 //!                 {
 240 //!                     // Shadow the structure so it cannot be used to return early. If a user
 241 //!                     // tries to write `return Ok(__InitOk)`, then they get a type error,
 242 //!                     // since that will refer to this struct instead of the one defined
 243 //!                     // above.
 244 //!                     struct __InitOk;
 245 //!                     // This is the expansion of `t,`, which is syntactic sugar for `t: t,`.
 246 //!                     {
 247 //!                         unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) };
 248 //!                     }
 249 //!                     // Since initialization could fail later (not in this case, since the
 250 //!                     // error type is `Infallible`) we will need to drop this field if there
 251 //!                     // is an error later. This `DropGuard` will drop the field when it gets
 252 //!                     // dropped and has not yet been forgotten.
 253 //!                     let __t_guard = unsafe {
 254 //!                         ::pinned_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t))
 255 //!                     };
 256 //!                     // Expansion of `x: 0,`:
 257 //!                     // Since this can be an arbitrary expression we cannot place it inside
 258 //!                     // of the `unsafe` block, so we bind it here.
 259 //!                     {
 260 //!                         let x = 0;
 261 //!                         unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) };
 262 //!                     }
 263 //!                     // We again create a `DropGuard`.
 264 //!                     let __x_guard = unsafe {
 265 //!                         ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x))
 266 //!                     };
 267 //!                     // Since initialization has successfully completed, we can now forget
 268 //!                     // the guards. This is not `mem::forget`, since we only have
 269 //!                     // `&DropGuard`.
 270 //!                     ::core::mem::forget(__x_guard);
 271 //!                     ::core::mem::forget(__t_guard);
 272 //!                     // Here we use the type checker to ensure that every field has been
 273 //!                     // initialized exactly once, since this is `if false` it will never get
 274 //!                     // executed, but still type-checked.
 275 //!                     // Additionally we abuse `slot` to automatically infer the correct type
 276 //!                     // for the struct. This is also another check that every field is
 277 //!                     // accessible from this scope.
 278 //!                     #[allow(unreachable_code, clippy::diverging_sub_expression)]
 279 //!                     let _ = || {
 280 //!                         unsafe {
 281 //!                             ::core::ptr::write(
 282 //!                                 slot,
 283 //!                                 Self {
 284 //!                                     // We only care about typecheck finding every field
 285 //!                                     // here, the expression does not matter, just conjure
 286 //!                                     // one using `panic!()`:
 287 //!                                     t: ::core::panic!(),
 288 //!                                     x: ::core::panic!(),
 289 //!                                 },
 290 //!                             );
 291 //!                         };
 292 //!                     };
 293 //!                 }
 294 //!                 // We leave the scope above and gain access to the previously shadowed
 295 //!                 // `__InitOk` that we need to return.
 296 //!                 Ok(__InitOk)
 297 //!             });
 298 //!             // Change the return type from `__InitOk` to `()`.
 299 //!             let init = move |
 300 //!                 slot,
 301 //!             | -> ::core::result::Result<(), ::core::convert::Infallible> {
 302 //!                 init(slot).map(|__InitOk| ())
 303 //!             };
 304 //!             // Construct the initializer.
 305 //!             let init = unsafe {
 306 //!                 ::kernel::init::pin_init_from_closure::<
 307 //!                     _,
 308 //!                     ::core::convert::Infallible,
 309 //!                 >(init)
 310 //!             };
 311 //!             init
 312 //!         }
 313 //!     }
 314 //! }
 315 //! ```
 316 //!
 317 //! ## `#[pin_data]` on `Foo`
 318 //!
 319 //! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the
 320 //! differences/new things in the expansion of the `Foo` definition:
 321 //!
 322 //! ```rust,ignore
 323 //! #[pin_data(PinnedDrop)]
 324 //! struct Foo {
 325 //!     a: usize,
 326 //!     #[pin]
 327 //!     b: Bar<u32>,
 328 //! }
 329 //! ```
 330 //!
 331 //! This expands to the following code:
 332 //!
 333 //! ```rust,ignore
 334 //! struct Foo {
 335 //!     a: usize,
 336 //!     b: Bar<u32>,
 337 //! }
 338 //! const _: () = {
 339 //!     struct __ThePinData {
 340 //!         __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
 341 //!     }
 342 //!     impl ::core::clone::Clone for __ThePinData {
 343 //!         fn clone(&self) -> Self {
 344 //!             *self
 345 //!         }
 346 //!     }
 347 //!     impl ::core::marker::Copy for __ThePinData {}
 348 //!     #[allow(dead_code)]
 349 //!     impl __ThePinData {
 350 //!         unsafe fn b<E>(
 351 //!             self,
 352 //!             slot: *mut Bar<u32>,
 353 //!             init: impl ::kernel::init::PinInit<Bar<u32>, E>,
 354 //!         ) -> ::core::result::Result<(), E> {
 355 //!             unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
 356 //!         }
 357 //!         unsafe fn a<E>(
 358 //!             self,
 359 //!             slot: *mut usize,
 360 //!             init: impl ::kernel::init::Init<usize, E>,
 361 //!         ) -> ::core::result::Result<(), E> {
 362 //!             unsafe { ::kernel::init::Init::__init(init, slot) }
 363 //!         }
 364 //!     }
 365 //!     unsafe impl ::kernel::init::__internal::HasPinData for Foo {
 366 //!         type PinData = __ThePinData;
 367 //!         unsafe fn __pin_data() -> Self::PinData {
 368 //!             __ThePinData {
 369 //!                 __phantom: ::core::marker::PhantomData,
 370 //!             }
 371 //!         }
 372 //!     }
 373 //!     unsafe impl ::kernel::init::__internal::PinData for __ThePinData {
 374 //!         type Datee = Foo;
 375 //!     }
 376 //!     #[allow(dead_code)]
 377 //!     struct __Unpin<'__pin> {
 378 //!         __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
 379 //!         __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
 380 //!         b: Bar<u32>,
 381 //!     }
 382 //!     #[doc(hidden)]
 383 //!     impl<'__pin> ::core::marker::Unpin for Foo
 384 //!     where
 385 //!         __Unpin<'__pin>: ::core::marker::Unpin,
 386 //!     {}
 387 //!     // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to
 388 //!     // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like
 389 //!     // before, instead we implement `Drop` here and delegate to `PinnedDrop`.
 390 //!     impl ::core::ops::Drop for Foo {
 391 //!         fn drop(&mut self) {
 392 //!             // Since we are getting dropped, no one else has a reference to `self` and thus we
 393 //!             // can assume that we never move.
 394 //!             let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
 395 //!             // Create the unsafe token that proves that we are inside of a destructor, this
 396 //!             // type is only allowed to be created in a destructor.
 397 //!             let token = unsafe { ::kernel::init::__internal::OnlyCallFromDrop::new() };
 398 //!             ::kernel::init::PinnedDrop::drop(pinned, token);
 399 //!         }
 400 //!     }
 401 //! };
 402 //! ```
 403 //!
 404 //! ## `#[pinned_drop]` on `impl PinnedDrop for Foo`
 405 //!
 406 //! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an
 407 //! extra parameter that should not be used at all. The macro hides that parameter.
 408 //!
 409 //! Here is the `PinnedDrop` impl for `Foo`:
 410 //!
 411 //! ```rust,ignore
 412 //! #[pinned_drop]
 413 //! impl PinnedDrop for Foo {
 414 //!     fn drop(self: Pin<&mut Self>) {
 415 //!         pr_info!("{self:p} is getting dropped.");
 416 //!     }
 417 //! }
 418 //! ```
 419 //!
 420 //! This expands to the following code:
 421 //!
 422 //! ```rust,ignore
 423 //! // `unsafe`, full path and the token parameter are added, everything else stays the same.
 424 //! unsafe impl ::kernel::init::PinnedDrop for Foo {
 425 //!     fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) {
 426 //!         pr_info!("{self:p} is getting dropped.");
 427 //!     }
 428 //! }
 429 //! ```
 430 //!
 431 //! ## `pin_init!` on `Foo`
 432 //!
 433 //! Since we already took a look at `pin_init!` on `Bar`, this section will only show the expansion
 434 //! of `pin_init!` on `Foo`:
 435 //!
 436 //! ```rust,ignore
 437 //! let a = 42;
 438 //! let initializer = pin_init!(Foo {
 439 //!     a,
 440 //!     b <- Bar::new(36),
 441 //! });
 442 //! ```
 443 //!
 444 //! This expands to the following code:
 445 //!
 446 //! ```rust,ignore
 447 //! let a = 42;
 448 //! let initializer = {
 449 //!     struct __InitOk;
 450 //!     let data = unsafe {
 451 //!         use ::kernel::init::__internal::HasPinData;
 452 //!         Foo::__pin_data()
 453 //!     };
 454 //!     let init = ::kernel::init::__internal::PinData::make_closure::<
 455 //!         _,
 456 //!         __InitOk,
 457 //!         ::core::convert::Infallible,
 458 //!     >(data, move |slot| {
 459 //!         {
 460 //!             struct __InitOk;
 461 //!             {
 462 //!                 unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) };
 463 //!             }
 464 //!             let __a_guard = unsafe {
 465 //!                 ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a))
 466 //!             };
 467 //!             let init = Bar::new(36);
 468 //!             unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? };
 469 //!             let __b_guard = unsafe {
 470 //!                 ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b))
 471 //!             };
 472 //!             ::core::mem::forget(__b_guard);
 473 //!             ::core::mem::forget(__a_guard);
 474 //!             #[allow(unreachable_code, clippy::diverging_sub_expression)]
 475 //!             let _ = || {
 476 //!                 unsafe {
 477 //!                     ::core::ptr::write(
 478 //!                         slot,
 479 //!                         Foo {
 480 //!                             a: ::core::panic!(),
 481 //!                             b: ::core::panic!(),
 482 //!                         },
 483 //!                     );
 484 //!                 };
 485 //!             };
 486 //!         }
 487 //!         Ok(__InitOk)
 488 //!     });
 489 //!     let init = move |
 490 //!         slot,
 491 //!     | -> ::core::result::Result<(), ::core::convert::Infallible> {
 492 //!         init(slot).map(|__InitOk| ())
 493 //!     };
 494 //!     let init = unsafe {
 495 //!         ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init)
 496 //!     };
 497 //!     init
 498 //! };
 499 //! ```
 500
 501 /// Creates a `unsafe impl<...> PinnedDrop for $type` block.
 502 ///
 503 /// See [`PinnedDrop`] for more information.
 504 #[doc(hidden)]
 505 #[macro_export]
 506 macro_rules! __pinned_drop {
 507     (
 508         @impl_sig($($impl_sig:tt)*),
 509         @impl_body(
 510             $(#[$($attr:tt)*])*
 511             fn drop($($sig:tt)*) {
 512                 $($inner:tt)*
 513             }
 514         ),
 515     ) => {
 516         // SAFETY: TODO.
 517         unsafe $($impl_sig)* {
 518             // Inherit all attributes and the type/ident tokens for the signature.
 519             $(#[$($attr)*])*
 520             fn drop($($sig)*, _: $crate::init::__internal::OnlyCallFromDrop) {
 521                 $($inner)*
 522             }
 523         }
 524     }
 525 }
 526
 527 /// This macro first parses the struct definition such that it separates pinned and not pinned
 528 /// fields. Afterwards it declares the struct and implement the `PinData` trait safely.
 529 #[doc(hidden)]
 530 #[macro_export]
 531 macro_rules! __pin_data {
 532     // Proc-macro entry point, this is supplied by the proc-macro pre-parsing.
 533     (parse_input:
 534         @args($($pinned_drop:ident)?),
 535         @sig(
 536             $(#[$($struct_attr:tt)*])*
 537             $vis:vis struct $name:ident
 538             $(where $($whr:tt)*)?
 539         ),
 540         @impl_generics($($impl_generics:tt)*),
 541         @ty_generics($($ty_generics:tt)*),
 542         @decl_generics($($decl_generics:tt)*),
 543         @body({ $($fields:tt)* }),
 544     ) => {
 545         // We now use token munching to iterate through all of the fields. While doing this we
 546         // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user
 547         // wants these to be structurally pinned. The rest of the fields are the
 548         // 'not pinned fields'. Additionally we collect all fields, since we need them in the right
 549         // order to declare the struct.
 550         //
 551         // In this call we also put some explaining comments for the parameters.
 552         $crate::__pin_data!(find_pinned_fields:
 553             // Attributes on the struct itself, these will just be propagated to be put onto the
 554             // struct definition.
 555             @struct_attrs($(#[$($struct_attr)*])*),
 556             // The visibility of the struct.
 557             @vis($vis),
 558             // The name of the struct.
 559             @name($name),
 560             // The 'impl generics', the generics that will need to be specified on the struct inside
 561             // of an `impl<$ty_generics>` block.
 562             @impl_generics($($impl_generics)*),
 563             // The 'ty generics', the generics that will need to be specified on the impl blocks.
 564             @ty_generics($($ty_generics)*),
 565             // The 'decl generics', the generics that need to be specified on the struct
 566             // definition.
 567             @decl_generics($($decl_generics)*),
 568             // The where clause of any impl block and the declaration.
 569             @where($($($whr)*)?),
 570             // The remaining fields tokens that need to be processed.
 571             // We add a `,` at the end to ensure correct parsing.
 572             @fields_munch($($fields)* ,),
 573             // The pinned fields.
 574             @pinned(),
 575             // The not pinned fields.
 576             @not_pinned(),
 577             // All fields.
 578             @fields(),
 579             // The accumulator containing all attributes already parsed.
 580             @accum(),
 581             // Contains `yes` or `` to indicate if `#[pin]` was found on the current field.
 582             @is_pinned(),
 583             // The proc-macro argument, this should be `PinnedDrop` or ``.
 584             @pinned_drop($($pinned_drop)?),
 585         );
 586     };
 587     (find_pinned_fields:
 588         @struct_attrs($($struct_attrs:tt)*),
 589         @vis($vis:vis),
 590         @name($name:ident),
 591         @impl_generics($($impl_generics:tt)*),
 592         @ty_generics($($ty_generics:tt)*),
 593         @decl_generics($($decl_generics:tt)*),
 594         @where($($whr:tt)*),
 595         // We found a PhantomPinned field, this should generally be pinned!
 596         @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*),
 597         @pinned($($pinned:tt)*),
 598         @not_pinned($($not_pinned:tt)*),
 599         @fields($($fields:tt)*),
 600         @accum($($accum:tt)*),
 601         // This field is not pinned.
 602         @is_pinned(),
 603         @pinned_drop($($pinned_drop:ident)?),
 604     ) => {
 605         ::core::compile_error!(concat!(
 606             "The field `",
 607             stringify!($field),
 608             "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.",
 609         ));
 610         $crate::__pin_data!(find_pinned_fields:
 611             @struct_attrs($($struct_attrs)*),
 612             @vis($vis),
 613             @name($name),
 614             @impl_generics($($impl_generics)*),
 615             @ty_generics($($ty_generics)*),
 616             @decl_generics($($decl_generics)*),
 617             @where($($whr)*),
 618             @fields_munch($($rest)*),
 619             @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,),
 620             @not_pinned($($not_pinned)*),
 621             @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,),
 622             @accum(),
 623             @is_pinned(),
 624             @pinned_drop($($pinned_drop)?),
 625         );
 626     };
 627     (find_pinned_fields:
 628         @struct_attrs($($struct_attrs:tt)*),
 629         @vis($vis:vis),
 630         @name($name:ident),
 631         @impl_generics($($impl_generics:tt)*),
 632         @ty_generics($($ty_generics:tt)*),
 633         @decl_generics($($decl_generics:tt)*),
 634         @where($($whr:tt)*),
 635         // We reached the field declaration.
 636         @fields_munch($field:ident : $type:ty, $($rest:tt)*),
 637         @pinned($($pinned:tt)*),
 638         @not_pinned($($not_pinned:tt)*),
 639         @fields($($fields:tt)*),
 640         @accum($($accum:tt)*),
 641         // This field is pinned.
 642         @is_pinned(yes),
 643         @pinned_drop($($pinned_drop:ident)?),
 644     ) => {
 645         $crate::__pin_data!(find_pinned_fields:
 646             @struct_attrs($($struct_attrs)*),
 647             @vis($vis),
 648             @name($name),
 649             @impl_generics($($impl_generics)*),
 650             @ty_generics($($ty_generics)*),
 651             @decl_generics($($decl_generics)*),
 652             @where($($whr)*),
 653             @fields_munch($($rest)*),
 654             @pinned($($pinned)* $($accum)* $field: $type,),
 655             @not_pinned($($not_pinned)*),
 656             @fields($($fields)* $($accum)* $field: $type,),
 657             @accum(),
 658             @is_pinned(),
 659             @pinned_drop($($pinned_drop)?),
 660         );
 661     };
 662     (find_pinned_fields:
 663         @struct_attrs($($struct_attrs:tt)*),
 664         @vis($vis:vis),
 665         @name($name:ident),
 666         @impl_generics($($impl_generics:tt)*),
 667         @ty_generics($($ty_generics:tt)*),
 668         @decl_generics($($decl_generics:tt)*),
 669         @where($($whr:tt)*),
 670         // We reached the field declaration.
 671         @fields_munch($field:ident : $type:ty, $($rest:tt)*),
 672         @pinned($($pinned:tt)*),
 673         @not_pinned($($not_pinned:tt)*),
 674         @fields($($fields:tt)*),
 675         @accum($($accum:tt)*),
 676         // This field is not pinned.
 677         @is_pinned(),
 678         @pinned_drop($($pinned_drop:ident)?),
 679     ) => {
 680         $crate::__pin_data!(find_pinned_fields:
 681             @struct_attrs($($struct_attrs)*),
 682             @vis($vis),
 683             @name($name),
 684             @impl_generics($($impl_generics)*),
 685             @ty_generics($($ty_generics)*),
 686             @decl_generics($($decl_generics)*),
 687             @where($($whr)*),
 688             @fields_munch($($rest)*),
 689             @pinned($($pinned)*),
 690             @not_pinned($($not_pinned)* $($accum)* $field: $type,),
 691             @fields($($fields)* $($accum)* $field: $type,),
 692             @accum(),
 693             @is_pinned(),
 694             @pinned_drop($($pinned_drop)?),
 695         );
 696     };
 697     (find_pinned_fields:
 698         @struct_attrs($($struct_attrs:tt)*),
 699         @vis($vis:vis),
 700         @name($name:ident),
 701         @impl_generics($($impl_generics:tt)*),
 702         @ty_generics($($ty_generics:tt)*),
 703         @decl_generics($($decl_generics:tt)*),
 704         @where($($whr:tt)*),
 705         // We found the `#[pin]` attr.
 706         @fields_munch(#[pin] $($rest:tt)*),
 707         @pinned($($pinned:tt)*),
 708         @not_pinned($($not_pinned:tt)*),
 709         @fields($($fields:tt)*),
 710         @accum($($accum:tt)*),
 711         @is_pinned($($is_pinned:ident)?),
 712         @pinned_drop($($pinned_drop:ident)?),
 713     ) => {
 714         $crate::__pin_data!(find_pinned_fields:
 715             @struct_attrs($($struct_attrs)*),
 716             @vis($vis),
 717             @name($name),
 718             @impl_generics($($impl_generics)*),
 719             @ty_generics($($ty_generics)*),
 720             @decl_generics($($decl_generics)*),
 721             @where($($whr)*),
 722             @fields_munch($($rest)*),
 723             // We do not include `#[pin]` in the list of attributes, since it is not actually an
 724             // attribute that is defined somewhere.
 725             @pinned($($pinned)*),
 726             @not_pinned($($not_pinned)*),
 727             @fields($($fields)*),
 728             @accum($($accum)*),
 729             // Set this to `yes`.
 730             @is_pinned(yes),
 731             @pinned_drop($($pinned_drop)?),
 732         );
 733     };
 734     (find_pinned_fields:
 735         @struct_attrs($($struct_attrs:tt)*),
 736         @vis($vis:vis),
 737         @name($name:ident),
 738         @impl_generics($($impl_generics:tt)*),
 739         @ty_generics($($ty_generics:tt)*),
 740         @decl_generics($($decl_generics:tt)*),
 741         @where($($whr:tt)*),
 742         // We reached the field declaration with visibility, for simplicity we only munch the
 743         // visibility and put it into `$accum`.
 744         @fields_munch($fvis:vis $field:ident $($rest:tt)*),
 745         @pinned($($pinned:tt)*),
 746         @not_pinned($($not_pinned:tt)*),
 747         @fields($($fields:tt)*),
 748         @accum($($accum:tt)*),
 749         @is_pinned($($is_pinned:ident)?),
 750         @pinned_drop($($pinned_drop:ident)?),
 751     ) => {
 752         $crate::__pin_data!(find_pinned_fields:
 753             @struct_attrs($($struct_attrs)*),
 754             @vis($vis),
 755             @name($name),
 756             @impl_generics($($impl_generics)*),
 757             @ty_generics($($ty_generics)*),
 758             @decl_generics($($decl_generics)*),
 759             @where($($whr)*),
 760             @fields_munch($field $($rest)*),
 761             @pinned($($pinned)*),
 762             @not_pinned($($not_pinned)*),
 763             @fields($($fields)*),
 764             @accum($($accum)* $fvis),
 765             @is_pinned($($is_pinned)?),
 766             @pinned_drop($($pinned_drop)?),
 767         );
 768     };
 769     (find_pinned_fields:
 770         @struct_attrs($($struct_attrs:tt)*),
 771         @vis($vis:vis),
 772         @name($name:ident),
 773         @impl_generics($($impl_generics:tt)*),
 774         @ty_generics($($ty_generics:tt)*),
 775         @decl_generics($($decl_generics:tt)*),
 776         @where($($whr:tt)*),
 777         // Some other attribute, just put it into `$accum`.
 778         @fields_munch(#[$($attr:tt)*] $($rest:tt)*),
 779         @pinned($($pinned:tt)*),
 780         @not_pinned($($not_pinned:tt)*),
 781         @fields($($fields:tt)*),
 782         @accum($($accum:tt)*),
 783         @is_pinned($($is_pinned:ident)?),
 784         @pinned_drop($($pinned_drop:ident)?),
 785     ) => {
 786         $crate::__pin_data!(find_pinned_fields:
 787             @struct_attrs($($struct_attrs)*),
 788             @vis($vis),
 789             @name($name),
 790             @impl_generics($($impl_generics)*),
 791             @ty_generics($($ty_generics)*),
 792             @decl_generics($($decl_generics)*),
 793             @where($($whr)*),
 794             @fields_munch($($rest)*),
 795             @pinned($($pinned)*),
 796             @not_pinned($($not_pinned)*),
 797             @fields($($fields)*),
 798             @accum($($accum)* #[$($attr)*]),
 799             @is_pinned($($is_pinned)?),
 800             @pinned_drop($($pinned_drop)?),
 801         );
 802     };
 803     (find_pinned_fields:
 804         @struct_attrs($($struct_attrs:tt)*),
 805         @vis($vis:vis),
 806         @name($name:ident),
 807         @impl_generics($($impl_generics:tt)*),
 808         @ty_generics($($ty_generics:tt)*),
 809         @decl_generics($($decl_generics:tt)*),
 810         @where($($whr:tt)*),
 811         // We reached the end of the fields, plus an optional additional comma, since we added one
 812         // before and the user is also allowed to put a trailing comma.
 813         @fields_munch($(,)?),
 814         @pinned($($pinned:tt)*),
 815         @not_pinned($($not_pinned:tt)*),
 816         @fields($($fields:tt)*),
 817         @accum(),
 818         @is_pinned(),
 819         @pinned_drop($($pinned_drop:ident)?),
 820     ) => {
 821         // Declare the struct with all fields in the correct order.
 822         $($struct_attrs)*
 823         $vis struct $name <$($decl_generics)*>
 824         where $($whr)*
 825         {
 826             $($fields)*
 827         }
 828
 829         // We put the rest into this const item, because it then will not be accessible to anything
 830         // outside.
 831         const _: () = {
 832             // We declare this struct which will host all of the projection function for our type.
 833             // it will be invariant over all generic parameters which are inherited from the
 834             // struct.
 835             $vis struct __ThePinData<$($impl_generics)*>
 836             where $($whr)*
 837             {
 838                 __phantom: ::core::marker::PhantomData<
 839                     fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
 840                 >,
 841             }
 842
 843             impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*>
 844             where $($whr)*
 845             {
 846                 fn clone(&self) -> Self { *self }
 847             }
 848
 849             impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*>
 850             where $($whr)*
 851             {}
 852
 853             // Make all projection functions.
 854             $crate::__pin_data!(make_pin_data:
 855                 @pin_data(__ThePinData),
 856                 @impl_generics($($impl_generics)*),
 857                 @ty_generics($($ty_generics)*),
 858                 @where($($whr)*),
 859                 @pinned($($pinned)*),
 860                 @not_pinned($($not_pinned)*),
 861             );
 862
 863             // SAFETY: We have added the correct projection functions above to `__ThePinData` and
 864             // we also use the least restrictive generics possible.
 865             unsafe impl<$($impl_generics)*>
 866                 $crate::init::__internal::HasPinData for $name<$($ty_generics)*>
 867             where $($whr)*
 868             {
 869                 type PinData = __ThePinData<$($ty_generics)*>;
 870
 871                 unsafe fn __pin_data() -> Self::PinData {
 872                     __ThePinData { __phantom: ::core::marker::PhantomData }
 873                 }
 874             }
 875
 876             // SAFETY: TODO.
 877             unsafe impl<$($impl_generics)*>
 878                 $crate::init::__internal::PinData for __ThePinData<$($ty_generics)*>
 879             where $($whr)*
 880             {
 881                 type Datee = $name<$($ty_generics)*>;
 882             }
 883
 884             // This struct will be used for the unpin analysis. Since only structurally pinned
 885             // fields are relevant whether the struct should implement `Unpin`.
 886             #[allow(dead_code)]
 887             struct __Unpin <'__pin, $($impl_generics)*>
 888             where $($whr)*
 889             {
 890                 __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
 891                 __phantom: ::core::marker::PhantomData<
 892                     fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
 893                 >,
 894                 // Only the pinned fields.
 895                 $($pinned)*
 896             }
 897
 898             #[doc(hidden)]
 899             impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*>
 900             where
 901                 __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin,
 902                 $($whr)*
 903             {}
 904
 905             // We need to disallow normal `Drop` implementation, the exact behavior depends on
 906             // whether `PinnedDrop` was specified as the parameter.
 907             $crate::__pin_data!(drop_prevention:
 908                 @name($name),
 909                 @impl_generics($($impl_generics)*),
 910                 @ty_generics($($ty_generics)*),
 911                 @where($($whr)*),
 912                 @pinned_drop($($pinned_drop)?),
 913             );
 914         };
 915     };
 916     // When no `PinnedDrop` was specified, then we have to prevent implementing drop.
 917     (drop_prevention:
 918         @name($name:ident),
 919         @impl_generics($($impl_generics:tt)*),
 920         @ty_generics($($ty_generics:tt)*),
 921         @where($($whr:tt)*),
 922         @pinned_drop(),
 923     ) => {
 924         // We prevent this by creating a trait that will be implemented for all types implementing
 925         // `Drop`. Additionally we will implement this trait for the struct leading to a conflict,
 926         // if it also implements `Drop`
 927         trait MustNotImplDrop {}
 928         #[expect(drop_bounds)]
 929         impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
 930         impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*>
 931         where $($whr)* {}
 932         // We also take care to prevent users from writing a useless `PinnedDrop` implementation.
 933         // They might implement `PinnedDrop` correctly for the struct, but forget to give
 934         // `PinnedDrop` as the parameter to `#[pin_data]`.
 935         #[expect(non_camel_case_types)]
 936         trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
 937         impl<T: $crate::init::PinnedDrop>
 938             UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
 939         impl<$($impl_generics)*>
 940             UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*>
 941         where $($whr)* {}
 942     };
 943     // When `PinnedDrop` was specified we just implement `Drop` and delegate.
 944     (drop_prevention:
 945         @name($name:ident),
 946         @impl_generics($($impl_generics:tt)*),
 947         @ty_generics($($ty_generics:tt)*),
 948         @where($($whr:tt)*),
 949         @pinned_drop(PinnedDrop),
 950     ) => {
 951         impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*>
 952         where $($whr)*
 953         {
 954             fn drop(&mut self) {
 955                 // SAFETY: Since this is a destructor, `self` will not move after this function
 956                 // terminates, since it is inaccessible.
 957                 let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
 958                 // SAFETY: Since this is a drop function, we can create this token to call the
 959                 // pinned destructor of this type.
 960                 let token = unsafe { $crate::init::__internal::OnlyCallFromDrop::new() };
 961                 $crate::init::PinnedDrop::drop(pinned, token);
 962             }
 963         }
 964     };
 965     // If some other parameter was specified, we emit a readable error.
 966     (drop_prevention:
 967         @name($name:ident),
 968         @impl_generics($($impl_generics:tt)*),
 969         @ty_generics($($ty_generics:tt)*),
 970         @where($($whr:tt)*),
 971         @pinned_drop($($rest:tt)*),
 972     ) => {
 973         compile_error!(
 974             "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.",
 975             stringify!($($rest)*),
 976         );
 977     };
 978     (make_pin_data:
 979         @pin_data($pin_data:ident),
 980         @impl_generics($($impl_generics:tt)*),
 981         @ty_generics($($ty_generics:tt)*),
 982         @where($($whr:tt)*),
 983         @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?),
 984         @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?),
 985     ) => {
 986         // For every field, we create a projection function according to its projection type. If a
 987         // field is structurally pinned, then it must be initialized via `PinInit`, if it is not
 988         // structurally pinned, then it can be initialized via `Init`.
 989         //
 990         // The functions are `unsafe` to prevent accidentally calling them.
 991         #[allow(dead_code)]
 992         #[expect(clippy::missing_safety_doc)]
 993         impl<$($impl_generics)*> $pin_data<$($ty_generics)*>
 994         where $($whr)*
 995         {
 996             $(
 997                 $(#[$($p_attr)*])*
 998                 $pvis unsafe fn $p_field<E>(
 999                     self,
1000                     slot: *mut $p_type,
1001                     init: impl $crate::init::PinInit<$p_type, E>,
1002                 ) -> ::core::result::Result<(), E> {
1003                     // SAFETY: TODO.
1004                     unsafe { $crate::init::PinInit::__pinned_init(init, slot) }
1005                 }
1006             )*
1007             $(
1008                 $(#[$($attr)*])*
1009                 $fvis unsafe fn $field<E>(
1010                     self,
1011                     slot: *mut $type,
1012                     init: impl $crate::init::Init<$type, E>,
1013                 ) -> ::core::result::Result<(), E> {
1014                     // SAFETY: TODO.
1015                     unsafe { $crate::init::Init::__init(init, slot) }
1016                 }
1017             )*
1018         }
1019     };
1020 }
1021
1022 /// The internal init macro. Do not call manually!
1023 ///
1024 /// This is called by the `{try_}{pin_}init!` macros with various inputs.
1025 ///
1026 /// This macro has multiple internal call configurations, these are always the very first ident:
1027 /// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros.
1028 /// - `with_update_parsed`: when the `..Zeroable::zeroed()` syntax has been handled.
1029 /// - `init_slot`: recursively creates the code that initializes all fields in `slot`.
1030 /// - `make_initializer`: recursively create the struct initializer that guarantees that every
1031 ///   field has been initialized exactly once.
1032 #[doc(hidden)]
1033 #[macro_export]
1034 macro_rules! __init_internal {
1035     (
1036         @this($($this:ident)?),
1037         @typ($t:path),
1038         @fields($($fields:tt)*),
1039         @error($err:ty),
1040         // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1041         // case.
1042         @data($data:ident, $($use_data:ident)?),
1043         // `HasPinData` or `HasInitData`.
1044         @has_data($has_data:ident, $get_data:ident),
1045         // `pin_init_from_closure` or `init_from_closure`.
1046         @construct_closure($construct_closure:ident),
1047         @munch_fields(),
1048     ) => {
1049         $crate::__init_internal!(with_update_parsed:
1050             @this($($this)?),
1051             @typ($t),
1052             @fields($($fields)*),
1053             @error($err),
1054             @data($data, $($use_data)?),
1055             @has_data($has_data, $get_data),
1056             @construct_closure($construct_closure),
1057             @zeroed(), // Nothing means default behavior.
1058         )
1059     };
1060     (
1061         @this($($this:ident)?),
1062         @typ($t:path),
1063         @fields($($fields:tt)*),
1064         @error($err:ty),
1065         // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1066         // case.
1067         @data($data:ident, $($use_data:ident)?),
1068         // `HasPinData` or `HasInitData`.
1069         @has_data($has_data:ident, $get_data:ident),
1070         // `pin_init_from_closure` or `init_from_closure`.
1071         @construct_closure($construct_closure:ident),
1072         @munch_fields(..Zeroable::zeroed()),
1073     ) => {
1074         $crate::__init_internal!(with_update_parsed:
1075             @this($($this)?),
1076             @typ($t),
1077             @fields($($fields)*),
1078             @error($err),
1079             @data($data, $($use_data)?),
1080             @has_data($has_data, $get_data),
1081             @construct_closure($construct_closure),
1082             @zeroed(()), // `()` means zero all fields not mentioned.
1083         )
1084     };
1085     (
1086         @this($($this:ident)?),
1087         @typ($t:path),
1088         @fields($($fields:tt)*),
1089         @error($err:ty),
1090         // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1091         // case.
1092         @data($data:ident, $($use_data:ident)?),
1093         // `HasPinData` or `HasInitData`.
1094         @has_data($has_data:ident, $get_data:ident),
1095         // `pin_init_from_closure` or `init_from_closure`.
1096         @construct_closure($construct_closure:ident),
1097         @munch_fields($ignore:tt $($rest:tt)*),
1098     ) => {
1099         $crate::__init_internal!(
1100             @this($($this)?),
1101             @typ($t),
1102             @fields($($fields)*),
1103             @error($err),
1104             @data($data, $($use_data)?),
1105             @has_data($has_data, $get_data),
1106             @construct_closure($construct_closure),
1107             @munch_fields($($rest)*),
1108         )
1109     };
1110     (with_update_parsed:
1111         @this($($this:ident)?),
1112         @typ($t:path),
1113         @fields($($fields:tt)*),
1114         @error($err:ty),
1115         // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
1116         // case.
1117         @data($data:ident, $($use_data:ident)?),
1118         // `HasPinData` or `HasInitData`.
1119         @has_data($has_data:ident, $get_data:ident),
1120         // `pin_init_from_closure` or `init_from_closure`.
1121         @construct_closure($construct_closure:ident),
1122         @zeroed($($init_zeroed:expr)?),
1123     ) => {{
1124         // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
1125         // type and shadow it later when we insert the arbitrary user code. That way there will be
1126         // no possibility of returning without `unsafe`.
1127         struct __InitOk;
1128         // Get the data about fields from the supplied type.
1129         //
1130         // SAFETY: TODO.
1131         let data = unsafe {
1132             use $crate::init::__internal::$has_data;
1133             // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1134             // information that is associated to already parsed fragments, so a path fragment
1135             // cannot be used in this position. Doing the retokenization results in valid rust
1136             // code.
1137             ::kernel::macros::paste!($t::$get_data())
1138         };
1139         // Ensure that `data` really is of type `$data` and help with type inference:
1140         let init = $crate::init::__internal::$data::make_closure::<_, __InitOk, $err>(
1141             data,
1142             move |slot| {
1143                 {
1144                     // Shadow the structure so it cannot be used to return early.
1145                     struct __InitOk;
1146                     // If `$init_zeroed` is present we should zero the slot now and not emit an
1147                     // error when fields are missing (since they will be zeroed). We also have to
1148                     // check that the type actually implements `Zeroable`.
1149                     $({
1150                         fn assert_zeroable<T: $crate::init::Zeroable>(_: *mut T) {}
1151                         // Ensure that the struct is indeed `Zeroable`.
1152                         assert_zeroable(slot);
1153                         // SAFETY: The type implements `Zeroable` by the check above.
1154                         unsafe { ::core::ptr::write_bytes(slot, 0, 1) };
1155                         $init_zeroed // This will be `()` if set.
1156                     })?
1157                     // Create the `this` so it can be referenced by the user inside of the
1158                     // expressions creating the individual fields.
1159                     $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
1160                     // Initialize every field.
1161                     $crate::__init_internal!(init_slot($($use_data)?):
1162                         @data(data),
1163                         @slot(slot),
1164                         @guards(),
1165                         @munch_fields($($fields)*,),
1166                     );
1167                     // We use unreachable code to ensure that all fields have been mentioned exactly
1168                     // once, this struct initializer will still be type-checked and complain with a
1169                     // very natural error message if a field is forgotten/mentioned more than once.
1170                     #[allow(unreachable_code, clippy::diverging_sub_expression)]
1171                     let _ = || {
1172                         $crate::__init_internal!(make_initializer:
1173                             @slot(slot),
1174                             @type_name($t),
1175                             @munch_fields($($fields)*,),
1176                             @acc(),
1177                         );
1178                     };
1179                 }
1180                 Ok(__InitOk)
1181             }
1182         );
1183         let init = move |slot| -> ::core::result::Result<(), $err> {
1184             init(slot).map(|__InitOk| ())
1185         };
1186         // SAFETY: TODO.
1187         let init = unsafe { $crate::init::$construct_closure::<_, $err>(init) };
1188         init
1189     }};
1190     (init_slot($($use_data:ident)?):
1191         @data($data:ident),
1192         @slot($slot:ident),
1193         @guards($($guards:ident,)*),
1194         @munch_fields($(..Zeroable::zeroed())? $(,)?),
1195     ) => {
1196         // Endpoint of munching, no fields are left. If execution reaches this point, all fields
1197         // have been initialized. Therefore we can now dismiss the guards by forgetting them.
1198         $(::core::mem::forget($guards);)*
1199     };
1200     (init_slot($use_data:ident): // `use_data` is present, so we use the `data` to init fields.
1201         @data($data:ident),
1202         @slot($slot:ident),
1203         @guards($($guards:ident,)*),
1204         // In-place initialization syntax.
1205         @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1206     ) => {
1207         let init = $val;
1208         // Call the initializer.
1209         //
1210         // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
1211         // return when an error/panic occurs.
1212         // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`.
1213         unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? };
1214         // Create the drop guard:
1215         //
1216         // We rely on macro hygiene to make it impossible for users to access this local variable.
1217         // We use `paste!` to create new hygiene for `$field`.
1218         ::kernel::macros::paste! {
1219             // SAFETY: We forget the guard later when initialization has succeeded.
1220             let [< __ $field _guard >] = unsafe {
1221                 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1222             };
1223
1224             $crate::__init_internal!(init_slot($use_data):
1225                 @data($data),
1226                 @slot($slot),
1227                 @guards([< __ $field _guard >], $($guards,)*),
1228                 @munch_fields($($rest)*),
1229             );
1230         }
1231     };
1232     (init_slot(): // No `use_data`, so we use `Init::__init` directly.
1233         @data($data:ident),
1234         @slot($slot:ident),
1235         @guards($($guards:ident,)*),
1236         // In-place initialization syntax.
1237         @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1238     ) => {
1239         let init = $val;
1240         // Call the initializer.
1241         //
1242         // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
1243         // return when an error/panic occurs.
1244         unsafe { $crate::init::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? };
1245         // Create the drop guard:
1246         //
1247         // We rely on macro hygiene to make it impossible for users to access this local variable.
1248         // We use `paste!` to create new hygiene for `$field`.
1249         ::kernel::macros::paste! {
1250             // SAFETY: We forget the guard later when initialization has succeeded.
1251             let [< __ $field _guard >] = unsafe {
1252                 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1253             };
1254
1255             $crate::__init_internal!(init_slot():
1256                 @data($data),
1257                 @slot($slot),
1258                 @guards([< __ $field _guard >], $($guards,)*),
1259                 @munch_fields($($rest)*),
1260             );
1261         }
1262     };
1263     (init_slot($($use_data:ident)?):
1264         @data($data:ident),
1265         @slot($slot:ident),
1266         @guards($($guards:ident,)*),
1267         // Init by-value.
1268         @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1269     ) => {
1270         {
1271             $(let $field = $val;)?
1272             // Initialize the field.
1273             //
1274             // SAFETY: The memory at `slot` is uninitialized.
1275             unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
1276         }
1277         // Create the drop guard:
1278         //
1279         // We rely on macro hygiene to make it impossible for users to access this local variable.
1280         // We use `paste!` to create new hygiene for `$field`.
1281         ::kernel::macros::paste! {
1282             // SAFETY: We forget the guard later when initialization has succeeded.
1283             let [< __ $field _guard >] = unsafe {
1284                 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
1285             };
1286
1287             $crate::__init_internal!(init_slot($($use_data)?):
1288                 @data($data),
1289                 @slot($slot),
1290                 @guards([< __ $field _guard >], $($guards,)*),
1291                 @munch_fields($($rest)*),
1292             );
1293         }
1294     };
1295     (make_initializer:
1296         @slot($slot:ident),
1297         @type_name($t:path),
1298         @munch_fields(..Zeroable::zeroed() $(,)?),
1299         @acc($($acc:tt)*),
1300     ) => {
1301         // Endpoint, nothing more to munch, create the initializer. Since the users specified
1302         // `..Zeroable::zeroed()`, the slot will already have been zeroed and all field that have
1303         // not been overwritten are thus zero and initialized. We still check that all fields are
1304         // actually accessible by using the struct update syntax ourselves.
1305         // We are inside of a closure that is never executed and thus we can abuse `slot` to
1306         // get the correct type inference here:
1307         #[allow(unused_assignments)]
1308         unsafe {
1309             let mut zeroed = ::core::mem::zeroed();
1310             // We have to use type inference here to make zeroed have the correct type. This does
1311             // not get executed, so it has no effect.
1312             ::core::ptr::write($slot, zeroed);
1313             zeroed = ::core::mem::zeroed();
1314             // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1315             // information that is associated to already parsed fragments, so a path fragment
1316             // cannot be used in this position. Doing the retokenization results in valid rust
1317             // code.
1318             ::kernel::macros::paste!(
1319                 ::core::ptr::write($slot, $t {
1320                     $($acc)*
1321                     ..zeroed
1322                 });
1323             );
1324         }
1325     };
1326     (make_initializer:
1327         @slot($slot:ident),
1328         @type_name($t:path),
1329         @munch_fields($(,)?),
1330         @acc($($acc:tt)*),
1331     ) => {
1332         // Endpoint, nothing more to munch, create the initializer.
1333         // Since we are in the closure that is never called, this will never get executed.
1334         // We abuse `slot` to get the correct type inference here:
1335         //
1336         // SAFETY: TODO.
1337         unsafe {
1338             // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
1339             // information that is associated to already parsed fragments, so a path fragment
1340             // cannot be used in this position. Doing the retokenization results in valid rust
1341             // code.
1342             ::kernel::macros::paste!(
1343                 ::core::ptr::write($slot, $t {
1344                     $($acc)*
1345                 });
1346             );
1347         }
1348     };
1349     (make_initializer:
1350         @slot($slot:ident),
1351         @type_name($t:path),
1352         @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
1353         @acc($($acc:tt)*),
1354     ) => {
1355         $crate::__init_internal!(make_initializer:
1356             @slot($slot),
1357             @type_name($t),
1358             @munch_fields($($rest)*),
1359             @acc($($acc)* $field: ::core::panic!(),),
1360         );
1361     };
1362     (make_initializer:
1363         @slot($slot:ident),
1364         @type_name($t:path),
1365         @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
1366         @acc($($acc:tt)*),
1367     ) => {
1368         $crate::__init_internal!(make_initializer:
1369             @slot($slot),
1370             @type_name($t),
1371             @munch_fields($($rest)*),
1372             @acc($($acc)* $field: ::core::panic!(),),
1373         );
1374     };
1375 }
1376
1377 #[doc(hidden)]
1378 #[macro_export]
1379 macro_rules! __derive_zeroable {
1380     (parse_input:
1381         @sig(
1382             $(#[$($struct_attr:tt)*])*
1383             $vis:vis struct $name:ident
1384             $(where $($whr:tt)*)?
1385         ),
1386         @impl_generics($($impl_generics:tt)*),
1387         @ty_generics($($ty_generics:tt)*),
1388         @body({
1389             $(
1390                 $(#[$($field_attr:tt)*])*
1391                 $field:ident : $field_ty:ty
1392             ),* $(,)?
1393         }),
1394     ) => {
1395         // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
1396         #[automatically_derived]
1397         unsafe impl<$($impl_generics)*> $crate::init::Zeroable for $name<$($ty_generics)*>
1398         where
1399             $($($whr)*)?
1400         {}
1401         const _: () = {
1402             fn assert_zeroable<T: ?::core::marker::Sized + $crate::init::Zeroable>() {}
1403             fn ensure_zeroable<$($impl_generics)*>()
1404                 where $($($whr)*)?
1405             {
1406                 $(assert_zeroable::<$field_ty>();)*
1407             }
1408         };
1409     };
1410 }