* [PATCH v5 1/8] rust: cpumask: Add a `Cpumask` iterator
2026-04-10 21:35 [PATCH v5 0/8] rust: Add Per-CPU Variable API Mitchell Levy
@ 2026-04-10 21:35 ` Mitchell Levy
2026-04-10 21:35 ` [PATCH v5 2/8] rust: cpumask: Add getters for globally defined cpumasks Mitchell Levy
` (6 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Mitchell Levy @ 2026-04-10 21:35 UTC (permalink / raw)
To: Miguel Ojeda, Alex Gaynor, Gary Guo, Björn Roy Baron,
Andreas Hindborg, Alice Ryhl, Trevor Gross, Andrew Morton,
Dennis Zhou, Tejun Heo, Christoph Lameter, Danilo Krummrich,
Benno Lossin, Yury Norov, Viresh Kumar, Boqun Feng
Cc: Tyler Hicks, Allen Pais, linux-kernel, rust-for-linux, linux-mm,
Mitchell Levy
Add an iterator for `Cpumask` making use of C's `cpumask_next`.
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
---
rust/helpers/cpumask.c | 6 ++++++
rust/kernel/cpumask.rs | 51 +++++++++++++++++++++++++++++++++++++++++++++++++-
2 files changed, 56 insertions(+), 1 deletion(-)
diff --git a/rust/helpers/cpumask.c b/rust/helpers/cpumask.c
index 5deced5b975e..76990e14dfdd 100644
--- a/rust/helpers/cpumask.c
+++ b/rust/helpers/cpumask.c
@@ -50,6 +50,12 @@ bool rust_helper_cpumask_full(struct cpumask *srcp)
return cpumask_full(srcp);
}
+__rust_helper
+unsigned int rust_helper_cpumask_next(int n, struct cpumask *srcp)
+{
+ return cpumask_next(n, srcp);
+}
+
__rust_helper
unsigned int rust_helper_cpumask_weight(struct cpumask *srcp)
{
diff --git a/rust/kernel/cpumask.rs b/rust/kernel/cpumask.rs
index 44bb36636ee3..b74a3fccf4b4 100644
--- a/rust/kernel/cpumask.rs
+++ b/rust/kernel/cpumask.rs
@@ -14,7 +14,10 @@
#[cfg(CONFIG_CPUMASK_OFFSTACK)]
use core::ptr::{self, NonNull};
-use core::ops::{Deref, DerefMut};
+use core::{
+ iter::FusedIterator,
+ ops::{Deref, DerefMut},
+};
/// A CPU Mask.
///
@@ -161,6 +164,52 @@ pub fn copy(&self, dstp: &mut Self) {
}
}
+/// Iterator for a `Cpumask`.
+pub struct CpumaskIter<'a> {
+ mask: &'a Cpumask,
+ /// [`None`] if no bits have been returned yet, or the index of the last bit returned by the
+ /// iterator. Equal to [`kernel::cpu::nr_cpu_ids()`] when the iterator has been exhausted.
+ last: Option<u32>,
+}
+
+impl<'a> CpumaskIter<'a> {
+ /// Creates a new `CpumaskIter` for the given `Cpumask`.
+ fn new(mask: &'a Cpumask) -> CpumaskIter<'a> {
+ Self { mask, last: None }
+ }
+}
+
+impl<'a> Iterator for CpumaskIter<'a> {
+ type Item = CpuId;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ // cpumask_next WARNs if the first argument is >= nr_cpu_ids when CONFIG_DEBUG_PER_CPU_MAPS
+ // is set. So, early out in that case
+ if let Some(last) = self.last {
+ if last >= kernel::cpu::nr_cpu_ids() {
+ return None;
+ }
+ }
+
+ // SAFETY: By the type invariant, `self.mask.as_raw` is a `struct cpumask *`.
+ let next = unsafe {
+ bindings::cpumask_next(self.last.map_or(-1, |l| l as i32), self.mask.as_raw())
+ };
+
+ self.last = Some(next);
+ CpuId::from_u32(next)
+ }
+}
+
+impl<'a> FusedIterator for CpumaskIter<'a> {}
+
+impl Cpumask {
+ /// Returns an iterator over the set bits in the cpumask.
+ pub fn iter(&self) -> CpumaskIter<'_> {
+ CpumaskIter::new(self)
+ }
+}
+
/// A CPU Mask pointer.
///
/// Rust abstraction for the C `struct cpumask_var_t`.
--
2.34.1
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH v5 2/8] rust: cpumask: Add getters for globally defined cpumasks
2026-04-10 21:35 [PATCH v5 0/8] rust: Add Per-CPU Variable API Mitchell Levy
2026-04-10 21:35 ` [PATCH v5 1/8] rust: cpumask: Add a `Cpumask` iterator Mitchell Levy
@ 2026-04-10 21:35 ` Mitchell Levy
2026-04-10 21:35 ` [PATCH v5 3/8] rust: percpu: Add C bindings for per-CPU variable API Mitchell Levy
` (5 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Mitchell Levy @ 2026-04-10 21:35 UTC (permalink / raw)
To: Miguel Ojeda, Alex Gaynor, Gary Guo, Björn Roy Baron,
Andreas Hindborg, Alice Ryhl, Trevor Gross, Andrew Morton,
Dennis Zhou, Tejun Heo, Christoph Lameter, Danilo Krummrich,
Benno Lossin, Yury Norov, Viresh Kumar, Boqun Feng
Cc: Tyler Hicks, Allen Pais, linux-kernel, rust-for-linux, linux-mm,
Mitchell Levy
Add getters for the global cpumasks documented in
`include/linux/cpumask.h`, specifically:
- cpu_possible_mask
- cpu_online_mask
- cpu_enabled_mask
- cpu_present_mask
- cpu_active_mask
Acked-by: Viresh Kumar <viresh.kumar@linaro.org>
Suggested-by: Yury Norov <yury.norov@gmail.com>
Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
---
rust/kernel/cpumask.rs | 46 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 46 insertions(+)
diff --git a/rust/kernel/cpumask.rs b/rust/kernel/cpumask.rs
index b74a3fccf4b4..2d61f1323837 100644
--- a/rust/kernel/cpumask.rs
+++ b/rust/kernel/cpumask.rs
@@ -80,6 +80,52 @@ pub unsafe fn from_raw<'a>(ptr: *const bindings::cpumask) -> &'a Self {
unsafe { &*ptr.cast() }
}
+ /// Get a CPU mask representing possible CPUs; has bit `cpu` set iff cpu is populatable
+ #[inline]
+ pub fn possible_cpus() -> &'static Self {
+ // SAFETY: `__cpu_possible_mask` is a valid global provided by the kernel that lives
+ // forever.
+ unsafe { Cpumask::from_raw(&raw const bindings::__cpu_possible_mask) }
+ }
+
+ /// Get a CPU mask representing online CPUs; has bit `cpu` set iff cpu available to the
+ /// scheduler
+ #[inline]
+ pub fn online_cpus() -> &'static Self {
+ // SAFETY: `__cpu_online_mask` is a valid global provided by the kernel that lives forever.
+ // Since we wrap the returned pointer in an `Opaque`, it's ok that `__cpu_online_mask`
+ // may change its value.
+ unsafe { Cpumask::from_raw(&raw const bindings::__cpu_online_mask) }
+ }
+
+ /// Get a CPU mask representing enabled CPUs; has bit `cpu` set iff cpu can be brought online
+ #[inline]
+ pub fn enabled_cpus() -> &'static Self {
+ // SAFETY: `__cpu_enabled_mask` is a valid global provided by the kernel that lives forever.
+ // Since we wrap the returned pointer in an `Opaque`, it's ok that `__cpu_enabled_mask`
+ // may change its value.
+ unsafe { Cpumask::from_raw(&raw const bindings::__cpu_enabled_mask) }
+ }
+
+ /// Get a CPU mask representing present CPUs; has bit `cpu` set iff cpu is populated
+ #[inline]
+ pub fn present_cpus() -> &'static Self {
+ // SAFETY: `__cpu_present_mask` is a valid global provided by the kernel that lives
+ // forever. Since we wrap the returned pointer in an `Opaque`, it's ok that
+ // `__cpu_present_mask` may change its value.
+ unsafe { Cpumask::from_raw(&raw const bindings::__cpu_present_mask) }
+ }
+
+ /// Get a CPU mask representing active CPUs; has bit `cpu` set iff cpu is available to
+ /// migration.
+ #[inline]
+ pub fn active_cpus() -> &'static Self {
+ // SAFETY: `__cpu_active_mask` is a valid global provided by the kernel that lives forever.
+ // Since we wrap the returned pointer in an `Opaque`, it's ok that `__cpu_active_mask`
+ // may change its value.
+ unsafe { Cpumask::from_raw(&raw const bindings::__cpu_active_mask) }
+ }
+
/// Obtain the raw `struct cpumask` pointer.
pub fn as_raw(&self) -> *mut bindings::cpumask {
let this: *const Self = self;
--
2.34.1
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH v5 3/8] rust: percpu: Add C bindings for per-CPU variable API
2026-04-10 21:35 [PATCH v5 0/8] rust: Add Per-CPU Variable API Mitchell Levy
2026-04-10 21:35 ` [PATCH v5 1/8] rust: cpumask: Add a `Cpumask` iterator Mitchell Levy
2026-04-10 21:35 ` [PATCH v5 2/8] rust: cpumask: Add getters for globally defined cpumasks Mitchell Levy
@ 2026-04-10 21:35 ` Mitchell Levy
2026-04-10 21:35 ` [PATCH v5 4/8] rust: percpu: introduce a rust API for static per-CPU variables Mitchell Levy
` (4 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Mitchell Levy @ 2026-04-10 21:35 UTC (permalink / raw)
To: Miguel Ojeda, Alex Gaynor, Gary Guo, Björn Roy Baron,
Andreas Hindborg, Alice Ryhl, Trevor Gross, Andrew Morton,
Dennis Zhou, Tejun Heo, Christoph Lameter, Danilo Krummrich,
Benno Lossin, Yury Norov, Viresh Kumar, Boqun Feng
Cc: Tyler Hicks, Allen Pais, linux-kernel, rust-for-linux, linux-mm,
Mitchell Levy
Add bindings necessary to implement a Rust per-CPU variable API,
specifically per-CPU variable allocation and management of CPU
preemption.
Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
---
rust/helpers/helpers.c | 2 ++
rust/helpers/percpu.c | 9 +++++++++
rust/helpers/preempt.c | 15 +++++++++++++++
3 files changed, 26 insertions(+)
diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c
index e05c6e7e4abb..af661bb9be34 100644
--- a/rust/helpers/helpers.c
+++ b/rust/helpers/helpers.c
@@ -70,9 +70,11 @@
#include "of.c"
#include "page.c"
#include "pci.c"
+#include "percpu.c"
#include "pid_namespace.c"
#include "platform.c"
#include "poll.c"
+#include "preempt.c"
#include "processor.c"
#include "property.c"
#include "pwm.c"
diff --git a/rust/helpers/percpu.c b/rust/helpers/percpu.c
new file mode 100644
index 000000000000..463186b8af9d
--- /dev/null
+++ b/rust/helpers/percpu.c
@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/percpu.h>
+
+__rust_helper
+void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align)
+{
+ return __alloc_percpu(sz, align);
+}
diff --git a/rust/helpers/preempt.c b/rust/helpers/preempt.c
new file mode 100644
index 000000000000..77dd5f10d8dc
--- /dev/null
+++ b/rust/helpers/preempt.c
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/preempt.h>
+
+__rust_helper
+void rust_helper_preempt_disable(void)
+{
+ preempt_disable();
+}
+
+__rust_helper
+void rust_helper_preempt_enable(void)
+{
+ preempt_enable();
+}
--
2.34.1
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH v5 4/8] rust: percpu: introduce a rust API for static per-CPU variables
2026-04-10 21:35 [PATCH v5 0/8] rust: Add Per-CPU Variable API Mitchell Levy
` (2 preceding siblings ...)
2026-04-10 21:35 ` [PATCH v5 3/8] rust: percpu: Add C bindings for per-CPU variable API Mitchell Levy
@ 2026-04-10 21:35 ` Mitchell Levy
2026-04-10 21:35 ` [PATCH v5 5/8] rust: percpu: introduce a rust API for dynamic " Mitchell Levy
` (3 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Mitchell Levy @ 2026-04-10 21:35 UTC (permalink / raw)
To: Miguel Ojeda, Alex Gaynor, Gary Guo, Björn Roy Baron,
Andreas Hindborg, Alice Ryhl, Trevor Gross, Andrew Morton,
Dennis Zhou, Tejun Heo, Christoph Lameter, Danilo Krummrich,
Benno Lossin, Yury Norov, Viresh Kumar, Boqun Feng
Cc: Tyler Hicks, Allen Pais, linux-kernel, rust-for-linux, linux-mm,
Mitchell Levy
Per-CPU variables are an important tool for reducing lock contention,
especially in systems with many processors. They also provide a
convenient way to handle data that are logically associated with a
particular CPU (e.g., the currently running task).
Therefore, add a Rust API to make use of statically-allocated per-CPU
variables. Add a RAII `CpuGuard` type for disabling CPU preemption.
Introduce unifying abstractions that can be reused for a Rust API for
dynamically-allocated per-CPU variables.
Co-developed-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
---
Using `rustc` 1.83.0, I can confirm that the symbols `__INIT_$id` are
optimized out by the compiler and do not appear in the final `.ko` file
when compiling `samples/rust/rust_percpu.rs` (introduced in a later
patch in this series).
---
rust/kernel/lib.rs | 3 +
rust/kernel/percpu.rs | 259 ++++++++++++++++++++++++++++++++++++++++
rust/kernel/percpu/cpu_guard.rs | 48 ++++++++
rust/kernel/percpu/static_.rs | 218 +++++++++++++++++++++++++++++++++
4 files changed, 528 insertions(+)
diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs
index 0fa9d820fe7c..b54aca69289c 100644
--- a/rust/kernel/lib.rs
+++ b/rust/kernel/lib.rs
@@ -97,6 +97,9 @@
pub mod page;
#[cfg(CONFIG_PCI)]
pub mod pci;
+// Only x86_64 is supported by percpu for now
+#[cfg(CONFIG_X86_64)]
+pub mod percpu;
pub mod pid_namespace;
pub mod platform;
pub mod prelude;
diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs
new file mode 100644
index 000000000000..615859d4daad
--- /dev/null
+++ b/rust/kernel/percpu.rs
@@ -0,0 +1,259 @@
+// SPDX-License-Identifier: GPL-2.0
+//! Per-CPU variables.
+//!
+//! See the [`crate::define_per_cpu!`] macro and the [`PerCpu<T>`] trait.
+
+pub mod cpu_guard;
+mod static_;
+
+#[doc(inline)]
+pub use static_::*;
+
+use crate::{
+ declare_extern_per_cpu,
+ percpu::cpu_guard::CpuGuard,
+ types::Opaque, //
+};
+
+use core::{
+ arch::asm,
+ cell::{
+ Cell,
+ RefCell,
+ UnsafeCell, //
+ },
+ mem::MaybeUninit, //
+};
+
+use ffi::c_void;
+
+/// A per-CPU pointer; that is, an offset into the per-CPU area.
+///
+/// Note that this type is NOT a smart pointer, it does not manage the allocation.
+pub struct PerCpuPtr<T>(*mut MaybeUninit<T>);
+
+/// Represents exclusive access to the memory location pointed at by a particular [`PerCpu<T>`].
+pub struct PerCpuToken<'a, T> {
+ // INVARIANT: the current CPU's memory location associated with the per-CPU variable pointed at
+ // by `ptr` (i.e., the entry in the per-CPU area on the current CPU) has been initialized.
+ _guard: CpuGuard,
+ ptr: &'a PerCpuPtr<T>,
+}
+
+/// Represents access to the memory location pointed at by a particular [`PerCpu<T>`] where the
+/// type `T` manages access to the underlying memory to avoid aliasing troubles.
+///
+/// For example, `T` might be a [`Cell`] or [`RefCell`].
+pub struct CheckedPerCpuToken<'a, T> {
+ // INVARIANT: the current CPU's memory location associated with the per-CPU variable pointed at
+ // by `ptr` (i.e., the entry in the per-CPU area on the current CPU) has been initialized.
+ _guard: CpuGuard,
+ ptr: &'a PerCpuPtr<T>,
+}
+
+impl<T> PerCpuPtr<T> {
+ /// Makes a new [`PerCpuPtr`] from a raw per-CPU pointer.
+ ///
+ /// Note that the returned [`PerCpuPtr`] is valid only as long as the given pointer is. This
+ /// also requires that the allocation pointed to by `ptr` must be correctly sized and aligned
+ /// to hold a `T`.
+ pub fn new(ptr: *mut MaybeUninit<T>) -> Self {
+ Self(ptr)
+ }
+
+ /// Get a [`&mut MaybeUninit<T>`](MaybeUninit) to the per-CPU variable on the current CPU
+ /// represented by `&self`
+ ///
+ /// # Safety
+ ///
+ /// The returned `&mut MaybeUninit<T>` must follow Rust's aliasing rules. That is, no other
+ /// `&(mut) MaybeUninit<T>` may exist that points to the same location in memory. In practice,
+ /// this means that [`Self::get_ref`] and [`Self::get_mut_ref`] must not be called on another
+ /// [`PerCpuPtr<T>`] that is a copy/clone of `&self` for as long as the returned reference
+ /// lives. If the per-CPU variable represented by `&self` is available to C, the caller of this
+ /// function must ensure that the C code does not modify the variable for as long as this
+ /// reference lives.
+ ///
+ /// `self` must point to a live allocation correctly sized and aligned to hold a `T`.
+ ///
+ /// CPU preemption must be disabled before calling this function and for the lifetime of the
+ /// returned reference. Otherwise, the returned reference might end up being a reference to a
+ /// different CPU's per-CPU area, causing the potential for a data race.
+ #[allow(clippy::mut_from_ref)] // Safety requirements prevent aliasing issues
+ pub unsafe fn get_mut_ref(&self) -> &mut MaybeUninit<T> {
+ // SAFETY: `self.get_ptr()` returns a valid pointer to a `MaybeUninit<T>` by its contract,
+ // and the safety requirements of this function ensure that the returned reference is
+ // exclusive.
+ unsafe { &mut *(self.get_ptr()) }
+ }
+
+ /// Get a [`&MaybeUninit<T>`](MaybeUninit) to the per-CPU variable on the current CPU
+ /// represented by `&self`
+ ///
+ /// # Safety
+ ///
+ /// The returned [`&MaybeUninit<T>`](MaybeUninit) must follow Rust's aliasing rules. That is,
+ /// no `&mut MaybeUninit<T>` may exist that points to the same location in memory. In practice,
+ /// this means that [`Self::get_mut_ref`] must not be called on another [`PerCpuPtr<T>`] that
+ /// is a copy/clone of `&self` for as long as the returned reference lives. If the per-CPU
+ /// variable represented by `&self` is available to C, the caller of this function must ensure
+ /// that the C code does not modify the variable for as long as this reference lives.
+ ///
+ /// `self` must point to a live allocation correctly sized and aligned to hold a `T`.
+ ///
+ /// CPU preemption must be disabled before calling this function and for the lifetime of the
+ /// returned reference. Otherwise, the returned reference might end up being a reference to a
+ /// different CPU's per-CPU area, causing the potential for a data race.
+ pub unsafe fn get_ref(&self) -> &MaybeUninit<T> {
+ // SAFETY: `self.get_ptr()` returns a valid pointer to a `MaybeUninit<T>` by its contract.
+ // The safety requirements of this function ensure that the returned reference isn't
+ // aliased by a `&mut MaybeUninit<T>`.
+ unsafe { &*self.get_ptr() }
+ }
+
+ /// Get a [`*mut MaybeUninit<T>`](MaybeUninit) to the per-CPU variable on the current CPU
+ /// represented by `&self`. Note that if CPU preemption is not disabled before calling this
+ /// function, use of the returned pointer may cause a data race without some other
+ /// synchronization mechanism. Buyer beware!
+ pub fn get_ptr(&self) -> *mut MaybeUninit<T> {
+ let this_cpu_off_pcpu = ExternStaticPerCpuSymbol::ptr(&raw const this_cpu_off);
+ let mut this_cpu_area: *mut c_void;
+ // SAFETY: gs + this_cpu_off_pcpu is guaranteed to be a valid pointer because `gs` points
+ // to the per-CPU area and this_cpu_off_pcpu is a valid per-CPU allocation.
+ unsafe {
+ asm!(
+ "mov {out}, gs:[{off_val}]",
+ off_val = in(reg) this_cpu_off_pcpu.0,
+ out = out(reg) this_cpu_area,
+ )
+ };
+
+ // This_cpu_area + self.0 is guaranteed to be a valid pointer by the per-CPU subsystem and
+ // the invariant that self.0 is a valid offset into the per-CPU area.
+ (this_cpu_area).wrapping_add(self.0 as usize).cast()
+ }
+}
+
+// SAFETY: Sending a [`PerCpuPtr<T>`] to another thread is safe because as soon as it's sent, the
+// pointer is logically referring to a different place in memory in the other CPU's per-CPU area.
+// In particular, this means that there are no restrictions on the type `T`.
+unsafe impl<T> Send for PerCpuPtr<T> {}
+
+// SAFETY: Two threads concurrently making use of a [`PerCpuPtr<T>`] will each see the `T` in their
+// own per-CPU area, so there's no potential for a data race (regardless of whether `T` is itself
+// `Sync`).
+unsafe impl<T> Sync for PerCpuPtr<T> {}
+
+impl<T> Clone for PerCpuPtr<T> {
+ fn clone(&self) -> Self {
+ *self
+ }
+}
+
+/// [`PerCpuPtr`] is just a wrapper around a pointer.
+impl<T> Copy for PerCpuPtr<T> {}
+
+/// A trait representing a per-CPU variable.
+///
+/// This is implemented for [`StaticPerCpu<T>`]. The main usage of this trait is to call
+/// [`Self::get_mut`] to get a [`PerCpuToken`] that can be used to access the underlying per-CPU
+/// variable.
+///
+/// See [`PerCpuToken::with`].
+pub trait PerCpu<T> {
+ /// Produces a token, asserting that the holder has exclusive access to the underlying memory
+ /// pointed to by `self`
+ ///
+ /// # Safety
+ ///
+ /// No other [`PerCpuToken`] or [`CheckedPerCpuToken`] may exist on the current CPU (which is a
+ /// sensible notion, since we keep a [`CpuGuard`] around) that is derived from the same
+ /// [`PerCpu<T>`] or a clone thereof.
+ unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T>;
+}
+
+/// A marker trait for types that are interior mutable. Types that implement this trait can be used
+/// to create "checked" per-CPU variables. See [`CheckedPerCpu<T>`].
+pub trait InteriorMutable {}
+
+impl<T> InteriorMutable for Cell<T> {}
+impl<T> InteriorMutable for RefCell<T> {}
+impl<T> InteriorMutable for UnsafeCell<T> {}
+impl<T> InteriorMutable for Opaque<T> {}
+
+/// A trait representing a per-CPU variable that is usable via a `&T`.
+///
+/// The unsafety of [`PerCpu<T>`] stems from the fact that the holder of a [`PerCpuToken`] can use
+/// it to get a `&mut T` to the underlying per-CPU variable. This is problematic because the
+/// existence of aliasing `&mut T` is undefined behavior in Rust. This type avoids that issue by
+/// only allowing access via a `&T`, with the tradeoff that then `T` must be interior mutable or
+/// the underlying per-CPU variable must be a constant for the lifetime of any corresponding
+/// [`CheckedPerCpuToken<T>`].
+///
+/// Currently, only the case where `T` is interior mutable has first-class support, though a custom
+/// implementation of [`PerCpu<T>`]/[`CheckedPerCpu<T>`] could be created for the const case.
+pub trait CheckedPerCpu<T>: PerCpu<T> {
+ /// Produces a token via which the holder can access the underlying per-CPU variable.
+ fn get(&self, guard: CpuGuard) -> CheckedPerCpuToken<'_, T>;
+}
+
+impl<'a, T> PerCpuToken<'a, T> {
+ /// Asserts that the holder has exclusive access to the initialized underlying memory pointed
+ /// to by `ptr` on the current CPU, producing a token.
+ ///
+ /// # Safety
+ ///
+ /// No other [`PerCpuToken`] or [`CheckedPerCpuToken`] may exist on the current CPU (which is a
+ /// sensible notion, since we keep a `CpuGuard` around) that uses the same [`PerCpuPtr<T>`].
+ ///
+ /// The current CPU's memory location associated with the per-CPU variable pointed at by `ptr`
+ /// (i.e., the entry in the per-CPU area on this CPU) must be initialized; `ptr` must be valid
+ /// (that is, pointing at a live per-CPU allocation correctly sized and aligned to hold a `T`)
+ /// for `'a`.
+ pub unsafe fn new(guard: CpuGuard, ptr: &'a PerCpuPtr<T>) -> PerCpuToken<'a, T> {
+ Self { _guard: guard, ptr }
+ }
+
+ /// Immediately invokes `func` with a `&mut T` that points at the underlying per-CPU variable
+ /// that `&mut self` represents. Forwards the return of `func`.
+ pub fn with<U, V>(&mut self, func: U) -> V
+ where
+ U: FnOnce(&mut T) -> V,
+ {
+ // SAFETY: The existence of a PerCpuToken means that the requirements for get_mut_ref are
+ // satisfied. Likewise, the requirements for assume_init_mut are satisfied because the
+ // invariants of this type ensure that on the current CPU (which is a sensible notion
+ // because we have a CpuGuard), the memory location pointed to by `ptr` is initialized.
+ func(unsafe { self.ptr.get_mut_ref().assume_init_mut() })
+ }
+}
+
+impl<'a, T> CheckedPerCpuToken<'a, T> {
+ /// Asserts that the memory pointed to by `ptr` is initialized on the current CPU, producing a
+ /// token.
+ ///
+ /// # Safety
+ ///
+ /// The current CPU's memory location associated with the per-CPU variable pointed at by `ptr`
+ /// (i.e., the entry in the per-CPU area on this CPU) must be initialized; `ptr` must be valid
+ /// (that is, pointing at a live per-CPU allocation correctly sized and aligned to hold a `T`)
+ /// for `'a`.
+ pub unsafe fn new(guard: CpuGuard, ptr: &'a PerCpuPtr<T>) -> CheckedPerCpuToken<'a, T> {
+ Self { _guard: guard, ptr }
+ }
+
+ /// Immediately invokes `func` with a `&T` that points at the underlying per-CPU variable that
+ /// `&mut self` represents. Forwards the return of `func`.
+ pub fn with<U, V>(&self, func: U) -> V
+ where
+ U: FnOnce(&T) -> V,
+ {
+ // SAFETY: The existence of a CheckedPerCpuToken means that the requirements for get_ref
+ // are satisfied. Likewise, the requirements for assume_init_ref are satisfied because the
+ // invariants of this type ensure that on the current CPU (which is a sensible notion
+ // because we have a CpuGuard), the memory location pointed to by `ptr` is initialized.
+ func(unsafe { self.ptr.get_ref().assume_init_ref() })
+ }
+}
+
+declare_extern_per_cpu!(this_cpu_off: *mut c_void);
diff --git a/rust/kernel/percpu/cpu_guard.rs b/rust/kernel/percpu/cpu_guard.rs
new file mode 100644
index 000000000000..aba776b64bbc
--- /dev/null
+++ b/rust/kernel/percpu/cpu_guard.rs
@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: GPL-2.0
+//! Contains abstractions for disabling CPU preemption. See [`CpuGuard`].
+
+/// A RAII guard for `bindings::preempt_disable` and `bindings::preempt_enable`.
+///
+/// Guarantees preemption is disabled for as long as this object exists.
+pub struct CpuGuard {
+ // Don't make one without using new()
+ _phantom: (),
+ /// [`CpuGuard`] is not [`Send`]: moving it to another CPU would break its guarantees and also
+ /// leave the CPU it was created on with preemption disabled indefinitely.
+ ///
+ /// It is also not [`Sync`]: preemption being disabled on one CPU is not relevant to other
+ /// CPUs, so we shouldn't be able to share a `&CpuGuard` between threads.
+ _not_send_sync: core::marker::PhantomData<*const ()>,
+}
+
+impl CpuGuard {
+ /// Create a new [`CpuGuard`]. Disables preemption for its lifetime.
+ #[inline]
+ pub fn new() -> Self {
+ // SAFETY: There are no preconditions required to call preempt_disable
+ unsafe {
+ bindings::preempt_disable();
+ }
+ CpuGuard {
+ _phantom: (),
+ _not_send_sync: core::marker::PhantomData,
+ }
+ }
+}
+
+impl Default for CpuGuard {
+ #[inline]
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl Drop for CpuGuard {
+ #[inline]
+ fn drop(&mut self) {
+ // SAFETY: There are no preconditions required to call preempt_enable
+ unsafe {
+ bindings::preempt_enable();
+ }
+ }
+}
diff --git a/rust/kernel/percpu/static_.rs b/rust/kernel/percpu/static_.rs
new file mode 100644
index 000000000000..70aca55295ca
--- /dev/null
+++ b/rust/kernel/percpu/static_.rs
@@ -0,0 +1,218 @@
+// SPDX-License-Identifier: GPL-2.0
+//! Statically allocated per-CPU variables.
+
+use super::*;
+
+/// A wrapper used for declaring static per-CPU variables.
+///
+/// These symbols are "virtual" in that the linker uses them to generate offsets into each CPU's
+/// per-CPU area, but shouldn't be read from/written to directly. The fact that the statics are
+/// immutable prevents them being written to (generally), this struct having _val be non-public
+/// prevents reading from them.
+///
+/// The end-user of the per-CPU API should make use of the [`crate::define_per_cpu!`] macro instead
+/// of declaring variables of this type directly. All instances of this type must be `static` and
+/// `#[link_section = ".data..percpu"]` (which the macro handles).
+#[repr(transparent)]
+pub struct StaticPerCpuSymbol<T> {
+ _val: T, // generate a correctly sized type
+}
+
+/// A wrapper for per-CPU variables declared in C.
+///
+/// As with [`StaticPerCpuSymbol`], this type should not be used directly. Instead, use the
+/// [`crate::declare_extern_per_cpu!`] macro.
+#[repr(transparent)]
+pub struct ExternStaticPerCpuSymbol<T>(StaticPerCpuSymbol<T>);
+
+/// Holds a statically-allocated per-CPU variable.
+#[derive(Clone)]
+pub struct StaticPerCpu<T>(pub(super) PerCpuPtr<T>);
+
+impl<T: PerCpuSyncMarkerType> StaticPerCpuSymbol<T> {
+ /// Removes the per-CPU marker type from a static per-CPU symbol.
+ ///
+ /// To declare a static per-CPU variable, Rust requires that the variable be `Sync`. However,
+ /// in the case of per-CPU variables, this is silly. See [`PerCpuSyncMarkerType`]. Thus, our
+ /// statics are actually of type `StaticPerCpuSymbol<PerCpuSyncMarkerType<T>>`, and this
+ /// produces a `StaticPerCpuSymbol<T>`.
+ pub fn forward(ptr: *const Self) -> *const StaticPerCpuSymbol<T::Inner> {
+ ptr.cast()
+ }
+}
+
+impl<T> ExternStaticPerCpuSymbol<T> {
+ /// Gets a [`PerCpuPtr`] to the per-CPU variable.
+ ///
+ /// Usage of this [`PerCpuPtr`] must be very careful to keep in mind what's happening in C. In
+ /// particular, the C code cannot modify the variable while any reference derived from the
+ /// returned pointer is live.
+ pub fn ptr(ptr: *const Self) -> PerCpuPtr<T> {
+ // These casts are OK because, ExternStaticPerCpuSymbol, StaticPerCpuSymbol, and
+ // MaybeUninit are transparent, everything is just a `T` from a memory layout perspective.
+ // Casting to `mut` is OK because any usage of the returned pointer must satisfy the
+ // soundness requirements for using it as such.
+ PerCpuPtr::new(ptr.cast::<StaticPerCpuSymbol<T>>().cast_mut().cast())
+ }
+}
+
+impl<T> StaticPerCpu<T> {
+ /// Creates a [`StaticPerCpu<T>`] from a [`StaticPerCpuSymbol<T>`].
+ ///
+ /// Users should probably declare static per-CPU variables with [`crate::define_per_cpu!`] and
+ /// then get instances of [`StaticPerCpu`] using [`crate::get_static_per_cpu!`].
+ ///
+ /// # Safety
+ /// You should probably be using [`crate::get_static_per_cpu!`] instead.
+ ///
+ /// `ptr` must be a valid offset into the per-CPU area, sized and aligned for access to a `T`;
+ /// typically this is the address of a static in the `.data..percpu` section, which is managed
+ /// by the per-CPU subsystem.
+ pub unsafe fn new(ptr: *const StaticPerCpuSymbol<T>) -> StaticPerCpu<T> {
+ let pcpu_ptr = PerCpuPtr::new(ptr.cast_mut().cast());
+ Self(pcpu_ptr)
+ }
+}
+
+impl<T> PerCpu<T> for StaticPerCpu<T> {
+ unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> {
+ // SAFETY:
+ // 1. By the requirements of `PerCpu::get_mut`, no other `[Checked]PerCpuToken` exists on
+ // the current CPU.
+ // 2. The per-CPU subsystem guarantees that each CPU's instance of a statically allocated
+ // variable begins with a copy of the contents of the corresponding symbol in
+ // `.data..percpu` and is therefore initialized.
+ // 3. The per-CPU subsystem guarantees that `self.0` is correctly aligned for a `T`.
+ // 4. The per-CPU subsystem guarantees that `self.0` lives forever as a per-CPU allocation,
+ // and that this allocation is the proper size for a `T`.
+ unsafe { PerCpuToken::new(guard, &self.0) }
+ }
+}
+
+impl<T: InteriorMutable> CheckedPerCpu<T> for StaticPerCpu<T> {
+ fn get(&self, guard: CpuGuard) -> CheckedPerCpuToken<'_, T> {
+ // SAFETY:
+ // 1. The per-CPU subsystem guarantees that each CPU's instance of a statically allocated
+ // variable begins with a copy of the contents of the corresponding symbol in
+ // `.data..percpu` and is therefore initialized.
+ // 2. The per-CPU subsystem guarantees that `self.0` is correctly aligned for a `T`.
+ // 3. The per-CPU subsystem guarantees that `self.0` lives forever as a per-CPU allocation,
+ // and that this allocation is the proper size for a `T`.
+ unsafe { CheckedPerCpuToken::new(guard, &self.0) }
+ }
+}
+
+/// Gets a [`StaticPerCpu<T>`] from a symbol declared with [`crate::define_per_cpu!`].
+///
+/// # Arguments
+///
+/// * `ident` - The identifier declared
+#[macro_export]
+macro_rules! get_static_per_cpu {
+ ($id:ident) => {
+ unsafe {
+ // SAFETY: The signature of `StaticPerCpuSymbol::forward` guarantees that `&raw const
+ // $id` is a `*const StaticPerCpuSymbol<PerCpuSyncMarkerType<T>>` if the macro
+ // invocation compiles.
+ //
+ // Values of type `StaticPerCpuSymbol<T>` must be created via `define_per_cpu`, and so
+ // the per-CPU subsystem guarantees that the requirements for `StaticPerCpu::new` are
+ // satisfied.
+ $crate::percpu::StaticPerCpu::new($crate::percpu::StaticPerCpuSymbol::forward(
+ &raw const $id,
+ ))
+ }
+ };
+}
+
+/// Declares an [`ExternStaticPerCpuSymbol`] corresponding to a per-CPU variable defined in C.
+#[macro_export]
+macro_rules! declare_extern_per_cpu {
+ ($id:ident: $ty:ty) => {
+ extern "C" {
+ static $id: ExternStaticPerCpuSymbol<$ty>;
+ }
+ };
+}
+
+/// A trait implemented by static per-CPU wrapper types.
+///
+/// Internally, static per-CPU variables are declared as `static` variables. However, Rust doesn't
+/// allow you to declare statics of a `!Sync` type. This trait is implemented by the marker type
+/// that is declared as `Sync` and used to declare the static. See [`crate::define_per_cpu!`] for
+/// the gory details.
+///
+/// # Safety
+///
+/// Implementations must be `#[repr(transparent)]` wrappers around an `Inner`. This trait should
+/// only be used from the [`crate::define_per_cpu!`] macro.
+pub unsafe trait PerCpuSyncMarkerType {
+ /// The "true" type of the per-CPU variable. It must always be valid to cast a `*const Self` to
+ /// a `*const Self::Inner`, which is guaranteed by this trait's safety requirement.
+ type Inner;
+}
+
+/// Declares and initializes a static per-CPU variable.
+///
+/// Analogous to the C `DEFINE_PER_CPU` macro.
+///
+/// See also [`crate::get_static_per_cpu!`] for how to get a [`StaticPerCpu`] from this
+/// declaration.
+///
+/// # Example
+/// ```
+/// use kernel::define_per_cpu;
+///
+/// define_per_cpu!(pub MY_PERCPU: u64 = 0);
+/// ```
+#[macro_export]
+macro_rules! define_per_cpu {
+ ($vis:vis $id:ident: $ty:ty = $expr:expr) => {
+ $crate::macros::paste! {
+ // We might want to have a per-CPU variable that doesn't implement `Sync` (not paying
+ // sync overhead costs is part of the point), but Rust won't let us declare a static of
+ // a `!Sync` type. Of course, we don't actually have any synchronization issues, since
+ // each CPU will see its own copy of the variable, so we cheat a little bit and tell
+ // Rust it's fine.
+ #[doc(hidden)]
+ #[allow(non_camel_case_types)]
+ #[repr(transparent)] // It needs to be the same size as $ty
+ struct [<__PRIVATE_TYPE_ $id>]($ty);
+
+ // SAFETY: [<__PRIVATE_TYPE_ $id>] is a `#[repr(transparent)]` wrapper around a `$ty`.
+ unsafe impl $crate::percpu::PerCpuSyncMarkerType for [<__PRIVATE_TYPE_ $id>] {
+ type Inner = $ty;
+ }
+
+ impl [<__PRIVATE_TYPE_ $id>] {
+ #[doc(hidden)]
+ const fn new(val: $ty) -> Self {
+ Self(val)
+ }
+ }
+
+ // Expand $expr outside of the unsafe block to avoid silently allowing unsafe code to be
+ // used without a user-facing unsafe block
+ #[doc(hidden)]
+ static [<__INIT_ $id>]: [<__PRIVATE_TYPE_ $id>] = [<__PRIVATE_TYPE_ $id>]::new($expr);
+
+ // SAFETY: This type will ONLY ever be used to declare a `StaticPerCpuSymbol`
+ // (which we then only ever use as input to `&raw`). Reading from the symbol is
+ // already UB, so we won't ever actually have any variables of this type where
+ // synchronization is a concern.
+ #[doc(hidden)]
+ unsafe impl Sync for [<__PRIVATE_TYPE_ $id>] {}
+
+ // SAFETY: StaticPerCpuSymbol<T> is #[repr(transparent)], so we can freely convert from
+ // [<__PRIVATE_TYPE_ $id>], which is also `#[repr(transparent)]` (i.e., everything is
+ // just a `$ty` from a memory layout perspective).
+ #[link_section = ".data..percpu"]
+ $vis static $id: $crate::percpu::StaticPerCpuSymbol<[<__PRIVATE_TYPE_ $id>]> = unsafe {
+ core::mem::transmute_copy::<
+ [<__PRIVATE_TYPE_ $id>],
+ $crate::percpu::StaticPerCpuSymbol<[<__PRIVATE_TYPE_ $id>]>
+ >(&[<__INIT_ $id>])
+ };
+ }
+ };
+}
--
2.34.1
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH v5 5/8] rust: percpu: introduce a rust API for dynamic per-CPU variables
2026-04-10 21:35 [PATCH v5 0/8] rust: Add Per-CPU Variable API Mitchell Levy
` (3 preceding siblings ...)
2026-04-10 21:35 ` [PATCH v5 4/8] rust: percpu: introduce a rust API for static per-CPU variables Mitchell Levy
@ 2026-04-10 21:35 ` Mitchell Levy
2026-04-10 21:35 ` [PATCH v5 6/8] rust: percpu: add a rust per-CPU variable sample Mitchell Levy
` (2 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Mitchell Levy @ 2026-04-10 21:35 UTC (permalink / raw)
To: Miguel Ojeda, Alex Gaynor, Gary Guo, Björn Roy Baron,
Andreas Hindborg, Alice Ryhl, Trevor Gross, Andrew Morton,
Dennis Zhou, Tejun Heo, Christoph Lameter, Danilo Krummrich,
Benno Lossin, Yury Norov, Viresh Kumar, Boqun Feng
Cc: Tyler Hicks, Allen Pais, linux-kernel, rust-for-linux, linux-mm,
Mitchell Levy
Dynamically allocated per-CPU variables are core to many of the
use-cases of per-CPU variables (e.g., ref counting). Add support for
them using the core `PerCpuPtr<T>` primitive, implementing the
`PerCpu<T>` and `CheckedPerCpu<T>` traits.
Co-developed-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
---
rust/helpers/percpu.c | 12 +++
rust/kernel/percpu.rs | 30 +++++-
rust/kernel/percpu/dynamic.rs | 217 ++++++++++++++++++++++++++++++++++++++++++
3 files changed, 255 insertions(+), 4 deletions(-)
diff --git a/rust/helpers/percpu.c b/rust/helpers/percpu.c
index 463186b8af9d..3b2f69a96c66 100644
--- a/rust/helpers/percpu.c
+++ b/rust/helpers/percpu.c
@@ -7,3 +7,15 @@ void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align)
{
return __alloc_percpu(sz, align);
}
+
+__rust_helper
+void *rust_helper_per_cpu_ptr(void __percpu *ptr, unsigned int cpu)
+{
+ return per_cpu_ptr(ptr, cpu);
+}
+
+__rust_helper
+void rust_helper_on_each_cpu(smp_call_func_t func, void *info, int wait)
+{
+ on_each_cpu(func, info, wait);
+}
diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs
index 615859d4daad..0ec1245038bb 100644
--- a/rust/kernel/percpu.rs
+++ b/rust/kernel/percpu.rs
@@ -1,15 +1,20 @@
// SPDX-License-Identifier: GPL-2.0
//! Per-CPU variables.
//!
-//! See the [`crate::define_per_cpu!`] macro and the [`PerCpu<T>`] trait.
+//! See the [`crate::define_per_cpu!`] macro, the [`DynamicPerCpu`] type, and the [`PerCpu<T>`]
+//! trait.
pub mod cpu_guard;
+mod dynamic;
mod static_;
+#[doc(inline)]
+pub use dynamic::*;
#[doc(inline)]
pub use static_::*;
use crate::{
+ cpu::CpuId,
declare_extern_per_cpu,
percpu::cpu_guard::CpuGuard,
types::Opaque, //
@@ -132,6 +137,23 @@ pub fn get_ptr(&self) -> *mut MaybeUninit<T> {
// the invariant that self.0 is a valid offset into the per-CPU area.
(this_cpu_area).wrapping_add(self.0 as usize).cast()
}
+
+ /// Get a [`*mut MaybeUninit<T>`](MaybeUninit) to the per-CPU variable on the CPU represented
+ /// by `cpu`. Note that without some kind of synchronization, use of the returned pointer may
+ /// cause a data race. It is the caller's responsibility to use the returned pointer in a
+ /// reasonable way.
+ ///
+ /// # Returns
+ /// - The returned pointer is valid only if `self` is (that is, it points to a live allocation
+ /// correctly sized and aligned to hold a `T`)
+ /// - The returned pointer is valid only if the bit corresponding to `cpu` is set in
+ /// [`kernel::cpumask::Cpumask::possible_cpus()`].
+ pub fn get_remote_ptr(&self, cpu: CpuId) -> *mut MaybeUninit<T> {
+ // SAFETY: `bindings::per_cpu_ptr` is just doing pointer arithmetic. The returned pointer
+ // may not be valid (under the conditions specified in this function's documentation), but
+ // the act of producing the pointer is safe.
+ unsafe { bindings::per_cpu_ptr(self.0.cast(), cpu.as_u32()) }.cast()
+ }
}
// SAFETY: Sending a [`PerCpuPtr<T>`] to another thread is safe because as soon as it's sent, the
@@ -155,9 +177,9 @@ impl<T> Copy for PerCpuPtr<T> {}
/// A trait representing a per-CPU variable.
///
-/// This is implemented for [`StaticPerCpu<T>`]. The main usage of this trait is to call
-/// [`Self::get_mut`] to get a [`PerCpuToken`] that can be used to access the underlying per-CPU
-/// variable.
+/// This is implemented for both [`StaticPerCpu<T>`] and [`DynamicPerCpu<T>`]. The main usage of
+/// this trait is to call [`Self::get_mut`] to get a [`PerCpuToken`] that can be used to access the
+/// underlying per-CPU variable.
///
/// See [`PerCpuToken::with`].
pub trait PerCpu<T> {
diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs
new file mode 100644
index 000000000000..40514704b3d0
--- /dev/null
+++ b/rust/kernel/percpu/dynamic.rs
@@ -0,0 +1,217 @@
+// SPDX-License-Identifier: GPL-2.0
+//! Dynamically allocated per-CPU variables.
+
+use super::*;
+
+use crate::{
+ alloc::Flags,
+ bindings::{
+ alloc_percpu,
+ free_percpu, //
+ },
+ cpumask::Cpumask,
+ prelude::*,
+ sync::Arc, //
+};
+
+use core::mem::{
+ align_of,
+ size_of,
+ MaybeUninit, //
+};
+
+/// Represents a dynamic allocation of a per-CPU variable via `alloc_percpu`. Calls `free_percpu`
+/// when dropped.
+///
+/// # Contents
+/// Note that the allocated memory need not be initialized, and this type does not track when/if
+/// the memory location on any particular CPU has been initialized. This means that it cannot tell
+/// whether it should drop the *contents* of the allocation when it is dropped. It is up to the
+/// user to do this via something like [`core::ptr::drop_in_place`].
+pub struct PerCpuAllocation<T>(PerCpuPtr<T>);
+
+impl<T: Zeroable> PerCpuAllocation<T> {
+ /// Dynamically allocates a space in the per-CPU area suitably sized and aligned to hold a `T`,
+ /// initially filled with the zero value for `T`.
+ ///
+ /// Returns [`None`] under the same circumstances the C function `alloc_percpu` returns `NULL`.
+ pub fn new_zero() -> Option<PerCpuAllocation<T>> {
+ let ptr: *mut MaybeUninit<T> =
+ // SAFETY: No preconditions to call `alloc_percpu`. The pointer is sized/aligned for a
+ // `T`, and `MaybeUninit<T>` is `#[repr(transparent)]` (and thus same size/align), so
+ // we can cast this `*mut c_void` to it.
+ unsafe { alloc_percpu(size_of::<T>(), align_of::<T>()) }.cast();
+ if ptr.is_null() {
+ return None;
+ }
+
+ // alloc_percpu returns zero'ed memory
+ Some(Self(PerCpuPtr::new(ptr)))
+ }
+}
+
+impl<T> PerCpuAllocation<T> {
+ /// Makes a per-CPU allocation sized and aligned to hold a `T`.
+ ///
+ /// Returns [`None`] under the same circumstances the C function `alloc_percpu` returns `NULL`.
+ pub fn new_uninit() -> Option<PerCpuAllocation<T>> {
+ let ptr: *mut MaybeUninit<T> =
+ // SAFETY: No preconditions to call `alloc_percpu`. The pointer is sized/aligned for a
+ // `T`, and `MaybeUninit<T>` is `#[repr(transparent)]` (and thus same size/align), so
+ // we can cast this `*mut c_void` to it.
+ unsafe { alloc_percpu(size_of::<T>(), align_of::<T>()) }.cast();
+ if ptr.is_null() {
+ return None;
+ }
+
+ Some(Self(PerCpuPtr::new(ptr)))
+ }
+}
+
+impl<T> Drop for PerCpuAllocation<T> {
+ fn drop(&mut self) {
+ // SAFETY: self.0.0 was returned by alloc_percpu, and so was a valid pointer into
+ // the percpu area, and has remained valid by the invariants of PerCpuAllocation<T>.
+ unsafe { free_percpu(self.0 .0.cast()) }
+ }
+}
+
+/// Holds a dynamically-allocated per-CPU variable.
+///
+/// # Examples
+/// ```
+/// # use kernel::prelude::*;
+/// # use kernel::percpu::*;
+/// # use kernel::percpu::cpu_guard::CpuGuard;
+///
+/// let mut pcpu: DynamicPerCpu<u32> =
+/// DynamicPerCpu::new_zero(GFP_KERNEL).expect("failed to allocate per-CPU variable");
+/// {
+/// let _guard = CpuGuard::new();
+/// // SAFETY: No other `pcpu` reference
+/// unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val| *val = 42);
+/// // SAFETY: No other `pcpu` reference
+/// unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val| assert!(*val == 42));
+/// }
+/// ```
+#[derive(Clone)]
+pub struct DynamicPerCpu<T> {
+ // INVARIANT: `alloc` is `Some` unless this object is in the process of being dropped.
+ // INVARIANT: The allocation held by `alloc` is sized and aligned for a `T`.
+ // INVARIANT: The memory location in each CPU's per-CPU area pointed at by the alloc is
+ // initialized.
+ alloc: Option<Arc<PerCpuAllocation<T>>>,
+}
+
+impl<T: Zeroable> DynamicPerCpu<T> {
+ /// Allocates a new per-CPU variable
+ ///
+ /// # Arguments
+ /// * `flags` - [`Flags`] used to allocate an [`Arc`] that keeps track of the underlying
+ /// [`PerCpuAllocation`].
+ pub fn new_zero(flags: Flags) -> Option<Self> {
+ let alloc: PerCpuAllocation<T> = PerCpuAllocation::new_zero()?;
+
+ let arc = Arc::new(alloc, flags).ok()?;
+
+ Some(Self { alloc: Some(arc) })
+ }
+}
+
+impl<T: Clone> DynamicPerCpu<T> {
+ /// Allocates a new per-CPU variable
+ ///
+ /// # Arguments
+ /// * `val` - The initial value of the per-CPU variable on all CPUs.
+ /// * `flags` - Flags used to allocate an [`Arc`] that keeps track of the underlying
+ /// [`PerCpuAllocation`].
+ pub fn new_with(val: &T, flags: Flags) -> Option<Self> {
+ Self::new_from(|_| val.clone(), flags)
+ }
+}
+
+impl<T> DynamicPerCpu<T> {
+ /// Allocates a new per-CPU variable
+ ///
+ /// # Arguments
+ /// * `initer` - A function that takes a `CpuId` and returns the initial value of the per-CPU
+ /// variable on the corresponding CPU. Called once for each possible CPU (i.e.,
+ /// `Cpumask::possible_cpus().weight()` times).
+ /// * `flags` - Flags used to allocate an [`Arc`] that keeps track of the underlying
+ /// [`PerCpuAllocation`].
+ pub fn new_from(mut initer: impl FnMut(CpuId) -> T, flags: Flags) -> Option<Self> {
+ let alloc: PerCpuAllocation<T> = PerCpuAllocation::new_uninit()?;
+ let ptr = alloc.0;
+
+ let arc = Arc::new(alloc, flags).ok()?;
+
+ for cpu in Cpumask::possible_cpus().iter() {
+ let remote_ptr = ptr.get_remote_ptr(cpu);
+ // SAFETY: `remote_ptr` is valid because `ptr` points to a live allocation and `cpu`
+ // appears in `Cpumask::possible_cpus()`.
+ //
+ // Each CPU's slot corresponding to `ptr` is currently uninitialized, and no one else
+ // has a reference to it. Therefore, we can freely write to it without worrying about
+ // the need to drop what was there or whether we're racing with someone else.
+ unsafe {
+ (*remote_ptr).write(initer(cpu));
+ }
+ }
+
+ Some(Self { alloc: Some(arc) })
+ }
+}
+
+impl<T> PerCpu<T> for DynamicPerCpu<T> {
+ unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> {
+ // SAFETY:
+ // 1. Invariants of this type assure that `alloc` is `Some`.
+ // 2. The requirements of `PerCpu::get_mut` ensure that no other `[Checked]PerCpuToken`
+ // exists on the current CPU.
+ // 3. The invariants of `DynamicPerCpu` ensure that the contents of the allocation are
+ // initialized on each CPU.
+ // 4. The existence of a reference to the `PerCpuAllocation` ensures that the allocation is
+ // live.
+ // 5. The invariants of `DynamicPerCpu` ensure that the allocation is sized and aligned for
+ // a `T`.
+ unsafe { PerCpuToken::new(guard, &self.alloc.as_ref().unwrap_unchecked().0) }
+ }
+}
+
+impl<T: InteriorMutable> CheckedPerCpu<T> for DynamicPerCpu<T> {
+ fn get(&self, guard: CpuGuard) -> CheckedPerCpuToken<'_, T> {
+ // SAFETY:
+ // 1. Invariants of this type assure that `alloc` is `Some`.
+ // 2. The invariants of `DynamicPerCpu` ensure that the contents of the allocation are
+ // initialized on each CPU.
+ // 3. The existence of a reference to the `PerCpuAllocation` ensures that the allocation is
+ // live.
+ // 4. The invariants of `DynamicPerCpu` ensure that the allocation is sized and aligned for
+ // a `T`.
+ unsafe { CheckedPerCpuToken::new(guard, &self.alloc.as_ref().unwrap_unchecked().0) }
+ }
+}
+
+impl<T> Drop for DynamicPerCpu<T> {
+ fn drop(&mut self) {
+ // SAFETY: This type's invariant ensures that `self.alloc` is `Some`.
+ let alloc = unsafe { self.alloc.take().unwrap_unchecked() };
+ if let Some(unique_alloc) = Arc::into_unique_or_drop(alloc) {
+ let ptr = unique_alloc.0;
+ for cpu in Cpumask::possible_cpus().iter() {
+ let remote_ptr = ptr.get_remote_ptr(cpu);
+ // SAFETY: `remote_ptr` is valid because the allocation it points to is still live,
+ // `cpu` appears in `Cpumask::possible_cpus()`, and the original allocation was
+ // sized and aligned for a `T`.
+ //
+ // This type's invariant ensures that the memory location in each CPU's per-CPU
+ // area pointed at by `alloc.0` has been initialized. We have a `UniqueArc`, so we
+ // know we're the only ones with a reference to the memory. These two facts
+ // together satisfy the requirements for `assume_init_drop`.
+ unsafe {
+ (*remote_ptr).assume_init_drop();
+ }
+ }
+ }
+ }
+}
--
2.34.1
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH v5 6/8] rust: percpu: add a rust per-CPU variable sample
2026-04-10 21:35 [PATCH v5 0/8] rust: Add Per-CPU Variable API Mitchell Levy
` (4 preceding siblings ...)
2026-04-10 21:35 ` [PATCH v5 5/8] rust: percpu: introduce a rust API for dynamic " Mitchell Levy
@ 2026-04-10 21:35 ` Mitchell Levy
2026-04-10 21:35 ` [PATCH v5 7/8] rust: percpu: Add pin-hole optimizations for numerics Mitchell Levy
2026-04-10 21:35 ` [PATCH v5 8/8] rust: percpu: cache per-CPU pointers in the dynamic case Mitchell Levy
7 siblings, 0 replies; 9+ messages in thread
From: Mitchell Levy @ 2026-04-10 21:35 UTC (permalink / raw)
To: Miguel Ojeda, Alex Gaynor, Gary Guo, Björn Roy Baron,
Andreas Hindborg, Alice Ryhl, Trevor Gross, Andrew Morton,
Dennis Zhou, Tejun Heo, Christoph Lameter, Danilo Krummrich,
Benno Lossin, Yury Norov, Viresh Kumar, Boqun Feng
Cc: Tyler Hicks, Allen Pais, linux-kernel, rust-for-linux, linux-mm,
Mitchell Levy
Add a short exercise for Rust's per-CPU variable API, modelled after
lib/percpu_test.c
Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
---
rust/helpers/percpu.c | 1 +
rust/kernel/percpu.rs | 2 +-
samples/rust/Kconfig | 9 ++
samples/rust/Makefile | 1 +
samples/rust/rust_percpu.rs | 278 ++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 290 insertions(+), 1 deletion(-)
diff --git a/rust/helpers/percpu.c b/rust/helpers/percpu.c
index 3b2f69a96c66..173b516cd813 100644
--- a/rust/helpers/percpu.c
+++ b/rust/helpers/percpu.c
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/percpu.h>
+#include <linux/smp.h>
__rust_helper
void __percpu *rust_helper_alloc_percpu(size_t sz, size_t align)
diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs
index 0ec1245038bb..72c83fef68ee 100644
--- a/rust/kernel/percpu.rs
+++ b/rust/kernel/percpu.rs
@@ -2,7 +2,7 @@
//! Per-CPU variables.
//!
//! See the [`crate::define_per_cpu!`] macro, the [`DynamicPerCpu`] type, and the [`PerCpu<T>`]
-//! trait.
+//! trait. Example usage can be found in `samples/rust/rust_percpu.rs`.
pub mod cpu_guard;
mod dynamic;
diff --git a/samples/rust/Kconfig b/samples/rust/Kconfig
index c49ab9106345..2616dbd7b37a 100644
--- a/samples/rust/Kconfig
+++ b/samples/rust/Kconfig
@@ -172,6 +172,15 @@ config SAMPLE_RUST_SOC
If unsure, say N.
+config SAMPLE_RUST_PERCPU
+ tristate "Per-CPU support"
+ depends on m
+ help
+ Enable this option to build a module which demonstrates Rust per-CPU
+ operations.
+
+ If unsure, say N.
+
config SAMPLE_RUST_HOSTPROGS
bool "Host programs"
help
diff --git a/samples/rust/Makefile b/samples/rust/Makefile
index 6c0aaa58cccc..1ce120ac0402 100644
--- a/samples/rust/Makefile
+++ b/samples/rust/Makefile
@@ -16,6 +16,7 @@ obj-$(CONFIG_SAMPLE_RUST_DRIVER_FAUX) += rust_driver_faux.o
obj-$(CONFIG_SAMPLE_RUST_DRIVER_AUXILIARY) += rust_driver_auxiliary.o
obj-$(CONFIG_SAMPLE_RUST_CONFIGFS) += rust_configfs.o
obj-$(CONFIG_SAMPLE_RUST_SOC) += rust_soc.o
+obj-$(CONFIG_SAMPLE_RUST_PERCPU) += rust_percpu.o
rust_print-y := rust_print_main.o rust_print_events.o
diff --git a/samples/rust/rust_percpu.rs b/samples/rust/rust_percpu.rs
new file mode 100644
index 000000000000..5adb30509bd4
--- /dev/null
+++ b/samples/rust/rust_percpu.rs
@@ -0,0 +1,278 @@
+// SPDX-License-Identifier: GPL-2.0
+//! A simple demonstration of the rust per-CPU API.
+
+use core::cell::RefCell;
+use core::ffi::c_void;
+
+use kernel::{
+ bindings::on_each_cpu,
+ cpu::CpuId,
+ define_per_cpu, get_static_per_cpu,
+ percpu::{cpu_guard::*, *},
+ pr_info,
+ prelude::*,
+ sync::Arc,
+};
+
+module! {
+ type: PerCpuMod,
+ name: "rust_percpu",
+ authors: ["Mitchell Levy"],
+ description: "Sample to demonstrate the Rust per-CPU API",
+ license: "GPL v2",
+}
+
+struct PerCpuMod;
+
+define_per_cpu!(PERCPU: i64 = 0);
+define_per_cpu!(UPERCPU: u64 = 0);
+define_per_cpu!(CHECKED: RefCell<u64> = RefCell::new(0));
+
+impl kernel::Module for PerCpuMod {
+ fn init(_module: &'static ThisModule) -> Result<Self, Error> {
+ pr_info!("rust percpu test start\n");
+
+ let mut native: i64 = 0;
+ let mut pcpu: StaticPerCpu<i64> = get_static_per_cpu!(PERCPU);
+
+ // SAFETY: We only have one PerCpu that points at PERCPU
+ unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut i64| {
+ pr_info!("The contents of pcpu are {}\n", *val);
+
+ native += -1;
+ *val += -1;
+ pr_info!("Native: {}, *pcpu: {}\n", native, *val);
+ assert!(native == *val && native == -1);
+
+ native += 1;
+ *val += 1;
+ pr_info!("Native: {}, *pcpu: {}\n", native, *val);
+ assert!(native == *val && native == 0);
+ });
+
+ let mut unative: u64 = 0;
+ let mut upcpu: StaticPerCpu<u64> = get_static_per_cpu!(UPERCPU);
+
+ // SAFETY: We only have one PerCpu pointing at UPERCPU
+ unsafe { upcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| {
+ unative += 1;
+ *val += 1;
+ pr_info!("Unative: {}, *upcpu: {}\n", unative, *val);
+ assert!(unative == *val && unative == 1);
+
+ unative = unative.wrapping_add((-1i64) as u64);
+ *val = val.wrapping_add((-1i64) as u64);
+ pr_info!("Unative: {}, *upcpu: {}\n", unative, *val);
+ assert!(unative == *val && unative == 0);
+
+ unative = unative.wrapping_add((-1i64) as u64);
+ *val = val.wrapping_add((-1i64) as u64);
+ pr_info!("Unative: {}, *upcpu: {}\n", unative, *val);
+ assert!(unative == *val && unative == (-1i64) as u64);
+
+ unative = 0;
+ *val = 0;
+
+ unative = unative.wrapping_sub(1);
+ *val = val.wrapping_sub(1);
+ pr_info!("Unative: {}, *upcpu: {}\n", unative, *val);
+ assert!(unative == *val && unative == (-1i64) as u64);
+ assert!(unative == *val && unative == u64::MAX);
+ });
+
+ let mut checked_native: u64 = 0;
+ let checked: StaticPerCpu<RefCell<u64>> = get_static_per_cpu!(CHECKED);
+ checked.get(CpuGuard::new()).with(|val: &RefCell<u64>| {
+ checked_native += 1;
+ *val.borrow_mut() += 1;
+ pr_info!(
+ "Checked native: {}, *checked: {}\n",
+ checked_native,
+ *val.borrow()
+ );
+ assert!(checked_native == *val.borrow() && checked_native == 1);
+
+ checked_native = checked_native.wrapping_add((-1i64) as u64);
+ val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64));
+ pr_info!(
+ "Checked native: {}, *checked: {}\n",
+ checked_native,
+ *val.borrow()
+ );
+ assert!(checked_native == *val.borrow() && checked_native == 0);
+
+ checked_native = checked_native.wrapping_add((-1i64) as u64);
+ val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64));
+ pr_info!(
+ "Checked native: {}, *checked: {}\n",
+ checked_native,
+ *val.borrow()
+ );
+ assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64);
+
+ checked_native = 0;
+ *val.borrow_mut() = 0;
+
+ checked_native = checked_native.wrapping_sub(1);
+ val.replace_with(|old: &mut u64| old.wrapping_sub(1));
+ pr_info!(
+ "Checked native: {}, *checked: {}\n",
+ checked_native,
+ *val.borrow()
+ );
+ assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64);
+ assert!(checked_native == *val.borrow() && checked_native == u64::MAX);
+ });
+
+ pr_info!("rust static percpu test done\n");
+
+ pr_info!("rust dynamic percpu test start\n");
+ let mut test: DynamicPerCpu<u64> = DynamicPerCpu::new_zero(GFP_KERNEL).unwrap();
+
+ // SAFETY: No prerequisites for on_each_cpu.
+ unsafe {
+ on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 0);
+ on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 0);
+ on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 0);
+ on_each_cpu(Some(inc_percpu_u64), (&raw mut test).cast(), 1);
+ on_each_cpu(Some(check_percpu_u64), (&raw mut test).cast(), 1);
+ }
+
+ let checked: DynamicPerCpu<RefCell<u64>> =
+ DynamicPerCpu::new_with(&RefCell::new(100), GFP_KERNEL).unwrap();
+
+ // SAFETY: No prerequisites for on_each_cpu.
+ unsafe {
+ on_each_cpu(
+ Some(inc_percpu_refcell_u64),
+ (&raw const checked) as *mut c_void,
+ 0,
+ );
+ on_each_cpu(
+ Some(inc_percpu_refcell_u64),
+ (&raw const checked) as *mut c_void,
+ 0,
+ );
+ on_each_cpu(
+ Some(inc_percpu_refcell_u64),
+ (&raw const checked) as *mut c_void,
+ 0,
+ );
+ on_each_cpu(
+ Some(inc_percpu_refcell_u64),
+ (&raw const checked) as *mut c_void,
+ 1,
+ );
+ on_each_cpu(
+ Some(check_percpu_refcell_u64),
+ (&raw const checked) as *mut c_void,
+ 1,
+ );
+ }
+
+ checked.get(CpuGuard::new()).with(|val: &RefCell<u64>| {
+ assert!(*val.borrow() == 104);
+
+ let mut checked_native = 0;
+ *val.borrow_mut() = 0;
+
+ checked_native += 1;
+ *val.borrow_mut() += 1;
+ pr_info!(
+ "Checked native: {}, *checked: {}\n",
+ checked_native,
+ *val.borrow()
+ );
+ assert!(checked_native == *val.borrow() && checked_native == 1);
+
+ checked_native = checked_native.wrapping_add((-1i64) as u64);
+ val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64));
+ pr_info!(
+ "Checked native: {}, *checked: {}\n",
+ checked_native,
+ *val.borrow()
+ );
+ assert!(checked_native == *val.borrow() && checked_native == 0);
+
+ checked_native = checked_native.wrapping_add((-1i64) as u64);
+ val.replace_with(|old: &mut u64| old.wrapping_add((-1i64) as u64));
+ pr_info!(
+ "Checked native: {}, *checked: {}\n",
+ checked_native,
+ *val.borrow()
+ );
+ assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64);
+
+ checked_native = 0;
+ *val.borrow_mut() = 0;
+
+ checked_native = checked_native.wrapping_sub(1);
+ val.replace_with(|old: &mut u64| old.wrapping_sub(1));
+ pr_info!(
+ "Checked native: {}, *checked: {}\n",
+ checked_native,
+ *val.borrow()
+ );
+ assert!(checked_native == *val.borrow() && checked_native == (-1i64) as u64);
+ assert!(checked_native == *val.borrow() && checked_native == u64::MAX);
+ });
+
+ let arc = Arc::new(0, GFP_KERNEL).unwrap();
+ {
+ let _arc_pcpu: DynamicPerCpu<Arc<u64>> =
+ DynamicPerCpu::new_with(&arc, GFP_KERNEL).unwrap();
+ }
+ // `arc` should be unique, since all the clones on each CPU should be dropped when
+ // `_arc_pcpu` is dropped
+ assert!(Arc::into_unique_or_drop(arc).is_some());
+
+ pr_info!("rust dynamic percpu test done\n");
+
+ // Return Err to unload the module
+ Result::Err(EINVAL)
+ }
+}
+
+extern "C" fn inc_percpu_u64(info: *mut c_void) {
+ // SAFETY: We know that info is a void *const DynamicPerCpu<u64> and DynamicPerCpu<u64> is Send.
+ let mut pcpu = unsafe { (*(info as *const DynamicPerCpu<u64>)).clone() };
+ pr_info!("Incrementing on {}\n", CpuId::current().as_u32());
+
+ // SAFETY: We don't have multiple clones of pcpu in scope
+ unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| *val += 1);
+}
+
+extern "C" fn check_percpu_u64(info: *mut c_void) {
+ // SAFETY: We know that info is a void *const DynamicPerCpu<u64> and DynamicPerCpu<u64> is Send.
+ let mut pcpu = unsafe { (*(info as *const DynamicPerCpu<u64>)).clone() };
+ pr_info!("Asserting on {}\n", CpuId::current().as_u32());
+
+ // SAFETY: We don't have multiple clones of pcpu in scope
+ unsafe { pcpu.get_mut(CpuGuard::new()) }.with(|val: &mut u64| assert!(*val == 4));
+}
+
+extern "C" fn inc_percpu_refcell_u64(info: *mut c_void) {
+ // SAFETY: We know that info is a void *const DynamicPerCpu<RefCell<u64>> and
+ // DynamicPerCpu<RefCell<u64>> is Send.
+ let pcpu = unsafe { (*(info as *const DynamicPerCpu<RefCell<u64>>)).clone() };
+ // SAFETY: smp_processor_id has no preconditions
+ pr_info!("Incrementing on {}\n", CpuId::current().as_u32());
+
+ pcpu.get(CpuGuard::new()).with(|val: &RefCell<u64>| {
+ let mut val = val.borrow_mut();
+ *val += 1;
+ });
+}
+
+extern "C" fn check_percpu_refcell_u64(info: *mut c_void) {
+ // SAFETY: We know that info is a void *const DynamicPerCpu<RefCell<u64>> and
+ // DynamicPerCpu<RefCell<u64>> is Send.
+ let pcpu = unsafe { (*(info as *const DynamicPerCpu<RefCell<u64>>)).clone() };
+ // SAFETY: smp_processor_id has no preconditions
+ pr_info!("Asserting on {}\n", CpuId::current().as_u32());
+
+ pcpu.get(CpuGuard::new()).with(|val: &RefCell<u64>| {
+ let val = val.borrow();
+ assert!(*val == 104);
+ });
+}
--
2.34.1
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH v5 7/8] rust: percpu: Add pin-hole optimizations for numerics
2026-04-10 21:35 [PATCH v5 0/8] rust: Add Per-CPU Variable API Mitchell Levy
` (5 preceding siblings ...)
2026-04-10 21:35 ` [PATCH v5 6/8] rust: percpu: add a rust per-CPU variable sample Mitchell Levy
@ 2026-04-10 21:35 ` Mitchell Levy
2026-04-10 21:35 ` [PATCH v5 8/8] rust: percpu: cache per-CPU pointers in the dynamic case Mitchell Levy
7 siblings, 0 replies; 9+ messages in thread
From: Mitchell Levy @ 2026-04-10 21:35 UTC (permalink / raw)
To: Miguel Ojeda, Alex Gaynor, Gary Guo, Björn Roy Baron,
Andreas Hindborg, Alice Ryhl, Trevor Gross, Andrew Morton,
Dennis Zhou, Tejun Heo, Christoph Lameter, Danilo Krummrich,
Benno Lossin, Yury Norov, Viresh Kumar, Boqun Feng
Cc: Tyler Hicks, Allen Pais, linux-kernel, rust-for-linux, linux-mm,
Mitchell Levy
The C implementations of `this_cpu_add`, `this_cpu_sub`, etc., are
optimized to save an instruction by avoiding having to compute
`this_cpu_ptr(&x)` for some per-CPU variable `x`. For example, rather
than
u64 *x_ptr = this_cpu_ptr(&x);
*x_ptr += 5;
the implementation of `this_cpu_add` is clever enough to make use of the
fact that per-CPU variables are implemented on x86 via segment
registers, and so we can use only a single instruction (where we assume
`&x` is already in `rax`)
add gs:[rax], 5
Add this optimization via a `PerCpuNumeric` type to enable code-reuse
between `DynamicPerCpu` and `StaticPerCpu`.
Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
---
rust/kernel/percpu.rs | 1 +
rust/kernel/percpu/dynamic.rs | 10 ++-
rust/kernel/percpu/numeric.rs | 138 ++++++++++++++++++++++++++++++++++++++++++
samples/rust/rust_percpu.rs | 36 +++++++++++
4 files changed, 184 insertions(+), 1 deletion(-)
diff --git a/rust/kernel/percpu.rs b/rust/kernel/percpu.rs
index 72c83fef68ee..ff04607ee047 100644
--- a/rust/kernel/percpu.rs
+++ b/rust/kernel/percpu.rs
@@ -6,6 +6,7 @@
pub mod cpu_guard;
mod dynamic;
+pub mod numeric;
mod static_;
#[doc(inline)]
diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs
index 40514704b3d0..a717138b93dc 100644
--- a/rust/kernel/percpu/dynamic.rs
+++ b/rust/kernel/percpu/dynamic.rs
@@ -28,7 +28,7 @@
/// the memory location on any particular CPU has been initialized. This means that it cannot tell
/// whether it should drop the *contents* of the allocation when it is dropped. It is up to the
/// user to do this via something like [`core::ptr::drop_in_place`].
-pub struct PerCpuAllocation<T>(PerCpuPtr<T>);
+pub struct PerCpuAllocation<T>(pub(super) PerCpuPtr<T>);
impl<T: Zeroable> PerCpuAllocation<T> {
/// Dynamically allocates a space in the per-CPU area suitably sized and aligned to hold a `T`,
@@ -162,6 +162,14 @@ pub fn new_from(mut initer: impl FnMut(CpuId) -> T, flags: Flags) -> Option<Self
}
}
+impl<T> DynamicPerCpu<T> {
+ /// Gets the allocation backing this per-CPU variable.
+ pub(crate) fn alloc(&self) -> &Arc<PerCpuAllocation<T>> {
+ // SAFETY: This type's invariant ensures that `self.alloc` is `Some`.
+ unsafe { self.alloc.as_ref().unwrap_unchecked() }
+ }
+}
+
impl<T> PerCpu<T> for DynamicPerCpu<T> {
unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> {
// SAFETY:
diff --git a/rust/kernel/percpu/numeric.rs b/rust/kernel/percpu/numeric.rs
new file mode 100644
index 000000000000..13b4ab4a794d
--- /dev/null
+++ b/rust/kernel/percpu/numeric.rs
@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: GPL-2.0
+//! Pin-hole optimizations for [`PerCpu<T>`] where T is a numeric type.
+
+use super::*;
+use core::arch::asm;
+
+/// Represents a per-CPU variable that can be manipulated with machine-intrinsic numeric
+/// operations.
+pub struct PerCpuNumeric<'a, T> {
+ // INVARIANT: `ptr.0` is a valid offset into the per-CPU area and is initialized on all CPUs
+ // (since we don't have a CPU guard, we have to be pessimistic and assume we could be on any
+ // CPU).
+ ptr: &'a PerCpuPtr<T>,
+}
+
+macro_rules! impl_ops {
+ ($ty:ty, $reg:tt) => {
+ impl DynamicPerCpu<$ty> {
+ /// Returns a [`PerCpuNumeric`] that can be used to manipulate the underlying per-CPU
+ /// variable.
+ #[inline]
+ pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> {
+ // The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that
+ // this pointer is valid and initialized on all CPUs.
+ PerCpuNumeric { ptr: &self.alloc().0 }
+ }
+ }
+ impl StaticPerCpu<$ty> {
+ /// Returns a [`PerCpuNumeric`] that can be used to manipulate the underlying per-CPU
+ /// variable.
+ #[inline]
+ pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> {
+ // The invariant is satisfied because `StaticPerCpu`'s invariant guarantees that
+ // this pointer is valid and initialized on all CPUs.
+ PerCpuNumeric { ptr: &self.0 }
+ }
+ }
+
+ impl PerCpuNumeric<'_, $ty> {
+ /// Adds `rhs` to the per-CPU variable.
+ #[inline]
+ pub fn add(&mut self, rhs: $ty) {
+ // SAFETY: `self.ptr.0` is a valid offset into the per-CPU area (i.e., valid as a
+ // pointer relative to the `gs` segment register) by the invariants of this type.
+ unsafe {
+ asm!(
+ concat!("add gs:[{off}], {val:", $reg, "}"),
+ off = in(reg) self.ptr.0.cast::<$ty>(),
+ val = in(reg) rhs,
+ );
+ }
+ }
+ }
+ impl PerCpuNumeric<'_, $ty> {
+ /// Subtracts `rhs` from the per-CPU variable.
+ #[inline]
+ pub fn sub(&mut self, rhs: $ty) {
+ // SAFETY: `self.ptr.0` is a valid offset into the per-CPU area (i.e., valid as a
+ // pointer relative to the `gs` segment register) by the invariants of this type.
+ unsafe {
+ asm!(
+ concat!("sub gs:[{off}], {val:", $reg, "}"),
+ off = in(reg) self.ptr.0.cast::<$ty>(),
+ val = in(reg) rhs,
+ );
+ }
+ }
+ }
+ };
+}
+
+macro_rules! impl_ops_byte {
+ ($ty:ty) => {
+ impl DynamicPerCpu<$ty> {
+ /// Returns a [`PerCpuNumeric`] that can be used to manipulate the underlying per-CPU
+ /// variable.
+ #[inline]
+ pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> {
+ // The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that
+ // this pointer is valid and initialized on all CPUs.
+ PerCpuNumeric { ptr: &self.alloc().0 }
+ }
+ }
+ impl StaticPerCpu<$ty> {
+ /// Returns a [`PerCpuNumeric`] that can be used to manipulate the underlying per-CPU
+ /// variable.
+ #[inline]
+ pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> {
+ // The invariant is satisfied because `StaticPerCpu`'s invariant guarantees that
+ // this pointer is valid and initialized on all CPUs.
+ PerCpuNumeric { ptr: &self.0 }
+ }
+ }
+
+ impl PerCpuNumeric<'_, $ty> {
+ /// Adds `rhs` to the per-CPU variable.
+ #[inline]
+ pub fn add(&mut self, rhs: $ty) {
+ // SAFETY: `self.ptr.0` is a valid offset into the per-CPU area (i.e., valid as a
+ // pointer relative to the `gs` segment register) by the invariants of this type.
+ unsafe {
+ asm!(
+ "add gs:[{off}], {val}",
+ off = in(reg) self.ptr.0.cast::<$ty>(),
+ val = in(reg_byte) rhs,
+ );
+ }
+ }
+ }
+ impl PerCpuNumeric<'_, $ty> {
+ /// Subtracts `rhs` from the per-CPU variable.
+ #[inline]
+ pub fn sub(&mut self, rhs: $ty) {
+ // SAFETY: `self.ptr.0` is a valid offset into the per-CPU area (i.e., valid as a
+ // pointer relative to the `gs` segment register) by the invariants of this type.
+ unsafe {
+ asm!(
+ "sub gs:[{off}], {val}",
+ off = in(reg) self.ptr.0.cast::<$ty>(),
+ val = in(reg_byte) rhs,
+ );
+ }
+ }
+ }
+ };
+}
+
+impl_ops_byte!(i8);
+impl_ops!(i16, "x");
+impl_ops!(i32, "e");
+impl_ops!(i64, "r");
+impl_ops!(isize, "r");
+
+impl_ops_byte!(u8);
+impl_ops!(u16, "x");
+impl_ops!(u32, "e");
+impl_ops!(u64, "r");
+impl_ops!(usize, "r");
diff --git a/samples/rust/rust_percpu.rs b/samples/rust/rust_percpu.rs
index 5adb30509bd4..90f5debd3c7a 100644
--- a/samples/rust/rust_percpu.rs
+++ b/samples/rust/rust_percpu.rs
@@ -28,6 +28,26 @@
define_per_cpu!(UPERCPU: u64 = 0);
define_per_cpu!(CHECKED: RefCell<u64> = RefCell::new(0));
+macro_rules! make_optimization_test {
+ ($ty:ty) => {
+ let mut test: DynamicPerCpu<$ty> = DynamicPerCpu::new_zero(GFP_KERNEL).unwrap();
+ {
+ let _guard = CpuGuard::new();
+ // SAFETY: No other usage of `test`
+ unsafe { test.get_mut(CpuGuard::new()) }.with(|val: &mut $ty| *val = 10);
+ test.num().add(1);
+ // SAFETY: No other usage of `test`
+ unsafe { test.get_mut(CpuGuard::new()) }.with(|val: &mut $ty| assert_eq!(*val, 11));
+ test.num().add(10);
+ // SAFETY: No other usage of `test`
+ unsafe { test.get_mut(CpuGuard::new()) }.with(|val: &mut $ty| assert_eq!(*val, 21));
+ test.num().sub(5);
+ // SAFETY: No other usage of `test`
+ unsafe { test.get_mut(CpuGuard::new()) }.with(|val: &mut $ty| assert_eq!(*val, 16));
+ }
+ };
+}
+
impl kernel::Module for PerCpuMod {
fn init(_module: &'static ThisModule) -> Result<Self, Error> {
pr_info!("rust percpu test start\n");
@@ -228,6 +248,22 @@ fn init(_module: &'static ThisModule) -> Result<Self, Error> {
pr_info!("rust dynamic percpu test done\n");
+ pr_info!("rust numeric optimizations test start\n");
+
+ make_optimization_test!(u8);
+ make_optimization_test!(u16);
+ make_optimization_test!(u32);
+ make_optimization_test!(u64);
+ make_optimization_test!(usize);
+
+ make_optimization_test!(i8);
+ make_optimization_test!(i16);
+ make_optimization_test!(i32);
+ make_optimization_test!(i64);
+ make_optimization_test!(isize);
+
+ pr_info!("rust numeric optimizations test done\n");
+
// Return Err to unload the module
Result::Err(EINVAL)
}
--
2.34.1
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH v5 8/8] rust: percpu: cache per-CPU pointers in the dynamic case
2026-04-10 21:35 [PATCH v5 0/8] rust: Add Per-CPU Variable API Mitchell Levy
` (6 preceding siblings ...)
2026-04-10 21:35 ` [PATCH v5 7/8] rust: percpu: Add pin-hole optimizations for numerics Mitchell Levy
@ 2026-04-10 21:35 ` Mitchell Levy
7 siblings, 0 replies; 9+ messages in thread
From: Mitchell Levy @ 2026-04-10 21:35 UTC (permalink / raw)
To: Miguel Ojeda, Alex Gaynor, Gary Guo, Björn Roy Baron,
Andreas Hindborg, Alice Ryhl, Trevor Gross, Andrew Morton,
Dennis Zhou, Tejun Heo, Christoph Lameter, Danilo Krummrich,
Benno Lossin, Yury Norov, Viresh Kumar, Boqun Feng
Cc: Tyler Hicks, Allen Pais, linux-kernel, rust-for-linux, linux-mm,
Mitchell Levy
Currently, the creation of a `PerCpuNumeric` requires a memory read via
the `Arc` managing the dynamic allocation. While the compiler might be
clever enough to consolidate these reads in some cases, the read must
happen *somewhere*, which, when we're concerning ourselves with
individual instructions, is a very high burden.
Instead, cache the `PerCpuPointer` inside the `DynamicPerCpu` structure;
then, the `Arc` is used solely to manage the allocation.
We might as well also use this speed-up in the standard `get` and
`get_mut` accessors that give a `{Checked,}PerCpuToken`
Signed-off-by: Mitchell Levy <levymitchell0@gmail.com>
---
rust/kernel/percpu/dynamic.rs | 34 ++++++++++++++++++----------------
rust/kernel/percpu/numeric.rs | 4 ++--
2 files changed, 20 insertions(+), 18 deletions(-)
diff --git a/rust/kernel/percpu/dynamic.rs b/rust/kernel/percpu/dynamic.rs
index a717138b93dc..675210098ed8 100644
--- a/rust/kernel/percpu/dynamic.rs
+++ b/rust/kernel/percpu/dynamic.rs
@@ -101,6 +101,9 @@ pub struct DynamicPerCpu<T> {
// INVARIANT: The memory location in each CPU's per-CPU area pointed at by the alloc is
// initialized.
alloc: Option<Arc<PerCpuAllocation<T>>>,
+ // INVARIANT: `ptr` is the per-CPU pointer managed by `alloc`, which does not change for the
+ // lifetime of `self`.
+ pub(super) ptr: PerCpuPtr<T>,
}
impl<T: Zeroable> DynamicPerCpu<T> {
@@ -112,9 +115,13 @@ impl<T: Zeroable> DynamicPerCpu<T> {
pub fn new_zero(flags: Flags) -> Option<Self> {
let alloc: PerCpuAllocation<T> = PerCpuAllocation::new_zero()?;
+ let ptr = alloc.0;
let arc = Arc::new(alloc, flags).ok()?;
- Some(Self { alloc: Some(arc) })
+ Some(Self {
+ alloc: Some(arc),
+ ptr,
+ })
}
}
@@ -158,15 +165,10 @@ pub fn new_from(mut initer: impl FnMut(CpuId) -> T, flags: Flags) -> Option<Self
}
}
- Some(Self { alloc: Some(arc) })
- }
-}
-
-impl<T> DynamicPerCpu<T> {
- /// Gets the allocation backing this per-CPU variable.
- pub(crate) fn alloc(&self) -> &Arc<PerCpuAllocation<T>> {
- // SAFETY: This type's invariant ensures that `self.alloc` is `Some`.
- unsafe { self.alloc.as_ref().unwrap_unchecked() }
+ Some(Self {
+ alloc: Some(arc),
+ ptr,
+ })
}
}
@@ -178,11 +180,11 @@ unsafe fn get_mut(&mut self, guard: CpuGuard) -> PerCpuToken<'_, T> {
// exists on the current CPU.
// 3. The invariants of `DynamicPerCpu` ensure that the contents of the allocation are
// initialized on each CPU.
- // 4. The existence of a reference to the `PerCpuAllocation` ensures that the allocation is
- // live.
+ // 4. `&mut self` holds a reference to the `PerCpuAllocation`, so the allocation it manages
+ // is live.
// 5. The invariants of `DynamicPerCpu` ensure that the allocation is sized and aligned for
// a `T`.
- unsafe { PerCpuToken::new(guard, &self.alloc.as_ref().unwrap_unchecked().0) }
+ unsafe { PerCpuToken::new(guard, &self.ptr) }
}
}
@@ -192,11 +194,11 @@ fn get(&self, guard: CpuGuard) -> CheckedPerCpuToken<'_, T> {
// 1. Invariants of this type assure that `alloc` is `Some`.
// 2. The invariants of `DynamicPerCpu` ensure that the contents of the allocation are
// initialized on each CPU.
- // 3. The existence of a reference to the `PerCpuAllocation` ensures that the allocation is
- // live.
+ // 3. `&mut self` holds a reference to the `PerCpuAllocation`, so the allocation it manages
+ // is live.
// 4. The invariants of `DynamicPerCpu` ensure that the allocation is sized and aligned for
// a `T`.
- unsafe { CheckedPerCpuToken::new(guard, &self.alloc.as_ref().unwrap_unchecked().0) }
+ unsafe { CheckedPerCpuToken::new(guard, &self.ptr) }
}
}
diff --git a/rust/kernel/percpu/numeric.rs b/rust/kernel/percpu/numeric.rs
index 13b4ab4a794d..4841843b05f7 100644
--- a/rust/kernel/percpu/numeric.rs
+++ b/rust/kernel/percpu/numeric.rs
@@ -22,7 +22,7 @@ impl DynamicPerCpu<$ty> {
pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> {
// The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that
// this pointer is valid and initialized on all CPUs.
- PerCpuNumeric { ptr: &self.alloc().0 }
+ PerCpuNumeric { ptr: &self.ptr }
}
}
impl StaticPerCpu<$ty> {
@@ -78,7 +78,7 @@ impl DynamicPerCpu<$ty> {
pub fn num(&mut self) -> PerCpuNumeric<'_, $ty> {
// The invariant is satisfied because `DynamicPerCpu`'s invariant guarantees that
// this pointer is valid and initialized on all CPUs.
- PerCpuNumeric { ptr: &self.alloc().0 }
+ PerCpuNumeric { ptr: &self.ptr }
}
}
impl StaticPerCpu<$ty> {
--
2.34.1
^ permalink raw reply [flat|nested] 9+ messages in thread