From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from kanga.kvack.org (kanga.kvack.org [205.233.56.17]) by smtp.lore.kernel.org (Postfix) with ESMTP id 93B02C83030 for ; Mon, 30 Jun 2025 22:16:47 +0000 (UTC) Received: by kanga.kvack.org (Postfix) id 24FCB6B00A9; Mon, 30 Jun 2025 18:16:47 -0400 (EDT) Received: by kanga.kvack.org (Postfix, from userid 40) id 2278C6B00AA; Mon, 30 Jun 2025 18:16:47 -0400 (EDT) X-Delivered-To: int-list-linux-mm@kvack.org Received: by kanga.kvack.org (Postfix, from userid 63042) id 13D5C6B00AB; Mon, 30 Jun 2025 18:16:47 -0400 (EDT) X-Delivered-To: linux-mm@kvack.org Received: from relay.hostedemail.com (smtprelay0016.hostedemail.com [216.40.44.16]) by kanga.kvack.org (Postfix) with ESMTP id F20976B00A9 for ; Mon, 30 Jun 2025 18:16:46 -0400 (EDT) Received: from smtpin13.hostedemail.com (a10.router.float.18 [10.200.18.1]) by unirelay01.hostedemail.com (Postfix) with ESMTP id 703ED1CC923 for ; Mon, 30 Jun 2025 22:16:46 +0000 (UTC) X-FDA: 83613477612.13.B4ABA76 Received: from mailrelay2-3.pub.mailoutpod3-cph3.one.com (mailrelay2-3.pub.mailoutpod3-cph3.one.com [46.30.212.1]) by imf14.hostedemail.com (Postfix) with ESMTP id 8770510000B for ; Mon, 30 Jun 2025 22:16:44 +0000 (UTC) Authentication-Results: imf14.hostedemail.com; dkim=pass header.d=konsulko.se header.s=rsa1 header.b=SxuljuAm; dkim=pass header.d=konsulko.se header.s=ed1 header.b="pARVt/Wg" ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=hostedemail.com; s=arc-20220608; t=1751321804; h=from:from:sender:reply-to:subject:subject:date:date: message-id:message-id:to:to:cc:cc:mime-version:mime-version: content-type:content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references:dkim-signature; bh=4iG1oP39anU2PhYaELXh16+r4+04PSVs269ka+LWjL8=; b=erlLk6HhNXzdqJvvyngItyaF+4ooKy9gyFYleJ7sc7FTxqtjVF5tmOkZyRuKmePH7RoavQ +giiFiyjESXBQgdDdazjxXV2IzCC6DrwmbWuakC7VkFG8W4c9NKgypCivyP7GSNMos+Pp1 6s9uloW8REbqxK43u/mZoT4i0wyXBbI= ARC-Authentication-Results: i=1; imf14.hostedemail.com; dkim=pass header.d=konsulko.se header.s=rsa1 header.b=SxuljuAm; dkim=pass header.d=konsulko.se header.s=ed1 header.b="pARVt/Wg"; spf=none (imf14.hostedemail.com: domain of vitaly.wool@konsulko.se has no SPF policy when checking 46.30.212.1) smtp.mailfrom=vitaly.wool@konsulko.se; dmarc=none ARC-Seal: i=1; s=arc-20220608; d=hostedemail.com; t=1751321804; a=rsa-sha256; cv=none; b=RE5G44ETXuUQ7qLIoHCCZpFmCpm4CYxKiqnTpm9+dbk4PwdDRxfX0mutlUCDcDsAg7q/1V 0ax9DOcpgR18dk3cMS682FwQIuBEJura1IF7YeouHEfXTsPmjEmdpr3MyWb8LMCPbeKISB wYJpY0GkpHkR4RAWznifoGb4V+STwkY= DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; t=1751321803; x=1751926603; d=konsulko.se; s=rsa1; h=content-transfer-encoding:mime-version:references:in-reply-to:message-id:date: subject:cc:to:from:from; bh=4iG1oP39anU2PhYaELXh16+r4+04PSVs269ka+LWjL8=; b=SxuljuAmBlE9t9buh7w+rdTfeItu3ji69WXveFPe07JwYE7NUgDuKim0X1zchgJ2yzDxM1+WH2I1h SzWGP5zL3slO8LrBukoZQWwoKA2jKjOiu7ato9DmWeQ5JBe3T2ltWTLtAGzYlDEGwDzRaKVjsmj6nP ZykxevgvTn5RezA65vVMPhkkKKnGM/fam+wbl/9cjL+llhpwN1AtkM2r2gBeaEsZB0SqlEUV5VmU3c xkBqoPdUCyTfaZkqC55BnU/qTARTZtmDxBWg/wJRaNu1oRqohbDSMOfI47on9MqMGMhypD800MDNQz lPmlPQF9cv2qJ7j5Q6fG2Or3/qUy1gQ== DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed; t=1751321803; x=1751926603; d=konsulko.se; s=ed1; h=content-transfer-encoding:mime-version:references:in-reply-to:message-id:date: subject:cc:to:from:from; bh=4iG1oP39anU2PhYaELXh16+r4+04PSVs269ka+LWjL8=; b=pARVt/WgGTUS9sqCKXOuC5xD3fUHXmoofwYKczSXHYJLvFWYIn1M/PLHvMY2JUVH92RihWjnddHJK xYCSxkfCg== X-HalOne-ID: e5e5d27a-55ff-11f0-aa72-b37c246f863f Received: from slottsdator.home (host-90-238-19-233.mobileonline.telia.com [90.238.19.233]) by mailrelay2.pub.mailoutpod3-cph3.one.com (Halon) with ESMTPSA id e5e5d27a-55ff-11f0-aa72-b37c246f863f; Mon, 30 Jun 2025 22:16:42 +0000 (UTC) From: Vitaly Wool To: linux-mm@kvack.org Cc: akpm@linux-foundation.org, linux-kernel@vger.kernel.org, Uladzislau Rezki , Danilo Krummrich , Alice Ryhl , rust-for-linux@vger.kernel.org, Vitaly Wool Subject: [PATCH v9 3/4] rust: add support for NUMA ids in allocations Date: Tue, 1 Jul 2025 00:16:40 +0200 Message-Id: <20250630221640.3325306-1-vitaly.wool@konsulko.se> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20250630221511.3325123-1-vitaly.wool@konsulko.se> References: <20250630221511.3325123-1-vitaly.wool@konsulko.se> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Stat-Signature: dsr7nbj4cign3okt3ozsnh37qdn73c5q X-Rspamd-Server: rspam08 X-Rspamd-Queue-Id: 8770510000B X-Rspam-User: X-HE-Tag: 1751321804-857179 X-HE-Meta: U2FsdGVkX19OmHLX1L9ugOplK1WicWkP0LuijmgkWYctNBUxsxxBGWs5CiYpu9WONN63K2HZXp+cF5+Kj/1hdvSQGwfBPB34HbqsEmCQIf/n7GyRWsF93/dn3ixRMsnE7i0iY4BnqAX2mf9D7CS8lLqN8kZ7YpGBPmQa4e5fD6xt7bNJ1hRRQIzXkFhhoMhGaaSJV8/o4P+TTfIF0Q6ngooBX7hTmi78zw9dwtKwLvx1x8JKtrTnfrTk+u6J1KTzveTEOiX+CYi7sF6eiQDF8WzGDOz/vTF+w/PWW4HwxpRl4tt/1gtaFBPlMC4snSEYXSO1RR0RfOiKIW4aNZVoF16WLHjkd8rNwgcvexQbVMEPoemvW/GJNtExeIX7kHZWKhaBEZlBT0Nv5oM48ej5SjSOObOD2RcePgty0L0Ciy0pqvFrKqDwgD0ABS84Dvf0RB/jLOEyRa0MlCJS6yE7U3PYymhPHYZjyOu3rRGAstUttTg6y/9o/qgdkJkVujaSYFteakBIHjTR1BHHQVURcRD0AA/lLqdrZIevO3BuxZb09H6yfND8bINSJ2dmF1RNKjJ8odjn20PKrK9gmwFiRMWoOJYK1kez0MZXu57iiW2vFbFhKof8RVBV4ll2z/XFq0dSOwmVYyv9Kc9/towxpcnK8QbzgXs2z5Zrc4gIgQz/hTaUHQ4aKZV+ZeUUzXMtIWkpLnILQeIrNYz/E95N55766W7p+38K2mNZMNbG1oqR5IF+bwnITjNvjxqNQzVuUie1pOWIjeo3w4bkX5BsKenhNZ4T11F5JuyRWubma82SDOPDG/fQRYuYeDAIi12O+YWXtdYckGbSVASuTc9n0qt6tWk4lvY6M3cNfz5yYtEwgWQZ7cPF3KWO30t7Zs8BVJUss3e4H0lHCIr/xTpa7f6MaK0vwM4WFU+9ORvppw/rMiUfEiOCd/6HYnxMU6omZlljWC5Q4j4gygDwREl uzOy4dum QSpFPQeXiuaTeCEco8U2EsEdB6PiJHTOuf5DLJ7MjJnnDIT/bW+nB6eIms4DHSs9hAvb0Ey1bhlHAYY4xRFP446d7snynSG4jEDDkzjumFur9ptWXpwIKzaJW6j4ZeToUK4OCtgRi3n1PlgiUtYrZY0f8dvNYYGCZcXMGPMpEK5wQ0XCDDXV2PhG76g== X-Bogosity: Ham, tests=bogofilter, spamicity=0.000000, version=1.2.4 Sender: owner-linux-mm@kvack.org Precedence: bulk X-Loop: owner-majordomo@kvack.org List-ID: List-Subscribe: List-Unsubscribe: Add a new type to support specifying NUMA identifiers in Rust allocators and extend the allocators to have NUMA id as a parameter. Thus, modify ReallocFunc to use the new extended realloc primitives from the C side of the kernel (i. e. k[v]realloc_node_align/vrealloc_node_align) and add the new function alloc_node to the Allocator trait while keeping the existing one (alloc) for backward compatibility. This will allow to specify node to use for allocation of e. g. {KV}Box, as well as for future NUMA aware users of the API. Signed-off-by: Vitaly Wool --- rust/helpers/slab.c | 8 ++-- rust/helpers/vmalloc.c | 4 +- rust/kernel/alloc.rs | 80 ++++++++++++++++++++++++++++++++-- rust/kernel/alloc/allocator.rs | 42 ++++++++++-------- 4 files changed, 107 insertions(+), 27 deletions(-) diff --git a/rust/helpers/slab.c b/rust/helpers/slab.c index a842bfbddcba..8472370a4338 100644 --- a/rust/helpers/slab.c +++ b/rust/helpers/slab.c @@ -3,13 +3,13 @@ #include void * __must_check __realloc_size(2) -rust_helper_krealloc(const void *objp, size_t new_size, gfp_t flags) +rust_helper_krealloc_node(const void *objp, size_t new_size, gfp_t flags, int node) { - return krealloc(objp, new_size, flags); + return krealloc_node(objp, new_size, flags, node); } void * __must_check __realloc_size(2) -rust_helper_kvrealloc(const void *p, size_t size, gfp_t flags) +rust_helper_kvrealloc_node(const void *p, size_t size, gfp_t flags, int node) { - return kvrealloc(p, size, flags); + return kvrealloc_node(p, size, flags, node); } diff --git a/rust/helpers/vmalloc.c b/rust/helpers/vmalloc.c index 80d34501bbc0..62d30db9a1a6 100644 --- a/rust/helpers/vmalloc.c +++ b/rust/helpers/vmalloc.c @@ -3,7 +3,7 @@ #include void * __must_check __realloc_size(2) -rust_helper_vrealloc(const void *p, size_t size, gfp_t flags) +rust_helper_vrealloc_node(const void *p, size_t size, gfp_t flags, int node) { - return vrealloc(p, size, flags); + return vrealloc_node(p, size, flags, node); } diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index a2c49e5494d3..e886ab31108f 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -28,7 +28,9 @@ /// Indicates an allocation error. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub struct AllocError; + use core::{alloc::Layout, ptr::NonNull}; +use crate::error::{code::EINVAL, Result}; /// Flags to be used when allocating memory. /// @@ -115,6 +117,31 @@ pub mod flags { pub const __GFP_NOWARN: Flags = Flags(bindings::__GFP_NOWARN); } +/// Non Uniform Memory Access (NUMA) node identifier +#[derive(Clone, Copy, PartialEq)] +pub struct NumaNode(i32); + +impl NumaNode { + /// create a new NUMA node identifer (non-negative integer) + /// returns EINVAL if a negative id or an id exceeding MAX_NUMNODES is specified + pub fn new(node: i32) -> Result { + // SAFETY: MAX_NUMNODES never exceeds 2**10 because NODES_SHIFT is 0..10 + if node < 0 || node >= bindings::MAX_NUMNODES as i32 { + return Err(EINVAL); + } + Ok(Self(node)) + } +} + +/// Specify necessary constant to pass the information to Allocator that the caller doesn't care +/// about the NUMA node to allocate memory from. +pub mod numa { + use super::NumaNode; + + /// No preference for NUMA node + pub const NUMA_NO_NODE: NumaNode = NumaNode(bindings::NUMA_NO_NODE); +} + /// The kernel's [`Allocator`] trait. /// /// An implementation of [`Allocator`] can allocate, re-allocate and free memory buffers described @@ -148,7 +175,7 @@ pub unsafe trait Allocator { /// /// When the return value is `Ok(ptr)`, then `ptr` is /// - valid for reads and writes for `layout.size()` bytes, until it is passed to - /// [`Allocator::free`] or [`Allocator::realloc`], + /// [`Allocator::free`], [`Allocator::realloc`] or [`Allocator::realloc_node`], /// - aligned to `layout.align()`, /// /// Additionally, `Flags` are honored as documented in @@ -159,7 +186,38 @@ fn alloc(layout: Layout, flags: Flags) -> Result, AllocError> { unsafe { Self::realloc(None, layout, Layout::new::<()>(), flags) } } - /// Re-allocate an existing memory allocation to satisfy the requested `layout`. + /// Allocate memory based on `layout`, `flags` and `nid`. + /// + /// On success, returns a buffer represented as `NonNull<[u8]>` that satisfies the layout + /// constraints (i.e. minimum size and alignment as specified by `layout`). + /// + /// This function is equivalent to `realloc_node` when called with `None`. + /// + /// # Guarantees + /// + /// When the return value is `Ok(ptr)`, then `ptr` is + /// - valid for reads and writes for `layout.size()` bytes, until it is passed to + /// [`Allocator::free`], [`Allocator::realloc`] or [`Allocator::realloc_node`], + /// - aligned to `layout.align()`, + /// + /// Additionally, `Flags` are honored as documented in + /// . + fn alloc_node(layout: Layout, flags: Flags, nid: NumaNode) + -> Result, AllocError> { + // SAFETY: Passing `None` to `realloc_node` is valid by its safety requirements and + // asks for a new memory allocation. + unsafe { Self::realloc_node(None, layout, Layout::new::<()>(), flags, nid) } + } + + /// Re-allocate an existing memory allocation to satisfy the requested `layout` and + /// a specific NUMA node request to allocate the memory for. + /// + /// Systems employing a Non Uniform Memory Access (NUMA) architecture contain collections of + /// hardware resources including processors, memory, and I/O buses, that comprise what is + /// commonly known as a NUMA node. + /// + /// `nid` stands for NUMA id, i. e. NUMA node identifier, which is a non-negative + /// integer if a node needs to be specified, or NUMA_NO_NODE if the caller doesn't care. /// /// If the requested size is zero, `realloc` behaves equivalent to `free`. /// @@ -191,13 +249,29 @@ fn alloc(layout: Layout, flags: Flags) -> Result, AllocError> { /// and old size, i.e. `ret_ptr[0..min(layout.size(), old_layout.size())] == /// p[0..min(layout.size(), old_layout.size())]`. /// - when the return value is `Err(AllocError)`, then `ptr` is still valid. - unsafe fn realloc( + unsafe fn realloc_node( ptr: Option>, layout: Layout, old_layout: Layout, flags: Flags, + nid: NumaNode, ) -> Result, AllocError>; + + /// Re-allocate an existing memory allocation to satisfy the requested `layout`. This + /// function works exactly as realloc_node() but it doesn't give the ability to specify + /// the NUMA node in the call. + unsafe fn realloc( + ptr: Option>, + layout: Layout, + old_layout: Layout, + flags: Flags, + ) -> Result, AllocError> { + // SAFETY: guaranteed by realloc_node() + unsafe { Self::realloc_node(ptr, layout, old_layout, flags, numa::NUMA_NO_NODE) } + } + + /// Free an existing memory allocation. /// /// # Safety diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index aa2dfa9dca4c..2e86e9839a1b 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -13,7 +13,7 @@ use core::ptr; use core::ptr::NonNull; -use crate::alloc::{AllocError, Allocator}; +use crate::alloc::{AllocError, Allocator, NumaNode}; use crate::bindings; use crate::pr_warn; @@ -58,18 +58,20 @@ fn aligned_size(new_layout: Layout) -> usize { /// /// One of the following: `krealloc`, `vrealloc`, `kvrealloc`. struct ReallocFunc( - unsafe extern "C" fn(*const crate::ffi::c_void, usize, u32) -> *mut crate::ffi::c_void, + unsafe extern "C" fn( + *const crate::ffi::c_void, usize, u32, crate::ffi::c_int, + ) -> *mut crate::ffi::c_void, ); impl ReallocFunc { - // INVARIANT: `krealloc` satisfies the type invariants. - const KREALLOC: Self = Self(bindings::krealloc); + // INVARIANT: `krealloc_node` satisfies the type invariants. + const KREALLOC: Self = Self(bindings::krealloc_node); - // INVARIANT: `vrealloc` satisfies the type invariants. - const VREALLOC: Self = Self(bindings::vrealloc); + // INVARIANT: `vrealloc_node` satisfies the type invariants. + const VREALLOC: Self = Self(bindings::vrealloc_node); - // INVARIANT: `kvrealloc` satisfies the type invariants. - const KVREALLOC: Self = Self(bindings::kvrealloc); + // INVARIANT: `kvrealloc_node` satisfies the type invariants. + const KVREALLOC: Self = Self(bindings::kvrealloc_node); /// # Safety /// @@ -87,6 +89,7 @@ unsafe fn call( layout: Layout, old_layout: Layout, flags: Flags, + nid: NumaNode, ) -> Result, AllocError> { let size = aligned_size(layout); let ptr = match ptr { @@ -110,7 +113,7 @@ unsafe fn call( // - Those functions provide the guarantees of this function. let raw_ptr = unsafe { // If `size == 0` and `ptr != NULL` the memory behind the pointer is freed. - self.0(ptr.cast(), size, flags.0).cast() + self.0(ptr.cast(), size, flags.0, nid.0).cast() }; let ptr = if size == 0 { @@ -123,34 +126,36 @@ unsafe fn call( } } -// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// SAFETY: `realloc_node` delegates to `ReallocFunc::call`, which guarantees that // - memory remains valid until it is explicitly freed, // - passing a pointer to a valid memory allocation is OK, // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. unsafe impl Allocator for Kmalloc { #[inline] - unsafe fn realloc( + unsafe fn realloc_node( ptr: Option>, layout: Layout, old_layout: Layout, flags: Flags, + nid: NumaNode, ) -> Result, AllocError> { // SAFETY: `ReallocFunc::call` has the same safety requirements as `Allocator::realloc`. - unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags) } + unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags, nid) } } } -// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// SAFETY: `realloc_node` delegates to `ReallocFunc::call`, which guarantees that // - memory remains valid until it is explicitly freed, // - passing a pointer to a valid memory allocation is OK, // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. unsafe impl Allocator for Vmalloc { #[inline] - unsafe fn realloc( + unsafe fn realloc_node( ptr: Option>, layout: Layout, old_layout: Layout, flags: Flags, + nid: NumaNode, ) -> Result, AllocError> { // TODO: Support alignments larger than PAGE_SIZE. if layout.align() > bindings::PAGE_SIZE { @@ -160,21 +165,22 @@ unsafe fn realloc( // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously // allocated with this `Allocator`. - unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags) } + unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags, nid) } } } -// SAFETY: `realloc` delegates to `ReallocFunc::call`, which guarantees that +// SAFETY: `realloc_node` delegates to `ReallocFunc::call`, which guarantees that // - memory remains valid until it is explicitly freed, // - passing a pointer to a valid memory allocation is OK, // - `realloc` satisfies the guarantees, since `ReallocFunc::call` has the same. unsafe impl Allocator for KVmalloc { #[inline] - unsafe fn realloc( + unsafe fn realloc_node( ptr: Option>, layout: Layout, old_layout: Layout, flags: Flags, + nid: NumaNode, ) -> Result, AllocError> { // TODO: Support alignments larger than PAGE_SIZE. if layout.align() > bindings::PAGE_SIZE { @@ -184,6 +190,6 @@ unsafe fn realloc( // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously // allocated with this `Allocator`. - unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags) } + unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags, nid) } } } -- 2.39.2