From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from kanga.kvack.org (kanga.kvack.org [205.233.56.17]) by smtp.lore.kernel.org (Postfix) with ESMTP id C81E7C77B7F for ; Tue, 24 Jun 2025 12:39:32 +0000 (UTC) Received: by kanga.kvack.org (Postfix) id 610466B009B; Tue, 24 Jun 2025 08:39:32 -0400 (EDT) Received: by kanga.kvack.org (Postfix, from userid 40) id 5C0C06B009C; Tue, 24 Jun 2025 08:39:32 -0400 (EDT) X-Delivered-To: int-list-linux-mm@kvack.org Received: by kanga.kvack.org (Postfix, from userid 63042) id 462236B009D; Tue, 24 Jun 2025 08:39:32 -0400 (EDT) X-Delivered-To: linux-mm@kvack.org Received: from relay.hostedemail.com (smtprelay0014.hostedemail.com [216.40.44.14]) by kanga.kvack.org (Postfix) with ESMTP id 327E66B009B for ; Tue, 24 Jun 2025 08:39:32 -0400 (EDT) Received: from smtpin21.hostedemail.com (a10.router.float.18 [10.200.18.1]) by unirelay07.hostedemail.com (Postfix) with ESMTP id D449C160728 for ; Tue, 24 Jun 2025 12:39:31 +0000 (UTC) X-FDA: 83590250142.21.6519B25 Received: from mailrelay5-3.pub.mailoutpod3-cph3.one.com (mailrelay5-3.pub.mailoutpod3-cph3.one.com [46.30.212.10]) by imf30.hostedemail.com (Postfix) with ESMTP id 23D0380013 for ; Tue, 24 Jun 2025 12:39:29 +0000 (UTC) Authentication-Results: imf30.hostedemail.com; dkim=pass header.d=konsulko.se header.s=rsa1 header.b=a3Nn7EtL; dkim=pass header.d=konsulko.se header.s=ed1 header.b=WLIrf7Nb; spf=none (imf30.hostedemail.com: domain of vitaly.wool@konsulko.se has no SPF policy when checking 46.30.212.10) smtp.mailfrom=vitaly.wool@konsulko.se; dmarc=none ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=hostedemail.com; s=arc-20220608; t=1750768770; h=from:from:sender:reply-to:subject:subject:date:date: message-id:message-id:to:to:cc:cc:mime-version:mime-version: content-type:content-transfer-encoding:content-transfer-encoding: in-reply-to:in-reply-to:references:references:dkim-signature; bh=fSXvnx+fgvE5GXly6XPTqjm0PPBXVKwM6V3gQyXQXoE=; b=AqBaUoaBcLQfc+KZp+jJBjRESYcTpaAAYRtKXrP2gKgYcPW60d2pRYasH4iWXSY31v+TOh wt9PhjkvwiYPjnIcbjj6l0M33gf/4vymyemuEnOPmNAH6Bd/FyurwHTcFPZiP3lsSyOpI3 k0mTeH8p05Bu6hG4VJEnP/Lf1L7eKRM= ARC-Authentication-Results: i=1; imf30.hostedemail.com; dkim=pass header.d=konsulko.se header.s=rsa1 header.b=a3Nn7EtL; dkim=pass header.d=konsulko.se header.s=ed1 header.b=WLIrf7Nb; spf=none (imf30.hostedemail.com: domain of vitaly.wool@konsulko.se has no SPF policy when checking 46.30.212.10) smtp.mailfrom=vitaly.wool@konsulko.se; dmarc=none ARC-Seal: i=1; s=arc-20220608; d=hostedemail.com; t=1750768770; a=rsa-sha256; cv=none; b=DUzrUqgbdpCfeNIw2Iq3vCDj4IkxRW3YPXO7wNHjsp+bUMyBCrZki7iEUT0euMSc8XeGsL 8xBRyn5BHGUIdRRxfd4CQ6BVjmpwIFmxEslIwf3+ysZ1mMYcF/zyN9Wsi/7hj9IMDue2yr GQPl+9scQgsfciPa/zhYnd3a9bj42R8= DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; t=1750768769; x=1751373569; d=konsulko.se; s=rsa1; h=content-transfer-encoding:mime-version:references:in-reply-to:message-id:date: subject:cc:to:from:from; bh=fSXvnx+fgvE5GXly6XPTqjm0PPBXVKwM6V3gQyXQXoE=; b=a3Nn7EtLsaOwYdMf6KyrQwD47cf6wvzuX1rboSdseCHX21hDhh5kWLB6FG+6/cJvgbczTwXwRatcu AqFRTKSsqdLOlGMX0BEtLNuj41NdIr4ECp+G0oRz58t4DTnT4Mz1lCWpSqcQwR87KobvcVd/+l87Nb uqoq9TlVWZ32ZHHtfqZy0oZX0h8CdGRjxgoUcDodykxBsNQr6CAP/VWJylirjZhVSkrS26MluP67AX A2hVluvWoGjl2+4ODBwVnxBEs2VYFeFLePFGFN/BVAUSquDWQrqLy5mLe4oKdLf+bU5/28hDkm34UA Sb0alqOsre9rQkFyQ+s1jqhoWX6LwnA== DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed; t=1750768769; x=1751373569; d=konsulko.se; s=ed1; h=content-transfer-encoding:mime-version:references:in-reply-to:message-id:date: subject:cc:to:from:from; bh=fSXvnx+fgvE5GXly6XPTqjm0PPBXVKwM6V3gQyXQXoE=; b=WLIrf7NbKfORLJhUNOxmbzsngSnWr4yfo8P110ZeLsm0QbKm7vIlCMNPe/eFPgxGRwuSEV2s7Vb23 qRrfrv1Cw== X-HalOne-ID: 4374bbea-50f8-11f0-afc3-d7c209f8bd06 Received: from slottsdator.home (host-90-238-19-233.mobileonline.telia.com [90.238.19.233]) by mailrelay5.pub.mailoutpod3-cph3.one.com (Halon) with ESMTPSA id 4374bbea-50f8-11f0-afc3-d7c209f8bd06; Tue, 24 Jun 2025 12:39:28 +0000 (UTC) From: Vitaly Wool To: linux-mm@kvack.org Cc: akpm@linux-foundation.org, linux-kernel@vger.kernel.org, Uladzislau Rezki , Danilo Krummrich , Alice Ryhl , rust-for-linux@vger.kernel.org, Vitaly Wool Subject: [PATCH v2 2/2] rust: support large align and NUMA ids in allocs Date: Tue, 24 Jun 2025 14:39:22 +0200 Message-Id: <20250624123922.3258215-1-vitaly.wool@konsulko.se> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20250624123743.3258032-1-vitaly.wool@konsulko.se> References: <20250624123743.3258032-1-vitaly.wool@konsulko.se> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Rspamd-Queue-Id: 23D0380013 X-Rspam-User: X-Rspamd-Server: rspam07 X-Stat-Signature: iks6h9ai5j75jfnmoh4xb1xfz4w6mh89 X-HE-Tag: 1750768769-961885 X-HE-Meta: U2FsdGVkX18tpbjEGLYbn4zX5FH4fCSTcz128XLEE1jb1EktpclVAz1dNHkj5AOMWvSUtnuLTQ3je5Y+6D9ePaVGxbNAOTnht+l0cjn2mE+ROqyDvUWWGXKe8Xvn070zKVNkpBYjlRNNMcTBQIx3Nh3xvAHOs9QAjZ1/y/74YR+1VK/MVeg1yzTJ5xAuXcLbT1tOGev5PNG23t6405takljRn8yFnEd5UW+9kDCzYF8qHFwaPxX+zQLRHTC7rBo1UKRk2MlqnTqiCpNMntf6cqJIcxv7Fhkk0cdTdTRJ7ObnTAPySCpuOKrfPxoiHLe8uJ3hZ/qCOh+5dgv7rSM/WgpFJk9OwfSNrjK69sHvxLDT37vxUnt8HmA22eJmP3Mli3sw1/i/x+CbtQ4atzzE+ZoeuTpCF8C4bQQWnRNZV+p8ZTkhO87RaOQmcg51tg3NEXIGUOZhwI+PZdIo7EdVZyNymvE/IgZYoE7RFLZllL9WYbNWQd79XBdLyu2npXYxScL2MUQIc+KBIMW7rflYJsNBvD9ANLwmbgeYB/EngrMPD+MUnY4irw1Lm75xRwVM5XQH6ZQAe35HCHiXU6Y5M51vz1rFNO+PZAyzjCvT0wiVq8buwBytphsdsNjXWbsjhz0kuKRQspCYmaH+K8HDgL7bNY73CKnK7glbuqiKKbJu9DY+AnvK9o+S+KhJwzXOZloye9RPYGUMdfoMpsX+RpMZfMrOoZPaNC8hIpbRkPCymoHVFr/zAQWLUlOD3N3O9p7UCm77sjgbVzieQLr4qGyVX9/1H38U2M5N7GbBPP6gsYE/Fn+SjFh5eb/YvmybL2UD/xaFo0pc5BFRfhDOts9sSNSt+hXsb5tuIWSpUTeT58rXh5QOGYzI8hUtakxfb3+nYXKzBrC1H1neUa6Q7RODIhyZ377MywzlnatL2JCMLjTTzKJchoyn7+/PYkeRSlAtkUiuraPHeyyHuWG AJIm41V4 6arzkvVLUzccMYyPs80lU2bCNfeDo4ywl7Ry2j7Nc0Ce7VOfoKiwnEBVqnN5y8j1APX9bXdwAw/eSoTQWV/EVhGgt/lkxcs0D59lyjUNB4xfkn8HKrCkdDLOQpEEJdvYna18gPaCvVng6h0H68fAs9EcqvYURTnefRiqmIIQVjL8HYIXzVakEUS+HnKM5Q78ErD8wytnYeSNwWhiA7TD8ALvT+hvADMuHO9b4jLEBjiLExQo= X-Bogosity: Ham, tests=bogofilter, spamicity=0.000000, version=1.2.4 Sender: owner-linux-mm@kvack.org Precedence: bulk X-Loop: owner-majordomo@kvack.org List-ID: List-Subscribe: List-Unsubscribe: Add support for large (> PAGE_SIZE) alignments in Rust allocators (Kmalloc support for large alignments is limited to the requested size, which is a reasonable limitation anyway). Besides, add support for NUMA id to Vmalloc. Signed-off-by: Vitaly Wool --- rust/helpers/slab.c | 8 +++++-- rust/helpers/vmalloc.c | 4 ++-- rust/kernel/alloc.rs | 28 ++++++++++++++++++++++-- rust/kernel/alloc/allocator.rs | 40 +++++++++++++++++++--------------- rust/kernel/alloc/kvec.rs | 3 ++- 5 files changed, 59 insertions(+), 24 deletions(-) diff --git a/rust/helpers/slab.c b/rust/helpers/slab.c index a842bfbddcba..221c517f57a1 100644 --- a/rust/helpers/slab.c +++ b/rust/helpers/slab.c @@ -3,13 +3,17 @@ #include void * __must_check __realloc_size(2) -rust_helper_krealloc(const void *objp, size_t new_size, gfp_t flags) +rust_helper_krealloc(const void *objp, size_t new_size, unsigned long align, gfp_t flags, int nid) { + if (WARN_ON(new_size & (align - 1))) + return NULL; return krealloc(objp, new_size, flags); } void * __must_check __realloc_size(2) -rust_helper_kvrealloc(const void *p, size_t size, gfp_t flags) +rust_helper_kvrealloc(const void *p, size_t size, unsigned long align, gfp_t flags, int nid) { + if (WARN_ON(size & (align - 1))) + return NULL; return kvrealloc(p, size, flags); } diff --git a/rust/helpers/vmalloc.c b/rust/helpers/vmalloc.c index 80d34501bbc0..9131279222fa 100644 --- a/rust/helpers/vmalloc.c +++ b/rust/helpers/vmalloc.c @@ -3,7 +3,7 @@ #include void * __must_check __realloc_size(2) -rust_helper_vrealloc(const void *p, size_t size, gfp_t flags) +rust_helper_vrealloc_node(const void *p, size_t size, unsigned long align, gfp_t flags, int node) { - return vrealloc(p, size, flags); + return vrealloc_node(p, size, align, flags, node); } diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index 2e377c52fa07..12a723bf6092 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -161,7 +161,30 @@ pub unsafe trait Allocator { fn alloc(layout: Layout, flags: Flags) -> Result, AllocError> { // SAFETY: Passing `None` to `realloc` is valid by its safety requirements and asks for a // new memory allocation. - unsafe { Self::realloc(None, layout, Layout::new::<()>(), flags) } + unsafe { Self::realloc(None, layout, Layout::new::<()>(), flags, None) } + } + + /// Allocate memory based on `layout`, `flags` and `nid`. + /// + /// On success, returns a buffer represented as `NonNull<[u8]>` that satisfies the layout + /// constraints (i.e. minimum size and alignment as specified by `layout`). + /// + /// This function is equivalent to `realloc` when called with `None`. + /// + /// # Guarantees + /// + /// When the return value is `Ok(ptr)`, then `ptr` is + /// - valid for reads and writes for `layout.size()` bytes, until it is passed to + /// [`Allocator::free`] or [`Allocator::realloc`], + /// - aligned to `layout.align()`, + /// + /// Additionally, `Flags` are honored as documented in + /// . + fn alloc_node(layout: Layout, flags: Flags, nid: Option) + -> Result, AllocError> { + // SAFETY: Passing `None` to `realloc` is valid by its safety requirements and asks for a + // new memory allocation. + unsafe { Self::realloc(None, layout, Layout::new::<()>(), flags, nid) } } /// Re-allocate an existing memory allocation to satisfy the requested `layout`. @@ -201,6 +224,7 @@ unsafe fn realloc( layout: Layout, old_layout: Layout, flags: Flags, + nid: Option, ) -> Result, AllocError>; /// Free an existing memory allocation. @@ -216,7 +240,7 @@ unsafe fn free(ptr: NonNull, layout: Layout) { // SAFETY: The caller guarantees that `ptr` points at a valid allocation created by this // allocator. We are passing a `Layout` with the smallest possible alignment, so it is // smaller than or equal to the alignment previously used with this allocation. - let _ = unsafe { Self::realloc(Some(ptr), Layout::new::<()>(), layout, Flags(0)) }; + let _ = unsafe { Self::realloc(Some(ptr), Layout::new::<()>(), layout, Flags(0), None) }; } } diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index aa2dfa9dca4c..91b36e128b92 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -58,7 +58,8 @@ fn aligned_size(new_layout: Layout) -> usize { /// /// One of the following: `krealloc`, `vrealloc`, `kvrealloc`. struct ReallocFunc( - unsafe extern "C" fn(*const crate::ffi::c_void, usize, u32) -> *mut crate::ffi::c_void, + unsafe extern "C" fn(*const crate::ffi::c_void, usize, usize, u32, i32) + -> *mut crate::ffi::c_void, ); impl ReallocFunc { @@ -66,7 +67,7 @@ impl ReallocFunc { const KREALLOC: Self = Self(bindings::krealloc); // INVARIANT: `vrealloc` satisfies the type invariants. - const VREALLOC: Self = Self(bindings::vrealloc); + const VREALLOC: Self = Self(bindings::vrealloc_node); // INVARIANT: `kvrealloc` satisfies the type invariants. const KVREALLOC: Self = Self(bindings::kvrealloc); @@ -87,6 +88,7 @@ unsafe fn call( layout: Layout, old_layout: Layout, flags: Flags, + nid: Option, ) -> Result, AllocError> { let size = aligned_size(layout); let ptr = match ptr { @@ -100,6 +102,11 @@ unsafe fn call( None => ptr::null(), }; + let c_nid = match nid { + None => bindings::NUMA_NO_NODE, + Some(n) => n, + }; + // SAFETY: // - `self.0` is one of `krealloc`, `vrealloc`, `kvrealloc` and thus only requires that // `ptr` is NULL or valid. @@ -110,7 +117,7 @@ unsafe fn call( // - Those functions provide the guarantees of this function. let raw_ptr = unsafe { // If `size == 0` and `ptr != NULL` the memory behind the pointer is freed. - self.0(ptr.cast(), size, flags.0).cast() + self.0(ptr.cast(), size, layout.align(), flags.0, c_nid).cast() }; let ptr = if size == 0 { @@ -134,9 +141,10 @@ unsafe fn realloc( layout: Layout, old_layout: Layout, flags: Flags, + _nid: Option, ) -> Result, AllocError> { // SAFETY: `ReallocFunc::call` has the same safety requirements as `Allocator::realloc`. - unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags) } + unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags, None) } } } @@ -151,16 +159,11 @@ unsafe fn realloc( layout: Layout, old_layout: Layout, flags: Flags, + nid: Option, ) -> Result, AllocError> { - // TODO: Support alignments larger than PAGE_SIZE. - if layout.align() > bindings::PAGE_SIZE { - pr_warn!("Vmalloc does not support alignments larger than PAGE_SIZE yet.\n"); - return Err(AllocError); - } - // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously // allocated with this `Allocator`. - unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags) } + unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags, nid) } } } @@ -175,15 +178,18 @@ unsafe fn realloc( layout: Layout, old_layout: Layout, flags: Flags, + _nid: Option, ) -> Result, AllocError> { - // TODO: Support alignments larger than PAGE_SIZE. - if layout.align() > bindings::PAGE_SIZE { - pr_warn!("KVmalloc does not support alignments larger than PAGE_SIZE yet.\n"); - return Err(AllocError); - } + // if the caller wants to have alignment bigger than PAGE_SIZE + // it's only vmalloc we can offer to satisfy that request + let alloc_func = if layout.align() > bindings::PAGE_SIZE { + ReallocFunc::VREALLOC + } else { + ReallocFunc::KVREALLOC + }; // SAFETY: If not `None`, `ptr` is guaranteed to point to valid memory, which was previously // allocated with this `Allocator`. - unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flags) } + unsafe { alloc_func.call(ptr, layout, old_layout, flags, None) } } } diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index 1a0dd852a468..ef4f977ba012 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -633,6 +633,7 @@ pub fn reserve(&mut self, additional: usize, flags: Flags) -> Result<(), AllocEr layout.into(), self.layout.into(), flags, + None, )? }; @@ -1058,7 +1059,7 @@ pub fn collect(self, flags: Flags) -> Vec { // the type invariant to be smaller than `cap`. Depending on `realloc` this operation // may shrink the buffer or leave it as it is. ptr = match unsafe { - A::realloc(Some(buf.cast()), layout.into(), old_layout.into(), flags) + A::realloc(Some(buf.cast()), layout.into(), old_layout.into(), flags, None) } { // If we fail to shrink, which likely can't even happen, continue with the existing // buffer. -- 2.39.2