From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from kanga.kvack.org (kanga.kvack.org [205.233.56.17]) by smtp.lore.kernel.org (Postfix) with ESMTP id 15554C282D0 for ; Tue, 4 Mar 2025 11:51:53 +0000 (UTC) Received: by kanga.kvack.org (Postfix) id 3C444280004; Tue, 4 Mar 2025 06:51:51 -0500 (EST) Received: by kanga.kvack.org (Postfix, from userid 40) id 34AF9280005; Tue, 4 Mar 2025 06:51:51 -0500 (EST) X-Delivered-To: int-list-linux-mm@kvack.org Received: by kanga.kvack.org (Postfix, from userid 63042) id 1028F280004; Tue, 4 Mar 2025 06:51:51 -0500 (EST) X-Delivered-To: linux-mm@kvack.org Received: from relay.hostedemail.com (smtprelay0016.hostedemail.com [216.40.44.16]) by kanga.kvack.org (Postfix) with ESMTP id DCB86280001 for ; Tue, 4 Mar 2025 06:51:50 -0500 (EST) Received: from smtpin02.hostedemail.com (a10.router.float.18 [10.200.18.1]) by unirelay03.hostedemail.com (Postfix) with ESMTP id 8D2BAA8651 for ; Tue, 4 Mar 2025 11:51:50 +0000 (UTC) X-FDA: 83183704380.02.892D7DE Received: from mail-wm1-f74.google.com (mail-wm1-f74.google.com [209.85.128.74]) by imf14.hostedemail.com (Postfix) with ESMTP id 9355A100002 for ; Tue, 4 Mar 2025 11:51:48 +0000 (UTC) Authentication-Results: imf14.hostedemail.com; dkim=pass header.d=google.com header.s=20230601 header.b=BA2SRr1N; dmarc=pass (policy=reject) header.from=google.com; spf=pass (imf14.hostedemail.com: domain of 3U-nGZwkKCIYkvsmo18rvqyyqvo.mywvsx47-wwu5kmu.y1q@flex--aliceryhl.bounces.google.com designates 209.85.128.74 as permitted sender) smtp.mailfrom=3U-nGZwkKCIYkvsmo18rvqyyqvo.mywvsx47-wwu5kmu.y1q@flex--aliceryhl.bounces.google.com ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=hostedemail.com; s=arc-20220608; t=1741089108; h=from:from:sender:reply-to:subject:subject:date:date: message-id:message-id:to:to:cc:cc:mime-version:mime-version: content-type:content-type:content-transfer-encoding: in-reply-to:in-reply-to:references:references:dkim-signature; bh=0QIrGJtgJ9WrxiYI0Ux/yttKhtDMbTNlh5iJ2ulSDEw=; b=PmhrOZldX3Q/0b2KWAb3sSMChwEgmiBBhIGqG7HiTd02xQMC/J9doxq//6WQY1Qf6tjgZD ecV8LJZTA3pgb0qjEj+a6slh2zIvhDSwpJjQBSzY8t1ZKXOXRZkYjLibmYUnKJtnRMcpF0 bADGgaQM/CeiBQsE0fuEMudZO3zXqd4= ARC-Authentication-Results: i=1; imf14.hostedemail.com; dkim=pass header.d=google.com header.s=20230601 header.b=BA2SRr1N; dmarc=pass (policy=reject) header.from=google.com; spf=pass (imf14.hostedemail.com: domain of 3U-nGZwkKCIYkvsmo18rvqyyqvo.mywvsx47-wwu5kmu.y1q@flex--aliceryhl.bounces.google.com designates 209.85.128.74 as permitted sender) smtp.mailfrom=3U-nGZwkKCIYkvsmo18rvqyyqvo.mywvsx47-wwu5kmu.y1q@flex--aliceryhl.bounces.google.com ARC-Seal: i=1; s=arc-20220608; d=hostedemail.com; t=1741089108; a=rsa-sha256; cv=none; b=LDmHwULwkiOoiBKBuZlXv8aHnsnClpTMBfnATIE2+wLjkM13JpAqAK3lt0vGAZllfC9knF PYWB1doLubOjyLjNxjORqaWJG/+WV7OdrE+kcU+oEzheAQjeSNqowffBxQkQbkrpqJ35Mz U/w585mZkNxKs0mikq46p0yJhSSOHz0= Received: by mail-wm1-f74.google.com with SMTP id 5b1f17b1804b1-43bca561111so5286115e9.3 for ; Tue, 04 Mar 2025 03:51:48 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20230601; t=1741089107; x=1741693907; darn=kvack.org; h=cc:to:from:subject:message-id:references:mime-version:in-reply-to :date:from:to:cc:subject:date:message-id:reply-to; bh=0QIrGJtgJ9WrxiYI0Ux/yttKhtDMbTNlh5iJ2ulSDEw=; b=BA2SRr1Nq57jHhaCJ5a2RRP78/WQM6Li9I6cF8akGigO7ehGp9Zv6WoUiwEQ9BOZjs u42zyqtYi+v5QojFyp2ojvQNclbQtUjDeqWkXAnCpXNFX+e9sYsQFqpk4MpzNdeiTVd9 sQgrs50U1cLkS4WbMkjiF4yoDxh7zQ24G6Z51KrT4ikqB4RAM/43t81KKv/pyGoF2PJ9 OM0ld6Dx/YanMeMbDpXDBQD3ss6S7OXcIi3cfqEHqgk6yORhO2eG2J+sKHe+VGcKuVQr G5gSC23BXy0NbfSIEUmmaNEaOgZIpJ9rup0KtYfFo6ph/JiGD0B9P+Hfk2ez7idawecB DsFA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1741089107; x=1741693907; h=cc:to:from:subject:message-id:references:mime-version:in-reply-to :date:x-gm-message-state:from:to:cc:subject:date:message-id:reply-to; bh=0QIrGJtgJ9WrxiYI0Ux/yttKhtDMbTNlh5iJ2ulSDEw=; b=qgBiYswDqxB3KbWMI53Htu0CLTNBEfNnqVifNYO6buG76KKKkQT20Vzxs8qY2emhqQ AXFbGxBnocq6HafGbz30UZfFEW+ovCae9qelPA95Gxwic6/csfcI+S3VflNgDJ11hc1C H9z1/BMCOn+2qZ+8ec/NfYmXthzrjQwjjW0ZkUA2+PZ1D2J4XQFIEHNIdsPx7Ed56+hY IAsJwwW8IIxcbSJqXTvR5tNKgkct/7v3+1PyEePxI2KmXAkts3XwrORf+wvIGi2dA1Iv d8GzdVPPMp4DgAJ0Vl/cxV2OLKfr/fBWeDCkxf+afa3HN17r0lSgvR3NL9EFIC2J8QSq KVTw== X-Forwarded-Encrypted: i=1; AJvYcCWF3HAoCh3GSR/eE0ixUYf0koriPyf0x5x6Fe03JV/FoJUh75/BIySFqfhh1oCTbbP1/UWODj4F2w==@kvack.org X-Gm-Message-State: AOJu0YyGrTZ4rpRqk2p9XGfpo7pFQJOBXEBnvi3m3/MwoYkYLc4SKZ3J g/YtGnLsCS2czrOB2asoBO5QR8op4iWoWy0aBOrmPMWoGyZHMkSCx9SfCTZ4OHWqjfI/1cg5czH DCPOR4HUS8G4LhQ== X-Google-Smtp-Source: AGHT+IH6uDE7kFpKO1QVfRSRVP5JbMp4n3OVboRInnn7ZaWP8qpMkUMUPL+jOVqAFfAOyRUiCPKTyKLwyY9/ckM= X-Received: from wmsp28.prod.google.com ([2002:a05:600c:1d9c:b0:439:47d8:90a7]) (user=aliceryhl job=prod-delivery.src-stubby-dispatcher) by 2002:a05:600c:1d02:b0:43b:bf6f:6eef with SMTP id 5b1f17b1804b1-43bbf6f718emr59000905e9.26.1741089107396; Tue, 04 Mar 2025 03:51:47 -0800 (PST) Date: Tue, 04 Mar 2025 11:49:55 +0000 In-Reply-To: <20250304-vma-v15-0-5c641b31980a@google.com> Mime-Version: 1.0 References: <20250304-vma-v15-0-5c641b31980a@google.com> X-Developer-Key: i=aliceryhl@google.com; a=openpgp; fpr=49F6C1FAA74960F43A5B86A1EE7A392FDE96209F X-Developer-Signature: v=1; a=openpgp-sha256; l=4299; i=aliceryhl@google.com; h=from:subject:message-id; bh=CJ1T+zPNUeGdU4wvXMNNlDKLq0tzZUD5m4mvMkDIXAA=; b=owEBbQKS/ZANAwAKAQRYvu5YxjlGAcsmYgBnxulEaSGw/31FVCS5/5VSXxZoaOUjXDpT1LrRR tDfSE0L5QqJAjMEAAEKAB0WIQSDkqKUTWQHCvFIvbIEWL7uWMY5RgUCZ8bpRAAKCRAEWL7uWMY5 RsEuD/4o/rKRXqBbfkuRQLFPGDy/juNfWz4I75JbEw2BitMOV9q6peNiHW/KVI0pzGPicY3eBFz zFOlfAZVOz9Y8l/OVFYrhFBYxNweJRZ4iP6tjagJGeEmnw3bNPINVGHYCviyuLhkN/f/Rf6PCLH WTZ+Qp225X5ZWQbT3lnG6OpqgCmVY4sGVkcyEhQy+MYMuCvGdSorUPTKDegf0lsWHnugChI933w ZQDhHKTlTVFn2qDoN+6A331S+guYm3vpVzcKQwEmNttw/j+EAWy65W9bJ9ohghPmwRtEXOsCN7w JWwaxbH8G2s0fdMV3k7PtY7X6pCsbgBWiZpTpzOn1dz5mX1QI9rNIq7SP8XS2jqlMIGZeOu+Ijh H7WpdZLUSHVXkFsk2ASi3a0DySn2MVw6U1W8yznu+0nukUfrZCzIFcyJOv77VGIrzaUg3CN6j/4 bT16j/9m3o0cuhLU7f6QhFRn3B0GeWBWtXn2y6627GG0V2xUTj/7Aomb7k2kKHlnbbuU7JfM4lD fDSeCVVdkUjfJF9PTT3ZAag2v8yD7/+AoTRyHVUg/JfCtq4eQB5VJ9jU966fx2AbSFcCNxPN15G EeNWMWKEXC8vRNy+nEw6XEMrOUknXmBnhDXEZi13FCs4NkyMMB8F4RsSaYKsPMbl2nHgDSMWNDe JxsF/6psbJFeXrg== X-Mailer: b4 0.14.2 Message-ID: <20250304-vma-v15-4-5c641b31980a@google.com> Subject: [PATCH v15 4/9] mm: rust: add lock_vma_under_rcu From: Alice Ryhl To: Miguel Ojeda , Matthew Wilcox , Lorenzo Stoakes , Vlastimil Babka , John Hubbard , "Liam R. Howlett" , Andrew Morton , Greg Kroah-Hartman , Arnd Bergmann , Jann Horn , Suren Baghdasaryan Cc: Alex Gaynor , Boqun Feng , Gary Guo , "=?utf-8?q?Bj=C3=B6rn_Roy_Baron?=" , Benno Lossin , Andreas Hindborg , Trevor Gross , Linus Torvalds , linux-kernel@vger.kernel.org, linux-mm@kvack.org, rust-for-linux@vger.kernel.org, Alice Ryhl Content-Type: text/plain; charset="utf-8" X-Rspamd-Server: rspam04 X-Rspamd-Queue-Id: 9355A100002 X-Rspam-User: X-Stat-Signature: scc78ws349dy6wy5pmtw5yya61kd9jy5 X-HE-Tag: 1741089108-742801 X-HE-Meta: U2FsdGVkX19cb1nz8fJb3pF4Bd0ARMrhKuKVAokVqwsDZ5XNHt+vjO95AQNEOzKQ5yJAu6AcwRWw+KVQUWzc5bVNBo71b7alb9e+C2vc2YJycpevwmyAHMWwxnqAjB8kj6i7e5SDd/kI5UMHbpbAmabR9dlOEKMYX0DGM9paWWyZXFDkXiXg9aIk+nONaDrVU0AcShfrAWbJnu+AdSrt7AvPTw3jtSv2knwQoonhD3BOUlGaLv7W9Vf7Cfsdkvq2fVESe/1WkbgsiX+jLEz/2Kfyn1388uQUw+dNcT70BOcgLqIOQGdH7Y67OwiJLIQNSkWe9EJ9X17SHroXGLE307itdeekPWwU0m73s9op6rqfiPkvhI4QGqPneETulhlxTj4mwe//KZQj5UT5CisRmpFktpqiz8U2VWridaEpB99tGuBxpxEikLa89rh6KTWuJZURfqBiKKHKD+Mx7jwjk85so2SdHtA94k5NUFqo6zJW/Mm/HCfTB2qlND/d5Fkdj1QR5jHUimU6ESyulCP/1slTyNaHQho97Yg1POIlomU6UpsbNFFRLheD5lfnv344M0D6/jhhD++EkdosRsO/V6UqKb8SaID+TQOrkyuZm7MNXyNzq1m4MBoNLOnNMh/7M35AvFd39n8YLizLeIM8iYJh32ArFuKnBAy6p6CGt220IccZG4/O4Yg+bSOCMCnZc9tzt3HN0h/SVFttAETf8EyMOdVxKVwaeUTxyMrsFcPF6zTgn4DZkjYXqepHNIGXCaOEhyQahcOw6o1/w5PHGuTiKuiaiKRAJb/tDCSJnQ46V5gWcoEYhxyot+PT2KfJhm/6Wy854HbX3WgdYiMlVYd29FaJDlVlFAzAlD8vZhSyvY2t3dDzkk4J606QE9M70R9QrhKEhxEkhN1B0zaJGtY4GgeM/lMVgEIoLLtd32+W37yckLjWH/l3EsTaR/z3VLR5cTomSoOeeB3ck1q bN1nM+AK RK/jEmAkorHiNqJPC530pnJaTqpdrXBJVOSPFf7fUmc7uYSsXnbTlN3GZCuZgUssddaRvuUcs22pkG2UL+d9XZBgn3gjNGtHKqc5QAgmzduvN8Q/1OAW2mNV2qU6YHohMEKfunva2BAewWKmAJe/RtC0YlN01sOS/9P0/GwIv0alt/ttRknr1na17bFr9RY1YCjKXUULmMi0ilYPeK2Wk8GZK6suPHXdjDhUKbo1wyeh6uk7MDJjSZtj7QEZyXjxf4naeI9ax36I2q3sYMdIXxgfWFbKsxI4Rx/2ieX2scjQYT4axi0AJWdb421JCb6nl+WGauV2yU6h2cKzDexmfMn3thIIOM0LSvKOtmDGnyXAau08TXt4sjOte+h4NaTq7fFGiEUWdtOfyGpm66j7KMNS2rAR/hxiCVzWJRzQa6VWy280yjxawZa3otMcFZqMMn0VTZCZlvxrpgiW7yBGjYb5WY326TBM8IjlbTlo7YuoQkS6sDrFCwgMUXQmlJ24De6Pf2LVeTSr0ZNODbg5Nku3jI4VwqU5TJR9rArzzCk336Brm4QxQozl2zxaU9kkrhiwi8aZWiBf/FQKR7yxlxu2zGrkTBq1w/hmvxA5B7LpF3x5UD8atIFxPkISTyn6tPgazfIPpJNmQhmyfEpIcTsWGVsPYX+r9fQFBflJ7tGUPtWwd62EWcCbM5YUgVN61qcHadsp5VzSBGzd17luwPgHhNAAAtf5urJAV/yBu2d5hD6xaOUQqgOu3cOUpWbgOLYSsdtnOz+h0s7je8rW7aKxlGM9Lp7hhoNOW X-Bogosity: Ham, tests=bogofilter, spamicity=0.334300, version=1.2.4 Sender: owner-linux-mm@kvack.org Precedence: bulk X-Loop: owner-majordomo@kvack.org List-ID: List-Subscribe: List-Unsubscribe: Currently, the binder driver always uses the mmap lock to make changes to its vma. Because the mmap lock is global to the process, this can involve significant contention. However, the kernel has a feature called per-vma locks, which can significantly reduce contention. For example, you can take a vma lock in parallel with an mmap write lock. This is important because contention on the mmap lock has been a long-term recurring challenge for the Binder driver. This patch introduces support for using `lock_vma_under_rcu` from Rust. The Rust Binder driver will be able to use this to reduce contention on the mmap lock. Acked-by: Lorenzo Stoakes Acked-by: Liam R. Howlett Reviewed-by: Jann Horn Reviewed-by: Andreas Hindborg Reviewed-by: Gary Guo Signed-off-by: Alice Ryhl --- rust/helpers/mm.c | 5 +++++ rust/kernel/mm.rs | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/rust/helpers/mm.c b/rust/helpers/mm.c index 7b72eb065a3e1173c920f02a440053cf6e93814e..81b510c96fd2692dcb7ab4705f790bd3a41a630e 100644 --- a/rust/helpers/mm.c +++ b/rust/helpers/mm.c @@ -43,3 +43,8 @@ struct vm_area_struct *rust_helper_vma_lookup(struct mm_struct *mm, { return vma_lookup(mm, addr); } + +void rust_helper_vma_end_read(struct vm_area_struct *vma) +{ + vma_end_read(vma); +} diff --git a/rust/kernel/mm.rs b/rust/kernel/mm.rs index 8b19dde249789b65d7b92159d131cbf38ba08149..618aa48e00a442c7790dc92085932f5aa6693729 100644 --- a/rust/kernel/mm.rs +++ b/rust/kernel/mm.rs @@ -18,6 +18,7 @@ use core::{ops::Deref, ptr::NonNull}; pub mod virt; +use virt::VmaRef; /// A wrapper for the kernel's `struct mm_struct`. /// @@ -160,6 +161,36 @@ pub unsafe fn from_raw<'a>(ptr: *const bindings::mm_struct) -> &'a MmWithUser { unsafe { &*ptr.cast() } } + /// Attempt to access a vma using the vma read lock. + /// + /// This is an optimistic trylock operation, so it may fail if there is contention. In that + /// case, you should fall back to taking the mmap read lock. + /// + /// When per-vma locks are disabled, this always returns `None`. + #[inline] + pub fn lock_vma_under_rcu(&self, vma_addr: usize) -> Option> { + #[cfg(CONFIG_PER_VMA_LOCK)] + { + // SAFETY: Calling `bindings::lock_vma_under_rcu` is always okay given an mm where + // `mm_users` is non-zero. + let vma = unsafe { bindings::lock_vma_under_rcu(self.as_raw(), vma_addr) }; + if !vma.is_null() { + return Some(VmaReadGuard { + // SAFETY: If `lock_vma_under_rcu` returns a non-null ptr, then it points at a + // valid vma. The vma is stable for as long as the vma read lock is held. + vma: unsafe { VmaRef::from_raw(vma) }, + _nts: NotThreadSafe, + }); + } + } + + // Silence warnings about unused variables. + #[cfg(not(CONFIG_PER_VMA_LOCK))] + let _ = vma_addr; + + None + } + /// Lock the mmap read lock. #[inline] pub fn mmap_read_lock(&self) -> MmapReadGuard<'_> { @@ -230,3 +261,32 @@ fn drop(&mut self) { unsafe { bindings::mmap_read_unlock(self.mm.as_raw()) }; } } + +/// A guard for the vma read lock. +/// +/// # Invariants +/// +/// This `VmaReadGuard` guard owns the vma read lock. +pub struct VmaReadGuard<'a> { + vma: &'a VmaRef, + // `vma_end_read` must be called on the same thread as where the lock was taken + _nts: NotThreadSafe, +} + +// Make all `VmaRef` methods available on `VmaReadGuard`. +impl Deref for VmaReadGuard<'_> { + type Target = VmaRef; + + #[inline] + fn deref(&self) -> &VmaRef { + self.vma + } +} + +impl Drop for VmaReadGuard<'_> { + #[inline] + fn drop(&mut self) { + // SAFETY: We hold the read lock by the type invariants. + unsafe { bindings::vma_end_read(self.vma.as_ptr()) }; + } +} -- 2.48.1.711.g2feabab25a-goog