From: Andreas Hindborg <a.hindborg@kernel.org>
To: "Boqun Feng" <boqun.feng@gmail.com>,
"Jens Axboe" <axboe@kernel.dk>, "Miguel Ojeda" <ojeda@kernel.org>,
"Gary Guo" <gary@garyguo.net>,
"Björn Roy Baron" <bjorn3_gh@protonmail.com>,
"Benno Lossin" <lossin@kernel.org>,
"Alice Ryhl" <aliceryhl@google.com>,
"Trevor Gross" <tmgross@umich.edu>,
"Danilo Krummrich" <dakr@kernel.org>,
"FUJITA Tomonori" <fujita.tomonori@gmail.com>,
"Frederic Weisbecker" <frederic@kernel.org>,
"Lyude Paul" <lyude@redhat.com>,
"Thomas Gleixner" <tglx@kernel.org>,
"Anna-Maria Behnsen" <anna-maria@linutronix.de>,
"John Stultz" <jstultz@google.com>,
"Stephen Boyd" <sboyd@kernel.org>,
"Lorenzo Stoakes" <lorenzo.stoakes@oracle.com>,
"Liam R. Howlett" <Liam.Howlett@oracle.com>
Cc: linux-block@vger.kernel.org, rust-for-linux@vger.kernel.org,
linux-kernel@vger.kernel.org, linux-mm@kvack.org,
Andreas Hindborg <a.hindborg@kernel.org>
Subject: [PATCH 51/79] block: rust: add polled completion support
Date: Mon, 16 Feb 2026 00:35:38 +0100 [thread overview]
Message-ID: <20260216-rnull-v6-19-rc5-send-v1-51-de9a7af4b469@kernel.org> (raw)
In-Reply-To: <20260216-rnull-v6-19-rc5-send-v1-0-de9a7af4b469@kernel.org>
Add support for polled I/O completion to the Rust block layer bindings.
This includes the `poll` callback in `Operations` and the
`IoCompletionBatch` type for batched completions.
The `poll` callback is invoked by the block layer when polling for
completed requests on poll queues. Drivers can use `IoCompletionBatch`
to batch multiple completions efficiently.
Signed-off-by: Andreas Hindborg <a.hindborg@kernel.org>
---
drivers/block/rnull/rnull.rs | 1 +
rust/helpers/blk.c | 7 +++
rust/kernel/block/mq.rs | 4 +-
rust/kernel/block/mq/operations.rs | 104 +++++++++++++++++++++++++++++++++++--
rust/kernel/block/mq/request.rs | 5 ++
5 files changed, 117 insertions(+), 4 deletions(-)
diff --git a/drivers/block/rnull/rnull.rs b/drivers/block/rnull/rnull.rs
index 48b2bd598304c..765bbc8101d10 100644
--- a/drivers/block/rnull/rnull.rs
+++ b/drivers/block/rnull/rnull.rs
@@ -733,6 +733,7 @@ fn queue_rq(
this: ArcBorrow<'_, Self>,
rq: Owned<mq::IdleRequest<Self>>,
_is_last: bool,
+ _is_poll: bool,
) -> BlkResult {
if this.bandwidth_limit != 0 {
if !this.bandwidth_timer.active() {
diff --git a/rust/helpers/blk.c b/rust/helpers/blk.c
index 53beba8c7782d..91b72131a04cd 100644
--- a/rust/helpers/blk.c
+++ b/rust/helpers/blk.c
@@ -20,3 +20,10 @@ void rust_helper_bio_advance_iter_single(const struct bio *bio,
{
bio_advance_iter_single(bio, iter, bytes);
}
+
+bool rust_helper_blk_mq_add_to_batch(struct request *req,
+ struct io_comp_batch *iob, bool is_error,
+ void (*complete)(struct io_comp_batch *))
+{
+ return blk_mq_add_to_batch(req, iob, is_error, complete);
+}
diff --git a/rust/kernel/block/mq.rs b/rust/kernel/block/mq.rs
index cd0bfbcbf317a..1d58eea971bd7 100644
--- a/rust/kernel/block/mq.rs
+++ b/rust/kernel/block/mq.rs
@@ -89,7 +89,8 @@
//! _hw_data: (),
//! _queue_data: (),
//! rq: Owned<IdleRequest<Self>>,
-//! _is_last: bool
+//! _is_last: bool,
+//! is_poll: bool
//! ) -> BlkResult {
//! rq.start().end_ok();
//! Ok(())
@@ -130,6 +131,7 @@
mod request_queue;
pub mod tag_set;
+pub use operations::IoCompletionBatch;
pub use operations::Operations;
pub use request::Command;
pub use request::IdleRequest;
diff --git a/rust/kernel/block/mq/operations.rs b/rust/kernel/block/mq/operations.rs
index 28dd4b28d203f..bccc1903a0d10 100644
--- a/rust/kernel/block/mq/operations.rs
+++ b/rust/kernel/block/mq/operations.rs
@@ -65,6 +65,7 @@ fn queue_rq(
queue_data: ForeignBorrowed<'_, Self::QueueData>,
rq: Owned<IdleRequest<Self>>,
is_last: bool,
+ is_poll: bool,
) -> BlkResult;
/// Called by the kernel to indicate that queued requests should be submitted.
@@ -84,7 +85,15 @@ fn init_hctx(
/// Called by the kernel to poll the device for completed requests. Only
/// used for poll queues.
- fn poll(_hw_data: ForeignBorrowed<'_, Self::HwData>) -> bool {
+ ///
+ /// Should return `Ok(true)` if any requests were completed during the call,
+ /// `Ok(false)` if no requests were completed, and `Err(e)` to signal an
+ /// error condition.
+ fn poll(
+ _hw_data: ForeignBorrowed<'_, Self::HwData>,
+ _queue_data: ForeignBorrowed<'_, Self::QueueData>,
+ _batch: &mut IoCompletionBatch<Self>,
+ ) -> Result<bool> {
build_error!(crate::error::VTABLE_DEFAULT_ERROR)
}
@@ -168,6 +177,11 @@ impl<T: Operations> OperationsVTable<T> {
// `into_foreign` in `Self::init_hctx_callback`.
let hw_data = unsafe { T::HwData::borrow((*hctx).driver_data) };
+ let is_poll = u32::from(
+ // SAFETY: `hctx` is valid as required by this function.
+ unsafe { (*hctx).type_ },
+ ) == bindings::hctx_type_HCTX_TYPE_POLL;
+
// SAFETY: `hctx` is valid as required by this function.
let queue_data = unsafe { (*(*hctx).queue).queuedata };
@@ -184,6 +198,7 @@ impl<T: Operations> OperationsVTable<T> {
// SAFETY: `bd` is valid as required by the safety requirement for
// this function.
unsafe { (*bd).last },
+ is_poll,
);
if let Err(e) = ret {
@@ -242,13 +257,32 @@ impl<T: Operations> OperationsVTable<T> {
/// previously initialized by a call to `init_hctx_callback`.
unsafe extern "C" fn poll_callback(
hctx: *mut bindings::blk_mq_hw_ctx,
- _iob: *mut bindings::io_comp_batch,
+ iob: *mut bindings::io_comp_batch,
) -> crate::ffi::c_int {
// SAFETY: By function safety requirement, `hctx` was initialized by
// `init_hctx_callback` and thus `driver_data` came from a call to
// `into_foreign`.
let hw_data = unsafe { T::HwData::borrow((*hctx).driver_data) };
- T::poll(hw_data).into()
+
+ // SAFETY: `hctx` is valid as required by this function.
+ let queue_data = unsafe { (*(*hctx).queue).queuedata };
+
+ // SAFETY: `queue.queuedata` was created by `GenDiskBuilder::build` with
+ // a call to `ForeignOwnable::into_foreign` to create `queuedata`.
+ // `ForeignOwnable::from_foreign` is only called when the tagset is
+ // dropped, which happens after we are dropped.
+ let queue_data = unsafe { T::QueueData::borrow(queue_data) };
+
+ let mut batch = IoCompletionBatch {
+ inner: iob,
+ _p: PhantomData,
+ };
+
+ let ret = T::poll(hw_data, queue_data, &mut batch);
+ match ret {
+ Ok(val) => val.into(),
+ Err(e) => e.to_errno(),
+ }
}
/// This function is called by the C kernel. A pointer to this function is
@@ -448,3 +482,67 @@ pub(crate) const fn build() -> &'static bindings::blk_mq_ops {
&Self::VTABLE
}
}
+
+/// A batch of I/O completions for polled I/O.
+///
+/// This struct wraps the C `struct io_comp_batch` and is used to batch
+/// multiple request completions together for improved efficiency during polled
+/// I/O operations.
+///
+/// When the kernel polls for completed requests via [`Operations::poll`], it
+/// passes an `IoCompletionBatch` to collect completed requests. The driver can
+/// add completed requests to the batch using [`add_request`], allowing the
+/// kernel to process multiple completions together rather than one at a time.
+///
+/// # Invariants
+///
+/// - `inner` must point to a valid `io_comp_batch`.
+///
+/// [`add_request`]: IoCompletionBatch::add_request
+#[repr(transparent)]
+pub struct IoCompletionBatch<T> {
+ inner: *mut bindings::io_comp_batch,
+ _p: PhantomData<T>,
+}
+
+impl<T: Operations> IoCompletionBatch<T> {
+ /// Attempt to add a completed request to this batch.
+ ///
+ /// This method tries to add `rq` to the batch for deferred completion. If
+ /// the request is successfully added, ownership is transferred to the batch
+ /// and the request will be completed later when the batch is processed.
+ ///
+ /// # Arguments
+ ///
+ /// - `rq`: The completed request to add to the batch.
+ /// - `error`: Set to `true` if the request completed with an error.
+ ///
+ /// # Return
+ ///
+ /// When this method returns `Err`, the caller is responsible for completing
+ /// the request through other means, such as calling
+ /// [`Request::complete`](super::Request::complete).
+ pub fn add_request(
+ &mut self,
+ rq: Owned<Request<T>>,
+ error: bool,
+ ) -> Result<(), Owned<Request<T>>> {
+ // SAFETY: By type invariant, `self.inner` is a valid `io_comp_batch`.
+ let ret = unsafe {
+ bindings::blk_mq_add_to_batch(
+ rq.as_raw(),
+ self.inner,
+ error,
+ Some(bindings::blk_mq_end_request_batch),
+ )
+ };
+
+ match ret {
+ true => {
+ core::mem::forget(rq);
+ Ok(())
+ }
+ false => Err(rq),
+ }
+ }
+}
diff --git a/rust/kernel/block/mq/request.rs b/rust/kernel/block/mq/request.rs
index 4e7adb10929f0..b4f5a98a97b16 100644
--- a/rust/kernel/block/mq/request.rs
+++ b/rust/kernel/block/mq/request.rs
@@ -135,6 +135,11 @@ pub fn queue(&self) -> &RequestQueue<T> {
// SAFETY: By type invariant, self.0 is guaranteed to be valid.
unsafe { RequestQueue::from_raw((*self.0.get()).q) }
}
+
+ /// Return a raw pointer to the underlying C structure.
+ pub fn as_raw(&self) -> *mut bindings::request {
+ self.0.get()
+ }
}
/// A wrapper around a blk-mq [`struct request`]. This represents an IO request.
--
2.51.2
next prev parent reply other threads:[~2026-02-16 4:14 UTC|newest]
Thread overview: 82+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-02-15 23:34 [PATCH 00/79] block: rnull: complete the rust null block driver Andreas Hindborg
2026-02-15 23:34 ` [PATCH 01/79] block: rnull: adopt new formatting guidelines Andreas Hindborg
2026-02-15 23:34 ` [PATCH 02/79] block: rnull: add module parameters Andreas Hindborg
2026-02-15 23:34 ` [PATCH 03/79] block: rnull: add macros to define configfs attributes Andreas Hindborg
2026-02-15 23:34 ` [PATCH 04/79] block: rust: fix generation of bindings to `BLK_STS_.*` Andreas Hindborg
2026-02-15 23:34 ` [PATCH 05/79] block: rust: change `queue_rq` request type to `Owned` Andreas Hindborg
2026-02-15 23:34 ` [PATCH 06/79] block: rust: add `Request` private data support Andreas Hindborg
2026-02-15 23:34 ` [PATCH 07/79] block: rust: allow `hrtimer::Timer` in `RequestData` Andreas Hindborg
2026-02-15 23:34 ` [PATCH 08/79] block: rnull: add timer completion mode Andreas Hindborg
2026-02-15 23:34 ` [PATCH 09/79] block: rust: introduce `kernel::block::bio` module Andreas Hindborg
2026-02-15 23:34 ` [PATCH 10/79] block: rust: add `command` getter to `Request` Andreas Hindborg
2026-02-15 23:34 ` [PATCH 11/79] block: rust: mq: use GFP_KERNEL from prelude Andreas Hindborg
2026-02-15 23:34 ` [PATCH 12/79] block: rust: add `TagSet` flags Andreas Hindborg
2026-02-15 23:35 ` [PATCH 13/79] block: rnull: add memory backing Andreas Hindborg
2026-02-15 23:35 ` [PATCH 14/79] block: rnull: add submit queue count config option Andreas Hindborg
2026-02-15 23:35 ` [PATCH 15/79] block: rnull: add `use_per_node_hctx` " Andreas Hindborg
2026-02-15 23:35 ` [PATCH 16/79] block: rust: allow specifying home node when constructing `TagSet` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 17/79] block: rnull: allow specifying the home numa node Andreas Hindborg
2026-02-15 23:35 ` [PATCH 18/79] block: rust: add Request::sectors() method Andreas Hindborg
2026-02-15 23:35 ` [PATCH 19/79] block: rust: mq: add max_hw_discard_sectors support to GenDiskBuilder Andreas Hindborg
2026-02-15 23:35 ` [PATCH 20/79] block: rnull: add discard support Andreas Hindborg
2026-02-15 23:35 ` [PATCH 21/79] block: rust: add `NoDefaultScheduler` flag for `TagSet` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 22/79] block: rnull: add no_sched module parameter and configfs attribute Andreas Hindborg
2026-02-15 23:35 ` [PATCH 23/79] block: rust: change sector type from usize to u64 Andreas Hindborg
2026-02-15 23:35 ` [PATCH 24/79] block: rust: add `BadBlocks` for bad block tracking Andreas Hindborg
2026-02-15 23:35 ` [PATCH 25/79] block: rust: mq: add Request::end() method for custom status codes Andreas Hindborg
2026-02-15 23:35 ` [PATCH 26/79] block: rnull: add badblocks support Andreas Hindborg
2026-02-15 23:35 ` [PATCH 27/79] block: rnull: add badblocks_once support Andreas Hindborg
2026-02-15 23:35 ` [PATCH 28/79] block: rnull: add partial I/O support for bad blocks Andreas Hindborg
2026-02-15 23:35 ` [PATCH 29/79] block: rust: add `TagSet` private data support Andreas Hindborg
2026-02-15 23:35 ` [PATCH 30/79] block: rust: add `hctx` " Andreas Hindborg
2026-02-15 23:35 ` [PATCH 31/79] block: rnull: add volatile cache emulation Andreas Hindborg
2026-02-15 23:35 ` [PATCH 32/79] block: rust: implement `Sync` for `GenDisk` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 33/79] block: rust: add a back reference feature to `GenDisk` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 34/79] block: rust: introduce an idle type state for `Request` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 35/79] block: rust: add a request queue abstraction Andreas Hindborg
2026-02-15 23:35 ` [PATCH 36/79] block: rust: add a method to get the request queue for a request Andreas Hindborg
2026-02-15 23:35 ` [PATCH 37/79] block: rust: introduce `kernel::block::error` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 38/79] block: rust: require `queue_rq` to return a `BlkResult` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 39/79] block: rust: add `GenDisk::queue_data` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 40/79] block: rnull: add bandwidth limiting Andreas Hindborg
2026-02-15 23:35 ` [PATCH 41/79] block: rnull: add blocking queue mode Andreas Hindborg
2026-02-15 23:35 ` [PATCH 42/79] block: rnull: add shared tags Andreas Hindborg
2026-02-15 23:35 ` [PATCH 43/79] block: rnull: add queue depth config option Andreas Hindborg
2026-02-15 23:35 ` [PATCH 44/79] block: rust: add an abstraction for `bindings::req_op` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 45/79] block: rust: add a method to set the target sector of a request Andreas Hindborg
2026-02-15 23:35 ` [PATCH 46/79] block: rust: move gendisk vtable construction to separate function Andreas Hindborg
2026-02-15 23:35 ` [PATCH 47/79] block: rust: add zoned block device support Andreas Hindborg
2026-02-15 23:35 ` [PATCH 48/79] block: rnull: add zoned storage support Andreas Hindborg
2026-02-15 23:35 ` [PATCH 49/79] block: rust: add `map_queues` support Andreas Hindborg
2026-02-15 23:35 ` [PATCH 50/79] block: rust: add an abstraction for `struct blk_mq_queue_map` Andreas Hindborg
2026-02-15 23:35 ` Andreas Hindborg [this message]
2026-02-15 23:35 ` [PATCH 52/79] block: rust: add accessors to `TagSet` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 53/79] block: rnull: add polled completion support Andreas Hindborg
2026-02-15 23:35 ` [PATCH 54/79] block: rnull: add REQ_OP_FLUSH support Andreas Hindborg
2026-02-15 23:35 ` [PATCH 55/79] block: rust: add request flags abstraction Andreas Hindborg
2026-02-15 23:35 ` [PATCH 56/79] block: rust: add abstraction for block queue feature flags Andreas Hindborg
2026-02-15 23:35 ` [PATCH 57/79] block: rust: allow setting write cache and FUA flags for `GenDisk` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 58/79] block: rust: add `Segment::copy_to_page_limit` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 59/79] block: rnull: add fua support Andreas Hindborg
2026-02-15 23:35 ` [PATCH 60/79] block: fix arg type in `blk_mq_update_nr_hw_queues` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 61/79] block: rust: add `GenDisk::tag_set` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 62/79] block: rust: add `TagSet::update_hw_queue_count` Andreas Hindborg
2026-02-16 23:59 ` Ken Kurematsu
2026-02-17 9:54 ` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 63/79] block: rnull: add an option to change the number of hardware queues Andreas Hindborg
2026-02-15 23:35 ` [PATCH 64/79] block: rust: add an abstraction for `struct rq_list` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 65/79] block: rust: add `queue_rqs` vtable hook Andreas Hindborg
2026-02-15 23:35 ` [PATCH 66/79] block: rnull: support queue_rqs Andreas Hindborg
2026-02-15 23:35 ` [PATCH 67/79] block: rust: remove the `is_poll` parameter from `queue_rq` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 68/79] block: rust: add a debug assert for refcounts Andreas Hindborg
2026-02-15 23:35 ` [PATCH 69/79] block: rust: add `TagSet::tag_to_rq` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 70/79] block: rust: add `Request::queue_index` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 71/79] block: rust: add `Request::requeue` Andreas Hindborg
2026-02-15 23:35 ` [PATCH 72/79] block: rust: add `request_timeout` hook Andreas Hindborg
2026-02-15 23:36 ` [PATCH 73/79] block: rnull: add fault injection support Andreas Hindborg
2026-02-15 23:36 ` [PATCH 74/79] block: rust: add max_sectors option to `GenDiskBuilder` Andreas Hindborg
2026-02-15 23:36 ` [PATCH 75/79] block: rnull: allow configuration of the maximum IO size Andreas Hindborg
2026-02-15 23:36 ` [PATCH 76/79] block: rust: add `virt_boundary_mask` option to `GenDiskBuilder` Andreas Hindborg
2026-02-15 23:36 ` [PATCH 77/79] block: rnull: add `virt_boundary` option Andreas Hindborg
2026-02-15 23:36 ` [PATCH 78/79] block: rnull: add `shared_tag_bitmap` config option Andreas Hindborg
2026-02-15 23:36 ` [PATCH 79/79] block: rnull: add zone offline and readonly configfs files Andreas Hindborg
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260216-rnull-v6-19-rc5-send-v1-51-de9a7af4b469@kernel.org \
--to=a.hindborg@kernel.org \
--cc=Liam.Howlett@oracle.com \
--cc=aliceryhl@google.com \
--cc=anna-maria@linutronix.de \
--cc=axboe@kernel.dk \
--cc=bjorn3_gh@protonmail.com \
--cc=boqun.feng@gmail.com \
--cc=dakr@kernel.org \
--cc=frederic@kernel.org \
--cc=fujita.tomonori@gmail.com \
--cc=gary@garyguo.net \
--cc=jstultz@google.com \
--cc=linux-block@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=lorenzo.stoakes@oracle.com \
--cc=lossin@kernel.org \
--cc=lyude@redhat.com \
--cc=ojeda@kernel.org \
--cc=rust-for-linux@vger.kernel.org \
--cc=sboyd@kernel.org \
--cc=tglx@kernel.org \
--cc=tmgross@umich.edu \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox