Re: [PATCH v3 0/4] Introduce QPW for per-cpu operations (v3)

[Leonardo Bras <leobras.c@gmail.com>]

Cc: Peter Zijlstra <peterz@infradead.org>

On Mon, Mar 23, 2026 at 02:55:44PM -0300, Marcelo Tosatti wrote:
> The problem:
> Some places in the kernel implement a parallel programming strategy
> consisting on local_locks() for most of the work, and some rare remote
> operations are scheduled on target cpu. This keeps cache bouncing low since
> cacheline tends to be mostly local, and avoids the cost of locks in non-RT
> kernels, even though the very few remote operations will be expensive due
> to scheduling overhead.
> 
> On the other hand, for RT workloads this can represent a problem: getting
> an important workload scheduled out to deal with remote requests is
> sure to introduce unexpected deadline misses.
> 
> The idea:
> Currently with PREEMPT_RT=y, local_locks() become per-cpu spinlocks.
> In this case, instead of scheduling work on a remote cpu, it should
> be safe to grab that remote cpu's per-cpu spinlock and run the required
> work locally. That major cost, which is un/locking in every local function,
> already happens in PREEMPT_RT.
> 
> Also, there is no need to worry about extra cache bouncing:
> The cacheline invalidation already happens due to schedule_work_on().
> 
> This will avoid schedule_work_on(), and thus avoid scheduling-out an
> RT workload.
> 
> Proposed solution:
> A new interface called Queue PerCPU Work (QPW), which should replace
> Work Queue in the above mentioned use case.
> 
> If CONFIG_QPW=n this interfaces just wraps the current
> local_locks + WorkQueue behavior, so no expected change in runtime.
> 
> If CONFIG_QPW=y, and qpw kernel boot option =1, 
> queue_percpu_work_on(cpu,...) will lock that cpu's per-cpu structure
> and perform work on it locally. This is possible because on 
> functions that can be used for performing remote work on remote 
> per-cpu structures, the local_lock (which is already
> a this_cpu spinlock()), will be replaced by a qpw_spinlock(), which
> is able to get the per_cpu spinlock() for the cpu passed as parameter.
> 
> v2->v3:
> - Use preempt_disable/preempt_enable on !CONFIG_PREEMPT_RT (Vlastimil Babka).
> - Improve documentation to include local_qpw_lock on operations table
>   (Leonardo Bras).
> - Enable qpw=1 automatically if CPU isolation is enabled (Vlastimil Babka).
> 
> v1->v2:
> - Introduce local_qpw_lock and unlock functions, move preempt_disable/
>   preempt_enable to it (Leonardo Bras). This reduces performance
>   overhead of the patch.
> - Documentation and changelog typo fixes (Leonardo Bras).
> - Fix places where preempt_disable/preempt_enable was not being
>   correctly performed.
> - Add performance measurements.
> 
> RFC->v1:
> 
> - Introduce CONFIG_QPW and qpw= kernel boot option to enable 
>   remote spinlocking and execution even on !CONFIG_PREEMPT_RT
>   kernels (Leonardo Bras).
> - Move buffer_head draining to separate workqueue (Marcelo Tosatti).
> - Convert mlock per-CPU page lists to QPW (Marcelo Tosatti).
> - Drop memcontrol convertion (as isolated CPUs are not targets
>   of queue_work_on anymore).
> - Rebase SLUB against Vlastimil's slab/next.
> - Add basic document for QPW (Waiman Long).
> 
> The performance numbers, as measured by the following test program,
> are as follows:
> 
> CONFIG_PREEMPT_DYNAMIC=y
> Unpatched kernel:                       60 cycles
> Patched kernel, CONFIG_QPW=n:           62 cycles
> Patched kernel, CONFIG_QPW=y, qpw=0:    62 cycles
> Patched kernel, CONFIG_QPW=y, qpw=1:    75 cycles
> 
> CONFIG_PREEMPT_RT:
> Unpatched kernel:                       95 cycles
> Patched kernel, CONFIG_QPW=y, qpw=0:    99 cycles
> Patched kernel, CONFIG_QPW=y, qpw=1:    97 cycles
> 
> kmalloc_bench.c:
> #include <linux/module.h>
> #include <linux/kernel.h>
> #include <linux/slab.h>
> #include <linux/timex.h>
> #include <linux/preempt.h>
> #include <linux/irqflags.h>
> #include <linux/vmalloc.h>
> 
> MODULE_LICENSE("GPL");
> MODULE_AUTHOR("Gemini AI");
> MODULE_DESCRIPTION("A simple kmalloc performance benchmark");
> 
> static int size = 64; // Default allocation size in bytes
> module_param(size, int, 0644);
> 
> static int iterations = 9000000; // Default number of iterations
> module_param(iterations, int, 0644);
> 
> static int __init kmalloc_bench_init(void) {
>     void **ptrs;
>     cycles_t start, end;
>     uint64_t total_cycles;
>     int i;
>     pr_info("kmalloc_bench: Starting test (size=%d, iterations=%d)\n", size, iterations);
> 
>     // Allocate an array to store pointers to avoid immediate kfree-reuse optimization
>     ptrs = vmalloc(sizeof(void *) * iterations);
>     if (!ptrs) {
>         pr_err("kmalloc_bench: Failed to allocate pointer array\n");
>         return -ENOMEM;
>     }
> 
>     preempt_disable();
>     start = get_cycles();
> 
>     for (i = 0; i < iterations; i++) {
>         ptrs[i] = kmalloc(size, GFP_ATOMIC);
>     }
> 
>     end = get_cycles();
> 
>     total_cycles = end - start;
>     preempt_enable();
> 
>     pr_info("kmalloc_bench: Total cycles for %d allocs: %llu\n", iterations, total_cycles);
>     pr_info("kmalloc_bench: Avg cycles per kmalloc: %llu\n", total_cycles / iterations);
> 
>     // Cleanup
>     for (i = 0; i < iterations; i++) {
>         kfree(ptrs[i]);
>     }
>     vfree(ptrs);
> 
>     return 0;
> }
> 
> static void __exit kmalloc_bench_exit(void) {
>     pr_info("kmalloc_bench: Module unloaded\n");
> }
> 
> module_init(kmalloc_bench_init);
> module_exit(kmalloc_bench_exit);
> 
> The following testcase triggers lru_add_drain_all on an isolated CPU
> (that does sys_write to a file before entering its realtime 
> loop).
> 
> /* 
>  * Simulates a low latency loop program that is interrupted
>  * due to lru_add_drain_all. To trigger lru_add_drain_all, run:
>  *
>  * blockdev --flushbufs /dev/sdX
>  *
>  */ 
> #define _GNU_SOURCE
> #include <fcntl.h>
> #include <stdio.h>
> #include <stdlib.h>
> #include <sys/mman.h>
> #include <string.h>
> #include <sys/types.h>
> #include <sys/stat.h>
> #include <stdlib.h>
> #include <stdarg.h>
> #include <pthread.h>
> #include <sched.h>
> #include <unistd.h>
> 
> int cpu;
> 
> static void *run(void *arg)
> {
> 	pthread_t current_thread;
> 	cpu_set_t cpuset;
> 	int ret, nrloops;
> 	struct sched_param sched_p;
> 	pid_t pid;
> 	int fd;
> 	char buf[] = "xxxxxxxxxxx";
> 
> 	CPU_ZERO(&cpuset);
> 	CPU_SET(cpu, &cpuset);
> 
> 	current_thread = pthread_self();    
> 	ret = pthread_setaffinity_np(current_thread, sizeof(cpu_set_t), &cpuset);
> 	if (ret) {
> 		perror("pthread_setaffinity_np failed\n");
> 		exit(0);
> 	}
> 
> 	memset(&sched_p, 0, sizeof(struct sched_param));
> 	sched_p.sched_priority = 1;
> 	pid = gettid();
> 	ret = sched_setscheduler(pid, SCHED_FIFO, &sched_p);
> 	if (ret) {
> 		perror("sched_setscheduler");
> 		exit(0);
> 	}
> 
> 	fd = open("/tmp/tmpfile", O_RDWR|O_CREAT|O_TRUNC);
> 	if (fd == -1) {
> 		perror("open");
> 		exit(0);
> 	}
> 
> 	ret = write(fd, buf, sizeof(buf));
> 	if (ret == -1) {
> 		perror("write");
> 		exit(0);
> 	}
> 
> 	do { 
> 		nrloops = nrloops+2;
> 		nrloops--;
> 	} while (1);
> }
> 
> int main(int argc, char *argv[])
> {
>         int fd, ret;
> 	pthread_t thread;
> 	long val;
> 	char *endptr, *str;
> 	struct sched_param sched_p;
> 	pid_t pid;
> 
> 	if (argc != 2) {
> 		printf("usage: %s cpu-nr\n", argv[0]);
> 		printf("where CPU number is the CPU to pin thread to\n");
> 		exit(0);
> 	}
> 	str = argv[1];
> 	cpu = strtol(str, &endptr, 10);
> 	if (cpu < 0) {
> 		printf("strtol returns %d\n", cpu);
> 		exit(0);
> 	}
> 	printf("cpunr=%d\n", cpu);
> 
> 	memset(&sched_p, 0, sizeof(struct sched_param));
> 	sched_p.sched_priority = 1;
> 	pid = getpid();
> 	ret = sched_setscheduler(pid, SCHED_FIFO, &sched_p);
> 	if (ret) {
> 		perror("sched_setscheduler");
> 		exit(0);
> 	}
> 
> 	pthread_create(&thread, NULL, run, NULL);
> 
> 	sleep(5000);
> 
> 	pthread_join(thread, NULL);
> }
> 
> 
> 
> 
>