Re: [PATCH v2] mm/slub: skip freelist construction for whole-slab bulk refill

[<hu.shengming@zte.com.cn>]

> On Wed, Apr 01, 2026 at 12:57:25PM +0800, hu.shengming@zte.com.cn wrote:
> > From: Shengming Hu <hu.shengming@zte.com.cn>
> > 
> > refill_objects() already notes that a whole-slab bulk refill could avoid
> > building a freelist that would be drained immediately.
> > 
> > When the remaining bulk allocation is large enough to consume an entire
> > new slab, building the freelist is unnecessary overhead. Instead,
> > allocate the slab without initializing its freelist and hand all objects
> > directly to the caller.
> > 
> > Handle CONFIG_SLAB_FREELIST_RANDOM=y as well by walking objects in the
> > randomized allocation order and placing them directly into the caller's
> > array, without constructing a temporary freelist.
> > 
> > Also mark setup_object() inline. After this optimization, the compiler no
> > longer consistently inlines this helper in the hot path, which can hurt
> > performance. Explicitly marking it inline restores the expected code
> > generation.
> > 
> > This reduces per-object overhead in bulk allocation paths and improves
> > allocation throughput significantly. In slub_bulk_bench, the time per
> > object drops by about 54% to 74% with CONFIG_SLAB_FREELIST_RANDOM=n, and
> > by about 62% to 74% with CONFIG_SLAB_FREELIST_RANDOM=y.
> 
> Thanks for the patch.
> Here are some quick review..
> 

Hi Hao,

Thanks for the quick review!

> > 
> > Benchmark results (slub_bulk_bench):
> > 
> > Machine: qemu-system-x86 -m 1024M -smp 8 -enable-kvm -cpu host
> > Kernel: Linux 7.0.0-rc6-next-20260330
> > Config: x86_64_defconfig
> > Cpu: 0
> > Rounds: 20
> > Total: 256MB
> > 
> > - CONFIG_SLAB_FREELIST_RANDOM=n -
> > 
> > obj_size=16, batch=256:
> > before: 5.29 +- 0.73 ns/object
> > after: 2.42 +- 0.05 ns/object
> > delta: -54.4%
> > 
> > obj_size=32, batch=128:
> > before: 7.65 +- 1.89 ns/object
> > after: 3.04 +- 0.03 ns/object
> > delta: -60.2%
> > 
> > obj_size=64, batch=64:
> > before: 11.07 +- 0.08 ns/object
> > after: 4.11 +- 0.04 ns/object
> > delta: -62.9%
> > 
> > obj_size=128, batch=32:
> > before: 19.95 +- 0.30 ns/object
> > after: 5.72 +- 0.05 ns/object
> > delta: -71.3%
> > 
> > obj_size=256, batch=32:
> > before: 24.31 +- 0.25 ns/object
> > after: 6.33 +- 0.14 ns/object
> > delta: -74.0%
> > 
> > obj_size=512, batch=32:
> > before: 22.48 +- 0.14 ns/object
> > after: 6.43 +- 0.10 ns/object
> > delta: -71.4%
> > 
> > - CONFIG_SLAB_FREELIST_RANDOM=y -
> > 
> > obj_size=16, batch=256:
> > before: 9.32 +- 1.26 ns/object
> > after: 3.51 +- 0.02 ns/object
> > delta: -62.4%
> > 
> > obj_size=32, batch=128:
> > before: 11.68 +- 0.15 ns/object
> > after: 4.18 +- 0.22 ns/object
> > delta: -64.2%
> > 
> > obj_size=64, batch=64:
> > before: 16.69 +- 1.36 ns/object
> > after: 5.22 +- 0.06 ns/object
> > delta: -68.7%
> > 
> > obj_size=128, batch=32:
> > before: 23.41 +- 0.23 ns/object
> > after: 7.40 +- 0.07 ns/object
> > delta: -68.4%
> > 
> > obj_size=256, batch=32:
> > before: 29.80 +- 0.44 ns/object
> > after: 7.98 +- 0.09 ns/object
> > delta: -73.2%
> > 
> > obj_size=512, batch=32:
> > before: 30.38 +- 0.36 ns/object
> > after: 8.01 +- 0.06 ns/object
> > delta: -73.6%
> > 
> > Link: https://github.com/HSM6236/slub_bulk_test.git
> > Signed-off-by: Shengming Hu <hu.shengming@zte.com.cn>
> > ---
> > Changes in v2:
> > - Handle CONFIG_SLAB_FREELIST_RANDOM=y and add benchmark results.
> > - Update the QEMU benchmark setup to use -enable-kvm -cpu host so benchmark results better reflect native CPU performance.
> > - Link to v1: https://lore.kernel.org/all/20260328125538341lvTGRpS62UNdRiAAz2gH3@zte.com.cn/
> > 
> > ---
> >  mm/slub.c | 155 +++++++++++++++++++++++++++++++++++++++++++++++-------
> >  1 file changed, 136 insertions(+), 19 deletions(-)
> > 
> > diff --git a/mm/slub.c b/mm/slub.c
> > index fb2c5c57bc4e..52da4a716b1b 100644
> > --- a/mm/slub.c
> > +++ b/mm/slub.c
> > @@ -2733,7 +2733,7 @@ bool slab_free_freelist_hook(struct kmem_cache *s, void **head, void **tail,
> >  	return *head != NULL;
> >  }
> >  
> > -static void *setup_object(struct kmem_cache *s, void *object)
> > +static inline void *setup_object(struct kmem_cache *s, void *object)
> >  {
> >  	setup_object_debug(s, object);
> >  	object = kasan_init_slab_obj(s, object);
> > @@ -3399,6 +3399,53 @@ static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab,
> >  
> >  	return true;
> >  }
> > +static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s,
> > +						   void *obj);
> > +
> > +static inline bool alloc_whole_from_new_slab_random(struct kmem_cache *s,
> > +					struct slab *slab, void **p,
> > +					bool allow_spin,
> > +					unsigned int *allocatedp)
> > +{
> > +	unsigned long pos, page_limit, freelist_count;
> > +	unsigned int allocated = 0;
> > +	void *next, *start;
> > +
> > +	if (slab->objects < 2 || !s->random_seq)
> > +		return false;
> > +
> > +	freelist_count = oo_objects(s->oo);
> > +
> > +	if (allow_spin) {
> > +		pos = get_random_u32_below(freelist_count);
> > +	} else {
> > +		struct rnd_state *state;
> > +
> > +		/*
> > +		 * An interrupt or NMI handler might interrupt and change
> > +		 * the state in the middle, but that's safe.
> > +		 */
> > +		state = &get_cpu_var(slab_rnd_state);
> > +		pos = prandom_u32_state(state) % freelist_count;
> > +		put_cpu_var(slab_rnd_state);
> > +	}
> > +
> > +	page_limit = slab->objects * s->size;
> > +	start = fixup_red_left(s, slab_address(slab));
> > +
> > +	while (allocated < slab->objects) {
> > +		next = next_freelist_entry(s, &pos, start, page_limit,
> > +					     freelist_count);
> > +		next = setup_object(s, next);
> > +		p[allocated] = next;
> > +		maybe_wipe_obj_freeptr(s, next);
> > +		allocated++;
> > +	}
> > +
> > +	*allocatedp = allocated;
> 
> It seems we does not need to return the allocated count through allocatedp,
> since the count should always be slab->objects.
> 

Agreed, I'll drop allocatedp.

> > +	return true;
> > +}
> > +
> >  #else
> >  static inline int init_cache_random_seq(struct kmem_cache *s)
> >  {
> > @@ -3410,6 +3457,14 @@ static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab,
> >  {
> >  	return false;
> >  }
> > +
> > +static inline bool alloc_whole_from_new_slab_random(struct kmem_cache *s,
> > +					struct slab *slab, void **p,
> > +					bool allow_spin,
> > +					unsigned int *allocatedp)
> > +{
> > +	return false;
> > +}
> >  #endif /* CONFIG_SLAB_FREELIST_RANDOM */
> >  
> >  static __always_inline void account_slab(struct slab *slab, int order,
> > @@ -3438,7 +3493,8 @@ static __always_inline void unaccount_slab(struct slab *slab, int order,
> >  			    -(PAGE_SIZE << order));
> >  }
> >  
> > -static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
> > +static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node,
> > +				  bool build_freelist, bool *allow_spinp)
> >  {
> >  	bool allow_spin = gfpflags_allow_spinning(flags);
> >  	struct slab *slab;
> > @@ -3446,7 +3502,10 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
> >  	gfp_t alloc_gfp;
> >  	void *start, *p, *next;
> >  	int idx;
> > -	bool shuffle;
> > +	bool shuffle = false;
> > +
> > +	if (allow_spinp)
> > +		*allow_spinp = allow_spin;
> 
> It seems unnecessary for allocate_slab() to compute allow_spin and return it
> via allow_spinp.
> We could instead calculate it directly in refill_objects() based on gfp.
> 

Yes, that makes sense. I'll compute allow_spin directly in refill_objects()
and remove the allow_spinp plumbing from allocate_slab()/new_slab().

> >  
> >  	flags &= gfp_allowed_mask;
> >  
> > @@ -3483,6 +3542,7 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
> >  	slab->frozen = 0;
> >  
> >  	slab->slab_cache = s;
> > +	slab->freelist = NULL;
> >  
> >  	kasan_poison_slab(slab);
> >  
> > @@ -3497,9 +3557,10 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
> >  	alloc_slab_obj_exts_early(s, slab);
> >  	account_slab(slab, oo_order(oo), s, flags);
> >  
> > -	shuffle = shuffle_freelist(s, slab, allow_spin);
> > +	if (build_freelist)
> > +		shuffle = shuffle_freelist(s, slab, allow_spin);
> >  
> > -	if (!shuffle) {
> > +	if (build_freelist && !shuffle) {
> >  		start = fixup_red_left(s, start);
> >  		start = setup_object(s, start);
> >  		slab->freelist = start;
> > @@ -3515,7 +3576,8 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
> >  	return slab;
> >  }
> >  
> > -static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node)
> > +static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node,
> > +			     bool build_freelist, bool *allow_spinp)
> >  {
> >  	if (unlikely(flags & GFP_SLAB_BUG_MASK))
> >  		flags = kmalloc_fix_flags(flags);
> > @@ -3523,7 +3585,8 @@ static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node)
> >  	WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO));
> >  
> >  	return allocate_slab(s,
> > -		flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
> > +			     flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK),
> > +			     node, build_freelist, allow_spinp);
> >  }
> >  
> >  static void __free_slab(struct kmem_cache *s, struct slab *slab, bool allow_spin)
> > @@ -4395,6 +4458,48 @@ static unsigned int alloc_from_new_slab(struct kmem_cache *s, struct slab *slab,
> >  	return allocated;
> >  }
> >  
> > +static unsigned int alloc_whole_from_new_slab(struct kmem_cache *s,
> > +		struct slab *slab, void **p, bool allow_spin)
> > +{
> > +
> > +	unsigned int allocated = 0;
> > +	void *object, *start;
> > +
> > +	if (alloc_whole_from_new_slab_random(s, slab, p, allow_spin,
> > +					     &allocated)) {
> > +		goto done;
> > +	}
> > +
> > +	start = fixup_red_left(s, slab_address(slab));
> > +	object = setup_object(s, start);
> > +
> > +	while (allocated < slab->objects - 1) {
> > +		p[allocated] = object;
> > +		maybe_wipe_obj_freeptr(s, object);
> > +
> > +		allocated++;
> > +		object += s->size;
> > +		object = setup_object(s, object);
> > +	}
> 
> Also, I feel the current patch contains some duplicated code like this loop.
> 
> Would it make sense to split allocate_slab() into two functions?
> 
> For example,
> the first part could be called allocate_slab_meta_setup() (just an example name)
> And, the second part could be allocate_slab_objects_setup(), with the core logic
> being the loop over objects. Then allocate_slab_objects_setup() could support
> two modes: one called BUILD_FREELIST, which builds the freelist, and another
> called EMIT_OBJECTS, which skips building the freelist and directly places the
> objects into the target array.
> 

I may be missing part of your idea here, so please correct me if I misunderstood.

Regarding the duplicated loop, this patch adds two loops: one in
alloc_whole_from_new_slab() and the other in
alloc_whole_from_new_slab_random(). I did not merge them because the
allocation path differs when CONFIG_SLAB_FREELIST_RANDOM is enabled versus
disabled.

As for allocate_slab(), my intention with the current build_freelist flag
was to keep the change small and reuse the existing allocate_slab() path,
since the only behavior difference here is whether we build the freelist for
the new slab.

Could you elaborate a bit more on the refactoring you have in mind?

> > +
> > +	p[allocated] = object;
> > +	maybe_wipe_obj_freeptr(s, object);
> > +	allocated++;
> > +
> > +done:
> > +	slab->freelist = NULL;
> > +	slab->inuse = slab->objects;
> > +	inc_slabs_node(s, slab_nid(slab), slab->objects);
> > +
> > +	return allocated;
> > +}
> > +
> > +static inline bool bulk_refill_consumes_whole_slab(struct kmem_cache *s,
> > +		unsigned int count)
> > +{
> > +	return count >= oo_objects(s->oo);
> 
> It seems using s->oo here may be a bit too strict. In allocate_slab(), the
> object count can fall back to s->min, so using s->objects might be more
> reasonable (If I understand correctly...).
> 

Good point. I do not see s->objects in current linux-next; did you mean
slab->objects?

oo_objects(s->oo) is the preferred-layout object count, while the actual
object count of a newly allocated slab is only known after allocate_slab(),
via slab->objects, since allocation can fall back to s->min.

So I used oo_objects(s->oo) because this check happens before slab
allocation. It is conservative, but safe.

I agree that slab->objects would be a more accurate basis if we move this
decision after slab allocation.

Thanks again for the review.

--
With Best Regards,
Shengming

> > +}
> > +
> >  /*
> >   * Slow path. We failed to allocate via percpu sheaves or they are not available
> >   * due to bootstrap or debugging enabled or SLUB_TINY.
> > @@ -4441,7 +4546,7 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
> >  	if (object)
> >  		goto success;
> >  
> > -	slab = new_slab(s, pc.flags, node);
> > +	slab = new_slab(s, pc.flags, node, true, NULL);
> >  
> >  	if (unlikely(!slab)) {
> >  		if (node != NUMA_NO_NODE && !(gfpflags & __GFP_THISNODE)
> > @@ -7244,18 +7349,30 @@ refill_objects(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min,
> >  
> >  new_slab:
> >  
> > -	slab = new_slab(s, gfp, local_node);
> > -	if (!slab)
> > -		goto out;
> > -
> > -	stat(s, ALLOC_SLAB);
> > -
> >  	/*
> > -	 * TODO: possible optimization - if we know we will consume the whole
> > -	 * slab we might skip creating the freelist?
> > +	 * If the remaining bulk allocation is large enough to consume
> > +	 * an entire slab, avoid building the freelist only to drain it
> > +	 * immediately. Instead, allocate a slab without a freelist and
> > +	 * hand out all objects directly.
> >  	 */
> > -	refilled += alloc_from_new_slab(s, slab, p + refilled, max - refilled,
> > -					/* allow_spin = */ true);
> > +	if (bulk_refill_consumes_whole_slab(s, max - refilled)) {
> > +		bool allow_spin;
> > +
> > +		slab = new_slab(s, gfp, local_node, false, &allow_spin);
> > +		if (!slab)
> > +			goto out;
> > +		stat(s, ALLOC_SLAB);
> > +		refilled += alloc_whole_from_new_slab(s, slab, p + refilled,
> > +						      allow_spin);
> > +	} else {
> > +		slab = new_slab(s, gfp, local_node, true, NULL);
> > +		if (!slab)
> > +			goto out;
> > +		stat(s, ALLOC_SLAB);
> > +		refilled += alloc_from_new_slab(s, slab, p + refilled,
> > +						max - refilled,
> > +						/* allow_spin = */ true);
> > +	}
> >  
> >  	if (refilled < min)
> >  		goto new_slab;
> > @@ -7587,7 +7704,7 @@ static void early_kmem_cache_node_alloc(int node)
> >  
> >  	BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
> >  
> > -	slab = new_slab(kmem_cache_node, GFP_NOWAIT, node);
> > +	slab = new_slab(kmem_cache_node, GFP_NOWAIT, node, true, NULL);
> >  
> >  	BUG_ON(!slab);
> >  	if (slab_nid(slab) != node) {
> > -- 
> > 2.25.1