Re: [PATCH v4 09/22] slab: handle kmalloc sheaves bootstrap

["Liam R. Howlett" <Liam.Howlett@oracle.com>]

* Vlastimil Babka <vbabka@suse.cz> [260123 01:53]:
> Enable sheaves for kmalloc caches. For other types than KMALLOC_NORMAL,
> we can simply allow them in calculate_sizes() as they are created later
> than KMALLOC_NORMAL caches and can allocate sheaves and barns from
> those.
> 
> For KMALLOC_NORMAL caches we perform additional step after first
> creating them without sheaves. Then bootstrap_cache_sheaves() simply
> allocates and initializes barns and sheaves and finally sets
> s->sheaf_capacity to make them actually used.
> 
> Afterwards the only caches left without sheaves (unless SLUB_TINY or
> debugging is enabled) are kmem_cache and kmem_cache_node. These are only
> used when creating or destroying other kmem_caches. Thus they are not
> performance critical and we can simply leave it that way.
> 
> Reviewed-by: Harry Yoo <harry.yoo@oracle.com>
> Reviewed-by: Hao Li <hao.li@linux.dev>
> Signed-off-by: Vlastimil Babka <vbabka@suse.cz>

Reviewed-by: Liam R. Howlett <Liam.Howlett@oracle.com>

> ---
>  mm/slub.c | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---
>  1 file changed, 84 insertions(+), 4 deletions(-)
> 
> diff --git a/mm/slub.c b/mm/slub.c
> index 4ca6bd944854..22acc249f9c0 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -2593,7 +2593,8 @@ static void *setup_object(struct kmem_cache *s, void *object)
>  	return object;
>  }
>  
> -static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp)
> +static struct slab_sheaf *__alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp,
> +					      unsigned int capacity)
>  {
>  	struct slab_sheaf *sheaf;
>  	size_t sheaf_size;
> @@ -2611,7 +2612,7 @@ static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp)
>  	if (s->flags & SLAB_KMALLOC)
>  		gfp |= __GFP_NO_OBJ_EXT;
>  
> -	sheaf_size = struct_size(sheaf, objects, s->sheaf_capacity);
> +	sheaf_size = struct_size(sheaf, objects, capacity);
>  	sheaf = kzalloc(sheaf_size, gfp);
>  
>  	if (unlikely(!sheaf))
> @@ -2624,6 +2625,12 @@ static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp)
>  	return sheaf;
>  }
>  
> +static inline struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s,
> +						   gfp_t gfp)
> +{
> +	return __alloc_empty_sheaf(s, gfp, s->sheaf_capacity);
> +}
> +
>  static void free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf)
>  {
>  	kfree(sheaf);
> @@ -8144,8 +8151,11 @@ static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s)
>  	if (s->flags & SLAB_RECLAIM_ACCOUNT)
>  		s->allocflags |= __GFP_RECLAIMABLE;
>  
> -	/* kmalloc caches need extra care to support sheaves */
> -	if (!is_kmalloc_cache(s))
> +	/*
> +	 * For KMALLOC_NORMAL caches we enable sheaves later by
> +	 * bootstrap_kmalloc_sheaves() to avoid recursion
> +	 */
> +	if (!is_kmalloc_normal(s))
>  		s->sheaf_capacity = calculate_sheaf_capacity(s, args);
>  
>  	/*
> @@ -8640,6 +8650,74 @@ static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
>  	return s;
>  }
>  
> +/*
> + * Finish the sheaves initialization done normally by init_percpu_sheaves() and
> + * init_kmem_cache_nodes(). For normal kmalloc caches we have to bootstrap it
> + * since sheaves and barns are allocated by kmalloc.
> + */
> +static void __init bootstrap_cache_sheaves(struct kmem_cache *s)
> +{
> +	struct kmem_cache_args empty_args = {};
> +	unsigned int capacity;
> +	bool failed = false;
> +	int node, cpu;
> +
> +	capacity = calculate_sheaf_capacity(s, &empty_args);
> +
> +	/* capacity can be 0 due to debugging or SLUB_TINY */
> +	if (!capacity)
> +		return;
> +
> +	for_each_node_mask(node, slab_nodes) {
> +		struct node_barn *barn;
> +
> +		barn = kmalloc_node(sizeof(*barn), GFP_KERNEL, node);
> +
> +		if (!barn) {
> +			failed = true;
> +			goto out;
> +		}
> +
> +		barn_init(barn);
> +		get_node(s, node)->barn = barn;
> +	}
> +
> +	for_each_possible_cpu(cpu) {
> +		struct slub_percpu_sheaves *pcs;
> +
> +		pcs = per_cpu_ptr(s->cpu_sheaves, cpu);
> +
> +		pcs->main = __alloc_empty_sheaf(s, GFP_KERNEL, capacity);
> +
> +		if (!pcs->main) {
> +			failed = true;
> +			break;
> +		}
> +	}
> +
> +out:
> +	/*
> +	 * It's still early in boot so treat this like same as a failure to
> +	 * create the kmalloc cache in the first place
> +	 */
> +	if (failed)
> +		panic("Out of memory when creating kmem_cache %s\n", s->name);
> +
> +	s->sheaf_capacity = capacity;
> +}
> +
> +static void __init bootstrap_kmalloc_sheaves(void)
> +{
> +	enum kmalloc_cache_type type;
> +
> +	for (type = KMALLOC_NORMAL; type <= KMALLOC_RANDOM_END; type++) {
> +		for (int idx = 0; idx < KMALLOC_SHIFT_HIGH + 1; idx++) {
> +			if (kmalloc_caches[type][idx])
> +				bootstrap_cache_sheaves(kmalloc_caches[type][idx]);
> +		}
> +	}
> +}
> +
>  void __init kmem_cache_init(void)
>  {
>  	static __initdata struct kmem_cache boot_kmem_cache,
> @@ -8683,6 +8761,8 @@ void __init kmem_cache_init(void)
>  	setup_kmalloc_cache_index_table();
>  	create_kmalloc_caches();
>  
> +	bootstrap_kmalloc_sheaves();
> +
>  	/* Setup random freelists for each cache */
>  	init_freelist_randomization();
>  
> 
> -- 
> 2.52.0
>