* CK2 [01/15] slab: Simplify bootstrap
[not found] <20120928191715.368450474@linux.com>
@ 2012-09-28 19:18 ` Christoph Lameter
2012-09-28 19:18 ` CK2 [02/15] create common functions for boot slab creation Christoph Lameter
` (13 subsequent siblings)
14 siblings, 0 replies; 21+ messages in thread
From: Christoph Lameter @ 2012-09-28 19:18 UTC (permalink / raw)
To: Pekka Enberg; +Cc: Joonsoo Kim, Glauber Costa, linux-mm, David Rientjes
The nodelists field in kmem_cache is pointing to the first unused
object in the array field when bootstrap is complete.
A problem with the current approach is that the statically sized
kmem_cache structure use on boot can only contain NR_CPUS entries.
If the number of nodes plus the number of cpus is greater then we
would overwrite memory following the kmem_cache_boot definition.
Increase the size of the array field to ensure that also the node
pointers fit into the array field.
Once we do that we no longer need the kmem_cache_nodelists
array and we can then also use that structure elsewhere.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
include/linux/slab_def.h | 2 +-
mm/slab.c | 18 +++++++++++++-----
2 files changed, 14 insertions(+), 6 deletions(-)
Index: linux/include/linux/slab_def.h
===================================================================
--- linux.orig/include/linux/slab_def.h 2012-09-12 10:35:59.867731044 -0500
+++ linux/include/linux/slab_def.h 2012-09-12 10:46:10.772621194 -0500
@@ -91,7 +91,7 @@ struct kmem_cache {
* is statically defined, so we reserve the max number of cpus.
*/
struct kmem_list3 **nodelists;
- struct array_cache *array[NR_CPUS];
+ struct array_cache *array[NR_CPUS + MAX_NUMNODES];
/*
* Do not add fields after array[]
*/
Index: linux/mm/slab.c
===================================================================
--- linux.orig/mm/slab.c 2012-09-12 10:46:09.980604396 -0500
+++ linux/mm/slab.c 2012-09-12 10:46:10.772621194 -0500
@@ -578,9 +578,7 @@ static struct arraycache_init initarray_
{ {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
/* internal cache of cache description objs */
-static struct kmem_list3 *kmem_cache_nodelists[MAX_NUMNODES];
static struct kmem_cache kmem_cache_boot = {
- .nodelists = kmem_cache_nodelists,
.batchcount = 1,
.limit = BOOT_CPUCACHE_ENTRIES,
.shared = 1,
@@ -1584,6 +1582,15 @@ static void __init set_up_list3s(struct
}
/*
+ * The memory after the last cpu cache pointer is used for the
+ * the nodelists pointer.
+ */
+static void setup_nodelists_pointer(struct kmem_cache *s)
+{
+ s->nodelists = (struct kmem_list3 **)&s->array[nr_cpu_ids];
+}
+
+/*
* Initialisation. Called after the page allocator have been initialised and
* before smp_init().
*/
@@ -1597,13 +1604,15 @@ void __init kmem_cache_init(void)
int node;
kmem_cache = &kmem_cache_boot;
+ setup_nodelists_pointer(kmem_cache);
if (num_possible_nodes() == 1)
use_alien_caches = 0;
+
for (i = 0; i < NUM_INIT_LISTS; i++) {
kmem_list3_init(&initkmem_list3[i]);
- if (i < MAX_NUMNODES)
+ if (i < nr_node_ids)
kmem_cache->nodelists[i] = NULL;
}
set_up_list3s(kmem_cache, CACHE_CACHE);
@@ -1643,7 +1652,6 @@ void __init kmem_cache_init(void)
list_add(&kmem_cache->list, &slab_caches);
kmem_cache->colour_off = cache_line_size();
kmem_cache->array[smp_processor_id()] = &initarray_cache.cache;
- kmem_cache->nodelists[node] = &initkmem_list3[CACHE_CACHE + node];
/*
* struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
@@ -2454,7 +2462,7 @@ __kmem_cache_create (struct kmem_cache *
else
gfp = GFP_NOWAIT;
- cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids];
+ setup_nodelists_pointer(cachep);
#if DEBUG
/*
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread* CK2 [02/15] create common functions for boot slab creation
[not found] <20120928191715.368450474@linux.com>
2012-09-28 19:18 ` CK2 [01/15] slab: Simplify bootstrap Christoph Lameter
@ 2012-09-28 19:18 ` Christoph Lameter
2012-09-28 19:23 ` CK2 [15/15] Move kmalloc_node functions to common code Christoph Lameter
` (12 subsequent siblings)
14 siblings, 0 replies; 21+ messages in thread
From: Christoph Lameter @ 2012-09-28 19:18 UTC (permalink / raw)
To: Pekka Enberg; +Cc: Joonsoo Kim, Glauber Costa, linux-mm, David Rientjes
Use a special function to create kmalloc caches and use that function in
SLAB and SLUB.
Reviewed-by: Glauber Costa <glommer@parallels.com>
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slab.c | 48 ++++++++++++++----------------------------------
mm/slab.h | 5 +++++
mm/slab_common.c | 32 ++++++++++++++++++++++++++++++++
mm/slub.c | 36 +++---------------------------------
4 files changed, 54 insertions(+), 67 deletions(-)
Index: linux/mm/slab.c
===================================================================
--- linux.orig/mm/slab.c 2012-09-28 12:24:33.141566539 -0500
+++ linux/mm/slab.c 2012-09-28 12:24:34.317591077 -0500
@@ -1686,23 +1686,13 @@ void __init kmem_cache_init(void)
* bug.
*/
- sizes[INDEX_AC].cs_cachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
- sizes[INDEX_AC].cs_cachep->name = names[INDEX_AC].name;
- sizes[INDEX_AC].cs_cachep->size = sizes[INDEX_AC].cs_size;
- sizes[INDEX_AC].cs_cachep->object_size = sizes[INDEX_AC].cs_size;
- sizes[INDEX_AC].cs_cachep->align = ARCH_KMALLOC_MINALIGN;
- __kmem_cache_create(sizes[INDEX_AC].cs_cachep, ARCH_KMALLOC_FLAGS|SLAB_PANIC);
- list_add(&sizes[INDEX_AC].cs_cachep->list, &slab_caches);
+ sizes[INDEX_AC].cs_cachep = create_kmalloc_cache(names[INDEX_AC].name,
+ sizes[INDEX_AC].cs_size, ARCH_KMALLOC_FLAGS);
- if (INDEX_AC != INDEX_L3) {
- sizes[INDEX_L3].cs_cachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
- sizes[INDEX_L3].cs_cachep->name = names[INDEX_L3].name;
- sizes[INDEX_L3].cs_cachep->size = sizes[INDEX_L3].cs_size;
- sizes[INDEX_L3].cs_cachep->object_size = sizes[INDEX_L3].cs_size;
- sizes[INDEX_L3].cs_cachep->align = ARCH_KMALLOC_MINALIGN;
- __kmem_cache_create(sizes[INDEX_L3].cs_cachep, ARCH_KMALLOC_FLAGS|SLAB_PANIC);
- list_add(&sizes[INDEX_L3].cs_cachep->list, &slab_caches);
- }
+ if (INDEX_AC != INDEX_L3)
+ sizes[INDEX_L3].cs_cachep =
+ create_kmalloc_cache(names[INDEX_L3].name,
+ sizes[INDEX_L3].cs_size, ARCH_KMALLOC_FLAGS);
slab_early_init = 0;
@@ -1714,24 +1704,14 @@ void __init kmem_cache_init(void)
* Note for systems short on memory removing the alignment will
* allow tighter packing of the smaller caches.
*/
- if (!sizes->cs_cachep) {
- sizes->cs_cachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
- sizes->cs_cachep->name = names->name;
- sizes->cs_cachep->size = sizes->cs_size;
- sizes->cs_cachep->object_size = sizes->cs_size;
- sizes->cs_cachep->align = ARCH_KMALLOC_MINALIGN;
- __kmem_cache_create(sizes->cs_cachep, ARCH_KMALLOC_FLAGS|SLAB_PANIC);
- list_add(&sizes->cs_cachep->list, &slab_caches);
- }
+ if (!sizes->cs_cachep)
+ sizes->cs_cachep = create_kmalloc_cache(names->name,
+ sizes->cs_size, ARCH_KMALLOC_FLAGS);
+
#ifdef CONFIG_ZONE_DMA
- sizes->cs_dmacachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
- sizes->cs_dmacachep->name = names->name_dma;
- sizes->cs_dmacachep->size = sizes->cs_size;
- sizes->cs_dmacachep->object_size = sizes->cs_size;
- sizes->cs_dmacachep->align = ARCH_KMALLOC_MINALIGN;
- __kmem_cache_create(sizes->cs_dmacachep,
- ARCH_KMALLOC_FLAGS|SLAB_CACHE_DMA| SLAB_PANIC);
- list_add(&sizes->cs_dmacachep->list, &slab_caches);
+ sizes->cs_dmacachep = create_kmalloc_cache(
+ names->name_dma, sizes->cs_size,
+ SLAB_CACHE_DMA|ARCH_KMALLOC_FLAGS);
#endif
sizes++;
names++;
Index: linux/mm/slab.h
===================================================================
--- linux.orig/mm/slab.h 2012-09-28 12:24:27.765454368 -0500
+++ linux/mm/slab.h 2012-09-28 12:24:34.317591077 -0500
@@ -35,6 +35,11 @@ extern struct kmem_cache *kmem_cache;
/* Functions provided by the slab allocators */
extern int __kmem_cache_create(struct kmem_cache *, unsigned long flags);
+extern struct kmem_cache *create_kmalloc_cache(const char *name, size_t size,
+ unsigned long flags);
+extern void create_boot_cache(struct kmem_cache *, const char *name,
+ size_t size, unsigned long flags);
+
#ifdef CONFIG_SLUB
struct kmem_cache *__kmem_cache_alias(const char *name, size_t size,
size_t align, unsigned long flags, void (*ctor)(void *));
Index: linux/mm/slab_common.c
===================================================================
--- linux.orig/mm/slab_common.c 2012-09-28 12:24:27.785454788 -0500
+++ linux/mm/slab_common.c 2012-09-28 12:24:53.153984094 -0500
@@ -189,3 +189,39 @@ int slab_is_available(void)
{
return slab_state >= UP;
}
+
+#ifndef CONFIG_SLOB
+/* Create a cache during boot when no slab services are available yet */
+void __init create_boot_cache(struct kmem_cache *s, const char *name, size_t size,
+ unsigned long flags)
+{
+ int err;
+
+ s->name = name;
+ s->size = s->object_size = size;
+ s->align = ARCH_KMALLOC_MINALIGN;
+ err = __kmem_cache_create(s, flags);
+
+ if (err)
+ panic("Creation of kmalloc slab %s size=%td failed. Reason %d\n",
+ name, size, err);
+
+ list_add(&s->list, &slab_caches);
+ s->refcount = -1; /* Exempt from merging for now */
+}
+
+struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size,
+ unsigned long flags)
+{
+ struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
+
+ if (!s)
+ panic("Out of memory when creating slab %s\n", name);
+
+ create_boot_cache(s, name, size, flags);
+ s->refcount = 1;
+ return s;
+}
+
+#endif /* !CONFIG_SLOB */
+
Index: linux/mm/slub.c
===================================================================
--- linux.orig/mm/slub.c 2012-09-28 12:24:27.797455035 -0500
+++ linux/mm/slub.c 2012-09-28 12:24:34.317591077 -0500
@@ -3253,32 +3253,6 @@ static int __init setup_slub_nomerge(cha
__setup("slub_nomerge", setup_slub_nomerge);
-static struct kmem_cache *__init create_kmalloc_cache(const char *name,
- int size, unsigned int flags)
-{
- struct kmem_cache *s;
-
- s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
-
- s->name = name;
- s->size = s->object_size = size;
- s->align = ARCH_KMALLOC_MINALIGN;
-
- /*
- * This function is called with IRQs disabled during early-boot on
- * single CPU so there's no need to take slab_mutex here.
- */
- if (kmem_cache_open(s, flags))
- goto panic;
-
- list_add(&s->list, &slab_caches);
- return s;
-
-panic:
- panic("Creation of kmalloc slab %s size=%d failed.\n", name, size);
- return NULL;
-}
-
/*
* Conversion table for small slabs sizes / 8 to the index in the
* kmalloc array. This is necessary for slabs < 192 since we have non power
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread* CK2 [15/15] Move kmalloc_node functions to common code
[not found] <20120928191715.368450474@linux.com>
2012-09-28 19:18 ` CK2 [01/15] slab: Simplify bootstrap Christoph Lameter
2012-09-28 19:18 ` CK2 [02/15] create common functions for boot slab creation Christoph Lameter
@ 2012-09-28 19:23 ` Christoph Lameter
2012-09-28 19:24 ` CK2 [07/15] slab: Use common kmalloc_index/kmalloc_size functions Christoph Lameter
` (11 subsequent siblings)
14 siblings, 0 replies; 21+ messages in thread
From: Christoph Lameter @ 2012-09-28 19:23 UTC (permalink / raw)
To: Pekka Enberg; +Cc: Joonsoo Kim, Glauber Costa, linux-mm, David Rientjes
Kmalloc_node functions are rather similar now so lets move them
into slab.h.
Signed-off-by: Christoph Lameter <cl@linux.com>
Index: linux/include/linux/slab.h
===================================================================
--- linux.orig/include/linux/slab.h 2012-09-28 13:41:04.541431894 -0500
+++ linux/include/linux/slab.h 2012-09-28 13:41:14.677643504 -0500
@@ -262,6 +262,41 @@ static __always_inline int kmalloc_size(
return 0;
}
+
+#ifdef CONFIG_NUMA
+void *__kmalloc_node(size_t size, gfp_t flags, int node);
+void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
+
+#ifdef CONFIG_TRACING
+extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
+ gfp_t gfpflags,
+ int node);
+#else
+static __always_inline void *
+kmem_cache_alloc_node_trace(struct kmem_cache *s,
+ gfp_t gfpflags,
+ int node)
+{
+ return kmem_cache_alloc_node(s, gfpflags, node);
+}
+#endif
+
+static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
+{
+ if (__builtin_constant_p(size) &&
+ size <= KMALLOC_MAX_CACHE_SIZE && !(flags & SLAB_CACHE_DMA)) {
+ int i = kmalloc_index(size);
+
+ if (!i)
+ return ZERO_SIZE_PTR;
+
+ return kmem_cache_alloc_node_trace(kmalloc_caches[i],
+ flags, node);
+ }
+ return __kmalloc_node(size, flags, node);
+}
+#endif
+
#endif /* !CONFIG_SLOB */
/*
Index: linux/include/linux/slub_def.h
===================================================================
--- linux.orig/include/linux/slub_def.h 2012-09-28 13:41:13.073610015 -0500
+++ linux/include/linux/slub_def.h 2012-09-28 13:41:14.677643504 -0500
@@ -200,37 +200,4 @@ static __always_inline void *kmalloc(siz
return __kmalloc(size, flags);
}
-#ifdef CONFIG_NUMA
-void *__kmalloc_node(size_t size, gfp_t flags, int node);
-void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
-
-#ifdef CONFIG_TRACING
-extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
- gfp_t gfpflags,
- int node, size_t size);
-#else
-static __always_inline void *
-kmem_cache_alloc_node_trace(struct kmem_cache *s,
- gfp_t gfpflags,
- int node, size_t size)
-{
- return kmem_cache_alloc_node(s, gfpflags, node);
-}
-#endif
-
-static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
-{
- if (__builtin_constant_p(size) &&
- size <= KMALLOC_MAX_CACHE_SIZE && !(flags & SLUB_DMA)) {
- struct kmem_cache *s = kmalloc_slab_inline(size);
-
- if (!s)
- return ZERO_SIZE_PTR;
-
- return kmem_cache_alloc_node_trace(s, flags, node, size);
- }
- return __kmalloc_node(size, flags, node);
-}
-#endif
-
#endif /* _LINUX_SLUB_DEF_H */
Index: linux/mm/slab.c
===================================================================
--- linux.orig/mm/slab.c 2012-09-28 13:41:13.069609936 -0500
+++ linux/mm/slab.c 2012-09-28 13:41:14.677643504 -0500
@@ -3673,8 +3673,7 @@ void *kmem_cache_alloc_node(struct kmem_
EXPORT_SYMBOL(kmem_cache_alloc_node);
#ifdef CONFIG_TRACING
-void *kmem_cache_alloc_node_trace(size_t size,
- struct kmem_cache *cachep,
+void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
gfp_t flags,
int nodeid)
{
@@ -3683,7 +3682,7 @@ void *kmem_cache_alloc_node_trace(size_t
ret = __cache_alloc_node(cachep, flags, nodeid,
__builtin_return_address(0));
trace_kmalloc_node(_RET_IP_, ret,
- size, slab_buffer_size(cachep),
+ s->size, slab_buffer_size(cachep),
flags, nodeid);
return ret;
}
@@ -3698,7 +3697,7 @@ __do_kmalloc_node(size_t size, gfp_t fla
cachep = kmalloc_slab(size, flags);
if (unlikely(ZERO_OR_NULL_PTR(cachep)))
return cachep;
- return kmem_cache_alloc_node_trace(size, cachep, flags, node);
+ return kmem_cache_alloc_node_trace(cachep, flags, node);
}
#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_TRACING)
Index: linux/include/linux/slab_def.h
===================================================================
--- linux.orig/include/linux/slab_def.h 2012-09-28 13:41:00.493347386 -0500
+++ linux/include/linux/slab_def.h 2012-09-28 13:41:14.681643583 -0500
@@ -144,50 +144,4 @@ static __always_inline void *kmalloc(siz
return __kmalloc(size, flags);
}
-#ifdef CONFIG_NUMA
-extern void *__kmalloc_node(size_t size, gfp_t flags, int node);
-extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
-
-#ifdef CONFIG_TRACING
-extern void *kmem_cache_alloc_node_trace(size_t size,
- struct kmem_cache *cachep,
- gfp_t flags,
- int nodeid);
-#else
-static __always_inline void *
-kmem_cache_alloc_node_trace(size_t size,
- struct kmem_cache *cachep,
- gfp_t flags,
- int nodeid)
-{
- return kmem_cache_alloc_node(cachep, flags, nodeid);
-}
-#endif
-
-static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
-{
- struct kmem_cache *cachep;
-
- if (__builtin_constant_p(size)) {
- int i;
-
- if (!size)
- return ZERO_SIZE_PTR;
-
- i = kmalloc_index(size);
-
-#ifdef CONFIG_ZONE_DMA
- if (flags & GFP_DMA)
- cachep = kmalloc_dma_caches[i];
- else
-#endif
- cachep = kmalloc_caches[i];
-
- return kmem_cache_alloc_node_trace(size, cachep, flags, node);
- }
- return __kmalloc_node(size, flags, node);
-}
-
-#endif /* CONFIG_NUMA */
-
#endif /* _LINUX_SLAB_DEF_H */
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread* CK2 [07/15] slab: Use common kmalloc_index/kmalloc_size functions
[not found] <20120928191715.368450474@linux.com>
` (2 preceding siblings ...)
2012-09-28 19:23 ` CK2 [15/15] Move kmalloc_node functions to common code Christoph Lameter
@ 2012-09-28 19:24 ` Christoph Lameter
2012-10-02 15:10 ` JoonSoo Kim
2012-09-28 19:24 ` CK2 [14/15] Common Kmalloc cache determination Christoph Lameter
` (10 subsequent siblings)
14 siblings, 1 reply; 21+ messages in thread
From: Christoph Lameter @ 2012-09-28 19:24 UTC (permalink / raw)
To: Pekka Enberg; +Cc: Joonsoo Kim, Glauber Costa, linux-mm, David Rientjes
Make slab use the common functions. We can get rid of a lot
of old ugly stuff as a results. Among them the sizes
array and the weird include/linux/kmalloc_sizes file and
some pretty bad #include statements in slab_def.h.
The one thing that is different in slab is that the 32 byte
cache will also be created for arches that have page sizes
larger than 4K. There are numerous smaller allocations that
SLOB and SLUB can handle better because of their support for
smaller allocation sizes so lets keep the 32 byte slab also
for arches with > 4K pages.
Reviewed-by: Glauber Costa <glommer@parallels.com>
Signed-off-by: Christoph Lameter <cl@linux.com>
Index: linux/mm/slab.c
===================================================================
--- linux.orig/mm/slab.c 2012-09-28 12:19:14.518918604 -0500
+++ linux/mm/slab.c 2012-09-28 12:20:12.732133191 -0500
@@ -334,34 +334,10 @@ static void free_block(struct kmem_cache
static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
static void cache_reap(struct work_struct *unused);
-/*
- * This function must be completely optimized away if a constant is passed to
- * it. Mostly the same as what is in linux/slab.h except it returns an index.
- */
-static __always_inline int index_of(const size_t size)
-{
- extern void __bad_size(void);
-
- if (__builtin_constant_p(size)) {
- int i = 0;
-
-#define CACHE(x) \
- if (size <=x) \
- return i; \
- else \
- i++;
-#include <linux/kmalloc_sizes.h>
-#undef CACHE
- __bad_size();
- } else
- __bad_size();
- return 0;
-}
-
static int slab_early_init = 1;
-#define INDEX_AC index_of(sizeof(struct arraycache_init))
-#define INDEX_L3 index_of(sizeof(struct kmem_list3))
+#define INDEX_AC kmalloc_index(sizeof(struct arraycache_init))
+#define INDEX_L3 kmalloc_index(sizeof(struct kmem_list3))
static void kmem_list3_init(struct kmem_list3 *parent)
{
@@ -548,30 +524,6 @@ static inline unsigned int obj_to_index(
return reciprocal_divide(offset, cache->reciprocal_buffer_size);
}
-/*
- * These are the default caches for kmalloc. Custom caches can have other sizes.
- */
-struct cache_sizes malloc_sizes[] = {
-#define CACHE(x) { .cs_size = (x) },
-#include <linux/kmalloc_sizes.h>
- CACHE(ULONG_MAX)
-#undef CACHE
-};
-EXPORT_SYMBOL(malloc_sizes);
-
-/* Must match cache_sizes above. Out of line to keep cache footprint low. */
-struct cache_names {
- char *name;
- char *name_dma;
-};
-
-static struct cache_names __initdata cache_names[] = {
-#define CACHE(x) { .name = "size-" #x, .name_dma = "size-" #x "(DMA)" },
-#include <linux/kmalloc_sizes.h>
- {NULL,}
-#undef CACHE
-};
-
static struct arraycache_init initarray_generic =
{ {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
@@ -649,19 +601,23 @@ static void slab_set_debugobj_lock_class
static void init_node_lock_keys(int q)
{
- struct cache_sizes *s = malloc_sizes;
+ int i;
if (slab_state < UP)
return;
- for (s = malloc_sizes; s->cs_size != ULONG_MAX; s++) {
+ for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
struct kmem_list3 *l3;
+ struct kmem_cache *cache = kmalloc_caches[i];
+
+ if (!cache)
+ continue;
- l3 = s->cs_cachep->nodelists[q];
- if (!l3 || OFF_SLAB(s->cs_cachep))
+ l3 = cache->nodelists[q];
+ if (!l3 || OFF_SLAB(cache))
continue;
- slab_set_lock_classes(s->cs_cachep, &on_slab_l3_key,
+ slab_set_lock_classes(cache, &on_slab_l3_key,
&on_slab_alc_key, q);
}
}
@@ -701,20 +657,19 @@ static inline struct array_cache *cpu_ca
static inline struct kmem_cache *__find_general_cachep(size_t size,
gfp_t gfpflags)
{
- struct cache_sizes *csizep = malloc_sizes;
+ int i;
#if DEBUG
/* This happens if someone tries to call
* kmem_cache_create(), or __kmalloc(), before
* the generic caches are initialized.
*/
- BUG_ON(malloc_sizes[INDEX_AC].cs_cachep == NULL);
+ BUG_ON(kmalloc_caches[INDEX_AC] == NULL);
#endif
if (!size)
return ZERO_SIZE_PTR;
- while (size > csizep->cs_size)
- csizep++;
+ i = kmalloc_index(size);
/*
* Really subtle: The last entry with cs->cs_size==ULONG_MAX
@@ -723,9 +678,9 @@ static inline struct kmem_cache *__find_
*/
#ifdef CONFIG_ZONE_DMA
if (unlikely(gfpflags & GFP_DMA))
- return csizep->cs_dmacachep;
+ return kmalloc_dma_caches[i];
#endif
- return csizep->cs_cachep;
+ return kmalloc_caches[i];
}
static struct kmem_cache *kmem_find_general_cachep(size_t size, gfp_t gfpflags)
@@ -1594,8 +1549,6 @@ static void setup_nodelists_pointer(stru
*/
void __init kmem_cache_init(void)
{
- struct cache_sizes *sizes;
- struct cache_names *names;
int i;
kmem_cache = &kmem_cache_boot;
@@ -1653,8 +1606,6 @@ void __init kmem_cache_init(void)
slab_state = PARTIAL;
/* 2+3) create the kmalloc caches */
- sizes = malloc_sizes;
- names = cache_names;
/*
* Initialize the caches that provide memory for the array cache and the
@@ -1662,35 +1613,39 @@ void __init kmem_cache_init(void)
* bug.
*/
- sizes[INDEX_AC].cs_cachep = create_kmalloc_cache(names[INDEX_AC].name,
- sizes[INDEX_AC].cs_size, ARCH_KMALLOC_FLAGS);
+ kmalloc_caches[INDEX_AC] = create_kmalloc_cache("kmalloc-ac",
+ kmalloc_size(INDEX_AC), ARCH_KMALLOC_FLAGS);
if (INDEX_AC != INDEX_L3)
- sizes[INDEX_L3].cs_cachep =
- create_kmalloc_cache(names[INDEX_L3].name,
- sizes[INDEX_L3].cs_size, ARCH_KMALLOC_FLAGS);
+ kmalloc_caches[INDEX_L3] =
+ create_kmalloc_cache("kmalloc-l3",
+ kmalloc_size(INDEX_L3), ARCH_KMALLOC_FLAGS);
slab_early_init = 0;
- while (sizes->cs_size != ULONG_MAX) {
- /*
- * For performance, all the general caches are L1 aligned.
- * This should be particularly beneficial on SMP boxes, as it
- * eliminates "false sharing".
- * Note for systems short on memory removing the alignment will
- * allow tighter packing of the smaller caches.
- */
- if (!sizes->cs_cachep)
- sizes->cs_cachep = create_kmalloc_cache(names->name,
- sizes->cs_size, ARCH_KMALLOC_FLAGS);
+ for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
+ size_t cs_size = kmalloc_size(i);
+
+ if (cs_size < KMALLOC_MIN_SIZE)
+ continue;
+
+ if (!kmalloc_caches[i]) {
+ /*
+ * For performance, all the general caches are L1 aligned.
+ * This should be particularly beneficial on SMP boxes, as it
+ * eliminates "false sharing".
+ * Note for systems short on memory removing the alignment will
+ * allow tighter packing of the smaller caches.
+ */
+ kmalloc_caches[i] = create_kmalloc_cache("kmalloc",
+ cs_size, ARCH_KMALLOC_FLAGS);
+ }
#ifdef CONFIG_ZONE_DMA
- sizes->cs_dmacachep = create_kmalloc_cache(
- names->name_dma, sizes->cs_size,
+ kmalloc_dma_caches[i] = create_kmalloc_cache(
+ "kmalloc-dma", cs_size,
SLAB_CACHE_DMA|ARCH_KMALLOC_FLAGS);
#endif
- sizes++;
- names++;
}
/* 4) Replace the bootstrap head arrays */
{
@@ -1709,17 +1664,16 @@ void __init kmem_cache_init(void)
ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
- BUG_ON(cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep)
+ BUG_ON(cpu_cache_get(kmalloc_caches[INDEX_AC])
!= &initarray_generic.cache);
- memcpy(ptr, cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep),
+ memcpy(ptr, cpu_cache_get(kmalloc_caches[INDEX_AC]),
sizeof(struct arraycache_init));
/*
* Do not assume that spinlocks can be initialized via memcpy:
*/
spin_lock_init(&ptr->lock);
- malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] =
- ptr;
+ kmalloc_caches[INDEX_AC]->array[smp_processor_id()] = ptr;
}
/* 5) Replace the bootstrap kmem_list3's */
{
@@ -1728,17 +1682,39 @@ void __init kmem_cache_init(void)
for_each_online_node(nid) {
init_list(kmem_cache, &initkmem_list3[CACHE_CACHE + nid], nid);
- init_list(malloc_sizes[INDEX_AC].cs_cachep,
+ init_list(kmalloc_caches[INDEX_AC],
&initkmem_list3[SIZE_AC + nid], nid);
if (INDEX_AC != INDEX_L3) {
- init_list(malloc_sizes[INDEX_L3].cs_cachep,
+ init_list(kmalloc_caches[INDEX_L3],
&initkmem_list3[SIZE_L3 + nid], nid);
}
}
}
slab_state = UP;
+
+ /* Create the proper names */
+ for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
+ char *s;
+ struct kmem_cache *c = kmalloc_caches[i];
+
+ if (!c)
+ continue;
+
+ s = kasprintf(GFP_NOWAIT, "kmalloc-%d", kmalloc_size(i));
+
+ BUG_ON(!s);
+ c->name = s;
+
+#ifdef CONFIG_ZONE_DMA
+ c = kmalloc_dma_caches[i];
+ BUG_ON(!c);
+ s = kasprintf(GFP_NOWAIT, "dma-kmalloc-%d", kmalloc_size(i));
+ BUG_ON(!s);
+ c->name = s;
+#endif
+ }
}
void __init kmem_cache_init_late(void)
@@ -2430,7 +2406,7 @@ __kmem_cache_create (struct kmem_cache *
size += BYTES_PER_WORD;
}
#if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC)
- if (size >= malloc_sizes[INDEX_L3 + 1].cs_size
+ if (size >= kmalloc_size(INDEX_L3 + 1))
&& cachep->object_size > cache_line_size() && ALIGN(size, align) < PAGE_SIZE) {
cachep->obj_offset += PAGE_SIZE - ALIGN(size, align);
size = PAGE_SIZE;
Index: linux/include/linux/kmalloc_sizes.h
===================================================================
--- linux.orig/include/linux/kmalloc_sizes.h 2012-09-28 12:19:14.498918200 -0500
+++ /dev/null 1970-01-01 00:00:00.000000000 +0000
@@ -1,45 +0,0 @@
-#if (PAGE_SIZE == 4096)
- CACHE(32)
-#endif
- CACHE(64)
-#if L1_CACHE_BYTES < 64
- CACHE(96)
-#endif
- CACHE(128)
-#if L1_CACHE_BYTES < 128
- CACHE(192)
-#endif
- CACHE(256)
- CACHE(512)
- CACHE(1024)
- CACHE(2048)
- CACHE(4096)
- CACHE(8192)
- CACHE(16384)
- CACHE(32768)
- CACHE(65536)
- CACHE(131072)
-#if KMALLOC_MAX_SIZE >= 262144
- CACHE(262144)
-#endif
-#if KMALLOC_MAX_SIZE >= 524288
- CACHE(524288)
-#endif
-#if KMALLOC_MAX_SIZE >= 1048576
- CACHE(1048576)
-#endif
-#if KMALLOC_MAX_SIZE >= 2097152
- CACHE(2097152)
-#endif
-#if KMALLOC_MAX_SIZE >= 4194304
- CACHE(4194304)
-#endif
-#if KMALLOC_MAX_SIZE >= 8388608
- CACHE(8388608)
-#endif
-#if KMALLOC_MAX_SIZE >= 16777216
- CACHE(16777216)
-#endif
-#if KMALLOC_MAX_SIZE >= 33554432
- CACHE(33554432)
-#endif
Index: linux/include/linux/slab_def.h
===================================================================
--- linux.orig/include/linux/slab_def.h 2012-09-28 12:19:14.510918425 -0500
+++ linux/include/linux/slab_def.h 2012-09-28 12:19:20.427041875 -0500
@@ -11,8 +11,6 @@
*/
#include <linux/init.h>
-#include <asm/page.h> /* kmalloc_sizes.h needs PAGE_SIZE */
-#include <asm/cache.h> /* kmalloc_sizes.h needs L1_CACHE_BYTES */
#include <linux/compiler.h>
/*
@@ -97,15 +95,8 @@ struct kmem_cache {
*/
};
-/* Size description struct for general caches. */
-struct cache_sizes {
- size_t cs_size;
- struct kmem_cache *cs_cachep;
-#ifdef CONFIG_ZONE_DMA
- struct kmem_cache *cs_dmacachep;
-#endif
-};
-extern struct cache_sizes malloc_sizes[];
+extern struct kmem_cache *kmalloc_caches[PAGE_SHIFT + MAX_ORDER];
+extern struct kmem_cache *kmalloc_dma_caches[PAGE_SHIFT + MAX_ORDER];
void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
void *__kmalloc(size_t size, gfp_t flags);
@@ -132,26 +123,19 @@ static __always_inline void *kmalloc(siz
void *ret;
if (__builtin_constant_p(size)) {
- int i = 0;
+ int i;
if (!size)
return ZERO_SIZE_PTR;
-#define CACHE(x) \
- if (size <= x) \
- goto found; \
- else \
- i++;
-#include <linux/kmalloc_sizes.h>
-#undef CACHE
- return NULL;
-found:
+ i = kmalloc_index(size);
+
#ifdef CONFIG_ZONE_DMA
if (flags & GFP_DMA)
- cachep = malloc_sizes[i].cs_dmacachep;
+ cachep = kmalloc_dma_caches[i];
else
#endif
- cachep = malloc_sizes[i].cs_cachep;
+ cachep = kmalloc_caches[i];
ret = kmem_cache_alloc_trace(size, cachep, flags);
@@ -185,26 +169,19 @@ static __always_inline void *kmalloc_nod
struct kmem_cache *cachep;
if (__builtin_constant_p(size)) {
- int i = 0;
+ int i;
if (!size)
return ZERO_SIZE_PTR;
-#define CACHE(x) \
- if (size <= x) \
- goto found; \
- else \
- i++;
-#include <linux/kmalloc_sizes.h>
-#undef CACHE
- return NULL;
-found:
+ i = kmalloc_index(size);
+
#ifdef CONFIG_ZONE_DMA
if (flags & GFP_DMA)
- cachep = malloc_sizes[i].cs_dmacachep;
+ cachep = kmalloc_dma_caches[i];
else
#endif
- cachep = malloc_sizes[i].cs_cachep;
+ cachep = kmalloc_caches[i];
return kmem_cache_alloc_node_trace(size, cachep, flags, node);
}
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: CK2 [07/15] slab: Use common kmalloc_index/kmalloc_size functions
2012-09-28 19:24 ` CK2 [07/15] slab: Use common kmalloc_index/kmalloc_size functions Christoph Lameter
@ 2012-10-02 15:10 ` JoonSoo Kim
2012-10-03 15:06 ` Christoph Lameter
0 siblings, 1 reply; 21+ messages in thread
From: JoonSoo Kim @ 2012-10-02 15:10 UTC (permalink / raw)
To: Christoph Lameter; +Cc: Pekka Enberg, Glauber Costa, linux-mm, David Rientjes
2012/9/29 Christoph Lameter <cl@linux.com>:
> Make slab use the common functions. We can get rid of a lot
> of old ugly stuff as a results. Among them the sizes
> array and the weird include/linux/kmalloc_sizes file and
> some pretty bad #include statements in slab_def.h.
>
> The one thing that is different in slab is that the 32 byte
> cache will also be created for arches that have page sizes
> larger than 4K. There are numerous smaller allocations that
> SLOB and SLUB can handle better because of their support for
> smaller allocation sizes so lets keep the 32 byte slab also
> for arches with > 4K pages.
>
> Reviewed-by: Glauber Costa <glommer@parallels.com>
> Signed-off-by: Christoph Lameter <cl@linux.com>
>
> Index: linux/mm/slab.c
> ===================================================================
> --- linux.orig/mm/slab.c 2012-09-28 12:19:14.518918604 -0500
> +++ linux/mm/slab.c 2012-09-28 12:20:12.732133191 -0500
> @@ -334,34 +334,10 @@ static void free_block(struct kmem_cache
> static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
> static void cache_reap(struct work_struct *unused);
>
> -/*
> - * This function must be completely optimized away if a constant is passed to
> - * it. Mostly the same as what is in linux/slab.h except it returns an index.
> - */
> -static __always_inline int index_of(const size_t size)
> -{
> - extern void __bad_size(void);
> -
> - if (__builtin_constant_p(size)) {
> - int i = 0;
> -
> -#define CACHE(x) \
> - if (size <=x) \
> - return i; \
> - else \
> - i++;
> -#include <linux/kmalloc_sizes.h>
> -#undef CACHE
> - __bad_size();
> - } else
> - __bad_size();
> - return 0;
> -}
> -
> static int slab_early_init = 1;
>
> -#define INDEX_AC index_of(sizeof(struct arraycache_init))
> -#define INDEX_L3 index_of(sizeof(struct kmem_list3))
> +#define INDEX_AC kmalloc_index(sizeof(struct arraycache_init))
> +#define INDEX_L3 kmalloc_index(sizeof(struct kmem_list3))
>
> static void kmem_list3_init(struct kmem_list3 *parent)
> {
> @@ -548,30 +524,6 @@ static inline unsigned int obj_to_index(
> return reciprocal_divide(offset, cache->reciprocal_buffer_size);
> }
>
> -/*
> - * These are the default caches for kmalloc. Custom caches can have other sizes.
> - */
> -struct cache_sizes malloc_sizes[] = {
> -#define CACHE(x) { .cs_size = (x) },
> -#include <linux/kmalloc_sizes.h>
> - CACHE(ULONG_MAX)
> -#undef CACHE
> -};
> -EXPORT_SYMBOL(malloc_sizes);
> -
> -/* Must match cache_sizes above. Out of line to keep cache footprint low. */
> -struct cache_names {
> - char *name;
> - char *name_dma;
> -};
> -
> -static struct cache_names __initdata cache_names[] = {
> -#define CACHE(x) { .name = "size-" #x, .name_dma = "size-" #x "(DMA)" },
> -#include <linux/kmalloc_sizes.h>
> - {NULL,}
> -#undef CACHE
> -};
> -
> static struct arraycache_init initarray_generic =
> { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
>
> @@ -649,19 +601,23 @@ static void slab_set_debugobj_lock_class
>
> static void init_node_lock_keys(int q)
> {
> - struct cache_sizes *s = malloc_sizes;
> + int i;
>
> if (slab_state < UP)
> return;
>
> - for (s = malloc_sizes; s->cs_size != ULONG_MAX; s++) {
> + for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
> struct kmem_list3 *l3;
> + struct kmem_cache *cache = kmalloc_caches[i];
> +
> + if (!cache)
> + continue;
>
> - l3 = s->cs_cachep->nodelists[q];
> - if (!l3 || OFF_SLAB(s->cs_cachep))
> + l3 = cache->nodelists[q];
> + if (!l3 || OFF_SLAB(cache))
> continue;
>
> - slab_set_lock_classes(s->cs_cachep, &on_slab_l3_key,
> + slab_set_lock_classes(cache, &on_slab_l3_key,
> &on_slab_alc_key, q);
> }
> }
> @@ -701,20 +657,19 @@ static inline struct array_cache *cpu_ca
> static inline struct kmem_cache *__find_general_cachep(size_t size,
> gfp_t gfpflags)
> {
> - struct cache_sizes *csizep = malloc_sizes;
> + int i;
>
> #if DEBUG
> /* This happens if someone tries to call
> * kmem_cache_create(), or __kmalloc(), before
> * the generic caches are initialized.
> */
> - BUG_ON(malloc_sizes[INDEX_AC].cs_cachep == NULL);
> + BUG_ON(kmalloc_caches[INDEX_AC] == NULL);
> #endif
> if (!size)
> return ZERO_SIZE_PTR;
>
> - while (size > csizep->cs_size)
> - csizep++;
> + i = kmalloc_index(size);
>
> /*
> * Really subtle: The last entry with cs->cs_size==ULONG_MAX
> @@ -723,9 +678,9 @@ static inline struct kmem_cache *__find_
> */
> #ifdef CONFIG_ZONE_DMA
> if (unlikely(gfpflags & GFP_DMA))
> - return csizep->cs_dmacachep;
> + return kmalloc_dma_caches[i];
> #endif
> - return csizep->cs_cachep;
> + return kmalloc_caches[i];
> }
>
> static struct kmem_cache *kmem_find_general_cachep(size_t size, gfp_t gfpflags)
> @@ -1594,8 +1549,6 @@ static void setup_nodelists_pointer(stru
> */
> void __init kmem_cache_init(void)
> {
> - struct cache_sizes *sizes;
> - struct cache_names *names;
> int i;
>
> kmem_cache = &kmem_cache_boot;
> @@ -1653,8 +1606,6 @@ void __init kmem_cache_init(void)
> slab_state = PARTIAL;
>
> /* 2+3) create the kmalloc caches */
> - sizes = malloc_sizes;
> - names = cache_names;
>
> /*
> * Initialize the caches that provide memory for the array cache and the
> @@ -1662,35 +1613,39 @@ void __init kmem_cache_init(void)
> * bug.
> */
>
> - sizes[INDEX_AC].cs_cachep = create_kmalloc_cache(names[INDEX_AC].name,
> - sizes[INDEX_AC].cs_size, ARCH_KMALLOC_FLAGS);
> + kmalloc_caches[INDEX_AC] = create_kmalloc_cache("kmalloc-ac",
> + kmalloc_size(INDEX_AC), ARCH_KMALLOC_FLAGS);
>
> if (INDEX_AC != INDEX_L3)
> - sizes[INDEX_L3].cs_cachep =
> - create_kmalloc_cache(names[INDEX_L3].name,
> - sizes[INDEX_L3].cs_size, ARCH_KMALLOC_FLAGS);
> + kmalloc_caches[INDEX_L3] =
> + create_kmalloc_cache("kmalloc-l3",
> + kmalloc_size(INDEX_L3), ARCH_KMALLOC_FLAGS);
>
> slab_early_init = 0;
>
> - while (sizes->cs_size != ULONG_MAX) {
> - /*
> - * For performance, all the general caches are L1 aligned.
> - * This should be particularly beneficial on SMP boxes, as it
> - * eliminates "false sharing".
> - * Note for systems short on memory removing the alignment will
> - * allow tighter packing of the smaller caches.
> - */
> - if (!sizes->cs_cachep)
> - sizes->cs_cachep = create_kmalloc_cache(names->name,
> - sizes->cs_size, ARCH_KMALLOC_FLAGS);
> + for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
> + size_t cs_size = kmalloc_size(i);
> +
> + if (cs_size < KMALLOC_MIN_SIZE)
> + continue;
> +
> + if (!kmalloc_caches[i]) {
> + /*
> + * For performance, all the general caches are L1 aligned.
> + * This should be particularly beneficial on SMP boxes, as it
> + * eliminates "false sharing".
> + * Note for systems short on memory removing the alignment will
> + * allow tighter packing of the smaller caches.
> + */
> + kmalloc_caches[i] = create_kmalloc_cache("kmalloc",
> + cs_size, ARCH_KMALLOC_FLAGS);
> + }
>
> #ifdef CONFIG_ZONE_DMA
> - sizes->cs_dmacachep = create_kmalloc_cache(
> - names->name_dma, sizes->cs_size,
> + kmalloc_dma_caches[i] = create_kmalloc_cache(
> + "kmalloc-dma", cs_size,
> SLAB_CACHE_DMA|ARCH_KMALLOC_FLAGS);
> #endif
> - sizes++;
> - names++;
> }
> /* 4) Replace the bootstrap head arrays */
> {
> @@ -1709,17 +1664,16 @@ void __init kmem_cache_init(void)
>
> ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
>
> - BUG_ON(cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep)
> + BUG_ON(cpu_cache_get(kmalloc_caches[INDEX_AC])
> != &initarray_generic.cache);
> - memcpy(ptr, cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep),
> + memcpy(ptr, cpu_cache_get(kmalloc_caches[INDEX_AC]),
> sizeof(struct arraycache_init));
> /*
> * Do not assume that spinlocks can be initialized via memcpy:
> */
> spin_lock_init(&ptr->lock);
>
> - malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] =
> - ptr;
> + kmalloc_caches[INDEX_AC]->array[smp_processor_id()] = ptr;
> }
> /* 5) Replace the bootstrap kmem_list3's */
> {
> @@ -1728,17 +1682,39 @@ void __init kmem_cache_init(void)
> for_each_online_node(nid) {
> init_list(kmem_cache, &initkmem_list3[CACHE_CACHE + nid], nid);
>
> - init_list(malloc_sizes[INDEX_AC].cs_cachep,
> + init_list(kmalloc_caches[INDEX_AC],
> &initkmem_list3[SIZE_AC + nid], nid);
>
> if (INDEX_AC != INDEX_L3) {
> - init_list(malloc_sizes[INDEX_L3].cs_cachep,
> + init_list(kmalloc_caches[INDEX_L3],
> &initkmem_list3[SIZE_L3 + nid], nid);
> }
> }
> }
>
> slab_state = UP;
> +
> + /* Create the proper names */
> + for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
> + char *s;
> + struct kmem_cache *c = kmalloc_caches[i];
> +
> + if (!c)
> + continue;
> +
> + s = kasprintf(GFP_NOWAIT, "kmalloc-%d", kmalloc_size(i));
> +
> + BUG_ON(!s);
> + c->name = s;
> +
> +#ifdef CONFIG_ZONE_DMA
> + c = kmalloc_dma_caches[i];
> + BUG_ON(!c);
> + s = kasprintf(GFP_NOWAIT, "dma-kmalloc-%d", kmalloc_size(i));
> + BUG_ON(!s);
> + c->name = s;
> +#endif
> + }
> }
>
> void __init kmem_cache_init_late(void)
> @@ -2430,7 +2406,7 @@ __kmem_cache_create (struct kmem_cache *
> size += BYTES_PER_WORD;
> }
> #if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC)
> - if (size >= malloc_sizes[INDEX_L3 + 1].cs_size
> + if (size >= kmalloc_size(INDEX_L3 + 1))
> && cachep->object_size > cache_line_size() && ALIGN(size, align) < PAGE_SIZE) {
> cachep->obj_offset += PAGE_SIZE - ALIGN(size, align);
> size = PAGE_SIZE;
> Index: linux/include/linux/kmalloc_sizes.h
> ===================================================================
> --- linux.orig/include/linux/kmalloc_sizes.h 2012-09-28 12:19:14.498918200 -0500
> +++ /dev/null 1970-01-01 00:00:00.000000000 +0000
> @@ -1,45 +0,0 @@
> -#if (PAGE_SIZE == 4096)
> - CACHE(32)
> -#endif
> - CACHE(64)
> -#if L1_CACHE_BYTES < 64
> - CACHE(96)
> -#endif
> - CACHE(128)
> -#if L1_CACHE_BYTES < 128
> - CACHE(192)
> -#endif
> - CACHE(256)
> - CACHE(512)
> - CACHE(1024)
> - CACHE(2048)
> - CACHE(4096)
> - CACHE(8192)
> - CACHE(16384)
> - CACHE(32768)
> - CACHE(65536)
> - CACHE(131072)
> -#if KMALLOC_MAX_SIZE >= 262144
> - CACHE(262144)
> -#endif
> -#if KMALLOC_MAX_SIZE >= 524288
> - CACHE(524288)
> -#endif
> -#if KMALLOC_MAX_SIZE >= 1048576
> - CACHE(1048576)
> -#endif
> -#if KMALLOC_MAX_SIZE >= 2097152
> - CACHE(2097152)
> -#endif
> -#if KMALLOC_MAX_SIZE >= 4194304
> - CACHE(4194304)
> -#endif
> -#if KMALLOC_MAX_SIZE >= 8388608
> - CACHE(8388608)
> -#endif
> -#if KMALLOC_MAX_SIZE >= 16777216
> - CACHE(16777216)
> -#endif
> -#if KMALLOC_MAX_SIZE >= 33554432
> - CACHE(33554432)
> -#endif
> Index: linux/include/linux/slab_def.h
> ===================================================================
> --- linux.orig/include/linux/slab_def.h 2012-09-28 12:19:14.510918425 -0500
> +++ linux/include/linux/slab_def.h 2012-09-28 12:19:20.427041875 -0500
> @@ -11,8 +11,6 @@
> */
>
> #include <linux/init.h>
> -#include <asm/page.h> /* kmalloc_sizes.h needs PAGE_SIZE */
> -#include <asm/cache.h> /* kmalloc_sizes.h needs L1_CACHE_BYTES */
> #include <linux/compiler.h>
>
> /*
> @@ -97,15 +95,8 @@ struct kmem_cache {
> */
> };
>
> -/* Size description struct for general caches. */
> -struct cache_sizes {
> - size_t cs_size;
> - struct kmem_cache *cs_cachep;
> -#ifdef CONFIG_ZONE_DMA
> - struct kmem_cache *cs_dmacachep;
> -#endif
> -};
> -extern struct cache_sizes malloc_sizes[];
> +extern struct kmem_cache *kmalloc_caches[PAGE_SHIFT + MAX_ORDER];
> +extern struct kmem_cache *kmalloc_dma_caches[PAGE_SHIFT + MAX_ORDER];
>
> void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
> void *__kmalloc(size_t size, gfp_t flags);
> @@ -132,26 +123,19 @@ static __always_inline void *kmalloc(siz
> void *ret;
>
> if (__builtin_constant_p(size)) {
> - int i = 0;
> + int i;
>
> if (!size)
> return ZERO_SIZE_PTR;
>
> -#define CACHE(x) \
> - if (size <= x) \
> - goto found; \
> - else \
> - i++;
> -#include <linux/kmalloc_sizes.h>
> -#undef CACHE
> - return NULL;
> -found:
> + i = kmalloc_index(size);
> +
> #ifdef CONFIG_ZONE_DMA
> if (flags & GFP_DMA)
> - cachep = malloc_sizes[i].cs_dmacachep;
> + cachep = kmalloc_dma_caches[i];
> else
> #endif
> - cachep = malloc_sizes[i].cs_cachep;
> + cachep = kmalloc_caches[i];
>
> ret = kmem_cache_alloc_trace(size, cachep, flags);
>
> @@ -185,26 +169,19 @@ static __always_inline void *kmalloc_nod
> struct kmem_cache *cachep;
>
> if (__builtin_constant_p(size)) {
> - int i = 0;
> + int i;
>
> if (!size)
> return ZERO_SIZE_PTR;
>
> -#define CACHE(x) \
> - if (size <= x) \
> - goto found; \
> - else \
> - i++;
> -#include <linux/kmalloc_sizes.h>
> -#undef CACHE
> - return NULL;
> -found:
> + i = kmalloc_index(size);
> +
> #ifdef CONFIG_ZONE_DMA
> if (flags & GFP_DMA)
> - cachep = malloc_sizes[i].cs_dmacachep;
> + cachep = kmalloc_dma_caches[i];
> else
> #endif
> - cachep = malloc_sizes[i].cs_cachep;
> + cachep = kmalloc_caches[i];
>
> return kmem_cache_alloc_node_trace(size, cachep, flags, node);
> }
>
Now, I am just reading the codes and I cannot find a defintion of the
kmalloc_caches in slab.c.
Please let me know where I can find definition of kmalloc_caches for slab.c.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: CK2 [07/15] slab: Use common kmalloc_index/kmalloc_size functions
2012-10-02 15:10 ` JoonSoo Kim
@ 2012-10-03 15:06 ` Christoph Lameter
0 siblings, 0 replies; 21+ messages in thread
From: Christoph Lameter @ 2012-10-03 15:06 UTC (permalink / raw)
To: JoonSoo Kim; +Cc: Pekka Enberg, Glauber Costa, linux-mm, David Rientjes
On Wed, 3 Oct 2012, JoonSoo Kim wrote:
> Now, I am just reading the codes and I cannot find a defintion of the
> kmalloc_caches in slab.c.
> Please let me know where I can find definition of kmalloc_caches for slab.c.
Its gone.... ;-) Did not pay enough attention to the early patches in
the last round it seems. Some hunks were moving around. Added the new
names to this patch and then removed them in the patch that moves the
definition to slab_common.c.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread
* CK2 [14/15] Common Kmalloc cache determination
[not found] <20120928191715.368450474@linux.com>
` (3 preceding siblings ...)
2012-09-28 19:24 ` CK2 [07/15] slab: Use common kmalloc_index/kmalloc_size functions Christoph Lameter
@ 2012-09-28 19:24 ` Christoph Lameter
2012-09-28 19:24 ` CK2 [12/15] Common function to create the kmalloc array Christoph Lameter
` (9 subsequent siblings)
14 siblings, 0 replies; 21+ messages in thread
From: Christoph Lameter @ 2012-09-28 19:24 UTC (permalink / raw)
To: Pekka Enberg; +Cc: Joonsoo Kim, Glauber Costa, linux-mm, David Rientjes
Extract the optimized lookup functions from slub and put them into
slab_common.c. Then make slab use these functions as well.
Signed-off-by: Christoph Lameter <cl@linux.com>
Index: linux/mm/slab.c
===================================================================
--- linux.orig/mm/slab.c 2012-09-28 13:23:51.907874005 -0500
+++ linux/mm/slab.c 2012-09-28 13:24:16.364384581 -0500
@@ -654,40 +654,6 @@ static inline struct array_cache *cpu_ca
return cachep->array[smp_processor_id()];
}
-static inline struct kmem_cache *__find_general_cachep(size_t size,
- gfp_t gfpflags)
-{
- int i;
-
-#if DEBUG
- /* This happens if someone tries to call
- * kmem_cache_create(), or __kmalloc(), before
- * the generic caches are initialized.
- */
- BUG_ON(kmalloc_caches[INDEX_AC] == NULL);
-#endif
- if (!size)
- return ZERO_SIZE_PTR;
-
- i = kmalloc_index(size);
-
- /*
- * Really subtle: The last entry with cs->cs_size==ULONG_MAX
- * has cs_{dma,}cachep==NULL. Thus no special case
- * for large kmalloc calls required.
- */
-#ifdef CONFIG_ZONE_DMA
- if (unlikely(gfpflags & GFP_DMA))
- return kmalloc_dma_caches[i];
-#endif
- return kmalloc_caches[i];
-}
-
-static struct kmem_cache *kmem_find_general_cachep(size_t size, gfp_t gfpflags)
-{
- return __find_general_cachep(size, gfpflags);
-}
-
static size_t slab_mgmt_size(size_t nr_objs, size_t align)
{
return ALIGN(sizeof(struct slab)+nr_objs*sizeof(kmem_bufctl_t), align);
@@ -2430,7 +2396,7 @@ __kmem_cache_create (struct kmem_cache *
cachep->reciprocal_buffer_size = reciprocal_value(size);
if (flags & CFLGS_OFF_SLAB) {
- cachep->slabp_cache = kmem_find_general_cachep(slab_size, 0u);
+ cachep->slabp_cache = kmalloc_slab(slab_size, 0u);
/*
* This is a possibility for one of the malloc_sizes caches.
* But since we go off slab only for object size greater than
@@ -3729,7 +3695,7 @@ __do_kmalloc_node(size_t size, gfp_t fla
{
struct kmem_cache *cachep;
- cachep = kmem_find_general_cachep(size, flags);
+ cachep = kmalloc_slab(size, flags);
if (unlikely(ZERO_OR_NULL_PTR(cachep)))
return cachep;
return kmem_cache_alloc_node_trace(size, cachep, flags, node);
@@ -3775,7 +3741,7 @@ static __always_inline void *__do_kmallo
* Then kmalloc uses the uninlined functions instead of the inline
* functions.
*/
- cachep = __find_general_cachep(size, flags);
+ cachep = kmalloc_slab(size, flags);
if (unlikely(ZERO_OR_NULL_PTR(cachep)))
return cachep;
ret = __cache_alloc(cachep, flags, caller);
Index: linux/mm/slab.h
===================================================================
--- linux.orig/mm/slab.h 2012-09-28 13:24:08.000000000 -0500
+++ linux/mm/slab.h 2012-09-28 13:24:54.497180671 -0500
@@ -38,6 +38,9 @@ unsigned long calculate_alignment(unsign
#ifndef CONFIG_SLOB
/* Kmalloc array related functions */
void create_kmalloc_caches(unsigned long);
+
+/* Find the kmalloc slab corresponding for a certain size */
+struct kmem_cache *kmalloc_slab(size_t, gfp_t);
#endif
Index: linux/mm/slab_common.c
===================================================================
--- linux.orig/mm/slab_common.c 2012-09-28 13:23:51.000000000 -0500
+++ linux/mm/slab_common.c 2012-09-28 13:24:16.364384581 -0500
@@ -260,6 +260,68 @@ EXPORT_SYMBOL(kmalloc_dma_caches);
#endif
/*
+ * Conversion table for small slabs sizes / 8 to the index in the
+ * kmalloc array. This is necessary for slabs < 192 since we have non power
+ * of two cache sizes there. The size of larger slabs can be determined using
+ * fls.
+ */
+static s8 size_index[24] = {
+ 3, /* 8 */
+ 4, /* 16 */
+ 5, /* 24 */
+ 5, /* 32 */
+ 6, /* 40 */
+ 6, /* 48 */
+ 6, /* 56 */
+ 6, /* 64 */
+ 1, /* 72 */
+ 1, /* 80 */
+ 1, /* 88 */
+ 1, /* 96 */
+ 7, /* 104 */
+ 7, /* 112 */
+ 7, /* 120 */
+ 7, /* 128 */
+ 2, /* 136 */
+ 2, /* 144 */
+ 2, /* 152 */
+ 2, /* 160 */
+ 2, /* 168 */
+ 2, /* 176 */
+ 2, /* 184 */
+ 2 /* 192 */
+};
+
+static inline int size_index_elem(size_t bytes)
+{
+ return (bytes - 1) / 8;
+}
+
+/*
+ * Find the kmem_cache structure that serves a given size of
+ * allocation
+ */
+struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
+{
+ int index;
+
+ if (size <= 192) {
+ if (!size)
+ return ZERO_SIZE_PTR;
+
+ index = size_index[size_index_elem(size)];
+ } else
+ index = fls(size - 1);
+
+#ifdef CONFIG_ZONE_DMA
+ if (unlikely((flags & SLAB_CACHE_DMA)))
+ return kmalloc_dma_caches[index];
+
+#endif
+ return kmalloc_caches[index];
+}
+
+/*
* Create the kmalloc array. Some of the regular kmalloc arrays
* may already have been created because they were needed to
* enable allocations for slab creation.
@@ -268,6 +330,47 @@ void __init create_kmalloc_caches(unsign
{
int i;
+ /*
+ * Patch up the size_index table if we have strange large alignment
+ * requirements for the kmalloc array. This is only the case for
+ * MIPS it seems. The standard arches will not generate any code here.
+ *
+ * Largest permitted alignment is 256 bytes due to the way we
+ * handle the index determination for the smaller caches.
+ *
+ * Make sure that nothing crazy happens if someone starts tinkering
+ * around with ARCH_KMALLOC_MINALIGN
+ */
+ BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
+ (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));
+
+ for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
+ int elem = size_index_elem(i);
+
+ if (elem >= ARRAY_SIZE(size_index))
+ break;
+ size_index[elem] = KMALLOC_SHIFT_LOW;
+ }
+
+ if (KMALLOC_MIN_SIZE >= 64) {
+ /*
+ * The 96 byte size cache is not used if the alignment
+ * is 64 byte.
+ */
+ for (i = 64 + 8; i <= 96; i += 8)
+ size_index[size_index_elem(i)] = 7;
+
+ }
+
+ if (KMALLOC_MIN_SIZE >= 128) {
+ /*
+ * The 192 byte sized cache is not used if the alignment
+ * is 128 byte. Redirect kmalloc to use the 256 byte cache
+ * instead.
+ */
+ for (i = 128 + 8; i <= 192; i += 8)
+ size_index[size_index_elem(i)] = 8;
+ }
/* Caches that are not of the two-to-the-power-of size */
if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1])
kmalloc_caches[1] = create_kmalloc_cache(NULL, 96, flags);
@@ -311,7 +414,5 @@ void __init create_kmalloc_caches(unsign
}
#endif
}
-
-
#endif /* !CONFIG_SLOB */
Index: linux/mm/slub.c
===================================================================
--- linux.orig/mm/slub.c 2012-09-28 13:23:51.907874005 -0500
+++ linux/mm/slub.c 2012-09-28 13:24:16.364384581 -0500
@@ -3210,64 +3210,6 @@ static int __init setup_slub_nomerge(cha
__setup("slub_nomerge", setup_slub_nomerge);
-/*
- * Conversion table for small slabs sizes / 8 to the index in the
- * kmalloc array. This is necessary for slabs < 192 since we have non power
- * of two cache sizes there. The size of larger slabs can be determined using
- * fls.
- */
-static s8 size_index[24] = {
- 3, /* 8 */
- 4, /* 16 */
- 5, /* 24 */
- 5, /* 32 */
- 6, /* 40 */
- 6, /* 48 */
- 6, /* 56 */
- 6, /* 64 */
- 1, /* 72 */
- 1, /* 80 */
- 1, /* 88 */
- 1, /* 96 */
- 7, /* 104 */
- 7, /* 112 */
- 7, /* 120 */
- 7, /* 128 */
- 2, /* 136 */
- 2, /* 144 */
- 2, /* 152 */
- 2, /* 160 */
- 2, /* 168 */
- 2, /* 176 */
- 2, /* 184 */
- 2 /* 192 */
-};
-
-static inline int size_index_elem(size_t bytes)
-{
- return (bytes - 1) / 8;
-}
-
-static struct kmem_cache *get_slab(size_t size, gfp_t flags)
-{
- int index;
-
- if (size <= 192) {
- if (!size)
- return ZERO_SIZE_PTR;
-
- index = size_index[size_index_elem(size)];
- } else
- index = fls(size - 1);
-
-#ifdef CONFIG_ZONE_DMA
- if (unlikely((flags & SLUB_DMA)))
- return kmalloc_dma_caches[index];
-
-#endif
- return kmalloc_caches[index];
-}
-
void *__kmalloc(size_t size, gfp_t flags)
{
struct kmem_cache *s;
@@ -3276,7 +3218,7 @@ void *__kmalloc(size_t size, gfp_t flags
if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
return kmalloc_large(size, flags);
- s = get_slab(size, flags);
+ s = kmalloc_slab(size, flags);
if (unlikely(ZERO_OR_NULL_PTR(s)))
return s;
@@ -3319,7 +3261,7 @@ void *__kmalloc_node(size_t size, gfp_t
return ret;
}
- s = get_slab(size, flags);
+ s = kmalloc_slab(size, flags);
if (unlikely(ZERO_OR_NULL_PTR(s)))
return s;
@@ -3627,8 +3569,6 @@ static __initdata struct kmem_cache boot
void __init kmem_cache_init(void)
{
- int i;
-
if (debug_guardpage_minorder())
slub_max_order = 0;
@@ -3663,45 +3603,6 @@ void __init kmem_cache_init(void)
kmem_cache_bootstrap_fixup(kmem_cache);
/* Now we can use the kmem_cache to allocate kmalloc slabs */
-
- /*
- * Patch up the size_index table if we have strange large alignment
- * requirements for the kmalloc array. This is only the case for
- * MIPS it seems. The standard arches will not generate any code here.
- *
- * Largest permitted alignment is 256 bytes due to the way we
- * handle the index determination for the smaller caches.
- *
- * Make sure that nothing crazy happens if someone starts tinkering
- * around with ARCH_KMALLOC_MINALIGN
- */
- BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
- (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));
-
- for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
- int elem = size_index_elem(i);
- if (elem >= ARRAY_SIZE(size_index))
- break;
- size_index[elem] = KMALLOC_SHIFT_LOW;
- }
-
- if (KMALLOC_MIN_SIZE == 64) {
- /*
- * The 96 byte size cache is not used if the alignment
- * is 64 byte.
- */
- for (i = 64 + 8; i <= 96; i += 8)
- size_index[size_index_elem(i)] = 7;
- } else if (KMALLOC_MIN_SIZE == 128) {
- /*
- * The 192 byte sized cache is not used if the alignment
- * is 128 byte. Redirect kmalloc to use the 256 byte cache
- * instead.
- */
- for (i = 128 + 8; i <= 192; i += 8)
- size_index[size_index_elem(i)] = 8;
- }
-
create_kmalloc_caches(0);
#ifdef CONFIG_SMP
@@ -3872,7 +3773,7 @@ void *__kmalloc_track_caller(size_t size
if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
return kmalloc_large(size, gfpflags);
- s = get_slab(size, gfpflags);
+ s = kmalloc_slab(size, gfpflags);
if (unlikely(ZERO_OR_NULL_PTR(s)))
return s;
@@ -3902,7 +3803,7 @@ void *__kmalloc_node_track_caller(size_t
return ret;
}
- s = get_slab(size, gfpflags);
+ s = kmalloc_slab(size, gfpflags);
if (unlikely(ZERO_OR_NULL_PTR(s)))
return s;
Index: linux/include/linux/slub_def.h
===================================================================
--- linux.orig/include/linux/slub_def.h 2012-09-28 13:23:34.000000000 -0500
+++ linux/include/linux/slub_def.h 2012-09-28 13:24:16.364384581 -0500
@@ -124,7 +124,7 @@ struct kmem_cache {
* This ought to end up with a global pointer to the right cache
* in kmalloc_caches.
*/
-static __always_inline struct kmem_cache *kmalloc_slab(size_t size)
+static __always_inline struct kmem_cache *kmalloc_slab_inline(size_t size)
{
int index = kmalloc_index(size);
@@ -189,7 +189,7 @@ static __always_inline void *kmalloc(siz
return kmalloc_large(size, flags);
if (!(flags & SLUB_DMA)) {
- struct kmem_cache *s = kmalloc_slab(size);
+ struct kmem_cache *s = kmalloc_slab_inline(size);
if (!s)
return ZERO_SIZE_PTR;
@@ -222,7 +222,7 @@ static __always_inline void *kmalloc_nod
{
if (__builtin_constant_p(size) &&
size <= KMALLOC_MAX_CACHE_SIZE && !(flags & SLUB_DMA)) {
- struct kmem_cache *s = kmalloc_slab(size);
+ struct kmem_cache *s = kmalloc_slab_inline(size);
if (!s)
return ZERO_SIZE_PTR;
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread* CK2 [12/15] Common function to create the kmalloc array
[not found] <20120928191715.368450474@linux.com>
` (4 preceding siblings ...)
2012-09-28 19:24 ` CK2 [14/15] Common Kmalloc cache determination Christoph Lameter
@ 2012-09-28 19:24 ` Christoph Lameter
2012-09-28 19:25 ` CK2 [05/15] Common alignment code Christoph Lameter
` (8 subsequent siblings)
14 siblings, 0 replies; 21+ messages in thread
From: Christoph Lameter @ 2012-09-28 19:24 UTC (permalink / raw)
To: Pekka Enberg; +Cc: Joonsoo Kim, Glauber Costa, linux-mm, David Rientjes
The kmalloc array is created in similar ways in both SLAB
and SLUB. Create a common function and have both allocators
call that function.
Reviewed-by: Glauber Costa <glommer@parallels.com>
Signed-off-by: Christoph Lameter <cl@linux.com>
Index: linux/mm/slab.c
===================================================================
--- linux.orig/mm/slab.c 2012-09-28 13:23:32.075459965 -0500
+++ linux/mm/slab.c 2012-09-28 13:23:51.907874005 -0500
@@ -1623,30 +1623,6 @@ void __init kmem_cache_init(void)
slab_early_init = 0;
- for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
- size_t cs_size = kmalloc_size(i);
-
- if (cs_size < KMALLOC_MIN_SIZE)
- continue;
-
- if (!kmalloc_caches[i]) {
- /*
- * For performance, all the general caches are L1 aligned.
- * This should be particularly beneficial on SMP boxes, as it
- * eliminates "false sharing".
- * Note for systems short on memory removing the alignment will
- * allow tighter packing of the smaller caches.
- */
- kmalloc_caches[i] = create_kmalloc_cache("kmalloc",
- cs_size, ARCH_KMALLOC_FLAGS);
- }
-
-#ifdef CONFIG_ZONE_DMA
- kmalloc_dma_caches[i] = create_kmalloc_cache(
- "kmalloc-dma", cs_size,
- SLAB_CACHE_DMA|ARCH_KMALLOC_FLAGS);
-#endif
- }
/* 4) Replace the bootstrap head arrays */
{
struct array_cache *ptr;
@@ -1692,29 +1668,7 @@ void __init kmem_cache_init(void)
}
}
- slab_state = UP;
-
- /* Create the proper names */
- for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
- char *s;
- struct kmem_cache *c = kmalloc_caches[i];
-
- if (!c)
- continue;
-
- s = kasprintf(GFP_NOWAIT, "kmalloc-%d", kmalloc_size(i));
-
- BUG_ON(!s);
- c->name = s;
-
-#ifdef CONFIG_ZONE_DMA
- c = kmalloc_dma_caches[i];
- BUG_ON(!c);
- s = kasprintf(GFP_NOWAIT, "dma-kmalloc-%d", kmalloc_size(i));
- BUG_ON(!s);
- c->name = s;
-#endif
- }
+ create_kmalloc_caches(ARCH_KMALLOC_FLAGS);
}
void __init kmem_cache_init_late(void)
Index: linux/mm/slab.h
===================================================================
--- linux.orig/mm/slab.h 2012-09-28 13:23:18.075167682 -0500
+++ linux/mm/slab.h 2012-09-28 13:24:08.536221146 -0500
@@ -35,6 +35,12 @@ extern struct kmem_cache *kmem_cache;
unsigned long calculate_alignment(unsigned long flags,
unsigned long align, unsigned long size);
+#ifndef CONFIG_SLOB
+/* Kmalloc array related functions */
+void create_kmalloc_caches(unsigned long);
+#endif
+
+
/* Functions provided by the slab allocators */
extern int __kmem_cache_create(struct kmem_cache *, unsigned long flags);
Index: linux/mm/slab_common.c
===================================================================
--- linux.orig/mm/slab_common.c 2012-09-28 13:23:34.851517919 -0500
+++ linux/mm/slab_common.c 2012-09-28 13:23:51.907874005 -0500
@@ -259,5 +259,59 @@ struct kmem_cache *kmalloc_dma_caches[KM
EXPORT_SYMBOL(kmalloc_dma_caches);
#endif
+/*
+ * Create the kmalloc array. Some of the regular kmalloc arrays
+ * may already have been created because they were needed to
+ * enable allocations for slab creation.
+ */
+void __init create_kmalloc_caches(unsigned long flags)
+{
+ int i;
+
+ /* Caches that are not of the two-to-the-power-of size */
+ if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1])
+ kmalloc_caches[1] = create_kmalloc_cache(NULL, 96, flags);
+
+ if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2])
+ kmalloc_caches[2] = create_kmalloc_cache(NULL, 192, flags);
+
+ for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++)
+ if (!kmalloc_caches[i])
+ kmalloc_caches[i] = create_kmalloc_cache(NULL,
+ 1 << i, flags);
+
+ /* Kmalloc array is now usable */
+ slab_state = UP;
+
+ for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
+ struct kmem_cache *s = kmalloc_caches[i];
+ char *n;
+
+ if (s) {
+ n = kasprintf(GFP_NOWAIT, "kmalloc-%d", kmalloc_size(i));
+
+ BUG_ON(!n);
+ s->name = n;
+ }
+ }
+
+#ifdef CONFIG_ZONE_DMA
+ for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
+ struct kmem_cache *s = kmalloc_caches[i];
+
+ if (s) {
+ int size = kmalloc_size(i);
+ char *n = kasprintf(GFP_NOWAIT,
+ "dma-kmalloc-%d", size);
+
+ BUG_ON(!n);
+ kmalloc_dma_caches[i] = create_kmalloc_cache(n,
+ size, SLAB_CACHE_DMA | flags);
+ }
+ }
+#endif
+}
+
+
#endif /* !CONFIG_SLOB */
Index: linux/mm/slub.c
===================================================================
--- linux.orig/mm/slub.c 2012-09-28 13:23:34.851517919 -0500
+++ linux/mm/slub.c 2012-09-28 13:23:51.907874005 -0500
@@ -3628,7 +3628,6 @@ static __initdata struct kmem_cache boot
void __init kmem_cache_init(void)
{
int i;
- int caches = 2;
if (debug_guardpage_minorder())
slub_max_order = 0;
@@ -3703,64 +3702,16 @@ void __init kmem_cache_init(void)
size_index[size_index_elem(i)] = 8;
}
- /* Caches that are not of the two-to-the-power-of size */
- if (KMALLOC_MIN_SIZE <= 32) {
- kmalloc_caches[1] = create_kmalloc_cache("kmalloc-96", 96, 0);
- caches++;
- }
-
- if (KMALLOC_MIN_SIZE <= 64) {
- kmalloc_caches[2] = create_kmalloc_cache("kmalloc-192", 192, 0);
- caches++;
- }
-
- for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
- kmalloc_caches[i] = create_kmalloc_cache("kmalloc", 1 << i, 0);
- caches++;
- }
-
- slab_state = UP;
-
- /* Provide the correct kmalloc names now that the caches are up */
- if (KMALLOC_MIN_SIZE <= 32) {
- kmalloc_caches[1]->name = kstrdup(kmalloc_caches[1]->name, GFP_NOWAIT);
- BUG_ON(!kmalloc_caches[1]->name);
- }
-
- if (KMALLOC_MIN_SIZE <= 64) {
- kmalloc_caches[2]->name = kstrdup(kmalloc_caches[2]->name, GFP_NOWAIT);
- BUG_ON(!kmalloc_caches[2]->name);
- }
-
- for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
- char *s = kasprintf(GFP_NOWAIT, "kmalloc-%d", 1 << i);
-
- BUG_ON(!s);
- kmalloc_caches[i]->name = s;
- }
+ create_kmalloc_caches(0);
#ifdef CONFIG_SMP
register_cpu_notifier(&slab_notifier);
#endif
-#ifdef CONFIG_ZONE_DMA
- for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
- struct kmem_cache *s = kmalloc_caches[i];
-
- if (s && s->size) {
- char *name = kasprintf(GFP_NOWAIT,
- "dma-kmalloc-%d", s->object_size);
-
- BUG_ON(!name);
- kmalloc_dma_caches[i] = create_kmalloc_cache(name,
- s->object_size, SLAB_CACHE_DMA);
- }
- }
-#endif
printk(KERN_INFO
- "SLUB: Genslabs=%d, HWalign=%d, Order=%d-%d, MinObjects=%d,"
+ "SLUB: HWalign=%d, Order=%d-%d, MinObjects=%d,"
" CPUs=%d, Nodes=%d\n",
- caches, cache_line_size(),
+ cache_line_size(),
slub_min_order, slub_max_order, slub_min_objects,
nr_cpu_ids, nr_node_ids);
}
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread* CK2 [05/15] Common alignment code
[not found] <20120928191715.368450474@linux.com>
` (5 preceding siblings ...)
2012-09-28 19:24 ` CK2 [12/15] Common function to create the kmalloc array Christoph Lameter
@ 2012-09-28 19:25 ` Christoph Lameter
2012-09-28 19:25 ` CK2 [04/15] slab: Use the new create_boot_cache function to simplify bootstrap Christoph Lameter
` (7 subsequent siblings)
14 siblings, 0 replies; 21+ messages in thread
From: Christoph Lameter @ 2012-09-28 19:25 UTC (permalink / raw)
To: Pekka Enberg; +Cc: Joonsoo Kim, Glauber Costa, linux-mm, David Rientjes
Extract the code to do object alignment from the allocators.
Do the alignment calculations in slab_common so that the
__kmem_cache_create functions of the allocators do not have
to deal with alignment.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slab.c | 20 --------------------
mm/slab.h | 3 +++
mm/slab_common.c | 32 ++++++++++++++++++++++++++++++--
mm/slob.c | 10 ----------
mm/slub.c | 38 +-------------------------------------
5 files changed, 34 insertions(+), 69 deletions(-)
Index: linux/mm/slab.h
===================================================================
--- linux.orig/mm/slab.h 2012-09-28 12:07:30.603667032 -0500
+++ linux/mm/slab.h 2012-09-28 12:08:53.613398708 -0500
@@ -32,6 +32,9 @@ extern struct list_head slab_caches;
/* The slab cache that manages slab cache information */
extern struct kmem_cache *kmem_cache;
+unsigned long calculate_alignment(unsigned long flags,
+ unsigned long align, unsigned long size);
+
/* Functions provided by the slab allocators */
extern int __kmem_cache_create(struct kmem_cache *, unsigned long flags);
Index: linux/mm/slab_common.c
===================================================================
--- linux.orig/mm/slab_common.c 2012-09-28 12:08:28.536875575 -0500
+++ linux/mm/slab_common.c 2012-09-28 12:08:53.613398708 -0500
@@ -71,6 +71,34 @@ static inline int kmem_cache_sanity_chec
#endif
/*
+ * Figure out what the alignment of the objects will be given a set of
+ * flags, a user specified alignment and the size of the objects.
+ */
+unsigned long calculate_alignment(unsigned long flags,
+ unsigned long align, unsigned long size)
+{
+ /*
+ * If the user wants hardware cache aligned objects then follow that
+ * suggestion if the object is sufficiently large.
+ *
+ * The hardware cache alignment cannot override the specified
+ * alignment though. If that is greater then use it.
+ */
+ if (flags & SLAB_HWCACHE_ALIGN) {
+ unsigned long ralign = cache_line_size();
+ while (size <= ralign / 2)
+ ralign /= 2;
+ align = max(align, ralign);
+ }
+
+ if (align < ARCH_SLAB_MINALIGN)
+ align = ARCH_SLAB_MINALIGN;
+
+ return ALIGN(align, sizeof(void *));
+}
+
+
+/*
* kmem_cache_create - Create a cache.
* @name: A string which is used in /proc/slabinfo to identify this cache.
* @size: The size of objects to be created in this cache.
@@ -115,7 +143,7 @@ struct kmem_cache *kmem_cache_create(con
s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
if (s) {
s->object_size = s->size = size;
- s->align = align;
+ s->align = calculate_alignment(flags, align, size);
s->ctor = ctor;
s->name = kstrdup(name, GFP_KERNEL);
if (!s->name) {
@@ -199,7 +227,7 @@ void __init create_boot_cache(struct kme
s->name = name;
s->size = s->object_size = size;
- s->align = ARCH_KMALLOC_MINALIGN;
+ s->align = calculate_alignment(flags, ARCH_KMALLOC_MINALIGN, size);
err = __kmem_cache_create(s, flags);
if (err)
Index: linux/mm/slob.c
===================================================================
--- linux.orig/mm/slob.c 2012-09-28 12:03:40.626869500 -0500
+++ linux/mm/slob.c 2012-09-28 12:08:53.613398708 -0500
@@ -124,7 +124,6 @@ static inline void clear_slob_page_free(
#define SLOB_UNIT sizeof(slob_t)
#define SLOB_UNITS(size) (((size) + SLOB_UNIT - 1)/SLOB_UNIT)
-#define SLOB_ALIGN L1_CACHE_BYTES
/*
* struct slob_rcu is inserted at the tail of allocated slob blocks, which
@@ -510,20 +509,11 @@ EXPORT_SYMBOL(ksize);
int __kmem_cache_create(struct kmem_cache *c, unsigned long flags)
{
- size_t align = c->size;
-
if (flags & SLAB_DESTROY_BY_RCU) {
/* leave room for rcu footer at the end of object */
c->size += sizeof(struct slob_rcu);
}
c->flags = flags;
- /* ignore alignment unless it's forced */
- c->align = (flags & SLAB_HWCACHE_ALIGN) ? SLOB_ALIGN : 0;
- if (c->align < ARCH_SLAB_MINALIGN)
- c->align = ARCH_SLAB_MINALIGN;
- if (c->align < align)
- c->align = align;
-
return 0;
}
Index: linux/mm/slub.c
===================================================================
--- linux.orig/mm/slub.c 2012-09-28 12:08:38.397081273 -0500
+++ linux/mm/slub.c 2012-09-28 12:08:53.613398708 -0500
@@ -2761,32 +2761,6 @@ static inline int calculate_order(int si
return -ENOSYS;
}
-/*
- * Figure out what the alignment of the objects will be.
- */
-static unsigned long calculate_alignment(unsigned long flags,
- unsigned long align, unsigned long size)
-{
- /*
- * If the user wants hardware cache aligned objects then follow that
- * suggestion if the object is sufficiently large.
- *
- * The hardware cache alignment cannot override the specified
- * alignment though. If that is greater then use it.
- */
- if (flags & SLAB_HWCACHE_ALIGN) {
- unsigned long ralign = cache_line_size();
- while (size <= ralign / 2)
- ralign /= 2;
- align = max(align, ralign);
- }
-
- if (align < ARCH_SLAB_MINALIGN)
- align = ARCH_SLAB_MINALIGN;
-
- return ALIGN(align, sizeof(void *));
-}
-
static void
init_kmem_cache_node(struct kmem_cache_node *n)
{
@@ -2920,7 +2894,6 @@ static int calculate_sizes(struct kmem_c
{
unsigned long flags = s->flags;
unsigned long size = s->object_size;
- unsigned long align = s->align;
int order;
/*
@@ -2992,19 +2965,11 @@ static int calculate_sizes(struct kmem_c
#endif
/*
- * Determine the alignment based on various parameters that the
- * user specified and the dynamic determination of cache line size
- * on bootup.
- */
- align = calculate_alignment(flags, align, s->object_size);
- s->align = align;
-
- /*
* SLUB stores one object immediately after another beginning from
* offset 0. In order to align the objects we have to simply size
* each object to conform to the alignment.
*/
- size = ALIGN(size, align);
+ size = ALIGN(size, s->align);
s->size = size;
if (forced_order >= 0)
order = forced_order;
@@ -3033,7 +2998,6 @@ static int calculate_sizes(struct kmem_c
s->max = s->oo;
return !!oo_objects(s->oo);
-
}
static int kmem_cache_open(struct kmem_cache *s, unsigned long flags)
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread* CK2 [04/15] slab: Use the new create_boot_cache function to simplify bootstrap
[not found] <20120928191715.368450474@linux.com>
` (6 preceding siblings ...)
2012-09-28 19:25 ` CK2 [05/15] Common alignment code Christoph Lameter
@ 2012-09-28 19:25 ` Christoph Lameter
2012-10-02 14:56 ` JoonSoo Kim
2012-09-28 19:25 ` CK2 [11/15] Common definition for the array of kmalloc caches Christoph Lameter
` (6 subsequent siblings)
14 siblings, 1 reply; 21+ messages in thread
From: Christoph Lameter @ 2012-09-28 19:25 UTC (permalink / raw)
To: Pekka Enberg; +Cc: Joonsoo Kim, Glauber Costa, linux-mm, David Rientjes
Simplify setup and reduce code in kmem_cache_init(). This allows us to
get rid of initarray_cache as well as the manual setup code for
the kmem_cache and kmem_cache_node arrays during bootstrap.
We introduce a new bootstrap state "PARTIAL" for slab that signals the
creation of a kmem_cache boot cache.
V1->V2: Get rid of initarray_cache as well.
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slab.c | 51 ++++++++++++++++++---------------------------------
1 file changed, 18 insertions(+), 33 deletions(-)
Index: linux/mm/slab.c
===================================================================
--- linux.orig/mm/slab.c 2012-09-19 09:21:14.422971030 -0500
+++ linux/mm/slab.c 2012-09-19 09:21:21.399115971 -0500
@@ -572,8 +572,6 @@ static struct cache_names __initdata cac
#undef CACHE
};
-static struct arraycache_init initarray_cache __initdata =
- { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
static struct arraycache_init initarray_generic =
{ {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
@@ -1596,12 +1594,9 @@ static void setup_nodelists_pointer(stru
*/
void __init kmem_cache_init(void)
{
- size_t left_over;
struct cache_sizes *sizes;
struct cache_names *names;
int i;
- int order;
- int node;
kmem_cache = &kmem_cache_boot;
setup_nodelists_pointer(kmem_cache);
@@ -1645,36 +1640,17 @@ void __init kmem_cache_init(void)
* 6) Resize the head arrays of the kmalloc caches to their final sizes.
*/
- node = numa_mem_id();
-
/* 1) create the kmem_cache */
- INIT_LIST_HEAD(&slab_caches);
- list_add(&kmem_cache->list, &slab_caches);
- kmem_cache->colour_off = cache_line_size();
- kmem_cache->array[smp_processor_id()] = &initarray_cache.cache;
/*
* struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
*/
- kmem_cache->size = offsetof(struct kmem_cache, array[nr_cpu_ids]) +
- nr_node_ids * sizeof(struct kmem_list3 *);
- kmem_cache->object_size = kmem_cache->size;
- kmem_cache->size = ALIGN(kmem_cache->object_size,
- cache_line_size());
- kmem_cache->reciprocal_buffer_size =
- reciprocal_value(kmem_cache->size);
-
- for (order = 0; order < MAX_ORDER; order++) {
- cache_estimate(order, kmem_cache->size,
- cache_line_size(), 0, &left_over, &kmem_cache->num);
- if (kmem_cache->num)
- break;
- }
- BUG_ON(!kmem_cache->num);
- kmem_cache->gfporder = order;
- kmem_cache->colour = left_over / kmem_cache->colour_off;
- kmem_cache->slab_size = ALIGN(kmem_cache->num * sizeof(kmem_bufctl_t) +
- sizeof(struct slab), cache_line_size());
+ create_boot_cache(kmem_cache, "kmem_cache",
+ offsetof(struct kmem_cache, array[nr_cpu_ids]) +
+ nr_node_ids * sizeof(struct kmem_list3 *),
+ SLAB_HWCACHE_ALIGN);
+
+ slab_state = PARTIAL;
/* 2+3) create the kmalloc caches */
sizes = malloc_sizes;
@@ -1722,7 +1698,6 @@ void __init kmem_cache_init(void)
ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
- BUG_ON(cpu_cache_get(kmem_cache) != &initarray_cache.cache);
memcpy(ptr, cpu_cache_get(kmem_cache),
sizeof(struct arraycache_init));
/*
@@ -2277,7 +2252,16 @@ static int __init_refok setup_cpu_cache(
if (slab_state == DOWN) {
/*
- * Note: the first kmem_cache_create must create the cache
+ * Note: Creation of first cache (kmem_cache).
+ * The setup_list3s is taken care
+ * of by the caller of __kmem_cache_create
+ */
+ cachep->array[smp_processor_id()] = &initarray_generic.cache;
+ slab_state = PARTIAL;
+ } else
+ if (slab_state == PARTIAL) {
+ /*
+ * Note: the second kmem_cache_create must create the cache
* that's used by kmalloc(24), otherwise the creation of
* further caches will BUG().
*/
@@ -2285,7 +2269,7 @@ static int __init_refok setup_cpu_cache(
/*
* If the cache that's used by kmalloc(sizeof(kmem_list3)) is
- * the first cache, then we need to set up all its list3s,
+ * the second cache, then we need to set up all its list3s,
* otherwise the creation of further caches will BUG().
*/
set_up_list3s(cachep, SIZE_AC);
@@ -2294,6 +2278,7 @@ static int __init_refok setup_cpu_cache(
else
slab_state = PARTIAL_ARRAYCACHE;
} else {
+ /* Remaining boot caches */
cachep->array[smp_processor_id()] =
kmalloc(sizeof(struct arraycache_init), gfp);
@@ -2389,22 +2374,6 @@ __kmem_cache_create (struct kmem_cache *
size &= ~(BYTES_PER_WORD - 1);
}
- /* calculate the final buffer alignment: */
-
- /* 1) arch recommendation: can be overridden for debug */
- if (flags & SLAB_HWCACHE_ALIGN) {
- /*
- * Default alignment: as specified by the arch code. Except if
- * an object is really small, then squeeze multiple objects into
- * one cacheline.
- */
- ralign = cache_line_size();
- while (size <= ralign / 2)
- ralign /= 2;
- } else {
- ralign = BYTES_PER_WORD;
- }
-
/*
* Redzoning and user store require word alignment or possibly larger.
* Note this will be overridden by architecture or caller mandated
@@ -2421,10 +2390,6 @@ __kmem_cache_create (struct kmem_cache *
size &= ~(REDZONE_ALIGN - 1);
}
- /* 2) arch mandated alignment */
- if (ralign < ARCH_SLAB_MINALIGN) {
- ralign = ARCH_SLAB_MINALIGN;
- }
/* 3) caller mandated alignment */
if (ralign < cachep->align) {
ralign = cachep->align;
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: CK2 [04/15] slab: Use the new create_boot_cache function to simplify bootstrap
2012-09-28 19:25 ` CK2 [04/15] slab: Use the new create_boot_cache function to simplify bootstrap Christoph Lameter
@ 2012-10-02 14:56 ` JoonSoo Kim
2012-10-03 14:55 ` Christoph Lameter
0 siblings, 1 reply; 21+ messages in thread
From: JoonSoo Kim @ 2012-10-02 14:56 UTC (permalink / raw)
To: Christoph Lameter; +Cc: Pekka Enberg, Glauber Costa, linux-mm, David Rientjes
2012/9/29 Christoph Lameter <cl@linux.com>:
> Simplify setup and reduce code in kmem_cache_init(). This allows us to
> get rid of initarray_cache as well as the manual setup code for
> the kmem_cache and kmem_cache_node arrays during bootstrap.
>
> We introduce a new bootstrap state "PARTIAL" for slab that signals the
> creation of a kmem_cache boot cache.
>
> V1->V2: Get rid of initarray_cache as well.
>
> Signed-off-by: Christoph Lameter <cl@linux.com>
> ---
> mm/slab.c | 51 ++++++++++++++++++---------------------------------
> 1 file changed, 18 insertions(+), 33 deletions(-)
>
> Index: linux/mm/slab.c
> ===================================================================
> --- linux.orig/mm/slab.c 2012-09-19 09:21:14.422971030 -0500
> +++ linux/mm/slab.c 2012-09-19 09:21:21.399115971 -0500
> @@ -572,8 +572,6 @@ static struct cache_names __initdata cac
> #undef CACHE
> };
>
> -static struct arraycache_init initarray_cache __initdata =
> - { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
> static struct arraycache_init initarray_generic =
> { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
>
> @@ -1596,12 +1594,9 @@ static void setup_nodelists_pointer(stru
> */
> void __init kmem_cache_init(void)
> {
> - size_t left_over;
> struct cache_sizes *sizes;
> struct cache_names *names;
> int i;
> - int order;
> - int node;
>
> kmem_cache = &kmem_cache_boot;
> setup_nodelists_pointer(kmem_cache);
> @@ -1645,36 +1640,17 @@ void __init kmem_cache_init(void)
> * 6) Resize the head arrays of the kmalloc caches to their final sizes.
> */
>
> - node = numa_mem_id();
> -
> /* 1) create the kmem_cache */
> - INIT_LIST_HEAD(&slab_caches);
> - list_add(&kmem_cache->list, &slab_caches);
> - kmem_cache->colour_off = cache_line_size();
> - kmem_cache->array[smp_processor_id()] = &initarray_cache.cache;
>
> /*
> * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
> */
> - kmem_cache->size = offsetof(struct kmem_cache, array[nr_cpu_ids]) +
> - nr_node_ids * sizeof(struct kmem_list3 *);
> - kmem_cache->object_size = kmem_cache->size;
> - kmem_cache->size = ALIGN(kmem_cache->object_size,
> - cache_line_size());
> - kmem_cache->reciprocal_buffer_size =
> - reciprocal_value(kmem_cache->size);
> -
> - for (order = 0; order < MAX_ORDER; order++) {
> - cache_estimate(order, kmem_cache->size,
> - cache_line_size(), 0, &left_over, &kmem_cache->num);
> - if (kmem_cache->num)
> - break;
> - }
> - BUG_ON(!kmem_cache->num);
> - kmem_cache->gfporder = order;
> - kmem_cache->colour = left_over / kmem_cache->colour_off;
> - kmem_cache->slab_size = ALIGN(kmem_cache->num * sizeof(kmem_bufctl_t) +
> - sizeof(struct slab), cache_line_size());
> + create_boot_cache(kmem_cache, "kmem_cache",
> + offsetof(struct kmem_cache, array[nr_cpu_ids]) +
> + nr_node_ids * sizeof(struct kmem_list3 *),
> + SLAB_HWCACHE_ALIGN);
> +
> + slab_state = PARTIAL;
>
> /* 2+3) create the kmalloc caches */
> sizes = malloc_sizes;
> @@ -1722,7 +1698,6 @@ void __init kmem_cache_init(void)
>
> ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
>
> - BUG_ON(cpu_cache_get(kmem_cache) != &initarray_cache.cache);
> memcpy(ptr, cpu_cache_get(kmem_cache),
> sizeof(struct arraycache_init));
> /*
> @@ -2277,7 +2252,16 @@ static int __init_refok setup_cpu_cache(
>
> if (slab_state == DOWN) {
> /*
> - * Note: the first kmem_cache_create must create the cache
> + * Note: Creation of first cache (kmem_cache).
> + * The setup_list3s is taken care
> + * of by the caller of __kmem_cache_create
> + */
> + cachep->array[smp_processor_id()] = &initarray_generic.cache;
> + slab_state = PARTIAL;
> + } else
> + if (slab_state == PARTIAL) {
> + /*
> + * Note: the second kmem_cache_create must create the cache
> * that's used by kmalloc(24), otherwise the creation of
> * further caches will BUG().
> */
> @@ -2285,7 +2269,7 @@ static int __init_refok setup_cpu_cache(
>
> /*
> * If the cache that's used by kmalloc(sizeof(kmem_list3)) is
> - * the first cache, then we need to set up all its list3s,
> + * the second cache, then we need to set up all its list3s,
> * otherwise the creation of further caches will BUG().
> */
> set_up_list3s(cachep, SIZE_AC);
> @@ -2294,6 +2278,7 @@ static int __init_refok setup_cpu_cache(
> else
> slab_state = PARTIAL_ARRAYCACHE;
> } else {
> + /* Remaining boot caches */
> cachep->array[smp_processor_id()] =
> kmalloc(sizeof(struct arraycache_init), gfp);
>
> @@ -2389,22 +2374,6 @@ __kmem_cache_create (struct kmem_cache *
> size &= ~(BYTES_PER_WORD - 1);
> }
>
> - /* calculate the final buffer alignment: */
> -
> - /* 1) arch recommendation: can be overridden for debug */
> - if (flags & SLAB_HWCACHE_ALIGN) {
> - /*
> - * Default alignment: as specified by the arch code. Except if
> - * an object is really small, then squeeze multiple objects into
> - * one cacheline.
> - */
> - ralign = cache_line_size();
> - while (size <= ralign / 2)
> - ralign /= 2;
> - } else {
> - ralign = BYTES_PER_WORD;
> - }
> -
> /*
> * Redzoning and user store require word alignment or possibly larger.
> * Note this will be overridden by architecture or caller mandated
> @@ -2421,10 +2390,6 @@ __kmem_cache_create (struct kmem_cache *
> size &= ~(REDZONE_ALIGN - 1);
> }
>
> - /* 2) arch mandated alignment */
> - if (ralign < ARCH_SLAB_MINALIGN) {
> - ralign = ARCH_SLAB_MINALIGN;
> - }
> /* 3) caller mandated alignment */
> if (ralign < cachep->align) {
> ralign = cachep->align;
>
With this patch, the slab allocator doesn't properly calculate an
alignment value for SLAB_HWCACHE_ALIGN flag.
Do we need to shuffle patches?
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: CK2 [04/15] slab: Use the new create_boot_cache function to simplify bootstrap
2012-10-02 14:56 ` JoonSoo Kim
@ 2012-10-03 14:55 ` Christoph Lameter
0 siblings, 0 replies; 21+ messages in thread
From: Christoph Lameter @ 2012-10-03 14:55 UTC (permalink / raw)
To: JoonSoo Kim; +Cc: Pekka Enberg, Glauber Costa, linux-mm, David Rientjes
On Tue, 2 Oct 2012, JoonSoo Kim wrote:
> With this patch, the slab allocator doesn't properly calculate an
> alignment value for SLAB_HWCACHE_ALIGN flag.
> Do we need to shuffle patches?
The last two hunks belong into the following patch.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread
* CK2 [11/15] Common definition for the array of kmalloc caches
[not found] <20120928191715.368450474@linux.com>
` (7 preceding siblings ...)
2012-09-28 19:25 ` CK2 [04/15] slab: Use the new create_boot_cache function to simplify bootstrap Christoph Lameter
@ 2012-09-28 19:25 ` Christoph Lameter
2012-09-28 19:34 ` CK2 [10/15] Common constants for kmalloc boundaries Christoph Lameter
` (5 subsequent siblings)
14 siblings, 0 replies; 21+ messages in thread
From: Christoph Lameter @ 2012-09-28 19:25 UTC (permalink / raw)
To: Pekka Enberg; +Cc: Joonsoo Kim, Glauber Costa, linux-mm, David Rientjes
Have a common definition fo the kmalloc cache arrays in
SLAB and SLUB
Signed-off-by: Christoph Lameter <cl@linux.com>
Index: linux/mm/slab_common.c
===================================================================
--- linux.orig/mm/slab_common.c 2012-09-28 13:31:22.809287368 -0500
+++ linux/mm/slab_common.c 2012-09-28 13:41:04.541431894 -0500
@@ -251,5 +251,13 @@ struct kmem_cache *__init create_kmalloc
return s;
}
+struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
+EXPORT_SYMBOL(kmalloc_caches);
+
+#ifdef CONFIG_ZONE_DMA
+struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1];
+EXPORT_SYMBOL(kmalloc_dma_caches);
+#endif
+
#endif /* !CONFIG_SLOB */
Index: linux/include/linux/slub_def.h
===================================================================
--- linux.orig/include/linux/slub_def.h 2012-09-28 13:41:01.125360578 -0500
+++ linux/include/linux/slub_def.h 2012-09-28 13:41:04.541431894 -0500
@@ -119,12 +119,6 @@ struct kmem_cache {
#endif
/*
- * We keep the general caches in an array of slab caches that are used for
- * 2^x bytes of allocations.
- */
-extern struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
-
-/*
* Find the slab cache for a given combination of allocation flags and size.
*
* This ought to end up with a global pointer to the right cache
Index: linux/mm/slub.c
===================================================================
--- linux.orig/mm/slub.c 2012-09-28 13:41:01.129360663 -0500
+++ linux/mm/slub.c 2012-09-28 13:41:04.541431894 -0500
@@ -3174,13 +3174,6 @@ int __kmem_cache_shutdown(struct kmem_ca
* Kmalloc subsystem
*******************************************************************/
-struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
-EXPORT_SYMBOL(kmalloc_caches);
-
-#ifdef CONFIG_ZONE_DMA
-static struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1];
-#endif
-
static int __init setup_slub_min_order(char *str)
{
get_option(&str, &slub_min_order);
Index: linux/include/linux/slab.h
===================================================================
--- linux.orig/include/linux/slab.h 2012-09-28 13:41:01.125360578 -0500
+++ linux/include/linux/slab.h 2012-09-28 13:41:04.541431894 -0500
@@ -189,6 +189,11 @@ struct kmem_cache {
#define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW)
#endif
+extern struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
+#ifdef CONFIG_ZONE_DMA
+extern struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1];
+#endif
+
/*
* Figure out which kmalloc slab an allocation of a certain size
* belongs to.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread* CK2 [10/15] Common constants for kmalloc boundaries
[not found] <20120928191715.368450474@linux.com>
` (8 preceding siblings ...)
2012-09-28 19:25 ` CK2 [11/15] Common definition for the array of kmalloc caches Christoph Lameter
@ 2012-09-28 19:34 ` Christoph Lameter
2012-09-28 19:34 ` CK2 [08/15] slab: Common name for the per node structures Christoph Lameter
` (4 subsequent siblings)
14 siblings, 0 replies; 21+ messages in thread
From: Christoph Lameter @ 2012-09-28 19:34 UTC (permalink / raw)
To: Pekka Enberg; +Cc: Joonsoo Kim, Glauber Costa, linux-mm, David Rientjes
Standardize the constants that describe the smallest and largest
object kept in the kmalloc arrays for SLAB and SLUB.
Differentiate between the maximum size for which a slab cache is used
(KMALLOC_MAX_CACHE_SIZE) and the maximum allocatable size
(KMALLOC_MAX_SIZE, KMALLOC_MAX_ORDER).
Signed-off-by: Christoph Lameter <cl@linux.com>
Index: linux/include/linux/slab.h
===================================================================
--- linux.orig/include/linux/slab.h 2012-09-28 13:40:49.017107804 -0500
+++ linux/include/linux/slab.h 2012-09-28 13:41:01.125360578 -0500
@@ -146,7 +146,12 @@ struct kmem_cache {
#else /* CONFIG_SLOB */
/*
- * The largest kmalloc size supported by the slab allocators is
+ * Kmalloc array related definitions
+ */
+
+#ifdef CONFIG_SLAB
+/*
+ * The largest kmalloc size supported by the SLAB allocators is
* 32 megabyte (2^25) or the maximum allocatable page order if that is
* less than 32 MB.
*
@@ -156,9 +161,24 @@ struct kmem_cache {
*/
#define KMALLOC_SHIFT_HIGH ((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \
(MAX_ORDER + PAGE_SHIFT - 1) : 25)
+#define KMALLOC_SHIFT_MAX KMALLOC_SHIFT_HIGH
+#define KMALLOC_SHIFT_LOW 5
+#else
+/*
+ * SLUB allocates up to order 2 pages directly and otherwise
+ * passes the request to the page allocator.
+ */
+#define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1)
+#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT)
+#define KMALLOC_SHIFT_LOW 3
+#endif
-#define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_HIGH)
-#define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_HIGH - PAGE_SHIFT)
+/* Maximum allocatable size */
+#define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_MAX)
+/* Maximum size for which we actually use a slab cache */
+#define KMALLOC_MAX_CACHE_SIZE (1UL << KMALLOC_SHIFT_HIGH)
+/* Maximum order allocatable via the slab allocagtor */
+#define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_MAX - PAGE_SHIFT)
/*
* Kmalloc subsystem.
@@ -166,15 +186,9 @@ struct kmem_cache {
#if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8
#define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN
#else
-#ifdef CONFIG_SLAB
-#define KMALLOC_MIN_SIZE 32
-#else
-#define KMALLOC_MIN_SIZE 8
-#endif
+#define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW)
#endif
-#define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE)
-
/*
* Figure out which kmalloc slab an allocation of a certain size
* belongs to.
Index: linux/include/linux/slub_def.h
===================================================================
--- linux.orig/include/linux/slub_def.h 2012-09-28 13:31:23.865309416 -0500
+++ linux/include/linux/slub_def.h 2012-09-28 13:41:01.125360578 -0500
@@ -111,19 +111,6 @@ struct kmem_cache {
struct kmem_cache_node *node[MAX_NUMNODES];
};
-/*
- * Maximum kmalloc object size handled by SLUB. Larger object allocations
- * are passed through to the page allocator. The page allocator "fastpath"
- * is relatively slow so we need this value sufficiently high so that
- * performance critical objects are allocated through the SLUB fastpath.
- *
- * This should be dropped to PAGE_SIZE / 2 once the page allocator
- * "fastpath" becomes competitive with the slab allocator fastpaths.
- */
-#define SLUB_MAX_SIZE (2 * PAGE_SIZE)
-
-#define SLUB_PAGE_SHIFT (PAGE_SHIFT + 2)
-
#ifdef CONFIG_ZONE_DMA
#define SLUB_DMA __GFP_DMA
#else
@@ -135,7 +122,7 @@ struct kmem_cache {
* We keep the general caches in an array of slab caches that are used for
* 2^x bytes of allocations.
*/
-extern struct kmem_cache *kmalloc_caches[SLUB_PAGE_SHIFT];
+extern struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
/*
* Find the slab cache for a given combination of allocation flags and size.
@@ -204,7 +191,7 @@ static __always_inline void *kmalloc_lar
static __always_inline void *kmalloc(size_t size, gfp_t flags)
{
if (__builtin_constant_p(size)) {
- if (size > SLUB_MAX_SIZE)
+ if (size > KMALLOC_MAX_CACHE_SIZE)
return kmalloc_large(size, flags);
if (!(flags & SLUB_DMA)) {
@@ -240,7 +227,7 @@ kmem_cache_alloc_node_trace(struct kmem_
static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
{
if (__builtin_constant_p(size) &&
- size <= SLUB_MAX_SIZE && !(flags & SLUB_DMA)) {
+ size <= KMALLOC_MAX_CACHE_SIZE && !(flags & SLUB_DMA)) {
struct kmem_cache *s = kmalloc_slab(size);
if (!s)
Index: linux/mm/slub.c
===================================================================
--- linux.orig/mm/slub.c 2012-09-28 13:31:22.809287368 -0500
+++ linux/mm/slub.c 2012-09-28 13:41:01.129360663 -0500
@@ -2777,7 +2777,7 @@ init_kmem_cache_node(struct kmem_cache_n
static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
{
BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
- SLUB_PAGE_SHIFT * sizeof(struct kmem_cache_cpu));
+ KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu));
/*
* Must align to double word boundary for the double cmpxchg
@@ -3174,11 +3174,11 @@ int __kmem_cache_shutdown(struct kmem_ca
* Kmalloc subsystem
*******************************************************************/
-struct kmem_cache *kmalloc_caches[SLUB_PAGE_SHIFT];
+struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
EXPORT_SYMBOL(kmalloc_caches);
#ifdef CONFIG_ZONE_DMA
-static struct kmem_cache *kmalloc_dma_caches[SLUB_PAGE_SHIFT];
+static struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1];
#endif
static int __init setup_slub_min_order(char *str)
@@ -3280,7 +3280,7 @@ void *__kmalloc(size_t size, gfp_t flags
struct kmem_cache *s;
void *ret;
- if (unlikely(size > SLUB_MAX_SIZE))
+ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
return kmalloc_large(size, flags);
s = get_slab(size, flags);
@@ -3316,7 +3316,7 @@ void *__kmalloc_node(size_t size, gfp_t
struct kmem_cache *s;
void *ret;
- if (unlikely(size > SLUB_MAX_SIZE)) {
+ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
ret = kmalloc_large_node(size, flags, node);
trace_kmalloc_node(_RET_IP_, ret,
@@ -3721,7 +3721,7 @@ void __init kmem_cache_init(void)
caches++;
}
- for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) {
+ for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
kmalloc_caches[i] = create_kmalloc_cache("kmalloc", 1 << i, 0);
caches++;
}
@@ -3739,7 +3739,7 @@ void __init kmem_cache_init(void)
BUG_ON(!kmalloc_caches[2]->name);
}
- for (i = KMALLOC_SHIFT_LOW; i < SLUB_PAGE_SHIFT; i++) {
+ for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
char *s = kasprintf(GFP_NOWAIT, "kmalloc-%d", 1 << i);
BUG_ON(!s);
@@ -3751,7 +3751,7 @@ void __init kmem_cache_init(void)
#endif
#ifdef CONFIG_ZONE_DMA
- for (i = 0; i < SLUB_PAGE_SHIFT; i++) {
+ for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
struct kmem_cache *s = kmalloc_caches[i];
if (s && s->size) {
@@ -3925,7 +3925,7 @@ void *__kmalloc_track_caller(size_t size
struct kmem_cache *s;
void *ret;
- if (unlikely(size > SLUB_MAX_SIZE))
+ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
return kmalloc_large(size, gfpflags);
s = get_slab(size, gfpflags);
@@ -3948,7 +3948,7 @@ void *__kmalloc_node_track_caller(size_t
struct kmem_cache *s;
void *ret;
- if (unlikely(size > SLUB_MAX_SIZE)) {
+ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
ret = kmalloc_large_node(size, gfpflags, node);
trace_kmalloc_node(caller, ret,
@@ -4307,7 +4307,7 @@ static void resiliency_test(void)
{
u8 *p;
- BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || SLUB_PAGE_SHIFT < 10);
+ BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || KMALLOC_SHIFT_HIGH < 10);
printk(KERN_ERR "SLUB resiliency testing\n");
printk(KERN_ERR "-----------------------\n");
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread* CK2 [08/15] slab: Common name for the per node structures
[not found] <20120928191715.368450474@linux.com>
` (9 preceding siblings ...)
2012-09-28 19:34 ` CK2 [10/15] Common constants for kmalloc boundaries Christoph Lameter
@ 2012-09-28 19:34 ` Christoph Lameter
2012-09-28 19:34 ` CK2 [03/15] slub: Use a statically allocated kmem_cache boot structure for bootstrap Christoph Lameter
` (3 subsequent siblings)
14 siblings, 0 replies; 21+ messages in thread
From: Christoph Lameter @ 2012-09-28 19:34 UTC (permalink / raw)
To: Pekka Enberg; +Cc: Joonsoo Kim, Glauber Costa, linux-mm, David Rientjes
Rename the structure used for the per node structures in slab
to have a name that expresses that fact.
Signed-off-by: Christoph Lameter <cl@linux.com>
Index: linux/include/linux/slab_def.h
===================================================================
--- linux.orig/include/linux/slab_def.h 2012-09-28 11:50:45.418698300 -0500
+++ linux/include/linux/slab_def.h 2012-09-28 11:50:54.606889979 -0500
@@ -88,7 +88,7 @@ struct kmem_cache {
* We still use [NR_CPUS] and not [1] or [0] because cache_cache
* is statically defined, so we reserve the max number of cpus.
*/
- struct kmem_list3 **nodelists;
+ struct kmem_cache_node **nodelists;
struct array_cache *array[NR_CPUS + MAX_NUMNODES];
/*
* Do not add fields after array[]
Index: linux/mm/slab.c
===================================================================
--- linux.orig/mm/slab.c 2012-09-28 11:50:45.430698559 -0500
+++ linux/mm/slab.c 2012-09-28 11:50:54.610890058 -0500
@@ -304,7 +304,7 @@ struct arraycache_init {
/*
* The slab lists for all objects.
*/
-struct kmem_list3 {
+struct kmem_cache_node {
struct list_head slabs_partial; /* partial list first, better asm code */
struct list_head slabs_full;
struct list_head slabs_free;
@@ -322,13 +322,13 @@ struct kmem_list3 {
* Need this for bootstrapping a per node allocator.
*/
#define NUM_INIT_LISTS (3 * MAX_NUMNODES)
-static struct kmem_list3 __initdata initkmem_list3[NUM_INIT_LISTS];
+static struct kmem_cache_node __initdata initkmem_list3[NUM_INIT_LISTS];
#define CACHE_CACHE 0
#define SIZE_AC MAX_NUMNODES
#define SIZE_L3 (2 * MAX_NUMNODES)
static int drain_freelist(struct kmem_cache *cache,
- struct kmem_list3 *l3, int tofree);
+ struct kmem_cache_node *l3, int tofree);
static void free_block(struct kmem_cache *cachep, void **objpp, int len,
int node);
static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
@@ -337,9 +337,9 @@ static void cache_reap(struct work_struc
static int slab_early_init = 1;
#define INDEX_AC kmalloc_index(sizeof(struct arraycache_init))
-#define INDEX_L3 kmalloc_index(sizeof(struct kmem_list3))
+#define INDEX_L3 kmalloc_index(sizeof(struct kmem_cache_node))
-static void kmem_list3_init(struct kmem_list3 *parent)
+static void kmem_list3_init(struct kmem_cache_node *parent)
{
INIT_LIST_HEAD(&parent->slabs_full);
INIT_LIST_HEAD(&parent->slabs_partial);
@@ -562,7 +562,7 @@ static void slab_set_lock_classes(struct
int q)
{
struct array_cache **alc;
- struct kmem_list3 *l3;
+ struct kmem_cache_node *l3;
int r;
l3 = cachep->nodelists[q];
@@ -607,7 +607,7 @@ static void init_node_lock_keys(int q)
return;
for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) {
- struct kmem_list3 *l3;
+ struct kmem_cache_node *l3;
struct kmem_cache *cache = kmalloc_caches[i];
if (!cache)
@@ -888,7 +888,7 @@ static inline bool is_slab_pfmemalloc(st
static void recheck_pfmemalloc_active(struct kmem_cache *cachep,
struct array_cache *ac)
{
- struct kmem_list3 *l3 = cachep->nodelists[numa_mem_id()];
+ struct kmem_cache_node *l3 = cachep->nodelists[numa_mem_id()];
struct slab *slabp;
unsigned long flags;
@@ -921,7 +921,7 @@ static void *__ac_get_obj(struct kmem_ca
/* Ensure the caller is allowed to use objects from PFMEMALLOC slab */
if (unlikely(is_obj_pfmemalloc(objp))) {
- struct kmem_list3 *l3;
+ struct kmem_cache_node *l3;
if (gfp_pfmemalloc_allowed(flags)) {
clear_obj_pfmemalloc(&objp);
@@ -1093,7 +1093,7 @@ static void free_alien_cache(struct arra
static void __drain_alien_cache(struct kmem_cache *cachep,
struct array_cache *ac, int node)
{
- struct kmem_list3 *rl3 = cachep->nodelists[node];
+ struct kmem_cache_node *rl3 = cachep->nodelists[node];
if (ac->avail) {
spin_lock(&rl3->list_lock);
@@ -1114,7 +1114,7 @@ static void __drain_alien_cache(struct k
/*
* Called from cache_reap() to regularly drain alien caches round robin.
*/
-static void reap_alien(struct kmem_cache *cachep, struct kmem_list3 *l3)
+static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *l3)
{
int node = __this_cpu_read(slab_reap_node);
@@ -1149,7 +1149,7 @@ static inline int cache_free_alien(struc
{
struct slab *slabp = virt_to_slab(objp);
int nodeid = slabp->nodeid;
- struct kmem_list3 *l3;
+ struct kmem_cache_node *l3;
struct array_cache *alien = NULL;
int node;
@@ -1194,8 +1194,8 @@ static inline int cache_free_alien(struc
static int init_cache_nodelists_node(int node)
{
struct kmem_cache *cachep;
- struct kmem_list3 *l3;
- const int memsize = sizeof(struct kmem_list3);
+ struct kmem_cache_node *l3;
+ const int memsize = sizeof(struct kmem_cache_node);
list_for_each_entry(cachep, &slab_caches, list) {
/*
@@ -1231,7 +1231,7 @@ static int init_cache_nodelists_node(int
static void __cpuinit cpuup_canceled(long cpu)
{
struct kmem_cache *cachep;
- struct kmem_list3 *l3 = NULL;
+ struct kmem_cache_node *l3 = NULL;
int node = cpu_to_mem(cpu);
const struct cpumask *mask = cpumask_of_node(node);
@@ -1296,7 +1296,7 @@ free_array_cache:
static int __cpuinit cpuup_prepare(long cpu)
{
struct kmem_cache *cachep;
- struct kmem_list3 *l3 = NULL;
+ struct kmem_cache_node *l3 = NULL;
int node = cpu_to_mem(cpu);
int err;
@@ -1447,7 +1447,7 @@ static int __meminit drain_cache_nodelis
int ret = 0;
list_for_each_entry(cachep, &slab_caches, list) {
- struct kmem_list3 *l3;
+ struct kmem_cache_node *l3;
l3 = cachep->nodelists[node];
if (!l3)
@@ -1500,15 +1500,15 @@ out:
/*
* swap the static kmem_list3 with kmalloced memory
*/
-static void __init init_list(struct kmem_cache *cachep, struct kmem_list3 *list,
+static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node *list,
int nodeid)
{
- struct kmem_list3 *ptr;
+ struct kmem_cache_node *ptr;
- ptr = kmalloc_node(sizeof(struct kmem_list3), GFP_NOWAIT, nodeid);
+ ptr = kmalloc_node(sizeof(struct kmem_cache_node), GFP_NOWAIT, nodeid);
BUG_ON(!ptr);
- memcpy(ptr, list, sizeof(struct kmem_list3));
+ memcpy(ptr, list, sizeof(struct kmem_cache_node));
/*
* Do not assume that spinlocks can be initialized via memcpy:
*/
@@ -1540,7 +1540,7 @@ static void __init set_up_list3s(struct
*/
static void setup_nodelists_pointer(struct kmem_cache *s)
{
- s->nodelists = (struct kmem_list3 **)&s->array[nr_cpu_ids];
+ s->nodelists = (struct kmem_cache_node **)&s->array[nr_cpu_ids];
}
/*
@@ -1600,7 +1600,7 @@ void __init kmem_cache_init(void)
*/
create_boot_cache(kmem_cache, "kmem_cache",
offsetof(struct kmem_cache, array[nr_cpu_ids]) +
- nr_node_ids * sizeof(struct kmem_list3 *),
+ nr_node_ids * sizeof(struct kmem_cache_node *),
SLAB_HWCACHE_ALIGN);
slab_state = PARTIAL;
@@ -1775,7 +1775,7 @@ __initcall(cpucache_init);
static noinline void
slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)
{
- struct kmem_list3 *l3;
+ struct kmem_cache_node *l3;
struct slab *slabp;
unsigned long flags;
int node;
@@ -2265,7 +2265,7 @@ static int __init_refok setup_cpu_cache(
int node;
for_each_online_node(node) {
cachep->nodelists[node] =
- kmalloc_node(sizeof(struct kmem_list3),
+ kmalloc_node(sizeof(struct kmem_cache_node),
gfp, node);
BUG_ON(!cachep->nodelists[node]);
kmem_list3_init(cachep->nodelists[node]);
@@ -2540,7 +2540,7 @@ static void check_spinlock_acquired_node
#define check_spinlock_acquired_node(x, y) do { } while(0)
#endif
-static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3,
+static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *l3,
struct array_cache *ac,
int force, int node);
@@ -2560,7 +2560,7 @@ static void do_drain(void *arg)
static void drain_cpu_caches(struct kmem_cache *cachep)
{
- struct kmem_list3 *l3;
+ struct kmem_cache_node *l3;
int node;
on_each_cpu(do_drain, cachep, 1);
@@ -2585,7 +2585,7 @@ static void drain_cpu_caches(struct kmem
* Returns the actual number of slabs released.
*/
static int drain_freelist(struct kmem_cache *cache,
- struct kmem_list3 *l3, int tofree)
+ struct kmem_cache_node *l3, int tofree)
{
struct list_head *p;
int nr_freed;
@@ -2623,7 +2623,7 @@ out:
static int __cache_shrink(struct kmem_cache *cachep)
{
int ret = 0, i = 0;
- struct kmem_list3 *l3;
+ struct kmem_cache_node *l3;
drain_cpu_caches(cachep);
@@ -2665,7 +2665,7 @@ EXPORT_SYMBOL(kmem_cache_shrink);
int __kmem_cache_shutdown(struct kmem_cache *cachep)
{
int i;
- struct kmem_list3 *l3;
+ struct kmem_cache_node *l3;
int rc = __cache_shrink(cachep);
if (rc)
@@ -2862,7 +2862,7 @@ static int cache_grow(struct kmem_cache
struct slab *slabp;
size_t offset;
gfp_t local_flags;
- struct kmem_list3 *l3;
+ struct kmem_cache_node *l3;
/*
* Be lazy and only check for valid flags here, keeping it out of the
@@ -3052,7 +3052,7 @@ static void *cache_alloc_refill(struct k
bool force_refill)
{
int batchcount;
- struct kmem_list3 *l3;
+ struct kmem_cache_node *l3;
struct array_cache *ac;
int node;
@@ -3384,7 +3384,7 @@ static void *____cache_alloc_node(struct
{
struct list_head *entry;
struct slab *slabp;
- struct kmem_list3 *l3;
+ struct kmem_cache_node *l3;
void *obj;
int x;
@@ -3575,7 +3575,7 @@ static void free_block(struct kmem_cache
int node)
{
int i;
- struct kmem_list3 *l3;
+ struct kmem_cache_node *l3;
for (i = 0; i < nr_objects; i++) {
void *objp;
@@ -3621,7 +3621,7 @@ static void free_block(struct kmem_cache
static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac)
{
int batchcount;
- struct kmem_list3 *l3;
+ struct kmem_cache_node *l3;
int node = numa_mem_id();
batchcount = ac->batchcount;
@@ -3918,7 +3918,7 @@ EXPORT_SYMBOL(kmem_cache_size);
static int alloc_kmemlist(struct kmem_cache *cachep, gfp_t gfp)
{
int node;
- struct kmem_list3 *l3;
+ struct kmem_cache_node *l3;
struct array_cache *new_shared;
struct array_cache **new_alien = NULL;
@@ -3963,7 +3963,7 @@ static int alloc_kmemlist(struct kmem_ca
free_alien_cache(new_alien);
continue;
}
- l3 = kmalloc_node(sizeof(struct kmem_list3), gfp, node);
+ l3 = kmalloc_node(sizeof(struct kmem_cache_node), gfp, node);
if (!l3) {
free_alien_cache(new_alien);
kfree(new_shared);
@@ -4120,7 +4120,7 @@ static int enable_cpucache(struct kmem_c
* necessary. Note that the l3 listlock also protects the array_cache
* if drain_array() is used on the shared array.
*/
-static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3,
+static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *l3,
struct array_cache *ac, int force, int node)
{
int tofree;
@@ -4159,7 +4159,7 @@ static void drain_array(struct kmem_cach
static void cache_reap(struct work_struct *w)
{
struct kmem_cache *searchp;
- struct kmem_list3 *l3;
+ struct kmem_cache_node *l3;
int node = numa_mem_id();
struct delayed_work *work = to_delayed_work(w);
@@ -4269,7 +4269,7 @@ static int s_show(struct seq_file *m, vo
const char *name;
char *error = NULL;
int node;
- struct kmem_list3 *l3;
+ struct kmem_cache_node *l3;
active_objs = 0;
num_slabs = 0;
@@ -4512,7 +4512,7 @@ static int leaks_show(struct seq_file *m
{
struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);
struct slab *slabp;
- struct kmem_list3 *l3;
+ struct kmem_cache_node *l3;
const char *name;
unsigned long *n = m->private;
int node;
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread* CK2 [03/15] slub: Use a statically allocated kmem_cache boot structure for bootstrap
[not found] <20120928191715.368450474@linux.com>
` (10 preceding siblings ...)
2012-09-28 19:34 ` CK2 [08/15] slab: Common name for the per node structures Christoph Lameter
@ 2012-09-28 19:34 ` Christoph Lameter
2012-10-02 14:47 ` JoonSoo Kim
2012-09-28 19:39 ` CK2 [13/15] stat: Use size_t for sizes instead of unsigned Christoph Lameter
` (2 subsequent siblings)
14 siblings, 1 reply; 21+ messages in thread
From: Christoph Lameter @ 2012-09-28 19:34 UTC (permalink / raw)
To: Pekka Enberg; +Cc: Joonsoo Kim, Glauber Costa, linux-mm, David Rientjes
Simplify bootstrap by statically allocated two kmem_cache structures. These are
freed after bootup is complete. Allows us to no longer worry about calculations
of sizes of kmem_cache structures during bootstrap.
V1->V2: Do not unlock mutexes that are not taken during early boot.
Reviewed-by: Glauber Costa <glommer@parallels.com>
Signed-off-by: Christoph Lameter <cl@linux.com>
---
mm/slub.c | 41 +++++++++++------------------------------
1 file changed, 11 insertions(+), 30 deletions(-)
Index: linux/mm/slub.c
===================================================================
--- linux.orig/mm/slub.c 2012-09-19 09:21:14.422971030 -0500
+++ linux/mm/slub.c 2012-09-19 09:21:18.403053765 -0500
@@ -3649,9 +3649,6 @@ static void __init kmem_cache_bootstrap_
{
int node;
- list_add(&s->list, &slab_caches);
- s->refcount = -1;
-
for_each_node_state(node, N_NORMAL_MEMORY) {
struct kmem_cache_node *n = get_node(s, node);
struct page *p;
@@ -3668,14 +3665,13 @@ static void __init kmem_cache_bootstrap_
}
}
+static __initdata struct kmem_cache boot_kmem_cache,
+ boot_kmem_cache_node;
+
void __init kmem_cache_init(void)
{
int i;
- int caches = 0;
- struct kmem_cache *temp_kmem_cache;
- int order;
- struct kmem_cache *temp_kmem_cache_node;
- unsigned long kmalloc_size;
+ int caches = 2;
if (debug_guardpage_minorder())
slub_max_order = 0;
@@ -3683,53 +3679,32 @@ void __init kmem_cache_init(void)
kmem_size = offsetof(struct kmem_cache, node) +
nr_node_ids * sizeof(struct kmem_cache_node *);
- /* Allocate two kmem_caches from the page allocator */
- kmalloc_size = ALIGN(kmem_size, cache_line_size());
- order = get_order(2 * kmalloc_size);
- kmem_cache = (void *)__get_free_pages(GFP_NOWAIT | __GFP_ZERO, order);
-
- /*
- * Must first have the slab cache available for the allocations of the
- * struct kmem_cache_node's. There is special bootstrap code in
- * kmem_cache_open for slab_state == DOWN.
- */
- kmem_cache_node = (void *)kmem_cache + kmalloc_size;
+ kmem_cache_node = &boot_kmem_cache_node;
- kmem_cache_node->name = "kmem_cache_node";
- kmem_cache_node->size = kmem_cache_node->object_size =
- sizeof(struct kmem_cache_node);
- kmem_cache_open(kmem_cache_node, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
+ create_boot_cache(kmem_cache_node, "kmem_cache_node",
+ sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN);
hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
/* Able to allocate the per node structures */
slab_state = PARTIAL;
- temp_kmem_cache = kmem_cache;
- kmem_cache->name = "kmem_cache";
- kmem_cache->size = kmem_cache->object_size = kmem_size;
- kmem_cache_open(kmem_cache, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
+ create_boot_cache(&boot_kmem_cache, "kmem_cache", kmem_size,
+ SLAB_HWCACHE_ALIGN);
- kmem_cache = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
- memcpy(kmem_cache, temp_kmem_cache, kmem_size);
+ kmem_cache = kmem_cache_alloc(&boot_kmem_cache, GFP_NOWAIT);
+ memcpy(kmem_cache, &boot_kmem_cache, kmem_size);
/*
* Allocate kmem_cache_node properly from the kmem_cache slab.
* kmem_cache_node is separately allocated so no need to
* update any list pointers.
*/
- temp_kmem_cache_node = kmem_cache_node;
-
kmem_cache_node = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
- memcpy(kmem_cache_node, temp_kmem_cache_node, kmem_size);
+ memcpy(kmem_cache_node, &boot_kmem_cache_node, kmem_size);
kmem_cache_bootstrap_fixup(kmem_cache_node);
-
- caches++;
kmem_cache_bootstrap_fixup(kmem_cache);
- caches++;
- /* Free temporary boot structure */
- free_pages((unsigned long)temp_kmem_cache, order);
/* Now we can use the kmem_cache to allocate kmalloc slabs */
@@ -3930,6 +3905,10 @@ int __kmem_cache_create(struct kmem_cach
if (err)
return err;
+ /* Mutex is not taken during early boot */
+ if (slab_state <= UP)
+ return 0;
+
mutex_unlock(&slab_mutex);
err = sysfs_slab_add(s);
mutex_lock(&slab_mutex);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: CK2 [03/15] slub: Use a statically allocated kmem_cache boot structure for bootstrap
2012-09-28 19:34 ` CK2 [03/15] slub: Use a statically allocated kmem_cache boot structure for bootstrap Christoph Lameter
@ 2012-10-02 14:47 ` JoonSoo Kim
2012-10-03 14:50 ` Christoph Lameter
0 siblings, 1 reply; 21+ messages in thread
From: JoonSoo Kim @ 2012-10-02 14:47 UTC (permalink / raw)
To: Christoph Lameter; +Cc: Pekka Enberg, Glauber Costa, linux-mm, David Rientjes
Hi, Christoph.
2012/9/29 Christoph Lameter <cl@linux.com>:
> @@ -3930,6 +3905,10 @@ int __kmem_cache_create(struct kmem_cach
> if (err)
> return err;
>
> + /* Mutex is not taken during early boot */
> + if (slab_state <= UP)
> + return 0;
> +
> mutex_unlock(&slab_mutex);
> err = sysfs_slab_add(s);
> mutex_lock(&slab_mutex);
This addition should go to previous patch "create common functions for
boot slab creation".
In there, create_boot_cache() call __kmem_cache_create(), and then
mutex_unlock() is called without taking mutext.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread
* Re: CK2 [03/15] slub: Use a statically allocated kmem_cache boot structure for bootstrap
2012-10-02 14:47 ` JoonSoo Kim
@ 2012-10-03 14:50 ` Christoph Lameter
0 siblings, 0 replies; 21+ messages in thread
From: Christoph Lameter @ 2012-10-03 14:50 UTC (permalink / raw)
To: JoonSoo Kim; +Cc: Pekka Enberg, Glauber Costa, linux-mm, David Rientjes
On Tue, 2 Oct 2012, JoonSoo Kim wrote:
> This addition should go to previous patch "create common functions for
> boot slab creation".
Ok. That will be the case in V3
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread
* CK2 [13/15] stat: Use size_t for sizes instead of unsigned
[not found] <20120928191715.368450474@linux.com>
` (11 preceding siblings ...)
2012-09-28 19:34 ` CK2 [03/15] slub: Use a statically allocated kmem_cache boot structure for bootstrap Christoph Lameter
@ 2012-09-28 19:39 ` Christoph Lameter
2012-09-28 19:39 ` CK2 [06/15] Common kmalloc slab index determination Christoph Lameter
2012-09-28 19:59 ` CK2 [09/15] slab: rename nodelists to node Christoph Lameter
14 siblings, 0 replies; 21+ messages in thread
From: Christoph Lameter @ 2012-09-28 19:39 UTC (permalink / raw)
To: Pekka Enberg; +Cc: Joonsoo Kim, Glauber Costa, linux-mm, David Rientjes
On some platforms (such as IA64) the large page size may results in
slab allocations to be allowed of numbers that do not fit in 32 bit.
Signed-off-by: Christoph Lameter <cl@linux.com>
Index: linux/fs/proc/stat.c
===================================================================
--- linux.orig/fs/proc/stat.c 2012-09-18 12:21:56.301459334 -0500
+++ linux/fs/proc/stat.c 2012-09-28 12:33:29.428759019 -0500
@@ -178,7 +178,7 @@ static int show_stat(struct seq_file *p,
static int stat_open(struct inode *inode, struct file *file)
{
- unsigned size = 1024 + 128 * num_possible_cpus();
+ size_t size = 1024 + 128 * num_possible_cpus();
char *buf;
struct seq_file *m;
int res;
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread* CK2 [06/15] Common kmalloc slab index determination
[not found] <20120928191715.368450474@linux.com>
` (12 preceding siblings ...)
2012-09-28 19:39 ` CK2 [13/15] stat: Use size_t for sizes instead of unsigned Christoph Lameter
@ 2012-09-28 19:39 ` Christoph Lameter
2012-09-28 19:59 ` CK2 [09/15] slab: rename nodelists to node Christoph Lameter
14 siblings, 0 replies; 21+ messages in thread
From: Christoph Lameter @ 2012-09-28 19:39 UTC (permalink / raw)
To: Pekka Enberg; +Cc: Joonsoo Kim, Glauber Costa, linux-mm, David Rientjes
Extract the function to determine the index of the slab within
the array of kmalloc caches as well as a function to determine
maximum object size from the nr of the kmalloc slab.
This is used here only to simplify slub bootstrap but will
be used later also for SLAB.
Signed-off-by: Christoph Lameter <cl@linux.com>
Index: linux/include/linux/slab.h
===================================================================
--- linux.orig/include/linux/slab.h 2012-09-28 13:31:11.021041273 -0500
+++ linux/include/linux/slab.h 2012-09-28 13:40:49.017107804 -0500
@@ -93,30 +93,6 @@
(unsigned long)ZERO_SIZE_PTR)
/*
- * Common fields provided in kmem_cache by all slab allocators
- * This struct is either used directly by the allocator (SLOB)
- * or the allocator must include definitions for all fields
- * provided in kmem_cache_common in their definition of kmem_cache.
- *
- * Once we can do anonymous structs (C11 standard) we could put a
- * anonymous struct definition in these allocators so that the
- * separate allocations in the kmem_cache structure of SLAB and
- * SLUB is no longer needed.
- */
-#ifdef CONFIG_SLOB
-struct kmem_cache {
- unsigned int object_size;/* The original size of the object */
- unsigned int size; /* The aligned/padded/added on size */
- unsigned int align; /* Alignment as calculated */
- unsigned long flags; /* Active flags on the slab */
- const char *name; /* Slab name for sysfs */
- int refcount; /* Use counter */
- void (*ctor)(void *); /* Called on object slot creation */
- struct list_head list; /* List of all slab caches on the system */
-};
-#endif
-
-/*
* struct kmem_cache related prototypes
*/
void __init kmem_cache_init(void);
@@ -142,6 +118,33 @@ unsigned int kmem_cache_size(struct kmem
sizeof(struct __struct), __alignof__(struct __struct),\
(__flags), NULL)
+#ifdef CONFIG_SLOB
+/*
+ * Common fields provided in kmem_cache by all slab allocators
+ * This struct is either used directly by the allocator (SLOB)
+ * or the allocator must include definitions for all fields
+ * provided in kmem_cache_common in their definition of kmem_cache.
+ *
+ * Once we can do anonymous structs (C11 standard) we could put a
+ * anonymous struct definition in these allocators so that the
+ * separate allocations in the kmem_cache structure of SLAB and
+ * SLUB is no longer needed.
+ */
+struct kmem_cache {
+ unsigned int object_size;/* The original size of the object */
+ unsigned int size; /* The aligned/padded/added on size */
+ unsigned int align; /* Alignment as calculated */
+ unsigned long flags; /* Active flags on the slab */
+ const char *name; /* Slab name for sysfs */
+ int refcount; /* Use counter */
+ void (*ctor)(void *); /* Called on object slot creation */
+ struct list_head list; /* List of all slab caches on the system */
+};
+
+#define KMALLOC_MAX_SIZE (1UL << 30)
+
+#else /* CONFIG_SLOB */
+
/*
* The largest kmalloc size supported by the slab allocators is
* 32 megabyte (2^25) or the maximum allocatable page order if that is
@@ -158,6 +161,91 @@ unsigned int kmem_cache_size(struct kmem
#define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_HIGH - PAGE_SHIFT)
/*
+ * Kmalloc subsystem.
+ */
+#if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8
+#define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN
+#else
+#ifdef CONFIG_SLAB
+#define KMALLOC_MIN_SIZE 32
+#else
+#define KMALLOC_MIN_SIZE 8
+#endif
+#endif
+
+#define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE)
+
+/*
+ * Figure out which kmalloc slab an allocation of a certain size
+ * belongs to.
+ * 0 = zero alloc
+ * 1 = 65 .. 96 bytes
+ * 2 = 120 .. 192 bytes
+ * n = 2^(n-1) .. 2^n -1
+ */
+static __always_inline int kmalloc_index(size_t size)
+{
+ if (!size)
+ return 0;
+
+ if (size <= KMALLOC_MIN_SIZE)
+ return KMALLOC_SHIFT_LOW;
+
+ if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96)
+ return 1;
+ if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192)
+ return 2;
+ if (size <= 8) return 3;
+ if (size <= 16) return 4;
+ if (size <= 32) return 5;
+ if (size <= 64) return 6;
+ if (size <= 128) return 7;
+ if (size <= 256) return 8;
+ if (size <= 512) return 9;
+ if (size <= 1024) return 10;
+ if (size <= 2 * 1024) return 11;
+ if (size <= 4 * 1024) return 12;
+ if (size <= 8 * 1024) return 13;
+ if (size <= 16 * 1024) return 14;
+ if (size <= 32 * 1024) return 15;
+ if (size <= 64 * 1024) return 16;
+ if (size <= 128 * 1024) return 17;
+ if (size <= 256 * 1024) return 18;
+ if (size <= 512 * 1024) return 19;
+ if (size <= 1024 * 1024) return 20;
+ if (size <= 2 * 1024 * 1024) return 21;
+ if (size <= 4 * 1024 * 1024) return 22;
+ if (size <= 8 * 1024 * 1024) return 23;
+ if (size <= 16 * 1024 * 1024) return 24;
+ if (size <= 32 * 1024 * 1024) return 25;
+ if (size <= 64 * 1024 * 1024) return 26;
+ BUG();
+
+ /* Will never be reached. Needed because the compiler may complain */
+ return -1;
+}
+
+/*
+ * Determine size used for the nth kmalloc cache.
+ * return size or 0 if a kmalloc cache for that
+ * size does not exist
+ */
+static __always_inline int kmalloc_size(int n)
+{
+ if (n > 2)
+ return 1 << n;
+
+ if (n == 1 && KMALLOC_MIN_SIZE <= 32)
+ return 96;
+
+ if (n == 2 && KMALLOC_MIN_SIZE <= 64)
+ return 192;
+
+ return 0;
+}
+#endif /* !CONFIG_SLOB */
+
+/*
* Some archs want to perform DMA into kmalloc caches and need a guaranteed
* alignment larger than the alignment of a 64-bit integer.
* Setting ARCH_KMALLOC_MINALIGN in arch headers allows that.
Index: linux/include/linux/slub_def.h
===================================================================
--- linux.orig/include/linux/slub_def.h 2012-09-28 13:31:11.033041519 -0500
+++ linux/include/linux/slub_def.h 2012-09-28 13:31:23.865309416 -0500
@@ -112,17 +112,6 @@ struct kmem_cache {
};
/*
- * Kmalloc subsystem.
- */
-#if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8
-#define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN
-#else
-#define KMALLOC_MIN_SIZE 8
-#endif
-
-#define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE)
-
-/*
* Maximum kmalloc object size handled by SLUB. Larger object allocations
* are passed through to the page allocator. The page allocator "fastpath"
* is relatively slow so we need this value sufficiently high so that
@@ -149,58 +138,6 @@ struct kmem_cache {
extern struct kmem_cache *kmalloc_caches[SLUB_PAGE_SHIFT];
/*
- * Sorry that the following has to be that ugly but some versions of GCC
- * have trouble with constant propagation and loops.
- */
-static __always_inline int kmalloc_index(size_t size)
-{
- if (!size)
- return 0;
-
- if (size <= KMALLOC_MIN_SIZE)
- return KMALLOC_SHIFT_LOW;
-
- if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96)
- return 1;
- if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192)
- return 2;
- if (size <= 8) return 3;
- if (size <= 16) return 4;
- if (size <= 32) return 5;
- if (size <= 64) return 6;
- if (size <= 128) return 7;
- if (size <= 256) return 8;
- if (size <= 512) return 9;
- if (size <= 1024) return 10;
- if (size <= 2 * 1024) return 11;
- if (size <= 4 * 1024) return 12;
-/*
- * The following is only needed to support architectures with a larger page
- * size than 4k. We need to support 2 * PAGE_SIZE here. So for a 64k page
- * size we would have to go up to 128k.
- */
- if (size <= 8 * 1024) return 13;
- if (size <= 16 * 1024) return 14;
- if (size <= 32 * 1024) return 15;
- if (size <= 64 * 1024) return 16;
- if (size <= 128 * 1024) return 17;
- if (size <= 256 * 1024) return 18;
- if (size <= 512 * 1024) return 19;
- if (size <= 1024 * 1024) return 20;
- if (size <= 2 * 1024 * 1024) return 21;
- BUG();
- return -1; /* Will never be reached */
-
-/*
- * What we really wanted to do and cannot do because of compiler issues is:
- * int i;
- * for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++)
- * if (size <= (1 << i))
- * return i;
- */
-}
-
-/*
* Find the slab cache for a given combination of allocation flags and size.
*
* This ought to end up with a global pointer to the right cache
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread* CK2 [09/15] slab: rename nodelists to node
[not found] <20120928191715.368450474@linux.com>
` (13 preceding siblings ...)
2012-09-28 19:39 ` CK2 [06/15] Common kmalloc slab index determination Christoph Lameter
@ 2012-09-28 19:59 ` Christoph Lameter
14 siblings, 0 replies; 21+ messages in thread
From: Christoph Lameter @ 2012-09-28 19:59 UTC (permalink / raw)
To: Pekka Enberg; +Cc: Joonsoo Kim, Glauber Costa, linux-mm, David Rientjes
Have a common naming between both slab caches for future changes.
Signed-off-by: Christoph Lameter <cl@linux.com>
Index: linux/include/linux/slab_def.h
===================================================================
--- linux.orig/include/linux/slab_def.h 2012-09-28 11:54:45.403704433 -0500
+++ linux/include/linux/slab_def.h 2012-09-28 11:54:46.243721959 -0500
@@ -88,7 +88,7 @@ struct kmem_cache {
* We still use [NR_CPUS] and not [1] or [0] because cache_cache
* is statically defined, so we reserve the max number of cpus.
*/
- struct kmem_cache_node **nodelists;
+ struct kmem_cache_node **node;
struct array_cache *array[NR_CPUS + MAX_NUMNODES];
/*
* Do not add fields after array[]
Index: linux/mm/slab.c
===================================================================
--- linux.orig/mm/slab.c 2012-09-28 11:54:45.403704433 -0500
+++ linux/mm/slab.c 2012-09-28 11:55:03.320078174 -0500
@@ -355,7 +355,7 @@ static void kmem_list3_init(struct kmem_
#define MAKE_LIST(cachep, listp, slab, nodeid) \
do { \
INIT_LIST_HEAD(listp); \
- list_splice(&(cachep->nodelists[nodeid]->slab), listp); \
+ list_splice(&(cachep->node[nodeid]->slab), listp); \
} while (0)
#define MAKE_ALL_LISTS(cachep, ptr, nodeid) \
@@ -565,7 +565,7 @@ static void slab_set_lock_classes(struct
struct kmem_cache_node *l3;
int r;
- l3 = cachep->nodelists[q];
+ l3 = cachep->node[q];
if (!l3)
return;
@@ -613,7 +613,7 @@ static void init_node_lock_keys(int q)
if (!cache)
continue;
- l3 = cache->nodelists[q];
+ l3 = cache->node[q];
if (!l3 || OFF_SLAB(cache))
continue;
@@ -888,7 +888,7 @@ static inline bool is_slab_pfmemalloc(st
static void recheck_pfmemalloc_active(struct kmem_cache *cachep,
struct array_cache *ac)
{
- struct kmem_cache_node *l3 = cachep->nodelists[numa_mem_id()];
+ struct kmem_cache_node *l3 = cachep->node[numa_mem_id()];
struct slab *slabp;
unsigned long flags;
@@ -943,7 +943,7 @@ static void *__ac_get_obj(struct kmem_ca
* If there are empty slabs on the slabs_free list and we are
* being forced to refill the cache, mark this one !pfmemalloc.
*/
- l3 = cachep->nodelists[numa_mem_id()];
+ l3 = cachep->node[numa_mem_id()];
if (!list_empty(&l3->slabs_free) && force_refill) {
struct slab *slabp = virt_to_slab(objp);
ClearPageSlabPfmemalloc(virt_to_head_page(slabp->s_mem));
@@ -1093,7 +1093,7 @@ static void free_alien_cache(struct arra
static void __drain_alien_cache(struct kmem_cache *cachep,
struct array_cache *ac, int node)
{
- struct kmem_cache_node *rl3 = cachep->nodelists[node];
+ struct kmem_cache_node *rl3 = cachep->node[node];
if (ac->avail) {
spin_lock(&rl3->list_lock);
@@ -1162,7 +1162,7 @@ static inline int cache_free_alien(struc
if (likely(slabp->nodeid == node))
return 0;
- l3 = cachep->nodelists[node];
+ l3 = cachep->node[node];
STATS_INC_NODEFREES(cachep);
if (l3->alien && l3->alien[nodeid]) {
alien = l3->alien[nodeid];
@@ -1174,24 +1174,24 @@ static inline int cache_free_alien(struc
ac_put_obj(cachep, alien, objp);
spin_unlock(&alien->lock);
} else {
- spin_lock(&(cachep->nodelists[nodeid])->list_lock);
+ spin_lock(&(cachep->node[nodeid])->list_lock);
free_block(cachep, &objp, 1, nodeid);
- spin_unlock(&(cachep->nodelists[nodeid])->list_lock);
+ spin_unlock(&(cachep->node[nodeid])->list_lock);
}
return 1;
}
#endif
/*
- * Allocates and initializes nodelists for a node on each slab cache, used for
+ * Allocates and initializes node for a node on each slab cache, used for
* either memory or cpu hotplug. If memory is being hot-added, the kmem_list3
* will be allocated off-node since memory is not yet online for the new node.
- * When hotplugging memory or a cpu, existing nodelists are not replaced if
+ * When hotplugging memory or a cpu, existing node are not replaced if
* already in use.
*
* Must hold slab_mutex.
*/
-static int init_cache_nodelists_node(int node)
+static int init_cache_node_node(int node)
{
struct kmem_cache *cachep;
struct kmem_cache_node *l3;
@@ -1203,7 +1203,7 @@ static int init_cache_nodelists_node(int
* begin anything. Make sure some other cpu on this
* node has not already allocated this
*/
- if (!cachep->nodelists[node]) {
+ if (!cachep->node[node]) {
l3 = kmalloc_node(memsize, GFP_KERNEL, node);
if (!l3)
return -ENOMEM;
@@ -1216,14 +1216,14 @@ static int init_cache_nodelists_node(int
* go. slab_mutex is sufficient
* protection here.
*/
- cachep->nodelists[node] = l3;
+ cachep->node[node] = l3;
}
- spin_lock_irq(&cachep->nodelists[node]->list_lock);
- cachep->nodelists[node]->free_limit =
+ spin_lock_irq(&cachep->node[node]->list_lock);
+ cachep->node[node]->free_limit =
(1 + nr_cpus_node(node)) *
cachep->batchcount + cachep->num;
- spin_unlock_irq(&cachep->nodelists[node]->list_lock);
+ spin_unlock_irq(&cachep->node[node]->list_lock);
}
return 0;
}
@@ -1243,7 +1243,7 @@ static void __cpuinit cpuup_canceled(lon
/* cpu is dead; no one can alloc from it. */
nc = cachep->array[cpu];
cachep->array[cpu] = NULL;
- l3 = cachep->nodelists[node];
+ l3 = cachep->node[node];
if (!l3)
goto free_array_cache;
@@ -1286,7 +1286,7 @@ free_array_cache:
* shrink each nodelist to its limit.
*/
list_for_each_entry(cachep, &slab_caches, list) {
- l3 = cachep->nodelists[node];
+ l3 = cachep->node[node];
if (!l3)
continue;
drain_freelist(cachep, l3, l3->free_objects);
@@ -1306,7 +1306,7 @@ static int __cpuinit cpuup_prepare(long
* kmalloc_node allows us to add the slab to the right
* kmem_list3 and not this cpu's kmem_list3
*/
- err = init_cache_nodelists_node(node);
+ err = init_cache_node_node(node);
if (err < 0)
goto bad;
@@ -1341,7 +1341,7 @@ static int __cpuinit cpuup_prepare(long
}
}
cachep->array[cpu] = nc;
- l3 = cachep->nodelists[node];
+ l3 = cachep->node[node];
BUG_ON(!l3);
spin_lock_irq(&l3->list_lock);
@@ -1441,7 +1441,7 @@ static struct notifier_block __cpuinitda
*
* Must hold slab_mutex.
*/
-static int __meminit drain_cache_nodelists_node(int node)
+static int __meminit drain_cache_node_node(int node)
{
struct kmem_cache *cachep;
int ret = 0;
@@ -1449,7 +1449,7 @@ static int __meminit drain_cache_nodelis
list_for_each_entry(cachep, &slab_caches, list) {
struct kmem_cache_node *l3;
- l3 = cachep->nodelists[node];
+ l3 = cachep->node[node];
if (!l3)
continue;
@@ -1478,12 +1478,12 @@ static int __meminit slab_memory_callbac
switch (action) {
case MEM_GOING_ONLINE:
mutex_lock(&slab_mutex);
- ret = init_cache_nodelists_node(nid);
+ ret = init_cache_node_node(nid);
mutex_unlock(&slab_mutex);
break;
case MEM_GOING_OFFLINE:
mutex_lock(&slab_mutex);
- ret = drain_cache_nodelists_node(nid);
+ ret = drain_cache_node_node(nid);
mutex_unlock(&slab_mutex);
break;
case MEM_ONLINE:
@@ -1515,7 +1515,7 @@ static void __init init_list(struct kmem
spin_lock_init(&ptr->list_lock);
MAKE_ALL_LISTS(cachep, ptr, nodeid);
- cachep->nodelists[nodeid] = ptr;
+ cachep->node[nodeid] = ptr;
}
/*
@@ -1527,8 +1527,8 @@ static void __init set_up_list3s(struct
int node;
for_each_online_node(node) {
- cachep->nodelists[node] = &initkmem_list3[index + node];
- cachep->nodelists[node]->next_reap = jiffies +
+ cachep->node[node] = &initkmem_list3[index + node];
+ cachep->node[node]->next_reap = jiffies +
REAPTIMEOUT_LIST3 +
((unsigned long)cachep) % REAPTIMEOUT_LIST3;
}
@@ -1536,11 +1536,11 @@ static void __init set_up_list3s(struct
/*
* The memory after the last cpu cache pointer is used for the
- * the nodelists pointer.
+ * the node pointer.
*/
-static void setup_nodelists_pointer(struct kmem_cache *s)
+static void setup_node_pointer(struct kmem_cache *s)
{
- s->nodelists = (struct kmem_cache_node **)&s->array[nr_cpu_ids];
+ s->node = (struct kmem_cache_node **)&s->array[nr_cpu_ids];
}
/*
@@ -1552,7 +1552,7 @@ void __init kmem_cache_init(void)
int i;
kmem_cache = &kmem_cache_boot;
- setup_nodelists_pointer(kmem_cache);
+ setup_node_pointer(kmem_cache);
if (num_possible_nodes() == 1)
use_alien_caches = 0;
@@ -1561,7 +1561,7 @@ void __init kmem_cache_init(void)
for (i = 0; i < NUM_INIT_LISTS; i++) {
kmem_list3_init(&initkmem_list3[i]);
if (i < nr_node_ids)
- kmem_cache->nodelists[i] = NULL;
+ kmem_cache->node[i] = NULL;
}
set_up_list3s(kmem_cache, CACHE_CACHE);
@@ -1745,7 +1745,7 @@ void __init kmem_cache_init_late(void)
#ifdef CONFIG_NUMA
/*
* Register a memory hotplug callback that initializes and frees
- * nodelists.
+ * node.
*/
hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
#endif
@@ -1790,7 +1790,7 @@ slab_out_of_memory(struct kmem_cache *ca
unsigned long active_objs = 0, num_objs = 0, free_objects = 0;
unsigned long active_slabs = 0, num_slabs = 0;
- l3 = cachep->nodelists[node];
+ l3 = cachep->node[node];
if (!l3)
continue;
@@ -2264,15 +2264,15 @@ static int __init_refok setup_cpu_cache(
} else {
int node;
for_each_online_node(node) {
- cachep->nodelists[node] =
+ cachep->node[node] =
kmalloc_node(sizeof(struct kmem_cache_node),
gfp, node);
- BUG_ON(!cachep->nodelists[node]);
- kmem_list3_init(cachep->nodelists[node]);
+ BUG_ON(!cachep->node[node]);
+ kmem_list3_init(cachep->node[node]);
}
}
}
- cachep->nodelists[numa_mem_id()]->next_reap =
+ cachep->node[numa_mem_id()]->next_reap =
jiffies + REAPTIMEOUT_LIST3 +
((unsigned long)cachep) % REAPTIMEOUT_LIST3;
@@ -2383,7 +2383,7 @@ __kmem_cache_create (struct kmem_cache *
else
gfp = GFP_NOWAIT;
- setup_nodelists_pointer(cachep);
+ setup_node_pointer(cachep);
#if DEBUG
/*
@@ -2521,7 +2521,7 @@ static void check_spinlock_acquired(stru
{
#ifdef CONFIG_SMP
check_irq_off();
- assert_spin_locked(&cachep->nodelists[numa_mem_id()]->list_lock);
+ assert_spin_locked(&cachep->node[numa_mem_id()]->list_lock);
#endif
}
@@ -2529,7 +2529,7 @@ static void check_spinlock_acquired_node
{
#ifdef CONFIG_SMP
check_irq_off();
- assert_spin_locked(&cachep->nodelists[node]->list_lock);
+ assert_spin_locked(&cachep->node[node]->list_lock);
#endif
}
@@ -2552,9 +2552,9 @@ static void do_drain(void *arg)
check_irq_off();
ac = cpu_cache_get(cachep);
- spin_lock(&cachep->nodelists[node]->list_lock);
+ spin_lock(&cachep->node[node]->list_lock);
free_block(cachep, ac->entry, ac->avail, node);
- spin_unlock(&cachep->nodelists[node]->list_lock);
+ spin_unlock(&cachep->node[node]->list_lock);
ac->avail = 0;
}
@@ -2566,13 +2566,13 @@ static void drain_cpu_caches(struct kmem
on_each_cpu(do_drain, cachep, 1);
check_irq_on();
for_each_online_node(node) {
- l3 = cachep->nodelists[node];
+ l3 = cachep->node[node];
if (l3 && l3->alien)
drain_alien_cache(cachep, l3->alien);
}
for_each_online_node(node) {
- l3 = cachep->nodelists[node];
+ l3 = cachep->node[node];
if (l3)
drain_array(cachep, l3, l3->shared, 1, node);
}
@@ -2629,7 +2629,7 @@ static int __cache_shrink(struct kmem_ca
check_irq_on();
for_each_online_node(i) {
- l3 = cachep->nodelists[i];
+ l3 = cachep->node[i];
if (!l3)
continue;
@@ -2676,7 +2676,7 @@ int __kmem_cache_shutdown(struct kmem_ca
/* NUMA: free the list3 structures */
for_each_online_node(i) {
- l3 = cachep->nodelists[i];
+ l3 = cachep->node[i];
if (l3) {
kfree(l3->shared);
free_alien_cache(l3->alien);
@@ -2873,7 +2873,7 @@ static int cache_grow(struct kmem_cache
/* Take the l3 list lock to change the colour_next on this node */
check_irq_off();
- l3 = cachep->nodelists[nodeid];
+ l3 = cachep->node[nodeid];
spin_lock(&l3->list_lock);
/* Get colour for the slab, and cal the next value. */
@@ -3071,7 +3071,7 @@ retry:
*/
batchcount = BATCHREFILL_LIMIT;
}
- l3 = cachep->nodelists[node];
+ l3 = cachep->node[node];
BUG_ON(ac->avail > 0 || !l3);
spin_lock(&l3->list_lock);
@@ -3293,7 +3293,7 @@ static void *alternate_node_alloc(struct
/*
* Fallback function if there was no memory available and no objects on a
* certain node and fall back is permitted. First we scan all the
- * available nodelists for available objects. If that fails then we
+ * available node for available objects. If that fails then we
* perform an allocation without specifying a node. This allows the page
* allocator to do its reclaim / fallback magic. We then insert the
* slab into the proper nodelist and then allocate from it.
@@ -3327,8 +3327,8 @@ retry:
nid = zone_to_nid(zone);
if (cpuset_zone_allowed_hardwall(zone, flags) &&
- cache->nodelists[nid] &&
- cache->nodelists[nid]->free_objects) {
+ cache->node[nid] &&
+ cache->node[nid]->free_objects) {
obj = ____cache_alloc_node(cache,
flags | GFP_THISNODE, nid);
if (obj)
@@ -3388,7 +3388,7 @@ static void *____cache_alloc_node(struct
void *obj;
int x;
- l3 = cachep->nodelists[nodeid];
+ l3 = cachep->node[nodeid];
BUG_ON(!l3);
retry:
@@ -3471,7 +3471,7 @@ __cache_alloc_node(struct kmem_cache *ca
if (nodeid == NUMA_NO_NODE)
nodeid = slab_node;
- if (unlikely(!cachep->nodelists[nodeid])) {
+ if (unlikely(!cachep->node[nodeid])) {
/* Node not bootstrapped yet */
ptr = fallback_alloc(cachep, flags);
goto out;
@@ -3585,7 +3585,7 @@ static void free_block(struct kmem_cache
objp = objpp[i];
slabp = virt_to_slab(objp);
- l3 = cachep->nodelists[node];
+ l3 = cachep->node[node];
list_del(&slabp->list);
check_spinlock_acquired_node(cachep, node);
check_slabp(cachep, slabp);
@@ -3629,7 +3629,7 @@ static void cache_flusharray(struct kmem
BUG_ON(!batchcount || batchcount > ac->avail);
#endif
check_irq_off();
- l3 = cachep->nodelists[node];
+ l3 = cachep->node[node];
spin_lock(&l3->list_lock);
if (l3->shared) {
struct array_cache *shared_array = l3->shared;
@@ -3941,7 +3941,7 @@ static int alloc_kmemlist(struct kmem_ca
}
}
- l3 = cachep->nodelists[node];
+ l3 = cachep->node[node];
if (l3) {
struct array_cache *shared = l3->shared;
@@ -3977,7 +3977,7 @@ static int alloc_kmemlist(struct kmem_ca
l3->alien = new_alien;
l3->free_limit = (1 + nr_cpus_node(node)) *
cachep->batchcount + cachep->num;
- cachep->nodelists[node] = l3;
+ cachep->node[node] = l3;
}
return 0;
@@ -3986,13 +3986,13 @@ fail:
/* Cache is not active yet. Roll back what we did */
node--;
while (node >= 0) {
- if (cachep->nodelists[node]) {
- l3 = cachep->nodelists[node];
+ if (cachep->node[node]) {
+ l3 = cachep->node[node];
kfree(l3->shared);
free_alien_cache(l3->alien);
kfree(l3);
- cachep->nodelists[node] = NULL;
+ cachep->node[node] = NULL;
}
node--;
}
@@ -4052,9 +4052,9 @@ static int do_tune_cpucache(struct kmem_
struct array_cache *ccold = new->new[i];
if (!ccold)
continue;
- spin_lock_irq(&cachep->nodelists[cpu_to_mem(i)]->list_lock);
+ spin_lock_irq(&cachep->node[cpu_to_mem(i)]->list_lock);
free_block(cachep, ccold->entry, ccold->avail, cpu_to_mem(i));
- spin_unlock_irq(&cachep->nodelists[cpu_to_mem(i)]->list_lock);
+ spin_unlock_irq(&cachep->node[cpu_to_mem(i)]->list_lock);
kfree(ccold);
}
kfree(new);
@@ -4175,7 +4175,7 @@ static void cache_reap(struct work_struc
* have established with reasonable certainty that
* we can do some work if the lock was obtained.
*/
- l3 = searchp->nodelists[node];
+ l3 = searchp->node[node];
reap_alien(searchp, l3);
@@ -4274,7 +4274,7 @@ static int s_show(struct seq_file *m, vo
active_objs = 0;
num_slabs = 0;
for_each_online_node(node) {
- l3 = cachep->nodelists[node];
+ l3 = cachep->node[node];
if (!l3)
continue;
@@ -4528,7 +4528,7 @@ static int leaks_show(struct seq_file *m
n[1] = 0;
for_each_online_node(node) {
- l3 = cachep->nodelists[node];
+ l3 = cachep->node[node];
if (!l3)
continue;
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 21+ messages in thread