[patch] slob: implement RCU freeing

[patch] slob: implement RCU freeing

[Nick Piggin]

[-- Attachment #1: Type: text/plain, Size: 4 bytes --]

-- 

[-- Attachment #2: slob-add-rcu.patch --]
[-- Type: text/plain, Size: 3053 bytes --]

SLOB allocator should implement SLAB_DESTROY_BY_RCU correctly, because even
on UP, RCU freeing semantics are not equivalent to simply freeing immediately.
This also allows SLOB to be used on SMP.

Signed-off-by: Nick Piggin <npiggin@suse.de>

Index: linux-2.6/init/Kconfig
===================================================================
--- linux-2.6.orig/init/Kconfig	2007-05-09 11:54:11.000000000 +1000
+++ linux-2.6/init/Kconfig	2007-05-09 11:54:50.000000000 +1000
@@ -476,7 +476,7 @@
 
 config SLAB
 	default y
-	bool "Use full SLAB allocator" if (EMBEDDED && !SMP && !SPARSEMEM)
+	bool "Use full SLAB allocator" if (EMBEDDED && !SPARSEMEM)
 	help
 	  Disabling this replaces the advanced SLAB allocator and
 	  kmalloc support with the drastically simpler SLOB allocator.
Index: linux-2.6/mm/slob.c
===================================================================
--- linux-2.6.orig/mm/slob.c	2007-05-09 11:54:11.000000000 +1000
+++ linux-2.6/mm/slob.c	2007-05-09 11:54:55.000000000 +1000
@@ -35,6 +35,7 @@
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/timer.h>
+#include <linux/rcupdate.h>
 
 struct slob_block {
 	int units;
@@ -53,6 +54,11 @@
 };
 typedef struct bigblock bigblock_t;
 
+struct slob_rcu {
+	struct rcu_head head;
+	int size;
+};
+
 static slob_t arena = { .next = &arena, .units = 1 };
 static slob_t *slobfree = &arena;
 static bigblock_t *bigblocks;
@@ -242,6 +248,7 @@
 
 struct kmem_cache {
 	unsigned int size, align;
+	unsigned long flags;
 	const char *name;
 	void (*ctor)(void *, struct kmem_cache *, unsigned long);
 	void (*dtor)(void *, struct kmem_cache *, unsigned long);
@@ -259,6 +266,12 @@
 	if (c) {
 		c->name = name;
 		c->size = size;
+		if (flags & SLAB_DESTROY_BY_RCU) {
+			BUG_ON(c->dtor);
+			/* leave room for rcu footer at the end of object */
+			c->size += sizeof(struct slob_rcu);
+		}
+		c->flags = flags;
 		c->ctor = ctor;
 		c->dtor = dtor;
 		/* ignore alignment unless it's forced */
@@ -303,15 +316,35 @@
 }
 EXPORT_SYMBOL(kmem_cache_zalloc);
 
-void kmem_cache_free(struct kmem_cache *c, void *b)
+static void __kmem_cache_free(void *b, int size)
 {
-	if (c->dtor)
-		c->dtor(b, c, 0);
-
-	if (c->size < PAGE_SIZE)
-		slob_free(b, c->size);
+	if (size < PAGE_SIZE)
+		slob_free(b, size);
 	else
-		free_pages((unsigned long)b, find_order(c->size));
+		free_pages((unsigned long)b, find_order(size));
+}
+
+static void kmem_rcu_free(struct rcu_head *head)
+{
+	struct slob_rcu *slob_rcu = (struct slob_rcu *)head;
+	void *b = (void *)slob_rcu - (slob_rcu->size - sizeof(struct slob_rcu));
+
+	__kmem_cache_free(b, slob_rcu->size);
+}
+
+void kmem_cache_free(struct kmem_cache *c, void *b)
+{
+	if (unlikely(c->flags & SLAB_DESTROY_BY_RCU)) {
+		struct slob_rcu *slob_rcu;
+		slob_rcu = b + (c->size - sizeof(struct slob_rcu));
+		INIT_RCU_HEAD(&slob_rcu->head);
+		slob_rcu->size = c->size;
+		call_rcu(&slob_rcu->head, kmem_rcu_free);
+	} else {
+		if (c->dtor)
+			c->dtor(b, c, 0);
+		__kmem_cache_free(b, c->size);
+	}
 }
 EXPORT_SYMBOL(kmem_cache_free);
 

[patch] slob: implement RCU freeing

[Nick Piggin]

[-- Attachment #1: Type: text/plain, Size: 231 bytes --]

Matt Mackall wrote:

> Looks good to me, but haven't had time to actually test it.
> 
> Acked-by: Matt Mackall <mpm@selenic.com>

Updated to current, added a comment, and test booted it again.
Works OK.

-- 
SUSE Labs, Novell Inc.

[-- Attachment #2: slob-add-rcu.patch --]
[-- Type: text/plain, Size: 3950 bytes --]

The SLOB allocator should implement SLAB_DESTROY_BY_RCU correctly, because even
on UP, RCU freeing semantics are not equivalent to simply freeing immediately.
This also allows SLOB to be used on SMP.

Signed-off-by: Nick Piggin <npiggin@suse.de>
Acked-by: Matt Mackall <mpm@selenic.com>

Index: linux-2.6/init/Kconfig
===================================================================
--- linux-2.6.orig/init/Kconfig	2007-05-10 14:17:38.000000000 +1000
+++ linux-2.6/init/Kconfig	2007-05-10 14:18:11.000000000 +1000
@@ -519,7 +519,8 @@
 	  slab allocator.
 
 config SLUB
-	depends on EXPERIMENTAL && !ARCH_USES_SLAB_PAGE_STRUCT
+#	depends on EXPERIMENTAL && !ARCH_USES_SLAB_PAGE_STRUCT
+	depends on EXPERIMENTAL
 	bool "SLUB (Unqueued Allocator)"
 	help
 	   SLUB is a slab allocator that minimizes cache line usage
@@ -529,15 +530,11 @@
 	   way and has enhanced diagnostics.
 
 config SLOB
-#
-#	SLOB cannot support SMP because SLAB_DESTROY_BY_RCU does not work
-#	properly.
-#
-	depends on EMBEDDED && !SMP && !SPARSEMEM
+	depends on EMBEDDED && !SPARSEMEM
 	bool "SLOB (Simple Allocator)"
 	help
 	   SLOB replaces the SLAB allocator with a drastically simpler
-	   allocator.  SLOB is more space efficient that SLAB but does not
+	   allocator.  SLOB is more space efficient than SLAB but does not
 	   scale well (single lock for all operations) and is more susceptible
 	   to fragmentation. SLOB it is a great choice to reduce
 	   memory usage and code size for embedded systems.
Index: linux-2.6/mm/slob.c
===================================================================
--- linux-2.6.orig/mm/slob.c	2007-05-10 14:17:38.000000000 +1000
+++ linux-2.6/mm/slob.c	2007-05-10 14:59:35.000000000 +1000
@@ -35,6 +35,7 @@
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/timer.h>
+#include <linux/rcupdate.h>
 
 struct slob_block {
 	int units;
@@ -53,6 +54,16 @@
 };
 typedef struct bigblock bigblock_t;
 
+/*
+ * struct slob_rcu is inserted at the tail of allocated slob blocks, which
+ * were created with a SLAB_DESTROY_BY_RCU slab. slob_rcu is used to free
+ * the block using call_rcu.
+ */
+struct slob_rcu {
+	struct rcu_head head;
+	int size;
+};
+
 static slob_t arena = { .next = &arena, .units = 1 };
 static slob_t *slobfree = &arena;
 static bigblock_t *bigblocks;
@@ -266,6 +277,7 @@
 
 struct kmem_cache {
 	unsigned int size, align;
+	unsigned long flags;
 	const char *name;
 	void (*ctor)(void *, struct kmem_cache *, unsigned long);
 	void (*dtor)(void *, struct kmem_cache *, unsigned long);
@@ -283,6 +295,12 @@
 	if (c) {
 		c->name = name;
 		c->size = size;
+		if (flags & SLAB_DESTROY_BY_RCU) {
+			BUG_ON(c->dtor);
+			/* leave room for rcu footer at the end of object */
+			c->size += sizeof(struct slob_rcu);
+		}
+		c->flags = flags;
 		c->ctor = ctor;
 		c->dtor = dtor;
 		/* ignore alignment unless it's forced */
@@ -328,15 +346,35 @@
 }
 EXPORT_SYMBOL(kmem_cache_zalloc);
 
-void kmem_cache_free(struct kmem_cache *c, void *b)
+static void __kmem_cache_free(void *b, int size)
 {
-	if (c->dtor)
-		c->dtor(b, c, 0);
-
-	if (c->size < PAGE_SIZE)
-		slob_free(b, c->size);
+	if (size < PAGE_SIZE)
+		slob_free(b, size);
 	else
-		free_pages((unsigned long)b, get_order(c->size));
+		free_pages((unsigned long)b, get_order(size));
+}
+
+static void kmem_rcu_free(struct rcu_head *head)
+{
+	struct slob_rcu *slob_rcu = (struct slob_rcu *)head;
+	void *b = (void *)slob_rcu - (slob_rcu->size - sizeof(struct slob_rcu));
+
+	__kmem_cache_free(b, slob_rcu->size);
+}
+
+void kmem_cache_free(struct kmem_cache *c, void *b)
+{
+	if (unlikely(c->flags & SLAB_DESTROY_BY_RCU)) {
+		struct slob_rcu *slob_rcu;
+		slob_rcu = b + (c->size - sizeof(struct slob_rcu));
+		INIT_RCU_HEAD(&slob_rcu->head);
+		slob_rcu->size = c->size;
+		call_rcu(&slob_rcu->head, kmem_rcu_free);
+	} else {
+		if (c->dtor)
+			c->dtor(b, c, 0);
+		__kmem_cache_free(b, c->size);
+	}
 }
 EXPORT_SYMBOL(kmem_cache_free);
 

Re: [patch] slob: implement RCU freeing

[Nick Piggin]

Nick Piggin wrote:
> Matt Mackall wrote:
> 
>> Looks good to me, but haven't had time to actually test it.
>>
>> Acked-by: Matt Mackall <mpm@selenic.com>
> 
> 
> Updated to current, added a comment, and test booted it again.
> Works OK.
> 
> 
> ------------------------------------------------------------------------
> 
> The SLOB allocator should implement SLAB_DESTROY_BY_RCU correctly, because even
> on UP, RCU freeing semantics are not equivalent to simply freeing immediately.
> This also allows SLOB to be used on SMP.
> 
> Signed-off-by: Nick Piggin <npiggin@suse.de>
> Acked-by: Matt Mackall <mpm@selenic.com>
> 
> Index: linux-2.6/init/Kconfig
> ===================================================================
> --- linux-2.6.orig/init/Kconfig	2007-05-10 14:17:38.000000000 +1000
> +++ linux-2.6/init/Kconfig	2007-05-10 14:18:11.000000000 +1000
> @@ -519,7 +519,8 @@
>  	  slab allocator.
>  
>  config SLUB
> -	depends on EXPERIMENTAL && !ARCH_USES_SLAB_PAGE_STRUCT
> +#	depends on EXPERIMENTAL && !ARCH_USES_SLAB_PAGE_STRUCT
> +	depends on EXPERIMENTAL
>  	bool "SLUB (Unqueued Allocator)"
>  	help
>  	   SLUB is a slab allocator that minimizes cache line usage
> @@ -529,15 +530,11 @@

Ooops, sorry this hunk leaked in because I wanted to test compile slub :P

-- 
SUSE Labs, Novell Inc.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [patch] slob: implement RCU freeing

[Peter Zijlstra]

On Thu, 2007-05-10 at 17:15 +1000, Nick Piggin wrote:

> @@ -283,6 +295,12 @@
>  	if (c) {
>  		c->name = name;
>  		c->size = size;
> +		if (flags & SLAB_DESTROY_BY_RCU) {
> +			BUG_ON(c->dtor);
> +			/* leave room for rcu footer at the end of object */
> +			c->size += sizeof(struct slob_rcu);
> +		}
> +		c->flags = flags;

might want to put this hunt below

> 		c->ctor = ctor;
>  		c->dtor = dtor;

here; for c->dtor is not initialised quite yet at the BUG_ON site.

> 		/* ignore alignment unless it's forced */



--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [patch] slob: implement RCU freeing

[Nick Piggin]

[-- Attachment #1: Type: text/plain, Size: 569 bytes --]

Peter Zijlstra wrote:
> On Thu, 2007-05-10 at 17:15 +1000, Nick Piggin wrote:
> 
> 
>>@@ -283,6 +295,12 @@
>> 	if (c) {
>> 		c->name = name;
>> 		c->size = size;
>>+		if (flags & SLAB_DESTROY_BY_RCU) {
>>+			BUG_ON(c->dtor);
>>+			/* leave room for rcu footer at the end of object */
>>+			c->size += sizeof(struct slob_rcu);
>>+		}
>>+		c->flags = flags;
> 
> 
> might want to put this hunt below
> 
> 
>>		c->ctor = ctor;
>> 		c->dtor = dtor;
> 
> 
> here; for c->dtor is not initialised quite yet at the BUG_ON site.

Indeed, how's this?

-- 
SUSE Labs, Novell Inc.

[-- Attachment #2: slob-add-rcu-fix.patch --]
[-- Type: text/plain, Size: 426 bytes --]

Index: linux-2.6/mm/slob.c
===================================================================
--- linux-2.6.orig/mm/slob.c
+++ linux-2.6/mm/slob.c
@@ -296,7 +296,7 @@ struct kmem_cache *kmem_cache_create(con
 		c->name = name;
 		c->size = size;
 		if (flags & SLAB_DESTROY_BY_RCU) {
-			BUG_ON(c->dtor);
+			BUG_ON(dtor);
 			/* leave room for rcu footer at the end of object */
 			c->size += sizeof(struct slob_rcu);
 		}

Re: [patch] slob: implement RCU freeing

[Peter Zijlstra]

On Thu, 2007-05-10 at 18:26 +1000, Nick Piggin wrote:

> Indeed, how's this?
> 
> plain text document attachment (slob-add-rcu-fix.patch)
> Index: linux-2.6/mm/slob.c
> ===================================================================
> --- linux-2.6.orig/mm/slob.c
> +++ linux-2.6/mm/slob.c
> @@ -296,7 +296,7 @@ struct kmem_cache *kmem_cache_create(con
>  		c->name = name;
>  		c->size = size;
>  		if (flags & SLAB_DESTROY_BY_RCU) {
> -			BUG_ON(c->dtor);
> +			BUG_ON(dtor);
>  			/* leave room for rcu footer at the end of object */
>  			c->size += sizeof(struct slob_rcu);
>  		}

works for me :-)

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>