* [RFC PATCH 1/3] Supporting hacks to be able to test slab allocated buffers in place of page_frag without rewriting lots of net code. We make several assumptions here, first that slab allocator is selected. Second, no one is doing get_page or put_page on pages marked PG_slab. Third we allocated all slabs page aligned that we do these calls on.
2014-01-16 23:17 [RFC PATCH 0/3] Use cached allocations in place of order-3 allocations for sk_page_frag_refill() and __netdev_alloc_frag() Debabrata Banerjee
@ 2014-01-16 23:17 ` Debabrata Banerjee
2014-01-16 23:17 ` [RFC PATCH 2/3] Use slab allocations for netdev page_frag receive buffers Debabrata Banerjee
2014-01-16 23:17 ` [RFC PATCH 3/3] Use slab allocations for sk page_frag send buffers Debabrata Banerjee
2 siblings, 0 replies; 4+ messages in thread
From: Debabrata Banerjee @ 2014-01-16 23:17 UTC (permalink / raw)
To: eric.dumazet, fw, netdev; +Cc: dbanerje, johunt, jbaron, davem, linux-mm
---
include/linux/mm.h | 6 ++++++
mm/slab.c | 8 ++++++++
mm/swap.c | 13 ++++++++++++-
3 files changed, 26 insertions(+), 1 deletion(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index e0c8528..de21a92 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -398,12 +398,18 @@ static inline void get_huge_page_tail(struct page *page)
}
extern bool __get_page_tail(struct page *page);
+extern struct page *slabpage_to_headpage(struct page *page);
static inline void get_page(struct page *page)
{
if (unlikely(PageTail(page)))
if (likely(__get_page_tail(page)))
return;
+
+ //Hack for slab page
+ if (unlikely(page->flags & (1L << PG_slab)))
+ page = slabpage_to_headpage(page);
+
/*
* Getting a normal page or the head of a compound page
* requires to already have an elevated page->_count.
diff --git a/mm/slab.c b/mm/slab.c
index bd88411..36d5176 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -483,6 +483,14 @@ static inline unsigned int obj_to_index(const struct kmem_cache *cache,
return reciprocal_divide(offset, cache->reciprocal_buffer_size);
}
+struct page *slabpage_to_headpage(struct page *page)
+{
+ //Hack to support get_page/put_page on slabs bigger than a page
+ unsigned int idx = obj_to_index(page->slab_cache, page->slab_page, page_address(page));
+ return virt_to_page(index_to_obj(page->slab_cache, page->slab_page, idx));
+}
+EXPORT_SYMBOL(slabpage_to_headpage);
+
static struct arraycache_init initarray_generic =
{ {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
diff --git a/mm/swap.c b/mm/swap.c
index 9f2225f..94c75bc 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -172,9 +172,20 @@ skip_lock_tail:
}
}
+extern struct page *slabpage_to_headpage(struct page *page);
+
void put_page(struct page *page)
{
- if (unlikely(PageCompound(page)))
+ if (unlikely(page->flags & (1L << PG_slab))) {
+ struct page *head_page = slabpage_to_headpage(page);
+ //Hack. Assume we have >PAGE_SIZE and aligned slabs, and no one is dumb enough
+ //to do a put_page to 0 on a slab page without meaning to free it from the slab.
+ if (put_page_testzero(head_page)) {
+ get_page(head_page); //restore 1 _count for slab
+ kmem_cache_free(page->slab_cache, page_address(head_page));
+ }
+ }
+ else if (unlikely(PageCompound(page)))
put_compound_page(page);
else if (put_page_testzero(page))
__put_single_page(page);
--
1.8.3.4
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 4+ messages in thread* [RFC PATCH 2/3] Use slab allocations for netdev page_frag receive buffers
2014-01-16 23:17 [RFC PATCH 0/3] Use cached allocations in place of order-3 allocations for sk_page_frag_refill() and __netdev_alloc_frag() Debabrata Banerjee
2014-01-16 23:17 ` [RFC PATCH 1/3] Supporting hacks to be able to test slab allocated buffers in place of page_frag without rewriting lots of net code. We make several assumptions here, first that slab allocator is selected. Second, no one is doing get_page or put_page on pages marked PG_slab. Third we allocated all slabs page aligned that we do these calls on Debabrata Banerjee
@ 2014-01-16 23:17 ` Debabrata Banerjee
2014-01-16 23:17 ` [RFC PATCH 3/3] Use slab allocations for sk page_frag send buffers Debabrata Banerjee
2 siblings, 0 replies; 4+ messages in thread
From: Debabrata Banerjee @ 2014-01-16 23:17 UTC (permalink / raw)
To: eric.dumazet, fw, netdev; +Cc: dbanerje, johunt, jbaron, davem, linux-mm
---
net/core/skbuff.c | 33 ++++++++++++++++++++++-----------
1 file changed, 22 insertions(+), 11 deletions(-)
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index d9e8736..7ecb7a8 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -368,6 +368,8 @@ struct netdev_alloc_cache {
};
static DEFINE_PER_CPU(struct netdev_alloc_cache, netdev_alloc_cache);
+struct kmem_cache *netdev_page_frag_cache;
+
static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
{
struct netdev_alloc_cache *nc;
@@ -379,18 +381,22 @@ static void *__netdev_alloc_frag(unsigned int fragsz, gfp_t gfp_mask)
nc = &__get_cpu_var(netdev_alloc_cache);
if (unlikely(!nc->frag.page)) {
refill:
- for (order = NETDEV_FRAG_PAGE_MAX_ORDER; ;) {
- gfp_t gfp = gfp_mask;
-
- if (order)
- gfp |= __GFP_COMP | __GFP_NOWARN;
- nc->frag.page = alloc_pages(gfp, order);
- if (likely(nc->frag.page))
- break;
- if (--order < 0)
- goto end;
+ if (NETDEV_FRAG_PAGE_MAX_ORDER > 0) {
+ void *kmem = kmem_cache_alloc(netdev_page_frag_cache, gfp_mask | __GFP_NOWARN);
+ if (likely(kmem)) {
+ nc->frag.page = virt_to_page(kmem);
+ nc->frag.size = PAGE_SIZE << NETDEV_FRAG_PAGE_MAX_ORDER;
+ goto recycle;
+ }
}
- nc->frag.size = PAGE_SIZE << order;
+
+ nc->frag.page = alloc_page(gfp_mask);
+
+ if (likely(nc->frag.page))
+ nc->frag.size = PAGE_SIZE;
+ else
+ goto end;
+
recycle:
atomic_set(&nc->frag.page->_count, NETDEV_PAGECNT_MAX_BIAS);
nc->pagecnt_bias = NETDEV_PAGECNT_MAX_BIAS;
@@ -3092,6 +3098,11 @@ void __init skb_init(void)
0,
SLAB_HWCACHE_ALIGN|SLAB_PANIC,
NULL);
+ netdev_page_frag_cache = kmem_cache_create("netdev_page_frag_cache",
+ PAGE_SIZE << NETDEV_FRAG_PAGE_MAX_ORDER,
+ PAGE_SIZE,
+ SLAB_HWCACHE_ALIGN,
+ NULL);
}
/**
--
1.8.3.4
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 4+ messages in thread* [RFC PATCH 3/3] Use slab allocations for sk page_frag send buffers
2014-01-16 23:17 [RFC PATCH 0/3] Use cached allocations in place of order-3 allocations for sk_page_frag_refill() and __netdev_alloc_frag() Debabrata Banerjee
2014-01-16 23:17 ` [RFC PATCH 1/3] Supporting hacks to be able to test slab allocated buffers in place of page_frag without rewriting lots of net code. We make several assumptions here, first that slab allocator is selected. Second, no one is doing get_page or put_page on pages marked PG_slab. Third we allocated all slabs page aligned that we do these calls on Debabrata Banerjee
2014-01-16 23:17 ` [RFC PATCH 2/3] Use slab allocations for netdev page_frag receive buffers Debabrata Banerjee
@ 2014-01-16 23:17 ` Debabrata Banerjee
2 siblings, 0 replies; 4+ messages in thread
From: Debabrata Banerjee @ 2014-01-16 23:17 UTC (permalink / raw)
To: eric.dumazet, fw, netdev; +Cc: dbanerje, johunt, jbaron, davem, linux-mm
---
net/core/sock.c | 33 ++++++++++++++++++++++-----------
1 file changed, 22 insertions(+), 11 deletions(-)
diff --git a/net/core/sock.c b/net/core/sock.c
index 6565431..dbbd2f9 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1792,10 +1792,12 @@ EXPORT_SYMBOL(sock_alloc_send_skb);
/* On 32bit arches, an skb frag is limited to 2^15 */
#define SKB_FRAG_PAGE_ORDER get_order(32768)
+struct kmem_cache *sk_page_frag_cache;
bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
{
int order;
+ gfp_t gfp_mask = sk->sk_allocation;
if (pfrag->page) {
if (atomic_read(&pfrag->page->_count) == 1) {
@@ -1807,21 +1809,25 @@ bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
put_page(pfrag->page);
}
- /* We restrict high order allocations to users that can afford to wait */
- order = (sk->sk_allocation & __GFP_WAIT) ? SKB_FRAG_PAGE_ORDER : 0;
+ order = SKB_FRAG_PAGE_ORDER;
- do {
- gfp_t gfp = sk->sk_allocation;
-
- if (order)
- gfp |= __GFP_COMP | __GFP_NOWARN;
- pfrag->page = alloc_pages(gfp, order);
- if (likely(pfrag->page)) {
+ if (order > 0) {
+ void *kmem = kmem_cache_alloc(sk_page_frag_cache, gfp_mask | __GFP_NOWARN);
+ if (likely(kmem)) {
+ pfrag->page = virt_to_page(kmem);
pfrag->offset = 0;
pfrag->size = PAGE_SIZE << order;
return true;
}
- } while (--order >= 0);
+ }
+
+ pfrag->page = alloc_page(gfp_mask);
+
+ if (likely(pfrag->page)) {
+ pfrag->offset = 0;
+ pfrag->size = PAGE_SIZE;
+ return true;
+ }
sk_enter_memory_pressure(sk);
sk_stream_moderate_sndbuf(sk);
@@ -2822,13 +2828,18 @@ static __net_init int proto_init_net(struct net *net)
{
if (!proc_create("protocols", S_IRUGO, net->proc_net, &proto_seq_fops))
return -ENOMEM;
-
+ sk_page_frag_cache = kmem_cache_create("sk_page_frag_cache",
+ PAGE_SIZE << SKB_FRAG_PAGE_ORDER,
+ PAGE_SIZE,
+ SLAB_HWCACHE_ALIGN,
+ NULL);
return 0;
}
static __net_exit void proto_exit_net(struct net *net)
{
remove_proc_entry("protocols", net->proc_net);
+ kmem_cache_destroy(sk_page_frag_cache);
}
--
1.8.3.4
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 4+ messages in thread