[PATCH] mm: exempt pcp alloc from watermarks

[Peter Zijlstra <a.p.zijlstra@chello.nl>]

On Sun, 2006-09-17 at 23:52 +1000, Nick Piggin wrote:

> What we could do then, is allocate pages in batches (we already do),
> but only check watermarks if we have to go to the buddly allocator
> (we don't currently do this, but really should anyway, considering
> that the watermark checks are based on pages in the buddy allocator
> rather than pages in buddy + pcp).

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
 mm/page_alloc.c |   57 ++++++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 53 insertions(+), 4 deletions(-)

Index: linux-2.6/mm/page_alloc.c
===================================================================
--- linux-2.6.orig/mm/page_alloc.c
+++ linux-2.6/mm/page_alloc.c
@@ -767,6 +767,42 @@ void split_page(struct page *page, unsig
 		set_page_refcounted(page + i);
 }
 
+static struct page *get_pcp_page(struct zonelist *zonelist, struct zone *zone,
+				 gfp_t gfp_mask, int order)
+{
+	unsigned long flags;
+	int cold, cpu;
+	struct per_cpu_pages *pcp;
+	struct page *page;
+
+	if (unlikely(order != 0))
+		return NULL;
+
+	cold = !!(gfp_mask & __GFP_COLD);
+again:
+	page = NULL;
+	cpu = get_cpu();
+	pcp = &zone_pcp(zone, cpu)->pcp[cold];
+	local_irq_save(flags);
+	if (pcp->count) {
+		page = list_entry(pcp->list.next, struct page, lru);
+		list_del(&page->lru);
+		pcp->count--;
+		__count_zone_vm_events(PGALLOC, zone, 1);
+		zone_statistics(zonelist, zone);
+	}
+	local_irq_restore(flags);
+	put_cpu();
+
+	if (page) {
+		BUG_ON(bad_range(zone, page));
+		if (prep_new_page(page, 0, gfp_mask))
+			goto again;
+	}
+
+	return page;
+}
+
 /*
  * Really, prep_compound_page() should be called from __rmqueue_bulk().  But
  * we cheat by calling it from here, in the order > 0 path.  Saves a branch
@@ -781,13 +817,17 @@ static struct page *buffered_rmqueue(str
 	int cpu;
 
 again:
-	cpu  = get_cpu();
+	cpu = get_cpu();
 	if (likely(order == 0)) {
 		struct per_cpu_pages *pcp;
 
 		pcp = &zone_pcp(zone, cpu)->pcp[cold];
 		local_irq_save(flags);
-		if (!pcp->count) {
+		/*
+		 * Even though we checked the pcps earlier we could have
+		 * been preempted and scheduled to another cpu.
+		 */
+		if (likely(!pcp->count)) {
 			pcp->count += rmqueue_bulk(zone, 0,
 						pcp->batch, &pcp->list);
 			if (unlikely(!pcp->count))
@@ -882,6 +922,16 @@ get_page_from_freelist(gfp_t gfp_mask, u
 
 		if (!(alloc_flags & ALLOC_NO_WATERMARKS)) {
 			unsigned long mark;
+
+			/*
+			 * If there are pages in the pcp lists take those
+			 * without checking the watermarks, since the zone
+			 * free_pages count is without the pcp count.
+			 */
+			page = get_pcp_page(zonelist, *z, order, gfp_mask);
+			if (page)
+				break;
+
 			if (alloc_flags & ALLOC_WMARK_MIN)
 				mark = (*z)->pages_min;
 			else if (alloc_flags & ALLOC_WMARK_LOW)
@@ -896,9 +946,8 @@ get_page_from_freelist(gfp_t gfp_mask, u
 		}
 
 		page = buffered_rmqueue(zonelist, *z, order, gfp_mask);
-		if (page) {
+		if (page)
 			break;
-		}
 	} while (*(++z) != NULL);
 	return page;
 }


--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>