[PATCH][8/8] mm: lru interface change

[PATCH][8/8] mm: lru interface change

[Stone Wang]

Add Wired list to LRU.
Add PG_Wired bit to page.

Signed-off-by: Shaoping Wang <pwstone@gmail.com>

--
 include/linux/mm_inline.h  |   25 +++++++++++++
 include/linux/mmzone.h     |    2 +
 include/linux/page-flags.h |    9 +++++
 include/linux/swap.h       |    2 +
 mm/page_alloc.c            |    4 +-
 mm/swap.c                  |   81 +++++++++++++++++++++++++++++++++++++--------
 6 files changed, 107 insertions(+), 16 deletions(-)

diff -urN  linux-2.6.15.orig/include/linux/mm_inline.h
linux-2.6.15/include/linux/mm_inline.h
--- linux-2.6.15.orig/include/linux/mm_inline.h	2006-01-02
22:21:10.000000000 -0500
+++ linux-2.6.15/include/linux/mm_inline.h	2006-03-07 01:56:10.000000000 -0500
@@ -1,3 +1,9 @@
+/*
+ * There are 3 per-zone lists in LRU:
+ * 			Active: pages which were accessed more frequently.
+ *			Inactive: pages accessed less frequently.
+ *			Wired: pages mlocked by some tasks.
+ */

 static inline void
 add_page_to_active_list(struct zone *zone, struct page *page)
@@ -14,6 +20,13 @@
 }

 static inline void
+add_page_to_wired_list(struct zone *zone, struct page *page)
+{
+	list_add(&page->lru, &zone->wired_list);
+	zone->nr_wired++;
+}
+
+static inline void
 del_page_from_active_list(struct zone *zone, struct page *page)
 {
 	list_del(&page->lru);
@@ -28,10 +41,20 @@
 }

 static inline void
+del_page_from_wired_list(struct zone *zone, struct page *page)
+{
+	list_del(&page->lru);
+	zone->nr_wired--;
+}
+
+static inline void
 del_page_from_lru(struct zone *zone, struct page *page)
 {
 	list_del(&page->lru);
-	if (PageActive(page)) {
+	if(PageWired(page)){
+		ClearPageWired(page);
+		zone->nr_wired--;
+	} else if (PageActive(page)) {
 		ClearPageActive(page);
 		zone->nr_active--;
 	} else {
diff -urN  linux-2.6.15.orig/include/linux/mmzone.h
linux-2.6.15/include/linux/mmzone.h
--- linux-2.6.15.orig/include/linux/mmzone.h	2006-01-02 22:21:10.000000000 -0500
+++ linux-2.6.15/include/linux/mmzone.h	2006-03-07 01:58:26.000000000 -0500
@@ -143,10 +143,12 @@
 	spinlock_t		lru_lock;	
 	struct list_head	active_list;
 	struct list_head	inactive_list;
+	struct list_head	wired_list; /* Pages wired. */
 	unsigned long		nr_scan_active;
 	unsigned long		nr_scan_inactive;
 	unsigned long		nr_active;
 	unsigned long		nr_inactive;
+	unsigned long		nr_wired;
 	unsigned long		pages_scanned;	   /* since last reclaim */
 	int			all_unreclaimable; /* All pages pinned */

diff -urN  linux-2.6.15.orig/include/linux/page-flags.h
linux-2.6.15/include/linux/page-flags.h
--- linux-2.6.15.orig/include/linux/page-flags.h	2006-01-02
22:21:10.000000000 -0500
+++ linux-2.6.15/include/linux/page-flags.h	2006-03-06 06:30:07.000000000 -0500
@@ -76,6 +76,8 @@
 #define PG_nosave_free		18	/* Free, should not be written */
 #define PG_uncached		19	/* Page has been mapped as uncached */

+#define PG_wired		20  /* Page is on Wired list */
+
 /*
  * Global page accounting.  One instance per CPU.  Only unsigned longs are
  * allowed.
@@ -198,7 +200,14 @@
 #define __ClearPageDirty(page)	__clear_bit(PG_dirty, &(page)->flags)
 #define TestClearPageDirty(page) test_and_clear_bit(PG_dirty, &(page)->flags)

+#define SetPageWired(page)	set_bit(PG_wired, &(page)->flags)
+#define ClearPageWired(page) clear_bit(PG_wired,&(page)->flags)
+#define PageWired(page)		test_bit(PG_wired, &(page)->flags)
+#define TestSetPageWired(page)	test_and_set_bit(PG_wired, &(page)->flags)
+#define TestClearPageWired(page)	test_and_clear_bit(PG_wired, &(page)->flags)
+
 #define SetPageLRU(page)	set_bit(PG_lru, &(page)->flags)
+#define ClearPageLRU(page)	clear_bit(PG_lru, &(page)->flags)
 #define PageLRU(page)		test_bit(PG_lru, &(page)->flags)
 #define TestSetPageLRU(page)	test_and_set_bit(PG_lru, &(page)->flags)
 #define TestClearPageLRU(page)	test_and_clear_bit(PG_lru, &(page)->flags)
diff -urN  linux-2.6.15.orig/include/linux/swap.h
linux-2.6.15/include/linux/swap.h
--- linux-2.6.15.orig/include/linux/swap.h	2006-01-02 22:21:10.000000000 -0500
+++ linux-2.6.15/include/linux/swap.h	2006-03-06 06:30:07.000000000 -0500
@@ -165,6 +165,8 @@
 extern void FASTCALL(lru_cache_add(struct page *));
 extern void FASTCALL(lru_cache_add_active(struct page *));
 extern void FASTCALL(activate_page(struct page *));
+extern void FASTCALL(wire_page(struct page *));
+extern void FASTCALL(unwire_page(struct page *));
 extern void FASTCALL(mark_page_accessed(struct page *));
 extern void lru_add_drain(void);
 extern int rotate_reclaimable_page(struct page *page);
diff -urN  linux-2.6.15.orig/mm/swap.c linux-2.6.15/mm/swap.c
--- linux-2.6.15.orig/mm/swap.c	2006-01-02 22:21:10.000000000 -0500
+++ linux-2.6.15/mm/swap.c	2006-03-07 11:45:37.000000000 -0500
@@ -110,6 +110,44 @@
 	spin_unlock_irq(&zone->lru_lock);
 }

+/* Wire the page; if the page is in LRU,
+ * try move it to Wired list.
+ */
+void fastcall wire_page(struct page *page)
+{
+	struct zone *zone = page_zone(page);
+
+	spin_lock_irq(&zone->lru_lock);
+	page->wired_count ++;
+	if(!PageWired(page)){
+		if(PageLRU(page)){
+			del_page_from_lru(zone, page);
+			add_page_to_wired_list(zone,page);
+			SetPageWired(page);
+		}
+	}
+	spin_unlock_irq(&zone->lru_lock);
+}
+
+/* Unwire the page.
+ * If it isnt wired by any process, try move it to active list.
+ */
+void fastcall unwire_page(struct page *page)
+{
+	struct zone *zone = page_zone(page);
+
+	spin_lock_irq(&zone->lru_lock);
+	page->wired_count --;
+	if(!page->wired_count){
+		if(PageLRU(page) && TestClearPageWired(page)){
+			del_page_from_wired_list(zone,page);
+			add_page_to_active_list(zone,page);
+			SetPageActive(page);
+		}
+	}
+	spin_unlock_irq(&zone->lru_lock);
+}
+
 /*
  * Mark a page as having seen activity.
  *
@@ -119,11 +157,13 @@
  */
 void fastcall mark_page_accessed(struct page *page)
 {
-	if (!PageActive(page) && PageReferenced(page) && PageLRU(page)) {
-		activate_page(page);
-		ClearPageReferenced(page);
-	} else if (!PageReferenced(page)) {
-		SetPageReferenced(page);
+	if(!PageWired(page)) {
+		if (!PageActive(page) && PageReferenced(page) && PageLRU(page)) {
+			activate_page(page);
+			ClearPageReferenced(page);
+		} else if (!PageReferenced(page)) {
+			SetPageReferenced(page);
+		}
 	}
 }

@@ -178,13 +218,15 @@
 	struct zone *zone = page_zone(page);

 	spin_lock_irqsave(&zone->lru_lock, flags);
-	if (TestClearPageLRU(page))
-		del_page_from_lru(zone, page);
-	if (page_count(page) != 0)
-		page = NULL;
+	if(!PageWired(page)) {
+		if (TestClearPageLRU(page))
+			del_page_from_lru(zone, page);
+		if (page_count(page) != 0)
+			page = NULL;
+		if (page)
+			free_hot_page(page);
+	}
 	spin_unlock_irqrestore(&zone->lru_lock, flags);
-	if (page)
-		free_hot_page(page);
 }

 EXPORT_SYMBOL(__page_cache_release);
@@ -214,7 +256,8 @@

 		if (!put_page_testzero(page))
 			continue;
-
+		if(PageWired(page))
+			continue;
 		pagezone = page_zone(page);
 		if (pagezone != zone) {
 			if (zone)
@@ -301,7 +344,12 @@
 		}
 		if (TestSetPageLRU(page))
 			BUG();
-		add_page_to_inactive_list(zone, page);
+		if(!page->wired_count)
+			add_page_to_inactive_list(zone, page);
+		else {
+			SetPageWired(page);
+			add_page_to_wired_list(zone,page);
+		}
 	}
 	if (zone)
 		spin_unlock_irq(&zone->lru_lock);
@@ -330,7 +378,12 @@
 			BUG();
 		if (TestSetPageActive(page))
 			BUG();
-		add_page_to_active_list(zone, page);
+		if(!page->wired_count)
+			add_page_to_active_list(zone, page);
+		else{
+			SetPageWired(page);
+			add_page_to_wired_list(zone,page);
+		}
 	}
 	if (zone)
 		spin_unlock_irq(&zone->lru_lock);
diff -urN  linux-2.6.15.orig/mm/page_alloc.c linux-2.6.15/mm/page_alloc.c
--- linux-2.6.15.orig/mm/page_alloc.c	2006-01-02 22:21:10.000000000 -0500
+++ linux-2.6.15/mm/page_alloc.c	2006-03-06 06:30:08.000000000 -0500
@@ -348,7 +348,8 @@
 			1 << PG_slab	|
 			1 << PG_swapcache |
 			1 << PG_writeback |
-			1 << PG_reserved )))
+			1 << PG_reserved |
+			1 << PG_wired )))
 		bad_page(function, page);
 	if (PageDirty(page))
 		__ClearPageDirty(page);
@@ -481,6 +482,7 @@
 			1 << PG_private	|
 			1 << PG_locked	|
 			1 << PG_active	|
+			1 << PG_wired   |
 			1 << PG_dirty	|
 			1 << PG_reclaim	|
 			1 << PG_slab    |

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH][8/8] mm: lru interface change

[Nick Piggin]

Stone Wang wrote:
> Add Wired list to LRU.
> Add PG_Wired bit to page.
> 

I may have missed something very trivial, but... why are they on a
list at all if they don't get scanned?


> diff -urN  linux-2.6.15.orig/mm/swap.c linux-2.6.15/mm/swap.c
> --- linux-2.6.15.orig/mm/swap.c	2006-01-02 22:21:10.000000000 -0500
> +++ linux-2.6.15/mm/swap.c	2006-03-07 11:45:37.000000000 -0500
> @@ -110,6 +110,44 @@
>  	spin_unlock_irq(&zone->lru_lock);
>  }
> 
> +/* Wire the page; if the page is in LRU,
> + * try move it to Wired list.
> + */
> +void fastcall wire_page(struct page *page)

Ahh, here's the elusive beast.

> +{
> +	struct zone *zone = page_zone(page);
> +
> +	spin_lock_irq(&zone->lru_lock);

This is a fairly heavy lock for something which is going into page fault
fastpaths and such. You might be better off making wired_count atomic and
not using a lock in the common case if possible (even if it is only for
mlocked pages, I'm sure someone will complain).

> +	page->wired_count ++;

Oh dear, I missed this change you made to struct page, tucked away in 5/8.
This alone pretty much makes it a showstopper, I'm afraid. You'll have to
work out some other way to do it so as not to penalise 99.999% of machines
which don't need this.

(Oh, and making the field a short usually won't help either, because of
alignment constraints).

> +	if(!PageWired(page)){
> +		if(PageLRU(page)){
> +			del_page_from_lru(zone, page);
> +			add_page_to_wired_list(zone,page);
> +			SetPageWired(page);
> +		}
> +	}
> +	spin_unlock_irq(&zone->lru_lock);
> +}
> +
> +/* Unwire the page.
> + * If it isnt wired by any process, try move it to active list.
> + */
> +void fastcall unwire_page(struct page *page)
> +{
> +	struct zone *zone = page_zone(page);
> +
> +	spin_lock_irq(&zone->lru_lock);
> +	page->wired_count --;
> +	if(!page->wired_count){
> +		if(PageLRU(page) && TestClearPageWired(page)){
> +			del_page_from_wired_list(zone,page);
> +			add_page_to_active_list(zone,page);
> +			SetPageActive(page);
> +		}
> +	}
> +	spin_unlock_irq(&zone->lru_lock);
> +}
> +
>  /*
>   * Mark a page as having seen activity.
>   *
> @@ -119,11 +157,13 @@
>   */
>  void fastcall mark_page_accessed(struct page *page)
>  {
> -	if (!PageActive(page) && PageReferenced(page) && PageLRU(page)) {
> -		activate_page(page);
> -		ClearPageReferenced(page);
> -	} else if (!PageReferenced(page)) {
> -		SetPageReferenced(page);
> +	if(!PageWired(page)) {
> +		if (!PageActive(page) && PageReferenced(page) && PageLRU(page)) {
> +			activate_page(page);
> +			ClearPageReferenced(page);
> +		} else if (!PageReferenced(page)) {
> +			SetPageReferenced(page);
> +		}
>  	}
>  }
> 

So, umm... what happens when the page gets wired before activate_page()?

> @@ -178,13 +218,15 @@
>  	struct zone *zone = page_zone(page);
> 
>  	spin_lock_irqsave(&zone->lru_lock, flags);
> -	if (TestClearPageLRU(page))
> -		del_page_from_lru(zone, page);
> -	if (page_count(page) != 0)
> -		page = NULL;
> +	if(!PageWired(page)) {
> +		if (TestClearPageLRU(page))
> +			del_page_from_lru(zone, page);
> +		if (page_count(page) != 0)
> +			page = NULL;
> +		if (page)
> +			free_hot_page(page);
> +	}
>  	spin_unlock_irqrestore(&zone->lru_lock, flags);
> -	if (page)
> -		free_hot_page(page);
>  }
> 

Hmm... how do PageWired pages get freed, then?

>  EXPORT_SYMBOL(__page_cache_release);
> @@ -214,7 +256,8 @@
> 
>  		if (!put_page_testzero(page))
>  			continue;
> -
> +		if(PageWired(page))
> +			continue;
>  		pagezone = page_zone(page);
>  		if (pagezone != zone) {
>  			if (zone)

Ditto.

-- 
SUSE Labs, Novell Inc.


Send instant messages to your online friends http://au.messenger.yahoo.com 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH][8/8] mm: lru interface change

[Arjan van de Ven]

> > +	page->wired_count ++;
> 
> Oh dear, I missed this change you made to struct page, tucked away in 5/8.
> This alone pretty much makes it a showstopper, I'm afraid. You'll have to
> work out some other way to do it so as not to penalise 99.999% of machines
> which don't need this.
> 
> (Oh, and making the field a short usually won't help either, because of
> alignment constraints).

it's not that hard even. All you need to do is make the vm be lazy about
it; if it encounters a pinned page during scanning, move it THEN to the
pinned list. If it then gets pinned more no issue. The first unpin then
moves it back to the normal list (yes it's still pinned), but the first
time the VM sees it it goes right back to the pinned list.
That way there's no need to keep a "pin depth" at all...



--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH][8/8] mm: lru interface change

[Stone Wang]

> I may have missed something very trivial, but... why are they on a
> list at all if they don't get scanned

Get the locked pages on a list is necessary for page management,
scatter the locked pages around isnt a good idea.

Also, we could add some kinds of scan to the locked pages,
if we find that it's necessary.


Shaoping Wang

2006/3/21, Nick Piggin <nickpiggin@yahoo.com.au>:
> Stone Wang wrote:
> > Add Wired list to LRU.
> > Add PG_Wired bit to page.
> >
>
> I may have missed something very trivial, but... why are they on a
> list at all if they don't get scanned?
>
>
> > diff -urN  linux-2.6.15.orig/mm/swap.c linux-2.6.15/mm/swap.c
> > --- linux-2.6.15.orig/mm/swap.c       2006-01-02 22:21:10.000000000 -0500
> > +++ linux-2.6.15/mm/swap.c    2006-03-07 11:45:37.000000000 -0500
> > @@ -110,6 +110,44 @@
> >       spin_unlock_irq(&zone->lru_lock);
> >  }
> >
> > +/* Wire the page; if the page is in LRU,
> > + * try move it to Wired list.
> > + */
> > +void fastcall wire_page(struct page *page)
>
> Ahh, here's the elusive beast.
>
> > +{
> > +     struct zone *zone = page_zone(page);
> > +
> > +     spin_lock_irq(&zone->lru_lock);
>
> This is a fairly heavy lock for something which is going into page fault
> fastpaths and such. You might be better off making wired_count atomic and
> not using a lock in the common case if possible (even if it is only for
> mlocked pages, I'm sure someone will complain).
>
> > +     page->wired_count ++;
>
> Oh dear, I missed this change you made to struct page, tucked away in 5/8.
> This alone pretty much makes it a showstopper, I'm afraid. You'll have to
> work out some other way to do it so as not to penalise 99.999% of machines
> which don't need this.
>
> (Oh, and making the field a short usually won't help either, because of
> alignment constraints).
>
> > +     if(!PageWired(page)){
> > +             if(PageLRU(page)){
> > +                     del_page_from_lru(zone, page);
> > +                     add_page_to_wired_list(zone,page);
> > +                     SetPageWired(page);
> > +             }
> > +     }
> > +     spin_unlock_irq(&zone->lru_lock);
> > +}
> > +
> > +/* Unwire the page.
> > + * If it isnt wired by any process, try move it to active list.
> > + */
> > +void fastcall unwire_page(struct page *page)
> > +{
> > +     struct zone *zone = page_zone(page);
> > +
> > +     spin_lock_irq(&zone->lru_lock);
> > +     page->wired_count --;
> > +     if(!page->wired_count){
> > +             if(PageLRU(page) && TestClearPageWired(page)){
> > +                     del_page_from_wired_list(zone,page);
> > +                     add_page_to_active_list(zone,page);
> > +                     SetPageActive(page);
> > +             }
> > +     }
> > +     spin_unlock_irq(&zone->lru_lock);
> > +}
> > +
> >  /*
> >   * Mark a page as having seen activity.
> >   *
> > @@ -119,11 +157,13 @@
> >   */
> >  void fastcall mark_page_accessed(struct page *page)
> >  {
> > -     if (!PageActive(page) && PageReferenced(page) && PageLRU(page)) {
> > -             activate_page(page);
> > -             ClearPageReferenced(page);
> > -     } else if (!PageReferenced(page)) {
> > -             SetPageReferenced(page);
> > +     if(!PageWired(page)) {
> > +             if (!PageActive(page) && PageReferenced(page) && PageLRU(page)) {
> > +                     activate_page(page);
> > +                     ClearPageReferenced(page);
> > +             } else if (!PageReferenced(page)) {
> > +                     SetPageReferenced(page);
> > +             }
> >       }
> >  }
> >
>
> So, umm... what happens when the page gets wired before activate_page()?
>
> > @@ -178,13 +218,15 @@
> >       struct zone *zone = page_zone(page);
> >
> >       spin_lock_irqsave(&zone->lru_lock, flags);
> > -     if (TestClearPageLRU(page))
> > -             del_page_from_lru(zone, page);
> > -     if (page_count(page) != 0)
> > -             page = NULL;
> > +     if(!PageWired(page)) {
> > +             if (TestClearPageLRU(page))
> > +                     del_page_from_lru(zone, page);
> > +             if (page_count(page) != 0)
> > +                     page = NULL;
> > +             if (page)
> > +                     free_hot_page(page);
> > +     }
> >       spin_unlock_irqrestore(&zone->lru_lock, flags);
> > -     if (page)
> > -             free_hot_page(page);
> >  }
> >
>
> Hmm... how do PageWired pages get freed, then?
>
> >  EXPORT_SYMBOL(__page_cache_release);
> > @@ -214,7 +256,8 @@
> >
> >               if (!put_page_testzero(page))
> >                       continue;
> > -
> > +             if(PageWired(page))
> > +                     continue;
> >               pagezone = page_zone(page);
> >               if (pagezone != zone) {
> >                       if (zone)
>
> Ditto.
>
> --
> SUSE Labs, Novell Inc.
>
>
> Send instant messages to your online friends http://au.messenger.yahoo.com
>
>

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH][8/8] mm: lru interface change

[Nick Piggin]

Stone Wang wrote:
>>I may have missed something very trivial, but... why are they on a
>>list at all if they don't get scanned
> 
> 
> Get the locked pages on a list is necessary for page management,
> scatter the locked pages around isnt a good idea.
> 

This doesn't make sense. Whether or not they're on a list does not
change the fact that they may be "scattered".

> Also, we could add some kinds of scan to the locked pages,
> if we find that it's necessary.
> 

This is a valid reason. But at present you don't scan them, do you?
So you should add them to a list in patch 9 where you add the
machanism to scan them as well, right?

-- 
SUSE Labs, Novell Inc.
Send instant messages to your online friends http://au.messenger.yahoo.com 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>