[RFC][PATCH 2/3] Move RSS accounting to page_xxxx_rmap() functions

[Balbir Singh <balbir@in.ibm.com>]

The accounting of RSS is moved from several places to the rmap functions
page_add_anon_rmap(), page_add_new_anon_rmap(), page_add_file_rmap()
and page_remove_rmap().



Signed-off-by: Balbir Singh <balbir@in.ibm.com>
---

 fs/exec.c            |    1 -
 include/linux/rmap.h |    8 ++++++--
 mm/filemap_xip.c     |    1 -
 mm/fremap.c          |   10 ++++------
 mm/memory.c          |   50 ++++++++++----------------------------------------
 mm/migrate.c         |    2 +-
 mm/rmap.c            |   17 ++++++++++-------
 mm/swapfile.c        |    1 -
 8 files changed, 31 insertions(+), 59 deletions(-)

diff -puN include/linux/rmap.h~move-accounting-to-rmap include/linux/rmap.h
--- linux-2.6.20-rc2/include/linux/rmap.h~move-accounting-to-rmap	2006-12-29 14:48:28.000000000 +0530
+++ linux-2.6.20-rc2-balbir/include/linux/rmap.h	2006-12-29 14:48:28.000000000 +0530
@@ -89,7 +89,7 @@ void __anon_vma_link(struct vm_area_stru
  */
 void page_add_anon_rmap(struct page *, struct vm_area_struct *, unsigned long);
 void page_add_new_anon_rmap(struct page *, struct vm_area_struct *, unsigned long);
-void page_add_file_rmap(struct page *);
+void page_add_file_rmap(struct page *, struct mm_struct *);
 void page_remove_rmap(struct page *, struct vm_area_struct *);
 
 /**
@@ -99,10 +99,14 @@ void page_remove_rmap(struct page *, str
  * For copy_page_range only: minimal extract from page_add_rmap,
  * avoiding unnecessary tests (already checked) so it's quicker.
  */
-static inline void page_dup_rmap(struct page *page)
+static inline void page_dup_rmap(struct page *page, struct mm_struct *mm)
 {
 	page_map_lock(page);
 	page_mapcount_inc(page);
+	if (PageAnon(page))
+		inc_mm_counter(mm, anon_rss);
+	else
+		inc_mm_counter(mm, file_rss);
 	page_map_unlock(page);
 }
 
diff -puN mm/rmap.c~move-accounting-to-rmap mm/rmap.c
--- linux-2.6.20-rc2/mm/rmap.c~move-accounting-to-rmap	2006-12-29 14:48:28.000000000 +0530
+++ linux-2.6.20-rc2-balbir/mm/rmap.c	2006-12-29 14:48:28.000000000 +0530
@@ -544,6 +544,7 @@ void page_add_anon_rmap(struct page *pag
 	page_map_lock(page);
 	if (page_mapcount_inc_and_test(page))
 		__page_set_anon_rmap(page, vma, address);
+	inc_mm_counter(vma->vm_mm, anon_rss);
 	page_map_unlock(page);
 }
 
@@ -562,6 +563,7 @@ void page_add_new_anon_rmap(struct page 
 	page_map_lock(page);
 	page_mapcount_set(page, 0); /* elevate count by 1 (starts at -1) */
 	__page_set_anon_rmap(page, vma, address);
+	inc_mm_counter(vma->vm_mm, anon_rss);
 	page_map_unlock(page);
 }
 
@@ -571,11 +573,12 @@ void page_add_new_anon_rmap(struct page 
  *
  * The caller needs to hold the pte lock.
  */
-void page_add_file_rmap(struct page *page)
+void page_add_file_rmap(struct page *page, struct mm_struct *mm)
 {
 	page_map_lock(page);
 	if (page_mapcount_inc_and_test(page))
 		__inc_zone_page_state(page, NR_FILE_MAPPED);
+	inc_mm_counter(mm, file_rss);
 	page_map_unlock(page);
 }
 
@@ -587,6 +590,7 @@ void page_add_file_rmap(struct page *pag
  */
 void page_remove_rmap(struct page *page, struct vm_area_struct *vma)
 {
+	int anon = PageAnon(page);
 	page_map_lock(page);
 	if (page_mapcount_add_negative(-1, page)) {
 		if (unlikely(page_mapcount(page) < 0)) {
@@ -615,8 +619,12 @@ void page_remove_rmap(struct page *page,
 		if (page_test_and_clear_dirty(page))
 			set_page_dirty(page);
 		__dec_zone_page_state(page,
-				PageAnon(page) ? NR_ANON_PAGES : NR_FILE_MAPPED);
+				anon ? NR_ANON_PAGES : NR_FILE_MAPPED);
 	}
+	if (anon)
+		dec_mm_counter(vma->vm_mm, anon_rss);
+	else
+		dec_mm_counter(vma->vm_mm, file_rss);
 	page_map_unlock(page);
 }
 
@@ -679,7 +687,6 @@ static int try_to_unmap_one(struct page 
 					list_add(&mm->mmlist, &init_mm.mmlist);
 				spin_unlock(&mmlist_lock);
 			}
-			dec_mm_counter(mm, anon_rss);
 #ifdef CONFIG_MIGRATION
 		} else {
 			/*
@@ -700,10 +707,7 @@ static int try_to_unmap_one(struct page 
 		swp_entry_t entry;
 		entry = make_migration_entry(page, pte_write(pteval));
 		set_pte_at(mm, address, pte, swp_entry_to_pte(entry));
-	} else
 #endif
-		dec_mm_counter(mm, file_rss);
-
 
 	page_remove_rmap(page, vma);
 	page_cache_release(page);
@@ -797,7 +801,6 @@ static void try_to_unmap_cluster(unsigne
 
 		page_remove_rmap(page, vma);
 		page_cache_release(page);
-		dec_mm_counter(mm, file_rss);
 		(*mapcount)--;
 	}
 	pte_unmap_unlock(pte - 1, ptl);
diff -puN fs/exec.c~move-accounting-to-rmap fs/exec.c
--- linux-2.6.20-rc2/fs/exec.c~move-accounting-to-rmap	2006-12-29 14:48:28.000000000 +0530
+++ linux-2.6.20-rc2-balbir/fs/exec.c	2006-12-29 14:48:28.000000000 +0530
@@ -321,7 +321,6 @@ void install_arg_page(struct vm_area_str
 		pte_unmap_unlock(pte, ptl);
 		goto out;
 	}
-	inc_mm_counter(mm, anon_rss);
 	lru_cache_add_active(page);
 	set_pte_at(mm, address, pte, pte_mkdirty(pte_mkwrite(mk_pte(
 					page, vma->vm_page_prot))));
diff -puN mm/mremap.c~move-accounting-to-rmap mm/mremap.c
diff -puN mm/memory.c~move-accounting-to-rmap mm/memory.c
--- linux-2.6.20-rc2/mm/memory.c~move-accounting-to-rmap	2006-12-29 14:48:28.000000000 +0530
+++ linux-2.6.20-rc2-balbir/mm/memory.c	2006-12-29 14:48:28.000000000 +0530
@@ -335,14 +335,6 @@ int __pte_alloc_kernel(pmd_t *pmd, unsig
 	return 0;
 }
 
-static inline void add_mm_rss(struct mm_struct *mm, int file_rss, int anon_rss)
-{
-	if (file_rss)
-		add_mm_counter(mm, file_rss, file_rss);
-	if (anon_rss)
-		add_mm_counter(mm, anon_rss, anon_rss);
-}
-
 /*
  * This function is called to print an error when a bad pte
  * is found. For example, we might have a PFN-mapped pte in
@@ -427,7 +419,7 @@ struct page *vm_normal_page(struct vm_ar
 static inline void
 copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 		pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
-		unsigned long addr, int *rss)
+		unsigned long addr)
 {
 	unsigned long vm_flags = vma->vm_flags;
 	pte_t pte = *src_pte;
@@ -481,8 +473,7 @@ copy_one_pte(struct mm_struct *dst_mm, s
 	page = vm_normal_page(vma, addr, pte);
 	if (page) {
 		get_page(page);
-		page_dup_rmap(page);
-		rss[!!PageAnon(page)]++;
+		page_dup_rmap(page, dst_mm);
 	}
 
 out_set_pte:
@@ -496,10 +487,8 @@ static int copy_pte_range(struct mm_stru
 	pte_t *src_pte, *dst_pte;
 	spinlock_t *src_ptl, *dst_ptl;
 	int progress = 0;
-	int rss[2];
 
 again:
-	rss[1] = rss[0] = 0;
 	dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
 	if (!dst_pte)
 		return -ENOMEM;
@@ -524,14 +513,13 @@ again:
 			progress++;
 			continue;
 		}
-		copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, vma, addr, rss);
+		copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, vma, addr);
 		progress += 8;
 	} while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
 
 	arch_leave_lazy_mmu_mode();
 	spin_unlock(src_ptl);
 	pte_unmap_nested(src_pte - 1);
-	add_mm_rss(dst_mm, rss[0], rss[1]);
 	pte_unmap_unlock(dst_pte - 1, dst_ptl);
 	cond_resched();
 	if (addr != end)
@@ -626,8 +614,6 @@ static unsigned long zap_pte_range(struc
 	struct mm_struct *mm = tlb->mm;
 	pte_t *pte;
 	spinlock_t *ptl;
-	int file_rss = 0;
-	int anon_rss = 0;
 
 	pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
 	arch_enter_lazy_mmu_mode();
@@ -672,14 +658,11 @@ static unsigned long zap_pte_range(struc
 						addr) != page->index)
 				set_pte_at(mm, addr, pte,
 					   pgoff_to_pte(page->index));
-			if (PageAnon(page))
-				anon_rss--;
-			else {
+			if (!PageAnon(page)) {
 				if (pte_dirty(ptent))
 					set_page_dirty(page);
 				if (pte_young(ptent))
 					mark_page_accessed(page);
-				file_rss--;
 			}
 			page_remove_rmap(page, vma);
 			tlb_remove_page(tlb, page);
@@ -696,7 +679,6 @@ static unsigned long zap_pte_range(struc
 		pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
 	} while (pte++, addr += PAGE_SIZE, (addr != end && *zap_work > 0));
 
-	add_mm_rss(mm, file_rss, anon_rss);
 	arch_leave_lazy_mmu_mode();
 	pte_unmap_unlock(pte - 1, ptl);
 
@@ -1126,8 +1108,7 @@ static int zeromap_pte_range(struct mm_s
 			break;
 		}
 		page_cache_get(page);
-		page_add_file_rmap(page);
-		inc_mm_counter(mm, file_rss);
+		page_add_file_rmap(page, mm);
 		set_pte_at(mm, addr, pte, zero_pte);
 	} while (pte++, addr += PAGE_SIZE, addr != end);
 	arch_leave_lazy_mmu_mode();
@@ -1233,8 +1214,7 @@ static int insert_page(struct mm_struct 
 
 	/* Ok, finally just insert the thing.. */
 	get_page(page);
-	inc_mm_counter(mm, file_rss);
-	page_add_file_rmap(page);
+	page_add_file_rmap(page, mm);
 	set_pte_at(mm, addr, pte, mk_pte(page, prot));
 
 	retval = 0;
@@ -1585,14 +1565,9 @@ gotten:
 	 */
 	page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
 	if (likely(pte_same(*page_table, orig_pte))) {
-		if (old_page) {
+		if (old_page)
 			page_remove_rmap(old_page, vma);
-			if (!PageAnon(old_page)) {
-				dec_mm_counter(mm, file_rss);
-				inc_mm_counter(mm, anon_rss);
-			}
-		} else
-			inc_mm_counter(mm, anon_rss);
+
 		flush_cache_page(vma, address, pte_pfn(orig_pte));
 		entry = mk_pte(new_page, vma->vm_page_prot);
 		entry = maybe_mkwrite(pte_mkdirty(entry), vma);
@@ -2038,7 +2013,6 @@ static int do_swap_page(struct mm_struct
 
 	/* The page isn't present yet, go ahead with the fault. */
 
-	inc_mm_counter(mm, anon_rss);
 	pte = mk_pte(page, vma->vm_page_prot);
 	if (write_access && can_share_swap_page(page)) {
 		pte = maybe_mkwrite(pte_mkdirty(pte), vma);
@@ -2104,7 +2078,6 @@ static int do_anonymous_page(struct mm_s
 		page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
 		if (!pte_none(*page_table))
 			goto release;
-		inc_mm_counter(mm, anon_rss);
 		lru_cache_add_active(page);
 		page_add_new_anon_rmap(page, vma, address);
 	} else {
@@ -2117,8 +2090,7 @@ static int do_anonymous_page(struct mm_s
 		spin_lock(ptl);
 		if (!pte_none(*page_table))
 			goto release;
-		inc_mm_counter(mm, file_rss);
-		page_add_file_rmap(page);
+		page_add_file_rmap(page, mm);
 	}
 
 	set_pte_at(mm, address, page_table, entry);
@@ -2251,12 +2223,10 @@ retry:
 			entry = maybe_mkwrite(pte_mkdirty(entry), vma);
 		set_pte_at(mm, address, page_table, entry);
 		if (anon) {
-			inc_mm_counter(mm, anon_rss);
 			lru_cache_add_active(new_page);
 			page_add_new_anon_rmap(new_page, vma, address);
 		} else {
-			inc_mm_counter(mm, file_rss);
-			page_add_file_rmap(new_page);
+			page_add_file_rmap(new_page, mm);
 			if (write_access) {
 				dirty_page = new_page;
 				get_page(dirty_page);
diff -puN mm/swapfile.c~move-accounting-to-rmap mm/swapfile.c
--- linux-2.6.20-rc2/mm/swapfile.c~move-accounting-to-rmap	2006-12-29 14:48:28.000000000 +0530
+++ linux-2.6.20-rc2-balbir/mm/swapfile.c	2006-12-29 14:48:28.000000000 +0530
@@ -503,7 +503,6 @@ unsigned int count_swap_pages(int type, 
 static void unuse_pte(struct vm_area_struct *vma, pte_t *pte,
 		unsigned long addr, swp_entry_t entry, struct page *page)
 {
-	inc_mm_counter(vma->vm_mm, anon_rss);
 	get_page(page);
 	set_pte_at(vma->vm_mm, addr, pte,
 		   pte_mkold(mk_pte(page, vma->vm_page_prot)));
diff -puN mm/filemap_xip.c~move-accounting-to-rmap mm/filemap_xip.c
--- linux-2.6.20-rc2/mm/filemap_xip.c~move-accounting-to-rmap	2006-12-29 14:48:28.000000000 +0530
+++ linux-2.6.20-rc2-balbir/mm/filemap_xip.c	2006-12-29 14:48:28.000000000 +0530
@@ -190,7 +190,6 @@ __xip_unmap (struct address_space * mapp
 			flush_cache_page(vma, address, pte_pfn(*pte));
 			pteval = ptep_clear_flush(vma, address, pte);
 			page_remove_rmap(page, vma);
-			dec_mm_counter(mm, file_rss);
 			BUG_ON(pte_dirty(pteval));
 			pte_unmap_unlock(pte, ptl);
 			page_cache_release(page);
diff -puN mm/fremap.c~move-accounting-to-rmap mm/fremap.c
--- linux-2.6.20-rc2/mm/fremap.c~move-accounting-to-rmap	2006-12-29 14:48:28.000000000 +0530
+++ linux-2.6.20-rc2-balbir/mm/fremap.c	2006-12-29 14:48:28.000000000 +0530
@@ -75,13 +75,13 @@ int install_page(struct mm_struct *mm, s
 	if (page_mapcount(page) > INT_MAX/2)
 		goto unlock;
 
-	if (pte_none(*pte) || !zap_pte(mm, vma, addr, pte))
-		inc_mm_counter(mm, file_rss);
+	if (!pte_none(*pte))
+		zap_pte(mm, vma, addr, pte);
 
 	flush_icache_page(vma, page);
 	pte_val = mk_pte(page, prot);
 	set_pte_at(mm, addr, pte, pte_val);
-	page_add_file_rmap(page);
+	page_add_file_rmap(page, mm);
 	update_mmu_cache(vma, addr, pte_val);
 	lazy_mmu_prot_update(pte_val);
 	err = 0;
@@ -107,10 +107,8 @@ int install_file_pte(struct mm_struct *m
 	if (!pte)
 		goto out;
 
-	if (!pte_none(*pte) && zap_pte(mm, vma, addr, pte)) {
+	if (!pte_none(*pte) && zap_pte(mm, vma, addr, pte))
 		update_hiwater_rss(mm);
-		dec_mm_counter(mm, file_rss);
-	}
 
 	set_pte_at(mm, addr, pte, pgoff_to_pte(pgoff));
 	/*
diff -puN mm/migrate.c~move-accounting-to-rmap mm/migrate.c
--- linux-2.6.20-rc2/mm/migrate.c~move-accounting-to-rmap	2006-12-29 14:48:28.000000000 +0530
+++ linux-2.6.20-rc2-balbir/mm/migrate.c	2006-12-29 14:48:28.000000000 +0530
@@ -177,7 +177,7 @@ static void remove_migration_pte(struct 
 	if (PageAnon(new))
 		page_add_anon_rmap(new, vma, addr);
 	else
-		page_add_file_rmap(new);
+		page_add_file_rmap(new, mm);
 
 	/* No need to invalidate - it was non-present before */
 	update_mmu_cache(vma, addr, pte);
_

-- 

	Balbir Singh,
	Linux Technology Center,
	IBM Software Labs

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>