[RFC MM] mmap_sem scaling: only scan cpus used by an mm

[Christoph Lameter <cl@linux-foundation.org>]

One way to reduce the cost of the writer lock is to track the cpus used
and loop over the processors in that bitmap.

---
 arch/x86/include/asm/mmu_context.h |    1 +
 include/linux/mm_types.h           |    3 ++-
 kernel/fork.c                      |    2 ++
 mm/init-mm.c                       |    1 +
 4 files changed, 6 insertions(+), 1 deletion(-)

Index: linux-2.6/arch/x86/include/asm/mmu_context.h
===================================================================
--- linux-2.6.orig/arch/x86/include/asm/mmu_context.h	2009-11-06 12:26:24.000000000 -0600
+++ linux-2.6/arch/x86/include/asm/mmu_context.h	2009-11-06 12:26:36.000000000 -0600
@@ -43,6 +43,7 @@ static inline void switch_mm(struct mm_s
 		percpu_write(cpu_tlbstate.active_mm, next);
 #endif
 		cpumask_set_cpu(cpu, mm_cpumask(next));
+		cpumask_set_cpu(cpu, &next->cpus_used);

 		/* Re-load page tables */
 		load_cr3(next->pgd);
Index: linux-2.6/include/linux/mm_types.h
===================================================================
--- linux-2.6.orig/include/linux/mm_types.h	2009-11-06 12:26:35.000000000 -0600
+++ linux-2.6/include/linux/mm_types.h	2009-11-06 12:26:36.000000000 -0600
@@ -241,6 +241,7 @@ struct mm_struct {
 	struct linux_binfmt *binfmt;

 	cpumask_t cpu_vm_mask;
+	cpumask_t cpus_used;

 	/* Architecture-specific MM context */
 	mm_context_t context;
@@ -291,7 +292,7 @@ static inline int mm_has_reader(struct m
 {
 	int cpu;

-	for_each_possible_cpu(cpu)
+	for_each_cpu(cpu, &mm->cpus_used)
 		if (per_cpu(mm->rss->readers, cpu))
 			return 1;

Index: linux-2.6/mm/init-mm.c
===================================================================
--- linux-2.6.orig/mm/init-mm.c	2009-11-06 12:26:35.000000000 -0600
+++ linux-2.6/mm/init-mm.c	2009-11-06 12:26:36.000000000 -0600
@@ -19,5 +19,6 @@ struct mm_struct init_mm = {
 	.page_table_lock =  __SPIN_LOCK_UNLOCKED(init_mm.page_table_lock),
 	.mmlist		= LIST_HEAD_INIT(init_mm.mmlist),
 	.cpu_vm_mask	= CPU_MASK_ALL,
+	.cpus_used	= CPU_MASK_ALL,
 	.rss		= &init_mm_counters,
 };
Index: linux-2.6/kernel/fork.c
===================================================================
--- linux-2.6.orig/kernel/fork.c	2009-11-06 12:26:35.000000000 -0600
+++ linux-2.6/kernel/fork.c	2009-11-06 12:26:40.000000000 -0600
@@ -297,6 +297,8 @@ static int dup_mmap(struct mm_struct *mm
 	mm->cached_hole_size = ~0UL;
 	mm->map_count = 0;
 	cpumask_clear(mm_cpumask(mm));
+	cpumask_clear(&mm->cpus_used);
+	cpumask_set_cpu(smp_processor_id(), &mm->cpus_used);
 	mm->mm_rb = RB_ROOT;
 	rb_link = &mm->mm_rb.rb_node;
 	rb_parent = NULL;

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>