[RFC MM 3/4] add mm version number

[KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>]

Add logical timestamp to mm_struct, which is incremented always
mmap_sem(write) is got and released. By this, it works like seqlock's
counter and indicates mm_struct is modified or not.

And this adds vma_cache to each thread. Each thread remember the last
faulted vma and grab reference count. Correctness of cache is checked by
mm->generation timestamp. (mm struct's vma cache is not very good
if mm is shared, I think)

Signed-off-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
---
 arch/x86/mm/fault.c       |   18 ++++++++++++++++--
 fs/exec.c                 |    4 ++++
 include/linux/init_task.h |    1 +
 include/linux/mm_types.h  |   11 ++++++++++-
 include/linux/sched.h     |    4 ++++
 kernel/exit.c             |    3 +++
 kernel/fork.c             |    5 ++++-
 7 files changed, 42 insertions(+), 4 deletions(-)

Index: mmotm-2.6.32-Nov2/include/linux/mm_types.h
===================================================================
--- mmotm-2.6.32-Nov2.orig/include/linux/mm_types.h
+++ mmotm-2.6.32-Nov2/include/linux/mm_types.h
@@ -216,6 +216,7 @@ struct mm_struct {
 	atomic_t mm_users;			/* How many users with user space? */
 	atomic_t mm_count;			/* How many references to "struct mm_struct" (users count as 1) */
 	int map_count;				/* number of VMAs */
+	unsigned int generation;		/* logical timestamp of last modification */
 	struct rw_semaphore sem;
 	spinlock_t page_table_lock;		/* Protects page tables and some counters */
 
@@ -308,16 +309,21 @@ static inline int mm_reader_trylock(stru
 static inline void mm_writer_lock(struct mm_struct *mm)
 {
 	down_write(&mm->sem);
+	mm->generation++;
 }
 
 static inline void mm_writer_unlock(struct mm_struct *mm)
 {
+	mm->generation++;
 	up_write(&mm->sem);
 }
 
 static inline int mm_writer_trylock(struct mm_struct *mm)
 {
-	return down_write_trylock(&mm->sem);
+	int ret = down_write_trylock(&mm->sem);
+	if (!ret)
+		mm->generation++;
+	return ret;
 }
 
 static inline int mm_locked(struct mm_struct *mm)
@@ -327,17 +333,20 @@ static inline int mm_locked(struct mm_st
 
 static inline void mm_writer_to_reader_lock(struct mm_struct *mm)
 {
+	mm->generation++;
 	downgrade_write(&mm->sem);
 }
 
 static inline void mm_writer_lock_nested(struct mm_struct *mm, int x)
 {
 	down_write_nested(&mm->sem, x);
+	mm->generation++;
 }
 
 static inline void mm_lock_init(struct mm_struct *mm)
 {
 	init_rwsem(&mm->sem);
+	mm->generation = 0;
 }
 
 static inline void mm_lock_prefetch(struct mm_struct *mm)
Index: mmotm-2.6.32-Nov2/arch/x86/mm/fault.c
===================================================================
--- mmotm-2.6.32-Nov2.orig/arch/x86/mm/fault.c
+++ mmotm-2.6.32-Nov2/arch/x86/mm/fault.c
@@ -952,6 +952,7 @@ do_page_fault(struct pt_regs *regs, unsi
 	struct mm_struct *mm;
 	int write;
 	int fault;
+	int cachehit = 0;
 
 	tsk = current;
 	mm = tsk->mm;
@@ -1071,8 +1072,13 @@ do_page_fault(struct pt_regs *regs, unsi
 		 */
 		might_sleep();
 	}
-
-	vma = find_vma(mm, address);
+	if ((mm->generation == current->mm_generation) && current->vma_cache) {
+		vma = current->vma_cache;
+		if ((vma->vm_start <= address) && (address < vma->vm_end))
+			cachehit = 1;
+	}
+	if (!cachehit)
+		vma = find_vma(mm, address);
 	if (unlikely(!vma)) {
 		bad_area(regs, error_code, address);
 		return;
@@ -1133,6 +1139,14 @@ good_area:
 		perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0,
 				     regs, address);
 	}
+	/* cache information */
+	if (!cachehit) {
+		if (current->vma_cache)
+			vma_put(current->vma_cache);
+		current->vma_cache = vma;
+		current->mm_generation = mm->generation;
+		vma_get(vma);
+	}
 
 	check_v8086_mode(regs, address, tsk);
 
Index: mmotm-2.6.32-Nov2/include/linux/sched.h
===================================================================
--- mmotm-2.6.32-Nov2.orig/include/linux/sched.h
+++ mmotm-2.6.32-Nov2/include/linux/sched.h
@@ -1370,6 +1370,10 @@ struct task_struct {
 /* hung task detection */
 	unsigned long last_switch_count;
 #endif
+/* For relaxing per-thread page fault, information is cached.*/
+	struct vm_area_struct *vma_cache;
+	unsigned int mm_generation;
+
 /* CPU-specific state of this task */
 	struct thread_struct thread;
 /* filesystem information */
Index: mmotm-2.6.32-Nov2/kernel/fork.c
===================================================================
--- mmotm-2.6.32-Nov2.orig/kernel/fork.c
+++ mmotm-2.6.32-Nov2/kernel/fork.c
@@ -264,6 +264,9 @@ static struct task_struct *dup_task_stru
 #endif
 	tsk->splice_pipe = NULL;
 
+	tsk->vma_cache = NULL;
+	tsk->mm_generation = 0;
+
 	account_kernel_stack(ti, 1);
 
 	return tsk;
@@ -289,7 +292,7 @@ static int dup_mmap(struct mm_struct *mm
 	 * Not linked in yet - no deadlock potential:
 	 */
 	mm_writer_lock_nested(mm, SINGLE_DEPTH_NESTING);
-
+	mm->generation = 0;
 	mm->locked_vm = 0;
 	mm->mmap = NULL;
 	mm->mmap_cache = NULL;
Index: mmotm-2.6.32-Nov2/kernel/exit.c
===================================================================
--- mmotm-2.6.32-Nov2.orig/kernel/exit.c
+++ mmotm-2.6.32-Nov2/kernel/exit.c
@@ -645,6 +645,9 @@ static void exit_mm(struct task_struct *
 	struct mm_struct *mm = tsk->mm;
 	struct core_state *core_state;
 
+	if (tsk->vma_cache)
+		vma_put(tsk->vma_cache);
+
 	mm_release(tsk, mm);
 	if (!mm)
 		return;
Index: mmotm-2.6.32-Nov2/fs/exec.c
===================================================================
--- mmotm-2.6.32-Nov2.orig/fs/exec.c
+++ mmotm-2.6.32-Nov2/fs/exec.c
@@ -720,6 +720,10 @@ static int exec_mmap(struct mm_struct *m
 			return -EINTR;
 		}
 	}
+	if (tsk->vma_cache) {
+		vma_put(tsk->vma_cache);
+		tsk->vma_cache = NULL;
+	}
 	task_lock(tsk);
 	active_mm = tsk->active_mm;
 	tsk->mm = mm;
Index: mmotm-2.6.32-Nov2/include/linux/init_task.h
===================================================================
--- mmotm-2.6.32-Nov2.orig/include/linux/init_task.h
+++ mmotm-2.6.32-Nov2/include/linux/init_task.h
@@ -156,6 +156,7 @@ extern struct cred init_cred;
 		 __MUTEX_INITIALIZER(tsk.cred_guard_mutex),		\
 	.comm		= "swapper",					\
 	.thread		= INIT_THREAD,					\
+	.vma_cache	= NULL,						\
 	.fs		= &init_fs,					\
 	.files		= &init_files,					\
 	.signal		= &init_signals,				\

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>