From: Martin Schwidefsky <schwidefsky@de.ibm.com>
To: davem@redhat.com
Cc: akpm@osdl.org, willy@debian.org, linux-arch@vger.kernel.org,
linux-mm@kvack.org
Subject: Re: TLB flush optimization on s/390.
Date: Thu, 9 Oct 2003 14:38:45 +0200 [thread overview]
Message-ID: <20031009123844.GA464@mschwid3.boeblingen.de.ibm.com> (raw)
> I have no real objections once you deal with the naming
> suggestions made by Matthew.
Ok, renamed three of the new functions. Patch @EOM.
blue skies,
Martin.
diff -urN linux-bk/include/asm-generic/pgtable.h linux-ptep/include/asm-generic/pgtable.h
--- linux-bk/include/asm-generic/pgtable.h Mon Oct 6 18:44:11 2003
+++ linux-ptep/include/asm-generic/pgtable.h Mon Oct 6 18:39:34 2003
@@ -1,6 +1,23 @@
#ifndef _ASM_GENERIC_PGTABLE_H
#define _ASM_GENERIC_PGTABLE_H
+#ifndef __HAVE_ARCH_PTEP_ESTABLISH
+/*
+ * Establish a new mapping:
+ * - flush the old one
+ * - update the page tables
+ * - inform the TLB about the new one
+ *
+ * We hold the mm semaphore for reading and vma->vm_mm->page_table_lock
+ */
+#define ptep_establish(__vma, __address, __ptep, __entry) \
+do { \
+ set_pte(__ptep, __entry); \
+ flush_tlb_page(__vma, __address); \
+} while (0)
+#endif
+
+#ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
static inline int ptep_test_and_clear_young(pte_t *ptep)
{
pte_t pte = *ptep;
@@ -9,7 +26,19 @@
set_pte(ptep, pte_mkold(pte));
return 1;
}
+#endif
+
+#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
+#define ptep_clear_flush_young(__vma, __address, __ptep) \
+({ \
+ int __young = ptep_test_and_clear_young(__ptep); \
+ if (__young) \
+ flush_tlb_page(__vma, __address); \
+ __young; \
+})
+#endif
+#ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
static inline int ptep_test_and_clear_dirty(pte_t *ptep)
{
pte_t pte = *ptep;
@@ -18,26 +47,54 @@
set_pte(ptep, pte_mkclean(pte));
return 1;
}
+#endif
+#ifndef __HAVE_ARCH_PTEP_CLEAR_DIRTY_FLUSH
+#define ptep_clear_flush_dirty(__vma, __address, __ptep) \
+({ \
+ int __dirty = ptep_test_and_clear_dirty(__ptep); \
+ if (__dirty) \
+ flush_tlb_page(__vma, __address); \
+ __dirty; \
+})
+#endif
+
+#ifndef __HAVE_PTEP_GET_AND_CLEAR
static inline pte_t ptep_get_and_clear(pte_t *ptep)
{
pte_t pte = *ptep;
pte_clear(ptep);
return pte;
}
+#endif
+
+#ifndef __HAVE_PTEP_CLEAR_FLUSH
+#define ptep_clear_flush(__vma, __address, __ptep) \
+({ \
+ pte_t __pte = ptep_get_and_clear(__ptep); \
+ flush_tlb_page(__vma, __address); \
+ __pte; \
+})
+#endif
+#ifndef __HAVE_PTEP_SET_WRPROTECT
static inline void ptep_set_wrprotect(pte_t *ptep)
{
pte_t old_pte = *ptep;
set_pte(ptep, pte_wrprotect(old_pte));
}
+#endif
+#ifndef __HAVE_PTEP_MKDIRTY
static inline void ptep_mkdirty(pte_t *ptep)
{
pte_t old_pte = *ptep;
set_pte(ptep, pte_mkdirty(old_pte));
}
+#endif
+#ifndef __HAVE_PTE_SAME
#define pte_same(A,B) (pte_val(A) == pte_val(B))
+#endif
#endif /* _ASM_GENERIC_PGTABLE_H */
diff -urN linux-bk/include/asm-i386/pgtable.h linux-ptep/include/asm-i386/pgtable.h
--- linux-bk/include/asm-i386/pgtable.h Mon Oct 6 18:44:11 2003
+++ linux-ptep/include/asm-i386/pgtable.h Mon Oct 6 18:14:00 2003
@@ -341,4 +341,12 @@
#define io_remap_page_range remap_page_range
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
+#define __HAVE_PTEP_GET_AND_CLEAR
+#define __HAVE_PTEP_SET_WRPROTECT
+#define __HAVE_PTEP_MKDIRTY
+#define __HAVE_PTE_SAME
+#include <asm-generic/pgtable.h>
+
#endif /* _I386_PGTABLE_H */
diff -urN linux-bk/include/asm-ia64/pgtable.h linux-ptep/include/asm-ia64/pgtable.h
--- linux-bk/include/asm-ia64/pgtable.h Mon Oct 6 18:44:11 2003
+++ linux-ptep/include/asm-ia64/pgtable.h Mon Oct 6 18:16:15 2003
@@ -501,4 +501,12 @@
#define FIXADDR_USER_START GATE_ADDR
#define FIXADDR_USER_END (GATE_ADDR + 2*PAGE_SIZE)
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
+#define __HAVE_PTEP_GET_AND_CLEAR
+#define __HAVE_PTEP_SET_WRPROTECT
+#define __HAVE_PTEP_MKDIRTY
+#define __HAVE_PTE_SAME
+#include <asm-generic/pgtable.h>
+
#endif /* _ASM_IA64_PGTABLE_H */
diff -urN linux-bk/include/asm-parisc/pgtable.h linux-ptep/include/asm-parisc/pgtable.h
--- linux-bk/include/asm-parisc/pgtable.h Mon Oct 6 18:44:11 2003
+++ linux-ptep/include/asm-parisc/pgtable.h Mon Oct 6 18:16:34 2003
@@ -460,4 +460,12 @@
#define HAVE_ARCH_UNMAPPED_AREA
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
+#define __HAVE_PTEP_GET_AND_CLEAR
+#define __HAVE_PTEP_SET_WRPROTECT
+#define __HAVE_PTEP_MKDIRTY
+#define __HAVE_PTE_SAME
+#include <asm-generic/pgtable.h>
+
#endif /* _PARISC_PGTABLE_H */
diff -urN linux-bk/include/asm-ppc/pgtable.h linux-ptep/include/asm-ppc/pgtable.h
--- linux-bk/include/asm-ppc/pgtable.h Mon Oct 6 18:44:11 2003
+++ linux-ptep/include/asm-ppc/pgtable.h Mon Oct 6 18:16:57 2003
@@ -661,5 +661,14 @@
typedef pte_t *pte_addr_t;
#endif /* !__ASSEMBLY__ */
+
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
+#define __HAVE_PTEP_GET_AND_CLEAR
+#define __HAVE_PTEP_SET_WRPROTECT
+#define __HAVE_PTEP_MKDIRTY
+#define __HAVE_PTE_SAME
+#include <asm-generic/pgtable.h>
+
#endif /* _PPC_PGTABLE_H */
#endif /* __KERNEL__ */
diff -urN linux-bk/include/asm-ppc64/pgtable.h linux-ptep/include/asm-ppc64/pgtable.h
--- linux-bk/include/asm-ppc64/pgtable.h Mon Oct 6 18:44:11 2003
+++ linux-ptep/include/asm-ppc64/pgtable.h Mon Oct 6 18:17:12 2003
@@ -414,4 +414,13 @@
unsigned long hpteflags, int bolted, int large);
#endif /* __ASSEMBLY__ */
+
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
+#define __HAVE_PTEP_GET_AND_CLEAR
+#define __HAVE_PTEP_SET_WRPROTECT
+#define __HAVE_PTEP_MKDIRTY
+#define __HAVE_PTE_SAME
+#include <asm-generic/pgtable.h>
+
#endif /* _PPC64_PGTABLE_H */
diff -urN linux-bk/include/asm-s390/pgalloc.h linux-ptep/include/asm-s390/pgalloc.h
--- linux-bk/include/asm-s390/pgalloc.h Mon Oct 6 18:44:11 2003
+++ linux-ptep/include/asm-s390/pgalloc.h Mon Oct 6 18:20:34 2003
@@ -155,29 +155,4 @@
*/
#define set_pgdir(addr,entry) do { } while(0)
-static inline pte_t ptep_invalidate(struct vm_area_struct *vma,
- unsigned long address, pte_t *ptep)
-{
- pte_t pte = *ptep;
-#ifndef __s390x__
- if (!(pte_val(pte) & _PAGE_INVALID)) {
- /* S390 has 1mb segments, we are emulating 4MB segments */
- pte_t *pto = (pte_t *) (((unsigned long) ptep) & 0x7ffffc00);
- __asm__ __volatile__ ("ipte %0,%1" : : "a" (pto), "a" (address));
- }
-#else /* __s390x__ */
- if (!(pte_val(pte) & _PAGE_INVALID))
- __asm__ __volatile__ ("ipte %0,%1" : : "a" (ptep), "a" (address));
-#endif /* __s390x__ */
- pte_clear(ptep);
- return pte;
-}
-
-static inline void ptep_establish(struct vm_area_struct *vma,
- unsigned long address, pte_t *ptep, pte_t entry)
-{
- ptep_invalidate(vma, address, ptep);
- set_pte(ptep, entry);
-}
-
#endif /* _S390_PGALLOC_H */
diff -urN linux-bk/include/asm-s390/pgtable.h linux-ptep/include/asm-s390/pgtable.h
--- linux-bk/include/asm-s390/pgtable.h Mon Oct 6 18:44:11 2003
+++ linux-ptep/include/asm-s390/pgtable.h Mon Oct 6 18:18:51 2003
@@ -32,6 +32,8 @@
#include <asm/processor.h>
#include <linux/threads.h>
+struct vm_area_struct; /* forward declaration (include/linux/mm.h) */
+
extern pgd_t swapper_pg_dir[] __attribute__ ((aligned (4096)));
extern void paging_init(void);
@@ -520,6 +522,14 @@
return ccode & 2;
}
+static inline int
+ptep_clear_flush_young(struct vm_area_struct *vma,
+ unsigned long address, pte_t *ptep)
+{
+ /* No need to flush TLB; bits are in storage key */
+ return ptep_test_and_clear_young(ptep);
+}
+
static inline int ptep_test_and_clear_dirty(pte_t *ptep)
{
int skey;
@@ -536,6 +546,14 @@
return 1;
}
+static inline int
+ptep_clear_flush_dirty(struct vm_area_struct *vma,
+ unsigned long address, pte_t *ptep)
+{
+ /* No need to flush TLB; bits are in storage key */
+ return ptep_test_and_clear_young(ptep);
+}
+
static inline pte_t ptep_get_and_clear(pte_t *ptep)
{
pte_t pte = *ptep;
@@ -543,6 +561,25 @@
return pte;
}
+static inline pte_t
+ptep_clear_flush(struct vm_area_struct *vma,
+ unsigned long address, pte_t *ptep)
+{
+ pte_t pte = *ptep;
+#ifndef __s390x__
+ if (!(pte_val(pte) & _PAGE_INVALID)) {
+ /* S390 has 1mb segments, we are emulating 4MB segments */
+ pte_t *pto = (pte_t *) (((unsigned long) ptep) & 0x7ffffc00);
+ __asm__ __volatile__ ("ipte %0,%1" : : "a" (pto), "a" (address));
+ }
+#else /* __s390x__ */
+ if (!(pte_val(pte) & _PAGE_INVALID))
+ __asm__ __volatile__ ("ipte %0,%1" : : "a" (ptep), "a" (address));
+#endif /* __s390x__ */
+ pte_clear(ptep);
+ return pte;
+}
+
static inline void ptep_set_wrprotect(pte_t *ptep)
{
pte_t old_pte = *ptep;
@@ -554,6 +591,14 @@
pte_mkdirty(*ptep);
}
+static inline void
+ptep_establish(struct vm_area_struct *vma,
+ unsigned long address, pte_t *ptep, pte_t entry)
+{
+ ptep_clear_flush(vma, address, ptep);
+ set_pte(ptep, entry);
+}
+
/*
* Conversion functions: convert a page and protection to a page entry,
* and a page entry and page directory to the page they refer to.
@@ -723,6 +768,18 @@
#ifdef __s390x__
# define HAVE_ARCH_UNMAPPED_AREA
#endif /* __s390x__ */
+
+#define __HAVE_ARCH_PTEP_ESTABLISH
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
+#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
+#define __HAVE_ARCH_PTEP_CLEAR_DIRTY_FLUSH
+#define __HAVE_PTEP_GET_AND_CLEAR
+#define __HAVE_PTEP_CLEAR_FLUSH
+#define __HAVE_PTEP_SET_WRPROTECT
+#define __HAVE_PTEP_MKDIRTY
+#define __HAVE_PTE_SAME
+#include <asm-generic/pgtable.h>
#endif /* _S390_PAGE_H */
diff -urN linux-bk/include/asm-sh/pgtable.h linux-ptep/include/asm-sh/pgtable.h
--- linux-bk/include/asm-sh/pgtable.h Mon Oct 6 18:44:11 2003
+++ linux-ptep/include/asm-sh/pgtable.h Mon Oct 6 18:21:07 2003
@@ -280,5 +280,13 @@
extern unsigned int kobjsize(const void *objp);
#endif /* !CONFIG_MMU */
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
+#define __HAVE_PTEP_GET_AND_CLEAR
+#define __HAVE_PTEP_SET_WRPROTECT
+#define __HAVE_PTEP_MKDIRTY
+#define __HAVE_PTE_SAME
+#include <asm-generic/pgtable.h>
+
#endif /* __ASM_SH_PAGE_H */
diff -urN linux-bk/include/asm-x86_64/pgtable.h linux-ptep/include/asm-x86_64/pgtable.h
--- linux-bk/include/asm-x86_64/pgtable.h Mon Oct 6 18:44:11 2003
+++ linux-ptep/include/asm-x86_64/pgtable.h Mon Oct 6 18:19:06 2003
@@ -408,4 +408,12 @@
#define kc_offset_to_vaddr(o) \
(((o) & (1UL << (__VIRTUAL_MASK_SHIFT-1))) ? ((o) | (~__VIRTUAL_MASK)) : (o))
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
+#define __HAVE_PTEP_GET_AND_CLEAR
+#define __HAVE_PTEP_SET_WRPROTECT
+#define __HAVE_PTEP_MKDIRTY
+#define __HAVE_PTE_SAME
+#include <asm-generic/pgtable.h>
+
#endif /* _X86_64_PGTABLE_H */
diff -urN linux-bk/mm/memory.c linux-ptep/mm/memory.c
--- linux-bk/mm/memory.c Mon Oct 6 18:44:11 2003
+++ linux-ptep/mm/memory.c Mon Oct 6 18:41:31 2003
@@ -941,28 +941,17 @@
}
/*
- * Establish a new mapping:
- * - flush the old one
- * - update the page tables
- * - inform the TLB about the new one
- *
- * We hold the mm semaphore for reading and vma->vm_mm->page_table_lock
- */
-static inline void establish_pte(struct vm_area_struct * vma, unsigned long address, pte_t *page_table, pte_t entry)
-{
- set_pte(page_table, entry);
- flush_tlb_page(vma, address);
- update_mmu_cache(vma, address, entry);
-}
-
-/*
* We hold the mm semaphore for reading and vma->vm_mm->page_table_lock
*/
static inline void break_cow(struct vm_area_struct * vma, struct page * new_page, unsigned long address,
pte_t *page_table)
{
+ pte_t entry;
+
flush_cache_page(vma, address);
- establish_pte(vma, address, page_table, pte_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot))));
+ entry = pte_mkwrite(pte_mkdirty(mk_pte(new_page, vma->vm_page_prot)));
+ ptep_establish(vma, address, page_table, entry);
+ update_mmu_cache(vma, address, entry);
}
/*
@@ -991,6 +980,7 @@
struct page *old_page, *new_page;
unsigned long pfn = pte_pfn(pte);
struct pte_chain *pte_chain = NULL;
+ pte_t entry;
int ret;
if (unlikely(!pfn_valid(pfn))) {
@@ -1011,8 +1001,9 @@
unlock_page(old_page);
if (reuse) {
flush_cache_page(vma, address);
- establish_pte(vma, address, page_table,
- pte_mkyoung(pte_mkdirty(pte_mkwrite(pte))));
+ entry = pte_mkyoung(pte_mkdirty(pte_mkwrite(pte)));
+ ptep_establish(vma, address, page_table, entry);
+ update_mmu_cache(vma, address, entry);
pte_unmap(page_table);
ret = VM_FAULT_MINOR;
goto out;
@@ -1571,7 +1562,8 @@
entry = pte_mkdirty(entry);
}
entry = pte_mkyoung(entry);
- establish_pte(vma, address, pte, entry);
+ ptep_establish(vma, address, pte, entry);
+ update_mmu_cache(vma, address, entry);
pte_unmap(pte);
spin_unlock(&mm->page_table_lock);
return VM_FAULT_MINOR;
diff -urN linux-bk/mm/mremap.c linux-ptep/mm/mremap.c
--- linux-bk/mm/mremap.c Mon Oct 6 18:44:11 2003
+++ linux-ptep/mm/mremap.c Mon Oct 6 18:42:43 2003
@@ -79,9 +79,9 @@
return pte;
}
-static int
-copy_one_pte(struct mm_struct *mm, pte_t *src, pte_t *dst,
- struct pte_chain **pte_chainp)
+static inline int
+copy_one_pte(struct vm_area_struct *vma, unsigned long old_addr,
+ pte_t *src, pte_t *dst, struct pte_chain **pte_chainp)
{
int error = 0;
pte_t pte;
@@ -93,7 +93,7 @@
if (!pte_none(*src)) {
if (page)
page_remove_rmap(page, src);
- pte = ptep_get_and_clear(src);
+ pte = ptep_clear_flush(vma, old_addr, src);
if (!dst) {
/* No dest? We must put it back. */
dst = src;
@@ -135,11 +135,15 @@
dst = alloc_one_pte_map(mm, new_addr);
if (src == NULL)
src = get_one_pte_map_nested(mm, old_addr);
- error = copy_one_pte(mm, src, dst, &pte_chain);
+ error = copy_one_pte(vma, old_addr, src, dst, &pte_chain);
pte_unmap_nested(src);
pte_unmap(dst);
- }
- flush_tlb_page(vma, old_addr);
+ } else
+ /*
+ * Why do we need this flush ? If there is no pte for
+ * old_addr, then there must not be a pte for it as well.
+ */
+ flush_tlb_page(vma, old_addr);
spin_unlock(&mm->page_table_lock);
pte_chain_free(pte_chain);
out:
diff -urN linux-bk/mm/msync.c linux-ptep/mm/msync.c
--- linux-bk/mm/msync.c Mon Oct 6 18:44:11 2003
+++ linux-ptep/mm/msync.c Mon Oct 6 18:43:11 2003
@@ -30,10 +30,9 @@
unsigned long pfn = pte_pfn(pte);
if (pfn_valid(pfn)) {
page = pfn_to_page(pfn);
- if (!PageReserved(page) && ptep_test_and_clear_dirty(ptep)) {
- flush_tlb_page(vma, address);
+ if (!PageReserved(page) &&
+ ptep_clear_flush_dirty(vma, address, ptep))
set_page_dirty(page);
- }
}
}
return 0;
diff -urN linux-bk/mm/rmap.c linux-ptep/mm/rmap.c
--- linux-bk/mm/rmap.c Mon Oct 6 18:44:11 2003
+++ linux-ptep/mm/rmap.c Mon Oct 6 18:43:25 2003
@@ -329,8 +329,7 @@
/* Nuke the page table entry. */
flush_cache_page(vma, address);
- pte = ptep_get_and_clear(ptep);
- flush_tlb_page(vma, address);
+ pte = ptep_clear_flush(vma, address, ptep);
if (PageSwapCache(page)) {
/*
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"aart@kvack.org"> aart@kvack.org </a>
next reply other threads:[~2003-10-09 12:38 UTC|newest]
Thread overview: 9+ messages / expand[flat|nested] mbox.gz Atom feed top
2003-10-09 12:38 Martin Schwidefsky [this message]
2003-10-10 7:00 ` David S. Miller
-- strict thread matches above, loose matches on Subject: below --
2003-10-10 8:37 Martin Schwidefsky
2003-10-10 8:20 Martin Schwidefsky
2003-10-10 8:50 ` David S. Miller
2003-10-10 9:01 ` Andi Kleen
2003-10-06 18:04 Martin Schwidefsky
2003-10-06 18:44 ` Matthew Wilcox
2003-10-07 8:29 ` David S. Miller
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20031009123844.GA464@mschwid3.boeblingen.de.ibm.com \
--to=schwidefsky@de.ibm.com \
--cc=akpm@osdl.org \
--cc=davem@redhat.com \
--cc=linux-arch@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=willy@debian.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox