From: Michel Lespinasse <walken@google.com>
To: Peter Zijlstra <peterz@infradead.org>,
Andrew Morton <akpm@linux-foundation.org>,
Laurent Dufour <ldufour@linux.ibm.com>,
Vlastimil Babka <vbabka@suse.cz>,
Matthew Wilcox <willy@infradead.org>,
"Liam R . Howlett" <Liam.Howlett@oracle.com>,
Jerome Glisse <jglisse@redhat.com>,
Davidlohr Bueso <dave@stgolabs.net>,
David Rientjes <rientjes@google.com>
Cc: linux-mm <linux-mm@kvack.org>, Michel Lespinasse <walken@google.com>
Subject: [RFC PATCH 08/24] mm/memory: allow specifying MM lock range to handle_mm_fault()
Date: Mon, 24 Feb 2020 12:30:41 -0800 [thread overview]
Message-ID: <20200224203057.162467-9-walken@google.com> (raw)
In-Reply-To: <20200224203057.162467-1-walken@google.com>
This change adds a new handle_mm_fault_range() function, which behaves
like handle_mm_fault() but specifies an explicit MM lock range.
handle_mm_fault() remains as an inline wrapper which passes the default
coarse locking range.
Signed-off-by: Michel Lespinasse <walken@google.com>
---
include/linux/hugetlb.h | 5 +++--
include/linux/mm.h | 11 +++++++++--
mm/hugetlb.c | 14 +++++++++-----
mm/memory.c | 16 +++++++++-------
4 files changed, 30 insertions(+), 16 deletions(-)
diff --git include/linux/hugetlb.h include/linux/hugetlb.h
index 31d4920994b9..75992d78289e 100644
--- include/linux/hugetlb.h
+++ include/linux/hugetlb.h
@@ -88,7 +88,8 @@ int hugetlb_report_node_meminfo(int, char *);
void hugetlb_show_meminfo(void);
unsigned long hugetlb_total_pages(void);
vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
- unsigned long address, unsigned int flags);
+ unsigned long address, unsigned int flags,
+ struct mm_lock_range *range);
int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, pte_t *dst_pte,
struct vm_area_struct *dst_vma,
unsigned long dst_addr,
@@ -307,7 +308,7 @@ static inline void __unmap_hugepage_range(struct mmu_gather *tlb,
static inline vm_fault_t hugetlb_fault(struct mm_struct *mm,
struct vm_area_struct *vma, unsigned long address,
- unsigned int flags)
+ unsigned int flags, struct mm_lock_range *range)
{
BUG();
return 0;
diff --git include/linux/mm.h include/linux/mm.h
index a1c9a0aa898b..1b6b022064b4 100644
--- include/linux/mm.h
+++ include/linux/mm.h
@@ -1460,8 +1460,15 @@ int generic_error_remove_page(struct address_space *mapping, struct page *page);
int invalidate_inode_page(struct page *page);
#ifdef CONFIG_MMU
-extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
- unsigned long address, unsigned int flags);
+extern vm_fault_t handle_mm_fault_range(struct vm_area_struct *vma,
+ unsigned long address, unsigned int flags,
+ struct mm_lock_range *range);
+static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
+ unsigned long address, unsigned int flags)
+{
+ return handle_mm_fault_range(vma, address, flags,
+ mm_coarse_lock_range());
+}
extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
unsigned long address, unsigned int fault_flags,
bool *unlocked);
diff --git mm/hugetlb.c mm/hugetlb.c
index 662f34b6c869..9d6fe9f291a7 100644
--- mm/hugetlb.c
+++ mm/hugetlb.c
@@ -3788,7 +3788,8 @@ int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
struct vm_area_struct *vma,
struct address_space *mapping, pgoff_t idx,
- unsigned long address, pte_t *ptep, unsigned int flags)
+ unsigned long address, pte_t *ptep, unsigned int flags,
+ struct mm_lock_range *range)
{
struct hstate *h = hstate_vma(vma);
vm_fault_t ret = VM_FAULT_SIGBUS;
@@ -3831,7 +3832,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
.vma = vma,
.address = haddr,
.flags = flags,
- .range = mm_coarse_lock_range(),
+ .range = range,
/*
* Hard to debug if it ends up being
* used by a callee that assumes
@@ -3997,7 +3998,8 @@ u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
#endif
vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
- unsigned long address, unsigned int flags)
+ unsigned long address, unsigned int flags,
+ struct mm_lock_range *range)
{
pte_t *ptep, entry;
spinlock_t *ptl;
@@ -4039,7 +4041,8 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
entry = huge_ptep_get(ptep);
if (huge_pte_none(entry)) {
- ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags);
+ ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep,
+ flags, range);
goto out_mutex;
}
@@ -4348,7 +4351,8 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
FAULT_FLAG_ALLOW_RETRY);
fault_flags |= FAULT_FLAG_TRIED;
}
- ret = hugetlb_fault(mm, vma, vaddr, fault_flags);
+ ret = hugetlb_fault(mm, vma, vaddr, fault_flags,
+ mm_coarse_lock_range());
if (ret & VM_FAULT_ERROR) {
err = vm_fault_to_errno(ret, flags);
remainder = 0;
diff --git mm/memory.c mm/memory.c
index 6cb3359f0857..bc24a6bdaa06 100644
--- mm/memory.c
+++ mm/memory.c
@@ -4039,7 +4039,8 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
* return value. See filemap_fault() and __lock_page_or_retry().
*/
static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
- unsigned long address, unsigned int flags)
+ unsigned long address, unsigned int flags,
+ struct mm_lock_range *range)
{
struct vm_fault vmf = {
.vma = vma,
@@ -4047,7 +4048,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
.flags = flags,
.pgoff = linear_page_index(vma, address),
.gfp_mask = __get_fault_gfp_mask(vma),
- .range = mm_coarse_lock_range(),
+ .range = range,
};
unsigned int dirty = flags & FAULT_FLAG_WRITE;
struct mm_struct *mm = vma->vm_mm;
@@ -4134,8 +4135,9 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
* The mmap_sem may have been released depending on flags and our
* return value. See filemap_fault() and __lock_page_or_retry().
*/
-vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
- unsigned int flags)
+vm_fault_t handle_mm_fault_range(struct vm_area_struct *vma,
+ unsigned long address, unsigned int flags,
+ struct mm_lock_range *range)
{
vm_fault_t ret;
@@ -4160,9 +4162,9 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
mem_cgroup_enter_user_fault();
if (unlikely(is_vm_hugetlb_page(vma)))
- ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
+ ret = hugetlb_fault(vma->vm_mm, vma, address, flags, range);
else
- ret = __handle_mm_fault(vma, address, flags);
+ ret = __handle_mm_fault(vma, address, flags, range);
if (flags & FAULT_FLAG_USER) {
mem_cgroup_exit_user_fault();
@@ -4178,7 +4180,7 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
return ret;
}
-EXPORT_SYMBOL_GPL(handle_mm_fault);
+EXPORT_SYMBOL_GPL(handle_mm_fault_range);
#ifndef __PAGETABLE_P4D_FOLDED
/*
--
2.25.0.341.g760bfbb309-goog
next prev parent reply other threads:[~2020-02-24 20:31 UTC|newest]
Thread overview: 28+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-02-24 20:30 [RFC PATCH 00/24] Fine grained MM locking Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 01/24] MM locking API: initial implementation as rwsem wrappers Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 02/24] MM locking API: use coccinelle to convert mmap_sem rwsem call sites Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 03/24] MM locking API: manual conversion of mmap_sem call sites missed by coccinelle Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 04/24] MM locking API: add range arguments Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 05/24] MM locking API: allow for sleeping during unlock Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 06/24] MM locking API: implement fine grained range locks Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 07/24] mm/memory: add range field to struct vm_fault Michel Lespinasse
2020-02-24 20:30 ` Michel Lespinasse [this message]
2020-02-24 20:30 ` [RFC PATCH 09/24] do_swap_page: use the vmf->range field when dropping mmap_sem Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 10/24] handle_userfault: " Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 11/24] x86 fault handler: merge bad_area() functions Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 12/24] x86 fault handler: use an explicit MM lock range Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 13/24] mm/memory: add prepare_mm_fault() function Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 14/24] mm/swap_state: disable swap vma readahead Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 15/24] x86 fault handler: use a pseudo-vma when operating on anonymous vmas Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 16/24] MM locking API: add vma locking API Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 17/24] x86 fault handler: implement range locking Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 18/24] shared file mappings: use the vmf->range field when dropping mmap_sem Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 19/24] mm: add field to annotate vm_operations that support range locking Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 20/24] x86 fault handler: extend range locking to supported file vmas Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 21/24] do_mmap: add locked argument Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 22/24] do_mmap: implement " Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 23/24] do_mmap: use locked=false in vm_mmap_pgoff() and aio_setup_ring() Michel Lespinasse
2020-02-24 20:30 ` [RFC PATCH 24/24] do_mmap: implement easiest cases of fine grained locking Michel Lespinasse
2022-03-20 22:08 ` [RFC PATCH 00/24] Fine grained MM locking Barry Song
2022-03-20 23:14 ` Matthew Wilcox
2022-03-21 0:20 ` Barry Song
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200224203057.162467-9-walken@google.com \
--to=walken@google.com \
--cc=Liam.Howlett@oracle.com \
--cc=akpm@linux-foundation.org \
--cc=dave@stgolabs.net \
--cc=jglisse@redhat.com \
--cc=ldufour@linux.ibm.com \
--cc=linux-mm@kvack.org \
--cc=peterz@infradead.org \
--cc=rientjes@google.com \
--cc=vbabka@suse.cz \
--cc=willy@infradead.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox