* [PATCH 1/5] hugetlb: Convert hugetlb_fault() to use struct vm_fault
2024-03-25 22:33 [PATCH 0/5] Define struct vm_fault in handle_mm_fault() Vishal Moola (Oracle)
@ 2024-03-25 22:33 ` Vishal Moola (Oracle)
2024-03-25 22:33 ` [PATCH 2/5] hugetlb: Convert hugetlb_no_page() " Vishal Moola (Oracle)
` (3 subsequent siblings)
4 siblings, 0 replies; 8+ messages in thread
From: Vishal Moola (Oracle) @ 2024-03-25 22:33 UTC (permalink / raw)
To: linux-mm; +Cc: linux-kernel, akpm, muchun.song, Vishal Moola (Oracle)
Now that hugetlb_fault() has a vm_fault available for fault tracking, use
it throughout. This cleans up the code by removing 2 variables, and
prepares hugetlb_fault() to take in a struct vm_fault argument.
Signed-off-by: Vishal Moola (Oracle) <vishal.moola@gmail.com>
---
mm/hugetlb.c | 84 +++++++++++++++++++++++++---------------------------
1 file changed, 41 insertions(+), 43 deletions(-)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 688017ca0cc2..81e8ade53b64 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -6330,8 +6330,6 @@ u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
unsigned long address, unsigned int flags)
{
- pte_t *ptep, entry;
- spinlock_t *ptl;
vm_fault_t ret;
u32 hash;
struct folio *folio = NULL;
@@ -6339,13 +6337,13 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
struct hstate *h = hstate_vma(vma);
struct address_space *mapping;
int need_wait_lock = 0;
- unsigned long haddr = address & huge_page_mask(h);
struct vm_fault vmf = {
.vma = vma,
- .address = haddr,
+ .address = address & huge_page_mask(h),
.real_address = address,
.flags = flags,
- .pgoff = vma_hugecache_offset(h, vma, haddr),
+ .pgoff = vma_hugecache_offset(h, vma,
+ address & huge_page_mask(h)),
/* TODO: Track hugetlb faults using vm_fault */
/*
@@ -6365,22 +6363,22 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
/*
* Acquire vma lock before calling huge_pte_alloc and hold
- * until finished with ptep. This prevents huge_pmd_unshare from
- * being called elsewhere and making the ptep no longer valid.
+ * until finished with vmf.pte. This prevents huge_pmd_unshare from
+ * being called elsewhere and making the vmf.pte no longer valid.
*/
hugetlb_vma_lock_read(vma);
- ptep = huge_pte_alloc(mm, vma, haddr, huge_page_size(h));
- if (!ptep) {
+ vmf.pte = huge_pte_alloc(mm, vma, vmf.address, huge_page_size(h));
+ if (!vmf.pte) {
hugetlb_vma_unlock_read(vma);
mutex_unlock(&hugetlb_fault_mutex_table[hash]);
return VM_FAULT_OOM;
}
- entry = huge_ptep_get(ptep);
- if (huge_pte_none_mostly(entry)) {
- if (is_pte_marker(entry)) {
+ vmf.orig_pte = huge_ptep_get(vmf.pte);
+ if (huge_pte_none_mostly(vmf.orig_pte)) {
+ if (is_pte_marker(vmf.orig_pte)) {
pte_marker marker =
- pte_marker_get(pte_to_swp_entry(entry));
+ pte_marker_get(pte_to_swp_entry(vmf.orig_pte));
if (marker & PTE_MARKER_POISONED) {
ret = VM_FAULT_HWPOISON_LARGE;
@@ -6395,20 +6393,20 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
* mutex internally, which make us return immediately.
*/
return hugetlb_no_page(mm, vma, mapping, vmf.pgoff, address,
- ptep, entry, flags, &vmf);
+ vmf.pte, vmf.orig_pte, flags, &vmf);
}
ret = 0;
/*
- * entry could be a migration/hwpoison entry at this point, so this
- * check prevents the kernel from going below assuming that we have
- * an active hugepage in pagecache. This goto expects the 2nd page
- * fault, and is_hugetlb_entry_(migration|hwpoisoned) check will
- * properly handle it.
+ * vmf.orig_pte could be a migration/hwpoison vmf.orig_pte at this
+ * point, so this check prevents the kernel from going below assuming
+ * that we have an active hugepage in pagecache. This goto expects
+ * the 2nd page fault, and is_hugetlb_entry_(migration|hwpoisoned)
+ * check will properly handle it.
*/
- if (!pte_present(entry)) {
- if (unlikely(is_hugetlb_entry_migration(entry))) {
+ if (!pte_present(vmf.orig_pte)) {
+ if (unlikely(is_hugetlb_entry_migration(vmf.orig_pte))) {
/*
* Release the hugetlb fault lock now, but retain
* the vma lock, because it is needed to guard the
@@ -6417,9 +6415,9 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
* be released there.
*/
mutex_unlock(&hugetlb_fault_mutex_table[hash]);
- migration_entry_wait_huge(vma, ptep);
+ migration_entry_wait_huge(vma, vmf.pte);
return 0;
- } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
+ } else if (unlikely(is_hugetlb_entry_hwpoisoned(vmf.orig_pte)))
ret = VM_FAULT_HWPOISON_LARGE |
VM_FAULT_SET_HINDEX(hstate_index(h));
goto out_mutex;
@@ -6433,13 +6431,13 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
* determine if a reservation has been consumed.
*/
if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
- !(vma->vm_flags & VM_MAYSHARE) && !huge_pte_write(entry)) {
- if (vma_needs_reservation(h, vma, haddr) < 0) {
+ !(vma->vm_flags & VM_MAYSHARE) && !huge_pte_write(vmf.orig_pte)) {
+ if (vma_needs_reservation(h, vma, vmf.address) < 0) {
ret = VM_FAULT_OOM;
goto out_mutex;
}
/* Just decrements count, does not deallocate */
- vma_end_reservation(h, vma, haddr);
+ vma_end_reservation(h, vma, vmf.address);
pagecache_folio = filemap_lock_hugetlb_folio(h, mapping,
vmf.pgoff);
@@ -6447,17 +6445,17 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
pagecache_folio = NULL;
}
- ptl = huge_pte_lock(h, mm, ptep);
+ vmf.ptl = huge_pte_lock(h, mm, vmf.pte);
/* Check for a racing update before calling hugetlb_wp() */
- if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
+ if (unlikely(!pte_same(vmf.orig_pte, huge_ptep_get(vmf.pte))))
goto out_ptl;
/* Handle userfault-wp first, before trying to lock more pages */
- if (userfaultfd_wp(vma) && huge_pte_uffd_wp(huge_ptep_get(ptep)) &&
- (flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
+ if (userfaultfd_wp(vma) && huge_pte_uffd_wp(huge_ptep_get(vmf.pte)) &&
+ (flags & FAULT_FLAG_WRITE) && !huge_pte_write(vmf.orig_pte)) {
if (!userfaultfd_wp_async(vma)) {
- spin_unlock(ptl);
+ spin_unlock(vmf.ptl);
if (pagecache_folio) {
folio_unlock(pagecache_folio);
folio_put(pagecache_folio);
@@ -6467,18 +6465,18 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
return handle_userfault(&vmf, VM_UFFD_WP);
}
- entry = huge_pte_clear_uffd_wp(entry);
- set_huge_pte_at(mm, haddr, ptep, entry,
+ vmf.orig_pte = huge_pte_clear_uffd_wp(vmf.orig_pte);
+ set_huge_pte_at(mm, vmf.address, vmf.pte, vmf.orig_pte,
huge_page_size(hstate_vma(vma)));
/* Fallthrough to CoW */
}
/*
- * hugetlb_wp() requires page locks of pte_page(entry) and
+ * hugetlb_wp() requires page locks of pte_page(vmf.orig_pte) and
* pagecache_folio, so here we need take the former one
* when folio != pagecache_folio or !pagecache_folio.
*/
- folio = page_folio(pte_page(entry));
+ folio = page_folio(pte_page(vmf.orig_pte));
if (folio != pagecache_folio)
if (!folio_trylock(folio)) {
need_wait_lock = 1;
@@ -6488,24 +6486,24 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
folio_get(folio);
if (flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
- if (!huge_pte_write(entry)) {
- ret = hugetlb_wp(mm, vma, address, ptep, flags,
- pagecache_folio, ptl, &vmf);
+ if (!huge_pte_write(vmf.orig_pte)) {
+ ret = hugetlb_wp(mm, vma, address, vmf.pte, flags,
+ pagecache_folio, vmf.ptl, &vmf);
goto out_put_page;
} else if (likely(flags & FAULT_FLAG_WRITE)) {
- entry = huge_pte_mkdirty(entry);
+ vmf.orig_pte = huge_pte_mkdirty(vmf.orig_pte);
}
}
- entry = pte_mkyoung(entry);
- if (huge_ptep_set_access_flags(vma, haddr, ptep, entry,
+ vmf.orig_pte = pte_mkyoung(vmf.orig_pte);
+ if (huge_ptep_set_access_flags(vma, vmf.address, vmf.pte, vmf.orig_pte,
flags & FAULT_FLAG_WRITE))
- update_mmu_cache(vma, haddr, ptep);
+ update_mmu_cache(vma, vmf.address, vmf.pte);
out_put_page:
if (folio != pagecache_folio)
folio_unlock(folio);
folio_put(folio);
out_ptl:
- spin_unlock(ptl);
+ spin_unlock(vmf.ptl);
if (pagecache_folio) {
folio_unlock(pagecache_folio);
--
2.43.0
^ permalink raw reply [flat|nested] 8+ messages in thread* [PATCH 2/5] hugetlb: Convert hugetlb_no_page() to use struct vm_fault
2024-03-25 22:33 [PATCH 0/5] Define struct vm_fault in handle_mm_fault() Vishal Moola (Oracle)
2024-03-25 22:33 ` [PATCH 1/5] hugetlb: Convert hugetlb_fault() to use struct vm_fault Vishal Moola (Oracle)
@ 2024-03-25 22:33 ` Vishal Moola (Oracle)
2024-03-25 22:33 ` [PATCH 3/5] hugetlb: Convert hugetlb_wp() " Vishal Moola (Oracle)
` (2 subsequent siblings)
4 siblings, 0 replies; 8+ messages in thread
From: Vishal Moola (Oracle) @ 2024-03-25 22:33 UTC (permalink / raw)
To: linux-mm; +Cc: linux-kernel, akpm, muchun.song, Vishal Moola (Oracle)
hugetlb_no_page() can use the struct vm_fault passed in from
hugetlb_fault(). This alleviates the stack by consolidating 7
variables into a single struct.
Signed-off-by: Vishal Moola (Oracle) <vishal.moola@gmail.com>
---
mm/hugetlb.c | 59 ++++++++++++++++++++++++++--------------------------
1 file changed, 29 insertions(+), 30 deletions(-)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 81e8ade53b64..819a6d067985 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -6096,9 +6096,7 @@ static bool hugetlb_pte_stable(struct hstate *h, struct mm_struct *mm,
static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
struct vm_area_struct *vma,
- struct address_space *mapping, pgoff_t idx,
- unsigned long address, pte_t *ptep,
- pte_t old_pte, unsigned int flags,
+ struct address_space *mapping,
struct vm_fault *vmf)
{
struct hstate *h = hstate_vma(vma);
@@ -6107,10 +6105,8 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
unsigned long size;
struct folio *folio;
pte_t new_pte;
- spinlock_t *ptl;
- unsigned long haddr = address & huge_page_mask(h);
bool new_folio, new_pagecache_folio = false;
- u32 hash = hugetlb_fault_mutex_hash(mapping, idx);
+ u32 hash = hugetlb_fault_mutex_hash(mapping, vmf->pgoff);
/*
* Currently, we are forced to kill the process in the event the
@@ -6129,10 +6125,10 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
* before we get page_table_lock.
*/
new_folio = false;
- folio = filemap_lock_hugetlb_folio(h, mapping, idx);
+ folio = filemap_lock_hugetlb_folio(h, mapping, vmf->pgoff);
if (IS_ERR(folio)) {
size = i_size_read(mapping->host) >> huge_page_shift(h);
- if (idx >= size)
+ if (vmf->pgoff >= size)
goto out;
/* Check for page in userfault range */
if (userfaultfd_missing(vma)) {
@@ -6153,7 +6149,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
* never happen on the page after UFFDIO_COPY has
* correctly installed the page and returned.
*/
- if (!hugetlb_pte_stable(h, mm, ptep, old_pte)) {
+ if (!hugetlb_pte_stable(h, mm, vmf->pte, vmf->orig_pte)) {
ret = 0;
goto out;
}
@@ -6162,7 +6158,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
VM_UFFD_MISSING);
}
- folio = alloc_hugetlb_folio(vma, haddr, 0);
+ folio = alloc_hugetlb_folio(vma, vmf->address, 0);
if (IS_ERR(folio)) {
/*
* Returning error will result in faulting task being
@@ -6176,18 +6172,20 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
* here. Before returning error, get ptl and make
* sure there really is no pte entry.
*/
- if (hugetlb_pte_stable(h, mm, ptep, old_pte))
+ if (hugetlb_pte_stable(h, mm, vmf->pte, vmf->orig_pte))
ret = vmf_error(PTR_ERR(folio));
else
ret = 0;
goto out;
}
- clear_huge_page(&folio->page, address, pages_per_huge_page(h));
+ clear_huge_page(&folio->page, vmf->real_address,
+ pages_per_huge_page(h));
__folio_mark_uptodate(folio);
new_folio = true;
if (vma->vm_flags & VM_MAYSHARE) {
- int err = hugetlb_add_to_page_cache(folio, mapping, idx);
+ int err = hugetlb_add_to_page_cache(folio, mapping,
+ vmf->pgoff);
if (err) {
/*
* err can't be -EEXIST which implies someone
@@ -6196,7 +6194,8 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
* to the page cache. So it's safe to call
* restore_reserve_on_error() here.
*/
- restore_reserve_on_error(h, vma, haddr, folio);
+ restore_reserve_on_error(h, vma, vmf->address,
+ folio);
folio_put(folio);
goto out;
}
@@ -6226,7 +6225,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
folio_unlock(folio);
folio_put(folio);
/* See comment in userfaultfd_missing() block above */
- if (!hugetlb_pte_stable(h, mm, ptep, old_pte)) {
+ if (!hugetlb_pte_stable(h, mm, vmf->pte, vmf->orig_pte)) {
ret = 0;
goto out;
}
@@ -6241,23 +6240,23 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
* any allocations necessary to record that reservation occur outside
* the spinlock.
*/
- if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
- if (vma_needs_reservation(h, vma, haddr) < 0) {
+ if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
+ if (vma_needs_reservation(h, vma, vmf->address) < 0) {
ret = VM_FAULT_OOM;
goto backout_unlocked;
}
/* Just decrements count, does not deallocate */
- vma_end_reservation(h, vma, haddr);
+ vma_end_reservation(h, vma, vmf->address);
}
- ptl = huge_pte_lock(h, mm, ptep);
+ vmf->ptl = huge_pte_lock(h, mm, vmf->pte);
ret = 0;
/* If pte changed from under us, retry */
- if (!pte_same(huge_ptep_get(ptep), old_pte))
+ if (!pte_same(huge_ptep_get(vmf->pte), vmf->orig_pte))
goto backout;
if (anon_rmap)
- hugetlb_add_new_anon_rmap(folio, vma, haddr);
+ hugetlb_add_new_anon_rmap(folio, vma, vmf->address);
else
hugetlb_add_file_rmap(folio);
new_pte = make_huge_pte(vma, &folio->page, ((vma->vm_flags & VM_WRITE)
@@ -6266,17 +6265,18 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
* If this pte was previously wr-protected, keep it wr-protected even
* if populated.
*/
- if (unlikely(pte_marker_uffd_wp(old_pte)))
+ if (unlikely(pte_marker_uffd_wp(vmf->orig_pte)))
new_pte = huge_pte_mkuffd_wp(new_pte);
- set_huge_pte_at(mm, haddr, ptep, new_pte, huge_page_size(h));
+ set_huge_pte_at(mm, vmf->address, vmf->pte, new_pte, huge_page_size(h));
hugetlb_count_add(pages_per_huge_page(h), mm);
- if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
+ if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
/* Optimization, do the COW without a second fault */
- ret = hugetlb_wp(mm, vma, address, ptep, flags, folio, ptl, vmf);
+ ret = hugetlb_wp(mm, vma, vmf->real_address, vmf->pte,
+ vmf->flags, folio, vmf->ptl, vmf);
}
- spin_unlock(ptl);
+ spin_unlock(vmf->ptl);
/*
* Only set hugetlb_migratable in newly allocated pages. Existing pages
@@ -6293,10 +6293,10 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
return ret;
backout:
- spin_unlock(ptl);
+ spin_unlock(vmf->ptl);
backout_unlocked:
if (new_folio && !new_pagecache_folio)
- restore_reserve_on_error(h, vma, haddr, folio);
+ restore_reserve_on_error(h, vma, vmf->address, folio);
folio_unlock(folio);
folio_put(folio);
@@ -6392,8 +6392,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
* hugetlb_no_page will drop vma lock and hugetlb fault
* mutex internally, which make us return immediately.
*/
- return hugetlb_no_page(mm, vma, mapping, vmf.pgoff, address,
- vmf.pte, vmf.orig_pte, flags, &vmf);
+ return hugetlb_no_page(mm, vma, mapping, &vmf);
}
ret = 0;
--
2.43.0
^ permalink raw reply [flat|nested] 8+ messages in thread* [PATCH 3/5] hugetlb: Convert hugetlb_wp() to use struct vm_fault
2024-03-25 22:33 [PATCH 0/5] Define struct vm_fault in handle_mm_fault() Vishal Moola (Oracle)
2024-03-25 22:33 ` [PATCH 1/5] hugetlb: Convert hugetlb_fault() to use struct vm_fault Vishal Moola (Oracle)
2024-03-25 22:33 ` [PATCH 2/5] hugetlb: Convert hugetlb_no_page() " Vishal Moola (Oracle)
@ 2024-03-25 22:33 ` Vishal Moola (Oracle)
2024-03-25 22:33 ` [PATCH 4/5] mm: Make pgoff non-const in " Vishal Moola (Oracle)
2024-03-25 22:33 ` [PATCH 5/5] memory: Define struct vm_fault in handle_mm_fault() Vishal Moola (Oracle)
4 siblings, 0 replies; 8+ messages in thread
From: Vishal Moola (Oracle) @ 2024-03-25 22:33 UTC (permalink / raw)
To: linux-mm; +Cc: linux-kernel, akpm, muchun.song, Vishal Moola (Oracle)
hugetlb_wp() can use the struct vm_fault passed in from hugetlb_fault().
This alleviates the stack by consolidating 5 variables into a single
struct.
Signed-off-by: Vishal Moola (Oracle) <vishal.moola@gmail.com>
---
mm/hugetlb.c | 61 ++++++++++++++++++++++++++--------------------------
1 file changed, 30 insertions(+), 31 deletions(-)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 819a6d067985..107b47329b9f 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -5825,18 +5825,16 @@ static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
* Keep the pte_same checks anyway to make transition from the mutex easier.
*/
static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,
- unsigned long address, pte_t *ptep, unsigned int flags,
- struct folio *pagecache_folio, spinlock_t *ptl,
+ struct folio *pagecache_folio,
struct vm_fault *vmf)
{
- const bool unshare = flags & FAULT_FLAG_UNSHARE;
- pte_t pte = huge_ptep_get(ptep);
+ const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
+ pte_t pte = huge_ptep_get(vmf->pte);
struct hstate *h = hstate_vma(vma);
struct folio *old_folio;
struct folio *new_folio;
int outside_reserve = 0;
vm_fault_t ret = 0;
- unsigned long haddr = address & huge_page_mask(h);
struct mmu_notifier_range range;
/*
@@ -5859,7 +5857,7 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,
/* Let's take out MAP_SHARED mappings first. */
if (vma->vm_flags & VM_MAYSHARE) {
- set_huge_ptep_writable(vma, haddr, ptep);
+ set_huge_ptep_writable(vma, vmf->address, vmf->pte);
return 0;
}
@@ -5878,7 +5876,7 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,
SetPageAnonExclusive(&old_folio->page);
}
if (likely(!unshare))
- set_huge_ptep_writable(vma, haddr, ptep);
+ set_huge_ptep_writable(vma, vmf->address, vmf->pte);
delayacct_wpcopy_end();
return 0;
@@ -5905,8 +5903,8 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,
* Drop page table lock as buddy allocator may be called. It will
* be acquired again before returning to the caller, as expected.
*/
- spin_unlock(ptl);
- new_folio = alloc_hugetlb_folio(vma, haddr, outside_reserve);
+ spin_unlock(vmf->ptl);
+ new_folio = alloc_hugetlb_folio(vma, vmf->address, outside_reserve);
if (IS_ERR(new_folio)) {
/*
@@ -5931,19 +5929,21 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,
*
* Reacquire both after unmap operation.
*/
- idx = vma_hugecache_offset(h, vma, haddr);
+ idx = vma_hugecache_offset(h, vma, vmf->address);
hash = hugetlb_fault_mutex_hash(mapping, idx);
hugetlb_vma_unlock_read(vma);
mutex_unlock(&hugetlb_fault_mutex_table[hash]);
- unmap_ref_private(mm, vma, &old_folio->page, haddr);
+ unmap_ref_private(mm, vma, &old_folio->page,
+ vmf->address);
mutex_lock(&hugetlb_fault_mutex_table[hash]);
hugetlb_vma_lock_read(vma);
- spin_lock(ptl);
- ptep = hugetlb_walk(vma, haddr, huge_page_size(h));
- if (likely(ptep &&
- pte_same(huge_ptep_get(ptep), pte)))
+ spin_lock(vmf->ptl);
+ vmf->pte = hugetlb_walk(vma, vmf->address,
+ huge_page_size(h));
+ if (likely(vmf->pte &&
+ pte_same(huge_ptep_get(vmf->pte), pte)))
goto retry_avoidcopy;
/*
* race occurs while re-acquiring page table
@@ -5965,37 +5965,38 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,
if (unlikely(ret))
goto out_release_all;
- if (copy_user_large_folio(new_folio, old_folio, address, vma)) {
+ if (copy_user_large_folio(new_folio, old_folio, vmf->real_address, vma)) {
ret = VM_FAULT_HWPOISON_LARGE;
goto out_release_all;
}
__folio_mark_uptodate(new_folio);
- mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, haddr,
- haddr + huge_page_size(h));
+ mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, vmf->address,
+ vmf->address + huge_page_size(h));
mmu_notifier_invalidate_range_start(&range);
/*
* Retake the page table lock to check for racing updates
* before the page tables are altered
*/
- spin_lock(ptl);
- ptep = hugetlb_walk(vma, haddr, huge_page_size(h));
- if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
+ spin_lock(vmf->ptl);
+ vmf->pte = hugetlb_walk(vma, vmf->address, huge_page_size(h));
+ if (likely(vmf->pte && pte_same(huge_ptep_get(vmf->pte), pte))) {
pte_t newpte = make_huge_pte(vma, &new_folio->page, !unshare);
/* Break COW or unshare */
- huge_ptep_clear_flush(vma, haddr, ptep);
+ huge_ptep_clear_flush(vma, vmf->address, vmf->pte);
hugetlb_remove_rmap(old_folio);
- hugetlb_add_new_anon_rmap(new_folio, vma, haddr);
+ hugetlb_add_new_anon_rmap(new_folio, vma, vmf->address);
if (huge_pte_uffd_wp(pte))
newpte = huge_pte_mkuffd_wp(newpte);
- set_huge_pte_at(mm, haddr, ptep, newpte, huge_page_size(h));
+ set_huge_pte_at(mm, vmf->address, vmf->pte, newpte,
+ huge_page_size(h));
folio_set_hugetlb_migratable(new_folio);
/* Make the old page be freed below */
new_folio = old_folio;
}
- spin_unlock(ptl);
+ spin_unlock(vmf->ptl);
mmu_notifier_invalidate_range_end(&range);
out_release_all:
/*
@@ -6003,12 +6004,12 @@ static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,
* unshare)
*/
if (new_folio != old_folio)
- restore_reserve_on_error(h, vma, haddr, new_folio);
+ restore_reserve_on_error(h, vma, vmf->address, new_folio);
folio_put(new_folio);
out_release_old:
folio_put(old_folio);
- spin_lock(ptl); /* Caller expects lock to be held */
+ spin_lock(vmf->ptl); /* Caller expects lock to be held */
delayacct_wpcopy_end();
return ret;
@@ -6272,8 +6273,7 @@ static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
hugetlb_count_add(pages_per_huge_page(h), mm);
if ((vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
/* Optimization, do the COW without a second fault */
- ret = hugetlb_wp(mm, vma, vmf->real_address, vmf->pte,
- vmf->flags, folio, vmf->ptl, vmf);
+ ret = hugetlb_wp(mm, vma, folio, vmf);
}
spin_unlock(vmf->ptl);
@@ -6486,8 +6486,7 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
if (flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
if (!huge_pte_write(vmf.orig_pte)) {
- ret = hugetlb_wp(mm, vma, address, vmf.pte, flags,
- pagecache_folio, vmf.ptl, &vmf);
+ ret = hugetlb_wp(mm, vma, pagecache_folio, &vmf);
goto out_put_page;
} else if (likely(flags & FAULT_FLAG_WRITE)) {
vmf.orig_pte = huge_pte_mkdirty(vmf.orig_pte);
--
2.43.0
^ permalink raw reply [flat|nested] 8+ messages in thread* [PATCH 4/5] mm: Make pgoff non-const in struct vm_fault
2024-03-25 22:33 [PATCH 0/5] Define struct vm_fault in handle_mm_fault() Vishal Moola (Oracle)
` (2 preceding siblings ...)
2024-03-25 22:33 ` [PATCH 3/5] hugetlb: Convert hugetlb_wp() " Vishal Moola (Oracle)
@ 2024-03-25 22:33 ` Vishal Moola (Oracle)
2024-03-26 2:38 ` Matthew Wilcox
2024-03-25 22:33 ` [PATCH 5/5] memory: Define struct vm_fault in handle_mm_fault() Vishal Moola (Oracle)
4 siblings, 1 reply; 8+ messages in thread
From: Vishal Moola (Oracle) @ 2024-03-25 22:33 UTC (permalink / raw)
To: linux-mm; +Cc: linux-kernel, akpm, muchun.song, Vishal Moola (Oracle)
Hugetlb calculates addresses and page offsets differently from the rest of
mm. In order to pass struct vm_fault through the fault pathway we will let
hugetlb_fault() and __handle_mm_fault() set those variables themselves
instead.
Signed-off-by: Vishal Moola (Oracle) <vishal.moola@gmail.com>
---
include/linux/mm.h | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index f5a97dec5169..c6874aa7b7f0 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -507,10 +507,11 @@ struct vm_fault {
const struct {
struct vm_area_struct *vma; /* Target VMA */
gfp_t gfp_mask; /* gfp mask to be used for allocations */
- pgoff_t pgoff; /* Logical page offset based on vma */
- unsigned long address; /* Faulting virtual address - masked */
unsigned long real_address; /* Faulting virtual address - unmasked */
};
+ unsigned long address; /* Faulting virtual address - masked */
+ pgoff_t pgoff; /* Logical page offset based on vma */
+
enum fault_flag flags; /* FAULT_FLAG_xxx flags
* XXX: should really be 'const' */
pmd_t *pmd; /* Pointer to pmd entry matching
--
2.43.0
^ permalink raw reply [flat|nested] 8+ messages in thread* Re: [PATCH 4/5] mm: Make pgoff non-const in struct vm_fault
2024-03-25 22:33 ` [PATCH 4/5] mm: Make pgoff non-const in " Vishal Moola (Oracle)
@ 2024-03-26 2:38 ` Matthew Wilcox
2024-03-26 20:06 ` Vishal Moola
0 siblings, 1 reply; 8+ messages in thread
From: Matthew Wilcox @ 2024-03-26 2:38 UTC (permalink / raw)
To: Vishal Moola (Oracle); +Cc: linux-mm, linux-kernel, akpm, muchun.song
On Mon, Mar 25, 2024 at 03:33:38PM -0700, Vishal Moola (Oracle) wrote:
> Hugetlb calculates addresses and page offsets differently from the rest of
> mm. In order to pass struct vm_fault through the fault pathway we will let
> hugetlb_fault() and __handle_mm_fault() set those variables themselves
> instead.
I don't think this is a great idea. I'd rather not do patch 5 than do
patch 4+5. If you look at the history, commits 742d33729a0df11 and
5857c9209ce58f show that drivers got into the bad habit of changing
address & pgoff, so they got made const to prevent that.
So can we make hugetlbfs OK with using addresses & pgoffsets that aren't
aligned to HPAGE boundaries? Worth playing with for a bit to see how
deep that assumption runs.
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH 4/5] mm: Make pgoff non-const in struct vm_fault
2024-03-26 2:38 ` Matthew Wilcox
@ 2024-03-26 20:06 ` Vishal Moola
0 siblings, 0 replies; 8+ messages in thread
From: Vishal Moola @ 2024-03-26 20:06 UTC (permalink / raw)
To: Matthew Wilcox; +Cc: linux-mm, linux-kernel, akpm, muchun.song
On Mon, Mar 25, 2024 at 7:38 PM Matthew Wilcox <willy@infradead.org> wrote:
>
> On Mon, Mar 25, 2024 at 03:33:38PM -0700, Vishal Moola (Oracle) wrote:
> > Hugetlb calculates addresses and page offsets differently from the rest of
> > mm. In order to pass struct vm_fault through the fault pathway we will let
> > hugetlb_fault() and __handle_mm_fault() set those variables themselves
> > instead.
>
> I don't think this is a great idea. I'd rather not do patch 5 than do
> patch 4+5. If you look at the history, commits 742d33729a0df11 and
> 5857c9209ce58f show that drivers got into the bad habit of changing
> address & pgoff, so they got made const to prevent that.
>
> So can we make hugetlbfs OK with using addresses & pgoffsets that aren't
> aligned to HPAGE boundaries? Worth playing with for a bit to see how
> deep that assumption runs.
Hmmm, I'll take a look. I don't think there should be too many issues
with that.
^ permalink raw reply [flat|nested] 8+ messages in thread
* [PATCH 5/5] memory: Define struct vm_fault in handle_mm_fault()
2024-03-25 22:33 [PATCH 0/5] Define struct vm_fault in handle_mm_fault() Vishal Moola (Oracle)
` (3 preceding siblings ...)
2024-03-25 22:33 ` [PATCH 4/5] mm: Make pgoff non-const in " Vishal Moola (Oracle)
@ 2024-03-25 22:33 ` Vishal Moola (Oracle)
4 siblings, 0 replies; 8+ messages in thread
From: Vishal Moola (Oracle) @ 2024-03-25 22:33 UTC (permalink / raw)
To: linux-mm; +Cc: linux-kernel, akpm, muchun.song, Vishal Moola (Oracle)
Define struct vm_fault in handle_mm_fault() to be passed throughout the
rest of the fault pathway. Pass it through to hugetlb_fault() and
__handle_mm_fault(), making any necessary trivial changes.
Signed-off-by: Vishal Moola (Oracle) <vishal.moola@gmail.com>
---
include/linux/hugetlb.h | 7 +--
mm/hugetlb.c | 106 +++++++++++++++++++---------------------
mm/memory.c | 87 +++++++++++++++++----------------
3 files changed, 98 insertions(+), 102 deletions(-)
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index c1ee640d87b1..0e0a93b4d9fc 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -147,8 +147,7 @@ void hugetlb_report_meminfo(struct seq_file *);
int hugetlb_report_node_meminfo(char *buf, int len, int nid);
void hugetlb_show_meminfo_node(int nid);
unsigned long hugetlb_total_pages(void);
-vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
- unsigned long address, unsigned int flags);
+vm_fault_t hugetlb_fault(struct vm_fault *vmf);
#ifdef CONFIG_USERFAULTFD
int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
struct vm_area_struct *dst_vma,
@@ -482,9 +481,7 @@ static inline void __unmap_hugepage_range(struct mmu_gather *tlb,
BUG();
}
-static inline vm_fault_t hugetlb_fault(struct mm_struct *mm,
- struct vm_area_struct *vma, unsigned long address,
- unsigned int flags)
+static inline vm_fault_t hugetlb_fault(struct vm_fault *vmf)
{
BUG();
return 0;
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 107b47329b9f..7ecc680f4681 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -6327,30 +6327,24 @@ u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
}
#endif
-vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
- unsigned long address, unsigned int flags)
+vm_fault_t hugetlb_fault(struct vm_fault *vmf)
{
vm_fault_t ret;
u32 hash;
+ struct vm_area_struct *vma = vmf->vma;
+ struct mm_struct *mm = vma->vm_mm;
struct folio *folio = NULL;
struct folio *pagecache_folio = NULL;
struct hstate *h = hstate_vma(vma);
struct address_space *mapping;
int need_wait_lock = 0;
- struct vm_fault vmf = {
- .vma = vma,
- .address = address & huge_page_mask(h),
- .real_address = address,
- .flags = flags,
- .pgoff = vma_hugecache_offset(h, vma,
- address & huge_page_mask(h)),
- /* TODO: Track hugetlb faults using vm_fault */
-
- /*
- * Some fields may not be initialized, be careful as it may
- * be hard to debug if called functions make assumptions
- */
- };
+ /*
+ * Some fields of vmf may not be initialized, be careful as it may
+ * be hard to debug if called functions make assumptions
+ */
+ vmf->address = vmf->real_address & huge_page_mask(h);
+ vmf->pgoff = vma_hugecache_offset(h, vma,
+ vmf->address & huge_page_mask(h));
/*
* Serialize hugepage allocation and instantiation, so that we don't
@@ -6358,27 +6352,27 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
* the same page in the page cache.
*/
mapping = vma->vm_file->f_mapping;
- hash = hugetlb_fault_mutex_hash(mapping, vmf.pgoff);
+ hash = hugetlb_fault_mutex_hash(mapping, vmf->pgoff);
mutex_lock(&hugetlb_fault_mutex_table[hash]);
/*
* Acquire vma lock before calling huge_pte_alloc and hold
- * until finished with vmf.pte. This prevents huge_pmd_unshare from
- * being called elsewhere and making the vmf.pte no longer valid.
+ * until finished with vmf->pte. This prevents huge_pmd_unshare from
+ * being called elsewhere and making the vmf->pte no longer valid.
*/
hugetlb_vma_lock_read(vma);
- vmf.pte = huge_pte_alloc(mm, vma, vmf.address, huge_page_size(h));
- if (!vmf.pte) {
+ vmf->pte = huge_pte_alloc(mm, vma, vmf->address, huge_page_size(h));
+ if (!vmf->pte) {
hugetlb_vma_unlock_read(vma);
mutex_unlock(&hugetlb_fault_mutex_table[hash]);
return VM_FAULT_OOM;
}
- vmf.orig_pte = huge_ptep_get(vmf.pte);
- if (huge_pte_none_mostly(vmf.orig_pte)) {
- if (is_pte_marker(vmf.orig_pte)) {
+ vmf->orig_pte = huge_ptep_get(vmf->pte);
+ if (huge_pte_none_mostly(vmf->orig_pte)) {
+ if (is_pte_marker(vmf->orig_pte)) {
pte_marker marker =
- pte_marker_get(pte_to_swp_entry(vmf.orig_pte));
+ pte_marker_get(pte_to_swp_entry(vmf->orig_pte));
if (marker & PTE_MARKER_POISONED) {
ret = VM_FAULT_HWPOISON_LARGE;
@@ -6392,20 +6386,20 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
* hugetlb_no_page will drop vma lock and hugetlb fault
* mutex internally, which make us return immediately.
*/
- return hugetlb_no_page(mm, vma, mapping, &vmf);
+ return hugetlb_no_page(mm, vma, mapping, vmf);
}
ret = 0;
/*
- * vmf.orig_pte could be a migration/hwpoison vmf.orig_pte at this
+ * vmf->orig_pte could be a migration/hwpoison vmf->orig_pte at this
* point, so this check prevents the kernel from going below assuming
* that we have an active hugepage in pagecache. This goto expects
* the 2nd page fault, and is_hugetlb_entry_(migration|hwpoisoned)
* check will properly handle it.
*/
- if (!pte_present(vmf.orig_pte)) {
- if (unlikely(is_hugetlb_entry_migration(vmf.orig_pte))) {
+ if (!pte_present(vmf->orig_pte)) {
+ if (unlikely(is_hugetlb_entry_migration(vmf->orig_pte))) {
/*
* Release the hugetlb fault lock now, but retain
* the vma lock, because it is needed to guard the
@@ -6414,9 +6408,9 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
* be released there.
*/
mutex_unlock(&hugetlb_fault_mutex_table[hash]);
- migration_entry_wait_huge(vma, vmf.pte);
+ migration_entry_wait_huge(vma, vmf->pte);
return 0;
- } else if (unlikely(is_hugetlb_entry_hwpoisoned(vmf.orig_pte)))
+ } else if (unlikely(is_hugetlb_entry_hwpoisoned(vmf->orig_pte)))
ret = VM_FAULT_HWPOISON_LARGE |
VM_FAULT_SET_HINDEX(hstate_index(h));
goto out_mutex;
@@ -6429,53 +6423,53 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
* spinlock. Also lookup the pagecache page now as it is used to
* determine if a reservation has been consumed.
*/
- if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
- !(vma->vm_flags & VM_MAYSHARE) && !huge_pte_write(vmf.orig_pte)) {
- if (vma_needs_reservation(h, vma, vmf.address) < 0) {
+ if ((vmf->flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
+ !(vma->vm_flags & VM_MAYSHARE) && !huge_pte_write(vmf->orig_pte)) {
+ if (vma_needs_reservation(h, vma, vmf->address) < 0) {
ret = VM_FAULT_OOM;
goto out_mutex;
}
/* Just decrements count, does not deallocate */
- vma_end_reservation(h, vma, vmf.address);
+ vma_end_reservation(h, vma, vmf->address);
pagecache_folio = filemap_lock_hugetlb_folio(h, mapping,
- vmf.pgoff);
+ vmf->pgoff);
if (IS_ERR(pagecache_folio))
pagecache_folio = NULL;
}
- vmf.ptl = huge_pte_lock(h, mm, vmf.pte);
+ vmf->ptl = huge_pte_lock(h, mm, vmf->pte);
/* Check for a racing update before calling hugetlb_wp() */
- if (unlikely(!pte_same(vmf.orig_pte, huge_ptep_get(vmf.pte))))
+ if (unlikely(!pte_same(vmf->orig_pte, huge_ptep_get(vmf->pte))))
goto out_ptl;
/* Handle userfault-wp first, before trying to lock more pages */
- if (userfaultfd_wp(vma) && huge_pte_uffd_wp(huge_ptep_get(vmf.pte)) &&
- (flags & FAULT_FLAG_WRITE) && !huge_pte_write(vmf.orig_pte)) {
+ if (userfaultfd_wp(vma) && huge_pte_uffd_wp(huge_ptep_get(vmf->pte)) &&
+ (vmf->flags & FAULT_FLAG_WRITE) && !huge_pte_write(vmf->orig_pte)) {
if (!userfaultfd_wp_async(vma)) {
- spin_unlock(vmf.ptl);
+ spin_unlock(vmf->ptl);
if (pagecache_folio) {
folio_unlock(pagecache_folio);
folio_put(pagecache_folio);
}
hugetlb_vma_unlock_read(vma);
mutex_unlock(&hugetlb_fault_mutex_table[hash]);
- return handle_userfault(&vmf, VM_UFFD_WP);
+ return handle_userfault(vmf, VM_UFFD_WP);
}
- vmf.orig_pte = huge_pte_clear_uffd_wp(vmf.orig_pte);
- set_huge_pte_at(mm, vmf.address, vmf.pte, vmf.orig_pte,
+ vmf->orig_pte = huge_pte_clear_uffd_wp(vmf->orig_pte);
+ set_huge_pte_at(mm, vmf->address, vmf->pte, vmf->orig_pte,
huge_page_size(hstate_vma(vma)));
/* Fallthrough to CoW */
}
/*
- * hugetlb_wp() requires page locks of pte_page(vmf.orig_pte) and
+ * hugetlb_wp() requires page locks of pte_page(vmf->orig_pte) and
* pagecache_folio, so here we need take the former one
* when folio != pagecache_folio or !pagecache_folio.
*/
- folio = page_folio(pte_page(vmf.orig_pte));
+ folio = page_folio(pte_page(vmf->orig_pte));
if (folio != pagecache_folio)
if (!folio_trylock(folio)) {
need_wait_lock = 1;
@@ -6484,24 +6478,24 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
folio_get(folio);
- if (flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
- if (!huge_pte_write(vmf.orig_pte)) {
- ret = hugetlb_wp(mm, vma, pagecache_folio, &vmf);
+ if (vmf->flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
+ if (!huge_pte_write(vmf->orig_pte)) {
+ ret = hugetlb_wp(mm, vma, pagecache_folio, vmf);
goto out_put_page;
- } else if (likely(flags & FAULT_FLAG_WRITE)) {
- vmf.orig_pte = huge_pte_mkdirty(vmf.orig_pte);
+ } else if (likely(vmf->flags & FAULT_FLAG_WRITE)) {
+ vmf->orig_pte = huge_pte_mkdirty(vmf->orig_pte);
}
}
- vmf.orig_pte = pte_mkyoung(vmf.orig_pte);
- if (huge_ptep_set_access_flags(vma, vmf.address, vmf.pte, vmf.orig_pte,
- flags & FAULT_FLAG_WRITE))
- update_mmu_cache(vma, vmf.address, vmf.pte);
+ vmf->orig_pte = pte_mkyoung(vmf->orig_pte);
+ if (huge_ptep_set_access_flags(vma, vmf->address, vmf->pte,
+ vmf->orig_pte, vmf->flags & FAULT_FLAG_WRITE))
+ update_mmu_cache(vma, vmf->address, vmf->pte);
out_put_page:
if (folio != pagecache_folio)
folio_unlock(folio);
folio_put(folio);
out_ptl:
- spin_unlock(vmf.ptl);
+ spin_unlock(vmf->ptl);
if (pagecache_folio) {
folio_unlock(pagecache_folio);
diff --git a/mm/memory.c b/mm/memory.c
index c93b058adfb2..a2fcb0322b11 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -5191,39 +5191,35 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
* the result, the mmap_lock is not held on exit. See filemap_fault()
* and __folio_lock_or_retry().
*/
-static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
- unsigned long address, unsigned int flags)
+static vm_fault_t __handle_mm_fault(struct vm_fault *vmf)
{
- struct vm_fault vmf = {
- .vma = vma,
- .address = address & PAGE_MASK,
- .real_address = address,
- .flags = flags,
- .pgoff = linear_page_index(vma, address),
- .gfp_mask = __get_fault_gfp_mask(vma),
- };
+ struct vm_area_struct *vma = vmf->vma;
struct mm_struct *mm = vma->vm_mm;
unsigned long vm_flags = vma->vm_flags;
+ const unsigned long address = vmf->real_address;
pgd_t *pgd;
p4d_t *p4d;
vm_fault_t ret;
+ vmf->address = address & PAGE_MASK;
+ vmf->pgoff = linear_page_index(vma, address);
pgd = pgd_offset(mm, address);
p4d = p4d_alloc(mm, pgd, address);
if (!p4d)
return VM_FAULT_OOM;
- vmf.pud = pud_alloc(mm, p4d, address);
- if (!vmf.pud)
+ vmf->pud = pud_alloc(mm, p4d, address);
+ if (!vmf->pud)
return VM_FAULT_OOM;
retry_pud:
- if (pud_none(*vmf.pud) &&
- thp_vma_allowable_order(vma, vm_flags, false, true, true, PUD_ORDER)) {
- ret = create_huge_pud(&vmf);
+ if (pud_none(*vmf->pud) &&
+ thp_vma_allowable_order(vma, vm_flags, false, true,
+ true, PUD_ORDER)) {
+ ret = create_huge_pud(vmf);
if (!(ret & VM_FAULT_FALLBACK))
return ret;
} else {
- pud_t orig_pud = *vmf.pud;
+ pud_t orig_pud = *vmf->pud;
barrier();
if (pud_trans_huge(orig_pud) || pud_devmap(orig_pud)) {
@@ -5232,57 +5228,60 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
* TODO once we support anonymous PUDs: NUMA case and
* FAULT_FLAG_UNSHARE handling.
*/
- if ((flags & FAULT_FLAG_WRITE) && !pud_write(orig_pud)) {
- ret = wp_huge_pud(&vmf, orig_pud);
+ if ((vmf->flags & FAULT_FLAG_WRITE) &&
+ !pud_write(orig_pud)) {
+ ret = wp_huge_pud(vmf, orig_pud);
if (!(ret & VM_FAULT_FALLBACK))
return ret;
} else {
- huge_pud_set_accessed(&vmf, orig_pud);
+ huge_pud_set_accessed(vmf, orig_pud);
return 0;
}
}
}
- vmf.pmd = pmd_alloc(mm, vmf.pud, address);
- if (!vmf.pmd)
+ vmf->pmd = pmd_alloc(mm, vmf->pud, address);
+ if (!vmf->pmd)
return VM_FAULT_OOM;
/* Huge pud page fault raced with pmd_alloc? */
- if (pud_trans_unstable(vmf.pud))
+ if (pud_trans_unstable(vmf->pud))
goto retry_pud;
- if (pmd_none(*vmf.pmd) &&
- thp_vma_allowable_order(vma, vm_flags, false, true, true, PMD_ORDER)) {
- ret = create_huge_pmd(&vmf);
+ if (pmd_none(*vmf->pmd) &&
+ thp_vma_allowable_order(vma, vm_flags, false, true,
+ true, PMD_ORDER)) {
+ ret = create_huge_pmd(vmf);
if (!(ret & VM_FAULT_FALLBACK))
return ret;
} else {
- vmf.orig_pmd = pmdp_get_lockless(vmf.pmd);
+ vmf->orig_pmd = pmdp_get_lockless(vmf->pmd);
- if (unlikely(is_swap_pmd(vmf.orig_pmd))) {
+ if (unlikely(is_swap_pmd(vmf->orig_pmd))) {
VM_BUG_ON(thp_migration_supported() &&
- !is_pmd_migration_entry(vmf.orig_pmd));
- if (is_pmd_migration_entry(vmf.orig_pmd))
- pmd_migration_entry_wait(mm, vmf.pmd);
+ !is_pmd_migration_entry(vmf->orig_pmd));
+ if (is_pmd_migration_entry(vmf->orig_pmd))
+ pmd_migration_entry_wait(mm, vmf->pmd);
return 0;
}
- if (pmd_trans_huge(vmf.orig_pmd) || pmd_devmap(vmf.orig_pmd)) {
- if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma))
- return do_huge_pmd_numa_page(&vmf);
-
- if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
- !pmd_write(vmf.orig_pmd)) {
- ret = wp_huge_pmd(&vmf);
+ if (pmd_trans_huge(vmf->orig_pmd) ||
+ pmd_devmap(vmf->orig_pmd)) {
+ if (pmd_protnone(vmf->orig_pmd) && vma_is_accessible(vma))
+ return do_huge_pmd_numa_page(vmf);
+
+ if ((vmf->flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE))
+ && !pmd_write(vmf->orig_pmd)) {
+ ret = wp_huge_pmd(vmf);
if (!(ret & VM_FAULT_FALLBACK))
return ret;
} else {
- huge_pmd_set_accessed(&vmf);
+ huge_pmd_set_accessed(vmf);
return 0;
}
}
}
- return handle_pte_fault(&vmf);
+ return handle_pte_fault(vmf);
}
/**
@@ -5421,6 +5420,12 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
/* If the fault handler drops the mmap_lock, vma may be freed */
struct mm_struct *mm = vma->vm_mm;
vm_fault_t ret;
+ struct vm_fault vmf = {
+ .vma = vma,
+ .real_address = address,
+ .flags = flags,
+ .gfp_mask = __get_fault_gfp_mask(vma),
+ };
__set_current_state(TASK_RUNNING);
@@ -5445,9 +5450,9 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
lru_gen_enter_fault(vma);
if (unlikely(is_vm_hugetlb_page(vma)))
- ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
+ ret = hugetlb_fault(&vmf);
else
- ret = __handle_mm_fault(vma, address, flags);
+ ret = __handle_mm_fault(&vmf);
lru_gen_exit_fault();
--
2.43.0
^ permalink raw reply [flat|nested] 8+ messages in thread