[PATCH v2] shmem: support huge_fault to avoid pmd split

[PATCH v2] shmem: support huge_fault to avoid pmd split

[Liu Zixian]

Transparent hugepage of tmpfs is useful to improve TLB miss, but
it will be split during cow memory fault.
This will happen if we mprotect and rewrite code segment (which is
private file map) to hotpatch a running process.

Users of huge= mount option prefer huge pages after cow.
We can avoid the splitting by adding a huge_fault function.

---
v2: removed redundant prep_transhuge_page

Signed-off-by: Liu Zixian <liuzixian4@huawei.com>
---
 mm/shmem.c | 45 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)

diff --git a/mm/shmem.c b/mm/shmem.c
index a6f565308..5074dff08 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -2120,6 +2120,50 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf)
 	return ret;
 }
 
+static vm_fault_t shmem_huge_fault(struct vm_fault *vmf, enum page_entry_size pe_size)
+{
+	vm_fault_t ret = VM_FAULT_FALLBACK;
+	unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
+	struct page *old_page, *new_page;
+	int gfp_flags = GFP_HIGHUSER_MOVABLE | __GFP_COMP;
+
+	/* read or shared fault will not split huge pmd */
+	if (!(vmf->flags & FAULT_FLAG_WRITE)
+			|| (vmf->vma->vm_flags & VM_SHARED))
+		return VM_FAULT_FALLBACK;
+	if (pe_size != PE_SIZE_PMD)
+		return VM_FAULT_FALLBACK;
+
+	if (pmd_none(*vmf->pmd)) {
+		if (shmem_fault(vmf) & VM_FAULT_ERROR)
+			goto out;
+		if (!PageTransHuge(vmf->page))
+			goto out;
+		old_page = vmf->page;
+	} else {
+		old_page = pmd_page(*vmf->pmd);
+		page_remove_rmap(old_page, vmf->vma, true);
+		pmdp_huge_clear_flush(vmf->vma, haddr, vmf->pmd);
+		add_mm_counter(vmf->vma->vm_mm, MM_SHMEMPAGES, -HPAGE_PMD_NR);
+	}
+
+	new_page = &vma_alloc_folio(gfp_flags, HPAGE_PMD_ORDER,
+			vmf->vma, haddr, true)->page;
+	if (!new_page)
+		goto out;
+	copy_user_huge_page(new_page, old_page, haddr, vmf->vma, HPAGE_PMD_NR);
+	__SetPageUptodate(new_page);
+
+	ret = do_set_pmd(vmf, new_page);
+
+out:
+	if (vmf->page) {
+		unlock_page(vmf->page);
+		put_page(vmf->page);
+	}
+	return ret;
+}
+
 unsigned long shmem_get_unmapped_area(struct file *file,
 				      unsigned long uaddr, unsigned long len,
 				      unsigned long pgoff, unsigned long flags)
@@ -3884,6 +3928,7 @@ static const struct super_operations shmem_ops = {
 
 static const struct vm_operations_struct shmem_vm_ops = {
 	.fault		= shmem_fault,
+	.huge_fault	= shmem_huge_fault,
 	.map_pages	= filemap_map_pages,
 #ifdef CONFIG_NUMA
 	.set_policy     = shmem_set_policy,
-- 
2.33.0

Re: [PATCH v2] shmem: support huge_fault to avoid pmd split

[kernel test robot]

Hi Liu,

Thank you for the patch! Yet something to improve:

[auto build test ERROR on akpm-mm/mm-everything]
[also build test ERROR on linus/master v5.19-rc8 next-20220726]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Liu-Zixian/shmem-support-huge_fault-to-avoid-pmd-split/20220726-212946
base:   https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-everything
config: hexagon-randconfig-r045-20220724 (https://download.01.org/0day-ci/archive/20220727/202207270709.lDgcRjnd-lkp@intel.com/config)
compiler: clang version 15.0.0 (https://github.com/llvm/llvm-project 83882606dbd7ffb0bdd3460356202d97705809c8)
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # https://github.com/intel-lab-lkp/linux/commit/40060ce82c97d38c8e2d71f9d0e4ee818596bb14
        git remote add linux-review https://github.com/intel-lab-lkp/linux
        git fetch --no-tags linux-review Liu-Zixian/shmem-support-huge_fault-to-avoid-pmd-split/20220726-212946
        git checkout 40060ce82c97d38c8e2d71f9d0e4ee818596bb14
        # save the config file
        mkdir build_dir && cp config build_dir/.config
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=hexagon SHELL=/bin/bash

If you fix the issue, kindly add following tag where applicable
Reported-by: kernel test robot <lkp@intel.com>

All errors (new ones prefixed by >>):

>> mm/shmem.c:2164:2: error: call to undeclared function 'copy_user_huge_page'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
           copy_user_huge_page(new_page, old_page, haddr, vmf->vma, HPAGE_PMD_NR);
           ^
   mm/shmem.c:2164:2: note: did you mean 'copy_user_highpage'?
   include/linux/highmem.h:307:20: note: 'copy_user_highpage' declared here
   static inline void copy_user_highpage(struct page *to, struct page *from,
                      ^
   1 error generated.


vim +/copy_user_huge_page +2164 mm/shmem.c

  2132	
  2133	static vm_fault_t shmem_huge_fault(struct vm_fault *vmf, enum page_entry_size pe_size)
  2134	{
  2135		vm_fault_t ret = VM_FAULT_FALLBACK;
  2136		unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
  2137		struct page *old_page, *new_page;
  2138		int gfp_flags = GFP_HIGHUSER_MOVABLE | __GFP_COMP;
  2139	
  2140		/* read or shared fault will not split huge pmd */
  2141		if (!(vmf->flags & FAULT_FLAG_WRITE)
  2142				|| (vmf->vma->vm_flags & VM_SHARED))
  2143			return VM_FAULT_FALLBACK;
  2144		if (pe_size != PE_SIZE_PMD)
  2145			return VM_FAULT_FALLBACK;
  2146	
  2147		if (pmd_none(*vmf->pmd)) {
  2148			if (shmem_fault(vmf) & VM_FAULT_ERROR)
  2149				goto out;
  2150			if (!PageTransHuge(vmf->page))
  2151				goto out;
  2152			old_page = vmf->page;
  2153		} else {
  2154			old_page = pmd_page(*vmf->pmd);
  2155			page_remove_rmap(old_page, vmf->vma, true);
  2156			pmdp_huge_clear_flush(vmf->vma, haddr, vmf->pmd);
  2157			add_mm_counter(vmf->vma->vm_mm, MM_SHMEMPAGES, -HPAGE_PMD_NR);
  2158		}
  2159	
  2160		new_page = &vma_alloc_folio(gfp_flags, HPAGE_PMD_ORDER,
  2161				vmf->vma, haddr, true)->page;
  2162		if (!new_page)
  2163			goto out;
> 2164		copy_user_huge_page(new_page, old_page, haddr, vmf->vma, HPAGE_PMD_NR);
  2165		__SetPageUptodate(new_page);
  2166	
  2167		ret = do_set_pmd(vmf, new_page);
  2168	
  2169	out:
  2170		if (vmf->page) {
  2171			unlock_page(vmf->page);
  2172			put_page(vmf->page);
  2173		}
  2174		return ret;
  2175	}
  2176	

-- 
0-DAY CI Kernel Test Service
https://01.org/lkp

Re: [PATCH v2] shmem: support huge_fault to avoid pmd split

[kernel test robot]

Hi Liu,

Thank you for the patch! Yet something to improve:

[auto build test ERROR on akpm-mm/mm-everything]
[also build test ERROR on linus/master v5.19-rc8 next-20220727]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Liu-Zixian/shmem-support-huge_fault-to-avoid-pmd-split/20220726-212946
base:   https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-everything
config: i386-randconfig-a004 (https://download.01.org/0day-ci/archive/20220728/202207281337.AGofErxA-lkp@intel.com/config)
compiler: clang version 15.0.0 (https://github.com/llvm/llvm-project 8dfaecc4c24494337933aff9d9166486ca0949f1)
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # https://github.com/intel-lab-lkp/linux/commit/40060ce82c97d38c8e2d71f9d0e4ee818596bb14
        git remote add linux-review https://github.com/intel-lab-lkp/linux
        git fetch --no-tags linux-review Liu-Zixian/shmem-support-huge_fault-to-avoid-pmd-split/20220726-212946
        git checkout 40060ce82c97d38c8e2d71f9d0e4ee818596bb14
        # save the config file
        mkdir build_dir && cp config build_dir/.config
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=i386 SHELL=/bin/bash

If you fix the issue, kindly add following tag where applicable
Reported-by: kernel test robot <lkp@intel.com>

All errors (new ones prefixed by >>):

>> mm/shmem.c:2136:39: error: call to __compiletime_assert_272 declared with 'error' attribute: BUILD_BUG failed
           unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
                                                ^
   include/linux/huge_mm.h:307:27: note: expanded from macro 'HPAGE_PMD_MASK'
   #define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })
                             ^
   include/linux/build_bug.h:59:21: note: expanded from macro 'BUILD_BUG'
   #define BUILD_BUG() BUILD_BUG_ON_MSG(1, "BUILD_BUG failed")
                       ^
   include/linux/build_bug.h:39:37: note: expanded from macro 'BUILD_BUG_ON_MSG'
   #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
                                       ^
   note: (skipping 1 expansions in backtrace; use -fmacro-backtrace-limit=0 to see all)
   include/linux/compiler_types.h:340:2: note: expanded from macro '_compiletime_assert'
           __compiletime_assert(condition, msg, prefix, suffix)
           ^
   include/linux/compiler_types.h:333:4: note: expanded from macro '__compiletime_assert'
                           prefix ## suffix();                             \
                           ^
   <scratch space>:41:1: note: expanded from here
   __compiletime_assert_272
   ^
   1 error generated.


vim +/error +2136 mm/shmem.c

  2132	
  2133	static vm_fault_t shmem_huge_fault(struct vm_fault *vmf, enum page_entry_size pe_size)
  2134	{
  2135		vm_fault_t ret = VM_FAULT_FALLBACK;
> 2136		unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
  2137		struct page *old_page, *new_page;
  2138		int gfp_flags = GFP_HIGHUSER_MOVABLE | __GFP_COMP;
  2139	
  2140		/* read or shared fault will not split huge pmd */
  2141		if (!(vmf->flags & FAULT_FLAG_WRITE)
  2142				|| (vmf->vma->vm_flags & VM_SHARED))
  2143			return VM_FAULT_FALLBACK;
  2144		if (pe_size != PE_SIZE_PMD)
  2145			return VM_FAULT_FALLBACK;
  2146	
  2147		if (pmd_none(*vmf->pmd)) {
  2148			if (shmem_fault(vmf) & VM_FAULT_ERROR)
  2149				goto out;
  2150			if (!PageTransHuge(vmf->page))
  2151				goto out;
  2152			old_page = vmf->page;
  2153		} else {
  2154			old_page = pmd_page(*vmf->pmd);
  2155			page_remove_rmap(old_page, vmf->vma, true);
  2156			pmdp_huge_clear_flush(vmf->vma, haddr, vmf->pmd);
  2157			add_mm_counter(vmf->vma->vm_mm, MM_SHMEMPAGES, -HPAGE_PMD_NR);
  2158		}
  2159	
  2160		new_page = &vma_alloc_folio(gfp_flags, HPAGE_PMD_ORDER,
  2161				vmf->vma, haddr, true)->page;
  2162		if (!new_page)
  2163			goto out;
  2164		copy_user_huge_page(new_page, old_page, haddr, vmf->vma, HPAGE_PMD_NR);
  2165		__SetPageUptodate(new_page);
  2166	
  2167		ret = do_set_pmd(vmf, new_page);
  2168	
  2169	out:
  2170		if (vmf->page) {
  2171			unlock_page(vmf->page);
  2172			put_page(vmf->page);
  2173		}
  2174		return ret;
  2175	}
  2176	

-- 
0-DAY CI Kernel Test Service
https://01.org/lkp