[PATCH] mm/gup: Remove the restriction on locked with FOLL_LONGTERM

[PATCH] mm/gup: Remove the restriction on locked with FOLL_LONGTERM

[Jason Gunthorpe]

This restriction was created because FOLL_LONGTERM used to scan the vma
list, so it could not tolerate becoming unlocked. That was fixed in commit
52650c8b466b ("mm/gup: remove the vma allocation from
gup_longterm_locked()") and the restriction on !vma was removed.

However, the locked restriction remained, even though it isn't necessary
anymore.

Adjust __gup_longterm_locked() so it can handle the mmap_read_lock()
becoming unlocked while it is looping for migration. Migration does not
require the mmap_read_sem because it is only handling struct pages.  If we
had to unlock then ensure the whole thing returns unlocked.

Remove __get_user_pages_remote() and __gup_longterm_unlocked(). These
cases can now just directly call other functions.

Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
---
 mm/gup.c | 109 ++++++++++++++-----------------------------------------
 1 file changed, 27 insertions(+), 82 deletions(-)

diff --git a/mm/gup.c b/mm/gup.c
index fe195d47de74a7..91251291698a08 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -2105,14 +2105,19 @@ static long __gup_longterm_locked(struct mm_struct *mm,
 				  unsigned long nr_pages,
 				  struct page **pages,
 				  struct vm_area_struct **vmas,
+				  int *locked,
 				  unsigned int gup_flags)
 {
+	bool must_unlock = false;
 	unsigned int flags;
 	long rc, nr_pinned_pages;
 
+	if (locked && WARN_ON_ONCE(!*locked))
+		return -EINVAL;
+
 	if (!(gup_flags & FOLL_LONGTERM))
 		return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
-					       NULL, gup_flags);
+					       locked, gup_flags);
 
 	/*
 	 * If we get to this point then FOLL_LONGTERM is set, and FOLL_LONGTERM
@@ -2126,8 +2131,13 @@ static long __gup_longterm_locked(struct mm_struct *mm,
 		return -EINVAL;
 	flags = memalloc_pin_save();
 	do {
+		if (locked && !*locked) {
+			mmap_read_lock(mm);
+			must_unlock = true;
+			*locked = 1;
+		}
 		nr_pinned_pages = __get_user_pages_locked(mm, start, nr_pages,
-							  pages, vmas, NULL,
+							  pages, vmas, locked,
 							  gup_flags);
 		if (nr_pinned_pages <= 0) {
 			rc = nr_pinned_pages;
@@ -2137,6 +2147,10 @@ static long __gup_longterm_locked(struct mm_struct *mm,
 	} while (rc == -EAGAIN);
 	memalloc_pin_restore(flags);
 
+	if (locked && *locked && must_unlock) {
+		mmap_read_unlock(mm);
+		*locked = 0;
+	}
 	return rc ? rc : nr_pinned_pages;
 }
 
@@ -2160,35 +2174,6 @@ static bool is_valid_gup_flags(unsigned int gup_flags)
 }
 
 #ifdef CONFIG_MMU
-static long __get_user_pages_remote(struct mm_struct *mm,
-				    unsigned long start, unsigned long nr_pages,
-				    unsigned int gup_flags, struct page **pages,
-				    struct vm_area_struct **vmas, int *locked)
-{
-	/*
-	 * Parts of FOLL_LONGTERM behavior are incompatible with
-	 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
-	 * vmas. However, this only comes up if locked is set, and there are
-	 * callers that do request FOLL_LONGTERM, but do not set locked. So,
-	 * allow what we can.
-	 */
-	if (gup_flags & FOLL_LONGTERM) {
-		if (WARN_ON_ONCE(locked))
-			return -EINVAL;
-		/*
-		 * This will check the vmas (even if our vmas arg is NULL)
-		 * and return -ENOTSUPP if DAX isn't allowed in this case:
-		 */
-		return __gup_longterm_locked(mm, start, nr_pages, pages,
-					     vmas, gup_flags | FOLL_TOUCH |
-					     FOLL_REMOTE);
-	}
-
-	return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
-				       locked,
-				       gup_flags | FOLL_TOUCH | FOLL_REMOTE);
-}
-
 /**
  * get_user_pages_remote() - pin user pages in memory
  * @mm:		mm_struct of target mm
@@ -2257,8 +2242,8 @@ long get_user_pages_remote(struct mm_struct *mm,
 	if (!is_valid_gup_flags(gup_flags))
 		return -EINVAL;
 
-	return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
-				       pages, vmas, locked);
+	return __gup_longterm_locked(mm, start, nr_pages, pages, vmas, locked,
+				     gup_flags | FOLL_TOUCH | FOLL_REMOTE);
 }
 EXPORT_SYMBOL(get_user_pages_remote);
 
@@ -2270,14 +2255,6 @@ long get_user_pages_remote(struct mm_struct *mm,
 {
 	return 0;
 }
-
-static long __get_user_pages_remote(struct mm_struct *mm,
-				    unsigned long start, unsigned long nr_pages,
-				    unsigned int gup_flags, struct page **pages,
-				    struct vm_area_struct **vmas, int *locked)
-{
-	return 0;
-}
 #endif /* !CONFIG_MMU */
 
 /**
@@ -2304,7 +2281,7 @@ long get_user_pages(unsigned long start, unsigned long nr_pages,
 		return -EINVAL;
 
 	return __gup_longterm_locked(current->mm, start, nr_pages,
-				     pages, vmas, gup_flags | FOLL_TOUCH);
+				     pages, vmas, NULL, gup_flags | FOLL_TOUCH);
 }
 EXPORT_SYMBOL(get_user_pages);
 
@@ -2330,18 +2307,9 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
 	int locked = 1;
 	long ret;
 
-	/*
-	 * FIXME: Current FOLL_LONGTERM behavior is incompatible with
-	 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
-	 * vmas.  As there are no users of this flag in this call we simply
-	 * disallow this option for now.
-	 */
-	if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
-		return -EINVAL;
-
 	mmap_read_lock(mm);
-	ret = __get_user_pages_locked(mm, start, nr_pages, pages, NULL,
-				      &locked, gup_flags | FOLL_TOUCH);
+	ret = __gup_longterm_locked(mm, start, nr_pages, pages, NULL, &locked,
+				    gup_flags | FOLL_TOUCH);
 	if (locked)
 		mmap_read_unlock(mm);
 	return ret;
@@ -2935,29 +2903,6 @@ static bool gup_fast_permitted(unsigned long start, unsigned long end)
 }
 #endif
 
-static int __gup_longterm_unlocked(unsigned long start, int nr_pages,
-				   unsigned int gup_flags, struct page **pages)
-{
-	int ret;
-
-	/*
-	 * FIXME: FOLL_LONGTERM does not work with
-	 * get_user_pages_unlocked() (see comments in that function)
-	 */
-	if (gup_flags & FOLL_LONGTERM) {
-		mmap_read_lock(current->mm);
-		ret = __gup_longterm_locked(current->mm,
-					    start, nr_pages,
-					    pages, NULL, gup_flags);
-		mmap_read_unlock(current->mm);
-	} else {
-		ret = get_user_pages_unlocked(start, nr_pages,
-					      pages, gup_flags);
-	}
-
-	return ret;
-}
-
 static unsigned long lockless_pages_from_mm(unsigned long start,
 					    unsigned long end,
 					    unsigned int gup_flags,
@@ -3041,8 +2986,8 @@ static int internal_get_user_pages_fast(unsigned long start,
 	/* Slow path: try to get the remaining pages with get_user_pages */
 	start += nr_pinned << PAGE_SHIFT;
 	pages += nr_pinned;
-	ret = __gup_longterm_unlocked(start, nr_pages - nr_pinned, gup_flags,
-				      pages);
+	ret = get_user_pages_unlocked(start, nr_pages - nr_pinned, pages,
+				      gup_flags);
 	if (ret < 0) {
 		/*
 		 * The caller has to unpin the pages we already pinned so
@@ -3241,9 +3186,9 @@ long pin_user_pages_remote(struct mm_struct *mm,
 	if (WARN_ON_ONCE(!pages))
 		return -EINVAL;
 
-	gup_flags |= FOLL_PIN;
-	return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
-				       pages, vmas, locked);
+	return __gup_longterm_locked(mm, start, nr_pages, pages, vmas, locked,
+				     gup_flags | FOLL_PIN | FOLL_TOUCH |
+					     FOLL_REMOTE);
 }
 EXPORT_SYMBOL(pin_user_pages_remote);
 
@@ -3277,7 +3222,7 @@ long pin_user_pages(unsigned long start, unsigned long nr_pages,
 
 	gup_flags |= FOLL_PIN;
 	return __gup_longterm_locked(current->mm, start, nr_pages,
-				     pages, vmas, gup_flags);
+				     pages, vmas, NULL, gup_flags);
 }
 EXPORT_SYMBOL(pin_user_pages);
 

base-commit: b1a4dd7dab677fa4bfc4967c99ee45ba4d1d05d6
-- 
2.38.1

Re: [PATCH] mm/gup: Remove the restriction on locked with FOLL_LONGTERM

[John Hubbard]

On 11/16/22 12:07, Jason Gunthorpe wrote:
> This restriction was created because FOLL_LONGTERM used to scan the vma
> list, so it could not tolerate becoming unlocked. That was fixed in commit
> 52650c8b466b ("mm/gup: remove the vma allocation from
> gup_longterm_locked()") and the restriction on !vma was removed.
> 
> However, the locked restriction remained, even though it isn't necessary
> anymore.
> 
> Adjust __gup_longterm_locked() so it can handle the mmap_read_lock()
> becoming unlocked while it is looping for migration. Migration does not
> require the mmap_read_sem because it is only handling struct pages.  If we
> had to unlock then ensure the whole thing returns unlocked.
> 
> Remove __get_user_pages_remote() and __gup_longterm_unlocked(). These
> cases can now just directly call other functions.
> 
> Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
> ---
>   mm/gup.c | 109 ++++++++++++++-----------------------------------------
>   1 file changed, 27 insertions(+), 82 deletions(-)

Looks nice.

Even after this cleanup, gup.c is still a bit of a ball of string, what
with things like __gup_longterm_locked() actually handling
!FOLL_LONGTERM, and a bunch of other vestigial stuff, but this takes us
in the right direction.


Reviewed-by: John Hubbard <jhubbard@nvidia.com>

thanks,
-- 
John Hubbard
NVIDIA

> 
> diff --git a/mm/gup.c b/mm/gup.c
> index fe195d47de74a7..91251291698a08 100644
> --- a/mm/gup.c
> +++ b/mm/gup.c
> @@ -2105,14 +2105,19 @@ static long __gup_longterm_locked(struct mm_struct *mm,
>   				  unsigned long nr_pages,
>   				  struct page **pages,
>   				  struct vm_area_struct **vmas,
> +				  int *locked,
>   				  unsigned int gup_flags)
>   {
> +	bool must_unlock = false;
>   	unsigned int flags;
>   	long rc, nr_pinned_pages;
>   
> +	if (locked && WARN_ON_ONCE(!*locked))
> +		return -EINVAL;
> +
>   	if (!(gup_flags & FOLL_LONGTERM))
>   		return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
> -					       NULL, gup_flags);
> +					       locked, gup_flags);
>   
>   	/*
>   	 * If we get to this point then FOLL_LONGTERM is set, and FOLL_LONGTERM
> @@ -2126,8 +2131,13 @@ static long __gup_longterm_locked(struct mm_struct *mm,
>   		return -EINVAL;
>   	flags = memalloc_pin_save();
>   	do {
> +		if (locked && !*locked) {
> +			mmap_read_lock(mm);
> +			must_unlock = true;
> +			*locked = 1;
> +		}
>   		nr_pinned_pages = __get_user_pages_locked(mm, start, nr_pages,
> -							  pages, vmas, NULL,
> +							  pages, vmas, locked,
>   							  gup_flags);
>   		if (nr_pinned_pages <= 0) {
>   			rc = nr_pinned_pages;
> @@ -2137,6 +2147,10 @@ static long __gup_longterm_locked(struct mm_struct *mm,
>   	} while (rc == -EAGAIN);
>   	memalloc_pin_restore(flags);
>   
> +	if (locked && *locked && must_unlock) {
> +		mmap_read_unlock(mm);
> +		*locked = 0;
> +	}
>   	return rc ? rc : nr_pinned_pages;
>   }
>   
> @@ -2160,35 +2174,6 @@ static bool is_valid_gup_flags(unsigned int gup_flags)
>   }
>   
>   #ifdef CONFIG_MMU
> -static long __get_user_pages_remote(struct mm_struct *mm,
> -				    unsigned long start, unsigned long nr_pages,
> -				    unsigned int gup_flags, struct page **pages,
> -				    struct vm_area_struct **vmas, int *locked)
> -{
> -	/*
> -	 * Parts of FOLL_LONGTERM behavior are incompatible with
> -	 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
> -	 * vmas. However, this only comes up if locked is set, and there are
> -	 * callers that do request FOLL_LONGTERM, but do not set locked. So,
> -	 * allow what we can.
> -	 */
> -	if (gup_flags & FOLL_LONGTERM) {
> -		if (WARN_ON_ONCE(locked))
> -			return -EINVAL;
> -		/*
> -		 * This will check the vmas (even if our vmas arg is NULL)
> -		 * and return -ENOTSUPP if DAX isn't allowed in this case:
> -		 */
> -		return __gup_longterm_locked(mm, start, nr_pages, pages,
> -					     vmas, gup_flags | FOLL_TOUCH |
> -					     FOLL_REMOTE);
> -	}
> -
> -	return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
> -				       locked,
> -				       gup_flags | FOLL_TOUCH | FOLL_REMOTE);
> -}
> -
>   /**
>    * get_user_pages_remote() - pin user pages in memory
>    * @mm:		mm_struct of target mm
> @@ -2257,8 +2242,8 @@ long get_user_pages_remote(struct mm_struct *mm,
>   	if (!is_valid_gup_flags(gup_flags))
>   		return -EINVAL;
>   
> -	return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
> -				       pages, vmas, locked);
> +	return __gup_longterm_locked(mm, start, nr_pages, pages, vmas, locked,
> +				     gup_flags | FOLL_TOUCH | FOLL_REMOTE);
>   }
>   EXPORT_SYMBOL(get_user_pages_remote);
>   
> @@ -2270,14 +2255,6 @@ long get_user_pages_remote(struct mm_struct *mm,
>   {
>   	return 0;
>   }
> -
> -static long __get_user_pages_remote(struct mm_struct *mm,
> -				    unsigned long start, unsigned long nr_pages,
> -				    unsigned int gup_flags, struct page **pages,
> -				    struct vm_area_struct **vmas, int *locked)
> -{
> -	return 0;
> -}
>   #endif /* !CONFIG_MMU */
>   
>   /**
> @@ -2304,7 +2281,7 @@ long get_user_pages(unsigned long start, unsigned long nr_pages,
>   		return -EINVAL;
>   
>   	return __gup_longterm_locked(current->mm, start, nr_pages,
> -				     pages, vmas, gup_flags | FOLL_TOUCH);
> +				     pages, vmas, NULL, gup_flags | FOLL_TOUCH);
>   }
>   EXPORT_SYMBOL(get_user_pages);
>   
> @@ -2330,18 +2307,9 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
>   	int locked = 1;
>   	long ret;
>   
> -	/*
> -	 * FIXME: Current FOLL_LONGTERM behavior is incompatible with
> -	 * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
> -	 * vmas.  As there are no users of this flag in this call we simply
> -	 * disallow this option for now.
> -	 */
> -	if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
> -		return -EINVAL;
> -
>   	mmap_read_lock(mm);
> -	ret = __get_user_pages_locked(mm, start, nr_pages, pages, NULL,
> -				      &locked, gup_flags | FOLL_TOUCH);
> +	ret = __gup_longterm_locked(mm, start, nr_pages, pages, NULL, &locked,
> +				    gup_flags | FOLL_TOUCH);
>   	if (locked)
>   		mmap_read_unlock(mm);
>   	return ret;
> @@ -2935,29 +2903,6 @@ static bool gup_fast_permitted(unsigned long start, unsigned long end)
>   }
>   #endif
>   
> -static int __gup_longterm_unlocked(unsigned long start, int nr_pages,
> -				   unsigned int gup_flags, struct page **pages)
> -{
> -	int ret;
> -
> -	/*
> -	 * FIXME: FOLL_LONGTERM does not work with
> -	 * get_user_pages_unlocked() (see comments in that function)
> -	 */
> -	if (gup_flags & FOLL_LONGTERM) {
> -		mmap_read_lock(current->mm);
> -		ret = __gup_longterm_locked(current->mm,
> -					    start, nr_pages,
> -					    pages, NULL, gup_flags);
> -		mmap_read_unlock(current->mm);
> -	} else {
> -		ret = get_user_pages_unlocked(start, nr_pages,
> -					      pages, gup_flags);
> -	}
> -
> -	return ret;
> -}
> -
>   static unsigned long lockless_pages_from_mm(unsigned long start,
>   					    unsigned long end,
>   					    unsigned int gup_flags,
> @@ -3041,8 +2986,8 @@ static int internal_get_user_pages_fast(unsigned long start,
>   	/* Slow path: try to get the remaining pages with get_user_pages */
>   	start += nr_pinned << PAGE_SHIFT;
>   	pages += nr_pinned;
> -	ret = __gup_longterm_unlocked(start, nr_pages - nr_pinned, gup_flags,
> -				      pages);
> +	ret = get_user_pages_unlocked(start, nr_pages - nr_pinned, pages,
> +				      gup_flags);
>   	if (ret < 0) {
>   		/*
>   		 * The caller has to unpin the pages we already pinned so
> @@ -3241,9 +3186,9 @@ long pin_user_pages_remote(struct mm_struct *mm,
>   	if (WARN_ON_ONCE(!pages))
>   		return -EINVAL;
>   
> -	gup_flags |= FOLL_PIN;
> -	return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
> -				       pages, vmas, locked);
> +	return __gup_longterm_locked(mm, start, nr_pages, pages, vmas, locked,
> +				     gup_flags | FOLL_PIN | FOLL_TOUCH |
> +					     FOLL_REMOTE);
>   }
>   EXPORT_SYMBOL(pin_user_pages_remote);
>   
> @@ -3277,7 +3222,7 @@ long pin_user_pages(unsigned long start, unsigned long nr_pages,
>   
>   	gup_flags |= FOLL_PIN;
>   	return __gup_longterm_locked(current->mm, start, nr_pages,
> -				     pages, vmas, gup_flags);
> +				     pages, vmas, NULL, gup_flags);
>   }
>   EXPORT_SYMBOL(pin_user_pages);
>   
> 
> base-commit: b1a4dd7dab677fa4bfc4967c99ee45ba4d1d05d6

Re: [PATCH] mm/gup: Remove the restriction on locked with FOLL_LONGTERM

[Jason Gunthorpe]

On Wed, Nov 16, 2022 at 02:11:04PM -0800, John Hubbard wrote:
> On 11/16/22 12:07, Jason Gunthorpe wrote:
> > This restriction was created because FOLL_LONGTERM used to scan the vma
> > list, so it could not tolerate becoming unlocked. That was fixed in commit
> > 52650c8b466b ("mm/gup: remove the vma allocation from
> > gup_longterm_locked()") and the restriction on !vma was removed.
> > 
> > However, the locked restriction remained, even though it isn't necessary
> > anymore.
> > 
> > Adjust __gup_longterm_locked() so it can handle the mmap_read_lock()
> > becoming unlocked while it is looping for migration. Migration does not
> > require the mmap_read_sem because it is only handling struct pages.  If we
> > had to unlock then ensure the whole thing returns unlocked.
> > 
> > Remove __get_user_pages_remote() and __gup_longterm_unlocked(). These
> > cases can now just directly call other functions.
> > 
> > Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
> > ---
> >   mm/gup.c | 109 ++++++++++++++-----------------------------------------
> >   1 file changed, 27 insertions(+), 82 deletions(-)
> 
> Looks nice.
> 
> Even after this cleanup, gup.c is still a bit of a ball of string, what
> with things like __gup_longterm_locked() actually handling
> !FOLL_LONGTERM, and a bunch of other vestigial stuff, but this takes us
> in the right direction.

What do you think of this:

https://github.com/jgunthorpe/linux/commits/gup

Jason

Re: [PATCH] mm/gup: Remove the restriction on locked with FOLL_LONGTERM

[John Hubbard]

On 11/18/22 12:00, Jason Gunthorpe wrote:
...
>> Even after this cleanup, gup.c is still a bit of a ball of string, what
>> with things like __gup_longterm_locked() actually handling
>> !FOLL_LONGTERM, and a bunch of other vestigial stuff, but this takes us
>> in the right direction.
> 
> What do you think of this:
> 
> https://github.com/jgunthorpe/linux/commits/gup
> 

I like it--this cleans things up a lot!

A couple of tweaks I'd suggest would be:

1) Add an enum for "1" to the new enum, and use it. That will clarify
the call sites in the same way that GUP_GET_LOCK helps.

2) Don't change the name of the is_valid_gup_flags(). Or at least,
leave some for is "is_" in there. Because that makes the call sites
very nice:

     if (!is_valid_gup_flags())
	...bail out, etc


thanks,
-- 
John Hubbard
NVIDIA