mm/filemap.c | 81 +++++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 28 deletions(-) diff --git a/mm/filemap.c b/mm/filemap.c index 13c5de94c884..64def0dd3b97 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2697,7 +2697,41 @@ static void filemap_end_dropbehind_read(struct folio *folio) } } -static bool filemap_read_fast(struct kiocb *iocb, struct iov_iter *iter, +static size_t inner_read_loop(struct kiocb *iocb, struct folio *folio, + void __user *dst, size_t dst_size, + char *buffer, size_t buffer_size, + struct address_space *mapping, unsigned int seq) +{ + size_t read = 0; + + if (can_do_masked_user_access()) + dst = masked_user_access_begin(dst); + else if (!user_access_begin(dst, dst_size)) + return 0; + + do { + size_t to_read = min(dst_size, buffer_size); + + to_read = memcpy_from_file_folio(buffer, folio, iocb->ki_pos, to_read); + + /* Give up and go to slow path if raced with page_cache_delete() */ + if (read_seqcount_retry(&mapping->i_pages_delete_seqcnt, seq)) + break; + + unsafe_copy_to_user(dst, buffer, to_read, Efault); + + dst += read; + dst_size -= read; + + iocb->ki_pos += read; + } while (dst_size && iocb->ki_pos % folio_size(folio)); + +Efault: + user_access_end(); + return read; +} + +static bool noinline filemap_read_fast(struct kiocb *iocb, struct iov_iter *iter, char *buffer, size_t buffer_size, ssize_t *already_read) { @@ -2719,14 +2753,12 @@ static bool filemap_read_fast(struct kiocb *iocb, struct iov_iter *iter, if (!raw_seqcount_try_begin(&mapping->i_pages_delete_seqcnt, seq)) return false; - if (!user_access_begin(iter->ubuf + iter->iov_offset, iter->count)) - return false; - rcu_read_lock(); pagefault_disable(); do { size_t to_read, read; + void __user *dst; XA_STATE(xas, &mapping->i_pages, iocb->ki_pos >> PAGE_SHIFT); xas_reset(&xas); @@ -2750,34 +2782,27 @@ static bool filemap_read_fast(struct kiocb *iocb, struct iov_iter *iter, /* i_size check must be after folio_test_uptodate() */ file_size = i_size_read(mapping->host); - do { - if (unlikely(iocb->ki_pos >= file_size)) - goto out; + if (unlikely(iocb->ki_pos >= file_size)) + break; + file_size -= iocb->ki_pos; + to_read = iov_iter_count(iter); + if (to_read > file_size) + to_read = file_size; - to_read = min(iov_iter_count(iter), buffer_size); - if (to_read > file_size - iocb->ki_pos) - to_read = file_size - iocb->ki_pos; - - read = memcpy_from_file_folio(buffer, folio, iocb->ki_pos, to_read); - - /* Give up and go to slow path if raced with page_cache_delete() */ - if (read_seqcount_retry(&mapping->i_pages_delete_seqcnt, seq)) - goto out; - - unsafe_copy_to_user(iter->ubuf + iter->iov_offset, buffer, - read, out); - - iter->iov_offset += read; - iter->count -= read; - *already_read += read; - iocb->ki_pos += read; - last_pos = iocb->ki_pos; - } while (iov_iter_count(iter) && iocb->ki_pos % folio_size(folio)); + dst = iter->ubuf + iter->iov_offset; + read = inner_read_loop(iocb, folio, + dst, to_read, buffer, buffer_size, + mapping, seq); + if (!read) + break; + iter->iov_offset += read; + iter->count -= read; + *already_read += read; + last_pos = iocb->ki_pos; } while (iov_iter_count(iter)); -out: + pagefault_enable(); rcu_read_unlock(); - user_access_end(); file_accessed(iocb->ki_filp); ra->prev_pos = last_pos;