* [PATCH mm-unstable v3 01/10] mm: add folio dtor and order setter functions
2022-11-17 21:14 [PATCH mm-unstable v3 00/10] convert core hugetlb functions to folios Sidhartha Kumar
@ 2022-11-17 21:14 ` Sidhartha Kumar
2022-11-17 21:14 ` [PATCH mm-unstable v3 02/10] mm/hugetlb: convert destroy_compound_gigantic_page() to folios Sidhartha Kumar
` (8 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Sidhartha Kumar @ 2022-11-17 21:14 UTC (permalink / raw)
To: linux-kernel, linux-mm
Cc: akpm, songmuchun, mike.kravetz, willy, almasrymina, linmiaohe,
hughd, Sidhartha Kumar
Add folio equivalents for set_compound_order() and set_compound_page_dtor().
Also remove extra new-lines introduced by mm/hugetlb: convert
move_hugetlb_state() to folios and mm/hugetlb_cgroup: convert
hugetlb_cgroup_uncharge_page() to folios.
Suggested-by: Mike Kravetz <mike.kravetz@oracle.com>
Suggested-by: Muchun Song <songmuchun@bytedance.com>
Signed-off-by: Sidhartha Kumar <sidhartha.kumar@oracle.com>
---
include/linux/mm.h | 16 ++++++++++++++++
mm/hugetlb.c | 4 +---
2 files changed, 17 insertions(+), 3 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index a48c5ad16a5e..2bdef8a5298a 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -972,6 +972,13 @@ static inline void set_compound_page_dtor(struct page *page,
page[1].compound_dtor = compound_dtor;
}
+static inline void folio_set_compound_dtor(struct folio *folio,
+ enum compound_dtor_id compound_dtor)
+{
+ VM_BUG_ON_FOLIO(compound_dtor >= NR_COMPOUND_DTORS, folio);
+ folio->_folio_dtor = compound_dtor;
+}
+
void destroy_large_folio(struct folio *folio);
static inline int head_compound_pincount(struct page *head)
@@ -987,6 +994,15 @@ static inline void set_compound_order(struct page *page, unsigned int order)
#endif
}
+static inline void folio_set_compound_order(struct folio *folio,
+ unsigned int order)
+{
+ folio->_folio_order = order;
+#ifdef CONFIG_64BIT
+ folio->_folio_nr_pages = order ? 1U << order : 0;
+#endif
+}
+
/* Returns the number of pages in this potentially compound page. */
static inline unsigned long compound_nr(struct page *page)
{
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 002337cc27b5..157f2392c64f 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -1780,7 +1780,7 @@ static void __prep_new_hugetlb_folio(struct hstate *h, struct folio *folio)
{
hugetlb_vmemmap_optimize(h, &folio->page);
INIT_LIST_HEAD(&folio->lru);
- folio->_folio_dtor = HUGETLB_PAGE_DTOR;
+ folio_set_compound_dtor(folio, HUGETLB_PAGE_DTOR);
hugetlb_set_folio_subpool(folio, NULL);
set_hugetlb_cgroup(folio, NULL);
set_hugetlb_cgroup_rsvd(folio, NULL);
@@ -2936,7 +2936,6 @@ struct page *alloc_huge_page(struct vm_area_struct *vma,
* a reservation exists for the allocation.
*/
page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg);
-
if (!page) {
spin_unlock_irq(&hugetlb_lock);
page = alloc_buddy_huge_page_with_mpol(h, vma, addr);
@@ -7349,7 +7348,6 @@ void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int re
int old_nid = folio_nid(old_folio);
int new_nid = folio_nid(new_folio);
-
folio_set_hugetlb_temporary(old_folio);
folio_clear_hugetlb_temporary(new_folio);
--
2.38.1
^ permalink raw reply [flat|nested] 13+ messages in thread* [PATCH mm-unstable v3 02/10] mm/hugetlb: convert destroy_compound_gigantic_page() to folios
2022-11-17 21:14 [PATCH mm-unstable v3 00/10] convert core hugetlb functions to folios Sidhartha Kumar
2022-11-17 21:14 ` [PATCH mm-unstable v3 01/10] mm: add folio dtor and order setter functions Sidhartha Kumar
@ 2022-11-17 21:14 ` Sidhartha Kumar
2022-11-17 21:14 ` [PATCH mm-unstable v3 03/10] mm/hugetlb: convert dissolve_free_huge_page() " Sidhartha Kumar
` (7 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Sidhartha Kumar @ 2022-11-17 21:14 UTC (permalink / raw)
To: linux-kernel, linux-mm
Cc: akpm, songmuchun, mike.kravetz, willy, almasrymina, linmiaohe,
hughd, Sidhartha Kumar
Convert page operations within __destroy_compound_gigantic_page() to the
corresponding folio operations.
Signed-off-by: Sidhartha Kumar <sidhartha.kumar@oracle.com>
---
mm/hugetlb.c | 43 +++++++++++++++++++++----------------------
1 file changed, 21 insertions(+), 22 deletions(-)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 157f2392c64f..5edb81541ede 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -1325,43 +1325,40 @@ static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
nr_nodes--)
/* used to demote non-gigantic_huge pages as well */
-static void __destroy_compound_gigantic_page(struct page *page,
+static void __destroy_compound_gigantic_folio(struct folio *folio,
unsigned int order, bool demote)
{
int i;
int nr_pages = 1 << order;
struct page *p;
- atomic_set(compound_mapcount_ptr(page), 0);
- atomic_set(subpages_mapcount_ptr(page), 0);
- atomic_set(compound_pincount_ptr(page), 0);
+ atomic_set(folio_mapcount_ptr(folio), 0);
+ atomic_set(folio_subpages_mapcount_ptr(folio), 0);
+ atomic_set(folio_pincount_ptr(folio), 0);
for (i = 1; i < nr_pages; i++) {
- p = nth_page(page, i);
+ p = folio_page(folio, i);
p->mapping = NULL;
clear_compound_head(p);
if (!demote)
set_page_refcounted(p);
}
- set_compound_order(page, 0);
-#ifdef CONFIG_64BIT
- page[1].compound_nr = 0;
-#endif
- __ClearPageHead(page);
+ folio_set_compound_order(folio, 0);
+ folio_clear_head(folio);
}
-static void destroy_compound_hugetlb_page_for_demote(struct page *page,
+static void destroy_compound_hugetlb_folio_for_demote(struct folio *folio,
unsigned int order)
{
- __destroy_compound_gigantic_page(page, order, true);
+ __destroy_compound_gigantic_folio(folio, order, true);
}
#ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE
-static void destroy_compound_gigantic_page(struct page *page,
+static void destroy_compound_gigantic_folio(struct folio *folio,
unsigned int order)
{
- __destroy_compound_gigantic_page(page, order, false);
+ __destroy_compound_gigantic_folio(folio, order, false);
}
static void free_gigantic_page(struct page *page, unsigned int order)
@@ -1430,7 +1427,7 @@ static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
return NULL;
}
static inline void free_gigantic_page(struct page *page, unsigned int order) { }
-static inline void destroy_compound_gigantic_page(struct page *page,
+static inline void destroy_compound_gigantic_folio(struct folio *folio,
unsigned int order) { }
#endif
@@ -1477,8 +1474,8 @@ static void __remove_hugetlb_page(struct hstate *h, struct page *page,
*
* For gigantic pages set the destructor to the null dtor. This
* destructor will never be called. Before freeing the gigantic
- * page destroy_compound_gigantic_page will turn the compound page
- * into a simple group of pages. After this the destructor does not
+ * page destroy_compound_gigantic_folio will turn the folio into a
+ * simple group of pages. After this the destructor does not
* apply.
*
* This handles the case where more than one ref is held when and
@@ -1559,6 +1556,7 @@ static void add_hugetlb_page(struct hstate *h, struct page *page,
static void __update_and_free_page(struct hstate *h, struct page *page)
{
int i;
+ struct folio *folio = page_folio(page);
struct page *subpage;
if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
@@ -1587,8 +1585,8 @@ static void __update_and_free_page(struct hstate *h, struct page *page)
* Move PageHWPoison flag from head page to the raw error pages,
* which makes any healthy subpages reusable.
*/
- if (unlikely(PageHWPoison(page)))
- hugetlb_clear_page_hwpoison(page);
+ if (unlikely(folio_test_hwpoison(folio)))
+ hugetlb_clear_page_hwpoison(&folio->page);
for (i = 0; i < pages_per_huge_page(h); i++) {
subpage = nth_page(page, i);
@@ -1604,7 +1602,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)
*/
if (hstate_is_gigantic(h) ||
hugetlb_cma_page(page, huge_page_order(h))) {
- destroy_compound_gigantic_page(page, huge_page_order(h));
+ destroy_compound_gigantic_folio(folio, huge_page_order(h));
free_gigantic_page(page, huge_page_order(h));
} else {
__free_pages(page, huge_page_order(h));
@@ -3435,6 +3433,7 @@ static int demote_free_huge_page(struct hstate *h, struct page *page)
{
int i, nid = page_to_nid(page);
struct hstate *target_hstate;
+ struct folio *folio = page_folio(page);
struct page *subpage;
int rc = 0;
@@ -3453,10 +3452,10 @@ static int demote_free_huge_page(struct hstate *h, struct page *page)
}
/*
- * Use destroy_compound_hugetlb_page_for_demote for all huge page
+ * Use destroy_compound_hugetlb_folio_for_demote for all huge page
* sizes as it will not ref count pages.
*/
- destroy_compound_hugetlb_page_for_demote(page, huge_page_order(h));
+ destroy_compound_hugetlb_folio_for_demote(folio, huge_page_order(h));
/*
* Taking target hstate mutex synchronizes with set_max_huge_pages.
--
2.38.1
^ permalink raw reply [flat|nested] 13+ messages in thread* [PATCH mm-unstable v3 03/10] mm/hugetlb: convert dissolve_free_huge_page() to folios
2022-11-17 21:14 [PATCH mm-unstable v3 00/10] convert core hugetlb functions to folios Sidhartha Kumar
2022-11-17 21:14 ` [PATCH mm-unstable v3 01/10] mm: add folio dtor and order setter functions Sidhartha Kumar
2022-11-17 21:14 ` [PATCH mm-unstable v3 02/10] mm/hugetlb: convert destroy_compound_gigantic_page() to folios Sidhartha Kumar
@ 2022-11-17 21:14 ` Sidhartha Kumar
2022-11-17 21:14 ` [PATCH mm-unstable v3 04/10] mm/hugetlb: convert remove_hugetlb_page() " Sidhartha Kumar
` (6 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Sidhartha Kumar @ 2022-11-17 21:14 UTC (permalink / raw)
To: linux-kernel, linux-mm
Cc: akpm, songmuchun, mike.kravetz, willy, almasrymina, linmiaohe,
hughd, Sidhartha Kumar
Removes compound_head() call by using a folio rather than a head page.
Signed-off-by: Sidhartha Kumar <sidhartha.kumar@oracle.com>
---
mm/hugetlb.c | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 5edb81541ede..ff609efe5497 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -2126,21 +2126,21 @@ static struct page *remove_pool_huge_page(struct hstate *h,
int dissolve_free_huge_page(struct page *page)
{
int rc = -EBUSY;
+ struct folio *folio = page_folio(page);
retry:
/* Not to disrupt normal path by vainly holding hugetlb_lock */
- if (!PageHuge(page))
+ if (!folio_test_hugetlb(folio))
return 0;
spin_lock_irq(&hugetlb_lock);
- if (!PageHuge(page)) {
+ if (!folio_test_hugetlb(folio)) {
rc = 0;
goto out;
}
- if (!page_count(page)) {
- struct page *head = compound_head(page);
- struct hstate *h = page_hstate(head);
+ if (!folio_ref_count(folio)) {
+ struct hstate *h = folio_hstate(folio);
if (!available_huge_pages(h))
goto out;
@@ -2148,7 +2148,7 @@ int dissolve_free_huge_page(struct page *page)
* We should make sure that the page is already on the free list
* when it is dissolved.
*/
- if (unlikely(!HPageFreed(head))) {
+ if (unlikely(!folio_test_hugetlb_freed(folio))) {
spin_unlock_irq(&hugetlb_lock);
cond_resched();
@@ -2163,7 +2163,7 @@ int dissolve_free_huge_page(struct page *page)
goto retry;
}
- remove_hugetlb_page(h, head, false);
+ remove_hugetlb_page(h, &folio->page, false);
h->max_huge_pages--;
spin_unlock_irq(&hugetlb_lock);
@@ -2175,12 +2175,12 @@ int dissolve_free_huge_page(struct page *page)
* Attempt to allocate vmemmmap here so that we can take
* appropriate action on failure.
*/
- rc = hugetlb_vmemmap_restore(h, head);
+ rc = hugetlb_vmemmap_restore(h, &folio->page);
if (!rc) {
- update_and_free_page(h, head, false);
+ update_and_free_page(h, &folio->page, false);
} else {
spin_lock_irq(&hugetlb_lock);
- add_hugetlb_page(h, head, false);
+ add_hugetlb_page(h, &folio->page, false);
h->max_huge_pages++;
spin_unlock_irq(&hugetlb_lock);
}
--
2.38.1
^ permalink raw reply [flat|nested] 13+ messages in thread* [PATCH mm-unstable v3 04/10] mm/hugetlb: convert remove_hugetlb_page() to folios
2022-11-17 21:14 [PATCH mm-unstable v3 00/10] convert core hugetlb functions to folios Sidhartha Kumar
` (2 preceding siblings ...)
2022-11-17 21:14 ` [PATCH mm-unstable v3 03/10] mm/hugetlb: convert dissolve_free_huge_page() " Sidhartha Kumar
@ 2022-11-17 21:14 ` Sidhartha Kumar
2022-11-17 21:14 ` [PATCH mm-unstable v3 05/10] mm/hugetlb: convert update_and_free_page() " Sidhartha Kumar
` (5 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Sidhartha Kumar @ 2022-11-17 21:14 UTC (permalink / raw)
To: linux-kernel, linux-mm
Cc: akpm, songmuchun, mike.kravetz, willy, almasrymina, linmiaohe,
hughd, Sidhartha Kumar
Removes page_folio() call by converting callers to directly pass a folio
into __remove_hugetlb_page().
Signed-off-by: Sidhartha Kumar <sidhartha.kumar@oracle.com>
---
mm/hugetlb.c | 48 +++++++++++++++++++++++++-----------------------
1 file changed, 25 insertions(+), 23 deletions(-)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index ff609efe5497..d9604c0dac54 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -1432,19 +1432,18 @@ static inline void destroy_compound_gigantic_folio(struct folio *folio,
#endif
/*
- * Remove hugetlb page from lists, and update dtor so that page appears
+ * Remove hugetlb folio from lists, and update dtor so that the folio appears
* as just a compound page.
*
- * A reference is held on the page, except in the case of demote.
+ * A reference is held on the folio, except in the case of demote.
*
* Must be called with hugetlb lock held.
*/
-static void __remove_hugetlb_page(struct hstate *h, struct page *page,
+static void __remove_hugetlb_folio(struct hstate *h, struct folio *folio,
bool adjust_surplus,
bool demote)
{
- int nid = page_to_nid(page);
- struct folio *folio = page_folio(page);
+ int nid = folio_nid(folio);
VM_BUG_ON_FOLIO(hugetlb_cgroup_from_folio(folio), folio);
VM_BUG_ON_FOLIO(hugetlb_cgroup_from_folio_rsvd(folio), folio);
@@ -1453,9 +1452,9 @@ static void __remove_hugetlb_page(struct hstate *h, struct page *page,
if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
return;
- list_del(&page->lru);
+ list_del(&folio->lru);
- if (HPageFreed(page)) {
+ if (folio_test_hugetlb_freed(folio)) {
h->free_huge_pages--;
h->free_huge_pages_node[nid]--;
}
@@ -1485,26 +1484,26 @@ static void __remove_hugetlb_page(struct hstate *h, struct page *page,
* be turned into a page of smaller size.
*/
if (!demote)
- set_page_refcounted(page);
+ folio_ref_unfreeze(folio, 1);
if (hstate_is_gigantic(h))
- set_compound_page_dtor(page, NULL_COMPOUND_DTOR);
+ folio_set_compound_dtor(folio, NULL_COMPOUND_DTOR);
else
- set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
+ folio_set_compound_dtor(folio, COMPOUND_PAGE_DTOR);
h->nr_huge_pages--;
h->nr_huge_pages_node[nid]--;
}
-static void remove_hugetlb_page(struct hstate *h, struct page *page,
+static void remove_hugetlb_folio(struct hstate *h, struct folio *folio,
bool adjust_surplus)
{
- __remove_hugetlb_page(h, page, adjust_surplus, false);
+ __remove_hugetlb_folio(h, folio, adjust_surplus, false);
}
-static void remove_hugetlb_page_for_demote(struct hstate *h, struct page *page,
+static void remove_hugetlb_folio_for_demote(struct hstate *h, struct folio *folio,
bool adjust_surplus)
{
- __remove_hugetlb_page(h, page, adjust_surplus, true);
+ __remove_hugetlb_folio(h, folio, adjust_surplus, true);
}
static void add_hugetlb_page(struct hstate *h, struct page *page,
@@ -1639,8 +1638,9 @@ static void free_hpage_workfn(struct work_struct *work)
/*
* The VM_BUG_ON_PAGE(!PageHuge(page), page) in page_hstate()
* is going to trigger because a previous call to
- * remove_hugetlb_page() will set_compound_page_dtor(page,
- * NULL_COMPOUND_DTOR), so do not use page_hstate() directly.
+ * remove_hugetlb_folio() will call folio_set_compound_dtor
+ * (folio, NULL_COMPOUND_DTOR), so do not use page_hstate()
+ * directly.
*/
h = size_to_hstate(page_size(page));
@@ -1749,12 +1749,12 @@ void free_huge_page(struct page *page)
h->resv_huge_pages++;
if (folio_test_hugetlb_temporary(folio)) {
- remove_hugetlb_page(h, page, false);
+ remove_hugetlb_folio(h, folio, false);
spin_unlock_irqrestore(&hugetlb_lock, flags);
update_and_free_page(h, page, true);
} else if (h->surplus_huge_pages_node[nid]) {
/* remove the page from active list */
- remove_hugetlb_page(h, page, true);
+ remove_hugetlb_folio(h, folio, true);
spin_unlock_irqrestore(&hugetlb_lock, flags);
update_and_free_page(h, page, true);
} else {
@@ -2090,6 +2090,7 @@ static struct page *remove_pool_huge_page(struct hstate *h,
{
int nr_nodes, node;
struct page *page = NULL;
+ struct folio *folio;
lockdep_assert_held(&hugetlb_lock);
for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
@@ -2101,7 +2102,8 @@ static struct page *remove_pool_huge_page(struct hstate *h,
!list_empty(&h->hugepage_freelists[node])) {
page = list_entry(h->hugepage_freelists[node].next,
struct page, lru);
- remove_hugetlb_page(h, page, acct_surplus);
+ folio = page_folio(page);
+ remove_hugetlb_folio(h, folio, acct_surplus);
break;
}
}
@@ -2163,7 +2165,7 @@ int dissolve_free_huge_page(struct page *page)
goto retry;
}
- remove_hugetlb_page(h, &folio->page, false);
+ remove_hugetlb_folio(h, folio, false);
h->max_huge_pages--;
spin_unlock_irq(&hugetlb_lock);
@@ -2801,7 +2803,7 @@ static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,
* and enqueue_huge_page() for new_page. The counters will remain
* stable since this happens under the lock.
*/
- remove_hugetlb_page(h, old_page, false);
+ remove_hugetlb_folio(h, old_folio, false);
/*
* Ref count on new page is already zero as it was dropped
@@ -3228,7 +3230,7 @@ static void try_to_free_low(struct hstate *h, unsigned long count,
goto out;
if (PageHighMem(page))
continue;
- remove_hugetlb_page(h, page, false);
+ remove_hugetlb_folio(h, page_folio(page), false);
list_add(&page->lru, &page_list);
}
}
@@ -3439,7 +3441,7 @@ static int demote_free_huge_page(struct hstate *h, struct page *page)
target_hstate = size_to_hstate(PAGE_SIZE << h->demote_order);
- remove_hugetlb_page_for_demote(h, page, false);
+ remove_hugetlb_folio_for_demote(h, folio, false);
spin_unlock_irq(&hugetlb_lock);
rc = hugetlb_vmemmap_restore(h, page);
--
2.38.1
^ permalink raw reply [flat|nested] 13+ messages in thread* [PATCH mm-unstable v3 05/10] mm/hugetlb: convert update_and_free_page() to folios
2022-11-17 21:14 [PATCH mm-unstable v3 00/10] convert core hugetlb functions to folios Sidhartha Kumar
` (3 preceding siblings ...)
2022-11-17 21:14 ` [PATCH mm-unstable v3 04/10] mm/hugetlb: convert remove_hugetlb_page() " Sidhartha Kumar
@ 2022-11-17 21:14 ` Sidhartha Kumar
2022-11-17 21:14 ` [PATCH mm-unstable v3 06/10] mm/hugetlb: convert add_hugetlb_page() to folios and add hugetlb_cma_folio() Sidhartha Kumar
` (4 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Sidhartha Kumar @ 2022-11-17 21:14 UTC (permalink / raw)
To: linux-kernel, linux-mm
Cc: akpm, songmuchun, mike.kravetz, willy, almasrymina, linmiaohe,
hughd, Sidhartha Kumar
Make more progress on converting the free_huge_page() destructor to
operate on folios by converting update_and_free_page() to folios.
Signed-off-by: Sidhartha Kumar <sidhartha.kumar@oracle.com>
---
mm/hugetlb.c | 30 ++++++++++++++++--------------
1 file changed, 16 insertions(+), 14 deletions(-)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index d9604c0dac54..80301fab56d8 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -1478,7 +1478,7 @@ static void __remove_hugetlb_folio(struct hstate *h, struct folio *folio,
* apply.
*
* This handles the case where more than one ref is held when and
- * after update_and_free_page is called.
+ * after update_and_free_hugetlb_folio is called.
*
* In the case of demote we do not ref count the page as it will soon
* be turned into a page of smaller size.
@@ -1609,7 +1609,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)
}
/*
- * As update_and_free_page() can be called under any context, so we cannot
+ * As update_and_free_hugetlb_folio() can be called under any context, so we cannot
* use GFP_KERNEL to allocate vmemmap pages. However, we can defer the
* actual freeing in a workqueue to prevent from using GFP_ATOMIC to allocate
* the vmemmap pages.
@@ -1657,11 +1657,11 @@ static inline void flush_free_hpage_work(struct hstate *h)
flush_work(&free_hpage_work);
}
-static void update_and_free_page(struct hstate *h, struct page *page,
+static void update_and_free_hugetlb_folio(struct hstate *h, struct folio *folio,
bool atomic)
{
- if (!HPageVmemmapOptimized(page) || !atomic) {
- __update_and_free_page(h, page);
+ if (!folio_test_hugetlb_vmemmap_optimized(folio) || !atomic) {
+ __update_and_free_page(h, &folio->page);
return;
}
@@ -1672,16 +1672,18 @@ static void update_and_free_page(struct hstate *h, struct page *page,
* empty. Otherwise, schedule_work() had been called but the workfn
* hasn't retrieved the list yet.
*/
- if (llist_add((struct llist_node *)&page->mapping, &hpage_freelist))
+ if (llist_add((struct llist_node *)&folio->mapping, &hpage_freelist))
schedule_work(&free_hpage_work);
}
static void update_and_free_pages_bulk(struct hstate *h, struct list_head *list)
{
struct page *page, *t_page;
+ struct folio *folio;
list_for_each_entry_safe(page, t_page, list, lru) {
- update_and_free_page(h, page, false);
+ folio = page_folio(page);
+ update_and_free_hugetlb_folio(h, folio, false);
cond_resched();
}
}
@@ -1751,12 +1753,12 @@ void free_huge_page(struct page *page)
if (folio_test_hugetlb_temporary(folio)) {
remove_hugetlb_folio(h, folio, false);
spin_unlock_irqrestore(&hugetlb_lock, flags);
- update_and_free_page(h, page, true);
+ update_and_free_hugetlb_folio(h, folio, true);
} else if (h->surplus_huge_pages_node[nid]) {
/* remove the page from active list */
remove_hugetlb_folio(h, folio, true);
spin_unlock_irqrestore(&hugetlb_lock, flags);
- update_and_free_page(h, page, true);
+ update_and_free_hugetlb_folio(h, folio, true);
} else {
arch_clear_hugepage_flags(page);
enqueue_huge_page(h, page);
@@ -2170,8 +2172,8 @@ int dissolve_free_huge_page(struct page *page)
spin_unlock_irq(&hugetlb_lock);
/*
- * Normally update_and_free_page will allocate required vmemmmap
- * before freeing the page. update_and_free_page will fail to
+ * Normally update_and_free_hugtlb_folio will allocate required vmemmmap
+ * before freeing the page. update_and_free_hugtlb_folio will fail to
* free the page if it can not allocate required vmemmap. We
* need to adjust max_huge_pages if the page is not freed.
* Attempt to allocate vmemmmap here so that we can take
@@ -2179,7 +2181,7 @@ int dissolve_free_huge_page(struct page *page)
*/
rc = hugetlb_vmemmap_restore(h, &folio->page);
if (!rc) {
- update_and_free_page(h, &folio->page, false);
+ update_and_free_hugetlb_folio(h, folio, false);
} else {
spin_lock_irq(&hugetlb_lock);
add_hugetlb_page(h, &folio->page, false);
@@ -2816,7 +2818,7 @@ static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,
* Pages have been replaced, we can safely free the old one.
*/
spin_unlock_irq(&hugetlb_lock);
- update_and_free_page(h, old_page, false);
+ update_and_free_hugetlb_folio(h, old_folio, false);
}
return ret;
@@ -2825,7 +2827,7 @@ static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,
spin_unlock_irq(&hugetlb_lock);
/* Page has a zero ref count, but needs a ref to be freed */
folio_ref_unfreeze(new_folio, 1);
- update_and_free_page(h, new_page, false);
+ update_and_free_hugetlb_folio(h, new_folio, false);
return ret;
}
--
2.38.1
^ permalink raw reply [flat|nested] 13+ messages in thread* [PATCH mm-unstable v3 06/10] mm/hugetlb: convert add_hugetlb_page() to folios and add hugetlb_cma_folio()
2022-11-17 21:14 [PATCH mm-unstable v3 00/10] convert core hugetlb functions to folios Sidhartha Kumar
` (4 preceding siblings ...)
2022-11-17 21:14 ` [PATCH mm-unstable v3 05/10] mm/hugetlb: convert update_and_free_page() " Sidhartha Kumar
@ 2022-11-17 21:14 ` Sidhartha Kumar
2022-11-19 18:59 ` Tarun Sahu
2022-11-17 21:14 ` [PATCH mm-unstable v3 07/10] mm/hugetlb: convert enqueue_huge_page() to folios Sidhartha Kumar
` (3 subsequent siblings)
9 siblings, 1 reply; 13+ messages in thread
From: Sidhartha Kumar @ 2022-11-17 21:14 UTC (permalink / raw)
To: linux-kernel, linux-mm
Cc: akpm, songmuchun, mike.kravetz, willy, almasrymina, linmiaohe,
hughd, Sidhartha Kumar
Convert add_hugetlb_page() to take in a folio, also convert
hugetlb_cma_page() to take in a folio.
Signed-off-by: Sidhartha Kumar <sidhartha.kumar@oracle.com>
---
mm/hugetlb.c | 40 ++++++++++++++++++++--------------------
1 file changed, 20 insertions(+), 20 deletions(-)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 80301fab56d8..bf36aa8e6072 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -54,13 +54,13 @@ struct hstate hstates[HUGE_MAX_HSTATE];
#ifdef CONFIG_CMA
static struct cma *hugetlb_cma[MAX_NUMNODES];
static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata;
-static bool hugetlb_cma_page(struct page *page, unsigned int order)
+static bool hugetlb_cma_folio(struct folio *folio, unsigned int order)
{
- return cma_pages_valid(hugetlb_cma[page_to_nid(page)], page,
+ return cma_pages_valid(hugetlb_cma[folio_nid(folio)], &folio->page,
1 << order);
}
#else
-static bool hugetlb_cma_page(struct page *page, unsigned int order)
+static bool hugetlb_cma_folio(struct folio *folio, unsigned int order)
{
return false;
}
@@ -1506,17 +1506,17 @@ static void remove_hugetlb_folio_for_demote(struct hstate *h, struct folio *foli
__remove_hugetlb_folio(h, folio, adjust_surplus, true);
}
-static void add_hugetlb_page(struct hstate *h, struct page *page,
+static void add_hugetlb_folio(struct hstate *h, struct folio *folio,
bool adjust_surplus)
{
int zeroed;
- int nid = page_to_nid(page);
+ int nid = folio_nid(folio);
- VM_BUG_ON_PAGE(!HPageVmemmapOptimized(page), page);
+ VM_BUG_ON_FOLIO(!folio_test_hugetlb_vmemmap_optimized(folio), folio);
lockdep_assert_held(&hugetlb_lock);
- INIT_LIST_HEAD(&page->lru);
+ INIT_LIST_HEAD(&folio->lru);
h->nr_huge_pages++;
h->nr_huge_pages_node[nid]++;
@@ -1525,21 +1525,21 @@ static void add_hugetlb_page(struct hstate *h, struct page *page,
h->surplus_huge_pages_node[nid]++;
}
- set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
- set_page_private(page, 0);
+ folio_set_compound_dtor(folio, HUGETLB_PAGE_DTOR);
+ folio_change_private(folio, 0);
/*
* We have to set HPageVmemmapOptimized again as above
- * set_page_private(page, 0) cleared it.
+ * folio_change_private(folio, 0) cleared it.
*/
- SetHPageVmemmapOptimized(page);
+ folio_set_hugetlb_vmemmap_optimized(folio);
/*
- * This page is about to be managed by the hugetlb allocator and
+ * This folio is about to be managed by the hugetlb allocator and
* should have no users. Drop our reference, and check for others
* just in case.
*/
- zeroed = put_page_testzero(page);
- if (!zeroed)
+ zeroed = folio_put_testzero(folio);
+ if (unlikely(!zeroed))
/*
* It is VERY unlikely soneone else has taken a ref on
* the page. In this case, we simply return as the
@@ -1548,8 +1548,8 @@ static void add_hugetlb_page(struct hstate *h, struct page *page,
*/
return;
- arch_clear_hugepage_flags(page);
- enqueue_huge_page(h, page);
+ arch_clear_hugepage_flags(&folio->page);
+ enqueue_huge_page(h, &folio->page);
}
static void __update_and_free_page(struct hstate *h, struct page *page)
@@ -1575,7 +1575,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)
* page and put the page back on the hugetlb free list and treat
* as a surplus page.
*/
- add_hugetlb_page(h, page, true);
+ add_hugetlb_folio(h, page_folio(page), true);
spin_unlock_irq(&hugetlb_lock);
return;
}
@@ -1600,7 +1600,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)
* need to be given back to CMA in free_gigantic_page.
*/
if (hstate_is_gigantic(h) ||
- hugetlb_cma_page(page, huge_page_order(h))) {
+ hugetlb_cma_folio(folio, huge_page_order(h))) {
destroy_compound_gigantic_folio(folio, huge_page_order(h));
free_gigantic_page(page, huge_page_order(h));
} else {
@@ -2184,7 +2184,7 @@ int dissolve_free_huge_page(struct page *page)
update_and_free_hugetlb_folio(h, folio, false);
} else {
spin_lock_irq(&hugetlb_lock);
- add_hugetlb_page(h, &folio->page, false);
+ add_hugetlb_folio(h, folio, false);
h->max_huge_pages++;
spin_unlock_irq(&hugetlb_lock);
}
@@ -3451,7 +3451,7 @@ static int demote_free_huge_page(struct hstate *h, struct page *page)
/* Allocation of vmemmmap failed, we can not demote page */
spin_lock_irq(&hugetlb_lock);
set_page_refcounted(page);
- add_hugetlb_page(h, page, false);
+ add_hugetlb_folio(h, page_folio(page), false);
return rc;
}
--
2.38.1
^ permalink raw reply [flat|nested] 13+ messages in thread* Re: [PATCH mm-unstable v3 06/10] mm/hugetlb: convert add_hugetlb_page() to folios and add hugetlb_cma_folio()
2022-11-17 21:14 ` [PATCH mm-unstable v3 06/10] mm/hugetlb: convert add_hugetlb_page() to folios and add hugetlb_cma_folio() Sidhartha Kumar
@ 2022-11-19 18:59 ` Tarun Sahu
2022-11-21 15:22 ` Sidhartha Kumar
0 siblings, 1 reply; 13+ messages in thread
From: Tarun Sahu @ 2022-11-19 18:59 UTC (permalink / raw)
To: Sidhartha Kumar
Cc: linux-kernel, linux-mm, akpm, songmuchun, mike.kravetz, willy,
almasrymina, linmiaohe, hughd
Hi,
Though it is already merged, it is just comment thing.
On Nov 17 2022, Sidhartha Kumar wrote:
> Convert add_hugetlb_page() to take in a folio, also convert
> hugetlb_cma_page() to take in a folio.
>
> Signed-off-by: Sidhartha Kumar <sidhartha.kumar@oracle.com>
> ---
> mm/hugetlb.c | 40 ++++++++++++++++++++--------------------
> 1 file changed, 20 insertions(+), 20 deletions(-)
>
> diff --git a/mm/hugetlb.c b/mm/hugetlb.c
> index 80301fab56d8..bf36aa8e6072 100644
> --- a/mm/hugetlb.c
> +++ b/mm/hugetlb.c
> @@ -54,13 +54,13 @@ struct hstate hstates[HUGE_MAX_HSTATE];
> #ifdef CONFIG_CMA
> static struct cma *hugetlb_cma[MAX_NUMNODES];
> static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata;
> -static bool hugetlb_cma_page(struct page *page, unsigned int order)
> +static bool hugetlb_cma_folio(struct folio *folio, unsigned int order)
> {
> - return cma_pages_valid(hugetlb_cma[page_to_nid(page)], page,
> + return cma_pages_valid(hugetlb_cma[folio_nid(folio)], &folio->page,
> 1 << order);
> }
> #else
> -static bool hugetlb_cma_page(struct page *page, unsigned int order)
> +static bool hugetlb_cma_folio(struct folio *folio, unsigned int order)
> {
> return false;
> }
> @@ -1506,17 +1506,17 @@ static void remove_hugetlb_folio_for_demote(struct hstate *h, struct folio *foli
> __remove_hugetlb_folio(h, folio, adjust_surplus, true);
> }
>
> -static void add_hugetlb_page(struct hstate *h, struct page *page,
> +static void add_hugetlb_folio(struct hstate *h, struct folio *folio,
> bool adjust_surplus)
> {
> int zeroed;
> - int nid = page_to_nid(page);
> + int nid = folio_nid(folio);
>
> - VM_BUG_ON_PAGE(!HPageVmemmapOptimized(page), page);
> + VM_BUG_ON_FOLIO(!folio_test_hugetlb_vmemmap_optimized(folio), folio);
>
> lockdep_assert_held(&hugetlb_lock);
>
> - INIT_LIST_HEAD(&page->lru);
> + INIT_LIST_HEAD(&folio->lru);
> h->nr_huge_pages++;
> h->nr_huge_pages_node[nid]++;
>
> @@ -1525,21 +1525,21 @@ static void add_hugetlb_page(struct hstate *h, struct page *page,
> h->surplus_huge_pages_node[nid]++;
> }
>
> - set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
> - set_page_private(page, 0);
> + folio_set_compound_dtor(folio, HUGETLB_PAGE_DTOR);
> + folio_change_private(folio, 0);
> /*
> * We have to set HPageVmemmapOptimized again as above
^
This can be changed to folio version of itself.
> - * set_page_private(page, 0) cleared it.
> + * folio_change_private(folio, 0) cleared it.
> */
> - SetHPageVmemmapOptimized(page);
> + folio_set_hugetlb_vmemmap_optimized(folio);
>
> /*
> - * This page is about to be managed by the hugetlb allocator and
> + * This folio is about to be managed by the hugetlb allocator and
> * should have no users. Drop our reference, and check for others
> * just in case.
> */
> - zeroed = put_page_testzero(page);
> - if (!zeroed)
> + zeroed = folio_put_testzero(folio);
> + if (unlikely(!zeroed))
> /*
> * It is VERY unlikely soneone else has taken a ref on
> * the page. In this case, we simply return as the
> @@ -1548,8 +1548,8 @@ static void add_hugetlb_page(struct hstate *h, struct page *page,
> */
> return;
>
> - arch_clear_hugepage_flags(page);
> - enqueue_huge_page(h, page);
> + arch_clear_hugepage_flags(&folio->page);
> + enqueue_huge_page(h, &folio->page);
> }
>
> static void __update_and_free_page(struct hstate *h, struct page *page)
> @@ -1575,7 +1575,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)
> * page and put the page back on the hugetlb free list and treat
> * as a surplus page.
> */
> - add_hugetlb_page(h, page, true);
> + add_hugetlb_folio(h, page_folio(page), true);
> spin_unlock_irq(&hugetlb_lock);
> return;
> }
> @@ -1600,7 +1600,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)
> * need to be given back to CMA in free_gigantic_page.
> */
> if (hstate_is_gigantic(h) ||
> - hugetlb_cma_page(page, huge_page_order(h))) {
> + hugetlb_cma_folio(folio, huge_page_order(h))) {
> destroy_compound_gigantic_folio(folio, huge_page_order(h));
> free_gigantic_page(page, huge_page_order(h));
> } else {
> @@ -2184,7 +2184,7 @@ int dissolve_free_huge_page(struct page *page)
> update_and_free_hugetlb_folio(h, folio, false);
> } else {
> spin_lock_irq(&hugetlb_lock);
> - add_hugetlb_page(h, &folio->page, false);
> + add_hugetlb_folio(h, folio, false);
> h->max_huge_pages++;
> spin_unlock_irq(&hugetlb_lock);
> }
> @@ -3451,7 +3451,7 @@ static int demote_free_huge_page(struct hstate *h, struct page *page)
> /* Allocation of vmemmmap failed, we can not demote page */
> spin_lock_irq(&hugetlb_lock);
> set_page_refcounted(page);
> - add_hugetlb_page(h, page, false);
> + add_hugetlb_folio(h, page_folio(page), false);
> return rc;
> }
>
> --
> 2.38.1
>
^ permalink raw reply [flat|nested] 13+ messages in thread* Re: [PATCH mm-unstable v3 06/10] mm/hugetlb: convert add_hugetlb_page() to folios and add hugetlb_cma_folio()
2022-11-19 18:59 ` Tarun Sahu
@ 2022-11-21 15:22 ` Sidhartha Kumar
0 siblings, 0 replies; 13+ messages in thread
From: Sidhartha Kumar @ 2022-11-21 15:22 UTC (permalink / raw)
To: Tarun Sahu
Cc: linux-kernel, linux-mm, akpm, songmuchun, mike.kravetz, willy,
almasrymina, linmiaohe, hughd
On 11/19/22 10:59 AM, Tarun Sahu wrote:
> Hi,
> Though it is already merged, it is just comment thing.
This series will need another version based on other feedback that I
got. I'll be sure to change the comment in the next version.
Thanks,
Sidhartha Kumar
> On Nov 17 2022, Sidhartha Kumar wrote:
>> Convert add_hugetlb_page() to take in a folio, also convert
>> hugetlb_cma_page() to take in a folio.
>>
>> Signed-off-by: Sidhartha Kumar <sidhartha.kumar@oracle.com>
>> ---
>> mm/hugetlb.c | 40 ++++++++++++++++++++--------------------
>> 1 file changed, 20 insertions(+), 20 deletions(-)
>>
>> diff --git a/mm/hugetlb.c b/mm/hugetlb.c
>> index 80301fab56d8..bf36aa8e6072 100644
>> --- a/mm/hugetlb.c
>> +++ b/mm/hugetlb.c
>> @@ -54,13 +54,13 @@ struct hstate hstates[HUGE_MAX_HSTATE];
>> #ifdef CONFIG_CMA
>> static struct cma *hugetlb_cma[MAX_NUMNODES];
>> static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata;
>> -static bool hugetlb_cma_page(struct page *page, unsigned int order)
>> +static bool hugetlb_cma_folio(struct folio *folio, unsigned int order)
>> {
>> - return cma_pages_valid(hugetlb_cma[page_to_nid(page)], page,
>> + return cma_pages_valid(hugetlb_cma[folio_nid(folio)], &folio->page,
>> 1 << order);
>> }
>> #else
>> -static bool hugetlb_cma_page(struct page *page, unsigned int order)
>> +static bool hugetlb_cma_folio(struct folio *folio, unsigned int order)
>> {
>> return false;
>> }
>> @@ -1506,17 +1506,17 @@ static void remove_hugetlb_folio_for_demote(struct hstate *h, struct folio *foli
>> __remove_hugetlb_folio(h, folio, adjust_surplus, true);
>> }
>>
>> -static void add_hugetlb_page(struct hstate *h, struct page *page,
>> +static void add_hugetlb_folio(struct hstate *h, struct folio *folio,
>> bool adjust_surplus)
>> {
>> int zeroed;
>> - int nid = page_to_nid(page);
>> + int nid = folio_nid(folio);
>>
>> - VM_BUG_ON_PAGE(!HPageVmemmapOptimized(page), page);
>> + VM_BUG_ON_FOLIO(!folio_test_hugetlb_vmemmap_optimized(folio), folio);
>>
>> lockdep_assert_held(&hugetlb_lock);
>>
>> - INIT_LIST_HEAD(&page->lru);
>> + INIT_LIST_HEAD(&folio->lru);
>> h->nr_huge_pages++;
>> h->nr_huge_pages_node[nid]++;
>>
>> @@ -1525,21 +1525,21 @@ static void add_hugetlb_page(struct hstate *h, struct page *page,
>> h->surplus_huge_pages_node[nid]++;
>> }
>>
>> - set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
>> - set_page_private(page, 0);
>> + folio_set_compound_dtor(folio, HUGETLB_PAGE_DTOR);
>> + folio_change_private(folio, 0);
>> /*
>> * We have to set HPageVmemmapOptimized again as above
> ^
> This can be changed to folio version of itself.
>
>> - * set_page_private(page, 0) cleared it.
>> + * folio_change_private(folio, 0) cleared it.
>> */
>> - SetHPageVmemmapOptimized(page);
>> + folio_set_hugetlb_vmemmap_optimized(folio);
>>
>> /*
>> - * This page is about to be managed by the hugetlb allocator and
>> + * This folio is about to be managed by the hugetlb allocator and
>> * should have no users. Drop our reference, and check for others
>> * just in case.
>> */
>> - zeroed = put_page_testzero(page);
>> - if (!zeroed)
>> + zeroed = folio_put_testzero(folio);
>> + if (unlikely(!zeroed))
>> /*
>> * It is VERY unlikely soneone else has taken a ref on
>> * the page. In this case, we simply return as the
>> @@ -1548,8 +1548,8 @@ static void add_hugetlb_page(struct hstate *h, struct page *page,
>> */
>> return;
>>
>> - arch_clear_hugepage_flags(page);
>> - enqueue_huge_page(h, page);
>> + arch_clear_hugepage_flags(&folio->page);
>> + enqueue_huge_page(h, &folio->page);
>> }
>>
>> static void __update_and_free_page(struct hstate *h, struct page *page)
>> @@ -1575,7 +1575,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)
>> * page and put the page back on the hugetlb free list and treat
>> * as a surplus page.
>> */
>> - add_hugetlb_page(h, page, true);
>> + add_hugetlb_folio(h, page_folio(page), true);
>> spin_unlock_irq(&hugetlb_lock);
>> return;
>> }
>> @@ -1600,7 +1600,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)
>> * need to be given back to CMA in free_gigantic_page.
>> */
>> if (hstate_is_gigantic(h) ||
>> - hugetlb_cma_page(page, huge_page_order(h))) {
>> + hugetlb_cma_folio(folio, huge_page_order(h))) {
>> destroy_compound_gigantic_folio(folio, huge_page_order(h));
>> free_gigantic_page(page, huge_page_order(h));
>> } else {
>> @@ -2184,7 +2184,7 @@ int dissolve_free_huge_page(struct page *page)
>> update_and_free_hugetlb_folio(h, folio, false);
>> } else {
>> spin_lock_irq(&hugetlb_lock);
>> - add_hugetlb_page(h, &folio->page, false);
>> + add_hugetlb_folio(h, folio, false);
>> h->max_huge_pages++;
>> spin_unlock_irq(&hugetlb_lock);
>> }
>> @@ -3451,7 +3451,7 @@ static int demote_free_huge_page(struct hstate *h, struct page *page)
>> /* Allocation of vmemmmap failed, we can not demote page */
>> spin_lock_irq(&hugetlb_lock);
>> set_page_refcounted(page);
>> - add_hugetlb_page(h, page, false);
>> + add_hugetlb_folio(h, page_folio(page), false);
>> return rc;
>> }
>>
>> --
>> 2.38.1
>>
^ permalink raw reply [flat|nested] 13+ messages in thread
* [PATCH mm-unstable v3 07/10] mm/hugetlb: convert enqueue_huge_page() to folios
2022-11-17 21:14 [PATCH mm-unstable v3 00/10] convert core hugetlb functions to folios Sidhartha Kumar
` (5 preceding siblings ...)
2022-11-17 21:14 ` [PATCH mm-unstable v3 06/10] mm/hugetlb: convert add_hugetlb_page() to folios and add hugetlb_cma_folio() Sidhartha Kumar
@ 2022-11-17 21:14 ` Sidhartha Kumar
2022-11-17 21:14 ` [PATCH mm-unstable v3 08/10] mm/hugetlb: convert free_gigantic_page() " Sidhartha Kumar
` (2 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Sidhartha Kumar @ 2022-11-17 21:14 UTC (permalink / raw)
To: linux-kernel, linux-mm
Cc: akpm, songmuchun, mike.kravetz, willy, almasrymina, linmiaohe,
hughd, Sidhartha Kumar
Convert callers of enqueue_huge_page() to pass in a folio, function is
renamed to enqueue_hugetlb_folio().
Signed-off-by: Sidhartha Kumar <sidhartha.kumar@oracle.com>
---
mm/hugetlb.c | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index bf36aa8e6072..4eb6f3d6f46e 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -1127,17 +1127,17 @@ static bool vma_has_reserves(struct vm_area_struct *vma, long chg)
return false;
}
-static void enqueue_huge_page(struct hstate *h, struct page *page)
+static void enqueue_hugetlb_folio(struct hstate *h, struct folio *folio)
{
- int nid = page_to_nid(page);
+ int nid = folio_nid(folio);
lockdep_assert_held(&hugetlb_lock);
- VM_BUG_ON_PAGE(page_count(page), page);
+ VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
- list_move(&page->lru, &h->hugepage_freelists[nid]);
+ list_move(&folio->lru, &h->hugepage_freelists[nid]);
h->free_huge_pages++;
h->free_huge_pages_node[nid]++;
- SetHPageFreed(page);
+ folio_set_hugetlb_freed(folio);
}
static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid)
@@ -1549,7 +1549,7 @@ static void add_hugetlb_folio(struct hstate *h, struct folio *folio,
return;
arch_clear_hugepage_flags(&folio->page);
- enqueue_huge_page(h, &folio->page);
+ enqueue_hugetlb_folio(h, folio);
}
static void __update_and_free_page(struct hstate *h, struct page *page)
@@ -1761,7 +1761,7 @@ void free_huge_page(struct page *page)
update_and_free_hugetlb_folio(h, folio, true);
} else {
arch_clear_hugepage_flags(page);
- enqueue_huge_page(h, page);
+ enqueue_hugetlb_folio(h, folio);
spin_unlock_irqrestore(&hugetlb_lock, flags);
}
}
@@ -2436,7 +2436,7 @@ static int gather_surplus_pages(struct hstate *h, long delta)
if ((--needed) < 0)
break;
/* Add the page to the hugetlb allocator */
- enqueue_huge_page(h, page);
+ enqueue_hugetlb_folio(h, page_folio(page));
}
free:
spin_unlock_irq(&hugetlb_lock);
@@ -2802,8 +2802,8 @@ static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,
* Ok, old_page is still a genuine free hugepage. Remove it from
* the freelist and decrease the counters. These will be
* incremented again when calling __prep_account_new_huge_page()
- * and enqueue_huge_page() for new_page. The counters will remain
- * stable since this happens under the lock.
+ * and enqueue_hugetlb_folio() for new_folio. The counters will
+ * remain stable since this happens under the lock.
*/
remove_hugetlb_folio(h, old_folio, false);
@@ -2812,7 +2812,7 @@ static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,
* earlier. It can be directly added to the pool free list.
*/
__prep_account_new_huge_page(h, nid);
- enqueue_huge_page(h, new_page);
+ enqueue_hugetlb_folio(h, new_folio);
/*
* Pages have been replaced, we can safely free the old one.
--
2.38.1
^ permalink raw reply [flat|nested] 13+ messages in thread* [PATCH mm-unstable v3 08/10] mm/hugetlb: convert free_gigantic_page() to folios
2022-11-17 21:14 [PATCH mm-unstable v3 00/10] convert core hugetlb functions to folios Sidhartha Kumar
` (6 preceding siblings ...)
2022-11-17 21:14 ` [PATCH mm-unstable v3 07/10] mm/hugetlb: convert enqueue_huge_page() to folios Sidhartha Kumar
@ 2022-11-17 21:14 ` Sidhartha Kumar
2022-11-17 21:15 ` [PATCH mm-unstable v3 09/10] mm/hugetlb: convert hugetlb prep functions " Sidhartha Kumar
2022-11-17 21:15 ` [PATCH mm-unstable v3 10/10] mm/hugetlb: change hugetlb allocation functions to return a folio Sidhartha Kumar
9 siblings, 0 replies; 13+ messages in thread
From: Sidhartha Kumar @ 2022-11-17 21:14 UTC (permalink / raw)
To: linux-kernel, linux-mm
Cc: akpm, songmuchun, mike.kravetz, willy, almasrymina, linmiaohe,
hughd, Sidhartha Kumar
Convert callers of free_gigantic_page() to use folios, function is then
renamed to free_gigantic_folio().
Signed-off-by: Sidhartha Kumar <sidhartha.kumar@oracle.com>
---
mm/hugetlb.c | 29 +++++++++++++++++------------
1 file changed, 17 insertions(+), 12 deletions(-)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 4eb6f3d6f46e..38c5ca015363 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -1361,18 +1361,20 @@ static void destroy_compound_gigantic_folio(struct folio *folio,
__destroy_compound_gigantic_folio(folio, order, false);
}
-static void free_gigantic_page(struct page *page, unsigned int order)
+static void free_gigantic_folio(struct folio *folio, unsigned int order)
{
/*
* If the page isn't allocated using the cma allocator,
* cma_release() returns false.
*/
#ifdef CONFIG_CMA
- if (cma_release(hugetlb_cma[page_to_nid(page)], page, 1 << order))
+ int nid = folio_nid(folio);
+
+ if (cma_release(hugetlb_cma[nid], &folio->page, 1 << order))
return;
#endif
- free_contig_range(page_to_pfn(page), 1 << order);
+ free_contig_range(folio_pfn(folio), 1 << order);
}
#ifdef CONFIG_CONTIG_ALLOC
@@ -1426,7 +1428,8 @@ static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
{
return NULL;
}
-static inline void free_gigantic_page(struct page *page, unsigned int order) { }
+static inline void free_gigantic_folio(struct folio *folio,
+ unsigned int order) { }
static inline void destroy_compound_gigantic_folio(struct folio *folio,
unsigned int order) { }
#endif
@@ -1565,7 +1568,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)
* If we don't know which subpages are hwpoisoned, we can't free
* the hugepage, so it's leaked intentionally.
*/
- if (HPageRawHwpUnreliable(page))
+ if (folio_test_hugetlb_raw_hwp_unreliable(folio))
return;
if (hugetlb_vmemmap_restore(h, page)) {
@@ -1575,7 +1578,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)
* page and put the page back on the hugetlb free list and treat
* as a surplus page.
*/
- add_hugetlb_folio(h, page_folio(page), true);
+ add_hugetlb_folio(h, folio, true);
spin_unlock_irq(&hugetlb_lock);
return;
}
@@ -1588,7 +1591,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)
hugetlb_clear_page_hwpoison(&folio->page);
for (i = 0; i < pages_per_huge_page(h); i++) {
- subpage = nth_page(page, i);
+ subpage = folio_page(folio, i);
subpage->flags &= ~(1 << PG_locked | 1 << PG_error |
1 << PG_referenced | 1 << PG_dirty |
1 << PG_active | 1 << PG_private |
@@ -1597,12 +1600,12 @@ static void __update_and_free_page(struct hstate *h, struct page *page)
/*
* Non-gigantic pages demoted from CMA allocated gigantic pages
- * need to be given back to CMA in free_gigantic_page.
+ * need to be given back to CMA in free_gigantic_folio.
*/
if (hstate_is_gigantic(h) ||
hugetlb_cma_folio(folio, huge_page_order(h))) {
destroy_compound_gigantic_folio(folio, huge_page_order(h));
- free_gigantic_page(page, huge_page_order(h));
+ free_gigantic_folio(folio, huge_page_order(h));
} else {
__free_pages(page, huge_page_order(h));
}
@@ -2023,6 +2026,7 @@ static struct page *alloc_fresh_huge_page(struct hstate *h,
nodemask_t *node_alloc_noretry)
{
struct page *page;
+ struct folio *folio;
bool retry = false;
retry:
@@ -2033,14 +2037,14 @@ static struct page *alloc_fresh_huge_page(struct hstate *h,
nid, nmask, node_alloc_noretry);
if (!page)
return NULL;
-
+ folio = page_folio(page);
if (hstate_is_gigantic(h)) {
if (!prep_compound_gigantic_page(page, huge_page_order(h))) {
/*
* Rare failure to convert pages to compound page.
* Free pages and try again - ONCE!
*/
- free_gigantic_page(page, huge_page_order(h));
+ free_gigantic_folio(folio, huge_page_order(h));
if (!retry) {
retry = true;
goto retry;
@@ -3048,6 +3052,7 @@ static void __init gather_bootmem_prealloc(void)
list_for_each_entry(m, &huge_boot_pages, list) {
struct page *page = virt_to_page(m);
+ struct folio *folio = page_folio(page);
struct hstate *h = m->hstate;
VM_BUG_ON(!hstate_is_gigantic(h));
@@ -3058,7 +3063,7 @@ static void __init gather_bootmem_prealloc(void)
free_huge_page(page); /* add to the hugepage allocator */
} else {
/* VERY unlikely inflated ref count on a tail page */
- free_gigantic_page(page, huge_page_order(h));
+ free_gigantic_folio(folio, huge_page_order(h));
}
/*
--
2.38.1
^ permalink raw reply [flat|nested] 13+ messages in thread* [PATCH mm-unstable v3 09/10] mm/hugetlb: convert hugetlb prep functions to folios
2022-11-17 21:14 [PATCH mm-unstable v3 00/10] convert core hugetlb functions to folios Sidhartha Kumar
` (7 preceding siblings ...)
2022-11-17 21:14 ` [PATCH mm-unstable v3 08/10] mm/hugetlb: convert free_gigantic_page() " Sidhartha Kumar
@ 2022-11-17 21:15 ` Sidhartha Kumar
2022-11-17 21:15 ` [PATCH mm-unstable v3 10/10] mm/hugetlb: change hugetlb allocation functions to return a folio Sidhartha Kumar
9 siblings, 0 replies; 13+ messages in thread
From: Sidhartha Kumar @ 2022-11-17 21:15 UTC (permalink / raw)
To: linux-kernel, linux-mm
Cc: akpm, songmuchun, mike.kravetz, willy, almasrymina, linmiaohe,
hughd, Sidhartha Kumar
Convert prep_new_huge_page() and __prep_compound_gigantic_page() to
folios.
Signed-off-by: Sidhartha Kumar <sidhartha.kumar@oracle.com>
---
mm/hugetlb.c | 61 +++++++++++++++++++++++++---------------------------
1 file changed, 29 insertions(+), 32 deletions(-)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 38c5ca015363..6ecf9874c521 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -1789,28 +1789,26 @@ static void __prep_new_hugetlb_folio(struct hstate *h, struct folio *folio)
set_hugetlb_cgroup_rsvd(folio, NULL);
}
-static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
+static void prep_new_hugetlb_folio(struct hstate *h, struct folio *folio, int nid)
{
- struct folio *folio = page_folio(page);
-
__prep_new_hugetlb_folio(h, folio);
spin_lock_irq(&hugetlb_lock);
__prep_account_new_huge_page(h, nid);
spin_unlock_irq(&hugetlb_lock);
}
-static bool __prep_compound_gigantic_page(struct page *page, unsigned int order,
- bool demote)
+static bool __prep_compound_gigantic_folio(struct folio *folio,
+ unsigned int order, bool demote)
{
int i, j;
int nr_pages = 1 << order;
struct page *p;
- /* we rely on prep_new_huge_page to set the destructor */
- set_compound_order(page, order);
- __SetPageHead(page);
+ /* we rely on prep_new_hugetlb_folio to set the destructor */
+ folio_set_compound_order(folio, order);
+ __SetPageHead(&folio->page);
for (i = 0; i < nr_pages; i++) {
- p = nth_page(page, i);
+ p = folio_page(folio, i);
/*
* For gigantic hugepages allocated through bootmem at
@@ -1851,43 +1849,41 @@ static bool __prep_compound_gigantic_page(struct page *page, unsigned int order,
VM_BUG_ON_PAGE(page_count(p), p);
}
if (i != 0)
- set_compound_head(p, page);
+ set_compound_head(p, &folio->page);
}
- atomic_set(compound_mapcount_ptr(page), -1);
- atomic_set(subpages_mapcount_ptr(page), 0);
- atomic_set(compound_pincount_ptr(page), 0);
+ atomic_set(folio_mapcount_ptr(folio), -1);
+ atomic_set(folio_subpages_mapcount_ptr(folio), 0);
+ atomic_set(folio_pincount_ptr(folio), 0);
return true;
out_error:
/* undo page modifications made above */
for (j = 0; j < i; j++) {
- p = nth_page(page, j);
+ p = folio_page(folio, j);
if (j != 0)
clear_compound_head(p);
set_page_refcounted(p);
}
/* need to clear PG_reserved on remaining tail pages */
for (; j < nr_pages; j++) {
- p = nth_page(page, j);
+ p = folio_page(folio, j);
__ClearPageReserved(p);
}
- set_compound_order(page, 0);
-#ifdef CONFIG_64BIT
- page[1].compound_nr = 0;
-#endif
- __ClearPageHead(page);
+ folio_set_compound_order(folio, 0);
+ folio_clear_head(folio);
return false;
}
-static bool prep_compound_gigantic_page(struct page *page, unsigned int order)
+static bool prep_compound_gigantic_folio(struct folio *folio,
+ unsigned int order)
{
- return __prep_compound_gigantic_page(page, order, false);
+ return __prep_compound_gigantic_folio(folio, order, false);
}
-static bool prep_compound_gigantic_page_for_demote(struct page *page,
+static bool prep_compound_gigantic_folio_for_demote(struct folio *folio,
unsigned int order)
{
- return __prep_compound_gigantic_page(page, order, true);
+ return __prep_compound_gigantic_folio(folio, order, true);
}
/*
@@ -2039,7 +2035,7 @@ static struct page *alloc_fresh_huge_page(struct hstate *h,
return NULL;
folio = page_folio(page);
if (hstate_is_gigantic(h)) {
- if (!prep_compound_gigantic_page(page, huge_page_order(h))) {
+ if (!prep_compound_gigantic_folio(folio, huge_page_order(h))) {
/*
* Rare failure to convert pages to compound page.
* Free pages and try again - ONCE!
@@ -2052,7 +2048,7 @@ static struct page *alloc_fresh_huge_page(struct hstate *h,
return NULL;
}
}
- prep_new_huge_page(h, page, page_to_nid(page));
+ prep_new_hugetlb_folio(h, folio, folio_nid(folio));
return page;
}
@@ -3056,10 +3052,10 @@ static void __init gather_bootmem_prealloc(void)
struct hstate *h = m->hstate;
VM_BUG_ON(!hstate_is_gigantic(h));
- WARN_ON(page_count(page) != 1);
- if (prep_compound_gigantic_page(page, huge_page_order(h))) {
- WARN_ON(PageReserved(page));
- prep_new_huge_page(h, page, page_to_nid(page));
+ WARN_ON(folio_ref_count(folio) != 1);
+ if (prep_compound_gigantic_folio(folio, huge_page_order(h))) {
+ WARN_ON(folio_test_reserved(folio));
+ prep_new_hugetlb_folio(h, folio, folio_nid(folio));
free_huge_page(page); /* add to the hugepage allocator */
} else {
/* VERY unlikely inflated ref count on a tail page */
@@ -3478,13 +3474,14 @@ static int demote_free_huge_page(struct hstate *h, struct page *page)
for (i = 0; i < pages_per_huge_page(h);
i += pages_per_huge_page(target_hstate)) {
subpage = nth_page(page, i);
+ folio = page_folio(subpage);
if (hstate_is_gigantic(target_hstate))
- prep_compound_gigantic_page_for_demote(subpage,
+ prep_compound_gigantic_folio_for_demote(folio,
target_hstate->order);
else
prep_compound_page(subpage, target_hstate->order);
set_page_private(subpage, 0);
- prep_new_huge_page(target_hstate, subpage, nid);
+ prep_new_hugetlb_folio(target_hstate, folio, nid);
free_huge_page(subpage);
}
mutex_unlock(&target_hstate->resize_lock);
--
2.38.1
^ permalink raw reply [flat|nested] 13+ messages in thread* [PATCH mm-unstable v3 10/10] mm/hugetlb: change hugetlb allocation functions to return a folio
2022-11-17 21:14 [PATCH mm-unstable v3 00/10] convert core hugetlb functions to folios Sidhartha Kumar
` (8 preceding siblings ...)
2022-11-17 21:15 ` [PATCH mm-unstable v3 09/10] mm/hugetlb: convert hugetlb prep functions " Sidhartha Kumar
@ 2022-11-17 21:15 ` Sidhartha Kumar
9 siblings, 0 replies; 13+ messages in thread
From: Sidhartha Kumar @ 2022-11-17 21:15 UTC (permalink / raw)
To: linux-kernel, linux-mm
Cc: akpm, songmuchun, mike.kravetz, willy, almasrymina, linmiaohe,
hughd, Sidhartha Kumar
Many hugetlb allocation helper functions have now been converting to
folios, update their higher level callers to be compatible with folios.
Signed-off-by: Sidhartha Kumar <sidhartha.kumar@oracle.com>
---
mm/hugetlb.c | 98 ++++++++++++++++++++++++----------------------------
1 file changed, 46 insertions(+), 52 deletions(-)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 6ecf9874c521..aa0a388bded4 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -1378,7 +1378,7 @@ static void free_gigantic_folio(struct folio *folio, unsigned int order)
}
#ifdef CONFIG_CONTIG_ALLOC
-static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
+static struct folio *alloc_gigantic_folio(struct hstate *h, gfp_t gfp_mask,
int nid, nodemask_t *nodemask)
{
unsigned long nr_pages = pages_per_huge_page(h);
@@ -1394,7 +1394,7 @@ static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
page = cma_alloc(hugetlb_cma[nid], nr_pages,
huge_page_order(h), true);
if (page)
- return page;
+ return page_folio(page);
}
if (!(gfp_mask & __GFP_THISNODE)) {
@@ -1405,17 +1405,16 @@ static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
page = cma_alloc(hugetlb_cma[node], nr_pages,
huge_page_order(h), true);
if (page)
- return page;
+ return page_folio(page);
}
}
}
#endif
-
- return alloc_contig_pages(nr_pages, gfp_mask, nid, nodemask);
+ return page_folio(alloc_contig_pages(nr_pages, gfp_mask, nid, nodemask));
}
#else /* !CONFIG_CONTIG_ALLOC */
-static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
+static struct folio *alloc_gigantic_folio(struct hstate *h, gfp_t gfp_mask,
int nid, nodemask_t *nodemask)
{
return NULL;
@@ -1423,7 +1422,7 @@ static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
#endif /* CONFIG_CONTIG_ALLOC */
#else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */
-static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
+static struct folio *alloc_gigantic_folio(struct hstate *h, gfp_t gfp_mask,
int nid, nodemask_t *nodemask)
{
return NULL;
@@ -1948,7 +1947,7 @@ pgoff_t hugetlb_basepage_index(struct page *page)
return (index << compound_order(page_head)) + compound_idx;
}
-static struct page *alloc_buddy_huge_page(struct hstate *h,
+static struct folio *alloc_buddy_hugetlb_folio(struct hstate *h,
gfp_t gfp_mask, int nid, nodemask_t *nmask,
nodemask_t *node_alloc_noretry)
{
@@ -2007,7 +2006,7 @@ static struct page *alloc_buddy_huge_page(struct hstate *h,
if (node_alloc_noretry && !page && alloc_try_hard)
node_set(nid, *node_alloc_noretry);
- return page;
+ return page_folio(page);
}
/*
@@ -2017,23 +2016,21 @@ static struct page *alloc_buddy_huge_page(struct hstate *h,
* Note that returned page is 'frozen': ref count of head page and all tail
* pages is zero.
*/
-static struct page *alloc_fresh_huge_page(struct hstate *h,
+static struct folio *alloc_fresh_hugetlb_folio(struct hstate *h,
gfp_t gfp_mask, int nid, nodemask_t *nmask,
nodemask_t *node_alloc_noretry)
{
- struct page *page;
struct folio *folio;
bool retry = false;
retry:
if (hstate_is_gigantic(h))
- page = alloc_gigantic_page(h, gfp_mask, nid, nmask);
+ folio = alloc_gigantic_folio(h, gfp_mask, nid, nmask);
else
- page = alloc_buddy_huge_page(h, gfp_mask,
+ folio = alloc_buddy_hugetlb_folio(h, gfp_mask,
nid, nmask, node_alloc_noretry);
- if (!page)
+ if (!folio)
return NULL;
- folio = page_folio(page);
if (hstate_is_gigantic(h)) {
if (!prep_compound_gigantic_folio(folio, huge_page_order(h))) {
/*
@@ -2050,7 +2047,7 @@ static struct page *alloc_fresh_huge_page(struct hstate *h,
}
prep_new_hugetlb_folio(h, folio, folio_nid(folio));
- return page;
+ return folio;
}
/*
@@ -2060,21 +2057,21 @@ static struct page *alloc_fresh_huge_page(struct hstate *h,
static int alloc_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
nodemask_t *node_alloc_noretry)
{
- struct page *page;
+ struct folio *folio;
int nr_nodes, node;
gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
- page = alloc_fresh_huge_page(h, gfp_mask, node, nodes_allowed,
- node_alloc_noretry);
- if (page)
+ folio = alloc_fresh_hugetlb_folio(h, gfp_mask, node,
+ nodes_allowed, node_alloc_noretry);
+ if (folio)
break;
}
- if (!page)
+ if (!folio)
return 0;
- free_huge_page(page); /* free it into the hugepage allocator */
+ free_huge_page(&folio->page); /* free it into the hugepage allocator */
return 1;
}
@@ -2235,7 +2232,7 @@ int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask,
int nid, nodemask_t *nmask)
{
- struct page *page = NULL;
+ struct folio *folio = NULL;
if (hstate_is_gigantic(h))
return NULL;
@@ -2245,8 +2242,8 @@ static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask,
goto out_unlock;
spin_unlock_irq(&hugetlb_lock);
- page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL);
- if (!page)
+ folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask, NULL);
+ if (!folio)
return NULL;
spin_lock_irq(&hugetlb_lock);
@@ -2258,43 +2255,42 @@ static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask,
* codeflow
*/
if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
- SetHPageTemporary(page);
+ folio_set_hugetlb_temporary(folio);
spin_unlock_irq(&hugetlb_lock);
- free_huge_page(page);
+ free_huge_page(&folio->page);
return NULL;
}
h->surplus_huge_pages++;
- h->surplus_huge_pages_node[page_to_nid(page)]++;
+ h->surplus_huge_pages_node[folio_nid(folio)]++;
out_unlock:
spin_unlock_irq(&hugetlb_lock);
- return page;
+ return &folio->page;
}
static struct page *alloc_migrate_huge_page(struct hstate *h, gfp_t gfp_mask,
int nid, nodemask_t *nmask)
{
- struct page *page;
+ struct folio *folio;
if (hstate_is_gigantic(h))
return NULL;
- page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL);
- if (!page)
+ folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid, nmask, NULL);
+ if (!folio)
return NULL;
/* fresh huge pages are frozen */
- set_page_refcounted(page);
-
+ folio_ref_unfreeze(folio, 1);
/*
* We do not account these pages as surplus because they are only
* temporary and will be released properly on the last reference
*/
- SetHPageTemporary(page);
+ folio_set_hugetlb_temporary(folio);
- return page;
+ return &folio->page;
}
/*
@@ -2743,19 +2739,18 @@ void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma,
}
/*
- * alloc_and_dissolve_huge_page - Allocate a new page and dissolve the old one
+ * alloc_and_dissolve_hugetlb_folio - Allocate a new folio and dissolve
+ * the old one
* @h: struct hstate old page belongs to
* @old_page: Old page to dissolve
* @list: List to isolate the page in case we need to
* Returns 0 on success, otherwise negated error.
*/
-static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,
- struct list_head *list)
+static int alloc_and_dissolve_hugetlb_folio(struct hstate *h,
+ struct folio *old_folio, struct list_head *list)
{
gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
- struct folio *old_folio = page_folio(old_page);
int nid = folio_nid(old_folio);
- struct page *new_page;
struct folio *new_folio;
int ret = 0;
@@ -2766,26 +2761,25 @@ static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,
* the pool. This simplifies and let us do most of the processing
* under the lock.
*/
- new_page = alloc_buddy_huge_page(h, gfp_mask, nid, NULL, NULL);
- if (!new_page)
+ new_folio = alloc_buddy_hugetlb_folio(h, gfp_mask, nid, NULL, NULL);
+ if (!new_folio)
return -ENOMEM;
- new_folio = page_folio(new_page);
__prep_new_hugetlb_folio(h, new_folio);
retry:
spin_lock_irq(&hugetlb_lock);
if (!folio_test_hugetlb(old_folio)) {
/*
- * Freed from under us. Drop new_page too.
+ * Freed from under us. Drop new_folio too.
*/
goto free_new;
} else if (folio_ref_count(old_folio)) {
/*
- * Someone has grabbed the page, try to isolate it here.
+ * Someone has grabbed the folio, try to isolate it here.
* Fail with -EBUSY if not possible.
*/
spin_unlock_irq(&hugetlb_lock);
- ret = isolate_hugetlb(old_page, list);
+ ret = isolate_hugetlb(&old_folio->page, list);
spin_lock_irq(&hugetlb_lock);
goto free_new;
} else if (!folio_test_hugetlb_freed(old_folio)) {
@@ -2863,7 +2857,7 @@ int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list)
if (folio_ref_count(folio) && !isolate_hugetlb(&folio->page, list))
ret = 0;
else if (!folio_ref_count(folio))
- ret = alloc_and_dissolve_huge_page(h, &folio->page, list);
+ ret = alloc_and_dissolve_hugetlb_folio(h, folio, list);
return ret;
}
@@ -3081,14 +3075,14 @@ static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid)
if (!alloc_bootmem_huge_page(h, nid))
break;
} else {
- struct page *page;
+ struct folio *folio;
gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
- page = alloc_fresh_huge_page(h, gfp_mask, nid,
+ folio = alloc_fresh_hugetlb_folio(h, gfp_mask, nid,
&node_states[N_MEMORY], NULL);
- if (!page)
+ if (!folio)
break;
- free_huge_page(page); /* free it into the hugepage allocator */
+ free_huge_page(&folio->page); /* free it into the hugepage allocator */
}
cond_resched();
}
--
2.38.1
^ permalink raw reply [flat|nested] 13+ messages in thread