From: Kefeng Wang <wangkefeng.wang@huawei.com>
To: Andrew Morton <akpm@linux-foundation.org>,
David Hildenbrand <david@redhat.com>,
Oscar Salvador <osalvador@suse.de>,
Muchun Song <muchun.song@linux.dev>, <linux-mm@kvack.org>
Cc: <sidhartha.kumar@oracle.com>, <jane.chu@oracle.com>,
Zi Yan <ziy@nvidia.com>, Vlastimil Babka <vbabka@suse.cz>,
Brendan Jackman <jackmanb@google.com>,
Johannes Weiner <hannes@cmpxchg.org>,
Matthew Wilcox <willy@infradead.org>,
Kefeng Wang <wangkefeng.wang@huawei.com>
Subject: [PATCH v5 6/6] mm: hugetlb: allocate frozen pages for gigantic allocation
Date: Tue, 30 Dec 2025 15:24:22 +0800 [thread overview]
Message-ID: <20251230072422.265265-7-wangkefeng.wang@huawei.com> (raw)
In-Reply-To: <20251230072422.265265-1-wangkefeng.wang@huawei.com>
The alloc_gigantic_folio() allocates a folio with refcount increated
and then freeze it, convert to allocate a frozen folio to remove the
atomic operation about folio refcount, and saving atomic operation
during __update_and_free_hugetlb_folio() too.
Besides, rename hugetlb_cma_{alloc,free}_folio(), alloc_gigantic_folio()
and alloc_buddy_hugetlb_folio() with frozen which make them more
self-explanatory.
Signed-off-by: Kefeng Wang <wangkefeng.wang@huawei.com>
---
mm/hugetlb.c | 75 +++++++++++++-----------------------------------
mm/hugetlb_cma.c | 9 ++----
mm/hugetlb_cma.h | 10 +++----
3 files changed, 28 insertions(+), 66 deletions(-)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index c990e439c32e..cb296c6912f6 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -121,16 +121,6 @@ static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
unsigned long start, unsigned long end, bool take_locks);
static struct resv_map *vma_resv_map(struct vm_area_struct *vma);
-static void hugetlb_free_folio(struct folio *folio)
-{
- if (folio_test_hugetlb_cma(folio)) {
- hugetlb_cma_free_folio(folio);
- return;
- }
-
- folio_put(folio);
-}
-
static inline bool subpool_is_free(struct hugepage_subpool *spool)
{
if (spool->count)
@@ -1417,52 +1407,25 @@ static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h,
return NULL;
}
-#ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE
-#ifdef CONFIG_CONTIG_ALLOC
-static struct folio *alloc_gigantic_folio(int order, gfp_t gfp_mask,
+#if defined(CONFIG_ARCH_HAS_GIGANTIC_PAGE) && defined(CONFIG_CONTIG_ALLOC)
+static struct folio *alloc_gigantic_frozen_folio(int order, gfp_t gfp_mask,
int nid, nodemask_t *nodemask)
{
struct folio *folio;
- bool retried = false;
-retry:
- folio = hugetlb_cma_alloc_folio(order, gfp_mask, nid, nodemask);
- if (!folio) {
- struct page *page;
-
- if (hugetlb_cma_exclusive_alloc())
- return NULL;
-
- page = alloc_contig_frozen_pages(1 << order, gfp_mask, nid, nodemask);
- if (!page)
- return NULL;
-
- set_page_refcounted(page);
- folio = page_folio(page);
- }
-
- if (folio_ref_freeze(folio, 1))
+ folio = hugetlb_cma_alloc_frozen_folio(order, gfp_mask, nid, nodemask);
+ if (folio)
return folio;
- pr_warn("HugeTLB: unexpected refcount on PFN %lu\n", folio_pfn(folio));
- hugetlb_free_folio(folio);
- if (!retried) {
- retried = true;
- goto retry;
- }
- return NULL;
-}
+ if (hugetlb_cma_exclusive_alloc())
+ return NULL;
-#else /* !CONFIG_CONTIG_ALLOC */
-static struct folio *alloc_gigantic_folio(int order, gfp_t gfp_mask, int nid,
- nodemask_t *nodemask)
-{
- return NULL;
+ folio = (struct folio *)alloc_contig_frozen_pages(1 << order, gfp_mask,
+ nid, nodemask);
+ return folio;
}
-#endif /* CONFIG_CONTIG_ALLOC */
-
-#else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */
-static struct folio *alloc_gigantic_folio(int order, gfp_t gfp_mask, int nid,
+#else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE || !CONFIG_CONTIG_ALLOC */
+static struct folio *alloc_gigantic_frozen_folio(int order, gfp_t gfp_mask, int nid,
nodemask_t *nodemask)
{
return NULL;
@@ -1592,9 +1555,11 @@ static void __update_and_free_hugetlb_folio(struct hstate *h,
if (unlikely(folio_test_hwpoison(folio)))
folio_clear_hugetlb_hwpoison(folio);
- folio_ref_unfreeze(folio, 1);
-
- hugetlb_free_folio(folio);
+ VM_BUG_ON_FOLIO(folio_ref_count(folio), folio);
+ if (folio_test_hugetlb_cma(folio))
+ hugetlb_cma_free_frozen_folio(folio);
+ else
+ free_frozen_pages(&folio->page, folio_order(folio));
}
/*
@@ -1874,7 +1839,7 @@ struct address_space *hugetlb_folio_mapping_lock_write(struct folio *folio)
return NULL;
}
-static struct folio *alloc_buddy_hugetlb_folio(int order, gfp_t gfp_mask,
+static struct folio *alloc_buddy_frozen_folio(int order, gfp_t gfp_mask,
int nid, nodemask_t *nmask, nodemask_t *node_alloc_noretry)
{
struct folio *folio;
@@ -1930,10 +1895,10 @@ static struct folio *only_alloc_fresh_hugetlb_folio(struct hstate *h,
nid = numa_mem_id();
if (order_is_gigantic(order))
- folio = alloc_gigantic_folio(order, gfp_mask, nid, nmask);
+ folio = alloc_gigantic_frozen_folio(order, gfp_mask, nid, nmask);
else
- folio = alloc_buddy_hugetlb_folio(order, gfp_mask, nid, nmask,
- node_alloc_noretry);
+ folio = alloc_buddy_frozen_folio(order, gfp_mask, nid, nmask,
+ node_alloc_noretry);
if (folio)
init_new_hugetlb_folio(folio);
return folio;
diff --git a/mm/hugetlb_cma.c b/mm/hugetlb_cma.c
index 9469a7bd673f..e9f63648df6d 100644
--- a/mm/hugetlb_cma.c
+++ b/mm/hugetlb_cma.c
@@ -18,16 +18,14 @@ static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata;
static bool hugetlb_cma_only;
static unsigned long hugetlb_cma_size __initdata;
-void hugetlb_cma_free_folio(struct folio *folio)
+void hugetlb_cma_free_frozen_folio(struct folio *folio)
{
- folio_ref_dec(folio);
-
WARN_ON_ONCE(!cma_release_frozen(hugetlb_cma[folio_nid(folio)],
&folio->page, folio_nr_pages(folio)));
}
-struct folio *hugetlb_cma_alloc_folio(int order, gfp_t gfp_mask,
- int nid, nodemask_t *nodemask)
+struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask,
+ int nid, nodemask_t *nodemask)
{
int node;
struct folio *folio;
@@ -50,7 +48,6 @@ struct folio *hugetlb_cma_alloc_folio(int order, gfp_t gfp_mask,
if (!page)
return NULL;
- set_page_refcounted(page);
folio = page_folio(page);
folio_set_hugetlb_cma(folio);
return folio;
diff --git a/mm/hugetlb_cma.h b/mm/hugetlb_cma.h
index 2c2ec8a7e134..3bc295c8c38e 100644
--- a/mm/hugetlb_cma.h
+++ b/mm/hugetlb_cma.h
@@ -3,8 +3,8 @@
#define _LINUX_HUGETLB_CMA_H
#ifdef CONFIG_CMA
-void hugetlb_cma_free_folio(struct folio *folio);
-struct folio *hugetlb_cma_alloc_folio(int order, gfp_t gfp_mask,
+void hugetlb_cma_free_frozen_folio(struct folio *folio);
+struct folio *hugetlb_cma_alloc_frozen_folio(int order, gfp_t gfp_mask,
int nid, nodemask_t *nodemask);
struct huge_bootmem_page *hugetlb_cma_alloc_bootmem(struct hstate *h, int *nid,
bool node_exact);
@@ -14,12 +14,12 @@ unsigned long hugetlb_cma_total_size(void);
void hugetlb_cma_validate_params(void);
bool hugetlb_early_cma(struct hstate *h);
#else
-static inline void hugetlb_cma_free_folio(struct folio *folio)
+static inline void hugetlb_cma_free_frozen_folio(struct folio *folio)
{
}
-static inline struct folio *hugetlb_cma_alloc_folio(int order, gfp_t gfp_mask,
- int nid, nodemask_t *nodemask)
+static inline struct folio *hugetlb_cma_alloc_frozen_folio(int order,
+ gfp_t gfp_mask, int nid, nodemask_t *nodemask)
{
return NULL;
}
--
2.27.0
next prev parent reply other threads:[~2025-12-30 7:25 UTC|newest]
Thread overview: 12+ messages / expand[flat|nested] mbox.gz Atom feed top
2025-12-30 7:24 [PATCH v5 mm-new 0/6] mm: hugetlb: allocate frozen gigantic folio Kefeng Wang
2025-12-30 7:24 ` [PATCH v5 1/6] mm: debug_vm_pgtable: add debug_vm_pgtable_free_huge_page() Kefeng Wang
2025-12-30 7:24 ` [PATCH v5 2/6] mm: page_alloc: add __split_page() Kefeng Wang
2025-12-30 7:24 ` [PATCH v5 3/6] mm: cma: kill cma_pages_valid() Kefeng Wang
2025-12-30 7:24 ` [PATCH v5 4/6] mm: page_alloc: add alloc_contig_frozen_{range,pages}() Kefeng Wang
2025-12-31 2:57 ` Zi Yan
2025-12-30 7:24 ` [PATCH v5 5/6] mm: cma: add cma_alloc_frozen{_compound}() Kefeng Wang
2025-12-31 2:59 ` Zi Yan
2025-12-30 7:24 ` Kefeng Wang [this message]
2025-12-31 2:50 ` [PATCH v5 6/6] mm: hugetlb: allocate frozen pages for gigantic allocation Muchun Song
2025-12-31 3:00 ` Zi Yan
2025-12-30 18:17 ` [PATCH v5 mm-new 0/6] mm: hugetlb: allocate frozen gigantic folio Andrew Morton
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20251230072422.265265-7-wangkefeng.wang@huawei.com \
--to=wangkefeng.wang@huawei.com \
--cc=akpm@linux-foundation.org \
--cc=david@redhat.com \
--cc=hannes@cmpxchg.org \
--cc=jackmanb@google.com \
--cc=jane.chu@oracle.com \
--cc=linux-mm@kvack.org \
--cc=muchun.song@linux.dev \
--cc=osalvador@suse.de \
--cc=sidhartha.kumar@oracle.com \
--cc=vbabka@suse.cz \
--cc=willy@infradead.org \
--cc=ziy@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox