* [RFC 2/4] mm, page_alloc: reduce number of alloc_pages* functions' parameters
2014-08-06 13:55 [RFC 0/4] Reducing parameters of alloc_pages* family of functions Vlastimil Babka
2014-08-06 13:55 ` [PATCH 1/4] mm: page_alloc: determine migratetype only once Vlastimil Babka
@ 2014-08-06 13:55 ` Vlastimil Babka
2014-08-06 13:55 ` [RFC 3/4] mm, page_alloc: make alloc_flags a separate parameter again Vlastimil Babka
2014-08-06 13:55 ` [RFC 4/4] mm, page_alloc: make gfp_mask " Vlastimil Babka
3 siblings, 0 replies; 6+ messages in thread
From: Vlastimil Babka @ 2014-08-06 13:55 UTC (permalink / raw)
To: linux-mm
Cc: linux-kernel, David Rientjes, Andrew Morton, Mel Gorman,
Rik van Riel, Johannes Weiner, Srivatsa S. Bhat, Hugh Dickins,
Minchan Kim, Joonsoo Kim, Vlastimil Babka
Introduce struct alloc_info to accumulate all the numerous parameters passed
between the alloc_pages* family of functions and get_page_from_freelist().
---
mm/page_alloc.c | 241 +++++++++++++++++++++++++++-----------------------------
1 file changed, 118 insertions(+), 123 deletions(-)
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index b2cd463..399d40d 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -231,6 +231,19 @@ EXPORT_SYMBOL(nr_node_ids);
EXPORT_SYMBOL(nr_online_nodes);
#endif
+struct alloc_info {
+ struct zonelist *zonelist;
+ nodemask_t *nodemask;
+ struct zone *preferred_zone;
+
+ unsigned int order;
+ gfp_t gfp_mask;
+ int alloc_flags;
+ int classzone_idx;
+ int migratetype;
+ enum zone_type high_zoneidx;
+};
+
int page_group_by_mobility_disabled __read_mostly;
void set_pageblock_migratetype(struct page *page, int migratetype)
@@ -1943,10 +1956,11 @@ static void reset_alloc_batches(struct zone *preferred_zone)
* a page.
*/
static struct page *
-get_page_from_freelist(gfp_t gfp_mask, nodemask_t *nodemask, unsigned int order,
- struct zonelist *zonelist, int high_zoneidx, int alloc_flags,
- struct zone *preferred_zone, int classzone_idx, int migratetype)
+get_page_from_freelist(const struct alloc_info *ai)
{
+ const unsigned int order = ai->order;
+ int alloc_flags = ai->alloc_flags;
+ struct zonelist *zonelist = ai->zonelist;
struct zoneref *z;
struct page *page = NULL;
struct zone *zone;
@@ -1954,7 +1968,7 @@ get_page_from_freelist(gfp_t gfp_mask, nodemask_t *nodemask, unsigned int order,
int zlc_active = 0; /* set if using zonelist_cache */
int did_zlc_setup = 0; /* just call zlc_setup() one time */
bool consider_zone_dirty = (alloc_flags & ALLOC_WMARK_LOW) &&
- (gfp_mask & __GFP_WRITE);
+ (ai->gfp_mask & __GFP_WRITE);
int nr_fair_skipped = 0;
bool zonelist_rescan;
@@ -1966,7 +1980,7 @@ zonelist_scan:
* See also __cpuset_node_allowed_softwall() comment in kernel/cpuset.c.
*/
for_each_zone_zonelist_nodemask(zone, z, zonelist,
- high_zoneidx, nodemask) {
+ ai->high_zoneidx, ai->nodemask) {
unsigned long mark;
if (IS_ENABLED(CONFIG_NUMA) && zlc_active &&
@@ -1974,7 +1988,7 @@ zonelist_scan:
continue;
if (cpusets_enabled() &&
(alloc_flags & ALLOC_CPUSET) &&
- !cpuset_zone_allowed_softwall(zone, gfp_mask))
+ !cpuset_zone_allowed_softwall(zone, ai->gfp_mask))
continue;
/*
* Distribute pages in proportion to the individual
@@ -1983,7 +1997,7 @@ zonelist_scan:
* time the page has in memory before being reclaimed.
*/
if (alloc_flags & ALLOC_FAIR) {
- if (!zone_local(preferred_zone, zone))
+ if (!zone_local(ai->preferred_zone, zone))
break;
if (zone_is_fair_depleted(zone)) {
nr_fair_skipped++;
@@ -2021,7 +2035,7 @@ zonelist_scan:
mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
if (!zone_watermark_ok(zone, order, mark,
- classzone_idx, alloc_flags)) {
+ ai->classzone_idx, alloc_flags)) {
int ret;
/* Checked here to keep the fast path fast */
@@ -2042,7 +2056,7 @@ zonelist_scan:
}
if (zone_reclaim_mode == 0 ||
- !zone_allows_reclaim(preferred_zone, zone))
+ !zone_allows_reclaim(ai->preferred_zone, zone))
goto this_zone_full;
/*
@@ -2053,7 +2067,7 @@ zonelist_scan:
!zlc_zone_worth_trying(zonelist, z, allowednodes))
continue;
- ret = zone_reclaim(zone, gfp_mask, order);
+ ret = zone_reclaim(zone, ai->gfp_mask, order);
switch (ret) {
case ZONE_RECLAIM_NOSCAN:
/* did not scan */
@@ -2064,7 +2078,7 @@ zonelist_scan:
default:
/* did we reclaim enough */
if (zone_watermark_ok(zone, order, mark,
- classzone_idx, alloc_flags))
+ ai->classzone_idx, alloc_flags))
goto try_this_zone;
/*
@@ -2085,8 +2099,8 @@ zonelist_scan:
}
try_this_zone:
- page = buffered_rmqueue(preferred_zone, zone, order,
- gfp_mask, migratetype);
+ page = buffered_rmqueue(ai->preferred_zone, zone, order,
+ ai->gfp_mask, ai->migratetype);
if (page)
break;
this_zone_full:
@@ -2118,7 +2132,7 @@ this_zone_full:
alloc_flags &= ~ALLOC_FAIR;
if (nr_fair_skipped) {
zonelist_rescan = true;
- reset_alloc_batches(preferred_zone);
+ reset_alloc_batches(ai->preferred_zone);
}
if (nr_online_nodes > 1)
zonelist_rescan = true;
@@ -2239,15 +2253,14 @@ should_alloc_retry(gfp_t gfp_mask, unsigned int order,
}
static inline struct page *
-__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
- struct zonelist *zonelist, enum zone_type high_zoneidx,
- nodemask_t *nodemask, struct zone *preferred_zone,
- int classzone_idx, int migratetype)
+__alloc_pages_may_oom(struct alloc_info *ai)
{
struct page *page;
+ const gfp_t gfp_mask = ai->gfp_mask;
+ const int alloc_flags_saved = ai->alloc_flags;
/* Acquire the per-zone oom lock for each zone */
- if (!oom_zonelist_trylock(zonelist, gfp_mask)) {
+ if (!oom_zonelist_trylock(ai->zonelist, ai->gfp_mask)) {
schedule_timeout_uninterruptible(1);
return NULL;
}
@@ -2257,19 +2270,21 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
* here, this is only to catch a parallel oom killing, we must fail if
* we're still under heavy pressure.
*/
- page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask,
- order, zonelist, high_zoneidx,
- ALLOC_WMARK_HIGH|ALLOC_CPUSET,
- preferred_zone, classzone_idx, migratetype);
+ ai->gfp_mask |= __GFP_HARDWALL;
+ ai->alloc_flags = ALLOC_WMARK_HIGH|ALLOC_CPUSET;
+ page = get_page_from_freelist(ai);
+ ai->gfp_mask = gfp_mask;
+ ai->alloc_flags = alloc_flags_saved;
+
if (page)
goto out;
if (!(gfp_mask & __GFP_NOFAIL)) {
/* The OOM killer will not help higher order allocs */
- if (order > PAGE_ALLOC_COSTLY_ORDER)
+ if (ai->order > PAGE_ALLOC_COSTLY_ORDER)
goto out;
/* The OOM killer does not needlessly kill tasks for lowmem */
- if (high_zoneidx < ZONE_NORMAL)
+ if (ai->high_zoneidx < ZONE_NORMAL)
goto out;
/*
* GFP_THISNODE contains __GFP_NORETRY and we never hit this.
@@ -2282,23 +2297,24 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
goto out;
}
/* Exhausted what can be done so it's blamo time */
- out_of_memory(zonelist, gfp_mask, order, nodemask, false);
+ out_of_memory(ai->zonelist, gfp_mask, ai->order, ai->nodemask, false);
out:
- oom_zonelist_unlock(zonelist, gfp_mask);
+ oom_zonelist_unlock(ai->zonelist, gfp_mask);
return page;
}
#ifdef CONFIG_COMPACTION
/* Try memory compaction for high-order allocations before reclaim */
static struct page *
-__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
- struct zonelist *zonelist, enum zone_type high_zoneidx,
- nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
- int classzone_idx, int migratetype, enum migrate_mode mode,
+__alloc_pages_direct_compact(struct alloc_info *ai, enum migrate_mode mode,
bool *contended_compaction, bool *deferred_compaction,
unsigned long *did_some_progress)
{
+ const unsigned int order = ai->order;
+ struct zone *preferred_zone = ai->preferred_zone;
+ const int alloc_flags_saved = ai->alloc_flags;
+
if (!order)
return NULL;
@@ -2308,8 +2324,8 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
}
current->flags |= PF_MEMALLOC;
- *did_some_progress = try_to_compact_pages(zonelist, order, gfp_mask,
- nodemask, mode,
+ *did_some_progress = try_to_compact_pages(ai->zonelist, order, ai->gfp_mask,
+ ai->nodemask, mode,
contended_compaction);
current->flags &= ~PF_MEMALLOC;
@@ -2320,10 +2336,10 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
drain_pages(get_cpu());
put_cpu();
- page = get_page_from_freelist(gfp_mask, nodemask,
- order, zonelist, high_zoneidx,
- alloc_flags & ~ALLOC_NO_WATERMARKS,
- preferred_zone, classzone_idx, migratetype);
+ ai->alloc_flags &= ~ALLOC_NO_WATERMARKS;
+ page = get_page_from_freelist(ai);
+ ai->alloc_flags = alloc_flags_saved;
+
if (page) {
preferred_zone->compact_blockskip_flush = false;
compaction_defer_reset(preferred_zone, order, true);
@@ -2352,12 +2368,9 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
}
#else
static inline struct page *
-__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
- struct zonelist *zonelist, enum zone_type high_zoneidx,
- nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
- int classzone_idx, int migratetype,
- enum migrate_mode mode, bool *contended_compaction,
- bool *deferred_compaction, unsigned long *did_some_progress)
+__alloc_pages_direct_compact(struct alloc_info *ai, enum migrate_mode mode,
+ bool *contended_compaction, bool *deferred_compaction,
+ unsigned long *did_some_progress)
{
return NULL;
}
@@ -2393,29 +2406,25 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order, struct zonelist *zonelist,
/* The really slow allocator path where we enter direct reclaim */
static inline struct page *
-__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
- struct zonelist *zonelist, enum zone_type high_zoneidx,
- nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
- int classzone_idx, int migratetype, unsigned long *did_some_progress)
+__alloc_pages_direct_reclaim(struct alloc_info *ai,
+ unsigned long *did_some_progress)
{
struct page *page = NULL;
bool drained = false;
+ const int alloc_flags_saved = ai->alloc_flags;
- *did_some_progress = __perform_reclaim(gfp_mask, order, zonelist,
- nodemask);
+ *did_some_progress = __perform_reclaim(ai->gfp_mask, ai->order,
+ ai->zonelist, ai->nodemask);
if (unlikely(!(*did_some_progress)))
return NULL;
/* After successful reclaim, reconsider all zones for allocation */
if (IS_ENABLED(CONFIG_NUMA))
- zlc_clear_zones_full(zonelist);
+ zlc_clear_zones_full(ai->zonelist);
+ ai->alloc_flags &= ~ALLOC_NO_WATERMARKS;
retry:
- page = get_page_from_freelist(gfp_mask, nodemask, order,
- zonelist, high_zoneidx,
- alloc_flags & ~ALLOC_NO_WATERMARKS,
- preferred_zone, classzone_idx,
- migratetype);
+ page = get_page_from_freelist(ai);
/*
* If an allocation failed after direct reclaim, it could be because
@@ -2427,6 +2436,7 @@ retry:
goto retry;
}
+ ai->alloc_flags = alloc_flags_saved;
return page;
}
@@ -2435,22 +2445,21 @@ retry:
* sufficient urgency to ignore watermarks and take other desperate measures
*/
static inline struct page *
-__alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order,
- struct zonelist *zonelist, enum zone_type high_zoneidx,
- nodemask_t *nodemask, struct zone *preferred_zone,
- int classzone_idx, int migratetype)
+__alloc_pages_high_priority(struct alloc_info *ai)
{
struct page *page;
+ const int alloc_flags_saved = ai->alloc_flags;
+ ai->alloc_flags = ALLOC_NO_WATERMARKS;
do {
- page = get_page_from_freelist(gfp_mask, nodemask, order,
- zonelist, high_zoneidx, ALLOC_NO_WATERMARKS,
- preferred_zone, classzone_idx, migratetype);
+ page = get_page_from_freelist(ai);
- if (!page && gfp_mask & __GFP_NOFAIL)
- wait_iff_congested(preferred_zone, BLK_RW_ASYNC, HZ/50);
- } while (!page && (gfp_mask & __GFP_NOFAIL));
+ if (!page && ai->gfp_mask & __GFP_NOFAIL)
+ wait_iff_congested(ai->preferred_zone, BLK_RW_ASYNC,
+ HZ/50);
+ } while (!page && (ai->gfp_mask & __GFP_NOFAIL));
+ ai->alloc_flags = alloc_flags_saved;
return page;
}
@@ -2521,11 +2530,10 @@ bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
}
static inline struct page *
-__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
- struct zonelist *zonelist, enum zone_type high_zoneidx,
- nodemask_t *nodemask, struct zone *preferred_zone,
- int classzone_idx, int migratetype)
+__alloc_pages_slowpath(struct alloc_info *ai)
{
+ const unsigned int order = ai->order;
+ const gfp_t gfp_mask = ai->gfp_mask;
const gfp_t wait = gfp_mask & __GFP_WAIT;
struct page *page = NULL;
int alloc_flags;
@@ -2560,7 +2568,8 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
restart:
if (!(gfp_mask & __GFP_NO_KSWAPD))
- wake_all_kswapds(order, zonelist, high_zoneidx, preferred_zone);
+ wake_all_kswapds(order, ai->zonelist, ai->high_zoneidx,
+ ai->preferred_zone);
/*
* OK, we're below the kswapd watermark and have kicked background
@@ -2573,33 +2582,33 @@ restart:
* Find the true preferred zone if the allocation is unconstrained by
* cpusets.
*/
- if (!(alloc_flags & ALLOC_CPUSET) && !nodemask) {
+ if (!(alloc_flags & ALLOC_CPUSET) && !ai->nodemask) {
struct zoneref *preferred_zoneref;
- preferred_zoneref = first_zones_zonelist(zonelist, high_zoneidx,
- NULL, &preferred_zone);
- classzone_idx = zonelist_zone_idx(preferred_zoneref);
+ preferred_zoneref = first_zones_zonelist(ai->zonelist,
+ ai->high_zoneidx, NULL, &ai->preferred_zone);
+ ai->classzone_idx = zonelist_zone_idx(preferred_zoneref);
}
rebalance:
/* This is the last chance, in general, before the goto nopage. */
- page = get_page_from_freelist(gfp_mask, nodemask, order, zonelist,
- high_zoneidx, alloc_flags & ~ALLOC_NO_WATERMARKS,
- preferred_zone, classzone_idx, migratetype);
+ ai->alloc_flags = alloc_flags & ~ALLOC_NO_WATERMARKS;
+ page = get_page_from_freelist(ai);
if (page)
goto got_pg;
/* Allocate without watermarks if the context allows */
if (alloc_flags & ALLOC_NO_WATERMARKS) {
+ /* We have removed ALLOC_NO_WATERMARKS from alloc_info */
+ ai->alloc_flags = alloc_flags;
/*
* Ignore mempolicies if ALLOC_NO_WATERMARKS on the grounds
* the allocation is high priority and these type of
* allocations are system rather than user orientated
*/
- zonelist = node_zonelist(numa_node_id(), gfp_mask);
+ ai->zonelist = node_zonelist(numa_node_id(), gfp_mask);
+
+ page = __alloc_pages_high_priority(ai);
- page = __alloc_pages_high_priority(gfp_mask, order,
- zonelist, high_zoneidx, nodemask,
- preferred_zone, classzone_idx, migratetype);
if (page) {
goto got_pg;
}
@@ -2628,11 +2637,8 @@ rebalance:
* Try direct compaction. The first pass is asynchronous. Subsequent
* attempts after direct reclaim are synchronous
*/
- page = __alloc_pages_direct_compact(gfp_mask, order, zonelist,
- high_zoneidx, nodemask, alloc_flags,
- preferred_zone,
- classzone_idx, migratetype,
- migration_mode, &contended_compaction,
+ page = __alloc_pages_direct_compact(ai, migration_mode,
+ &contended_compaction,
&deferred_compaction,
&did_some_progress);
if (page)
@@ -2658,12 +2664,7 @@ rebalance:
migration_mode = MIGRATE_SYNC_LIGHT;
/* Try direct reclaim and then allocating */
- page = __alloc_pages_direct_reclaim(gfp_mask, order,
- zonelist, high_zoneidx,
- nodemask,
- alloc_flags, preferred_zone,
- classzone_idx, migratetype,
- &did_some_progress);
+ page = __alloc_pages_direct_reclaim(ai, &did_some_progress);
if (page)
goto got_pg;
@@ -2679,10 +2680,7 @@ rebalance:
if ((current->flags & PF_DUMPCORE) &&
!(gfp_mask & __GFP_NOFAIL))
goto nopage;
- page = __alloc_pages_may_oom(gfp_mask, order,
- zonelist, high_zoneidx,
- nodemask, preferred_zone,
- classzone_idx, migratetype);
+ page = __alloc_pages_may_oom(ai);
if (page)
goto got_pg;
@@ -2700,7 +2698,7 @@ rebalance:
* allocations to prevent needlessly killing
* innocent tasks.
*/
- if (high_zoneidx < ZONE_NORMAL)
+ if (ai->high_zoneidx < ZONE_NORMAL)
goto nopage;
}
@@ -2713,7 +2711,7 @@ rebalance:
if (should_alloc_retry(gfp_mask, order, did_some_progress,
pages_reclaimed)) {
/* Wait for some write requests to complete then retry */
- wait_iff_congested(preferred_zone, BLK_RW_ASYNC, HZ/50);
+ wait_iff_congested(ai->preferred_zone, BLK_RW_ASYNC, HZ/50);
goto rebalance;
} else {
/*
@@ -2721,11 +2719,8 @@ rebalance:
* direct reclaim and reclaim/compaction depends on compaction
* being called after reclaim so call directly if necessary
*/
- page = __alloc_pages_direct_compact(gfp_mask, order, zonelist,
- high_zoneidx, nodemask, alloc_flags,
- preferred_zone,
- classzone_idx, migratetype,
- migration_mode, &contended_compaction,
+ page = __alloc_pages_direct_compact(ai, migration_mode,
+ &contended_compaction,
&deferred_compaction,
&did_some_progress);
if (page)
@@ -2749,14 +2744,17 @@ struct page *
__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
struct zonelist *zonelist, nodemask_t *nodemask)
{
- enum zone_type high_zoneidx = gfp_zone(gfp_mask);
- struct zone *preferred_zone;
struct zoneref *preferred_zoneref;
struct page *page = NULL;
- int migratetype = allocflags_to_migratetype(gfp_mask);
unsigned int cpuset_mems_cookie;
- int alloc_flags = ALLOC_WMARK_LOW|ALLOC_CPUSET|ALLOC_FAIR;
- int classzone_idx;
+ struct alloc_info ai = {
+ .order = order,
+ .alloc_flags = ALLOC_WMARK_LOW|ALLOC_CPUSET|ALLOC_FAIR,
+ .zonelist = zonelist,
+ .high_zoneidx = gfp_zone(gfp_mask),
+ .nodemask = nodemask,
+ .migratetype = allocflags_to_migratetype(gfp_mask),
+ };
gfp_mask &= gfp_allowed_mask;
@@ -2775,37 +2773,34 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
if (unlikely(!zonelist->_zonerefs->zone))
return NULL;
- if (IS_ENABLED(CONFIG_CMA) && migratetype == MIGRATE_MOVABLE)
- alloc_flags |= ALLOC_CMA;
+ if (IS_ENABLED(CONFIG_CMA) && ai.migratetype == MIGRATE_MOVABLE)
+ ai.alloc_flags |= ALLOC_CMA;
retry_cpuset:
cpuset_mems_cookie = read_mems_allowed_begin();
/* The preferred zone is used for statistics later */
- preferred_zoneref = first_zones_zonelist(zonelist, high_zoneidx,
- nodemask ? : &cpuset_current_mems_allowed,
- &preferred_zone);
- if (!preferred_zone)
+ preferred_zoneref = first_zones_zonelist(ai.zonelist, ai.high_zoneidx,
+ ai.nodemask ? : &cpuset_current_mems_allowed,
+ &ai.preferred_zone);
+ if (!ai.preferred_zone)
goto out;
- classzone_idx = zonelist_zone_idx(preferred_zoneref);
+ ai.classzone_idx = zonelist_zone_idx(preferred_zoneref);
/* First allocation attempt */
- page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order,
- zonelist, high_zoneidx, alloc_flags,
- preferred_zone, classzone_idx, migratetype);
+ ai.gfp_mask = gfp_mask|__GFP_HARDWALL;
+ page = get_page_from_freelist(&ai);
if (unlikely(!page)) {
/*
* Runtime PM, block IO and its error handling path
* can deadlock because I/O on the device might not
* complete.
*/
- gfp_mask = memalloc_noio_flags(gfp_mask);
- page = __alloc_pages_slowpath(gfp_mask, order,
- zonelist, high_zoneidx, nodemask,
- preferred_zone, classzone_idx, migratetype);
+ ai.gfp_mask = memalloc_noio_flags(gfp_mask);
+ page = __alloc_pages_slowpath(&ai);
}
- trace_mm_page_alloc(page, order, gfp_mask, migratetype);
+ trace_mm_page_alloc(page, order, ai.gfp_mask, ai.migratetype);
out:
/*
--
1.8.4.5
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 6+ messages in thread* [RFC 3/4] mm, page_alloc: make alloc_flags a separate parameter again
2014-08-06 13:55 [RFC 0/4] Reducing parameters of alloc_pages* family of functions Vlastimil Babka
2014-08-06 13:55 ` [PATCH 1/4] mm: page_alloc: determine migratetype only once Vlastimil Babka
2014-08-06 13:55 ` [RFC 2/4] mm, page_alloc: reduce number of alloc_pages* functions' parameters Vlastimil Babka
@ 2014-08-06 13:55 ` Vlastimil Babka
2014-08-06 13:55 ` [RFC 4/4] mm, page_alloc: make gfp_mask " Vlastimil Babka
3 siblings, 0 replies; 6+ messages in thread
From: Vlastimil Babka @ 2014-08-06 13:55 UTC (permalink / raw)
To: linux-mm
Cc: linux-kernel, David Rientjes, Andrew Morton, Mel Gorman,
Rik van Riel, Johannes Weiner, Srivatsa S. Bhat, Hugh Dickins,
Minchan Kim, Joonsoo Kim, Vlastimil Babka
The alloc_flags parameter often changes between the various alloc_pages*
function so make it separate again to prevent the need for saving it etc.
It is also heavily used so it might benefit from being passed by register.
---
mm/page_alloc.c | 62 +++++++++++++++++++++------------------------------------
1 file changed, 23 insertions(+), 39 deletions(-)
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 399d40d..c06ad53 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -238,7 +238,6 @@ struct alloc_info {
unsigned int order;
gfp_t gfp_mask;
- int alloc_flags;
int classzone_idx;
int migratetype;
enum zone_type high_zoneidx;
@@ -1956,10 +1955,9 @@ static void reset_alloc_batches(struct zone *preferred_zone)
* a page.
*/
static struct page *
-get_page_from_freelist(const struct alloc_info *ai)
+get_page_from_freelist(int alloc_flags, const struct alloc_info *ai)
{
const unsigned int order = ai->order;
- int alloc_flags = ai->alloc_flags;
struct zonelist *zonelist = ai->zonelist;
struct zoneref *z;
struct page *page = NULL;
@@ -2253,11 +2251,10 @@ should_alloc_retry(gfp_t gfp_mask, unsigned int order,
}
static inline struct page *
-__alloc_pages_may_oom(struct alloc_info *ai)
+__alloc_pages_may_oom(int alloc_flags, struct alloc_info *ai)
{
struct page *page;
const gfp_t gfp_mask = ai->gfp_mask;
- const int alloc_flags_saved = ai->alloc_flags;
/* Acquire the per-zone oom lock for each zone */
if (!oom_zonelist_trylock(ai->zonelist, ai->gfp_mask)) {
@@ -2271,10 +2268,8 @@ __alloc_pages_may_oom(struct alloc_info *ai)
* we're still under heavy pressure.
*/
ai->gfp_mask |= __GFP_HARDWALL;
- ai->alloc_flags = ALLOC_WMARK_HIGH|ALLOC_CPUSET;
- page = get_page_from_freelist(ai);
+ page = get_page_from_freelist(ALLOC_WMARK_HIGH|ALLOC_CPUSET, ai);
ai->gfp_mask = gfp_mask;
- ai->alloc_flags = alloc_flags_saved;
if (page)
goto out;
@@ -2307,13 +2302,12 @@ out:
#ifdef CONFIG_COMPACTION
/* Try memory compaction for high-order allocations before reclaim */
static struct page *
-__alloc_pages_direct_compact(struct alloc_info *ai, enum migrate_mode mode,
- bool *contended_compaction, bool *deferred_compaction,
- unsigned long *did_some_progress)
+__alloc_pages_direct_compact(int alloc_flags, const struct alloc_info *ai,
+ enum migrate_mode mode, bool *contended_compaction,
+ bool *deferred_compaction, unsigned long *did_some_progress)
{
const unsigned int order = ai->order;
struct zone *preferred_zone = ai->preferred_zone;
- const int alloc_flags_saved = ai->alloc_flags;
if (!order)
return NULL;
@@ -2336,9 +2330,8 @@ __alloc_pages_direct_compact(struct alloc_info *ai, enum migrate_mode mode,
drain_pages(get_cpu());
put_cpu();
- ai->alloc_flags &= ~ALLOC_NO_WATERMARKS;
- page = get_page_from_freelist(ai);
- ai->alloc_flags = alloc_flags_saved;
+ page = get_page_from_freelist(alloc_flags &
+ ~ALLOC_NO_WATERMARKS, ai);
if (page) {
preferred_zone->compact_blockskip_flush = false;
@@ -2368,9 +2361,9 @@ __alloc_pages_direct_compact(struct alloc_info *ai, enum migrate_mode mode,
}
#else
static inline struct page *
-__alloc_pages_direct_compact(struct alloc_info *ai, enum migrate_mode mode,
- bool *contended_compaction, bool *deferred_compaction,
- unsigned long *did_some_progress)
+__alloc_pages_direct_compact(int alloc_flags, const struct alloc_info *ai,
+ enum migrate_mode mode, bool *contended_compaction,
+ bool *deferred_compaction, unsigned long *did_some_progress)
{
return NULL;
}
@@ -2406,12 +2399,11 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order, struct zonelist *zonelist,
/* The really slow allocator path where we enter direct reclaim */
static inline struct page *
-__alloc_pages_direct_reclaim(struct alloc_info *ai,
+__alloc_pages_direct_reclaim(int alloc_flags, const struct alloc_info *ai,
unsigned long *did_some_progress)
{
struct page *page = NULL;
bool drained = false;
- const int alloc_flags_saved = ai->alloc_flags;
*did_some_progress = __perform_reclaim(ai->gfp_mask, ai->order,
ai->zonelist, ai->nodemask);
@@ -2422,9 +2414,8 @@ __alloc_pages_direct_reclaim(struct alloc_info *ai,
if (IS_ENABLED(CONFIG_NUMA))
zlc_clear_zones_full(ai->zonelist);
- ai->alloc_flags &= ~ALLOC_NO_WATERMARKS;
retry:
- page = get_page_from_freelist(ai);
+ page = get_page_from_freelist(alloc_flags & ~ALLOC_NO_WATERMARKS, ai);
/*
* If an allocation failed after direct reclaim, it could be because
@@ -2436,7 +2427,6 @@ retry:
goto retry;
}
- ai->alloc_flags = alloc_flags_saved;
return page;
}
@@ -2445,21 +2435,18 @@ retry:
* sufficient urgency to ignore watermarks and take other desperate measures
*/
static inline struct page *
-__alloc_pages_high_priority(struct alloc_info *ai)
+__alloc_pages_high_priority(const struct alloc_info *ai)
{
struct page *page;
- const int alloc_flags_saved = ai->alloc_flags;
- ai->alloc_flags = ALLOC_NO_WATERMARKS;
do {
- page = get_page_from_freelist(ai);
+ page = get_page_from_freelist(ALLOC_NO_WATERMARKS, ai);
if (!page && ai->gfp_mask & __GFP_NOFAIL)
wait_iff_congested(ai->preferred_zone, BLK_RW_ASYNC,
HZ/50);
} while (!page && (ai->gfp_mask & __GFP_NOFAIL));
- ai->alloc_flags = alloc_flags_saved;
return page;
}
@@ -2591,15 +2578,12 @@ restart:
rebalance:
/* This is the last chance, in general, before the goto nopage. */
- ai->alloc_flags = alloc_flags & ~ALLOC_NO_WATERMARKS;
- page = get_page_from_freelist(ai);
+ page = get_page_from_freelist(alloc_flags & ~ALLOC_NO_WATERMARKS, ai);
if (page)
goto got_pg;
/* Allocate without watermarks if the context allows */
if (alloc_flags & ALLOC_NO_WATERMARKS) {
- /* We have removed ALLOC_NO_WATERMARKS from alloc_info */
- ai->alloc_flags = alloc_flags;
/*
* Ignore mempolicies if ALLOC_NO_WATERMARKS on the grounds
* the allocation is high priority and these type of
@@ -2637,7 +2621,7 @@ rebalance:
* Try direct compaction. The first pass is asynchronous. Subsequent
* attempts after direct reclaim are synchronous
*/
- page = __alloc_pages_direct_compact(ai, migration_mode,
+ page = __alloc_pages_direct_compact(alloc_flags, ai, migration_mode,
&contended_compaction,
&deferred_compaction,
&did_some_progress);
@@ -2664,7 +2648,7 @@ rebalance:
migration_mode = MIGRATE_SYNC_LIGHT;
/* Try direct reclaim and then allocating */
- page = __alloc_pages_direct_reclaim(ai, &did_some_progress);
+ page = __alloc_pages_direct_reclaim(alloc_flags, ai, &did_some_progress);
if (page)
goto got_pg;
@@ -2680,7 +2664,7 @@ rebalance:
if ((current->flags & PF_DUMPCORE) &&
!(gfp_mask & __GFP_NOFAIL))
goto nopage;
- page = __alloc_pages_may_oom(ai);
+ page = __alloc_pages_may_oom(alloc_flags, ai);
if (page)
goto got_pg;
@@ -2719,7 +2703,7 @@ rebalance:
* direct reclaim and reclaim/compaction depends on compaction
* being called after reclaim so call directly if necessary
*/
- page = __alloc_pages_direct_compact(ai, migration_mode,
+ page = __alloc_pages_direct_compact(alloc_flags, ai, migration_mode,
&contended_compaction,
&deferred_compaction,
&did_some_progress);
@@ -2747,9 +2731,9 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
struct zoneref *preferred_zoneref;
struct page *page = NULL;
unsigned int cpuset_mems_cookie;
+ int alloc_flags = ALLOC_WMARK_LOW|ALLOC_CPUSET|ALLOC_FAIR;
struct alloc_info ai = {
.order = order,
- .alloc_flags = ALLOC_WMARK_LOW|ALLOC_CPUSET|ALLOC_FAIR,
.zonelist = zonelist,
.high_zoneidx = gfp_zone(gfp_mask),
.nodemask = nodemask,
@@ -2774,7 +2758,7 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
return NULL;
if (IS_ENABLED(CONFIG_CMA) && ai.migratetype == MIGRATE_MOVABLE)
- ai.alloc_flags |= ALLOC_CMA;
+ alloc_flags |= ALLOC_CMA;
retry_cpuset:
cpuset_mems_cookie = read_mems_allowed_begin();
@@ -2789,7 +2773,7 @@ retry_cpuset:
/* First allocation attempt */
ai.gfp_mask = gfp_mask|__GFP_HARDWALL;
- page = get_page_from_freelist(&ai);
+ page = get_page_from_freelist(alloc_flags, &ai);
if (unlikely(!page)) {
/*
* Runtime PM, block IO and its error handling path
--
1.8.4.5
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 6+ messages in thread* [RFC 4/4] mm, page_alloc: make gfp_mask a separate parameter again
2014-08-06 13:55 [RFC 0/4] Reducing parameters of alloc_pages* family of functions Vlastimil Babka
` (2 preceding siblings ...)
2014-08-06 13:55 ` [RFC 3/4] mm, page_alloc: make alloc_flags a separate parameter again Vlastimil Babka
@ 2014-08-06 13:55 ` Vlastimil Babka
3 siblings, 0 replies; 6+ messages in thread
From: Vlastimil Babka @ 2014-08-06 13:55 UTC (permalink / raw)
To: linux-mm
Cc: linux-kernel, David Rientjes, Andrew Morton, Mel Gorman,
Rik van Riel, Johannes Weiner, Srivatsa S. Bhat, Hugh Dickins,
Minchan Kim, Joonsoo Kim, Vlastimil Babka
Similarly to alloc_flags, gfp_mask sometimes changes and is heavily used
so it might benefit from being passed in a register.
---
mm/page_alloc.c | 85 +++++++++++++++++++++++++++++----------------------------
1 file changed, 43 insertions(+), 42 deletions(-)
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index c06ad53..b5c5944 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -237,7 +237,6 @@ struct alloc_info {
struct zone *preferred_zone;
unsigned int order;
- gfp_t gfp_mask;
int classzone_idx;
int migratetype;
enum zone_type high_zoneidx;
@@ -1955,7 +1954,7 @@ static void reset_alloc_batches(struct zone *preferred_zone)
* a page.
*/
static struct page *
-get_page_from_freelist(int alloc_flags, const struct alloc_info *ai)
+get_page_from_freelist(gfp_t gfp_mask, int alloc_flags, const struct alloc_info *ai)
{
const unsigned int order = ai->order;
struct zonelist *zonelist = ai->zonelist;
@@ -1966,7 +1965,7 @@ get_page_from_freelist(int alloc_flags, const struct alloc_info *ai)
int zlc_active = 0; /* set if using zonelist_cache */
int did_zlc_setup = 0; /* just call zlc_setup() one time */
bool consider_zone_dirty = (alloc_flags & ALLOC_WMARK_LOW) &&
- (ai->gfp_mask & __GFP_WRITE);
+ (gfp_mask & __GFP_WRITE);
int nr_fair_skipped = 0;
bool zonelist_rescan;
@@ -1986,7 +1985,7 @@ zonelist_scan:
continue;
if (cpusets_enabled() &&
(alloc_flags & ALLOC_CPUSET) &&
- !cpuset_zone_allowed_softwall(zone, ai->gfp_mask))
+ !cpuset_zone_allowed_softwall(zone, gfp_mask))
continue;
/*
* Distribute pages in proportion to the individual
@@ -2065,7 +2064,7 @@ zonelist_scan:
!zlc_zone_worth_trying(zonelist, z, allowednodes))
continue;
- ret = zone_reclaim(zone, ai->gfp_mask, order);
+ ret = zone_reclaim(zone, gfp_mask, order);
switch (ret) {
case ZONE_RECLAIM_NOSCAN:
/* did not scan */
@@ -2098,7 +2097,7 @@ zonelist_scan:
try_this_zone:
page = buffered_rmqueue(ai->preferred_zone, zone, order,
- ai->gfp_mask, ai->migratetype);
+ gfp_mask, ai->migratetype);
if (page)
break;
this_zone_full:
@@ -2251,13 +2250,12 @@ should_alloc_retry(gfp_t gfp_mask, unsigned int order,
}
static inline struct page *
-__alloc_pages_may_oom(int alloc_flags, struct alloc_info *ai)
+__alloc_pages_may_oom(gfp_t gfp_mask, int alloc_flags, const struct alloc_info *ai)
{
struct page *page;
- const gfp_t gfp_mask = ai->gfp_mask;
/* Acquire the per-zone oom lock for each zone */
- if (!oom_zonelist_trylock(ai->zonelist, ai->gfp_mask)) {
+ if (!oom_zonelist_trylock(ai->zonelist, gfp_mask)) {
schedule_timeout_uninterruptible(1);
return NULL;
}
@@ -2267,10 +2265,8 @@ __alloc_pages_may_oom(int alloc_flags, struct alloc_info *ai)
* here, this is only to catch a parallel oom killing, we must fail if
* we're still under heavy pressure.
*/
- ai->gfp_mask |= __GFP_HARDWALL;
- page = get_page_from_freelist(ALLOC_WMARK_HIGH|ALLOC_CPUSET, ai);
- ai->gfp_mask = gfp_mask;
-
+ page = get_page_from_freelist(gfp_mask | __GFP_HARDWALL,
+ ALLOC_WMARK_HIGH|ALLOC_CPUSET, ai);
if (page)
goto out;
@@ -2302,9 +2298,10 @@ out:
#ifdef CONFIG_COMPACTION
/* Try memory compaction for high-order allocations before reclaim */
static struct page *
-__alloc_pages_direct_compact(int alloc_flags, const struct alloc_info *ai,
- enum migrate_mode mode, bool *contended_compaction,
- bool *deferred_compaction, unsigned long *did_some_progress)
+__alloc_pages_direct_compact(gfp_t gfp_mask, int alloc_flags,
+ const struct alloc_info *ai, enum migrate_mode mode,
+ bool *contended_compaction, bool *deferred_compaction,
+ unsigned long *did_some_progress)
{
const unsigned int order = ai->order;
struct zone *preferred_zone = ai->preferred_zone;
@@ -2318,7 +2315,7 @@ __alloc_pages_direct_compact(int alloc_flags, const struct alloc_info *ai,
}
current->flags |= PF_MEMALLOC;
- *did_some_progress = try_to_compact_pages(ai->zonelist, order, ai->gfp_mask,
+ *did_some_progress = try_to_compact_pages(ai->zonelist, order, gfp_mask,
ai->nodemask, mode,
contended_compaction);
current->flags &= ~PF_MEMALLOC;
@@ -2330,7 +2327,7 @@ __alloc_pages_direct_compact(int alloc_flags, const struct alloc_info *ai,
drain_pages(get_cpu());
put_cpu();
- page = get_page_from_freelist(alloc_flags &
+ page = get_page_from_freelist(gfp_mask, alloc_flags &
~ALLOC_NO_WATERMARKS, ai);
if (page) {
@@ -2361,9 +2358,10 @@ __alloc_pages_direct_compact(int alloc_flags, const struct alloc_info *ai,
}
#else
static inline struct page *
-__alloc_pages_direct_compact(int alloc_flags, const struct alloc_info *ai,
- enum migrate_mode mode, bool *contended_compaction,
- bool *deferred_compaction, unsigned long *did_some_progress)
+__alloc_pages_direct_compact(gfp_t gfp_mask, int alloc_flags,
+ const struct alloc_info *ai, enum migrate_mode mode,
+ bool *contended_compaction, bool *deferred_compaction,
+ unsigned long *did_some_progress)
{
return NULL;
}
@@ -2399,13 +2397,13 @@ __perform_reclaim(gfp_t gfp_mask, unsigned int order, struct zonelist *zonelist,
/* The really slow allocator path where we enter direct reclaim */
static inline struct page *
-__alloc_pages_direct_reclaim(int alloc_flags, const struct alloc_info *ai,
- unsigned long *did_some_progress)
+__alloc_pages_direct_reclaim(gfp_t gfp_mask, int alloc_flags,
+ const struct alloc_info *ai, unsigned long *did_some_progress)
{
struct page *page = NULL;
bool drained = false;
- *did_some_progress = __perform_reclaim(ai->gfp_mask, ai->order,
+ *did_some_progress = __perform_reclaim(gfp_mask, ai->order,
ai->zonelist, ai->nodemask);
if (unlikely(!(*did_some_progress)))
return NULL;
@@ -2415,7 +2413,8 @@ __alloc_pages_direct_reclaim(int alloc_flags, const struct alloc_info *ai,
zlc_clear_zones_full(ai->zonelist);
retry:
- page = get_page_from_freelist(alloc_flags & ~ALLOC_NO_WATERMARKS, ai);
+ page = get_page_from_freelist(gfp_mask,
+ alloc_flags & ~ALLOC_NO_WATERMARKS, ai);
/*
* If an allocation failed after direct reclaim, it could be because
@@ -2435,17 +2434,17 @@ retry:
* sufficient urgency to ignore watermarks and take other desperate measures
*/
static inline struct page *
-__alloc_pages_high_priority(const struct alloc_info *ai)
+__alloc_pages_high_priority(gfp_t gfp_mask, const struct alloc_info *ai)
{
struct page *page;
do {
- page = get_page_from_freelist(ALLOC_NO_WATERMARKS, ai);
+ page = get_page_from_freelist(gfp_mask, ALLOC_NO_WATERMARKS, ai);
- if (!page && ai->gfp_mask & __GFP_NOFAIL)
+ if (!page && gfp_mask & __GFP_NOFAIL)
wait_iff_congested(ai->preferred_zone, BLK_RW_ASYNC,
HZ/50);
- } while (!page && (ai->gfp_mask & __GFP_NOFAIL));
+ } while (!page && (gfp_mask & __GFP_NOFAIL));
return page;
}
@@ -2517,10 +2516,9 @@ bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
}
static inline struct page *
-__alloc_pages_slowpath(struct alloc_info *ai)
+__alloc_pages_slowpath(gfp_t gfp_mask, struct alloc_info *ai)
{
const unsigned int order = ai->order;
- const gfp_t gfp_mask = ai->gfp_mask;
const gfp_t wait = gfp_mask & __GFP_WAIT;
struct page *page = NULL;
int alloc_flags;
@@ -2578,7 +2576,8 @@ restart:
rebalance:
/* This is the last chance, in general, before the goto nopage. */
- page = get_page_from_freelist(alloc_flags & ~ALLOC_NO_WATERMARKS, ai);
+ page = get_page_from_freelist(gfp_mask,
+ alloc_flags & ~ALLOC_NO_WATERMARKS, ai);
if (page)
goto got_pg;
@@ -2591,7 +2590,7 @@ rebalance:
*/
ai->zonelist = node_zonelist(numa_node_id(), gfp_mask);
- page = __alloc_pages_high_priority(ai);
+ page = __alloc_pages_high_priority(gfp_mask, ai);
if (page) {
goto got_pg;
@@ -2621,7 +2620,8 @@ rebalance:
* Try direct compaction. The first pass is asynchronous. Subsequent
* attempts after direct reclaim are synchronous
*/
- page = __alloc_pages_direct_compact(alloc_flags, ai, migration_mode,
+ page = __alloc_pages_direct_compact(gfp_mask, alloc_flags, ai,
+ migration_mode,
&contended_compaction,
&deferred_compaction,
&did_some_progress);
@@ -2648,7 +2648,8 @@ rebalance:
migration_mode = MIGRATE_SYNC_LIGHT;
/* Try direct reclaim and then allocating */
- page = __alloc_pages_direct_reclaim(alloc_flags, ai, &did_some_progress);
+ page = __alloc_pages_direct_reclaim(gfp_mask, alloc_flags, ai,
+ &did_some_progress);
if (page)
goto got_pg;
@@ -2664,7 +2665,7 @@ rebalance:
if ((current->flags & PF_DUMPCORE) &&
!(gfp_mask & __GFP_NOFAIL))
goto nopage;
- page = __alloc_pages_may_oom(alloc_flags, ai);
+ page = __alloc_pages_may_oom(gfp_mask, alloc_flags, ai);
if (page)
goto got_pg;
@@ -2703,7 +2704,8 @@ rebalance:
* direct reclaim and reclaim/compaction depends on compaction
* being called after reclaim so call directly if necessary
*/
- page = __alloc_pages_direct_compact(alloc_flags, ai, migration_mode,
+ page = __alloc_pages_direct_compact(gfp_mask, alloc_flags, ai,
+ migration_mode,
&contended_compaction,
&deferred_compaction,
&did_some_progress);
@@ -2772,19 +2774,18 @@ retry_cpuset:
ai.classzone_idx = zonelist_zone_idx(preferred_zoneref);
/* First allocation attempt */
- ai.gfp_mask = gfp_mask|__GFP_HARDWALL;
- page = get_page_from_freelist(alloc_flags, &ai);
+ page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, alloc_flags, &ai);
if (unlikely(!page)) {
/*
* Runtime PM, block IO and its error handling path
* can deadlock because I/O on the device might not
* complete.
*/
- ai.gfp_mask = memalloc_noio_flags(gfp_mask);
- page = __alloc_pages_slowpath(&ai);
+ gfp_mask = memalloc_noio_flags(gfp_mask);
+ page = __alloc_pages_slowpath(gfp_mask, &ai);
}
- trace_mm_page_alloc(page, order, ai.gfp_mask, ai.migratetype);
+ trace_mm_page_alloc(page, order, gfp_mask, ai.migratetype);
out:
/*
--
1.8.4.5
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 6+ messages in thread