From: Roman Gushchin <guroan@gmail.com>
To: linux-mm@kvack.org, kernel-team@fb.com
Cc: linux-kernel@vger.kernel.org, Tejun Heo <tj@kernel.org>,
Rik van Riel <riel@surriel.com>,
Johannes Weiner <hannes@cmpxchg.org>,
Michal Hocko <mhocko@kernel.org>, Roman Gushchin <guro@fb.com>
Subject: [PATCH v3 1/6] mm: prepare to premature release of memcg->vmstats_percpu
Date: Wed, 13 Mar 2019 11:39:48 -0700 [thread overview]
Message-ID: <20190313183953.17854-2-guro@fb.com> (raw)
In-Reply-To: <20190313183953.17854-1-guro@fb.com>
Prepare to handle premature release of memcg->vmstats_percpu data.
Currently it's a generic pointer which is expected to be non-NULL
during the whole life time of a memcg. Switch over to the
rcu-protected pointer, and carefully check it for being non-NULL.
This change is a required step towards dynamic premature release
of percpu memcg data.
Signed-off-by: Roman Gushchin <guro@fb.com>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
---
include/linux/memcontrol.h | 40 +++++++++++++++++-------
mm/memcontrol.c | 62 +++++++++++++++++++++++++++++---------
2 files changed, 77 insertions(+), 25 deletions(-)
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 534267947664..05ca77767c6a 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -274,7 +274,7 @@ struct mem_cgroup {
struct task_struct *move_lock_task;
/* memory.stat */
- struct memcg_vmstats_percpu __percpu *vmstats_percpu;
+ struct memcg_vmstats_percpu __rcu /* __percpu */ *vmstats_percpu;
MEMCG_PADDING(_pad2_);
@@ -597,17 +597,26 @@ static inline unsigned long memcg_page_state(struct mem_cgroup *memcg,
static inline void __mod_memcg_state(struct mem_cgroup *memcg,
int idx, int val)
{
+ struct memcg_vmstats_percpu __percpu *vmstats_percpu;
long x;
if (mem_cgroup_disabled())
return;
- x = val + __this_cpu_read(memcg->vmstats_percpu->stat[idx]);
- if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
- atomic_long_add(x, &memcg->vmstats[idx]);
- x = 0;
+ rcu_read_lock();
+ vmstats_percpu = (struct memcg_vmstats_percpu __percpu *)
+ rcu_dereference(memcg->vmstats_percpu);
+ if (likely(vmstats_percpu)) {
+ x = val + __this_cpu_read(vmstats_percpu->stat[idx]);
+ if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
+ atomic_long_add(x, &memcg->vmstats[idx]);
+ x = 0;
+ }
+ __this_cpu_write(vmstats_percpu->stat[idx], x);
+ } else {
+ atomic_long_add(val, &memcg->vmstats[idx]);
}
- __this_cpu_write(memcg->vmstats_percpu->stat[idx], x);
+ rcu_read_unlock();
}
/* idx can be of type enum memcg_stat_item or node_stat_item */
@@ -740,17 +749,26 @@ static inline void __count_memcg_events(struct mem_cgroup *memcg,
enum vm_event_item idx,
unsigned long count)
{
+ struct memcg_vmstats_percpu __percpu *vmstats_percpu;
unsigned long x;
if (mem_cgroup_disabled())
return;
- x = count + __this_cpu_read(memcg->vmstats_percpu->events[idx]);
- if (unlikely(x > MEMCG_CHARGE_BATCH)) {
- atomic_long_add(x, &memcg->vmevents[idx]);
- x = 0;
+ rcu_read_lock();
+ vmstats_percpu = (struct memcg_vmstats_percpu __percpu *)
+ rcu_dereference(memcg->vmstats_percpu);
+ if (likely(vmstats_percpu)) {
+ x = count + __this_cpu_read(vmstats_percpu->events[idx]);
+ if (unlikely(x > MEMCG_CHARGE_BATCH)) {
+ atomic_long_add(x, &memcg->vmevents[idx]);
+ x = 0;
+ }
+ __this_cpu_write(vmstats_percpu->events[idx], x);
+ } else {
+ atomic_long_add(count, &memcg->vmevents[idx]);
}
- __this_cpu_write(memcg->vmstats_percpu->events[idx], x);
+ rcu_read_unlock();
}
static inline void count_memcg_events(struct mem_cgroup *memcg,
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index c532f8685aa3..803c772f354b 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -697,6 +697,8 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
struct page *page,
bool compound, int nr_pages)
{
+ struct memcg_vmstats_percpu __percpu *vmstats_percpu;
+
/*
* Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
* counted as CACHE even if it's on ANON LRU.
@@ -722,7 +724,12 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
nr_pages = -nr_pages; /* for event */
}
- __this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages);
+ rcu_read_lock();
+ vmstats_percpu = (struct memcg_vmstats_percpu __percpu *)
+ rcu_dereference(memcg->vmstats_percpu);
+ if (likely(vmstats_percpu))
+ __this_cpu_add(vmstats_percpu->nr_page_events, nr_pages);
+ rcu_read_unlock();
}
unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
@@ -756,10 +763,18 @@ static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
enum mem_cgroup_events_target target)
{
+ struct memcg_vmstats_percpu __percpu *vmstats_percpu;
unsigned long val, next;
+ bool ret = false;
- val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events);
- next = __this_cpu_read(memcg->vmstats_percpu->targets[target]);
+ rcu_read_lock();
+ vmstats_percpu = (struct memcg_vmstats_percpu __percpu *)
+ rcu_dereference(memcg->vmstats_percpu);
+ if (!vmstats_percpu)
+ goto out;
+
+ val = __this_cpu_read(vmstats_percpu->nr_page_events);
+ next = __this_cpu_read(vmstats_percpu->targets[target]);
/* from time_after() in jiffies.h */
if ((long)(next - val) < 0) {
switch (target) {
@@ -775,10 +790,12 @@ static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
default:
break;
}
- __this_cpu_write(memcg->vmstats_percpu->targets[target], next);
- return true;
+ __this_cpu_write(vmstats_percpu->targets[target], next);
+ ret = true;
}
- return false;
+out:
+ rcu_read_unlock();
+ return ret;
}
/*
@@ -2104,22 +2121,29 @@ static void drain_all_stock(struct mem_cgroup *root_memcg)
static int memcg_hotplug_cpu_dead(unsigned int cpu)
{
+ struct memcg_vmstats_percpu __percpu *vmstats_percpu;
struct memcg_stock_pcp *stock;
struct mem_cgroup *memcg;
stock = &per_cpu(memcg_stock, cpu);
drain_stock(stock);
+ rcu_read_lock();
for_each_mem_cgroup(memcg) {
int i;
+ vmstats_percpu = (struct memcg_vmstats_percpu __percpu *)
+ rcu_dereference(memcg->vmstats_percpu);
+
for (i = 0; i < MEMCG_NR_STAT; i++) {
int nid;
long x;
- x = this_cpu_xchg(memcg->vmstats_percpu->stat[i], 0);
- if (x)
- atomic_long_add(x, &memcg->vmstats[i]);
+ if (vmstats_percpu) {
+ x = this_cpu_xchg(vmstats_percpu->stat[i], 0);
+ if (x)
+ atomic_long_add(x, &memcg->vmstats[i]);
+ }
if (i >= NR_VM_NODE_STAT_ITEMS)
continue;
@@ -2137,11 +2161,14 @@ static int memcg_hotplug_cpu_dead(unsigned int cpu)
for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
long x;
- x = this_cpu_xchg(memcg->vmstats_percpu->events[i], 0);
- if (x)
- atomic_long_add(x, &memcg->vmevents[i]);
+ if (vmstats_percpu) {
+ x = this_cpu_xchg(vmstats_percpu->events[i], 0);
+ if (x)
+ atomic_long_add(x, &memcg->vmevents[i]);
+ }
}
}
+ rcu_read_unlock();
return 0;
}
@@ -4464,7 +4491,8 @@ static struct mem_cgroup *mem_cgroup_alloc(void)
if (memcg->id.id < 0)
goto fail;
- memcg->vmstats_percpu = alloc_percpu(struct memcg_vmstats_percpu);
+ rcu_assign_pointer(memcg->vmstats_percpu,
+ alloc_percpu(struct memcg_vmstats_percpu));
if (!memcg->vmstats_percpu)
goto fail;
@@ -6054,6 +6082,7 @@ static void uncharge_batch(const struct uncharge_gather *ug)
{
unsigned long nr_pages = ug->nr_anon + ug->nr_file + ug->nr_kmem;
unsigned long flags;
+ struct memcg_vmstats_percpu __percpu *vmstats_percpu;
if (!mem_cgroup_is_root(ug->memcg)) {
page_counter_uncharge(&ug->memcg->memory, nr_pages);
@@ -6070,7 +6099,12 @@ static void uncharge_batch(const struct uncharge_gather *ug)
__mod_memcg_state(ug->memcg, MEMCG_RSS_HUGE, -ug->nr_huge);
__mod_memcg_state(ug->memcg, NR_SHMEM, -ug->nr_shmem);
__count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
- __this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, nr_pages);
+ rcu_read_lock();
+ vmstats_percpu = (struct memcg_vmstats_percpu __percpu *)
+ rcu_dereference(ug->memcg->vmstats_percpu);
+ if (likely(vmstats_percpu))
+ __this_cpu_add(vmstats_percpu->nr_page_events, nr_pages);
+ rcu_read_unlock();
memcg_check_events(ug->memcg, ug->dummy_page);
local_irq_restore(flags);
--
2.20.1
next prev parent reply other threads:[~2019-03-13 18:40 UTC|newest]
Thread overview: 8+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-03-13 18:39 [PATCH v3 0/6] mm: reduce the memory footprint of dying memory cgroups Roman Gushchin
2019-03-13 18:39 ` Roman Gushchin [this message]
2019-03-13 18:39 ` [PATCH v3 2/6] mm: prepare to premature release of per-node lruvec_stat_cpu Roman Gushchin
2019-03-13 18:39 ` [PATCH v3 3/6] mm: release memcg percpu data prematurely Roman Gushchin
2019-03-13 18:39 ` [PATCH v3 4/6] mm: release per-node " Roman Gushchin
2019-03-13 18:39 ` [PATCH v3 5/6] mm: flush memcg percpu stats and events before releasing Roman Gushchin
2019-03-13 18:39 ` [PATCH v3 6/6] mm: refactor memcg_hotplug_cpu_dead() to use memcg_flush_offline_percpu() Roman Gushchin
2019-03-13 19:48 ` Johannes Weiner
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190313183953.17854-2-guro@fb.com \
--to=guroan@gmail.com \
--cc=guro@fb.com \
--cc=hannes@cmpxchg.org \
--cc=kernel-team@fb.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=mhocko@kernel.org \
--cc=riel@surriel.com \
--cc=tj@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox