Re: [PATCH v4 29/31] mm: memcontrol: prepare for reparenting non-hierarchical stats

[Shakeel Butt <shakeel.butt@linux.dev>]

On Thu, Feb 05, 2026 at 05:01:48PM +0800, Qi Zheng wrote:
> From: Qi Zheng <zhengqi.arch@bytedance.com>
> 
> To resolve the dying memcg issue, we need to reparent LRU folios of child
> memcg to its parent memcg. This could cause problems for non-hierarchical
> stats.
> 
> As Yosry Ahmed pointed out:
> 
> ```
> In short, if memory is charged to a dying cgroup at the time of
> reparenting, when the memory gets uncharged the stats updates will occur
> at the parent. This will update both hierarchical and non-hierarchical
> stats of the parent, which would corrupt the parent's non-hierarchical
> stats (because those counters were never incremented when the memory was
> charged).
> ```
> 
> Now we have the following two types of non-hierarchical stats, and they
> are only used in CONFIG_MEMCG_V1:
> 
> a. memcg->vmstats->state_local[i]
> b. pn->lruvec_stats->state_local[i]
> 
> To ensure that these non-hierarchical stats work properly, we need to
> reparent these non-hierarchical stats after reparenting LRU folios. To
> this end, this commit makes the following preparations:
> 
> 1. implement reparent_state_local() to reparent non-hierarchical stats
> 2. make css_killed_work_fn() to be called in rcu work, and implement
>    get_non_dying_memcg_start() and get_non_dying_memcg_end() to avoid race
>    between mod_memcg_state()/mod_memcg_lruvec_state()
>    and reparent_state_local()
> 3. change these non-hierarchical stats to atomic_long_t type to avoid race
>    between mem_cgroup_stat_aggregate() and reparent_state_local()
> 
> Signed-off-by: Qi Zheng <zhengqi.arch@bytedance.com>

Overall looks good just a couple of comments.

> ---
>  include/linux/memcontrol.h |   4 ++
>  kernel/cgroup/cgroup.c     |   8 +--
>  mm/memcontrol-v1.c         |  16 ++++++
>  mm/memcontrol-v1.h         |   3 +
>  mm/memcontrol.c            | 113 ++++++++++++++++++++++++++++++++++---
>  5 files changed, 132 insertions(+), 12 deletions(-)
> 
> diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
> index 3970c102fe741..a4f6ab7eb98d6 100644
> --- a/include/linux/memcontrol.h
> +++ b/include/linux/memcontrol.h
> @@ -957,12 +957,16 @@ static inline void mod_memcg_page_state(struct page *page,
>  
>  unsigned long memcg_events(struct mem_cgroup *memcg, int event);
>  unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx);
> +void reparent_memcg_state_local(struct mem_cgroup *memcg,
> +				struct mem_cgroup *parent, int idx);

Put the above in mm/memcontrol-v1.h file.

>  unsigned long memcg_page_state_output(struct mem_cgroup *memcg, int item);
>  bool memcg_stat_item_valid(int idx);
>  bool memcg_vm_event_item_valid(enum vm_event_item idx);
>  unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx);
>  unsigned long lruvec_page_state_local(struct lruvec *lruvec,
>  				      enum node_stat_item idx);
> +void reparent_memcg_lruvec_state_local(struct mem_cgroup *memcg,
> +				       struct mem_cgroup *parent, int idx);

Put the above in mm/memcontrol-v1.h file.

>  
>  void mem_cgroup_flush_stats(struct mem_cgroup *memcg);
>  void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg);
> diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
> index 94788bd1fdf0e..dbf94a77018e6 100644
> --- a/kernel/cgroup/cgroup.c
> +++ b/kernel/cgroup/cgroup.c
> @@ -6043,8 +6043,8 @@ int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode)
>   */
>  static void css_killed_work_fn(struct work_struct *work)
>  {
> -	struct cgroup_subsys_state *css =
> -		container_of(work, struct cgroup_subsys_state, destroy_work);
> +	struct cgroup_subsys_state *css = container_of(to_rcu_work(work),
> +				struct cgroup_subsys_state, destroy_rwork);
>  
>  	cgroup_lock();
>  
> @@ -6065,8 +6065,8 @@ static void css_killed_ref_fn(struct percpu_ref *ref)
>  		container_of(ref, struct cgroup_subsys_state, refcnt);
>  
>  	if (atomic_dec_and_test(&css->online_cnt)) {
> -		INIT_WORK(&css->destroy_work, css_killed_work_fn);
> -		queue_work(cgroup_offline_wq, &css->destroy_work);
> +		INIT_RCU_WORK(&css->destroy_rwork, css_killed_work_fn);
> +		queue_rcu_work(cgroup_offline_wq, &css->destroy_rwork);
>  	}
>  }
>  
> diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c
> index c6078cd7f7e53..a427bb205763b 100644
> --- a/mm/memcontrol-v1.c
> +++ b/mm/memcontrol-v1.c
> @@ -1887,6 +1887,22 @@ static const unsigned int memcg1_events[] = {
>  	PGMAJFAULT,
>  };
>  
> +void reparent_memcg1_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent)
> +{
> +	int i;
> +
> +	for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++)
> +		reparent_memcg_state_local(memcg, parent, memcg1_stats[i]);
> +}
> +
> +void reparent_memcg1_lruvec_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent)
> +{
> +	int i;
> +
> +	for (i = 0; i < NR_LRU_LISTS; i++)
> +		reparent_memcg_lruvec_state_local(memcg, parent, i);
> +}
> +
>  void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
>  {
>  	unsigned long memory, memsw;
> diff --git a/mm/memcontrol-v1.h b/mm/memcontrol-v1.h
> index eb3c3c1056574..45528195d3578 100644
> --- a/mm/memcontrol-v1.h
> +++ b/mm/memcontrol-v1.h
> @@ -41,6 +41,7 @@ static inline bool do_memsw_account(void)
>  
>  unsigned long memcg_events_local(struct mem_cgroup *memcg, int event);
>  unsigned long memcg_page_state_local(struct mem_cgroup *memcg, int idx);
> +void mod_memcg_page_state_local(struct mem_cgroup *memcg, int idx, unsigned long val);
>  unsigned long memcg_page_state_local_output(struct mem_cgroup *memcg, int item);
>  bool memcg1_alloc_events(struct mem_cgroup *memcg);
>  void memcg1_free_events(struct mem_cgroup *memcg);
> @@ -73,6 +74,8 @@ void memcg1_uncharge_batch(struct mem_cgroup *memcg, unsigned long pgpgout,
>  			   unsigned long nr_memory, int nid);
>  
>  void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s);
> +void reparent_memcg1_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent);
> +void reparent_memcg1_lruvec_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent);
>  
>  void memcg1_account_kmem(struct mem_cgroup *memcg, int nr_pages);
>  static inline bool memcg1_tcpmem_active(struct mem_cgroup *memcg)
> diff --git a/mm/memcontrol.c b/mm/memcontrol.c
> index c9b5dfd822d0a..e7d4e4ff411b6 100644
> --- a/mm/memcontrol.c
> +++ b/mm/memcontrol.c
> @@ -225,6 +225,26 @@ static inline struct obj_cgroup *__memcg_reparent_objcgs(struct mem_cgroup *memc
>  	return objcg;
>  }
>  
> +#ifdef CONFIG_MEMCG_V1
> +static void __mem_cgroup_flush_stats(struct mem_cgroup *memcg, bool force);
> +
> +static inline void reparent_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent)
> +{
> +	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
> +		return;
> +
> +	__mem_cgroup_flush_stats(memcg, true);
> +
> +	/* The following counts are all non-hierarchical and need to be reparented. */
> +	reparent_memcg1_state_local(memcg, parent);
> +	reparent_memcg1_lruvec_state_local(memcg, parent);
> +}
> +#else
> +static inline void reparent_state_local(struct mem_cgroup *memcg, struct mem_cgroup *parent)
> +{
> +}
> +#endif
> +
>  static inline void reparent_locks(struct mem_cgroup *memcg, struct mem_cgroup *parent)
>  {
>  	spin_lock_irq(&objcg_lock);
> @@ -407,7 +427,7 @@ struct lruvec_stats {
>  	long state[NR_MEMCG_NODE_STAT_ITEMS];
>  
>  	/* Non-hierarchical (CPU aggregated) state */
> -	long state_local[NR_MEMCG_NODE_STAT_ITEMS];
> +	atomic_long_t state_local[NR_MEMCG_NODE_STAT_ITEMS];
>  
>  	/* Pending child counts during tree propagation */
>  	long state_pending[NR_MEMCG_NODE_STAT_ITEMS];
> @@ -450,7 +470,7 @@ unsigned long lruvec_page_state_local(struct lruvec *lruvec,
>  		return 0;
>  
>  	pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
> -	x = READ_ONCE(pn->lruvec_stats->state_local[i]);
> +	x = atomic_long_read(&(pn->lruvec_stats->state_local[i]));
>  #ifdef CONFIG_SMP
>  	if (x < 0)
>  		x = 0;
> @@ -458,6 +478,27 @@ unsigned long lruvec_page_state_local(struct lruvec *lruvec,
>  	return x;
>  }
>  

Please put the following function under CONFIG_MEMCG_V1. Just move it in
the same block as reparent_state_local().

> +void reparent_memcg_lruvec_state_local(struct mem_cgroup *memcg,
> +				       struct mem_cgroup *parent, int idx)
> +{
> +	int i = memcg_stats_index(idx);
> +	int nid;
> +
> +	if (WARN_ONCE(BAD_STAT_IDX(i), "%s: missing stat item %d\n", __func__, idx))
> +		return;
> +
> +	for_each_node(nid) {
> +		struct lruvec *child_lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
> +		struct lruvec *parent_lruvec = mem_cgroup_lruvec(parent, NODE_DATA(nid));
> +		struct mem_cgroup_per_node *parent_pn;
> +		unsigned long value = lruvec_page_state_local(child_lruvec, idx);
> +
> +		parent_pn = container_of(parent_lruvec, struct mem_cgroup_per_node, lruvec);
> +
> +		atomic_long_add(value, &(parent_pn->lruvec_stats->state_local[i]));
> +	}
> +}
> +

[...]

>  
> +#ifdef CONFIG_MEMCG_V1
> +/*
> + * Used in mod_memcg_state() and mod_memcg_lruvec_state() to avoid race with
> + * reparenting of non-hierarchical state_locals.
> + */
> +static inline struct mem_cgroup *get_non_dying_memcg_start(struct mem_cgroup *memcg)
> +{
> +	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
> +		return memcg;
> +
> +	rcu_read_lock();
> +
> +	while (memcg_is_dying(memcg))
> +		memcg = parent_mem_cgroup(memcg);
> +
> +	return memcg;
> +}
> +
> +static inline void get_non_dying_memcg_end(void)
> +{
> +	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
> +		return;
> +
> +	rcu_read_unlock();
> +}
> +#else
> +static inline struct mem_cgroup *get_non_dying_memcg_start(struct mem_cgroup *memcg)
> +{
> +	return memcg;
> +}
> +
> +static inline void get_non_dying_memcg_end(void)
> +{
> +}
> +#endif

Add the usage of these start and end functions in mod_memcg_state() and
mod_memcg_lruvec_state() in this patch.