[PATCH 11/11] cgroup: separate rstat list pointers from base stats

[JP Kobryn <inwardvessel@gmail.com>]

A majority of the cgroup_rstat_cpu struct size is made up of the base stat
entities. Since only the "self" subsystem state makes use of these, move
them into a struct of their own. This allows for a new compact
cgroup_rstat_cpu struct that the formal subsystems can make use of.
Where applicable, decide on whether to allocate the compact or the full
struct including the base stats.

Signed-off-by: JP Kobryn <inwardvessel@gmail.com>
---
 include/linux/cgroup_rstat.h |  8 +++++-
 kernel/cgroup/rstat.c        | 55 +++++++++++++++++++++++++++---------
 2 files changed, 48 insertions(+), 15 deletions(-)

diff --git a/include/linux/cgroup_rstat.h b/include/linux/cgroup_rstat.h
index 780b826ea364..fc26c0aa91ef 100644
--- a/include/linux/cgroup_rstat.h
+++ b/include/linux/cgroup_rstat.h
@@ -27,7 +27,10 @@ struct cgroup_rstat_cpu;
  * resource statistics on top of it - bsync, bstat and last_bstat.
  */
 struct cgroup_rstat {
-	struct cgroup_rstat_cpu __percpu *rstat_cpu;
+	union {
+		struct cgroup_rstat_cpu __percpu *rstat_cpu;
+		struct cgroup_rstat_base_cpu __percpu *rstat_base_cpu;
+	};
 
 	/*
 	 * Add padding to separate the read mostly rstat_cpu and
@@ -60,7 +63,10 @@ struct cgroup_rstat_cpu {
 	 */
 	struct cgroup_rstat *updated_children;	/* terminated by self */
 	struct cgroup_rstat *updated_next;		/* NULL if not on the list */
+};
 
+struct cgroup_rstat_base_cpu {
+	struct cgroup_rstat_cpu self;
 	/*
 	 * ->bsync protects ->bstat.  These are the only fields which get
 	 * updated in the hot path.
diff --git a/kernel/cgroup/rstat.c b/kernel/cgroup/rstat.c
index 93b97bddec9c..6b14241d0924 100644
--- a/kernel/cgroup/rstat.c
+++ b/kernel/cgroup/rstat.c
@@ -33,6 +33,12 @@ static struct cgroup_rstat_cpu *rstat_cpu(struct cgroup_rstat *rstat, int cpu)
 	return per_cpu_ptr(rstat->rstat_cpu, cpu);
 }
 
+static struct cgroup_rstat_base_cpu *rstat_base_cpu(
+		struct cgroup_rstat *rstat, int cpu)
+{
+	return per_cpu_ptr(rstat->rstat_base_cpu, cpu);
+}
+
 static inline bool is_base_css(struct cgroup_subsys_state *css)
 {
 	/* css for base stats has no subsystem */
@@ -597,6 +603,18 @@ static void __cgroup_rstat_init(struct cgroup_rstat *rstat)
 		struct cgroup_rstat_cpu *rstatc = rstat_cpu(rstat, cpu);
 
 		rstatc->updated_children = rstat;
+	}
+}
+
+static void __cgroup_rstat_base_init(struct cgroup_rstat *rstat)
+{
+	int cpu;
+
+	/* ->updated_children list is self terminated */
+	for_each_possible_cpu(cpu) {
+		struct cgroup_rstat_base_cpu *rstatc = rstat_base_cpu(rstat, cpu);
+
+		rstatc->self.updated_children = rstat;
 		u64_stats_init(&rstatc->bsync);
 	}
 }
@@ -607,13 +625,21 @@ int cgroup_rstat_init(struct cgroup_subsys_state *css)
 
 	/* the root cgrp has rstat_cpu preallocated */
 	if (!rstat->rstat_cpu) {
-		rstat->rstat_cpu = alloc_percpu(struct cgroup_rstat_cpu);
-		if (!rstat->rstat_cpu)
-			return -ENOMEM;
+		if (is_base_css(css)) {
+			rstat->rstat_base_cpu = alloc_percpu(struct cgroup_rstat_base_cpu);
+			if (!rstat->rstat_base_cpu)
+				return -ENOMEM;
+
+			__cgroup_rstat_base_init(rstat);
+		} else {
+			rstat->rstat_cpu = alloc_percpu(struct cgroup_rstat_cpu);
+			if (!rstat->rstat_cpu)
+				return -ENOMEM;
+
+			__cgroup_rstat_init(rstat);
+		}
 	}
 
-	__cgroup_rstat_init(rstat);
-
 	return 0;
 }
 
@@ -718,9 +744,10 @@ static void cgroup_base_stat_sub(struct cgroup_base_stat *dst_bstat,
 
 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)
 {
-	struct cgroup_rstat_cpu *rstatc = rstat_cpu(&(cgrp->self.rstat), cpu);
+	struct cgroup_rstat_base_cpu *rstatc = rstat_base_cpu(
+			&(cgrp->self.rstat), cpu);
 	struct cgroup *parent = cgroup_parent(cgrp);
-	struct cgroup_rstat_cpu *prstatc;
+	struct cgroup_rstat_base_cpu *prstatc;
 	struct cgroup_base_stat delta;
 	unsigned seq;
 
@@ -748,25 +775,25 @@ static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)
 		cgroup_base_stat_add(&cgrp->last_bstat, &delta);
 
 		delta = rstatc->subtree_bstat;
-		prstatc = rstat_cpu(&(parent->self.rstat), cpu);
+		prstatc = rstat_base_cpu(&(parent->self.rstat), cpu);
 		cgroup_base_stat_sub(&delta, &rstatc->last_subtree_bstat);
 		cgroup_base_stat_add(&prstatc->subtree_bstat, &delta);
 		cgroup_base_stat_add(&rstatc->last_subtree_bstat, &delta);
 	}
 }
 
-static struct cgroup_rstat_cpu *
+static struct cgroup_rstat_base_cpu *
 cgroup_base_stat_cputime_account_begin(struct cgroup *cgrp, unsigned long *flags)
 {
-	struct cgroup_rstat_cpu *rstatc;
+	struct cgroup_rstat_base_cpu *rstatc;
 
-	rstatc = get_cpu_ptr(cgrp->self.rstat.rstat_cpu);
+	rstatc = get_cpu_ptr(cgrp->self.rstat.rstat_base_cpu);
 	*flags = u64_stats_update_begin_irqsave(&rstatc->bsync);
 	return rstatc;
 }
 
 static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp,
-						 struct cgroup_rstat_cpu *rstatc,
+						 struct cgroup_rstat_base_cpu *rstatc,
 						 unsigned long flags)
 {
 	u64_stats_update_end_irqrestore(&rstatc->bsync, flags);
@@ -776,7 +803,7 @@ static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp,
 
 void __cgroup_account_cputime(struct cgroup *cgrp, u64 delta_exec)
 {
-	struct cgroup_rstat_cpu *rstatc;
+	struct cgroup_rstat_base_cpu *rstatc;
 	unsigned long flags;
 
 	rstatc = cgroup_base_stat_cputime_account_begin(cgrp, &flags);
@@ -787,7 +814,7 @@ void __cgroup_account_cputime(struct cgroup *cgrp, u64 delta_exec)
 void __cgroup_account_cputime_field(struct cgroup *cgrp,
 				    enum cpu_usage_stat index, u64 delta_exec)
 {
-	struct cgroup_rstat_cpu *rstatc;
+	struct cgroup_rstat_base_cpu *rstatc;
 	unsigned long flags;
 
 	rstatc = cgroup_base_stat_cputime_account_begin(cgrp, &flags);
-- 
2.48.1