[RFC PATCH 5/7] mm/percpu: allow (un)charging objects without alloc/free

[Harry Yoo <harry.yoo@oracle.com>]

With a slab ctor/dtor pair, slab objects can retain a pointer to percpu
memory that remains allocated until the slab destructor frees it.

In such cases, the charging and uncharging of percpu memory should be
invoked when slab objects are allocated and freed. Allow explicit
(un)charging of percpu memory to ensure accurate memory accounting
for the slab destructor users.

Note that these APIs only (un)charge memory only for memory cgroups.
They do not affect memory allocation profiling. Memory allocation
profiling records percpu memory only when it is actually allocated or
freed.

Signed-off-by: Harry Yoo <harry.yoo@oracle.com>
---
 include/linux/percpu.h | 10 ++++++
 mm/percpu.c            | 79 +++++++++++++++++++++++++++++-------------
 2 files changed, 64 insertions(+), 25 deletions(-)

diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 52b5ea663b9f..2d13ef0885d6 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -140,6 +140,16 @@ extern void __init setup_per_cpu_areas(void);
 extern void __percpu *pcpu_alloc_noprof(size_t size, size_t align, bool reserved,
 				   gfp_t gfp) __alloc_size(1);
 
+#ifdef CONFIG_MEMCG
+extern bool pcpu_charge(void __percpu *__pdata, size_t size, gfp_t gfp);
+extern void pcpu_uncharge(void __percpu *__pdata, size_t size);
+#else
+static inline bool pcpu_charge(void __percpu *__pdata, size_t size, gfp_t gfp)
+{
+	return true;
+}
+static inline void pcpu_uncharge(void __percpu *__pdata, size_t size) { }
+#endif
 #define __alloc_percpu_gfp(_size, _align, _gfp)				\
 	alloc_hooks(pcpu_alloc_noprof(_size, _align, false, _gfp))
 #define __alloc_percpu(_size, _align)					\
diff --git a/mm/percpu.c b/mm/percpu.c
index b35494c8ede2..069d8e593164 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -1606,6 +1606,32 @@ static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr)
 	return pcpu_get_page_chunk(pcpu_addr_to_page(addr));
 }
 
+#ifdef CONFIG_MEM_ALLOC_PROFILING
+static void pcpu_alloc_tag_alloc_hook(struct pcpu_chunk *chunk, int off,
+				      size_t size)
+{
+	if (mem_alloc_profiling_enabled() && likely(chunk->obj_exts)) {
+		alloc_tag_add(&chunk->obj_exts[off >> PCPU_MIN_ALLOC_SHIFT].tag,
+			      current->alloc_tag, size);
+	}
+}
+
+static void pcpu_alloc_tag_free_hook(struct pcpu_chunk *chunk, int off, size_t size)
+{
+	if (mem_alloc_profiling_enabled() && likely(chunk->obj_exts))
+		alloc_tag_sub(&chunk->obj_exts[off >> PCPU_MIN_ALLOC_SHIFT].tag, size);
+}
+#else
+static void pcpu_alloc_tag_alloc_hook(struct pcpu_chunk *chunk, int off,
+				      size_t size)
+{
+}
+
+static void pcpu_alloc_tag_free_hook(struct pcpu_chunk *chunk, int off, size_t size)
+{
+}
+#endif
+
 #ifdef CONFIG_MEMCG
 static bool pcpu_memcg_pre_alloc_hook(size_t size, gfp_t gfp,
 				      struct obj_cgroup **objcgp)
@@ -1667,7 +1693,35 @@ static void pcpu_memcg_free_hook(struct pcpu_chunk *chunk, int off, size_t size)
 
 	obj_cgroup_put(objcg);
 }
+bool pcpu_charge(void *ptr, size_t size, gfp_t gfp)
+{
+	struct obj_cgroup *objcg = NULL;
+	void *addr;
+	struct pcpu_chunk *chunk;
+	int off;
+
+	addr = __pcpu_ptr_to_addr(ptr);
+	chunk = pcpu_chunk_addr_search(addr);
+	off = addr - chunk->base_addr;
+
+	if (!pcpu_memcg_pre_alloc_hook(size, gfp, &objcg))
+		return false;
+	pcpu_memcg_post_alloc_hook(objcg, chunk, off, size);
+	return true;
+}
+
+void pcpu_uncharge(void *ptr, size_t size)
+{
+	void *addr;
+	struct pcpu_chunk *chunk;
+	int off;
+
+	addr = __pcpu_ptr_to_addr(ptr);
+	chunk = pcpu_chunk_addr_search(addr);
+	off = addr - chunk->base_addr;
 
+	pcpu_memcg_free_hook(chunk, off, size);
+}
 #else /* CONFIG_MEMCG */
 static bool
 pcpu_memcg_pre_alloc_hook(size_t size, gfp_t gfp, struct obj_cgroup **objcgp)
@@ -1686,31 +1740,6 @@ static void pcpu_memcg_free_hook(struct pcpu_chunk *chunk, int off, size_t size)
 }
 #endif /* CONFIG_MEMCG */
 
-#ifdef CONFIG_MEM_ALLOC_PROFILING
-static void pcpu_alloc_tag_alloc_hook(struct pcpu_chunk *chunk, int off,
-				      size_t size)
-{
-	if (mem_alloc_profiling_enabled() && likely(chunk->obj_exts)) {
-		alloc_tag_add(&chunk->obj_exts[off >> PCPU_MIN_ALLOC_SHIFT].tag,
-			      current->alloc_tag, size);
-	}
-}
-
-static void pcpu_alloc_tag_free_hook(struct pcpu_chunk *chunk, int off, size_t size)
-{
-	if (mem_alloc_profiling_enabled() && likely(chunk->obj_exts))
-		alloc_tag_sub(&chunk->obj_exts[off >> PCPU_MIN_ALLOC_SHIFT].tag, size);
-}
-#else
-static void pcpu_alloc_tag_alloc_hook(struct pcpu_chunk *chunk, int off,
-				      size_t size)
-{
-}
-
-static void pcpu_alloc_tag_free_hook(struct pcpu_chunk *chunk, int off, size_t size)
-{
-}
-#endif
 
 /**
  * pcpu_alloc - the percpu allocator
-- 
2.43.0