From: Yafang Shao <laoar.shao@gmail.com>
To: ast@kernel.org, daniel@iogearbox.net, andrii@kernel.org,
kafai@fb.com, songliubraving@fb.com, yhs@fb.com,
john.fastabend@gmail.com, kpsingh@kernel.org, sdf@google.com,
haoluo@google.com, jolsa@kernel.org, tj@kernel.org,
dennis@kernel.org, cl@linux.com, akpm@linux-foundation.org,
penberg@kernel.org, rientjes@google.com, iamjoonsoo.kim@lge.com,
vbabka@suse.cz, roman.gushchin@linux.dev, 42.hyeyoo@gmail.com
Cc: linux-mm@kvack.org, bpf@vger.kernel.org,
Yafang Shao <laoar.shao@gmail.com>
Subject: [RFC PATCH bpf-next 9/9] bpf: Use active vm to account bpf map memory usage
Date: Mon, 12 Dec 2022 00:37:11 +0000 [thread overview]
Message-ID: <20221212003711.24977-10-laoar.shao@gmail.com> (raw)
In-Reply-To: <20221212003711.24977-1-laoar.shao@gmail.com>
The memory allocation via bpf_map_*alloc or bpf memalloc are accounted
with active vm. We only need to annotate the allocation. These memory
will automatically unaccount when they are freed.
Signed-off-by: Yafang Shao <laoar.shao@gmail.com>
---
fs/proc/meminfo.c | 3 +++
include/linux/active_vm.h | 10 +++++-----
kernel/bpf/memalloc.c | 5 +++++
kernel/bpf/ringbuf.c | 8 ++++++--
kernel/bpf/syscall.c | 25 +++++++++++++++++++++++--
5 files changed, 42 insertions(+), 9 deletions(-)
diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c
index 440960110a42..efe1fbd6a80e 100644
--- a/fs/proc/meminfo.c
+++ b/fs/proc/meminfo.c
@@ -16,6 +16,7 @@
#ifdef CONFIG_CMA
#include <linux/cma.h>
#endif
+#include <linux/active_vm.h>
#include <asm/page.h>
#include "internal.h"
@@ -159,6 +160,8 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
arch_report_meminfo(m);
+ seq_printf(m, "BPF: %8lu kB\n",
+ active_vm_item_sum(ACTIVE_VM_BPF) >> 10);
return 0;
}
diff --git a/include/linux/active_vm.h b/include/linux/active_vm.h
index 21f9aaca12c4..e26edfb3654e 100644
--- a/include/linux/active_vm.h
+++ b/include/linux/active_vm.h
@@ -2,6 +2,11 @@
#ifndef __INCLUDE_ACTIVE_VM_H
#define __INCLUDE_ACTIVE_VM_H
+enum active_vm_item {
+ ACTIVE_VM_BPF = 1,
+ NR_ACTIVE_VM_ITEM = ACTIVE_VM_BPF,
+};
+
#ifdef CONFIG_ACTIVE_VM
#include <linux/jump_label.h>
#include <linux/preempt.h>
@@ -18,11 +23,6 @@ static inline bool active_vm_enabled(void)
return true;
}
-enum active_vm_item {
- DUMMY_ITEM = 1,
- NR_ACTIVE_VM_ITEM = DUMMY_ITEM,
-};
-
struct active_vm_stat {
long stat[NR_ACTIVE_VM_ITEM];
};
diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c
index ebcc3dd0fa19..403ae0d83241 100644
--- a/kernel/bpf/memalloc.c
+++ b/kernel/bpf/memalloc.c
@@ -7,6 +7,8 @@
#include <linux/bpf_mem_alloc.h>
#include <linux/memcontrol.h>
#include <asm/local.h>
+#include <linux/page_ext.h>
+#include <linux/active_vm.h>
/* Any context (including NMI) BPF specific memory allocator.
*
@@ -165,11 +167,13 @@ static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node)
{
struct mem_cgroup *memcg = NULL, *old_memcg;
unsigned long flags;
+ int old_active_vm;
void *obj;
int i;
memcg = get_memcg(c);
old_memcg = set_active_memcg(memcg);
+ old_active_vm = active_vm_item_set(ACTIVE_VM_BPF);
for (i = 0; i < cnt; i++) {
/*
* free_by_rcu is only manipulated by irq work refill_work().
@@ -209,6 +213,7 @@ static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node)
if (IS_ENABLED(CONFIG_PREEMPT_RT))
local_irq_restore(flags);
}
+ active_vm_item_set(old_active_vm);
set_active_memcg(old_memcg);
mem_cgroup_put(memcg);
}
diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c
index 3264bf509c68..7575f078eb34 100644
--- a/kernel/bpf/ringbuf.c
+++ b/kernel/bpf/ringbuf.c
@@ -11,6 +11,7 @@
#include <linux/kmemleak.h>
#include <uapi/linux/btf.h>
#include <linux/btf_ids.h>
+#include <linux/active_vm.h>
#define RINGBUF_CREATE_FLAG_MASK (BPF_F_NUMA_NODE)
@@ -107,16 +108,18 @@ static struct page **bpf_ringbuf_pages_alloc(int nr_meta_pages,
{
int nr_pages = nr_meta_pages + nr_data_pages;
struct page **pages, *page;
+ int old_active_vm;
int array_size;
int i;
+ old_active_vm = active_vm_item_set(ACTIVE_VM_BPF);
array_size = (nr_meta_pages + 2 * nr_data_pages) * sizeof(*pages);
pages = bpf_map_area_alloc(array_size, numa_node);
if (!pages)
goto err;
for (i = 0; i < nr_pages; i++) {
- page = alloc_pages_node(numa_node, flags, 0);
+ page = alloc_pages_node(numa_node, flags | __GFP_ACCOUNT, 0);
if (!page) {
nr_pages = i;
goto err_free_pages;
@@ -125,12 +128,13 @@ static struct page **bpf_ringbuf_pages_alloc(int nr_meta_pages,
if (i >= nr_meta_pages)
pages[nr_data_pages + i] = page;
}
-
+ active_vm_item_set(old_active_vm);
return pages;
err_free_pages:
bpf_ringbuf_pages_free(pages, nr_pages);
err:
+ active_vm_item_set(old_active_vm);
return NULL;
}
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index c38875d6aea4..92572d4a09fb 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -35,6 +35,8 @@
#include <linux/rcupdate_trace.h>
#include <linux/memcontrol.h>
#include <linux/trace_events.h>
+#include <linux/page_ext.h>
+#include <linux/active_vm.h>
#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \
(map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \
@@ -312,11 +314,14 @@ static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable)
const gfp_t gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_ACCOUNT;
unsigned int flags = 0;
unsigned long align = 1;
+ int old_active_vm;
void *area;
+ void *ptr;
if (size >= SIZE_MAX)
return NULL;
+ old_active_vm = active_vm_item_set(ACTIVE_VM_BPF);
/* kmalloc()'ed memory can't be mmap()'ed */
if (mmapable) {
BUG_ON(!PAGE_ALIGNED(size));
@@ -325,13 +330,17 @@ static void *__bpf_map_area_alloc(u64 size, int numa_node, bool mmapable)
} else if (size <= (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) {
area = kmalloc_node(size, gfp | GFP_USER | __GFP_NORETRY,
numa_node);
- if (area != NULL)
+ if (area != NULL) {
+ active_vm_item_set(old_active_vm);
return area;
+ }
}
- return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
+ ptr = __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
gfp | GFP_KERNEL | __GFP_RETRY_MAYFAIL, PAGE_KERNEL,
flags, numa_node, __builtin_return_address(0));
+ active_vm_item_set(old_active_vm);
+ return ptr;
}
void *bpf_map_area_alloc(u64 size, int numa_node)
@@ -445,11 +454,14 @@ void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
int node)
{
struct mem_cgroup *memcg, *old_memcg;
+ int old_active_vm;
void *ptr;
memcg = bpf_map_get_memcg(map);
old_memcg = set_active_memcg(memcg);
+ old_active_vm = active_vm_item_set(ACTIVE_VM_BPF);
ptr = kmalloc_node(size, flags | __GFP_ACCOUNT, node);
+ active_vm_item_set(old_active_vm);
set_active_memcg(old_memcg);
mem_cgroup_put(memcg);
@@ -459,11 +471,14 @@ void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags,
void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags)
{
struct mem_cgroup *memcg, *old_memcg;
+ int old_active_vm;
void *ptr;
memcg = bpf_map_get_memcg(map);
old_memcg = set_active_memcg(memcg);
+ old_active_vm = active_vm_item_set(ACTIVE_VM_BPF);
ptr = kzalloc(size, flags | __GFP_ACCOUNT);
+ active_vm_item_set(old_active_vm);
set_active_memcg(old_memcg);
mem_cgroup_put(memcg);
@@ -474,11 +489,14 @@ void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size,
gfp_t flags)
{
struct mem_cgroup *memcg, *old_memcg;
+ int old_active_vm;
void *ptr;
memcg = bpf_map_get_memcg(map);
old_memcg = set_active_memcg(memcg);
+ old_active_vm = active_vm_item_set(ACTIVE_VM_BPF);
ptr = kvcalloc(n, size, flags | __GFP_ACCOUNT);
+ active_vm_item_set(old_active_vm);
set_active_memcg(old_memcg);
mem_cgroup_put(memcg);
@@ -490,10 +508,13 @@ void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size,
{
struct mem_cgroup *memcg, *old_memcg;
void __percpu *ptr;
+ int old_active_vm;
memcg = bpf_map_get_memcg(map);
old_memcg = set_active_memcg(memcg);
+ old_active_vm = active_vm_item_set(ACTIVE_VM_BPF);
ptr = __alloc_percpu_gfp(size, align, flags | __GFP_ACCOUNT);
+ active_vm_item_set(old_active_vm);
set_active_memcg(old_memcg);
mem_cgroup_put(memcg);
--
2.30.1 (Apple Git-130)
next prev parent reply other threads:[~2022-12-12 0:38 UTC|newest]
Thread overview: 20+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-12-12 0:37 [RFC PATCH bpf-next 0/9] mm, bpf: Add BPF into /proc/meminfo Yafang Shao
2022-12-12 0:37 ` [RFC PATCH bpf-next 1/9] mm: Introduce active vm item Yafang Shao
2022-12-12 0:37 ` [RFC PATCH bpf-next 2/9] mm: Allow using active vm in all contexts Yafang Shao
2022-12-12 0:37 ` [RFC PATCH bpf-next 3/9] mm: percpu: Account active vm for percpu Yafang Shao
2022-12-12 0:37 ` [RFC PATCH bpf-next 4/9] mm: slab: Account active vm for slab Yafang Shao
2022-12-12 0:37 ` [RFC PATCH bpf-next 5/9] mm: Account active vm for page Yafang Shao
2022-12-12 0:37 ` [RFC PATCH bpf-next 6/9] bpf: Introduce new helpers bpf_ringbuf_pages_{alloc,free} Yafang Shao
2022-12-12 0:37 ` [RFC PATCH bpf-next 7/9] bpf: Use bpf_map_kzalloc in arraymap Yafang Shao
2022-12-12 0:37 ` [RFC PATCH bpf-next 8/9] bpf: Use bpf_map_kvcalloc in bpf_local_storage Yafang Shao
2022-12-12 0:37 ` Yafang Shao [this message]
2022-12-14 8:45 ` [RFC PATCH bpf-next 9/9] bpf: Use active vm to account bpf map memory usage kernel test robot
2022-12-14 12:01 ` Yafang Shao
2022-12-12 17:54 ` [RFC PATCH bpf-next 0/9] mm, bpf: Add BPF into /proc/meminfo Vlastimil Babka
2022-12-13 11:52 ` Yafang Shao
2022-12-13 14:56 ` Hyeonggon Yoo
2022-12-13 15:52 ` Vlastimil Babka
2022-12-13 19:21 ` Paul E. McKenney
2022-12-14 10:46 ` Yafang Shao
2022-12-14 10:43 ` Yafang Shao
2022-12-14 10:34 ` Yafang Shao
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20221212003711.24977-10-laoar.shao@gmail.com \
--to=laoar.shao@gmail.com \
--cc=42.hyeyoo@gmail.com \
--cc=akpm@linux-foundation.org \
--cc=andrii@kernel.org \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=cl@linux.com \
--cc=daniel@iogearbox.net \
--cc=dennis@kernel.org \
--cc=haoluo@google.com \
--cc=iamjoonsoo.kim@lge.com \
--cc=john.fastabend@gmail.com \
--cc=jolsa@kernel.org \
--cc=kafai@fb.com \
--cc=kpsingh@kernel.org \
--cc=linux-mm@kvack.org \
--cc=penberg@kernel.org \
--cc=rientjes@google.com \
--cc=roman.gushchin@linux.dev \
--cc=sdf@google.com \
--cc=songliubraving@fb.com \
--cc=tj@kernel.org \
--cc=vbabka@suse.cz \
--cc=yhs@fb.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox