Re: [Bug 201699] New: kmemleak in memcg_create_kmem_cache

[段熊春 <duanxiongchun@bytedance.com>]

[-- Attachment #1: Type: text/plain, Size: 11072 bytes --]

I had double check on the newest version 4.20-rc3

I had wrote a small test .


Test service code 

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
int main()
{
	struct stat buf;
	int fd ;
	fd = open("/var/log/test",O_RDWR|O_CREAT|O_APPEND|O_CLOEXEC,0644);
	sleep(1);
	fd = open("/var/log/test.1", O_RDWR|O_CREAT|O_APPEND|O_CLOEXEC|O_SYNC, 0644);
	char log[4096] = {'a'};
	if (fd > 0) {
		write(fd, log, 4096);
		close(fd);
	}

	return 1;
}


Test.service 

[Service]
ExecStart=/usr/bin/test
Restart=always
RestartSec=100ms
MemoryLimit=1G
StartLimitInterval=0
[Install]
WantedBy=default.target

Probe code 

Get test.1 node address kretprobe  code 

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/kprobes.h>
#include <linux/seq_file.h>
#include <linux/proc_fs.h>
#include <linux/spinlock.h>

static struct kretprobe kprobe_ret_object = {
    .kp.symbol_name    = "d_lookup",
};

static int handler_d_lookup_pre(struct kretprobe_instance *p, struct pt_regs *regs)
{
	int *tmp;
	struct qstr * name =(struct qstr *)regs->si;
	tmp=(int *)p->data;
	*tmp=0;
	if(strcmp("test.1",name->name)==0)
		*tmp=1;
	return 0;
}

static int ret_handler_d_lookup_pre(struct kretprobe_instance *p,struct pt_regs *regs)
{
	int *tmp;
	struct dentry * tmp_dentry = (struct dentry *)regs_return_value(regs);
	tmp = (int *)p->data;
	if(*tmp == 1)
		printk(KERN_INFO "return dentry address %px,inode address %px\n",
			tmp_dentry,tmp_dentry->d_inode);
	return 0;
}
static int __init kprobe_init(void)
{
    int ret;
    kprobe_ret_object.entry_handler = handler_d_lookup_pre;
    kprobe_ret_object.handler = ret_handler_d_lookup_pre;
    kprobe_ret_object.maxactive = 0;
    kprobe_ret_object.data_size = sizeof(int);

    ret = register_kretprobe(&kprobe_ret_object);
    if (ret < 0) {
        printk(KERN_INFO "register_kprobe failed, returned %d\n", ret);
        return ret;
    }
    printk(KERN_INFO "Planted kprobe at %p\n", kprobe_ret_object.kp.addr);
    return 0;
}

static void __exit kprobe_exit(void)
{
    unregister_kretprobe(&kprobe_ret_object);
    printk(KERN_INFO "kprobe at %p unregistered\n", kprobe_ret_object.kp.addr);
}

module_init(kprobe_init)
module_exit(kprobe_exit)
MODULE_LICENSE("GPL”);

Get unreleased mem_cgroup address

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/kprobes.h>
#include <linux/seq_file.h>
#include <linux/proc_fs.h>
#include <linux/spinlock.h>

static struct kretprobe css_alloc = {
    .kp.symbol_name    = "mem_cgroup_css_alloc",
};

static struct kprobe css_free = {
    .symbol_name    = "mem_cgroup_css_free",
};
static struct kprobe css_released = {
    .symbol_name    = "mem_cgroup_css_released",
};
static struct kprobe css_offline = {
    .symbol_name    = "mem_cgroup_css_offline",
};
static struct kprobe trycharge = {
    .symbol_name    = "page_counter_try_charge",
};
static struct kprobe charge = {
    .symbol_name    = "page_counter_charge",
};
static struct kprobe uncharge = {
    .symbol_name    = "page_counter_uncharge",
};

atomic_t cssalloc;
atomic_t cssfree;
atomic_t cssreleased;
atomic_t cssoffline;
static spinlock_t my_lock = __SPIN_LOCK_UNLOCKED();
void * css_addr=0;
void * memory_addr=0;

static int handler_trycharge(struct kprobe *p,struct pt_regs *regs)
{
    if (memory_addr == (void *)(regs->di)){
	printk(KERN_INFO"trycharge_memory %px nr %px",(void *)memory_addr,(void *)regs->si);
	spin_lock(&my_lock);
	dump_stack();
	spin_unlock(&my_lock);
    }
    return 0;
}
static int handler_charge(struct kprobe *p,struct pt_regs *regs)
{
    if (memory_addr == (void *)(regs->di)){
	printk(KERN_INFO"charge_memory %px,nr %px",(void *)memory_addr,(void *)regs->si);
	spin_lock(&my_lock);
	dump_stack();
	spin_unlock(&my_lock);
    }
    return 0;
}
static int handler_uncharge(struct kprobe *p,struct pt_regs *regs)
{
    if (memory_addr == (void *)(regs->di)){
	printk(KERN_INFO"uncharge_memory %px,nr %px",(void *)memory_addr,(void *)regs->si);
	spin_lock(&my_lock);
	dump_stack();
	spin_unlock(&my_lock);
    }
    return 0;
}

static int handler_cssalloc_pre(struct kretprobe_instance *p, struct pt_regs *regs)
{
    atomic_inc(&cssalloc);
    return 0;
}

static int ret_handler_cssalloc_pre(struct kretprobe_instance *p,struct pt_regs *regs)
{
	if (css_addr==0){
		css_addr=(void *)regs_return_value(regs);
		memory_addr=(void *)(regs_return_value(regs)+192);
	}
	return 0;
}

static int handler_cssfree_pre(struct kprobe *p,struct pt_regs *regs)
{
   atomic_inc(&cssfree);
    if (css_addr == (void *)(regs->di))
	css_addr = 0;
    return 0;
}
static int handler_cssreleased_pre(struct kprobe *p,struct pt_regs *regs)
{
   atomic_inc(&cssreleased);
    return 0;
}
static int handler_cssoffline_pre(struct kprobe *p,struct pt_regs *regs)
{
   atomic_inc(&cssoffline);
    return 0;
}

static void handler_post(struct kprobe *p, struct pt_regs *regs,
                unsigned long flags)
{
}

static int handler_fault(struct kprobe *p, struct pt_regs *regs, int trapnr)
{
    return 0;
}

static int myleak_read(struct seq_file *m, void *v)
{
        seq_printf(m,"alloc %d  offline %d release %d free %d trace addr %px\n",atomic_read(&cssalloc),atomic_read(&cssoffline),
		atomic_read(&cssreleased),atomic_read(&cssfree),css_addr);
        return 0;
}

static int myleak_open(struct inode *inode, struct file *file)
{
        return single_open(file, myleak_read, NULL);
}

ssize_t myleak_write(struct file *filp,const char *buf,size_t count,loff_t *offp){
	css_addr = 0;
	return count;
}

static const struct file_operations myleak = {
        .open =myleak_open,
        .read = seq_read,
	.write = myleak_write,
        .llseek = seq_lseek,
        .release = single_release,
};

static int __init kprobe_init(void)
{
    int ret;
    css_alloc.entry_handler = handler_cssalloc_pre;
    css_alloc.handler = ret_handler_cssalloc_pre;
    css_alloc.maxactive = 0;

    css_free.pre_handler = handler_cssfree_pre;
    css_free.post_handler = handler_post;
    css_free.fault_handler = handler_fault;

    css_released.pre_handler = handler_cssreleased_pre;
    css_released.post_handler = handler_post;
    css_released.fault_handler = handler_fault;

    css_offline.pre_handler = handler_cssoffline_pre;
    css_offline.post_handler = handler_post;
    css_offline.fault_handler = handler_fault;

    trycharge.pre_handler = handler_trycharge;
    trycharge.post_handler = handler_post;
    trycharge.fault_handler = handler_fault;

    charge.pre_handler = handler_charge;
    charge.post_handler = handler_post;
    charge.fault_handler = handler_fault;

    uncharge.pre_handler = handler_uncharge;
    uncharge.post_handler = handler_post;
    uncharge.fault_handler = handler_fault;
    atomic_set(&cssalloc,0);
    atomic_set(&cssfree,0);
    atomic_set(&cssreleased,0);
    atomic_set(&cssoffline,0);

    ret = register_kretprobe(&css_alloc);
    if (ret < 0) {
        printk(KERN_INFO "register_kprobe failed, returned %d\n", ret);
        return ret;
    }
    ret = register_kprobe(&css_free);
    if (ret < 0) {
        printk(KERN_INFO "register_kprobe failed, returned %d\n", ret);
        return ret;
    }
    ret = register_kprobe(&css_released);
    if (ret < 0) {
        printk(KERN_INFO "register_kprobe failed, returned %d\n", ret);
        return ret;
    }
    ret = register_kprobe(&css_offline);
    if (ret < 0) {
        printk(KERN_INFO "register_kprobe failed, returned %d\n", ret);
        return ret;
    }
    ret = register_kprobe(&trycharge);
    if (ret < 0) {
        printk(KERN_INFO "register_kprobe failed, returned %d\n", ret);
        return ret;
    }
    ret = register_kprobe(&charge);
    if (ret < 0) {
        printk(KERN_INFO "register_kprobe failed, returned %d\n", ret);
        return ret;
    }
    ret = register_kprobe(&uncharge);
    if (ret < 0) {
        printk(KERN_INFO "register_kprobe failed, returned %d\n", ret);
        return ret;
    }
    proc_create("cgroup_leak", 0, NULL, &myleak);
    printk(KERN_INFO "Planted kprobe at %p\n", css_alloc.kp.addr);
    printk(KERN_INFO "Planted kprobe at %p\n", css_free.addr);
    printk(KERN_INFO "Planted kprobe at %p\n", css_released.addr);
    printk(KERN_INFO "Planted kprobe at %p\n", css_offline.addr);
    printk(KERN_INFO "Planted kprobe at %p\n", trycharge.addr);
    printk(KERN_INFO "Planted kprobe at %p\n", charge.addr);
    printk(KERN_INFO "Planted kprobe at %p\n", uncharge.addr);
    return 0;
}

static void __exit kprobe_exit(void)
{
    unregister_kretprobe(&css_alloc);
    unregister_kprobe(&css_free);
    unregister_kprobe(&css_released);
    unregister_kprobe(&css_offline);
    unregister_kprobe(&trycharge);
    unregister_kprobe(&charge);
    unregister_kprobe(&uncharge);
    printk(KERN_INFO "kprobe at %p unregistered\n", css_alloc.kp.addr);
    printk(KERN_INFO "kprobe at %p unregistered\n", css_free.addr);
    printk(KERN_INFO "kprobe at %p unregistered\n", css_released.addr);
    printk(KERN_INFO "kprobe at %p unregistered\n", css_offline.addr);
    printk(KERN_INFO "kprobe at %p unregistered\n", trycharge.addr);
    printk(KERN_INFO "kprobe at %p unregistered\n", charge.addr);
    printk(KERN_INFO "kprobe at %p unregistered\n", uncharge.addr);
    remove_proc_entry("cgroup_leak",NULL);
}

module_init(kprobe_init)
module_exit(kprobe_exit)
MODULE_LICENSE("GPL");




First delete /var/log/test /var/log/test.1

Then run command systemctl start test ,After three second run command systemctl stop test 

Then write a python script open /var/log/test.1
Import time
f=open("/var/log/test.1”)
Time.sleep(1000)

Then in other console echo 3 > /proc/sys/vm/drop_caches

after that we find mem_cgroup object  still unreleased。

if we close the python process，then echo 3 >  /proc/sys/vm/drop_caches。
the mem_cgroup was released。

I think because the inode of test.1 is hold by python process ， so drop_caches is no used。

I do not think this is a real bug。 but programer should care about   the memory used。 -:)

Thanks for reply
bytedance.net
段熊春
duanxiongchun@bytedance.com




> On Nov 22, 2018, at 3:34 PM, Michal Hocko <mhocko@kernel.org> wrote:
> 
> On Thu 22-11-18 10:56:04, 段熊春 wrote:
>> After long time dig, we find their lots of offline but not release memcg object in memory eating lots of memory.
>> Why this memcg not release? Because the inode pagecache use  some page which is charged to those memcg,
> 
> As already explained these objects should be reclaimed under memory
> pressure. If they are not then there is a bug. And Roman has fixed some
> of those recently.
> 
> Which kernel version are you using?
> -- 
> Michal Hocko
> SUSE Labs


[-- Attachment #2: Type: text/html, Size: 24593 bytes --]