* [PATCH 1/3] zsmalloc: make struct can be migrated
2015-11-27 12:12 [PATCH v3 0/3] zsmalloc: make its pages can be migrated Hui Zhu
@ 2015-11-27 12:12 ` Hui Zhu
2015-11-27 12:12 ` [PATCH 2/3] zsmalloc: make its page "PageMobile" Hui Zhu
2015-11-27 12:12 ` [PATCH 3/3] zram: make create "__GFP_MOVABLE" pool Hui Zhu
2 siblings, 0 replies; 4+ messages in thread
From: Hui Zhu @ 2015-11-27 12:12 UTC (permalink / raw)
To: Minchan Kim, Nitin Gupta, Sergey Senozhatsky, linux-kernel, linux-mm
Cc: teawater, Hui Zhu
After "[RFC zsmalloc 0/4] meta diet" [1], the struct it close to
be migrated.
But the LRU is still used. And to use the migration frame in [2], need
a way to get class through page struct.
So this patch add a new struct zs_migration and store it in struct page.
[1] https://lkml.org/lkml/2015/8/10/90
[2] https://lkml.org/lkml/2015/7/7/21
Signed-off-by: Hui Zhu <zhuhui@xiaomi.com>
---
mm/zsmalloc.c | 178 ++++++++++++++++++++++++++++++++++------------------------
1 file changed, 104 insertions(+), 74 deletions(-)
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index 1b18144..57c91a5 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -17,10 +17,10 @@
*
* Usage of struct page fields:
* page->first_page: points to the first component (0-order) page
- * page->index (union with page->freelist): offset of the first object
- * starting in this page.
- * page->lru: links together all component pages (except the first page)
- * of a zspage
+ * ZS_MIGRATION(page)->index: offset of the first object starting in
+ * this page
+ * ZS_MIGRATION(page)->lru: links together all component pages (except
+ * the first page) of a zspage
*
* For _first_ page only:
*
@@ -28,9 +28,9 @@
* component page after the first page
* If the page is first_page for huge object, it stores handle.
* Look at size_class->huge.
- * page->lru: links together first pages of various zspages.
+ * ZS_MIGRATION(page)->lru: links together first pages of various zspages.
* Basically forming list of zspages in a fullness group.
- * page->freelist: override by struct zs_meta
+ * ZS_MIGRATION(page)->index: override by struct zs_meta
*
* Usage of struct page flags:
* PG_private: identifies the first component page
@@ -136,7 +136,7 @@
#define INUSE_BITS 11
#define INUSE_MASK ((1 << INUSE_BITS) - 1)
#define ETC_BITS ((sizeof(unsigned long) * 8) - FREE_OBJ_IDX_BITS - \
- CLASS_IDX_BITS - FULLNESS_BITS - INUSE_BITS)
+ FULLNESS_BITS - INUSE_BITS)
/*
* On systems with 4K page size, this gives 255 size classes! There is a
* trader-off here:
@@ -266,12 +266,21 @@ struct zs_pool {
*/
struct zs_meta {
unsigned long free_idx:FREE_OBJ_IDX_BITS;
- unsigned long class_idx:CLASS_IDX_BITS;
unsigned long fullness:FULLNESS_BITS;
unsigned long inuse:INUSE_BITS;
unsigned long etc:ETC_BITS;
};
+struct zs_migration {
+ unsigned long index;
+ struct size_class *class;
+ struct list_head lru;
+ struct page *page;
+};
+
+#define ZS_MIGRATION(p) ((struct zs_migration *)((p)->freelist))
+#define ZS_META(p) ((struct zs_meta *)&(ZS_MIGRATION(p)->index))
+
struct mapping_area {
#ifdef CONFIG_PGTABLE_MAPPING
struct vm_struct *vm; /* vm area for mapping object that span pages */
@@ -311,6 +320,19 @@ static void record_obj(unsigned long handle, unsigned long obj)
*(unsigned long *)handle = obj;
}
+struct kmem_cache *zs_migration_cachep;
+
+static struct migration *alloc_migration(gfp_t flags)
+{
+ return (struct migration *)kmem_cache_alloc(zs_migration_cachep,
+ flags & ~__GFP_HIGHMEM);
+}
+
+static void free_migration(struct migration *migration)
+{
+ kmem_cache_free(zs_migration_cachep, (void *)migration);
+}
+
/* zpool driver */
#ifdef CONFIG_ZPOOL
@@ -414,7 +436,7 @@ static int get_inuse_obj(struct page *page)
BUG_ON(!is_first_page(page));
- m = (struct zs_meta *)&page->freelist;
+ m = ZS_META(page);
return m->inuse;
}
@@ -425,48 +447,22 @@ static void set_inuse_obj(struct page *page, int inc)
BUG_ON(!is_first_page(page));
- m = (struct zs_meta *)&page->freelist;
+ m = ZS_META(page);
m->inuse += inc;
}
static void set_free_obj_idx(struct page *first_page, int idx)
{
- struct zs_meta *m = (struct zs_meta *)&first_page->freelist;
+ struct zs_meta *m = ZS_META(first_page);
m->free_idx = idx;
}
static unsigned long get_free_obj_idx(struct page *first_page)
{
- struct zs_meta *m = (struct zs_meta *)&first_page->freelist;
+ struct zs_meta *m = ZS_META(first_page);
return m->free_idx;
}
-static void get_zspage_mapping(struct page *page, unsigned int *class_idx,
- enum fullness_group *fullness)
-{
- struct zs_meta *m;
- BUG_ON(!is_first_page(page));
-
- m = (struct zs_meta *)&page->freelist;
- *fullness = m->fullness;
- *class_idx = m->class_idx;
-}
-
-static void set_zspage_mapping(struct page *page, unsigned int class_idx,
- enum fullness_group fullness)
-{
- struct zs_meta *m;
-
- BUG_ON(!is_first_page(page));
-
- BUG_ON(class_idx >= (1 << CLASS_IDX_BITS));
- BUG_ON(fullness >= (1 << FULLNESS_BITS));
-
- m = (struct zs_meta *)&page->freelist;
- m->fullness = fullness;
- m->class_idx = class_idx;
-}
-
/*
* zsmalloc divides the pool into various size classes where each
* class maintains a list of zspages where each zspage is divided
@@ -698,7 +694,7 @@ static void insert_zspage(struct page *page, struct size_class *class,
* We want to see more ZS_FULL pages and less almost
* empty/full. Put pages with higher inuse first.
*/
- list_add_tail(&page->lru, &(*head)->lru);
+ list_add_tail(&ZS_MIGRATION(page)->lru, &ZS_MIGRATION(*head)->lru);
if (get_inuse_obj(page) >= get_inuse_obj(*head))
*head = page;
}
@@ -719,13 +715,18 @@ static void remove_zspage(struct page *page, struct size_class *class,
head = &class->fullness_list[fullness];
BUG_ON(!*head);
- if (list_empty(&(*head)->lru))
+ if (list_empty(&ZS_MIGRATION(*head)->lru))
*head = NULL;
- else if (*head == page)
- *head = (struct page *)list_entry((*head)->lru.next,
- struct page, lru);
+ else if (*head == page) {
+ struct zs_migration *m;
- list_del_init(&page->lru);
+ m = (struct zs_migration *)
+ list_entry(ZS_MIGRATION(*head)->lru.next,
+ struct zs_migration, lru);
+ *head = m->page;
+ }
+
+ list_del_init(&ZS_MIGRATION(page)->lru);
zs_stat_dec(class, fullness == ZS_ALMOST_EMPTY ?
CLASS_ALMOST_EMPTY : CLASS_ALMOST_FULL, 1);
}
@@ -742,19 +743,18 @@ static void remove_zspage(struct page *page, struct size_class *class,
static enum fullness_group fix_fullness_group(struct size_class *class,
struct page *page)
{
- int class_idx;
enum fullness_group currfg, newfg;
BUG_ON(!is_first_page(page));
- get_zspage_mapping(page, &class_idx, &currfg);
+ currfg = ZS_META(page)->fullness;
newfg = get_fullness_group(class, page);
if (newfg == currfg)
goto out;
remove_zspage(page, class, currfg);
insert_zspage(page, class, newfg);
- set_zspage_mapping(page, class_idx, newfg);
+ ZS_META(page)->fullness = newfg;
out:
return newfg;
@@ -817,8 +817,14 @@ static struct page *get_next_page(struct page *page)
next = NULL;
else if (is_first_page(page))
next = (struct page *)page_private(page);
- else
- next = list_entry(page->lru.next, struct page, lru);
+ else {
+ struct zs_migration *m;
+
+ m = (struct zs_migration *)
+ list_entry(ZS_MIGRATION(page)->lru.next,
+ struct zs_migration, lru);
+ next = m->page;
+ }
return next;
}
@@ -908,13 +914,15 @@ static void reset_page(struct page *page)
clear_bit(PG_private, &page->flags);
clear_bit(PG_private_2, &page->flags);
set_page_private(page, 0);
+ free_migration(page->freelist);
page->freelist = NULL;
page_mapcount_reset(page);
}
static void free_zspage(struct page *first_page)
{
- struct page *nextp, *tmp, *head_extra;
+ struct zs_migration *tmp, *nextm;
+ struct page *nextp, *head_extra;
BUG_ON(!is_first_page(first_page));
BUG_ON(get_inuse_obj(first_page));
@@ -928,8 +936,10 @@ static void free_zspage(struct page *first_page)
if (!head_extra)
return;
- list_for_each_entry_safe(nextp, tmp, &head_extra->lru, lru) {
- list_del(&nextp->lru);
+ list_for_each_entry_safe(nextm, tmp, &ZS_MIGRATION(head_extra)->lru,
+ lru) {
+ nextp = nextm->page;
+ list_del(&ZS_MIGRATION(nextp)->lru);
reset_page(nextp);
__free_page(nextp);
}
@@ -951,15 +961,14 @@ static void init_zspage(struct page *first_page, struct size_class *class)
void *vaddr;
/*
- * page->index stores offset of first object starting
- * in the page.
+ * ZS_MIGRATION(page)->index stores offset of first object
+ * starting in the page.
*/
if (page != first_page)
- page->index = off;
+ ZS_MIGRATION(page)->index = off;
vaddr = kmap_atomic(page);
link = (struct link_free *)vaddr + off / sizeof(*link);
-
while ((off += class->size) < PAGE_SIZE) {
link->next = (obj_idx++ << OBJ_ALLOCATED_TAG);
link += class->size / sizeof(*link);
@@ -994,13 +1003,13 @@ static struct page *alloc_zspage(struct size_class *class, gfp_t flags)
/*
* Allocate individual pages and link them together as:
* 1. first page->private = first sub-page
- * 2. all sub-pages are linked together using page->lru
+ * 2. all sub-pages are linked together using ZS_MIGRATION(page)->lru
* 3. each sub-page is linked to the first page using page->first_page
*
* For each size class, First/Head pages are linked together using
- * page->lru. Also, we set PG_private to identify the first page
- * (i.e. no other sub-page has this flag set) and PG_private_2 to
- * identify the last page.
+ * ZS_MIGRATION(page)->lru. Also, we set PG_private to identify the
+ * first page (i.e. no other sub-page has this flag set) and
+ * PG_private_2 to identify the last page.
*/
error = -ENOMEM;
for (i = 0; i < class->pages_per_zspage; i++) {
@@ -1009,8 +1018,17 @@ static struct page *alloc_zspage(struct size_class *class, gfp_t flags)
page = alloc_page(flags);
if (!page)
goto cleanup;
+ page->freelist = alloc_migration(flags);
+ if (!page->freelist) {
+ __free_page(page);
+ goto cleanup;
+ }
INIT_LIST_HEAD(&page->lru);
+ INIT_LIST_HEAD(&ZS_MIGRATION(page)->lru);
+ ZS_MIGRATION(page)->index = 0;
+ ZS_MIGRATION(page)->page = page;
+ ZS_MIGRATION(page)->class = class;
if (i == 0) { /* first page */
SetPagePrivate(page);
set_page_private(page, 0);
@@ -1022,7 +1040,8 @@ static struct page *alloc_zspage(struct size_class *class, gfp_t flags)
if (i >= 1)
page->first_page = first_page;
if (i >= 2)
- list_add(&page->lru, &prev_page->lru);
+ list_add(&ZS_MIGRATION(page)->lru,
+ &ZS_MIGRATION(prev_page)->lru);
if (i == class->pages_per_zspage - 1) /* last page */
SetPagePrivate2(page);
prev_page = page;
@@ -1292,7 +1311,6 @@ void *zs_map_object(struct zs_pool *pool, unsigned long handle,
struct page *obj_page, *first_page;
unsigned long obj, obj_idx, obj_ofs;
- unsigned int class_idx;
enum fullness_group fg;
struct size_class *class;
struct mapping_area *area;
@@ -1315,9 +1333,10 @@ void *zs_map_object(struct zs_pool *pool, unsigned long handle,
obj_to_obj_idx(obj, &obj_page, &obj_idx);
first_page = get_first_page(obj_page);
- get_zspage_mapping(first_page, &class_idx, &fg);
- class = pool->size_class[class_idx];
+ fg = ZS_META(first_page)->fullness;
+ class = ZS_MIGRATION(first_page)->class;
+
obj_ofs = (class->size * obj_idx) % PAGE_SIZE;
area = &get_cpu_var(zs_map_area);
@@ -1348,7 +1367,6 @@ void zs_unmap_object(struct zs_pool *pool, unsigned long handle)
struct page *obj_page, *first_page;
unsigned long obj, obj_idx, obj_ofs;
- unsigned int class_idx;
enum fullness_group fg;
struct size_class *class;
struct mapping_area *area;
@@ -1359,8 +1377,8 @@ void zs_unmap_object(struct zs_pool *pool, unsigned long handle)
obj_to_obj_idx(obj, &obj_page, &obj_idx);
first_page = get_first_page(obj_page);
- get_zspage_mapping(first_page, &class_idx, &fg);
- class = pool->size_class[class_idx];
+ fg = ZS_META(first_page)->fullness;
+ class = ZS_MIGRATION(first_page)->class;
obj_ofs = (class->size * obj_idx) % PAGE_SIZE;
area = this_cpu_ptr(&zs_map_area);
@@ -1450,7 +1468,8 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size)
return 0;
}
- set_zspage_mapping(first_page, class->index, ZS_EMPTY);
+ ZS_META(first_page)->fullness = ZS_EMPTY;
+ ZS_MIGRATION(first_page)->class = class;
atomic_long_add(class->pages_per_zspage,
&pool->pages_allocated);
@@ -1501,7 +1520,6 @@ void zs_free(struct zs_pool *pool, unsigned long handle)
{
struct page *first_page, *obj_page;
unsigned long obj, obj_idx;
- int class_idx;
struct size_class *class;
enum fullness_group fullness;
@@ -1513,8 +1531,8 @@ void zs_free(struct zs_pool *pool, unsigned long handle)
obj_to_obj_idx(obj, &obj_page, &obj_idx);
first_page = get_first_page(obj_page);
- get_zspage_mapping(first_page, &class_idx, &fullness);
- class = pool->size_class[class_idx];
+ fullness = ZS_META(first_page)->fullness;
+ class = ZS_MIGRATION(first_page)->class;
spin_lock(&class->lock);
obj_free(pool, class, obj);
@@ -1611,7 +1629,7 @@ static unsigned long find_alloced_obj(struct page *page, int index,
void *addr = kmap_atomic(page);
if (!is_first_page(page))
- offset = page->index;
+ offset = ZS_MIGRATION(page)->index;
offset += class->size * index;
while (offset < PAGE_SIZE) {
@@ -1719,7 +1737,8 @@ static enum fullness_group putback_zspage(struct zs_pool *pool,
fullness = get_fullness_group(class, first_page);
insert_zspage(first_page, class, fullness);
- set_zspage_mapping(first_page, class->index, fullness);
+ ZS_META(first_page)->fullness = fullness;
+ ZS_MIGRATION(first_page)->class = class;
if (fullness == ZS_EMPTY) {
zs_stat_dec(class, OBJ_ALLOCATED, get_maxobj_per_zspage(
@@ -2041,7 +2060,7 @@ static int __init zs_init(void)
goto notifier_fail;
BUILD_BUG_ON(sizeof(unsigned long) * 8 < (FREE_OBJ_IDX_BITS + \
- CLASS_IDX_BITS + FULLNESS_BITS + INUSE_BITS + ETC_BITS));
+ FULLNESS_BITS + INUSE_BITS + ETC_BITS));
init_zs_size_classes();
@@ -2054,6 +2073,15 @@ static int __init zs_init(void)
pr_err("zs stat initialization failed\n");
goto stat_fail;
}
+
+ zs_migration_cachep = kmem_cache_create("zs_migration",
+ sizeof(struct zs_migration),
+ 0, 0, NULL);
+ if (!zs_migration_cachep) {
+ pr_err("zs migration initialization failed\n");
+ goto stat_fail;
+ }
+
return 0;
stat_fail:
@@ -2068,6 +2096,8 @@ notifier_fail:
static void __exit zs_exit(void)
{
+ kmem_cache_destroy(zs_migration_cachep);
+
#ifdef CONFIG_ZPOOL
zpool_unregister_driver(&zs_zpool_driver);
#endif
--
1.9.1
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 4+ messages in thread* [PATCH 2/3] zsmalloc: make its page "PageMobile"
2015-11-27 12:12 [PATCH v3 0/3] zsmalloc: make its pages can be migrated Hui Zhu
2015-11-27 12:12 ` [PATCH 1/3] zsmalloc: make struct " Hui Zhu
@ 2015-11-27 12:12 ` Hui Zhu
2015-11-27 12:12 ` [PATCH 3/3] zram: make create "__GFP_MOVABLE" pool Hui Zhu
2 siblings, 0 replies; 4+ messages in thread
From: Hui Zhu @ 2015-11-27 12:12 UTC (permalink / raw)
To: Minchan Kim, Nitin Gupta, Sergey Senozhatsky, linux-kernel, linux-mm
Cc: teawater, Hui Zhu
The idea of this patch is same with prev version [1]. But it use the
migration frame in [1].
[1] http://comments.gmane.org/gmane.linux.kernel.mm/140014
[2] https://lkml.org/lkml/2015/7/7/21
Signed-off-by: Hui Zhu <zhuhui@xiaomi.com>
---
mm/zsmalloc.c | 214 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 209 insertions(+), 5 deletions(-)
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index 57c91a5..5034aac 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -53,10 +53,13 @@
#include <linux/vmalloc.h>
#include <linux/hardirq.h>
#include <linux/spinlock.h>
+#include <linux/rwlock.h>
#include <linux/types.h>
#include <linux/debugfs.h>
#include <linux/zsmalloc.h>
#include <linux/zpool.h>
+#include <linux/migrate.h>
+#include <linux/anon_inodes.h>
/*
* This must be power of 2 and greater than of equal to sizeof(link_free).
@@ -217,6 +220,8 @@ struct size_class {
/* huge object: pages_per_zspage == 1 && maxobj_per_zspage == 1 */
bool huge;
+
+ atomic_t count;
};
/*
@@ -281,6 +286,10 @@ struct zs_migration {
#define ZS_MIGRATION(p) ((struct zs_migration *)((p)->freelist))
#define ZS_META(p) ((struct zs_meta *)&(ZS_MIGRATION(p)->index))
+static struct inode *zs_inode;
+static DEFINE_SPINLOCK(zs_migration_lock);
+static DEFINE_RWLOCK(zs_tag_rwlock);
+
struct mapping_area {
#ifdef CONFIG_PGTABLE_MAPPING
struct vm_struct *vm; /* vm area for mapping object that span pages */
@@ -307,7 +316,7 @@ static void destroy_handle_cache(struct zs_pool *pool)
static unsigned long alloc_handle(struct zs_pool *pool)
{
return (unsigned long)kmem_cache_alloc(pool->handle_cachep,
- pool->flags & ~__GFP_HIGHMEM);
+ pool->flags & ~(__GFP_HIGHMEM | __GFP_MOVABLE));
}
static void free_handle(struct zs_pool *pool, unsigned long handle)
@@ -914,9 +923,12 @@ static void reset_page(struct page *page)
clear_bit(PG_private, &page->flags);
clear_bit(PG_private_2, &page->flags);
set_page_private(page, 0);
- free_migration(page->freelist);
- page->freelist = NULL;
+ if (page->freelist) {
+ free_migration(page->freelist);
+ page->freelist = NULL;
+ }
page_mapcount_reset(page);
+ page->mapping = NULL;
}
static void free_zspage(struct page *first_page)
@@ -927,6 +939,8 @@ static void free_zspage(struct page *first_page)
BUG_ON(!is_first_page(first_page));
BUG_ON(get_inuse_obj(first_page));
+ spin_lock(&zs_migration_lock);
+
head_extra = (struct page *)page_private(first_page);
reset_page(first_page);
@@ -934,7 +948,7 @@ static void free_zspage(struct page *first_page)
/* zspage with only 1 system page */
if (!head_extra)
- return;
+ goto out;
list_for_each_entry_safe(nextm, tmp, &ZS_MIGRATION(head_extra)->lru,
lru) {
@@ -945,6 +959,9 @@ static void free_zspage(struct page *first_page)
}
reset_page(head_extra);
__free_page(head_extra);
+
+out:
+ spin_unlock(&zs_migration_lock);
}
/* Initialize a newly allocated zspage */
@@ -1018,6 +1035,7 @@ static struct page *alloc_zspage(struct size_class *class, gfp_t flags)
page = alloc_page(flags);
if (!page)
goto cleanup;
+ page->mapping = zs_inode->i_mapping;
page->freelist = alloc_migration(flags);
if (!page->freelist) {
__free_page(page);
@@ -1327,6 +1345,7 @@ void *zs_map_object(struct zs_pool *pool, unsigned long handle,
BUG_ON(in_interrupt());
/* From now on, migration cannot move the object */
+ read_lock(&zs_tag_rwlock);
pin_tag(handle);
obj = handle_to_obj(handle);
@@ -1395,6 +1414,7 @@ void zs_unmap_object(struct zs_pool *pool, unsigned long handle)
}
put_cpu_var(zs_map_area);
unpin_tag(handle);
+ read_unlock(&zs_tag_rwlock);
}
EXPORT_SYMBOL_GPL(zs_unmap_object);
@@ -1431,6 +1451,16 @@ static unsigned long obj_malloc(struct page *first_page,
}
+static void set_zspage_mobile(struct size_class *class, struct page *page)
+{
+ BUG_ON(!is_first_page(page));
+
+ while (page) {
+ __SetPageMobile(page);
+ page = get_next_page(page);
+ }
+}
+
/**
* zs_malloc - Allocate block of given size from pool.
* @pool: pool to allocate from
@@ -1474,6 +1504,7 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size)
&pool->pages_allocated);
spin_lock(&class->lock);
+ set_zspage_mobile(class, first_page);
zs_stat_inc(class, OBJ_ALLOCATED, get_maxobj_per_zspage(
class->size, class->pages_per_zspage));
}
@@ -1526,6 +1557,7 @@ void zs_free(struct zs_pool *pool, unsigned long handle)
if (unlikely(!handle))
return;
+ read_lock(&zs_tag_rwlock);
pin_tag(handle);
obj = handle_to_obj(handle);
@@ -1546,6 +1578,7 @@ void zs_free(struct zs_pool *pool, unsigned long handle)
}
spin_unlock(&class->lock);
unpin_tag(handle);
+ read_unlock(&zs_tag_rwlock);
free_handle(pool, handle);
}
@@ -1919,6 +1952,19 @@ static int zs_register_shrinker(struct zs_pool *pool)
return register_shrinker(&pool->shrinker);
}
+static void
+get_class(struct size_class *class)
+{
+ atomic_inc(&class->count);
+}
+
+static void
+put_class(struct size_class *class)
+{
+ if (atomic_dec_and_test(&class->count))
+ kfree(class);
+}
+
/**
* zs_create_pool - Creates an allocation pool to work from.
* @flags: allocation flags used to allocate pool metadata
@@ -1995,6 +2041,8 @@ struct zs_pool *zs_create_pool(char *name, gfp_t flags)
get_maxobj_per_zspage(size, pages_per_zspage) == 1)
class->huge = true;
spin_lock_init(&class->lock);
+ atomic_set(&class->count, 0);
+ get_class(class);
pool->size_class[i] = class;
prev_class = class;
@@ -2042,7 +2090,9 @@ void zs_destroy_pool(struct zs_pool *pool)
class->size, fg);
}
}
- kfree(class);
+ spin_lock(&zs_migration_lock);
+ put_class(class);
+ spin_unlock(&zs_migration_lock);
}
destroy_handle_cache(pool);
@@ -2052,6 +2102,151 @@ void zs_destroy_pool(struct zs_pool *pool)
}
EXPORT_SYMBOL_GPL(zs_destroy_pool);
+bool zs_isolatepage(struct page *page, isolate_mode_t mode)
+{
+ bool ret = false;
+
+ spin_lock(&zs_migration_lock);
+
+ if (!get_page_unless_zero(page))
+ /* The zspage is released. */
+ goto out;
+ if (page_count(page) != 2)
+ /* The page is isolated by others or just freed. */
+ goto put_out;
+ if (page->freelist == NULL)
+ goto put_out;
+
+ ret = true;
+out:
+ spin_unlock(&zs_migration_lock);
+ return ret;
+
+put_out:
+ put_page(page);
+ goto out;
+}
+
+void zs_putbackpage(struct page *page)
+{
+ put_page(page);
+}
+
+int
+zs_migratepage(struct address_space *mapping,
+ struct page *newpage, struct page *page,
+ enum migrate_mode mode)
+{
+ int ret = -EAGAIN;
+ struct size_class *class;
+ struct page *first_page;
+
+ /* Get class. */
+ spin_lock(&zs_migration_lock);
+ if (page->freelist == NULL || page_count(page) <= 1) {
+ spin_unlock(&zs_migration_lock);
+ return ret;
+ }
+ class = ZS_MIGRATION(page)->class;
+ get_class(class);
+ spin_unlock(&zs_migration_lock);
+
+ write_lock(&zs_tag_rwlock);
+ spin_lock(&class->lock);
+
+ if (page->freelist == NULL || page_count(page) <= 1)
+ goto out; /* The zspage is released. */
+
+ first_page = get_first_page(page);
+
+ INIT_LIST_HEAD(&ZS_MIGRATION(newpage)->lru);
+ newpage->mapping = zs_inode->i_mapping;
+ if (page == first_page) { /* first page */
+ struct page **head;
+
+ SetPagePrivate(newpage);
+
+ if (class->huge) {
+ unsigned long handle = page_private(page);
+
+ if (handle != 0) {
+ void *addr, *newaddr;
+ unsigned long obj = obj_idx_to_obj(newpage, 0);
+
+ /* The page is allocated. */
+ handle = handle & ~OBJ_ALLOCATED_TAG;
+ record_obj(handle, obj);
+ addr = kmap_atomic(page);
+ newaddr = kmap_atomic(newpage);
+ memcpy(newaddr, addr, class->size);
+ kunmap_atomic(newaddr);
+ kunmap_atomic(addr);
+ } else
+ set_free_obj_idx(first_page, 0);
+ set_page_private(newpage, handle);
+ } else {
+ struct page *tmp_page = get_next_page(page);
+
+ while (tmp_page) {
+ tmp_page->first_page = newpage;
+ tmp_page = get_next_page(tmp_page);
+ }
+ set_page_private(newpage, page_private(page));
+ }
+
+ head = &class->fullness_list[ZS_META(first_page)->fullness];
+ BUG_ON(!*head);
+ if (*head == page)
+ *head = newpage;
+ } else
+ newpage->first_page = page->first_page;
+
+ if (is_last_page(page)) /* last page */
+ SetPagePrivate2(newpage);
+
+ if (!class->huge) {
+ void *addr, *newaddr;
+
+ addr = kmap_atomic(page);
+ newaddr = kmap_atomic(newpage);
+ copy_page(newaddr, addr);
+ kunmap_atomic(newaddr);
+ kunmap_atomic(addr);
+ }
+
+ /* Add newpage to zspage. */
+ if (first_page == page)
+ first_page = newpage;
+ else {
+ if ((struct page *)page_private(first_page) == page)
+ set_page_private(first_page, (unsigned long)newpage);
+ }
+ newpage->freelist = page->freelist;
+ ZS_MIGRATION(newpage)->page = newpage;
+
+ get_page(newpage);
+ __SetPageMobile(newpage);
+
+ spin_lock(&zs_migration_lock);
+ page->freelist = NULL;
+ reset_page(page);
+ put_page(page);
+ spin_unlock(&zs_migration_lock);
+
+ ret = MIGRATEPAGE_SUCCESS;
+out:
+ spin_unlock(&class->lock);
+ write_unlock(&zs_tag_rwlock);
+ put_class(class);
+ return ret;
+}
+
+const struct address_space_operations zs_mobile_aops = {
+ .migratepage = zs_migratepage,
+ .isolatepage = zs_isolatepage,
+ .putbackpage = zs_putbackpage,
+};
+
static int __init zs_init(void)
{
int ret = zs_register_cpu_notifier();
@@ -2082,6 +2277,13 @@ static int __init zs_init(void)
goto stat_fail;
}
+ zs_inode = anon_inode_new();
+ if (IS_ERR(zs_inode)) {
+ pr_err("zs inode initialization failed\n");
+ ret = PTR_ERR(zs_inode);
+ }
+ zs_inode->i_mapping->a_ops = &zs_mobile_aops;
+
return 0;
stat_fail:
@@ -2096,6 +2298,8 @@ notifier_fail:
static void __exit zs_exit(void)
{
+ iput(zs_inode);
+
kmem_cache_destroy(zs_migration_cachep);
#ifdef CONFIG_ZPOOL
--
1.9.1
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 4+ messages in thread