* [RFC zsmalloc 1/4] zsmalloc: keep max_object in size_class
2015-08-10 7:12 [RFC zsmalloc 0/4] meta diet Minchan Kim
@ 2015-08-10 7:12 ` Minchan Kim
2015-08-10 8:06 ` Sergey Senozhatsky
2015-08-10 7:12 ` [RFC zsmalloc 2/4] zsmalloc: squeeze inuse into page->mapping Minchan Kim
` (2 subsequent siblings)
3 siblings, 1 reply; 7+ messages in thread
From: Minchan Kim @ 2015-08-10 7:12 UTC (permalink / raw)
To: Andrew Morton
Cc: gioh.kim, Sergey Senozhatsky, linux-kernel, linux-mm, Minchan Kim
Every zspage in a size_class has same max_objects so we could
move it to a size_class.
Signed-off-by: Minchan Kim <minchan@kernel.org>
---
mm/zsmalloc.c | 22 ++++++++++------------
1 file changed, 10 insertions(+), 12 deletions(-)
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index f135b1b..491491a 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -33,8 +33,6 @@
* page->freelist: points to the first free object in zspage.
* Free objects are linked together using in-place
* metadata.
- * page->objects: maximum number of objects we can store in this
- * zspage (class->zspage_order * PAGE_SIZE / class->size)
* page->lru: links together first pages of various zspages.
* Basically forming list of zspages in a fullness group.
* page->mapping: class index and fullness group of the zspage
@@ -206,6 +204,7 @@ struct size_class {
* of ZS_ALIGN.
*/
int size;
+ int max_objects;
unsigned int index;
/* Number of PAGE_SIZE sized pages to combine to form a 'zspage' */
@@ -606,14 +605,15 @@ static inline void zs_pool_stat_destroy(struct zs_pool *pool)
* the pool (not yet implemented). This function returns fullness
* status of the given page.
*/
-static enum fullness_group get_fullness_group(struct page *page)
+static enum fullness_group get_fullness_group(struct size_class *class,
+ struct page *page)
{
int inuse, max_objects;
enum fullness_group fg;
BUG_ON(!is_first_page(page));
inuse = page->inuse;
- max_objects = page->objects;
+ max_objects = class->max_objects;
if (inuse == 0)
fg = ZS_EMPTY;
@@ -706,7 +706,7 @@ static enum fullness_group fix_fullness_group(struct size_class *class,
BUG_ON(!is_first_page(page));
get_zspage_mapping(page, &class_idx, &currfg);
- newfg = get_fullness_group(page);
+ newfg = get_fullness_group(class, page);
if (newfg == currfg)
goto out;
@@ -985,9 +985,6 @@ static struct page *alloc_zspage(struct size_class *class, gfp_t flags)
init_zspage(first_page, class);
first_page->freelist = location_to_obj(first_page, 0);
- /* Maximum number of objects we can store in this zspage */
- first_page->objects = class->pages_per_zspage * PAGE_SIZE / class->size;
-
error = 0; /* Success */
cleanup:
@@ -1217,11 +1214,11 @@ static bool can_merge(struct size_class *prev, int size, int pages_per_zspage)
return true;
}
-static bool zspage_full(struct page *page)
+static bool zspage_full(struct size_class *class, struct page *page)
{
BUG_ON(!is_first_page(page));
- return page->inuse == page->objects;
+ return page->inuse == class->max_objects;
}
unsigned long zs_get_total_pages(struct zs_pool *pool)
@@ -1619,7 +1616,7 @@ static int migrate_zspage(struct zs_pool *pool, struct size_class *class,
}
/* Stop if there is no more space */
- if (zspage_full(d_page)) {
+ if (zspage_full(class, d_page)) {
unpin_tag(handle);
ret = -ENOMEM;
break;
@@ -1673,7 +1670,7 @@ static enum fullness_group putback_zspage(struct zs_pool *pool,
BUG_ON(!is_first_page(first_page));
- fullness = get_fullness_group(first_page);
+ fullness = get_fullness_group(class, first_page);
insert_zspage(first_page, class, fullness);
set_zspage_mapping(first_page, class->index, fullness);
@@ -1927,6 +1924,7 @@ struct zs_pool *zs_create_pool(char *name, gfp_t flags)
class->size = size;
class->index = i;
class->pages_per_zspage = pages_per_zspage;
+ class->max_objects = class->pages_per_zspage * PAGE_SIZE / class->size;
if (pages_per_zspage == 1 &&
get_maxobj_per_zspage(size, pages_per_zspage) == 1)
class->huge = true;
--
1.9.1
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 7+ messages in thread* [RFC zsmalloc 2/4] zsmalloc: squeeze inuse into page->mapping
2015-08-10 7:12 [RFC zsmalloc 0/4] meta diet Minchan Kim
2015-08-10 7:12 ` [RFC zsmalloc 1/4] zsmalloc: keep max_object in size_class Minchan Kim
@ 2015-08-10 7:12 ` Minchan Kim
2015-08-10 8:53 ` Sergey Senozhatsky
2015-08-10 7:12 ` [RFC zsmalloc 3/4] zsmalloc: squeeze freelist " Minchan Kim
2015-08-10 7:12 ` [RFC zsmalloc 4/4] zsmalloc: move struct zs_meta from mapping to somewhere Minchan Kim
3 siblings, 1 reply; 7+ messages in thread
From: Minchan Kim @ 2015-08-10 7:12 UTC (permalink / raw)
To: Andrew Morton
Cc: gioh.kim, Sergey Senozhatsky, linux-kernel, linux-mm, Minchan Kim
Currently, we store class:fullness into page->mapping.
The number of class we can support is 255 and fullness is 4 so
10bit is enough to represent them.
IOW, we have room (sizeof(void *) * 8 - 10) in mapping.
Meanwhile, the bits we need to store in-use objects in zspage
is just 11bit like below.
For example, 64K page system, class_size 32, in this case
class->pages_per_zspage is 1 so max_objects is 2048.
So, we could squeeze inuse object count to page->mapping.
Signed-off-by: Minchan Kim <minchan@kernel.org>
---
mm/zsmalloc.c | 104 +++++++++++++++++++++++++++++++++++++++-------------------
1 file changed, 71 insertions(+), 33 deletions(-)
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index 491491a..75fefba 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -35,7 +35,7 @@
* metadata.
* page->lru: links together first pages of various zspages.
* Basically forming list of zspages in a fullness group.
- * page->mapping: class index and fullness group of the zspage
+ * page->mapping: override by struct zs_meta
*
* Usage of struct page flags:
* PG_private: identifies the first component page
@@ -132,6 +132,14 @@
/* each chunk includes extra space to keep handle */
#define ZS_MAX_ALLOC_SIZE PAGE_SIZE
+#define CLASS_IDX_BITS 8
+#define CLASS_IDX_MASK ((1 << CLASS_IDX_BITS) - 1)
+#define FULLNESS_BITS 2
+#define FULLNESS_MASK ((1 << FULLNESS_BITS) - 1)
+#define INUSE_BITS 11
+#define INUSE_MASK ((1 << INUSE_BITS) - 1)
+#define ETC_BITS ((sizeof(unsigned long) * 8) - CLASS_IDX_BITS \
+ - FULLNESS_BITS - INUSE_BITS)
/*
* On systems with 4K page size, this gives 255 size classes! There is a
* trader-off here:
@@ -145,16 +153,15 @@
* ZS_MIN_ALLOC_SIZE and ZS_SIZE_CLASS_DELTA must be multiple of ZS_ALIGN
* (reason above)
*/
-#define ZS_SIZE_CLASS_DELTA (PAGE_SIZE >> 8)
+#define ZS_SIZE_CLASS_DELTA (PAGE_SIZE >> CLASS_IDX_BITS)
/*
* We do not maintain any list for completely empty or full pages
+ * Don't reorder.
*/
enum fullness_group {
- ZS_ALMOST_FULL,
+ ZS_ALMOST_FULL = 0,
ZS_ALMOST_EMPTY,
- _ZS_NR_FULLNESS_GROUPS,
-
ZS_EMPTY,
ZS_FULL
};
@@ -198,7 +205,7 @@ static const int fullness_threshold_frac = 4;
struct size_class {
spinlock_t lock;
- struct page *fullness_list[_ZS_NR_FULLNESS_GROUPS];
+ struct page *fullness_list[ZS_EMPTY];
/*
* Size of objects stored in this class. Must be multiple
* of ZS_ALIGN.
@@ -259,13 +266,16 @@ struct zs_pool {
};
/*
- * A zspage's class index and fullness group
- * are encoded in its (first)page->mapping
+ * In this implementation, a zspage's class index, fullness group,
+ * inuse object count are encoded in its (first)page->mapping
+ * sizeof(struct zs_meta) should be equal to sizeof(unsigned long).
*/
-#define CLASS_IDX_BITS 28
-#define FULLNESS_BITS 4
-#define CLASS_IDX_MASK ((1 << CLASS_IDX_BITS) - 1)
-#define FULLNESS_MASK ((1 << FULLNESS_BITS) - 1)
+struct zs_meta {
+ unsigned long class_idx:CLASS_IDX_BITS;
+ unsigned long fullness:FULLNESS_BITS;
+ unsigned long inuse:INUSE_BITS;
+ unsigned long etc:ETC_BITS;
+};
struct mapping_area {
#ifdef CONFIG_PGTABLE_MAPPING
@@ -403,26 +413,51 @@ static int is_last_page(struct page *page)
return PagePrivate2(page);
}
+static int get_inuse_obj(struct page *page)
+{
+ struct zs_meta *m;
+
+ BUG_ON(!is_first_page(page));
+
+ m = (struct zs_meta *)&page->mapping;
+
+ return m->inuse;
+}
+
+static void set_inuse_obj(struct page *page, int inc)
+{
+ struct zs_meta *m;
+
+ BUG_ON(!is_first_page(page));
+
+ m = (struct zs_meta *)&page->mapping;
+ m->inuse += inc;
+}
+
static void get_zspage_mapping(struct page *page, unsigned int *class_idx,
enum fullness_group *fullness)
{
- unsigned long m;
+ struct zs_meta *m;
BUG_ON(!is_first_page(page));
- m = (unsigned long)page->mapping;
- *fullness = m & FULLNESS_MASK;
- *class_idx = (m >> FULLNESS_BITS) & CLASS_IDX_MASK;
+ m = (struct zs_meta *)&page->mapping;
+ *fullness = m->fullness;
+ *class_idx = m->class_idx;
}
static void set_zspage_mapping(struct page *page, unsigned int class_idx,
enum fullness_group fullness)
{
- unsigned long m;
+ struct zs_meta *m;
+
BUG_ON(!is_first_page(page));
- m = ((class_idx & CLASS_IDX_MASK) << FULLNESS_BITS) |
- (fullness & FULLNESS_MASK);
- page->mapping = (struct address_space *)m;
+ BUG_ON(class_idx >= (1 << CLASS_IDX_BITS));
+ BUG_ON(fullness >= (1 << FULLNESS_BITS));
+
+ m = (struct zs_meta *)&page->mapping;
+ m->fullness = fullness;
+ m->class_idx = class_idx;
}
/*
@@ -612,7 +647,7 @@ static enum fullness_group get_fullness_group(struct size_class *class,
enum fullness_group fg;
BUG_ON(!is_first_page(page));
- inuse = page->inuse;
+ inuse = get_inuse_obj(page);
max_objects = class->max_objects;
if (inuse == 0)
@@ -640,7 +675,7 @@ static void insert_zspage(struct page *page, struct size_class *class,
BUG_ON(!is_first_page(page));
- if (fullness >= _ZS_NR_FULLNESS_GROUPS)
+ if (fullness >= ZS_EMPTY)
return;
zs_stat_inc(class, fullness == ZS_ALMOST_EMPTY ?
@@ -654,10 +689,10 @@ static void insert_zspage(struct page *page, struct size_class *class,
/*
* We want to see more ZS_FULL pages and less almost
- * empty/full. Put pages with higher ->inuse first.
+ * empty/full. Put pages with higher inuse first.
*/
list_add_tail(&page->lru, &(*head)->lru);
- if (page->inuse >= (*head)->inuse)
+ if (get_inuse_obj(page) >= get_inuse_obj(*head))
*head = page;
}
@@ -672,7 +707,7 @@ static void remove_zspage(struct page *page, struct size_class *class,
BUG_ON(!is_first_page(page));
- if (fullness >= _ZS_NR_FULLNESS_GROUPS)
+ if (fullness >= ZS_EMPTY)
return;
head = &class->fullness_list[fullness];
@@ -874,7 +909,7 @@ static void free_zspage(struct page *first_page)
struct page *nextp, *tmp, *head_extra;
BUG_ON(!is_first_page(first_page));
- BUG_ON(first_page->inuse);
+ BUG_ON(get_inuse_obj(first_page));
head_extra = (struct page *)page_private(first_page);
@@ -969,7 +1004,7 @@ static struct page *alloc_zspage(struct size_class *class, gfp_t flags)
SetPagePrivate(page);
set_page_private(page, 0);
first_page = page;
- first_page->inuse = 0;
+ set_inuse_obj(page, 0);
}
if (i == 1)
set_page_private(first_page, (unsigned long)page);
@@ -1001,7 +1036,7 @@ static struct page *find_get_zspage(struct size_class *class)
int i;
struct page *page;
- for (i = 0; i < _ZS_NR_FULLNESS_GROUPS; i++) {
+ for (i = 0; i < ZS_EMPTY; i++) {
page = class->fullness_list[i];
if (page)
break;
@@ -1218,7 +1253,7 @@ static bool zspage_full(struct size_class *class, struct page *page)
{
BUG_ON(!is_first_page(page));
- return page->inuse == class->max_objects;
+ return get_inuse_obj(page) == class->max_objects;
}
unsigned long zs_get_total_pages(struct zs_pool *pool)
@@ -1355,7 +1390,7 @@ static unsigned long obj_malloc(struct page *first_page,
/* record handle in first_page->private */
set_page_private(first_page, handle);
kunmap_atomic(vaddr);
- first_page->inuse++;
+ set_inuse_obj(first_page, 1);
zs_stat_inc(class, OBJ_USED, 1);
return obj;
@@ -1446,7 +1481,7 @@ static void obj_free(struct zs_pool *pool, struct size_class *class,
set_page_private(first_page, 0);
kunmap_atomic(vaddr);
first_page->freelist = (void *)obj;
- first_page->inuse--;
+ set_inuse_obj(first_page, -1);
zs_stat_dec(class, OBJ_USED, 1);
}
@@ -1643,7 +1678,7 @@ static struct page *isolate_target_page(struct size_class *class)
int i;
struct page *page;
- for (i = 0; i < _ZS_NR_FULLNESS_GROUPS; i++) {
+ for (i = 0; i < ZS_EMPTY; i++) {
page = class->fullness_list[i];
if (page) {
remove_zspage(page, class, i);
@@ -1970,7 +2005,7 @@ void zs_destroy_pool(struct zs_pool *pool)
if (class->index != i)
continue;
- for (fg = 0; fg < _ZS_NR_FULLNESS_GROUPS; fg++) {
+ for (fg = 0; fg < ZS_EMPTY; fg++) {
if (class->fullness_list[fg]) {
pr_info("Freeing non-empty class with size %db, fullness group %d\n",
class->size, fg);
@@ -1993,6 +2028,9 @@ static int __init zs_init(void)
if (ret)
goto notifier_fail;
+ BUILD_BUG_ON(sizeof(unsigned long) * 8 < (CLASS_IDX_BITS + \
+ FULLNESS_BITS + INUSE_BITS + ETC_BITS));
+
init_zs_size_classes();
#ifdef CONFIG_ZPOOL
--
1.9.1
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 7+ messages in thread* [RFC zsmalloc 3/4] zsmalloc: squeeze freelist into page->mapping
2015-08-10 7:12 [RFC zsmalloc 0/4] meta diet Minchan Kim
2015-08-10 7:12 ` [RFC zsmalloc 1/4] zsmalloc: keep max_object in size_class Minchan Kim
2015-08-10 7:12 ` [RFC zsmalloc 2/4] zsmalloc: squeeze inuse into page->mapping Minchan Kim
@ 2015-08-10 7:12 ` Minchan Kim
2015-08-10 7:12 ` [RFC zsmalloc 4/4] zsmalloc: move struct zs_meta from mapping to somewhere Minchan Kim
3 siblings, 0 replies; 7+ messages in thread
From: Minchan Kim @ 2015-08-10 7:12 UTC (permalink / raw)
To: Andrew Morton
Cc: gioh.kim, Sergey Senozhatsky, linux-kernel, linux-mm, Minchan Kim
Zsmalloc stores free object's position into first_page's freelist
in each zspage. If we change it with object offset from first_page
instead of free object's location, we could squeeze it into page->
mapping because the number of bit we need to store it is at most
11bit.
For example, 64K page system, class_size 32, in this case
class->pages_per_zspage is 1 so max_objects is 2048.
Signed-off-by: Minchan Kim <minchan@kernel.org>
---
mm/zsmalloc.c | 232 ++++++++++++++++++++++++++++++----------------------------
1 file changed, 122 insertions(+), 110 deletions(-)
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index 75fefba..55dc066 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -18,9 +18,7 @@
* Usage of struct page fields:
* page->first_page: points to the first component (0-order) page
* page->index (union with page->freelist): offset of the first object
- * starting in this page. For the first page, this is
- * always 0, so we use this field (aka freelist) to point
- * to the first free object in zspage.
+ * starting in this page.
* page->lru: links together all component pages (except the first page)
* of a zspage
*
@@ -30,9 +28,6 @@
* component page after the first page
* If the page is first_page for huge object, it stores handle.
* Look at size_class->huge.
- * page->freelist: points to the first free object in zspage.
- * Free objects are linked together using in-place
- * metadata.
* page->lru: links together first pages of various zspages.
* Basically forming list of zspages in a fullness group.
* page->mapping: override by struct zs_meta
@@ -132,14 +127,16 @@
/* each chunk includes extra space to keep handle */
#define ZS_MAX_ALLOC_SIZE PAGE_SIZE
+#define FREE_OBJ_IDX_BITS 11
+#define FREE_OBJ_IDX_MASK ((1 << FREE_OBJ_IDX_BITS) - 1)
#define CLASS_IDX_BITS 8
#define CLASS_IDX_MASK ((1 << CLASS_IDX_BITS) - 1)
#define FULLNESS_BITS 2
#define FULLNESS_MASK ((1 << FULLNESS_BITS) - 1)
#define INUSE_BITS 11
#define INUSE_MASK ((1 << INUSE_BITS) - 1)
-#define ETC_BITS ((sizeof(unsigned long) * 8) - CLASS_IDX_BITS \
- - FULLNESS_BITS - INUSE_BITS)
+#define ETC_BITS ((sizeof(unsigned long) * 8) - FREE_OBJ_IDX_BITS - \
+ CLASS_IDX_BITS - FULLNESS_BITS - INUSE_BITS)
/*
* On systems with 4K page size, this gives 255 size classes! There is a
* trader-off here:
@@ -224,17 +221,14 @@ struct size_class {
/*
* Placed within free objects to form a singly linked list.
- * For every zspage, first_page->freelist gives head of this list.
+ * For every zspage, first_page->free_obj_idx gives head of this list.
*
* This must be power of 2 and less than or equal to ZS_ALIGN
*/
struct link_free {
union {
- /*
- * Position of next free chunk (encodes <PFN, obj_idx>)
- * It's valid for non-allocated object
- */
- void *next;
+ /* Next free object index from first page */
+ unsigned long next;
/*
* Handle of allocated object.
*/
@@ -266,11 +260,12 @@ struct zs_pool {
};
/*
- * In this implementation, a zspage's class index, fullness group,
+ * In this implementation, a free_idx, zspage's class index, fullness group,
* inuse object count are encoded in its (first)page->mapping
* sizeof(struct zs_meta) should be equal to sizeof(unsigned long).
*/
struct zs_meta {
+ unsigned long free_idx:FREE_OBJ_IDX_BITS;
unsigned long class_idx:CLASS_IDX_BITS;
unsigned long fullness:FULLNESS_BITS;
unsigned long inuse:INUSE_BITS;
@@ -434,6 +429,18 @@ static void set_inuse_obj(struct page *page, int inc)
m->inuse += inc;
}
+static void set_free_obj_idx(struct page *first_page, int idx)
+{
+ struct zs_meta *m = (struct zs_meta *)&first_page->mapping;
+ m->free_idx = idx;
+}
+
+static unsigned long get_free_obj_idx(struct page *first_page)
+{
+ struct zs_meta *m = (struct zs_meta *)&first_page->mapping;
+ return m->free_idx;
+}
+
static void get_zspage_mapping(struct page *page, unsigned int *class_idx,
enum fullness_group *fullness)
{
@@ -816,37 +823,50 @@ static struct page *get_next_page(struct page *page)
return next;
}
-/*
- * Encode <page, obj_idx> as a single handle value.
- * We use the least bit of handle for tagging.
- */
-static void *location_to_obj(struct page *page, unsigned long obj_idx)
+static void obj_idx_to_location(struct size_class *class, struct page *first_page,
+ unsigned long obj_idx, struct page **obj_page,
+ unsigned long *ofs_in_page)
{
- unsigned long obj;
+ int i;
+ unsigned long ofs;
+ struct page *cursor;
+ int nr_page;
- if (!page) {
- BUG_ON(obj_idx);
- return NULL;
- }
+ BUG_ON(!is_first_page(first_page));
- obj = page_to_pfn(page) << OBJ_INDEX_BITS;
- obj |= ((obj_idx) & OBJ_INDEX_MASK);
- obj <<= OBJ_TAG_BITS;
+ ofs = obj_idx * class->size;
+ cursor = first_page;
+ nr_page = ofs / PAGE_SIZE;
- return (void *)obj;
+ *ofs_in_page = ofs % PAGE_SIZE;
+
+ for ( i = 0; i < nr_page; i++)
+ cursor = get_next_page(cursor);
+
+ *obj_page = cursor;
}
-/*
- * Decode <page, obj_idx> pair from the given object handle. We adjust the
- * decoded obj_idx back to its original value since it was adjusted in
- * location_to_obj().
- */
-static void obj_to_location(unsigned long obj, struct page **page,
+
+static void obj_to_obj_idx(unsigned long obj, struct page **obj_page,
unsigned long *obj_idx)
{
obj >>= OBJ_TAG_BITS;
- *page = pfn_to_page(obj >> OBJ_INDEX_BITS);
- *obj_idx = (obj & OBJ_INDEX_MASK);
+ *obj_idx = obj & OBJ_INDEX_MASK;
+
+ obj >>= OBJ_INDEX_BITS;
+ *obj_page = pfn_to_page(obj);
+}
+
+static unsigned long obj_idx_to_obj(struct page *obj_page,
+ unsigned long obj_idx)
+{
+ unsigned long obj;
+
+ obj = page_to_pfn(obj_page) << OBJ_INDEX_BITS;
+ obj |= ((obj_idx) & OBJ_INDEX_MASK);
+ obj <<= OBJ_TAG_BITS;
+
+ return obj;
}
static unsigned long handle_to_obj(unsigned long handle)
@@ -864,17 +884,6 @@ static unsigned long obj_to_head(struct size_class *class, struct page *page,
return *(unsigned long *)obj;
}
-static unsigned long obj_idx_to_offset(struct page *page,
- unsigned long obj_idx, int class_size)
-{
- unsigned long off = 0;
-
- if (!is_first_page(page))
- off = page->index;
-
- return off + obj_idx * class_size;
-}
-
static inline int trypin_tag(unsigned long handle)
{
unsigned long *ptr = (unsigned long *)handle;
@@ -900,7 +909,6 @@ static void reset_page(struct page *page)
clear_bit(PG_private_2, &page->flags);
set_page_private(page, 0);
page->mapping = NULL;
- page->freelist = NULL;
page_mapcount_reset(page);
}
@@ -932,6 +940,7 @@ static void free_zspage(struct page *first_page)
/* Initialize a newly allocated zspage */
static void init_zspage(struct page *first_page, struct size_class *class)
{
+ int obj_idx = 1;
unsigned long off = 0;
struct page *page = first_page;
@@ -939,14 +948,11 @@ static void init_zspage(struct page *first_page, struct size_class *class)
while (page) {
struct page *next_page;
struct link_free *link;
- unsigned int i = 1;
void *vaddr;
/*
* page->index stores offset of first object starting
- * in the page. For the first page, this is always 0,
- * so we use first_page->index (aka ->freelist) to store
- * head of corresponding zspage's freelist.
+ * in the page.
*/
if (page != first_page)
page->index = off;
@@ -955,7 +961,7 @@ static void init_zspage(struct page *first_page, struct size_class *class)
link = (struct link_free *)vaddr + off / sizeof(*link);
while ((off += class->size) < PAGE_SIZE) {
- link->next = location_to_obj(page, i++);
+ link->next = (obj_idx++ << OBJ_ALLOCATED_TAG);
link += class->size / sizeof(*link);
}
@@ -965,11 +971,16 @@ static void init_zspage(struct page *first_page, struct size_class *class)
* page (if present)
*/
next_page = get_next_page(page);
- link->next = location_to_obj(next_page, 0);
+ if (next_page)
+ link->next = (obj_idx++ << OBJ_ALLOCATED_TAG);
+ else
+ link->next = (-1 << OBJ_ALLOCATED_TAG);
kunmap_atomic(vaddr);
page = next_page;
off %= PAGE_SIZE;
}
+
+ set_free_obj_idx(first_page, 0);
}
/*
@@ -1019,7 +1030,6 @@ static struct page *alloc_zspage(struct size_class *class, gfp_t flags)
init_zspage(first_page, class);
- first_page->freelist = location_to_obj(first_page, 0);
error = 0; /* Success */
cleanup:
@@ -1279,8 +1289,8 @@ EXPORT_SYMBOL_GPL(zs_get_total_pages);
void *zs_map_object(struct zs_pool *pool, unsigned long handle,
enum zs_mapmode mm)
{
- struct page *page;
- unsigned long obj, obj_idx, off;
+ struct page *obj_page, *first_page;
+ unsigned long obj, obj_idx, obj_ofs;
unsigned int class_idx;
enum fullness_group fg;
@@ -1302,26 +1312,29 @@ void *zs_map_object(struct zs_pool *pool, unsigned long handle,
pin_tag(handle);
obj = handle_to_obj(handle);
- obj_to_location(obj, &page, &obj_idx);
- get_zspage_mapping(get_first_page(page), &class_idx, &fg);
+ obj_to_obj_idx(obj, &obj_page, &obj_idx);
+
+ first_page = get_first_page(obj_page);
+ get_zspage_mapping(first_page, &class_idx, &fg);
+
class = pool->size_class[class_idx];
- off = obj_idx_to_offset(page, obj_idx, class->size);
+ obj_ofs = (class->size * obj_idx) % PAGE_SIZE;
area = &get_cpu_var(zs_map_area);
area->vm_mm = mm;
- if (off + class->size <= PAGE_SIZE) {
+ if (obj_ofs + class->size <= PAGE_SIZE) {
/* this object is contained entirely within a page */
- area->vm_addr = kmap_atomic(page);
- ret = area->vm_addr + off;
+ area->vm_addr = kmap_atomic(obj_page);
+ ret = area->vm_addr + obj_ofs;
goto out;
}
/* this object spans two pages */
- pages[0] = page;
- pages[1] = get_next_page(page);
+ pages[0] = obj_page;
+ pages[1] = get_next_page(obj_page);
BUG_ON(!pages[1]);
- ret = __zs_map_object(area, pages, off, class->size);
+ ret = __zs_map_object(area, pages, obj_ofs, class->size);
out:
if (!class->huge)
ret += ZS_HANDLE_SIZE;
@@ -1332,8 +1345,8 @@ EXPORT_SYMBOL_GPL(zs_map_object);
void zs_unmap_object(struct zs_pool *pool, unsigned long handle)
{
- struct page *page;
- unsigned long obj, obj_idx, off;
+ struct page *obj_page, *first_page;
+ unsigned long obj, obj_idx, obj_ofs;
unsigned int class_idx;
enum fullness_group fg;
@@ -1343,22 +1356,24 @@ void zs_unmap_object(struct zs_pool *pool, unsigned long handle)
BUG_ON(!handle);
obj = handle_to_obj(handle);
- obj_to_location(obj, &page, &obj_idx);
- get_zspage_mapping(get_first_page(page), &class_idx, &fg);
+
+ obj_to_obj_idx(obj, &obj_page, &obj_idx);
+ first_page = get_first_page(obj_page);
+ get_zspage_mapping(first_page, &class_idx, &fg);
class = pool->size_class[class_idx];
- off = obj_idx_to_offset(page, obj_idx, class->size);
+ obj_ofs = (class->size * obj_idx) % PAGE_SIZE;
area = this_cpu_ptr(&zs_map_area);
- if (off + class->size <= PAGE_SIZE)
+ if (obj_ofs + class->size <= PAGE_SIZE)
kunmap_atomic(area->vm_addr);
else {
struct page *pages[2];
- pages[0] = page;
- pages[1] = get_next_page(page);
+ pages[0] = obj_page;
+ pages[1] = get_next_page(obj_page);
BUG_ON(!pages[1]);
- __zs_unmap_object(area, pages, off, class->size);
+ __zs_unmap_object(area, pages, obj_ofs, class->size);
}
put_cpu_var(zs_map_area);
unpin_tag(handle);
@@ -1368,21 +1383,20 @@ EXPORT_SYMBOL_GPL(zs_unmap_object);
static unsigned long obj_malloc(struct page *first_page,
struct size_class *class, unsigned long handle)
{
- unsigned long obj;
+ unsigned long obj_idx, obj;
struct link_free *link;
- struct page *m_page;
- unsigned long m_objidx, m_offset;
+ struct page *obj_page;
+ unsigned long obj_ofs;
void *vaddr;
handle |= OBJ_ALLOCATED_TAG;
- obj = (unsigned long)first_page->freelist;
- obj_to_location(obj, &m_page, &m_objidx);
- m_offset = obj_idx_to_offset(m_page, m_objidx, class->size);
+ obj_idx = get_free_obj_idx(first_page);
+ obj_idx_to_location(class, first_page, obj_idx, &obj_page, &obj_ofs);
- vaddr = kmap_atomic(m_page);
- link = (struct link_free *)vaddr + m_offset / sizeof(*link);
- first_page->freelist = link->next;
+ vaddr = kmap_atomic(obj_page);
+ link = (struct link_free *)vaddr + obj_ofs / sizeof(*link);
+ set_free_obj_idx(first_page, link->next >> OBJ_ALLOCATED_TAG);
if (!class->huge)
/* record handle in the header of allocated chunk */
link->handle = handle;
@@ -1393,6 +1407,8 @@ static unsigned long obj_malloc(struct page *first_page,
set_inuse_obj(first_page, 1);
zs_stat_inc(class, OBJ_USED, 1);
+ obj = obj_idx_to_obj(obj_page, obj_idx);
+
return obj;
}
@@ -1457,38 +1473,34 @@ static void obj_free(struct zs_pool *pool, struct size_class *class,
unsigned long obj)
{
struct link_free *link;
- struct page *first_page, *f_page;
- unsigned long f_objidx, f_offset;
+ struct page *first_page, *obj_page;
+ unsigned long obj_idx, obj_ofs;
void *vaddr;
- int class_idx;
- enum fullness_group fullness;
BUG_ON(!obj);
- obj &= ~OBJ_ALLOCATED_TAG;
- obj_to_location(obj, &f_page, &f_objidx);
- first_page = get_first_page(f_page);
+ obj_to_obj_idx(obj, &obj_page, &obj_idx);
+ obj_ofs = (class->size * obj_idx) % PAGE_SIZE;
+ first_page = get_first_page(obj_page);
- get_zspage_mapping(first_page, &class_idx, &fullness);
- f_offset = obj_idx_to_offset(f_page, f_objidx, class->size);
-
- vaddr = kmap_atomic(f_page);
+ vaddr = kmap_atomic(obj_page);
/* Insert this object in containing zspage's freelist */
- link = (struct link_free *)(vaddr + f_offset);
- link->next = first_page->freelist;
+ link = (struct link_free *)(vaddr + obj_ofs);
+ link->next = get_free_obj_idx(first_page) << OBJ_ALLOCATED_TAG;
if (class->huge)
set_page_private(first_page, 0);
kunmap_atomic(vaddr);
- first_page->freelist = (void *)obj;
+ set_free_obj_idx(first_page, obj_idx);
set_inuse_obj(first_page, -1);
zs_stat_dec(class, OBJ_USED, 1);
+
}
void zs_free(struct zs_pool *pool, unsigned long handle)
{
- struct page *first_page, *f_page;
- unsigned long obj, f_objidx;
+ struct page *first_page, *obj_page;
+ unsigned long obj, obj_idx;
int class_idx;
struct size_class *class;
enum fullness_group fullness;
@@ -1498,9 +1510,9 @@ void zs_free(struct zs_pool *pool, unsigned long handle)
pin_tag(handle);
obj = handle_to_obj(handle);
- obj_to_location(obj, &f_page, &f_objidx);
- first_page = get_first_page(f_page);
+ obj_to_obj_idx(obj, &obj_page, &obj_idx);
+ first_page = get_first_page(obj_page);
get_zspage_mapping(first_page, &class_idx, &fullness);
class = pool->size_class[class_idx];
@@ -1521,7 +1533,7 @@ void zs_free(struct zs_pool *pool, unsigned long handle)
}
EXPORT_SYMBOL_GPL(zs_free);
-static void zs_object_copy(unsigned long dst, unsigned long src,
+static void zs_object_copy(unsigned long dst_obj, unsigned long src_obj,
struct size_class *class)
{
struct page *s_page, *d_page;
@@ -1533,11 +1545,11 @@ static void zs_object_copy(unsigned long dst, unsigned long src,
s_size = d_size = class->size;
- obj_to_location(src, &s_page, &s_objidx);
- obj_to_location(dst, &d_page, &d_objidx);
+ obj_to_obj_idx(src_obj, &s_page, &s_objidx);
+ obj_to_obj_idx(dst_obj, &d_page, &d_objidx);
- s_off = obj_idx_to_offset(s_page, s_objidx, class->size);
- d_off = obj_idx_to_offset(d_page, d_objidx, class->size);
+ s_off = (class->size * s_objidx) % PAGE_SIZE;
+ d_off = (class->size * d_objidx) % PAGE_SIZE;
if (s_off + class->size > PAGE_SIZE)
s_size = PAGE_SIZE - s_off;
@@ -2028,8 +2040,8 @@ static int __init zs_init(void)
if (ret)
goto notifier_fail;
- BUILD_BUG_ON(sizeof(unsigned long) * 8 < (CLASS_IDX_BITS + \
- FULLNESS_BITS + INUSE_BITS + ETC_BITS));
+ BUILD_BUG_ON(sizeof(unsigned long) * 8 < (FREE_OBJ_IDX_BITS + \
+ CLASS_IDX_BITS + FULLNESS_BITS + INUSE_BITS + ETC_BITS));
init_zs_size_classes();
--
1.9.1
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply [flat|nested] 7+ messages in thread