[PATCH v2 05/23] mm/vma: use new VMA flags for sticky flags logic

Lorenzo Stoakes (Oracle)
2026-03-16T13:09:12Z

Use the new vma_flags_t flags implementation to perform the logic around
sticky flags and what flags are ignored on VMA merge.

We make use of the new vma_flags_empty(), vma_flags_diff_pair(), and
vma_flags_and_mask() functionality.

Note that we cannot rely on VM_NONE convenience any longer, so have to
explicitly check for cases where VMA flags would not be specified.

Also update the VMA tests accordingly.

Signed-off-by: Lorenzo Stoakes (Oracle) <ljs@kernel.org>
---
include/linux/mm.h | 32 +++++++++++---------
mm/vma.c | 47 ++++++++++++++++++++++--------
tools/testing/vma/include/custom.h | 5 ----
tools/testing/vma/include/dup.h | 9 ++++--
4 files changed, 61 insertions(+), 32 deletions(-)

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 6d2c4bd2c61d..b75e089dfd65 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -540,6 +540,7 @@ enum {

/* VMA basic access permission flags */
#define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC)
+#define VMA_ACCESS_FLAGS mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT)

/*
* Special vmas that are non-mergable, non-mlock()able.
@@ -585,27 +586,32 @@ enum {
* possesses it but the other does not, the merged VMA should nonetheless have
* applied to it:
*
- * VM_SOFTDIRTY - if a VMA is marked soft-dirty, that is has not had its
- * references cleared via /proc/$pid/clear_refs, any merged VMA
- * should be considered soft-dirty also as it operates at a VMA
- * granularity.
+ * VMA_SOFTDIRTY_BIT - if a VMA is marked soft-dirty, that is has not had its
+ * references cleared via /proc/$pid/clear_refs, any
+ * merged VMA should be considered soft-dirty also as it
+ * operates at a VMA granularity.
*
- * VM_MAYBE_GUARD - If a VMA may have guard regions in place it implies that
- * mapped page tables may contain metadata not described by the
- * VMA and thus any merged VMA may also contain this metadata,
- * and thus we must make this flag sticky.
+ * VMA_MAYBE_GUARD_BIT - If a VMA may have guard regions in place it implies
+ * that mapped page tables may contain metadata not
+ * described by the VMA and thus any merged VMA may also
+ * contain this metadata, and thus we must make this flag
+ * sticky.
*/
-#define VM_STICKY (VM_SOFTDIRTY | VM_MAYBE_GUARD)
+#ifdef CONFIG_MEM_SOFT_DIRTY
+#define VMA_STICKY_FLAGS mk_vma_flags(VMA_SOFTDIRTY_BIT, VMA_MAYBE_GUARD_BIT)
+#else
+#define VMA_STICKY_FLAGS mk_vma_flags(VMA_MAYBE_GUARD_BIT)
+#endif

/*
* VMA flags we ignore for the purposes of merge, i.e. one VMA possessing one
* of these flags and the other not does not preclude a merge.
*
- * VM_STICKY - When merging VMAs, VMA flags must match, unless they are
- * 'sticky'. If any sticky flags exist in either VMA, we simply
- * set all of them on the merged VMA.
+ * VMA_STICKY_FLAGS - When merging VMAs, VMA flags must match, unless they
+ * are 'sticky'. If any sticky flags exist in either VMA,
+ * we simply set all of them on the merged VMA.
*/
-#define VM_IGNORE_MERGE VM_STICKY
+#define VMA_IGNORE_MERGE_FLAGS VMA_STICKY_FLAGS

/*
* Flags which should result in page tables being copied on fork. These are
diff --git a/mm/vma.c b/mm/vma.c
index 4d21e7d8e93c..15d643eee97f 100644
--- a/mm/vma.c
+++ b/mm/vma.c
@@ -86,10 +86,15 @@ static bool vma_is_fork_child(struct vm_area_struct *vma)
static inline bool is_mergeable_vma(struct vma_merge_struct *vmg, bool merge_next)
{
struct vm_area_struct *vma = merge_next ? vmg->next : vmg->prev;
+ vma_flags_t diff;

if (!mpol_equal(vmg->policy, vma_policy(vma)))
return false;
- if ((vma->vm_flags ^ vmg->vm_flags) & ~VM_IGNORE_MERGE)
+
+ diff = vma_flags_diff_pair(&vma->flags, &vmg->vma_flags);
+ vma_flags_clear_mask(&diff, VMA_IGNORE_MERGE_FLAGS);
+
+ if (!vma_flags_empty(&diff))
return false;
if (vma->vm_file != vmg->file)
return false;
@@ -805,7 +810,8 @@ static bool can_merge_remove_vma(struct vm_area_struct *vma)
static __must_check struct vm_area_struct *vma_merge_existing_range(
struct vma_merge_struct *vmg)
{
- vm_flags_t sticky_flags = vmg->vm_flags & VM_STICKY;
+ vma_flags_t sticky_flags = vma_flags_and_mask(&vmg->vma_flags,
+ VMA_STICKY_FLAGS);
struct vm_area_struct *middle = vmg->middle;
struct vm_area_struct *prev = vmg->prev;
struct vm_area_struct *next;
@@ -898,15 +904,21 @@ static __must_check struct vm_area_struct *vma_merge_existing_range(
vma_start_write(middle);

if (merge_right) {
+ const vma_flags_t next_sticky =
+ vma_flags_and_mask(&next->flags, VMA_STICKY_FLAGS);
+
vma_start_write(next);
vmg->target = next;
- sticky_flags |= (next->vm_flags & VM_STICKY);
+ vma_flags_set_mask(&sticky_flags, next_sticky);
}

if (merge_left) {
+ const vma_flags_t prev_sticky =
+ vma_flags_and_mask(&prev->flags, VMA_STICKY_FLAGS);
+
vma_start_write(prev);
vmg->target = prev;
- sticky_flags |= (prev->vm_flags & VM_STICKY);
+ vma_flags_set_mask(&sticky_flags, prev_sticky);
}

if (merge_both) {
@@ -976,7 +988,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range(
if (err || commit_merge(vmg))
goto abort;

- vm_flags_set(vmg->target, sticky_flags);
+ vma_set_flags_mask(vmg->target, sticky_flags);
khugepaged_enter_vma(vmg->target, vmg->vm_flags);
vmg->state = VMA_MERGE_SUCCESS;
return vmg->target;
@@ -1154,7 +1166,10 @@ int vma_expand(struct vma_merge_struct *vmg)
struct vm_area_struct *target = vmg->target;
struct vm_area_struct *next = vmg->next;
bool remove_next = false;
- vm_flags_t sticky_flags;
+ vma_flags_t sticky_flags =
+ vma_flags_and_mask(&vmg->vma_flags, VMA_STICKY_FLAGS);
+ const vma_flags_t target_sticky =
+ vma_flags_and_mask(&target->flags, VMA_STICKY_FLAGS);
int ret = 0;

mmap_assert_write_locked(vmg->mm);
@@ -1174,10 +1189,13 @@ int vma_expand(struct vma_merge_struct *vmg)
VM_WARN_ON_VMG(target->vm_start < vmg->start ||
target->vm_end > vmg->end, vmg);

- sticky_flags = vmg->vm_flags & VM_STICKY;
- sticky_flags |= target->vm_flags & VM_STICKY;
- if (remove_next)
- sticky_flags |= next->vm_flags & VM_STICKY;
+ vma_flags_set_mask(&sticky_flags, target_sticky);
+ if (remove_next) {
+ const vma_flags_t next_sticky =
+ vma_flags_and_mask(&next->flags, VMA_STICKY_FLAGS);
+
+ vma_flags_set_mask(&sticky_flags, next_sticky);
+ }

/*
* If we are removing the next VMA or copying from a VMA
@@ -1200,7 +1218,7 @@ int vma_expand(struct vma_merge_struct *vmg)
if (commit_merge(vmg))
goto nomem;

- vm_flags_set(target, sticky_flags);
+ vma_set_flags_mask(target, sticky_flags);
return 0;

nomem:
@@ -1950,10 +1968,15 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
*/
static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
{
+ vma_flags_t diff = vma_flags_diff_pair(&a->flags, &b->flags);
+
+ vma_flags_clear_mask(&diff, VMA_ACCESS_FLAGS);
+ vma_flags_clear_mask(&diff, VMA_IGNORE_MERGE_FLAGS);
+
return a->vm_end == b->vm_start &&
mpol_equal(vma_policy(a), vma_policy(b)) &&
a->vm_file == b->vm_file &&
- !((a->vm_flags ^ b->vm_flags) & ~(VM_ACCESS_FLAGS | VM_IGNORE_MERGE)) &&
+ vma_flags_empty(&diff) &&
b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
}

diff --git a/tools/testing/vma/include/custom.h b/tools/testing/vma/include/custom.h
index 6200f938e586..7cdd0f60600a 100644
--- a/tools/testing/vma/include/custom.h
+++ b/tools/testing/vma/include/custom.h
@@ -134,8 +134,3 @@ static __always_inline bool vma_flags_same_mask(vma_flags_t *flags,
vma_flags_same_mask(flags, mk_vma_flags(__VA_ARGS__))
#define VMA_SPECIAL_FLAGS mk_vma_flags(VMA_IO_BIT, VMA_DONTEXPAND_BIT, \
VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT)
-#ifdef CONFIG_MEM_SOFT_DIRTY
-#define VMA_STICKY_FLAGS mk_vma_flags(VMA_SOFTDIRTY_BIT, VMA_MAYBE_GUARD_BIT)
-#else
-#define VMA_STICKY_FLAGS mk_vma_flags(VMA_MAYBE_GUARD_BIT)
-#endif
diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h
index 44f77453ee85..e5fdf6f5a033 100644
--- a/tools/testing/vma/include/dup.h
+++ b/tools/testing/vma/include/dup.h
@@ -338,6 +338,7 @@ enum {

/* VMA basic access permission flags */
#define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC)
+#define VMA_ACCESS_FLAGS mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT)

/*
* Special vmas that are non-mergable, non-mlock()able.
@@ -363,9 +364,13 @@ enum {

#define CAP_IPC_LOCK 14

-#define VM_STICKY (VM_SOFTDIRTY | VM_MAYBE_GUARD)
+#ifdef CONFIG_MEM_SOFT_DIRTY
+#define VMA_STICKY_FLAGS mk_vma_flags(VMA_SOFTDIRTY_BIT, VMA_MAYBE_GUARD_BIT)
+#else
+#define VMA_STICKY_FLAGS mk_vma_flags(VMA_MAYBE_GUARD_BIT)
+#endif

-#define VM_IGNORE_MERGE VM_STICKY
+#define VMA_IGNORE_MERGE_FLAGS VMA_STICKY_FLAGS

#define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_MAYBE_GUARD)

--
2.53.0