* (no subject)
2024-04-16 17:12 mbland
@ 2024-04-16 17:12 ` mbland
0 siblings, 0 replies; 2+ messages in thread
From: mbland @ 2024-04-16 17:12 UTC (permalink / raw)
To: linux-mm; +Cc: Maxwell Bland
linux-kernel@vger.kernel.org,
Andrew Morton <akpm@linux-foundation.org>,
Uladzislau Rezki <urezki@gmail.com>,
Christoph Hellwig <hch@infradead.org>,
Lorenzo Stoakes <lstoakes@gmail.com>
From: Maxwell Bland <mbland@motorola.com>
Date: Tue, 2 Apr 2024 15:15:01 -0500
Subject: [PATCH 1/5] mm: allow arch refinement/skip for vmap alloc
Makes red black tree allocation more flexible on a per-architecture
basis by introducing an optional hooks to refine the red-black tree
structuring and exposing vmalloc functions for clipping vmap areas,
finding vmap areas, and inserting vmap areas.
With this patch, the red-black vmap tree can be refined to account for
architecture-specific memory management operations, most notably address
space layout randomization, as these features conflict with generic
management of a single vmalloc_start to vmalloc_end range as given by
mm/vmalloc.c.
For example, x86 is forced to restrict aslr to 1024 possible locations,
which is a very, very small number, and arm64 breaks standard code/data
partitioning altogether, which prevents the enforcement of performant
immmutability on kernel page tables.
Signed-off-by: Maxwell Bland <mbland@motorola.com>
---
include/linux/vmalloc.h | 24 ++++++++++++++++++++++++
mm/vmalloc.c | 16 ++++++++++------
2 files changed, 34 insertions(+), 6 deletions(-)
diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
index 98ea90e90439..3c5ce7ee0bea 100644
--- a/include/linux/vmalloc.h
+++ b/include/linux/vmalloc.h
@@ -12,6 +12,7 @@
#include <asm/vmalloc.h>
+struct kmem_cache;
struct vm_area_struct; /* vma defining user mapping in mm_types.h */
struct notifier_block; /* in notifier.h */
struct iov_iter; /* in uio.h */
@@ -125,6 +126,21 @@ static inline pgprot_t arch_vmap_pgprot_tagged(pgprot_t prot)
}
#endif
+#ifndef arch_skip_va
+static inline bool arch_skip_va(struct vmap_area *va, unsigned long vstart)
+{
+ return false;
+}
+#endif
+
+#ifndef arch_refine_vmap_space
+static inline void arch_refine_vmap_space(struct rb_root *root,
+ struct list_head *head,
+ struct kmem_cache *cachep)
+{
+}
+#endif
+
/*
* Highlevel APIs for driver use
*/
@@ -214,6 +230,14 @@ extern struct vm_struct *__get_vm_area_caller(unsigned long size,
void free_vm_area(struct vm_struct *area);
extern struct vm_struct *remove_vm_area(const void *addr);
extern struct vm_struct *find_vm_area(const void *addr);
+extern void insert_vmap_area_augment(struct vmap_area *va, struct rb_node *from,
+ struct rb_root *root,
+ struct list_head *head);
+extern int va_clip(struct rb_root *root, struct list_head *head,
+ struct vmap_area *va, unsigned long nva_start_addr,
+ unsigned long size);
+extern struct vmap_area *__find_vmap_area(unsigned long addr,
+ struct rb_root *root);
struct vmap_area *find_vmap_area(unsigned long addr);
static inline bool is_vm_area_hugepages(const void *addr)
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 68fa001648cc..de4577a3708e 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -989,7 +989,7 @@ unsigned long vmalloc_nr_pages(void)
return atomic_long_read(&nr_vmalloc_pages);
}
-static struct vmap_area *__find_vmap_area(unsigned long addr, struct rb_root *root)
+struct vmap_area *__find_vmap_area(unsigned long addr, struct rb_root *root)
{
struct rb_node *n = root->rb_node;
@@ -1322,7 +1322,7 @@ insert_vmap_area(struct vmap_area *va,
link_va(va, root, parent, link, head);
}
-static void
+void
insert_vmap_area_augment(struct vmap_area *va,
struct rb_node *from, struct rb_root *root,
struct list_head *head)
@@ -1501,7 +1501,7 @@ find_vmap_lowest_match(struct rb_root *root, unsigned long size,
vstart < va->va_start) {
node = node->rb_left;
} else {
- if (is_within_this_va(va, size, align, vstart))
+ if (!arch_skip_va(va, vstart) && is_within_this_va(va, size, align, vstart))
return va;
/*
@@ -1522,7 +1522,8 @@ find_vmap_lowest_match(struct rb_root *root, unsigned long size,
*/
while ((node = rb_parent(node))) {
va = rb_entry(node, struct vmap_area, rb_node);
- if (is_within_this_va(va, size, align, vstart))
+ if (!arch_skip_va(va, vstart) &&
+ is_within_this_va(va, size, align, vstart))
return va;
if (get_subtree_max_size(node->rb_right) >= length &&
@@ -1554,7 +1555,7 @@ find_vmap_lowest_linear_match(struct list_head *head, unsigned long size,
struct vmap_area *va;
list_for_each_entry(va, head, list) {
- if (!is_within_this_va(va, size, align, vstart))
+ if (arch_skip_va(va, vstart) || !is_within_this_va(va, size, align, vstart))
continue;
return va;
@@ -1617,7 +1618,7 @@ classify_va_fit_type(struct vmap_area *va,
return type;
}
-static __always_inline int
+__always_inline int
va_clip(struct rb_root *root, struct list_head *head,
struct vmap_area *va, unsigned long nva_start_addr,
unsigned long size)
@@ -5129,4 +5130,7 @@ void __init vmalloc_init(void)
vmap_node_shrinker->count_objects = vmap_node_shrink_count;
vmap_node_shrinker->scan_objects = vmap_node_shrink_scan;
shrinker_register(vmap_node_shrinker);
+
+ arch_refine_vmap_space(&free_vmap_area_root, &free_vmap_area_list,
+ vmap_area_cachep);
}
--
2.39.2
^ permalink raw reply [flat|nested] 2+ messages in thread
* (no subject)
@ 2024-04-16 17:12 mbland
2024-04-16 17:12 ` mbland
0 siblings, 1 reply; 2+ messages in thread
From: mbland @ 2024-04-16 17:12 UTC (permalink / raw)
To: linux-mm; +Cc: Maxwell Bland
linux-kernel@vger.kernel.org,
linux-arm-kernel@lists.infradead.org,
linux-riscv@lists.infradead.org,
linuxppc-dev@lists.ozlabs.org,
Mark Rutland <mark.rutland@arm.com>,
Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
Christoph Hellwig <hch@infradead.org>,
Christophe Leroy <christophe.leroy@csgroup.eu>,
David Hildenbrand <david@redhat.com>,
Conor Dooley <conor.dooley@microchip.com>
From: Maxwell Bland <mbland@motorola.com>
Date: Mon, 15 Apr 2024 15:16:08 -0500
Subject: [PATCH 0/5] mm: code and data partitioning improvements
Managing allocations to ensure code and data pages are not interleaved
is not possible prior to this patch, as ASLR requires programming a
dynamic _text offset while the vmalloc infrastructure maintains static
VMALLOC_START and VMALLOC_END constants.
In systems where code and data are interleaved at a PTE granularity,
kernel improvements targeting the prevention of exploit stages which
modify page tables are inefficient and less effective as individual PTE
updates occur at high frequency and cannot be coarsely grouped at the
PMD level or greater.
This patch adds minimal arch-specific callbacks to the initialization of
vmalloc and when deciding whether to use a specific virtual memory area
to satisfy a vmalloc request to provide the capability to prevent the
allocation of specific virtual addresses under specific system states.
By default these hooks are unimplemented.
To further support the practical use of these callbacks, this patch also
adds a virtual address parameter to pmd_populate_kernel, so that this
interface matches the equivalent pte-level interface and architectures
are not required to perform a reverse page table lookup to determine the
vaddr being allocated during pmd creation.
To demonstrate the impact and value of these changes, this patch
implements support for dynamic PXNTable under aarch64 in 71 lines of
code (a single "if" check during memory allocation), by checking the
virtual address of a given vmalloc call to determine whether it is code
or data. From experience in trying to implement kernel page table
immutability and protections in KVM to prevent recent CVEs, e.g.
CVE-2024-1086, this is a necessary first step.
To better help maintainers and future developers, this patch expands
ptdump.c so that non-leaf page table descriptors can be more easily
noted in debug output by setting a note_non_leaf bool in the ptdump
state.
Signed-off-by: Maxwell Bland <mbland@motorola.com>
---
First, thank you to a number of maintainers (Mark Rutland, Greg KH,
Christoph Hellwig, Christophe Leroy, David Hildenbrand, Conor Dooley)
for their feedback on
<20240220203256.31153-1-mbland@motorola.com>
and
<CAP5Mv+ydhk=Ob4b40ZahGMgT-5+-VEHxtmA=-LkJiEOOU+K6hw@mail.gmail.com>
This patch is a further refinement and overhaul of these prior two
attempts. Also, apologies for the roughly two months delay between patch
submissions! I had Motorola work to do.
In support of testing this patch (but not included in this patch), I set
note_non_leaf to true under arch/arm64/mm/ptdump.c and added
PMD_TABLE_PXN to pte_bits to print out whether the PXNTable bit was set.
The txt files under the following directory can be diff'ed to see the
result:
github.com/maxwell-bland/linux-patch-data/tree/main/code_data_parting/ptdump
I also created a script to fetch and cross-compile the kernel for each
of the 21 subarchitectures which required fixes to provide a virtual
address to pmd_populate_kernel. I have no idea if it is useful and maybe
one already exists, but it worked well for me over some alternatives
(xcross, buildroot):
github.com/maxwell-bland/x-linux
As with the last patchset, I also measured performance using Torvald's
test-tlb program on an aarch64 QEMU instance, with results here:
github.com/maxwell-bland/linux-patch-data/tree/main/code_data_parting/tlbperf
As all changes to other arches are effectively no-ops, performance
impacts in those domains are negligible.
Maxwell Bland (5):
mm: allow arch refinement/skip for vmap alloc
arm64: mm: code and data partitioning for aslr
mm: add vaddr param to pmd_populate_kernel
arm64: dynamic enforcement of PXNTable
ptdump: add state parameter for non-leaf callback
arch/alpha/include/asm/pgalloc.h | 5 +-
arch/arc/include/asm/pgalloc.h | 3 +-
arch/arc/mm/highmem.c | 2 +-
arch/arm/include/asm/kfence.h | 2 +-
arch/arm/include/asm/pgalloc.h | 3 +-
arch/arm/mm/kasan_init.c | 2 +-
arch/arm/mm/mmu.c | 2 +-
arch/arm64/include/asm/module.h | 12 ++++
arch/arm64/include/asm/pgalloc.h | 15 ++++-
arch/arm64/include/asm/vmalloc.h | 17 ++++-
arch/arm64/kernel/Makefile | 2 +-
arch/arm64/kernel/module.c | 7 +-
arch/arm64/kernel/probes/kprobes.c | 7 +-
arch/arm64/kernel/setup.c | 4 ++
arch/arm64/kernel/vmalloc.c | 71 ++++++++++++++++++++
arch/arm64/mm/ptdump.c | 10 +--
arch/arm64/mm/trans_pgd.c | 2 +-
arch/arm64/net/bpf_jit_comp.c | 8 ++-
arch/csky/include/asm/pgalloc.h | 2 +-
arch/hexagon/include/asm/pgalloc.h | 2 +-
arch/loongarch/include/asm/pgalloc.h | 3 +-
arch/loongarch/mm/init.c | 2 +-
arch/loongarch/mm/kasan_init.c | 2 +-
arch/m68k/include/asm/mcf_pgalloc.h | 2 +-
arch/m68k/include/asm/motorola_pgalloc.h | 3 +-
arch/m68k/include/asm/sun3_pgalloc.h | 3 +-
arch/microblaze/include/asm/pgalloc.h | 2 +-
arch/mips/include/asm/pgalloc.h | 2 +-
arch/mips/kvm/mmu.c | 2 +-
arch/nios2/include/asm/pgalloc.h | 2 +-
arch/openrisc/include/asm/pgalloc.h | 2 +-
arch/parisc/include/asm/pgalloc.h | 5 +-
arch/parisc/mm/init.c | 6 +-
arch/powerpc/include/asm/book3s/32/pgalloc.h | 2 +-
arch/powerpc/include/asm/book3s/64/pgalloc.h | 2 +-
arch/powerpc/include/asm/nohash/32/pgalloc.h | 2 +-
arch/powerpc/include/asm/nohash/64/pgalloc.h | 2 +-
arch/powerpc/mm/book3s64/radix_pgtable.c | 2 +-
arch/powerpc/mm/kasan/init_32.c | 4 +-
arch/powerpc/mm/kasan/init_book3e_64.c | 9 ++-
arch/powerpc/mm/kasan/init_book3s_64.c | 7 +-
arch/powerpc/mm/nohash/book3e_pgtable.c | 2 +-
arch/powerpc/mm/pgtable_32.c | 4 +-
arch/powerpc/mm/ptdump/ptdump.c | 2 +
arch/riscv/include/asm/pgalloc.h | 2 +-
arch/riscv/kernel/hibernate.c | 2 +-
arch/riscv/mm/ptdump.c | 6 +-
arch/s390/include/asm/pgalloc.h | 2 +-
arch/s390/mm/dump_pagetables.c | 6 +-
arch/sh/include/asm/pgalloc.h | 2 +-
arch/sh/mm/init.c | 2 +-
arch/sparc/include/asm/pgalloc_32.h | 3 +-
arch/sparc/include/asm/pgalloc_64.h | 4 +-
arch/sparc/mm/init_64.c | 8 +--
arch/um/include/asm/pgalloc.h | 4 +-
arch/x86/include/asm/pgalloc.h | 3 +-
arch/x86/mm/dump_pagetables.c | 3 +-
arch/x86/mm/init_64.c | 14 +++-
arch/x86/mm/ioremap.c | 2 +-
arch/x86/mm/kasan_init_64.c | 2 +-
arch/xtensa/include/asm/pgalloc.h | 2 +-
include/linux/mm.h | 4 +-
include/linux/ptdump.h | 1 +
include/linux/vmalloc.h | 24 +++++++
mm/hugetlb_vmemmap.c | 4 +-
mm/kasan/init.c | 14 ++--
mm/memory.c | 4 +-
mm/percpu.c | 2 +-
mm/pgalloc-track.h | 3 +-
mm/ptdump.c | 13 ++++
mm/sparse-vmemmap.c | 2 +-
mm/vmalloc.c | 16 +++--
72 files changed, 299 insertions(+), 107 deletions(-)
create mode 100644 arch/arm64/kernel/vmalloc.c
base-commit: 0bbac3facb5d6cc0171c45c9873a2dc96bea9680
--
2.39.2
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2024-04-16 17:12 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2024-04-16 17:12 mbland
2024-04-16 17:12 ` mbland
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox