Re: [PATCH v4 15/45] mm: kmsan: call KMSAN hooks from SLUB code

[Marco Elver <elver@google.com>]

On Fri, 1 Jul 2022 at 16:23, 'Alexander Potapenko' via kasan-dev
<kasan-dev@googlegroups.com> wrote:
>
> In order to report uninitialized memory coming from heap allocations
> KMSAN has to poison them unless they're created with __GFP_ZERO.
>
> It's handy that we need KMSAN hooks in the places where
> init_on_alloc/init_on_free initialization is performed.
>
> In addition, we apply __no_kmsan_checks to get_freepointer_safe() to
> suppress reports when accessing freelist pointers that reside in freed
> objects.
>
> Signed-off-by: Alexander Potapenko <glider@google.com>

Reviewed-by: Marco Elver <elver@google.com>

But see comment below.

> ---
> v2:
>  -- move the implementation of SLUB hooks here
>
> v4:
>  -- change sizeof(type) to sizeof(*ptr)
>  -- swap mm: and kmsan: in the subject
>  -- get rid of kmsan_init(), replace it with __no_kmsan_checks
>
> Link: https://linux-review.googlesource.com/id/I6954b386c5c5d7f99f48bb6cbcc74b75136ce86e
> ---
>  include/linux/kmsan.h | 57 ++++++++++++++++++++++++++++++
>  mm/kmsan/hooks.c      | 80 +++++++++++++++++++++++++++++++++++++++++++
>  mm/slab.h             |  1 +
>  mm/slub.c             | 18 ++++++++++
>  4 files changed, 156 insertions(+)
>
> diff --git a/include/linux/kmsan.h b/include/linux/kmsan.h
> index 699fe4f5b3bee..fd76cea338878 100644
> --- a/include/linux/kmsan.h
> +++ b/include/linux/kmsan.h
> @@ -15,6 +15,7 @@
>  #include <linux/types.h>
>
>  struct page;
> +struct kmem_cache;
>
>  #ifdef CONFIG_KMSAN
>
> @@ -72,6 +73,44 @@ void kmsan_free_page(struct page *page, unsigned int order);
>   */
>  void kmsan_copy_page_meta(struct page *dst, struct page *src);
>
> +/**
> + * kmsan_slab_alloc() - Notify KMSAN about a slab allocation.
> + * @s:      slab cache the object belongs to.
> + * @object: object pointer.
> + * @flags:  GFP flags passed to the allocator.
> + *
> + * Depending on cache flags and GFP flags, KMSAN sets up the metadata of the
> + * newly created object, marking it as initialized or uninitialized.
> + */
> +void kmsan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags);
> +
> +/**
> + * kmsan_slab_free() - Notify KMSAN about a slab deallocation.
> + * @s:      slab cache the object belongs to.
> + * @object: object pointer.
> + *
> + * KMSAN marks the freed object as uninitialized.
> + */
> +void kmsan_slab_free(struct kmem_cache *s, void *object);
> +
> +/**
> + * kmsan_kmalloc_large() - Notify KMSAN about a large slab allocation.
> + * @ptr:   object pointer.
> + * @size:  object size.
> + * @flags: GFP flags passed to the allocator.
> + *
> + * Similar to kmsan_slab_alloc(), but for large allocations.
> + */
> +void kmsan_kmalloc_large(const void *ptr, size_t size, gfp_t flags);
> +
> +/**
> + * kmsan_kfree_large() - Notify KMSAN about a large slab deallocation.
> + * @ptr: object pointer.
> + *
> + * Similar to kmsan_slab_free(), but for large allocations.
> + */
> +void kmsan_kfree_large(const void *ptr);
> +
>  /**
>   * kmsan_map_kernel_range_noflush() - Notify KMSAN about a vmap.
>   * @start:     start of vmapped range.
> @@ -138,6 +177,24 @@ static inline void kmsan_copy_page_meta(struct page *dst, struct page *src)
>  {
>  }
>
> +static inline void kmsan_slab_alloc(struct kmem_cache *s, void *object,
> +                                   gfp_t flags)
> +{
> +}
> +
> +static inline void kmsan_slab_free(struct kmem_cache *s, void *object)
> +{
> +}
> +
> +static inline void kmsan_kmalloc_large(const void *ptr, size_t size,
> +                                      gfp_t flags)
> +{
> +}
> +
> +static inline void kmsan_kfree_large(const void *ptr)
> +{
> +}
> +
>  static inline void kmsan_vmap_pages_range_noflush(unsigned long start,
>                                                   unsigned long end,
>                                                   pgprot_t prot,
> diff --git a/mm/kmsan/hooks.c b/mm/kmsan/hooks.c
> index 070756be70e3a..052e17b7a717d 100644
> --- a/mm/kmsan/hooks.c
> +++ b/mm/kmsan/hooks.c
> @@ -26,6 +26,86 @@
>   * skipping effects of functions like memset() inside instrumented code.
>   */
>
> +void kmsan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags)
> +{
> +       if (unlikely(object == NULL))
> +               return;
> +       if (!kmsan_enabled || kmsan_in_runtime())
> +               return;
> +       /*
> +        * There's a ctor or this is an RCU cache - do nothing. The memory
> +        * status hasn't changed since last use.
> +        */
> +       if (s->ctor || (s->flags & SLAB_TYPESAFE_BY_RCU))
> +               return;
> +
> +       kmsan_enter_runtime();
> +       if (flags & __GFP_ZERO)
> +               kmsan_internal_unpoison_memory(object, s->object_size,
> +                                              KMSAN_POISON_CHECK);
> +       else
> +               kmsan_internal_poison_memory(object, s->object_size, flags,
> +                                            KMSAN_POISON_CHECK);
> +       kmsan_leave_runtime();
> +}
> +EXPORT_SYMBOL(kmsan_slab_alloc);
> +
> +void kmsan_slab_free(struct kmem_cache *s, void *object)
> +{
> +       if (!kmsan_enabled || kmsan_in_runtime())
> +               return;
> +
> +       /* RCU slabs could be legally used after free within the RCU period */
> +       if (unlikely(s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)))
> +               return;
> +       /*
> +        * If there's a constructor, freed memory must remain in the same state
> +        * until the next allocation. We cannot save its state to detect
> +        * use-after-free bugs, instead we just keep it unpoisoned.
> +        */
> +       if (s->ctor)
> +               return;
> +       kmsan_enter_runtime();
> +       kmsan_internal_poison_memory(object, s->object_size, GFP_KERNEL,
> +                                    KMSAN_POISON_CHECK | KMSAN_POISON_FREE);
> +       kmsan_leave_runtime();
> +}
> +EXPORT_SYMBOL(kmsan_slab_free);
> +
> +void kmsan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
> +{
> +       if (unlikely(ptr == NULL))
> +               return;
> +       if (!kmsan_enabled || kmsan_in_runtime())
> +               return;
> +       kmsan_enter_runtime();
> +       if (flags & __GFP_ZERO)
> +               kmsan_internal_unpoison_memory((void *)ptr, size,
> +                                              /*checked*/ true);
> +       else
> +               kmsan_internal_poison_memory((void *)ptr, size, flags,
> +                                            KMSAN_POISON_CHECK);
> +       kmsan_leave_runtime();
> +}
> +EXPORT_SYMBOL(kmsan_kmalloc_large);
> +
> +void kmsan_kfree_large(const void *ptr)
> +{
> +       struct page *page;
> +
> +       if (!kmsan_enabled || kmsan_in_runtime())
> +               return;
> +       kmsan_enter_runtime();
> +       page = virt_to_head_page((void *)ptr);
> +       KMSAN_WARN_ON(ptr != page_address(page));
> +       kmsan_internal_poison_memory((void *)ptr,
> +                                    PAGE_SIZE << compound_order(page),
> +                                    GFP_KERNEL,
> +                                    KMSAN_POISON_CHECK | KMSAN_POISON_FREE);
> +       kmsan_leave_runtime();
> +}
> +EXPORT_SYMBOL(kmsan_kfree_large);
> +
>  static unsigned long vmalloc_shadow(unsigned long addr)
>  {
>         return (unsigned long)kmsan_get_metadata((void *)addr,
> diff --git a/mm/slab.h b/mm/slab.h
> index db9fb5c8dae73..d0de8195873d8 100644
> --- a/mm/slab.h
> +++ b/mm/slab.h
> @@ -752,6 +752,7 @@ static inline void slab_post_alloc_hook(struct kmem_cache *s,
>                         memset(p[i], 0, s->object_size);
>                 kmemleak_alloc_recursive(p[i], s->object_size, 1,
>                                          s->flags, flags);
> +               kmsan_slab_alloc(s, p[i], flags);
>         }
>
>         memcg_slab_post_alloc_hook(s, objcg, flags, size, p);
> diff --git a/mm/slub.c b/mm/slub.c
> index b1281b8654bd3..b8b601f165087 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -22,6 +22,7 @@
>  #include <linux/proc_fs.h>
>  #include <linux/seq_file.h>
>  #include <linux/kasan.h>
> +#include <linux/kmsan.h>
>  #include <linux/cpu.h>
>  #include <linux/cpuset.h>
>  #include <linux/mempolicy.h>
> @@ -359,6 +360,17 @@ static void prefetch_freepointer(const struct kmem_cache *s, void *object)
>         prefetchw(object + s->offset);
>  }
>
> +/*
> + * When running under KMSAN, get_freepointer_safe() may return an uninitialized
> + * pointer value in the case the current thread loses the race for the next
> + * memory chunk in the freelist. In that case this_cpu_cmpxchg_double() in
> + * slab_alloc_node() will fail, so the uninitialized value won't be used, but
> + * KMSAN will still check all arguments of cmpxchg because of imperfect
> + * handling of inline assembly.
> + * To work around this problem, we apply __no_kmsan_checks to ensure that
> + * get_freepointer_safe() returns initialized memory.
> + */
> +__no_kmsan_checks
>  static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
>  {
>         unsigned long freepointer_addr;
> @@ -1709,6 +1721,7 @@ static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
>         ptr = kasan_kmalloc_large(ptr, size, flags);
>         /* As ptr might get tagged, call kmemleak hook after KASAN. */
>         kmemleak_alloc(ptr, size, 1, flags);
> +       kmsan_kmalloc_large(ptr, size, flags);
>         return ptr;
>  }
>
> @@ -1716,12 +1729,14 @@ static __always_inline void kfree_hook(void *x)
>  {
>         kmemleak_free(x);
>         kasan_kfree_large(x);
> +       kmsan_kfree_large(x);
>  }
>
>  static __always_inline bool slab_free_hook(struct kmem_cache *s,
>                                                 void *x, bool init)
>  {
>         kmemleak_free_recursive(x, s->flags);
> +       kmsan_slab_free(s, x);
>
>         debug_check_no_locks_freed(x, s->object_size);
>
> @@ -3756,6 +3771,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
>          */
>         slab_post_alloc_hook(s, objcg, flags, size, p,
>                                 slab_want_init_on_alloc(flags, s));
> +

Remove unnecessary whitespace change.