Re: [RFC PATCH 09/14] khwasan: add hooks implementation

[Alexander Potapenko <glider@google.com>]

On Fri, Mar 2, 2018 at 8:44 PM, Andrey Konovalov <andreyknvl@google.com> wrote:
> This commit adds KHWASAN hooks implementation.
>
> 1. When a new slab cache is created, KHWASAN rounds up the size of the
>    objects in this cache to KASAN_SHADOW_SCALE_SIZE (== 16).
>
> 2. On each kmalloc KHWASAN generates a random tag, sets the shadow memory,
>    that corresponds to this object to this tag, and embeds this tag value
>    into the top byte of the returned pointer.
>
> 3. On each kfree KHWASAN poisons the shadow memory with a random tag to
>    allow detection of use-after-free bugs.
>
> The rest of the logic of the hook implementation is very much similar to
> the one provided by KASAN. KHWASAN saves allocation and free stack metadata
> to the slab object the same was KASAN does this.
> ---
>  mm/kasan/khwasan.c | 178 ++++++++++++++++++++++++++++++++++++++++++++-
>  1 file changed, 175 insertions(+), 3 deletions(-)
>
> diff --git a/mm/kasan/khwasan.c b/mm/kasan/khwasan.c
> index 21a2221e3368..09d6f0a72266 100644
> --- a/mm/kasan/khwasan.c
> +++ b/mm/kasan/khwasan.c
> @@ -78,69 +78,238 @@ void *khwasan_reset_tag(void *addr)
>         return reset_tag(addr);
>  }
>
> +void kasan_poison_shadow(const void *address, size_t size, u8 value)
> +{
> +       void *shadow_start, *shadow_end;
> +
> +       /* Perform shadow offset calculation based on untagged address */
> +       address = reset_tag((void *)address);
> +
> +       shadow_start = kasan_mem_to_shadow(address);
> +       shadow_end = kasan_mem_to_shadow(address + size);
> +
> +       memset(shadow_start, value, shadow_end - shadow_start);
> +}
> +
>  void kasan_unpoison_shadow(const void *address, size_t size)
>  {
> +       /* KHWASAN only allows 16-byte granularity */
> +       size = round_up(size, KASAN_SHADOW_SCALE_SIZE);
> +       kasan_poison_shadow(address, size, get_tag(address));
>  }
>
>  void check_memory_region(unsigned long addr, size_t size, bool write,
>                                 unsigned long ret_ip)
>  {
> +       u8 tag;
> +       u8 *shadow_first, *shadow_last, *shadow;
> +       void *untagged_addr;
> +
> +       tag = get_tag((void *)addr);
> +       untagged_addr = reset_tag((void *)addr);
> +       shadow_first = (u8 *)kasan_mem_to_shadow(untagged_addr);
> +       shadow_last = (u8 *)kasan_mem_to_shadow(untagged_addr + size - 1);
> +
> +       for (shadow = shadow_first; shadow <= shadow_last; shadow++) {
> +               if (*shadow != tag) {
> +                       /* Report invalid-access bug here */
> +                       return;
> +               }
> +       }
>  }
>
>  void kasan_free_pages(struct page *page, unsigned int order)
>  {
> +       if (likely(!PageHighMem(page)))
> +               kasan_poison_shadow(page_address(page),
> +                               PAGE_SIZE << order,
> +                               khwasan_random_tag());
>  }
>
>  void kasan_cache_create(struct kmem_cache *cache, size_t *size,
>                 slab_flags_t *flags)
>  {
> +       int orig_size = *size;
> +
> +       cache->kasan_info.alloc_meta_offset = *size;
> +       *size += sizeof(struct kasan_alloc_meta);
> +
> +       if (*size % KASAN_SHADOW_SCALE_SIZE != 0)
> +               *size = round_up(*size, KASAN_SHADOW_SCALE_SIZE);
> +
> +
> +       if (*size > KMALLOC_MAX_SIZE) {
> +               *size = orig_size;
> +               return;
> +       }
> +
> +       cache->align = round_up(cache->align, KASAN_SHADOW_SCALE_SIZE);
> +
> +       *flags |= SLAB_KASAN;
>  }
>
>  void kasan_poison_slab(struct page *page)
>  {
> +       kasan_poison_shadow(page_address(page),
> +                       PAGE_SIZE << compound_order(page),
> +                       khwasan_random_tag());
>  }
>
>  void kasan_poison_object_data(struct kmem_cache *cache, void *object)
>  {
> +       kasan_poison_shadow(object,
> +                       round_up(cache->object_size, KASAN_SHADOW_SCALE_SIZE),
> +                       khwasan_random_tag());
>  }
>
>  void *kasan_slab_alloc(struct kmem_cache *cache, void *object, gfp_t flags)
>  {
> +       if (!READ_ONCE(khwasan_enabled))
> +               return object;
> +       object = kasan_kmalloc(cache, object, cache->object_size, flags);
> +       if (unlikely(cache->ctor)) {
> +               // Cache constructor might use object's pointer value to
> +               // initialize some of its fields.
> +               cache->ctor(object);
> +       }
>         return object;
>  }
>
> -bool kasan_slab_free(struct kmem_cache *cache, void *object, unsigned long ip)
> +static bool __kasan_slab_free(struct kmem_cache *cache, void *object,
> +                               unsigned long ip)
>  {
> +       u8 shadow_byte;
> +       u8 tag;
> +       unsigned long rounded_up_size;
> +       void *untagged_addr = reset_tag(object);
> +
> +       if (unlikely(nearest_obj(cache, virt_to_head_page(untagged_addr),
> +                       untagged_addr) != untagged_addr)) {
> +               /* Report invalid-free here */
> +               return true;
> +       }
> +
> +       /* RCU slabs could be legally used after free within the RCU period */
> +       if (unlikely(cache->flags & SLAB_TYPESAFE_BY_RCU))
> +               return false;
> +
> +       shadow_byte = READ_ONCE(*(u8 *)kasan_mem_to_shadow(untagged_addr));
> +       tag = get_tag(object);
> +       if (tag != shadow_byte) {
> +               /* Report invalid-free here */
> +               return true;
> +       }
> +
> +       rounded_up_size = round_up(cache->object_size, KASAN_SHADOW_SCALE_SIZE);
> +       kasan_poison_shadow(object, rounded_up_size, khwasan_random_tag());
> +
> +       if (unlikely(!(cache->flags & SLAB_KASAN)))
> +               return false;
> +
> +       set_track(&get_alloc_info(cache, object)->free_track, GFP_NOWAIT);
>         return false;
>  }
>
> +bool kasan_slab_free(struct kmem_cache *cache, void *object, unsigned long ip)
> +{
> +       return __kasan_slab_free(cache, object, ip);
> +}
> +
>  void *kasan_kmalloc(struct kmem_cache *cache, const void *object,
>                         size_t size, gfp_t flags)
>  {
> -       return (void *)object;
> +       unsigned long redzone_start, redzone_end;
> +       u8 tag;
> +
> +       if (!READ_ONCE(khwasan_enabled))
> +               return (void *)object;
> +
> +       if (unlikely(object == NULL))
> +               return NULL;
> +
> +       redzone_start = round_up((unsigned long)(object + size),
> +                               KASAN_SHADOW_SCALE_SIZE);
> +       redzone_end = round_up((unsigned long)(object + cache->object_size),
> +                               KASAN_SHADOW_SCALE_SIZE);
> +
> +       tag = khwasan_random_tag();
> +       kasan_poison_shadow(object, redzone_start - (unsigned long)object, tag);
> +       kasan_poison_shadow((void *)redzone_start, redzone_end - redzone_start,
> +               khwasan_random_tag());
> +
> +       if (cache->flags & SLAB_KASAN)
> +               set_track(&get_alloc_info(cache, object)->alloc_track, flags);
> +
> +       return set_tag((void *)object, tag);
>  }
>  EXPORT_SYMBOL(kasan_kmalloc);
>
>  void *kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
>  {
> -       return (void *)ptr;
> +       unsigned long redzone_start, redzone_end;
> +       u8 tag;
> +       struct page *page;
> +
> +       if (!READ_ONCE(khwasan_enabled))
> +               return (void *)ptr;
> +
> +       if (unlikely(ptr == NULL))
> +               return NULL;
> +
> +       page = virt_to_page(ptr);
> +       redzone_start = round_up((unsigned long)(ptr + size),
> +                               KASAN_SHADOW_SCALE_SIZE);
> +       redzone_end = (unsigned long)ptr + (PAGE_SIZE << compound_order(page));
> +
> +       tag = khwasan_random_tag();
> +       kasan_poison_shadow(ptr, redzone_start - (unsigned long)ptr, tag);
> +       kasan_poison_shadow((void *)redzone_start, redzone_end - redzone_start,
> +               khwasan_random_tag());
Am I understanding right that the object and the redzone may receive
identical tags here?
Does it make sense to generate the redzone tag from the object tag
(e.g. by addding 1 to it)?
> +       return set_tag((void *)ptr, tag);
>  }
>
>  void kasan_poison_kfree(void *ptr, unsigned long ip)
>  {
> +       struct page *page;
> +
> +       page = virt_to_head_page(ptr);
> +
> +       if (unlikely(!PageSlab(page))) {
> +               if (reset_tag(ptr) != page_address(page)) {
> +                       /* Report invalid-free here */
> +                       return;
> +               }
> +               kasan_poison_shadow(ptr, PAGE_SIZE << compound_order(page),
> +                                       khwasan_random_tag());
> +       } else {
> +               __kasan_slab_free(page->slab_cache, ptr, ip);
> +       }
>  }
>
>  void kasan_kfree_large(void *ptr, unsigned long ip)
>  {
> +       struct page *page = virt_to_page(ptr);
> +       struct page *head_page = virt_to_head_page(ptr);
> +
> +       if (reset_tag(ptr) != page_address(head_page)) {
> +               /* Report invalid-free here */
> +               return;
> +       }
> +
> +       kasan_poison_shadow(ptr, PAGE_SIZE << compound_order(page),
> +                       khwasan_random_tag());
>  }
>
>  #define DEFINE_HWASAN_LOAD_STORE(size)                                 \
>         void __hwasan_load##size##_noabort(unsigned long addr)          \
>         {                                                               \
> +               check_memory_region(addr, size, false, _RET_IP_);       \
>         }                                                               \
>         EXPORT_SYMBOL(__hwasan_load##size##_noabort);                   \
>         void __hwasan_store##size##_noabort(unsigned long addr)         \
>         {                                                               \
> +               check_memory_region(addr, size, true, _RET_IP_);        \
>         }                                                               \
>         EXPORT_SYMBOL(__hwasan_store##size##_noabort)
>
> @@ -152,15 +321,18 @@ DEFINE_HWASAN_LOAD_STORE(16);
>
>  void __hwasan_loadN_noabort(unsigned long addr, unsigned long size)
>  {
> +       check_memory_region(addr, size, false, _RET_IP_);
>  }
>  EXPORT_SYMBOL(__hwasan_loadN_noabort);
>
>  void __hwasan_storeN_noabort(unsigned long addr, unsigned long size)
>  {
> +       check_memory_region(addr, size, true, _RET_IP_);
>  }
>  EXPORT_SYMBOL(__hwasan_storeN_noabort);
>
>  void __hwasan_tag_memory(unsigned long addr, u8 tag, unsigned long size)
>  {
> +       kasan_poison_shadow((void *)addr, size, tag);
>  }
>  EXPORT_SYMBOL(__hwasan_tag_memory);
> --
> 2.16.2.395.g2e18187dfd-goog
>



-- 
Alexander Potapenko
Software Engineer

Google Germany GmbH
Erika-Mann-Straße, 33
80636 München

Geschäftsführer: Paul Manicle, Halimah DeLaine Prado
Registergericht und -nummer: Hamburg, HRB 86891
Sitz der Gesellschaft: Hamburg