Commit c0054c56 authored by Andrey Konovalov's avatar Andrey Konovalov Committed by Linus Torvalds
Browse files

kasan: inline kasan_reset_tag for tag-based modes 

Using kasan_reset_tag() currently results in a function call. As it's
called quite often from the allocator code, this leads to a noticeable
slowdown. Move it to include/linux/kasan.h and turn it into a static
inline function. Also remove the now unneeded reset_tag() internal KASAN
macro and use kasan_reset_tag() instead.

Link: https://lkml.kernel.org/r/6940383a3a9dfb416134d338d8fac97a9ebb8686.1606162397.git.andreyknvl@google.com
Link: https://linux-review.googlesource.com/id/I4d2061acfe91d480a75df00b07c22d8494ef14b5


Signed-off-by: default avatarAndrey Konovalov <andreyknvl@google.com>
Reviewed-by: default avatarMarco Elver <elver@google.com>
Reviewed-by: default avatarDmitry Vyukov <dvyukov@google.com>
Tested-by: default avatarVincenzo Frascino <vincenzo.frascino@arm.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Ryabinin <aryabinin@virtuozzo.com>
Cc: Branislav Rankov <Branislav.Rankov@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Evgenii Stepanov <eugenis@google.com>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 77f57c98

include/linux/kasan.h

+4 −1
Original line number Diff line number Diff line
@@ -194,7 +194,10 @@ static inline void kasan_record_aux_stack(void *ptr) {} 


#if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)



void *kasan_reset_tag(const void *addr);

static inline void *kasan_reset_tag(const void *addr)

{

	return (void *)arch_kasan_reset_tag(addr);

}



bool kasan_report(unsigned long addr, size_t size,

		bool is_write, unsigned long ip);

mm/kasan/common.c

+3 −3
Original line number Diff line number Diff line
@@ -179,14 +179,14 @@ size_t kasan_metadata_size(struct kmem_cache *cache) 
struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache,

					      const void *object)

{

	return (void *)reset_tag(object) + cache->kasan_info.alloc_meta_offset;

	return kasan_reset_tag(object) + cache->kasan_info.alloc_meta_offset;

}



struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache,

					    const void *object)

{

	BUILD_BUG_ON(sizeof(struct kasan_free_meta) > 32);

	return (void *)reset_tag(object) + cache->kasan_info.free_meta_offset;

	return kasan_reset_tag(object) + cache->kasan_info.free_meta_offset;

}



void kasan_poison_slab(struct page *page)
@@ -283,7 +283,7 @@ static bool __kasan_slab_free(struct kmem_cache *cache, void *object, 


	tag = get_tag(object);

	tagged_object = object;

	object = reset_tag(object);

	object = kasan_reset_tag(object);



	if (unlikely(nearest_obj(cache, virt_to_head_page(object), object) !=

	    object)) {

mm/kasan/hw_tags.c

+2 −7
Original line number Diff line number Diff line
@@ -30,20 +30,15 @@ void __init kasan_init_hw_tags(void) 
	pr_info("KernelAddressSanitizer initialized\n");

}



void *kasan_reset_tag(const void *addr)

{

	return reset_tag(addr);

}



void poison_range(const void *address, size_t size, u8 value)

{

	hw_set_mem_tag_range(reset_tag(address),

	hw_set_mem_tag_range(kasan_reset_tag(address),

			round_up(size, KASAN_GRANULE_SIZE), value);

}



void unpoison_range(const void *address, size_t size)

{

	hw_set_mem_tag_range(reset_tag(address),

	hw_set_mem_tag_range(kasan_reset_tag(address),

			round_up(size, KASAN_GRANULE_SIZE), get_tag(address));

}

mm/kasan/kasan.h

+0 −4
Original line number Diff line number Diff line
@@ -248,15 +248,11 @@ static inline const void *arch_kasan_set_tag(const void *addr, u8 tag) 
	return addr;

}

#endif

#ifndef arch_kasan_reset_tag

#define arch_kasan_reset_tag(addr)	((void *)(addr))

#endif

#ifndef arch_kasan_get_tag

#define arch_kasan_get_tag(addr)	0

#endif



#define set_tag(addr, tag)	((void *)arch_kasan_set_tag((addr), (tag)))

#define reset_tag(addr)		((void *)arch_kasan_reset_tag(addr))

#define get_tag(addr)		arch_kasan_get_tag(addr)



#ifdef CONFIG_KASAN_HW_TAGS

mm/kasan/report.c

+2 −2
Original line number Diff line number Diff line
@@ -328,7 +328,7 @@ void kasan_report_invalid_free(void *object, unsigned long ip) 
	unsigned long flags;

	u8 tag = get_tag(object);



	object = reset_tag(object);

	object = kasan_reset_tag(object);



#if IS_ENABLED(CONFIG_KUNIT)

	if (current->kunit_test)
@@ -361,7 +361,7 @@ static void __kasan_report(unsigned long addr, size_t size, bool is_write, 
	disable_trace_on_warning();



	tagged_addr = (void *)addr;

	untagged_addr = reset_tag(tagged_addr);

	untagged_addr = kasan_reset_tag(tagged_addr);



	info.access_addr = tagged_addr;

	if (addr_has_metadata(untagged_addr))

CRA Git | Maintained and supported by SUSTech CRA and CCSE