Commit aa5c219c authored by Andrey Konovalov's avatar Andrey Konovalov Committed by Linus Torvalds
Browse files

kasan: init memory in kasan_(un)poison for HW_TAGS 

This change adds an argument to kasan_poison() and kasan_unpoison() that
allows initializing memory along with setting the tags for HW_TAGS.

Combining setting allocation tags with memory initialization will improve
HW_TAGS KASAN performance when init_on_alloc/free is enabled.

This change doesn't integrate memory initialization with KASAN, this is
done is subsequent patches in this series.

Link: https://lkml.kernel.org/r/3054314039fa64510947e674180d675cab1b4c41.1615296150.git.andreyknvl@google.com


Signed-off-by: default avatarAndrey Konovalov <andreyknvl@google.com>
Reviewed-by: default avatarMarco Elver <elver@google.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Ryabinin <aryabinin@virtuozzo.com>
Cc: Branislav Rankov <Branislav.Rankov@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Evgenii Stepanov <eugenis@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Kevin Brodsky <kevin.brodsky@arm.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Peter Collingbourne <pcc@google.com>
Cc: Vincenzo Frascino <vincenzo.frascino@arm.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent d9b6f907

lib/test_kasan.c

+2 −2
Original line number Diff line number Diff line
@@ -1049,14 +1049,14 @@ static void match_all_mem_tag(struct kunit *test) 
			continue;



		/* Mark the first memory granule with the chosen memory tag. */

		kasan_poison(ptr, KASAN_GRANULE_SIZE, (u8)tag);

		kasan_poison(ptr, KASAN_GRANULE_SIZE, (u8)tag, false);



		/* This access must cause a KASAN report. */

		KUNIT_EXPECT_KASAN_FAIL(test, *ptr = 0);

	}



	/* Recover the memory tag and free. */

	kasan_poison(ptr, KASAN_GRANULE_SIZE, get_tag(ptr));

	kasan_poison(ptr, KASAN_GRANULE_SIZE, get_tag(ptr), false);

	kfree(ptr);

}

mm/kasan/common.c

+14 −14
Original line number Diff line number Diff line
@@ -60,7 +60,7 @@ void kasan_disable_current(void) 


void __kasan_unpoison_range(const void *address, size_t size)

{

	kasan_unpoison(address, size);

	kasan_unpoison(address, size, false);

}



#ifdef CONFIG_KASAN_STACK
@@ -69,7 +69,7 @@ void kasan_unpoison_task_stack(struct task_struct *task) 
{

	void *base = task_stack_page(task);



	kasan_unpoison(base, THREAD_SIZE);

	kasan_unpoison(base, THREAD_SIZE, false);

}



/* Unpoison the stack for the current task beyond a watermark sp value. */
@@ -82,7 +82,7 @@ asmlinkage void kasan_unpoison_task_stack_below(const void *watermark) 
	 */

	void *base = (void *)((unsigned long)watermark & ~(THREAD_SIZE - 1));



	kasan_unpoison(base, watermark - base);

	kasan_unpoison(base, watermark - base, false);

}

#endif /* CONFIG_KASAN_STACK */
@@ -108,14 +108,14 @@ void __kasan_alloc_pages(struct page *page, unsigned int order) 
	tag = kasan_random_tag();

	for (i = 0; i < (1 << order); i++)

		page_kasan_tag_set(page + i, tag);

	kasan_unpoison(page_address(page), PAGE_SIZE << order);

	kasan_unpoison(page_address(page), PAGE_SIZE << order, false);

}



void __kasan_free_pages(struct page *page, unsigned int order)

{

	if (likely(!PageHighMem(page)))

		kasan_poison(page_address(page), PAGE_SIZE << order,

			     KASAN_FREE_PAGE);

			     KASAN_FREE_PAGE, false);

}



/*
@@ -251,18 +251,18 @@ void __kasan_poison_slab(struct page *page) 
	for (i = 0; i < compound_nr(page); i++)

		page_kasan_tag_reset(page + i);

	kasan_poison(page_address(page), page_size(page),

		     KASAN_KMALLOC_REDZONE);

		     KASAN_KMALLOC_REDZONE, false);

}



void __kasan_unpoison_object_data(struct kmem_cache *cache, void *object)

{

	kasan_unpoison(object, cache->object_size);

	kasan_unpoison(object, cache->object_size, false);

}



void __kasan_poison_object_data(struct kmem_cache *cache, void *object)

{

	kasan_poison(object, round_up(cache->object_size, KASAN_GRANULE_SIZE),

			KASAN_KMALLOC_REDZONE);

			KASAN_KMALLOC_REDZONE, false);

}



/*
@@ -351,7 +351,7 @@ static inline bool ____kasan_slab_free(struct kmem_cache *cache, 
	}



	kasan_poison(object, round_up(cache->object_size, KASAN_GRANULE_SIZE),

			KASAN_KMALLOC_FREE);

			KASAN_KMALLOC_FREE, false);



	if ((IS_ENABLED(CONFIG_KASAN_GENERIC) && !quarantine))

		return false;
@@ -407,7 +407,7 @@ void __kasan_slab_free_mempool(void *ptr, unsigned long ip) 
	if (unlikely(!PageSlab(page))) {

		if (____kasan_kfree_large(ptr, ip))

			return;

		kasan_poison(ptr, page_size(page), KASAN_FREE_PAGE);

		kasan_poison(ptr, page_size(page), KASAN_FREE_PAGE, false);

	} else {

		____kasan_slab_free(page->slab_cache, ptr, ip, false);

	}
@@ -453,7 +453,7 @@ void * __must_check __kasan_slab_alloc(struct kmem_cache *cache, 
	 * Unpoison the whole object.

	 * For kmalloc() allocations, kasan_kmalloc() will do precise poisoning.

	 */

	kasan_unpoison(tagged_object, cache->object_size);

	kasan_unpoison(tagged_object, cache->object_size, false);



	/* Save alloc info (if possible) for non-kmalloc() allocations. */

	if (kasan_stack_collection_enabled())
@@ -496,7 +496,7 @@ static inline void *____kasan_kmalloc(struct kmem_cache *cache, 
	redzone_end = round_up((unsigned long)(object + cache->object_size),

				KASAN_GRANULE_SIZE);

	kasan_poison((void *)redzone_start, redzone_end - redzone_start,

			   KASAN_KMALLOC_REDZONE);

			   KASAN_KMALLOC_REDZONE, false);



	/*

	 * Save alloc info (if possible) for kmalloc() allocations.
@@ -546,7 +546,7 @@ void * __must_check __kasan_kmalloc_large(const void *ptr, size_t size, 
				KASAN_GRANULE_SIZE);

	redzone_end = (unsigned long)ptr + page_size(virt_to_page(ptr));

	kasan_poison((void *)redzone_start, redzone_end - redzone_start,

		     KASAN_PAGE_REDZONE);

		     KASAN_PAGE_REDZONE, false);



	return (void *)ptr;

}
@@ -563,7 +563,7 @@ void * __must_check __kasan_krealloc(const void *object, size_t size, gfp_t flag 
	 * Part of it might already have been unpoisoned, but it's unknown

	 * how big that part is.

	 */

	kasan_unpoison(object, size);

	kasan_unpoison(object, size, false);



	page = virt_to_head_page(object);

mm/kasan/generic.c

+6 −6
Original line number Diff line number Diff line
@@ -208,11 +208,11 @@ static void register_global(struct kasan_global *global) 
{

	size_t aligned_size = round_up(global->size, KASAN_GRANULE_SIZE);



	kasan_unpoison(global->beg, global->size);

	kasan_unpoison(global->beg, global->size, false);



	kasan_poison(global->beg + aligned_size,

		     global->size_with_redzone - aligned_size,

		     KASAN_GLOBAL_REDZONE);

		     KASAN_GLOBAL_REDZONE, false);

}



void __asan_register_globals(struct kasan_global *globals, size_t size)
@@ -292,11 +292,11 @@ void __asan_alloca_poison(unsigned long addr, size_t size) 
	WARN_ON(!IS_ALIGNED(addr, KASAN_ALLOCA_REDZONE_SIZE));



	kasan_unpoison((const void *)(addr + rounded_down_size),

			size - rounded_down_size);

			size - rounded_down_size, false);

	kasan_poison(left_redzone, KASAN_ALLOCA_REDZONE_SIZE,

		     KASAN_ALLOCA_LEFT);

		     KASAN_ALLOCA_LEFT, false);

	kasan_poison(right_redzone, padding_size + KASAN_ALLOCA_REDZONE_SIZE,

		     KASAN_ALLOCA_RIGHT);

		     KASAN_ALLOCA_RIGHT, false);

}

EXPORT_SYMBOL(__asan_alloca_poison);
@@ -306,7 +306,7 @@ void __asan_allocas_unpoison(const void *stack_top, const void *stack_bottom) 
	if (unlikely(!stack_top || stack_top > stack_bottom))

		return;



	kasan_unpoison(stack_top, stack_bottom - stack_top);

	kasan_unpoison(stack_top, stack_bottom - stack_top, false);

}

EXPORT_SYMBOL(__asan_allocas_unpoison);

mm/kasan/kasan.h

+8 −6
Original line number Diff line number Diff line
@@ -359,7 +359,7 @@ static inline u8 kasan_random_tag(void) { return 0; } 


#ifdef CONFIG_KASAN_HW_TAGS



static inline void kasan_poison(const void *addr, size_t size, u8 value)

static inline void kasan_poison(const void *addr, size_t size, u8 value, bool init)

{

	addr = kasan_reset_tag(addr);
@@ -372,10 +372,10 @@ static inline void kasan_poison(const void *addr, size_t size, u8 value) 
	if (WARN_ON(size & KASAN_GRANULE_MASK))

		return;



	hw_set_mem_tag_range((void *)addr, size, value, false);

	hw_set_mem_tag_range((void *)addr, size, value, init);

}



static inline void kasan_unpoison(const void *addr, size_t size)

static inline void kasan_unpoison(const void *addr, size_t size, bool init)

{

	u8 tag = get_tag(addr);
@@ -389,7 +389,7 @@ static inline void kasan_unpoison(const void *addr, size_t size) 
		return;

	size = round_up(size, KASAN_GRANULE_SIZE);



	hw_set_mem_tag_range((void *)addr, size, tag, false);

	hw_set_mem_tag_range((void *)addr, size, tag, init);

}



static inline bool kasan_byte_accessible(const void *addr)
@@ -407,22 +407,24 @@ static inline bool kasan_byte_accessible(const void *addr) 
 * @addr - range start address, must be aligned to KASAN_GRANULE_SIZE

 * @size - range size, must be aligned to KASAN_GRANULE_SIZE

 * @value - value that's written to metadata for the range

 * @init - whether to initialize the memory range (only for hardware tag-based)

 *

 * The size gets aligned to KASAN_GRANULE_SIZE before marking the range.

 */

void kasan_poison(const void *addr, size_t size, u8 value);

void kasan_poison(const void *addr, size_t size, u8 value, bool init);



/**

 * kasan_unpoison - mark the memory range as accessible

 * @addr - range start address, must be aligned to KASAN_GRANULE_SIZE

 * @size - range size, can be unaligned

 * @init - whether to initialize the memory range (only for hardware tag-based)

 *

 * For the tag-based modes, the @size gets aligned to KASAN_GRANULE_SIZE before

 * marking the range.

 * For the generic mode, the last granule of the memory range gets partially

 * unpoisoned based on the @size.

 */

void kasan_unpoison(const void *addr, size_t size);

void kasan_unpoison(const void *addr, size_t size, bool init);



bool kasan_byte_accessible(const void *addr);

mm/kasan/shadow.c

+5 −5
Original line number Diff line number Diff line
@@ -69,7 +69,7 @@ void *memcpy(void *dest, const void *src, size_t len) 
	return __memcpy(dest, src, len);

}



void kasan_poison(const void *addr, size_t size, u8 value)

void kasan_poison(const void *addr, size_t size, u8 value, bool init)

{

	void *shadow_start, *shadow_end;
@@ -106,7 +106,7 @@ void kasan_poison_last_granule(const void *addr, size_t size) 
}

#endif



void kasan_unpoison(const void *addr, size_t size)

void kasan_unpoison(const void *addr, size_t size, bool init)

{

	u8 tag = get_tag(addr);
@@ -129,7 +129,7 @@ void kasan_unpoison(const void *addr, size_t size) 
		return;



	/* Unpoison all granules that cover the object. */

	kasan_poison(addr, round_up(size, KASAN_GRANULE_SIZE), tag);

	kasan_poison(addr, round_up(size, KASAN_GRANULE_SIZE), tag, false);



	/* Partially poison the last granule for the generic mode. */

	if (IS_ENABLED(CONFIG_KASAN_GENERIC))
@@ -344,7 +344,7 @@ void kasan_poison_vmalloc(const void *start, unsigned long size) 
		return;



	size = round_up(size, KASAN_GRANULE_SIZE);

	kasan_poison(start, size, KASAN_VMALLOC_INVALID);

	kasan_poison(start, size, KASAN_VMALLOC_INVALID, false);

}



void kasan_unpoison_vmalloc(const void *start, unsigned long size)
@@ -352,7 +352,7 @@ void kasan_unpoison_vmalloc(const void *start, unsigned long size) 
	if (!is_vmalloc_or_module_addr(start))

		return;



	kasan_unpoison(start, size);

	kasan_unpoison(start, size, false);

}



static int kasan_depopulate_vmalloc_pte(pte_t *ptep, unsigned long addr,