Commit 53ae233c authored by Andrey Konovalov's avatar Andrey Konovalov Committed by Linus Torvalds
Browse files

kasan, page_alloc: allow skipping unpoisoning for HW_TAGS 

Add a new GFP flag __GFP_SKIP_KASAN_UNPOISON that allows skipping KASAN
poisoning for page_alloc allocations.  The flag is only effective with
HW_TAGS KASAN.

This flag will be used by vmalloc code for page_alloc allocations backing
vmalloc() mappings in a following patch.  The reason to skip KASAN
poisoning for these pages in page_alloc is because vmalloc code will be
poisoning them instead.

Also reword the comment for __GFP_SKIP_KASAN_POISON.

Link: https://lkml.kernel.org/r/35c97d77a704f6ff971dd3bfe4be95855744108e.1643047180.git.andreyknvl@google.com


Signed-off-by: default avatarAndrey Konovalov <andreyknvl@google.com>
Acked-by: default avatarMarco Elver <elver@google.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Evgenii Stepanov <eugenis@google.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Peter Collingbourne <pcc@google.com>
Cc: Vincenzo Frascino <vincenzo.frascino@arm.com>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent f49d9c5b

include/linux/gfp.h

+13 −8
Original line number Diff line number Diff line
@@ -55,12 +55,14 @@ struct vm_area_struct; 
#define ___GFP_ACCOUNT		0x400000u

#define ___GFP_ZEROTAGS		0x800000u

#ifdef CONFIG_KASAN_HW_TAGS

#define ___GFP_SKIP_KASAN_POISON	0x1000000u

#define ___GFP_SKIP_KASAN_UNPOISON	0x1000000u

#define ___GFP_SKIP_KASAN_POISON	0x2000000u

#else

#define ___GFP_SKIP_KASAN_UNPOISON	0

#define ___GFP_SKIP_KASAN_POISON	0

#endif

#ifdef CONFIG_LOCKDEP

#define ___GFP_NOLOCKDEP	0x2000000u

#define ___GFP_NOLOCKDEP	0x4000000u

#else

#define ___GFP_NOLOCKDEP	0

#endif
@@ -241,14 +243,17 @@ struct vm_area_struct; 
 * intended for optimization: setting memory tags at the same time as zeroing

 * memory has minimal additional performace impact.

 *

 * %__GFP_SKIP_KASAN_POISON returns a page which does not need to be poisoned

 * on deallocation. Typically used for userspace pages. Currently only has an

 * effect in HW tags mode.

 * %__GFP_SKIP_KASAN_UNPOISON makes KASAN skip unpoisoning on page allocation.

 * Only effective in HW_TAGS mode.

 *

 * %__GFP_SKIP_KASAN_POISON makes KASAN skip poisoning on page deallocation.

 * Typically, used for userspace pages. Only effective in HW_TAGS mode.

 */

#define __GFP_NOWARN	((__force gfp_t)___GFP_NOWARN)

#define __GFP_COMP	((__force gfp_t)___GFP_COMP)

#define __GFP_ZERO	((__force gfp_t)___GFP_ZERO)

#define __GFP_ZEROTAGS	((__force gfp_t)___GFP_ZEROTAGS)

#define __GFP_SKIP_KASAN_UNPOISON ((__force gfp_t)___GFP_SKIP_KASAN_UNPOISON)

#define __GFP_SKIP_KASAN_POISON   ((__force gfp_t)___GFP_SKIP_KASAN_POISON)



/* Disable lockdep for GFP context tracking */
@@ -256,7 +261,7 @@ struct vm_area_struct; 


/* Room for N __GFP_FOO bits */

#define __GFP_BITS_SHIFT (24 +						\

			  IS_ENABLED(CONFIG_KASAN_HW_TAGS) +	\

			  2 * IS_ENABLED(CONFIG_KASAN_HW_TAGS) +	\

			  IS_ENABLED(CONFIG_LOCKDEP))

#define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))

include/trace/events/mmflags.h

+3 −2
Original line number Diff line number Diff line
@@ -52,8 +52,9 @@ 
	{(unsigned long)__GFP_ZEROTAGS,		"__GFP_ZEROTAGS"}	\



#ifdef CONFIG_KASAN_HW_TAGS

#define __def_gfpflag_names_kasan					      \

	, {(unsigned long)__GFP_SKIP_KASAN_POISON, "__GFP_SKIP_KASAN_POISON"}

#define __def_gfpflag_names_kasan ,					       \

	{(unsigned long)__GFP_SKIP_KASAN_POISON,   "__GFP_SKIP_KASAN_POISON"}, \

	{(unsigned long)__GFP_SKIP_KASAN_UNPOISON, "__GFP_SKIP_KASAN_UNPOISON"}

#else

#define __def_gfpflag_names_kasan

#endif

mm/page_alloc.c

+22 −9
Original line number Diff line number Diff line
@@ -2343,6 +2343,26 @@ static inline bool check_new_pcp(struct page *page, unsigned int order) 
}

#endif /* CONFIG_DEBUG_VM */



static inline bool should_skip_kasan_unpoison(gfp_t flags, bool init_tags)

{

	/* Don't skip if a software KASAN mode is enabled. */

	if (IS_ENABLED(CONFIG_KASAN_GENERIC) ||

	    IS_ENABLED(CONFIG_KASAN_SW_TAGS))

		return false;



	/* Skip, if hardware tag-based KASAN is not enabled. */

	if (!kasan_hw_tags_enabled())

		return true;



	/*

	 * With hardware tag-based KASAN enabled, skip if either:

	 *

	 * 1. Memory tags have already been cleared via tag_clear_highpage().

	 * 2. Skipping has been requested via __GFP_SKIP_KASAN_UNPOISON.

	 */

	return init_tags || (flags & __GFP_SKIP_KASAN_UNPOISON);

}



inline void post_alloc_hook(struct page *page, unsigned int order,

				gfp_t gfp_flags)

{
@@ -2382,15 +2402,8 @@ inline void post_alloc_hook(struct page *page, unsigned int order, 
		/* Note that memory is already initialized by the loop above. */

		init = false;

	}

	/*

	 * If either a software KASAN mode is enabled, or,

	 * in the case of hardware tag-based KASAN,

	 * if memory tags have not been cleared via tag_clear_highpage().

	 */

	if (IS_ENABLED(CONFIG_KASAN_GENERIC) ||

	    IS_ENABLED(CONFIG_KASAN_SW_TAGS) ||

	    kasan_hw_tags_enabled() && !init_tags) {

		/* Mark shadow memory or set memory tags. */

	if (!should_skip_kasan_unpoison(gfp_flags, init_tags)) {

		/* Unpoison shadow memory or set memory tags. */

		kasan_unpoison_pages(page, order, init);



		/* Note that memory is already initialized by KASAN. */