Commit 0a54864f authored by Peter Collingbourne's avatar Peter Collingbourne Committed by Andrew Morton
Browse files

kasan: remove PG_skip_kasan_poison flag 

Code inspection reveals that PG_skip_kasan_poison is redundant with
kasantag, because the former is intended to be set iff the latter is the
match-all tag.  It can also be observed that it's basically pointless to
poison pages which have kasantag=0, because any pages with this tag would
have been pointed to by pointers with match-all tags, so poisoning the
pages would have little to no effect in terms of bug detection. 
Therefore, change the condition in should_skip_kasan_poison() to check
kasantag instead, and remove PG_skip_kasan_poison and associated flags.

Link: https://lkml.kernel.org/r/20230310042914.3805818-3-pcc@google.com
Link: https://linux-review.googlesource.com/id/I57f825f2eaeaf7e8389d6cf4597c8a5821359838


Signed-off-by: default avatarPeter Collingbourne <pcc@google.com>
Reviewed-by: default avatarAndrey Konovalov <andreyknvl@gmail.com>
Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Evgenii Stepanov <eugenis@google.com>
Cc: Vincenzo Frascino <vincenzo.frascino@arm.com>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 7eb16f23

include/linux/gfp_types.h

+13 −17
Original line number Diff line number Diff line
@@ -48,15 +48,13 @@ typedef unsigned int __bitwise gfp_t; 
#define ___GFP_ZEROTAGS		0x800000u

#ifdef CONFIG_KASAN_HW_TAGS

#define ___GFP_SKIP_ZERO	0x1000000u

#define ___GFP_SKIP_KASAN_UNPOISON	0x2000000u

#define ___GFP_SKIP_KASAN_POISON	0x4000000u

#define ___GFP_SKIP_KASAN	0x2000000u

#else

#define ___GFP_SKIP_ZERO	0

#define ___GFP_SKIP_KASAN_UNPOISON	0

#define ___GFP_SKIP_KASAN_POISON	0

#define ___GFP_SKIP_KASAN	0

#endif

#ifdef CONFIG_LOCKDEP

#define ___GFP_NOLOCKDEP	0x8000000u

#define ___GFP_NOLOCKDEP	0x4000000u

#else

#define ___GFP_NOLOCKDEP	0

#endif
@@ -234,25 +232,24 @@ typedef unsigned int __bitwise gfp_t; 
 * memory tags at the same time as zeroing memory has minimal additional

 * performace impact.

 *

 * %__GFP_SKIP_KASAN_UNPOISON makes KASAN skip unpoisoning on page allocation.

 * Only effective in HW_TAGS mode.

 *

 * %__GFP_SKIP_KASAN_POISON makes KASAN skip poisoning on page deallocation.

 * Typically, used for userspace pages. Only effective in HW_TAGS mode.

 * %__GFP_SKIP_KASAN makes KASAN skip unpoisoning on page allocation.

 * Used for userspace and vmalloc pages; the latter are unpoisoned by

 * kasan_unpoison_vmalloc instead. For userspace pages, results in

 * poisoning being skipped as well, see should_skip_kasan_poison for

 * details. Only effective in HW_TAGS mode.

 */

#define __GFP_NOWARN	((__force gfp_t)___GFP_NOWARN)

#define __GFP_COMP	((__force gfp_t)___GFP_COMP)

#define __GFP_ZERO	((__force gfp_t)___GFP_ZERO)

#define __GFP_ZEROTAGS	((__force gfp_t)___GFP_ZEROTAGS)

#define __GFP_SKIP_ZERO ((__force gfp_t)___GFP_SKIP_ZERO)

#define __GFP_SKIP_KASAN_UNPOISON ((__force gfp_t)___GFP_SKIP_KASAN_UNPOISON)

#define __GFP_SKIP_KASAN_POISON   ((__force gfp_t)___GFP_SKIP_KASAN_POISON)

#define __GFP_SKIP_KASAN ((__force gfp_t)___GFP_SKIP_KASAN)



/* Disable lockdep for GFP context tracking */

#define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP)



/* Room for N __GFP_FOO bits */

#define __GFP_BITS_SHIFT (27 + IS_ENABLED(CONFIG_LOCKDEP))

#define __GFP_BITS_SHIFT (26 + IS_ENABLED(CONFIG_LOCKDEP))

#define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))



/**
@@ -335,8 +332,7 @@ typedef unsigned int __bitwise gfp_t; 
#define GFP_DMA		__GFP_DMA

#define GFP_DMA32	__GFP_DMA32

#define GFP_HIGHUSER	(GFP_USER | __GFP_HIGHMEM)

#define GFP_HIGHUSER_MOVABLE	(GFP_HIGHUSER | __GFP_MOVABLE | \

			 __GFP_SKIP_KASAN_POISON | __GFP_SKIP_KASAN_UNPOISON)

#define GFP_HIGHUSER_MOVABLE	(GFP_HIGHUSER | __GFP_MOVABLE | __GFP_SKIP_KASAN)

#define GFP_TRANSHUGE_LIGHT	((GFP_HIGHUSER_MOVABLE | __GFP_COMP | \

			 __GFP_NOMEMALLOC | __GFP_NOWARN) & ~__GFP_RECLAIM)

#define GFP_TRANSHUGE	(GFP_TRANSHUGE_LIGHT | __GFP_DIRECT_RECLAIM)

include/linux/page-flags.h

+0 −9
Original line number Diff line number Diff line
@@ -135,9 +135,6 @@ enum pageflags { 
#ifdef CONFIG_ARCH_USES_PG_ARCH_X

	PG_arch_2,

	PG_arch_3,

#endif

#ifdef CONFIG_KASAN_HW_TAGS

	PG_skip_kasan_poison,

#endif

	__NR_PAGEFLAGS,
@@ -594,12 +591,6 @@ TESTCLEARFLAG(Young, young, PF_ANY) 
PAGEFLAG(Idle, idle, PF_ANY)

#endif



#ifdef CONFIG_KASAN_HW_TAGS

PAGEFLAG(SkipKASanPoison, skip_kasan_poison, PF_HEAD)

#else

PAGEFLAG_FALSE(SkipKASanPoison, skip_kasan_poison)

#endif



/*

 * PageReported() is used to track reported free pages within the Buddy

 * allocator. We can use the non-atomic version of the test and set

include/trace/events/mmflags.h

+2 −11
Original line number Diff line number Diff line
@@ -55,8 +55,7 @@ 
#ifdef CONFIG_KASAN_HW_TAGS

#define __def_gfpflag_names_kasan ,			\

	gfpflag_string(__GFP_SKIP_ZERO),		\

	gfpflag_string(__GFP_SKIP_KASAN_POISON),	\

	gfpflag_string(__GFP_SKIP_KASAN_UNPOISON)

	gfpflag_string(__GFP_SKIP_KASAN)

#else

#define __def_gfpflag_names_kasan

#endif
@@ -96,13 +95,6 @@ 
#define IF_HAVE_PG_ARCH_X(_name)

#endif



#ifdef CONFIG_KASAN_HW_TAGS

#define IF_HAVE_PG_SKIP_KASAN_POISON(_name) \

	,{1UL << PG_##_name, __stringify(_name)}

#else

#define IF_HAVE_PG_SKIP_KASAN_POISON(_name)

#endif



#define DEF_PAGEFLAG_NAME(_name) { 1UL <<  PG_##_name, __stringify(_name) }



#define __def_pageflag_names						\
@@ -133,8 +125,7 @@ IF_HAVE_PG_HWPOISON(hwpoison) \ 
IF_HAVE_PG_IDLE(idle)							\

IF_HAVE_PG_IDLE(young)							\

IF_HAVE_PG_ARCH_X(arch_2)						\

IF_HAVE_PG_ARCH_X(arch_3)						\

IF_HAVE_PG_SKIP_KASAN_POISON(skip_kasan_poison)

IF_HAVE_PG_ARCH_X(arch_3)



#define show_page_flags(flags)						\

	(flags) ? __print_flags(flags, "|",				\

mm/kasan/hw_tags.c

+1 −1
Original line number Diff line number Diff line
@@ -318,7 +318,7 @@ void *__kasan_unpoison_vmalloc(const void *start, unsigned long size, 
	 * Thus, for VM_ALLOC mappings, hardware tag-based KASAN only tags

	 * the first virtual mapping, which is created by vmalloc().

	 * Tagging the page_alloc memory backing that vmalloc() allocation is

	 * skipped, see ___GFP_SKIP_KASAN_UNPOISON.

	 * skipped, see ___GFP_SKIP_KASAN.

	 *

	 * For non-VM_ALLOC allocations, page_alloc memory is tagged as usual.

	 */

mm/page_alloc.c

+30 −51
Original line number Diff line number Diff line
@@ -112,17 +112,6 @@ typedef int __bitwise fpi_t; 
 */

#define FPI_TO_TAIL		((__force fpi_t)BIT(1))



/*

 * Don't poison memory with KASAN (only for the tag-based modes).

 * During boot, all non-reserved memblock memory is exposed to page_alloc.

 * Poisoning all that memory lengthens boot time, especially on systems with

 * large amount of RAM. This flag is used to skip that poisoning.

 * This is only done for the tag-based KASAN modes, as those are able to

 * detect memory corruptions with the memory tags assigned by default.

 * All memory allocated normally after boot gets poisoned as usual.

 */

#define FPI_SKIP_KASAN_POISON	((__force fpi_t)BIT(2))



/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */

static DEFINE_MUTEX(pcp_batch_high_lock);

#define MIN_PERCPU_PAGELIST_HIGH_FRACTION (8)
@@ -1370,13 +1359,19 @@ static int free_tail_pages_check(struct page *head_page, struct page *page) 
/*

 * Skip KASAN memory poisoning when either:

 *

 * 1. Deferred memory initialization has not yet completed,

 *    see the explanation below.

 * 2. Skipping poisoning is requested via FPI_SKIP_KASAN_POISON,

 *    see the comment next to it.

 * 3. Skipping poisoning is requested via __GFP_SKIP_KASAN_POISON,

 *    see the comment next to it.

 * 4. The allocation is excluded from being checked due to sampling,

 * 1. For generic KASAN: deferred memory initialization has not yet completed.

 *    Tag-based KASAN modes skip pages freed via deferred memory initialization

 *    using page tags instead (see below).

 * 2. For tag-based KASAN modes: the page has a match-all KASAN tag, indicating

 *    that error detection is disabled for accesses via the page address.

 *

 * Pages will have match-all tags in the following circumstances:

 *

 * 1. Pages are being initialized for the first time, including during deferred

 *    memory init; see the call to page_kasan_tag_reset in __init_single_page.

 * 2. The allocation was not unpoisoned due to __GFP_SKIP_KASAN, with the

 *    exception of pages unpoisoned by kasan_unpoison_vmalloc.

 * 3. The allocation was excluded from being checked due to sampling,

 *    see the call to kasan_unpoison_pages.

 *

 * Poisoning pages during deferred memory init will greatly lengthen the
@@ -1392,10 +1387,10 @@ static int free_tail_pages_check(struct page *head_page, struct page *page) 
 */

static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags)

{

	return deferred_pages_enabled() ||

	       (!IS_ENABLED(CONFIG_KASAN_GENERIC) &&

		(fpi_flags & FPI_SKIP_KASAN_POISON)) ||

	       PageSkipKASanPoison(page);

	if (IS_ENABLED(CONFIG_KASAN_GENERIC))

		return deferred_pages_enabled();



	return page_kasan_tag(page) == 0xff;

}



static void kernel_init_pages(struct page *page, int numpages)
@@ -1730,7 +1725,7 @@ void __free_pages_core(struct page *page, unsigned int order) 
	 * Bypass PCP and place fresh pages right to the tail, primarily

	 * relevant for memory onlining.

	 */

	__free_pages_ok(page, order, FPI_TO_TAIL | FPI_SKIP_KASAN_POISON);

	__free_pages_ok(page, order, FPI_TO_TAIL);

}



#ifdef CONFIG_NUMA
@@ -2396,9 +2391,9 @@ static inline bool should_skip_kasan_unpoison(gfp_t flags) 


	/*

	 * With hardware tag-based KASAN enabled, skip if this has been

	 * requested via __GFP_SKIP_KASAN_UNPOISON.

	 * requested via __GFP_SKIP_KASAN.

	 */

	return flags & __GFP_SKIP_KASAN_UNPOISON;

	return flags & __GFP_SKIP_KASAN;

}



static inline bool should_skip_init(gfp_t flags)
@@ -2417,7 +2412,6 @@ inline void post_alloc_hook(struct page *page, unsigned int order, 
	bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) &&

			!should_skip_init(gfp_flags);

	bool zero_tags = init && (gfp_flags & __GFP_ZEROTAGS);

	bool reset_tags = true;

	int i;



	set_page_private(page, 0);
@@ -2451,37 +2445,22 @@ inline void post_alloc_hook(struct page *page, unsigned int order, 
		/* Take note that memory was initialized by the loop above. */

		init = false;

	}

	if (!should_skip_kasan_unpoison(gfp_flags)) {

		/* Try unpoisoning (or setting tags) and initializing memory. */

		if (kasan_unpoison_pages(page, order, init)) {

	if (!should_skip_kasan_unpoison(gfp_flags) &&

	    kasan_unpoison_pages(page, order, init)) {

		/* Take note that memory was initialized by KASAN. */

		if (kasan_has_integrated_init())

			init = false;

			/* Take note that memory tags were set by KASAN. */

			reset_tags = false;

	} else {

		/*

			 * KASAN decided to exclude this allocation from being

			 * (un)poisoned due to sampling. Make KASAN skip

			 * poisoning when the allocation is freed.

			 */

			SetPageSkipKASanPoison(page);

		}

	}

	/*

	 * If memory tags have not been set by KASAN, reset the page tags to

	 * ensure page_address() dereferencing does not fault.

		 * If memory tags have not been set by KASAN, reset the page

		 * tags to ensure page_address() dereferencing does not fault.

		 */

	if (reset_tags) {

		for (i = 0; i != 1 << order; ++i)

			page_kasan_tag_reset(page + i);

	}

	/* If memory is still not initialized, initialize it now. */

	if (init)

		kernel_init_pages(page, 1 << order);

	/* Propagate __GFP_SKIP_KASAN_POISON to page flags. */

	if (kasan_hw_tags_enabled() && (gfp_flags & __GFP_SKIP_KASAN_POISON))

		SetPageSkipKASanPoison(page);



	set_page_owner(page, order, gfp_flags);

	page_table_check_alloc(page, order);