Commit fdceddb0 authored by Will Deacon's avatar Will Deacon
Browse files

Merge branch 'for-next/mte' into for-next/core 

KASAN optimisations for the hardware tagging (MTE) implementation.

* for-next/mte:
  kasan: disable freed user page poisoning with HW tags
  arm64: mte: handle tags zeroing at page allocation time
  kasan: use separate (un)poison implementation for integrated init
  mm: arch: remove indirection level in alloc_zeroed_user_highpage_movable()
  kasan: speed up mte_set_mem_tag_range
parents 81ad4bb1 c275c5c6

arch/alpha/include/asm/page.h

+3 −3
Original line number Diff line number Diff line
@@ -17,9 +17,9 @@ 
extern void clear_page(void *page);

#define clear_user_page(page, vaddr, pg)	clear_page(page)



#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \

	alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vmaddr)

#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE

#define alloc_zeroed_user_highpage_movable(vma, vaddr) \

	alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, vma, vmaddr)

#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE



extern void copy_page(void * _to, void * _from);

#define copy_user_page(to, from, vaddr, pg)	copy_page(to, from)

arch/arm64/include/asm/mte-kasan.h

+67 −26
Original line number Diff line number Diff line
@@ -48,43 +48,84 @@ static inline u8 mte_get_random_tag(void) 
	return mte_get_ptr_tag(addr);

}



static inline u64 __stg_post(u64 p)

{

	asm volatile(__MTE_PREAMBLE "stg %0, [%0], #16"

		     : "+r"(p)

		     :

		     : "memory");

	return p;

}



static inline u64 __stzg_post(u64 p)

{

	asm volatile(__MTE_PREAMBLE "stzg %0, [%0], #16"

		     : "+r"(p)

		     :

		     : "memory");

	return p;

}



static inline void __dc_gva(u64 p)

{

	asm volatile(__MTE_PREAMBLE "dc gva, %0" : : "r"(p) : "memory");

}



static inline void __dc_gzva(u64 p)

{

	asm volatile(__MTE_PREAMBLE "dc gzva, %0" : : "r"(p) : "memory");

}



/*

 * Assign allocation tags for a region of memory based on the pointer tag.

 * Note: The address must be non-NULL and MTE_GRANULE_SIZE aligned and

 * size must be non-zero and MTE_GRANULE_SIZE aligned.

 * size must be MTE_GRANULE_SIZE aligned.

 */

static inline void mte_set_mem_tag_range(void *addr, size_t size,

						u8 tag, bool init)

static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag,

					 bool init)

{

	u64 curr, end;

	u64 curr, mask, dczid_bs, end1, end2, end3;



	if (!size)

		return;

	/* Read DC G(Z)VA block size from the system register. */

	dczid_bs = 4ul << (read_cpuid(DCZID_EL0) & 0xf);



	curr = (u64)__tag_set(addr, tag);

	end = curr + size;

	mask = dczid_bs - 1;

	/* STG/STZG up to the end of the first block. */

	end1 = curr | mask;

	end3 = curr + size;

	/* DC GVA / GZVA in [end1, end2) */

	end2 = end3 & ~mask;



	/*

	 * 'asm volatile' is required to prevent the compiler to move

	 * the statement outside of the loop.

	 * The following code uses STG on the first DC GVA block even if the

	 * start address is aligned - it appears to be faster than an alignment

	 * check + conditional branch. Also, if the range size is at least 2 DC

	 * GVA blocks, the first two loops can use post-condition to save one

	 * branch each.

	 */

	if (init) {

		do {

			asm volatile(__MTE_PREAMBLE "stzg %0, [%0]"

				     :

				     : "r" (curr)

				     : "memory");

			curr += MTE_GRANULE_SIZE;

		} while (curr != end);

	} else {

		do {

			asm volatile(__MTE_PREAMBLE "stg %0, [%0]"

				     :

				     : "r" (curr)

				     : "memory");

			curr += MTE_GRANULE_SIZE;

		} while (curr != end);

	}

#define SET_MEMTAG_RANGE(stg_post, dc_gva)		\

	do {						\

		if (size >= 2 * dczid_bs) {		\

			do {				\

				curr = stg_post(curr);	\

			} while (curr < end1);		\

							\

			do {				\

				dc_gva(curr);		\

				curr += dczid_bs;	\

			} while (curr < end2);		\

		}					\

							\

		while (curr < end3)			\

			curr = stg_post(curr);		\

	} while (0)



	if (init)

		SET_MEMTAG_RANGE(__stzg_post, __dc_gzva);

	else

		SET_MEMTAG_RANGE(__stg_post, __dc_gva);

#undef SET_MEMTAG_RANGE

}



void mte_enable_kernel_sync(void);

arch/arm64/include/asm/mte.h

+4 −0
Original line number Diff line number Diff line
@@ -37,6 +37,7 @@ void mte_free_tag_storage(char *storage); 
/* track which pages have valid allocation tags */

#define PG_mte_tagged	PG_arch_2



void mte_zero_clear_page_tags(void *addr);

void mte_sync_tags(pte_t *ptep, pte_t pte);

void mte_copy_page_tags(void *kto, const void *kfrom);

void mte_thread_init_user(void);
@@ -53,6 +54,9 @@ int mte_ptrace_copy_tags(struct task_struct *child, long request, 
/* unused if !CONFIG_ARM64_MTE, silence the compiler */

#define PG_mte_tagged	0



static inline void mte_zero_clear_page_tags(void *addr)

{

}

static inline void mte_sync_tags(pte_t *ptep, pte_t pte)

{

}

arch/arm64/include/asm/page.h

+7 −3
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ 
#ifndef __ASSEMBLY__



#include <linux/personality.h> /* for READ_IMPLIES_EXEC */

#include <linux/types.h> /* for gfp_t */

#include <asm/pgtable-types.h>



struct page;
@@ -28,9 +29,12 @@ void copy_user_highpage(struct page *to, struct page *from, 
void copy_highpage(struct page *to, struct page *from);

#define __HAVE_ARCH_COPY_HIGHPAGE



#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \

	alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr)

#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE

struct page *alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma,

						unsigned long vaddr);

#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE



void tag_clear_highpage(struct page *to);

#define __HAVE_ARCH_TAG_CLEAR_HIGHPAGE



#define clear_user_page(page, vaddr, pg)	clear_page(page)

#define copy_user_page(to, from, vaddr, pg)	copy_page(to, from)

arch/arm64/lib/mte.S

+20 −0
Original line number Diff line number Diff line
@@ -36,6 +36,26 @@ SYM_FUNC_START(mte_clear_page_tags) 
	ret

SYM_FUNC_END(mte_clear_page_tags)



/*

 * Zero the page and tags at the same time

 *

 * Parameters:

 *	x0 - address to the beginning of the page

 */

SYM_FUNC_START(mte_zero_clear_page_tags)

	mrs	x1, dczid_el0

	and	w1, w1, #0xf

	mov	x2, #4

	lsl	x1, x2, x1

	and	x0, x0, #(1 << MTE_TAG_SHIFT) - 1	// clear the tag



1:	dc	gzva, x0

	add	x0, x0, x1

	tst	x0, #(PAGE_SIZE - 1)

	b.ne	1b

	ret

SYM_FUNC_END(mte_zero_clear_page_tags)



/*

 * Copy the tags from the source page to the destination one

 *   x0 - address of the destination page