Commit 685e2564 authored by Reiji Watanabe's avatar Reiji Watanabe Committed by Catalin Marinas
Browse files

arm64: mte: DC {GVA,GZVA} shouldn't be used when DCZID_EL0.DZP == 1 



Currently, mte_set_mem_tag_range() and mte_zero_clear_page_tags() use
DC {GVA,GZVA} unconditionally.  But, they should make sure that
DCZID_EL0.DZP, which indicates whether or not use of those instructions
is prohibited, is zero when using those instructions.
Use ST{G,ZG,Z2G} instead when DCZID_EL0.DZP == 1.

Fixes: 013bb59d ("arm64: mte: handle tags zeroing at page allocation time")
Fixes: 3d0cca0b ("kasan: speed up mte_set_mem_tag_range")
Signed-off-by: default avatarReiji Watanabe <reijiw@google.com>
Link: https://lore.kernel.org/r/20211206004736.1520989-3-reijiw@google.com


Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent f0616abd

arch/arm64/include/asm/mte-kasan.h

+5 −3
Original line number Diff line number Diff line
@@ -84,10 +84,12 @@ static inline void __dc_gzva(u64 p) 
static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag,

					 bool init)

{

	u64 curr, mask, dczid_bs, end1, end2, end3;

	u64 curr, mask, dczid, dczid_bs, dczid_dzp, end1, end2, end3;



	/* Read DC G(Z)VA block size from the system register. */

	dczid_bs = 4ul << (read_cpuid(DCZID_EL0) & 0xf);

	dczid = read_cpuid(DCZID_EL0);

	dczid_bs = 4ul << (dczid & 0xf);

	dczid_dzp = (dczid >> 4) & 1;



	curr = (u64)__tag_set(addr, tag);

	mask = dczid_bs - 1;
@@ -106,7 +108,7 @@ static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag, 
	 */

#define SET_MEMTAG_RANGE(stg_post, dc_gva)		\

	do {						\

		if (size >= 2 * dczid_bs) {		\

		if (!dczid_dzp && size >= 2 * dczid_bs) {\

			do {				\

				curr = stg_post(curr);	\

			} while (curr < end1);		\

arch/arm64/lib/mte.S

+7 −1
Original line number Diff line number Diff line
@@ -43,17 +43,23 @@ SYM_FUNC_END(mte_clear_page_tags) 
 *	x0 - address to the beginning of the page

 */

SYM_FUNC_START(mte_zero_clear_page_tags)

	and	x0, x0, #(1 << MTE_TAG_SHIFT) - 1	// clear the tag

	mrs	x1, dczid_el0

	tbnz	x1, #4, 2f	// Branch if DC GZVA is prohibited

	and	w1, w1, #0xf

	mov	x2, #4

	lsl	x1, x2, x1

	and	x0, x0, #(1 << MTE_TAG_SHIFT) - 1	// clear the tag



1:	dc	gzva, x0

	add	x0, x0, x1

	tst	x0, #(PAGE_SIZE - 1)

	b.ne	1b

	ret



2:	stz2g	x0, [x0], #(MTE_GRANULE_SIZE * 2)

	tst	x0, #(PAGE_SIZE - 1)

	b.ne	2b

	ret

SYM_FUNC_END(mte_zero_clear_page_tags)



/*