Commit 85c653fc authored by Marc Zyngier's avatar Marc Zyngier
Browse files

Merge branch arm64/for-next/caches into kvmarm-master/next 



arm64 cache management function cleanup from Fuad Tabba,
shared with the arm64 tree.

* arm64/for-next/caches:
  arm64: Rename arm64-internal cache maintenance functions
  arm64: Fix cache maintenance function comments
  arm64: sync_icache_aliases to take end parameter instead of size
  arm64: __clean_dcache_area_pou to take end parameter instead of size
  arm64: __clean_dcache_area_pop to take end parameter instead of size
  arm64: __clean_dcache_area_poc to take end parameter instead of size
  arm64: __flush_dcache_area to take end parameter instead of size
  arm64: dcache_by_line_op to take end parameter instead of size
  arm64: __inval_dcache_area to take end parameter instead of size
  arm64: Fix comments to refer to correct function __flush_icache_range
  arm64: Move documentation of dcache_by_line_op
  arm64: assembler: remove user_alt
  arm64: Downgrade flush_icache_range to invalidate
  arm64: Do not enable uaccess for invalidate_icache_range
  arm64: Do not enable uaccess for flush_icache_range
  arm64: Apply errata to swsusp_arch_suspend_exit
  arm64: assembler: add conditional cache fixups
  arm64: assembler: replace `kaddr` with `addr`

Signed-off-by: default avatarMarc Zyngier <maz@kernel.org>
parents cb5faa8c fade9c2c

arch/arm64/include/asm/alternative-macros.h

+0 −5
Original line number Diff line number Diff line
@@ -197,11 +197,6 @@ alternative_endif 
#define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...)	\

	alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)



.macro user_alt, label, oldinstr, newinstr, cond

9999:	alternative_insn "\oldinstr", "\newinstr", \cond

	_asm_extable 9999b, \label

.endm



#endif  /*  __ASSEMBLY__  */



/*

arch/arm64/include/asm/arch_gicv3.h

+2 −1
Original line number Diff line number Diff line
@@ -124,7 +124,8 @@ static inline u32 gic_read_rpr(void) 
#define gic_read_lpir(c)		readq_relaxed(c)

#define gic_write_lpir(v, c)		writeq_relaxed(v, c)



#define gic_flush_dcache_to_poc(a,l)	__flush_dcache_area((a), (l))

#define gic_flush_dcache_to_poc(a,l)	\

	dcache_clean_inval_poc((unsigned long)(a), (unsigned long)(a)+(l))



#define gits_read_baser(c)		readq_relaxed(c)

#define gits_write_baser(v, c)		writeq_relaxed(v, c)

arch/arm64/include/asm/assembler.h

+48 −32
Original line number Diff line number Diff line
@@ -130,15 +130,27 @@ alternative_endif 
	.endm



/*

 * Emit an entry into the exception table

 * Create an exception table entry for `insn`, which will branch to `fixup`

 * when an unhandled fault is taken.

 */

	.macro		_asm_extable, from, to

	.macro		_asm_extable, insn, fixup

	.pushsection	__ex_table, "a"

	.align		3

	.long		(\from - .), (\to - .)

	.long		(\insn - .), (\fixup - .)

	.popsection

	.endm



/*

 * Create an exception table entry for `insn` if `fixup` is provided. Otherwise

 * do nothing.

 */

	.macro		_cond_extable, insn, fixup

	.ifnc		\fixup,

	_asm_extable	\insn, \fixup

	.endif

	.endm





#define USER(l, x...)				\

9999:	x;					\

	_asm_extable	9999b, l
@@ -375,51 +387,53 @@ alternative_cb_end 
	bfi	\tcr, \tmp0, \pos, #3

	.endm



	.macro __dcache_op_workaround_clean_cache, op, addr

alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE

	dc	\op, \addr

alternative_else

	dc	civac, \addr

alternative_endif

	.endm



/*

 * Macro to perform a data cache maintenance for the interval

 * [kaddr, kaddr + size)

 * [start, end)

 *

 * 	op:		operation passed to dc instruction

 * 	domain:		domain used in dsb instruciton

 * 	kaddr:		starting virtual address of the region

 * 	size:		size of the region

 * 	Corrupts:	kaddr, size, tmp1, tmp2

 * 	start:          starting virtual address of the region

 * 	end:            end virtual address of the region

 * 	fixup:		optional label to branch to on user fault

 * 	Corrupts:       start, end, tmp1, tmp2

 */

	.macro __dcache_op_workaround_clean_cache, op, kaddr

alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE

	dc	\op, \kaddr

alternative_else

	dc	civac, \kaddr

alternative_endif

	.endm



	.macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2

	.macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup

	dcache_line_size \tmp1, \tmp2

	add	\size, \kaddr, \size

	sub	\tmp2, \tmp1, #1

	bic	\kaddr, \kaddr, \tmp2

9998:

	bic	\start, \start, \tmp2

.Ldcache_op\@:

	.ifc	\op, cvau

	__dcache_op_workaround_clean_cache \op, \kaddr

	__dcache_op_workaround_clean_cache \op, \start

	.else

	.ifc	\op, cvac

	__dcache_op_workaround_clean_cache \op, \kaddr

	__dcache_op_workaround_clean_cache \op, \start

	.else

	.ifc	\op, cvap

	sys	3, c7, c12, 1, \kaddr	// dc cvap

	sys	3, c7, c12, 1, \start	// dc cvap

	.else

	.ifc	\op, cvadp

	sys	3, c7, c13, 1, \kaddr	// dc cvadp

	sys	3, c7, c13, 1, \start	// dc cvadp

	.else

	dc	\op, \kaddr

	dc	\op, \start

	.endif

	.endif

	.endif

	.endif

	add	\kaddr, \kaddr, \tmp1

	cmp	\kaddr, \size

	b.lo	9998b

	add	\start, \start, \tmp1

	cmp	\start, \end

	b.lo	.Ldcache_op\@

	dsb	\domain



	_cond_extable .Ldcache_op\@, \fixup

	.endm



/*
@@ -427,20 +441,22 @@ alternative_endif 
 * [start, end)

 *

 * 	start, end:	virtual addresses describing the region

 *	label:		A label to branch to on user fault.

 *	fixup:		optional label to branch to on user fault

 * 	Corrupts:	tmp1, tmp2

 */

	.macro invalidate_icache_by_line start, end, tmp1, tmp2, label

	.macro invalidate_icache_by_line start, end, tmp1, tmp2, fixup

	icache_line_size \tmp1, \tmp2

	sub	\tmp2, \tmp1, #1

	bic	\tmp2, \start, \tmp2

9997:

USER(\label, ic	ivau, \tmp2)			// invalidate I line PoU

.Licache_op\@:

	ic	ivau, \tmp2			// invalidate I line PoU

	add	\tmp2, \tmp2, \tmp1

	cmp	\tmp2, \end

	b.lo	9997b

	b.lo	.Licache_op\@

	dsb	ish

	isb



	_cond_extable .Licache_op\@, \fixup

	.endm



/*

arch/arm64/include/asm/cacheflush.h

+42 −29
Original line number Diff line number Diff line
@@ -30,45 +30,58 @@ 
 *	the implementation assumes non-aliasing VIPT D-cache and (aliasing)

 *	VIPT I-cache.

 *

 *	flush_icache_range(start, end)

 *	All functions below apply to the interval [start, end)

 *		- start  - virtual start address (inclusive)

 *		- end    - virtual end address (exclusive)

 *

 *		Ensure coherency between the I-cache and the D-cache in the

 *		region described by start, end.

 *		- start  - virtual start address

 *		- end    - virtual end address

 *	caches_clean_inval_pou(start, end)

 *

 *	invalidate_icache_range(start, end)

 *		Ensure coherency between the I-cache and the D-cache region to

 *		the Point of Unification.

 *

 *		Invalidate the I-cache in the region described by start, end.

 *		- start  - virtual start address

 *		- end    - virtual end address

 *	caches_clean_inval_user_pou(start, end)

 *

 *	__flush_cache_user_range(start, end)

 *		Ensure coherency between the I-cache and the D-cache region to

 *		the Point of Unification.

 *		Use only if the region might access user memory.

 *

 *		Ensure coherency between the I-cache and the D-cache in the

 *		region described by start, end.

 *		- start  - virtual start address

 *		- end    - virtual end address

 *	icache_inval_pou(start, end)

 *

 *	__flush_dcache_area(kaddr, size)

 *		Invalidate I-cache region to the Point of Unification.

 *

 *		Ensure that the data held in page is written back.

 *		- kaddr  - page address

 *		- size   - region size

 *	dcache_clean_inval_poc(start, end)

 *

 *		Clean and invalidate D-cache region to the Point of Coherency.

 *

 *	dcache_inval_poc(start, end)

 *

 *		Invalidate D-cache region to the Point of Coherency.

 *

 *	dcache_clean_poc(start, end)

 *

 *		Clean D-cache region to the Point of Coherency.

 *

 *	dcache_clean_pop(start, end)

 *

 *		Clean D-cache region to the Point of Persistence.

 *

 *	dcache_clean_pou(start, end)

 *

 *		Clean D-cache region to the Point of Unification.

 */

extern void __flush_icache_range(unsigned long start, unsigned long end);

extern int  invalidate_icache_range(unsigned long start, unsigned long end);

extern void __flush_dcache_area(void *addr, size_t len);

extern void __inval_dcache_area(void *addr, size_t len);

extern void __clean_dcache_area_poc(void *addr, size_t len);

extern void __clean_dcache_area_pop(void *addr, size_t len);

extern void __clean_dcache_area_pou(void *addr, size_t len);

extern long __flush_cache_user_range(unsigned long start, unsigned long end);

extern void sync_icache_aliases(void *kaddr, unsigned long len);

extern void caches_clean_inval_pou(unsigned long start, unsigned long end);

extern void icache_inval_pou(unsigned long start, unsigned long end);

extern void dcache_clean_inval_poc(unsigned long start, unsigned long end);

extern void dcache_inval_poc(unsigned long start, unsigned long end);

extern void dcache_clean_poc(unsigned long start, unsigned long end);

extern void dcache_clean_pop(unsigned long start, unsigned long end);

extern void dcache_clean_pou(unsigned long start, unsigned long end);

extern long caches_clean_inval_user_pou(unsigned long start, unsigned long end);

extern void sync_icache_aliases(unsigned long start, unsigned long end);



static inline void flush_icache_range(unsigned long start, unsigned long end)

{

	__flush_icache_range(start, end);

	caches_clean_inval_pou(start, end);



	/*

	 * IPI all online CPUs so that they undergo a context synchronization
@@ -122,7 +135,7 @@ extern void copy_to_user_page(struct vm_area_struct *, struct page *, 
#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1

extern void flush_dcache_page(struct page *);



static __always_inline void __flush_icache_all(void)

static __always_inline void icache_inval_all_pou(void)

{

	if (cpus_have_const_cap(ARM64_HAS_CACHE_DIC))

		return;

arch/arm64/include/asm/efi.h

+1 −1
Original line number Diff line number Diff line
@@ -137,7 +137,7 @@ void efi_virtmap_unload(void); 


static inline void efi_capsule_flush_cache_range(void *addr, int size)

{

	__flush_dcache_area(addr, size);

	dcache_clean_inval_poc((unsigned long)addr, (unsigned long)addr + size);

}



#endif /* _ASM_EFI_H */