Commit 4c0bd995 authored by Mark Rutland's avatar Mark Rutland Committed by Catalin Marinas
Browse files

arm64: alternatives: have callbacks take a cap 



Today, callback alternatives are special-cased within
__apply_alternatives(), and are applied alongside patching for system
capabilities as ARM64_NCAPS is not part of the boot_capabilities feature
mask.

This special-casing is less than ideal. Giving special meaning to
ARM64_NCAPS for this requires some structures and loops to use
ARM64_NCAPS + 1 (AKA ARM64_NPATCHABLE), while others use ARM64_NCAPS.
It's also not immediately clear callback alternatives are only applied
when applying alternatives for system-wide features.

To make this a bit clearer, changes the way that callback alternatives
are identified to remove the special-casing of ARM64_NCAPS, and to allow
callback alternatives to be associated with a cpucap as with all other
alternatives.

New cpucaps, ARM64_ALWAYS_BOOT and ARM64_ALWAYS_SYSTEM are added which
are always detected alongside boot cpu capabilities and system
capabilities respectively. All existing callback alternatives are made
to use ARM64_ALWAYS_SYSTEM, and so will be patched at the same point
during the boot flow as before.

Subsequent patches will make more use of these new cpucaps.

There should be no functional change as a result of this patch.

Signed-off-by: default avatarMark Rutland <mark.rutland@arm.com>
Cc: Ard Biesheuvel <ardb@kernel.org>
Cc: James Morse <james.morse@arm.com>
Cc: Joey Gouly <joey.gouly@arm.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Will Deacon <will@kernel.org>
Reviewed-by: default avatarArd Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20220912162210.3626215-7-mark.rutland@arm.com


Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent b723edf3

arch/arm64/include/asm/alternative-macros.h

+12 −6
Original line number Diff line number Diff line
@@ -2,10 +2,16 @@ 
#ifndef __ASM_ALTERNATIVE_MACROS_H

#define __ASM_ALTERNATIVE_MACROS_H



#include <linux/const.h>



#include <asm/cpucaps.h>

#include <asm/insn-def.h>



#define ARM64_CB_PATCH ARM64_NCAPS

#define ARM64_CB_BIT	(UL(1) << 15)



#if ARM64_NCAPS >= ARM64_CB_BIT

#error "cpucaps have overflown ARM64_CB_BIT"

#endif



#ifndef __ASSEMBLY__
@@ -73,8 +79,8 @@ 
#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...)	\

	__ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg))



#define ALTERNATIVE_CB(oldinstr, cb) \

	__ALTERNATIVE_CFG_CB(oldinstr, ARM64_CB_PATCH, 1, cb)

#define ALTERNATIVE_CB(oldinstr, feature, cb) \

	__ALTERNATIVE_CFG_CB(oldinstr, ARM64_CB_BIT | (feature), 1, cb)

#else



#include <asm/assembler.h>
@@ -82,7 +88,7 @@ 
.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len

	.word \orig_offset - .

	.word \alt_offset - .

	.hword \feature

	.hword (\feature)

	.byte \orig_len

	.byte \alt_len

.endm
@@ -141,10 +147,10 @@ 
661:

.endm



.macro alternative_cb cb

.macro alternative_cb cap, cb

	.set .Lasm_alt_mode, 0

	.pushsection .altinstructions, "a"

	altinstruction_entry 661f, \cb, ARM64_CB_PATCH, 662f-661f, 0

	altinstruction_entry 661f, \cb, ARM64_CB_BIT | \cap, 662f-661f, 0

	.popsection

661:

.endm

arch/arm64/include/asm/assembler.h

+5 −5
Original line number Diff line number Diff line
@@ -293,7 +293,7 @@ alternative_endif 
alternative_if_not ARM64_KVM_PROTECTED_MODE

	ASM_BUG()

alternative_else_nop_endif

alternative_cb kvm_compute_final_ctr_el0

alternative_cb ARM64_ALWAYS_SYSTEM, kvm_compute_final_ctr_el0

	movz	\reg, #0

	movk	\reg, #0, lsl #16

	movk	\reg, #0, lsl #32
@@ -877,7 +877,7 @@ alternative_endif 


	.macro __mitigate_spectre_bhb_loop      tmp

#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY

alternative_cb  spectre_bhb_patch_loop_iter

alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_loop_iter

	mov	\tmp, #32		// Patched to correct the immediate

alternative_cb_end

.Lspectre_bhb_loop\@:
@@ -890,7 +890,7 @@ alternative_cb_end 


	.macro mitigate_spectre_bhb_loop	tmp

#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY

alternative_cb	spectre_bhb_patch_loop_mitigation_enable

alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_loop_mitigation_enable

	b	.L_spectre_bhb_loop_done\@	// Patched to NOP

alternative_cb_end

	__mitigate_spectre_bhb_loop	\tmp
@@ -904,7 +904,7 @@ alternative_cb_end 
	stp	x0, x1, [sp, #-16]!

	stp	x2, x3, [sp, #-16]!

	mov	w0, #ARM_SMCCC_ARCH_WORKAROUND_3

alternative_cb	smccc_patch_fw_mitigation_conduit

alternative_cb ARM64_ALWAYS_SYSTEM, smccc_patch_fw_mitigation_conduit

	nop					// Patched to SMC/HVC #0

alternative_cb_end

	ldp	x2, x3, [sp], #16
@@ -914,7 +914,7 @@ alternative_cb_end 


	.macro mitigate_spectre_bhb_clear_insn

#ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY

alternative_cb	spectre_bhb_patch_clearbhb

alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_clearbhb

	/* Patched to NOP when not supported */

	clearbhb

	isb

arch/arm64/include/asm/cpufeature.h

+1 −3
Original line number Diff line number Diff line
@@ -422,9 +422,7 @@ extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); 
extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS];

extern struct static_key_false arm64_const_caps_ready;



/* ARM64 CAPS + alternative_cb */

#define ARM64_NPATCHABLE (ARM64_NCAPS + 1)

extern DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE);

extern DECLARE_BITMAP(boot_capabilities, ARM64_NCAPS);



#define for_each_available_cap(cap)		\

	for_each_set_bit(cap, cpu_hwcaps, ARM64_NCAPS)

arch/arm64/include/asm/kvm_mmu.h

+3 −2
Original line number Diff line number Diff line
@@ -63,7 +63,7 @@ 
 * specific registers encoded in the instructions).

 */

.macro kern_hyp_va	reg

alternative_cb kvm_update_va_mask

alternative_cb ARM64_ALWAYS_SYSTEM, kvm_update_va_mask

	and     \reg, \reg, #1		/* mask with va_mask */

	ror	\reg, \reg, #1		/* rotate to the first tag bit */

	add	\reg, \reg, #0		/* insert the low 12 bits of the tag */
@@ -97,7 +97,7 @@ alternative_cb_end 
	hyp_pa	\reg, \tmp



	/* Load kimage_voffset. */

alternative_cb kvm_get_kimage_voffset

alternative_cb ARM64_ALWAYS_SYSTEM, kvm_get_kimage_voffset

	movz	\tmp, #0

	movk	\tmp, #0, lsl #16

	movk	\tmp, #0, lsl #32
@@ -131,6 +131,7 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v) 
				    "add %0, %0, #0\n"

				    "add %0, %0, #0, lsl 12\n"

				    "ror %0, %0, #63\n",

				    ARM64_ALWAYS_SYSTEM,

				    kvm_update_va_mask)

		     : "+r" (v));

	return v;

arch/arm64/kernel/alternative.c

+14 −12
Original line number Diff line number Diff line
@@ -21,6 +21,9 @@ 
#define ALT_ORIG_PTR(a)		__ALT_PTR(a, orig_offset)

#define ALT_REPL_PTR(a)		__ALT_PTR(a, alt_offset)



#define ALT_CAP(a)		((a)->cpufeature & ~ARM64_CB_BIT)

#define ALT_HAS_CB(a)		((a)->cpufeature & ARM64_CB_BIT)



/* Volatile, as we may be patching the guts of READ_ONCE() */

static volatile int all_alternatives_applied;
@@ -143,16 +146,15 @@ static void __nocfi __apply_alternatives(const struct alt_region *region, 


	for (alt = region->begin; alt < region->end; alt++) {

		int nr_inst;

		int cap = ALT_CAP(alt);



		if (!test_bit(alt->cpufeature, feature_mask))

		if (!test_bit(cap, feature_mask))

			continue;



		/* Use ARM64_CB_PATCH as an unconditional patch */

		if (alt->cpufeature < ARM64_CB_PATCH &&

		    !cpus_have_cap(alt->cpufeature))

		if (!cpus_have_cap(cap))

			continue;



		if (alt->cpufeature == ARM64_CB_PATCH)

		if (ALT_HAS_CB(alt))

			BUG_ON(alt->alt_len != 0);

		else

			BUG_ON(alt->alt_len != alt->orig_len);
@@ -161,10 +163,10 @@ static void __nocfi __apply_alternatives(const struct alt_region *region, 
		updptr = is_module ? origptr : lm_alias(origptr);

		nr_inst = alt->orig_len / AARCH64_INSN_SIZE;



		if (alt->cpufeature < ARM64_CB_PATCH)

			alt_cb = patch_alternative;

		else

		if (ALT_HAS_CB(alt))

			alt_cb  = ALT_REPL_PTR(alt);

		else

			alt_cb = patch_alternative;



		alt_cb(alt, origptr, updptr, nr_inst);
@@ -208,10 +210,10 @@ static int __apply_alternatives_multi_stop(void *unused) 
			cpu_relax();

		isb();

	} else {

		DECLARE_BITMAP(remaining_capabilities, ARM64_NPATCHABLE);

		DECLARE_BITMAP(remaining_capabilities, ARM64_NCAPS);



		bitmap_complement(remaining_capabilities, boot_capabilities,

				  ARM64_NPATCHABLE);

				  ARM64_NCAPS);



		BUG_ON(all_alternatives_applied);

		__apply_alternatives(&kernel_alternatives, false,
@@ -254,9 +256,9 @@ void apply_alternatives_module(void *start, size_t length) 
		.begin	= start,

		.end	= start + length,

	};

	DECLARE_BITMAP(all_capabilities, ARM64_NPATCHABLE);

	DECLARE_BITMAP(all_capabilities, ARM64_NCAPS);



	bitmap_fill(all_capabilities, ARM64_NPATCHABLE);

	bitmap_fill(all_capabilities, ARM64_NCAPS);



	__apply_alternatives(&region, true, &all_capabilities[0]);

}