Commit e35123d8 authored by Will Deacon's avatar Will Deacon
Browse files

arm64: lto: Strengthen READ_ONCE() to acquire when CONFIG_LTO=y 



When building with LTO, there is an increased risk of the compiler
converting an address dependency headed by a READ_ONCE() invocation
into a control dependency and consequently allowing for harmful
reordering by the CPU.

Ensure that such transformations are harmless by overriding the generic
READ_ONCE() definition with one that provides acquire semantics when
building with LTO.

Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: default avatarMark Rutland <mark.rutland@arm.com>
Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent 5af76fb4

arch/arm64/include/asm/rwonce.h

0 → 100644
+73 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

/*

 * Copyright (C) 2020 Google LLC.

 */

#ifndef __ASM_RWONCE_H

#define __ASM_RWONCE_H



#ifdef CONFIG_LTO



#include <linux/compiler_types.h>

#include <asm/alternative-macros.h>



#ifndef BUILD_VDSO



#ifdef CONFIG_AS_HAS_LDAPR

#define __LOAD_RCPC(sfx, regs...)					\

	ALTERNATIVE(							\

		"ldar"	#sfx "\t" #regs,				\

		".arch_extension rcpc\n"				\

		"ldapr"	#sfx "\t" #regs,				\

	ARM64_HAS_LDAPR)

#else

#define __LOAD_RCPC(sfx, regs...)	"ldar" #sfx "\t" #regs

#endif /* CONFIG_AS_HAS_LDAPR */



/*

 * When building with LTO, there is an increased risk of the compiler

 * converting an address dependency headed by a READ_ONCE() invocation

 * into a control dependency and consequently allowing for harmful

 * reordering by the CPU.

 *

 * Ensure that such transformations are harmless by overriding the generic

 * READ_ONCE() definition with one that provides RCpc acquire semantics

 * when building with LTO.

 */

#define __READ_ONCE(x)							\

({									\

	typeof(&(x)) __x = &(x);					\

	int atomic = 1;							\

	union { __unqual_scalar_typeof(*__x) __val; char __c[1]; } __u;	\

	switch (sizeof(x)) {						\

	case 1:								\

		asm volatile(__LOAD_RCPC(b, %w0, %1)			\

			: "=r" (*(__u8 *)__u.__c)			\

			: "Q" (*__x) : "memory");			\

		break;							\

	case 2:								\

		asm volatile(__LOAD_RCPC(h, %w0, %1)			\

			: "=r" (*(__u16 *)__u.__c)			\

			: "Q" (*__x) : "memory");			\

		break;							\

	case 4:								\

		asm volatile(__LOAD_RCPC(, %w0, %1)			\

			: "=r" (*(__u32 *)__u.__c)			\

			: "Q" (*__x) : "memory");			\

		break;							\

	case 8:								\

		asm volatile(__LOAD_RCPC(, %0, %1)			\

			: "=r" (*(__u64 *)__u.__c)			\

			: "Q" (*__x) : "memory");			\

		break;							\

	default:							\

		atomic = 0;						\

	}								\

	atomic ? (typeof(*__x))__u.__val : (*(volatile typeof(__x))__x);\

})



#endif	/* !BUILD_VDSO */

#endif	/* CONFIG_LTO */



#include <asm-generic/rwonce.h>



#endif	/* __ASM_RWONCE_H */

arch/arm64/kernel/vdso/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -28,7 +28,7 @@ ldflags-y := -shared -nostdlib -soname=linux-vdso.so.1 --hash-style=sysv \ 
	     $(btildflags-y) -T



ccflags-y := -fno-common -fno-builtin -fno-stack-protector -ffixed-x18

ccflags-y += -DDISABLE_BRANCH_PROFILING

ccflags-y += -DDISABLE_BRANCH_PROFILING -DBUILD_VDSO



CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) $(GCC_PLUGINS_CFLAGS)

KASAN_SANITIZE			:= n

arch/arm64/kernel/vdso32/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -48,7 +48,7 @@ cc32-as-instr = $(call try-run,\ 
# As a result we set our own flags here.



# KBUILD_CPPFLAGS and NOSTDINC_FLAGS from top-level Makefile

VDSO_CPPFLAGS := -D__KERNEL__ -nostdinc -isystem $(shell $(CC_COMPAT) -print-file-name=include)

VDSO_CPPFLAGS := -DBUILD_VDSO -D__KERNEL__ -nostdinc -isystem $(shell $(CC_COMPAT) -print-file-name=include)

VDSO_CPPFLAGS += $(LINUXINCLUDE)



# Common C and assembly flags

arch/arm64/kernel/vmlinux.lds.S

+1 −1
Original line number Diff line number Diff line
@@ -201,7 +201,7 @@ SECTIONS 
		INIT_CALLS

		CON_INITCALL

		INIT_RAM_FS

		*(.init.rodata.* .init.bss)	/* from the EFI stub */

		*(.init.altinstructions .init.rodata.* .init.bss)	/* from the EFI stub */

	}

	.exit.data : {

		EXIT_DATA