Commit bea75b33 authored by Thomas Gleixner's avatar Thomas Gleixner Committed by Peter Zijlstra
Browse files

x86/Kconfig: Introduce function padding 



Now that all functions are 16 byte aligned, add 16 bytes of NOP
padding in front of each function. This prepares things for software
call stack tracking and kCFI/FineIBT.

This significantly increases kernel .text size, around 5.1% on a
x86_64-defconfig-ish build.

However, per the random access argument used for alignment, these 16
extra bytes are code that wouldn't be used. Performance measurements
back this up by showing no significant performance regressions.

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20220915111146.950884492@infradead.org
parent 8f7c0d8b

arch/x86/Kconfig

+19 −1
Original line number Diff line number Diff line
@@ -2446,9 +2446,27 @@ config CC_HAS_SLS 
config CC_HAS_RETURN_THUNK

	def_bool $(cc-option,-mfunction-return=thunk-extern)



config CC_HAS_ENTRY_PADDING

	def_bool $(cc-option,-fpatchable-function-entry=16,16)



config FUNCTION_PADDING_CFI

	int

	default 59 if FUNCTION_ALIGNMENT_64B

	default 27 if FUNCTION_ALIGNMENT_32B

	default 11 if FUNCTION_ALIGNMENT_16B

	default  3 if FUNCTION_ALIGNMENT_8B

	default  0



# Basically: FUNCTION_ALIGNMENT - 5*CFI_CLANG

# except Kconfig can't do arithmetic :/

config FUNCTION_PADDING_BYTES

	int

	default FUNCTION_PADDING_CFI if CFI_CLANG

	default FUNCTION_ALIGNMENT



config HAVE_CALL_THUNKS

	def_bool y

	depends on RETHUNK && OBJTOOL

	depends on CC_HAS_ENTRY_PADDING && RETHUNK && OBJTOOL



config CALL_THUNKS

	def_bool n

arch/x86/Makefile

+6 −0
Original line number Diff line number Diff line
@@ -208,6 +208,12 @@ ifdef CONFIG_SLS 
  KBUILD_CFLAGS += -mharden-sls=all

endif



ifdef CONFIG_CALL_THUNKS

PADDING_CFLAGS := -fpatchable-function-entry=$(CONFIG_FUNCTION_PADDING_BYTES),$(CONFIG_FUNCTION_PADDING_BYTES)

KBUILD_CFLAGS += $(PADDING_CFLAGS)

export PADDING_CFLAGS

endif



KBUILD_LDFLAGS += -m elf_$(UTS_MACHINE)



ifdef CONFIG_LTO_CLANG

arch/x86/entry/vdso/Makefile

+2 −1
Original line number Diff line number Diff line
@@ -95,7 +95,7 @@ ifneq ($(RETPOLINE_VDSO_CFLAGS),) 
endif

endif



$(vobjs): KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL)

$(vobjs): KBUILD_CFLAGS := $(filter-out $(PADDING_CFLAGS) $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL)

$(vobjs): KBUILD_AFLAGS += -DBUILD_VDSO



#
@@ -158,6 +158,7 @@ KBUILD_CFLAGS_32 := $(filter-out $(GCC_PLUGINS_CFLAGS),$(KBUILD_CFLAGS_32)) 
KBUILD_CFLAGS_32 := $(filter-out $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS_32))

KBUILD_CFLAGS_32 := $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS_32))

KBUILD_CFLAGS_32 := $(filter-out $(CC_FLAGS_CFI),$(KBUILD_CFLAGS_32))

KBUILD_CFLAGS_32 := $(filter-out $(PADDING_CFLAGS),$(KBUILD_CFLAGS_32))

KBUILD_CFLAGS_32 += -m32 -msoft-float -mregparm=0 -fpic

KBUILD_CFLAGS_32 += -fno-stack-protector

KBUILD_CFLAGS_32 += $(call cc-option, -foptimize-sibling-calls)

arch/x86/include/asm/linkage.h

+48 −3
Original line number Diff line number Diff line
@@ -15,8 +15,19 @@ 
#define __ALIGN		.balign CONFIG_FUNCTION_ALIGNMENT, 0x90;

#define __ALIGN_STR	__stringify(__ALIGN)



#define ASM_FUNC_ALIGN		__ALIGN_STR

#if defined(CONFIG_CALL_THUNKS) && !defined(__DISABLE_EXPORTS) && !defined(BUILD_VDSO)

#define FUNCTION_PADDING	.skip CONFIG_FUNCTION_ALIGNMENT, 0x90;

#else

#define FUNCTION_PADDING

#endif



#if (CONFIG_FUNCTION_ALIGNMENT > 8) && !defined(__DISABLE_EXPORTS) && !defined(BULID_VDSO)

# define __FUNC_ALIGN		__ALIGN; FUNCTION_PADDING

#else

# define __FUNC_ALIGN		__ALIGN

#endif



#define ASM_FUNC_ALIGN		__stringify(__FUNC_ALIGN)

#define SYM_F_ALIGN		__FUNC_ALIGN



#ifdef __ASSEMBLY__
@@ -45,11 +56,45 @@ 


#endif /* __ASSEMBLY__ */



/*

 * Depending on -fpatchable-function-entry=N,N usage (CONFIG_CALL_THUNKS) the

 * CFI symbol layout changes.

 *

 * Without CALL_THUNKS:

 *

 * 	.align	FUNCTION_ALIGNMENT

 * __cfi_##name:

 * 	.skip	FUNCTION_PADDING, 0x90

 * 	.byte   0xb8

 * 	.long	__kcfi_typeid_##name

 * name:

 *

 * With CALL_THUNKS:

 *

 * 	.align FUNCTION_ALIGNMENT

 * __cfi_##name:

 * 	.byte	0xb8

 * 	.long	__kcfi_typeid_##name

 * 	.skip	FUNCTION_PADDING, 0x90

 * name:

 *

 * In both cases the whole thing is FUNCTION_ALIGNMENT aligned and sized.

 */



#ifdef CONFIG_CALL_THUNKS

#define CFI_PRE_PADDING

#define CFI_POST_PADDING	.skip	CONFIG_FUNCTION_PADDING_BYTES, 0x90;

#else

#define CFI_PRE_PADDING		.skip	CONFIG_FUNCTION_PADDING_BYTES, 0x90;

#define CFI_POST_PADDING

#endif



#define __CFI_TYPE(name)					\

	SYM_START(__cfi_##name, SYM_L_LOCAL, SYM_A_NONE)	\

	.fill 11, 1, 0x90 ASM_NL				\

	CFI_PRE_PADDING						\

	.byte 0xb8 ASM_NL					\

	.long __kcfi_typeid_##name ASM_NL			\

	CFI_POST_PADDING					\

	SYM_FUNC_END(__cfi_##name)



/* SYM_TYPED_FUNC_START -- use for indirectly called globals, w/ CFI type */

include/linux/bpf.h

+4 −0
Original line number Diff line number Diff line
@@ -984,7 +984,11 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func 
}



#ifdef CONFIG_X86_64

#ifdef CONFIG_CALL_THUNKS

#define BPF_DISPATCHER_ATTRIBUTES __attribute__((patchable_function_entry(5+CONFIG_FUNCTION_PADDING_BYTES,CONFIG_FUNCTION_PADDING_BYTES)))

#else

#define BPF_DISPATCHER_ATTRIBUTES __attribute__((patchable_function_entry(5)))

#endif

#else

#define BPF_DISPATCHER_ATTRIBUTES

#endif