Commit e506ea45 authored by Will Deacon's avatar Will Deacon
Browse files

compiler.h: Split {READ,WRITE}_ONCE definitions out into rwonce.h 



In preparation for allowing architectures to define their own
implementation of the READ_ONCE() macro, move the generic
{READ,WRITE}_ONCE() definitions out of the unwieldy 'linux/compiler.h'
file and into a new 'rwonce.h' header under 'asm-generic'.

Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: default avatarPaul E. McKenney <paulmck@kernel.org>
Signed-off-by: default avatarWill Deacon <will@kernel.org>
parent f143c11b

include/asm-generic/Kbuild

+1 −0
Original line number Diff line number Diff line
@@ -45,6 +45,7 @@ mandatory-y += pci.h 
mandatory-y += percpu.h

mandatory-y += pgalloc.h

mandatory-y += preempt.h

mandatory-y += rwonce.h

mandatory-y += sections.h

mandatory-y += serial.h

mandatory-y += shmparam.h

include/asm-generic/barrier.h

+1 −1
Original line number Diff line number Diff line
@@ -13,7 +13,7 @@ 


#ifndef __ASSEMBLY__



#include <linux/compiler.h>

#include <asm/rwonce.h>



#ifndef nop

#define nop()	asm volatile ("nop")

include/asm-generic/rwonce.h

0 → 100644
+101 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 */

/*

 * Prevent the compiler from merging or refetching reads or writes. The

 * compiler is also forbidden from reordering successive instances of

 * READ_ONCE and WRITE_ONCE, but only when the compiler is aware of some

 * particular ordering. One way to make the compiler aware of ordering is to

 * put the two invocations of READ_ONCE or WRITE_ONCE in different C

 * statements.

 *

 * These two macros will also work on aggregate data types like structs or

 * unions.

 *

 * Their two major use cases are: (1) Mediating communication between

 * process-level code and irq/NMI handlers, all running on the same CPU,

 * and (2) Ensuring that the compiler does not fold, spindle, or otherwise

 * mutilate accesses that either do not require ordering or that interact

 * with an explicit memory barrier or atomic instruction that provides the

 * required ordering.

 */

#ifndef __ASM_GENERIC_RWONCE_H

#define __ASM_GENERIC_RWONCE_H



#ifndef __ASSEMBLY__



#include <linux/compiler_types.h>

#include <linux/kasan-checks.h>

#include <linux/kcsan-checks.h>



#include <asm/barrier.h>



/*

 * Yes, this permits 64-bit accesses on 32-bit architectures. These will

 * actually be atomic in some cases (namely Armv7 + LPAE), but for others we

 * rely on the access being split into 2x32-bit accesses for a 32-bit quantity

 * (e.g. a virtual address) and a strong prevailing wind.

 */

#define compiletime_assert_rwonce_type(t)					\

	compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long),	\

		"Unsupported access size for {READ,WRITE}_ONCE().")



/*

 * Use __READ_ONCE() instead of READ_ONCE() if you do not require any

 * atomicity or dependency ordering guarantees. Note that this may result

 * in tears!

 */

#define __READ_ONCE(x)	(*(const volatile __unqual_scalar_typeof(x) *)&(x))



#define __READ_ONCE_SCALAR(x)						\

({									\

	__unqual_scalar_typeof(x) __x = __READ_ONCE(x);			\

	smp_read_barrier_depends();					\

	(typeof(x))__x;							\

})



#define READ_ONCE(x)							\

({									\

	compiletime_assert_rwonce_type(x);				\

	__READ_ONCE_SCALAR(x);						\

})



#define __WRITE_ONCE(x, val)						\

do {									\

	*(volatile typeof(x) *)&(x) = (val);				\

} while (0)



#define WRITE_ONCE(x, val)						\

do {									\

	compiletime_assert_rwonce_type(x);				\

	__WRITE_ONCE(x, val);						\

} while (0)



static __no_sanitize_or_inline

unsigned long __read_once_word_nocheck(const void *addr)

{

	return __READ_ONCE(*(unsigned long *)addr);

}



/*

 * Use READ_ONCE_NOCHECK() instead of READ_ONCE() if you need to load a

 * word from memory atomically but without telling KASAN/KCSAN. This is

 * usually used by unwinding code when walking the stack of a running process.

 */

#define READ_ONCE_NOCHECK(x)						\

({									\

	unsigned long __x;						\

	compiletime_assert(sizeof(x) == sizeof(__x),			\

		"Unsupported access size for READ_ONCE_NOCHECK().");	\

	__x = __read_once_word_nocheck(&(x));				\

	smp_read_barrier_depends();					\

	(typeof(x))__x;							\

})



static __no_kasan_or_inline

unsigned long read_word_at_a_time(const void *addr)

{

	kasan_check_read(addr, 1);

	return *(unsigned long *)addr;

}



#endif /* __ASSEMBLY__ */

#endif	/* __ASM_GENERIC_RWONCE_H */

include/linux/compiler.h

+2 −91
Original line number Diff line number Diff line
@@ -230,28 +230,6 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, 
# define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __LINE__)

#endif



/*

 * Prevent the compiler from merging or refetching reads or writes. The

 * compiler is also forbidden from reordering successive instances of

 * READ_ONCE and WRITE_ONCE, but only when the compiler is aware of some

 * particular ordering. One way to make the compiler aware of ordering is to

 * put the two invocations of READ_ONCE or WRITE_ONCE in different C

 * statements.

 *

 * These two macros will also work on aggregate data types like structs or

 * unions.

 *

 * Their two major use cases are: (1) Mediating communication between

 * process-level code and irq/NMI handlers, all running on the same CPU,

 * and (2) Ensuring that the compiler does not fold, spindle, or otherwise

 * mutilate accesses that either do not require ordering or that interact

 * with an explicit memory barrier or atomic instruction that provides the

 * required ordering.

 */

#include <asm/barrier.h>

#include <linux/kasan-checks.h>

#include <linux/kcsan-checks.h>



/**

 * data_race - mark an expression as containing intentional data races

 *
@@ -272,65 +250,6 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, 
	__v;								\

})



/*

 * Use __READ_ONCE() instead of READ_ONCE() if you do not require any

 * atomicity or dependency ordering guarantees. Note that this may result

 * in tears!

 */

#define __READ_ONCE(x)	(*(const volatile __unqual_scalar_typeof(x) *)&(x))



#define __READ_ONCE_SCALAR(x)						\

({									\

	__unqual_scalar_typeof(x) __x = __READ_ONCE(x);			\

	smp_read_barrier_depends();					\

	(typeof(x))__x;							\

})



#define READ_ONCE(x)							\

({									\

	compiletime_assert_rwonce_type(x);				\

	__READ_ONCE_SCALAR(x);						\

})



#define __WRITE_ONCE(x, val)						\

do {									\

	*(volatile typeof(x) *)&(x) = (val);				\

} while (0)



#define WRITE_ONCE(x, val)						\

do {									\

	compiletime_assert_rwonce_type(x);				\

	__WRITE_ONCE(x, val);						\

} while (0)



static __no_sanitize_or_inline

unsigned long __read_once_word_nocheck(const void *addr)

{

	return __READ_ONCE(*(unsigned long *)addr);

}



/*

 * Use READ_ONCE_NOCHECK() instead of READ_ONCE() if you need to load a

 * word from memory atomically but without telling KASAN/KCSAN. This is

 * usually used by unwinding code when walking the stack of a running process.

 */

#define READ_ONCE_NOCHECK(x)						\

({									\

	unsigned long __x;						\

	compiletime_assert(sizeof(x) == sizeof(__x),			\

		"Unsupported access size for READ_ONCE_NOCHECK().");	\

	__x = __read_once_word_nocheck(&(x));				\

	smp_read_barrier_depends();					\

	(typeof(x))__x;							\

})



static __no_kasan_or_inline

unsigned long read_word_at_a_time(const void *addr)

{

	kasan_check_read(addr, 1);

	return *(unsigned long *)addr;

}



#endif /* __KERNEL__ */



/*
@@ -395,16 +314,6 @@ static inline void *offset_to_ptr(const int *off) 
	compiletime_assert(__native_word(t),				\

		"Need native word sized stores/loads for atomicity.")



/*

 * Yes, this permits 64-bit accesses on 32-bit architectures. These will

 * actually be atomic in some cases (namely Armv7 + LPAE), but for others we

 * rely on the access being split into 2x32-bit accesses for a 32-bit quantity

 * (e.g. a virtual address) and a strong prevailing wind.

 */

#define compiletime_assert_rwonce_type(t)					\

	compiletime_assert(__native_word(t) || sizeof(t) == sizeof(long long),	\

		"Unsupported access size for {READ,WRITE}_ONCE().")



/* &a[0] degrades to a pointer: a different type from an array */

#define __must_be_array(a)	BUILD_BUG_ON_ZERO(__same_type((a), &(a)[0]))
@@ -414,4 +323,6 @@ static inline void *offset_to_ptr(const int *off) 
 */

#define prevent_tail_call_optimization()	mb()



#include <asm/rwonce.h>



#endif /* __LINUX_COMPILER_H */