Commit c82f2c50 authored by Mikulas Patocka's avatar Mikulas Patocka Committed by Baokun Li
Browse files

wait_on_bit: add an acquire memory barrier 

mainline inclusion
from mainline-v6.0-rc3
commit 8238b457
category: bugfix
bugzilla: https://gitee.com/openeuler/kernel/issues/IB5UKT



--------------------------------

There are several places in the kernel where wait_on_bit is not followed
by a memory barrier (for example, in drivers/md/dm-bufio.c:new_read).

On architectures with weak memory ordering, it may happen that memory
accesses that follow wait_on_bit are reordered before wait_on_bit and
they may return invalid data.

Fix this class of bugs by introducing a new function "test_bit_acquire"
that works like test_bit, but has acquire memory ordering semantics.

Signed-off-by: default avatarMikulas Patocka <mpatocka@redhat.com>
Acked-by: default avatarWill Deacon <will@kernel.org>
Cc: stable@vger.kernel.org
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>

Conflicts:
	arch/x86/include/asm/bitops.h
	include/asm-generic/bitops/non-atomic.h
	include/asm-generic/bitops/instrumented-non-atomic.h
	include/asm-generic/bitops/non-instrumented-non-atomic.h
	include/linux/bitops.h
	include/linux/buffer_head.h
	include/asm-generic/bitops/generic-non-atomic.h
	Documentation/atomic_bitops.txt
[Due to the large number of patch modifications and conflicts, patch
adaptation is performed in the original contexts]
Signed-off-by: default avatarZizhi Wo <wozizhi@huawei.com>
Signed-off-by: default avatarBaokun Li <libaokun1@huawei.com>
parent 45ee5410

Documentation/atomic_bitops.txt

+4 −6
Original line number Diff line number Diff line
@@ -58,13 +58,11 @@ Like with atomic_t, the rule of thumb is: 


 - RMW operations that have a return value are fully ordered.



 - RMW operations that are conditional are unordered on FAILURE,

   otherwise the above rules apply. In the case of test_and_{}_bit() operations,

   if the bit in memory is unchanged by the operation then it is deemed to have

   failed.

 - RMW operations that are conditional are fully ordered.



Except for a successful test_and_set_bit_lock() which has ACQUIRE semantics and

clear_bit_unlock() which has RELEASE semantics.

Except for a successful test_and_set_bit_lock() which has ACQUIRE semantics,

clear_bit_unlock() which has RELEASE semantics and test_bit_acquire which has

ACQUIRE semantics.



Since a platform only has a single means of achieving atomic operations

the same barriers as for atomic_t are used, see atomic_t.txt.

arch/x86/include/asm/bitops.h

+21 −0
Original line number Diff line number Diff line
@@ -207,6 +207,20 @@ static __always_inline bool constant_test_bit(long nr, const volatile unsigned l 
		(addr[nr >> _BITOPS_LONG_SHIFT])) != 0;

}



static __always_inline bool constant_test_bit_acquire(long nr, const volatile unsigned long *addr)

{

	bool oldbit;



	asm volatile("testb %2,%1"

		     CC_SET(nz)

		     : CC_OUT(nz) (oldbit)

		     : "m" (((unsigned char *)addr)[nr >> 3]),

		       "i" (1 << (nr & 7))

		     : "memory");



	return oldbit;

}



static __always_inline bool variable_test_bit(long nr, volatile const unsigned long *addr)

{

	bool oldbit;
@@ -224,6 +238,13 @@ static __always_inline bool variable_test_bit(long nr, volatile const unsigned l 
	 ? constant_test_bit((nr), (addr))	\

	 : variable_test_bit((nr), (addr)))



static __always_inline bool

arch_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)

{

	return __builtin_constant_p(nr) ? constant_test_bit_acquire(nr, addr) :

					  variable_test_bit(nr, addr);

}



/**

 * __ffs - find first set bit in word

 * @word: The word to search

include/asm-generic/bitops/instrumented-non-atomic.h

+12 −0
Original line number Diff line number Diff line
@@ -135,4 +135,16 @@ static inline bool test_bit(long nr, const volatile unsigned long *addr) 
	return arch_test_bit(nr, addr);

}



/**

 * test_bit_acquire - Determine, with acquire semantics, whether a bit is set

 * @nr: bit number to test

 * @addr: Address to start counting from

 */

static __always_inline bool

test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)

{

	instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));

	return arch_test_bit_acquire(nr, addr);

}



#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */

include/asm-generic/bitops/non-atomic.h

+14 −0
Original line number Diff line number Diff line
@@ -3,6 +3,7 @@ 
#define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_



#include <asm/types.h>

#include <asm/barrier.h>



/**

 * __set_bit - Set a bit in memory
@@ -106,4 +107,17 @@ static inline int test_bit(int nr, const volatile unsigned long *addr) 
	return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));

}



/**

 *  arch_test_bit_acquire - Determine, with acquire semantics, whether a bit is set

 *  @nr: bit number to test

 *  @addr: Address to start counting from

 */

static __always_inline bool

arch_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)

{

	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);

	return 1UL & (smp_load_acquire(p) >> (nr & (BITS_PER_LONG-1)));

}

#define test_bit_acquire arch_test_bit_acquire



#endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */

include/linux/wait_bit.h

+4 −4
Original line number Diff line number Diff line
@@ -71,7 +71,7 @@ static inline int 
wait_on_bit(unsigned long *word, int bit, unsigned mode)

{

	might_sleep();

	if (!test_bit(bit, word))

	if (!test_bit_acquire(bit, word))

		return 0;

	return out_of_line_wait_on_bit(word, bit,

				       bit_wait,
@@ -96,7 +96,7 @@ static inline int 
wait_on_bit_io(unsigned long *word, int bit, unsigned mode)

{

	might_sleep();

	if (!test_bit(bit, word))

	if (!test_bit_acquire(bit, word))

		return 0;

	return out_of_line_wait_on_bit(word, bit,

				       bit_wait_io,
@@ -123,7 +123,7 @@ wait_on_bit_timeout(unsigned long *word, int bit, unsigned mode, 
		    unsigned long timeout)

{

	might_sleep();

	if (!test_bit(bit, word))

	if (!test_bit_acquire(bit, word))

		return 0;

	return out_of_line_wait_on_bit_timeout(word, bit,

					       bit_wait_timeout,
@@ -151,7 +151,7 @@ wait_on_bit_action(unsigned long *word, int bit, wait_bit_action_f *action, 
		   unsigned mode)

{

	might_sleep();

	if (!test_bit(bit, word))

	if (!test_bit_acquire(bit, word))

		return 0;

	return out_of_line_wait_on_bit(word, bit, action, mode);

}