Commit 8238b457 authored by Mikulas Patocka's avatar Mikulas Patocka Committed by Linus Torvalds
Browse files

wait_on_bit: add an acquire memory barrier 



There are several places in the kernel where wait_on_bit is not followed
by a memory barrier (for example, in drivers/md/dm-bufio.c:new_read).

On architectures with weak memory ordering, it may happen that memory
accesses that follow wait_on_bit are reordered before wait_on_bit and
they may return invalid data.

Fix this class of bugs by introducing a new function "test_bit_acquire"
that works like test_bit, but has acquire memory ordering semantics.

Signed-off-by: default avatarMikulas Patocka <mpatocka@redhat.com>
Acked-by: default avatarWill Deacon <will@kernel.org>
Cc: stable@vger.kernel.org
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 4c612826

Documentation/atomic_bitops.txt

+4 −6
Original line number Diff line number Diff line
@@ -58,13 +58,11 @@ Like with atomic_t, the rule of thumb is: 


 - RMW operations that have a return value are fully ordered.



 - RMW operations that are conditional are unordered on FAILURE,

   otherwise the above rules apply. In the case of test_and_set_bit_lock(),

   if the bit in memory is unchanged by the operation then it is deemed to have

   failed.

 - RMW operations that are conditional are fully ordered.



Except for a successful test_and_set_bit_lock() which has ACQUIRE semantics and

clear_bit_unlock() which has RELEASE semantics.

Except for a successful test_and_set_bit_lock() which has ACQUIRE semantics,

clear_bit_unlock() which has RELEASE semantics and test_bit_acquire which has

ACQUIRE semantics.



Since a platform only has a single means of achieving atomic operations

the same barriers as for atomic_t are used, see atomic_t.txt.

arch/x86/include/asm/bitops.h

+21 −0
Original line number Diff line number Diff line
@@ -207,6 +207,20 @@ static __always_inline bool constant_test_bit(long nr, const volatile unsigned l 
		(addr[nr >> _BITOPS_LONG_SHIFT])) != 0;

}



static __always_inline bool constant_test_bit_acquire(long nr, const volatile unsigned long *addr)

{

	bool oldbit;



	asm volatile("testb %2,%1"

		     CC_SET(nz)

		     : CC_OUT(nz) (oldbit)

		     : "m" (((unsigned char *)addr)[nr >> 3]),

		       "i" (1 << (nr & 7))

		     :"memory");



	return oldbit;

}



static __always_inline bool variable_test_bit(long nr, volatile const unsigned long *addr)

{

	bool oldbit;
@@ -226,6 +240,13 @@ arch_test_bit(unsigned long nr, const volatile unsigned long *addr) 
					  variable_test_bit(nr, addr);

}



static __always_inline bool

arch_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)

{

	return __builtin_constant_p(nr) ? constant_test_bit_acquire(nr, addr) :

					  variable_test_bit(nr, addr);

}



/**

 * __ffs - find first set bit in word

 * @word: The word to search

include/asm-generic/bitops/generic-non-atomic.h

+14 −0
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ 
#define __ASM_GENERIC_BITOPS_GENERIC_NON_ATOMIC_H



#include <linux/bits.h>

#include <asm/barrier.h>



#ifndef _LINUX_BITOPS_H

#error only <linux/bitops.h> can be included directly
@@ -127,6 +128,18 @@ generic_test_bit(unsigned long nr, const volatile unsigned long *addr) 
	return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));

}



/**

 * generic_test_bit_acquire - Determine, with acquire semantics, whether a bit is set

 * @nr: bit number to test

 * @addr: Address to start counting from

 */

static __always_inline bool

generic_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)

{

	unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr);

	return 1UL & (smp_load_acquire(p) >> (nr & (BITS_PER_LONG-1)));

}



/*

 * const_*() definitions provide good compile-time optimizations when

 * the passed arguments can be resolved at compile time.
@@ -137,6 +150,7 @@ generic_test_bit(unsigned long nr, const volatile unsigned long *addr) 
#define const___test_and_set_bit	generic___test_and_set_bit

#define const___test_and_clear_bit	generic___test_and_clear_bit

#define const___test_and_change_bit	generic___test_and_change_bit

#define const_test_bit_acquire		generic_test_bit_acquire



/**

 * const_test_bit - Determine whether a bit is set

include/asm-generic/bitops/instrumented-non-atomic.h

+12 −0
Original line number Diff line number Diff line
@@ -142,4 +142,16 @@ _test_bit(unsigned long nr, const volatile unsigned long *addr) 
	return arch_test_bit(nr, addr);

}



/**

 * _test_bit_acquire - Determine, with acquire semantics, whether a bit is set

 * @nr: bit number to test

 * @addr: Address to start counting from

 */

static __always_inline bool

_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)

{

	instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));

	return arch_test_bit_acquire(nr, addr);

}



#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */

include/asm-generic/bitops/non-atomic.h

+1 −0
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ 
#define arch___test_and_change_bit generic___test_and_change_bit



#define arch_test_bit generic_test_bit

#define arch_test_bit_acquire generic_test_bit_acquire



#include <asm-generic/bitops/non-instrumented-non-atomic.h>