Commit f68edc92 authored by Yury Norov's avatar Yury Norov
Browse files

lib: add find_first_and_bit() 



Currently find_first_and_bit() is an alias to find_next_and_bit(). However,
it is widely used in cpumask, so it worth to optimize it. This patch adds
its own implementation for find_first_and_bit().

On x86_64 find_bit_benchmark says:

Before (#define find_first_and_bit(...) find_next_and_bit(..., 0):
Start testing find_bit() with random-filled bitmap
[  140.291468] find_first_and_bit:           46890919 ns,  32671 iterations
Start testing find_bit() with sparse bitmap
[  140.295028] find_first_and_bit:               7103 ns,      1 iterations

After:
Start testing find_bit() with random-filled bitmap
[  162.574907] find_first_and_bit:           25045813 ns,  32846 iterations
Start testing find_bit() with sparse bitmap
[  162.578458] find_first_and_bit:               4900 ns,      1 iterations

(Thanks to Alexey Klimov for thorough testing.)

Signed-off-by: default avatarYury Norov <yury.norov@gmail.com>
Tested-by: default avatarWolfram Sang <wsa+renesas@sang-engineering.com>
Tested-by: default avatarAlexey Klimov <aklimov@redhat.com>
parent c126a53c

include/linux/find.h

+27 −0
Original line number Diff line number Diff line
@@ -12,6 +12,8 @@ extern unsigned long _find_next_bit(const unsigned long *addr1, 
		const unsigned long *addr2, unsigned long nbits,

		unsigned long start, unsigned long invert, unsigned long le);

extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long size);

extern unsigned long _find_first_and_bit(const unsigned long *addr1,

					 const unsigned long *addr2, unsigned long size);

extern unsigned long _find_first_zero_bit(const unsigned long *addr, unsigned long size);

extern unsigned long _find_last_bit(const unsigned long *addr, unsigned long size);
@@ -123,6 +125,31 @@ unsigned long find_first_bit(const unsigned long *addr, unsigned long size) 
}

#endif



#ifndef find_first_and_bit

/**

 * find_first_and_bit - find the first set bit in both memory regions

 * @addr1: The first address to base the search on

 * @addr2: The second address to base the search on

 * @size: The bitmap size in bits

 *

 * Returns the bit number for the next set bit

 * If no bits are set, returns @size.

 */

static inline

unsigned long find_first_and_bit(const unsigned long *addr1,

				 const unsigned long *addr2,

				 unsigned long size)

{

	if (small_const_nbits(size)) {

		unsigned long val = *addr1 & *addr2 & GENMASK(size - 1, 0);



		return val ? __ffs(val) : size;

	}



	return _find_first_and_bit(addr1, addr2, size);

}

#endif



#ifndef find_first_zero_bit

/**

 * find_first_zero_bit - find the first cleared bit in a memory region

lib/find_bit.c

+21 −0
Original line number Diff line number Diff line
@@ -89,6 +89,27 @@ unsigned long _find_first_bit(const unsigned long *addr, unsigned long size) 
EXPORT_SYMBOL(_find_first_bit);

#endif



#ifndef find_first_and_bit

/*

 * Find the first set bit in two memory regions.

 */

unsigned long _find_first_and_bit(const unsigned long *addr1,

				  const unsigned long *addr2,

				  unsigned long size)

{

	unsigned long idx, val;



	for (idx = 0; idx * BITS_PER_LONG < size; idx++) {

		val = addr1[idx] & addr2[idx];

		if (val)

			return min(idx * BITS_PER_LONG + __ffs(val), size);

	}



	return size;

}

EXPORT_SYMBOL(_find_first_and_bit);

#endif



#ifndef find_first_zero_bit

/*

 * Find the first cleared bit in a memory region.

lib/find_bit_benchmark.c

+21 −0
Original line number Diff line number Diff line
@@ -49,6 +49,25 @@ static int __init test_find_first_bit(void *bitmap, unsigned long len) 
	return 0;

}



static int __init test_find_first_and_bit(void *bitmap, const void *bitmap2, unsigned long len)

{

	static DECLARE_BITMAP(cp, BITMAP_LEN) __initdata;

	unsigned long i, cnt;

	ktime_t time;



	bitmap_copy(cp, bitmap, BITMAP_LEN);



	time = ktime_get();

	for (cnt = i = 0; i < len; cnt++) {

		i = find_first_and_bit(cp, bitmap2, len);

		__clear_bit(i, cp);

	}

	time = ktime_get() - time;

	pr_err("find_first_and_bit: %18llu ns, %6ld iterations\n", time, cnt);



	return 0;

}



static int __init test_find_next_bit(const void *bitmap, unsigned long len)

{

	unsigned long i, cnt;
@@ -129,6 +148,7 @@ static int __init find_bit_test(void) 
	 * traverse only part of bitmap to avoid soft lockup.

	 */

	test_find_first_bit(bitmap, BITMAP_LEN / 10);

	test_find_first_and_bit(bitmap, bitmap2, BITMAP_LEN / 2);

	test_find_next_and_bit(bitmap, bitmap2, BITMAP_LEN);



	pr_err("\nStart testing find_bit() with sparse bitmap\n");
@@ -145,6 +165,7 @@ static int __init find_bit_test(void) 
	test_find_next_zero_bit(bitmap, BITMAP_LEN);

	test_find_last_bit(bitmap, BITMAP_LEN);

	test_find_first_bit(bitmap, BITMAP_LEN);

	test_find_first_and_bit(bitmap, bitmap2, BITMAP_LEN);

	test_find_next_and_bit(bitmap, bitmap2, BITMAP_LEN);



	/*