Commit 6cc18331 authored by Yury Norov's avatar Yury Norov
Browse files

lib/find_bit: add find_next{,_and}_bit_wrap 



The helper is better optimized for the worst case: in case of empty
cpumask, current code traverses 2 * size:

  next = cpumask_next_and(prev, src1p, src2p);
  if (next >= nr_cpu_ids)
  	next = cpumask_first_and(src1p, src2p);

At bitmap level we can stop earlier after checking 'size + offset' bits.

Signed-off-by: default avatarYury Norov <yury.norov@gmail.com>
parent 33e67710

include/linux/find.h

+46 −0
Original line number Original line Diff line number Diff line
@@ -290,6 +290,52 @@ unsigned long find_last_bit(const unsigned long *addr, unsigned long size) 
}

}

#endif

#endif





/**

 * find_next_and_bit_wrap - find the next set bit in both memory regions

 * @addr1: The first address to base the search on

 * @addr2: The second address to base the search on

 * @size: The bitmap size in bits

 * @offset: The bitnumber to start searching at

 *

 * Returns the bit number for the next set bit, or first set bit up to @offset

 * If no bits are set, returns @size.

 */

static inline

unsigned long find_next_and_bit_wrap(const unsigned long *addr1,

					const unsigned long *addr2,

					unsigned long size, unsigned long offset)

{

	unsigned long bit = find_next_and_bit(addr1, addr2, size, offset);



	if (bit < size)

		return bit;



	bit = find_first_and_bit(addr1, addr2, offset);

	return bit < offset ? bit : size;

}



/**

 * find_next_bit_wrap - find the next set bit in both memory regions

 * @addr: The first address to base the search on

 * @size: The bitmap size in bits

 * @offset: The bitnumber to start searching at

 *

 * Returns the bit number for the next set bit, or first set bit up to @offset

 * If no bits are set, returns @size.

 */

static inline

unsigned long find_next_bit_wrap(const unsigned long *addr,

					unsigned long size, unsigned long offset)

{

	unsigned long bit = find_next_bit(addr, size, offset);



	if (bit < size)

		return bit;



	bit = find_first_bit(addr, offset);

	return bit < offset ? bit : size;

}



/**

/**

 * find_next_clump8 - find next 8-bit clump with set bits in a memory region

 * find_next_clump8 - find next 8-bit clump with set bits in a memory region

 * @clump: location to store copy of found clump

 * @clump: location to store copy of found clump

lib/cpumask.c

+3 −9
Original line number Original line Diff line number Diff line
@@ -166,10 +166,8 @@ unsigned int cpumask_any_and_distribute(const struct cpumask *src1p, 
	/* NOTE: our first selection will skip 0. */

	/* NOTE: our first selection will skip 0. */

	prev = __this_cpu_read(distribute_cpu_mask_prev);

	prev = __this_cpu_read(distribute_cpu_mask_prev);





	next = cpumask_next_and(prev, src1p, src2p);

	next = find_next_and_bit_wrap(cpumask_bits(src1p), cpumask_bits(src2p),

	if (next >= nr_cpu_ids)

					nr_cpumask_bits, prev + 1);

		next = cpumask_first_and(src1p, src2p);



	if (next < nr_cpu_ids)

	if (next < nr_cpu_ids)

		__this_cpu_write(distribute_cpu_mask_prev, next);

		__this_cpu_write(distribute_cpu_mask_prev, next);
@@ -183,11 +181,7 @@ unsigned int cpumask_any_distribute(const struct cpumask *srcp) 




	/* NOTE: our first selection will skip 0. */

	/* NOTE: our first selection will skip 0. */

	prev = __this_cpu_read(distribute_cpu_mask_prev);

	prev = __this_cpu_read(distribute_cpu_mask_prev);



	next = find_next_bit_wrap(cpumask_bits(srcp), nr_cpumask_bits, prev + 1);

	next = cpumask_next(prev, srcp);

	if (next >= nr_cpu_ids)

		next = cpumask_first(srcp);



	if (next < nr_cpu_ids)

	if (next < nr_cpu_ids)

		__this_cpu_write(distribute_cpu_mask_prev, next);

		__this_cpu_write(distribute_cpu_mask_prev, next);