Commit 801a5736 authored by Yury Norov's avatar Yury Norov
Browse files

mm/percpu: micro-optimize pcpu_is_populated() 



bitmap_next_clear_region() calls find_next_zero_bit() and find_next_bit()
sequentially to find a range of clear bits. In case of pcpu_is_populated()
there's a chance to return earlier if bitmap has all bits set.

Signed-off-by: default avatarYury Norov <yury.norov@gmail.com>
Tested-by: default avatarWolfram Sang <wsa+renesas@sang-engineering.com>
Acked-by: default avatarDennis Zhou <dennis@kernel.org>
parent 749443de

mm/percpu.c

+8 −7
Original line number Diff line number Diff line
@@ -1070,17 +1070,18 @@ static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off, 
static bool pcpu_is_populated(struct pcpu_chunk *chunk, int bit_off, int bits,

			      int *next_off)

{

	unsigned int page_start, page_end, rs, re;

	unsigned int start, end;



	page_start = PFN_DOWN(bit_off * PCPU_MIN_ALLOC_SIZE);

	page_end = PFN_UP((bit_off + bits) * PCPU_MIN_ALLOC_SIZE);

	start = PFN_DOWN(bit_off * PCPU_MIN_ALLOC_SIZE);

	end = PFN_UP((bit_off + bits) * PCPU_MIN_ALLOC_SIZE);



	rs = page_start;

	bitmap_next_clear_region(chunk->populated, &rs, &re, page_end);

	if (rs >= page_end)

	start = find_next_zero_bit(chunk->populated, end, start);

	if (start >= end)

		return true;



	*next_off = re * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE;

	end = find_next_bit(chunk->populated, end, start + 1);



	*next_off = end * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE;

	return false;

}