Commit 9672b0d4 authored by Jens Axboe's avatar Jens Axboe
Browse files

sbitmap: add __sbitmap_queue_get_batch() 



The block layer tag allocation batching still calls into sbitmap to get
each tag, but we can improve on that. Add __sbitmap_queue_get_batch(),
which returns a mask of tags all at once, along with an offset for
those tags.

An example return would be 0xff, where bits 0..7 are set, with
tag_offset == 128. The valid tags in this case would be 128..135.

A batch is specific to an individual sbitmap_map, hence it cannot be
larger than that. The requested number of tags is automatically reduced
to the max that can be satisfied with a single map.

On failure, 0 is returned. Caller should fall back to single tag
allocation at that point/

Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 8971a3b7

include/linux/sbitmap.h

+13 −0
Original line number Diff line number Diff line
@@ -426,6 +426,19 @@ void sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth); 
 */

int __sbitmap_queue_get(struct sbitmap_queue *sbq);



/**

 * __sbitmap_queue_get_batch() - Try to allocate a batch of free bits

 * @sbq: Bitmap queue to allocate from.

 * @nr_tags: number of tags requested

 * @offset: offset to add to returned bits

 *

 * Return: Mask of allocated tags, 0 if none are found. Each tag allocated is

 * a bit in the mask returned, and the caller must add @offset to the value to

 * get the absolute tag value.

 */

unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags,

					unsigned int *offset);



/**

 * __sbitmap_queue_get_shallow() - Try to allocate a free bit from a &struct

 * sbitmap_queue, limiting the depth used from each word, with preemption

lib/sbitmap.c

+51 −0
Original line number Diff line number Diff line
@@ -489,6 +489,57 @@ int __sbitmap_queue_get(struct sbitmap_queue *sbq) 
}

EXPORT_SYMBOL_GPL(__sbitmap_queue_get);



unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags,

					unsigned int *offset)

{

	struct sbitmap *sb = &sbq->sb;

	unsigned int hint, depth;

	unsigned long index, nr;

	int i;



	if (unlikely(sb->round_robin))

		return 0;



	depth = READ_ONCE(sb->depth);

	hint = update_alloc_hint_before_get(sb, depth);



	index = SB_NR_TO_INDEX(sb, hint);



	for (i = 0; i < sb->map_nr; i++) {

		struct sbitmap_word *map = &sb->map[index];

		unsigned long get_mask;



		sbitmap_deferred_clear(map);

		if (map->word == (1UL << (map->depth - 1)) - 1)

			continue;



		nr = find_first_zero_bit(&map->word, map->depth);

		if (nr + nr_tags <= map->depth) {

			atomic_long_t *ptr = (atomic_long_t *) &map->word;

			int map_tags = min_t(int, nr_tags, map->depth);

			unsigned long val, ret;



			get_mask = ((1UL << map_tags) - 1) << nr;

			do {

				val = READ_ONCE(map->word);

				ret = atomic_long_cmpxchg(ptr, val, get_mask | val);

			} while (ret != val);

			get_mask = (get_mask & ~ret) >> nr;

			if (get_mask) {

				*offset = nr + (index << sb->shift);

				update_alloc_hint_after_get(sb, depth, hint,

							*offset + map_tags - 1);

				return get_mask;

			}

		}

		/* Jump to next index. */

		if (++index >= sb->map_nr)

			index = 0;

	}



	return 0;

}



int __sbitmap_queue_get_shallow(struct sbitmap_queue *sbq,

				unsigned int shallow_depth)

{