Commit 7a800a20 authored by Christoph Hellwig's avatar Christoph Hellwig Committed by Jens Axboe
Browse files

block: use bi_max_vecs to find the bvec pool 



Instead of encoding of the bvec pool using magic bio flags, just use
a helper to find the pool based on the max_vecs value.

Signed-off-by: default avatarChristoph Hellwig <hch@lst.de>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 72b04365

block/bio-integrity.c

+4 −7
Original line number Diff line number Diff line
@@ -28,7 +28,7 @@ static void __bio_integrity_free(struct bio_set *bs, 
	if (bs && mempool_initialized(&bs->bio_integrity_pool)) {

		if (bip->bip_vec)

			bvec_free(&bs->bvec_integrity_pool, bip->bip_vec,

				  bip->bip_slab);

				  bip->bip_max_vcnt);

		mempool_free(bip, &bs->bio_integrity_pool);

	} else {

		kfree(bip);
@@ -70,14 +70,11 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, 
	memset(bip, 0, sizeof(*bip));



	if (nr_vecs > inline_vecs) {

		unsigned long idx = 0;



		bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx,

					  &bs->bvec_integrity_pool);

		bip->bip_max_vcnt = nr_vecs;

		bip->bip_vec = bvec_alloc(&bs->bvec_integrity_pool,

					  &bip->bip_max_vcnt, gfp_mask);

		if (!bip->bip_vec)

			goto err;

		bip->bip_max_vcnt = bvec_nr_vecs(idx);

		bip->bip_slab = idx;

	} else {

		bip->bip_vec = bip->bip_inline_vecs;

		bip->bip_max_vcnt = inline_vecs;

block/bio.c

+43 −61
Original line number Diff line number Diff line
@@ -36,6 +36,24 @@ static struct biovec_slab { 
	{ .nr_vecs = BIO_MAX_PAGES, .name = "biovec-max" },

};



static struct biovec_slab *biovec_slab(unsigned short nr_vecs)

{

	switch (nr_vecs) {

	/* smaller bios use inline vecs */

	case 5 ... 16:

		return &bvec_slabs[0];

	case 17 ... 64:

		return &bvec_slabs[1];

	case 65 ... 128:

		return &bvec_slabs[2];

	case 129 ... BIO_MAX_PAGES:

		return &bvec_slabs[3];

	default:

		BUG();

		return NULL;

	}

}



/*

 * fs_bio_set is the bio_set containing bio and iovec memory pools used by

 * IO code that does not need private memory pools.
@@ -131,26 +149,14 @@ static void bio_put_slab(struct bio_set *bs) 
	mutex_unlock(&bio_slab_lock);

}



unsigned int bvec_nr_vecs(unsigned short idx)

void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs)

{

	return bvec_slabs[--idx].nr_vecs;

}



void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx)

{

	if (!idx)

		return;

	idx--;

	BIO_BUG_ON(nr_vecs > BIO_MAX_PAGES);



	BIO_BUG_ON(idx >= BVEC_POOL_NR);



	if (idx == BVEC_POOL_MAX) {

	if (nr_vecs == BIO_MAX_PAGES)

		mempool_free(bv, pool);

	} else {

		struct biovec_slab *bvs = bvec_slabs + idx;



		kmem_cache_free(bvs->slab, bv);

	}

	else if (nr_vecs > BIO_INLINE_VECS)

		kmem_cache_free(biovec_slab(nr_vecs)->slab, bv);

}



/*
@@ -163,48 +169,34 @@ static inline gfp_t bvec_alloc_gfp(gfp_t gfp) 
		__GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;

}



struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,

			   mempool_t *pool)

struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs,

		gfp_t gfp_mask)

{

	struct biovec_slab *bvs = biovec_slab(*nr_vecs);



	if (WARN_ON_ONCE(!bvs))

		return NULL;



	/*

	 * see comment near bvec_array define!

	 * Upgrade the nr_vecs request to take full advantage of the allocation.

	 * We also rely on this in the bvec_free path.

	 */

	switch (nr) {

	/* smaller bios use inline vecs */

	case 5 ... 16:

		*idx = 2;

		break;

	case 17 ... 64:

		*idx = 3;

		break;

	case 65 ... 128:

		*idx = 4;

		break;

	case 129 ... BIO_MAX_PAGES:

		*idx = 5;

		break;

	default:

		return NULL;

	}

	*nr_vecs = bvs->nr_vecs;



	/*

	 * Try a slab allocation first for all smaller allocations.  If that

	 * fails and __GFP_DIRECT_RECLAIM is set retry with the mempool.

	 * The mempool is sized to handle up to BIO_MAX_PAGES entries.

	 */

	if (*idx < BVEC_POOL_MAX) {

		struct biovec_slab *bvs = bvec_slabs + *idx;

	if (*nr_vecs < BIO_MAX_PAGES) {

		struct bio_vec *bvl;



		bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask));

		if (likely(bvl) || !(gfp_mask & __GFP_DIRECT_RECLAIM)) {

			(*idx)++;

		if (likely(bvl) || !(gfp_mask & __GFP_DIRECT_RECLAIM))

			return bvl;

		}

		*idx = BVEC_POOL_MAX;

		*nr_vecs = BIO_MAX_PAGES;

	}



	(*idx)++;

	return mempool_alloc(pool, gfp_mask);

}
@@ -231,7 +223,7 @@ static void bio_free(struct bio *bio) 
	bio_uninit(bio);



	if (bs) {

		bvec_free(&bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));

		bvec_free(&bs->bvec_pool, bio->bi_io_vec, bio->bi_max_vecs);



		/*

		 * If we have front padding, adjust the bio pointer before freeing
@@ -275,12 +267,8 @@ EXPORT_SYMBOL(bio_init); 
 */

void bio_reset(struct bio *bio)

{

	unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS);



	bio_uninit(bio);



	memset(bio, 0, BIO_RESET_BYTES);

	bio->bi_flags = flags;

	atomic_set(&bio->__bi_remaining, 1);

}

EXPORT_SYMBOL(bio_reset);
@@ -453,22 +441,18 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned short nr_iovecs, 


	bio = p + bs->front_pad;

	if (nr_iovecs > BIO_INLINE_VECS) {

		unsigned long idx = 0;

		struct bio_vec *bvl = NULL;



		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);

		bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask);

		if (!bvl && gfp_mask != saved_gfp) {

			punt_bios_to_rescuer(bs);

			gfp_mask = saved_gfp;

			bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx,

					 &bs->bvec_pool);

			bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask);

		}



		if (unlikely(!bvl))

			goto err_free;



		bio_init(bio, bvl, bvec_nr_vecs(idx));

		bio->bi_flags |= idx << BVEC_POOL_OFFSET;

		bio_init(bio, bvl, nr_iovecs);

	} else if (nr_iovecs) {

		bio_init(bio, bio->bi_inline_vecs, BIO_INLINE_VECS);

	} else {
@@ -644,7 +628,7 @@ EXPORT_SYMBOL(bio_put); 
 */

void __bio_clone_fast(struct bio *bio, struct bio *bio_src)

{

	BUG_ON(bio->bi_pool && BVEC_POOL_IDX(bio));

	WARN_ON_ONCE(bio->bi_pool && bio->bi_max_vecs);



	/*

	 * most users will be overriding ->bi_bdev with a new target,
@@ -934,7 +918,7 @@ EXPORT_SYMBOL_GPL(bio_release_pages); 


static int bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter)

{

	WARN_ON_ONCE(BVEC_POOL_IDX(bio) != 0);

	WARN_ON_ONCE(bio->bi_max_vecs);



	bio->bi_vcnt = iter->nr_segs;

	bio->bi_io_vec = (struct bio_vec *)iter->bvec;
@@ -1495,7 +1479,7 @@ EXPORT_SYMBOL_GPL(bio_trim); 
 */

int biovec_init_pool(mempool_t *pool, int pool_entries)

{

	struct biovec_slab *bp = bvec_slabs + BVEC_POOL_MAX;

	struct biovec_slab *bp = bvec_slabs + ARRAY_SIZE(bvec_slabs) - 1;



	return mempool_init_slab_pool(pool, pool_entries, bp->slab);

}
@@ -1605,8 +1589,6 @@ static int __init init_bio(void) 
{

	int i;



	BUILD_BUG_ON(BIO_FLAG_LAST > BVEC_POOL_OFFSET);



	bio_integrity_init();



	for (i = 0; i < ARRAY_SIZE(bvec_slabs); i++) {

block/blk.h

+3 −3
Original line number Diff line number Diff line
@@ -56,9 +56,9 @@ void blk_free_flush_queue(struct blk_flush_queue *q); 
void blk_freeze_queue(struct request_queue *q);



#define BIO_INLINE_VECS 4

struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *);

void bvec_free(mempool_t *, struct bio_vec *, unsigned int);

unsigned int bvec_nr_vecs(unsigned short idx);

struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs,

		gfp_t gfp_mask);

void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs);



static inline bool biovec_phys_mergeable(struct request_queue *q,

		struct bio_vec *vec1, struct bio_vec *vec2)

include/linux/bio.h

+0 −1
Original line number Diff line number Diff line
@@ -329,7 +329,6 @@ struct bio_integrity_payload { 


	struct bvec_iter	bip_iter;



	unsigned short		bip_slab;	/* slab the bip came from */

	unsigned short		bip_vcnt;	/* # of integrity bio_vecs */

	unsigned short		bip_max_vcnt;	/* integrity bio_vec slots */

	unsigned short		bip_flags;	/* control flags */

include/linux/blk_types.h

+1 −28
Original line number Diff line number Diff line
@@ -227,7 +227,7 @@ struct bio { 
						 * top bits REQ_OP. Use

						 * accessors.

						 */

	unsigned short		bi_flags;	/* status, etc and bvec pool number */

	unsigned short		bi_flags;	/* BIO_* below */

	unsigned short		bi_ioprio;

	unsigned short		bi_write_hint;

	blk_status_t		bi_status;
@@ -307,33 +307,6 @@ enum { 
	BIO_FLAG_LAST

};



/* See BVEC_POOL_OFFSET below before adding new flags */



/*

 * We support 6 different bvec pools, the last one is magic in that it

 * is backed by a mempool.

 */

#define BVEC_POOL_NR		6

#define BVEC_POOL_MAX		(BVEC_POOL_NR - 1)



/*

 * Top 3 bits of bio flags indicate the pool the bvecs came from.  We add

 * 1 to the actual index so that 0 indicates that there are no bvecs to be

 * freed.

 */

#define BVEC_POOL_BITS		(3)

#define BVEC_POOL_OFFSET	(16 - BVEC_POOL_BITS)

#define BVEC_POOL_IDX(bio)	((bio)->bi_flags >> BVEC_POOL_OFFSET)

#if (1<< BVEC_POOL_BITS) < (BVEC_POOL_NR+1)

# error "BVEC_POOL_BITS is too small"

#endif



/*

 * Flags starting here get preserved by bio_reset() - this includes

 * only BVEC_POOL_IDX()

 */

#define BIO_RESET_BITS	BVEC_POOL_OFFSET



typedef __u32 __bitwise blk_mq_req_flags_t;



/*