block: use bi_max_vecs to find the bvec pool (958384da) · Commits · EulixOS / Software / Kernel

block/bio-integrity.c

+4 −7

Original line number	Diff line number	Diff line
		@@ -28,7 +28,7 @@ static void __bio_integrity_free(struct bio_set *bs,
		if (bs && mempool_initialized(&bs->bio_integrity_pool)) {
		if (bip->bip_vec)
		bvec_free(&bs->bvec_integrity_pool, bip->bip_vec,
		bip->bip_slab);
		bip->bip_max_vcnt);
		mempool_free(bip, &bs->bio_integrity_pool);
		} else {
		kfree(bip);
		@@ -70,14 +70,11 @@ struct bio_integrity_payload bio_integrity_alloc(struct bio bio,
		memset(bip, 0, sizeof(*bip));

		if (nr_vecs > inline_vecs) {
		unsigned long idx = 0;

		bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx,
		&bs->bvec_integrity_pool);
		bip->bip_max_vcnt = nr_vecs;
		bip->bip_vec = bvec_alloc(&bs->bvec_integrity_pool,
		&bip->bip_max_vcnt, gfp_mask);
		if (!bip->bip_vec)
		goto err;
		bip->bip_max_vcnt = bvec_nr_vecs(idx);
		bip->bip_slab = idx;
		} else {
		bip->bip_vec = bip->bip_inline_vecs;
		bip->bip_max_vcnt = inline_vecs;

block/bio.c

+41 −57

Original line number	Diff line number	Diff line
		@@ -35,6 +35,24 @@ static struct biovec_slab {
		{ .nr_vecs = BIO_MAX_PAGES, .name = "biovec-max" },
		};

		static struct biovec_slab *biovec_slab(unsigned short nr_vecs)
		{
		switch (nr_vecs) {
		/* smaller bios use inline vecs */
		case 5 ... 16:
		return &bvec_slabs[0];
		case 17 ... 64:
		return &bvec_slabs[1];
		case 65 ... 128:
		return &bvec_slabs[2];
		case 129 ... BIO_MAX_PAGES:
		return &bvec_slabs[3];
		default:
		BUG();
		return NULL;
		}
		}

		/*
		* fs_bio_set is the bio_set containing bio and iovec memory pools used by
		* IO code that does not need private memory pools.
		@@ -140,26 +158,14 @@ static void bio_put_slab(struct bio_set *bs)
		mutex_unlock(&bio_slab_lock);
		}

		unsigned int bvec_nr_vecs(unsigned short idx)
		void bvec_free(mempool_t pool, struct bio_vec bv, unsigned short nr_vecs)
		{
		return bvec_slabs[--idx].nr_vecs;
		}

		void bvec_free(mempool_t pool, struct bio_vec bv, unsigned int idx)
		{
		if (!idx)
		return;
		idx--;
		BIO_BUG_ON(nr_vecs > BIO_MAX_PAGES);

		BIO_BUG_ON(idx >= BVEC_POOL_NR);

		if (idx == BVEC_POOL_MAX) {
		if (nr_vecs == BIO_MAX_PAGES)
		mempool_free(bv, pool);
		} else {
		struct biovec_slab *bvs = bvec_slabs + idx;

		kmem_cache_free(bvs->slab, bv);
		}
		else if (nr_vecs > BIO_INLINE_VECS)
		kmem_cache_free(biovec_slab(nr_vecs)->slab, bv);
		}

		/*
		@@ -172,48 +178,34 @@ static inline gfp_t bvec_alloc_gfp(gfp_t gfp)
		__GFP_NOMEMALLOC \| __GFP_NORETRY \| __GFP_NOWARN;
		}

		struct bio_vec bvec_alloc(gfp_t gfp_mask, int nr, unsigned long idx,
		mempool_t *pool)
		struct bio_vec bvec_alloc(mempool_t pool, unsigned short *nr_vecs,
		gfp_t gfp_mask)
		{
		struct biovec_slab bvs = biovec_slab(nr_vecs);

		if (WARN_ON_ONCE(!bvs))
		return NULL;

		/*
		* see comment near bvec_array define!
		* Upgrade the nr_vecs request to take full advantage of the allocation.
		* We also rely on this in the bvec_free path.
		*/
		switch (nr) {
		/* smaller bios use inline vecs */
		case 5 ... 16:
		*idx = 2;
		break;
		case 17 ... 64:
		*idx = 3;
		break;
		case 65 ... 128:
		*idx = 4;
		break;
		case 129 ... BIO_MAX_PAGES:
		*idx = 5;
		break;
		default:
		return NULL;
		}
		*nr_vecs = bvs->nr_vecs;

		/*
		* Try a slab allocation first for all smaller allocations. If that
		* fails and __GFP_DIRECT_RECLAIM is set retry with the mempool.
		* The mempool is sized to handle up to BIO_MAX_PAGES entries.
		*/
		if (*idx < BVEC_POOL_MAX) {
		struct biovec_slab bvs = bvec_slabs + idx;
		if (*nr_vecs < BIO_MAX_PAGES) {
		struct bio_vec *bvl;

		bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask));
		if (likely(bvl) \|\| !(gfp_mask & __GFP_DIRECT_RECLAIM)) {
		(*idx)++;
		if (likely(bvl) \|\| !(gfp_mask & __GFP_DIRECT_RECLAIM))
		return bvl;
		}
		*idx = BVEC_POOL_MAX;
		*nr_vecs = BIO_MAX_PAGES;
		}

		(*idx)++;
		return mempool_alloc(pool, gfp_mask);
		}

		@@ -240,7 +232,7 @@ static void bio_free(struct bio *bio)
		bio_uninit(bio);

		if (bs) {
		bvec_free(&bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));
		bvec_free(&bs->bvec_pool, bio->bi_io_vec, bio->bi_max_vecs);

		/*
		* If we have front padding, adjust the bio pointer before freeing
		@@ -284,12 +276,8 @@ EXPORT_SYMBOL(bio_init);
		*/
		void bio_reset(struct bio *bio)
		{
		unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS);

		bio_uninit(bio);

		memset(bio, 0, BIO_RESET_BYTES);
		bio->bi_flags = flags;
		atomic_set(&bio->__bi_remaining, 1);
		}
		EXPORT_SYMBOL(bio_reset);
		@@ -486,19 +474,17 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned short nr_iovecs,
		bio_init(bio, NULL, 0);

		if (nr_iovecs > inline_vecs) {
		unsigned long idx = 0;

		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);
		bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask);
		if (!bvl && gfp_mask != saved_gfp) {
		punt_bios_to_rescuer(bs);
		gfp_mask = saved_gfp;
		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);
		bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask);
		}

		if (unlikely(!bvl))
		goto err_free;

		bio->bi_flags \|= idx << BVEC_POOL_OFFSET;
		} else if (nr_iovecs) {
		bvl = bio->bi_inline_vecs;
		}
		@@ -659,7 +645,7 @@ EXPORT_SYMBOL(bio_put);
		*/
		void __bio_clone_fast(struct bio bio, struct bio bio_src)
		{
		BUG_ON(bio->bi_pool && BVEC_POOL_IDX(bio));
		WARN_ON_ONCE(bio->bi_pool && bio->bi_max_vecs);

		/*
		* most users will be overriding ->bi_disk with a new target,
		@@ -1527,7 +1513,7 @@ EXPORT_SYMBOL_GPL(bio_trim);
		*/
		int biovec_init_pool(mempool_t *pool, int pool_entries)
		{
		struct biovec_slab *bp = bvec_slabs + BVEC_POOL_MAX;
		struct biovec_slab *bp = bvec_slabs + ARRAY_SIZE(bvec_slabs) - 1;

		return mempool_init_slab_pool(pool, pool_entries, bp->slab);
		}
		@@ -1639,8 +1625,6 @@ static int __init init_bio(void)
		bio_slabs = kcalloc(bio_slab_max, sizeof(struct bio_slab),
		GFP_KERNEL);

		BUILD_BUG_ON(BIO_FLAG_LAST > BVEC_POOL_OFFSET);

		if (!bio_slabs)
		panic("bio: can't allocate bios\n");

include/linux/bio.h

+3 −3

Original line number	Diff line number	Diff line
		@@ -312,7 +312,6 @@ struct bio_integrity_payload {

		struct bvec_iter bip_iter;

		unsigned short bip_slab; /* slab the bip came from */
		unsigned short bip_vcnt; /* # of integrity bio_vecs */
		unsigned short bip_max_vcnt; /* integrity bio_vec slots */
		unsigned short bip_flags; /* control flags */
		@@ -466,8 +465,9 @@ static inline void zero_fill_bio(struct bio *bio)
		zero_fill_bio_iter(bio, bio->bi_iter);
		}

		extern struct bio_vec bvec_alloc(gfp_t, int, unsigned long , mempool_t *);
		extern void bvec_free(mempool_t , struct bio_vec , unsigned int);
		extern struct bio_vec bvec_alloc(mempool_t pool, unsigned short *nr_vecs,
		gfp_t gfp_mask);
		extern void bvec_free(mempool_t pool, struct bio_vec bv, unsigned short nr_vecs);
		extern unsigned int bvec_nr_vecs(unsigned short idx);
		extern const char bio_devname(struct bio bio, char *buffer);

include/linux/blk_types.h

+1 −28

Original line number	Diff line number	Diff line
		@@ -223,7 +223,7 @@ struct bio {
		* top bits REQ_OP. Use
		* accessors.
		*/
		unsigned short bi_flags; /* status, etc and bvec pool number */
		unsigned short bi_flags; /* BIO_* below */
		unsigned short bi_ioprio;
		unsigned short bi_write_hint;
		blk_status_t bi_status;
		@@ -310,33 +310,6 @@ enum {
		BIO_FLAG_LAST
		};

		/* See BVEC_POOL_OFFSET below before adding new flags */

		/*
		* We support 6 different bvec pools, the last one is magic in that it
		* is backed by a mempool.
		*/
		#define BVEC_POOL_NR 6
		#define BVEC_POOL_MAX (BVEC_POOL_NR - 1)

		/*
		* Top 3 bits of bio flags indicate the pool the bvecs came from. We add
		* 1 to the actual index so that 0 indicates that there are no bvecs to be
		* freed.
		*/
		#define BVEC_POOL_BITS (3)
		#define BVEC_POOL_OFFSET (16 - BVEC_POOL_BITS)
		#define BVEC_POOL_IDX(bio) ((bio)->bi_flags >> BVEC_POOL_OFFSET)
		#if (1<< BVEC_POOL_BITS) < (BVEC_POOL_NR+1)
		# error "BVEC_POOL_BITS is too small"
		#endif

		/*
		* Flags starting here get preserved by bio_reset() - this includes
		* only BVEC_POOL_IDX()
		*/
		#define BIO_RESET_BITS BVEC_POOL_OFFSET

		typedef __u32 __bitwise blk_mq_req_flags_t;

		/*