block: split bio_kmalloc from bio_alloc_bioset (3175199a) · Commits · EulixOS / Software / Kernel

block/bio.c

+85 −82

Original line number	Diff line number	Diff line
		@@ -396,24 +396,22 @@ static void punt_bios_to_rescuer(struct bio_set *bs)
		* @nr_iovecs: number of iovecs to pre-allocate
		* @bs: the bio_set to allocate from.
		*
		* Description:
		* If @bs is NULL, uses kmalloc() to allocate the bio; else the allocation is
		* backed by the @bs's mempool.
		* Allocate a bio from the mempools in @bs.
		*
		* When @bs is not NULL, if %__GFP_DIRECT_RECLAIM is set then bio_alloc will
		* always be able to allocate a bio. This is due to the mempool guarantees.
		* To make this work, callers must never allocate more than 1 bio at a time
		* from this pool. Callers that need to allocate more than 1 bio must always
		* submit the previously allocated bio for IO before attempting to allocate
		* a new one. Failure to do so can cause deadlocks under memory pressure.
		* If %__GFP_DIRECT_RECLAIM is set then bio_alloc will always be able to
		* allocate a bio. This is due to the mempool guarantees. To make this work,
		* callers must never allocate more than 1 bio at a time from the general pool.
		* Callers that need to allocate more than 1 bio must always submit the
		* previously allocated bio for IO before attempting to allocate a new one.
		* Failure to do so can cause deadlocks under memory pressure.
		*
		* Note that when running under submit_bio_noacct() (i.e. any block
		* driver), bios are not submitted until after you return - see the code in
		* Note that when running under submit_bio_noacct() (i.e. any block driver),
		* bios are not submitted until after you return - see the code in
		* submit_bio_noacct() that converts recursion into iteration, to prevent
		* stack overflows.
		*
		* This would normally mean allocating multiple bios under
		* submit_bio_noacct() would be susceptible to deadlocks, but we have
		* This would normally mean allocating multiple bios under submit_bio_noacct()
		* would be susceptible to deadlocks, but we have
		* deadlock avoidance code that resubmits any blocked bios from a rescuer
		* thread.
		*
		@@ -422,52 +420,37 @@ static void punt_bios_to_rescuer(struct bio_set *bs)
		* submit_bio_noacct() should be avoided - instead, use bio_set's front_pad
		* for per bio allocations.
		*
		* RETURNS:
		* Pointer to new bio on success, NULL on failure.
		* Returns: Pointer to new bio on success, NULL on failure.
		*/
		struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,
		struct bio_set *bs)
		{
		gfp_t saved_gfp = gfp_mask;
		unsigned front_pad;
		unsigned inline_vecs;
		struct bio_vec *bvl = NULL;
		struct bio *bio;
		void *p;

		if (!bs) {
		if (nr_iovecs > UIO_MAXIOV)
		return NULL;

		p = kmalloc(struct_size(bio, bi_inline_vecs, nr_iovecs), gfp_mask);
		front_pad = 0;
		inline_vecs = nr_iovecs;
		} else {
		/* should not use nobvec bioset for nr_iovecs > 0 */
		if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) &&
		nr_iovecs > 0))
		if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && nr_iovecs > 0))
		return NULL;

		/*
		* submit_bio_noacct() converts recursion to iteration; this
		* means if we're running beneath it, any bios we allocate and
		* submit will not be submitted (and thus freed) until after we
		* return.
		* submit_bio_noacct() converts recursion to iteration; this means if
		* we're running beneath it, any bios we allocate and submit will not be
		* submitted (and thus freed) until after we return.
		*
		* This exposes us to a potential deadlock if we allocate
		* multiple bios from the same bio_set() while running
		* underneath submit_bio_noacct(). If we were to allocate
		* multiple bios (say a stacking block driver that was splitting
		* bios), we would deadlock if we exhausted the mempool's
		* reserve.
		* This exposes us to a potential deadlock if we allocate multiple bios
		* from the same bio_set() while running underneath submit_bio_noacct().
		* If we were to allocate multiple bios (say a stacking block driver
		* that was splitting bios), we would deadlock if we exhausted the
		* mempool's reserve.
		*
		* We solve this, and guarantee forward progress, with a rescuer
		* workqueue per bio_set. If we go to allocate and there are
		* bios on current->bio_list, we first try the allocation
		* without __GFP_DIRECT_RECLAIM; if that fails, we punt those
		* bios we would be blocking to the rescuer workqueue before
		* we retry with the original gfp_flags.
		* workqueue per bio_set. If we go to allocate and there are bios on
		* current->bio_list, we first try the allocation without
		* __GFP_DIRECT_RECLAIM; if that fails, we punt those bios we would be
		* blocking to the rescuer workqueue before we retry with the original
		* gfp_flags.
		*/

		if (current->bio_list &&
		(!bio_list_empty(&current->bio_list[0]) \|\|
		!bio_list_empty(&current->bio_list[1])) &&
		@@ -480,39 +463,34 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,
		gfp_mask = saved_gfp;
		p = mempool_alloc(&bs->bio_pool, gfp_mask);
		}

		front_pad = bs->front_pad;
		inline_vecs = BIO_INLINE_VECS;
		}

		if (unlikely(!p))
		return NULL;

		bio = p + front_pad;
		bio_init(bio, NULL, 0);

		if (nr_iovecs > inline_vecs) {
		bio = p + bs->front_pad;
		if (nr_iovecs > BIO_INLINE_VECS) {
		unsigned long idx = 0;
		struct bio_vec *bvl = NULL;

		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);
		if (!bvl && gfp_mask != saved_gfp) {
		punt_bios_to_rescuer(bs);
		gfp_mask = saved_gfp;
		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);
		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx,
		&bs->bvec_pool);
		}

		if (unlikely(!bvl))
		goto err_free;

		bio->bi_flags \|= idx << BVEC_POOL_OFFSET;
		bio->bi_max_vecs = bvec_nr_vecs(idx);
		bio_init(bio, bvl, bvec_nr_vecs(idx));
		} else if (nr_iovecs) {
		bvl = bio->bi_inline_vecs;
		bio->bi_max_vecs = inline_vecs;
		bio_init(bio, bio->bi_inline_vecs, BIO_INLINE_VECS);
		} else {
		bio_init(bio, NULL, 0);
		}

		bio->bi_pool = bs;
		bio->bi_io_vec = bvl;
		return bio;

		err_free:
		@@ -521,6 +499,31 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,
		}
		EXPORT_SYMBOL(bio_alloc_bioset);

		/**
		* bio_kmalloc - kmalloc a bio for I/O
		* @gfp_mask: the GFP_* mask given to the slab allocator
		* @nr_iovecs: number of iovecs to pre-allocate
		*
		* Use kmalloc to allocate and initialize a bio.
		*
		* Returns: Pointer to new bio on success, NULL on failure.
		*/
		struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs)
		{
		struct bio *bio;

		if (nr_iovecs > UIO_MAXIOV)
		return NULL;

		bio = kmalloc(struct_size(bio, bi_inline_vecs, nr_iovecs), gfp_mask);
		if (unlikely(!bio))
		return NULL;
		bio_init(bio, nr_iovecs ? bio->bi_inline_vecs : NULL, nr_iovecs);
		bio->bi_pool = NULL;
		return bio;
		}
		EXPORT_SYMBOL(bio_kmalloc);

		void zero_fill_bio_iter(struct bio *bio, struct bvec_iter start)
		{
		unsigned long flags;

include/linux/bio.h

+1 −5

Original line number	Diff line number	Diff line
		@@ -408,6 +408,7 @@ extern int biovec_init_pool(mempool_t *pool, int pool_entries);
		extern int bioset_init_from_src(struct bio_set bs, struct bio_set src);

		extern struct bio bio_alloc_bioset(gfp_t, unsigned int, struct bio_set );
		struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs);
		extern void bio_put(struct bio *);

		extern void __bio_clone_fast(struct bio , struct bio );
		@@ -420,11 +421,6 @@ static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
		return bio_alloc_bioset(gfp_mask, nr_iovecs, &fs_bio_set);
		}

		static inline struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs)
		{
		return bio_alloc_bioset(gfp_mask, nr_iovecs, NULL);
		}

		extern blk_qc_t submit_bio(struct bio *);

		extern void bio_endio(struct bio *);