Commit f0093fb1 authored by Jason Gunthorpe's avatar Jason Gunthorpe
Browse files

RDMA/mlx5: Move mlx5_ib_cont_pages() to the creation of the mlx5_ib_mr 

For the user MR path, instead of calling this after getting the umem, call
it as part of creating the struct mlx5_ib_mr and distill its output to a
single page_shift stored inside the mr.

This avoids passing around the tuple of its output. Based on the umem and
page_shift, the output arguments can be computed using:

  count == ib_umem_num_pages(mr->umem)
  shift == mr->page_shift
  ncont == ib_umem_num_dma_blocks(mr->umem, 1 << mr->page_shift)
  order == order_base_2(ncont)

And since mr->page_shift == umem_odp->page_shift then ncont ==
ib_umem_num_dma_blocks() == ib_umem_odp_num_pages() for ODP umems.

Link: https://lore.kernel.org/r/20201026131936.1335664-5-leon@kernel.org


Signed-off-by: default avatarLeon Romanovsky <leonro@nvidia.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@nvidia.com>
parent 1c3d247e

drivers/infiniband/hw/mlx5/mem.c

+11 −0
Original line number Diff line number Diff line
@@ -57,6 +57,17 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, 
	struct scatterlist *sg;

	int entry;



	if (umem->is_odp) {

		struct ib_umem_odp *odp = to_ib_umem_odp(umem);



		*shift = odp->page_shift;

		*ncont = ib_umem_odp_num_pages(odp);

		*count = *ncont << (*shift - PAGE_SHIFT);

		if (order)

			*order = ilog2(roundup_pow_of_two(*count));

		return;

	}



	addr = addr >> PAGE_SHIFT;

	tmp = (unsigned long)addr;

	m = find_first_bit(&tmp, BITS_PER_LONG);

drivers/infiniband/hw/mlx5/mlx5_ib.h

+1 −0
Original line number Diff line number Diff line
@@ -597,6 +597,7 @@ struct mlx5_ib_mr { 
	int			max_descs;

	int			desc_size;

	int			access_mode;

	unsigned int		page_shift;

	struct mlx5_core_mkey	mmkey;

	struct ib_umem	       *umem;

	struct mlx5_shared_mr_info	*smr_info;

drivers/infiniband/hw/mlx5/mr.c

+57 −78
Original line number Diff line number Diff line
@@ -868,14 +868,11 @@ static int mr_cache_max_order(struct mlx5_ib_dev *dev) 
	return MLX5_MAX_UMR_SHIFT;

}



static int mr_umem_get(struct mlx5_ib_dev *dev, u64 start, u64 length,

		       int access_flags, struct ib_umem **umem, int *npages,

		       int *page_shift, int *ncont, int *order)

static struct ib_umem *mr_umem_get(struct mlx5_ib_dev *dev, u64 start,

				   u64 length, int access_flags)

{

	struct ib_umem *u;



	*umem = NULL;



	if (access_flags & IB_ACCESS_ON_DEMAND) {

		struct ib_umem_odp *odp;
@@ -884,39 +881,17 @@ static int mr_umem_get(struct mlx5_ib_dev *dev, u64 start, u64 length, 
		if (IS_ERR(odp)) {

			mlx5_ib_dbg(dev, "umem get failed (%ld)\n",

				    PTR_ERR(odp));

			return PTR_ERR(odp);

			return ERR_CAST(odp);

		}

		return &odp->umem;

	}



		u = &odp->umem;



		*page_shift = odp->page_shift;

		*ncont = ib_umem_odp_num_pages(odp);

		*npages = *ncont << (*page_shift - PAGE_SHIFT);

		if (order)

			*order = ilog2(roundup_pow_of_two(*ncont));

	} else {

	u = ib_umem_get(&dev->ib_dev, start, length, access_flags);

	if (IS_ERR(u)) {

		mlx5_ib_dbg(dev, "umem get failed (%ld)\n", PTR_ERR(u));

			return PTR_ERR(u);

		return u;

	}



		mlx5_ib_cont_pages(u, start, MLX5_MKEY_PAGE_SHIFT_MASK, npages,

				   page_shift, ncont, order);

	}



	if (!*npages) {

		mlx5_ib_warn(dev, "avoid zero region\n");

		ib_umem_release(u);

		return -EINVAL;

	}



	*umem = u;



	mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n",

		    *npages, *ncont, *order, *page_shift);



	return 0;

	return u;

}



static void mlx5_ib_umr_done(struct ib_cq *cq, struct ib_wc *wc)
@@ -975,15 +950,21 @@ static struct mlx5_cache_ent *mr_cache_ent_from_order(struct mlx5_ib_dev *dev, 
	return &cache->ent[order];

}



static struct mlx5_ib_mr *

alloc_mr_from_cache(struct ib_pd *pd, struct ib_umem *umem, u64 virt_addr,

		    u64 len, int npages, int page_shift, unsigned int order,

static struct mlx5_ib_mr *alloc_mr_from_cache(struct ib_pd *pd,

					      struct ib_umem *umem, u64 iova,

					      int access_flags)

{

	struct mlx5_ib_dev *dev = to_mdev(pd->device);

	struct mlx5_cache_ent *ent = mr_cache_ent_from_order(dev, order);

	struct mlx5_cache_ent *ent;

	struct mlx5_ib_mr *mr;

	int npages;

	int page_shift;

	int ncont;

	int order;



	mlx5_ib_cont_pages(umem, iova, MLX5_MKEY_PAGE_SHIFT_MASK, &npages,

			   &page_shift, &ncont, &order);

	ent = mr_cache_ent_from_order(dev, order);

	if (!ent)

		return ERR_PTR(-E2BIG);
@@ -1002,9 +983,10 @@ alloc_mr_from_cache(struct ib_pd *pd, struct ib_umem *umem, u64 virt_addr, 
	mr->umem = umem;

	mr->access_flags = access_flags;

	mr->desc_size = sizeof(struct mlx5_mtt);

	mr->mmkey.iova = virt_addr;

	mr->mmkey.size = len;

	mr->mmkey.iova = iova;

	mr->mmkey.size = umem->length;

	mr->mmkey.pd = to_mpd(pd)->pdn;

	mr->page_shift = page_shift;



	return mr;

}
@@ -1163,14 +1145,15 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, 
 * Else, the given ibmr will be used.

 */

static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,

				     u64 virt_addr, u64 length,

				     struct ib_umem *umem, int npages,

				     int page_shift, int access_flags,

				     bool populate)

				     struct ib_umem *umem, u64 iova,

				     int access_flags, bool populate)

{

	struct mlx5_ib_dev *dev = to_mdev(pd->device);

	struct mlx5_ib_mr *mr;

	int page_shift;

	__be64 *pas;

	int npages;

	int ncont;

	void *mkc;

	int inlen;

	u32 *in;
@@ -1181,6 +1164,10 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd, 
	if (!mr)

		return ERR_PTR(-ENOMEM);



	mlx5_ib_cont_pages(umem, iova, MLX5_MKEY_PAGE_SHIFT_MASK, &npages,

			   &page_shift, &ncont, NULL);



	mr->page_shift = page_shift;

	mr->ibmr.pd = pd;

	mr->access_flags = access_flags;
@@ -1207,20 +1194,20 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd, 
	MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap));



	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);

	set_mkc_access_pd_addr_fields(mkc, access_flags, virt_addr,

	set_mkc_access_pd_addr_fields(mkc, access_flags, iova,

				      populate ? pd : dev->umrc.pd);

	MLX5_SET(mkc, mkc, free, !populate);

	MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);

	MLX5_SET(mkc, mkc, umr_en, 1);



	MLX5_SET64(mkc, mkc, len, length);

	MLX5_SET64(mkc, mkc, len, umem->length);

	MLX5_SET(mkc, mkc, bsf_octword_size, 0);

	MLX5_SET(mkc, mkc, translations_octword_size,

		 get_octo_len(virt_addr, length, page_shift));

		 get_octo_len(iova, umem->length, page_shift));

	MLX5_SET(mkc, mkc, log_page_size, page_shift);

	if (populate) {

		MLX5_SET(create_mkey_in, in, translations_octword_actual_size,

			 get_octo_len(virt_addr, length, page_shift));

			 get_octo_len(iova, umem->length, page_shift));

	}



	err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
@@ -1358,10 +1345,6 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 
	struct mlx5_ib_mr *mr = NULL;

	bool xlt_with_umr;

	struct ib_umem *umem;

	int page_shift;

	int npages;

	int ncont;

	int order;

	int err;



	if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM))
@@ -1389,23 +1372,20 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 
		return &mr->ibmr;

	}



	err = mr_umem_get(dev, start, length, access_flags, &umem,

			  &npages, &page_shift, &ncont, &order);



	if (err < 0)

		return ERR_PTR(err);

	umem = mr_umem_get(dev, start, length, access_flags);

	if (IS_ERR(umem))

		return ERR_CAST(umem);



	if (xlt_with_umr) {

		mr = alloc_mr_from_cache(pd, umem, virt_addr, length, ncont,

					 page_shift, order, access_flags);

		mr = alloc_mr_from_cache(pd, umem, virt_addr, access_flags);

		if (IS_ERR(mr))

			mr = NULL;

	}



	if (!mr) {

		mutex_lock(&dev->slow_path_mutex);

		mr = reg_create(NULL, pd, virt_addr, length, umem, ncont,

				page_shift, access_flags, !xlt_with_umr);

		mr = reg_create(NULL, pd, umem, virt_addr, access_flags,

				!xlt_with_umr);

		mutex_unlock(&dev->slow_path_mutex);

	}
@@ -1428,8 +1408,10 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 
		 */

		int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE;



		err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift,

					 update_xlt_flags);

		err = mlx5_ib_update_xlt(

			mr, 0,

			ib_umem_num_dma_blocks(umem, 1UL << mr->page_shift),

			mr->page_shift, update_xlt_flags);

		if (err) {

			dereg_mr(dev, mr);

			return ERR_PTR(err);
@@ -1519,11 +1501,7 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, 
	int access_flags = flags & IB_MR_REREG_ACCESS ?

			    new_access_flags :

			    mr->access_flags;

	int page_shift = 0;

	int upd_flags = 0;

	int npages = 0;

	int ncont = 0;

	int order = 0;

	u64 addr, len;

	int err;
@@ -1553,12 +1531,12 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, 
		atomic_sub(ib_umem_num_pages(mr->umem),

			   &dev->mdev->priv.reg_pages);

		ib_umem_release(mr->umem);

		mr->umem = mr_umem_get(dev, addr, len, access_flags);

		if (IS_ERR(mr->umem)) {

			err = PTR_ERR(mr->umem);

			mr->umem = NULL;



		err = mr_umem_get(dev, addr, len, access_flags, &mr->umem,

				  &npages, &page_shift, &ncont, &order);

		if (err)

			goto err;

		}

		atomic_add(ib_umem_num_pages(mr->umem),

			   &dev->mdev->priv.reg_pages);

	}
@@ -1577,9 +1555,7 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, 
		if (err)

			goto err;



		mr = reg_create(ib_mr, pd, addr, len, mr->umem, ncont,

				page_shift, access_flags, true);



		mr = reg_create(ib_mr, pd, mr->umem, addr, access_flags, true);

		if (IS_ERR(mr)) {

			err = PTR_ERR(mr);

			mr = to_mmr(ib_mr);
@@ -1601,8 +1577,11 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, 
				upd_flags |= MLX5_IB_UPD_XLT_PD;

			if (flags & IB_MR_REREG_ACCESS)

				upd_flags |= MLX5_IB_UPD_XLT_ACCESS;

			err = mlx5_ib_update_xlt(mr, 0, npages, page_shift,

						 upd_flags);

			err = mlx5_ib_update_xlt(

				mr, 0,

				ib_umem_num_dma_blocks(mr->umem,

						       1UL << mr->page_shift),

				mr->page_shift, upd_flags);

		} else {

			err = rereg_umr(pd, mr, access_flags, flags);

		}