Commit 6e8484c5 authored by Max Gurtovoy's avatar Max Gurtovoy Committed by Doug Ledford
Browse files

RDMA/mlx5: set UMR wqe fence according to HCA cap 



Cache the needed umr_fence and set the wqe ctrl segmennt
accordingly.

Signed-off-by: default avatarMax Gurtovoy <maxg@mellanox.com>
Acked-by: default avatarLeon Romanovsky <leon@kernel.org>
Reviewed-by: default avatarSagi Grimberg <sagi@grimberg.me>
Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parent 1410a90a

drivers/infiniband/hw/mlx5/main.c

+14 −0
Original line number Diff line number Diff line
@@ -2979,6 +2979,18 @@ error_0: 
	return ret;

}



static u8 mlx5_get_umr_fence(u8 umr_fence_cap)

{

	switch (umr_fence_cap) {

	case MLX5_CAP_UMR_FENCE_NONE:

		return MLX5_FENCE_MODE_NONE;

	case MLX5_CAP_UMR_FENCE_SMALL:

		return MLX5_FENCE_MODE_INITIATOR_SMALL;

	default:

		return MLX5_FENCE_MODE_STRONG_ORDERING;

	}

}



static int create_dev_resources(struct mlx5_ib_resources *devr)

{

	struct ib_srq_init_attr attr;
@@ -3693,6 +3705,8 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev) 


	mlx5_ib_internal_fill_odp_caps(dev);



	dev->umr_fence = mlx5_get_umr_fence(MLX5_CAP_GEN(mdev, umr_fence));



	if (MLX5_CAP_GEN(mdev, imaicl)) {

		dev->ib_dev.alloc_mw		= mlx5_ib_alloc_mw;

		dev->ib_dev.dealloc_mw		= mlx5_ib_dealloc_mw;

drivers/infiniband/hw/mlx5/mlx5_ib.h

+2 −1
Original line number Diff line number Diff line
@@ -349,7 +349,7 @@ struct mlx5_ib_qp { 
	struct mlx5_ib_wq	rq;



	u8			sq_signal_bits;

	u8			fm_cache;

	u8			next_fence;

	struct mlx5_ib_wq	sq;



	/* serialize qp state modifications
@@ -654,6 +654,7 @@ struct mlx5_ib_dev { 
	struct mlx5_ib_port	*port;

	struct mlx5_sq_bfreg     bfreg;

	struct mlx5_sq_bfreg     fp_bfreg;

	u8				umr_fence;

};



static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)

drivers/infiniband/hw/mlx5/qp.c

+23 −36
Original line number Diff line number Diff line
@@ -3738,24 +3738,6 @@ static void dump_wqe(struct mlx5_ib_qp *qp, int idx, int size_16) 
	}

}



static u8 get_fence(u8 fence, struct ib_send_wr *wr)

{

	if (unlikely(wr->opcode == IB_WR_LOCAL_INV &&

		     wr->send_flags & IB_SEND_FENCE))

		return MLX5_FENCE_MODE_STRONG_ORDERING;



	if (unlikely(fence)) {

		if (wr->send_flags & IB_SEND_FENCE)

			return MLX5_FENCE_MODE_SMALL_AND_FENCE;

		else

			return fence;

	} else if (unlikely(wr->send_flags & IB_SEND_FENCE)) {

		return MLX5_FENCE_MODE_FENCE;

	}



	return 0;

}



static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,

		     struct mlx5_wqe_ctrl_seg **ctrl,

		     struct ib_send_wr *wr, unsigned *idx,
@@ -3784,8 +3766,7 @@ static int begin_wqe(struct mlx5_ib_qp *qp, void **seg, 
static void finish_wqe(struct mlx5_ib_qp *qp,

		       struct mlx5_wqe_ctrl_seg *ctrl,

		       u8 size, unsigned idx, u64 wr_id,

		       int nreq, u8 fence, u8 next_fence,

		       u32 mlx5_opcode)

		       int nreq, u8 fence, u32 mlx5_opcode)

{

	u8 opmod = 0;
@@ -3793,7 +3774,6 @@ static void finish_wqe(struct mlx5_ib_qp *qp, 
					     mlx5_opcode | ((u32)opmod << 24));

	ctrl->qpn_ds = cpu_to_be32(size | (qp->trans_qp.base.mqp.qpn << 8));

	ctrl->fm_ce_se |= fence;

	qp->fm_cache = next_fence;

	if (unlikely(qp->wq_sig))

		ctrl->signature = wq_sig(ctrl);
@@ -3853,7 +3833,6 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 
			goto out;

		}



		fence = qp->fm_cache;

		num_sge = wr->num_sge;

		if (unlikely(num_sge > qp->sq.max_gs)) {

			mlx5_ib_warn(dev, "\n");
@@ -3870,6 +3849,19 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 
			goto out;

		}



		if (wr->opcode == IB_WR_LOCAL_INV ||

		    wr->opcode == IB_WR_REG_MR) {

			fence = dev->umr_fence;

			next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;

		} else if (wr->send_flags & IB_SEND_FENCE) {

			if (qp->next_fence)

				fence = MLX5_FENCE_MODE_SMALL_AND_FENCE;

			else

				fence = MLX5_FENCE_MODE_FENCE;

		} else {

			fence = qp->next_fence;

		}



		switch (ibqp->qp_type) {

		case IB_QPT_XRC_INI:

			xrc = seg;
@@ -3896,7 +3888,6 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 
				goto out;



			case IB_WR_LOCAL_INV:

				next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;

				qp->sq.wr_data[idx] = IB_WR_LOCAL_INV;

				ctrl->imm = cpu_to_be32(wr->ex.invalidate_rkey);

				set_linv_wr(qp, &seg, &size);
@@ -3904,7 +3895,6 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 
				break;



			case IB_WR_REG_MR:

				next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;

				qp->sq.wr_data[idx] = IB_WR_REG_MR;

				ctrl->imm = cpu_to_be32(reg_wr(wr)->key);

				err = set_reg_wr(qp, reg_wr(wr), &seg, &size);
@@ -3927,9 +3917,8 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 
					goto out;

				}



				finish_wqe(qp, ctrl, size, idx, wr->wr_id,

					   nreq, get_fence(fence, wr),

					   next_fence, MLX5_OPCODE_UMR);

				finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,

					   fence, MLX5_OPCODE_UMR);

				/*

				 * SET_PSV WQEs are not signaled and solicited

				 * on error
@@ -3954,9 +3943,8 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 
					goto out;

				}



				finish_wqe(qp, ctrl, size, idx, wr->wr_id,

					   nreq, get_fence(fence, wr),

					   next_fence, MLX5_OPCODE_SET_PSV);

				finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,

					   fence, MLX5_OPCODE_SET_PSV);

				err = begin_wqe(qp, &seg, &ctrl, wr,

						&idx, &size, nreq);

				if (err) {
@@ -3966,7 +3954,6 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 
					goto out;

				}



				next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;

				err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->wire,

						 mr->sig->psv_wire.psv_idx, &seg,

						 &size);
@@ -3976,9 +3963,9 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 
					goto out;

				}



				finish_wqe(qp, ctrl, size, idx, wr->wr_id,

					   nreq, get_fence(fence, wr),

					   next_fence, MLX5_OPCODE_SET_PSV);

				finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,

					   fence, MLX5_OPCODE_SET_PSV);

				qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL;

				num_sge = 0;

				goto skip_psv;
@@ -4089,8 +4076,8 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 
			}

		}



		finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq,

			   get_fence(fence, wr), next_fence,

		qp->next_fence = next_fence;

		finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq, fence,

			   mlx5_ib_opcode[wr->opcode]);

skip_psv:

		if (0)