Commit f9debea2 authored by Chengchang Tang's avatar Chengchang Tang Committed by 岳国风
Browse files

RDMA/hns: Fix cpu stuck by printings during reset 

driver inclusion
category: bugfix
bugzilla: https://gitee.com/openeuler/kernel/issues/I9SI44



----------------------------------------------------------------------

During the reset, issuing the cmdq command to destroy resources
such as qp, cq, and mr may fail, and the destruction failure
information is printed. When a large number of resources fail to
be destroyed, printk() causes multiple threads to compete for a
global log buffer spinlock.  As a result, the CPU performance
deteriorates and the CPU is stuck. To fix it, use a function like
printk_limit_rate() to limit the print frequency of printk().

Fixes: 9a443537 ("IB/hns: Add driver files for hns RoCE driver")
Fixes: c7bcb134 ("RDMA/hns: Add SRQ support for hip08 kernel mode")
Fixes: 626903e9 ("RDMA/hns: Add support for reporting wc as software mode")
Fixes: 6eef5242 ("RDMA/hns: Replace not intuitive function/macro names")
Fixes: 6f5f556d ("RDMA/hns: Use the reserved loopback QPs to free MR before destroying MPT")
Fixes: 357f3429 ("RDMA/hns: Simplify the state judgment code of qp")
Fixes: 926a01dc ("RDMA/hns: Add QP operations support for hip08 SoC")
Fixes: 0273952c ("RDMA/hns: Add method to detach WQE buffer")
Fixes: 9c9edf68 ("RDMA/hns: Remove redundant DFX file and DFX ops structure")
Signed-off-by: default avatarChengchang Tang <tangchengchang@huawei.com>
Signed-off-by: default avatarGuofeng Yue <yueguofeng@h-partners.com>
parent ef32fef1

drivers/infiniband/hw/hns/hns_roce_cq.c

+3 −2
Original line number Diff line number Diff line
@@ -179,8 +179,9 @@ static void free_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) 
	ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_CQC,

				      hr_cq->cqn);

	if (ret)

		dev_err(dev, "DESTROY_CQ failed (%d) for CQN %06lx\n", ret,

			hr_cq->cqn);

		dev_err_ratelimited(dev,

				    "DESTROY_CQ failed (%d) for CQN %06lx\n",

				    ret, hr_cq->cqn);

	if (ret == -EBUSY)

		hr_cq->delayed_destroy_flag = true;

drivers/infiniband/hw/hns/hns_roce_hem.c

+3 −2
Original line number Diff line number Diff line
@@ -712,7 +712,8 @@ void hns_roce_table_put(struct hns_roce_dev *hr_dev, 


	ret = hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT);

	if (ret)

		dev_warn(dev, "failed to clear HEM base address, ret = %d.\n",

		dev_warn_ratelimited(dev,

			"failed to clear HEM base address, ret = %d.\n",

			ret);



	hns_roce_free_hem(hr_dev, table->hem[i]);

drivers/infiniband/hw/hns/hns_roce_hw_v2.c

+29 −21
Original line number Diff line number Diff line
@@ -451,11 +451,13 @@ static int check_send_valid(struct hns_roce_dev *hr_dev, 
	if (unlikely(hr_qp->state == IB_QPS_RESET ||

		     hr_qp->state == IB_QPS_INIT ||

		     hr_qp->state == IB_QPS_RTR)) {

		ibdev_err(ibdev, "failed to post WQE, QP state %u!\n",

		ibdev_err_ratelimited(ibdev,

				      "failed to post WQE, QP state %u!\n",

				      hr_qp->state);

		return -EINVAL;

	} else if (unlikely(hr_dev->state >= HNS_ROCE_DEVICE_STATE_RST_DOWN)) {

		ibdev_err(ibdev, "failed to post WQE, dev state %d!\n",

		ibdev_err_ratelimited(ibdev,

				      "failed to post WQE, dev state %d!\n",

				      hr_dev->state);

		return -EIO;

	}
@@ -463,7 +465,7 @@ static int check_send_valid(struct hns_roce_dev *hr_dev, 
	if (check_dca_attach_enable(hr_qp)) {

		ret = dca_attach_qp_buf(hr_dev, hr_qp);

		if (unlikely(ret)) {

			ibdev_err(ibdev,

			ibdev_err_ratelimited(ibdev,

				  "failed to attach DCA for QP-%ld send!\n",

				  hr_qp->qpn);

			return ret;
@@ -3818,7 +3820,8 @@ static int free_mr_post_send_lp_wqe(struct hns_roce_qp *hr_qp) 


	ret = hns_roce_v2_post_send(&hr_qp->ibqp, send_wr, &bad_wr);

	if (ret) {

		ibdev_err(ibdev, "failed to post wqe for free mr, ret = %d.\n",

		ibdev_err_ratelimited(ibdev,

			"failed to post wqe for free mr, ret = %d.\n",

			ret);

		return ret;

	}
@@ -3858,7 +3861,7 @@ static void free_mr_send_cmd_to_hw(struct hns_roce_dev *hr_dev) 


		ret = free_mr_post_send_lp_wqe(hr_qp);

		if (ret) {

			ibdev_err(ibdev,

			ibdev_err_ratelimited(ibdev,

				  "failed to send wqe (qp:0x%lx) for free mr, ret = %d.\n",

				  hr_qp->qpn, ret);

			break;
@@ -3871,14 +3874,14 @@ static void free_mr_send_cmd_to_hw(struct hns_roce_dev *hr_dev) 
	while (cqe_cnt) {

		npolled = hns_roce_v2_poll_cq(&free_mr->rsv_cq->ib_cq, cqe_cnt, wc);

		if (npolled < 0) {

			ibdev_err(ibdev,

			ibdev_err_ratelimited(ibdev,

				  "failed to poll cqe for free mr, remain %d cqe.\n",

				  cqe_cnt);

			goto out;

		}



		if (time_after(jiffies, end)) {

			ibdev_err(ibdev,

			ibdev_err_ratelimited(ibdev,

				  "failed to poll cqe for free mr and timeout, remain %d cqe.\n",

				  cqe_cnt);

			goto out;
@@ -5541,7 +5544,8 @@ static int hns_roce_v2_set_abs_fields(struct ib_qp *ibqp, 
	int ret = 0;



	if (!check_qp_state(cur_state, new_state)) {

		ibdev_err(&hr_dev->ib_dev, "Illegal state for QP!\n");

		ibdev_err_ratelimited(&hr_dev->ib_dev,

				      "Illegal state for QP!\n");

		return -EINVAL;

	}
@@ -5801,7 +5805,7 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, 
	/* SW pass context to HW */

	ret = hns_roce_v2_qp_modify(hr_dev, context, qpc_mask, hr_qp);

	if (ret) {

		ibdev_err(ibdev, "failed to modify QP, ret = %d.\n", ret);

		ibdev_err_ratelimited(ibdev, "failed to modify QP, ret = %d.\n", ret);

		goto out;

	}
@@ -5986,7 +5990,9 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, 


	ret = hns_roce_v2_query_qpc(hr_dev, hr_qp->qpn, &context);

	if (ret) {

		ibdev_err(ibdev, "failed to query QPC, ret = %d.\n", ret);

		ibdev_err_ratelimited(ibdev,

				      "failed to query QPC, ret = %d.\n",

				      ret);

		ret = -EINVAL;

		goto out;

	}
@@ -5994,7 +6000,7 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, 
	state = hr_reg_read(&context, QPC_QP_ST);

	tmp_qp_state = to_ib_qp_st((enum hns_roce_v2_qp_state)state);

	if (tmp_qp_state == -1) {

		ibdev_err(ibdev, "Illegal ib_qp_state\n");

		ibdev_err_ratelimited(ibdev, "Illegal ib_qp_state\n");

		ret = -EINVAL;

		goto out;

	}
@@ -6076,7 +6082,9 @@ static bool hns_roce_v2_chk_dca_buf_inactive(struct hns_roce_dev *hr_dev, 


	ret = hns_roce_v2_query_qpc(hr_dev, hr_qp->qpn, &context);

	if (ret) {

		ibdev_err(ibdev, "failed to query DCA QPC, ret = %d.\n", ret);

		ibdev_err_ratelimited(ibdev,

				      "failed to query DCA QPC, ret = %d.\n",

				      ret);

		return false;

	}
@@ -6120,7 +6128,7 @@ int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, 
		ret = hns_roce_v2_modify_qp(&hr_qp->ibqp, NULL, 0,

					    hr_qp->state, IB_QPS_RESET, udata);

		if (ret)

			ibdev_err(ibdev,

			ibdev_err_ratelimited(ibdev,

				  "failed to modify QP to RST, ret = %d.\n",

				  ret);

	}
@@ -6158,7 +6166,7 @@ int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 


	ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata);

	if (ret)

		ibdev_err(&hr_dev->ib_dev,

		ibdev_err_ratelimited(&hr_dev->ib_dev,

			  "failed to destroy QP, QPN = 0x%06lx, ret = %d.\n",

			  hr_qp->qpn, ret);
@@ -6457,7 +6465,7 @@ static int hns_roce_v2_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period) 
				HNS_ROCE_CMD_MODIFY_CQC, hr_cq->cqn);

	hns_roce_free_cmd_mailbox(hr_dev, mailbox);

	if (ret)

		ibdev_err(&hr_dev->ib_dev,

		ibdev_err_ratelimited(&hr_dev->ib_dev,

			  "failed to process cmd when modifying CQ, ret = %d.\n",

			  ret);
@@ -6483,7 +6491,7 @@ static int hns_roce_v2_query_cqc(struct hns_roce_dev *hr_dev, u32 cqn, 
	ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma,

				HNS_ROCE_CMD_QUERY_CQC, cqn);

	if (ret) {

		ibdev_err(&hr_dev->ib_dev,

		ibdev_err_ratelimited(&hr_dev->ib_dev,

			"failed to process cmd when querying CQ, ret = %d.\n",

			ret);

		goto err_mailbox;

drivers/infiniband/hw/hns/hns_roce_mr.c

+1 −1
Original line number Diff line number Diff line
@@ -151,7 +151,7 @@ static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr 
					      key_to_hw_index(mr->key) &

					      (hr_dev->caps.num_mtpts - 1));

		if (ret)

			ibdev_warn(ibdev, "failed to destroy mpt, ret = %d.\n",

			ibdev_warn_ratelimited(ibdev, "failed to destroy mpt, ret = %d.\n",

				   ret);

		if (ret == -EBUSY)

			mr->delayed_destroy_flag = true;

drivers/infiniband/hw/hns/hns_roce_srq.c

+3 −2
Original line number Diff line number Diff line
@@ -162,7 +162,8 @@ static void free_srqc(struct hns_roce_dev *hr_dev, struct hns_roce_srq *srq) 
	ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_SRQ,

				      srq->srqn);

	if (ret)

		dev_err(hr_dev->dev, "DESTROY_SRQ failed (%d) for SRQN %06lx\n",

		dev_err_ratelimited(hr_dev->dev,

				    "DESTROY_SRQ failed (%d) for SRQN %06lx\n",

				    ret, srq->srqn);

	if (ret == -EBUSY)

		srq->delayed_destroy_flag = true;