Commit cb95709e authored by Chandramohan Akula's avatar Chandramohan Akula Committed by Leon Romanovsky
Browse files

bnxt_re: Update the hw counters for resource stats 



Report the additional resource counters which enables
better debugging. Includes active RC/UD QPs,
Watermark of the resources and a count that indicates the
resize cq operations after driver load.

Signed-off-by: default avatarChandramohan Akula <chandramohan.akula@broadcom.com>
Signed-off-by: default avatarSelvin Xavier <selvin.xavier@broadcom.com>
Link: https://lore.kernel.org/r/1690383081-15033-3-git-send-email-selvin.xavier@broadcom.com


Signed-off-by: default avatarLeon Romanovsky <leon@kernel.org>
parent 063975fe

drivers/infiniband/hw/bnxt_re/hw_counters.c

+24 −0
Original line number Diff line number Diff line
@@ -61,10 +61,22 @@ static const struct rdma_stat_desc bnxt_re_stat_descs[] = { 
	[BNXT_RE_ACTIVE_PD].name		=  "active_pds",

	[BNXT_RE_ACTIVE_AH].name		=  "active_ahs",

	[BNXT_RE_ACTIVE_QP].name		=  "active_qps",

	[BNXT_RE_ACTIVE_RC_QP].name             =  "active_rc_qps",

	[BNXT_RE_ACTIVE_UD_QP].name             =  "active_ud_qps",

	[BNXT_RE_ACTIVE_SRQ].name		=  "active_srqs",

	[BNXT_RE_ACTIVE_CQ].name		=  "active_cqs",

	[BNXT_RE_ACTIVE_MR].name		=  "active_mrs",

	[BNXT_RE_ACTIVE_MW].name		=  "active_mws",

	[BNXT_RE_WATERMARK_PD].name             =  "watermark_pds",

	[BNXT_RE_WATERMARK_AH].name             =  "watermark_ahs",

	[BNXT_RE_WATERMARK_QP].name             =  "watermark_qps",

	[BNXT_RE_WATERMARK_RC_QP].name          =  "watermark_rc_qps",

	[BNXT_RE_WATERMARK_UD_QP].name          =  "watermark_ud_qps",

	[BNXT_RE_WATERMARK_SRQ].name            =  "watermark_srqs",

	[BNXT_RE_WATERMARK_CQ].name             =  "watermark_cqs",

	[BNXT_RE_WATERMARK_MR].name             =  "watermark_mrs",

	[BNXT_RE_WATERMARK_MW].name             =  "watermark_mws",

	[BNXT_RE_RESIZE_CQ_CNT].name            =  "resize_cq_cnt",

	[BNXT_RE_RX_PKTS].name		=  "rx_pkts",

	[BNXT_RE_RX_BYTES].name		=  "rx_bytes",

	[BNXT_RE_TX_PKTS].name		=  "tx_pkts",
@@ -264,12 +276,24 @@ int bnxt_re_ib_get_hw_stats(struct ib_device *ibdev, 
		return -EINVAL;



	stats->value[BNXT_RE_ACTIVE_QP] = atomic_read(&res_s->qp_count);

	stats->value[BNXT_RE_ACTIVE_RC_QP] = atomic_read(&res_s->rc_qp_count);

	stats->value[BNXT_RE_ACTIVE_UD_QP] = atomic_read(&res_s->ud_qp_count);

	stats->value[BNXT_RE_ACTIVE_SRQ] = atomic_read(&res_s->srq_count);

	stats->value[BNXT_RE_ACTIVE_CQ] = atomic_read(&res_s->cq_count);

	stats->value[BNXT_RE_ACTIVE_MR] = atomic_read(&res_s->mr_count);

	stats->value[BNXT_RE_ACTIVE_MW] = atomic_read(&res_s->mw_count);

	stats->value[BNXT_RE_ACTIVE_PD] = atomic_read(&res_s->pd_count);

	stats->value[BNXT_RE_ACTIVE_AH] = atomic_read(&res_s->ah_count);

	stats->value[BNXT_RE_WATERMARK_QP] = res_s->qp_watermark;

	stats->value[BNXT_RE_WATERMARK_RC_QP] = res_s->rc_qp_watermark;

	stats->value[BNXT_RE_WATERMARK_UD_QP] = res_s->ud_qp_watermark;

	stats->value[BNXT_RE_WATERMARK_SRQ] = res_s->srq_watermark;

	stats->value[BNXT_RE_WATERMARK_CQ] = res_s->cq_watermark;

	stats->value[BNXT_RE_WATERMARK_MR] = res_s->mr_watermark;

	stats->value[BNXT_RE_WATERMARK_MW] = res_s->mw_watermark;

	stats->value[BNXT_RE_WATERMARK_PD] = res_s->pd_watermark;

	stats->value[BNXT_RE_WATERMARK_AH] = res_s->ah_watermark;

	stats->value[BNXT_RE_RESIZE_CQ_CNT] = atomic_read(&res_s->resize_count);



	if (hw_stats) {

		stats->value[BNXT_RE_RECOVERABLE_ERRORS] =

drivers/infiniband/hw/bnxt_re/hw_counters.h

+24 −0
Original line number Diff line number Diff line
@@ -44,10 +44,22 @@ enum bnxt_re_hw_stats { 
	BNXT_RE_ACTIVE_PD,

	BNXT_RE_ACTIVE_AH,

	BNXT_RE_ACTIVE_QP,

	BNXT_RE_ACTIVE_RC_QP,

	BNXT_RE_ACTIVE_UD_QP,

	BNXT_RE_ACTIVE_SRQ,

	BNXT_RE_ACTIVE_CQ,

	BNXT_RE_ACTIVE_MR,

	BNXT_RE_ACTIVE_MW,

	BNXT_RE_WATERMARK_PD,

	BNXT_RE_WATERMARK_AH,

	BNXT_RE_WATERMARK_QP,

	BNXT_RE_WATERMARK_RC_QP,

	BNXT_RE_WATERMARK_UD_QP,

	BNXT_RE_WATERMARK_SRQ,

	BNXT_RE_WATERMARK_CQ,

	BNXT_RE_WATERMARK_MR,

	BNXT_RE_WATERMARK_MW,

	BNXT_RE_RESIZE_CQ_CNT,

	BNXT_RE_RX_PKTS,

	BNXT_RE_RX_BYTES,

	BNXT_RE_TX_PKTS,
@@ -115,12 +127,24 @@ enum bnxt_re_hw_stats { 


struct bnxt_re_res_cntrs {

	atomic_t qp_count;

	atomic_t rc_qp_count;

	atomic_t ud_qp_count;

	atomic_t cq_count;

	atomic_t srq_count;

	atomic_t mr_count;

	atomic_t mw_count;

	atomic_t ah_count;

	atomic_t pd_count;

	atomic_t resize_count;

	u64 qp_watermark;

	u64 rc_qp_watermark;

	u64 ud_qp_watermark;

	u64 cq_watermark;

	u64 srq_watermark;

	u64 mr_watermark;

	u64 mw_watermark;

	u64 ah_watermark;

	u64 pd_watermark;

};



struct bnxt_re_rstat {

drivers/infiniband/hw/bnxt_re/ib_verbs.c

+46 −9
Original line number Diff line number Diff line
@@ -615,6 +615,7 @@ int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 
		udata, struct bnxt_re_ucontext, ib_uctx);

	struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ib_pd);

	struct bnxt_re_user_mmap_entry *entry = NULL;

	u32 active_pds;

	int rc = 0;



	pd->rdev = rdev;
@@ -665,7 +666,9 @@ int bnxt_re_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 
		if (bnxt_re_create_fence_mr(pd))

			ibdev_warn(&rdev->ibdev,

				   "Failed to create Fence-MR\n");

	atomic_inc(&rdev->stats.res.pd_count);

	active_pds = atomic_inc_return(&rdev->stats.res.pd_count);

	if (active_pds > rdev->stats.res.pd_watermark)

		rdev->stats.res.pd_watermark = active_pds;



	return 0;

dbfail:
@@ -725,6 +728,7 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr, 
	const struct ib_gid_attr *sgid_attr;

	struct bnxt_re_gid_ctx *ctx;

	struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);

	u32 active_ahs;

	u8 nw_type;

	int rc;
@@ -777,7 +781,9 @@ int bnxt_re_create_ah(struct ib_ah *ib_ah, struct rdma_ah_init_attr *init_attr, 
		wmb(); /* make sure cache is updated. */

		spin_unlock_irqrestore(&uctx->sh_lock, flag);

	}

	atomic_inc(&rdev->stats.res.ah_count);

	active_ahs = atomic_inc_return(&rdev->stats.res.ah_count);

	if (active_ahs > rdev->stats.res.ah_watermark)

		rdev->stats.res.ah_watermark = active_ahs;



	return 0;

}
@@ -1487,6 +1493,7 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, 
	struct bnxt_re_dev *rdev = pd->rdev;

	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;

	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);

	u32 active_qps;

	int rc;



	rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr);
@@ -1535,7 +1542,18 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr, 
	mutex_lock(&rdev->qp_lock);

	list_add_tail(&qp->list, &rdev->qp_list);

	mutex_unlock(&rdev->qp_lock);

	atomic_inc(&rdev->stats.res.qp_count);

	active_qps = atomic_inc_return(&rdev->stats.res.qp_count);

	if (active_qps > rdev->stats.res.qp_watermark)

		rdev->stats.res.qp_watermark = active_qps;

	if (qp_init_attr->qp_type == IB_QPT_RC) {

		active_qps = atomic_inc_return(&rdev->stats.res.rc_qp_count);

		if (active_qps > rdev->stats.res.rc_qp_watermark)

			rdev->stats.res.rc_qp_watermark = active_qps;

	} else if (qp_init_attr->qp_type == IB_QPT_UD) {

		active_qps = atomic_inc_return(&rdev->stats.res.ud_qp_count);

		if (active_qps > rdev->stats.res.ud_qp_watermark)

			rdev->stats.res.ud_qp_watermark = active_qps;

	}



	return 0;

qp_destroy:
@@ -1686,6 +1704,7 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq, 
	struct bnxt_re_srq *srq;

	struct bnxt_re_pd *pd;

	struct ib_pd *ib_pd;

	u32 active_srqs;

	int rc, entries;



	ib_pd = ib_srq->pd;
@@ -1750,7 +1769,9 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq, 
	}

	if (nq)

		nq->budget++;

	atomic_inc(&rdev->stats.res.srq_count);

	active_srqs = atomic_inc_return(&rdev->stats.res.srq_count);

	if (active_srqs > rdev->stats.res.srq_watermark)

		rdev->stats.res.srq_watermark = active_srqs;

	spin_lock_init(&srq->lock);



	return 0;
@@ -2892,6 +2913,7 @@ int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 
	int cqe = attr->cqe;

	struct bnxt_qplib_nq *nq = NULL;

	unsigned int nq_alloc_cnt;

	u32 active_cqs;



	if (attr->flags)

		return -EOPNOTSUPP;
@@ -2960,7 +2982,9 @@ int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 
	cq->cq_period = cq->qplib_cq.period;

	nq->budget++;



	atomic_inc(&rdev->stats.res.cq_count);

	active_cqs = atomic_inc_return(&rdev->stats.res.cq_count);

	if (active_cqs > rdev->stats.res.cq_watermark)

		rdev->stats.res.cq_watermark = active_cqs;

	spin_lock_init(&cq->cq_lock);



	if (udata) {
@@ -3073,6 +3097,7 @@ int bnxt_re_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata) 
	}



	cq->ib_cq.cqe = cq->resize_cqe;

	atomic_inc(&rdev->stats.res.resize_count);



	return 0;
@@ -3758,6 +3783,7 @@ struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) 
	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);

	struct bnxt_re_dev *rdev = pd->rdev;

	struct bnxt_re_mr *mr;

	u32 active_mrs;

	int rc;



	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
@@ -3785,7 +3811,9 @@ struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) 
	if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |

			       IB_ACCESS_REMOTE_ATOMIC))

		mr->ib_mr.rkey = mr->ib_mr.lkey;

	atomic_inc(&rdev->stats.res.mr_count);

	active_mrs = atomic_inc_return(&rdev->stats.res.mr_count);

	if (active_mrs > rdev->stats.res.mr_watermark)

		rdev->stats.res.mr_watermark = active_mrs;



	return &mr->ib_mr;
@@ -3848,6 +3876,7 @@ struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, 
	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);

	struct bnxt_re_dev *rdev = pd->rdev;

	struct bnxt_re_mr *mr = NULL;

	u32 active_mrs;

	int rc;



	if (type != IB_MR_TYPE_MEM_REG) {
@@ -3886,7 +3915,9 @@ struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, 
		goto fail_mr;

	}



	atomic_inc(&rdev->stats.res.mr_count);

	active_mrs = atomic_inc_return(&rdev->stats.res.mr_count);

	if (active_mrs > rdev->stats.res.mr_watermark)

		rdev->stats.res.mr_watermark = active_mrs;

	return &mr->ib_mr;



fail_mr:
@@ -3904,6 +3935,7 @@ struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, 
	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);

	struct bnxt_re_dev *rdev = pd->rdev;

	struct bnxt_re_mw *mw;

	u32 active_mws;

	int rc;



	mw = kzalloc(sizeof(*mw), GFP_KERNEL);
@@ -3922,7 +3954,9 @@ struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, 
	}

	mw->ib_mw.rkey = mw->qplib_mw.rkey;



	atomic_inc(&rdev->stats.res.mw_count);

	active_mws = atomic_inc_return(&rdev->stats.res.mw_count);

	if (active_mws > rdev->stats.res.mw_watermark)

		rdev->stats.res.mw_watermark = active_mws;

	return &mw->ib_mw;



fail:
@@ -3958,6 +3992,7 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, 
	struct ib_umem *umem;

	unsigned long page_size;

	int umem_pgs, rc;

	u32 active_mrs;



	if (length > BNXT_RE_MAX_MR_SIZE) {

		ibdev_err(&rdev->ibdev, "MR Size: %lld > Max supported:%lld\n",
@@ -4010,7 +4045,9 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, 


	mr->ib_mr.lkey = mr->qplib_mr.lkey;

	mr->ib_mr.rkey = mr->qplib_mr.lkey;

	atomic_inc(&rdev->stats.res.mr_count);

	active_mrs = atomic_inc_return(&rdev->stats.res.mr_count);

	if (active_mrs > rdev->stats.res.mr_watermark)

		rdev->stats.res.mr_watermark = active_mrs;



	return &mr->ib_mr;

free_umem: