Commit 58fb0b56 authored by Bernard Metzler's avatar Bernard Metzler Committed by Jason Gunthorpe
Browse files

RDMA/siw: Simplify QP representation 

Change siw_qp to contain ib_qp. Use rdma_is_kernel_res() on contained
ib_qp to distinguish kernel level from user level applications
resources. Apply same mechanism for kernel/user level application
detection to completion queues.

Link: https://lore.kernel.org/r/20191210161729.31598-1-bmt@zurich.ibm.com


Signed-off-by: default avatarBernard Metzler <bmt@zurich.ibm.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@mellanox.com>
parent 47688202

drivers/infiniband/sw/siw/siw.h

+7 −19
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ 
#define _SIW_H



#include <rdma/ib_verbs.h>

#include <rdma/restrack.h>

#include <linux/socket.h>

#include <linux/skbuff.h>

#include <crypto/hash.h>
@@ -209,7 +210,6 @@ struct siw_cq { 
	u32 cq_put;

	u32 cq_get;

	u32 num_cqe;

	bool kernel_verbs;

	struct rdma_user_mmap_entry *cq_entry; /* mmap info for CQE array */

	u32 id; /* For debugging only */

};
@@ -254,8 +254,8 @@ struct siw_srq { 
	u32 rq_get;

	u32 num_rqe; /* max # of wqe's allowed */

	struct rdma_user_mmap_entry *srq_entry; /* mmap info for SRQ array */

	char armed; /* inform user if limit hit */

	char kernel_verbs; /* '1' if kernel client */

	bool armed:1; /* inform user if limit hit */

	bool is_kernel_res:1; /* true if kernel client */

};



struct siw_qp_attrs {
@@ -418,13 +418,11 @@ struct siw_iwarp_tx { 
};



struct siw_qp {

	struct ib_qp base_qp;

	struct siw_device *sdev;

	struct ib_qp *ib_qp;

	struct kref ref;

	u32 qp_num;

	struct list_head devq;

	int tx_cpu;

	bool kernel_verbs;

	struct siw_qp_attrs attrs;



	struct siw_cep *cep;
@@ -472,11 +470,6 @@ struct siw_qp { 
	struct rcu_head rcu;

};



struct siw_base_qp {

	struct ib_qp base_qp;

	struct siw_qp *qp;

};



/* helper macros */

#define rx_qp(rx) container_of(rx, struct siw_qp, rx_stream)

#define tx_qp(tx) container_of(tx, struct siw_qp, tx_ctx)
@@ -572,14 +565,9 @@ static inline struct siw_ucontext *to_siw_ctx(struct ib_ucontext *base_ctx) 
	return container_of(base_ctx, struct siw_ucontext, base_ucontext);

}



static inline struct siw_base_qp *to_siw_base_qp(struct ib_qp *base_qp)

{

	return container_of(base_qp, struct siw_base_qp, base_qp);

}



static inline struct siw_qp *to_siw_qp(struct ib_qp *base_qp)

{

	return to_siw_base_qp(base_qp)->qp;

	return container_of(base_qp, struct siw_qp, base_qp);

}



static inline struct siw_cq *to_siw_cq(struct ib_cq *base_cq)
@@ -624,7 +612,7 @@ static inline struct siw_qp *siw_qp_id2obj(struct siw_device *sdev, int id) 


static inline u32 qp_id(struct siw_qp *qp)

{

	return qp->qp_num;

	return qp->base_qp.qp_num;

}



static inline void siw_qp_get(struct siw_qp *qp)

drivers/infiniband/sw/siw/siw_cq.c

+1 −1
Original line number Diff line number Diff line
@@ -65,7 +65,7 @@ int siw_reap_cqe(struct siw_cq *cq, struct ib_wc *wc) 
		 * reaped here, which do not hold a QP reference

		 * and do not qualify for memory extension verbs.

		 */

		if (likely(cq->kernel_verbs)) {

		if (likely(rdma_is_kernel_res(&cq->base_cq.res))) {

			if (cqe->flags & SIW_WQE_REM_INVAL) {

				wc->ex.invalidate_rkey = cqe->inval_stag;

				wc->wc_flags = IB_WC_WITH_INVALIDATE;

drivers/infiniband/sw/siw/siw_main.c

+1 −1
Original line number Diff line number Diff line
@@ -244,7 +244,7 @@ static struct ib_qp *siw_get_base_qp(struct ib_device *base_dev, int id) 
		 * siw_qp_id2obj() increments object reference count

		 */

		siw_qp_put(qp);

		return qp->ib_qp;

		return &qp->base_qp;

	}

	return NULL;

}

drivers/infiniband/sw/siw/siw_qp.c

+5 −8
Original line number Diff line number Diff line
@@ -1070,8 +1070,8 @@ int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes, 
			cqe->imm_data = 0;

			cqe->bytes = bytes;



			if (cq->kernel_verbs)

				cqe->base_qp = qp->ib_qp;

			if (rdma_is_kernel_res(&cq->base_cq.res))

				cqe->base_qp = &qp->base_qp;

			else

				cqe->qp_id = qp_id(qp);
@@ -1128,8 +1128,8 @@ int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes, 
			cqe->imm_data = 0;

			cqe->bytes = bytes;



			if (cq->kernel_verbs) {

				cqe->base_qp = qp->ib_qp;

			if (rdma_is_kernel_res(&cq->base_cq.res)) {

				cqe->base_qp = &qp->base_qp;

				if (inval_stag) {

					cqe_flags |= SIW_WQE_REM_INVAL;

					cqe->inval_stag = inval_stag;
@@ -1297,13 +1297,12 @@ void siw_rq_flush(struct siw_qp *qp) 


int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp)

{

	int rv = xa_alloc(&sdev->qp_xa, &qp->ib_qp->qp_num, qp, xa_limit_32b,

	int rv = xa_alloc(&sdev->qp_xa, &qp->base_qp.qp_num, qp, xa_limit_32b,

			  GFP_KERNEL);



	if (!rv) {

		kref_init(&qp->ref);

		qp->sdev = sdev;

		qp->qp_num = qp->ib_qp->qp_num;

		siw_dbg_qp(qp, "new QP\n");

	}

	return rv;
@@ -1312,7 +1311,6 @@ int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp) 
void siw_free_qp(struct kref *ref)

{

	struct siw_qp *found, *qp = container_of(ref, struct siw_qp, ref);

	struct siw_base_qp *siw_base_qp = to_siw_base_qp(qp->ib_qp);

	struct siw_device *sdev = qp->sdev;

	unsigned long flags;
@@ -1335,5 +1333,4 @@ void siw_free_qp(struct kref *ref) 
	atomic_dec(&sdev->num_qp);

	siw_dbg_qp(qp, "free QP\n");

	kfree_rcu(qp, rcu);

	kfree(siw_base_qp);

}

drivers/infiniband/sw/siw/siw_qp_rx.c

+3 −3
Original line number Diff line number Diff line
@@ -68,7 +68,7 @@ static int siw_rx_umem(struct siw_rx_stream *srx, struct siw_umem *umem, 
			return -EFAULT;

		}

		if (srx->mpa_crc_hd) {

			if (rx_qp(srx)->kernel_verbs) {

			if (rdma_is_kernel_res(&rx_qp(srx)->base_qp.res)) {

				crypto_shash_update(srx->mpa_crc_hd,

					(u8 *)(dest + pg_off), bytes);

				kunmap_atomic(dest);
@@ -388,7 +388,7 @@ static struct siw_wqe *siw_rqe_get(struct siw_qp *qp) 
				struct siw_rqe *rqe2 = &srq->recvq[off];



				if (!(rqe2->flags & SIW_WQE_VALID)) {

					srq->armed = 0;

					srq->armed = false;

					srq_event = true;

				}

			}
@@ -1264,7 +1264,7 @@ static int siw_rdmap_complete(struct siw_qp *qp, int error) 


			if (wc_status == SIW_WC_SUCCESS)

				wc_status = SIW_WC_GENERAL_ERR;

		} else if (qp->kernel_verbs &&

		} else if (rdma_is_kernel_res(&qp->base_qp.res) &&

			   rx_type(wqe) == SIW_OP_READ_LOCAL_INV) {

			/*

			 * Handle any STag invalidation request