Commit 8b42cfab authored by Kamal Heib's avatar Kamal Heib Committed by Marcel Apfelbaum
Browse files

hw/rdma: Modify create/destroy QP to support SRQ 



Modify create/destroy QP to support shared receive queue and rearrange
the destroy_qp() code to avoid touching the QP after calling
rdma_rm_dealloc_qp().

Signed-off-by: default avatarKamal Heib <kamalheib1@gmail.com>
Message-Id: <20190403113343.26384-4-kamalheib1@gmail.com>
Reviewed-by: default avatarYuval Shaia <yuval.shaia@oracle.com>
Signed-off-by: default avatarMarcel Apfelbaum <marcel.apfelbaum@gmail.com>
parent cdc84058

hw/rdma/rdma_backend.c

+6 −3
Original line number Diff line number Diff line
@@ -794,9 +794,9 @@ void rdma_backend_destroy_cq(RdmaBackendCQ *cq) 


int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t qp_type,

                           RdmaBackendPD *pd, RdmaBackendCQ *scq,

                           RdmaBackendCQ *rcq, uint32_t max_send_wr,

                           uint32_t max_recv_wr, uint32_t max_send_sge,

                           uint32_t max_recv_sge)

                           RdmaBackendCQ *rcq, RdmaBackendSRQ *srq,

                           uint32_t max_send_wr, uint32_t max_recv_wr,

                           uint32_t max_send_sge, uint32_t max_recv_sge)

{

    struct ibv_qp_init_attr attr = {};
@@ -824,6 +824,9 @@ int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t qp_type, 
    attr.cap.max_recv_wr = max_recv_wr;

    attr.cap.max_send_sge = max_send_sge;

    attr.cap.max_recv_sge = max_recv_sge;

    if (srq) {

        attr.srq = srq->ibsrq;

    }



    qp->ibqp = ibv_create_qp(pd->ibpd, &attr);

    if (!qp->ibqp) {

hw/rdma/rdma_backend.h

+3 −3
Original line number Diff line number Diff line
@@ -89,9 +89,9 @@ void rdma_backend_poll_cq(RdmaDeviceResources *rdma_dev_res, RdmaBackendCQ *cq); 


int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t qp_type,

                           RdmaBackendPD *pd, RdmaBackendCQ *scq,

                           RdmaBackendCQ *rcq, uint32_t max_send_wr,

                           uint32_t max_recv_wr, uint32_t max_send_sge,

                           uint32_t max_recv_sge);

                           RdmaBackendCQ *rcq, RdmaBackendSRQ *srq,

                           uint32_t max_send_wr, uint32_t max_recv_wr,

                           uint32_t max_send_sge, uint32_t max_recv_sge);

int rdma_backend_qp_state_init(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,

                               uint8_t qp_type, uint32_t qkey);

int rdma_backend_qp_state_rtr(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,

hw/rdma/rdma_rm.c

+19 −3
Original line number Diff line number Diff line
@@ -386,12 +386,14 @@ int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res, uint32_t pd_handle, 
                     uint8_t qp_type, uint32_t max_send_wr,

                     uint32_t max_send_sge, uint32_t send_cq_handle,

                     uint32_t max_recv_wr, uint32_t max_recv_sge,

                     uint32_t recv_cq_handle, void *opaque, uint32_t *qpn)

                     uint32_t recv_cq_handle, void *opaque, uint32_t *qpn,

                     uint8_t is_srq, uint32_t srq_handle)

{

    int rc;

    RdmaRmQP *qp;

    RdmaRmCQ *scq, *rcq;

    RdmaRmPD *pd;

    RdmaRmSRQ *srq = NULL;

    uint32_t rm_qpn;



    pd = rdma_rm_get_pd(dev_res, pd_handle);
@@ -408,6 +410,16 @@ int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res, uint32_t pd_handle, 
        return -EINVAL;

    }



    if (is_srq) {

        srq = rdma_rm_get_srq(dev_res, srq_handle);

        if (!srq) {

            rdma_error_report("Invalid srqn %d", srq_handle);

            return -EINVAL;

        }



        srq->recv_cq_handle = recv_cq_handle;

    }



    if (qp_type == IBV_QPT_GSI) {

        scq->notify = CNT_SET;

        rcq->notify = CNT_SET;
@@ -424,10 +436,14 @@ int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res, uint32_t pd_handle, 
    qp->send_cq_handle = send_cq_handle;

    qp->recv_cq_handle = recv_cq_handle;

    qp->opaque = opaque;

    qp->is_srq = is_srq;



    rc = rdma_backend_create_qp(&qp->backend_qp, qp_type, &pd->backend_pd,

                                &scq->backend_cq, &rcq->backend_cq, max_send_wr,

                                max_recv_wr, max_send_sge, max_recv_sge);

                                &scq->backend_cq, &rcq->backend_cq,

                                is_srq ? &srq->backend_srq : NULL,

                                max_send_wr, max_recv_wr, max_send_sge,

                                max_recv_sge);



    if (rc) {

        rc = -EIO;

        goto out_dealloc_qp;

hw/rdma/rdma_rm.h

+2 −1
Original line number Diff line number Diff line
@@ -53,7 +53,8 @@ int rdma_rm_alloc_qp(RdmaDeviceResources *dev_res, uint32_t pd_handle, 
                     uint8_t qp_type, uint32_t max_send_wr,

                     uint32_t max_send_sge, uint32_t send_cq_handle,

                     uint32_t max_recv_wr, uint32_t max_recv_sge,

                     uint32_t recv_cq_handle, void *opaque, uint32_t *qpn);

                     uint32_t recv_cq_handle, void *opaque, uint32_t *qpn,

                     uint8_t is_srq, uint32_t srq_handle);

RdmaRmQP *rdma_rm_get_qp(RdmaDeviceResources *dev_res, uint32_t qpn);

int rdma_rm_modify_qp(RdmaDeviceResources *dev_res, RdmaBackendDev *backend_dev,

                      uint32_t qp_handle, uint32_t attr_mask, uint8_t sgid_idx,

hw/rdma/rdma_rm_defs.h

+1 −0
Original line number Diff line number Diff line
@@ -88,6 +88,7 @@ typedef struct RdmaRmQP { 
    uint32_t send_cq_handle;

    uint32_t recv_cq_handle;

    enum ibv_qp_state qp_state;

    uint8_t is_srq;

} RdmaRmQP;



typedef struct RdmaRmSRQ {