Commit ef6d4ccd authored by Yuval Shaia's avatar Yuval Shaia Committed by Marcel Apfelbaum
Browse files

hw/rdma: Implementation of generic rdma device layers 



This layer is composed of two sub-modules, backend and resource manager.
Backend sub-module is responsible for all the interaction with IB layers
such as ibverbs and umad (external libraries).
Resource manager is a collection of functions and structures to manage
RDMA resources such as QPs, CQs and MRs.

Reviewed-by: default avatarDotan Barak <dotanb@mellanox.com>
Reviewed-by: default avatarZhu Yanjun <yanjun.zhu@oracle.com>
Signed-off-by: default avatarYuval Shaia <yuval.shaia@oracle.com>
Signed-off-by: default avatarMarcel Apfelbaum <marcel@redhat.com>
parent b3a92277

Makefile.objs

+1 −0
Original line number Diff line number Diff line
@@ -130,6 +130,7 @@ trace-events-subdirs += hw/block/dataplane 
trace-events-subdirs += hw/char

trace-events-subdirs += hw/intc

trace-events-subdirs += hw/net

trace-events-subdirs += hw/rdma

trace-events-subdirs += hw/virtio

trace-events-subdirs += hw/audio

trace-events-subdirs += hw/misc

configure

+5 −4
Original line number Diff line number Diff line
@@ -1572,7 +1572,7 @@ disabled with --disable-FEATURE, default is enabled if available: 
  hax             HAX acceleration support

  hvf             Hypervisor.framework acceleration support

  whpx            Windows Hypervisor Platform acceleration support

  rdma            RDMA-based migration support

  rdma            Enable RDMA-based migration and PVRDMA support

  vde             support for vde network

  netmap          support for netmap network

  linux-aio       Linux AIO support
@@ -2923,15 +2923,16 @@ if test "$rdma" != "no" ; then 
#include <rdma/rdma_cma.h>

int main(void) { return 0; }

EOF

  rdma_libs="-lrdmacm -libverbs"

  rdma_libs="-lrdmacm -libverbs -libumad"

  if compile_prog "" "$rdma_libs" ; then

    rdma="yes"

    libs_softmmu="$libs_softmmu $rdma_libs"

  else

    if test "$rdma" = "yes" ; then

        error_exit \

            " OpenFabrics librdmacm/libibverbs not present." \

            " OpenFabrics librdmacm/libibverbs/libibumad not present." \

            " Your options:" \

            "  (1) Fast: Install infiniband packages from your distro." \

            "  (1) Fast: Install infiniband packages (devel) from your distro." \

            "  (2) Cleanest: Install libraries from www.openfabrics.org" \

            "  (3) Also: Install softiwarp if you don't have RDMA hardware"

    fi

hw/rdma/Makefile.objs

+1 −1
Original line number Diff line number Diff line 
ifeq ($(CONFIG_RDMA),y)

obj-$(CONFIG_PCI) += rdma_utils.o

obj-$(CONFIG_PCI) += rdma_utils.o rdma_backend.o rdma_rm.o

endif

hw/rdma/rdma_backend.c

0 → 100644
+818 −0

File added.

Preview size limit exceeded, changes collapsed.

hw/rdma/rdma_backend.h

0 → 100644
+98 −0
Original line number Diff line number Diff line 
/*

 *  RDMA device: Definitions of Backend Device functions

 *

 * Copyright (C) 2018 Oracle

 * Copyright (C) 2018 Red Hat Inc

 *

 * Authors:

 *     Yuval Shaia <yuval.shaia@oracle.com>

 *     Marcel Apfelbaum <marcel@redhat.com>

 *

 * This work is licensed under the terms of the GNU GPL, version 2 or later.

 * See the COPYING file in the top-level directory.

 *

 */



#ifndef RDMA_BACKEND_H

#define RDMA_BACKEND_H



#include <qapi/error.h>

#include "rdma_rm_defs.h"

#include "rdma_backend_defs.h"



/* Add definition for QP0 and QP1 as there is no userspace enums for them */

enum ibv_special_qp_type {

    IBV_QPT_SMI = 0,

    IBV_QPT_GSI = 1,

};



static inline union ibv_gid *rdma_backend_gid(RdmaBackendDev *dev)

{

    return &dev->gid;

}



static inline uint32_t rdma_backend_qpn(const RdmaBackendQP *qp)

{

    return qp->ibqp ? qp->ibqp->qp_num : 0;

}



static inline uint32_t rdma_backend_mr_lkey(const RdmaBackendMR *mr)

{

    return mr->ibmr ? mr->ibmr->lkey : 0;

}



static inline uint32_t rdma_backend_mr_rkey(const RdmaBackendMR *mr)

{

    return mr->ibmr ? mr->ibmr->rkey : 0;

}



int rdma_backend_init(RdmaBackendDev *backend_dev,

                      RdmaDeviceResources *rdma_dev_res,

                      const char *backend_device_name, uint8_t port_num,

                      uint8_t backend_gid_idx, struct ibv_device_attr *dev_attr,

                      Error **errp);

void rdma_backend_fini(RdmaBackendDev *backend_dev);

void rdma_backend_register_comp_handler(void (*handler)(int status,

                                        unsigned int vendor_err, void *ctx));

void rdma_backend_unregister_comp_handler(void);



int rdma_backend_query_port(RdmaBackendDev *backend_dev,

                            struct ibv_port_attr *port_attr);

int rdma_backend_create_pd(RdmaBackendDev *backend_dev, RdmaBackendPD *pd);

void rdma_backend_destroy_pd(RdmaBackendPD *pd);



int rdma_backend_create_mr(RdmaBackendMR *mr, RdmaBackendPD *pd, uint64_t addr,

                           size_t length, int access);

void rdma_backend_destroy_mr(RdmaBackendMR *mr);



int rdma_backend_create_cq(RdmaBackendDev *backend_dev, RdmaBackendCQ *cq,

                           int cqe);

void rdma_backend_destroy_cq(RdmaBackendCQ *cq);

void rdma_backend_poll_cq(RdmaDeviceResources *rdma_dev_res, RdmaBackendCQ *cq);



int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t qp_type,

                           RdmaBackendPD *pd, RdmaBackendCQ *scq,

                           RdmaBackendCQ *rcq, uint32_t max_send_wr,

                           uint32_t max_recv_wr, uint32_t max_send_sge,

                           uint32_t max_recv_sge);

int rdma_backend_qp_state_init(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,

                               uint8_t qp_type, uint32_t qkey);

int rdma_backend_qp_state_rtr(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,

                              uint8_t qp_type, union ibv_gid *dgid,

                              uint32_t dqpn, uint32_t rq_psn, uint32_t qkey,

                              bool use_qkey);

int rdma_backend_qp_state_rts(RdmaBackendQP *qp, uint8_t qp_type,

                              uint32_t sq_psn, uint32_t qkey, bool use_qkey);

void rdma_backend_destroy_qp(RdmaBackendQP *qp);



void rdma_backend_post_send(RdmaBackendDev *backend_dev,

                            RdmaBackendQP *qp, uint8_t qp_type,

                            struct ibv_sge *sge, uint32_t num_sge,

                            union ibv_gid *dgid, uint32_t dqpn, uint32_t dqkey,

                            void *ctx);

void rdma_backend_post_recv(RdmaBackendDev *backend_dev,

                            RdmaDeviceResources *rdma_dev_res,

                            RdmaBackendQP *qp, uint8_t qp_type,

                            struct ibv_sge *sge, uint32_t num_sge, void *ctx);



#endif