Commit 66f4817b authored by Gal Pressman's avatar Gal Pressman Committed by Jason Gunthorpe
Browse files

RDMA/efa: Add support for dmabuf memory regions 

Implement a dmabuf importer for the EFA driver. As ODP is not supported,
the pinned dmabuf are used to prevent the move_notify callback from being
called.

Link: https://lore.kernel.org/r/20211012120903.96933-4-galpress@amazon.com


Signed-off-by: default avatarGal Pressman <galpress@amazon.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@nvidia.com>
parent 1e4df4a2

drivers/infiniband/hw/efa/efa.h

+4 −0
Original line number Diff line number Diff line
@@ -154,6 +154,10 @@ int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 
struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,

			 u64 virt_addr, int access_flags,

			 struct ib_udata *udata);

struct ib_mr *efa_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start,

				     u64 length, u64 virt_addr,

				     int fd, int access_flags,

				     struct ib_udata *udata);

int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);

int efa_get_port_immutable(struct ib_device *ibdev, u32 port_num,

			   struct ib_port_immutable *immutable);

drivers/infiniband/hw/efa/efa_main.c

+1 −0
Original line number Diff line number Diff line
@@ -380,6 +380,7 @@ static const struct ib_device_ops efa_dev_ops = { 
	.query_port = efa_query_port,

	.query_qp = efa_query_qp,

	.reg_user_mr = efa_reg_mr,

	.reg_user_mr_dmabuf = efa_reg_user_mr_dmabuf,



	INIT_RDMA_OBJ_SIZE(ib_ah, efa_ah, ibah),

	INIT_RDMA_OBJ_SIZE(ib_cq, efa_cq, ibcq),

drivers/infiniband/hw/efa/efa_verbs.c

+96 −31
Original line number Diff line number Diff line
@@ -3,6 +3,8 @@ 
 * Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved.

 */



#include <linux/dma-buf.h>

#include <linux/dma-resv.h>

#include <linux/vmalloc.h>

#include <linux/log2.h>
@@ -1544,26 +1546,18 @@ static int efa_create_pbl(struct efa_dev *dev, 
	return 0;

}



struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,

			 u64 virt_addr, int access_flags,

static struct efa_mr *efa_alloc_mr(struct ib_pd *ibpd, int access_flags,

				   struct ib_udata *udata)

{

	struct efa_dev *dev = to_edev(ibpd->device);

	struct efa_com_reg_mr_params params = {};

	struct efa_com_reg_mr_result result = {};

	struct pbl_context pbl;

	int supp_access_flags;

	unsigned int pg_sz;

	struct efa_mr *mr;

	int inline_size;

	int err;



	if (udata && udata->inlen &&

	    !ib_is_udata_cleared(udata, 0, sizeof(udata->inlen))) {

		ibdev_dbg(&dev->ibdev,

			  "Incompatible ABI params, udata not cleared\n");

		err = -EINVAL;

		goto err_out;

		return ERR_PTR(-EINVAL);

	}



	supp_access_flags =
@@ -1575,24 +1569,27 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, 
		ibdev_dbg(&dev->ibdev,

			  "Unsupported access flags[%#x], supported[%#x]\n",

			  access_flags, supp_access_flags);

		err = -EOPNOTSUPP;

		goto err_out;

		return ERR_PTR(-EOPNOTSUPP);

	}



	mr = kzalloc(sizeof(*mr), GFP_KERNEL);

	if (!mr) {

		err = -ENOMEM;

		goto err_out;

	}

	if (!mr)

		return ERR_PTR(-ENOMEM);



	mr->umem = ib_umem_get(ibpd->device, start, length, access_flags);

	if (IS_ERR(mr->umem)) {

		err = PTR_ERR(mr->umem);

		ibdev_dbg(&dev->ibdev,

			  "Failed to pin and map user space memory[%d]\n", err);

		goto err_free;

	return mr;

}



static int efa_register_mr(struct ib_pd *ibpd, struct efa_mr *mr, u64 start,

			   u64 length, u64 virt_addr, int access_flags)

{

	struct efa_dev *dev = to_edev(ibpd->device);

	struct efa_com_reg_mr_params params = {};

	struct efa_com_reg_mr_result result = {};

	struct pbl_context pbl;

	unsigned int pg_sz;

	int inline_size;

	int err;



	params.pd = to_epd(ibpd)->pdn;

	params.iova = virt_addr;

	params.mr_length_in_bytes = length;
@@ -1602,10 +1599,9 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, 
				       dev->dev_attr.page_size_cap,

				       virt_addr);

	if (!pg_sz) {

		err = -EOPNOTSUPP;

		ibdev_dbg(&dev->ibdev, "Failed to find a suitable page size in page_size_cap %#llx\n",

			  dev->dev_attr.page_size_cap);

		goto err_unmap;

		return -EOPNOTSUPP;

	}



	params.page_shift = order_base_2(pg_sz);
@@ -1619,21 +1615,21 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, 
	if (params.page_num <= inline_size) {

		err = efa_create_inline_pbl(dev, mr, &params);

		if (err)

			goto err_unmap;

			return err;



		err = efa_com_register_mr(&dev->edev, &params, &result);

		if (err)

			goto err_unmap;

			return err;

	} else {

		err = efa_create_pbl(dev, &pbl, mr, &params);

		if (err)

			goto err_unmap;

			return err;



		err = efa_com_register_mr(&dev->edev, &params, &result);

		pbl_destroy(dev, &pbl);



		if (err)

			goto err_unmap;

			return err;

	}



	mr->ibmr.lkey = result.l_key;
@@ -1641,9 +1637,78 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, 
	mr->ibmr.length = length;

	ibdev_dbg(&dev->ibdev, "Registered mr[%d]\n", mr->ibmr.lkey);



	return 0;

}



struct ib_mr *efa_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start,

				     u64 length, u64 virt_addr,

				     int fd, int access_flags,

				     struct ib_udata *udata)

{

	struct efa_dev *dev = to_edev(ibpd->device);

	struct ib_umem_dmabuf *umem_dmabuf;

	struct efa_mr *mr;

	int err;



	mr = efa_alloc_mr(ibpd, access_flags, udata);

	if (IS_ERR(mr)) {

		err = PTR_ERR(mr);

		goto err_out;

	}



	umem_dmabuf = ib_umem_dmabuf_get_pinned(ibpd->device, start, length, fd,

						access_flags);

	if (IS_ERR(umem_dmabuf)) {

		err = PTR_ERR(umem_dmabuf);

		ibdev_dbg(&dev->ibdev, "Failed to get dmabuf umem[%d]\n", err);

		goto err_free;

	}



	mr->umem = &umem_dmabuf->umem;

	err = efa_register_mr(ibpd, mr, start, length, virt_addr, access_flags);

	if (err)

		goto err_release;



	return &mr->ibmr;



err_release:

	ib_umem_release(mr->umem);

err_free:

	kfree(mr);

err_out:

	atomic64_inc(&dev->stats.reg_mr_err);

	return ERR_PTR(err);

}



struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,

			 u64 virt_addr, int access_flags,

			 struct ib_udata *udata)

{

	struct efa_dev *dev = to_edev(ibpd->device);

	struct efa_mr *mr;

	int err;



	mr = efa_alloc_mr(ibpd, access_flags, udata);

	if (IS_ERR(mr)) {

		err = PTR_ERR(mr);

		goto err_out;

	}



	mr->umem = ib_umem_get(ibpd->device, start, length, access_flags);

	if (IS_ERR(mr->umem)) {

		err = PTR_ERR(mr->umem);

		ibdev_dbg(&dev->ibdev,

			  "Failed to pin and map user space memory[%d]\n", err);

		goto err_free;

	}



	err = efa_register_mr(ibpd, mr, start, length, virt_addr, access_flags);

	if (err)

		goto err_release;



	return &mr->ibmr;



err_unmap:

err_release:

	ib_umem_release(mr->umem);

err_free:

	kfree(mr);