Commit 28ad5f65 authored by Leon Romanovsky's avatar Leon Romanovsky Committed by Jason Gunthorpe
Browse files

RDMA: Move XRCD to be under ib_core responsibility 

Update the code to allocate and free ib_xrcd structure in the
ib_core instead of inside drivers.

Link: https://lore.kernel.org/r/20200630101855.368895-4-leon@kernel.org


Signed-off-by: default avatarLeon Romanovsky <leonro@mellanox.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@nvidia.com>
parent 3b023e1b

drivers/infiniband/core/device.c

+1 −0
Original line number Diff line number Diff line
@@ -2692,6 +2692,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) 
	SET_OBJ_SIZE(dev_ops, ib_pd);

	SET_OBJ_SIZE(dev_ops, ib_srq);

	SET_OBJ_SIZE(dev_ops, ib_ucontext);

	SET_OBJ_SIZE(dev_ops, ib_xrcd);

}

EXPORT_SYMBOL(ib_set_device_ops);

drivers/infiniband/core/verbs.c

+19 −9
Original line number Diff line number Diff line
@@ -2293,20 +2293,28 @@ struct ib_xrcd *ib_alloc_xrcd_user(struct ib_device *device, 
				   struct inode *inode, struct ib_udata *udata)

{

	struct ib_xrcd *xrcd;

	int ret;



	if (!device->ops.alloc_xrcd)

		return ERR_PTR(-EOPNOTSUPP);



	xrcd = device->ops.alloc_xrcd(device, udata);

	if (!IS_ERR(xrcd)) {

	xrcd = rdma_zalloc_drv_obj(device, ib_xrcd);

	if (!xrcd)

		return ERR_PTR(-ENOMEM);



	xrcd->device = device;

	xrcd->inode = inode;

	atomic_set(&xrcd->usecnt, 0);

	init_rwsem(&xrcd->tgt_qps_rwsem);

	xa_init(&xrcd->tgt_qps);

	}



	ret = device->ops.alloc_xrcd(xrcd, udata);

	if (ret)

		goto err;

	return xrcd;

err:

	kfree(xrcd);

	return ERR_PTR(ret);

}

EXPORT_SYMBOL(ib_alloc_xrcd_user);
@@ -2321,7 +2329,9 @@ int ib_dealloc_xrcd_user(struct ib_xrcd *xrcd, struct ib_udata *udata) 
		return -EBUSY;



	WARN_ON(!xa_empty(&xrcd->tgt_qps));

	return xrcd->device->ops.dealloc_xrcd(xrcd, udata);

	xrcd->device->ops.dealloc_xrcd(xrcd, udata);

	kfree(xrcd);

	return 0;

}

EXPORT_SYMBOL(ib_dealloc_xrcd_user);

drivers/infiniband/hw/mlx4/main.c

+15 −22
Original line number Diff line number Diff line
@@ -1219,56 +1219,47 @@ static void mlx4_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) 
	mlx4_pd_free(to_mdev(pd->device)->dev, to_mpd(pd)->pdn);

}



static struct ib_xrcd *mlx4_ib_alloc_xrcd(struct ib_device *ibdev,

					  struct ib_udata *udata)

static int mlx4_ib_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)

{

	struct mlx4_ib_xrcd *xrcd;

	struct mlx4_ib_dev *dev = to_mdev(ibxrcd->device);

	struct mlx4_ib_xrcd *xrcd = to_mxrcd(ibxrcd);

	struct ib_cq_init_attr cq_attr = {};

	int err;



	if (!(to_mdev(ibdev)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))

		return ERR_PTR(-ENOSYS);



	xrcd = kmalloc(sizeof *xrcd, GFP_KERNEL);

	if (!xrcd)

		return ERR_PTR(-ENOMEM);

	if (!(dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))

		return -EOPNOTSUPP;



	err = mlx4_xrcd_alloc(to_mdev(ibdev)->dev, &xrcd->xrcdn);

	err = mlx4_xrcd_alloc(dev->dev, &xrcd->xrcdn);

	if (err)

		goto err1;

		return err;



	xrcd->pd = ib_alloc_pd(ibdev, 0);

	xrcd->pd = ib_alloc_pd(ibxrcd->device, 0);

	if (IS_ERR(xrcd->pd)) {

		err = PTR_ERR(xrcd->pd);

		goto err2;

	}



	cq_attr.cqe = 1;

	xrcd->cq = ib_create_cq(ibdev, NULL, NULL, xrcd, &cq_attr);

	xrcd->cq = ib_create_cq(ibxrcd->device, NULL, NULL, xrcd, &cq_attr);

	if (IS_ERR(xrcd->cq)) {

		err = PTR_ERR(xrcd->cq);

		goto err3;

	}



	return &xrcd->ibxrcd;

	return 0;



err3:

	ib_dealloc_pd(xrcd->pd);

err2:

	mlx4_xrcd_free(to_mdev(ibdev)->dev, xrcd->xrcdn);

err1:

	kfree(xrcd);

	return ERR_PTR(err);

	mlx4_xrcd_free(dev->dev, xrcd->xrcdn);

	return err;

}



static int mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)

static void mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)

{

	ib_destroy_cq(to_mxrcd(xrcd)->cq);

	ib_dealloc_pd(to_mxrcd(xrcd)->pd);

	mlx4_xrcd_free(to_mdev(xrcd->device)->dev, to_mxrcd(xrcd)->xrcdn);

	kfree(xrcd);



	return 0;

}



static int add_gid_entry(struct ib_qp *ibqp, union ib_gid *gid)
@@ -2607,6 +2598,8 @@ static const struct ib_device_ops mlx4_ib_dev_mw_ops = { 
static const struct ib_device_ops mlx4_ib_dev_xrc_ops = {

	.alloc_xrcd = mlx4_ib_alloc_xrcd,

	.dealloc_xrcd = mlx4_ib_dealloc_xrcd,



	INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx4_ib_xrcd, ibxrcd),

};



static const struct ib_device_ops mlx4_ib_dev_fs_ops = {

drivers/infiniband/hw/mlx5/main.c

+2 −0
Original line number Diff line number Diff line
@@ -6671,6 +6671,8 @@ static const struct ib_device_ops mlx5_ib_dev_mw_ops = { 
static const struct ib_device_ops mlx5_ib_dev_xrc_ops = {

	.alloc_xrcd = mlx5_ib_alloc_xrcd,

	.dealloc_xrcd = mlx5_ib_dealloc_xrcd,



	INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx5_ib_xrcd, ibxrcd),

};



static const struct ib_device_ops mlx5_ib_dev_dm_ops = {

drivers/infiniband/hw/mlx5/mlx5_ib.h

+2 −3
Original line number Diff line number Diff line
@@ -1224,9 +1224,8 @@ int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, 
			const struct ib_wc *in_wc, const struct ib_grh *in_grh,

			const struct ib_mad *in, struct ib_mad *out,

			size_t *out_mad_size, u16 *out_mad_pkey_index);

struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,

				   struct ib_udata *udata);

int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);

int mlx5_ib_alloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);

void mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);

int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset);

int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port);

int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev,