Commit 04876c12 authored by Aharon Landau's avatar Aharon Landau Committed by Jason Gunthorpe
Browse files

RDMA/mlx5: Move init and cleanup of UMR to umr.c 

The first patch in a series to split UMR logic to a dedicated file.  As a
start, move the init and cleanup of UMR resources to umr.c.

Link: https://lore.kernel.org/r/849e632dd1945a2534712a320cc5779f2149ba96.1649747695.git.leonro@nvidia.com


Signed-off-by: default avatarAharon Landau <aharonl@nvidia.com>
Reviewed-by: default avatarMichael Guralnik <michaelgur@nvidia.com>
Signed-off-by: default avatarLeon Romanovsky <leonro@nvidia.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@nvidia.com>
parent b5a93e79

drivers/infiniband/hw/mlx5/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -19,6 +19,7 @@ mlx5_ib-y := ah.o \ 
	     restrack.o \

	     srq.o \

	     srq_cmd.o \

	     umr.o \

	     wr.o



mlx5_ib-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += odp.o

drivers/infiniband/hw/mlx5/main.c

+6 −103
Original line number Diff line number Diff line
@@ -41,6 +41,7 @@ 
#include "wr.h"

#include "restrack.h"

#include "counters.h"

#include "umr.h"

#include <rdma/uverbs_std_types.h>

#include <rdma/uverbs_ioctl.h>

#include <rdma/mlx5_user_ioctl_verbs.h>
@@ -4004,12 +4005,7 @@ static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev) 
	if (err)

		mlx5_ib_warn(dev, "mr cache cleanup failed\n");



	if (dev->umrc.qp)

		ib_destroy_qp(dev->umrc.qp);

	if (dev->umrc.cq)

		ib_free_cq(dev->umrc.cq);

	if (dev->umrc.pd)

		ib_dealloc_pd(dev->umrc.pd);

	mlx5r_umr_resource_cleanup(dev);

}



static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)
@@ -4017,112 +4013,19 @@ static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev) 
	ib_unregister_device(&dev->ib_dev);

}



enum {

	MAX_UMR_WR = 128,

};



static int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev)

{

	struct ib_qp_init_attr *init_attr = NULL;

	struct ib_qp_attr *attr = NULL;

	struct ib_pd *pd;

	struct ib_cq *cq;

	struct ib_qp *qp;

	int ret;



	attr = kzalloc(sizeof(*attr), GFP_KERNEL);

	init_attr = kzalloc(sizeof(*init_attr), GFP_KERNEL);

	if (!attr || !init_attr) {

		ret = -ENOMEM;

		goto error_0;

	}



	pd = ib_alloc_pd(&dev->ib_dev, 0);

	if (IS_ERR(pd)) {

		mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n");

		ret = PTR_ERR(pd);

		goto error_0;

	}



	cq = ib_alloc_cq(&dev->ib_dev, NULL, 128, 0, IB_POLL_SOFTIRQ);

	if (IS_ERR(cq)) {

		mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n");

		ret = PTR_ERR(cq);

		goto error_2;

	}



	init_attr->send_cq = cq;

	init_attr->recv_cq = cq;

	init_attr->sq_sig_type = IB_SIGNAL_ALL_WR;

	init_attr->cap.max_send_wr = MAX_UMR_WR;

	init_attr->cap.max_send_sge = 1;

	init_attr->qp_type = MLX5_IB_QPT_REG_UMR;

	init_attr->port_num = 1;

	qp = ib_create_qp(pd, init_attr);

	if (IS_ERR(qp)) {

		mlx5_ib_dbg(dev, "Couldn't create sync UMR QP\n");

		ret = PTR_ERR(qp);

		goto error_3;

	}



	attr->qp_state = IB_QPS_INIT;

	attr->port_num = 1;

	ret = ib_modify_qp(qp, attr,

			   IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT);

	if (ret) {

		mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n");

		goto error_4;

	}



	memset(attr, 0, sizeof(*attr));

	attr->qp_state = IB_QPS_RTR;

	attr->path_mtu = IB_MTU_256;



	ret = ib_modify_qp(qp, attr, IB_QP_STATE);

	if (ret) {

		mlx5_ib_dbg(dev, "Couldn't modify umr QP to rtr\n");

		goto error_4;

	}



	memset(attr, 0, sizeof(*attr));

	attr->qp_state = IB_QPS_RTS;

	ret = ib_modify_qp(qp, attr, IB_QP_STATE);

	if (ret) {

		mlx5_ib_dbg(dev, "Couldn't modify umr QP to rts\n");

		goto error_4;

	}



	dev->umrc.qp = qp;

	dev->umrc.cq = cq;

	dev->umrc.pd = pd;

	ret = mlx5r_umr_resource_init(dev);

	if (ret)

		return ret;



	sema_init(&dev->umrc.sem, MAX_UMR_WR);

	ret = mlx5_mr_cache_init(dev);

	if (ret) {

		mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);

		goto error_4;

		mlx5r_umr_resource_cleanup(dev);

	}



	kfree(attr);

	kfree(init_attr);



	return 0;



error_4:

	ib_destroy_qp(qp);

	dev->umrc.qp = NULL;



error_3:

	ib_free_cq(cq);

	dev->umrc.cq = NULL;



error_2:

	ib_dealloc_pd(pd);

	dev->umrc.pd = NULL;



error_0:

	kfree(attr);

	kfree(init_attr);

	return ret;

}

drivers/infiniband/hw/mlx5/umr.c

0 → 100644
+106 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB

/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. */



#include "mlx5_ib.h"

#include "umr.h"



enum {

	MAX_UMR_WR = 128,

};



static int mlx5r_umr_qp_rst2rts(struct mlx5_ib_dev *dev, struct ib_qp *qp)

{

	struct ib_qp_attr attr = {};

	int ret;



	attr.qp_state = IB_QPS_INIT;

	attr.port_num = 1;

	ret = ib_modify_qp(qp, &attr,

			   IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT);

	if (ret) {

		mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n");

		return ret;

	}



	memset(&attr, 0, sizeof(attr));

	attr.qp_state = IB_QPS_RTR;



	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);

	if (ret) {

		mlx5_ib_dbg(dev, "Couldn't modify umr QP to rtr\n");

		return ret;

	}



	memset(&attr, 0, sizeof(attr));

	attr.qp_state = IB_QPS_RTS;

	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);

	if (ret) {

		mlx5_ib_dbg(dev, "Couldn't modify umr QP to rts\n");

		return ret;

	}



	return 0;

}



int mlx5r_umr_resource_init(struct mlx5_ib_dev *dev)

{

	struct ib_qp_init_attr init_attr = {};

	struct ib_pd *pd;

	struct ib_cq *cq;

	struct ib_qp *qp;

	int ret;



	pd = ib_alloc_pd(&dev->ib_dev, 0);

	if (IS_ERR(pd)) {

		mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n");

		return PTR_ERR(pd);

	}



	cq = ib_alloc_cq(&dev->ib_dev, NULL, 128, 0, IB_POLL_SOFTIRQ);

	if (IS_ERR(cq)) {

		mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n");

		ret = PTR_ERR(cq);

		goto destroy_pd;

	}



	init_attr.send_cq = cq;

	init_attr.recv_cq = cq;

	init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;

	init_attr.cap.max_send_wr = MAX_UMR_WR;

	init_attr.cap.max_send_sge = 1;

	init_attr.qp_type = MLX5_IB_QPT_REG_UMR;

	init_attr.port_num = 1;

	qp = ib_create_qp(pd, &init_attr);

	if (IS_ERR(qp)) {

		mlx5_ib_dbg(dev, "Couldn't create sync UMR QP\n");

		ret = PTR_ERR(qp);

		goto destroy_cq;

	}



	ret = mlx5r_umr_qp_rst2rts(dev, qp);

	if (ret)

		goto destroy_qp;



	dev->umrc.qp = qp;

	dev->umrc.cq = cq;

	dev->umrc.pd = pd;



	sema_init(&dev->umrc.sem, MAX_UMR_WR);



	return 0;



destroy_qp:

	ib_destroy_qp(qp);

destroy_cq:

	ib_free_cq(cq);

destroy_pd:

	ib_dealloc_pd(pd);

	return ret;

}



void mlx5r_umr_resource_cleanup(struct mlx5_ib_dev *dev)

{

	ib_destroy_qp(dev->umrc.qp);

	ib_free_cq(dev->umrc.cq);

	ib_dealloc_pd(dev->umrc.pd);

}

drivers/infiniband/hw/mlx5/umr.h

0 → 100644
+12 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */

/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. */



#ifndef _MLX5_IB_UMR_H

#define _MLX5_IB_UMR_H



#include "mlx5_ib.h"



int mlx5r_umr_resource_init(struct mlx5_ib_dev *dev);

void mlx5r_umr_resource_cleanup(struct mlx5_ib_dev *dev);



#endif /* _MLX5_IB_UMR_H */