Commit 831df883 authored by Maor Gottlieb's avatar Maor Gottlieb Committed by Jason Gunthorpe
Browse files

RDMA/mlx5: Move all DM logic to separate file 

Move all device memory related code to a separate file.

Link: https://lore.kernel.org/r/20210411122924.60230-4-leon@kernel.org


Signed-off-by: default avatarMaor Gottlieb <maorg@nvidia.com>
Signed-off-by: default avatarLeon Romanovsky <leonro@nvidia.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@nvidia.com>
parent 7ca2b8a3

drivers/infiniband/hw/mlx5/Makefile

+1 −0
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ mlx5_ib-y := ah.o \ 
	     cong.o \

	     counters.o \

	     cq.o \

	     dm.o \

	     doorbell.o \

	     gsi.o \

	     ib_virt.o \

drivers/infiniband/hw/mlx5/cmd.c

+0 −101
Original line number Diff line number Diff line
@@ -47,107 +47,6 @@ int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point, 
	return mlx5_cmd_exec_inout(dev, query_cong_params, in, out);

}



int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,

			 u64 length, u32 alignment)

{

	struct mlx5_core_dev *dev = dm->dev;

	u64 num_memic_hw_pages = MLX5_CAP_DEV_MEM(dev, memic_bar_size)

					>> PAGE_SHIFT;

	u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);

	u32 max_alignment = MLX5_CAP_DEV_MEM(dev, log_max_memic_addr_alignment);

	u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);

	u32 out[MLX5_ST_SZ_DW(alloc_memic_out)] = {};

	u32 in[MLX5_ST_SZ_DW(alloc_memic_in)] = {};

	u32 mlx5_alignment;

	u64 page_idx = 0;

	int ret = 0;



	if (!length || (length & MLX5_MEMIC_ALLOC_SIZE_MASK))

		return -EINVAL;



	/* mlx5 device sets alignment as 64*2^driver_value

	 * so normalizing is needed.

	 */

	mlx5_alignment = (alignment < MLX5_MEMIC_BASE_ALIGN) ? 0 :

			 alignment - MLX5_MEMIC_BASE_ALIGN;

	if (mlx5_alignment > max_alignment)

		return -EINVAL;



	MLX5_SET(alloc_memic_in, in, opcode, MLX5_CMD_OP_ALLOC_MEMIC);

	MLX5_SET(alloc_memic_in, in, range_size, num_pages * PAGE_SIZE);

	MLX5_SET(alloc_memic_in, in, memic_size, length);

	MLX5_SET(alloc_memic_in, in, log_memic_addr_alignment,

		 mlx5_alignment);



	while (page_idx < num_memic_hw_pages) {

		spin_lock(&dm->lock);

		page_idx = bitmap_find_next_zero_area(dm->memic_alloc_pages,

						      num_memic_hw_pages,

						      page_idx,

						      num_pages, 0);



		if (page_idx < num_memic_hw_pages)

			bitmap_set(dm->memic_alloc_pages,

				   page_idx, num_pages);



		spin_unlock(&dm->lock);



		if (page_idx >= num_memic_hw_pages)

			break;



		MLX5_SET64(alloc_memic_in, in, range_start_addr,

			   hw_start_addr + (page_idx * PAGE_SIZE));



		ret = mlx5_cmd_exec_inout(dev, alloc_memic, in, out);

		if (ret) {

			spin_lock(&dm->lock);

			bitmap_clear(dm->memic_alloc_pages,

				     page_idx, num_pages);

			spin_unlock(&dm->lock);



			if (ret == -EAGAIN) {

				page_idx++;

				continue;

			}



			return ret;

		}



		*addr = dev->bar_addr +

			MLX5_GET64(alloc_memic_out, out, memic_start_addr);



		return 0;

	}



	return -ENOMEM;

}



void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length)

{

	struct mlx5_core_dev *dev = dm->dev;

	u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);

	u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);

	u32 in[MLX5_ST_SZ_DW(dealloc_memic_in)] = {};

	u64 start_page_idx;

	int err;



	addr -= dev->bar_addr;

	start_page_idx = (addr - hw_start_addr) >> PAGE_SHIFT;



	MLX5_SET(dealloc_memic_in, in, opcode, MLX5_CMD_OP_DEALLOC_MEMIC);

	MLX5_SET64(dealloc_memic_in, in, memic_start_addr, addr);

	MLX5_SET(dealloc_memic_in, in, memic_size, length);



	err =  mlx5_cmd_exec_in(dev, dealloc_memic, in);

	if (err)

		return;



	spin_lock(&dm->lock);

	bitmap_clear(dm->memic_alloc_pages,

		     start_page_idx, num_pages);

	spin_unlock(&dm->lock);

}



void mlx5_cmd_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u16 uid)

{

	u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {};

drivers/infiniband/hw/mlx5/cmd.h

+0 −3
Original line number Diff line number Diff line
@@ -41,9 +41,6 @@ int mlx5_cmd_dump_fill_mkey(struct mlx5_core_dev *dev, u32 *mkey); 
int mlx5_cmd_null_mkey(struct mlx5_core_dev *dev, u32 *null_mkey);

int mlx5_cmd_query_cong_params(struct mlx5_core_dev *dev, int cong_point,

			       void *out);

int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,

			 u64 length, u32 alignment);

void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, u64 length);

int mlx5_cmd_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid);

void mlx5_cmd_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u16 uid);

void mlx5_cmd_destroy_tis(struct mlx5_core_dev *dev, u32 tisn, u16 uid);

drivers/infiniband/hw/mlx5/dm.c

0 → 100644
+342 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB

/*

 * Copyright (c) 2021, Mellanox Technologies inc. All rights reserved.

 */



#include <rdma/uverbs_std_types.h>

#include "dm.h"



#define UVERBS_MODULE_NAME mlx5_ib

#include <rdma/uverbs_named_ioctl.h>



static int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,

				u64 length, u32 alignment)

{

	struct mlx5_core_dev *dev = dm->dev;

	u64 num_memic_hw_pages = MLX5_CAP_DEV_MEM(dev, memic_bar_size)

					>> PAGE_SHIFT;

	u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);

	u32 max_alignment = MLX5_CAP_DEV_MEM(dev, log_max_memic_addr_alignment);

	u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);

	u32 out[MLX5_ST_SZ_DW(alloc_memic_out)] = {};

	u32 in[MLX5_ST_SZ_DW(alloc_memic_in)] = {};

	u32 mlx5_alignment;

	u64 page_idx = 0;

	int ret = 0;



	if (!length || (length & MLX5_MEMIC_ALLOC_SIZE_MASK))

		return -EINVAL;



	/* mlx5 device sets alignment as 64*2^driver_value

	 * so normalizing is needed.

	 */

	mlx5_alignment = (alignment < MLX5_MEMIC_BASE_ALIGN) ? 0 :

			 alignment - MLX5_MEMIC_BASE_ALIGN;

	if (mlx5_alignment > max_alignment)

		return -EINVAL;



	MLX5_SET(alloc_memic_in, in, opcode, MLX5_CMD_OP_ALLOC_MEMIC);

	MLX5_SET(alloc_memic_in, in, range_size, num_pages * PAGE_SIZE);

	MLX5_SET(alloc_memic_in, in, memic_size, length);

	MLX5_SET(alloc_memic_in, in, log_memic_addr_alignment,

		 mlx5_alignment);



	while (page_idx < num_memic_hw_pages) {

		spin_lock(&dm->lock);

		page_idx = bitmap_find_next_zero_area(dm->memic_alloc_pages,

						      num_memic_hw_pages,

						      page_idx,

						      num_pages, 0);



		if (page_idx < num_memic_hw_pages)

			bitmap_set(dm->memic_alloc_pages,

				   page_idx, num_pages);



		spin_unlock(&dm->lock);



		if (page_idx >= num_memic_hw_pages)

			break;



		MLX5_SET64(alloc_memic_in, in, range_start_addr,

			   hw_start_addr + (page_idx * PAGE_SIZE));



		ret = mlx5_cmd_exec_inout(dev, alloc_memic, in, out);

		if (ret) {

			spin_lock(&dm->lock);

			bitmap_clear(dm->memic_alloc_pages,

				     page_idx, num_pages);

			spin_unlock(&dm->lock);



			if (ret == -EAGAIN) {

				page_idx++;

				continue;

			}



			return ret;

		}



		*addr = dev->bar_addr +

			MLX5_GET64(alloc_memic_out, out, memic_start_addr);



		return 0;

	}



	return -ENOMEM;

}



void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr,

			    u64 length)

{

	struct mlx5_core_dev *dev = dm->dev;

	u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr);

	u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE);

	u32 in[MLX5_ST_SZ_DW(dealloc_memic_in)] = {};

	u64 start_page_idx;

	int err;



	addr -= dev->bar_addr;

	start_page_idx = (addr - hw_start_addr) >> PAGE_SHIFT;



	MLX5_SET(dealloc_memic_in, in, opcode, MLX5_CMD_OP_DEALLOC_MEMIC);

	MLX5_SET64(dealloc_memic_in, in, memic_start_addr, addr);

	MLX5_SET(dealloc_memic_in, in, memic_size, length);



	err =  mlx5_cmd_exec_in(dev, dealloc_memic, in);

	if (err)

		return;



	spin_lock(&dm->lock);

	bitmap_clear(dm->memic_alloc_pages,

		     start_page_idx, num_pages);

	spin_unlock(&dm->lock);

}



static int add_dm_mmap_entry(struct ib_ucontext *context,

			     struct mlx5_ib_dm *mdm, u64 address)

{

	mdm->mentry.mmap_flag = MLX5_IB_MMAP_TYPE_MEMIC;

	mdm->mentry.address = address;

	return rdma_user_mmap_entry_insert_range(

		context, &mdm->mentry.rdma_entry, mdm->size,

		MLX5_IB_MMAP_DEVICE_MEM << 16,

		(MLX5_IB_MMAP_DEVICE_MEM << 16) + (1UL << 16) - 1);

}



static inline int check_dm_type_support(struct mlx5_ib_dev *dev, u32 type)

{

	switch (type) {

	case MLX5_IB_UAPI_DM_TYPE_MEMIC:

		if (!MLX5_CAP_DEV_MEM(dev->mdev, memic))

			return -EOPNOTSUPP;

		break;

	case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:

	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:

		if (!capable(CAP_SYS_RAWIO) || !capable(CAP_NET_RAW))

			return -EPERM;



		if (!(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner) ||

		      MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, sw_owner) ||

		      MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner_v2) ||

		      MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, sw_owner_v2)))

			return -EOPNOTSUPP;

		break;

	}



	return 0;

}



static int handle_alloc_dm_memic(struct ib_ucontext *ctx, struct mlx5_ib_dm *dm,

				 struct ib_dm_alloc_attr *attr,

				 struct uverbs_attr_bundle *attrs)

{

	struct mlx5_dm *dm_db = &to_mdev(ctx->device)->dm;

	u64 start_offset;

	u16 page_idx;

	int err;

	u64 address;



	dm->size = roundup(attr->length, MLX5_MEMIC_BASE_SIZE);



	err = mlx5_cmd_alloc_memic(dm_db, &dm->dev_addr,

				   dm->size, attr->alignment);

	if (err) {

		kfree(dm);

		return err;

	}



	address = dm->dev_addr & PAGE_MASK;

	err = add_dm_mmap_entry(ctx, dm, address);

	if (err) {

		mlx5_cmd_dealloc_memic(dm_db, dm->dev_addr, dm->size);

		kfree(dm);

		return err;

	}



	page_idx = dm->mentry.rdma_entry.start_pgoff & 0xFFFF;

	err = uverbs_copy_to(attrs,

			     MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,

			     &page_idx,

			     sizeof(page_idx));

	if (err)

		goto err_copy;



	start_offset = dm->dev_addr & ~PAGE_MASK;

	err = uverbs_copy_to(attrs,

			     MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,

			     &start_offset, sizeof(start_offset));

	if (err)

		goto err_copy;



	return 0;



err_copy:

	rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry);



	return err;

}



static int handle_alloc_dm_sw_icm(struct ib_ucontext *ctx,

				  struct mlx5_ib_dm *dm,

				  struct ib_dm_alloc_attr *attr,

				  struct uverbs_attr_bundle *attrs, int type)

{

	struct mlx5_core_dev *dev = to_mdev(ctx->device)->mdev;

	u64 act_size;

	int err;



	/* Allocation size must a multiple of the basic block size

	 * and a power of 2.

	 */

	act_size = round_up(attr->length, MLX5_SW_ICM_BLOCK_SIZE(dev));

	act_size = roundup_pow_of_two(act_size);



	dm->size = act_size;

	err = mlx5_dm_sw_icm_alloc(dev, type, act_size, attr->alignment,

				   to_mucontext(ctx)->devx_uid, &dm->dev_addr,

				   &dm->icm_dm.obj_id);

	if (err)

		return err;



	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,

			     &dm->dev_addr, sizeof(dm->dev_addr));

	if (err)

		mlx5_dm_sw_icm_dealloc(dev, type, dm->size,

				       to_mucontext(ctx)->devx_uid,

				       dm->dev_addr, dm->icm_dm.obj_id);



	return err;

}



struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev,

			       struct ib_ucontext *context,

			       struct ib_dm_alloc_attr *attr,

			       struct uverbs_attr_bundle *attrs)

{

	struct mlx5_ib_dm *dm;

	enum mlx5_ib_uapi_dm_type type;

	int err;



	err = uverbs_get_const_default(&type, attrs,

				       MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,

				       MLX5_IB_UAPI_DM_TYPE_MEMIC);

	if (err)

		return ERR_PTR(err);



	mlx5_ib_dbg(to_mdev(ibdev), "alloc_dm req: dm_type=%d user_length=0x%llx log_alignment=%d\n",

		    type, attr->length, attr->alignment);



	err = check_dm_type_support(to_mdev(ibdev), type);

	if (err)

		return ERR_PTR(err);



	dm = kzalloc(sizeof(*dm), GFP_KERNEL);

	if (!dm)

		return ERR_PTR(-ENOMEM);



	dm->type = type;



	switch (type) {

	case MLX5_IB_UAPI_DM_TYPE_MEMIC:

		err = handle_alloc_dm_memic(context, dm,

					    attr,

					    attrs);

		break;

	case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:

		err = handle_alloc_dm_sw_icm(context, dm,

					     attr, attrs,

					     MLX5_SW_ICM_TYPE_STEERING);

		break;

	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:

		err = handle_alloc_dm_sw_icm(context, dm,

					     attr, attrs,

					     MLX5_SW_ICM_TYPE_HEADER_MODIFY);

		break;

	default:

		err = -EOPNOTSUPP;

	}



	if (err)

		goto err_free;



	return &dm->ibdm;



err_free:

	kfree(dm);

	return ERR_PTR(err);

}



int mlx5_ib_dealloc_dm(struct ib_dm *ibdm, struct uverbs_attr_bundle *attrs)

{

	struct mlx5_ib_ucontext *ctx = rdma_udata_to_drv_context(

		&attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);

	struct mlx5_core_dev *dev = to_mdev(ibdm->device)->mdev;

	struct mlx5_ib_dm *dm = to_mdm(ibdm);

	int ret;



	switch (dm->type) {

	case MLX5_IB_UAPI_DM_TYPE_MEMIC:

		rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry);

		return 0;

	case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM:

		ret = mlx5_dm_sw_icm_dealloc(dev, MLX5_SW_ICM_TYPE_STEERING,

					     dm->size, ctx->devx_uid,

					     dm->dev_addr, dm->icm_dm.obj_id);

		if (ret)

			return ret;

		break;

	case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM:

		ret = mlx5_dm_sw_icm_dealloc(dev,

					     MLX5_SW_ICM_TYPE_HEADER_MODIFY,

					     dm->size, ctx->devx_uid,

					     dm->dev_addr, dm->icm_dm.obj_id);

		if (ret)

			return ret;

		break;

	default:

		return -EOPNOTSUPP;

	}



	kfree(dm);



	return 0;

}



ADD_UVERBS_ATTRIBUTES_SIMPLE(

	mlx5_ib_dm, UVERBS_OBJECT_DM, UVERBS_METHOD_DM_ALLOC,

	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET,

			    UVERBS_ATTR_TYPE(u64), UA_MANDATORY),

	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,

			    UVERBS_ATTR_TYPE(u16), UA_OPTIONAL),

	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,

			     enum mlx5_ib_uapi_dm_type, UA_OPTIONAL));



const struct uapi_definition mlx5_ib_dm_defs[] = {

	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DM, &mlx5_ib_dm),

	{},

};



const struct ib_device_ops mlx5_ib_dev_dm_ops = {

	.alloc_dm = mlx5_ib_alloc_dm,

	.dealloc_dm = mlx5_ib_dealloc_dm,

	.reg_dm_mr = mlx5_ib_reg_dm_mr,

};

drivers/infiniband/hw/mlx5/dm.h

0 → 100644
+42 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */

/*

 * Copyright (c) 2021, Mellanox Technologies inc. All rights reserved.

 */



#ifndef _MLX5_IB_DM_H

#define _MLX5_IB_DM_H



#include "mlx5_ib.h"





extern const struct ib_device_ops mlx5_ib_dev_dm_ops;

extern const struct uapi_definition mlx5_ib_dm_defs[];



struct mlx5_ib_dm {

	struct ib_dm		ibdm;

	phys_addr_t		dev_addr;

	u32			type;

	size_t			size;

	union {

		struct {

			u32	obj_id;

		} icm_dm;

		/* other dm types specific params should be added here */

	};

	struct mlx5_user_mmap_entry mentry;

};



static inline struct mlx5_ib_dm *to_mdm(struct ib_dm *ibdm)

{

	return container_of(ibdm, struct mlx5_ib_dm, ibdm);

}



struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev,

			       struct ib_ucontext *context,

			       struct ib_dm_alloc_attr *attr,

			       struct uverbs_attr_bundle *attrs);

int mlx5_ib_dealloc_dm(struct ib_dm *ibdm, struct uverbs_attr_bundle *attrs);

void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr,

			    u64 length);



#endif /* _MLX5_IB_DM_H */