Commit 8a8a5d37 authored by Aharon Landau's avatar Aharon Landau Committed by Jason Gunthorpe
Browse files

RDMA/mlx5: Move mkey ctrl segment logic to umr.c 

Move set_reg_umr_segment() and its helpers to umr.c.

Link: https://lore.kernel.org/r/5a7fac8ae8543521d19d174663245ae84b910310.1649747695.git.leonro@nvidia.com


Signed-off-by: default avatarAharon Landau <aharonl@nvidia.com>
Reviewed-by: default avatarMichael Guralnik <michaelgur@nvidia.com>
Signed-off-by: default avatarLeon Romanovsky <leonro@nvidia.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@nvidia.com>
parent f49c856a

drivers/infiniband/hw/mlx5/mlx5_ib.h

+0 −3
Original line number Diff line number Diff line
@@ -311,9 +311,6 @@ struct mlx5_ib_flow_db { 
#define MLX5_IB_QPT_DCT		IB_QPT_RESERVED4

#define MLX5_IB_WR_UMR		IB_WR_RESERVED1



#define MLX5_IB_UMR_OCTOWORD	       16

#define MLX5_IB_UMR_XLT_ALIGNMENT      64



#define MLX5_IB_UPD_XLT_ZAP	      BIT(0)

#define MLX5_IB_UPD_XLT_ENABLE	      BIT(1)

#define MLX5_IB_UPD_XLT_ATOMIC	      BIT(2)

drivers/infiniband/hw/mlx5/qp.c

+1 −0
Original line number Diff line number Diff line
@@ -40,6 +40,7 @@ 
#include "ib_rep.h"

#include "counters.h"

#include "cmd.h"

#include "umr.h"

#include "qp.h"

#include "wr.h"

drivers/infiniband/hw/mlx5/umr.c

+129 −0
Original line number Diff line number Diff line
@@ -4,6 +4,135 @@ 
#include "mlx5_ib.h"

#include "umr.h"



static __be64 get_umr_enable_mr_mask(void)

{

	u64 result;



	result = MLX5_MKEY_MASK_KEY |

		 MLX5_MKEY_MASK_FREE;



	return cpu_to_be64(result);

}



static __be64 get_umr_disable_mr_mask(void)

{

	u64 result;



	result = MLX5_MKEY_MASK_FREE;



	return cpu_to_be64(result);

}



static __be64 get_umr_update_translation_mask(void)

{

	u64 result;



	result = MLX5_MKEY_MASK_LEN |

		 MLX5_MKEY_MASK_PAGE_SIZE |

		 MLX5_MKEY_MASK_START_ADDR;



	return cpu_to_be64(result);

}



static __be64 get_umr_update_access_mask(int atomic,

					 int relaxed_ordering_write,

					 int relaxed_ordering_read)

{

	u64 result;



	result = MLX5_MKEY_MASK_LR |

		 MLX5_MKEY_MASK_LW |

		 MLX5_MKEY_MASK_RR |

		 MLX5_MKEY_MASK_RW;



	if (atomic)

		result |= MLX5_MKEY_MASK_A;



	if (relaxed_ordering_write)

		result |= MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE;



	if (relaxed_ordering_read)

		result |= MLX5_MKEY_MASK_RELAXED_ORDERING_READ;



	return cpu_to_be64(result);

}



static __be64 get_umr_update_pd_mask(void)

{

	u64 result;



	result = MLX5_MKEY_MASK_PD;



	return cpu_to_be64(result);

}



static int umr_check_mkey_mask(struct mlx5_ib_dev *dev, u64 mask)

{

	if (mask & MLX5_MKEY_MASK_PAGE_SIZE &&

	    MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled))

		return -EPERM;



	if (mask & MLX5_MKEY_MASK_A &&

	    MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled))

		return -EPERM;



	if (mask & MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE &&

	    !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr))

		return -EPERM;



	if (mask & MLX5_MKEY_MASK_RELAXED_ORDERING_READ &&

	    !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr))

		return -EPERM;



	return 0;

}



int mlx5r_umr_set_umr_ctrl_seg(struct mlx5_ib_dev *dev,

			       struct mlx5_wqe_umr_ctrl_seg *umr,

			       const struct ib_send_wr *wr)

{

	const struct mlx5_umr_wr *umrwr = umr_wr(wr);



	memset(umr, 0, sizeof(*umr));



	if (!umrwr->ignore_free_state) {

		if (wr->send_flags & MLX5_IB_SEND_UMR_FAIL_IF_FREE)

			 /* fail if free */

			umr->flags = MLX5_UMR_CHECK_FREE;

		else

			/* fail if not free */

			umr->flags = MLX5_UMR_CHECK_NOT_FREE;

	}



	umr->xlt_octowords =

		cpu_to_be16(mlx5r_umr_get_xlt_octo(umrwr->xlt_size));

	if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_XLT) {

		u64 offset = mlx5r_umr_get_xlt_octo(umrwr->offset);



		umr->xlt_offset = cpu_to_be16(offset & 0xffff);

		umr->xlt_offset_47_16 = cpu_to_be32(offset >> 16);

		umr->flags |= MLX5_UMR_TRANSLATION_OFFSET_EN;

	}

	if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION)

		umr->mkey_mask |= get_umr_update_translation_mask();

	if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS) {

		umr->mkey_mask |= get_umr_update_access_mask(

			!!MLX5_CAP_GEN(dev->mdev, atomic),

			!!MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr),

			!!MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr));

		umr->mkey_mask |= get_umr_update_pd_mask();

	}

	if (wr->send_flags & MLX5_IB_SEND_UMR_ENABLE_MR)

		umr->mkey_mask |= get_umr_enable_mr_mask();

	if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)

		umr->mkey_mask |= get_umr_disable_mr_mask();



	if (!wr->num_sge)

		umr->flags |= MLX5_UMR_INLINE;



	return umr_check_mkey_mask(dev, be64_to_cpu(umr->mkey_mask));

}



enum {

	MAX_UMR_WR = 128,

};

drivers/infiniband/hw/mlx5/umr.h

+13 −0
Original line number Diff line number Diff line
@@ -10,6 +10,9 @@ 
#define MLX5_MAX_UMR_SHIFT 16

#define MLX5_MAX_UMR_PAGES (1 << MLX5_MAX_UMR_SHIFT)



#define MLX5_IB_UMR_OCTOWORD	       16

#define MLX5_IB_UMR_XLT_ALIGNMENT      64



int mlx5r_umr_resource_init(struct mlx5_ib_dev *dev);

void mlx5r_umr_resource_cleanup(struct mlx5_ib_dev *dev);
@@ -66,4 +69,14 @@ static inline bool mlx5r_umr_can_reconfig(struct mlx5_ib_dev *dev, 
	return true;

}



static inline u64 mlx5r_umr_get_xlt_octo(u64 bytes)

{

	return ALIGN(bytes, MLX5_IB_UMR_XLT_ALIGNMENT) /

	       MLX5_IB_UMR_OCTOWORD;

}



int mlx5r_umr_set_umr_ctrl_seg(struct mlx5_ib_dev *dev,

			       struct mlx5_wqe_umr_ctrl_seg *umr,

			       const struct ib_send_wr *wr);



#endif /* _MLX5_IB_UMR_H */

drivers/infiniband/hw/mlx5/wr.c

+4 −138
Original line number Diff line number Diff line
@@ -166,12 +166,6 @@ static void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ib_sge *sg) 
	dseg->addr       = cpu_to_be64(sg->addr);

}



static u64 get_xlt_octo(u64 bytes)

{

	return ALIGN(bytes, MLX5_IB_UMR_XLT_ALIGNMENT) /

	       MLX5_IB_UMR_OCTOWORD;

}



static __be64 frwr_mkey_mask(bool atomic)

{

	u64 result;
@@ -223,7 +217,7 @@ static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr, 
	memset(umr, 0, sizeof(*umr));



	umr->flags = flags;

	umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));

	umr->xlt_octowords = cpu_to_be16(mlx5r_umr_get_xlt_octo(size));

	umr->mkey_mask = frwr_mkey_mask(atomic);

}
@@ -234,134 +228,6 @@ static void set_linv_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr) 
	umr->flags = MLX5_UMR_INLINE;

}



static __be64 get_umr_enable_mr_mask(void)

{

	u64 result;



	result = MLX5_MKEY_MASK_KEY |

		 MLX5_MKEY_MASK_FREE;



	return cpu_to_be64(result);

}



static __be64 get_umr_disable_mr_mask(void)

{

	u64 result;



	result = MLX5_MKEY_MASK_FREE;



	return cpu_to_be64(result);

}



static __be64 get_umr_update_translation_mask(void)

{

	u64 result;



	result = MLX5_MKEY_MASK_LEN |

		 MLX5_MKEY_MASK_PAGE_SIZE |

		 MLX5_MKEY_MASK_START_ADDR;



	return cpu_to_be64(result);

}



static __be64 get_umr_update_access_mask(int atomic,

					 int relaxed_ordering_write,

					 int relaxed_ordering_read)

{

	u64 result;



	result = MLX5_MKEY_MASK_LR |

		 MLX5_MKEY_MASK_LW |

		 MLX5_MKEY_MASK_RR |

		 MLX5_MKEY_MASK_RW;



	if (atomic)

		result |= MLX5_MKEY_MASK_A;



	if (relaxed_ordering_write)

		result |= MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE;



	if (relaxed_ordering_read)

		result |= MLX5_MKEY_MASK_RELAXED_ORDERING_READ;



	return cpu_to_be64(result);

}



static __be64 get_umr_update_pd_mask(void)

{

	u64 result;



	result = MLX5_MKEY_MASK_PD;



	return cpu_to_be64(result);

}



static int umr_check_mkey_mask(struct mlx5_ib_dev *dev, u64 mask)

{

	if (mask & MLX5_MKEY_MASK_PAGE_SIZE &&

	    MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled))

		return -EPERM;



	if (mask & MLX5_MKEY_MASK_A &&

	    MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled))

		return -EPERM;



	if (mask & MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE &&

	    !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr))

		return -EPERM;



	if (mask & MLX5_MKEY_MASK_RELAXED_ORDERING_READ &&

	    !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr))

		return -EPERM;



	return 0;

}



static int set_reg_umr_segment(struct mlx5_ib_dev *dev,

			       struct mlx5_wqe_umr_ctrl_seg *umr,

			       const struct ib_send_wr *wr)

{

	const struct mlx5_umr_wr *umrwr = umr_wr(wr);



	memset(umr, 0, sizeof(*umr));



	if (!umrwr->ignore_free_state) {

		if (wr->send_flags & MLX5_IB_SEND_UMR_FAIL_IF_FREE)

			 /* fail if free */

			umr->flags = MLX5_UMR_CHECK_FREE;

		else

			/* fail if not free */

			umr->flags = MLX5_UMR_CHECK_NOT_FREE;

	}



	umr->xlt_octowords = cpu_to_be16(get_xlt_octo(umrwr->xlt_size));

	if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_XLT) {

		u64 offset = get_xlt_octo(umrwr->offset);



		umr->xlt_offset = cpu_to_be16(offset & 0xffff);

		umr->xlt_offset_47_16 = cpu_to_be32(offset >> 16);

		umr->flags |= MLX5_UMR_TRANSLATION_OFFSET_EN;

	}

	if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_TRANSLATION)

		umr->mkey_mask |= get_umr_update_translation_mask();

	if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS) {

		umr->mkey_mask |= get_umr_update_access_mask(

			!!(MLX5_CAP_GEN(dev->mdev, atomic)),

			!!(MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr)),

			!!(MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr)));

		umr->mkey_mask |= get_umr_update_pd_mask();

	}

	if (wr->send_flags & MLX5_IB_SEND_UMR_ENABLE_MR)

		umr->mkey_mask |= get_umr_enable_mr_mask();

	if (wr->send_flags & MLX5_IB_SEND_UMR_DISABLE_MR)

		umr->mkey_mask |= get_umr_disable_mr_mask();



	if (!wr->num_sge)

		umr->flags |= MLX5_UMR_INLINE;



	return umr_check_mkey_mask(dev, be64_to_cpu(umr->mkey_mask));

}



static u8 get_umr_flags(int acc)

{

	return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC       : 0) |
@@ -761,7 +627,7 @@ static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg, 
	seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 |

				    MLX5_MKEY_BSF_EN | pdn);

	seg->len = cpu_to_be64(length);

	seg->xlt_oct_size = cpu_to_be32(get_xlt_octo(size));

	seg->xlt_oct_size = cpu_to_be32(mlx5r_umr_get_xlt_octo(size));

	seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE);

}
@@ -771,7 +637,7 @@ static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr, 
	memset(umr, 0, sizeof(*umr));



	umr->flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE;

	umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));

	umr->xlt_octowords = cpu_to_be16(mlx5r_umr_get_xlt_octo(size));

	umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE);

	umr->mkey_mask = sig_mkey_mask();

}
@@ -1262,7 +1128,7 @@ static int handle_qpt_reg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp, 


	qp->sq.wr_data[idx] = MLX5_IB_WR_UMR;

	(*ctrl)->imm = cpu_to_be32(umr_wr(wr)->mkey);

	err = set_reg_umr_segment(dev, *seg, wr);

	err = mlx5r_umr_set_umr_ctrl_seg(dev, *seg, wr);

	if (unlikely(err))

		goto out;

	*seg += sizeof(struct mlx5_wqe_umr_ctrl_seg);