Commit 8ad11fb6 authored by Jack Morgenstein's avatar Jack Morgenstein Committed by Roland Dreier
Browse files

IB/mlx4: Implement FMRs 



Implement FMRs for mlx4.  This is an adaptation of code from mthca.

Signed-off-by: default avatarJack Morgenstein <jackm@dev.mellanox.co.il>
Signed-off-by: default avatarMichael S. Tsirkin <mst@dev.mellanox.co.il>
Signed-off-by: default avatarRoland Dreier <rolandd@cisco.com>
parent d7bb58fb

drivers/infiniband/hw/mlx4/main.c

+5 −0
Original line number Diff line number Diff line
@@ -607,6 +607,11 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) 
	ibdev->ib_dev.detach_mcast	= mlx4_ib_mcg_detach;

	ibdev->ib_dev.process_mad	= mlx4_ib_process_mad;



	ibdev->ib_dev.alloc_fmr		= mlx4_ib_fmr_alloc;

	ibdev->ib_dev.map_phys_fmr	= mlx4_ib_map_phys_fmr;

	ibdev->ib_dev.unmap_fmr		= mlx4_ib_unmap_fmr;

	ibdev->ib_dev.dealloc_fmr	= mlx4_ib_fmr_dealloc;



	if (init_node_data(ibdev))

		goto err_map;

drivers/infiniband/hw/mlx4/mlx4_ib.h

+16 −0
Original line number Diff line number Diff line
@@ -93,6 +93,11 @@ struct mlx4_ib_mr { 
	struct ib_umem	       *umem;

};



struct mlx4_ib_fmr {

	struct ib_fmr           ibfmr;

	struct mlx4_fmr         mfmr;

};



struct mlx4_ib_wq {

	u64		       *wrid;

	spinlock_t		lock;
@@ -199,6 +204,10 @@ static inline struct mlx4_ib_mr *to_mmr(struct ib_mr *ibmr) 
	return container_of(ibmr, struct mlx4_ib_mr, ibmr);

}



static inline struct mlx4_ib_fmr *to_mfmr(struct ib_fmr *ibfmr)

{

	return container_of(ibfmr, struct mlx4_ib_fmr, ibfmr);

}

static inline struct mlx4_ib_qp *to_mqp(struct ib_qp *ibqp)

{

	return container_of(ibqp, struct mlx4_ib_qp, ibqp);
@@ -284,6 +293,13 @@ int mlx4_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, 
int mlx4_ib_mad_init(struct mlx4_ib_dev *dev);

void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev);



struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int mr_access_flags,

				  struct ib_fmr_attr *fmr_attr);

int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, int npages,

			 u64 iova);

int mlx4_ib_unmap_fmr(struct list_head *fmr_list);

int mlx4_ib_fmr_dealloc(struct ib_fmr *fmr);



static inline int mlx4_ib_ah_grh_present(struct mlx4_ib_ah *ah)

{

	return !!(ah->av.g_slid & 0x80);

drivers/infiniband/hw/mlx4/mr.c

+93 −0
Original line number Diff line number Diff line
@@ -181,3 +181,96 @@ int mlx4_ib_dereg_mr(struct ib_mr *ibmr) 


	return 0;

}



struct ib_fmr *mlx4_ib_fmr_alloc(struct ib_pd *pd, int acc,

				 struct ib_fmr_attr *fmr_attr)

{

	struct mlx4_ib_dev *dev = to_mdev(pd->device);

	struct mlx4_ib_fmr *fmr;

	int err = -ENOMEM;



	fmr = kmalloc(sizeof *fmr, GFP_KERNEL);

	if (!fmr)

		return ERR_PTR(-ENOMEM);



	err = mlx4_fmr_alloc(dev->dev, to_mpd(pd)->pdn, convert_access(acc),

			     fmr_attr->max_pages, fmr_attr->max_maps,

			     fmr_attr->page_shift, &fmr->mfmr);

	if (err)

		goto err_free;



	err = mlx4_mr_enable(to_mdev(pd->device)->dev, &fmr->mfmr.mr);

	if (err)

		goto err_mr;



	fmr->ibfmr.rkey = fmr->ibfmr.lkey = fmr->mfmr.mr.key;



	return &fmr->ibfmr;



err_mr:

	mlx4_mr_free(to_mdev(pd->device)->dev, &fmr->mfmr.mr);



err_free:

	kfree(fmr);



	return ERR_PTR(err);

}



int mlx4_ib_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list,

		      int npages, u64 iova)

{

	struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);

	struct mlx4_ib_dev *dev = to_mdev(ifmr->ibfmr.device);



	return mlx4_map_phys_fmr(dev->dev, &ifmr->mfmr, page_list, npages, iova,

				 &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey);

}



int mlx4_ib_unmap_fmr(struct list_head *fmr_list)

{

	struct ib_fmr *ibfmr;

	int err;

	struct mlx4_dev *mdev = NULL;



	list_for_each_entry(ibfmr, fmr_list, list) {

		if (mdev && to_mdev(ibfmr->device)->dev != mdev)

			return -EINVAL;

		mdev = to_mdev(ibfmr->device)->dev;

	}



	if (!mdev)

		return 0;



	list_for_each_entry(ibfmr, fmr_list, list) {

		struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);



		mlx4_fmr_unmap(mdev, &ifmr->mfmr, &ifmr->ibfmr.lkey, &ifmr->ibfmr.rkey);

	}



	/*

	 * Make sure all MPT status updates are visible before issuing

	 * SYNC_TPT firmware command.

	 */

	wmb();



	err = mlx4_SYNC_TPT(mdev);

	if (err)

		printk(KERN_WARNING "mlx4_ib: SYNC_TPT error %d when "

		       "unmapping FMRs\n", err);



	return 0;

}



int mlx4_ib_fmr_dealloc(struct ib_fmr *ibfmr)

{

	struct mlx4_ib_fmr *ifmr = to_mfmr(ibfmr);

	struct mlx4_ib_dev *dev = to_mdev(ibfmr->device);

	int err;



	err = mlx4_fmr_free(dev->dev, &ifmr->mfmr);



	if (!err)

		kfree(ifmr);



	return err;

}

drivers/net/mlx4/mr.c

+165 −0
Original line number Diff line number Diff line
@@ -68,6 +68,9 @@ struct mlx4_mpt_entry { 


#define MLX4_MTT_FLAG_PRESENT		1



#define MLX4_MPT_STATUS_SW		0xF0

#define MLX4_MPT_STATUS_HW		0x00



static u32 mlx4_buddy_alloc(struct mlx4_buddy *buddy, int order)

{

	int o;
@@ -469,3 +472,165 @@ void mlx4_cleanup_mr_table(struct mlx4_dev *dev) 
	mlx4_buddy_cleanup(&mr_table->mtt_buddy);

	mlx4_bitmap_cleanup(&mr_table->mpt_bitmap);

}



static inline int mlx4_check_fmr(struct mlx4_fmr *fmr, u64 *page_list,

				  int npages, u64 iova)

{

	int i, page_mask;



	if (npages > fmr->max_pages)

		return -EINVAL;



	page_mask = (1 << fmr->page_shift) - 1;



	/* We are getting page lists, so va must be page aligned. */

	if (iova & page_mask)

		return -EINVAL;



	/* Trust the user not to pass misaligned data in page_list */

	if (0)

		for (i = 0; i < npages; ++i) {

			if (page_list[i] & ~page_mask)

				return -EINVAL;

		}



	if (fmr->maps >= fmr->max_maps)

		return -EINVAL;



	return 0;

}



int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list,

		      int npages, u64 iova, u32 *lkey, u32 *rkey)

{

	u32 key;

	int i, err;



	err = mlx4_check_fmr(fmr, page_list, npages, iova);

	if (err)

		return err;



	++fmr->maps;



	key = key_to_hw_index(fmr->mr.key);

	key += dev->caps.num_mpts;

	*lkey = *rkey = fmr->mr.key = hw_index_to_key(key);



	*(u8 *) fmr->mpt = MLX4_MPT_STATUS_SW;



	/* Make sure MPT status is visible before writing MTT entries */

	wmb();



	for (i = 0; i < npages; ++i)

		fmr->mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT);



	dma_sync_single(&dev->pdev->dev, fmr->dma_handle,

			npages * sizeof(u64), DMA_TO_DEVICE);



	fmr->mpt->key    = cpu_to_be32(key);

	fmr->mpt->lkey   = cpu_to_be32(key);

	fmr->mpt->length = cpu_to_be64(npages * (1ull << fmr->page_shift));

	fmr->mpt->start  = cpu_to_be64(iova);



	/* Make MTT entries are visible before setting MPT status */

	wmb();



	*(u8 *) fmr->mpt = MLX4_MPT_STATUS_HW;



	/* Make sure MPT status is visible before consumer can use FMR */

	wmb();



	return 0;

}

EXPORT_SYMBOL_GPL(mlx4_map_phys_fmr);



int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages,

		   int max_maps, u8 page_shift, struct mlx4_fmr *fmr)

{

	struct mlx4_priv *priv = mlx4_priv(dev);

	u64 mtt_seg;

	int err = -ENOMEM;



	if (page_shift < 12 || page_shift >= 32)

		return -EINVAL;



	/* All MTTs must fit in the same page */

	if (max_pages * sizeof *fmr->mtts > PAGE_SIZE)

		return -EINVAL;



	fmr->page_shift = page_shift;

	fmr->max_pages  = max_pages;

	fmr->max_maps   = max_maps;

	fmr->maps = 0;



	err = mlx4_mr_alloc(dev, pd, 0, 0, access, max_pages,

			    page_shift, &fmr->mr);

	if (err)

		return err;



	mtt_seg = fmr->mr.mtt.first_seg * dev->caps.mtt_entry_sz;



	fmr->mtts = mlx4_table_find(&priv->mr_table.mtt_table,

				    fmr->mr.mtt.first_seg,

				    &fmr->dma_handle);

	if (!fmr->mtts) {

		err = -ENOMEM;

		goto err_free;

	}



	fmr->mpt = mlx4_table_find(&priv->mr_table.dmpt_table,

				    key_to_hw_index(fmr->mr.key), NULL);

	if (!fmr->mpt) {

		err = -ENOMEM;

		goto err_free;

	}



	return 0;



err_free:

	mlx4_mr_free(dev, &fmr->mr);

	return err;

}

EXPORT_SYMBOL_GPL(mlx4_fmr_alloc);



int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr)

{

	return mlx4_mr_enable(dev, &fmr->mr);

}

EXPORT_SYMBOL_GPL(mlx4_fmr_enable);



void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr,

		    u32 *lkey, u32 *rkey)

{

	u32 key;



	if (!fmr->maps)

		return;



	key = key_to_hw_index(fmr->mr.key);

	key &= dev->caps.num_mpts - 1;

	*lkey = *rkey = fmr->mr.key = hw_index_to_key(key);



	fmr->maps = 0;



	*(u8 *) fmr->mpt = MLX4_MPT_STATUS_SW;

}

EXPORT_SYMBOL_GPL(mlx4_fmr_unmap);



int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr)

{

	if (fmr->maps)

		return -EBUSY;



	fmr->mr.enabled = 0;

	mlx4_mr_free(dev, &fmr->mr);



	return 0;

}

EXPORT_SYMBOL_GPL(mlx4_fmr_free);



int mlx4_SYNC_TPT(struct mlx4_dev *dev)

{

	return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_SYNC_TPT, 1000);

}

EXPORT_SYMBOL_GPL(mlx4_SYNC_TPT);

include/linux/mlx4/device.h

+21 −0
Original line number Diff line number Diff line
@@ -214,6 +214,17 @@ struct mlx4_mr { 
	int			enabled;

};



struct mlx4_fmr {

	struct mlx4_mr		mr;

	struct mlx4_mpt_entry  *mpt;

	__be64		       *mtts;

	dma_addr_t		dma_handle;

	int			max_pages;

	int			max_maps;

	int			maps;

	u8			page_shift;

};



struct mlx4_uar {

	unsigned long		pfn;

	int			index;
@@ -337,4 +348,14 @@ int mlx4_CLOSE_PORT(struct mlx4_dev *dev, int port); 
int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16]);

int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16]);



int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list,

		      int npages, u64 iova, u32 *lkey, u32 *rkey);

int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages,

		   int max_maps, u8 page_shift, struct mlx4_fmr *fmr);

int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr);

void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr,

		    u32 *lkey, u32 *rkey);

int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr);

int mlx4_SYNC_TPT(struct mlx4_dev *dev);



#endif /* MLX4_DEVICE_H */