Commit b9584517 authored by Michael Guralnik's avatar Michael Guralnik Committed by Jason Gunthorpe
Browse files

RDMA/mlx5: Change the cache structure to an RB-tree 

Currently, the cache structure is a static linear array. Therefore, his
size is limited to the number of entries in it and is not expandable.  The
entries are dedicated to mkeys of size 2^x and no access_flags. Mkeys with
different properties are not cacheable.

In this patch, we change the cache structure to an RB-tree.  This will
allow to extend the cache to support more entries with different mkey
properties.

Link: https://lore.kernel.org/r/20230125222807.6921-4-michaelgur@nvidia.com


Signed-off-by: default avatarMichael Guralnik <michaelgur@nvidia.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@nvidia.com>
parent 18b1746b

drivers/infiniband/hw/mlx5/mlx5_ib.h

+9 −2
Original line number Diff line number Diff line
@@ -741,6 +741,8 @@ struct mlx5_cache_ent { 
	u32			access_mode;

	unsigned int		ndescs;



	struct rb_node		node;



	u8 disabled:1;

	u8 fill_to_high_water:1;
@@ -770,8 +772,9 @@ struct mlx5r_async_create_mkey { 


struct mlx5_mkey_cache {

	struct workqueue_struct *wq;

	struct mlx5_cache_ent	ent[MAX_MKEY_CACHE_ENTRIES];

	struct dentry		*root;

	struct rb_root		rb_root;

	struct mutex		rb_lock;

	struct dentry		*fs_root;

	unsigned long		last_add;

};
@@ -1316,11 +1319,15 @@ void mlx5_ib_copy_pas(u64 *old, u64 *new, int step, int num); 
int mlx5_ib_get_cqe_size(struct ib_cq *ibcq);

int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev);

int mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev);

struct mlx5_cache_ent *mlx5r_cache_create_ent(struct mlx5_ib_dev *dev,

					      int order);



struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,

				       struct mlx5_cache_ent *ent,

				       int access_flags);



struct mlx5_ib_mr *mlx5_mr_cache_alloc_order(struct mlx5_ib_dev *dev, u32 order,

					     int access_flags);

int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,

			    struct ib_mr_status *mr_status);

struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,

drivers/infiniband/hw/mlx5/mr.c

+120 −40
Original line number Diff line number Diff line
@@ -515,18 +515,22 @@ static const struct file_operations limit_fops = { 


static bool someone_adding(struct mlx5_mkey_cache *cache)

{

	unsigned int i;



	for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) {

		struct mlx5_cache_ent *ent = &cache->ent[i];

	struct mlx5_cache_ent *ent;

	struct rb_node *node;

	bool ret;



	mutex_lock(&cache->rb_lock);

	for (node = rb_first(&cache->rb_root); node; node = rb_next(node)) {

		ent = rb_entry(node, struct mlx5_cache_ent, node);

		xa_lock_irq(&ent->mkeys);

		ret = ent->stored < ent->limit;

		xa_unlock_irq(&ent->mkeys);

		if (ret)

		if (ret) {

			mutex_unlock(&cache->rb_lock);

			return true;

		}

	}

	mutex_unlock(&cache->rb_lock);

	return false;

}
@@ -637,6 +641,59 @@ static void delayed_cache_work_func(struct work_struct *work) 
	__cache_work_func(ent);

}



static int mlx5_cache_ent_insert(struct mlx5_mkey_cache *cache,

				 struct mlx5_cache_ent *ent)

{

	struct rb_node **new = &cache->rb_root.rb_node, *parent = NULL;

	struct mlx5_cache_ent *cur;



	mutex_lock(&cache->rb_lock);

	/* Figure out where to put new node */

	while (*new) {

		cur = rb_entry(*new, struct mlx5_cache_ent, node);

		parent = *new;

		if (ent->order < cur->order)

			new = &((*new)->rb_left);

		if (ent->order > cur->order)

			new = &((*new)->rb_right);

		if (ent->order == cur->order) {

			mutex_unlock(&cache->rb_lock);

			return -EEXIST;

		}

	}



	/* Add new node and rebalance tree. */

	rb_link_node(&ent->node, parent, new);

	rb_insert_color(&ent->node, &cache->rb_root);



	mutex_unlock(&cache->rb_lock);

	return 0;

}



static struct mlx5_cache_ent *mkey_cache_ent_from_order(struct mlx5_ib_dev *dev,

							unsigned int order)

{

	struct rb_node *node = dev->cache.rb_root.rb_node;

	struct mlx5_cache_ent *cur, *smallest = NULL;



	/*

	 * Find the smallest ent with order >= requested_order.

	 */

	while (node) {

		cur = rb_entry(node, struct mlx5_cache_ent, node);

		if (cur->order > order) {

			smallest = cur;

			node = node->rb_left;

		}

		if (cur->order < order)

			node = node->rb_right;

		if (cur->order == order)

			return cur;

	}



	return smallest;

}



struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,

				       struct mlx5_cache_ent *ent,

				       int access_flags)
@@ -677,10 +734,16 @@ struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, 
	return mr;

}



static void clean_keys(struct mlx5_ib_dev *dev, int c)

struct mlx5_ib_mr *mlx5_mr_cache_alloc_order(struct mlx5_ib_dev *dev,

					     u32 order, int access_flags)

{

	struct mlx5_cache_ent *ent = mkey_cache_ent_from_order(dev, order);



	return mlx5_mr_cache_alloc(dev, ent, access_flags);

}



static void clean_keys(struct mlx5_ib_dev *dev, struct mlx5_cache_ent *ent)

{

	struct mlx5_mkey_cache *cache = &dev->cache;

	struct mlx5_cache_ent *ent = &cache->ent[c];

	u32 mkey;



	cancel_delayed_work(&ent->dwork);
@@ -699,8 +762,8 @@ static void mlx5_mkey_cache_debugfs_cleanup(struct mlx5_ib_dev *dev) 
	if (!mlx5_debugfs_root || dev->is_rep)

		return;



	debugfs_remove_recursive(dev->cache.root);

	dev->cache.root = NULL;

	debugfs_remove_recursive(dev->cache.fs_root);

	dev->cache.fs_root = NULL;

}



static void mlx5_mkey_cache_debugfs_init(struct mlx5_ib_dev *dev)
@@ -713,12 +776,13 @@ static void mlx5_mkey_cache_debugfs_init(struct mlx5_ib_dev *dev) 
	if (!mlx5_debugfs_root || dev->is_rep)

		return;



	cache->root = debugfs_create_dir("mr_cache", mlx5_debugfs_get_dev_root(dev->mdev));

	dir = mlx5_debugfs_get_dev_root(dev->mdev);

	cache->fs_root = debugfs_create_dir("mr_cache", dir);



	for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) {

		ent = &cache->ent[i];

		ent = mkey_cache_ent_from_order(dev, i);

		sprintf(ent->name, "%d", ent->order);

		dir = debugfs_create_dir(ent->name, cache->root);

		dir = debugfs_create_dir(ent->name, cache->fs_root);

		debugfs_create_file("size", 0600, dir, ent, &size_fops);

		debugfs_create_file("limit", 0600, dir, ent, &limit_fops);

		debugfs_create_ulong("cur", 0400, dir, &ent->stored);
@@ -733,6 +797,30 @@ static void delay_time_func(struct timer_list *t) 
	WRITE_ONCE(dev->fill_delay, 0);

}



struct mlx5_cache_ent *mlx5r_cache_create_ent(struct mlx5_ib_dev *dev,

					      int order)

{

	struct mlx5_cache_ent *ent;

	int ret;



	ent = kzalloc(sizeof(*ent), GFP_KERNEL);

	if (!ent)

		return ERR_PTR(-ENOMEM);



	xa_init_flags(&ent->mkeys, XA_FLAGS_LOCK_IRQ);

	ent->order = order;

	ent->dev = dev;



	INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);



	ret = mlx5_cache_ent_insert(&dev->cache, ent);

	if (ret) {

		kfree(ent);

		return ERR_PTR(ret);

	}

	return ent;

}



int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev)

{

	struct mlx5_mkey_cache *cache = &dev->cache;
@@ -740,6 +828,8 @@ int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev) 
	int i;



	mutex_init(&dev->slow_path_mutex);

	mutex_init(&dev->cache.rb_lock);

	dev->cache.rb_root = RB_ROOT;

	cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM);

	if (!cache->wq) {

		mlx5_ib_warn(dev, "failed to create work queue\n");
@@ -749,13 +839,7 @@ int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev) 
	mlx5_cmd_init_async_ctx(dev->mdev, &dev->async_ctx);

	timer_setup(&dev->delay_timer, delay_time_func, 0);

	for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) {

		ent = &cache->ent[i];

		xa_init_flags(&ent->mkeys, XA_FLAGS_LOCK_IRQ);

		ent->order = i + 2;

		ent->dev = dev;

		ent->limit = 0;



		INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func);

		ent = mlx5r_cache_create_ent(dev, i);



		if (i > MKEY_CACHE_LAST_STD_ENTRY) {

			mlx5_odp_init_mkey_cache_entry(ent);
@@ -785,14 +869,16 @@ int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev) 


int mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev)

{

	unsigned int i;

	struct rb_root *root = &dev->cache.rb_root;

	struct mlx5_cache_ent *ent;

	struct rb_node *node;



	if (!dev->cache.wq)

		return 0;



	for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++) {

		struct mlx5_cache_ent *ent = &dev->cache.ent[i];



	mutex_lock(&dev->cache.rb_lock);

	for (node = rb_first(root); node; node = rb_next(node)) {

		ent = rb_entry(node, struct mlx5_cache_ent, node);

		xa_lock_irq(&ent->mkeys);

		ent->disabled = true;

		xa_unlock_irq(&ent->mkeys);
@@ -802,8 +888,15 @@ int mlx5_mkey_cache_cleanup(struct mlx5_ib_dev *dev) 
	mlx5_mkey_cache_debugfs_cleanup(dev);

	mlx5_cmd_cleanup_async_ctx(&dev->async_ctx);



	for (i = 0; i < MAX_MKEY_CACHE_ENTRIES; i++)

		clean_keys(dev, i);

	node = rb_first(root);

	while (node) {

		ent = rb_entry(node, struct mlx5_cache_ent, node);

		node = rb_next(node);

		clean_keys(dev, ent);

		rb_erase(&ent->node, root);

		kfree(ent);

	}

	mutex_unlock(&dev->cache.rb_lock);



	destroy_workqueue(dev->cache.wq);

	del_timer_sync(&dev->delay_timer);
@@ -876,19 +969,6 @@ static int mkey_cache_max_order(struct mlx5_ib_dev *dev) 
	return MLX5_MAX_UMR_SHIFT;

}



static struct mlx5_cache_ent *mkey_cache_ent_from_order(struct mlx5_ib_dev *dev,

							unsigned int order)

{

	struct mlx5_mkey_cache *cache = &dev->cache;



	if (order < cache->ent[0].order)

		return &cache->ent[0];

	order = order - cache->ent[0].order;

	if (order > MKEY_CACHE_LAST_STD_ENTRY)

		return NULL;

	return &cache->ent[order];

}



static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,

			  u64 length, int access_flags, u64 iova)

{

drivers/infiniband/hw/mlx5/odp.c

+3 −5
Original line number Diff line number Diff line
@@ -419,8 +419,7 @@ static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr, 
		return ERR_CAST(odp);



	BUILD_BUG_ON(order > MKEY_CACHE_LAST_STD_ENTRY);

	mr = mlx5_mr_cache_alloc(dev, &dev->cache.ent[order],

				 imr->access_flags);

	mr = mlx5_mr_cache_alloc_order(dev, order, imr->access_flags);

	if (IS_ERR(mr)) {

		ib_umem_odp_release(odp);

		return mr;
@@ -494,8 +493,7 @@ struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, 
	if (IS_ERR(umem_odp))

		return ERR_CAST(umem_odp);



	imr = mlx5_mr_cache_alloc(dev,

				  &dev->cache.ent[MLX5_IMR_KSM_CACHE_ENTRY],

	imr = mlx5_mr_cache_alloc_order(dev, MLX5_IMR_KSM_CACHE_ENTRY,

					access_flags);

	if (IS_ERR(imr)) {

		ib_umem_odp_release(umem_odp);