Commit e4e3f24b authored by Leon Romanovsky's avatar Leon Romanovsky Committed by Saeed Mahameed
Browse files

net/mlx5: Provide cpumask at EQ creation phase 



The users of EQ are running their code on different CPUs and with
various affinity patterns. Move the cpumask setting close to their
actual usage.

Signed-off-by: default avatarLeon Romanovsky <leonro@nvidia.com>
Reviewed-by: default avatarShay Drory <shayd@nvidia.com>
Signed-off-by: default avatarSaeed Mahameed <saeedm@nvidia.com>
parent 3b43190b

drivers/infiniband/hw/mlx5/odp.c

+5 −0
Original line number Diff line number Diff line
@@ -1564,7 +1564,12 @@ int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 
		.nent = MLX5_IB_NUM_PF_EQE,

	};

	param.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_FAULT;

	if (!zalloc_cpumask_var(&param.affinity, GFP_KERNEL)) {

		err = -ENOMEM;

		goto err_wq;

	}

	eq->core = mlx5_eq_create_generic(dev->mdev, &param);

	free_cpumask_var(param.affinity);

	if (IS_ERR(eq->core)) {

		err = PTR_ERR(eq->core);

		goto err_wq;

drivers/net/ethernet/mellanox/mlx5/core/eq.c

+20 −7
Original line number Diff line number Diff line
@@ -310,7 +310,7 @@ create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, 
	mlx5_init_fbc(eq->frag_buf.frags, log_eq_stride, log_eq_size, &eq->fbc);

	init_eq_buf(eq);



	eq->irq = mlx5_irq_request(dev, vecidx);

	eq->irq = mlx5_irq_request(dev, vecidx, param->affinity);

	if (IS_ERR(eq->irq)) {

		err = PTR_ERR(eq->irq);

		goto err_buf;
@@ -621,8 +621,11 @@ setup_async_eq(struct mlx5_core_dev *dev, struct mlx5_eq_async *eq, 


	eq->irq_nb.notifier_call = mlx5_eq_async_int;

	spin_lock_init(&eq->lock);

	if (!zalloc_cpumask_var(&param->affinity, GFP_KERNEL))

		return -ENOMEM;



	err = create_async_eq(dev, &eq->core, param);

	free_cpumask_var(param->affinity);

	if (err) {

		mlx5_core_warn(dev, "failed to create %s EQ %d\n", name, err);

		return err;
@@ -740,6 +743,9 @@ mlx5_eq_create_generic(struct mlx5_core_dev *dev, 
	struct mlx5_eq *eq = kvzalloc(sizeof(*eq), GFP_KERNEL);

	int err;



	if (!param->affinity)

		return ERR_PTR(-EINVAL);



	if (!eq)

		return ERR_PTR(-ENOMEM);
@@ -850,16 +856,21 @@ static int create_comp_eqs(struct mlx5_core_dev *dev) 
			.irq_index = vecidx,

			.nent = nent,

		};

		err = create_map_eq(dev, &eq->core, &param);

		if (err) {

			kfree(eq);

			goto clean;



		if (!zalloc_cpumask_var(&param.affinity, GFP_KERNEL)) {

			err = -ENOMEM;

			goto clean_eq;

		}

		cpumask_set_cpu(cpumask_local_spread(i, dev->priv.numa_node),

				param.affinity);

		err = create_map_eq(dev, &eq->core, &param);

		free_cpumask_var(param.affinity);

		if (err)

			goto clean_eq;

		err = mlx5_eq_enable(dev, &eq->core, &eq->irq_nb);

		if (err) {

			destroy_unmap_eq(dev, &eq->core);

			kfree(eq);

			goto clean;

			goto clean_eq;

		}



		mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->core.eqn);
@@ -868,6 +879,8 @@ static int create_comp_eqs(struct mlx5_core_dev *dev) 
	}



	return 0;

clean_eq:

	kfree(eq);

clean:

	destroy_comp_eqs(dev);

	return err;

drivers/net/ethernet/mellanox/mlx5/core/mlx5_irq.h

+2 −1
Original line number Diff line number Diff line
@@ -20,7 +20,8 @@ int mlx5_set_msix_vec_count(struct mlx5_core_dev *dev, int devfn, 
			    int msix_vec_count);

int mlx5_get_default_msix_vec_count(struct mlx5_core_dev *dev, int num_vfs);



struct mlx5_irq *mlx5_irq_request(struct mlx5_core_dev *dev, int vecidx);

struct mlx5_irq *mlx5_irq_request(struct mlx5_core_dev *dev, int vecidx,

				  struct cpumask *affinity);

void mlx5_irq_release(struct mlx5_irq *irq);

int mlx5_irq_attach_nb(struct mlx5_irq *irq, struct notifier_block *nb);

int mlx5_irq_detach_nb(struct mlx5_irq *irq, struct notifier_block *nb);

drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c

+21 −82
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ struct mlx5_irq { 
	struct atomic_notifier_head nh;

	cpumask_var_t mask;

	char name[MLX5_MAX_IRQ_NAME];

	spinlock_t lock; /* protects affinity assignment */

	struct kref kref;

	int irqn;

};
@@ -153,6 +154,8 @@ static void irq_release(struct kref *kref) 
{

	struct mlx5_irq *irq = container_of(kref, struct mlx5_irq, kref);



	irq_set_affinity_hint(irq->irqn, NULL);

	free_cpumask_var(irq->mask);

	free_irq(irq->irqn, &irq->nh);

}
@@ -189,7 +192,8 @@ void mlx5_irq_release(struct mlx5_irq *irq) 
	irq_put(irq);

}



struct mlx5_irq *mlx5_irq_request(struct mlx5_core_dev *dev, int vecidx)

struct mlx5_irq *mlx5_irq_request(struct mlx5_core_dev *dev, int vecidx,

				  struct cpumask *affinity)

{

	struct mlx5_irq_table *table = mlx5_irq_table_get(dev);

	struct mlx5_irq *irq = &table->irq[vecidx];
@@ -199,6 +203,16 @@ struct mlx5_irq *mlx5_irq_request(struct mlx5_core_dev *dev, int vecidx) 
	if (!err)

		return ERR_PTR(-ENOENT);



	spin_lock(&irq->lock);

	if (!cpumask_empty(irq->mask)) {

		/* already configured */

		spin_unlock(&irq->lock);

		return irq;

	}



	cpumask_copy(irq->mask, affinity);

	irq_set_affinity_hint(irq->irqn, irq->mask);

	spin_unlock(&irq->lock);

	return irq;

}
@@ -239,6 +253,12 @@ static int request_irqs(struct mlx5_core_dev *dev, int nvec) 
			mlx5_core_err(dev, "Failed to request irq\n");

			goto err_request_irq;

		}

		if (!zalloc_cpumask_var(&irq->mask, GFP_KERNEL)) {

			mlx5_core_warn(dev, "zalloc_cpumask_var failed\n");

			err = -ENOMEM;

			goto err_request_irq;

		}

		spin_lock_init(&irq->lock);

		kref_init(&irq->kref);

	}

	return 0;
@@ -294,69 +314,6 @@ static int irq_set_rmap(struct mlx5_core_dev *mdev) 
	return err;

}



/* Completion IRQ vectors */



static int set_comp_irq_affinity_hint(struct mlx5_core_dev *mdev, int i)

{

	int vecidx = MLX5_IRQ_VEC_COMP_BASE + i;

	struct mlx5_irq *irq;



	irq = mlx5_irq_get(mdev, vecidx);

	if (!zalloc_cpumask_var(&irq->mask, GFP_KERNEL)) {

		mlx5_core_warn(mdev, "zalloc_cpumask_var failed");

		return -ENOMEM;

	}



	cpumask_set_cpu(cpumask_local_spread(i, mdev->priv.numa_node),

			irq->mask);

	if (IS_ENABLED(CONFIG_SMP) &&

	    irq_set_affinity_hint(irq->irqn, irq->mask))

		mlx5_core_warn(mdev, "irq_set_affinity_hint failed, irq 0x%.4x",

			       irq->irqn);



	return 0;

}



static void clear_comp_irq_affinity_hint(struct mlx5_core_dev *mdev, int i)

{

	int vecidx = MLX5_IRQ_VEC_COMP_BASE + i;

	struct mlx5_irq *irq;



	irq = mlx5_irq_get(mdev, vecidx);

	irq_set_affinity_hint(irq->irqn, NULL);

	free_cpumask_var(irq->mask);

}



static int set_comp_irq_affinity_hints(struct mlx5_core_dev *mdev)

{

	int nvec = mlx5_irq_get_num_comp(mdev->priv.irq_table);

	int err;

	int i;



	for (i = 0; i < nvec; i++) {

		err = set_comp_irq_affinity_hint(mdev, i);

		if (err)

			goto err_out;

	}



	return 0;



err_out:

	for (i--; i >= 0; i--)

		clear_comp_irq_affinity_hint(mdev, i);



	return err;

}



static void clear_comp_irqs_affinity_hints(struct mlx5_core_dev *mdev)

{

	int nvec = mlx5_irq_get_num_comp(mdev->priv.irq_table);

	int i;



	for (i = 0; i < nvec; i++)

		clear_comp_irq_affinity_hint(mdev, i);

}



struct cpumask *

mlx5_irq_get_affinity_mask(struct mlx5_irq_table *irq_table, int vecidx)

{
@@ -370,15 +327,6 @@ struct cpu_rmap *mlx5_irq_get_rmap(struct mlx5_irq_table *irq_table) 
}

#endif



static void unrequest_irqs(struct mlx5_core_dev *dev)

{

	struct mlx5_irq_table *table = dev->priv.irq_table;

	int i;



	for (i = 0; i < table->nvec; i++)

		irq_put(mlx5_irq_get(dev, i));

}



int mlx5_irq_table_create(struct mlx5_core_dev *dev)

{

	struct mlx5_priv *priv = &dev->priv;
@@ -419,16 +367,8 @@ int mlx5_irq_table_create(struct mlx5_core_dev *dev) 
	if (err)

		goto err_request_irqs;



	err = set_comp_irq_affinity_hints(dev);

	if (err) {

		mlx5_core_err(dev, "Failed to alloc affinity hint cpumask\n");

		goto err_set_affinity;

	}



	return 0;



err_set_affinity:

	unrequest_irqs(dev);

err_request_irqs:

	irq_clear_rmap(dev);

err_set_rmap:
@@ -451,7 +391,6 @@ void mlx5_irq_table_destroy(struct mlx5_core_dev *dev) 
	 * which should be called after alloc_irq but before request_irq.

	 */

	irq_clear_rmap(dev);

	clear_comp_irqs_affinity_hints(dev);

	for (i = 0; i < table->nvec; i++)

		irq_release(&mlx5_irq_get(dev, i)->kref);

	pci_free_irq_vectors(dev->pdev);

include/linux/mlx5/eq.h

+1 −0
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ struct mlx5_eq_param { 
	u8             irq_index;

	int            nent;

	u64            mask[4];

	cpumask_var_t  affinity;

};



struct mlx5_eq *