Commit f2b77012 authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge branch 'mlx5-next' of git://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux 

Saeed Mahameed says:

====================
mlx5-next 2022-22-02

The following PR includes updates to mlx5-next branch:

Headlines:
==========

1) Jakub cleans up unused static inline functions

2) I did some low level firmware command interface return status changes to
provide the caller with full visibility on the error/status returned by
the Firmware.

3) Use the new command interface in RDMA DEVX usecases to avoid flooding
dmesg with some "expected" user error prone use cases.

4) Moshe also uses the new command interface to grab the specific error
code from MFRL register command to provide the exact error reason for
why SW reset couldn't perform internally in FW.

5) From Mark Bloch: Lag, drop packets in hardware when possible

In active-backup mode the inactive interface's packets are dropped by the
bond device. In switchdev where TC rules are offloaded to the FDB
this can lead to packets being hit in the FDB where without offload
they would have been dropped before reaching TC rules in the kernel.

Create a drop rule to make sure packets on inactive ports are dropped
before reaching the FDB.

Listen on NETDEV_CHANGEUPPER / NETDEV_CHANGEINFODATA events and record
the inactive state and offload accordingly.

* 'mlx5-next' of git://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux:
  net/mlx5: Add clarification on sync reset failure
  net/mlx5: Add reset_state field to MFRL register
  RDMA/mlx5: Use new command interface API
  net/mlx5: cmdif, Refactor error handling and reporting of async commands
  net/mlx5: Use mlx5_cmd_do() in core create_{cq,dct}
  net/mlx5: cmdif, Add new api for command execution
  net/mlx5: cmdif, cmd_check refactoring
  net/mlx5: cmdif, Return value improvements
  net/mlx5: Lag, offload active-backup drops to hardware
  net/mlx5: Lag, record inactive state of bond device
  net/mlx5: Lag, don't use magic numbers for ports
  net/mlx5: Lag, use local variable already defined to access E-Switch
  net/mlx5: E-switch, add drop rule support to ingress ACL
  net/mlx5: E-switch, remove special uplink ingress ACL handling
  net/mlx5: E-Switch, reserve and use same uplink metadata across ports
  net/mlx5: Add ability to insert to specific flow group
  mlx5: remove unused static inlines
====================

Link: https://lore.kernel.org/r/20220223233930.319301-1-saeed@kernel.org


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents 0b9e69e1 45fee8ed

drivers/infiniband/hw/mlx5/devx.c

+35 −26
Original line number Diff line number Diff line
@@ -1055,7 +1055,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)( 
	int cmd_out_len = uverbs_attr_get_len(attrs,

					MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT);

	void *cmd_out;

	int err;

	int err, err2;

	int uid;



	c = devx_ufile2uctx(attrs);
@@ -1076,14 +1076,16 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)( 
		return PTR_ERR(cmd_out);



	MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);

	err = mlx5_cmd_exec(dev->mdev, cmd_in,

	err = mlx5_cmd_do(dev->mdev, cmd_in,

			  uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN),

			  cmd_out, cmd_out_len);

	if (err)

	if (err && err != -EREMOTEIO)

		return err;



	return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out,

	err2 = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out,

			      cmd_out_len);



	return err2 ?: err;

}



static void devx_obj_build_destroy_cmd(void *in, void *out, void *din,
@@ -1457,7 +1459,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)( 
	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];

	struct devx_obj *obj;

	u16 obj_type = 0;

	int err;

	int err, err2 = 0;

	int uid;

	u32 obj_id;

	u16 opcode;
@@ -1497,15 +1499,18 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)( 
		   !is_apu_cq(dev, cmd_in)) {

		obj->flags |= DEVX_OBJ_FLAGS_CQ;

		obj->core_cq.comp = devx_cq_comp;

		err = mlx5_core_create_cq(dev->mdev, &obj->core_cq,

		err = mlx5_create_cq(dev->mdev, &obj->core_cq,

				     cmd_in, cmd_in_len, cmd_out,

				     cmd_out_len);

	} else {

		err = mlx5_cmd_exec(dev->mdev, cmd_in,

				    cmd_in_len,

		err = mlx5_cmd_do(dev->mdev, cmd_in, cmd_in_len,

				  cmd_out, cmd_out_len);

	}



	if (err == -EREMOTEIO)

		err2 = uverbs_copy_to(attrs,

				      MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,

				      cmd_out, cmd_out_len);

	if (err)

		goto obj_free;
@@ -1548,7 +1553,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)( 
			      sizeof(out));

obj_free:

	kfree(obj);

	return err;

	return err2 ?: err;

}



static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)(
@@ -1563,7 +1568,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)( 
		&attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);

	struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);

	void *cmd_out;

	int err;

	int err, err2;

	int uid;



	if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
@@ -1586,14 +1591,16 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)( 
	MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);

	devx_set_umem_valid(cmd_in);



	err = mlx5_cmd_exec(mdev->mdev, cmd_in,

	err = mlx5_cmd_do(mdev->mdev, cmd_in,

			  uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN),

			  cmd_out, cmd_out_len);

	if (err)

	if (err && err != -EREMOTEIO)

		return err;



	return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,

	err2 = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,

			      cmd_out, cmd_out_len);



	return err2 ?: err;

}



static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)(
@@ -1607,7 +1614,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)( 
	struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(

		&attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);

	void *cmd_out;

	int err;

	int err, err2;

	int uid;

	struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
@@ -1629,14 +1636,16 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)( 
		return PTR_ERR(cmd_out);



	MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);

	err = mlx5_cmd_exec(mdev->mdev, cmd_in,

	err = mlx5_cmd_do(mdev->mdev, cmd_in,

			  uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN),

			  cmd_out, cmd_out_len);

	if (err)

	if (err && err != -EREMOTEIO)

		return err;



	return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,

	err2 = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,

			      cmd_out, cmd_out_len);



	return err2 ?: err;

}



struct devx_async_event_queue {

drivers/infiniband/hw/mlx5/mr.c

+14 −1
Original line number Diff line number Diff line
@@ -140,6 +140,19 @@ static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 
	return mlx5_core_destroy_mkey(dev->mdev, mr->mmkey.key);

}



static void create_mkey_warn(struct mlx5_ib_dev *dev, int status, void *out)

{

	if (status == -ENXIO) /* core driver is not available */

		return;



	mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status);

	if (status != -EREMOTEIO) /* driver specific failure */

		return;



	/* Failed in FW, print cmd out failure details */

	mlx5_cmd_out_err(dev->mdev, MLX5_CMD_OP_CREATE_MKEY, 0, out);

}



static void create_mkey_callback(int status, struct mlx5_async_work *context)

{

	struct mlx5_ib_mr *mr =
@@ -149,7 +162,7 @@ static void create_mkey_callback(int status, struct mlx5_async_work *context) 
	unsigned long flags;



	if (status) {

		mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status);

		create_mkey_warn(dev, status, mr->out);

		kfree(mr);

		spin_lock_irqsave(&ent->lock, flags);

		ent->pending--;

drivers/infiniband/hw/mlx5/qp.c

+1 −0
Original line number Diff line number Diff line
@@ -4465,6 +4465,7 @@ static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr, 
		err = mlx5_core_create_dct(dev, &qp->dct.mdct, qp->dct.in,

					   MLX5_ST_SZ_BYTES(create_dct_in), out,

					   sizeof(out));

		err = mlx5_cmd_check(dev->mdev, err, qp->dct.in, out);

		if (err)

			return err;

		resp.dctn = qp->dct.mdct.mqp.qpn;

drivers/infiniband/hw/mlx5/qpc.c

+1 −1
Original line number Diff line number Diff line
@@ -220,7 +220,7 @@ int mlx5_core_create_dct(struct mlx5_ib_dev *dev, struct mlx5_core_dct *dct, 
	init_completion(&dct->drained);

	MLX5_SET(create_dct_in, in, opcode, MLX5_CMD_OP_CREATE_DCT);



	err = mlx5_cmd_exec(dev->mdev, in, inlen, out, outlen);

	err = mlx5_cmd_do(dev->mdev, in, inlen, out, outlen);

	if (err)

		return err;

drivers/net/ethernet/mellanox/mlx5/core/cmd.c

+207 −121
Original line number Diff line number Diff line
@@ -190,10 +190,10 @@ static int verify_block_sig(struct mlx5_cmd_prot_block *block) 
	int xor_len = sizeof(*block) - sizeof(block->data) - 1;



	if (xor8_buf(block, rsvd0_off, xor_len) != 0xff)

		return -EINVAL;

		return -EHWPOISON;



	if (xor8_buf(block, 0, sizeof(*block)) != 0xff)

		return -EINVAL;

		return -EHWPOISON;



	return 0;

}
@@ -259,12 +259,12 @@ static int verify_signature(struct mlx5_cmd_work_ent *ent) 


	sig = xor8_buf(ent->lay, 0, sizeof(*ent->lay));

	if (sig != 0xff)

		return -EINVAL;

		return -EHWPOISON;



	for (i = 0; i < n && next; i++) {

		err = verify_block_sig(next->buf);

		if (err)

			return err;

			return -EHWPOISON;



		next = next->next;

	}
@@ -479,7 +479,7 @@ static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op, 
	case MLX5_CMD_OP_ALLOC_SF:

		*status = MLX5_DRIVER_STATUS_ABORTED;

		*synd = MLX5_DRIVER_SYND;

		return -EIO;

		return -ENOLINK;

	default:

		mlx5_core_err(dev, "Unknown FW command (%d)\n", op);

		return -EINVAL;
@@ -760,44 +760,72 @@ struct mlx5_ifc_mbox_in_bits { 
	u8         reserved_at_40[0x40];

};



void mlx5_cmd_mbox_status(void *out, u8 *status, u32 *syndrome)

void mlx5_cmd_out_err(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out)

{

	*status = MLX5_GET(mbox_out, out, status);

	*syndrome = MLX5_GET(mbox_out, out, syndrome);

	u32 syndrome = MLX5_GET(mbox_out, out, syndrome);

	u8 status = MLX5_GET(mbox_out, out, status);



	mlx5_core_err_rl(dev,

			 "%s(0x%x) op_mod(0x%x) failed, status %s(0x%x), syndrome (0x%x), err(%d)\n",

			 mlx5_command_str(opcode), opcode, op_mod,

			 cmd_status_str(status), status, syndrome, cmd_status_to_err(status));

}

EXPORT_SYMBOL(mlx5_cmd_out_err);



static int mlx5_cmd_check(struct mlx5_core_dev *dev, void *in, void *out)

static void cmd_status_print(struct mlx5_core_dev *dev, void *in, void *out)

{

	u16 opcode, op_mod;

	u32 syndrome;

	u8  status;

	u16 opcode;

	u16 op_mod;

	u16 uid;

	int err;



	mlx5_cmd_mbox_status(out, &status, &syndrome);

	if (!status)

		return 0;

	syndrome = MLX5_GET(mbox_out, out, syndrome);

	status = MLX5_GET(mbox_out, out, status);



	opcode = MLX5_GET(mbox_in, in, opcode);

	op_mod = MLX5_GET(mbox_in, in, op_mod);

	uid    = MLX5_GET(mbox_in, in, uid);



	err = cmd_status_to_err(status);



	if (!uid && opcode != MLX5_CMD_OP_DESTROY_MKEY)

		mlx5_core_err_rl(dev,

			"%s(0x%x) op_mod(0x%x) failed, status %s(0x%x), syndrome (0x%x)\n",

			mlx5_command_str(opcode), opcode, op_mod,

			cmd_status_str(status), status, syndrome);

		mlx5_cmd_out_err(dev, opcode, op_mod, out);

	else

		mlx5_core_dbg(dev,

		      "%s(0x%x) op_mod(0x%x) failed, status %s(0x%x), syndrome (0x%x)\n",

		      mlx5_command_str(opcode),

		      opcode, op_mod,

		      cmd_status_str(status),

		      status,

		      syndrome);

			"%s(0x%x) op_mod(0x%x) uid(%d) failed, status %s(0x%x), syndrome (0x%x), err(%d)\n",

			mlx5_command_str(opcode), opcode, op_mod, uid,

			cmd_status_str(status), status, syndrome, err);

}



int mlx5_cmd_check(struct mlx5_core_dev *dev, int err, void *in, void *out)

{

	/* aborted due to PCI error or via reset flow mlx5_cmd_trigger_completions() */

	if (err == -ENXIO) {

		u16 opcode = MLX5_GET(mbox_in, in, opcode);

		u32 syndrome;

		u8 status;



		/* PCI Error, emulate command return status, for smooth reset */

		err = mlx5_internal_err_ret_value(dev, opcode, &syndrome, &status);

		MLX5_SET(mbox_out, out, status, status);

		MLX5_SET(mbox_out, out, syndrome, syndrome);

		if (!err)

			return 0;

	}



	/* driver or FW delivery error */

	if (err != -EREMOTEIO && err)

		return err;



	/* check outbox status */

	err = cmd_status_to_err(MLX5_GET(mbox_out, out, status));

	if (err)

		cmd_status_print(dev, in, out);



	return cmd_status_to_err(status);

	return err;

}

EXPORT_SYMBOL(mlx5_cmd_check);



static void dump_command(struct mlx5_core_dev *dev,

			 struct mlx5_cmd_work_ent *ent, int input)
@@ -980,13 +1008,7 @@ static void cmd_work_handler(struct work_struct *work) 


	/* Skip sending command to fw if internal error */

	if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, ent->op)) {

		u8 status = 0;

		u32 drv_synd;



		ent->ret = mlx5_internal_err_ret_value(dev, msg_to_opcode(ent->in), &drv_synd, &status);

		MLX5_SET(mbox_out, ent->out, status, status);

		MLX5_SET(mbox_out, ent->out, syndrome, drv_synd);



		ent->ret = -ENXIO;

		mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);

		return;

	}
@@ -1005,6 +1027,31 @@ static void cmd_work_handler(struct work_struct *work) 
	}

}



static int deliv_status_to_err(u8 status)

{

	switch (status) {

	case MLX5_CMD_DELIVERY_STAT_OK:

	case MLX5_DRIVER_STATUS_ABORTED:

		return 0;

	case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR:

	case MLX5_CMD_DELIVERY_STAT_TOK_ERR:

		return -EBADR;

	case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR:

	case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR:

	case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR:

		return -EFAULT; /* Bad address */

	case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR:

	case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR:

	case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR:

	case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR:

		return -ENOMSG;

	case MLX5_CMD_DELIVERY_STAT_FW_ERR:

		return -EIO;

	default:

		return -EINVAL;

	}

}



static const char *deliv_status_to_str(u8 status)

{

	switch (status) {
@@ -1101,16 +1148,27 @@ static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent) 
/*  Notes:

 *    1. Callback functions may not sleep

 *    2. page queue commands do not support asynchrous completion

 *

 * return value in case (!callback):

 *	ret < 0 : Command execution couldn't be submitted by driver

 *	ret > 0 : Command execution couldn't be performed by firmware

 *	ret == 0: Command was executed by FW, Caller must check FW outbox status.

 *

 * return value in case (callback):

 *	ret < 0 : Command execution couldn't be submitted by driver

 *	ret == 0: Command will be submitted to FW for execution

 *		  and the callback will be called for further status updates

 */

static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in,

			   struct mlx5_cmd_msg *out, void *uout, int uout_size,

			   mlx5_cmd_cbk_t callback,

			   void *context, int page_queue, u8 *status,

			   void *context, int page_queue,

			   u8 token, bool force_polling)

{

	struct mlx5_cmd *cmd = &dev->cmd;

	struct mlx5_cmd_work_ent *ent;

	struct mlx5_cmd_stats *stats;

	u8 status = 0;

	int err = 0;

	s64 ds;

	u16 op;
@@ -1141,12 +1199,12 @@ static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in, 
		cmd_work_handler(&ent->work);

	} else if (!queue_work(cmd->wq, &ent->work)) {

		mlx5_core_warn(dev, "failed to queue work\n");

		err = -ENOMEM;

		err = -EALREADY;

		goto out_free;

	}



	if (callback)

		goto out; /* mlx5_cmd_comp_handler() will put(ent) */

		return 0; /* mlx5_cmd_comp_handler() will put(ent) */



	err = wait_func(dev, ent);

	if (err == -ETIMEDOUT || err == -ECANCELED)
@@ -1164,12 +1222,11 @@ static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in, 
	mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_TIME,

			   "fw exec time for %s is %lld nsec\n",

			   mlx5_command_str(op), ds);

	*status = ent->status;



out_free:

	status = ent->status;

	cmd_ent_put(ent);

out:

	return err;

	return err ? : status;

}



static ssize_t dbg_write(struct file *filp, const char __user *buf,
@@ -1612,14 +1669,14 @@ static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool force 
			ent->ts2 = ktime_get_ns();

			memcpy(ent->out->first.data, ent->lay->out, sizeof(ent->lay->out));

			dump_command(dev, ent, 0);

			if (!ent->ret) {



			if (vec & MLX5_TRIGGERED_CMD_COMP)

				ent->ret = -ENXIO;



			if (!ent->ret) { /* Command completed by FW */

				if (!cmd->checksum_disabled)

					ent->ret = verify_signature(ent);

				else

					ent->ret = 0;

				if (vec & MLX5_TRIGGERED_CMD_COMP)

					ent->status = MLX5_DRIVER_STATUS_ABORTED;

				else



				ent->status = ent->lay->status_own >> 1;



				mlx5_core_dbg(dev, "command completed. ret 0x%x, delivery status %s(0x%x)\n",
@@ -1638,21 +1695,18 @@ static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool force 


				callback = ent->callback;

				context = ent->context;

				err = ent->ret;

				if (!err) {

				err = ent->ret ? : ent->status;

				if (err > 0) /* Failed in FW, command didn't execute */

					err = deliv_status_to_err(err);



				if (!err)

					err = mlx5_copy_from_msg(ent->uout,

								 ent->out,

								 ent->uout_size);



					err = err ? err : mlx5_cmd_check(dev,

									ent->in->first.data,

									ent->uout);

				}



				mlx5_free_cmd_msg(dev, ent->out);

				free_msg(dev, ent->in);



				err = err ? err : ent->status;

				/* final consumer is done, release ent */

				cmd_ent_put(ent);

				callback(err, context);
@@ -1719,31 +1773,6 @@ void mlx5_cmd_flush(struct mlx5_core_dev *dev) 
		up(&cmd->sem);

}



static int status_to_err(u8 status)

{

	switch (status) {

	case MLX5_CMD_DELIVERY_STAT_OK:

	case MLX5_DRIVER_STATUS_ABORTED:

		return 0;

	case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR:

	case MLX5_CMD_DELIVERY_STAT_TOK_ERR:

		return -EBADR;

	case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR:

	case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR:

	case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR:

		return -EFAULT; /* Bad address */

	case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR:

	case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR:

	case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR:

	case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR:

		return -ENOMSG;

	case MLX5_CMD_DELIVERY_STAT_FW_ERR:

		return -EIO;

	default:

		return -EINVAL;

	}

}



static struct mlx5_cmd_msg *alloc_msg(struct mlx5_core_dev *dev, int in_size,

				      gfp_t gfp)

{
@@ -1787,27 +1816,23 @@ static int is_manage_pages(void *in) 
	return MLX5_GET(mbox_in, in, opcode) == MLX5_CMD_OP_MANAGE_PAGES;

}



/*  Notes:

 *    1. Callback functions may not sleep

 *    2. Page queue commands do not support asynchrous completion

 */

static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,

		    int out_size, mlx5_cmd_cbk_t callback, void *context,

		    bool force_polling)

{

	struct mlx5_cmd_msg *inb;

	struct mlx5_cmd_msg *outb;

	u16 opcode = MLX5_GET(mbox_in, in, opcode);

	struct mlx5_cmd_msg *inb, *outb;

	int pages_queue;

	gfp_t gfp;

	int err;

	u8 status = 0;

	u32 drv_synd;

	u16 opcode;

	u8 token;

	int err;



	opcode = MLX5_GET(mbox_in, in, opcode);

	if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, opcode)) {

		err = mlx5_internal_err_ret_value(dev, opcode, &drv_synd, &status);

		MLX5_SET(mbox_out, out, status, status);

		MLX5_SET(mbox_out, out, syndrome, drv_synd);

		return err;

	}

	if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, opcode))

		return -ENXIO;



	pages_queue = is_manage_pages(in);

	gfp = callback ? GFP_ATOMIC : GFP_KERNEL;
@@ -1833,39 +1858,108 @@ static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out, 
	}



	err = mlx5_cmd_invoke(dev, inb, outb, out, out_size, callback, context,

			      pages_queue, &status, token, force_polling);

	if (err)

		goto out_out;

			      pages_queue, token, force_polling);

	if (callback)

		return err;



	if (err > 0) /* Failed in FW, command didn't execute */

		err = deliv_status_to_err(err);



	mlx5_core_dbg(dev, "err %d, status %d\n", err, status);

	if (status) {

		err = status_to_err(status);

	if (err)

		goto out_out;

	}



	if (!callback)

	/* command completed by FW */

	err = mlx5_copy_from_msg(out, outb, out_size);



out_out:

	if (!callback)

	mlx5_free_cmd_msg(dev, outb);



out_in:

	if (!callback)

	free_msg(dev, inb);

	return err;

}



/* preserve -EREMOTEIO for outbox.status != OK, otherwise return err as is */

static int cmd_status_err(int err, void *out)

{

	if (err) /* -EREMOTEIO is preserved */

		return err == -EREMOTEIO ? -EIO : err;



	if (MLX5_GET(mbox_out, out, status) != MLX5_CMD_STAT_OK)

		return -EREMOTEIO;



	return 0;

}



/**

 * mlx5_cmd_do - Executes a fw command, wait for completion.

 * Unlike mlx5_cmd_exec, this function will not translate or intercept

 * outbox.status and will return -EREMOTEIO when

 * outbox.status != MLX5_CMD_STAT_OK

 *

 * @dev: mlx5 core device

 * @in: inbox mlx5_ifc command buffer

 * @in_size: inbox buffer size

 * @out: outbox mlx5_ifc buffer

 * @out_size: outbox size

 *

 * @return:

 * -EREMOTEIO : Command executed by FW, outbox.status != MLX5_CMD_STAT_OK.

 *              Caller must check FW outbox status.

 *   0 : Command execution successful, outbox.status == MLX5_CMD_STAT_OK.

 * < 0 : Command execution couldn't be performed by firmware or driver

 */

int mlx5_cmd_do(struct mlx5_core_dev *dev, void *in, int in_size, void *out, int out_size)

{

	int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, false);



	return cmd_status_err(err, out);

}

EXPORT_SYMBOL(mlx5_cmd_do);



/**

 * mlx5_cmd_exec - Executes a fw command, wait for completion

 *

 * @dev: mlx5 core device

 * @in: inbox mlx5_ifc command buffer

 * @in_size: inbox buffer size

 * @out: outbox mlx5_ifc buffer

 * @out_size: outbox size

 *

 * @return: 0 if no error, FW command execution was successful

 *          and outbox status is ok.

 */

int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,

		  int out_size)

{

	int err;

	int err = mlx5_cmd_do(dev, in, in_size, out, out_size);



	err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, false);

	return err ? : mlx5_cmd_check(dev, in, out);

	return mlx5_cmd_check(dev, err, in, out);

}

EXPORT_SYMBOL(mlx5_cmd_exec);



/**

 * mlx5_cmd_exec_polling - Executes a fw command, poll for completion

 *	Needed for driver force teardown, when command completion EQ

 *	will not be available to complete the command

 *

 * @dev: mlx5 core device

 * @in: inbox mlx5_ifc command buffer

 * @in_size: inbox buffer size

 * @out: outbox mlx5_ifc buffer

 * @out_size: outbox size

 *

 * @return: 0 if no error, FW command execution was successful

 *          and outbox status is ok.

 */

int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size,

			  void *out, int out_size)

{

	int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, true);



	err = cmd_status_err(err, out);

	return mlx5_cmd_check(dev, err, in, out);

}

EXPORT_SYMBOL(mlx5_cmd_exec_polling);



void mlx5_cmd_init_async_ctx(struct mlx5_core_dev *dev,

			     struct mlx5_async_ctx *ctx)

{
@@ -1894,8 +1988,10 @@ EXPORT_SYMBOL(mlx5_cmd_cleanup_async_ctx); 
static void mlx5_cmd_exec_cb_handler(int status, void *_work)

{

	struct mlx5_async_work *work = _work;

	struct mlx5_async_ctx *ctx = work->ctx;

	struct mlx5_async_ctx *ctx;



	ctx = work->ctx;

	status = cmd_status_err(status, work->out);

	work->user_callback(status, work);

	if (atomic_dec_and_test(&ctx->num_inflight))

		wake_up(&ctx->wait);
@@ -1909,6 +2005,7 @@ int mlx5_cmd_exec_cb(struct mlx5_async_ctx *ctx, void *in, int in_size, 


	work->ctx = ctx;

	work->user_callback = callback;

	work->out = out;

	if (WARN_ON(!atomic_inc_not_zero(&ctx->num_inflight)))

		return -EIO;

	ret = cmd_exec(ctx->dev, in, in_size, out, out_size,
@@ -1920,17 +2017,6 @@ int mlx5_cmd_exec_cb(struct mlx5_async_ctx *ctx, void *in, int in_size, 
}

EXPORT_SYMBOL(mlx5_cmd_exec_cb);



int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size,

			  void *out, int out_size)

{

	int err;



	err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, true);



	return err ? : mlx5_cmd_check(dev, in, out);

}

EXPORT_SYMBOL(mlx5_cmd_exec_polling);



static void destroy_msg_cache(struct mlx5_core_dev *dev)

{

	struct cmd_msg_cache *ch;