Commit 7930d855 authored by Dave Jiang's avatar Dave Jiang Committed by Vinod Koul
Browse files

dmaengine: idxd: add knob for enqcmds retries 



Add a sysfs knob to allow tuning of retries for the kernel ENQCMDS
descriptor submission. While on host, it is not as likely that ENQCMDS
return busy during normal operations due to the driver controlling the
number of descriptors allocated for submission. However, when the driver is
operating as a guest driver, the chance of retry goes up significantly due
to sharing a wq with multiple VMs. A default value is provided with the
system admin being able to tune the value on a per WQ basis.

Suggested-by: default avatarSanjay Kumar <sanjay.k.kumar@intel.com>
Signed-off-by: default avatarDave Jiang <dave.jiang@intel.com>
Link: https://lore.kernel.org/r/163820629464.2702134.7577370098568297574.stgit@djiang5-desk3.ch.intel.com


Signed-off-by: default avatarVinod Koul <vkoul@kernel.org>
parent 92452a72

Documentation/ABI/stable/sysfs-driver-dma-idxd

+7 −0
Original line number Diff line number Diff line
@@ -220,6 +220,13 @@ Contact: dmaengine@vger.kernel.org 
Description:	Show the current number of entries in this WQ if WQ Occupancy

		Support bit WQ capabilities is 1.



What:		/sys/bus/dsa/devices/wq<m>.<n>/enqcmds_retries

Date		Oct 29, 2021

KernelVersion:	5.17.0

Contact:	dmaengine@vger.kernel.org

Description:	Indicate the number of retires for an enqcmds submission on a shared wq.

		A max value to set attribute is capped at 64.



What:           /sys/bus/dsa/devices/engine<m>.<n>/group_id

Date:           Oct 25, 2019

KernelVersion:  5.6.0

drivers/dma/idxd/device.c

+1 −0
Original line number Diff line number Diff line
@@ -387,6 +387,7 @@ static void idxd_wq_disable_cleanup(struct idxd_wq *wq) 
	wq->threshold = 0;

	wq->priority = 0;

	wq->ats_dis = 0;

	wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES;

	clear_bit(WQ_FLAG_DEDICATED, &wq->flags);

	clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags);

	memset(wq->name, 0, WQ_NAME_SIZE);

drivers/dma/idxd/idxd.h

+5 −0
Original line number Diff line number Diff line
@@ -52,6 +52,9 @@ enum idxd_type { 
#define IDXD_NAME_SIZE		128

#define IDXD_PMU_EVENT_MAX	64



#define IDXD_ENQCMDS_RETRIES		32

#define IDXD_ENQCMDS_MAX_RETRIES	64



struct idxd_device_driver {

	const char *name;

	enum idxd_dev_type *type;
@@ -173,6 +176,7 @@ struct idxd_dma_chan { 
struct idxd_wq {

	void __iomem *portal;

	u32 portal_offset;

	unsigned int enqcmds_retries;

	struct percpu_ref wq_active;

	struct completion wq_dead;

	struct completion wq_resurrect;
@@ -584,6 +588,7 @@ int idxd_wq_init_percpu_ref(struct idxd_wq *wq); 
int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);

struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype);

void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc);

int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc);



/* dmaengine */

int idxd_register_dma_device(struct idxd_device *idxd);

drivers/dma/idxd/init.c

+1 −0
Original line number Diff line number Diff line
@@ -248,6 +248,7 @@ static int idxd_setup_wqs(struct idxd_device *idxd) 
		init_completion(&wq->wq_resurrect);

		wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER;

		wq->max_batch_size = WQ_DEFAULT_MAX_BATCH;

		wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES;

		wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev));

		if (!wq->wqcfg) {

			put_device(conf_dev);

drivers/dma/idxd/irq.c

+1 −1
Original line number Diff line number Diff line
@@ -98,7 +98,7 @@ static void idxd_int_handle_revoke_drain(struct idxd_irq_entry *ie) 
	if (wq_dedicated(wq)) {

		iosubmit_cmds512(portal, &desc, 1);

	} else {

		rc = enqcmds(portal, &desc);

		rc = idxd_enqcmds(wq, portal, &desc);

		/* This should not fail unless hardware failed. */

		if (rc < 0)

			dev_warn(dev, "Failed to submit drain desc on wq %d\n", wq->id);