Commit 89c72f42 authored Sep 03, 2020 by Quinn Tran Committed by Martin K. Petersen Sep 22, 2020

scsi: qla2xxx: Add IOCB resource tracking

This patch tracks number of IOCB resources used in the I/O fast path. If
the number of used IOCBs reach a high water limit, driver would return the
I/O as busy and let upper layer retry. This prevents over subscription of
IOCB resources where any future error recovery command is unable to cut
through.  Enable IOCB throttling by default.

Link: https://lore.kernel.org/r/20200904045128.23631-12-njavali@marvell.com


Reviewed-by: Himanshu Madhani <himanshu.madhani@oracle.com>
Signed-off-by: Quinn Tran <qutran@marvell.com>
Signed-off-by: Arun Easi <aeasi@marvell.com>
Signed-off-by: Nilesh Javali <njavali@marvell.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>

parent 6152d20f

drivers/scsi/qla2xxx/qla_def.h

+17 −0

Original line number	Diff line number	Diff line
		@@ -624,6 +624,12 @@ enum {
		TYPE_TGT_TMCMD, /* task management */
		};

		struct iocb_resource {
		u8 res_type;
		u8 pad;
		u16 iocb_cnt;
		};

		typedef struct srb {
		/*
		* Do not move cmd_type field, it needs to
		@@ -631,6 +637,7 @@ typedef struct srb {
		*/
		uint8_t cmd_type;
		uint8_t pad[3];
		struct iocb_resource iores;
		struct kref cmd_kref; /* need to migrate ref_count over to this */
		void *priv;
		wait_queue_head_t nvme_ls_waitq;
		@@ -3577,6 +3584,15 @@ struct req_que {
		uint8_t req_pkt[REQUEST_ENTRY_SIZE];
		};

		struct qla_fw_resources {
		u16 iocbs_total;
		u16 iocbs_limit;
		u16 iocbs_qp_limit;
		u16 iocbs_used;
		};

		#define QLA_IOCB_PCT_LIMIT 95

		/Queue pair data structure /
		struct qla_qpair {
		spinlock_t qp_lock;
		@@ -3629,6 +3645,7 @@ struct qla_qpair {
		uint64_t retry_term_jiff;
		struct qla_tgt_counters tgt_counters;
		uint16_t cpuid;
		struct qla_fw_resources fwres ____cacheline_aligned;
		};

		/* Place holder for FW buffer parameters */

drivers/scsi/qla2xxx/qla_dfs.c

+14 −0

Original line number	Diff line number	Diff line
		@@ -261,6 +261,8 @@ qla_dfs_fw_resource_cnt_show(struct seq_file s, void unused)
		struct scsi_qla_host *vha = s->private;
		uint16_t mb[MAX_IOCB_MB_REG];
		int rc;
		struct qla_hw_data *ha = vha->hw;
		u16 iocbs_used, i;

		rc = qla24xx_res_count_wait(vha, mb, SIZEOF_IOCB_MB_REG);
		if (rc != QLA_SUCCESS) {
		@@ -285,6 +287,18 @@ qla_dfs_fw_resource_cnt_show(struct seq_file s, void unused)
		mb[23]);
		}

		if (ql2xenforce_iocb_limit) {
		/* lock is not require. It's an estimate. */
		iocbs_used = ha->base_qpair->fwres.iocbs_used;
		for (i = 0; i < ha->max_qpairs; i++) {
		if (ha->queue_pair_map[i])
		iocbs_used += ha->queue_pair_map[i]->fwres.iocbs_used;
		}

		seq_printf(s, "Driver: estimate iocb used [%d] high water limit [%d]\n",
		iocbs_used, ha->base_qpair->fwres.iocbs_limit);
		}

		return 0;
		}

drivers/scsi/qla2xxx/qla_gbl.h

+3 −0

Original line number	Diff line number	Diff line
		@@ -129,6 +129,8 @@ int qla2x00_reserve_mgmt_server_loop_id(scsi_qla_host_t *);
		void qla_rscn_replay(fc_port_t *fcport);
		void qla24xx_free_purex_item(struct purex_item *item);
		extern bool qla24xx_risc_firmware_invalid(uint32_t *);
		void qla_init_iocb_limit(scsi_qla_host_t *);


		/*
		* Global Data in qla_os.c source file.
		@@ -175,6 +177,7 @@ extern int qla2xuseresexchforels;
		extern int ql2xexlogins;
		extern int ql2xdifbundlinginternalbuffers;
		extern int ql2xfulldump_on_mpifail;
		extern int ql2xenforce_iocb_limit;

		extern int qla2x00_loop_reset(scsi_qla_host_t *);
		extern void qla2x00_abort_all_cmds(scsi_qla_host_t *, int);

drivers/scsi/qla2xxx/qla_init.c

+26 −0

Original line number	Diff line number	Diff line
		@@ -3632,6 +3632,31 @@ qla24xx_detect_sfp(scsi_qla_host_t *vha)
		return ha->flags.lr_detected;
		}

		void qla_init_iocb_limit(scsi_qla_host_t *vha)
		{
		u16 i, num_qps;
		u32 limit;
		struct qla_hw_data *ha = vha->hw;

		num_qps = ha->num_qpairs + 1;
		limit = (ha->orig_fw_iocb_count * QLA_IOCB_PCT_LIMIT) / 100;

		ha->base_qpair->fwres.iocbs_total = ha->orig_fw_iocb_count;
		ha->base_qpair->fwres.iocbs_limit = limit;
		ha->base_qpair->fwres.iocbs_qp_limit = limit / num_qps;
		ha->base_qpair->fwres.iocbs_used = 0;
		for (i = 0; i < ha->max_qpairs; i++) {
		if (ha->queue_pair_map[i]) {
		ha->queue_pair_map[i]->fwres.iocbs_total =
		ha->orig_fw_iocb_count;
		ha->queue_pair_map[i]->fwres.iocbs_limit = limit;
		ha->queue_pair_map[i]->fwres.iocbs_qp_limit =
		limit / num_qps;
		ha->queue_pair_map[i]->fwres.iocbs_used = 0;
		}
		}
		}

		/**
		* qla2x00_setup_chip() - Load and start RISC firmware.
		* @vha: HA context
		@@ -3731,6 +3756,7 @@ qla2x00_setup_chip(scsi_qla_host_t *vha)
		MIN_MULTI_ID_FABRIC - 1;
		}
		qla2x00_get_resource_cnts(vha);
		qla_init_iocb_limit(vha);

		/*
		* Allocate the array of outstanding commands

drivers/scsi/qla2xxx/qla_inline.h

+55 −0

Original line number	Diff line number	Diff line
		@@ -378,3 +378,58 @@ qla2xxx_get_fc4_priority(struct scsi_qla_host *vha)

		return (data >> 6) & BIT_0 ? FC4_PRIORITY_FCP : FC4_PRIORITY_NVME;
		}

		enum {
		RESOURCE_NONE,
		RESOURCE_INI,
		};

		static inline int
		qla_get_iocbs(struct qla_qpair qp, struct iocb_resource iores)
		{
		u16 iocbs_used, i;
		struct qla_hw_data *ha = qp->vha->hw;

		if (!ql2xenforce_iocb_limit) {
		iores->res_type = RESOURCE_NONE;
		return 0;
		}

		if ((iores->iocb_cnt + qp->fwres.iocbs_used) < qp->fwres.iocbs_qp_limit) {
		qp->fwres.iocbs_used += iores->iocb_cnt;
		return 0;
		} else {
		/* no need to acquire qpair lock. It's just rough calculation */
		iocbs_used = ha->base_qpair->fwres.iocbs_used;
		for (i = 0; i < ha->max_qpairs; i++) {
		if (ha->queue_pair_map[i])
		iocbs_used += ha->queue_pair_map[i]->fwres.iocbs_used;
		}

		if ((iores->iocb_cnt + iocbs_used) < qp->fwres.iocbs_limit) {
		qp->fwres.iocbs_used += iores->iocb_cnt;
		return 0;
		} else {
		iores->res_type = RESOURCE_NONE;
		return -ENOSPC;
		}
		}
		}

		static inline void
		qla_put_iocbs(struct qla_qpair qp, struct iocb_resource iores)
		{
		switch (iores->res_type) {
		case RESOURCE_NONE:
		break;
		default:
		if (qp->fwres.iocbs_used >= iores->iocb_cnt) {
		qp->fwres.iocbs_used -= iores->iocb_cnt;
		} else {
		// should not happen
		qp->fwres.iocbs_used = 0;
		}
		break;
		}
		iores->res_type = RESOURCE_NONE;
		}