Unverified Commit 55c10867 authored by openeuler-ci-bot's avatar openeuler-ci-bot Committed by Gitee
Browse files

!799 SCSI: SSSRAID: fix the issue that consider the scenario of HDD will occur... 

!799 SCSI: SSSRAID: fix the issue that consider the scenario of HDD will occur unexpected high latency when pressure, concurrent, time all big enough

Merge Pull Request from: @steven-song3 
 
[issue description]
HDD will occur unexpected high latency when pressure, concurrent, time all big enough.
[steps to reproduce the problem]
1. 16 HDDs create 16 bare disks, each disk starts a thread, each thread
   is bound to a core, and the queue depth is 64
2. Issue read & write mixed I/O to the drive letters corresponding to
   rawdrive
3. After the long read and write IO period is completed, observe the
   maximum delay in the printout of the fio tool
[Probability of occurrence]
100%
[Expected results]
The maximum delay in the printout of the FIO tool does not exceed 10 seconds
[Root cause]
Muti-queue implementation will cause high concurrent in each different hardware
dispatch queue is not suitable for HDD.
[Solution]
Introduce work_mode module parameter to fastened total I/O queue number
equal to 1 and maximum concurrent I/O is 4k (according to RAID cured performance)
 
 
Link:https://gitee.com/openeuler/kernel/pulls/799

 

Reviewed-by: default avatarJialin Zhang <zhangjialin11@huawei.com>
Signed-off-by: default avatarJialin Zhang <zhangjialin11@huawei.com>
parents e00500cc 220dfe07

drivers/scsi/sssraid/sssraid.h

+19 −3
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ 


#define SSSRAID_NAME_LENGTH 32

#define BSG_NAME_SIZE 15



/*

 * SSSRAID Vendor ID and Device IDs

 */
@@ -90,7 +91,7 @@ extern u32 admin_tmout; 
#define ADMIN_TIMEOUT		(admin_tmout * HZ)



#define SSSRAID_WAIT_ABNL_CMD_TIMEOUT	6



#define SSSRAID_WAIT_RST_IO_TIMEOUT 10

#define SSSRAID_DMA_MSK_BIT_MAX	64



enum {
@@ -325,7 +326,7 @@ struct sssraid_ioc { 
	int logging_level;



	char name[SSSRAID_NAME_LENGTH];

	int cpu_count;

	u32 cpu_count;

	/*

	 * before_affinity_msix_cnt is

	 * min("FW support IO Queue count", num_online_cpus)+1
@@ -363,6 +364,11 @@ struct sssraid_ioc { 
	struct list_head fwevt_list;



	struct sssraid_fwevt *current_event;



	void *senses;

	dma_addr_t sense_dma_addr;

	u32 last_qcnt;

	u8 hdd_dispatch;

};



/*
@@ -790,6 +796,7 @@ struct sssraid_squeue { 
	void *sense;

	dma_addr_t sense_dma_addr;

	struct dma_pool *prp_small_pool;

	atomic_t inflight;

};



struct sssraid_cqueue {
@@ -813,6 +820,7 @@ struct sssraid_iod { 
	enum sssraid_cmd_state state;

	int npages;

	u32 nsge;

	u16 cid;

	u32 length;

	bool use_sgl;

	dma_addr_t first_dma;
@@ -850,6 +858,11 @@ enum { 
	SSSRAID_NVME_SSD_PD = 0x16,

};



enum {

	DISPATCH_BY_CPU,

	DISPATCH_BY_DISK,

};



/*

 * define the SSSRAID scsi device queue depth

 */
@@ -929,11 +942,14 @@ struct sssraid_sdev_hostdata { 
	u8 attr;

	u8 flag;

	u8 rg_id;

	u8 rsvd[3];

	u8 hwq;

	u16 pend_count;

};



extern unsigned char small_pool_num;

extern u32 io_queue_depth;

extern u32 max_hwq_num;

extern bool work_mode;

irqreturn_t sssraid_isr_poll(int irq, void *privdata);

bool sssraid_poll_cq(struct sssraid_ioc *sdioc, u16 qidx, int cid);

void sssraid_submit_cmd(struct sssraid_squeue *sqinfo, const void *cmd);

drivers/scsi/sssraid/sssraid_fw.c

+86 −57
Original line number Diff line number Diff line
@@ -194,7 +194,7 @@ static int sssraid_alloc_resources(struct sssraid_ioc *sdioc) 
	}



	/* not num_online_cpus */

	nqueue = num_possible_cpus() + 1;

	nqueue = min(num_possible_cpus(), max_hwq_num) + 1;

	sdioc->cqinfo = kcalloc_node(nqueue, sizeof(struct sssraid_cqueue),

			    GFP_KERNEL, sdioc->numa_node);

	if (!sdioc->cqinfo) {
@@ -282,7 +282,6 @@ static int sssraid_setup_resources(struct sssraid_ioc *sdioc) 


	sdioc->cap = lo_hi_readq(sdioc->bar + SSSRAID_REG_CAP);

	sdioc->ioq_depth = min_t(u32, SSSRAID_CAP_MQES(sdioc->cap) + 1, io_queue_depth);

	sdioc->scsi_qd = sdioc->ioq_depth - SSSRAID_PTCMDS_PERQ;

	sdioc->db_stride = 1 << SSSRAID_CAP_STRIDE(sdioc->cap);



	maskbit = SSSRAID_CAP_DMAMASK(sdioc->cap);
@@ -357,10 +356,8 @@ static int sssraid_alloc_qpair(struct sssraid_ioc *sdioc, u16 qidx, u16 depth) 


	cqinfo->cqes = dma_alloc_coherent(&sdioc->pdev->dev, CQ_SIZE(depth),

					   &cqinfo->cq_dma_addr, GFP_KERNEL | __GFP_ZERO);

	if (!cqinfo->cqes) {

		ioc_err(sdioc, "failure at alloc dma space for cqueue.\n");

	if (!cqinfo->cqes)

		return -ENOMEM;

	}



	sqinfo->sq_cmds = dma_alloc_coherent(&sdioc->pdev->dev, SQ_SIZE(qidx, depth),

					      &sqinfo->sq_dma_addr, GFP_KERNEL);
@@ -370,6 +367,13 @@ static int sssraid_alloc_qpair(struct sssraid_ioc *sdioc, u16 qidx, u16 depth) 
		goto  free_cqes;

	}



	/*

	 * if single hw queue, we do not need to alloc sense buffer for every queue,

	 * we have alloced all on hiraid_alloc_resources.

	 */

	if (work_mode)

		goto initq;



	/* alloc sense buffer */

	sqinfo->sense = dma_alloc_coherent(&sdioc->pdev->dev, SENSE_SIZE(depth),

					    &sqinfo->sense_dma_addr, GFP_KERNEL | __GFP_ZERO);
@@ -379,6 +383,7 @@ static int sssraid_alloc_qpair(struct sssraid_ioc *sdioc, u16 qidx, u16 depth) 
		goto free_sq_cmds;

	}



initq:

	spin_lock_init(&sqinfo->sq_lock);

	spin_lock_init(&cqinfo->cq_lock);

	cqinfo->sdioc = sdioc;
@@ -413,6 +418,8 @@ static void sssraid_init_queue(struct sssraid_ioc *sdioc, u16 qidx) 
	memset((void *)cqinfo->cqes, 0, CQ_SIZE(sqinfo->q_depth));



	sqinfo->sq_tail = 0;

	atomic_set(&sqinfo->inflight, 0);



	cqinfo->cq_head = 0;

	cqinfo->cq_phase = 1;

	sqinfo->q_db = &sdioc->dbs[qidx * 2 * sdioc->db_stride];
@@ -819,6 +826,8 @@ int sssraid_init_ctrl_info(struct sssraid_ioc *sdioc) 
	if (sdioc->ctrl_info->aerl > SSSRAID_NR_AEN_CMDS)

		sdioc->ctrl_info->aerl = SSSRAID_NR_AEN_CMDS;



	sdioc->scsi_qd = work_mode ? le16_to_cpu(sdioc->ctrl_info->max_cmds) :

					 sdioc->ioq_depth - SSSRAID_PTCMDS_PERQ;

	return 0;

}
@@ -902,6 +911,8 @@ static int sssraid_create_io_cq(struct sssraid_ioc *sdioc, u16 qidx) 
	if (retval)

		return retval;



	if (!sdioc->last_qcnt)

		sdioc->last_qcnt = sdioc->init_done_queue_cnt;

	/*

	 * cqinfo initialization at sssraid_init_queue

	 */
@@ -1093,9 +1104,18 @@ static int sssraid_setup_io_qpair(struct sssraid_ioc *sdioc) 
		}

	}



	ioc_info(sdioc, "init_done_queue_cnt[%d], intr_info_count[%d] num_queues[%d]",

	if (work_mode && !sdioc->senses) {

		sdioc->senses = dma_alloc_coherent(&sdioc->pdev->dev,

			SENSE_SIZE(SSSRAID_IO_BLK_MQ_DEPTH + max_hwq_num *

			SSSRAID_PTCMDS_PERQ), &sdioc->sense_dma_addr,

			GFP_KERNEL | __GFP_ZERO);

		if (!sdioc->senses)

			return -ENOMEM;

	}



	ioc_info(sdioc, "init_done_queue_cnt[%d], intr_info_count[%d] num_queues[%d], last_online[%d]",

		 sdioc->init_done_queue_cnt,

		 sdioc->intr_info_count, num_queues);

		 sdioc->intr_info_count, num_queues, sdioc->last_qcnt);



	return retval >= 0 ? 0 : retval;

}
@@ -1194,10 +1214,8 @@ static int sssraid_disk_list_init(struct sssraid_ioc *sdioc) 


	sdioc->devices = kzalloc_node(nd * sizeof(struct sssraid_dev_info),

				     GFP_KERNEL, sdioc->numa_node);

	if (!sdioc->devices) {

		ioc_err(sdioc, "err: failed to alloc memory for device info.\n");

	if (!sdioc->devices)

		return -ENOMEM;

	}



	return 0;

}
@@ -1230,7 +1248,6 @@ int sssraid_init_ioc(struct sssraid_ioc *sdioc, u8 re_init) 
			    retval);

			goto out_nocleanup;

		}

	}



		/* reset need re-setup */

		retval = sssraid_setup_resources(sdioc);
@@ -1240,7 +1257,6 @@ int sssraid_init_ioc(struct sssraid_ioc *sdioc, u8 re_init) 
			goto out_failed;

		}



	if (!re_init) {

		retval = sssraid_alloc_admin_cmds(sdioc);

		if (retval) {

			ioc_err(sdioc, "Err: Failed to alloc admin cmds, ret %d\n",
@@ -1259,11 +1275,8 @@ int sssraid_init_ioc(struct sssraid_ioc *sdioc, u8 re_init) 
	}



	retval = sssraid_setup_admin_qpair(sdioc);

	if (retval) {

		ioc_err(sdioc, "Err: Failed to setup admin queue, ret %d\n",

		    retval);

	if (retval)

		goto out_failed;

	}



	/* 1. unregister all interrupt

	 * 2. admin interrupt registry
@@ -1275,14 +1288,16 @@ int sssraid_init_ioc(struct sssraid_ioc *sdioc, u8 re_init) 
		goto out_failed;

	}



	if (!re_init) {

		retval = sssraid_init_ctrl_info(sdioc);

		if (retval) {

			ioc_err(sdioc, "Failed to get ctrl info error %d\n",

					retval);

			goto out_failed;

		}

	}



	nr_ioqs = sdioc->cpu_count;

	nr_ioqs = min(sdioc->cpu_count, max_hwq_num);

	retval = sssraid_set_queue_cnt(sdioc, &nr_ioqs);

	if (retval) {

		ioc_err(sdioc, "Failed to set queue cnt error %d\n",
@@ -1303,6 +1318,9 @@ int sssraid_init_ioc(struct sssraid_ioc *sdioc, u8 re_init) 
		goto out_failed;

	}



	/* num_vecs no sense, abandon */



	if (!re_init) {

		/* remap */

		bar_size = SSSRAID_REG_DBS + ((nr_ioqs + 1) * 8 * sdioc->db_stride);

		retval = sssraid_remap_bar(sdioc, bar_size);
@@ -1313,9 +1331,6 @@ int sssraid_init_ioc(struct sssraid_ioc *sdioc, u8 re_init) 
		}

		sdioc->sqinfo[0].q_db = sdioc->dbs;



	/* num_vecs no sense, abandon */



	if (!re_init) {

		for (i = sdioc->init_done_queue_cnt; i < sdioc->intr_info_count; i++) {

			retval = sssraid_alloc_qpair(sdioc, i, sdioc->ioq_depth);

			if (retval) {
@@ -1403,6 +1418,15 @@ static void sssraid_free_ioq_ptcmds(struct sssraid_ioc *sdioc) 
	INIT_LIST_HEAD(&sdioc->ioq_pt_list);

}



static void sssraid_free_sense_buffer(struct sssraid_ioc *sdioc)

{

	if (sdioc->senses) {

		dma_free_coherent(&sdioc->pdev->dev, SENSE_SIZE(SSSRAID_IO_BLK_MQ_DEPTH +

			max_hwq_num * SSSRAID_PTCMDS_PERQ), sdioc->senses, sdioc->sense_dma_addr);

		sdioc->senses = NULL;

	}

}



static void sssraid_delete_io_queues(struct sssraid_ioc *sdioc)

{

	u16 queues = sdioc->init_done_queue_cnt - SSSRAID_ADM_QUEUE_NUM;
@@ -1419,6 +1443,7 @@ static void sssraid_delete_io_queues(struct sssraid_ioc *sdioc) 
		return;

	}



	sssraid_free_sense_buffer(sdioc);

	for (pass = 0; pass < 2; pass++) {

		for (i = queues; i > 0; i--)

			if (sssraid_delete_queue(sdioc, opcode, i))
@@ -1516,6 +1541,11 @@ static void sssraid_complete_ioq_cmnd(struct sssraid_ioc *sdioc, u16 qidx, 
	struct request *req;

	unsigned long elapsed;



	atomic_dec(&sqinfo->inflight);



	if (work_mode)

		tags = sdioc->shost->tag_set.tags[0];

	else

		tags = sdioc->shost->tag_set.tags[sqinfo->qidx - 1];



	req = blk_mq_tag_to_rq(tags, le16_to_cpu(cqe->cmd_id));
@@ -1556,8 +1586,8 @@ static void sssraid_complete_ioq_cmnd(struct sssraid_ioc *sdioc, u16 qidx, 
	scmd->scsi_done(scmd);

}



static void sssraid_process_admin_cq(struct sssraid_ioc *sdioc,

	struct sssraid_squeue *sqinfo,

static

void sssraid_process_admin_cq(struct sssraid_ioc *sdioc, struct sssraid_squeue *sqinfo,

							  struct sssraid_completion *cqe)

{

	struct sssraid_fwevt *fwevt = NULL;
@@ -1584,8 +1614,7 @@ static void sssraid_process_admin_cq(struct sssraid_ioc *sdioc, 
	}

}



static void sssraid_process_io_cq(struct sssraid_ioc *sdioc,

	struct sssraid_squeue *sqinfo,

static void sssraid_process_io_cq(struct sssraid_ioc *sdioc, struct sssraid_squeue *sqinfo,

								  struct sssraid_completion *cqe)

{

	u16 cid = le16_to_cpu(cqe->cmd_id);
@@ -1604,7 +1633,7 @@ static inline void sssraid_handle_cqe(struct sssraid_ioc *sdioc, u16 mdix, u16 d 
	struct sssraid_completion *cqe = &cqinfo->cqes[didx];

	u16 cid = le16_to_cpu(cqe->cmd_id);



	if (unlikely(cid >= sqinfo->q_depth)) {

	if (unlikely(!work_mode && (cid >= sqinfo->q_depth))) {

		ioc_err(sdioc, "Err: invalid command id[%d] completed on queue %d\n",

			cid, cqe->sq_id);

		return;
@@ -1670,6 +1699,7 @@ static void sssraid_free_all_queues(struct sssraid_ioc *sdioc) 
				(void *)cqinfo->cqes, cqinfo->cq_dma_addr);

		dma_free_coherent(&sdioc->pdev->dev, SQ_SIZE(sqinfo->qidx, sqinfo->q_depth),

				sqinfo->sq_cmds, sqinfo->sq_dma_addr);

		if (!work_mode)

			dma_free_coherent(&sdioc->pdev->dev, SENSE_SIZE(sqinfo->q_depth),

							  sqinfo->sense, sqinfo->sense_dma_addr);

	}
@@ -1732,11 +1762,11 @@ int sssraid_soft_reset_handler(struct sssraid_ioc *sdioc) 
	 * i.e. sssraid_cleanup_ioc(1)

	 */

	if (sdioc->ctrl_config & SSSRAID_CC_ENABLE) {

		ioc_info(sdioc, "start disable admin queue\n");

		ioc_info(sdioc, "\n");

		retval = sssraid_disable_admin_queue(sdioc, 0);

	}



	sssraid_cleanup_resources(sdioc);

	sssraid_cleanup_isr(sdioc);



	/* realize above here:

	 * sssraid_dev_disable -> sssraid_back_all_io
@@ -1747,14 +1777,13 @@ int sssraid_soft_reset_handler(struct sssraid_ioc *sdioc) 
		goto host_reset_failed;



	retval = sssraid_init_ioc(sdioc, 1);

	if (retval)

		goto cleanup_resources;

	if (retval || sdioc->last_qcnt != sdioc->init_done_queue_cnt)

		goto host_reset_failed;



	sssraid_scan_disk(sdioc);

	sssraid_change_host_state(sdioc, SSSRAID_LIVE);

	return 0;



cleanup_resources:

	sssraid_cleanup_resources(sdioc);

host_reset_failed:

	sssraid_change_host_state(sdioc, SSSRAID_DEAD);

	ioc_err(sdioc, "err, host reset failed\n");

drivers/scsi/sssraid/sssraid_os.c

+282 −103

File changed.

Preview size limit exceeded, changes collapsed.