Commit 03cbbd7c authored by James Smart's avatar James Smart Committed by Martin K. Petersen
Browse files

scsi: lpfc: Fix SCSI I/O completion and abort handler deadlock 

During stress I/O tests with 500+ vports, hard LOCKUP call traces are
observed.

CPU A:
 native_queued_spin_lock_slowpath+0x192
 _raw_spin_lock_irqsave+0x32
 lpfc_handle_fcp_err+0x4c6
 lpfc_fcp_io_cmd_wqe_cmpl+0x964
 lpfc_sli4_fp_handle_cqe+0x266
 __lpfc_sli4_process_cq+0x105
 __lpfc_sli4_hba_process_cq+0x3c
 lpfc_cq_poll_hdler+0x16
 irq_poll_softirq+0x76
 __softirqentry_text_start+0xe4
 irq_exit+0xf7
 do_IRQ+0x7f

CPU B:
 native_queued_spin_lock_slowpath+0x5b
 _raw_spin_lock+0x1c
 lpfc_abort_handler+0x13e
 scmd_eh_abort_handler+0x85
 process_one_work+0x1a7
 worker_thread+0x30
 kthread+0x112
 ret_from_fork+0x1f

Diagram of lockup:

CPUA                            CPUB
----                            ----
lpfc_cmd->buf_lock
                            phba->hbalock
                            lpfc_cmd->buf_lock
phba->hbalock

Fix by reordering the taking of the lpfc_cmd->buf_lock and phba->hbalock in
lpfc_abort_handler routine so that it tries to take the lpfc_cmd->buf_lock
first before phba->hbalock.

Link: https://lore.kernel.org/r/20220412222008.126521-7-jsmart2021@gmail.com


Co-developed-by: default avatarJustin Tee <justin.tee@broadcom.com>
Signed-off-by: default avatarJustin Tee <justin.tee@broadcom.com>
Signed-off-by: default avatarJames Smart <jsmart2021@gmail.com>
Signed-off-by: default avatarMartin K. Petersen <martin.petersen@oracle.com>
parent b6474465

drivers/scsi/lpfc/lpfc_scsi.c

+15 −18
Original line number Diff line number Diff line
@@ -5865,25 +5865,25 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd) 
	if (!lpfc_cmd)

		return ret;



	spin_lock_irqsave(&phba->hbalock, flags);

	/* Guard against IO completion being called at same time */

	spin_lock_irqsave(&lpfc_cmd->buf_lock, flags);



	spin_lock(&phba->hbalock);

	/* driver queued commands are in process of being flushed */

	if (phba->hba_flag & HBA_IOQ_FLUSH) {

		lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,

			"3168 SCSI Layer abort requested I/O has been "

			"flushed by LLD.\n");

		ret = FAILED;

		goto out_unlock;

		goto out_unlock_hba;

	}



	/* Guard against IO completion being called at same time */

	spin_lock(&lpfc_cmd->buf_lock);



	if (!lpfc_cmd->pCmd) {

		lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,

			 "2873 SCSI Layer I/O Abort Request IO CMPL Status "

			 "x%x ID %d LUN %llu\n",

			 SUCCESS, cmnd->device->id, cmnd->device->lun);

		goto out_unlock_buf;

		goto out_unlock_hba;

	}



	iocb = &lpfc_cmd->cur_iocbq;
@@ -5891,7 +5891,7 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd) 
		pring_s4 = phba->sli4_hba.hdwq[iocb->hba_wqidx].io_wq->pring;

		if (!pring_s4) {

			ret = FAILED;

			goto out_unlock_buf;

			goto out_unlock_hba;

		}

		spin_lock(&pring_s4->ring_lock);

	}
@@ -5924,8 +5924,8 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd) 
			 "3389 SCSI Layer I/O Abort Request is pending\n");

		if (phba->sli_rev == LPFC_SLI_REV4)

			spin_unlock(&pring_s4->ring_lock);

		spin_unlock(&lpfc_cmd->buf_lock);

		spin_unlock_irqrestore(&phba->hbalock, flags);

		spin_unlock(&phba->hbalock);

		spin_unlock_irqrestore(&lpfc_cmd->buf_lock, flags);

		goto wait_for_cmpl;

	}
@@ -5946,15 +5946,13 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd) 
	if (ret_val != IOCB_SUCCESS) {

		/* Indicate the IO is not being aborted by the driver. */

		lpfc_cmd->waitq = NULL;

		spin_unlock(&lpfc_cmd->buf_lock);

		spin_unlock_irqrestore(&phba->hbalock, flags);

		ret = FAILED;

		goto out;

		goto out_unlock_hba;

	}



	/* no longer need the lock after this point */

	spin_unlock(&lpfc_cmd->buf_lock);

	spin_unlock_irqrestore(&phba->hbalock, flags);

	spin_unlock(&phba->hbalock);

	spin_unlock_irqrestore(&lpfc_cmd->buf_lock, flags);



	if (phba->cfg_poll & DISABLE_FCP_RING_INT)

		lpfc_sli_handle_fast_ring_event(phba,
@@ -5989,10 +5987,9 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd) 
out_unlock_ring:

	if (phba->sli_rev == LPFC_SLI_REV4)

		spin_unlock(&pring_s4->ring_lock);

out_unlock_buf:

	spin_unlock(&lpfc_cmd->buf_lock);

out_unlock:

	spin_unlock_irqrestore(&phba->hbalock, flags);

out_unlock_hba:

	spin_unlock(&phba->hbalock);

	spin_unlock_irqrestore(&lpfc_cmd->buf_lock, flags);

out:

	lpfc_printf_vlog(vport, KERN_WARNING, LOG_FCP,

			 "0749 SCSI Layer I/O Abort Request Status x%x ID %d "