Commit 2fff76f8 authored by Daejun Park's avatar Daejun Park Committed by Martin K. Petersen
Browse files

scsi: ufs: ufshpb: Prepare HPB read for cached sub-region 

If the logical address of a read I/O belongs to an active sub-region, the
HPB driver modifies the read I/O command to an HPB read. The driver
modifies the UFS UPIU instead of modifying the existing SCSI command.

In HPB version 1.0, the maximum read I/O size that can be converted to HPB
read is 4KB.

The dirty map of the active sub-region prevents an incorrect HPB read that
has stale physical page number which is updated by previous write I/O.

[mkp: REQ_OP_DRV_* and blk_rq_is_passthrough()]

Link: https://lore.kernel.org/r/20210712085936epcms2p4b0ec5c8cecdeea6cc043d684363842b6@epcms2p4


Tested-by: default avatarBean Huo <beanhuo@micron.com>
Tested-by: default avatarCan Guo <cang@codeaurora.org>
Tested-by: default avatarStanley Chu <stanley.chu@mediatek.com>
Reviewed-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>
Reviewed-by: default avatarCan Guo <cang@codeaurora.org>
Reviewed-by: default avatarBart Van Assche <bvanassche@acm.org>
Reviewed-by: default avatarBean Huo <beanhuo@micron.com>
Reviewed-by: default avatarStanley Chu <stanley.chu@mediatek.com>
Acked-by: default avatarAvri Altman <Avri.Altman@wdc.com>
Signed-off-by: default avatarDaejun Park <daejun7.park@samsung.com>
Signed-off-by: default avatarMartin K. Petersen <martin.petersen@oracle.com>
parent 4b5f4907

drivers/scsi/ufs/ufshcd.c

+2 −0
Original line number Diff line number Diff line
@@ -2788,6 +2788,8 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) 


	lrbp->req_abort_skip = false;



	ufshpb_prep(hba, lrbp);



	ufshcd_comp_scsi_upiu(hba, lrbp);



	err = ufshcd_map_sg(hba, lrbp);

drivers/scsi/ufs/ufshpb.c

+256 −3
Original line number Diff line number Diff line
@@ -46,6 +46,29 @@ static void ufshpb_set_state(struct ufshpb_lu *hpb, int state) 
	atomic_set(&hpb->hpb_state, state);

}



static int ufshpb_is_valid_srgn(struct ufshpb_region *rgn,

				struct ufshpb_subregion *srgn)

{

	return rgn->rgn_state != HPB_RGN_INACTIVE &&

		srgn->srgn_state == HPB_SRGN_VALID;

}



static bool ufshpb_is_read_cmd(struct scsi_cmnd *cmd)

{

	return req_op(cmd->request) == REQ_OP_READ;

}



static bool ufshpb_is_write_or_discard(struct scsi_cmnd *cmd)

{

	return op_is_write(req_op(cmd->request)) ||

	       op_is_discard(req_op(cmd->request));

}



static bool ufshpb_is_supported_chunk(int transfer_len)

{

	return transfer_len <= HPB_MULTI_CHUNK_HIGH;

}



static bool ufshpb_is_general_lun(int lun)

{

	return lun < UFS_UPIU_MAX_UNIT_NUM_ID;
@@ -107,6 +130,236 @@ static bool ufshpb_is_hpb_rsp_valid(struct ufs_hba *hba, 
	return true;

}



static void ufshpb_set_ppn_dirty(struct ufshpb_lu *hpb, int rgn_idx,

				 int srgn_idx, int srgn_offset, int cnt)

{

	struct ufshpb_region *rgn;

	struct ufshpb_subregion *srgn;

	int set_bit_len;

	int bitmap_len;



next_srgn:

	rgn = hpb->rgn_tbl + rgn_idx;

	srgn = rgn->srgn_tbl + srgn_idx;



	if (likely(!srgn->is_last))

		bitmap_len = hpb->entries_per_srgn;

	else

		bitmap_len = hpb->last_srgn_entries;



	if ((srgn_offset + cnt) > bitmap_len)

		set_bit_len = bitmap_len - srgn_offset;

	else

		set_bit_len = cnt;



	if (rgn->rgn_state != HPB_RGN_INACTIVE &&

	    srgn->srgn_state == HPB_SRGN_VALID)

		bitmap_set(srgn->mctx->ppn_dirty, srgn_offset, set_bit_len);



	srgn_offset = 0;

	if (++srgn_idx == hpb->srgns_per_rgn) {

		srgn_idx = 0;

		rgn_idx++;

	}



	cnt -= set_bit_len;

	if (cnt > 0)

		goto next_srgn;

}



static bool ufshpb_test_ppn_dirty(struct ufshpb_lu *hpb, int rgn_idx,

				  int srgn_idx, int srgn_offset, int cnt)

{

	struct ufshpb_region *rgn;

	struct ufshpb_subregion *srgn;

	int bitmap_len;

	int bit_len;



next_srgn:

	rgn = hpb->rgn_tbl + rgn_idx;

	srgn = rgn->srgn_tbl + srgn_idx;



	if (likely(!srgn->is_last))

		bitmap_len = hpb->entries_per_srgn;

	else

		bitmap_len = hpb->last_srgn_entries;



	if (!ufshpb_is_valid_srgn(rgn, srgn))

		return true;



	/*

	 * If the region state is active, mctx must be allocated.

	 * In this case, check whether the region is evicted or

	 * mctx allcation fail.

	 */

	if (unlikely(!srgn->mctx)) {

		dev_err(&hpb->sdev_ufs_lu->sdev_dev,

			"no mctx in region %d subregion %d.\n",

			srgn->rgn_idx, srgn->srgn_idx);

		return true;

	}



	if ((srgn_offset + cnt) > bitmap_len)

		bit_len = bitmap_len - srgn_offset;

	else

		bit_len = cnt;



	if (find_next_bit(srgn->mctx->ppn_dirty, bit_len + srgn_offset,

			  srgn_offset) < bit_len + srgn_offset)

		return true;



	srgn_offset = 0;

	if (++srgn_idx == hpb->srgns_per_rgn) {

		srgn_idx = 0;

		rgn_idx++;

	}



	cnt -= bit_len;

	if (cnt > 0)

		goto next_srgn;



	return false;

}



static int ufshpb_fill_ppn_from_page(struct ufshpb_lu *hpb,

				     struct ufshpb_map_ctx *mctx, int pos,

				     int len, __be64 *ppn_buf)

{

	struct page *page;

	int index, offset;

	int copied;



	index = pos / (PAGE_SIZE / HPB_ENTRY_SIZE);

	offset = pos % (PAGE_SIZE / HPB_ENTRY_SIZE);



	if ((offset + len) <= (PAGE_SIZE / HPB_ENTRY_SIZE))

		copied = len;

	else

		copied = (PAGE_SIZE / HPB_ENTRY_SIZE) - offset;



	page = mctx->m_page[index];

	if (unlikely(!page)) {

		dev_err(&hpb->sdev_ufs_lu->sdev_dev,

			"error. cannot find page in mctx\n");

		return -ENOMEM;

	}



	memcpy(ppn_buf, page_address(page) + (offset * HPB_ENTRY_SIZE),

	       copied * HPB_ENTRY_SIZE);



	return copied;

}



static void

ufshpb_get_pos_from_lpn(struct ufshpb_lu *hpb, unsigned long lpn, int *rgn_idx,

			int *srgn_idx, int *offset)

{

	int rgn_offset;



	*rgn_idx = lpn >> hpb->entries_per_rgn_shift;

	rgn_offset = lpn & hpb->entries_per_rgn_mask;

	*srgn_idx = rgn_offset >> hpb->entries_per_srgn_shift;

	*offset = rgn_offset & hpb->entries_per_srgn_mask;

}



static void

ufshpb_set_hpb_read_to_upiu(struct ufshpb_lu *hpb, struct ufshcd_lrb *lrbp,

			    u32 lpn, __be64 ppn, u8 transfer_len)

{

	unsigned char *cdb = lrbp->cmd->cmnd;



	cdb[0] = UFSHPB_READ;



	/* ppn value is stored as big-endian in the host memory */

	memcpy(&cdb[6], &ppn, sizeof(__be64));

	cdb[14] = transfer_len;



	lrbp->cmd->cmd_len = UFS_CDB_SIZE;

}



/*

 * This function will set up HPB read command using host-side L2P map data.

 * In HPB v1.0, maximum size of HPB read command is 4KB.

 */

void ufshpb_prep(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)

{

	struct ufshpb_lu *hpb;

	struct ufshpb_region *rgn;

	struct ufshpb_subregion *srgn;

	struct scsi_cmnd *cmd = lrbp->cmd;

	u32 lpn;

	__be64 ppn;

	unsigned long flags;

	int transfer_len, rgn_idx, srgn_idx, srgn_offset;

	int err = 0;



	hpb = ufshpb_get_hpb_data(cmd->device);

	if (!hpb)

		return;



	if (ufshpb_get_state(hpb) == HPB_INIT)

		return;



	if (ufshpb_get_state(hpb) != HPB_PRESENT) {

		dev_notice(&hpb->sdev_ufs_lu->sdev_dev,

			   "%s: ufshpb state is not PRESENT", __func__);

		return;

	}



	if (blk_rq_is_passthrough(cmd->request) ||

	    (!ufshpb_is_write_or_discard(cmd) &&

	     !ufshpb_is_read_cmd(cmd)))

		return;



	transfer_len = sectors_to_logical(cmd->device,

					  blk_rq_sectors(cmd->request));

	if (unlikely(!transfer_len))

		return;



	lpn = sectors_to_logical(cmd->device, blk_rq_pos(cmd->request));

	ufshpb_get_pos_from_lpn(hpb, lpn, &rgn_idx, &srgn_idx, &srgn_offset);

	rgn = hpb->rgn_tbl + rgn_idx;

	srgn = rgn->srgn_tbl + srgn_idx;



	/* If command type is WRITE or DISCARD, set bitmap as drity */

	if (ufshpb_is_write_or_discard(cmd)) {

		spin_lock_irqsave(&hpb->rgn_state_lock, flags);

		ufshpb_set_ppn_dirty(hpb, rgn_idx, srgn_idx, srgn_offset,

				 transfer_len);

		spin_unlock_irqrestore(&hpb->rgn_state_lock, flags);

		return;

	}



	if (!ufshpb_is_supported_chunk(transfer_len))

		return;



	WARN_ON_ONCE(transfer_len > HPB_MULTI_CHUNK_HIGH);



	spin_lock_irqsave(&hpb->rgn_state_lock, flags);

	if (ufshpb_test_ppn_dirty(hpb, rgn_idx, srgn_idx, srgn_offset,

				   transfer_len)) {

		hpb->stats.miss_cnt++;

		spin_unlock_irqrestore(&hpb->rgn_state_lock, flags);

		return;

	}



	err = ufshpb_fill_ppn_from_page(hpb, srgn->mctx, srgn_offset, 1, &ppn);

	spin_unlock_irqrestore(&hpb->rgn_state_lock, flags);

	if (unlikely(err < 0)) {

		/*

		 * In this case, the region state is active,

		 * but the ppn table is not allocated.

		 * Make sure that ppn table must be allocated on

		 * active state.

		 */

		dev_err(hba->dev, "get ppn failed. err %d\n", err);

		return;

	}



	ufshpb_set_hpb_read_to_upiu(hpb, lrbp, lpn, ppn, transfer_len);



	hpb->stats.hit_cnt++;

}

static struct ufshpb_req *ufshpb_get_map_req(struct ufshpb_lu *hpb,

					     struct ufshpb_subregion *srgn)

{

drivers/scsi/ufs/ufshpb.h

+2 −0
Original line number Diff line number Diff line
@@ -201,6 +201,7 @@ struct ufs_hba; 
struct ufshcd_lrb;



#ifndef CONFIG_SCSI_UFS_HPB

static void ufshpb_prep(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) {}

static void ufshpb_rsp_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) {}

static void ufshpb_resume(struct ufs_hba *hba) {}

static void ufshpb_suspend(struct ufs_hba *hba) {}
@@ -214,6 +215,7 @@ static bool ufshpb_is_allowed(struct ufs_hba *hba) { return false; } 
static void ufshpb_get_geo_info(struct ufs_hba *hba, u8 *geo_buf) {}

static void ufshpb_get_dev_info(struct ufs_hba *hba, u8 *desc_buf) {}

#else

void ufshpb_prep(struct ufs_hba *hba, struct ufshcd_lrb *lrbp);

void ufshpb_rsp_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp);

void ufshpb_resume(struct ufs_hba *hba);

void ufshpb_suspend(struct ufs_hba *hba);