Commit 433dc0ef authored by Max Gurtovoy's avatar Max Gurtovoy Committed by Jason Gunthorpe
Browse files

IB/iser: Don't suppress send completions 

In order to complete a scsi command and guarantee that the HCA will never
perform an access violation when retrying a send operation we must
complete a scsi request only when both send and receive completions has
arrived. This is a preparation commit that remove the send completions
suppression. Next step will be taking care of the local invalidation
mechanism and adding a reference counter for commands.  Currently, we
don't do anything upon getting the send completion and just "consume" it.

Link: https://lore.kernel.org/r/20211215135721.3662-5-mgurtovoy@nvidia.com


Signed-off-by: default avatarMax Gurtovoy <mgurtovoy@nvidia.com>
Reviewed-by: default avatarIsrael Rukshin <israelr@nvidia.com>
Signed-off-by: default avatarJason Gunthorpe <jgg@nvidia.com>
parent cf9962cf

drivers/infiniband/ulp/iser/iscsi_iser.h

+1 −6
Original line number Diff line number Diff line
@@ -146,8 +146,6 @@ 
					 - ISER_MAX_RX_MISC_PDUS) /	\

					 (1 + ISER_INFLIGHT_DATAOUTS))



#define ISER_SIGNAL_CMD_COUNT 32



/* Constant PDU lengths calculations */

#define ISER_HEADERS_LEN	(sizeof(struct iser_ctrl) + sizeof(struct iscsi_hdr))
@@ -364,7 +362,6 @@ struct iser_fr_pool { 
 * @qp:                  Connection Queue-pair

 * @cq:                  Connection completion queue

 * @cq_size:             The number of max outstanding completions

 * @sig_count:           send work request signal count

 * @device:              reference to iser device

 * @fr_pool:             connection fast registration poool

 * @pi_support:          Indicate device T10-PI support
@@ -375,7 +372,6 @@ struct ib_conn { 
	struct ib_qp	            *qp;

	struct ib_cq		    *cq;

	u32			    cq_size;

	u8                           sig_count;

	struct iser_device          *device;

	struct iser_fr_pool          fr_pool;

	bool			     pi_support;
@@ -532,8 +528,7 @@ int iser_connect(struct iser_conn *iser_conn, 
int  iser_post_recvl(struct iser_conn *iser_conn);

int  iser_post_recvm(struct iser_conn *iser_conn,

		     struct iser_rx_desc *rx_desc);

int  iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,

		    bool signal);

int  iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc);



int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,

			   struct iser_data_buf *data,

drivers/infiniband/ulp/iser/iser_initiator.c

+3 −10
Original line number Diff line number Diff line
@@ -350,11 +350,6 @@ static int iser_post_rx_bufs(struct iscsi_conn *conn, struct iscsi_hdr *req) 
	return err;

}



static inline bool iser_signal_comp(u8 sig_count)

{

	return ((sig_count % ISER_SIGNAL_CMD_COUNT) == 0);

}



/**

 * iser_send_command - send command PDU

 * @conn: link to matching iscsi connection
@@ -371,7 +366,6 @@ int iser_send_command(struct iscsi_conn *conn, 
	struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)task->hdr;

	struct scsi_cmnd *sc  =  task->sc;

	struct iser_tx_desc *tx_desc = &iser_task->desc;

	u8 sig_count = ++iser_conn->ib_conn.sig_count;



	edtl = ntohl(hdr->data_length);
@@ -418,8 +412,7 @@ int iser_send_command(struct iscsi_conn *conn, 


	iser_task->status = ISER_TASK_STATUS_STARTED;



	err = iser_post_send(&iser_conn->ib_conn, tx_desc,

			     iser_signal_comp(sig_count));

	err = iser_post_send(&iser_conn->ib_conn, tx_desc);

	if (!err)

		return 0;
@@ -487,7 +480,7 @@ int iser_send_data_out(struct iscsi_conn *conn, 
		 itt, buf_offset, data_seg_len);





	err = iser_post_send(&iser_conn->ib_conn, tx_desc, true);

	err = iser_post_send(&iser_conn->ib_conn, tx_desc);

	if (!err)

		return 0;
@@ -550,7 +543,7 @@ int iser_send_control(struct iscsi_conn *conn, 
			goto send_control_error;

	}



	err = iser_post_send(&iser_conn->ib_conn, mdesc, true);

	err = iser_post_send(&iser_conn->ib_conn, mdesc);

	if (!err)

		return 0;

drivers/infiniband/ulp/iser/iser_verbs.c

+2 −4
Original line number Diff line number Diff line
@@ -871,12 +871,10 @@ int iser_post_recvm(struct iser_conn *iser_conn, struct iser_rx_desc *rx_desc) 
 * iser_post_send - Initiate a Send DTO operation

 * @ib_conn: connection RDMA resources

 * @tx_desc: iSER TX descriptor

 * @signal: true to send work request as SIGNALED

 *

 * Return: 0 on success, -1 on failure

 */

int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,

		   bool signal)

int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc)

{

	struct ib_send_wr *wr = &tx_desc->send_wr;

	struct ib_send_wr *first_wr;
@@ -891,7 +889,7 @@ int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc, 
	wr->sg_list = tx_desc->tx_sg;

	wr->num_sge = tx_desc->num_sge;

	wr->opcode = IB_WR_SEND;

	wr->send_flags = signal ? IB_SEND_SIGNALED : 0;

	wr->send_flags = IB_SEND_SIGNALED;



	if (tx_desc->inv_wr.next)

		first_wr = &tx_desc->inv_wr;