Commit e33c2482 authored by Varun Prakash's avatar Varun Prakash Committed by Martin K. Petersen
Browse files

scsi: cxgb4i: Add support for iSCSI segmentation offload 

T5/T6 adapters support iSCSI segmentation offload.

To transmit iSCSI PDUs using ISO driver provides iSCSI header and data
scatterlist to the adapter, adapter forms multiple iSCSI PDUs and transmits
them.

[mkp: checkpatch]

Link: https://lore.kernel.org/r/1593448871-2972-1-git-send-email-varun@chelsio.com


Signed-off-by: default avatarVarun Prakash <varun@chelsio.com>
Signed-off-by: default avatarMartin K. Petersen <martin.petersen@oracle.com>
parent 0a765665

drivers/scsi/cxgbi/cxgb3i/cxgb3i.c

+2 −2
Original line number Diff line number Diff line
@@ -361,7 +361,7 @@ static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb, 
	/* len includes the length of any HW ULP additions */

	req->len = htonl(len);

	/* V_TX_ULP_SUBMODE sets both the mode and submode */

	req->flags = htonl(V_TX_ULP_SUBMODE(cxgbi_skcb_ulp_mode(skb)) |

	req->flags = htonl(V_TX_ULP_SUBMODE(cxgbi_skcb_tx_ulp_mode(skb)) |

			   V_TX_SHOVE((skb_peek(&csk->write_queue) ? 0 : 1)));

	req->sndseq = htonl(csk->snd_nxt);

	req->param = htonl(V_TX_PORT(l2t->smt_idx));
@@ -442,7 +442,7 @@ static int push_tx_frames(struct cxgbi_sock *csk, int req_completion) 
				req_completion = 1;

				csk->wr_una_cred = 0;

			}

			len += cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb));

			len += cxgbi_ulp_extra_len(cxgbi_skcb_tx_ulp_mode(skb));

			make_tx_data_wr(csk, skb, len, req_completion);

			csk->snd_nxt += len;

			cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR);

drivers/scsi/cxgbi/cxgb4i/cxgb4i.c

+184 −58
Original line number Diff line number Diff line
@@ -197,7 +197,10 @@ static inline bool is_ofld_imm(const struct sk_buff *skb) 
	if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR)))

		len += sizeof(struct fw_ofld_tx_data_wr);



	return len <= MAX_IMM_TX_PKT_LEN;

	if  (likely(cxgbi_skcb_test_flag((struct sk_buff *)skb, SKCBF_TX_ISO)))

		len += sizeof(struct cpl_tx_data_iso);



	return (len <= MAX_IMM_OFLD_TX_DATA_WR_LEN);

}



static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb,
@@ -641,7 +644,10 @@ static inline int send_tx_flowc_wr(struct cxgbi_sock *csk) 
	flowc->mnemval[8].mnemonic = 0;

	flowc->mnemval[8].val = 0;

	flowc->mnemval[8].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX;

	flowc->mnemval[8].val = 16384;

	if (csk->cdev->skb_iso_txhdr)

		flowc->mnemval[8].val = cpu_to_be32(CXGBI_MAX_ISO_DATA_IN_SKB);

	else

		flowc->mnemval[8].val = cpu_to_be32(16128);

#ifdef CONFIG_CHELSIO_T4_DCB

	flowc->mnemval[9].mnemonic = FW_FLOWC_MNEM_DCBPRIO;

	if (vlan == CPL_L2T_VLAN_NONE) {
@@ -667,38 +673,86 @@ static inline int send_tx_flowc_wr(struct cxgbi_sock *csk) 
	return flowclen16;

}



static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb,

				   int dlen, int len, u32 credits, int compl)

static void

cxgb4i_make_tx_iso_cpl(struct sk_buff *skb, struct cpl_tx_data_iso *cpl)

{

	struct cxgbi_iso_info *info = (struct cxgbi_iso_info *)skb->head;

	u32 imm_en = !!(info->flags & CXGBI_ISO_INFO_IMM_ENABLE);

	u32 fslice = !!(info->flags & CXGBI_ISO_INFO_FSLICE);

	u32 lslice = !!(info->flags & CXGBI_ISO_INFO_LSLICE);

	u32 pdu_type = (info->op == ISCSI_OP_SCSI_CMD) ? 0 : 1;

	u32 submode = cxgbi_skcb_tx_ulp_mode(skb) & 0x3;



	cpl->op_to_scsi = cpu_to_be32(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) |

				CPL_TX_DATA_ISO_FIRST_V(fslice) |

				CPL_TX_DATA_ISO_LAST_V(lslice) |

				CPL_TX_DATA_ISO_CPLHDRLEN_V(0) |

				CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) |

				CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) |

				CPL_TX_DATA_ISO_IMMEDIATE_V(imm_en) |

				CPL_TX_DATA_ISO_SCSI_V(pdu_type));



	cpl->ahs_len = info->ahs;

	cpl->mpdu = cpu_to_be16(DIV_ROUND_UP(info->mpdu, 4));

	cpl->burst_size = cpu_to_be32(info->burst_size);

	cpl->len = cpu_to_be32(info->len);

	cpl->reserved2_seglen_offset =

	     cpu_to_be32(CPL_TX_DATA_ISO_SEGLEN_OFFSET_V(info->segment_offset));

	cpl->datasn_offset = cpu_to_be32(info->datasn_offset);

	cpl->buffer_offset = cpu_to_be32(info->buffer_offset);

	cpl->reserved3 = cpu_to_be32(0);

	log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX,

		  "iso: flags 0x%x, op %u, ahs %u, num_pdu %u, mpdu %u, "

		  "burst_size %u, iso_len %u\n",

		  info->flags, info->op, info->ahs, info->num_pdu,

		  info->mpdu, info->burst_size << 2, info->len);

}



static void

cxgb4i_make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb, int dlen,

		       int len, u32 credits, int compl)

{

	struct cxgbi_device *cdev = csk->cdev;

	struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);

	struct fw_ofld_tx_data_wr *req;

	unsigned int submode = cxgbi_skcb_ulp_mode(skb) & 3;

	unsigned int wr_ulp_mode = 0, val;

	bool imm = is_ofld_imm(skb);

	struct cpl_tx_data_iso *cpl;

	u32 submode = cxgbi_skcb_tx_ulp_mode(skb) & 0x3;

	u32 wr_ulp_mode = 0;

	u32 hdr_size = sizeof(*req);

	u32 opcode = FW_OFLD_TX_DATA_WR;

	u32 immlen = 0;

	u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) :

						T6_TX_FORCE_F;



	if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)) {

		hdr_size += sizeof(struct cpl_tx_data_iso);

		opcode = FW_ISCSI_TX_DATA_WR;

		immlen += sizeof(struct cpl_tx_data_iso);

		submode |= 8;

	}



	req = __skb_push(skb, sizeof(*req));

	if (is_ofld_imm(skb))

		immlen += dlen;



	if (imm) {

		req->op_to_immdlen = htonl(FW_WR_OP_V(FW_OFLD_TX_DATA_WR) |

					FW_WR_COMPL_F |

					FW_WR_IMMDLEN_V(dlen));

		req->flowid_len16 = htonl(FW_WR_FLOWID_V(csk->tid) |

	req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, hdr_size);

	req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) |

					 FW_WR_COMPL_V(compl) |

					 FW_WR_IMMDLEN_V(immlen));

	req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |

					FW_WR_LEN16_V(credits));

	} else {

		req->op_to_immdlen =

			cpu_to_be32(FW_WR_OP_V(FW_OFLD_TX_DATA_WR) |

					FW_WR_COMPL_F |

					FW_WR_IMMDLEN_V(0));

		req->flowid_len16 =

			cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |

					FW_WR_LEN16_V(credits));

	}

	req->plen = cpu_to_be32(len);

	cpl =  (struct cpl_tx_data_iso *)(req + 1);



	if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)))

		cxgb4i_make_tx_iso_cpl(skb, cpl);



	if (submode)

		wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP2_MODE_ISCSI) |

			      FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode);

	val = skb_peek(&csk->write_queue) ? 0 : 1;

	req->tunnel_to_proxy = htonl(wr_ulp_mode |

				     FW_OFLD_TX_DATA_WR_SHOVE_V(val));

	req->plen = htonl(len);



	req->tunnel_to_proxy = cpu_to_be32(wr_ulp_mode | force |

					   FW_OFLD_TX_DATA_WR_SHOVE_V(1U));



	if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT))

		cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);

}
@@ -722,24 +776,28 @@ static int push_tx_frames(struct cxgbi_sock *csk, int req_completion) 
		return 0;

	}



	while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) {

		int dlen = skb->len;

		int len = skb->len;

		unsigned int credits_needed;

		int flowclen16 = 0;

	while (csk->wr_cred && ((skb = skb_peek(&csk->write_queue)) != NULL)) {

		struct cxgbi_iso_info *iso_cpl;

		u32 dlen = skb->len;

		u32 len = skb->len;

		u32 iso_cpl_len = 0;

		u32 flowclen16 = 0;

		u32 credits_needed;

		u32 num_pdu = 1, hdr_len;



		if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO))

			iso_cpl_len = sizeof(struct cpl_tx_data_iso);



		skb_reset_transport_header(skb);

		if (is_ofld_imm(skb))

			credits_needed = DIV_ROUND_UP(dlen, 16);

			credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16);

		else

			credits_needed = DIV_ROUND_UP(

						8 * calc_tx_flits_ofld(skb),

						16);

			credits_needed =

				DIV_ROUND_UP((8 * calc_tx_flits_ofld(skb)) +

					     iso_cpl_len, 16);



		if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR)))

			credits_needed += DIV_ROUND_UP(

					sizeof(struct fw_ofld_tx_data_wr),

					16);

			credits_needed +=

			   DIV_ROUND_UP(sizeof(struct fw_ofld_tx_data_wr), 16);



		/*

		 * Assumes the initial credits is large enough to support
@@ -757,11 +815,16 @@ static int push_tx_frames(struct cxgbi_sock *csk, int req_completion) 
				  "csk 0x%p, skb %u/%u, wr %d < %u.\n",

				  csk, skb->len, skb->data_len,

				  credits_needed, csk->wr_cred);



			csk->no_tx_credits++;

			break;

		}



		csk->no_tx_credits = 0;



		__skb_unlink(skb, &csk->write_queue);

		set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);

		skb->csum = credits_needed + flowclen16;

		skb->csum = (__force __wsum)(credits_needed + flowclen16);

		csk->wr_cred -= credits_needed;

		csk->wr_una_cred += credits_needed;

		cxgbi_sock_enqueue_wr(csk, skb);
@@ -771,25 +834,42 @@ static int push_tx_frames(struct cxgbi_sock *csk, int req_completion) 
			csk, skb->len, skb->data_len, credits_needed,

			csk->wr_cred, csk->wr_una_cred);



		if (!req_completion &&

		    ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) ||

		     after(csk->write_seq, (csk->snd_una + csk->snd_win / 2))))

			req_completion = 1;



		if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) {

			len += cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb));

			make_tx_data_wr(csk, skb, dlen, len, credits_needed,

					req_completion);

			u32 ulp_mode = cxgbi_skcb_tx_ulp_mode(skb);



			if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)) {

				iso_cpl = (struct cxgbi_iso_info *)skb->head;

				num_pdu = iso_cpl->num_pdu;

				hdr_len = cxgbi_skcb_tx_iscsi_hdrlen(skb);

				len += (cxgbi_ulp_extra_len(ulp_mode) * num_pdu) +

				       (hdr_len * (num_pdu - 1));

			} else {

				len += cxgbi_ulp_extra_len(ulp_mode);

			}



			cxgb4i_make_tx_data_wr(csk, skb, dlen, len,

					       credits_needed, req_completion);

			csk->snd_nxt += len;

			cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR);

		} else if (cxgbi_skcb_test_flag(skb, SKCBF_TX_FLAG_COMPL) &&

			   (csk->wr_una_cred >= (csk->wr_max_cred / 2))) {

			struct cpl_close_con_req *req =

				(struct cpl_close_con_req *)skb->data;

			req->wr.wr_hi |= htonl(FW_WR_COMPL_F);



			req->wr.wr_hi |= cpu_to_be32(FW_WR_COMPL_F);

		}



		total_size += skb->truesize;

		t4_set_arp_err_handler(skb, csk, arp_failure_skb_discard);



		log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,

			  "csk 0x%p,%u,0x%lx,%u, skb 0x%p, %u.\n",

			  csk, csk->state, csk->flags, csk->tid, skb, len);



		cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t);

	}

	return total_size;
@@ -2111,10 +2191,30 @@ static int cxgb4i_ddp_init(struct cxgbi_device *cdev) 
	return 0;

}



static bool is_memfree(struct adapter *adap)

{

	u32 io;



	io = t4_read_reg(adap, MA_TARGET_MEM_ENABLE_A);

	if (is_t5(adap->params.chip)) {

		if ((io & EXT_MEM0_ENABLE_F) || (io & EXT_MEM1_ENABLE_F))

			return false;

	} else if (io & EXT_MEM_ENABLE_F) {

		return false;

	}



	return true;

}



static void *t4_uld_add(const struct cxgb4_lld_info *lldi)

{

	struct cxgbi_device *cdev;

	struct port_info *pi;

	struct net_device *ndev;

	struct adapter *adap;

	struct tid_info *t;

	u32 max_cmds = CXGB4I_SCSI_HOST_QDEPTH;

	u32 max_conn = CXGBI_MAX_CONN;

	int i, rc;



	cdev = cxgbi_device_register(sizeof(*lldi), lldi->nports);
@@ -2154,13 +2254,39 @@ static void *t4_uld_add(const struct cxgb4_lld_info *lldi) 
		pr_info("t4 0x%p ddp init failed %d.\n", cdev, rc);

		goto err_out;

	}



	ndev = cdev->ports[0];

	adap = netdev2adap(ndev);

	if (adap) {

		t = &adap->tids;

		if (t->ntids <= CXGBI_MAX_CONN)

			max_conn = t->ntids;



		if (is_memfree(adap)) {

			cdev->flags |=	CXGBI_FLAG_DEV_ISO_OFF;

			max_cmds = CXGB4I_SCSI_HOST_QDEPTH >> 2;



			pr_info("%s: 0x%p, tid %u, SO adapter.\n",

				ndev->name, cdev, t->ntids);

		}

	} else {

		pr_info("%s, 0x%p, NO adapter struct.\n", ndev->name, cdev);

	}



	/* ISO is enabled in T5/T6 firmware version >= 1.13.43.0 */

	if (!is_t4(lldi->adapter_type) &&

	    (lldi->fw_vers >= 0x10d2b00) &&

	    !(cdev->flags & CXGBI_FLAG_DEV_ISO_OFF))

		cdev->skb_iso_txhdr = sizeof(struct cpl_tx_data_iso);



	rc = cxgb4i_ofld_init(cdev);

	if (rc) {

		pr_info("t4 0x%p ofld init failed.\n", cdev);

		goto err_out;

	}



	rc = cxgbi_hbas_add(cdev, CXGB4I_MAX_LUN, CXGBI_MAX_CONN,

	cxgb4i_host_template.can_queue = max_cmds;

	rc = cxgbi_hbas_add(cdev, CXGB4I_MAX_LUN, max_conn,

			    &cxgb4i_host_template, cxgb4i_stt);

	if (rc)

		goto err_out;

drivers/scsi/cxgbi/libcxgbi.c

+498 −167

File changed.

Preview size limit exceeded, changes collapsed.

drivers/scsi/cxgbi/libcxgbi.h

+42 −4
Original line number Diff line number Diff line
@@ -76,6 +76,14 @@ do { \ 
#define ULP2_MAX_PDU_PAYLOAD	\

	(ULP2_MAX_PKT_SIZE - ISCSI_PDU_NONPAYLOAD_LEN)



#define CXGBI_ULP2_MAX_ISO_PAYLOAD	65535



#define CXGBI_MAX_ISO_DATA_IN_SKB	\

	min_t(u32, MAX_SKB_FRAGS << PAGE_SHIFT, CXGBI_ULP2_MAX_ISO_PAYLOAD)



#define cxgbi_is_iso_config(csk)	((csk)->cdev->skb_iso_txhdr)

#define cxgbi_is_iso_disabled(csk)	((csk)->disable_iso)



/*

 * For iscsi connections HW may inserts digest bytes into the pdu. Those digest

 * bytes are not sent by the host but are part of the TCP payload and therefore
@@ -162,6 +170,10 @@ struct cxgbi_sock { 
	u32 write_seq;

	u32 snd_win;

	u32 rcv_win;



	bool disable_iso;

	u32 no_tx_credits;

	unsigned long prev_iso_ts;

};



/*
@@ -203,6 +215,8 @@ struct cxgbi_skb_tx_cb { 
	void *handle;

	void *arp_err_handler;

	struct sk_buff *wr_next;

	u16 iscsi_hdr_len;

	u8 ulp_mode;

};



enum cxgbi_skcb_flags {
@@ -218,6 +232,7 @@ enum cxgbi_skcb_flags { 
	SKCBF_RX_HCRC_ERR,	/* header digest error */

	SKCBF_RX_DCRC_ERR,	/* data digest error */

	SKCBF_RX_PAD_ERR,	/* padding byte error */

	SKCBF_TX_ISO,		/* iso cpl in tx skb */

};



struct cxgbi_skb_cb {
@@ -225,18 +240,18 @@ struct cxgbi_skb_cb { 
		struct cxgbi_skb_rx_cb rx;

		struct cxgbi_skb_tx_cb tx;

	};

	unsigned char ulp_mode;

	unsigned long flags;

	unsigned int seq;

};



#define CXGBI_SKB_CB(skb)	((struct cxgbi_skb_cb *)&((skb)->cb[0]))

#define cxgbi_skcb_flags(skb)		(CXGBI_SKB_CB(skb)->flags)

#define cxgbi_skcb_ulp_mode(skb)	(CXGBI_SKB_CB(skb)->ulp_mode)

#define cxgbi_skcb_tcp_seq(skb)		(CXGBI_SKB_CB(skb)->seq)

#define cxgbi_skcb_rx_ddigest(skb)	(CXGBI_SKB_CB(skb)->rx.ddigest)

#define cxgbi_skcb_rx_pdulen(skb)	(CXGBI_SKB_CB(skb)->rx.pdulen)

#define cxgbi_skcb_tx_wr_next(skb)	(CXGBI_SKB_CB(skb)->tx.wr_next)

#define cxgbi_skcb_tx_iscsi_hdrlen(skb)	(CXGBI_SKB_CB(skb)->tx.iscsi_hdr_len)

#define cxgbi_skcb_tx_ulp_mode(skb)	(CXGBI_SKB_CB(skb)->tx.ulp_mode)



static inline void cxgbi_skcb_set_flag(struct sk_buff *skb,

					enum cxgbi_skcb_flags flag)
@@ -458,6 +473,7 @@ struct cxgbi_ports_map { 
#define CXGBI_FLAG_IPV4_SET		0x10

#define CXGBI_FLAG_USE_PPOD_OFLDQ       0x40

#define CXGBI_FLAG_DDP_OFF		0x100

#define CXGBI_FLAG_DEV_ISO_OFF		0x400



struct cxgbi_device {

	struct list_head list_head;
@@ -477,6 +493,7 @@ struct cxgbi_device { 
	unsigned int pfvf;

	unsigned int rx_credit_thres;

	unsigned int skb_tx_rsvd;

	u32 skb_iso_txhdr;

	unsigned int skb_rx_extra;	/* for msg coalesced mode */

	unsigned int tx_max_size;

	unsigned int rx_max_size;
@@ -523,20 +540,41 @@ struct cxgbi_endpoint { 
	struct cxgbi_sock *csk;

};



#define MAX_PDU_FRAGS	((ULP2_MAX_PDU_PAYLOAD + 512 - 1) / 512)

struct cxgbi_task_data {

#define CXGBI_TASK_SGL_CHECKED	0x1

#define CXGBI_TASK_SGL_COPY	0x2

	u8 flags;

	unsigned short nr_frags;

	struct page_frag frags[MAX_PDU_FRAGS];

	struct page_frag frags[MAX_SKB_FRAGS];

	struct sk_buff *skb;

	unsigned int dlen;

	unsigned int offset;

	unsigned int count;

	unsigned int sgoffset;

	u32 total_count;

	u32 total_offset;

	u32 max_xmit_dlength;

	struct cxgbi_task_tag_info ttinfo;

};

#define iscsi_task_cxgbi_data(task) \

	((task)->dd_data + sizeof(struct iscsi_tcp_task))



struct cxgbi_iso_info {

#define CXGBI_ISO_INFO_FSLICE		0x1

#define CXGBI_ISO_INFO_LSLICE		0x2

#define CXGBI_ISO_INFO_IMM_ENABLE	0x4

	u8 flags;

	u8 op;

	u8 ahs;

	u8 num_pdu;

	u32 mpdu;

	u32 burst_size;

	u32 len;

	u32 segment_offset;

	u32 datasn_offset;

	u32 buffer_offset;

};



static inline void *cxgbi_alloc_big_mem(unsigned int size,

					gfp_t gfp)

{