Commit dca508cd authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'nfp-ipsec-offload' 



Simon Horman says:

====================
nfp: IPsec offload support

Huanhuan Wang says:

this series adds support for IPsec offload to the NFP driver.

It covers three enhancements:

1. Patches 1/3:
   - Extend the capability word and control word to to support
     new features.

2. Patch 2/3:
   - Add framework to support IPsec offloading for NFP driver,
     but IPsec offload control plane interface xfrm callbacks which
     interact with upper layer are not implemented in this patch.

3. Patch 3/3:
   - IPsec control plane interface xfrm callbacks are implemented
     in this patch.

Changes since v3
* Remove structure fields that describe firmware but
  are not used for Kernel offload
* Add WARN_ON(!xa_empty()) before call to xa_destroy()
* Added helpers for hash methods

Changes since v2
* OFFLOAD_HANDLE_ERROR macro and the associated code removed
* Unnecessary logging removed
* Hook function xdo_dev_state_free in struct xfrmdev_ops removed
* Use Xarray to maintain SA entries

Changes since v1
* Explicitly return failure when XFRM_STATE_ESN is set
* Fix the issue that AEAD algorithm is not correctly offloaded
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 8719a1c3 859a497f

drivers/net/ethernet/netronome/Kconfig

+11 −0
Original line number Diff line number Diff line
@@ -54,6 +54,17 @@ config NFP_APP_ABM_NIC 
	  functionality.

	  Code will be built into the nfp.ko driver.



config NFP_NET_IPSEC

	bool "NFP IPsec crypto offload support"

	depends on NFP

	depends on XFRM_OFFLOAD

	default y

	help

	  Enable driver support IPsec crypto offload on NFP NIC.

	  Say Y, if you are planning to make use of IPsec crypto

	  offload. NOTE that IPsec crypto offload on NFP NIC

	  requires specific FW to work.



config NFP_DEBUG

	bool "Debug support for Netronome(R) NFP4000/NFP6000 NIC drivers"

	depends on NFP

drivers/net/ethernet/netronome/nfp/Makefile

+2 −0
Original line number Diff line number Diff line
@@ -80,4 +80,6 @@ nfp-objs += \ 
	    abm/main.o

endif



nfp-$(CONFIG_NFP_NET_IPSEC) += crypto/ipsec.o nfd3/ipsec.o



nfp-$(CONFIG_NFP_DEBUG) += nfp_net_debugfs.o

drivers/net/ethernet/netronome/nfp/crypto/crypto.h

+23 −0
Original line number Diff line number Diff line
@@ -39,4 +39,27 @@ nfp_net_tls_rx_resync_req(struct net_device *netdev, 
}

#endif



/* IPsec related structures and functions */

struct nfp_ipsec_offload {

	u32 seq_hi;

	u32 seq_low;

	u32 handle;

};



#ifndef CONFIG_NFP_NET_IPSEC

static inline void nfp_net_ipsec_init(struct nfp_net *nn)

{

}



static inline void nfp_net_ipsec_clean(struct nfp_net *nn)

{

}

#else

void nfp_net_ipsec_init(struct nfp_net *nn);

void nfp_net_ipsec_clean(struct nfp_net *nn);

bool nfp_net_ipsec_tx_prep(struct nfp_net_dp *dp, struct sk_buff *skb,

			   struct nfp_ipsec_offload *offload_info);

int nfp_net_ipsec_rx(struct nfp_meta_parsed *meta, struct sk_buff *skb);

#endif



#endif

drivers/net/ethernet/netronome/nfp/crypto/ipsec.c

0 → 100644
+587 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)

/* Copyright (C) 2018 Netronome Systems, Inc */

/* Copyright (C) 2021 Corigine, Inc */



#include <linux/module.h>

#include <linux/kernel.h>

#include <linux/init.h>

#include <linux/netdevice.h>

#include <asm/unaligned.h>

#include <linux/ktime.h>

#include <net/xfrm.h>



#include "../nfp_net_ctrl.h"

#include "../nfp_net.h"

#include "crypto.h"



#define NFP_NET_IPSEC_MAX_SA_CNT  (16 * 1024) /* Firmware support a maximum of 16K SA offload */



/* IPsec config message cmd codes */

enum nfp_ipsec_cfg_mssg_cmd_codes {

	NFP_IPSEC_CFG_MSSG_ADD_SA,	 /* Add a new SA */

	NFP_IPSEC_CFG_MSSG_INV_SA	 /* Invalidate an existing SA */

};



/* IPsec config message response codes */

enum nfp_ipsec_cfg_mssg_rsp_codes {

	NFP_IPSEC_CFG_MSSG_OK,

	NFP_IPSEC_CFG_MSSG_FAILED,

	NFP_IPSEC_CFG_MSSG_SA_VALID,

	NFP_IPSEC_CFG_MSSG_SA_HASH_ADD_FAILED,

	NFP_IPSEC_CFG_MSSG_SA_HASH_DEL_FAILED,

	NFP_IPSEC_CFG_MSSG_SA_INVALID_CMD

};



/* Protocol */

enum nfp_ipsec_sa_prot {

	NFP_IPSEC_PROTOCOL_AH = 0,

	NFP_IPSEC_PROTOCOL_ESP = 1

};



/* Mode */

enum nfp_ipsec_sa_mode {

	NFP_IPSEC_PROTMODE_TRANSPORT = 0,

	NFP_IPSEC_PROTMODE_TUNNEL = 1

};



/* Cipher types */

enum nfp_ipsec_sa_cipher {

	NFP_IPSEC_CIPHER_NULL,

	NFP_IPSEC_CIPHER_3DES,

	NFP_IPSEC_CIPHER_AES128,

	NFP_IPSEC_CIPHER_AES192,

	NFP_IPSEC_CIPHER_AES256,

	NFP_IPSEC_CIPHER_AES128_NULL,

	NFP_IPSEC_CIPHER_AES192_NULL,

	NFP_IPSEC_CIPHER_AES256_NULL,

	NFP_IPSEC_CIPHER_CHACHA20

};



/* Cipher modes */

enum nfp_ipsec_sa_cipher_mode {

	NFP_IPSEC_CIMODE_ECB,

	NFP_IPSEC_CIMODE_CBC,

	NFP_IPSEC_CIMODE_CFB,

	NFP_IPSEC_CIMODE_OFB,

	NFP_IPSEC_CIMODE_CTR

};



/* Hash types */

enum nfp_ipsec_sa_hash_type {

	NFP_IPSEC_HASH_NONE,

	NFP_IPSEC_HASH_MD5_96,

	NFP_IPSEC_HASH_SHA1_96,

	NFP_IPSEC_HASH_SHA256_96,

	NFP_IPSEC_HASH_SHA384_96,

	NFP_IPSEC_HASH_SHA512_96,

	NFP_IPSEC_HASH_MD5_128,

	NFP_IPSEC_HASH_SHA1_80,

	NFP_IPSEC_HASH_SHA256_128,

	NFP_IPSEC_HASH_SHA384_192,

	NFP_IPSEC_HASH_SHA512_256,

	NFP_IPSEC_HASH_GF128_128,

	NFP_IPSEC_HASH_POLY1305_128

};



/* IPSEC_CFG_MSSG_ADD_SA */

struct nfp_ipsec_cfg_add_sa {

	u32 ciph_key[8];		  /* Cipher Key */

	union {

		u32 auth_key[16];	  /* Authentication Key */

		struct nfp_ipsec_aesgcm { /* AES-GCM-ESP fields */

			u32 salt;	  /* Initialized with SA */

			u32 resv[15];

		} aesgcm_fields;

	};

	struct sa_ctrl_word {

		uint32_t hash   :4;	  /* From nfp_ipsec_sa_hash_type */

		uint32_t cimode :4;	  /* From nfp_ipsec_sa_cipher_mode */

		uint32_t cipher :4;	  /* From nfp_ipsec_sa_cipher */

		uint32_t mode   :2;	  /* From nfp_ipsec_sa_mode */

		uint32_t proto  :2;	  /* From nfp_ipsec_sa_prot */

		uint32_t dir :1;	  /* SA direction */

		uint32_t resv0 :12;

		uint32_t encap_dsbl:1;	  /* Encap/Decap disable */

		uint32_t resv1 :2;	  /* Must be set to 0 */

	} ctrl_word;

	u32 spi;			  /* SPI Value */

	uint32_t pmtu_limit :16;          /* PMTU Limit */

	uint32_t resv0 :5;

	uint32_t ipv6       :1;		  /* Outbound IPv6 addr format */

	uint32_t resv1	 :10;

	u32 resv2[2];

	u32 src_ip[4];			  /* Src IP addr */

	u32 dst_ip[4];			  /* Dst IP addr */

	u32 resv3[6];

};



/* IPSEC_CFG_MSSG */

struct nfp_ipsec_cfg_mssg {

	union {

		struct{

			uint32_t cmd:16;     /* One of nfp_ipsec_cfg_mssg_cmd_codes */

			uint32_t rsp:16;     /* One of nfp_ipsec_cfg_mssg_rsp_codes */

			uint32_t sa_idx:16;  /* SA table index */

			uint32_t spare0:16;

			struct nfp_ipsec_cfg_add_sa cfg_add_sa;

		};

		u32 raw[64];

	};

};



static int nfp_ipsec_cfg_cmd_issue(struct nfp_net *nn, int type, int saidx,

				   struct nfp_ipsec_cfg_mssg *msg)

{

	int i, msg_size, ret;



	msg->cmd = type;

	msg->sa_idx = saidx;

	msg->rsp = 0;

	msg_size = ARRAY_SIZE(msg->raw);



	for (i = 0; i < msg_size; i++)

		nn_writel(nn, NFP_NET_CFG_MBOX_VAL + 4 * i, msg->raw[i]);



	ret = nfp_net_mbox_reconfig(nn, NFP_NET_CFG_MBOX_CMD_IPSEC);

	if (ret < 0)

		return ret;



	/* For now we always read the whole message response back */

	for (i = 0; i < msg_size; i++)

		msg->raw[i] = nn_readl(nn, NFP_NET_CFG_MBOX_VAL + 4 * i);



	switch (msg->rsp) {

	case NFP_IPSEC_CFG_MSSG_OK:

		return 0;

	case NFP_IPSEC_CFG_MSSG_SA_INVALID_CMD:

		return -EINVAL;

	case NFP_IPSEC_CFG_MSSG_SA_VALID:

		return -EEXIST;

	case NFP_IPSEC_CFG_MSSG_FAILED:

	case NFP_IPSEC_CFG_MSSG_SA_HASH_ADD_FAILED:

	case NFP_IPSEC_CFG_MSSG_SA_HASH_DEL_FAILED:

		return -EIO;

	default:

		return -EINVAL;

	}

}



static int set_aes_keylen(struct nfp_ipsec_cfg_add_sa *cfg, int alg, int keylen)

{

	bool aes_gmac = (alg == SADB_X_EALG_NULL_AES_GMAC);



	switch (keylen) {

	case 128:

		cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES128_NULL :

						   NFP_IPSEC_CIPHER_AES128;

		break;

	case 192:

		cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES192_NULL :

						   NFP_IPSEC_CIPHER_AES192;

		break;

	case 256:

		cfg->ctrl_word.cipher = aes_gmac ? NFP_IPSEC_CIPHER_AES256_NULL :

						   NFP_IPSEC_CIPHER_AES256;

		break;

	default:

		return -EINVAL;

	}



	return 0;

}



static void set_md5hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)

{

	switch (*trunc_len) {

	case 96:

		cfg->ctrl_word.hash = NFP_IPSEC_HASH_MD5_96;

		break;

	case 128:

		cfg->ctrl_word.hash = NFP_IPSEC_HASH_MD5_128;

		break;

	default:

		*trunc_len = 0;

	}

}



static void set_sha1hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)

{

	switch (*trunc_len) {

	case 96:

		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA1_96;

		break;

	case 80:

		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA1_80;

		break;

	default:

		*trunc_len = 0;

	}

}



static void set_sha2_256hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)

{

	switch (*trunc_len) {

	case 96:

		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA256_96;

		break;

	case 128:

		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA256_128;

		break;

	default:

		*trunc_len = 0;

	}

}



static void set_sha2_384hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)

{

	switch (*trunc_len) {

	case 96:

		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA384_96;

		break;

	case 192:

		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA384_192;

		break;

	default:

		*trunc_len = 0;

	}

}



static void set_sha2_512hmac(struct nfp_ipsec_cfg_add_sa *cfg, int *trunc_len)

{

	switch (*trunc_len) {

	case 96:

		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA512_96;

		break;

	case 256:

		cfg->ctrl_word.hash = NFP_IPSEC_HASH_SHA512_256;

		break;

	default:

		*trunc_len = 0;

	}

}



static int nfp_net_xfrm_add_state(struct xfrm_state *x)

{

	struct net_device *netdev = x->xso.dev;

	struct nfp_ipsec_cfg_mssg msg = {};

	int i, key_len, trunc_len, err = 0;

	struct nfp_ipsec_cfg_add_sa *cfg;

	struct nfp_net *nn;

	unsigned int saidx;



	nn = netdev_priv(netdev);

	cfg = &msg.cfg_add_sa;



	/* General */

	switch (x->props.mode) {

	case XFRM_MODE_TUNNEL:

		cfg->ctrl_word.mode = NFP_IPSEC_PROTMODE_TUNNEL;

		break;

	case XFRM_MODE_TRANSPORT:

		cfg->ctrl_word.mode = NFP_IPSEC_PROTMODE_TRANSPORT;

		break;

	default:

		nn_err(nn, "Unsupported mode for xfrm offload\n");

		return -EINVAL;

	}



	switch (x->id.proto) {

	case IPPROTO_ESP:

		cfg->ctrl_word.proto = NFP_IPSEC_PROTOCOL_ESP;

		break;

	case IPPROTO_AH:

		cfg->ctrl_word.proto = NFP_IPSEC_PROTOCOL_AH;

		break;

	default:

		nn_err(nn, "Unsupported protocol for xfrm offload\n");

		return -EINVAL;

	}



	if (x->props.flags & XFRM_STATE_ESN) {

		nn_err(nn, "Unsupported XFRM_REPLAY_MODE_ESN for xfrm offload\n");

		return -EINVAL;

	}



	cfg->spi = ntohl(x->id.spi);



	/* Hash/Authentication */

	if (x->aalg)

		trunc_len = x->aalg->alg_trunc_len;

	else

		trunc_len = 0;



	switch (x->props.aalgo) {

	case SADB_AALG_NONE:

		if (x->aead) {

			trunc_len = -1;

		} else {

			nn_err(nn, "Unsupported authentication algorithm\n");

			return -EINVAL;

		}

		break;

	case SADB_X_AALG_NULL:

		cfg->ctrl_word.hash = NFP_IPSEC_HASH_NONE;

		trunc_len = -1;

		break;

	case SADB_AALG_MD5HMAC:

		set_md5hmac(cfg, &trunc_len);

		break;

	case SADB_AALG_SHA1HMAC:

		set_sha1hmac(cfg, &trunc_len);

		break;

	case SADB_X_AALG_SHA2_256HMAC:

		set_sha2_256hmac(cfg, &trunc_len);

		break;

	case SADB_X_AALG_SHA2_384HMAC:

		set_sha2_384hmac(cfg, &trunc_len);

		break;

	case SADB_X_AALG_SHA2_512HMAC:

		set_sha2_512hmac(cfg, &trunc_len);

		break;

	default:

		nn_err(nn, "Unsupported authentication algorithm\n");

		return -EINVAL;

	}



	if (!trunc_len) {

		nn_err(nn, "Unsupported authentication algorithm trunc length\n");

		return -EINVAL;

	}



	if (x->aalg) {

		key_len = DIV_ROUND_UP(x->aalg->alg_key_len, BITS_PER_BYTE);

		if (key_len > sizeof(cfg->auth_key)) {

			nn_err(nn, "Insufficient space for offloaded auth key\n");

			return -EINVAL;

		}

		for (i = 0; i < key_len / sizeof(cfg->auth_key[0]) ; i++)

			cfg->auth_key[i] = get_unaligned_be32(x->aalg->alg_key +

							      sizeof(cfg->auth_key[0]) * i);

	}



	/* Encryption */

	switch (x->props.ealgo) {

	case SADB_EALG_NONE:

	case SADB_EALG_NULL:

		cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC;

		cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_NULL;

		break;

	case SADB_EALG_3DESCBC:

		cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC;

		cfg->ctrl_word.cipher = NFP_IPSEC_CIPHER_3DES;

		break;

	case SADB_X_EALG_AES_GCM_ICV16:

	case SADB_X_EALG_NULL_AES_GMAC:

		if (!x->aead) {

			nn_err(nn, "Invalid AES key data\n");

			return -EINVAL;

		}



		if (x->aead->alg_icv_len != 128) {

			nn_err(nn, "ICV must be 128bit with SADB_X_EALG_AES_GCM_ICV16\n");

			return -EINVAL;

		}

		cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CTR;

		cfg->ctrl_word.hash = NFP_IPSEC_HASH_GF128_128;



		/* Aead->alg_key_len includes 32-bit salt */

		if (set_aes_keylen(cfg, x->props.ealgo, x->aead->alg_key_len - 32)) {

			nn_err(nn, "Unsupported AES key length %d\n", x->aead->alg_key_len);

			return -EINVAL;

		}

		break;

	case SADB_X_EALG_AESCBC:

		cfg->ctrl_word.cimode = NFP_IPSEC_CIMODE_CBC;

		if (!x->ealg) {

			nn_err(nn, "Invalid AES key data\n");

			return -EINVAL;

		}

		if (set_aes_keylen(cfg, x->props.ealgo, x->ealg->alg_key_len) < 0) {

			nn_err(nn, "Unsupported AES key length %d\n", x->ealg->alg_key_len);

			return -EINVAL;

		}

		break;

	default:

		nn_err(nn, "Unsupported encryption algorithm for offload\n");

		return -EINVAL;

	}



	if (x->aead) {

		int salt_len = 4;



		key_len = DIV_ROUND_UP(x->aead->alg_key_len, BITS_PER_BYTE);

		key_len -= salt_len;



		if (key_len > sizeof(cfg->ciph_key)) {

			nn_err(nn, "aead: Insufficient space for offloaded key\n");

			return -EINVAL;

		}



		for (i = 0; i < key_len / sizeof(cfg->ciph_key[0]) ; i++)

			cfg->ciph_key[i] = get_unaligned_be32(x->aead->alg_key +

							      sizeof(cfg->ciph_key[0]) * i);



		/* Load up the salt */

		cfg->aesgcm_fields.salt = get_unaligned_be32(x->aead->alg_key + key_len);

	}



	if (x->ealg) {

		key_len = DIV_ROUND_UP(x->ealg->alg_key_len, BITS_PER_BYTE);



		if (key_len > sizeof(cfg->ciph_key)) {

			nn_err(nn, "ealg: Insufficient space for offloaded key\n");

			return -EINVAL;

		}

		for (i = 0; i < key_len / sizeof(cfg->ciph_key[0]) ; i++)

			cfg->ciph_key[i] = get_unaligned_be32(x->ealg->alg_key +

							      sizeof(cfg->ciph_key[0]) * i);

	}



	/* IP related info */

	switch (x->props.family) {

	case AF_INET:

		cfg->ipv6 = 0;

		cfg->src_ip[0] = ntohl(x->props.saddr.a4);

		cfg->dst_ip[0] = ntohl(x->id.daddr.a4);

		break;

	case AF_INET6:

		cfg->ipv6 = 1;

		for (i = 0; i < 4; i++) {

			cfg->src_ip[i] = ntohl(x->props.saddr.a6[i]);

			cfg->dst_ip[i] = ntohl(x->id.daddr.a6[i]);

		}

		break;

	default:

		nn_err(nn, "Unsupported address family\n");

		return -EINVAL;

	}



	/* Maximum nic IPsec code could handle. Other limits may apply. */

	cfg->pmtu_limit = 0xffff;

	cfg->ctrl_word.encap_dsbl = 1;



	/* SA direction */

	cfg->ctrl_word.dir = x->xso.dir;



	/* Find unused SA data*/

	err = xa_alloc(&nn->xa_ipsec, &saidx, x,

		       XA_LIMIT(0, NFP_NET_IPSEC_MAX_SA_CNT - 1), GFP_KERNEL);

	if (err < 0) {

		nn_err(nn, "Unable to get sa_data number for IPsec\n");

		return err;

	}



	/* Allocate saidx and commit the SA */

	err = nfp_ipsec_cfg_cmd_issue(nn, NFP_IPSEC_CFG_MSSG_ADD_SA, saidx, &msg);

	if (err) {

		xa_erase(&nn->xa_ipsec, saidx);

		nn_err(nn, "Failed to issue IPsec command err ret=%d\n", err);

		return err;

	}



	/* 0 is invalid offload_handle for kernel */

	x->xso.offload_handle = saidx + 1;

	return 0;

}



static void nfp_net_xfrm_del_state(struct xfrm_state *x)

{

	struct net_device *netdev = x->xso.dev;

	struct nfp_ipsec_cfg_mssg msg;

	struct nfp_net *nn;

	int err;



	nn = netdev_priv(netdev);

	err = nfp_ipsec_cfg_cmd_issue(nn, NFP_IPSEC_CFG_MSSG_INV_SA,

				      x->xso.offload_handle - 1, &msg);

	if (err)

		nn_warn(nn, "Failed to invalidate SA in hardware\n");



	xa_erase(&nn->xa_ipsec, x->xso.offload_handle - 1);

}



static bool nfp_net_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x)

{

	if (x->props.family == AF_INET)

		/* Offload with IPv4 options is not supported yet */

		return ip_hdr(skb)->ihl == 5;



	/* Offload with IPv6 extension headers is not support yet */

	return !(ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr));

}



static const struct xfrmdev_ops nfp_net_ipsec_xfrmdev_ops = {

	.xdo_dev_state_add = nfp_net_xfrm_add_state,

	.xdo_dev_state_delete = nfp_net_xfrm_del_state,

	.xdo_dev_offload_ok = nfp_net_ipsec_offload_ok,

};



void nfp_net_ipsec_init(struct nfp_net *nn)

{

	if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC))

		return;



	xa_init_flags(&nn->xa_ipsec, XA_FLAGS_ALLOC);

	nn->dp.netdev->xfrmdev_ops = &nfp_net_ipsec_xfrmdev_ops;

}



void nfp_net_ipsec_clean(struct nfp_net *nn)

{

	if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC))

		return;



	WARN_ON(!xa_empty(&nn->xa_ipsec));

	xa_destroy(&nn->xa_ipsec);

}



bool nfp_net_ipsec_tx_prep(struct nfp_net_dp *dp, struct sk_buff *skb,

			   struct nfp_ipsec_offload *offload_info)

{

	struct xfrm_offload *xo = xfrm_offload(skb);

	struct xfrm_state *x;



	x = xfrm_input_state(skb);

	if (!x)

		return false;



	offload_info->seq_hi = xo->seq.hi;

	offload_info->seq_low = xo->seq.low;

	offload_info->handle = x->xso.offload_handle;



	return true;

}



int nfp_net_ipsec_rx(struct nfp_meta_parsed *meta, struct sk_buff *skb)

{

	struct net_device *netdev = skb->dev;

	struct xfrm_offload *xo;

	struct xfrm_state *x;

	struct sec_path *sp;

	struct nfp_net *nn;

	u32 saidx;



	nn = netdev_priv(netdev);



	saidx = meta->ipsec_saidx - 1;

	if (saidx >= NFP_NET_IPSEC_MAX_SA_CNT)

		return -EINVAL;



	sp = secpath_set(skb);

	if (unlikely(!sp))

		return -ENOMEM;



	xa_lock(&nn->xa_ipsec);

	x = xa_load(&nn->xa_ipsec, saidx);

	xa_unlock(&nn->xa_ipsec);

	if (!x)

		return -EINVAL;



	xfrm_state_hold(x);

	sp->xvec[sp->len++] = x;

	sp->olen++;

	xo = xfrm_offload(skb);

	xo->flags = CRYPTO_DONE;

	xo->status = CRYPTO_SUCCESS;



	return 0;

}

drivers/net/ethernet/netronome/nfp/nfd3/dp.c

+48 −10
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ 
#include <linux/bpf_trace.h>

#include <linux/netdevice.h>

#include <linux/bitfield.h>

#include <net/xfrm.h>



#include "../nfp_app.h"

#include "../nfp_net.h"
@@ -167,28 +168,34 @@ nfp_nfd3_tx_csum(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec, 
	u64_stats_update_end(&r_vec->tx_sync);

}



static int nfp_nfd3_prep_tx_meta(struct nfp_net_dp *dp, struct sk_buff *skb, u64 tls_handle)

static int nfp_nfd3_prep_tx_meta(struct nfp_net_dp *dp, struct sk_buff *skb,

				 u64 tls_handle, bool *ipsec)

{

	struct metadata_dst *md_dst = skb_metadata_dst(skb);

	struct nfp_ipsec_offload offload_info;

	unsigned char *data;

	bool vlan_insert;

	u32 meta_id = 0;

	int md_bytes;



	if (unlikely(md_dst || tls_handle)) {

#ifdef CONFIG_NFP_NET_IPSEC

	if (xfrm_offload(skb))

		*ipsec = nfp_net_ipsec_tx_prep(dp, skb, &offload_info);

#endif



	if (unlikely(md_dst && md_dst->type != METADATA_HW_PORT_MUX))

		md_dst = NULL;

	}



	vlan_insert = skb_vlan_tag_present(skb) && (dp->ctrl & NFP_NET_CFG_CTRL_TXVLAN_V2);



	if (!(md_dst || tls_handle || vlan_insert))

	if (!(md_dst || tls_handle || vlan_insert || *ipsec))

		return 0;



	md_bytes = sizeof(meta_id) +

		   !!md_dst * NFP_NET_META_PORTID_SIZE +

		   !!tls_handle * NFP_NET_META_CONN_HANDLE_SIZE +

		   vlan_insert * NFP_NET_META_VLAN_SIZE;

		   vlan_insert * NFP_NET_META_VLAN_SIZE +

		   *ipsec * NFP_NET_META_IPSEC_FIELD_SIZE; /* IPsec has 12 bytes of metadata */



	if (unlikely(skb_cow_head(skb, md_bytes)))

		return -ENOMEM;
@@ -218,6 +225,19 @@ static int nfp_nfd3_prep_tx_meta(struct nfp_net_dp *dp, struct sk_buff *skb, u64 
		meta_id <<= NFP_NET_META_FIELD_SIZE;

		meta_id |= NFP_NET_META_VLAN;

	}

	if (*ipsec) {

		/* IPsec has three consecutive 4-bit IPsec metadata types,

		 * so in total IPsec has three 4 bytes of metadata.

		 */

		data -= NFP_NET_META_IPSEC_SIZE;

		put_unaligned_be32(offload_info.seq_hi, data);

		data -= NFP_NET_META_IPSEC_SIZE;

		put_unaligned_be32(offload_info.seq_low, data);

		data -= NFP_NET_META_IPSEC_SIZE;

		put_unaligned_be32(offload_info.handle - 1, data);

		meta_id <<= NFP_NET_META_IPSEC_FIELD_SIZE;

		meta_id |= NFP_NET_META_IPSEC << 8 | NFP_NET_META_IPSEC << 4 | NFP_NET_META_IPSEC;

	}



	data -= sizeof(meta_id);

	put_unaligned_be32(meta_id, data);
@@ -246,6 +266,7 @@ netdev_tx_t nfp_nfd3_tx(struct sk_buff *skb, struct net_device *netdev) 
	dma_addr_t dma_addr;

	unsigned int fsize;

	u64 tls_handle = 0;

	bool ipsec = false;

	u16 qidx;



	dp = &nn->dp;
@@ -273,7 +294,7 @@ netdev_tx_t nfp_nfd3_tx(struct sk_buff *skb, struct net_device *netdev) 
		return NETDEV_TX_OK;

	}



	md_bytes = nfp_nfd3_prep_tx_meta(dp, skb, tls_handle);

	md_bytes = nfp_nfd3_prep_tx_meta(dp, skb, tls_handle, &ipsec);

	if (unlikely(md_bytes < 0))

		goto err_flush;
@@ -312,6 +333,8 @@ netdev_tx_t nfp_nfd3_tx(struct sk_buff *skb, struct net_device *netdev) 
		txd->vlan = cpu_to_le16(skb_vlan_tag_get(skb));

	}



	if (ipsec)

		nfp_nfd3_ipsec_tx(txd, skb);

	/* Gather DMA */

	if (nr_frags > 0) {

		__le64 second_half;
@@ -764,6 +787,15 @@ nfp_nfd3_parse_meta(struct net_device *netdev, struct nfp_meta_parsed *meta, 
				return false;

			data += sizeof(struct nfp_net_tls_resync_req);

			break;

#ifdef CONFIG_NFP_NET_IPSEC

		case NFP_NET_META_IPSEC:

			/* Note: IPsec packet will have zero saidx, so need add 1

			 * to indicate packet is IPsec packet within driver.

			 */

			meta->ipsec_saidx = get_unaligned_be32(data) + 1;

			data += 4;

			break;

#endif

		default:

			return true;

		}
@@ -876,12 +908,11 @@ static int nfp_nfd3_rx(struct nfp_net_rx_ring *rx_ring, int budget) 
	struct nfp_net_dp *dp = &r_vec->nfp_net->dp;

	struct nfp_net_tx_ring *tx_ring;

	struct bpf_prog *xdp_prog;

	int idx, pkts_polled = 0;

	bool xdp_tx_cmpl = false;

	unsigned int true_bufsz;

	struct sk_buff *skb;

	int pkts_polled = 0;

	struct xdp_buff xdp;

	int idx;



	xdp_prog = READ_ONCE(dp->xdp_prog);

	true_bufsz = xdp_prog ? PAGE_SIZE : dp->fl_bufsz;
@@ -1081,6 +1112,13 @@ static int nfp_nfd3_rx(struct nfp_net_rx_ring *rx_ring, int budget) 
			continue;

		}



#ifdef CONFIG_NFP_NET_IPSEC

		if (meta.ipsec_saidx != 0 && unlikely(nfp_net_ipsec_rx(&meta, skb))) {

			nfp_nfd3_rx_drop(dp, r_vec, rx_ring, NULL, skb);

			continue;

		}

#endif



		if (meta_len_xdp)

			skb_metadata_set(skb, meta_len_xdp);