Commit 402f2d6b authored by David S. Miller's avatar David S. Miller
Browse files

Merge tag 'mlx5-updates-2022-05-03' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux 



mlx5-updates-2022-05-03

Leon Romanovsky Says:
=====================

Extra IPsec cleanup

After FPGA IPsec removal, we can go further and make sure that flow
steering logic is aligned to mlx5_core standard together with deep
cleaning of whole IPsec path.

=====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 6a9b3de8 656d3389

drivers/net/ethernet/mellanox/mlx5/core/en/fs.h

+0 −1
Original line number Diff line number Diff line
@@ -164,7 +164,6 @@ struct mlx5e_ptp_fs; 


struct mlx5e_flow_steering {

	struct mlx5_flow_namespace      *ns;

	struct mlx5_flow_namespace      *egress_ns;

#ifdef CONFIG_MLX5_EN_RXNFC

	struct mlx5e_ethtool_steering   ethtool;

#endif

drivers/net/ethernet/mellanox/mlx5/core/en/params.c

+1 −1
Original line number Diff line number Diff line
@@ -5,7 +5,7 @@ 
#include "en/txrx.h"

#include "en/port.h"

#include "en_accel/en_accel.h"

#include "en_accel/ipsec_offload.h"

#include "en_accel/ipsec.h"



static bool mlx5e_rx_is_xdp(struct mlx5e_params *params,

			    struct mlx5e_xsk_param *xsk)

drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c

+48 −126
Original line number Diff line number Diff line
@@ -37,23 +37,12 @@ 
#include <linux/netdevice.h>



#include "en.h"

#include "en_accel/ipsec.h"

#include "en_accel/ipsec_rxtx.h"

#include "en_accel/ipsec_fs.h"

#include "ipsec.h"

#include "ipsec_rxtx.h"



static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(struct xfrm_state *x)

{

	struct mlx5e_ipsec_sa_entry *sa;



	if (!x)

		return NULL;



	sa = (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle;

	if (!sa)

		return NULL;



	WARN_ON(sa->x != x);

	return sa;

	return (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle;

}



struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *ipsec,
@@ -74,9 +63,9 @@ struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *ipsec, 
	return ret;

}



static int  mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry,

				    unsigned int handle)

static int mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry)

{

	unsigned int handle = sa_entry->ipsec_obj_id;

	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;

	struct mlx5e_ipsec_sa_entry *_sa_entry;

	unsigned long flags;
@@ -148,7 +137,7 @@ mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, 
				   struct mlx5_accel_esp_xfrm_attrs *attrs)

{

	struct xfrm_state *x = sa_entry->x;

	struct aes_gcm_keymat *aes_gcm = &attrs->keymat.aes_gcm;

	struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm;

	struct aead_geniv_ctx *geniv_ctx;

	struct crypto_aead *aead;

	unsigned int crypto_data_len, key_len;
@@ -182,12 +171,6 @@ mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, 
			attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;

	}



	/* rx handle */

	attrs->sa_handle = sa_entry->handle;



	/* algo type */

	attrs->keymat_type = MLX5_ACCEL_ESP_KEYMAT_AES_GCM;



	/* action */

	attrs->action = (!(x->xso.flags & XFRM_OFFLOAD_INBOUND)) ?

			MLX5_ACCEL_ESP_ACTION_ENCRYPT :
@@ -198,7 +181,7 @@ mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry, 
			MLX5_ACCEL_ESP_FLAGS_TUNNEL;



	/* spi */

	attrs->spi = x->id.spi;

	attrs->spi = be32_to_cpu(x->id.spi);



	/* source , destination ips */

	memcpy(&attrs->saddr, x->props.saddr.a6, sizeof(attrs->saddr));
@@ -226,7 +209,7 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) 
		return -EINVAL;

	}

	if (x->props.flags & XFRM_STATE_ESN &&

	    !(mlx5_ipsec_device_caps(priv->mdev) & MLX5_ACCEL_IPSEC_CAP_ESN)) {

	    !(mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_ESN)) {

		netdev_info(netdev, "Cannot offload ESN xfrm states\n");

		return -EINVAL;

	}
@@ -273,39 +256,29 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x) 
		netdev_info(netdev, "Cannot offload xfrm states with geniv other than seqiv\n");

		return -EINVAL;

	}

	if (x->props.family == AF_INET6 &&

	    !(mlx5_ipsec_device_caps(priv->mdev) & MLX5_ACCEL_IPSEC_CAP_IPV6)) {

		netdev_info(netdev, "IPv6 xfrm state offload is not supported by this device\n");

		return -EINVAL;

	}

	return 0;

}



static int mlx5e_xfrm_fs_add_rule(struct mlx5e_priv *priv,

				  struct mlx5e_ipsec_sa_entry *sa_entry)

static void _update_xfrm_state(struct work_struct *work)

{

	return mlx5e_accel_ipsec_fs_add_rule(priv, &sa_entry->xfrm->attrs,

					     sa_entry->ipsec_obj_id,

					     &sa_entry->ipsec_rule);

}

	struct mlx5e_ipsec_modify_state_work *modify_work =

		container_of(work, struct mlx5e_ipsec_modify_state_work, work);

	struct mlx5e_ipsec_sa_entry *sa_entry = container_of(

		modify_work, struct mlx5e_ipsec_sa_entry, modify_work);



static void mlx5e_xfrm_fs_del_rule(struct mlx5e_priv *priv,

				   struct mlx5e_ipsec_sa_entry *sa_entry)

{

	mlx5e_accel_ipsec_fs_del_rule(priv, &sa_entry->xfrm->attrs,

				      &sa_entry->ipsec_rule);

	mlx5_accel_esp_modify_xfrm(sa_entry, &modify_work->attrs);

}



static int mlx5e_xfrm_add_state(struct xfrm_state *x)

{

	struct mlx5e_ipsec_sa_entry *sa_entry = NULL;

	struct net_device *netdev = x->xso.real_dev;

	struct mlx5_accel_esp_xfrm_attrs attrs;

	struct mlx5e_priv *priv;

	unsigned int sa_handle;

	int err;



	priv = netdev_priv(netdev);

	if (!priv->ipsec)

		return -EOPNOTSUPP;



	err = mlx5e_xfrm_validate_state(x);

	if (err)
@@ -323,31 +296,18 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) 
	/* check esn */

	mlx5e_ipsec_update_esn_state(sa_entry);



	/* create xfrm */

	mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &attrs);

	sa_entry->xfrm = mlx5_accel_esp_create_xfrm(priv->mdev, &attrs);

	if (IS_ERR(sa_entry->xfrm)) {

		err = PTR_ERR(sa_entry->xfrm);

		goto err_sa_entry;

	}



	mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &sa_entry->attrs);

	/* create hw context */

	sa_entry->hw_context =

			mlx5_accel_esp_create_hw_context(priv->mdev,

							 sa_entry->xfrm,

							 &sa_handle);

	if (IS_ERR(sa_entry->hw_context)) {

		err = PTR_ERR(sa_entry->hw_context);

	err = mlx5_ipsec_create_sa_ctx(sa_entry);

	if (err)

		goto err_xfrm;

	}



	sa_entry->ipsec_obj_id = sa_handle;

	err = mlx5e_xfrm_fs_add_rule(priv, sa_entry);

	err = mlx5e_accel_ipsec_fs_add_rule(priv, sa_entry);

	if (err)

		goto err_hw_ctx;



	if (x->xso.flags & XFRM_OFFLOAD_INBOUND) {

		err = mlx5e_ipsec_sadb_rx_add(sa_entry, sa_handle);

		err = mlx5e_ipsec_sadb_rx_add(sa_entry);

		if (err)

			goto err_add_rule;

	} else {
@@ -355,18 +315,16 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x) 
				mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv;

	}



	INIT_WORK(&sa_entry->modify_work.work, _update_xfrm_state);

	x->xso.offload_handle = (unsigned long)sa_entry;

	goto out;



err_add_rule:

	mlx5e_xfrm_fs_del_rule(priv, sa_entry);

	mlx5e_accel_ipsec_fs_del_rule(priv, sa_entry);

err_hw_ctx:

	mlx5_accel_esp_free_hw_context(priv->mdev, sa_entry->hw_context);

	mlx5_ipsec_free_sa_ctx(sa_entry);

err_xfrm:

	mlx5_accel_esp_destroy_xfrm(sa_entry->xfrm);

err_sa_entry:

	kfree(sa_entry);



out:

	return err;

}
@@ -375,9 +333,6 @@ static void mlx5e_xfrm_del_state(struct xfrm_state *x) 
{

	struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);



	if (!sa_entry)

		return;



	if (x->xso.flags & XFRM_OFFLOAD_INBOUND)

		mlx5e_ipsec_sadb_rx_del(sa_entry);

}
@@ -387,22 +342,16 @@ static void mlx5e_xfrm_free_state(struct xfrm_state *x) 
	struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);

	struct mlx5e_priv *priv = netdev_priv(x->xso.dev);



	if (!sa_entry)

		return;



	if (sa_entry->hw_context) {

		flush_workqueue(sa_entry->ipsec->wq);

		mlx5e_xfrm_fs_del_rule(priv, sa_entry);

		mlx5_accel_esp_free_hw_context(sa_entry->xfrm->mdev, sa_entry->hw_context);

		mlx5_accel_esp_destroy_xfrm(sa_entry->xfrm);

	}



	cancel_work_sync(&sa_entry->modify_work.work);

	mlx5e_accel_ipsec_fs_del_rule(priv, sa_entry);

	mlx5_ipsec_free_sa_ctx(sa_entry);

	kfree(sa_entry);

}



int mlx5e_ipsec_init(struct mlx5e_priv *priv)

{

	struct mlx5e_ipsec *ipsec = NULL;

	struct mlx5e_ipsec *ipsec;

	int ret;



	if (!mlx5_ipsec_device_caps(priv->mdev)) {

		netdev_dbg(priv->netdev, "Not an IPSec offload device\n");
@@ -415,18 +364,27 @@ int mlx5e_ipsec_init(struct mlx5e_priv *priv) 


	hash_init(ipsec->sadb_rx);

	spin_lock_init(&ipsec->sadb_rx_lock);

	ipsec->en_priv = priv;

	ipsec->mdev = priv->mdev;

	ipsec->wq = alloc_ordered_workqueue("mlx5e_ipsec: %s", 0,

					    priv->netdev->name);

	if (!ipsec->wq) {

		kfree(ipsec);

		return -ENOMEM;

		ret = -ENOMEM;

		goto err_wq;

	}



	ret = mlx5e_accel_ipsec_fs_init(ipsec);

	if (ret)

		goto err_fs_init;



	priv->ipsec = ipsec;

	mlx5e_accel_ipsec_fs_init(priv);

	netdev_dbg(priv->netdev, "IPSec attached to netdevice\n");

	return 0;



err_fs_init:

	destroy_workqueue(ipsec->wq);

err_wq:

	kfree(ipsec);

	return (ret != -EOPNOTSUPP) ? ret : 0;

}



void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv)
@@ -436,9 +394,8 @@ void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv) 
	if (!ipsec)

		return;



	mlx5e_accel_ipsec_fs_cleanup(priv);

	mlx5e_accel_ipsec_fs_cleanup(ipsec);

	destroy_workqueue(ipsec->wq);



	kfree(ipsec);

	priv->ipsec = NULL;

}
@@ -458,50 +415,19 @@ static bool mlx5e_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x) 
	return true;

}



struct mlx5e_ipsec_modify_state_work {

	struct work_struct		work;

	struct mlx5_accel_esp_xfrm_attrs attrs;

	struct mlx5e_ipsec_sa_entry	*sa_entry;

};



static void _update_xfrm_state(struct work_struct *work)

{

	int ret;

	struct mlx5e_ipsec_modify_state_work *modify_work =

		container_of(work, struct mlx5e_ipsec_modify_state_work, work);

	struct mlx5e_ipsec_sa_entry *sa_entry = modify_work->sa_entry;



	ret = mlx5_accel_esp_modify_xfrm(sa_entry->xfrm,

					 &modify_work->attrs);

	if (ret)

		netdev_warn(sa_entry->ipsec->en_priv->netdev,

			    "Not an IPSec offload device\n");



	kfree(modify_work);

}



static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x)

{

	struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);

	struct mlx5e_ipsec_modify_state_work *modify_work;

	struct mlx5e_ipsec_modify_state_work *modify_work =

		&sa_entry->modify_work;

	bool need_update;



	if (!sa_entry)

		return;



	need_update = mlx5e_ipsec_update_esn_state(sa_entry);

	if (!need_update)

		return;



	modify_work = kzalloc(sizeof(*modify_work), GFP_ATOMIC);

	if (!modify_work)

		return;



	mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &modify_work->attrs);

	modify_work->sa_entry = sa_entry;



	INIT_WORK(&modify_work->work, _update_xfrm_state);

	WARN_ON(!queue_work(sa_entry->ipsec->wq, &modify_work->work));

	queue_work(sa_entry->ipsec->wq, &modify_work->work);

}



static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = {
@@ -517,11 +443,8 @@ void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) 
	struct mlx5_core_dev *mdev = priv->mdev;

	struct net_device *netdev = priv->netdev;



	if (!(mlx5_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_ESP) ||

	    !MLX5_CAP_ETH(mdev, swp)) {

		mlx5_core_dbg(mdev, "mlx5e: ESP and SWP offload not supported\n");

	if (!mlx5_ipsec_device_caps(mdev))

		return;

	}



	mlx5_core_info(mdev, "mlx5e: IPSec ESP acceleration enabled\n");

	netdev->xfrmdev_ops = &mlx5e_ipsec_xfrmdev_ops;
@@ -536,8 +459,7 @@ void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) 
	netdev->features |= NETIF_F_HW_ESP_TX_CSUM;

	netdev->hw_enc_features |= NETIF_F_HW_ESP_TX_CSUM;



	if (!(mlx5_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_LSO) ||

	    !MLX5_CAP_ETH(mdev, swp_lso)) {

	if (!MLX5_CAP_ETH(mdev, swp_lso)) {

		mlx5_core_dbg(mdev, "mlx5e: ESP LSO not supported\n");

		return;

	}

drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h

+80 −5
Original line number Diff line number Diff line
@@ -40,11 +40,56 @@ 
#include <net/xfrm.h>

#include <linux/idr.h>



#include "ipsec_offload.h"



#define MLX5E_IPSEC_SADB_RX_BITS 10

#define MLX5E_IPSEC_ESN_SCOPE_MID 0x80000000L



enum mlx5_accel_esp_flags {

	MLX5_ACCEL_ESP_FLAGS_TUNNEL            = 0,    /* Default */

	MLX5_ACCEL_ESP_FLAGS_TRANSPORT         = 1UL << 0,

	MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED     = 1UL << 1,

	MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP = 1UL << 2,

};



enum mlx5_accel_esp_action {

	MLX5_ACCEL_ESP_ACTION_DECRYPT,

	MLX5_ACCEL_ESP_ACTION_ENCRYPT,

};



struct aes_gcm_keymat {

	u64   seq_iv;



	u32   salt;

	u32   icv_len;



	u32   key_len;

	u32   aes_key[256 / 32];

};



struct mlx5_accel_esp_xfrm_attrs {

	enum mlx5_accel_esp_action action;

	u32   esn;

	u32   spi;

	u32   flags;

	struct aes_gcm_keymat aes_gcm;



	union {

		__be32 a4;

		__be32 a6[4];

	} saddr;



	union {

		__be32 a4;

		__be32 a6[4];

	} daddr;



	u8 is_ipv6;

};



enum mlx5_ipsec_cap {

	MLX5_IPSEC_CAP_CRYPTO		= 1 << 0,

	MLX5_IPSEC_CAP_ESN		= 1 << 1,

};



struct mlx5e_priv;



struct mlx5e_ipsec_sw_stats {
@@ -61,7 +106,7 @@ struct mlx5e_accel_fs_esp; 
struct mlx5e_ipsec_tx;



struct mlx5e_ipsec {

	struct mlx5e_priv *en_priv;

	struct mlx5_core_dev *mdev;

	DECLARE_HASHTABLE(sadb_rx, MLX5E_IPSEC_SADB_RX_BITS);

	spinlock_t sadb_rx_lock; /* Protects sadb_rx */

	struct mlx5e_ipsec_sw_stats sw_stats;
@@ -81,18 +126,24 @@ struct mlx5e_ipsec_rule { 
	struct mlx5_modify_hdr *set_modify_hdr;

};



struct mlx5e_ipsec_modify_state_work {

	struct work_struct		work;

	struct mlx5_accel_esp_xfrm_attrs attrs;

};



struct mlx5e_ipsec_sa_entry {

	struct hlist_node hlist; /* Item in SADB_RX hashtable */

	struct mlx5e_ipsec_esn_state esn_state;

	unsigned int handle; /* Handle in SADB_RX */

	struct xfrm_state *x;

	struct mlx5e_ipsec *ipsec;

	struct mlx5_accel_esp_xfrm *xfrm;

	void *hw_context;

	struct mlx5_accel_esp_xfrm_attrs attrs;

	void (*set_iv_op)(struct sk_buff *skb, struct xfrm_state *x,

			  struct xfrm_offload *xo);

	u32 ipsec_obj_id;

	u32 enc_key_id;

	struct mlx5e_ipsec_rule ipsec_rule;

	struct mlx5e_ipsec_modify_state_work modify_work;

};



int mlx5e_ipsec_init(struct mlx5e_priv *priv);
@@ -102,6 +153,26 @@ void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv); 
struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *dev,

					      unsigned int handle);



void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec);

int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec);

int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_priv *priv,

				  struct mlx5e_ipsec_sa_entry *sa_entry);

void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_priv *priv,

				   struct mlx5e_ipsec_sa_entry *sa_entry);



int mlx5_ipsec_create_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry);

void mlx5_ipsec_free_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry);



u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev);



void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry,

				const struct mlx5_accel_esp_xfrm_attrs *attrs);



static inline struct mlx5_core_dev *

mlx5e_ipsec_sa2dev(struct mlx5e_ipsec_sa_entry *sa_entry)

{

	return sa_entry->ipsec->mdev;

}

#else

static inline int mlx5e_ipsec_init(struct mlx5e_priv *priv)

{
@@ -116,6 +187,10 @@ static inline void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv) 
{

}



static inline u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev)

{

	return 0;

}

#endif



#endif	/* __MLX5E_IPSEC_H__ */

drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c

+126 −236

File changed.

Preview size limit exceeded, changes collapsed.