Merge branch 'mlx5 IPsec packet offload support (Part II)' (1de8fda4) · Commits · EulixOS / Software / Kernel

drivers/net/ethernet/mellanox/mlx5/core/en/fs.h

+2 −1

Original line number	Diff line number	Diff line
		@@ -84,7 +84,8 @@ enum {
		MLX5E_ARFS_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1,
		#endif
		#ifdef CONFIG_MLX5_EN_IPSEC
		MLX5E_ACCEL_FS_ESP_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1,
		MLX5E_ACCEL_FS_POL_FT_LEVEL = MLX5E_INNER_TTC_FT_LEVEL + 1,
		MLX5E_ACCEL_FS_ESP_FT_LEVEL,
		MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL,
		#endif
		};

drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c

+238 −74

Original line number	Diff line number	Diff line
		@@ -45,55 +45,9 @@ static struct mlx5e_ipsec_sa_entry to_ipsec_sa_entry(struct xfrm_state x)
		return (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle;
		}

		struct xfrm_state mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec ipsec,
		unsigned int handle)
		static struct mlx5e_ipsec_pol_entry to_ipsec_pol_entry(struct xfrm_policy x)
		{
		struct mlx5e_ipsec_sa_entry *sa_entry;
		struct xfrm_state *ret = NULL;

		rcu_read_lock();
		hash_for_each_possible_rcu(ipsec->sadb_rx, sa_entry, hlist, handle)
		if (sa_entry->handle == handle) {
		ret = sa_entry->x;
		xfrm_state_hold(ret);
		break;
		}
		rcu_read_unlock();

		return ret;
		}

		static int mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry)
		{
		unsigned int handle = sa_entry->ipsec_obj_id;
		struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
		struct mlx5e_ipsec_sa_entry *_sa_entry;
		unsigned long flags;

		rcu_read_lock();
		hash_for_each_possible_rcu(ipsec->sadb_rx, _sa_entry, hlist, handle)
		if (_sa_entry->handle == handle) {
		rcu_read_unlock();
		return -EEXIST;
		}
		rcu_read_unlock();

		spin_lock_irqsave(&ipsec->sadb_rx_lock, flags);
		sa_entry->handle = handle;
		hash_add_rcu(ipsec->sadb_rx, &sa_entry->hlist, sa_entry->handle);
		spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags);

		return 0;
		}

		static void mlx5e_ipsec_sadb_rx_del(struct mlx5e_ipsec_sa_entry *sa_entry)
		{
		struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
		unsigned long flags;

		spin_lock_irqsave(&ipsec->sadb_rx_lock, flags);
		hash_del_rcu(&sa_entry->hlist);
		spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags);
		return (struct mlx5e_ipsec_pol_entry *)x->xdo.offload_handle;
		}

		static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry)
		@@ -129,8 +83,32 @@ static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry)
		return false;
		}

		static void
		mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,
		static void mlx5e_ipsec_init_limits(struct mlx5e_ipsec_sa_entry *sa_entry,
		struct mlx5_accel_esp_xfrm_attrs *attrs)
		{
		struct xfrm_state *x = sa_entry->x;

		attrs->hard_packet_limit = x->lft.hard_packet_limit;
		if (x->lft.soft_packet_limit == XFRM_INF)
		return;

		/* Hardware decrements hard_packet_limit counter through
		* the operation. While fires an event when soft_packet_limit
		* is reached. It emans that we need substitute the numbers
		* in order to properly count soft limit.
		*
		* As an example:
		* XFRM user sets soft limit is 2 and hard limit is 9 and
		* expects to see soft event after 2 packets and hard event
		* after 9 packets. In our case, the hard limit will be set
		* to 9 and soft limit is comparator to 7 so user gets the
		* soft event after 2 packeta
		*/
		attrs->soft_packet_limit =
		x->lft.hard_packet_limit - x->lft.soft_packet_limit;
		}

		void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,
		struct mlx5_accel_esp_xfrm_attrs *attrs)
		{
		struct xfrm_state *x = sa_entry->x;
		@@ -157,6 +135,8 @@ mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,
		memcpy(&aes_gcm->salt, x->aead->alg_key + key_len,
		sizeof(aes_gcm->salt));

		attrs->authsize = crypto_aead_authsize(aead) / 4; /* in dwords */

		/* iv len */
		aes_gcm->icv_len = x->aead->alg_icv_len;

		@@ -177,6 +157,9 @@ mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,
		memcpy(&attrs->daddr, x->id.daddr.a6, sizeof(attrs->daddr));
		attrs->family = x->props.family;
		attrs->type = x->xso.type;
		attrs->reqid = x->props.reqid;

		mlx5e_ipsec_init_limits(sa_entry, attrs);
		}

		static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x)
		@@ -208,11 +191,6 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x)
		netdev_info(netdev, "Only IPv4/6 xfrm states may be offloaded\n");
		return -EINVAL;
		}
		if (x->props.mode != XFRM_MODE_TRANSPORT &&
		x->props.mode != XFRM_MODE_TUNNEL) {
		dev_info(&netdev->dev, "Only transport and tunnel xfrm states may be offloaded\n");
		return -EINVAL;
		}
		if (x->id.proto != IPPROTO_ESP) {
		netdev_info(netdev, "Only ESP xfrm state may be offloaded\n");
		return -EINVAL;
		@@ -246,11 +224,32 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x)
		netdev_info(netdev, "Cannot offload xfrm states with geniv other than seqiv\n");
		return -EINVAL;
		}
		if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) {
		netdev_info(netdev, "Unsupported xfrm offload type\n");
		switch (x->xso.type) {
		case XFRM_DEV_OFFLOAD_CRYPTO:
		if (!(mlx5_ipsec_device_caps(priv->mdev) &
		MLX5_IPSEC_CAP_CRYPTO)) {
		netdev_info(netdev, "Crypto offload is not supported\n");
		return -EINVAL;
		}

		if (x->props.mode != XFRM_MODE_TRANSPORT &&
		x->props.mode != XFRM_MODE_TUNNEL) {
		netdev_info(netdev, "Only transport and tunnel xfrm states may be offloaded\n");
		return -EINVAL;
		}
		break;
		case XFRM_DEV_OFFLOAD_PACKET:
		if (!(mlx5_ipsec_device_caps(priv->mdev) &
		MLX5_IPSEC_CAP_PACKET_OFFLOAD)) {
		netdev_info(netdev, "Packet offload is not supported\n");
		return -EINVAL;
		}

		if (x->props.mode != XFRM_MODE_TRANSPORT) {
		netdev_info(netdev, "Only transport xfrm states may be offloaded in packet mode\n");
		return -EINVAL;
		}
		if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) {

		if (x->replay_esn && x->replay_esn->replay_window != 32 &&
		x->replay_esn->replay_window != 64 &&
		x->replay_esn->replay_window != 128 &&
		@@ -260,6 +259,31 @@ static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x)
		x->replay_esn->replay_window);
		return -EINVAL;
		}

		if (!x->props.reqid) {
		netdev_info(netdev, "Cannot offload without reqid\n");
		return -EINVAL;
		}

		if (x->lft.hard_byte_limit != XFRM_INF \|\|
		x->lft.soft_byte_limit != XFRM_INF) {
		netdev_info(netdev,
		"Device doesn't support limits in bytes\n");
		return -EINVAL;
		}

		if (x->lft.soft_packet_limit >= x->lft.hard_packet_limit &&
		x->lft.hard_packet_limit != XFRM_INF) {
		/* XFRM stack doesn't prevent such configuration :(. */
		netdev_info(netdev,
		"Hard packet limit must be greater than soft one\n");
		return -EINVAL;
		}
		break;
		default:
		netdev_info(netdev, "Unsupported xfrm offload type %d\n",
		x->xso.type);
		return -EINVAL;
		}
		return 0;
		}
		@@ -278,6 +302,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x)
		{
		struct mlx5e_ipsec_sa_entry *sa_entry = NULL;
		struct net_device *netdev = x->xso.real_dev;
		struct mlx5e_ipsec *ipsec;
		struct mlx5e_priv *priv;
		int err;

		@@ -285,6 +310,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x)
		if (!priv->ipsec)
		return -EOPNOTSUPP;

		ipsec = priv->ipsec;
		err = mlx5e_xfrm_validate_state(x);
		if (err)
		return err;
		@@ -296,7 +322,7 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x)
		}

		sa_entry->x = x;
		sa_entry->ipsec = priv->ipsec;
		sa_entry->ipsec = ipsec;

		/* check esn */
		mlx5e_ipsec_update_esn_state(sa_entry);
		@@ -311,18 +337,22 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x)
		if (err)
		goto err_hw_ctx;

		if (x->xso.dir == XFRM_DEV_OFFLOAD_IN) {
		err = mlx5e_ipsec_sadb_rx_add(sa_entry);
		/* We use *_bh() variant because xfrm_timer_handler(), which runs
		* in softirq context, can reach our state delete logic and we need
		* xa_erase_bh() there.
		*/
		err = xa_insert_bh(&ipsec->sadb, sa_entry->ipsec_obj_id, sa_entry,
		GFP_KERNEL);
		if (err)
		goto err_add_rule;
		} else {

		if (x->xso.dir == XFRM_DEV_OFFLOAD_OUT)
		sa_entry->set_iv_op = (x->props.flags & XFRM_STATE_ESN) ?
		mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv;
		}

		INIT_WORK(&sa_entry->modify_work.work, _update_xfrm_state);
		x->xso.offload_handle = (unsigned long)sa_entry;
		goto out;
		return 0;

		err_add_rule:
		mlx5e_accel_ipsec_fs_del_rule(sa_entry);
		@@ -337,9 +367,11 @@ static int mlx5e_xfrm_add_state(struct xfrm_state *x)
		static void mlx5e_xfrm_del_state(struct xfrm_state *x)
		{
		struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
		struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
		struct mlx5e_ipsec_sa_entry *old;

		if (x->xso.dir == XFRM_DEV_OFFLOAD_IN)
		mlx5e_ipsec_sadb_rx_del(sa_entry);
		old = xa_erase_bh(&ipsec->sadb, sa_entry->ipsec_obj_id);
		WARN_ON(old != sa_entry);
		}

		static void mlx5e_xfrm_free_state(struct xfrm_state *x)
		@@ -366,8 +398,7 @@ void mlx5e_ipsec_init(struct mlx5e_priv *priv)
		if (!ipsec)
		return;

		hash_init(ipsec->sadb_rx);
		spin_lock_init(&ipsec->sadb_rx_lock);
		xa_init_flags(&ipsec->sadb, XA_FLAGS_ALLOC);
		ipsec->mdev = priv->mdev;
		ipsec->wq = alloc_ordered_workqueue("mlx5e_ipsec: %s", 0,
		priv->netdev->name);
		@@ -446,6 +477,122 @@ static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x)
		queue_work(sa_entry->ipsec->wq, &modify_work->work);
		}

		static void mlx5e_xfrm_update_curlft(struct xfrm_state *x)
		{
		struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
		int err;

		lockdep_assert_held(&x->lock);

		if (sa_entry->attrs.soft_packet_limit == XFRM_INF)
		/* Limits are not configured, as soft limit
		* must be lowever than hard limit.
		*/
		return;

		err = mlx5e_ipsec_aso_query(sa_entry, NULL);
		if (err)
		return;

		mlx5e_ipsec_aso_update_curlft(sa_entry, &x->curlft.packets);
		}

		static int mlx5e_xfrm_validate_policy(struct xfrm_policy *x)
		{
		struct net_device *netdev = x->xdo.real_dev;

		if (x->type != XFRM_POLICY_TYPE_MAIN) {
		netdev_info(netdev, "Cannot offload non-main policy types\n");
		return -EINVAL;
		}

		/* Please pay attention that we support only one template */
		if (x->xfrm_nr > 1) {
		netdev_info(netdev, "Cannot offload more than one template\n");
		return -EINVAL;
		}

		if (x->xdo.dir != XFRM_DEV_OFFLOAD_IN &&
		x->xdo.dir != XFRM_DEV_OFFLOAD_OUT) {
		netdev_info(netdev, "Cannot offload forward policy\n");
		return -EINVAL;
		}

		if (!x->xfrm_vec[0].reqid) {
		netdev_info(netdev, "Cannot offload policy without reqid\n");
		return -EINVAL;
		}

		if (x->xdo.type != XFRM_DEV_OFFLOAD_PACKET) {
		netdev_info(netdev, "Unsupported xfrm offload type\n");
		return -EINVAL;
		}

		return 0;
		}

		static void
		mlx5e_ipsec_build_accel_pol_attrs(struct mlx5e_ipsec_pol_entry *pol_entry,
		struct mlx5_accel_pol_xfrm_attrs *attrs)
		{
		struct xfrm_policy *x = pol_entry->x;
		struct xfrm_selector *sel;

		sel = &x->selector;
		memset(attrs, 0, sizeof(*attrs));

		memcpy(&attrs->saddr, sel->saddr.a6, sizeof(attrs->saddr));
		memcpy(&attrs->daddr, sel->daddr.a6, sizeof(attrs->daddr));
		attrs->family = sel->family;
		attrs->dir = x->xdo.dir;
		attrs->action = x->action;
		attrs->type = XFRM_DEV_OFFLOAD_PACKET;
		attrs->reqid = x->xfrm_vec[0].reqid;
		}

		static int mlx5e_xfrm_add_policy(struct xfrm_policy *x)
		{
		struct net_device *netdev = x->xdo.real_dev;
		struct mlx5e_ipsec_pol_entry *pol_entry;
		struct mlx5e_priv *priv;
		int err;

		priv = netdev_priv(netdev);
		if (!priv->ipsec)
		return -EOPNOTSUPP;

		err = mlx5e_xfrm_validate_policy(x);
		if (err)
		return err;

		pol_entry = kzalloc(sizeof(*pol_entry), GFP_KERNEL);
		if (!pol_entry)
		return -ENOMEM;

		pol_entry->x = x;
		pol_entry->ipsec = priv->ipsec;

		mlx5e_ipsec_build_accel_pol_attrs(pol_entry, &pol_entry->attrs);
		err = mlx5e_accel_ipsec_fs_add_pol(pol_entry);
		if (err)
		goto err_fs;

		x->xdo.offload_handle = (unsigned long)pol_entry;
		return 0;

		err_fs:
		kfree(pol_entry);
		return err;
		}

		static void mlx5e_xfrm_free_policy(struct xfrm_policy *x)
		{
		struct mlx5e_ipsec_pol_entry *pol_entry = to_ipsec_pol_entry(x);

		mlx5e_accel_ipsec_fs_del_pol(pol_entry);
		kfree(pol_entry);
		}

		static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = {
		.xdo_dev_state_add = mlx5e_xfrm_add_state,
		.xdo_dev_state_delete = mlx5e_xfrm_del_state,
		@@ -454,6 +601,18 @@ static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = {
		.xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state,
		};

		static const struct xfrmdev_ops mlx5e_ipsec_packet_xfrmdev_ops = {
		.xdo_dev_state_add = mlx5e_xfrm_add_state,
		.xdo_dev_state_delete = mlx5e_xfrm_del_state,
		.xdo_dev_state_free = mlx5e_xfrm_free_state,
		.xdo_dev_offload_ok = mlx5e_ipsec_offload_ok,
		.xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state,

		.xdo_dev_state_update_curlft = mlx5e_xfrm_update_curlft,
		.xdo_dev_policy_add = mlx5e_xfrm_add_policy,
		.xdo_dev_policy_free = mlx5e_xfrm_free_policy,
		};

		void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv)
		{
		struct mlx5_core_dev *mdev = priv->mdev;
		@@ -463,7 +622,12 @@ void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv)
		return;

		mlx5_core_info(mdev, "mlx5e: IPSec ESP acceleration enabled\n");

		if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)
		netdev->xfrmdev_ops = &mlx5e_ipsec_packet_xfrmdev_ops;
		else
		netdev->xfrmdev_ops = &mlx5e_ipsec_xfrmdev_ops;

		netdev->features \|= NETIF_F_HW_ESP;
		netdev->hw_enc_features \|= NETIF_F_HW_ESP;

drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.h

+75 −9

Original line number	Diff line number	Diff line
		@@ -34,8 +34,6 @@
		#ifndef __MLX5E_IPSEC_H__
		#define __MLX5E_IPSEC_H__

		#ifdef CONFIG_MLX5_EN_IPSEC

		#include <linux/mlx5/device.h>
		#include <net/xfrm.h>
		#include <linux/idr.h>
		@@ -76,6 +74,10 @@ struct mlx5_accel_esp_xfrm_attrs {
		u8 type : 2;
		u8 family;
		u32 replay_window;
		u32 authsize;
		u32 reqid;
		u64 hard_packet_limit;
		u64 soft_packet_limit;
		};

		enum mlx5_ipsec_cap {
		@@ -86,6 +88,17 @@ enum mlx5_ipsec_cap {

		struct mlx5e_priv;

		struct mlx5e_ipsec_hw_stats {
		u64 ipsec_rx_pkts;
		u64 ipsec_rx_bytes;
		u64 ipsec_rx_drop_pkts;
		u64 ipsec_rx_drop_bytes;
		u64 ipsec_tx_pkts;
		u64 ipsec_tx_bytes;
		u64 ipsec_tx_drop_pkts;
		u64 ipsec_tx_drop_bytes;
		};

		struct mlx5e_ipsec_sw_stats {
		atomic64_t ipsec_rx_drop_sp_alloc;
		atomic64_t ipsec_rx_drop_sadb_miss;
		@@ -99,23 +112,35 @@ struct mlx5e_ipsec_sw_stats {
		struct mlx5e_ipsec_rx;
		struct mlx5e_ipsec_tx;

		struct mlx5e_ipsec_work {
		struct work_struct work;
		struct mlx5e_ipsec *ipsec;
		u32 id;
		};

		struct mlx5e_ipsec_aso {
		u8 ctx[MLX5_ST_SZ_BYTES(ipsec_aso)];
		dma_addr_t dma_addr;
		struct mlx5_aso *aso;
		/* IPsec ASO caches data on every query call,
		* so in nested calls, we can use this boolean to save
		* recursive calls to mlx5e_ipsec_aso_query()
		*/
		u8 use_cache : 1;
		};

		struct mlx5e_ipsec {
		struct mlx5_core_dev *mdev;
		DECLARE_HASHTABLE(sadb_rx, MLX5E_IPSEC_SADB_RX_BITS);
		spinlock_t sadb_rx_lock; /* Protects sadb_rx */
		struct xarray sadb;
		struct mlx5e_ipsec_sw_stats sw_stats;
		struct mlx5e_ipsec_hw_stats hw_stats;
		struct workqueue_struct *wq;
		struct mlx5e_flow_steering *fs;
		struct mlx5e_ipsec_rx *rx_ipv4;
		struct mlx5e_ipsec_rx *rx_ipv6;
		struct mlx5e_ipsec_tx *tx;
		struct mlx5e_ipsec_aso *aso;
		struct notifier_block nb;
		};

		struct mlx5e_ipsec_esn_state {
		@@ -127,6 +152,7 @@ struct mlx5e_ipsec_esn_state {
		struct mlx5e_ipsec_rule {
		struct mlx5_flow_handle *rule;
		struct mlx5_modify_hdr *modify_hdr;
		struct mlx5_pkt_reformat *pkt_reformat;
		};

		struct mlx5e_ipsec_modify_state_work {
		@@ -135,9 +161,7 @@ struct mlx5e_ipsec_modify_state_work {
		};

		struct mlx5e_ipsec_sa_entry {
		struct hlist_node hlist; /* Item in SADB_RX hashtable */
		struct mlx5e_ipsec_esn_state esn_state;
		unsigned int handle; /* Handle in SADB_RX */
		struct xfrm_state *x;
		struct mlx5e_ipsec *ipsec;
		struct mlx5_accel_esp_xfrm_attrs attrs;
		@@ -149,17 +173,43 @@ struct mlx5e_ipsec_sa_entry {
		struct mlx5e_ipsec_modify_state_work modify_work;
		};

		struct mlx5_accel_pol_xfrm_attrs {
		union {
		__be32 a4;
		__be32 a6[4];
		} saddr;

		union {
		__be32 a4;
		__be32 a6[4];
		} daddr;

		u8 family;
		u8 action;
		u8 type : 2;
		u8 dir : 2;
		u32 reqid;
		};

		struct mlx5e_ipsec_pol_entry {
		struct xfrm_policy *x;
		struct mlx5e_ipsec *ipsec;
		struct mlx5e_ipsec_rule ipsec_rule;
		struct mlx5_accel_pol_xfrm_attrs attrs;
		};

		#ifdef CONFIG_MLX5_EN_IPSEC

		void mlx5e_ipsec_init(struct mlx5e_priv *priv);
		void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv);
		void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv);

		struct xfrm_state mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec dev,
		unsigned int handle);

		void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec);
		int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec);
		int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry);
		void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry);
		int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry);
		void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry);

		int mlx5_ipsec_create_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry);
		void mlx5_ipsec_free_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry);
		@@ -172,11 +222,27 @@ void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry,
		int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec);
		void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec);

		int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry,
		struct mlx5_wqe_aso_ctrl_seg *data);
		void mlx5e_ipsec_aso_update_curlft(struct mlx5e_ipsec_sa_entry *sa_entry,
		u64 *packets);

		void mlx5e_accel_ipsec_fs_read_stats(struct mlx5e_priv *priv,
		void *ipsec_stats);

		void mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,
		struct mlx5_accel_esp_xfrm_attrs *attrs);
		static inline struct mlx5_core_dev *
		mlx5e_ipsec_sa2dev(struct mlx5e_ipsec_sa_entry *sa_entry)
		{
		return sa_entry->ipsec->mdev;
		}

		static inline struct mlx5_core_dev *
		mlx5e_ipsec_pol2dev(struct mlx5e_ipsec_pol_entry *pol_entry)
		{
		return pol_entry->ipsec->mdev;
		}
		#else
		static inline void mlx5e_ipsec_init(struct mlx5e_priv *priv)
		{

drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c

+460 −22

File changed.

Preview size limit exceeded, changes collapsed.

drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c

+196 −0

Original line number	Diff line number	Diff line
		@@ -6,6 +6,10 @@
		#include "ipsec.h"
		#include "lib/mlx5.h"

		enum {
		MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET,
		};

		u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev)
		{
		u32 caps = 0;
		@@ -83,6 +87,20 @@ static void mlx5e_ipsec_packet_setup(void *obj, u32 pdn,
		MLX5_SET(ipsec_obj, obj, aso_return_reg, MLX5_IPSEC_ASO_REG_C_4_5);
		if (attrs->dir == XFRM_DEV_OFFLOAD_OUT)
		MLX5_SET(ipsec_aso, aso_ctx, mode, MLX5_IPSEC_ASO_INC_SN);

		if (attrs->hard_packet_limit != XFRM_INF) {
		MLX5_SET(ipsec_aso, aso_ctx, remove_flow_pkt_cnt,
		lower_32_bits(attrs->hard_packet_limit));
		MLX5_SET(ipsec_aso, aso_ctx, hard_lft_arm, 1);
		MLX5_SET(ipsec_aso, aso_ctx, remove_flow_enable, 1);
		}

		if (attrs->soft_packet_limit != XFRM_INF) {
		MLX5_SET(ipsec_aso, aso_ctx, remove_flow_soft_lft,
		lower_32_bits(attrs->soft_packet_limit));

		MLX5_SET(ipsec_aso, aso_ctx, soft_lft_arm, 1);
		}
		}

		static int mlx5_create_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry)
		@@ -246,6 +264,113 @@ void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry,
		memcpy(&sa_entry->attrs, attrs, sizeof(sa_entry->attrs));
		}

		static void
		mlx5e_ipsec_aso_update_esn(struct mlx5e_ipsec_sa_entry *sa_entry,
		const struct mlx5_accel_esp_xfrm_attrs *attrs)
		{
		struct mlx5_wqe_aso_ctrl_seg data = {};

		data.data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BITWISE_64BIT << 6;
		data.condition_1_0_operand = MLX5_ASO_ALWAYS_TRUE \| MLX5_ASO_ALWAYS_TRUE
		<< 4;
		data.data_offset_condition_operand = MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET;
		data.bitwise_data = cpu_to_be64(BIT_ULL(54));
		data.data_mask = data.bitwise_data;

		mlx5e_ipsec_aso_query(sa_entry, &data);
		}

		static void mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry,
		u32 mode_param)
		{
		struct mlx5_accel_esp_xfrm_attrs attrs = {};

		if (mode_param < MLX5E_IPSEC_ESN_SCOPE_MID) {
		sa_entry->esn_state.esn++;
		sa_entry->esn_state.overlap = 0;
		} else {
		sa_entry->esn_state.overlap = 1;
		}

		mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &attrs);
		mlx5_accel_esp_modify_xfrm(sa_entry, &attrs);
		mlx5e_ipsec_aso_update_esn(sa_entry, &attrs);
		}

		static void mlx5e_ipsec_handle_event(struct work_struct *_work)
		{
		struct mlx5e_ipsec_work *work =
		container_of(_work, struct mlx5e_ipsec_work, work);
		struct mlx5_accel_esp_xfrm_attrs *attrs;
		struct mlx5e_ipsec_sa_entry *sa_entry;
		struct mlx5e_ipsec_aso *aso;
		struct mlx5e_ipsec *ipsec;
		int ret;

		sa_entry = xa_load(&work->ipsec->sadb, work->id);
		if (!sa_entry)
		goto out;

		ipsec = sa_entry->ipsec;
		aso = ipsec->aso;
		attrs = &sa_entry->attrs;

		spin_lock(&sa_entry->x->lock);
		ret = mlx5e_ipsec_aso_query(sa_entry, NULL);
		if (ret)
		goto unlock;

		aso->use_cache = true;
		if (attrs->esn_trigger &&
		!MLX5_GET(ipsec_aso, aso->ctx, esn_event_arm)) {
		u32 mode_param = MLX5_GET(ipsec_aso, aso->ctx, mode_parameter);

		mlx5e_ipsec_update_esn_state(sa_entry, mode_param);
		}

		if (attrs->soft_packet_limit != XFRM_INF)
		if (!MLX5_GET(ipsec_aso, aso->ctx, soft_lft_arm) \|\|
		!MLX5_GET(ipsec_aso, aso->ctx, hard_lft_arm) \|\|
		!MLX5_GET(ipsec_aso, aso->ctx, remove_flow_enable))
		xfrm_state_check_expire(sa_entry->x);
		aso->use_cache = false;

		unlock:
		spin_unlock(&sa_entry->x->lock);
		out:
		kfree(work);
		}

		static int mlx5e_ipsec_event(struct notifier_block *nb, unsigned long event,
		void *data)
		{
		struct mlx5e_ipsec *ipsec = container_of(nb, struct mlx5e_ipsec, nb);
		struct mlx5_eqe_obj_change *object;
		struct mlx5e_ipsec_work *work;
		struct mlx5_eqe *eqe = data;
		u16 type;

		if (event != MLX5_EVENT_TYPE_OBJECT_CHANGE)
		return NOTIFY_DONE;

		object = &eqe->data.obj_change;
		type = be16_to_cpu(object->obj_type);

		if (type != MLX5_GENERAL_OBJECT_TYPES_IPSEC)
		return NOTIFY_DONE;

		work = kmalloc(sizeof(*work), GFP_ATOMIC);
		if (!work)
		return NOTIFY_DONE;

		INIT_WORK(&work->work, mlx5e_ipsec_handle_event);
		work->ipsec = ipsec;
		work->id = be32_to_cpu(object->obj_id);

		queue_work(ipsec->wq, &work->work);
		return NOTIFY_OK;
		}

		int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec)
		{
		struct mlx5_core_dev *mdev = ipsec->mdev;
		@@ -273,6 +398,9 @@ int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec)
		goto err_aso_create;
		}

		ipsec->nb.notifier_call = mlx5e_ipsec_event;
		mlx5_notifier_register(mdev, &ipsec->nb);

		ipsec->aso = aso;
		return 0;

		@@ -293,8 +421,76 @@ void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec)
		aso = ipsec->aso;
		pdev = mlx5_core_dma_dev(mdev);

		mlx5_notifier_unregister(mdev, &ipsec->nb);
		mlx5_aso_destroy(aso->aso);
		dma_unmap_single(pdev, aso->dma_addr, sizeof(aso->ctx),
		DMA_BIDIRECTIONAL);
		kfree(aso);
		}

		static void mlx5e_ipsec_aso_copy(struct mlx5_wqe_aso_ctrl_seg *ctrl,
		struct mlx5_wqe_aso_ctrl_seg *data)
		{
		if (!data)
		return;

		ctrl->data_mask_mode = data->data_mask_mode;
		ctrl->condition_1_0_operand = data->condition_1_0_operand;
		ctrl->condition_1_0_offset = data->condition_1_0_offset;
		ctrl->data_offset_condition_operand = data->data_offset_condition_operand;
		ctrl->condition_0_data = data->condition_0_data;
		ctrl->condition_0_mask = data->condition_0_mask;
		ctrl->condition_1_data = data->condition_1_data;
		ctrl->condition_1_mask = data->condition_1_mask;
		ctrl->bitwise_data = data->bitwise_data;
		ctrl->data_mask = data->data_mask;
		}

		int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry,
		struct mlx5_wqe_aso_ctrl_seg *data)
		{
		struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
		struct mlx5e_ipsec_aso *aso = ipsec->aso;
		struct mlx5_core_dev *mdev = ipsec->mdev;
		struct mlx5_wqe_aso_ctrl_seg *ctrl;
		struct mlx5e_hw_objs *res;
		struct mlx5_aso_wqe *wqe;
		u8 ds_cnt;

		lockdep_assert_held(&sa_entry->x->lock);
		if (aso->use_cache)
		return 0;

		res = &mdev->mlx5e_res.hw_objs;

		memset(aso->ctx, 0, sizeof(aso->ctx));
		wqe = mlx5_aso_get_wqe(aso->aso);
		ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
		mlx5_aso_build_wqe(aso->aso, ds_cnt, wqe, sa_entry->ipsec_obj_id,
		MLX5_ACCESS_ASO_OPC_MOD_IPSEC);

		ctrl = &wqe->aso_ctrl;
		ctrl->va_l =
		cpu_to_be32(lower_32_bits(aso->dma_addr) \| ASO_CTRL_READ_EN);
		ctrl->va_h = cpu_to_be32(upper_32_bits(aso->dma_addr));
		ctrl->l_key = cpu_to_be32(res->mkey);
		mlx5e_ipsec_aso_copy(ctrl, data);

		mlx5_aso_post_wqe(aso->aso, false, &wqe->ctrl);
		return mlx5_aso_poll_cq(aso->aso, false);
		}

		void mlx5e_ipsec_aso_update_curlft(struct mlx5e_ipsec_sa_entry *sa_entry,
		u64 *packets)
		{
		struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
		struct mlx5e_ipsec_aso *aso = ipsec->aso;
		u64 hard_cnt;

		hard_cnt = MLX5_GET(ipsec_aso, aso->ctx, remove_flow_pkt_cnt);
		/* HW decresases the limit till it reaches zero to fire an avent.
		* We need to fix the calculations, so the returned count is a total
		* number of passed packets and not how much left.
		*/
		*packets = sa_entry->attrs.hard_packet_limit - hard_cnt;
		}