Commit 05b953a5 authored by Jakub Kicinski's avatar Jakub Kicinski
Browse files

Merge tag 'mlx5-updates-2023-02-15' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux 

Saeed Mahameed says:

====================
mlx5-updates-2023-02-15

1) From Gal Tariq and Parav, Few cleanups for mlx5 driver.

2) From Vlad: Allow offloading of ct 'new' match based on [1]

[1] https://lore.kernel.org/netdev/20230201163100.1001180-1-vladbu@nvidia.com/

* tag 'mlx5-updates-2023-02-15' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux:
  net/mlx5e: RX, Remove doubtful unlikely call
  net/mlx5e: Fix outdated TLS comment
  net/mlx5e: Remove unused function mlx5e_sq_xmit_simple
  net/mlx5e: Allow offloading of ct 'new' match
  net/mlx5e: Implement CT entry update
  net/mlx5: Simplify eq list traversal
  net/mlx5e: Remove redundant page argument in mlx5e_xdp_handle()
  net/mlx5e: Remove redundant page argument in mlx5e_xmit_xdp_buff()
  net/mlx5e: Switch to using napi_build_skb()
====================

Link: https://lore.kernel.org/r/20230218090513.284718-1-saeed@kernel.org


Signed-off-by: default avatarJakub Kicinski <kuba@kernel.org>
parents 981f4045 993fd9bd

drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c

+126 −13
Original line number Diff line number Diff line
@@ -35,6 +35,7 @@ 
#define MLX5_CT_STATE_REPLY_BIT BIT(4)

#define MLX5_CT_STATE_RELATED_BIT BIT(5)

#define MLX5_CT_STATE_INVALID_BIT BIT(6)

#define MLX5_CT_STATE_NEW_BIT BIT(7)



#define MLX5_CT_LABELS_BITS MLX5_REG_MAPPING_MBITS(LABELS_TO_REG)

#define MLX5_CT_LABELS_MASK MLX5_REG_MAPPING_MASK(LABELS_TO_REG)
@@ -721,12 +722,14 @@ mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv *ct_priv, 
	DECLARE_MOD_HDR_ACTS_ACTIONS(actions_arr, MLX5_CT_MIN_MOD_ACTS);

	DECLARE_MOD_HDR_ACTS(mod_acts, actions_arr);

	struct flow_action_entry *meta;

	enum ip_conntrack_info ctinfo;

	u16 ct_state = 0;

	int err;



	meta = mlx5_tc_ct_get_ct_metadata_action(flow_rule);

	if (!meta)

		return -EOPNOTSUPP;

	ctinfo = meta->ct_metadata.cookie & NFCT_INFOMASK;



	err = mlx5_get_label_mapping(ct_priv, meta->ct_metadata.labels,

				     &attr->ct_attr.ct_labels_id);
@@ -742,7 +745,8 @@ mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv *ct_priv, 
		ct_state |= MLX5_CT_STATE_NAT_BIT;

	}



	ct_state |= MLX5_CT_STATE_ESTABLISHED_BIT | MLX5_CT_STATE_TRK_BIT;

	ct_state |= MLX5_CT_STATE_TRK_BIT;

	ct_state |= ctinfo == IP_CT_NEW ? MLX5_CT_STATE_NEW_BIT : MLX5_CT_STATE_ESTABLISHED_BIT;

	ct_state |= meta->ct_metadata.orig_dir ? 0 : MLX5_CT_STATE_REPLY_BIT;

	err = mlx5_tc_ct_entry_set_registers(ct_priv, &mod_acts,

					     ct_state,
@@ -871,6 +875,68 @@ mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv *ct_priv, 
	return err;

}



static int

mlx5_tc_ct_entry_replace_rule(struct mlx5_tc_ct_priv *ct_priv,

			      struct flow_rule *flow_rule,

			      struct mlx5_ct_entry *entry,

			      bool nat, u8 zone_restore_id)

{

	struct mlx5_ct_zone_rule *zone_rule = &entry->zone_rules[nat];

	struct mlx5_flow_attr *attr = zone_rule->attr, *old_attr;

	struct mlx5e_mod_hdr_handle *mh;

	struct mlx5_ct_fs_rule *rule;

	struct mlx5_flow_spec *spec;

	int err;



	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);

	if (!spec)

		return -ENOMEM;



	old_attr = mlx5_alloc_flow_attr(ct_priv->ns_type);

	if (!old_attr) {

		err = -ENOMEM;

		goto err_attr;

	}

	*old_attr = *attr;



	err = mlx5_tc_ct_entry_create_mod_hdr(ct_priv, attr, flow_rule, &mh, zone_restore_id,

					      nat, mlx5_tc_ct_entry_has_nat(entry));

	if (err) {

		ct_dbg("Failed to create ct entry mod hdr");

		goto err_mod_hdr;

	}



	mlx5_tc_ct_set_tuple_match(ct_priv, spec, flow_rule);

	mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG, entry->tuple.zone, MLX5_CT_ZONE_MASK);



	rule = ct_priv->fs_ops->ct_rule_add(ct_priv->fs, spec, attr, flow_rule);

	if (IS_ERR(rule)) {

		err = PTR_ERR(rule);

		ct_dbg("Failed to add replacement ct entry rule, nat: %d", nat);

		goto err_rule;

	}



	ct_priv->fs_ops->ct_rule_del(ct_priv->fs, zone_rule->rule);

	zone_rule->rule = rule;

	mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, old_attr, zone_rule->mh);

	zone_rule->mh = mh;



	kfree(old_attr);

	kvfree(spec);

	ct_dbg("Replaced ct entry rule in zone %d", entry->tuple.zone);



	return 0;



err_rule:

	mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, zone_rule->attr, mh);

	mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id);

err_mod_hdr:

	kfree(old_attr);

err_attr:

	kvfree(spec);

	return err;

}



static bool

mlx5_tc_ct_entry_valid(struct mlx5_ct_entry *entry)

{
@@ -1065,6 +1131,52 @@ mlx5_tc_ct_entry_add_rules(struct mlx5_tc_ct_priv *ct_priv, 
	return err;

}



static int

mlx5_tc_ct_entry_replace_rules(struct mlx5_tc_ct_priv *ct_priv,

			       struct flow_rule *flow_rule,

			       struct mlx5_ct_entry *entry,

			       u8 zone_restore_id)

{

	int err;



	err = mlx5_tc_ct_entry_replace_rule(ct_priv, flow_rule, entry, false,

					    zone_restore_id);

	if (err)

		return err;



	err = mlx5_tc_ct_entry_replace_rule(ct_priv, flow_rule, entry, true,

					    zone_restore_id);

	if (err)

		mlx5_tc_ct_entry_del_rule(ct_priv, entry, false);

	return err;

}



static int

mlx5_tc_ct_block_flow_offload_replace(struct mlx5_ct_ft *ft, struct flow_rule *flow_rule,

				      struct mlx5_ct_entry *entry, unsigned long cookie)

{

	struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv;

	int err;



	err = mlx5_tc_ct_entry_replace_rules(ct_priv, flow_rule, entry, ft->zone_restore_id);

	if (!err)

		return 0;



	/* If failed to update the entry, then look it up again under ht_lock

	 * protection and properly delete it.

	 */

	spin_lock_bh(&ct_priv->ht_lock);

	entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params);

	if (entry) {

		rhashtable_remove_fast(&ft->ct_entries_ht, &entry->node, cts_ht_params);

		spin_unlock_bh(&ct_priv->ht_lock);

		mlx5_tc_ct_entry_put(entry);

	} else {

		spin_unlock_bh(&ct_priv->ht_lock);

	}

	return err;

}



static int

mlx5_tc_ct_block_flow_offload_add(struct mlx5_ct_ft *ft,

				  struct flow_cls_offload *flow)
@@ -1073,24 +1185,28 @@ mlx5_tc_ct_block_flow_offload_add(struct mlx5_ct_ft *ft, 
	struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv;

	struct flow_action_entry *meta_action;

	unsigned long cookie = flow->cookie;

	enum ip_conntrack_info ctinfo;

	struct mlx5_ct_entry *entry;

	int err;



	meta_action = mlx5_tc_ct_get_ct_metadata_action(flow_rule);

	if (!meta_action)

		return -EOPNOTSUPP;

	ctinfo = meta_action->ct_metadata.cookie & NFCT_INFOMASK;

	if (ctinfo == IP_CT_NEW)

		return -EOPNOTSUPP;



	spin_lock_bh(&ct_priv->ht_lock);

	entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params);

	if (entry && refcount_inc_not_zero(&entry->refcnt)) {

		if (entry->restore_cookie == meta_action->ct_metadata.cookie) {

			spin_unlock_bh(&ct_priv->ht_lock);

			mlx5_tc_ct_entry_put(entry);

			return -EEXIST;

		}

		entry->restore_cookie = meta_action->ct_metadata.cookie;

		spin_unlock_bh(&ct_priv->ht_lock);



		err = mlx5_tc_ct_block_flow_offload_replace(ft, flow_rule, entry, cookie);

		mlx5_tc_ct_entry_put(entry);

		return err;

	}

	spin_unlock_bh(&ct_priv->ht_lock);



	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
@@ -1327,7 +1443,7 @@ mlx5_tc_ct_match_add(struct mlx5_tc_ct_priv *priv, 
		     struct mlx5_ct_attr *ct_attr,

		     struct netlink_ext_ack *extack)

{

	bool trk, est, untrk, unest, new, rpl, unrpl, rel, unrel, inv, uninv;

	bool trk, est, untrk, unnew, unest, new, rpl, unrpl, rel, unrel, inv, uninv;

	struct flow_rule *rule = flow_cls_offload_flow_rule(f);

	struct flow_dissector_key_ct *mask, *key;

	u32 ctstate = 0, ctstate_mask = 0;
@@ -1373,15 +1489,18 @@ mlx5_tc_ct_match_add(struct mlx5_tc_ct_priv *priv, 
	rel = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_RELATED;

	inv = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_INVALID;

	untrk = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_TRACKED;

	unnew = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_NEW;

	unest = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED;

	unrpl = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_REPLY;

	unrel = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_RELATED;

	uninv = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_INVALID;



	ctstate |= trk ? MLX5_CT_STATE_TRK_BIT : 0;

	ctstate |= new ? MLX5_CT_STATE_NEW_BIT : 0;

	ctstate |= est ? MLX5_CT_STATE_ESTABLISHED_BIT : 0;

	ctstate |= rpl ? MLX5_CT_STATE_REPLY_BIT : 0;

	ctstate_mask |= (untrk || trk) ? MLX5_CT_STATE_TRK_BIT : 0;

	ctstate_mask |= (unnew || new) ? MLX5_CT_STATE_NEW_BIT : 0;

	ctstate_mask |= (unest || est) ? MLX5_CT_STATE_ESTABLISHED_BIT : 0;

	ctstate_mask |= (unrpl || rpl) ? MLX5_CT_STATE_REPLY_BIT : 0;

	ctstate_mask |= unrel ? MLX5_CT_STATE_RELATED_BIT : 0;
@@ -1399,12 +1518,6 @@ mlx5_tc_ct_match_add(struct mlx5_tc_ct_priv *priv, 
		return -EOPNOTSUPP;

	}



	if (new) {

		NL_SET_ERR_MSG_MOD(extack,

				   "matching on ct_state +new isn't supported");

		return -EOPNOTSUPP;

	}



	if (mask->ct_zone)

		mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG,

					    key->ct_zone, MLX5_CT_ZONE_MASK);

drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h

+0 −1
Original line number Diff line number Diff line
@@ -320,7 +320,6 @@ mlx5e_tx_dma_unmap(struct device *pdev, struct mlx5e_sq_dma *dma) 
	}

}



void mlx5e_sq_xmit_simple(struct mlx5e_txqsq *sq, struct sk_buff *skb, bool xmit_more);

void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq);



static inline bool mlx5e_tx_mpwqe_is_full(struct mlx5e_tx_mpwqe *session, u8 max_sq_mpw_wqebbs)

drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c

+5 −4
Original line number Diff line number Diff line
@@ -57,8 +57,9 @@ int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk) 


static inline bool

mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq,

		    struct page *page, struct xdp_buff *xdp)

		    struct xdp_buff *xdp)

{

	struct page *page = virt_to_page(xdp->data);

	struct skb_shared_info *sinfo = NULL;

	struct mlx5e_xmit_data xdptxd;

	struct mlx5e_xdp_info xdpi;
@@ -185,7 +186,7 @@ const struct xdp_metadata_ops mlx5e_xdp_metadata_ops = { 
};



/* returns true if packet was consumed by xdp */

bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct page *page,

bool mlx5e_xdp_handle(struct mlx5e_rq *rq,

		      struct bpf_prog *prog, struct mlx5e_xdp_buff *mxbuf)

{

	struct xdp_buff *xdp = &mxbuf->xdp;
@@ -197,7 +198,7 @@ bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct page *page, 
	case XDP_PASS:

		return false;

	case XDP_TX:

		if (unlikely(!mlx5e_xmit_xdp_buff(rq->xdpsq, rq, page, xdp)))

		if (unlikely(!mlx5e_xmit_xdp_buff(rq->xdpsq, rq, xdp)))

			goto xdp_abort;

		__set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags); /* non-atomic */

		return true;
@@ -209,7 +210,7 @@ bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct page *page, 
		__set_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags);

		__set_bit(MLX5E_RQ_FLAG_XDP_REDIRECT, rq->flags);

		if (xdp->rxq->mem.type != MEM_TYPE_XSK_BUFF_POOL)

			mlx5e_page_dma_unmap(rq, page);

			mlx5e_page_dma_unmap(rq, virt_to_page(xdp->data));

		rq->stats->xdp_redirect++;

		return true;

	default:

drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h

+1 −1
Original line number Diff line number Diff line
@@ -52,7 +52,7 @@ struct mlx5e_xdp_buff { 


struct mlx5e_xsk_param;

int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk);

bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct page *page,

bool mlx5e_xdp_handle(struct mlx5e_rq *rq,

		      struct bpf_prog *prog, struct mlx5e_xdp_buff *mlctx);

void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq);

bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq);

drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c

+2 −2
Original line number Diff line number Diff line
@@ -289,7 +289,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, 
	 */



	prog = rcu_dereference(rq->xdp_prog);

	if (likely(prog && mlx5e_xdp_handle(rq, NULL, prog, mxbuf))) {

	if (likely(prog && mlx5e_xdp_handle(rq, prog, mxbuf))) {

		if (likely(__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)))

			__set_bit(page_idx, wi->xdp_xmit_bitmap); /* non-atomic */

		return NULL; /* page/packet was consumed by XDP */
@@ -323,7 +323,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq, 
	net_prefetch(mxbuf->xdp.data);



	prog = rcu_dereference(rq->xdp_prog);

	if (likely(prog && mlx5e_xdp_handle(rq, NULL, prog, mxbuf)))

	if (likely(prog && mlx5e_xdp_handle(rq, prog, mxbuf)))

		return NULL; /* page/packet was consumed by XDP */



	/* XDP_PASS: copy the data from the UMEM to a new SKB. The frame reuse