Commit e8822565 authored by David S. Miller's avatar David S. Miller
Browse files

Merge branch 'sfc-TC-offload-counters' 



Edward Cree says:

====================
sfc: TC offload counters

EF100 hardware supports attaching counters to action-sets in the MAE.
Use these counters to implement stats for TC flower offload.

The counters are delivered to the host over a special hardware RX queue
 which should only ever receive counter update messages, not 'real'
 network packets.
====================

Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parents 7d63b21d 50f8f2f7

drivers/net/ethernet/sfc/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -9,7 +9,7 @@ sfc-y += efx.o efx_common.o efx_channels.o nic.o \ 
			   ef100_ethtool.o ef100_rx.o ef100_tx.o

sfc-$(CONFIG_SFC_MTD)	+= mtd.o

sfc-$(CONFIG_SFC_SRIOV)	+= sriov.o ef10_sriov.o ef100_sriov.o ef100_rep.o \

                           mae.o tc.o tc_bindings.o

                           mae.o tc.o tc_bindings.o tc_counters.o



obj-$(CONFIG_SFC)	+= sfc.o

drivers/net/ethernet/sfc/ef100_rx.c

+19 −4
Original line number Diff line number Diff line
@@ -21,7 +21,7 @@ 
/* Get the value of a field in the RX prefix */

#define PREFIX_OFFSET_W(_f)	(ESF_GZ_RX_PREFIX_ ## _f ## _LBN / 32)

#define PREFIX_OFFSET_B(_f)	(ESF_GZ_RX_PREFIX_ ## _f ## _LBN % 32)

#define PREFIX_WIDTH_MASK(_f)	((1UL << ESF_GZ_RX_PREFIX_ ## _f ## _WIDTH) - 1)

#define PREFIX_WIDTH_MASK(_f)	((1ULL << ESF_GZ_RX_PREFIX_ ## _f ## _WIDTH) - 1)

#define PREFIX_WORD(_p, _f)	le32_to_cpu((__force __le32)(_p)[PREFIX_OFFSET_W(_f)])

#define PREFIX_FIELD(_p, _f)	((PREFIX_WORD(_p, _f) >> PREFIX_OFFSET_B(_f)) & \

				 PREFIX_WIDTH_MASK(_f))
@@ -67,6 +67,13 @@ void __ef100_rx_packet(struct efx_channel *channel) 


	prefix = (u32 *)(eh - ESE_GZ_RX_PKT_PREFIX_LEN);



	if (channel->type->receive_raw) {

		u32 mark = PREFIX_FIELD(prefix, USER_MARK);



		if (channel->type->receive_raw(rx_queue, mark))

			return; /* packet was consumed */

	}



	if (ef100_has_fcs_error(channel, prefix) &&

	    unlikely(!(efx->net_dev->features & NETIF_F_RXALL)))

		goto out;
@@ -183,24 +190,32 @@ void efx_ef100_ev_rx(struct efx_channel *channel, const efx_qword_t *p_event) 


void ef100_rx_write(struct efx_rx_queue *rx_queue)

{

	unsigned int notified_count = rx_queue->notified_count;

	struct efx_rx_buffer *rx_buf;

	unsigned int idx;

	efx_qword_t *rxd;

	efx_dword_t rxdb;



	while (rx_queue->notified_count != rx_queue->added_count) {

		idx = rx_queue->notified_count & rx_queue->ptr_mask;

	while (notified_count != rx_queue->added_count) {

		idx = notified_count & rx_queue->ptr_mask;

		rx_buf = efx_rx_buffer(rx_queue, idx);

		rxd = efx_rx_desc(rx_queue, idx);



		EFX_POPULATE_QWORD_1(*rxd, ESF_GZ_RX_BUF_ADDR, rx_buf->dma_addr);



		++rx_queue->notified_count;

		++notified_count;

	}

	if (notified_count == rx_queue->notified_count)

		return;



	wmb();

	EFX_POPULATE_DWORD_1(rxdb, ERF_GZ_RX_RING_PIDX,

			     rx_queue->added_count & rx_queue->ptr_mask);

	efx_writed_page(rx_queue->efx, &rxdb,

			ER_GZ_RX_RING_DOORBELL, efx_rx_queue_index(rx_queue));

	if (rx_queue->grant_credits)

		wmb();

	rx_queue->notified_count = notified_count;

	if (rx_queue->grant_credits)

		schedule_work(&rx_queue->grant_work);

}

drivers/net/ethernet/sfc/efx_channels.c

+8 −1
Original line number Diff line number Diff line
@@ -1119,6 +1119,8 @@ void efx_start_channels(struct efx_nic *efx) 
	struct efx_channel *channel;



	efx_for_each_channel_rev(channel, efx) {

		if (channel->type->start)

			channel->type->start(channel);

		efx_for_each_channel_tx_queue(tx_queue, channel) {

			efx_init_tx_queue(tx_queue);

			atomic_inc(&efx->active_queues);
@@ -1143,8 +1145,13 @@ void efx_stop_channels(struct efx_nic *efx) 
	struct efx_channel *channel;

	int rc = 0;



	/* Stop RX refill */

	/* Stop special channels and RX refill.

	 * The channel's stop has to be called first, since it might wait

	 * for a sentinel RX to indicate the channel has fully drained.

	 */

	efx_for_each_channel(channel, efx) {

		if (channel->type->stop)

			channel->type->stop(channel);

		efx_for_each_channel_rx_queue(rx_queue, channel)

			rx_queue->refill_enabled = false;

	}

drivers/net/ethernet/sfc/mae.c

+168 −2
Original line number Diff line number Diff line
@@ -112,6 +112,117 @@ int efx_mae_lookup_mport(struct efx_nic *efx, u32 selector, u32 *id) 
	return 0;

}



int efx_mae_start_counters(struct efx_nic *efx, struct efx_rx_queue *rx_queue)

{

	MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTERS_STREAM_START_V2_IN_LEN);

	MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_COUNTERS_STREAM_START_OUT_LEN);

	u32 out_flags;

	size_t outlen;

	int rc;



	MCDI_SET_WORD(inbuf, MAE_COUNTERS_STREAM_START_V2_IN_QID,

		      efx_rx_queue_index(rx_queue));

	MCDI_SET_WORD(inbuf, MAE_COUNTERS_STREAM_START_V2_IN_PACKET_SIZE,

		      efx->net_dev->mtu);

	MCDI_SET_DWORD(inbuf, MAE_COUNTERS_STREAM_START_V2_IN_COUNTER_TYPES_MASK,

		       BIT(MAE_COUNTER_TYPE_AR) | BIT(MAE_COUNTER_TYPE_CT) |

		       BIT(MAE_COUNTER_TYPE_OR));

	rc = efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTERS_STREAM_START,

			  inbuf, sizeof(inbuf), outbuf, sizeof(outbuf), &outlen);

	if (rc)

		return rc;

	if (outlen < sizeof(outbuf))

		return -EIO;

	out_flags = MCDI_DWORD(outbuf, MAE_COUNTERS_STREAM_START_OUT_FLAGS);

	if (out_flags & BIT(MC_CMD_MAE_COUNTERS_STREAM_START_OUT_USES_CREDITS_OFST)) {

		netif_dbg(efx, drv, efx->net_dev,

			  "MAE counter stream uses credits\n");

		rx_queue->grant_credits = true;

		out_flags &= ~BIT(MC_CMD_MAE_COUNTERS_STREAM_START_OUT_USES_CREDITS_OFST);

	}

	if (out_flags) {

		netif_err(efx, drv, efx->net_dev,

			  "MAE counter stream start: unrecognised flags %x\n",

			  out_flags);

		goto out_stop;

	}

	return 0;

out_stop:

	efx_mae_stop_counters(efx, rx_queue);

	return -EOPNOTSUPP;

}



static bool efx_mae_counters_flushed(u32 *flush_gen, u32 *seen_gen)

{

	int i;



	for (i = 0; i < EFX_TC_COUNTER_TYPE_MAX; i++)

		if ((s32)(flush_gen[i] - seen_gen[i]) > 0)

			return false;

	return true;

}



int efx_mae_stop_counters(struct efx_nic *efx, struct efx_rx_queue *rx_queue)

{

	MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_COUNTERS_STREAM_STOP_V2_OUT_LENMAX);

	MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTERS_STREAM_STOP_IN_LEN);

	size_t outlen;

	int rc, i;



	MCDI_SET_WORD(inbuf, MAE_COUNTERS_STREAM_STOP_IN_QID,

		      efx_rx_queue_index(rx_queue));

	rc = efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTERS_STREAM_STOP,

			  inbuf, sizeof(inbuf), outbuf, sizeof(outbuf), &outlen);



	if (rc)

		return rc;



	netif_dbg(efx, drv, efx->net_dev, "Draining counters:\n");

	/* Only process received generation counts */

	for (i = 0; (i < (outlen / 4)) && (i < EFX_TC_COUNTER_TYPE_MAX); i++) {

		efx->tc->flush_gen[i] = MCDI_ARRAY_DWORD(outbuf,

							 MAE_COUNTERS_STREAM_STOP_V2_OUT_GENERATION_COUNT,

							 i);

		netif_dbg(efx, drv, efx->net_dev,

			  "\ttype %u, awaiting gen %u\n", i,

			  efx->tc->flush_gen[i]);

	}



	efx->tc->flush_counters = true;



	/* Drain can take up to 2 seconds owing to FWRIVERHD-2884; whatever

	 * timeout we use, that delay is added to unload on nonresponsive

	 * hardware, so 2500ms seems like a reasonable compromise.

	 */

	if (!wait_event_timeout(efx->tc->flush_wq,

				efx_mae_counters_flushed(efx->tc->flush_gen,

							 efx->tc->seen_gen),

				msecs_to_jiffies(2500)))

		netif_warn(efx, drv, efx->net_dev,

			   "Failed to drain counters RXQ, FW may be unhappy\n");



	efx->tc->flush_counters = false;



	return rc;

}



void efx_mae_counters_grant_credits(struct work_struct *work)

{

	MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS_IN_LEN);

	struct efx_rx_queue *rx_queue = container_of(work, struct efx_rx_queue,

						     grant_work);

	struct efx_nic *efx = rx_queue->efx;

	unsigned int credits;



	BUILD_BUG_ON(MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS_OUT_LEN);

	credits = READ_ONCE(rx_queue->notified_count) - rx_queue->granted_count;

	MCDI_SET_DWORD(inbuf, MAE_COUNTERS_STREAM_GIVE_CREDITS_IN_NUM_CREDITS,

		       credits);

	if (!efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS,

			  inbuf, sizeof(inbuf), NULL, 0, NULL))

		rx_queue->granted_count += credits;

}



static int efx_mae_get_basic_caps(struct efx_nic *efx, struct mae_caps *caps)

{

	MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_GET_CAPS_OUT_LEN);
@@ -323,6 +434,57 @@ int efx_mae_match_check_caps(struct efx_nic *efx, 
#undef CHECK_BIT

#undef CHECK



int efx_mae_allocate_counter(struct efx_nic *efx, struct efx_tc_counter *cnt)

{

	MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_COUNTER_ALLOC_OUT_LEN(1));

	MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTER_ALLOC_V2_IN_LEN);

	size_t outlen;

	int rc;



	if (!cnt)

		return -EINVAL;



	MCDI_SET_DWORD(inbuf, MAE_COUNTER_ALLOC_V2_IN_REQUESTED_COUNT, 1);

	MCDI_SET_DWORD(inbuf, MAE_COUNTER_ALLOC_V2_IN_COUNTER_TYPE, cnt->type);

	rc = efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTER_ALLOC, inbuf, sizeof(inbuf),

			  outbuf, sizeof(outbuf), &outlen);

	if (rc)

		return rc;

	/* pcol says this can't happen, since count is 1 */

	if (outlen < sizeof(outbuf))

		return -EIO;

	cnt->fw_id = MCDI_DWORD(outbuf, MAE_COUNTER_ALLOC_OUT_COUNTER_ID);

	cnt->gen = MCDI_DWORD(outbuf, MAE_COUNTER_ALLOC_OUT_GENERATION_COUNT);

	return 0;

}



int efx_mae_free_counter(struct efx_nic *efx, struct efx_tc_counter *cnt)

{

	MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_COUNTER_FREE_OUT_LEN(1));

	MCDI_DECLARE_BUF(inbuf, MC_CMD_MAE_COUNTER_FREE_V2_IN_LEN);

	size_t outlen;

	int rc;



	MCDI_SET_DWORD(inbuf, MAE_COUNTER_FREE_V2_IN_COUNTER_ID_COUNT, 1);

	MCDI_SET_DWORD(inbuf, MAE_COUNTER_FREE_V2_IN_FREE_COUNTER_ID, cnt->fw_id);

	MCDI_SET_DWORD(inbuf, MAE_COUNTER_FREE_V2_IN_COUNTER_TYPE, cnt->type);

	rc = efx_mcdi_rpc(efx, MC_CMD_MAE_COUNTER_FREE, inbuf, sizeof(inbuf),

			  outbuf, sizeof(outbuf), &outlen);

	if (rc)

		return rc;

	/* pcol says this can't happen, since count is 1 */

	if (outlen < sizeof(outbuf))

		return -EIO;

	/* FW freed a different ID than we asked for, should also never happen.

	 * Warn because it means we've now got a different idea to the FW of

	 * what counters exist, which could cause mayhem later.

	 */

	if (WARN_ON(MCDI_DWORD(outbuf, MAE_COUNTER_FREE_OUT_FREED_COUNTER_ID) !=

		    cnt->fw_id))

		return -EIO;

	return 0;

}



static bool efx_mae_asl_id(u32 id)

{

	return !!(id & BIT(31));
@@ -339,6 +501,10 @@ int efx_mae_alloc_action_set(struct efx_nic *efx, struct efx_tc_action_set *act) 
		       MC_CMD_MAE_MAC_ADDR_ALLOC_OUT_MAC_ID_NULL);

	MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_DST_MAC_ID,

		       MC_CMD_MAE_MAC_ADDR_ALLOC_OUT_MAC_ID_NULL);

	if (act->count && !WARN_ON(!act->count->cnt))

		MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_COUNTER_ID,

			       act->count->cnt->fw_id);

	else

		MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_COUNTER_ID,

			       MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_NULL);

	MCDI_SET_DWORD(inbuf, MAE_ACTION_SET_ALLOC_IN_COUNTER_LIST_ID,

drivers/net/ethernet/sfc/mae.h

+7 −0
Original line number Diff line number Diff line
@@ -27,6 +27,10 @@ void efx_mae_mport_mport(struct efx_nic *efx, u32 mport_id, u32 *out); 


int efx_mae_lookup_mport(struct efx_nic *efx, u32 selector, u32 *id);



int efx_mae_start_counters(struct efx_nic *efx, struct efx_rx_queue *rx_queue);

int efx_mae_stop_counters(struct efx_nic *efx, struct efx_rx_queue *rx_queue);

void efx_mae_counters_grant_credits(struct work_struct *work);



#define MAE_NUM_FIELDS	(MAE_FIELD_ENC_VNET_ID + 1)



struct mae_caps {
@@ -41,6 +45,9 @@ int efx_mae_match_check_caps(struct efx_nic *efx, 
			     const struct efx_tc_match_fields *mask,

			     struct netlink_ext_ack *extack);



int efx_mae_allocate_counter(struct efx_nic *efx, struct efx_tc_counter *cnt);

int efx_mae_free_counter(struct efx_nic *efx, struct efx_tc_counter *cnt);



int efx_mae_alloc_action_set(struct efx_nic *efx, struct efx_tc_action_set *act);

int efx_mae_free_action_set(struct efx_nic *efx, u32 fw_id);