Commit 583024cf authored by Paolo Abeni's avatar Paolo Abeni
Browse files

Merge branch 'netdevsim-support-for-l3-hw-stats' 

Petr Machata says:

====================
netdevsim: Support for L3 HW stats

"L3 stats" is a suite of interface statistics aimed at reflecting traffic
taking place in a HW device, on an object corresponding to some software
netdevice. Support for this stats suite has been added recently, in commit
ca0a53dc ("Merge branch 'net-hw-counters-for-soft-devices'").

In this patch set:

- Patch #1 adds support for L3 stats to netdevsim.

  Real devices can have various conditions for when an L3 counter is
  available. To simulate this, netdevsim maintains a list of devices
  suitable for HW stats collection. Only when l3_stats is enabled on both a
  netdevice itself, and in netdevsim, will netdevsim contribute values to
  L3 stats.

  This enablement and disablement is done via debugfs:

    # echo $ifindex > /sys/kernel/debug/netdevsim/$DEV/hwstats/l3/enable_ifindex
    # echo $ifindex > /sys/kernel/debug/netdevsim/$DEV/hwstats/l3/disable_ifindex

  Besides this, there is a third toggle to mark a device for future failure:

    # echo $ifindex > /sys/kernel/debug/netdevsim/$DEV/hwstats/l3/fail_next_enable

- This allows HW-independent testing of stats reporting and in-kernel APIs,
  as well as a test for enablement rollback, which is difficult to do
  otherwise. This netdevsim-specific selftest is added in patch #2.

- Patch #3 adds another driver-specific selftest, namely a test aimed at
  checking mlxsw-induced stats monitoring events.

====================

Link: https://lore.kernel.org/r/cover.1647265833.git.petrm@nvidia.com


Signed-off-by: default avatarPaolo Abeni <pabeni@redhat.com>
parents 231fdac3 ed2ae69c

drivers/net/netdevsim/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -3,7 +3,7 @@ 
obj-$(CONFIG_NETDEVSIM) += netdevsim.o



netdevsim-objs := \

	netdev.o dev.o ethtool.o fib.o bus.o health.o udp_tunnels.o

	netdev.o dev.o ethtool.o fib.o bus.o health.o hwstats.o udp_tunnels.o



ifeq ($(CONFIG_BPF_SYSCALL),y)

netdevsim-objs += \

drivers/net/netdevsim/dev.c

+15 −2
Original line number Diff line number Diff line
@@ -1498,10 +1498,14 @@ static int nsim_dev_reload_create(struct nsim_dev *nsim_dev, 
	if (err)

		goto err_health_exit;



	err = nsim_dev_port_add_all(nsim_dev, nsim_bus_dev->port_count);

	err = nsim_dev_hwstats_init(nsim_dev);

	if (err)

		goto err_psample_exit;



	err = nsim_dev_port_add_all(nsim_dev, nsim_bus_dev->port_count);

	if (err)

		goto err_hwstats_exit;



	nsim_dev->take_snapshot = debugfs_create_file("take_snapshot",

						      0200,

						      nsim_dev->ddir,
@@ -1509,6 +1513,8 @@ static int nsim_dev_reload_create(struct nsim_dev *nsim_dev, 
						&nsim_dev_take_snapshot_fops);

	return 0;



err_hwstats_exit:

	nsim_dev_hwstats_exit(nsim_dev);

err_psample_exit:

	nsim_dev_psample_exit(nsim_dev);

err_health_exit:
@@ -1595,15 +1601,21 @@ int nsim_drv_probe(struct nsim_bus_dev *nsim_bus_dev) 
	if (err)

		goto err_bpf_dev_exit;



	err = nsim_dev_port_add_all(nsim_dev, nsim_bus_dev->port_count);

	err = nsim_dev_hwstats_init(nsim_dev);

	if (err)

		goto err_psample_exit;



	err = nsim_dev_port_add_all(nsim_dev, nsim_bus_dev->port_count);

	if (err)

		goto err_hwstats_exit;



	nsim_dev->esw_mode = DEVLINK_ESWITCH_MODE_LEGACY;

	devlink_set_features(devlink, DEVLINK_F_RELOAD);

	devlink_register(devlink);

	return 0;



err_hwstats_exit:

	nsim_dev_hwstats_exit(nsim_dev);

err_psample_exit:

	nsim_dev_psample_exit(nsim_dev);

err_bpf_dev_exit:
@@ -1648,6 +1660,7 @@ static void nsim_dev_reload_destroy(struct nsim_dev *nsim_dev) 
	mutex_unlock(&nsim_dev->vfs_lock);



	nsim_dev_port_del_all(nsim_dev);

	nsim_dev_hwstats_exit(nsim_dev);

	nsim_dev_psample_exit(nsim_dev);

	nsim_dev_health_exit(nsim_dev);

	nsim_fib_destroy(devlink, nsim_dev->fib_data);

drivers/net/netdevsim/hwstats.c

0 → 100644
+486 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0



#include <linux/debugfs.h>



#include "netdevsim.h"



#define NSIM_DEV_HWSTATS_TRAFFIC_MS	100



static struct list_head *

nsim_dev_hwstats_get_list_head(struct nsim_dev_hwstats *hwstats,

			       enum netdev_offload_xstats_type type)

{

	switch (type) {

	case NETDEV_OFFLOAD_XSTATS_TYPE_L3:

		return &hwstats->l3_list;

	}



	WARN_ON_ONCE(1);

	return NULL;

}



static void nsim_dev_hwstats_traffic_bump(struct nsim_dev_hwstats *hwstats,

					  enum netdev_offload_xstats_type type)

{

	struct nsim_dev_hwstats_netdev *hwsdev;

	struct list_head *hwsdev_list;



	hwsdev_list = nsim_dev_hwstats_get_list_head(hwstats, type);

	if (WARN_ON(!hwsdev_list))

		return;



	list_for_each_entry(hwsdev, hwsdev_list, list) {

		if (hwsdev->enabled) {

			hwsdev->stats.rx_packets += 1;

			hwsdev->stats.tx_packets += 2;

			hwsdev->stats.rx_bytes += 100;

			hwsdev->stats.tx_bytes += 300;

		}

	}

}



static void nsim_dev_hwstats_traffic_work(struct work_struct *work)

{

	struct nsim_dev_hwstats *hwstats;



	hwstats = container_of(work, struct nsim_dev_hwstats, traffic_dw.work);

	mutex_lock(&hwstats->hwsdev_list_lock);

	nsim_dev_hwstats_traffic_bump(hwstats, NETDEV_OFFLOAD_XSTATS_TYPE_L3);

	mutex_unlock(&hwstats->hwsdev_list_lock);



	schedule_delayed_work(&hwstats->traffic_dw,

			      msecs_to_jiffies(NSIM_DEV_HWSTATS_TRAFFIC_MS));

}



static struct nsim_dev_hwstats_netdev *

nsim_dev_hwslist_find_hwsdev(struct list_head *hwsdev_list,

			     int ifindex)

{

	struct nsim_dev_hwstats_netdev *hwsdev;



	list_for_each_entry(hwsdev, hwsdev_list, list) {

		if (hwsdev->netdev->ifindex == ifindex)

			return hwsdev;

	}



	return NULL;

}



static int nsim_dev_hwsdev_enable(struct nsim_dev_hwstats_netdev *hwsdev,

				  struct netlink_ext_ack *extack)

{

	if (hwsdev->fail_enable) {

		hwsdev->fail_enable = false;

		NL_SET_ERR_MSG_MOD(extack, "Stats enablement set to fail");

		return -ECANCELED;

	}



	hwsdev->enabled = true;

	return 0;

}



static void nsim_dev_hwsdev_disable(struct nsim_dev_hwstats_netdev *hwsdev)

{

	hwsdev->enabled = false;

	memset(&hwsdev->stats, 0, sizeof(hwsdev->stats));

}



static int

nsim_dev_hwsdev_report_delta(struct nsim_dev_hwstats_netdev *hwsdev,

			     struct netdev_notifier_offload_xstats_info *info)

{

	netdev_offload_xstats_report_delta(info->report_delta, &hwsdev->stats);

	memset(&hwsdev->stats, 0, sizeof(hwsdev->stats));

	return 0;

}



static void

nsim_dev_hwsdev_report_used(struct nsim_dev_hwstats_netdev *hwsdev,

			    struct netdev_notifier_offload_xstats_info *info)

{

	if (hwsdev->enabled)

		netdev_offload_xstats_report_used(info->report_used);

}



static int nsim_dev_hwstats_event_off_xstats(struct nsim_dev_hwstats *hwstats,

					     struct net_device *dev,

					     unsigned long event, void *ptr)

{

	struct netdev_notifier_offload_xstats_info *info;

	struct nsim_dev_hwstats_netdev *hwsdev;

	struct list_head *hwsdev_list;

	int err = 0;



	info = ptr;

	hwsdev_list = nsim_dev_hwstats_get_list_head(hwstats, info->type);

	if (!hwsdev_list)

		return 0;



	mutex_lock(&hwstats->hwsdev_list_lock);



	hwsdev = nsim_dev_hwslist_find_hwsdev(hwsdev_list, dev->ifindex);

	if (!hwsdev)

		goto out;



	switch (event) {

	case NETDEV_OFFLOAD_XSTATS_ENABLE:

		err = nsim_dev_hwsdev_enable(hwsdev, info->info.extack);

		break;

	case NETDEV_OFFLOAD_XSTATS_DISABLE:

		nsim_dev_hwsdev_disable(hwsdev);

		break;

	case NETDEV_OFFLOAD_XSTATS_REPORT_USED:

		nsim_dev_hwsdev_report_used(hwsdev, info);

		break;

	case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA:

		err = nsim_dev_hwsdev_report_delta(hwsdev, info);

		break;

	}



out:

	mutex_unlock(&hwstats->hwsdev_list_lock);

	return err;

}



static void nsim_dev_hwsdev_fini(struct nsim_dev_hwstats_netdev *hwsdev)

{

	dev_put(hwsdev->netdev);

	kfree(hwsdev);

}



static void

__nsim_dev_hwstats_event_unregister(struct nsim_dev_hwstats *hwstats,

				    struct net_device *dev,

				    enum netdev_offload_xstats_type type)

{

	struct nsim_dev_hwstats_netdev *hwsdev;

	struct list_head *hwsdev_list;



	hwsdev_list = nsim_dev_hwstats_get_list_head(hwstats, type);

	if (WARN_ON(!hwsdev_list))

		return;



	hwsdev = nsim_dev_hwslist_find_hwsdev(hwsdev_list, dev->ifindex);

	if (!hwsdev)

		return;



	list_del(&hwsdev->list);

	nsim_dev_hwsdev_fini(hwsdev);

}



static void nsim_dev_hwstats_event_unregister(struct nsim_dev_hwstats *hwstats,

					      struct net_device *dev)

{

	mutex_lock(&hwstats->hwsdev_list_lock);

	__nsim_dev_hwstats_event_unregister(hwstats, dev,

					    NETDEV_OFFLOAD_XSTATS_TYPE_L3);

	mutex_unlock(&hwstats->hwsdev_list_lock);

}



static int nsim_dev_hwstats_event(struct nsim_dev_hwstats *hwstats,

				  struct net_device *dev,

				  unsigned long event, void *ptr)

{

	switch (event) {

	case NETDEV_OFFLOAD_XSTATS_ENABLE:

	case NETDEV_OFFLOAD_XSTATS_DISABLE:

	case NETDEV_OFFLOAD_XSTATS_REPORT_USED:

	case NETDEV_OFFLOAD_XSTATS_REPORT_DELTA:

		return nsim_dev_hwstats_event_off_xstats(hwstats, dev,

							 event, ptr);

	case NETDEV_UNREGISTER:

		nsim_dev_hwstats_event_unregister(hwstats, dev);

		break;

	}



	return 0;

}



static int nsim_dev_netdevice_event(struct notifier_block *nb,

				    unsigned long event, void *ptr)

{

	struct net_device *dev = netdev_notifier_info_to_dev(ptr);

	struct nsim_dev_hwstats *hwstats;

	int err = 0;



	hwstats = container_of(nb, struct nsim_dev_hwstats, netdevice_nb);

	err = nsim_dev_hwstats_event(hwstats, dev, event, ptr);

	if (err)

		return notifier_from_errno(err);



	return NOTIFY_OK;

}



static int

nsim_dev_hwstats_enable_ifindex(struct nsim_dev_hwstats *hwstats,

				int ifindex,

				enum netdev_offload_xstats_type type,

				struct list_head *hwsdev_list)

{

	struct nsim_dev_hwstats_netdev *hwsdev;

	struct nsim_dev *nsim_dev;

	struct net_device *netdev;

	bool notify = false;

	struct net *net;

	int err = 0;



	nsim_dev = container_of(hwstats, struct nsim_dev, hwstats);

	net = nsim_dev_net(nsim_dev);



	rtnl_lock();

	mutex_lock(&hwstats->hwsdev_list_lock);

	hwsdev = nsim_dev_hwslist_find_hwsdev(hwsdev_list, ifindex);

	if (hwsdev)

		goto out_unlock_list;



	netdev = dev_get_by_index(net, ifindex);

	if (!netdev) {

		err = -ENODEV;

		goto out_unlock_list;

	}



	hwsdev = kzalloc(sizeof(*hwsdev), GFP_KERNEL);

	if (!hwsdev) {

		err = -ENOMEM;

		goto out_put_netdev;

	}



	hwsdev->netdev = netdev;

	list_add_tail(&hwsdev->list, hwsdev_list);

	mutex_unlock(&hwstats->hwsdev_list_lock);



	if (netdev_offload_xstats_enabled(netdev, type)) {

		nsim_dev_hwsdev_enable(hwsdev, NULL);

		notify = true;

	}



	if (notify)

		rtnl_offload_xstats_notify(netdev);

	rtnl_unlock();

	return err;



out_put_netdev:

	dev_put(netdev);

out_unlock_list:

	mutex_unlock(&hwstats->hwsdev_list_lock);

	rtnl_unlock();

	return err;

}



static int

nsim_dev_hwstats_disable_ifindex(struct nsim_dev_hwstats *hwstats,

				 int ifindex,

				 enum netdev_offload_xstats_type type,

				 struct list_head *hwsdev_list)

{

	struct nsim_dev_hwstats_netdev *hwsdev;

	int err = 0;



	rtnl_lock();

	mutex_lock(&hwstats->hwsdev_list_lock);

	hwsdev = nsim_dev_hwslist_find_hwsdev(hwsdev_list, ifindex);

	if (hwsdev)

		list_del(&hwsdev->list);

	mutex_unlock(&hwstats->hwsdev_list_lock);



	if (!hwsdev) {

		err = -ENOENT;

		goto unlock_out;

	}



	if (netdev_offload_xstats_enabled(hwsdev->netdev, type)) {

		netdev_offload_xstats_push_delta(hwsdev->netdev, type,

						 &hwsdev->stats);

		rtnl_offload_xstats_notify(hwsdev->netdev);

	}

	nsim_dev_hwsdev_fini(hwsdev);



unlock_out:

	rtnl_unlock();

	return err;

}



static int

nsim_dev_hwstats_fail_ifindex(struct nsim_dev_hwstats *hwstats,

			      int ifindex,

			      enum netdev_offload_xstats_type type,

			      struct list_head *hwsdev_list)

{

	struct nsim_dev_hwstats_netdev *hwsdev;

	int err = 0;



	mutex_lock(&hwstats->hwsdev_list_lock);



	hwsdev = nsim_dev_hwslist_find_hwsdev(hwsdev_list, ifindex);

	if (!hwsdev) {

		err = -ENOENT;

		goto err_hwsdev_list_unlock;

	}



	hwsdev->fail_enable = true;



err_hwsdev_list_unlock:

	mutex_unlock(&hwstats->hwsdev_list_lock);

	return err;

}



enum nsim_dev_hwstats_do {

	NSIM_DEV_HWSTATS_DO_DISABLE,

	NSIM_DEV_HWSTATS_DO_ENABLE,

	NSIM_DEV_HWSTATS_DO_FAIL,

};



struct nsim_dev_hwstats_fops {

	const struct file_operations fops;

	enum nsim_dev_hwstats_do action;

	enum netdev_offload_xstats_type type;

};



static ssize_t

nsim_dev_hwstats_do_write(struct file *file,

			  const char __user *data,

			  size_t count, loff_t *ppos)

{

	struct nsim_dev_hwstats *hwstats = file->private_data;

	struct nsim_dev_hwstats_fops *hwsfops;

	struct list_head *hwsdev_list;

	int ifindex;

	int err;



	hwsfops = container_of(debugfs_real_fops(file),

			       struct nsim_dev_hwstats_fops, fops);



	err = kstrtoint_from_user(data, count, 0, &ifindex);

	if (err)

		return err;



	hwsdev_list = nsim_dev_hwstats_get_list_head(hwstats, hwsfops->type);

	if (WARN_ON(!hwsdev_list))

		return -EINVAL;



	switch (hwsfops->action) {

	case NSIM_DEV_HWSTATS_DO_DISABLE:

		err = nsim_dev_hwstats_disable_ifindex(hwstats, ifindex,

						       hwsfops->type,

						       hwsdev_list);

		break;

	case NSIM_DEV_HWSTATS_DO_ENABLE:

		err = nsim_dev_hwstats_enable_ifindex(hwstats, ifindex,

						      hwsfops->type,

						      hwsdev_list);

		break;

	case NSIM_DEV_HWSTATS_DO_FAIL:

		err = nsim_dev_hwstats_fail_ifindex(hwstats, ifindex,

						    hwsfops->type,

						    hwsdev_list);

		break;

	}

	if (err)

		return err;



	return count;

}



#define NSIM_DEV_HWSTATS_FOPS(ACTION, TYPE)			\

	{							\

		.fops = {					\

			.open = simple_open,			\

			.write = nsim_dev_hwstats_do_write,	\

			.llseek = generic_file_llseek,		\

			.owner = THIS_MODULE,			\

		},						\

		.action = ACTION,				\

		.type = TYPE,					\

	}



static const struct nsim_dev_hwstats_fops nsim_dev_hwstats_l3_disable_fops =

	NSIM_DEV_HWSTATS_FOPS(NSIM_DEV_HWSTATS_DO_DISABLE,

			      NETDEV_OFFLOAD_XSTATS_TYPE_L3);



static const struct nsim_dev_hwstats_fops nsim_dev_hwstats_l3_enable_fops =

	NSIM_DEV_HWSTATS_FOPS(NSIM_DEV_HWSTATS_DO_ENABLE,

			      NETDEV_OFFLOAD_XSTATS_TYPE_L3);



static const struct nsim_dev_hwstats_fops nsim_dev_hwstats_l3_fail_fops =

	NSIM_DEV_HWSTATS_FOPS(NSIM_DEV_HWSTATS_DO_FAIL,

			      NETDEV_OFFLOAD_XSTATS_TYPE_L3);



#undef NSIM_DEV_HWSTATS_FOPS



int nsim_dev_hwstats_init(struct nsim_dev *nsim_dev)

{

	struct nsim_dev_hwstats *hwstats = &nsim_dev->hwstats;

	struct net *net = nsim_dev_net(nsim_dev);

	int err;



	mutex_init(&hwstats->hwsdev_list_lock);

	INIT_LIST_HEAD(&hwstats->l3_list);



	hwstats->netdevice_nb.notifier_call = nsim_dev_netdevice_event;

	err = register_netdevice_notifier_net(net, &hwstats->netdevice_nb);

	if (err)

		goto err_mutex_destroy;



	hwstats->ddir = debugfs_create_dir("hwstats", nsim_dev->ddir);

	if (IS_ERR(hwstats->ddir)) {

		err = PTR_ERR(hwstats->ddir);

		goto err_unregister_notifier;

	}



	hwstats->l3_ddir = debugfs_create_dir("l3", hwstats->ddir);

	if (IS_ERR(hwstats->l3_ddir)) {

		err = PTR_ERR(hwstats->l3_ddir);

		goto err_remove_hwstats_recursive;

	}



	debugfs_create_file("enable_ifindex", 0600, hwstats->l3_ddir, hwstats,

			    &nsim_dev_hwstats_l3_enable_fops.fops);

	debugfs_create_file("disable_ifindex", 0600, hwstats->l3_ddir, hwstats,

			    &nsim_dev_hwstats_l3_disable_fops.fops);

	debugfs_create_file("fail_next_enable", 0600, hwstats->l3_ddir, hwstats,

			    &nsim_dev_hwstats_l3_fail_fops.fops);



	INIT_DELAYED_WORK(&hwstats->traffic_dw,

			  &nsim_dev_hwstats_traffic_work);

	schedule_delayed_work(&hwstats->traffic_dw,

			      msecs_to_jiffies(NSIM_DEV_HWSTATS_TRAFFIC_MS));

	return 0;



err_remove_hwstats_recursive:

	debugfs_remove_recursive(hwstats->ddir);

err_unregister_notifier:

	unregister_netdevice_notifier_net(net, &hwstats->netdevice_nb);

err_mutex_destroy:

	mutex_destroy(&hwstats->hwsdev_list_lock);

	return err;

}



static void nsim_dev_hwsdev_list_wipe(struct nsim_dev_hwstats *hwstats,

				      enum netdev_offload_xstats_type type)

{

	struct nsim_dev_hwstats_netdev *hwsdev, *tmp;

	struct list_head *hwsdev_list;



	hwsdev_list = nsim_dev_hwstats_get_list_head(hwstats, type);

	if (WARN_ON(!hwsdev_list))

		return;



	mutex_lock(&hwstats->hwsdev_list_lock);

	list_for_each_entry_safe(hwsdev, tmp, hwsdev_list, list) {

		list_del(&hwsdev->list);

		nsim_dev_hwsdev_fini(hwsdev);

	}

	mutex_unlock(&hwstats->hwsdev_list_lock);

}



void nsim_dev_hwstats_exit(struct nsim_dev *nsim_dev)

{

	struct nsim_dev_hwstats *hwstats = &nsim_dev->hwstats;

	struct net *net = nsim_dev_net(nsim_dev);



	cancel_delayed_work_sync(&hwstats->traffic_dw);

	debugfs_remove_recursive(hwstats->ddir);

	unregister_netdevice_notifier_net(net, &hwstats->netdevice_nb);

	nsim_dev_hwsdev_list_wipe(hwstats, NETDEV_OFFLOAD_XSTATS_TYPE_L3);

	mutex_destroy(&hwstats->hwsdev_list_lock);

}

drivers/net/netdevsim/netdevsim.h

+23 −0
Original line number Diff line number Diff line
@@ -184,6 +184,28 @@ struct nsim_dev_health { 
int nsim_dev_health_init(struct nsim_dev *nsim_dev, struct devlink *devlink);

void nsim_dev_health_exit(struct nsim_dev *nsim_dev);



struct nsim_dev_hwstats_netdev {

	struct list_head list;

	struct net_device *netdev;

	struct rtnl_hw_stats64 stats;

	bool enabled;

	bool fail_enable;

};



struct nsim_dev_hwstats {

	struct dentry *ddir;

	struct dentry *l3_ddir;



	struct mutex hwsdev_list_lock; /* protects hwsdev list(s) */

	struct list_head l3_list;



	struct notifier_block netdevice_nb;

	struct delayed_work traffic_dw;

};



int nsim_dev_hwstats_init(struct nsim_dev *nsim_dev);

void nsim_dev_hwstats_exit(struct nsim_dev *nsim_dev);



#if IS_ENABLED(CONFIG_PSAMPLE)

int nsim_dev_psample_init(struct nsim_dev *nsim_dev);

void nsim_dev_psample_exit(struct nsim_dev *nsim_dev);
@@ -261,6 +283,7 @@ struct nsim_dev { 
	bool fail_reload;

	struct devlink_region *dummy_region;

	struct nsim_dev_health health;

	struct nsim_dev_hwstats hwstats;

	struct flow_action_cookie *fa_cookie;

	spinlock_t fa_cookie_lock; /* protects fa_cookie */

	bool fail_trap_group_set;

tools/testing/selftests/drivers/net/mlxsw/hw_stats_l3.sh

0 → 100755
+31 −0
Original line number Diff line number Diff line 
#!/bin/bash

# SPDX-License-Identifier: GPL-2.0



lib_dir=$(dirname $0)/../../../net/forwarding



ALL_TESTS="

	l3_monitor_test

"

NUM_NETIFS=0

source $lib_dir/lib.sh



swp=$NETIF_NO_CABLE



cleanup()

{

	pre_cleanup

}



l3_monitor_test()

{

	hw_stats_monitor_test $swp l3		    \

		"ip addr add dev $swp 192.0.2.1/28" \

		"ip addr del dev $swp 192.0.2.1/28"

}



trap cleanup EXIT



setup_wait

tests_run



exit $EXIT_STATUS