Commit 577c34b3 authored by Andrii Nakryiko's avatar Andrii Nakryiko
Browse files

Merge branch 'selftests/bpf: test_progs can read test lists from file' 

Stephen Veiss says:

====================

BPF selftests have ALLOWLIST and DENYLIST files, used to control which
tests are run in CI. These files are currently parsed by a shell
script. [1]

This patchset allows those files to be specified directly on the
test_progs command line (eg, as -a @ALLOWLIST).

This also fixes a bug in the existing test filter code causing
unnecessary duplicate top-level test filter entries to be created.

[1] https://github.com/kernel-patches/vmtest/blob/57feb460047b69f891cf4afe3cc860794a2ced17/ci/vmtest/run_selftests.sh#L21-L27
---

v2:
 - error handling style changes per reviewer comments
 - fdopen return value checking in test_parse_test_list_file

v1:
 https://lore.kernel.org/bpf/20230425225401.1075796-1-sveiss@meta.com/


====================

Signed-off-by: default avatarAndrii Nakryiko <andrii@kernel.org>
parents c39028b3 64276f01

tools/testing/selftests/bpf/prog_tests/arg_parsing.c

+68 −0
Original line number Diff line number Diff line
@@ -96,12 +96,80 @@ static void test_parse_test_list(void) 
		goto error;

	ASSERT_OK(strcmp("*bpf_cookie*", set.tests[0].name), "test name");

	ASSERT_OK(strcmp("*trace*", set.tests[0].subtests[0]), "subtest name");

	free_test_filter_set(&set);



	ASSERT_OK(parse_test_list("t/subtest1,t/subtest2", &set, true),

		  "parsing");

	if (!ASSERT_EQ(set.cnt, 1, "count of test filters"))

		goto error;

	if (!ASSERT_OK_PTR(set.tests, "test filters initialized"))

		goto error;

	if (!ASSERT_EQ(set.tests[0].subtest_cnt, 2, "subtest filters count"))

		goto error;

	ASSERT_OK(strcmp("t", set.tests[0].name), "test name");

	ASSERT_OK(strcmp("subtest1", set.tests[0].subtests[0]), "subtest name");

	ASSERT_OK(strcmp("subtest2", set.tests[0].subtests[1]), "subtest name");

error:

	free_test_filter_set(&set);

}



static void test_parse_test_list_file(void)

{

	struct test_filter_set set;

	char tmpfile[80];

	FILE *fp;

	int fd;



	snprintf(tmpfile, sizeof(tmpfile), "/tmp/bpf_arg_parsing_test.XXXXXX");

	fd = mkstemp(tmpfile);

	if (!ASSERT_GE(fd, 0, "create tmp"))

		return;



	fp = fdopen(fd, "w");

	if (!ASSERT_NEQ(fp, NULL, "fdopen tmp")) {

		close(fd);

		goto out_remove;

	}



	fprintf(fp, "# comment\n");

	fprintf(fp, "  test_with_spaces    \n");

	fprintf(fp, "testA/subtest    # comment\n");

	fprintf(fp, "testB#comment with no space\n");

	fprintf(fp, "testB # duplicate\n");

	fprintf(fp, "testA/subtest # subtest duplicate\n");

	fprintf(fp, "testA/subtest2\n");

	fprintf(fp, "testC_no_eof_newline");

	fflush(fp);



	if (!ASSERT_OK(ferror(fp), "prepare tmp"))

		goto out_fclose;



	init_test_filter_set(&set);



	ASSERT_OK(parse_test_list_file(tmpfile, &set, true), "parse file");



	ASSERT_EQ(set.cnt, 4, "test  count");

	ASSERT_OK(strcmp("test_with_spaces", set.tests[0].name), "test 0 name");

	ASSERT_EQ(set.tests[0].subtest_cnt, 0, "test 0 subtest count");

	ASSERT_OK(strcmp("testA", set.tests[1].name), "test 1 name");

	ASSERT_EQ(set.tests[1].subtest_cnt, 2, "test 1 subtest count");

	ASSERT_OK(strcmp("subtest", set.tests[1].subtests[0]), "test 1 subtest 0");

	ASSERT_OK(strcmp("subtest2", set.tests[1].subtests[1]), "test 1 subtest 1");

	ASSERT_OK(strcmp("testB", set.tests[2].name), "test 2 name");

	ASSERT_OK(strcmp("testC_no_eof_newline", set.tests[3].name), "test 3 name");



	free_test_filter_set(&set);



out_fclose:

	fclose(fp);

out_remove:

	remove(tmpfile);

}



void test_arg_parsing(void)

{

	if (test__start_subtest("test_parse_test_list"))

		test_parse_test_list();

	if (test__start_subtest("test_parse_test_list_file"))

		test_parse_test_list_file();

}

tools/testing/selftests/bpf/test_progs.c

+27 −10
Original line number Diff line number Diff line
@@ -714,7 +714,13 @@ static struct test_state test_states[ARRAY_SIZE(prog_test_defs)]; 


const char *argp_program_version = "test_progs 0.1";

const char *argp_program_bug_address = "<bpf@vger.kernel.org>";

static const char argp_program_doc[] = "BPF selftests test runner";

static const char argp_program_doc[] =

"BPF selftests test runner\v"

"Options accepting the NAMES parameter take either a comma-separated list\n"

"of test names, or a filename prefixed with @. The file contains one name\n"

"(or wildcard pattern) per line, and comments beginning with # are ignored.\n"

"\n"

"These options can be passed repeatedly to read multiple files.\n";



enum ARG_KEYS {

	ARG_TEST_NUM = 'n',
@@ -797,6 +803,7 @@ extern int extra_prog_load_log_flags; 
static error_t parse_arg(int key, char *arg, struct argp_state *state)

{

	struct test_env *env = state->input;

	int err = 0;



	switch (key) {

	case ARG_TEST_NUM: {
@@ -821,18 +828,28 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) 
	}

	case ARG_TEST_NAME_GLOB_ALLOWLIST:

	case ARG_TEST_NAME: {

		if (parse_test_list(arg,

		if (arg[0] == '@')

			err = parse_test_list_file(arg + 1,

						   &env->test_selector.whitelist,

				    key == ARG_TEST_NAME_GLOB_ALLOWLIST))

			return -ENOMEM;

						   key == ARG_TEST_NAME_GLOB_ALLOWLIST);

		else

			err = parse_test_list(arg,

					      &env->test_selector.whitelist,

					      key == ARG_TEST_NAME_GLOB_ALLOWLIST);



		break;

	}

	case ARG_TEST_NAME_GLOB_DENYLIST:

	case ARG_TEST_NAME_BLACKLIST: {

		if (parse_test_list(arg,

		if (arg[0] == '@')

			err = parse_test_list_file(arg + 1,

						   &env->test_selector.blacklist,

				    key == ARG_TEST_NAME_GLOB_DENYLIST))

			return -ENOMEM;

						   key == ARG_TEST_NAME_GLOB_DENYLIST);

		else

			err = parse_test_list(arg,

					      &env->test_selector.blacklist,

					      key == ARG_TEST_NAME_GLOB_DENYLIST);



		break;

	}

	case ARG_VERIFIER_STATS:
@@ -900,7 +917,7 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) 
	default:

		return ARGP_ERR_UNKNOWN;

	}

	return 0;

	return err;

}



/*

tools/testing/selftests/bpf/testing_helpers.c

+142 −65
Original line number Diff line number Diff line 
// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)

/* Copyright (C) 2019 Netronome Systems, Inc. */

/* Copyright (C) 2020 Facebook, Inc. */

#include <ctype.h>

#include <stdlib.h>

#include <string.h>

#include <errno.h>
@@ -70,30 +71,66 @@ int parse_num_list(const char *s, bool **num_set, int *num_set_len) 
	return 0;

}



int parse_test_list(const char *s,

		    struct test_filter_set *set,

		    bool is_glob_pattern)

static int do_insert_test(struct test_filter_set *set,

			  char *test_str,

			  char *subtest_str)

{

	char *input, *state = NULL, *next;

	struct test_filter *tmp, *tests = NULL;

	int i, j, cnt = 0;

	struct test_filter *tmp, *test;

	char **ctmp;

	int i;



	input = strdup(s);

	if (!input)

		return -ENOMEM;

	for (i = 0; i < set->cnt; i++) {

		test = &set->tests[i];



	while ((next = strtok_r(state ? NULL : input, ",", &state))) {

		char *subtest_str = strchr(next, '/');

		char *pattern = NULL;

		int glob_chars = 0;

		if (strcmp(test_str, test->name) == 0) {

			free(test_str);

			goto subtest;

		}

	}



		tmp = realloc(tests, sizeof(*tests) * (cnt + 1));

	tmp = realloc(set->tests, sizeof(*test) * (set->cnt + 1));

	if (!tmp)

			goto err;

		tests = tmp;

		return -ENOMEM;



	set->tests = tmp;

	test = &set->tests[set->cnt];



	test->name = test_str;

	test->subtests = NULL;

	test->subtest_cnt = 0;



	set->cnt++;



subtest:

	if (!subtest_str)

		return 0;



	for (i = 0; i < test->subtest_cnt; i++) {

		if (strcmp(subtest_str, test->subtests[i]) == 0) {

			free(subtest_str);

			return 0;

		}

	}



	ctmp = realloc(test->subtests,

		       sizeof(*test->subtests) * (test->subtest_cnt + 1));

	if (!ctmp)

		return -ENOMEM;



	test->subtests = ctmp;

	test->subtests[test->subtest_cnt] = subtest_str;



	test->subtest_cnt++;



		tests[cnt].subtest_cnt = 0;

		tests[cnt].subtests = NULL;

	return 0;

}



static int insert_test(struct test_filter_set *set,

		       char *test_spec,

		       bool is_glob_pattern)

{

	char *pattern, *subtest_str, *ext_test_str, *ext_subtest_str = NULL;

	int glob_chars = 0;



	if (is_glob_pattern) {

		pattern = "%s";
@@ -102,60 +139,100 @@ int parse_test_list(const char *s, 
		glob_chars = 2;

	}



	subtest_str = strchr(test_spec, '/');

	if (subtest_str) {

			char **tmp_subtests = NULL;

			int subtest_cnt = tests[cnt].subtest_cnt;



		*subtest_str = '\0';

		subtest_str += 1;

			tmp_subtests = realloc(tests[cnt].subtests,

					       sizeof(*tmp_subtests) *

					       (subtest_cnt + 1));

			if (!tmp_subtests)

	}



	ext_test_str = malloc(strlen(test_spec) + glob_chars + 1);

	if (!ext_test_str)

		goto err;

			tests[cnt].subtests = tmp_subtests;



			tests[cnt].subtests[subtest_cnt] =

				malloc(strlen(subtest_str) + glob_chars + 1);

			if (!tests[cnt].subtests[subtest_cnt])

	sprintf(ext_test_str, pattern, test_spec);



	if (subtest_str) {

		ext_subtest_str = malloc(strlen(subtest_str) + glob_chars + 1);

		if (!ext_subtest_str)

			goto err;

			sprintf(tests[cnt].subtests[subtest_cnt],

				pattern,

				subtest_str);



			tests[cnt].subtest_cnt++;

		sprintf(ext_subtest_str, pattern, subtest_str);

	}



		tests[cnt].name = malloc(strlen(next) + glob_chars + 1);

		if (!tests[cnt].name)

			goto err;

		sprintf(tests[cnt].name, pattern, next);

	return do_insert_test(set, ext_test_str, ext_subtest_str);



		cnt++;

err:

	free(ext_test_str);

	free(ext_subtest_str);



	return -ENOMEM;

}



	tmp = realloc(set->tests, sizeof(*tests) * (cnt + set->cnt));

	if (!tmp)

		goto err;

int parse_test_list_file(const char *path,

			 struct test_filter_set *set,

			 bool is_glob_pattern)

{

	char *buf = NULL, *capture_start, *capture_end, *scan_end;

	size_t buflen = 0;

	int err = 0;

	FILE *f;



	memcpy(tmp +  set->cnt, tests, sizeof(*tests) * cnt);

	set->tests = tmp;

	set->cnt += cnt;

	f = fopen(path, "r");

	if (!f) {

		err = -errno;

		fprintf(stderr, "Failed to open '%s': %d\n", path, err);

		return err;

	}



	free(tests);

	free(input);

	return 0;

	while (getline(&buf, &buflen, f) != -1) {

		capture_start = buf;



err:

	for (i = 0; i < cnt; i++) {

		for (j = 0; j < tests[i].subtest_cnt; j++)

			free(tests[i].subtests[j]);

		while (isspace(*capture_start))

			++capture_start;



		capture_end = capture_start;

		scan_end = capture_start;



		free(tests[i].name);

		while (*scan_end && *scan_end != '#') {

			if (!isspace(*scan_end))

				capture_end = scan_end;



			++scan_end;

		}

	free(tests);

	free(input);



		if (capture_end == capture_start)

			continue;



		*(++capture_end) = '\0';



		err = insert_test(set, capture_start, is_glob_pattern);

		if (err)

			break;

	}



	fclose(f);

	return err;

}



int parse_test_list(const char *s,

		    struct test_filter_set *set,

		    bool is_glob_pattern)

{

	char *input, *state = NULL, *test_spec;

	int err = 0;



	input = strdup(s);

	if (!input)

		return -ENOMEM;



	while ((test_spec = strtok_r(state ? NULL : input, ",", &state))) {

		err = insert_test(set, test_spec, is_glob_pattern);

		if (err)

			break;

	}



	free(input);

	return err;

}



__u32 link_info_prog_id(const struct bpf_link *link, struct bpf_link_info *info)

tools/testing/selftests/bpf/testing_helpers.h

+3 −0
Original line number Diff line number Diff line
@@ -20,5 +20,8 @@ struct test_filter_set; 
int parse_test_list(const char *s,

		    struct test_filter_set *test_set,

		    bool is_glob_pattern);

int parse_test_list_file(const char *path,

			 struct test_filter_set *test_set,

			 bool is_glob_pattern);



__u64 read_perf_max_sample_freq(void);