Commit dbcf7639 authored by Mark Brown's avatar Mark Brown Committed by Shuah Khan
Browse files

selftests/ftrace: Improve integration with kselftest runner 



The ftrace selftests do not currently produce KTAP output, they produce a
custom format much nicer for human consumption. This means that when run in
automated test systems we just get a single result for the suite as a whole
rather than recording results for individual test cases, making it harder
to look at the test data and masking things like inappropriate skips.

Address this by adding support for KTAP output to the ftracetest script and
providing a trivial wrapper which will be invoked by the kselftest runner
to generate output in this format by default, users using ftracetest
directly will continue to get the existing output.

This is not the most elegant solution but it is simple and effective. I
did consider implementing this by post processing the existing output
format but that felt more complex and likely to result in all output being
lost if something goes seriously wrong during the run which would not be
helpful. I did also consider just writing a separate runner script but
there's enough going on with things like the signal handling for that to
seem like it would be duplicating too much.

Acked-by: default avatarSteven Rostedt (Google) <rostedt@goodmis.org>
Acked-by: default avatarMasami Hiramatsu (Google) <mhiramat@kernel.org>
Tested-by: default avatarMasami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: default avatarMark Brown <broonie@kernel.org>
Signed-off-by: default avatarShuah Khan <skhan@linuxfoundation.org>
parent 4b79f769

tools/testing/selftests/ftrace/Makefile

+2 −1
Original line number Diff line number Diff line 
# SPDX-License-Identifier: GPL-2.0

all:



TEST_PROGS := ftracetest

TEST_PROGS_EXTENDED := ftracetest

TEST_PROGS := ftracetest-ktap

TEST_FILES := test.d settings

EXTRA_CLEAN := $(OUTPUT)/logs/*

tools/testing/selftests/ftrace/ftracetest

+60 −3
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ echo "Usage: ftracetest [options] [testcase(s)] [testcase-directory(s)]" 
echo " Options:"

echo "		-h|--help  Show help message"

echo "		-k|--keep  Keep passed test logs"

echo "		-K|--ktap  Output in KTAP format"

echo "		-v|--verbose Increase verbosity of test messages"

echo "		-vv        Alias of -v -v (Show all results in stdout)"

echo "		-vvv       Alias of -v -v -v (Show all commands immediately)"
@@ -85,6 +86,10 @@ parse_opts() { # opts 
      KEEP_LOG=1

      shift 1

    ;;

    --ktap|-K)

      KTAP=1

      shift 1

    ;;

    --verbose|-v|-vv|-vvv)

      if [ $VERBOSE -eq -1 ]; then

	usage "--console can not use with --verbose"
@@ -178,6 +183,7 @@ TEST_DIR=$TOP_DIR/test.d 
TEST_CASES=`find_testcases $TEST_DIR`

LOG_DIR=$TOP_DIR/logs/`date +%Y%m%d-%H%M%S`/

KEEP_LOG=0

KTAP=0

DEBUG=0

VERBOSE=0

UNSUPPORTED_RESULT=0
@@ -229,7 +235,7 @@ prlog() { # messages 
    newline=

    shift

  fi

  printf "$*$newline"

  [ "$KTAP" != "1" ] && printf "$*$newline"

  [ "$LOG_FILE" ] && printf "$*$newline" | strip_esc >> $LOG_FILE

}

catlog() { #file
@@ -260,11 +266,11 @@ TOTAL_RESULT=0 


INSTANCE=

CASENO=0

CASENAME=



testcase() { # testfile

  CASENO=$((CASENO+1))

  desc=`grep "^#[ \t]*description:" $1 | cut -f2- -d:`

  prlog -n "[$CASENO]$INSTANCE$desc"

  CASENAME=`grep "^#[ \t]*description:" $1 | cut -f2- -d:`

}



checkreq() { # testfile
@@ -277,40 +283,68 @@ test_on_instance() { # testfile 
  grep -q "^#[ \t]*flags:.*instance" $1

}



ktaptest() { # result comment

  if [ "$KTAP" != "1" ]; then

    return

  fi



  local result=

  if [ "$1" = "1" ]; then

    result="ok"

  else

    result="not ok"

  fi

  shift



  local comment=$*

  if [ "$comment" != "" ]; then

    comment="# $comment"

  fi



  echo $CASENO $result $INSTANCE$CASENAME $comment

}



eval_result() { # sigval

  case $1 in

    $PASS)

      prlog "	[${color_green}PASS${color_reset}]"

      ktaptest 1

      PASSED_CASES="$PASSED_CASES $CASENO"

      return 0

    ;;

    $FAIL)

      prlog "	[${color_red}FAIL${color_reset}]"

      ktaptest 0

      FAILED_CASES="$FAILED_CASES $CASENO"

      return 1 # this is a bug.

    ;;

    $UNRESOLVED)

      prlog "	[${color_blue}UNRESOLVED${color_reset}]"

      ktaptest 0 UNRESOLVED

      UNRESOLVED_CASES="$UNRESOLVED_CASES $CASENO"

      return $UNRESOLVED_RESULT # depends on use case

    ;;

    $UNTESTED)

      prlog "	[${color_blue}UNTESTED${color_reset}]"

      ktaptest 1 SKIP

      UNTESTED_CASES="$UNTESTED_CASES $CASENO"

      return 0

    ;;

    $UNSUPPORTED)

      prlog "	[${color_blue}UNSUPPORTED${color_reset}]"

      ktaptest 1 SKIP

      UNSUPPORTED_CASES="$UNSUPPORTED_CASES $CASENO"

      return $UNSUPPORTED_RESULT # depends on use case

    ;;

    $XFAIL)

      prlog "	[${color_green}XFAIL${color_reset}]"

      ktaptest 1 XFAIL

      XFAILED_CASES="$XFAILED_CASES $CASENO"

      return 0

    ;;

    *)

      prlog "	[${color_blue}UNDEFINED${color_reset}]"

      ktaptest 0 error

      UNDEFINED_CASES="$UNDEFINED_CASES $CASENO"

      return 1 # this must be a test bug

    ;;
@@ -371,6 +405,7 @@ __run_test() { # testfile 
run_test() { # testfile

  local testname=`basename $1`

  testcase $1

  prlog -n "[$CASENO]$INSTANCE$CASENAME"

  if [ ! -z "$LOG_FILE" ] ; then

    local testlog=`mktemp $LOG_DIR/${CASENO}-${testname}-log.XXXXXX`

  else
@@ -405,6 +440,17 @@ run_test() { # testfile 
# load in the helper functions

. $TEST_DIR/functions



if [ "$KTAP" = "1" ]; then

  echo "TAP version 13"



  casecount=`echo $TEST_CASES | wc -w`

  for t in $TEST_CASES; do

    test_on_instance $t || continue

    casecount=$((casecount+1))

  done

  echo "1..${casecount}"

fi



# Main loop

for t in $TEST_CASES; do

  run_test $t
@@ -439,6 +485,17 @@ prlog "# of unsupported: " `echo $UNSUPPORTED_CASES | wc -w` 
prlog "# of xfailed: " `echo $XFAILED_CASES | wc -w`

prlog "# of undefined(test bug): " `echo $UNDEFINED_CASES | wc -w`



if [ "$KTAP" = "1" ]; then

  echo -n "# Totals:"

  echo -n " pass:"`echo $PASSED_CASES | wc -w`

  echo -n " faii:"`echo $FAILED_CASES | wc -w`

  echo -n " xfail:"`echo $XFAILED_CASES | wc -w`

  echo -n " xpass:0"

  echo -n " skip:"`echo $UNTESTED_CASES $UNSUPPORTED_CASES | wc -w`

  echo -n " error:"`echo $UNRESOLVED_CASES $UNDEFINED_CASES | wc -w`

  echo

fi



cleanup



# if no error, return 0

tools/testing/selftests/ftrace/ftracetest-ktap

0 → 100755
+8 −0
Original line number Diff line number Diff line 
#!/bin/sh -e

# SPDX-License-Identifier: GPL-2.0-only

#

# ftracetest-ktap: Wrapper to integrate ftracetest with the kselftest runner

#

# Copyright (C) Arm Ltd., 2023



./ftracetest -K