testme.sh

   1 #!/bin/bash
   2 #
   3 # return values of this script are:
   4 #   0  success
   5 # 128  a test failed
   6 #  >0  the number of timed-out tests
   7 # 255  parsing of parameters failed
   8
   9 set -e
  10
  11 if [ -f /proc/cpuinfo ]
  12 then
  13   MAKE_JOBS=$(( ($(cat /proc/cpuinfo | grep -E '^processor[[:space:]]*:' | tail -n -1 | cut -d':' -f2) + 1) * 2 + 1 ))
  14 else
  15   MAKE_JOBS=8
  16 fi
  17
  18 ret=0
  19 TEST_CFLAGS=""
  20
  21 _help()
  22 {
  23   cat << EOF
  24 Usage options for $(basename $0) [--with-cc=arg [other options]]
  25
  26 Executing this script without any parameter will only run the default
  27 configuration that has automatically been determined for the
  28 architecture you're running.
  29
  30     --with-cc=*             The compiler(s) to use for the tests
  31                             This is an option that will be iterated.
  32
  33     --test-vs-mtest=*       Run test vs. mtest for '*' operations.
  34                             Only the first of each options will be
  35                             taken into account.
  36
  37 To be able to specify options a compiler has to be given with
  38 the option --with-cc=compilername
  39 All other options will be tested with all MP_xBIT configurations.
  40
  41     --with-{m64,m32,mx32}   The architecture(s) to build and test
  42                             for, e.g. --with-mx32.
  43                             This is an option that will be iterated,
  44                             multiple selections are possible.
  45                             The mx32 architecture is not supported
  46                             by clang and will not be executed.
  47
  48     --cflags=*              Give an option to the compiler,
  49                             e.g. --cflags=-g
  50                             This is an option that will always be
  51                             passed as parameter to CC.
  52
  53     --make-option=*         Give an option to make,
  54                             e.g. --make-option="-f makefile.shared"
  55                             This is an option that will always be
  56                             passed as parameter to make.
  57
  58     --with-low-mp           Also build&run tests with -DMP_{8,16,32}BIT.
  59
  60     --mtest-real-rand       Use real random data when running mtest.
  61
  62     --with-valgrind
  63     --with-valgrind=*       Run in valgrind (slow!).
  64
  65     --limit-valgrind        Run with valgrind on CI only on specific branches.
  66
  67     --valgrind-options      Additional Valgrind options
  68                             Some of the options like e.g.:
  69                             --track-origins=yes add a lot of extra
  70                             runtime and may trigger the 30 minutes
  71                             timeout.
  72
  73     --multithread           Run tests in multi-threaded mode (via pthread).
  74
  75 Godmode:
  76
  77     --all                   Choose all architectures and gcc and clang
  78                             as compilers but does not run valgrind.
  79
  80     --format                Runs the various source-code formatters
  81                             and generators and checks if the sources
  82                             are clean.
  83
  84     -h
  85     --help                  This message
  86
  87     -v
  88     --version               Prints the version. It is just the number
  89                             of git commits to this file, no deeper
  90                             meaning attached
  91 EOF
  92   exit 0
  93 }
  94
  95 _die()
  96 {
  97   echo "error $2 while $1"
  98   if [ "$2" != "124" ]
  99   then
 100     exit 128
 101   else
 102     echo "assuming timeout while running test - continue"
 103     local _tail=""
 104     which tail >/dev/null && _tail="tail -n 1 test_${suffix}.log" && \
 105     echo "last line of test_"${suffix}".log was:" && $_tail && echo ""
 106     ret=$(( $ret + 1 ))
 107   fi
 108 }
 109
 110 _fixup_cflags() {
 111   compiler_version=$(echo "$1="$($1 -dumpversion))
 112   case "$compiler_version" in
 113     clang*=4.2.1)
 114       # one of my versions of clang complains about some stuff in stdio.h and stdarg.h ...
 115       TEST_CFLAGS="-Wno-typedef-redefinition"
 116     ;;
 117     gcc*=9)
 118       # gcc 9 seems to sometimes think that variables are uninitialized, but they are.
 119       TEST_CFLAGS="-Wno-maybe-uninitialized"
 120     ;;
 121     *)
 122       TEST_CFLAGS=""
 123     ;;
 124   esac
 125   echo $compiler_version
 126 }
 127
 128 _make()
 129 {
 130   echo -ne " Compile $1 $2"
 131   suffix=$(echo ${1}${2}  | tr ' ' '_')
 132   _fixup_cflags "$1"
 133   CC="$1" CFLAGS="$2 $TEST_CFLAGS" LFLAGS="$4" LDFLAGS="$5" make -j$MAKE_JOBS $3 $MAKE_OPTIONS 2>gcc_errors_${suffix}.log
 134   errcnt=$(wc -l < gcc_errors_${suffix}.log)
 135   if [[ ${errcnt} -gt 1 ]]; then
 136     echo " failed"
 137     cat gcc_errors_${suffix}.log
 138     exit 128
 139   fi
 140 }
 141
 142
 143 _runtest()
 144 {
 145   make clean > /dev/null
 146   local _timeout=""
 147   which timeout >/dev/null && _timeout="timeout --foreground 90"
 148   if [[ "$MAKE_OPTIONS" =~ "tune" ]]
 149   then
 150     # "make tune" will run "tune_it.sh" automatically, hence "autotune", but it cannot
 151     # get switched off without some effort, so we just let it run twice for testing purposes
 152     echo -e "\rRun autotune $1 $2"
 153     _make "$1" "$2" "" "$3" "$4"
 154     $_timeout $TUNE_CMD > test_${suffix}.log || _die "running autotune" $?
 155   else
 156     _make "$1" "$2" "test" "$3" "$4"
 157     echo -e "\rRun test $1 $2"
 158     $_timeout ./test > test_${suffix}.log || _die "running tests" $?
 159   fi
 160 }
 161
 162 # This is not much more of a C&P of _runtest with a different timeout
 163 # and the additional valgrind call.
 164 # TODO: merge
 165 _runvalgrind()
 166 {
 167   make clean > /dev/null
 168   local _timeout=""
 169   # 30 minutes? Yes. Had it at 20 minutes and the Valgrind run needed over 25 minutes.
 170   # A bit too close for comfort.
 171   which timeout >/dev/null && _timeout="timeout --foreground 1800"
 172 echo "MAKE_OPTIONS = \"$MAKE_OPTIONS\""
 173   if [[ "$MAKE_OPTIONS" =~ "tune"  ]]
 174   then
 175 echo "autotune branch"
 176     _make "$1" "$2" "" "$3" "$4"
 177     # The shell used for /bin/sh is DASH 0.5.7-4ubuntu1 on the author's machine which fails valgrind, so
 178     # we just run on instance of etc/tune with the same options as in etc/tune_it.sh
 179     echo -e "\rRun etc/tune $1 $2 once inside valgrind"
 180     $_timeout $VALGRIND_BIN $VALGRIND_OPTS $TUNE_CMD > test_${suffix}.log || _die "running etc/tune" $?
 181   else
 182     _make "$1" "$2" "test" "$3" "$4"
 183     echo -e "\rRun test $1 $2 inside valgrind"
 184     $_timeout $VALGRIND_BIN $VALGRIND_OPTS ./test > test_${suffix}.log || _die "running tests" $?
 185   fi
 186 }
 187
 188
 189 _banner()
 190 {
 191   echo "uname="$(uname -a)
 192   [[ "$#" != "0" ]] && (echo $1=$($1 -dumpversion)) || true
 193 }
 194
 195 _exit()
 196 {
 197   if [ "$ret" == "0" ]
 198   then
 199     echo "Tests successful"
 200   else
 201     echo "$ret tests timed out"
 202   fi
 203
 204   exit $ret
 205 }
 206
 207 ARCHFLAGS=""
 208 COMPILERS=""
 209 CFLAGS=""
 210 WITH_LOW_MP=""
 211 TEST_VS_MTEST=""
 212 MTEST_RAND=""
 213 # timed with an AMD A8-6600K
 214 # 25 minutes
 215 #VALGRIND_OPTS=" --track-origins=yes --leak-check=full --show-leak-kinds=all --error-exitcode=1 "
 216 # 9 minutes (14 minutes with --test-vs-mtest=333333 --mtest-real-rand)
 217 VALGRIND_OPTS=" --leak-check=full --show-leak-kinds=all --error-exitcode=1 "
 218 #VALGRIND_OPTS=""
 219 VALGRIND_BIN=""
 220 CHECK_FORMAT=""
 221 CHECK_SYMBOLS=""
 222 C89=""
 223 C89_C99_ROUNDTRIP=""
 224 TUNE_CMD="./etc/tune -t -r 10 -L 3"
 225
 226 alive_pid=0
 227
 228 function kill_alive() {
 229   disown $alive_pid || true
 230   kill $alive_pid 2>/dev/null
 231 }
 232
 233 function start_alive_printing() {
 234   [ "$alive_pid" == "0" ] || return 0;
 235   for i in `seq 1 10` ; do sleep 300 && echo "Tests still in Progress..."; done &
 236   alive_pid=$!
 237   trap kill_alive EXIT
 238 }
 239
 240 while [ $# -gt 0 ];
 241 do
 242   case $1 in
 243     "--with-m64" | "--with-m32" | "--with-mx32")
 244       ARCHFLAGS="$ARCHFLAGS ${1:6}"
 245     ;;
 246     --c89)
 247       C89="1"
 248     ;;
 249     --c89-c99-roundtrip)
 250       C89_C99_ROUNDTRIP="1"
 251     ;;
 252     --with-cc=*)
 253       COMPILERS="$COMPILERS ${1#*=}"
 254     ;;
 255     --cflags=*)
 256       CFLAGS="$CFLAGS ${1#*=}"
 257     ;;
 258     --valgrind-options=*)
 259       VALGRIND_OPTS="$VALGRIND_OPTS ${1#*=}"
 260     ;;
 261     --with-valgrind*)
 262       if [[ ${1#*d} != "" ]]
 263       then
 264         VALGRIND_BIN="${1#*=}"
 265       else
 266         VALGRIND_BIN="valgrind"
 267       fi
 268       start_alive_printing
 269     ;;
 270     --limit-valgrind*)
 271       if [[ ("$GITHUB_BASE_REF" == "develop" && "$PR_NUMBER" == "") || "$GITHUB_REF_NAME" == *"valgrind"* || "$COMMIT_MESSAGE" == *"valgrind"* ]]
 272       then
 273         if [[ ${1#*d} != "" ]]
 274         then
 275           VALGRIND_BIN="${1#*=}"
 276         else
 277           VALGRIND_BIN="valgrind"
 278         fi
 279         start_alive_printing
 280       fi
 281     ;;
 282     --make-option=*)
 283       MAKE_OPTIONS="$MAKE_OPTIONS ${1#*=}"
 284     ;;
 285     --with-low-mp)
 286       WITH_LOW_MP="1"
 287     ;;
 288     --test-vs-mtest=*)
 289       TEST_VS_MTEST="${1#*=}"
 290       if ! [ "$TEST_VS_MTEST" -eq "$TEST_VS_MTEST" ] 2> /dev/null
 291       then
 292          echo "--test-vs-mtest Parameter has to be int"
 293          exit 255
 294       fi
 295       start_alive_printing
 296     ;;
 297     --mtest-real-rand)
 298       MTEST_RAND="-DLTM_MTEST_REAL_RAND"
 299     ;;
 300     --format)
 301       CHECK_FORMAT="1"
 302     ;;
 303     --symbols)
 304       CHECK_SYMBOLS="1"
 305     ;;
 306     --multithread)
 307       CFLAGS="$CFLAGS -DLTM_TEST_MULTITHREAD"
 308       LFLAGS="$LFLAGS -pthread"
 309       LDFLAGS="$LDFLAGS -pthread"
 310     ;;
 311     --all)
 312       COMPILERS="gcc clang"
 313       ARCHFLAGS="-m64 -m32 -mx32"
 314     ;;
 315     --help | -h)
 316       _help
 317     ;;
 318     --version | -v)
 319       echo $(git rev-list HEAD --count -- testme.sh) || echo "Unknown. Please run in original libtommath git repository."
 320       exit 0
 321     ;;
 322     *)
 323       echo "Ignoring option ${1}"
 324     ;;
 325   esac
 326   shift
 327 done
 328
 329 function _check_git() {
 330   git update-index --refresh >/dev/null || true
 331   git diff-index --quiet HEAD -- . || ( echo "FAILURE: $*" && exit 1 )
 332 }
 333
 334 [[ "$C89" == "1" ]] && make c89
 335
 336 if [[ "$C89_C99_ROUNDTRIP" == "1" ]]
 337 then
 338   make c89
 339   make c99
 340   _check_git "make c89; make c99"
 341   exit $?
 342 fi
 343
 344 if [[ "$CHECK_SYMBOLS" == "1" ]]
 345 then
 346   make -f makefile.shared
 347   cat << EOF
 348
 349
 350 The following list shows the discrepancy between
 351 the shared library and the Windows dynamic library.
 352 To fix this error, one of the following things
 353 has to be done:
 354 * the script which generates tommath.def has to be modified
 355     (function generate_def() in helper.pl).
 356 * The exported symbols are really different for some reason
 357     This has to be manually investigated.
 358
 359 EOF
 360   exit $(comm -3 <(nm -D --defined-only .libs/libtommath.so | cut -d' ' -f3 | grep -v '^_' | sort) <(tail -n+9 tommath.def | tr -d ' ' | sort) | tee /dev/tty | wc -l)
 361 fi
 362
 363 if [[ "$CHECK_FORMAT" == "1" ]]
 364 then
 365   make astyle
 366   _check_git "make astyle"
 367   perl helper.pl --update-files
 368   _check_git "helper.pl --update-files"
 369   perl helper.pl --check-all
 370   _check_git "helper.pl --check-all"
 371   exit $?
 372 fi
 373
 374 [[ "$VALGRIND_BIN" == "" ]] && VALGRIND_OPTS=""
 375
 376 # default to CC environment variable if no compiler is defined but some other options
 377 if [[ "$COMPILERS" == "" ]] && [[ "$ARCHFLAGS$MAKE_OPTIONS$CFLAGS" != "" ]]
 378 then
 379    COMPILERS="${CC:-cc}"
 380 # default to CC environment variable and run only default config if no option is given
 381 elif [[ "$COMPILERS" == "" ]]
 382 then
 383   _banner "$CC"
 384   if [[ "$VALGRIND_BIN" != "" ]]
 385   then
 386     _runvalgrind "$CC" "" "$LFLAGS"  "$LDFLAGS"
 387   else
 388     _runtest "$CC" ""  "$LFLAGS"  "$LDFLAGS"
 389   fi
 390   _exit
 391 fi
 392
 393
 394 archflags=( $ARCHFLAGS )
 395 compilers=( $COMPILERS )
 396
 397 # choosing a compiler without specifying an architecture will use the default architecture
 398 if [ "${#archflags[@]}" == "0" ]
 399 then
 400   archflags[0]=" "
 401 fi
 402
 403 _banner
 404
 405 if [[ "$TEST_VS_MTEST" != "" ]]
 406 then
 407    make clean > /dev/null
 408    _make "${compilers[0]}" "${archflags[0]} $CFLAGS" "mtest_opponent" "$LFLAGS" "$LDFLAGS"
 409    echo
 410    _make "gcc" "$MTEST_RAND" "mtest" "$LFLAGS" "$LDFLAGS"
 411    echo
 412    echo "Run test vs. mtest for $TEST_VS_MTEST iterations"
 413    _timeout=""
 414    which timeout >/dev/null && _timeout="timeout --foreground 1800"
 415    $_timeout ./mtest/mtest $TEST_VS_MTEST | $VALGRIND_BIN $VALGRIND_OPTS  ./mtest_opponent > valgrind_test.log 2> test_vs_mtest_err.log
 416    retval=$?
 417    head -n 5 valgrind_test.log
 418    tail -n 2 valgrind_test.log
 419    exit $retval
 420 fi
 421
 422 for i in "${compilers[@]}"
 423 do
 424   if [ -z "$(which $i)" ]
 425   then
 426     echo "Skipped compiler $i, file not found"
 427     continue
 428   fi
 429
 430   for a in "${archflags[@]}"
 431   do
 432     if [[ $(expr "$i" : "clang") -ne 0 && "$a" == "-mx32" ]]
 433     then
 434       echo "clang -mx32 tests skipped"
 435       continue
 436     fi
 437     if [[ "$VALGRIND_BIN" != "" ]]
 438     then
 439       _runvalgrind "$i" "$a $CFLAGS" "$LFLAGS" "$LDFLAGS"
 440       [ "$WITH_LOW_MP" != "1" ] && continue
 441       _runvalgrind "$i" "$a -DMP_16BIT $CFLAGS" "$LFLAGS" "$LDFLAGS"
 442       _runvalgrind "$i" "$a -DMP_32BIT $CFLAGS" "$LFLAGS" "$LDFLAGS"
 443     else
 444       _runtest "$i" "$a $CFLAGS" "$LFLAGS" "$LDFLAGS"
 445       [ "$WITH_LOW_MP" != "1" ] && continue
 446       _runtest "$i" "$a -DMP_16BIT $CFLAGS" "$LFLAGS" "$LDFLAGS"
 447       _runtest "$i" "$a -DMP_32BIT $CFLAGS" "$LFLAGS" "$LDFLAGS"
 448     fi
 449   done
 450 done
 451
 452 _exit