build-aux/check.mk

   1 ## Vaucanson, a generic library for finite state machines.
   2 ## Copyright (C) 2006, 2007 The Vaucanson Group.
   3 ##
   4 ## This program is free software; you can redistribute it and/or
   5 ## modify it under the terms of the GNU General Public License
   6 ## as published by the Free Software Foundation; either version 2
   7 ## of the License, or (at your option) any later version.
   8 ##
   9 ## The complete GNU General Public Licence Notice can be found as the
  10 ## `COPYING' file in the root directory.
  11
  12 ## Override the definition from Automake to generate a log file with
  13 ## failed tests.  It also supports parallel make checks.
  14 ##
  15 ## This file provides special support for "unit tests", that is to
  16 ## say, tests that (once run) no longer need to be re-compiled and
  17 ## re-run at each "make check", unless their sources changed.  To
  18 ## enable unit-test supports, define LAZY_TEST_SUITE.  In such a
  19 ## setting, that heavily relies on correct dependencies, its users may
  20 ## prefer to define EXTRA_PROGRAMS instead of check_PROGRAMS, because
  21 ## it allows intertwined compilation and execution of the tests.
  22 ## Sometimes this helps catching errors earlier (you don't have to
  23 ## wait for all the tests to be compiled).
  24 ##
  25 ## Define TEST_SUITE_LOG to be the name of the global log to create.
  26 ## Define TEST_LOGS to the set of logs to include in it.  One possibility
  27 ## is $(TESTS:.test=.log).
  28 ##
  29 ## In addition to the magic "exit 77 means SKIP" feature (which was
  30 ## imported from automake), there is a magic "exit 177 means FAIL" feature
  31 ## which is useful if you need to issue a hard error no matter whether the
  32 ## test is XFAIL or not.
  33
  34 # Use a POSIX-compatible shell if available, as this file uses
  35 # features of the POSIX shell that are not supported by some standard
  36 # shell implementations (e.g., Solaris 10 /bin/sh).
  37 SHELL = $(PREFERABLY_POSIX_SHELL)
  38
  39 # Set this to `false' to disable hard errors.
  40 ENABLE_HARD_ERRORS = :
  41
  42 ## We use GNU Make extensions (%-rules) inside GNU_MAKE checks,
  43 ## and we override check-TESTS.
  44 AUTOMAKE_OPTIONS += -Wno-portability -Wno-override
  45
  46 # Restructured Text title and section.
  47 am__rst_title   = sed 's/.*/   &   /;h;s/./=/g;p;x;p;g;p;s/.*//'
  48 am__rst_section = sed 'p;s/./=/g;p;g'
  49
  50 # Put stdin (possibly several lines separated by ".  ") in a box.
  51 am__text_box = $(AWK) '{gsub ("\\.  ", "\n"); print $$0; }' |   \
  52 $(AWK) '                                                        \
  53 max < length($$0) {                                             \
  54   final= final (final ? "\n" : "") " " $$0;                     \
  55   max = length($$0);                                            \
  56 }                                                               \
  57 END {                                                           \
  58   for (i = 0; i < max + 2 ; ++i)                                \
  59     line = line "=";                                            \
  60   print line;                                                   \
  61   print final;                                                  \
  62   print line;                                                   \
  63 }'
  64
  65 # If stdout is a tty and TERM is smart then use colors.  If test -t or
  66 # tput are not supported then this fails; a conservative approach.  Of
  67 # course do not redirect stdout here, just stderr...
  68 am__tty_colors =                                \
  69 red=;                                           \
  70 grn=;                                           \
  71 lgn=;                                           \
  72 blu=;                                           \
  73 std=;                                           \
  74 test "X$$TERM" != Xdumb &&                      \
  75 test -t 1 2>/dev/null &&                        \
  76 tput bold 1 >/dev/null 2>&1 &&                  \
  77 tput setaf 1 >/dev/null 2>&1 &&                 \
  78 tput sgr0 >/dev/null 2>&1 &&                    \
  79 {                                               \
  80     red=$$(tput setaf 1);                       \
  81     grn=$$(tput setaf 2);                       \
  82     lgn=$$(tput bold)$$(tput setaf 2);          \
  83     blu=$$(tput setaf 4);                       \
  84     std=$$(tput sgr0);                          \
  85 }
  86
  87 # Solaris 10 'make', and several other traditional 'make' implementations,
  88 # pass "-e" to $(SHELL).  This contradicts POSIX.  Work around the problem
  89 # by disabling -e (using the XSI extension "set +e") if it's set.
  90 SH_E_WORKAROUND = case $$- in *e*) set +e;; esac
  91
  92 # Emulate dirname with sed.
  93 _d_no_slash       = s,^[^/]*$$,.,
  94 _d_strip_trailing = s,\([^/]\)//*$$,\1,
  95 _d_abs_trivial    = s,^//*[^/]*$$,/,
  96 _d_rm_basename    = s,\([^/]\)//*[^/]*$$,\1,
  97 _dirname = \
  98   sed '$(_d_no_slash);$(_d_strip_trailing);$(_d_abs_trivial);$(_d_rm_basename)'
  99
 100 # To be inserted before the command running the test.  Creates the
 101 # directory for the log if needed.  Stores in $dir the directory
 102 # containing $src, and passes TESTS_ENVIRONMENT.
 103 # Save and restore TERM around use of TESTS_ENVIRONMENT,
 104 # in case that unsets it.
 105 am__check_pre =                                 \
 106 $(SH_E_WORKAROUND);                             \
 107 tst=`echo "$$src" | sed 's|^.*/||'`;            \
 108 rm -f $@-t;                                     \
 109 trap 'st=$$?; rm -f '\''$(abs_builddir)/$@-t'\''; (exit $$st); exit $$st' \
 110   1 2 13 15;                                    \
 111 $(mkdir_p) "$$(echo '$@'|$(_dirname))" || exit; \
 112 if test -f "./$$src"; then dir=./;              \
 113 elif test -f "$$src"; then dir=;                \
 114 else dir="$(srcdir)/"; fi;                      \
 115 __SAVED_TERM=$$TERM;                            \
 116 $(TESTS_ENVIRONMENT)
 117
 118 # To be appended to the command running the test.  Handles the stdout
 119 # and stderr redirection, and catch the exit status.
 120 am__check_post =                                        \
 121 >$@-t 2>&1;                                             \
 122 estatus=$$?;                                            \
 123 if test $$estatus -eq 177; then                         \
 124   $(ENABLE_HARD_ERRORS) || estatus=1;                   \
 125 fi;                                                     \
 126 TERM=$$__SAVED_TERM; export TERM;                       \
 127 $(am__tty_colors);                                      \
 128 xfailed=PASS;                                           \
 129 for xfail in : $(XFAIL_TESTS); do                       \
 130   case $$src in                                         \
 131     $$xfail | */$$xfail) xfailed=XFAIL; break;          \
 132   esac;                                                 \
 133 done;                                                   \
 134 case $$estatus:$$xfailed in                             \
 135     0:XFAIL) col=$$red; res=XPASS;;                     \
 136     0:*)     col=$$grn; res=PASS ;;                     \
 137     77:*)    col=$$blu; res=SKIP ;;                     \
 138     177:*)   col=$$red; res=FAIL ;;                     \
 139     *:XFAIL) col=$$lgn; res=XFAIL;;                     \
 140     *:*)     col=$$red; res=FAIL ;;                     \
 141 esac;                                                   \
 142 echo "$${col}$$res$${std}: $@";                         \
 143 echo "$$res: $@ (exit: $$estatus)" |                    \
 144   $(am__rst_section) >$@;                               \
 145 cat $@-t >>$@;                                          \
 146 rm $@-t
 147
 148 SUFFIXES += .html .log
 149
 150 # From a test (with no extension) to a log file.
 151 if GNU_MAKE
 152 %.log: %
 153         @src='$<'; $(am__check_pre) "$$dir$$src" $(am__check_post)
 154 else
 155 # With POSIX 'make', inference rules cannot have FOO.log depend on FOO.
 156 # Work around the problem by calculating the dependency dynamically, and
 157 # then invoking a submake with the calculated dependency.
 158 CHECK-FORCE:
 159 DEPENDENCY = CHECK-FORCE
 160 $(TEST_LOGS): $(DEPENDENCY)
 161         @if test '$(DEPENDENCY)' = CHECK-FORCE; then                    \
 162           dst=$@; src=$${dst%.log};                                     \
 163           test -x "$$src" || src='$(srcdir)'/$$src;                     \
 164           exec $(MAKE) $(AM_MAKEFLAGS) DEPENDENCY="$$src" $@;           \
 165         else                                                            \
 166           src='$(DEPENDENCY)';                                          \
 167           $(am__check_pre) "$$dir$$src" $(am__check_post);              \
 168         fi
 169 endif
 170
 171 #TEST_LOGS = $(TESTS:.test=.log)
 172 TEST_SUITE_LOG = test-suite.log
 173
 174 $(TEST_SUITE_LOG): $(TEST_LOGS)
 175         @$(SH_E_WORKAROUND);                                            \
 176         results=$$(for f in $(TEST_LOGS); do sed 1q $$f; done);         \
 177         all=$$(echo "$$results" | wc -l | sed -e 's/^[ \t]*//');        \
 178         fail=$$(echo "$$results" | grep -c '^FAIL');                    \
 179         pass=$$(echo "$$results" | grep -c '^PASS');                    \
 180         skip=$$(echo "$$results" | grep -c '^SKIP');                    \
 181         xfail=$$(echo "$$results" | grep -c '^XFAIL');                  \
 182         xpass=$$(echo "$$results" | grep -c '^XPASS');                  \
 183         failures=$$(expr $$fail + $$xpass);                             \
 184         case fail=$$fail:xpass=$$xpass:xfail=$$xfail in                 \
 185           fail=0:xpass=0:xfail=0)                                       \
 186             msg="All $$all tests passed.  ";                            \
 187             exit=true;;                                                 \
 188           fail=0:xpass=0:xfail=*)                                       \
 189             msg="All $$all tests behaved as expected";                  \
 190             msg="$$msg ($$xfail expected failures).  ";                 \
 191             exit=true;;                                                 \
 192           fail=*:xpass=0:xfail=*)                                       \
 193             msg="$$fail of $$all tests failed.  ";                      \
 194             exit=false;;                                                \
 195           fail=*:xpass=*:xfail=*)                                       \
 196             msg="$$failures of $$all tests did not behave as expected"; \
 197             msg="$$msg ($$xpass unexpected passes).  ";                 \
 198             exit=false;;                                                \
 199           *)                                                            \
 200             echo >&2 "incorrect case"; exit 4;;                         \
 201         esac;                                                           \
 202         if test "$$skip" -ne 0; then                                    \
 203           msg="$$msg($$skip tests were not run).  ";                    \
 204         fi;                                                             \
 205         if test "$$failures" -ne 0; then                                \
 206           {                                                             \
 207             echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" |     \
 208               $(am__rst_title);                                         \
 209             echo "$$msg";                                               \
 210             echo;                                                       \
 211             echo ".. contents:: :depth: 2";                             \
 212             echo;                                                       \
 213             for f in $(TEST_LOGS);                                      \
 214             do                                                          \
 215               case $$(sed 1q $$f) in                                    \
 216                 SKIP:*|PASS:*|XFAIL:*);;                                \
 217                 *) echo; cat $$f;;                                      \
 218               esac;                                                     \
 219             done;                                                       \
 220           } >$(TEST_SUITE_LOG).tmp;                                     \
 221           mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG);                   \
 222           msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG).  ";              \
 223           if test -n "$(PACKAGE_BUGREPORT)"; then                       \
 224             msg="$${msg}Please report it to $(PACKAGE_BUGREPORT).  ";   \
 225           fi;                                                           \
 226         fi;                                                             \
 227         $(am__tty_colors);                                              \
 228         if $$exit; then echo $$grn; else echo $$red; fi;                \
 229           echo "$$msg" | $(am__text_box);                               \
 230         echo $$std;                                                     \
 231         test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG);       \
 232         $$exit
 233
 234 # if test x"$$VERBOSE" != x && ! $exit; then
 235
 236 # Run all the tests.
 237 check-TESTS:
 238         @if test -z '$(LAZY_TEST_SUITE)'; then  \
 239           rm -f $(TEST_SUITE_LOG) $(TEST_LOGS); \
 240         fi
 241         @$(MAKE) $(TEST_SUITE_LOG)
 242
 243
 244 ## -------------- ##
 245 ## Produce HTML.  ##
 246 ## -------------- ##
 247
 248 TEST_SUITE_HTML = $(TEST_SUITE_LOG:.log=.html)
 249
 250 .log.html:
 251         @for r2h in $(RST2HTML) $$RST2HTML rst2html rst2html.py;        \
 252         do                                                              \
 253           if ($$r2h --version) >/dev/null 2>&1; then                    \
 254             R2H=$$r2h;                                                  \
 255           fi;                                                           \
 256         done;                                                           \
 257         if test -z "$$R2H"; then                                        \
 258           echo >&2 "cannot find rst2html, cannot create $@";            \
 259           exit 2;                                                       \
 260         fi;                                                             \
 261         $$R2H $< >$@.tmp
 262         @mv $@.tmp $@
 263
 264 # Be sure to run check-TESTS first, and then to convert the result.
 265 # Beware of concurrent executions.  And expect check-TESTS to fail.
 266 check-html:
 267         @if $(MAKE) $(AM_MAKEFLAGS) check-TESTS; then :; else   \
 268           rv=$$?;                                               \
 269           $(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_HTML);           \
 270           exit $$rv;                                            \
 271         fi
 272
 273 .PHONY: check-html
 274
 275
 276 ## ------- ##
 277 ## Clean.  ##
 278 ## ------- ##
 279
 280 check-clean:
 281         rm -f $(CHECK_CLEANFILES) $(TEST_SUITE_LOG) $(TEST_SUITE_HTML) $(TEST_LOGS)
 282 .PHONY: check-clean
 283 clean-local: check-clean