sdcc/support/regression/Makefile.in

   1 # Test suite Makefile
   2 # Part of SDCC - http://sdcc.sourceforge.net/
   3 # Michael Hope <michaelh@juju.net.nz> 2001
   4 # refactored: Felix Salfelder 2022
   5 #
   6 # This Makefile builds and runs the test suites under tests/ for each
   7 # one of the supported ports located under ports/.  The test suite
   8 # results are summarised and individual test failures are logged.  The
   9 # expected result is a single line per port summarising the number of
  10 # failures, test points, and test cases.  The philosophy is that
  11 # checked in code should always pass the suite with no failures, as
  12 # then if there are failures then it is in the current developers code.
  13 #
  14 # Only the required suites are run.  Changing sdcc causes all to be
  15 # re-run.  Changing one suite causes just that to be run.  Changing
  16 # one of the library files should cause all to re-run
  17 #
  18 # Test individual ports using the command
  19 # $ make test-$port
  20 # For port=rrz80 port=rrgbz80, tests will use rrz80 and rrgbz80 simulators
  21 # respectively.
  22 # mos6502 uses sim65.
  23 # pic1{4,6} uses gpsim.
  24 # port=host does not simulate.
  25 # The others use uCsim.
  26
  27 # Dependencies:
  28 #   * The sdcc-extra package, available from svn.
  29 #       o svn co https://sdcc.svn.sourceforge.net/svnroot/sdcc/trunk/sdcc-extra
  30 #       o Provides the emulators
  31 #   * The gbdk-lib package from gbdk.
  32 #       o cvs -d cvs.gbdk.sourceforge.net:/cvsroot/gbdk co gbdk-lib
  33 #       o Provides mul, div, and include files for the z80 tests.
  34 #   * python 1.5 or above
  35 #   * uCsim for the mcs51 port
  36 #
  37 # The paths below assume that sdcc, sdcc-extra, and gbdk-lib all reside in
  38 # the same directory.
  39
  40 # Old notes:
  41 # Starting at the bottom
  42 # Set of source test suites
  43 # Each source suite is processed producing multiple device specific test suites.
  44 # Each device specific test suite is compiled.
  45 # Each device specific test suite is run, and the output recorded.
  46 # The output from each device specific test suite derived from a source
  47 # test suite are collated.
  48
  49 # set V=0 for normal verbosity
  50 # set V=1 for more verbosity
  51 V ?= silent
  52 ifeq (${V},silent)
  53 MAKEFLAGS += --silent
  54 endif
  55
  56 # TEST_PREFIX: select tests matching a pattern
  57 # All tests (default)
  58 TEST_PREFIX =
  59 #
  60 # only run m4 test
  61 # TEST_PREFIX = m4_
  62
  63 # TEST_SLICE. Subselect tests from list of all tests.
  64 # This is Python syntax for slicing, start:stop:step
  65 #
  66 # All tests, 0 1 2... (default)
  67 TEST_SLICE = 0::1
  68 #
  69 # Select first 10
  70 # TEST_SLICE = 0:10
  71 #
  72 # Select 2 9 16...
  73 # TEST_SLICE = 2::7
  74
  75 # delete failed attempts (so make will retry...)
  76 .DELETE_ON_ERROR:
  77
  78 CC = @CC@
  79 CFLAGS = @CFLAGS@
  80 CPPFLAGS = @CPPFLAGS@ -DNO_VARARGS
  81 EXEEXT = @EXEEXT@
  82 PYTHON = @PYTHON@
  83 M4 = @M4@
  84
  85 MAKE_SUB_FLAGS = --no-print-directory
  86
  87 srcdir       = @srcdir@
  88 top_srcdir   = @top_srcdir@
  89 top_builddir = @top_builddir@
  90
  91 DEFAULT_VERBOSITY = silent
  92 M_V_at = $(m__v_at_$(V))
  93 m__v_at_ = $(m__v_at_$(DEFAULT_VERBOSITY))
  94 m__v_at_silent = @
  95 m__v_at_0 = @
  96 m__v_at_1 =
  97
  98 M_V_GEN = $(m__v_GEN_$(V))
  99 m__v_GEN_ = $(m__v_GEN_$(DEFAULT_VERBOSITY))
 100 m__v_GEN_silent = @
 101 m__v_GEN_0 = @echo "  GEN     " $@;
 102 m__v_GEN_1 =
 103
 104 M_V_EMU = $(m__v_EMU_$(V))
 105 m__v_EMU_silent =
 106 m__v_EMU_ = $(m__v_EMU_$(DEFAULT_VERBOSITY))
 107 m__v_EMU_0 = echo "  EMU     " $@;
 108 m__v_EMU_1 =
 109
 110 M_V_echo = $(m__v_echo_$(V))
 111 m__v_echo_ = $(m__v_EMU_$(DEFAULT_VERBOSITY))
 112 m__v_echo_silent = echo > /dev/null
 113 m__v_echo_0 = echo > /dev/null
 114 m__v_echo_1 = echo
 115
 116 M_V_CC = $(m__v_CC_$(V))
 117 m__v_CC_ = $(m__v_CC_$(DEFAULT_VERBOSITY))
 118 m__v_CC_silent = @
 119 m__v_CC_0 = @echo "  CC      " $@;
 120 m__v_CC_1 =
 121
 122 M_V_CCLD = $(m__v_CCLD_$(V))
 123 m__v_CCLD_ = $(m__v_CCLD_$(DEFAULT_VERBOSITY))
 124 m__v_CCLD_silent = @
 125 m__v_CCLD_0 = @echo "  CCLD    " $@;
 126 m__v_CCLD_1 =
 127
 128 # All original tests live in TESTS_DIR and below
 129 TESTS_DIR = $(srcdir)/tests
 130 # All suite results go in RESULTS_DIR
 131 RESULTS_DIR = results
 132 # All data relating to supported ports live in their own directory
 133 # under PORTS_DIR.
 134 PORTS_DIR = $(srcdir)/ports
 135
 136 # Intermediate data directories
 137 # Directory where test cases and the generated object code go.
 138 CASES_DIR = cases
 139 TMP_DIR = gen
 140
 141 # The directories in PORTS_DIR correspond to ports.
 142 PORTS = $(filter-out .svn, $(notdir $(wildcard $(PORTS_DIR)/*)))
 143
 144 # not a port.
 145 EXCLUDE_PORTS = \
 146         mcs51-common
 147
 148 # unstable
 149 EXCLUDE_PORTS += \
 150         mcs51-xstack-auto \
 151         uc6502-stack-auto
 152
 153 # not enough ram
 154 EXCLUDE_PORTS += \
 155         pdk13
 156
 157 # simulator may be unavailable
 158 EXCLUDE_PORTS += \
 159         mos6502 \
 160         mos6502-stack-auto
 161
 162 # unstable
 163 EXCLUDE_PORTS += \
 164         pic14 \
 165         pic16
 166
 167 # redundant, uCSim works better.
 168 EXCLUDE_PORTS += \
 169         rrz80 \
 170         rrgbz80
 171
 172 # Generate the list of ports to test. Each port has a spec.mk with build and
 173 # emulator flags. Some ports are tested by default, these are the tests
 174 # associated to the "all" target.
 175 ALL_PORTS = $(filter-out ${EXCLUDE_PORTS}, $(PORTS))
 176
 177 # These  ports will be cleaned with 'make clean'
 178 CLEAN_PORTS = $(filter-out mcs51-common host, $(PORTS))
 179
 180 MAKE_LIBRARY =
 181
 182 # support VPATH:
 183 vpath %.c $(CASES_DIR)
 184 vpath %.c.in $(TESTS_DIR)
 185
 186 all: print-summary
 187
 188 print-summary: test-ports
 189         @cat $(RESULTS_DIR)/*.sum
 190
 191 .PHONY: print-summary
 192
 193 # Test all ports that you normally would want to test,
 194 # normally a subset of all ports.
 195 test-ports:
 196         $(MAKE) $(MAKE_SUB_FLAGS) test-common
 197         for i in $(ALL_PORTS); \
 198           do $(MAKE) $(MAKE_SUB_FLAGS) make_library test-port PORT=$$i; \
 199         done
 200
 201 # Helper rule for testing various ports
 202 ${PORTS:%=test-%} : test-%: test-common
 203         $(MAKE) test-port PORT=$*
 204         cat $(RESULTS_DIR)/$*.sum
 205
 206 # Shorthand for testing the mcs51 port only
 207 TEST_MCS51 = \
 208         test-mcs51-small \
 209         test-mcs51-medium \
 210         test-mcs51-large \
 211         test-mcs51-huge \
 212         test-mcs51-small-stack-auto \
 213         test-mcs51-large-stack-auto \
 214         test-mcs51-xstack-auto
 215 test-mcs51: $(filter-out ${EXCLUDE_PORTS:%=test-%}, $(TEST_MCS51))
 216
 217 # Shorthand for testing the z80 related ports only
 218 test-z80: test-ucz80 test-ucz80-undoc test-ucz80-resiy test-ucz180 test-ucz180-resiy test-ucz80n test-ucgbz80 test-ucr2k test-ucr2ka test-ucr3ka test-ez80-z80 test-tlcs90 test-ucr800
 219
 220 # Shorthand for testing the pdk ports only
 221 test-pdk: test-pdk14 test-pdk15 test-pdk15-stack-auto
 222
 223 # what is this?
 224 test-host2:
 225         $(MAKE) $(MAKE_SUB_FLAGS) test-common
 226         $(MAKE) $(MAKE_SUB_FLAGS) test-port PORT=host
 227
 228 # generate test list (and possibly expand tests)
 229 cases/MakeList:
 230         $(MAKE) -C cases MakeList
 231
 232 ifdef PORT
 233 include cases/MakeList
 234 endif
 235
 236 # Intermediate directory
 237 PORT_TMP_DIR = $(TMP_DIR)/$(PORT)
 238 # BUG. used in spec.mk
 239 PORT_CASES_DIR = $(TMP_DIR)/$(PORT)
 240 # PORT_CASES_DIRR = $(TMP_DIR)/$(PORT)
 241 PORT_RESULTS_DIR = $(RESULTS_DIR)/$(PORT)
 242
 243 ##### per-port exclude (cases, not tests) #####
 244 ## BUG/Feature: after excluding tests, manually delete .sum
 245 include $(srcdir)/MakeList
 246
 247 SDCC_EXTRA_DIR = $(top_builddir)/../sdcc-extra
 248
 249 # Defaults.  Override in spec.mk if required.
 250 # Simulation timeout in seconds.
 251 SIM_TIMEOUT = 120
 252 PORT_BASE = $(PORT)
 253 # Path to SDCC
 254 ifdef SDCC_BIN_PATH
 255   SDCC = export PATH=$(SDCC_BIN_PATH); sdcc$(EXEEXT)
 256 else
 257   SDCC = $(top_builddir)/bin/sdcc$(EXEEXT)
 258   INC_DIR ?= $(top_srcdir)/device/include
 259 endif
 260 # Base flags.
 261 SDCCFLAGS += --fverbose-asm -DNO_VARARGS
 262 # Extension of object intermediate files
 263 OBJEXT = .rel
 264 # Extension of files that can be run in the emulator
 265 BINEXT = .bin
 266 # Currently unused.  Extension to append to intermediate directories.
 267 DIREXT =
 268
 269 CC_FOR_BUILD = $(CC)
 270
 271 # Only include if we're in a per-port call.
 272 ifdef PORT
 273
 274 EMU_INPUT = < $(PORTS_DIR)/$(PORT_BASE)/uCsim.cmd
 275
 276 # run simulator with SIM_TIMEOUT seconds timeout
 277   # include ./$(PORT)/spec.mk if exists, else include $(PORTS_DIR)/$(PORT)/spec.mk
 278   ifeq ($(shell if test -f ./ports/$(PORT)/spec.mk; then echo OK; fi),OK)
 279     include ./ports/$(PORT)/spec.mk
 280   else
 281     include $(PORTS_DIR)/$(PORT)/spec.mk
 282   endif
 283
 284 $(RESULTS_DIR)/$(PORT)/%.out: $(CASES_DIR)/%.c $(TMP_DIR)/timeout
 285         @mkdir -p $(dir $@)
 286         @# BUG: need "Running" in .out, regardless of whether we run anything.
 287         @echo --- Running: $< > $@;
 288
 289         @# BUG: does not respect --dry-run
 290         -${M_V_at}if $(MAKE) $(MAKE_SUB_FLAGS) $(TMP_DIR)/$(PORT)/$*$(BINEXT) PORT=$(PORT); then \
 291           ${M_V_echo} "$(TMP_DIR)/timeout $(SIM_TIMEOUT) $(EMU) $(EMU_PORT_FLAG) $(EMU_FLAGS) $(TMP_DIR)/$(PORT)/$*$(BINEXT) ${EMU_INPUT} > $@"; \
 292           ${M_V_EMU}$(TMP_DIR)/timeout $(SIM_TIMEOUT) $(EMU) $(EMU_PORT_FLAG) $(EMU_FLAGS) $(TMP_DIR)/$(PORT)/$*$(BINEXT) ${EMU_INPUT} > $@; \
 293           if [ 0 -ne $$? ]; then \
 294             echo --- FAIL: \"timeout, simulation killed\" in $< >> $@; \
 295             echo --- Summary: 1/1/1: timeout >> $@; \
 296           fi \
 297         else \
 298           echo --- FAIL: cannot compile/link $< >> $@; \
 299           echo --- Summary: 1/1/1: compile >> $@; \
 300         fi
 301
 302         @# always count ticks?
 303         @$(PYTHON) $(srcdir)/get_ticks.py < $@ >> $@
 304         @grep -n FAIL $@ /dev/null | head -n 20 || true
 305
 306 endif # PORT
 307
 308 # For each test we produce a result log file, %.log
 309 ifndef CASES
 310 TEST_STEMS_FILTERED_ = $(filter-out ${EXCLUDE_${PORT}}, $(TEST_STEMS))
 311 TEST_STEMS_FILTERED__ = $(filter ${TEST_PREFIX}%, $(TEST_STEMS_FILTERED_))
 312 TEST_STEMS_FILTERED = $(shell ${PYTHON} -c 'for i in "${TEST_STEMS_FILTERED__}".split(" ")[${TEST_SLICE}]: print(i)')
 313 PORT_RESULTS = $(TEST_STEMS_FILTERED:%=$(PORT_RESULTS_DIR)/%.log)
 314 endif
 315
 316 make_library: $(MAKE_LIBRARY)
 317
 318 SDCCFLAGS += -I$(srcdir)/fwk/include -I$(srcdir)/tests
 319 ifdef INC_DIR
 320   SDCCFLAGS += -I$(INC_DIR)
 321 endif
 322
 323 # List of intermediate files to keep.  Pretty much keep everything as
 324 # disk space is free.
 325 .SECONDARY: $(RESULTS_DIR) $(CASES_DIR)/% $(PORT_TMP_DIR)/% %$(OBJEXT) %$(EXEEXT) %$(BINEXT) \
 326         $(CASES_DIR)/%/stamp \
 327         ${TEST_STEMS:%=$(RESULTS_DIR)/$(PORT)/%.out}
 328
 329 # $(TMP_DIR)/stamp is really a proxy for $(TMP_DIR). We shouldn't use
 330 # $(TMP_DIR) as a dependency directly, because as a directory its time
 331 # stamp updates every time something in the directory is updated (which
 332 # can then cause spurious rebuilds of other targets that can conflict
 333 # with parallel make jobs)
 334 $(TMP_DIR)/stamp:
 335         mkdir -p $(TMP_DIR)
 336         touch $@
 337
 338 # Rule linking the combined results log to all of the files in the
 339 # iteration directory.
 340 $(PORT_RESULTS_DIR)/%.out: $(CASES_DIR)/%/stamp
 341         @ # mkdir -p $(dir $@)/$*
 342         ${M_V_at}$(MAKE) $(MAKE_SUB_FLAGS) iterations PORT=$(PORT) CASES=$*/
 343
 344 $(PORT_RESULTS_DIR)/%.log: $(PORT_RESULTS_DIR)/%.out
 345         @# compact-results prints the stats. Do once per test.
 346         ${M_V_GEN}$(PYTHON) $(srcdir)/compact-results.py $* < $< | tee $@
 347
 348 # Rule to summarise the results for one port after all of the tests
 349 # have been run.
 350 $(RESULTS_DIR)/$(PORT).sum: $(PORT_RESULTS) cases/MakeList
 351         ${M_V_at}echo | cat - $(PORT_RESULTS:%.log=%.out) | $(PYTHON) $(srcdir)/collate-results.py $(PORT) > $@
 352
 353 port-fwklib: port-dirs $(EXTRAS) $(FWKLIB)
 354
 355 port-dirs:
 356         ${M_V_GEN}mkdir -p $(PORT_TMP_DIR) $(PORT_RESULTS_DIR)
 357
 358 # Files shared between all ports need to be built by the test-common target,
 359 # which should always be built before the port specific targets.
 360 test-common: $(TMP_DIR)/timeout
 361         $(MAKE) -C cases
 362
 363 ifdef PORT
 364 test-port: test-common port-dirs
 365         @# recurse: force vpath to re-read the $(CASES_DIR)
 366         ${M_V_at}$(MAKE) $(MAKE_SUB_FLAGS) port-fwklib PORT=$(PORT) V=${V}
 367         @echo Running ${PORT} regression tests
 368         ${M_V_at}$(MAKE) $(MAKE_SUB_FLAGS) $(RESULTS_DIR)/$(PORT).sum PORT=$(PORT)
 369 endif
 370
 371 # Begin rules that process each iteration generated from the source
 372 # test
 373
 374 # List of all of the generated iteration source files.
 375 SUB_CASES = $(sort $(wildcard $(CASES_DIR)/$(CASES)*.c))
 376 # List of all the sub result logs generated from the iterations.
 377 # these are stored in TMP_DIR...
 378 SUB_RESULTS_ = $(SUB_CASES:%.c=%.out)
 379 SUB_RESULTS = $(SUB_RESULTS_:cases%=$(RESULTS_DIR)/$(PORT)%)
 380 SUB_RESULTS_FILES = $(SUB_RESULTS_:cases/%=%)
 381 SUB_RESULTS_DIR = $(RESULTS_DIR)/$(PORT)
 382 # Overall target.  Concatenation of all of the sub results.
 383 RESULTS = $(CASES:%/$(DIREXT)=$(PORT_RESULTS_DIR)/%.out)
 384
 385 iterations: $(RESULTS)
 386
 387 # Rule to generate the overall target from the sub results.
 388 $(RESULTS): $(SUB_RESULTS)
 389         ${M_V_at}( cd $(SUB_RESULTS_DIR); echo | cat - $(SUB_RESULTS_FILES) ) > $@ # subresult rule
 390
 391 $(TMP_DIR)/timeout: $(srcdir)/fwk/lib/timeout.c $(TMP_DIR)/stamp
 392         $(CC_FOR_BUILD) $(CFLAGS) $< -o $@
 393
 394 # The remainder of the rules are in $PORT/spec.mk.  The port needs to
 395 # be able to turn an iterated test suite into a sub result, normally
 396 # by:
 397 #    1. Compile the required library files
 398 #    2. Compile this test suite.
 399 #    3. Link 1, 2, and any required stdlib into an executable.
 400 #    4. Run the executable inside an emulator, and capture the text
 401 #    output into %.out.
 402 #
 403 # The emulator must exit when main() returns.
 404
 405 # BeginGeneric rules
 406
 407 clean: clean-results
 408         $(MAKE) -C $(CASES_DIR) clean
 409
 410 clean-results:
 411         rm -rf $(TMP_DIR) $(RESULTS_DIR) *.pyc __pycache__
 412         for i in $(CLEAN_PORTS); do \
 413           $(MAKE) $(MAKE_RECURSE_FLAGS) -f $(PORTS_DIR)/$$i/spec.mk _clean PORTS_DIR=$(PORTS_DIR) PORT=$$i srcdir=$(srcdir); \
 414         done
 415
 416 distclean: clean
 417         $(MAKE) -C $(CASES_DIR) distclean
 418         rm -f $(PORTS_DIR)/host/spec.mk
 419         rm -f Makefile
 420
 421 # Rule to link into .ihx
 422 %$(BINEXT): %$(OBJEXT) $(EXTRAS) $(FWKLIB) $(PORT_CASES_DIR)/fwk.lib
 423         ${M_V_CCLD}$(SDCC) $(SDCCFLAGS) $(LINKFLAGS) $(EXTRAS) $(PORT_CASES_DIR)/fwk.lib $< -o $@
 424
 425 # ignore (preprocessor) warnings
 426 gen/%/tst_p99-conformance.rel: EXTRA_COMPILE_HACK=2>/dev/null
 427
 428 $(PORT_CASES_DIR)/%$(OBJEXT): %.c
 429         @mkdir -p $(dir $@) # BUG?
 430         ${M_V_CC}$(SDCC) $(SDCCFLAGS) -c $< -o $@ ${EXTRA_COMPILE_HACK}
 431
 432 $(PORT_CASES_DIR)/%$(OBJEXT): $(PORTS_DIR)/$(PORT_BASE)/%.c
 433         ${M_V_CC}$(SDCC) $(SDCCFLAGS) -c $< -o $@
 434
 435 $(PORT_CASES_DIR)/%$(OBJEXT): $(srcdir)/fwk/lib/%.c
 436         ${M_V_CC}$(SDCC) $(SDCCFLAGS) -c $< -o $@
 437
 438 $(PORT_CASES_DIR)/fwk.lib: $(srcdir)/fwk/lib/fwk.lib ${SPEC_LIB}
 439         ${M_V_GEN}cat $(srcdir)/fwk/lib/fwk.lib ${SPEC_LIB} > $@
 440
 441 Makefile: $(srcdir)/Makefile.in
 442          cd $(top_builddir); ./config.status support/regression/Makefile
 443