util/testing/Makefile.mk

   1 ## SPDX-License-Identifier: GPL-2.0-only
   2
   3 test-help help::
   4         @echo  '*** coreboot test targets ***'
   5         @echo  '  what-jenkins-does    - Run platform build tests with junit output'
   6         @echo  '                         Skip tests by setting env vars JENKINS_SKIP_x_TESTS'
   7         @echo  '                         to 'y' where x is: CLANG, GCC, LINT, TOOL, or UNIT'
   8         @echo  '  lint / lint-stable   - Run coreboot lint tools (all / minimal)'
   9         @echo  '  test-basic           - Run standard build tests. All expected to pass.'
  10         @echo  '  test-lint            - Basic: Run stable and extended lint tests.'
  11         @echo  '  test-tools           - Basic: Tests a basic list of tools.'
  12         @echo  '  test-abuild          - Basic: Builds all platforms'
  13         @echo  '  test-payloads        - Basic: Builds internal payloads'
  14         @echo  '  test-cleanup         - Basic: Cleans coreboot directories'
  15         @echo
  16
  17 # junit.xml is a helper target to wrap builds that don't create junit.xml output
  18 # BLD = The name of the build
  19 # BLD_DIR = Top path from coreboot to the build subdirectory
  20 # MAKETARGET = target to build
  21 junit.xml:
  22         echo "Building $(BLD)"
  23         echo '<?xml version="1.0" encoding="utf-8"?><testsuite>' > $@.tmp
  24         echo "<testcase classname='$(BLD)' name='$(BLD)'>" >> $@.tmp
  25         -if [ -z "$(BLD_DIR)" ]; then                                          \
  26                 projdir="";                                                    \
  27                 xmlfile=junit_$(BLD).xml;                                      \
  28         else                                                                   \
  29                 projdir="-C $(BLD_DIR)";                                       \
  30                 xmlfile="$(BLD_DIR)/$@";                                       \
  31         fi;                                                                    \
  32         if [ -z "$(SKIP_DISTCLEAN)" ]; then                                    \
  33                 $(MAKE) -j $(CPUS) $$projdir clean distclean > $@.tmp.2 2>&1 ; \
  34         fi;                                                                    \
  35         if $(MAKE) -j $(CPUS) $$projdir $(MAKETARGET) >> $@.tmp.2 2>&1; then   \
  36                 type="system-out";                                             \
  37                 echo "<$$type>" >> $@.tmp;                                     \
  38                 echo "Building $(BLD) Succeeded";                              \
  39         else                                                                   \
  40                 type="failure";                                                \
  41                 echo "<failure type='buildFailed'>" >> $@.tmp;                 \
  42                 echo "Building $(BLD) Failed";                                 \
  43         fi;                                                                    \
  44         cat $@.tmp.2;                                                          \
  45         echo '<![CDATA[' >> $@.tmp;                                            \
  46         cat $@.tmp.2 >> $@.tmp;                                                \
  47         echo "]]></$$type>" >>$@.tmp;                                          \
  48         rm -f $@.tmp.2;                                                        \
  49         echo "</testcase>" >> $@.tmp;                                          \
  50         echo "</testsuite>" >> $@.tmp;                                         \
  51         mv $@.tmp "$$xmlfile"
  52         echo
  53
  54 TOOLLIST= \
  55 amdfwtool \
  56 cbfstool \
  57 cbmem \
  58 ectool \
  59 futility \
  60 ifdtool \
  61 intelmetool \
  62 inteltool \
  63 intelvbttool \
  64 nvramtool \
  65 post \
  66 superiotool
  67
  68 TEST_PAYLOADLIST_INTERNAL= \
  69 coreinfo \
  70 nvramcui
  71
  72 JENKINS_PAYLOAD?=none
  73 TEST_PAYLOAD?=$(JENKINS_PAYLOAD)
  74 CPUS?=4
  75
  76 lint lint-stable lint-extended:
  77         util/lint/lint $@
  78
  79 ABUILD_OPTIONS=--blobs --junit --cpus $(CPUS) --payloads $(JENKINS_PAYLOAD)
  80 ABUILD_OPTIONS+=$(if $(V),--verbose,)
  81 ABUILD_OPTIONS+=$(JENKINS_ABUILD_OPT)
  82
  83 CCACHE_OPTIONS=$(if $(JENKINS_NOCCACHE),,--ccache)
  84
  85 COREBOOT_BUILD_DIR?=coreboot-builds
  86
  87 validate_sec_tools:
  88         if [ ! -f 3rdparty/intel-sec-tools/go.mod ]; then \
  89                 echo "Please download 3rdparty/intel-sec-tools/:"; \
  90                 echo "git submodule update --init 3rdparty/intel-sec-tools"; \
  91                 exit 1; \
  92         fi
  93
  94 what-jenkins-does: test-cleanup validate_sec_tools
  95 ifneq ($(JENKINS_SKIP_LINT_TESTS),y)
  96         JUNIT=--junit $(MAKE) test-lint
  97 endif
  98         cd 3rdparty/intel-sec-tools/ ; go mod vendor
  99         cd util/goswid ; go mod vendor
 100         $(MAKE) test-abuild
 101 ifneq  ($(JENKINS_SKIP_TOOL_TESTS),y)
 102         $(MAKE) test-tools
 103 endif
 104 ifneq  ($(JENKINS_SKIP_UNIT_TESTS),y)
 105         +$(MAKE) junit.xml-unit-tests COV=1
 106         +(cd payloads/libpayload; unset COREBOOT_BUILD_DIR; $(MAKE) junit.xml-unit-tests COV=1)
 107         +(cd payloads/libpayload; unset COREBOOT_BUILD_DIR; $(MAKE) coverage-report COV=1)
 108         +$(MAKE) coverage-report JUNIT_OUTPUT=y COV=1
 109         find . -name 'tests.info' -exec cat {} + >$(COREBOOT_BUILD_DIR)/coverage.info
 110 endif
 111
 112 test-basic: test-lint test-tools test-abuild test-payloads test-cleanup
 113
 114 test-lint:
 115         util/lint/lint lint-stable $(JUNIT)
 116         util/lint/lint lint-extended $(JUNIT)
 117
 118 test-abuild:
 119 ifneq ($(JENKINS_SKIP_SCANBUILD_TEST),y)
 120         NAME=scanbuild; SCANBUILD_ARGS='-k -plist-html -maxloop 10' util/abuild/abuild -o $(COREBOOT_BUILD_DIR)/$${NAME} $(ABUILD_OPTIONS) --scan-build --target EMULATION_QEMU_X86_Q35 --exitcode --name $${NAME}
 121 endif
 122 ifneq ($(JENKINS_SKIP_GCC_TESTS),y)
 123 ifneq ($(JENKINS_SKIP_ALLTHREAD_TEST),y)
 124         NAME=emu_gcc_allthreads; util/abuild/abuild -o $(COREBOOT_BUILD_DIR)/$${NAME} $(ABUILD_OPTIONS) $(CCACHE_OPTIONS) --name $${NAME} --target EMULATION_QEMU_X86_Q35 -b EMULATION_QEMU_X86_Q35 --exitcode --clean
 125         NAME=amd_gcc_allthreads; util/abuild/abuild -o $(COREBOOT_BUILD_DIR)/$${NAME} $(ABUILD_OPTIONS) $(CCACHE_OPTIONS) --name $${NAME} --target GOOGLE_SKYRIM -b GOOGLE_SKYRIM --exitcode --clean
 126         NAME=intel_gcc_allthreads; util/abuild/abuild -o $(COREBOOT_BUILD_DIR)/$${NAME} $(ABUILD_OPTIONS) $(CCACHE_OPTIONS) --name $${NAME} --target GOOGLE_REX0 -b GOOGLE_REX0 --exitcode --clean
 127         NAME=mediatek_gcc_allthreads; util/abuild/abuild -o $(COREBOOT_BUILD_DIR)/$${NAME} $(ABUILD_OPTIONS) $(CCACHE_OPTIONS) --name $${NAME} --target GOOGLE_CHERRY -b GOOGLE_CHERRY --exitcode --clean
 128 endif
 129         NAME=gcc-chromeos; util/abuild/abuild -o $(COREBOOT_BUILD_DIR)/$${NAME} $(ABUILD_OPTIONS) $(CCACHE_OPTIONS) -x --name $${NAME} --clean
 130         NAME=gcc; util/abuild/abuild -o $(COREBOOT_BUILD_DIR)/$${NAME} $(ABUILD_OPTIONS) $(CCACHE_OPTIONS) --name $${NAME} --clean-somewhat
 131 endif
 132 ifneq ($(JENKINS_SKIP_CLANG_TESTS),y)
 133 ifneq ($(JENKINS_SKIP_ALLTHREAD_TEST),y)
 134         NAME=emu_clang_allthreads; util/abuild/abuild -o $(COREBOOT_BUILD_DIR)/$${NAME} $(ABUILD_OPTIONS) $(CCACHE_OPTIONS) --clang --name $${NAME} --target EMULATION_QEMU_X86_Q35 -b EMULATION_QEMU_X86_Q35 --exitcode --clean
 135         NAME=amd_clang_allthreads; util/abuild/abuild -o $(COREBOOT_BUILD_DIR)/$${NAME} $(ABUILD_OPTIONS) $(CCACHE_OPTIONS) --clang --name $${NAME} --target GOOGLE_SKYRIM -b GOOGLE_SKYRIM --exitcode --clean
 136         NAME=intel_clang_allthreads; util/abuild/abuild -o $(COREBOOT_BUILD_DIR)/$${NAME} $(ABUILD_OPTIONS) $(CCACHE_OPTIONS) --clang --name $${NAME} --target GOOGLE_REX0 -b GOOGLE_REX0 --exitcode --clean
 137         NAME=mediatek_clang_allthreads; util/abuild/abuild -o $(COREBOOT_BUILD_DIR)/$${NAME} $(ABUILD_OPTIONS) $(CCACHE_OPTIONS) --clang --name $${NAME} --target GOOGLE_CHERRY -b GOOGLE_CHERRY --exitcode --clean
 138 endif
 139         NAME=clang-chromeos; util/abuild/abuild -o $(COREBOOT_BUILD_DIR)/$${NAME} $(ABUILD_OPTIONS) $(CCACHE_OPTIONS) -L -x --name $${NAME} --clean
 140         NAME=clang; util/abuild/abuild -o $(COREBOOT_BUILD_DIR)/$${NAME} $(ABUILD_OPTIONS) $(CCACHE_OPTIONS) -L --name $${NAME} --clean
 141 endif
 142
 143 test-payloads:
 144         $(MAKE) -C payloads/libpayload test-configs -j $(CPUS) V=$(V) Q=$(Q) MFLAGS= MAKEFLAGS= $(if $(TEST_NOCCACHE),,CONFIG_LP_CCACHE=y)
 145         export MFLAGS=;export MAKEFLAGS=; \
 146         $(foreach payload, $(TEST_PAYLOADLIST_INTERNAL), \
 147                 echo "*** Making payload $(payload) ***"; \
 148                 $(MAKE) -C payloads/$(payload) distclean ;\
 149                 $(MAKE) $(payload) -j $(CPUS) V=$(V) Q=$(Q)\
 150                 || exit 1; )
 151
 152 test-tools:
 153         @echo "Build testing $(TOOLLIST)"
 154         $(foreach tool, $(TOOLLIST),  echo "Building $(tool)";$(MAKE) CPUS=$(CPUS) V=$(V) Q=$(Q) BLD_DIR="util/$(tool)" BLD="$(tool)" MFLAGS= MAKEFLAGS= MAKETARGET= junit.xml; )
 155         unset COREBOOT_BUILD_DIR;$(MAKE) CPUS=$(CPUS) V=$(V) Q=$(Q) BLD_DIR=payloads/nvramcui BLD=nvramcui MFLAGS= MAKEFLAGS= MAKETARGET=all junit.xml
 156         unset COREBOOT_BUILD_DIR;$(MAKE) CPUS=$(CPUS) V=$(V) Q=$(Q) BLD_DIR=payloads/coreinfo BLD=coreinfo MFLAGS= MAKEFLAGS= MAKETARGET=defaultbuild junit.xml
 157         (cd payloads/libpayload; unset COREBOOT_BUILD_DIR; $(MAKE) $(if $(JENKINS_NOCCACHE),,CONFIG_LP_CCACHE=y) V=$(V) Q=$(Q) junit.xml)
 158         echo "The tegra builds need to be fixed.  Disabled until someone fixes them."
 159         # The tegra targets don't use the standard build system, so we need to provide an xcompile to them.
 160         # $(MAKE) xcompile=$(COREBOOT_BUILD_DIR)/xcompile $(COREBOOT_BUILD_DIR)/xcompile
 161         # $(MAKE) CPUS=$(CPUS) V=$(V) Q=$(Q) BLD_DIR=src/soc/nvidia/tegra124/lp0 BLD=tegra124_lp0 MFLAGS= MAKEFLAGS=xcompile=$(COREBOOT_BUILD_DIR)/xcompile MAKETARGET=all junit.xml
 162         # $(MAKE) CPUS=$(CPUS) V=$(V) Q=$(Q) BLD_DIR=src/soc/nvidia/tegra210/lp0 BLD=tegra120_lp0 MFLAGS= MAKEFLAGS=xcompile=$(COREBOOT_BUILD_DIR)/xcompile MAKETARGET=all junit.xml
 163         #
 164         echo "The gitconfig tests are flaky and are disabled until someone fixes them."
 165         # @echo "Running gitconfig tests"
 166         # @for test in $$(find util/gitconfig/test -maxdepth 1 \
 167         #       -type f -executable); do \
 168         #       echo "$${test}"; \
 169         #       "$${test}" || exit $${?}; \
 170         # done
 171
 172 test-cleanup:
 173         rm -rf $(COREBOOT_BUILD_DIR)/chromeos $(COREBOOT_BUILD_DIR)/default
 174         rm -rf $(COREBOOT_BUILD_DIR)/chromeos-clang $(COREBOOT_BUILD_DIR)/default-clang
 175         rm -rf $(COREBOOT_BUILD_DIR)/scanbuild
 176         $(MAKE) clean
 177         $(foreach tool, $(TOOLLIST), $(MAKE) -C util/$(tool) clean ; )
 178         $(MAKE) -C src/soc/nvidia/tegra124/lp0 clean
 179         $(MAKE) -C src/soc/nvidia/tegra210/lp0 clean
 180
 181 .PHONY: test-basic test-lint test-abuild test-payloads
 182 .PHONY: test-tools test-cleanup test-help
 183 .PHONY: lint lint-stable what-jenkins-does