1 # CI configuration file for CI utilizing cirrus-ci.org
3 # For instructions on how to enable the CI integration in a repository and
4 # further details, see src/tools/ci/README
8 # The lower depth accelerates git clone. Use a bit of depth so that
9 # concurrent tasks and retrying older jobs have a chance of working.
10 CIRRUS_CLONE_DEPTH: 500
11 # Useful to be able to analyse what in a script takes long
12 CIRRUS_LOG_TIMESTAMP: true
14 CCACHE_MAXSIZE: "250M"
16 # target to test, for all but windows
17 CHECK: check-world PROVE_FLAGS=$PROVE_FLAGS
20 MTEST_ARGS: --print-errorlogs --no-rebuild -C build
21 PGCTLTIMEOUT: 120 # avoids spurious failures during parallel tests
22 TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/src/tools/ci/pg_ci_base.conf
23 PG_TEST_EXTRA: kerberos ldap ssl libpq_encryption load_balance
26 # What files to preserve in case tests fail
27 on_failure_ac: &on_failure_ac
35 on_failure_meson: &on_failure_meson
38 - "build*/testrun/**/*.log"
39 - "build*/testrun/**/*.diffs"
40 - "build*/testrun/**/regress_log_*"
43 # In theory it'd be nice to upload the junit files meson generates, so that
44 # cirrus will nicely annotate the commit. Unfortunately the files don't
45 # contain identifiable file + line numbers right now, so the annotations
46 # don't end up useful. We could probably improve on that with a some custom
47 # conversion script, but ...
49 path: "build*/meson-logs/*.txt"
53 # To avoid unnecessarily spinning up a lot of VMs / containers for entirely
54 # broken commits, have a minimal task that all others depend on.
58 # If a specific OS is requested, don't run the sanity check. This shortens
59 # push-wait-for-ci cycle time a bit when debugging operating system specific
60 # failures. Uses skip instead of only_if, as cirrus otherwise warns about
61 # only_if conditions not matching.
62 skip: $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:.*'
68 IMAGE_FAMILY: pg-ci-bookworm
69 CCACHE_DIR: ${CIRRUS_WORKING_DIR}/ccache_dir
70 # no options enabled, should be small
71 CCACHE_MAXSIZE: "150M"
73 # While containers would start up a bit quicker, building is a bit
74 # slower. This way we don't have to maintain a container image.
75 <<: *linux_task_template
82 chown -R postgres:postgres .
83 mkdir -p ${CCACHE_DIR}
84 chown -R postgres:postgres ${CCACHE_DIR}
85 echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf
86 su postgres -c "ulimit -l -H && ulimit -l -S"
87 # Can't change container's kernel.core_pattern. Postgres user can't write
88 # to / normally. Change that.
96 --auto-features=disabled \
102 ninja -C build -j${BUILD_JOBS}
104 upload_caches: ccache
106 # Run a minimal set of tests. The main regression tests take too long for
107 # this purpose. For now this is a random quick pg_regress style test, and a
108 # tap test that exercises both a frontend binary and the backend.
109 test_minimal_script: |
112 meson test $MTEST_ARGS --suite setup
113 meson test $MTEST_ARGS --num-processes ${TEST_JOBS} \
114 cube/regress pg_ctl/001_start_stop
118 <<: *on_failure_meson
120 mkdir -m 770 /tmp/cores
121 find / -maxdepth 1 -type f -name 'core*' -exec mv '{}' /tmp/cores/ \;
122 src/tools/ci/cores_backtrace.sh linux /tmp/cores
126 name: FreeBSD - 13 - Meson
132 IMAGE_FAMILY: pg-ci-freebsd-13
135 CCACHE_DIR: /tmp/ccache_dir
136 CPPFLAGS: -DRELCACHE_FORCE_RELEASE -DCOPY_PARSE_PLAN_TREES -DWRITE_READ_PARSE_PLAN_TREES -DRAW_EXPRESSION_COVERAGE_TEST -DENFORCE_REGRESSION_TEST_NAME_RESTRICTIONS
139 PG_TEST_PG_UPGRADE_MODE: --link
141 <<: *freebsd_task_template
143 depends_on: SanityCheck
144 only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
149 ulimit -a -H && ulimit -a -S
154 # Work around performance issues due to 32KB block size
155 repartition_script: src/tools/ci/gcp_freebsd_repartition.sh
156 create_user_script: |
158 chown -R postgres:postgres .
159 mkdir -p ${CCACHE_DIR}
160 chown -R postgres:postgres ${CCACHE_DIR}
161 setup_core_files_script: |
162 mkdir -m 770 /tmp/cores
163 chown root:postgres /tmp/cores
164 sysctl kern.corefile='/tmp/cores/%N.%P.core'
165 setup_additional_packages_script: |
168 # NB: Intentionally build without -Dllvm. The freebsd image size is already
169 # large enough to make VM startup slow, and even without llvm freebsd
170 # already takes longer than other platforms except for windows.
175 -Dcassert=true -Dinjection_points=true \
176 -Duuid=bsd -Dtcl_version=tcl86 -Ddtrace=auto \
177 -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
178 -Dextra_lib_dirs=/usr/local/lib -Dextra_include_dirs=/usr/local/include/ \
181 build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS}'
182 upload_caches: ccache
187 meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
190 # test runningcheck, freebsd chosen because it's currently fast enough
191 test_running_script: |
195 meson test $MTEST_ARGS --quiet --suite setup
196 export LD_LIBRARY_PATH="$(pwd)/build/tmp_install/usr/local/pgsql/lib/:$LD_LIBRARY_PATH"
197 mkdir -p build/testrun
198 build/tmp_install/usr/local/pgsql/bin/initdb -N build/runningcheck --no-instructions -A trust
199 echo "include '$(pwd)/src/tools/ci/pg_ci_base.conf'" >> build/runningcheck/postgresql.conf
200 build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
201 meson test $MTEST_ARGS --num-processes ${TEST_JOBS} --setup running
202 build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
206 # if the server continues running, it often causes cirrus-ci to fail
207 # during upload, as it doesn't expect artifacts to change size
208 stop_running_script: |
210 build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop || true
212 <<: *on_failure_meson
213 cores_script: src/tools/ci/cores_backtrace.sh freebsd /tmp/cores
216 # configure feature flags, shared between the task running the linux tests and
217 # the CompilerWarnings task
218 LINUX_CONFIGURE_FEATURES: &LINUX_CONFIGURE_FEATURES >-
232 --with-tcl --with-tclconfig=/usr/lib/tcl8.6/
236 LINUX_MESON_FEATURES: &LINUX_MESON_FEATURES >-
245 TEST_JOBS: 8 # experimentally derived to be a decent choice
246 IMAGE_FAMILY: pg-ci-bookworm
248 CCACHE_DIR: /tmp/ccache_dir
249 DEBUGINFOD_URLS: "https://debuginfod.debian.net"
251 # Enable a reasonable set of sanitizers. Use the linux task for that, as
252 # it's one of the fastest tasks (without sanitizers). Also several of the
253 # sanitizers work best on linux.
255 # The overhead of alignment sanitizer is low, undefined behaviour has
256 # moderate overhead. Test alignment sanitizer in the meson task, as it
257 # does both 32 and 64 bit builds and is thus more likely to expose
260 # Address sanitizer in contrast is somewhat expensive. Enable it in the
261 # autoconf task, as the meson task tests both 32 and 64bit.
263 # disable_coredump=0, abort_on_error=1: for useful backtraces in case of crashes
264 # print_stacktraces=1,verbosity=2, duh
265 # detect_leaks=0: too many uninteresting leak errors in short-lived binaries
266 UBSAN_OPTIONS: print_stacktrace=1:disable_coredump=0:abort_on_error=1:verbosity=2
267 ASAN_OPTIONS: print_stacktrace=1:disable_coredump=0:abort_on_error=1:detect_leaks=0
269 # SANITIZER_FLAGS is set in the tasks below
270 CFLAGS: -Og -ggdb -fno-sanitize-recover=all $SANITIZER_FLAGS
272 LDFLAGS: $SANITIZER_FLAGS
276 LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
277 LINUX_MESON_FEATURES: *LINUX_MESON_FEATURES
279 <<: *linux_task_template
281 depends_on: SanityCheck
282 only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
285 folder: ${CCACHE_DIR}
291 ulimit -a -H && ulimit -a -S
293 create_user_script: |
295 chown -R postgres:postgres .
296 mkdir -p ${CCACHE_DIR}
297 chown -R postgres:postgres ${CCACHE_DIR}
298 echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf
299 su postgres -c "ulimit -l -H && ulimit -l -S"
300 setup_core_files_script: |
301 mkdir -m 770 /tmp/cores
302 chown root:postgres /tmp/cores
303 sysctl kernel.core_pattern='/tmp/cores/%e-%s-%p.core'
305 setup_hosts_file_script: |
306 cat >> /etc/hosts <<-EOF
307 127.0.0.1 pg-loadbalancetest
308 127.0.0.2 pg-loadbalancetest
309 127.0.0.3 pg-loadbalancetest
312 setup_additional_packages_script: |
314 #DEBIAN_FRONTEND=noninteractive apt-get -y install ...
317 - name: Linux - Debian Bookworm - Autoconf
320 SANITIZER_FLAGS: -fsanitize=address
322 # Normally, the "relation segment" code basically has no coverage in our
323 # tests, because we (quite reasonably) don't generate tables large
324 # enough in tests. We've had plenty bugs that we didn't notice due the
325 # code not being exercised much. Thus specify a very small segment size
326 # here. Use a non-power-of-two segment size, given we currently allow
331 --enable-cassert --enable-injection-points --enable-debug \
332 --enable-tap-tests --enable-nls \
333 --with-segsize-blocks=6 \
335 ${LINUX_CONFIGURE_FEATURES} \
339 build_script: su postgres -c "make -s -j${BUILD_JOBS} world-bin"
340 upload_caches: ccache
344 ulimit -c unlimited # default is 0
345 make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS}
351 - name: Linux - Debian Bookworm - Meson
354 CCACHE_MAXSIZE: "400M" # tests two different builds
355 SANITIZER_FLAGS: -fsanitize=alignment,undefined
361 -Dcassert=true -Dinjection_points=true \
362 ${LINUX_MESON_FEATURES} \
363 -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
367 # Also build & test in a 32bit build - it's gotten rare to test that
369 configure_32_script: |
371 export CC='ccache gcc -m32'
374 -Dcassert=true -Dinjection_points=true \
375 ${LINUX_MESON_FEATURES} \
377 --pkg-config-path /usr/lib/i386-linux-gnu/pkgconfig/ \
378 -DPERL=perl5.36-i386-linux-gnu \
379 -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
383 build_script: su postgres -c 'ninja -C build -j${BUILD_JOBS}'
384 build_32_script: su postgres -c 'ninja -C build-32 -j${BUILD_JOBS}'
386 upload_caches: ccache
391 meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
393 # so that we don't upload 64bit logs if 32bit fails
396 # There's currently no coverage of icu with LANG=C in the buildfarm. We
397 # can easily provide some here by running one of the sets of tests that
398 # way. Newer versions of python insist on changing the LC_CTYPE away
399 # from C, prevent that with PYTHONCOERCECLOCALE.
400 test_world_32_script: |
403 PYTHONCOERCECLOCALE=0 LANG=C meson test $MTEST_ARGS -C build-32 --num-processes ${TEST_JOBS}
407 <<: *on_failure_meson
410 cores_script: src/tools/ci/cores_backtrace.sh linux /tmp/cores
414 name: macOS - Sonoma - Meson
417 CPUS: 4 # always get that much for cirrusci macOS instances
419 # Test performance regresses noticably when using all cores. 8 seems to
421 # https://postgr.es/m/20220927040208.l3shfcidovpzqxfh%40awork3.anarazel.de
423 IMAGE: ghcr.io/cirruslabs/macos-runner:sonoma
425 CIRRUS_WORKING_DIR: ${HOME}/pgsql/
426 CCACHE_DIR: ${HOME}/ccache
427 MACPORTS_CACHE: ${HOME}/macports-cache
434 PG_TEST_PG_UPGRADE_MODE: --clone
436 <<: *macos_task_template
438 depends_on: SanityCheck
439 only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
444 ulimit -a -H && ulimit -a -S
447 setup_core_files_script:
448 - mkdir ${HOME}/cores
449 - sudo sysctl kern.corefile="${HOME}/cores/core.%P"
451 # Use macports, even though homebrew is installed. The installation
452 # of the additional packages we need would take quite a while with
453 # homebrew, even if we cache the downloads. We can't cache all of
454 # homebrew, because it's already large. So we use macports. To cache
455 # the installation we create a .dmg file that we mount if it already
457 # XXX: The reason for the direct p5.34* references is that we'd need
458 # the large macport tree around to figure out that p5-io-tty is
459 # actually p5.34-io-tty. Using the unversioned name works, but
460 # updates macports every time.
462 folder: ${MACPORTS_CACHE}
463 fingerprint_script: |
464 # Include the OS major version in the cache key. If the OS image changes
465 # to a different major version, we need to reinstall.
466 sw_vers -productVersion | sed 's/\..*//'
467 # Also start afresh if we change our MacPorts install script.
468 md5 src/tools/ci/ci_macports_packages.sh
469 reupload_on_changes: true
470 setup_additional_packages_script: |
471 sh src/tools/ci/ci_macports_packages.sh \
484 # system python doesn't provide headers
485 sudo /opt/local/bin/port select python3 python312
486 # Make macports install visible for subsequent steps
487 echo PATH=/opt/local/sbin/:/opt/local/bin/:$PATH >> $CIRRUS_ENV
488 upload_caches: macports
493 export PKG_CONFIG_PATH="/opt/local/lib/pkgconfig/"
496 -Dextra_include_dirs=/opt/local/include \
497 -Dextra_lib_dirs=/opt/local/lib \
498 -Dcassert=true -Dinjection_points=true \
499 -Duuid=e2fs -Ddtrace=auto \
500 -DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
503 build_script: ninja -C build -j${BUILD_JOBS}
504 upload_caches: ccache
507 ulimit -c unlimited # default is 0
508 ulimit -n 1024 # default is 256, pretty low
509 meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
512 <<: *on_failure_meson
513 cores_script: src/tools/ci/cores_backtrace.sh macos "${HOME}/cores"
516 WINDOWS_ENVIRONMENT_BASE: &WINDOWS_ENVIRONMENT_BASE
518 # Half the allowed per-user CPU cores
521 # The default cirrus working dir is in a directory msbuild complains about
522 CIRRUS_WORKING_DIR: "c:/cirrus"
523 # git's tar doesn't deal with drive letters, see
524 # https://postgr.es/m/b6782dc3-a7b0-ed56-175f-f8f54cb08d67%40dunslane.net
525 TAR: "c:/windows/system32/tar.exe"
526 # Avoids port conflicts between concurrent tap test runs
527 PG_TEST_USE_UNIX_SOCKETS: 1
528 PG_REGRESS_SOCK_DIR: "c:/cirrus/"
530 IMAGE_FAMILY: pg-ci-windows-ci
535 powershell -Command get-psdrive -psprovider filesystem
540 name: Windows - Server 2019, VS 2019 - Meson & ninja
541 << : *WINDOWS_ENVIRONMENT_BASE
544 TEST_JOBS: 8 # wild guess, data based value welcome
546 # Cirrus defaults to SetErrorMode(SEM_NOGPFAULTERRORBOX | ...). That
547 # prevents crash reporting from working unless binaries do SetErrorMode()
548 # themselves. Furthermore, it appears that either python or, more likely,
549 # the C runtime has a bug where SEM_NOGPFAULTERRORBOX can very
550 # occasionally *trigger* a crash on process exit - which is hard to debug,
551 # given that it explicitly prevents crash dumps from working...
552 # 0x8001 is SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX
553 CIRRUS_WINDOWS_ERROR_MODE: 0x8001
555 <<: *windows_task_template
557 depends_on: SanityCheck
558 only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
560 setup_additional_packages_script: |
561 REM choco install -y --no-progress ...
563 setup_hosts_file_script: |
564 echo 127.0.0.1 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts
565 echo 127.0.0.2 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts
566 echo 127.0.0.3 pg-loadbalancetest >> c:\Windows\System32\Drivers\etc\hosts
567 type c:\Windows\System32\Drivers\etc\hosts
569 # Use /DEBUG:FASTLINK to avoid high memory usage during linking
572 meson setup --backend ninja --buildtype debug -Dc_link_args=/DEBUG:FASTLINK -Dcassert=true -Dinjection_points=true -Db_pch=true -Dextra_lib_dirs=c:\openssl\1.1\lib -Dextra_include_dirs=c:\openssl\1.1\include -DTAR=%TAR% -DPG_TEST_EXTRA="%PG_TEST_EXTRA%" build
578 check_world_script: |
580 meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
583 <<: *on_failure_meson
585 path: "crashlog-*.txt"
590 << : *WINDOWS_ENVIRONMENT_BASE
591 name: Windows - Server 2019, MinGW64 - Meson
593 # due to resource constraints we don't run this task by default for now
595 # worth using only_if despite being manual, otherwise this task will show up
596 # when e.g. ci-os-only: linux is used.
597 only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*mingw.*'
598 # otherwise it'll be sorted before other tasks
599 depends_on: SanityCheck
602 TEST_JOBS: 4 # higher concurrency causes occasional failures
603 CCACHE_DIR: C:/msys64/ccache
604 CCACHE_MAXSIZE: "500M"
605 CCACHE_SLOPPINESS: pch_defines,time_macros
607 # for some reason mingw plpython cannot find its installation without this
608 PYTHONHOME: C:/msys64/ucrt64
609 # prevents MSYS bash from resetting error mode
611 # Start bash in current working directory
613 BASH: C:\msys64\usr\bin\bash.exe -l
615 <<: *windows_task_template
618 folder: ${CCACHE_DIR}
620 setup_additional_packages_script: |
621 REM C:\msys64\usr\bin\pacman.exe -S --noconfirm ...
624 %BASH% -c "where gcc"
625 %BASH% -c "gcc --version"
626 %BASH% -c "where perl"
627 %BASH% -c "perl --version"
629 # disable -Dnls as the number of files it creates cause a noticable slowdown
631 %BASH% -c "meson setup -Ddebug=true -Doptimization=g -Dcassert=true -Dinjection_points=true -Db_pch=true -Dnls=disabled -DTAR=%TAR% build"
634 %BASH% -c "ninja -C build"
636 upload_caches: ccache
639 %BASH% -c "meson test %MTEST_ARGS% --num-processes %TEST_JOBS%"
642 <<: *on_failure_meson
644 path: "crashlog-*.txt"
649 name: CompilerWarnings
651 # To limit unnecessary work only run this once the SanityCheck
652 # succeeds. This is particularly important for this task as we intentionally
653 # use always: to continue after failures. Task that did not run count as a
654 # success, so we need to recheck SanityChecks's condition here ...
655 depends_on: SanityCheck
656 only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*'
661 IMAGE_FAMILY: pg-ci-bookworm
663 # Use larger ccache cache, as this task compiles with multiple compilers /
666 CCACHE_DIR: "/tmp/ccache_dir"
668 LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
669 LINUX_MESON_FEATURES: *LINUX_MESON_FEATURES
671 <<: *linux_task_template
677 ulimit -a -H && ulimit -a -S
685 setup_additional_packages_script: |
687 #DEBIAN_FRONTEND=noninteractive apt-get -y install ...
690 # Test that code can be built with gcc/clang without warnings
693 setup_script: echo "COPT=-Werror" > src/Makefile.custom
695 # Trace probes have a history of getting accidentally broken. Use the
696 # different compilers to build with different combinations of dtrace on/off
697 # and cassert on/off.
699 # gcc, cassert off, dtrace on
701 gcc_warning_script: |
705 ${LINUX_CONFIGURE_FEATURES} \
706 CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
707 make -s -j${BUILD_JOBS} clean
708 time make -s -j${BUILD_JOBS} world-bin
710 # gcc, cassert on, dtrace off
712 gcc_a_warning_script: |
716 ${LINUX_CONFIGURE_FEATURES} \
717 CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"
718 make -s -j${BUILD_JOBS} clean
719 time make -s -j${BUILD_JOBS} world-bin
721 # clang, cassert off, dtrace off
723 clang_warning_script: |
725 --cache clang.cache \
726 ${LINUX_CONFIGURE_FEATURES} \
727 CC="ccache clang" CXX="ccache clang++" CLANG="ccache clang"
728 make -s -j${BUILD_JOBS} clean
729 time make -s -j${BUILD_JOBS} world-bin
731 # clang, cassert on, dtrace on
733 clang_a_warning_script: |
735 --cache clang.cache \
738 ${LINUX_CONFIGURE_FEATURES} \
739 CC="ccache clang" CXX="ccache clang++" CLANG="ccache clang"
740 make -s -j${BUILD_JOBS} clean
741 time make -s -j${BUILD_JOBS} world-bin
743 # cross-compile to windows
745 mingw_cross_warning_script: |
747 --host=x86_64-w64-mingw32 \
750 CC="ccache x86_64-w64-mingw32-gcc" \
751 CXX="ccache x86_64-w64-mingw32-g++"
752 make -s -j${BUILD_JOBS} clean
753 time make -s -j${BUILD_JOBS} world-bin
756 # Verify docs can be built
758 # XXX: Only do this if there have been changes in doc/ since last build
766 make -s -j${BUILD_JOBS} clean
767 time make -s -j${BUILD_JOBS} -C doc
770 # Verify headerscheck / cpluspluscheck succeed
772 # - Don't use ccache, the files are uncacheable, polluting ccache's
774 # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
775 # - XXX have to disable ICU to avoid errors:
776 # https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
779 headers_headerscheck_script: |
781 ${LINUX_CONFIGURE_FEATURES} \
784 CC="gcc" CXX"=g++" CLANG="clang"
785 make -s -j${BUILD_JOBS} clean
786 time make -s headerscheck EXTRAFLAGS='-fmax-errors=10'
787 headers_cpluspluscheck_script: |
788 time make -s cpluspluscheck EXTRAFLAGS='-fmax-errors=10'
791 upload_caches: ccache