Jenkinsfile-dynamatrix

   1 #!/usr/bin/env groovy
   2 // ^^^ For syntax highlighters
   3
   4 /* Typical Keep this build description formula for custom replayed builds (see below):
   5
   6 Kept for reference: build of commit https://github.com/networkupstools/nut/commit/86a32237c7df45c5aba640746f7afc4de09505a1
   7 PR https://github.com/networkupstools/nut/pull/1047
   8 A milestone of "fightwarn" effort attacking actual warnings in codebase Jun 2021
   9
  10 def buildCommit = '86a32237c7df45c5aba640746f7afc4de09505a1'
  11 */
  12
  13 // See https://github.com/networkupstools/jenkins-dynamatrix/ for the lib
  14 // Agent setup evolves at https://ci.networkupstools.org/computer/
  15 // NOTE: The "${BRANCH_NAME}" below IS NOT A VARIABLE!
  16 //       Special notation per custom plugin build including changes from
  17 //       https://github.com/jenkinsci/pipeline-groovy-lib-plugin/pull/19/
  18 @Library('jenkins-dynamatrix@${BRANCH_NAME}') _
  19 import org.nut.dynamatrix.dynamatrixGlobalState;
  20 import org.nut.dynamatrix.*;
  21
  22     // dynacfgBase = Base configuration for Dynamatrix for this pipeline
  23     // dynacfgPipeline = Step-dependent setup in sub-maps
  24     def dynacfgBase = [:]
  25     def dynacfgPipeline = [:]
  26
  27     // NOTE: These can be further disabled or active in different combo specs
  28     // below based on branch names. Also note that the values are somewhat
  29     // "inversed" -- e.g. that "disabledSomething = false" means "enable it".
  30     dynacfgPipeline.disableSlowBuildAutotools = false
  31     dynacfgPipeline.disableSlowBuildCIBuild = false
  32     dynacfgPipeline.disableSlowBuildCIBuildExperimental = false
  33
  34     // NOTE: Disabled by default because with -std=c* the compiler and linker
  35     // (at least on environments NUT CI farm has) do not "see" many things,
  36     // and do not even define WIN32, and this is unrelated to NUT codebase.
  37     // This toggle aims to only disable 'c' builds in the scenario; but the
  38     //'gnu' ones should still happen if it is enabled overall.
  39     dynacfgPipeline.disableStrictCIBuild_CrossWindows = true
  40
  41     // At this time, GCC succeeds building C89/GNU89 mode for NUT
  42     // while CLANG complains about things we can't fix easily.
  43     dynacfgPipeline.axisCombos_COMPILER_GCC = [~/COMPILER=GCC/]
  44     dynacfgPipeline.axisCombos_COMPILER_NOT_GCC = [~/COMPILER=(?!GCC)/]
  45
  46     // Avoid requiring success on GCC so old we can't manage warnings by CLI or pragmas:
  47     dynacfgPipeline.axisCombos_COMPILER_GCC_TOO_OLD = [~/COMPILER=GCC/, ~/GCCVER=([0123]\.|4\.[0123])/]
  48
  49     // Beside the flag here, the pre-defined C89/C90/ANSI scenarios
  50     // should only get considered in branches named ~/fightwarn.*89.*/
  51     // or PRs to those (for non-GCC builds):
  52     dynacfgPipeline.disableSlowBuildCIBuildExperimentalANSI = false
  53
  54     // The NUT CI farm offers a few other architectures in containers backed
  55     // by QEMU virtual CPUs. Running builds in these is expensive (takes a
  56     // lot of time and can lag the NUT pipeline), so we would run just a few
  57     // scenarios there to ensure code compatibility with headers and libs,
  58     // but not exhaustive tests that can be done elsewhere. Currently this
  59     // workload hits the Linux builder that completes its usual work earlier
  60     // than some other builders, so a few minutes more would not hit pipeline
  61     // wallclock time frame much. The slowBuild filter rules rely on separate
  62     // label patterns with "qemu-nut-builder" and/or "qemu-nut-builder:alldrv".
  63     dynacfgPipeline.disableSlowBuildCIBuild_QEMU = true
  64     //if ( env?.BRANCH_NAME ==~ /master|main|stable|.*qemu.*/ ) {
  65     if ( env?.BRANCH_NAME ==~ /.*qemu.*/ ) {
  66         dynacfgPipeline.disableSlowBuildCIBuild_QEMU = false
  67     }
  68
  69     dynacfgPipeline.traceBuildShell_configureEnvvars = false // true
  70     dynacfgPipeline.traceBuildShell = false // true
  71
  72     //if (false) // <<< (Un-)comment away in select runs/branches
  73     //if (true)  // <<< (Un-)comment away in select runs/branches
  74     if ( env?.BRANCH_NAME ==~ /.*fightwarn-verbose.*/ )
  75     {
  76         dynacfgPipeline.traceBuildShell_configureEnvvars = true // false
  77         dynacfgPipeline.traceBuildShell = true // false
  78     }
  79
  80     dynacfgPipeline.failFast = //true //
  81         false
  82
  83     // How long can a single "slow-build stage" run before we
  84     // consider that the build agent is stuck or network dropped?
  85     // The dynamatrix should try to re-schedule this scenario then.
  86     dynacfgPipeline.dsbcStageTimeoutSettings = [
  87         time: 2,
  88         unit: 'HOURS'
  89         ]
  90
  91     // Note: this setting causes a lot of noise in build summary page and
  92     // parent job definition (PR, branch...) overview page on Jenkins,
  93     // by reporting dozens of lines for each analyzer ID ever published.
  94     // Do not enable instant (non-delayed, "false" here) reports for the
  95     // "master" and equivalent branch builds.
  96     dynacfgPipeline.delayedIssueAnalysis = //false //
  97         true
  98
  99     // In modern builds, use the ci_build.sh recipe which first checks
 100     // quietly for things that succeed, and summarizes errors in the end
 101     dynacfgPipeline['spellcheck_prepconf'] = false
 102     dynacfgPipeline['spellcheck_configure'] = false
 103     dynacfgPipeline['spellcheck'] = '(BUILD_TYPE=default-spellcheck ./ci_build.sh)'
 104
 105 /*
 106     // For older builds, with only autotools in the tree:
 107     dynacfgPipeline['spellcheck'] = //false //true
 108         // '( \${MAKE} VERBOSE=1 SPELLCHECK_ERROR_FATAL=yes spellcheck )'
 109 */
 110
 111     //dynacfgPipeline['shellcheck'] = true
 112     // Check shell scripts as well as make implementations registered on
 113     // CI farm -- that they do not fundamentally reject our Makefile syntax.
 114     // Note that if MAKE=something does not get into envvars, defaultTools
 115     // are used (just assigning it among build agent labels is not enough).
 116     dynacfgPipeline['shellcheck'] = [
 117         //'stageNameFunc': null,
 118         //'dynamatrixAxesLabels': [~/^OS_.+/, 'MAKE'],
 119         'dynamatrixAxesLabels': ['OS_FAMILY', 'OS_DISTRO', 'MAKE'],
 120         'single': '( if [ x"\${MAKE-}" = x ]; then echo "WARNING: MAKE is somehow unset, defaulting!" >&2; MAKE=make; fi; \${MAKE} shellcheck )',
 121         'multi': '(cd tests && SERVICE_FRAMEWORK="selftest" SHELL_PROGS="$SHELL_PROGS" ./nut-driver-enumerator-test.sh )',
 122         'multiLabel': 'SHELL_PROGS',
 123         'skipShells': [ 'zsh', 'tcsh', 'csh' ]
 124     ]
 125
 126 /*
 127     // Examples for custom checkouts instead of following a branch/PR that triggered the build:
 128     //dynacfgPipeline.bodyStashCmd = { git (url: "https://github.com/networkupstools/nut", branch: "fightwarn") }
 129
 130     //def buildCommit = '86a32237c7df45c5aba640746f7afc4de09505a1'
 131     def buildCommit = 'refs/tags/v2.7.4'
 132
 133     dynacfgPipeline.bodyStashCmd = { checkout([
 134         $class: 'GitSCM', branches: [[name: buildCommit]],
 135         doGenerateSubmoduleConfigurations: false,
 136         extensions: [[$class: 'SubmoduleOption', disableSubmodules: false, parentCredentials: false, recursiveSubmodules: false, reference: '', trackingSubmodules: false]],
 137         submoduleCfg: [],
 138         userRemoteConfigs: [[url: "https://github.com/networkupstools/nut"]]
 139         ])
 140     }
 141
 142     // While building older release (2.7.4) disable recipes that did not exist back then
 143     dynacfgPipeline['stylecheck'] = false //true
 144     dynacfgPipeline['spellcheck'] = false //true
 145     dynacfgPipeline['shellcheck'] = false //true
 146
 147     dynacfgPipeline.disableSlowBuildCIBuild = true
 148     dynacfgPipeline.disableSlowBuildCIBuildExperimental = true
 149 */
 150
 151     dynacfgBase['commonLabelExpr'] = 'nut-builder'
 152     dynacfgBase['dynamatrixAxesLabels'] = //[~/^OS_.+/]
 153         ['OS_FAMILY', 'OS_DISTRO', '${COMPILER}VER', 'ARCH${ARCH_BITS}']
 154     dynacfgBase['dynamatrixAxesCommonEnv'] = [ ['LANG=C', 'LC_ALL=C', 'TZ=UTC'] ]
 155
 156     dynacfgPipeline.stashnameSrc = 'nut-ci-src'
 157
 158     // These platforms do not serve a functional cppunit for gcc,
 159     // so a diverse C++ build matrix on them is pointless; thus
 160     // so far we allow-failure (or avoid C++11 and newer builds)
 161     // on OpenIndiana (cppcheck pkg seems flawed, at least in
 162     // various versions of GCC builds) and BSD (also just for GCC):
 163     dynacfgPipeline.axisCombos_CPPUNIT = [~/OS_DISTRO=(openindiana|freebsd).*/, ~/CSTDVERSION_cxx=[12].+/, ~/COMPILER=GCC/]
 164
 165     // Avoid mix-up of bitness-related requests and abilities
 166     dynacfgPipeline.axisCombos_ARCH32x64 = [~/BITS=32/, ~/ARCH_BITS=64/]
 167     dynacfgPipeline.axisCombos_ARCH64x32 = [~/BITS=64/, ~/ARCH_BITS=32/]
 168
 169     // Some (but not all) builds skip strict-C standard due to
 170     // current build failures with its requirements
 171     dynacfgPipeline.axisCombos_STRICT_C = [~/CSTDVARIANT=c/]
 172     dynacfgPipeline.axisCombos_GNU_C = [~/CSTDVARIANT=gnu/]
 173
 174     // Here we consider native-platform builds on a Windows box
 175     // (possibly with need for "bat" instead of "sh" steps):
 176     dynacfgPipeline.axisCombos_WINDOWS = [~/OS_FAMILY=windows/]
 177     dynacfgPipeline.axisCombos_NOT_WINDOWS = [~/OS_FAMILY=(?!windows)/]
 178
 179     // TODO: some cross-build enviroments like Linux with mingw?
 180     // Currently done as an explicit scenario below...
 181     dynacfgPipeline.axisCombos_WINDOWS_CROSS = [~/OS_FAMILY=(mingw|mingw32|mingw64|msys2)/]
 182     dynacfgPipeline.axisCombos_NOT_WINDOWS_CROSS = [~/OS_FAMILY=(?!(mingw|mingw32|mingw64|msys2))/]
 183
 184     // At a minimum, we don't want to mess up our arches:
 185     dynacfgPipeline.excludeCombos_DEFAULT = [
 186         dynacfgPipeline.axisCombos_ARCH32x64,
 187         dynacfgPipeline.axisCombos_ARCH64x32
 188         ]
 189
 190     dynacfgPipeline.excludeCombos_DEFAULT_CPPUNIT = [
 191         dynacfgPipeline.axisCombos_CPPUNIT,
 192         dynacfgPipeline.axisCombos_ARCH32x64,
 193         dynacfgPipeline.axisCombos_ARCH64x32
 194         ]
 195
 196     dynacfgPipeline.excludeCombos_DEFAULT_STRICT_C = [
 197         dynacfgPipeline.axisCombos_STRICT_C,
 198         dynacfgPipeline.axisCombos_ARCH32x64,
 199         dynacfgPipeline.axisCombos_ARCH64x32
 200         ]
 201
 202     dynacfgPipeline.excludeCombos_DEFAULT_CPPUNIT_STRICT_C = [
 203         dynacfgPipeline.axisCombos_CPPUNIT,
 204         dynacfgPipeline.axisCombos_STRICT_C,
 205         dynacfgPipeline.axisCombos_ARCH32x64,
 206         dynacfgPipeline.axisCombos_ARCH64x32
 207         ]
 208
 209     dynacfgPipeline.branchStableRegex = ~/^(master|main|stable)$/
 210     if ( env?.BRANCH_NAME ==~ /^fightwarn.*$/ && (!env?.CHANGE_TARGET) ) {
 211         dynamatrixGlobalState.branchDefaultStable = 'fightwarn'
 212     }
 213
 214     if ( env?.BRANCH_NAME ==~ dynacfgPipeline.branchStableRegex ) {
 215         // For our main branches we want all builds in full,
 216         // to keep reference for warnings-ng up to date, so do
 217         // not limit ny appliesToChangedFilesRegex categories
 218
 219         // Be sure to not make noise in long-lived branches'
 220         // overview pages by enabling quick analysis publishing
 221         // (by default above, or in a Replay):
 222         dynacfgPipeline.delayedIssueAnalysis = true
 223     } else {
 224         // First list the building blocks as lists of files;
 225         // final regexes are arranged below:
 226         // TODO: Technically the slash should be the Path.Separator,
 227         // but at least modern windows can handle a sh step with that
 228         // character, so no big deal for NUT supported platforms
 229         dynacfgPipeline.appliesToChangedFilesRegex_FILES_PREFIX = ~/^(\/*|.*\/)/
 230         dynacfgPipeline.appliesToChangedFilesRegex_FILES_RECIPE = ~/(configure\.ac|.*\.m4|C?Make.*|ci_.*\.sh|autogen\.sh|\.git.*|Jenkinsfile.*|.*\.groovy)/
 231         dynacfgPipeline.appliesToChangedFilesRegex_FILES_C = ~/^(.*\.h|.*\.hpp|.*\.c|.*\.cxx|.*\.cpp)/
 232         dynacfgPipeline.appliesToChangedFilesRegex_FILES_TXT = ~/^(.*\.txt|.*\.dict|asciidoc.*|.*\.xsl|.*\.css|AUTHORS.*|CHANGELOG.*|COPYING.*|INSTALL.*|LICENSE.*|MAINT.*|NEWS.*|README.*|TODO.*|UPGRAD.*)/
 233         dynacfgPipeline.appliesToChangedFilesRegex_FILES_IMG = ~/^(.*\.svg|.*\.png|.*\.jpg|.*\.jpeg|.*\.gif)/
 234         dynacfgPipeline.appliesToChangedFilesRegex_FILES_PY = ~/^(.*\.py|scripts\/python\/app\/Nut-Monitor)/
 235
 236         // Recipe changes and C source changes go here:
 237         dynacfgPipeline.appliesToChangedFilesRegex_RECIPE = ~/${dynacfgPipeline.appliesToChangedFilesRegex_FILES_PREFIX}${dynacfgPipeline.appliesToChangedFilesRegex_FILES_RECIPE}(|\.in)$/
 238         dynacfgPipeline.appliesToChangedFilesRegex_C = ~/${dynacfgPipeline.appliesToChangedFilesRegex_FILES_PREFIX}(${dynacfgPipeline.appliesToChangedFilesRegex_FILES_RECIPE}|${dynacfgPipeline.appliesToChangedFilesRegex_FILES_C})(|\.in)$/
 239         dynacfgPipeline.appliesToChangedFilesRegex_PY = ~/${dynacfgPipeline.appliesToChangedFilesRegex_FILES_PREFIX}(${dynacfgPipeline.appliesToChangedFilesRegex_FILES_RECIPE}|${dynacfgPipeline.appliesToChangedFilesRegex_FILES_PY})(|\.in)$/
 240
 241         // Recipe changes and docs source changes go here:
 242         dynacfgPipeline.appliesToChangedFilesRegex_TXT = ~/${dynacfgPipeline.appliesToChangedFilesRegex_FILES_PREFIX}(${dynacfgPipeline.appliesToChangedFilesRegex_FILES_RECIPE}|${dynacfgPipeline.appliesToChangedFilesRegex_FILES_TXT}(|\.in)$)/
 243         dynacfgPipeline.appliesToChangedFilesRegex_DOC = ~/${dynacfgPipeline.appliesToChangedFilesRegex_FILES_PREFIX}(${dynacfgPipeline.appliesToChangedFilesRegex_FILES_RECIPE}|${dynacfgPipeline.appliesToChangedFilesRegex_FILES_TXT}|${dynacfgPipeline.appliesToChangedFilesRegex_FILES_IMG})(|\.in)$/
 244
 245         // TODO: Similar for shell files but based on some logic
 246         // like in shellcheck to find the script files (not only *.sh)?
 247     }
 248
 249     // Do not override DISTCHECK_CONFIGURE_FLAGS as default implem
 250     // does, that breaks custom proto-dir installs and tries to go
 251     // into (not writeable) system paths:
 252     if (!dynacfgPipeline.containsKey('buildPhases')) {
 253         dynacfgPipeline.buildPhases = [:]
 254     }
 255
 256     // Imported from jenkins-dynamatrix JSL vars/autotools.groovy:
 257     // a workaround for the cases of curiously missing MAKE envvar...
 258     dynacfgPipeline.buildPhases['distcheck'] = """( if [ x"\${MAKE-}" = x ]; then echo "WARNING: MAKE is somehow unset, defaulting!" >&2; MAKE=make; fi; eval \${CONFIG_ENVVARS} time \${MAKE} \${MAKE_OPTS} distcheck DISTCHECK_FLAGS=\${CONFIG_OPTS:+\\"\$CONFIG_OPTS\\"} )"""
 259
 260     // Note: shellcheck/spellcheck/... require autotools currently
 261     // or need to be redefined with respective BUILD_TYPE
 262     //dynacfgPipeline.buildSystem = 'ci_build.sh'
 263
 264     //dynacfgPipeline.slowBuildDefaultBody = { echo "Running default custom build" }
 265     dynacfgPipeline.slowBuildDefaultBody_autotools = { def delegate -> setDelegate(delegate)
 266         // Be sure to have a fixed resolved String here ASAP:
 267         String stageNameClone = "${stageName}"
 268         def dsbcClone = dsbc.clone()
 269
 270         stage('Investigate envvars (Autotools DEBUG)') {
 271             echo "Running default custom build for '${stageNameClone}' ==> ${dsbcClone.toString()}" +
 272                 (dynacfgPipeline?.configureEnvvars ? "" : " (note: has no dynacfgPipeline.configureEnvvars)")
 273             // Trick about endianness via ELF binary header picked up from https://serverfault.com/a/749469/490516
 274             sh label: 'Inspect initial envvars', script: """ hostname; date -u; uname -a
 275 echo "LONG_BIT:`getconf LONG_BIT` WORD_BIT:`getconf WORD_BIT`" || true
 276 if command -v xxd >/dev/null ; then xxd -c 1 -l 6 | tail -1; else if command -v od >/dev/null; then od -N 1 -j 5 -b | head -1 ; else hexdump -s 5 -n 1 -C | head -1; fi; fi < /bin/ls 2>/dev/null | awk '(\$2 == 1){print "Endianness: LE"}; (\$2 == 2){print "Endianness: BE"}' || true
 277 echo "\${MATRIX_TAG}"
 278 set | sort -n """
 279             if (dynacfgPipeline?.configureEnvvars) {
 280                 sh label: 'Apply CONFIG_ENVVARS', script: """ set +x
 281 echo "Applying CONFIG_ENVVARS:"
 282 #set -xv
 283 ${dynacfgPipeline.configureEnvvars}
 284 set | sort -n """
 285             }
 286         }
 287
 288         withEnvOptional(dynacfgPipeline.defaultTools) {
 289             stage('Unstash sources') {
 290                 unstashCleanSrc(dynacfgPipeline.stashnameSrc)
 291             }
 292
 293             buildMatrixCellCI(dynacfgPipeline, dsbcClone, stageNameClone)
 294             //buildMatrixCellCI(dynacfgPipeline, dsbc, stageName)
 295         }
 296     }
 297
 298     dynacfgPipeline.slowBuildDefaultBody_ci_build = { def delegate -> setDelegate(delegate)
 299         // Be sure to have a fixed resolved String here ASAP:
 300         String stageNameClone = "${stageName}"
 301         def dsbcClone = dsbc.clone()
 302
 303         stage('Investigate envvars (CI_Build DEBUG)') {
 304             echo "Running default custom build for '${stageNameClone}' ==> ${dsbcClone.toString()}" +
 305                 (dynacfgPipeline?.configureEnvvars ? "" : " (note: has no dynacfgPipeline.configureEnvvars)")
 306             // Trick about endianness via ELF binary header picked up from https://serverfault.com/a/749469/490516
 307             sh label: 'Inspect initial envvars', script: """ hostname; date -u; uname -a
 308 echo "LONG_BIT:`getconf LONG_BIT` WORD_BIT:`getconf WORD_BIT`" || true
 309 if command -v xxd >/dev/null ; then xxd -c 1 -l 6 | tail -1; else if command -v od >/dev/null; then od -N 1 -j 5 -b | head -1 ; else hexdump -s 5 -n 1 -C | head -1; fi; fi < /bin/ls 2>/dev/null | awk '(\$2 == 1){print "Endianness: LE"}; (\$2 == 2){print "Endianness: BE"}' || true
 310 echo "\${MATRIX_TAG}"
 311 set | sort -n """
 312             if (dynacfgPipeline?.configureEnvvars) {
 313                 sh label: 'Apply CONFIG_ENVVARS', script: """ set +x
 314 echo "Applying CONFIG_ENVVARS:"
 315 #set -xv
 316 ${dynacfgPipeline.configureEnvvars}
 317 set | sort -n """
 318             }
 319         }
 320
 321         withEnvOptional(dynacfgPipeline.defaultTools) {
 322             stage('Unstash sources') {
 323                 unstashCleanSrc(dynacfgPipeline.stashnameSrc)
 324             }
 325
 326             def dynacfgPipeline_ciBuild = dynacfgPipeline.clone()
 327             dynacfgPipeline_ciBuild.buildSystem = 'ci_build.sh'
 328             dynacfgPipeline_ciBuild.buildPhases = [:]
 329             dynacfgPipeline_ciBuild = ci_build.sanityCheckDynacfgPipeline(dynacfgPipeline_ciBuild)
 330
 331             buildMatrixCellCI(dynacfgPipeline_ciBuild, dsbcClone, stageNameClone)
 332             //buildMatrixCellCI(dynacfgPipeline_ciBuild, dsbc, stageName)
 333         }
 334     }
 335
 336     dynacfgPipeline.slowBuildDefaultBody = dynacfgPipeline.slowBuildDefaultBody_autotools
 337
 338     /* By default, the master/main/stable branch and PRs against it
 339      * is built with as few scenarios as possible, allowing for fast
 340      * turnaround and avoiding redundant work (e.g. documentation
 341      * rendering, distchecks that are more about recipes than code),
 342      * and stricter warnings that current codebase would fail so far.
 343      * In particular, this saves CI farm resources - allowing more
 344      * PRs per day to be checked in practice.
 345      * Conversely, a branch with "fightwarn" in the name (or PR to it)
 346      * would enjoy many more build scenarios, covering both autotools
 347      * directly and ci_build.sh with stricter warnings, in particular.
 348      */
 349     dynacfgPipeline.slowBuild = [
 350         [name: 'Default autotools driven build with default warning levels (gnu99/gnu++11)',
 351          disabled: dynacfgPipeline.disableSlowBuildAutotools,
 352          branchRegexSource: ~/^(PR-.+|fightwarn.*)$/,
 353          //branchRegexTarget: dynacfgPipeline.branchStableRegex,
 354          branchRegexTarget: ~/fightwarn/,
 355          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
 356          'getParStages': { def dynamatrix, Closure body ->
 357             return dynamatrix.generateBuild([
 358                 //commonLabelExpr: dynacfgBase.commonLabelExpr,
 359                 //defaultDynamatrixConfig: dynacfgBase.defaultDynamatrixConfig,
 360                 requiredNodelabels: [],
 361                 excludedNodelabels: [],
 362
 363                 dynamatrixAxesVirtualLabelsMap: [
 364                     'BITS': [32, 64],
 365                     // 'CSTDVERSION': ['03', '2a'],
 366                     //'CSTDVERSION_${KEY}': [ ['c': '03', 'cxx': '03'], ['c': '99', 'cxx': '98'], ['c': '17', 'cxx': '2a'], 'ansi' ],
 367                     //'CSTDVERSION_${KEY}': [ ['c': '03', 'cxx': '03'], ['c': '99', 'cxx': '98'], ['c': '17', 'cxx': '2a'] ],
 368                     'CSTDVERSION_${KEY}': [ ['c': '99', 'cxx': '11'] ],
 369                     'CSTDVARIANT': ['gnu']
 370                     ],
 371
 372                 mergeMode: [ 'dynamatrixAxesVirtualLabelsMap': 'replace', 'excludeCombos': 'merge' ],
 373                 allowedFailure: [
 374                     dynacfgPipeline.axisCombos_WINDOWS,
 375                     dynacfgPipeline.axisCombos_STRICT_C
 376                     ],
 377                 runAllowedFailure: true,
 378                 //dynamatrixAxesLabels: ['OS_FAMILY', 'OS_DISTRO', '${COMPILER}VER', 'ARCH${ARCH_BITS}'],
 379                 //dynamatrixAxesLabels: [~/^OS/, '${COMPILER}VER', 'ARCH${ARCH_BITS}'],
 380                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_STRICT_C
 381                     + [dynacfgPipeline.axisCombos_WINDOWS_CROSS]
 382                 ], body)
 383             }, // getParStages
 384         //'bodyParStages': {}
 385         ] // one slowBuild filter configuration, autotools-minimal
 386
 387         ,[name: 'Default autotools driven build with max warnings and varied C/C++ revisions (allowed to fail)',
 388          disabled: dynacfgPipeline.disableSlowBuildAutotools,
 389          branchRegexSource: ~/^(PR-.+|fightwarn.*)$/,
 390          //branchRegexTarget: dynacfgPipeline.branchStableRegex,
 391          branchRegexTarget: ~/fightwarn/,
 392          // NOTE: For fightwarn, we want some schenarios that would always build to test
 393          //appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
 394          'getParStages': { def dynamatrix, Closure body ->
 395             return dynamatrix.generateBuild([
 396                 //commonLabelExpr: dynacfgBase.commonLabelExpr,
 397                 //defaultDynamatrixConfig: dynacfgBase.defaultDynamatrixConfig,
 398                 requiredNodelabels: [],
 399                 excludedNodelabels: [],
 400
 401                 dynamatrixAxesVirtualLabelsMap: [
 402                     'BITS': [32, 64],
 403                     // 'CSTDVERSION': ['03', '2a'],
 404                     //'CSTDVERSION_${KEY}': [ ['c': '03', 'cxx': '03'], ['c': '99', 'cxx': '98'], ['c': '17', 'cxx': '2a'], 'ansi' ],
 405                     //'CSTDVERSION_${KEY}': [ ['c': '03', 'cxx': '03'], ['c': '99', 'cxx': '98'], ['c': '17', 'cxx': '2a'] ],
 406                     //'CSTDVERSION_${KEY}': [ ['c': '99', 'cxx': '11'] ],
 407                     'CSTDVERSION_${KEY}': [ ['c': '99', 'cxx': '98'], ['c': '99', 'cxx': '11'], ['c': '11', 'cxx': '11'], ['c': '17', 'cxx': '17'] ],
 408                     'CSTDVARIANT': ['gnu']
 409                     ],
 410
 411                 mergeMode: [ 'dynamatrixAxesVirtualLabelsMap': 'replace', 'dynamatrixAxesCommonEnv': 'replace', 'excludeCombos': 'merge' ],
 412                 dynamatrixAxesCommonEnv: [
 413                     //['LANG=C','LC_ALL=C','TZ=UTC', 'CFLAGS=-Wall\\ -Wextra\\ -Werror', 'CXXFLAGS=-Wall\\ -Wextra\\ -Werror']
 414                     ['LANG=C','LC_ALL=C','TZ=UTC', 'CFLAGS=-Wall', 'CXXFLAGS=-Wall']
 415                     ],
 416                 allowedFailure: [
 417                     dynacfgPipeline.axisCombos_WINDOWS,
 418                     dynacfgPipeline.axisCombos_STRICT_C,
 419                     [~/C.*FLAGS=.+Werror/]
 420                     ],
 421                 runAllowedFailure: true,
 422                 //dynamatrixAxesLabels: ['OS_FAMILY', 'OS_DISTRO', '${COMPILER}VER', 'ARCH${ARCH_BITS}'],
 423                 //dynamatrixAxesLabels: [~/^OS/, '${COMPILER}VER', 'ARCH${ARCH_BITS}'],
 424                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_STRICT_C
 425                     + [dynacfgPipeline.axisCombos_WINDOWS_CROSS]
 426                 ], body)
 427             }, // getParStages
 428         //'bodyParStages': {}
 429         ] // one slowBuild filter configuration, autotools-Wall
 430
 431         ,[name: 'Default autotools driven build with default configuration, bitness and warning levels on each NUT CI farm platform (but with fatal warnings as of gnu99/gnu++11, must pass where enabled)',
 432          disabled: dynacfgPipeline.disableSlowBuildAutotools,
 433          //branchRegexSource: ~/^(PR-.+|fightwarn.*)$/,
 434          //branchRegexTarget: dynacfgPipeline.branchStableRegex,
 435          //branchRegexTarget: ~/fightwarn/,
 436          //appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
 437          'getParStages': { def dynamatrix, Closure body ->
 438             return dynamatrix.generateBuild([
 439                 //commonLabelExpr: dynacfgBase.commonLabelExpr,
 440                 //defaultDynamatrixConfig: dynacfgBase.defaultDynamatrixConfig,
 441                 requiredNodelabels: [],
 442                 excludedNodelabels: [],
 443
 444                 dynamatrixAxesVirtualLabelsMap: [
 445                     //'BITS': [32, 64],
 446                     // 'CSTDVERSION': ['03', '2a'],
 447                     //'CSTDVERSION_${KEY}': [ ['c': '03', 'cxx': '03'], ['c': '99', 'cxx': '98'], ['c': '17', 'cxx': '2a'], 'ansi' ],
 448                     //'CSTDVERSION_${KEY}': [ ['c': '03', 'cxx': '03'], ['c': '99', 'cxx': '98'], ['c': '17', 'cxx': '2a'] ],
 449                     'CSTDVERSION_${KEY}': [ ['c': '99', 'cxx': '11'] ],
 450                     'CSTDVARIANT': ['gnu']
 451                     ],
 452                 dynamatrixAxesCommonEnv: [
 453                     // One set of several simultaneously exported envvars!
 454                     // CONFIG_OPTS are picked up by our dynamatrix configuration
 455                     // and substituted into shell "as is" for normal builds
 456                     // (so splitting into many tokens), or quoted as a single
 457                     // token DISTCHECK_FLAGS in its stage (split by make later).
 458                     ['LANG=C','LC_ALL=C','TZ=UTC',
 459                      'CONFIG_OPTS=--with-all=auto --with-docs=auto --with-ssl=auto --enable-Werror --enable-warnings --disable-Wcolor --enable-silent-rules'
 460                     ]
 461                     ],
 462
 463                 mergeMode: [ 'dynamatrixAxesVirtualLabelsMap': 'replace', 'excludeCombos': 'merge' ],
 464                 allowedFailure: [
 465                     dynacfgPipeline.axisCombos_WINDOWS,
 466                     dynacfgPipeline.axisCombos_STRICT_C
 467                     ],
 468                 runAllowedFailure: true,
 469                 //dynamatrixAxesLabels: ['OS_FAMILY', 'OS_DISTRO', '${COMPILER}VER', 'ARCH${ARCH_BITS}'],
 470                 //dynamatrixAxesLabels: [~/^OS/, '${COMPILER}VER', 'ARCH${ARCH_BITS}'],
 471                 dynamatrixAxesLabels: [~/^OS_DISTRO/, 'COMPILER'],
 472                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_STRICT_C
 473                     + [dynacfgPipeline.axisCombos_WINDOWS_CROSS]
 474                     + [[~/OS_DISTRO=openbsd-6\./, ~/COMPILER=GCC/]]
 475                     // Here we picked just OSes and compilers (gcc or clang),
 476                     // so exclude systems which have e.g. gcc-4.2.1 which claims
 477                     // type range comparison warnings despite pragma fencing.
 478                     // gcc-4.8.x on CentOS 7 and Ubuntu 14.04 looks already okay.
 479                     + [[~/OS_DISTRO=macos/]]
 480                     // MacOS (at least agents prepared with HomeBrew packages)
 481                     // requires a few pkg-config and CFLAGS pre-sets which are
 482                     // done in ci_build.sh and defeat the purpose of this stage.
 483                     // So it is easier and more honest to just skip it.
 484                 ], body)
 485             }, // getParStages
 486         //'bodyParStages': {}
 487         ] // one slowBuild filter configuration, autotools-everywhere
 488
 489         ,[name: 'Various non-docs distchecked target builds with main and ~newest supported C/C++ revisions (must pass on all platforms)',
 490          disabled: dynacfgPipeline.disableSlowBuildCIBuild,
 491          //branchRegexSource: ~/^(PR-.+|fightwarn.*)$/,
 492          //branchRegexTarget: dynacfgPipeline.branchStableRegex,
 493          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
 494          'getParStages': { def dynamatrix, Closure body ->
 495             return dynamatrix.generateBuild([
 496                 requiredNodelabels: [],
 497                 excludedNodelabels: [],
 498
 499                 dynamatrixAxesVirtualLabelsMap: [
 500                     'BITS': [32, 64],
 501                     'CSTDVERSION_${KEY}': [ ['c': '99', 'cxx': '98'], ['c': '17', 'cxx': '17'] ],
 502                     'CSTDVARIANT': ['gnu'],
 503                     'BUILD_TYPE': ['default-nodoc']
 504                     // BUILD_TYPE=default-withdoc:man
 505                     // BUILD_TYPE=default-tgt:distcheck-light == --with-all=auto --with-ssl=auto --with-doc=auto
 506                     // BUILD_TYPE=default-tgt:distcheck-light + NO_PKG_CONFIG=true ?
 507                     ],
 508                 dynamatrixAxesCommonEnv: [
 509                     ['LANG=C','LC_ALL=C','TZ=UTC','BUILD_WARNFATAL=yes'
 510                      //,'BUILD_WARNOPT=hard'
 511                     ]
 512                     ],
 513                 allowedFailure: [
 514                     dynacfgPipeline.axisCombos_WINDOWS
 515                     ],
 516                 runAllowedFailure: true,
 517                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
 518                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_CPPUNIT_STRICT_C +
 519                     [dynacfgPipeline.axisCombos_COMPILER_GCC_TOO_OLD]
 520                     + [dynacfgPipeline.axisCombos_WINDOWS_CROSS]
 521                 ], body)
 522             }, // getParStages
 523         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
 524         ] // one slowBuild filter configuration
 525
 526         ,[name: 'Valgrind+distchecked target builds with main and ~newest supported C/C++ revisions (must pass on all platforms)',
 527          disabled: dynacfgPipeline.disableSlowBuildCIBuild,
 528          //branchRegexSource: ~/^(PR-.+|fightwarn.*)$/,
 529          //branchRegexTarget: dynacfgPipeline.branchStableRegex,
 530          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
 531          'getParStages': { def dynamatrix, Closure body ->
 532             return dynamatrix.generateBuild([
 533                 requiredNodelabels: ["(NUT_BUILD_CAPS=valgrind=yes||NUT_BUILD_CAPS=valgrind)"],
 534                 excludedNodelabels: ["NUT_BUILD_CAPS=valgrind=no"],
 535
 536                 dynamatrixAxesVirtualLabelsMap: [
 537                     'BITS': [32, 64],
 538                     'CSTDVERSION_${KEY}': [ ['c': '99', 'cxx': '98'], ['c': '17', 'cxx': '17'] ],
 539                     'CSTDVARIANT': ['gnu'],
 540                     'BUILD_TYPE': ['default-tgt:distcheck-valgrind']
 541                     // BUILD_TYPE=default-withdoc:man
 542                     // BUILD_TYPE=default-tgt:distcheck-light == --with-all=auto --with-ssl=auto --with-doc=auto
 543                     // BUILD_TYPE=default-tgt:distcheck-light + NO_PKG_CONFIG=true ?
 544                     ],
 545                 dynamatrixAxesCommonEnv: [
 546                     ['LANG=C','LC_ALL=C','TZ=UTC','BUILD_WARNFATAL=yes'
 547                      //,'BUILD_WARNOPT=hard'
 548                     ]
 549                     ],
 550                 allowedFailure: [
 551                     dynacfgPipeline.axisCombos_WINDOWS
 552                     ],
 553                 runAllowedFailure: true,
 554                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
 555                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_CPPUNIT_STRICT_C
 556                     + [dynacfgPipeline.axisCombos_WINDOWS_CROSS]
 557                 ], body)
 558             }, // getParStages
 559         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
 560         ] // one slowBuild filter configuration
 561
 562         ,[name: 'A cppcheck analysis build',
 563          disabled: dynacfgPipeline.disableSlowBuildCIBuild,
 564          // NOTE: At the time of this posting, there are a handful of "high"
 565          // priority issues, and hundreds of lesser problems which may overlap
 566          // or not with those reported by other analysers. Having issues to
 567          // report does not mark the builds FAILED nor UNSTABLE however, so
 568          // this should be safe to enable for all branches now. But it makes
 569          // a lot of noise and (bug?) falls into common warnings category,
 570          // not a separate analysis group as was intended by dynamatrix.
 571          // So for now this applies only to fightwarn-related branches.
 572          branchRegexSource: ~/^(PR-.+|fightwarn.*)$/,
 573          //branchRegexTarget: dynacfgPipeline.branchStableRegex,
 574          branchRegexTarget: ~/fightwarn/,
 575          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
 576          'getParStages': { def dynamatrix, Closure body ->
 577             return dynamatrix.generateBuild([
 578                 dynamatrixAxesLabels: ['OS_FAMILY'], // + [ 'OS_DISTRO', 'MAKE'],
 579                 requiredNodelabels: ["(NUT_BUILD_CAPS=cppcheck||NUT_BUILD_CAPS=cppcheck=yes)"],
 580                 excludedNodelabels: ["NUT_BUILD_CAPS=cppcheck=no"],
 581
 582                 dynamatrixAxesVirtualLabelsMap: [
 583                     // For cppcheck we do not iterate BITS,
 584                     // doing one (default) hit per system:
 585                     //'BITS': [32, 64],
 586
 587                     // Take systems that CAN build C; do not really
 588                     // care about revision here since it is hardcoded
 589                     // in the Makefile to create two analysis XMLs now:
 590                     'CSTDVERSION_${KEY}': [ ['c': '17', 'cxx': '17'] ],
 591                     'CSTDVARIANT': ['gnu'],
 592                     'BUILD_TYPE': ['default-tgt:cppcheck']
 593                     ],
 594                 dynamatrixAxesCommonEnv: [
 595                     ['LANG=C','LC_ALL=C','TZ=UTC', 'DO_CLEAN_CHECK=no', 'BUILD_WARNFATAL=yes'
 596                      //,'BUILD_WARNOPT=hard'
 597                     ]
 598                     ],
 599                 allowedFailure: [
 600                     dynacfgPipeline.axisCombos_WINDOWS
 601                     ],
 602                 runAllowedFailure: true,
 603                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
 604                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_CPPUNIT_STRICT_C
 605                     + [dynacfgPipeline.axisCombos_WINDOWS_CROSS]
 606                 ], body)
 607             }, // getParStages
 608         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
 609         ] // one slowBuild filter configuration
 610
 611         ,[name: 'A build with all driver types on capable systems with distcheck for main supported C/C++ revision (must pass)',
 612          // NOTE: Here we constrain distcheck builds (more CI stress load)
 613          // to run as few combos as possible; arguably this filter config
 614          // is more about recipes distributing those drivers than about
 615          // directly codebase quality.
 616          disabled: dynacfgPipeline.disableSlowBuildCIBuild,
 617          //branchRegexSource: ~/^(PR-.+|fightwarn.*)$/,
 618          //branchRegexTarget: dynacfgPipeline.branchStableRegex,
 619          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
 620          'getParStages': { def dynamatrix, Closure body ->
 621             return dynamatrix.generateBuild([
 622                 //commonLabelExpr: "nut-builder:alldrv",
 623                 requiredNodelabels: ["(NUT_BUILD_CAPS=drivers:all||nut-builder:alldrv)"],
 624                 excludedNodelabels: [],
 625
 626                 dynamatrixAxesVirtualLabelsMap: [
 627                     // TODO: Find a way to constrain these builds to one
 628                     // per OS type, whatever bitness(es) supported there,
 629                     // so we really primarily only test the dist-ability.
 630                     'BITS': [32, 64],
 631                     'CSTDVERSION_${KEY}': [ ['c': '99', 'cxx': '98'] ],
 632                     'CSTDVARIANT': ['gnu'],
 633                     'BUILD_TYPE': ['default-alldrv']
 634                     ],
 635                 dynamatrixAxesCommonEnv: [
 636                     ['LANG=C','LC_ALL=C','TZ=UTC','BUILD_WARNFATAL=yes'
 637                      //,'BUILD_WARNOPT=hard'
 638                     ]
 639                     ],
 640                 // On some systems, pkg-config for net-snmp includes CFLAGS values not supported by gcc-4.9 and older
 641                 allowedFailure: [
 642                     dynacfgPipeline.axisCombos_WINDOWS,
 643                     [~/GCCVER=[01234].+/, ~/BUILD_TYPE=default-alldrv/]
 644                     ],
 645                 runAllowedFailure: true,
 646                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
 647                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_STRICT_C
 648                 ], body)
 649             }, // getParStages
 650         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
 651         ] // one slowBuild filter configuration
 652
 653         ,[name: 'A build with all driver types on capable systems without distcheck for other C/C++ revisions (must pass)',
 654          // NOTE: We reduce the build load here since the Makefile recipes
 655          // (for distcheck part) are deemed tested above with the supported
 656          // C/C++ standard revision
 657          disabled: dynacfgPipeline.disableSlowBuildCIBuild,
 658          //branchRegexSource: ~/^(PR-.+|fightwarn.*)$/,
 659          //branchRegexTarget: dynacfgPipeline.branchStableRegex,
 660          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
 661          'getParStages': { def dynamatrix, Closure body ->
 662             return dynamatrix.generateBuild([
 663                 //commonLabelExpr: "nut-builder:alldrv",
 664                 requiredNodelabels: ["(NUT_BUILD_CAPS=drivers:all||nut-builder:alldrv)"],
 665                 excludedNodelabels: [],
 666
 667                 dynamatrixAxesVirtualLabelsMap: [
 668                     'BITS': [32, 64],
 669                     'CSTDVERSION_${KEY}': [ ['c': '99', 'cxx': '98'], ['c': '17', 'cxx': '17'] ],
 670                     'CSTDVARIANT': ['gnu'],
 671                     'BUILD_TYPE': ['default-alldrv:no-distcheck']
 672                     ],
 673                 dynamatrixAxesCommonEnv: [
 674                     ['LANG=C','LC_ALL=C','TZ=UTC','BUILD_WARNFATAL=yes'
 675                      //,'BUILD_WARNOPT=hard'
 676                     ]
 677                     ],
 678                 // On some systems, pkg-config for net-snmp includes CFLAGS values not supported by gcc-4.9 and older
 679                 allowedFailure: [
 680                     dynacfgPipeline.axisCombos_WINDOWS,
 681                     [~/GCCVER=[01234].+/, ~/BUILD_TYPE=default-alldr(v|v:no-distcheck)/]
 682                     ],
 683                 runAllowedFailure: true,
 684                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
 685                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_STRICT_C
 686                     + [dynacfgPipeline.axisCombos_WINDOWS_CROSS]
 687                 ], body)
 688             }, // getParStages
 689         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
 690         ] // one slowBuild filter configuration
 691
 692         ,[name: 'A build with all driver types on capable systems with distcheck and fatal warnings (allowed to fail)',
 693          disabled: dynacfgPipeline.disableSlowBuildCIBuildExperimental,
 694          branchRegexSource: ~/^(PR-.+|.*fightwarn.*)$/,
 695          branchRegexTarget: ~/fightwarn/,
 696          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
 697          'getParStages': { def dynamatrix, Closure body ->
 698             return dynamatrix.generateBuild([
 699                 //commonLabelExpr: "nut-builder:alldrv",
 700                 requiredNodelabels: ["(NUT_BUILD_CAPS=drivers:all||nut-builder:alldrv)"],
 701                 excludedNodelabels: [],
 702
 703                 dynamatrixAxesVirtualLabelsMap: [
 704                     'BITS': [32, 64],
 705                     'CSTDVERSION_${KEY}': [ ['c': '99', 'cxx': '98'], ['c': '99', 'cxx': '11'], ['c': '11', 'cxx': '11'], ['c': '17', 'cxx': '17'] ],
 706                     'CSTDVARIANT': ['gnu'],
 707                     'BUILD_TYPE': ['default-alldrv']
 708                     ],
 709                 dynamatrixAxesCommonEnv: [
 710                     ['LANG=C','LC_ALL=C','TZ=UTC',
 711                      'BUILD_WARNFATAL=yes','BUILD_WARNOPT=hard'
 712                     ]
 713                     ],
 714                 // On some systems, pkg-config for net-snmp includes CFLAGS values not supported by gcc-4.9 and older
 715                 allowedFailure: [
 716                     dynacfgPipeline.axisCombos_WINDOWS,
 717                     [~/BUILD_WARNOPT=hard/],
 718                     [~/GCCVER=[01234].+/, ~/BUILD_TYPE=default-alldrv/]
 719                     ],
 720                 runAllowedFailure: true,
 721                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
 722                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_CPPUNIT_STRICT_C +
 723                     [dynacfgPipeline.axisCombos_COMPILER_GCC_TOO_OLD]
 724                     + [dynacfgPipeline.axisCombos_WINDOWS_CROSS]
 725                 ], body)
 726             }, // getParStages
 727         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
 728         ] // one slowBuild filter configuration
 729
 730         ,[name: 'Various build combos with Python versions for helper scripts',
 731          // This is a recipe (and target OS) test for ability to use helper
 732          // scripts with various Python interpreter versions.
 733          disabled: dynacfgPipeline.disableSlowBuildCIBuild,
 734          //branchRegexSource: ~/^(PR-.+|fightwarn.*)$/,
 735          //branchRegexTarget: dynacfgPipeline.branchStableRegex,
 736          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_PY,
 737          'getParStages': { def dynamatrix, Closure body ->
 738             return dynamatrix.generateBuild([
 739                 //commonLabelExpr: "nut-builder:alldrv",
 740                 // TOTHINK: Should we also vary compilers here?
 741                 dynamatrixAxesLabels: ['OS_FAMILY', 'OS_DISTRO', 'MAKE', 'PYTHON'],
 742                 requiredNodelabels: ["(NUT_BUILD_CAPS=drivers:all||nut-builder:alldrv)"],
 743                 excludedNodelabels: [],
 744
 745                 dynamatrixAxesVirtualLabelsMap: [
 746                     //'BITS': [32, 64],
 747                     'CSTDVERSION_${KEY}': [ ['c': '99', 'cxx': '98'] ],
 748                     'CSTDVARIANT': ['gnu'],
 749                     'BUILD_TYPE': ['default-alldrv']
 750                     ],
 751                 dynamatrixAxesCommonEnv: [
 752                     ['LANG=C','LC_ALL=C','TZ=UTC',
 753                      'BUILD_WARNFATAL=yes'
 754                      //,'BUILD_WARNOPT=medium'
 755                     ]
 756                     ],
 757                 allowedFailure: [
 758                     dynacfgPipeline.axisCombos_WINDOWS
 759                     ],
 760                 runAllowedFailure: true,
 761                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesLabels': 'replace', 'commonLabelExpr': 'replace', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
 762                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_CPPUNIT_STRICT_C +
 763                     [dynacfgPipeline.axisCombos_COMPILER_GCC_TOO_OLD]
 764                     + [dynacfgPipeline.axisCombos_WINDOWS_CROSS]
 765                 ], body)
 766             }, // getParStages
 767         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
 768         ] // one slowBuild filter configuration
 769
 770         ,[name: 'An out-of-tree build with all docs types on capable systems, and a distcheck (must pass)',
 771          // TODO: This is a recipe (and target OS) test for ability to build
 772          // the docs without error; it should not iterate compilers (maybe
 773          // iterate docs tools though, if we were to support many backends?)
 774          disabled: dynacfgPipeline.disableSlowBuildCIBuild,
 775          //branchRegexSource: ~/^(PR-.+|fightwarn.*)$/,
 776          //branchRegexTarget: dynacfgPipeline.branchStableRegex,
 777          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_DOC,
 778          'getParStages': { def dynamatrix, Closure body ->
 779             return dynamatrix.generateBuild([
 780                 //commonLabelExpr: dynacfgBase.commonLabelExpr + " && doc-builder",
 781                 //commonLabelExpr: infra.labelDocumentationWorker(),
 782                 dynamatrixAxesLabels: ['OS_FAMILY', 'OS_DISTRO', 'MAKE'],
 783                 requiredNodelabels: ["NUT_BUILD_CAPS=docs:all"],
 784                 excludedNodelabels: [],
 785
 786                 dynamatrixAxesVirtualLabelsMap: [
 787                     //'BITS': [32, 64],
 788                     'CSTDVERSION_${KEY}': [ ['c': '99', 'cxx': '98'] ],
 789                     'CSTDVARIANT': ['gnu'],
 790                     'BUILD_TYPE': ['default-withdoc']
 791                     ],
 792                 dynamatrixAxesCommonEnv: [
 793                     ['LANG=C','LC_ALL=C','TZ=UTC',
 794                      // Build in a subdirectory to check that out-of-dir
 795                      // builds are healthy too.
 796                      // NOTE: It would be useful to also have a recipe to build
 797                      // "completely out-of-tree", in a different filesystem (to
 798                      // make sure we do not rely on hard-links, relative paths,
 799                      // etc.)
 800                      'CI_BUILDDIR=obj',
 801                      'BUILD_WARNFATAL=yes'
 802                      //,'BUILD_WARNOPT=minimal'
 803                     ]
 804                     ],
 805                 allowedFailure: [
 806                     dynacfgPipeline.axisCombos_WINDOWS
 807                     ],
 808                 runAllowedFailure: true,
 809                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesLabels': 'replace', 'commonLabelExpr': 'replace', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
 810                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_CPPUNIT_STRICT_C +
 811                     [dynacfgPipeline.axisCombos_COMPILER_GCC_TOO_OLD]
 812                     + [dynacfgPipeline.axisCombos_WINDOWS_CROSS]
 813                 ], body)
 814             }, // getParStages
 815         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
 816         ] // one slowBuild filter configuration
 817
 818         ,[name: 'A build with manpage docs on all systems that did not build "all docs", and a distcheck (allowed to fail - e.g. no tools even for that)',
 819          // TODO: This is a recipe (and target OS) test for ability to build
 820          // the docs without error; it should not iterate compilers; see above
 821          disabled: dynacfgPipeline.disableSlowBuildCIBuild,
 822          //branchRegexSource: ~/^(PR-.+|fightwarn.*)$/,
 823          //branchRegexTarget: dynacfgPipeline.branchStableRegex,
 824          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_TXT,
 825          'getParStages': { def dynamatrix, Closure body ->
 826             return dynamatrix.generateBuild([
 827                 //commonLabelExpr: dynacfgBase.commonLabelExpr + " && doc-builder",
 828                 //commonLabelExpr: infra.labelDocumentationWorker(),
 829                 dynamatrixAxesLabels: ['OS_FAMILY', 'OS_DISTRO', 'MAKE'],
 830                 requiredNodelabels: ["NUT_BUILD_CAPS=docs:man"],
 831                 excludedNodelabels: ["NUT_BUILD_CAPS=docs:all"],
 832
 833                 dynamatrixAxesVirtualLabelsMap: [
 834                     //'BITS': [32, 64],
 835                     'CSTDVERSION_${KEY}': [ ['c': '99', 'cxx': '98'] ],
 836                     'CSTDVARIANT': ['gnu'],
 837                     'BUILD_TYPE': ['default-withdoc:man']
 838                     ],
 839                 dynamatrixAxesCommonEnv: [
 840                     ['LANG=C','LC_ALL=C','TZ=UTC',
 841                      'BUILD_WARNFATAL=yes'
 842                      //,'BUILD_WARNOPT=minimal'
 843                     ]
 844                     ],
 845                 allowedFailure: [
 846                     dynacfgPipeline.axisCombos_WINDOWS,
 847                     [~/BUILD_TYPE=default-withdoc:man/]
 848                     ],
 849                 runAllowedFailure: true,
 850                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesLabels': 'replace', 'commonLabelExpr': 'replace', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
 851                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_CPPUNIT_STRICT_C +
 852                     [dynacfgPipeline.axisCombos_COMPILER_GCC_TOO_OLD]
 853                     + [dynacfgPipeline.axisCombos_WINDOWS_CROSS]
 854                 ], body)
 855             }, // getParStages
 856         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
 857         ] // one slowBuild filter configuration
 858
 859         ,[name: 'GNU C standard builds with non-fatal warnings, without distcheck and docs (must pass)',
 860          disabled: dynacfgPipeline.disableSlowBuildCIBuild,
 861          //branchRegexSource: ~/^(PR-.+|fightwarn.*)$/,
 862          //branchRegexTarget: dynacfgPipeline.branchStableRegex,
 863          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
 864          'getParStages': { def dynamatrix, Closure body ->
 865             return dynamatrix.generateBuild([
 866                 requiredNodelabels: [],
 867                 excludedNodelabels: [],
 868
 869                 // NOTE: C89 not included here as its warnings are quite
 870                 // noisy as in "not relevant for more modern revisions".
 871                 // It has a separate slowBuild filter configuration below.
 872                 dynamatrixAxesVirtualLabelsMap: [
 873                     'BITS': [32, 64],
 874                     'CSTDVERSION_${KEY}': [ ['c': '99', 'cxx': '11'], ['c': '11', 'cxx': '11'], ['c': '17', 'cxx': '17'] ],
 875                     'CSTDVARIANT': ['gnu'],
 876                     ],
 877                 dynamatrixAxesCommonEnv: [
 878                     ['LANG=C','LC_ALL=C','TZ=UTC',
 879                      'BUILD_TYPE=default-all-errors',
 880                      'BUILD_WARNFATAL=no','BUILD_WARNOPT=auto'
 881                     ]
 882                     ],
 883                 allowedFailure: [
 884                     dynacfgPipeline.axisCombos_WINDOWS
 885                     ],
 886                 runAllowedFailure: true,
 887                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
 888                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_CPPUNIT_STRICT_C
 889                     + [dynacfgPipeline.axisCombos_WINDOWS_CROSS]
 890                 ], body)
 891             }, // getParStages
 892         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
 893         ] // one slowBuild filter configuration
 894
 895         ,[name: 'GNU C89 standard builds with non-fatal warnings and GCC toolkits, without distcheck and docs (must pass)',
 896          disabled: dynacfgPipeline.disableSlowBuildCIBuild,
 897          //disabled: dynacfgPipeline.disableSlowBuildCIBuildExperimentalANSI,
 898          //branchRegexSource: ~/^(PR-.+|.*fightwarn.*89.*)$/,
 899          //branchRegexTarget: ~/fightwarn.*89.*/,
 900          //disabled: dynacfgPipeline.disableSlowBuildCIBuildExperimental,
 901          //branchRegexSource: ~/^(PR-.+|.*fightwarn.*)$/,
 902          //branchRegexTarget: ~/fightwarn.*/,
 903          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
 904          'getParStages': { def dynamatrix, Closure body ->
 905             return dynamatrix.generateBuild([
 906                 requiredNodelabels: [],
 907                 excludedNodelabels: [],
 908
 909                 dynamatrixAxesVirtualLabelsMap: [
 910                     'BITS': [32, 64],
 911                     'CSTDVERSION_${KEY}': [ ['c': '89', 'cxx': '98'] ],
 912                     'CSTDVARIANT': ['gnu'],
 913                     ],
 914                 dynamatrixAxesCommonEnv: [
 915                     ['LANG=C','LC_ALL=C','TZ=UTC',
 916                      'BUILD_TYPE=default-all-errors',
 917                      'BUILD_WARNFATAL=no','BUILD_WARNOPT=auto'
 918                     ]
 919                     ],
 920                 allowedFailure: [
 921                     dynacfgPipeline.axisCombos_WINDOWS
 922                     ],
 923                 runAllowedFailure: true,
 924                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
 925                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_CPPUNIT_STRICT_C +
 926                     [dynacfgPipeline.axisCombos_COMPILER_NOT_GCC]
 927                     + [dynacfgPipeline.axisCombos_WINDOWS_CROSS]
 928                 ], body)
 929             }, // getParStages
 930         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
 931         ] // one slowBuild filter configuration
 932
 933         ,[name: 'GNU C89 standard builds with non-fatal warnings and non-GCC toolkits, without distcheck and docs (must pass)',
 934          disabled: dynacfgPipeline.disableSlowBuildCIBuildExperimentalANSI,
 935          branchRegexSource: ~/^(PR-.+|.*fightwarn.*89.*)$/,
 936          branchRegexTarget: ~/fightwarn.*89.*/,
 937          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
 938          'getParStages': { def dynamatrix, Closure body ->
 939             return dynamatrix.generateBuild([
 940                 requiredNodelabels: [],
 941                 excludedNodelabels: [],
 942
 943                 dynamatrixAxesVirtualLabelsMap: [
 944                     'BITS': [32, 64],
 945                     'CSTDVERSION_${KEY}': [ ['c': '89', 'cxx': '98'] ],
 946                     'CSTDVARIANT': ['gnu'],
 947                     ],
 948                 dynamatrixAxesCommonEnv: [
 949                     ['LANG=C','LC_ALL=C','TZ=UTC',
 950                      'BUILD_TYPE=default-all-errors',
 951                      'BUILD_WARNFATAL=no','BUILD_WARNOPT=auto'
 952                     ]
 953                     ],
 954                 allowedFailure: [
 955                     dynacfgPipeline.axisCombos_WINDOWS
 956                     ],
 957                 runAllowedFailure: true,
 958                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
 959                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_CPPUNIT_STRICT_C +
 960                     [dynacfgPipeline.axisCombos_COMPILER_GCC]
 961                     + [dynacfgPipeline.axisCombos_WINDOWS_CROSS]
 962                 ], body)
 963             }, // getParStages
 964         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
 965         ] // one slowBuild filter configuration
 966
 967         ,[name: 'GNU C standard builds with non-fatal warnings, without distcheck and docs, one compiler with main supported C/C++ revision on slower QEMU builders (may fail due to those workers)',
 968          disabled: dynacfgPipeline.disableSlowBuildCIBuild_QEMU,
 969          //branchRegexSource: ~/^(PR-.+|fightwarn.*|.*qemu.*)$/,
 970          //branchRegexTarget: ~/^(master|main|stable|.*qemu.*)$/,
 971          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
 972          'getParStages': { def dynamatrix, Closure body ->
 973             return dynamatrix.generateBuild([
 974                 //commonLabelExpr: "qemu-" + dynacfgBase.commonLabelExpr,
 975                 commonLabelExpr: "qemu-nut-builder || ssh-qemu-nut-builder",
 976                 dynamatrixAxesLabels: ['OS_FAMILY', 'OS_DISTRO', 'ARCH${ARCH_BITS}', 'COMPILER'],
 977                 requiredNodelabels: ["(NUT_BUILD_CAPS=drivers:all||qemu-nut-builder:alldrv)"],
 978                 excludedNodelabels: [],
 979
 980                 dynamatrixAxesVirtualLabelsMap: [
 981                     'BITS': [32, 64],
 982                     'CSTDVERSION_${KEY}': [ ['c': '99', 'cxx': '11'] ],
 983                     'CSTDVARIANT': ['gnu'],
 984                     ],
 985                 dynamatrixAxesCommonEnv: [
 986                     ['LANG=C','LC_ALL=C','TZ=UTC',
 987                      'BUILD_TYPE=default-all-errors',
 988                      'BUILD_WARNFATAL=no','BUILD_WARNOPT=auto'
 989                     ]
 990                     ],
 991                 allowedFailure: [
 992                     //[~/ARCH(32|64)=(?!i386|amd64))/],
 993                     //[~/OS_FAMILY=windows/]
 994                     [~/OS_FAMILY=.+/]
 995                     ],
 996                 runAllowedFailure: true,
 997                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesCommonEnv': 'replace', 'dynamatrixAxesLabels': 'replace', 'commonLabelExpr': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
 998                 excludeCombos: [
 999                     [~/BITS=32/, ~/ARCH_BITS=64/], [~/BITS=64/, ~/ARCH_BITS=32/],
1000                     [~/CSTDVARIANT=c/],
1001                     [~/OS_DISTRO=(openindiana|freebsd).*/, ~/CSTDVERSION_cxx=[12].+/, ~/COMPILER=GCC/],
1002                     dynacfgPipeline.axisCombos_WINDOWS_CROSS
1003                     ]
1004                 ], body)
1005             }, // getParStages
1006         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
1007         ] // one slowBuild filter configuration
1008
1009         ,[name: 'GNU C standard builds with fatal warnings, without distcheck and docs (allowed to fail with non-GCC compilers)',
1010          disabled: dynacfgPipeline.disableSlowBuildCIBuildExperimental,
1011          branchRegexSource: ~/^(PR-.+|.*fightwarn.*)$/,
1012          branchRegexTarget: ~/fightwarn/,
1013          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
1014          'getParStages': { def dynamatrix, Closure body ->
1015             return dynamatrix.generateBuild([
1016                 requiredNodelabels: [],
1017                 excludedNodelabels: [],
1018
1019                 // NOTE: C89 not included here as its warnings are quite
1020                 // noisy as in "not relevant for more modern revisions".
1021                 // It has a separate slowBuild filter configuration below.
1022                 dynamatrixAxesVirtualLabelsMap: [
1023                     'BITS': [32, 64],
1024                     'CSTDVERSION_${KEY}': [ ['c': '99', 'cxx': '98'], ['c': '99', 'cxx': '11'], ['c': '11', 'cxx': '11'], ['c': '17', 'cxx': '17'] ],
1025                     'CSTDVARIANT': ['gnu'],
1026                     ],
1027                 dynamatrixAxesCommonEnv: [
1028                     ['LANG=C','LC_ALL=C','TZ=UTC',
1029                      'BUILD_TYPE=default-all-errors',
1030                      'BUILD_WARNFATAL=yes','BUILD_WARNOPT=hard'
1031                     ]
1032                     ],
1033                 allowedFailure: [
1034                     dynacfgPipeline.axisCombos_WINDOWS,
1035                     [~/BUILD_WARNOPT=hard/]
1036                     ],
1037                 runAllowedFailure: true,
1038                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
1039                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_CPPUNIT_STRICT_C +
1040                     [dynacfgPipeline.axisCombos_COMPILER_GCC_TOO_OLD]
1041                     + [dynacfgPipeline.axisCombos_WINDOWS_CROSS]
1042                     //+ [ [~/COMPILER=GCC/, ~/CSTDVERSION_KEY=(?!89)/] ]
1043                 ], body)
1044             }, // getParStages
1045         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
1046         ] // one slowBuild filter configuration
1047
1048         ,[name: 'GNU C89 standard builds with fatal warnings with non-GCC compilers, without distcheck and docs (allowed to fail)',
1049          // NOTE: This build scenario is quite noisy with regard to warnings
1050          // analysis and not too beneficial unless someone looking at the
1051          // logs is actively fixing the C89 compatibility. So off by default,
1052          // and would only run for a PR against a "fightwarn.*89.*" named
1053          // branch or for a build of such branch.
1054          disabled: dynacfgPipeline.disableSlowBuildCIBuildExperimentalANSI,
1055          branchRegexSource: ~/^(PR-.+|.*fightwarn.*89.*)$/,
1056          branchRegexTarget: ~/fightwarn.*89.*/,
1057          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
1058          'getParStages': { def dynamatrix, Closure body ->
1059             return dynamatrix.generateBuild([
1060                 requiredNodelabels: [],
1061                 excludedNodelabels: [],
1062
1063                 dynamatrixAxesVirtualLabelsMap: [
1064                     'BITS': [32, 64],
1065                     'CSTDVERSION_${KEY}': [ ['c': '89', 'cxx': '98'] ],
1066                     'CSTDVARIANT': ['gnu'],
1067                     ],
1068                 dynamatrixAxesCommonEnv: [
1069                     ['LANG=C','LC_ALL=C','TZ=UTC',
1070                      'BUILD_TYPE=default-all-errors',
1071                      'BUILD_WARNFATAL=yes','BUILD_WARNOPT=hard'
1072                     ]
1073                     ],
1074                 allowedFailure: [
1075                     dynacfgPipeline.axisCombos_WINDOWS,
1076                     [~/BUILD_WARNOPT=hard/]
1077                     ],
1078                 runAllowedFailure: true,
1079                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
1080                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_CPPUNIT_STRICT_C + [
1081                     [~/COMPILER=GCC/, ~/CSTDVERSION_KEY=(?!89)/]
1082                     ] +
1083                     [dynacfgPipeline.axisCombos_COMPILER_GCC_TOO_OLD]
1084                     + [dynacfgPipeline.axisCombos_WINDOWS_CROSS]
1085                 ], body)
1086             }, // getParStages
1087         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
1088         ] // one slowBuild filter configuration
1089
1090         ,[name: 'GNU C standard out-of-tree builds with fatal warnings with GCC, without distcheck and docs (must pass)',
1091          disabled: dynacfgPipeline.disableSlowBuildCIBuild,
1092          //branchRegexSource: ~/^(PR-.+|fightwarn.*)$/,
1093          //branchRegexTarget: dynacfgPipeline.branchStableRegex,
1094          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
1095          'getParStages': { def dynamatrix, Closure body ->
1096             return dynamatrix.generateBuild([
1097                 requiredNodelabels: [],
1098                 excludedNodelabels: [],
1099
1100                 dynamatrixAxesVirtualLabelsMap: [
1101                     'BITS': [32, 64],
1102                     'CSTDVERSION_${KEY}': [ ['c': '99', 'cxx': '98'], ['c': '99', 'cxx': '11'], ['c': '11', 'cxx': '11'], ['c': '17', 'cxx': '17'] ],
1103                     'CSTDVARIANT': ['gnu'],
1104                     ],
1105                 dynamatrixAxesCommonEnv: [
1106                     ['LANG=C','LC_ALL=C','TZ=UTC',
1107                      'BUILD_TYPE=default-all-errors',
1108                      'BUILD_WARNFATAL=yes',
1109                      // Build in a subdirectory to check that out-of-dir
1110                      // builds are healthy too
1111                      'CI_BUILDDIR=obj',
1112                      // NOTE: "gcc-hard" warnings are still not as picky
1113                      // as "clang-hard" and do not differ much from the
1114                      // "gcc-medium" definition currently:
1115                      'BUILD_WARNOPT=hard'
1116                     ]
1117                     ],
1118                 allowedFailure: [
1119                     dynacfgPipeline.axisCombos_WINDOWS
1120                     ],
1121                 runAllowedFailure: true,
1122                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
1123                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_CPPUNIT_STRICT_C + [
1124                     [~/COMPILER=(?!GCC)/]
1125                     ] +
1126                     [dynacfgPipeline.axisCombos_COMPILER_GCC_TOO_OLD]
1127                     + [dynacfgPipeline.axisCombos_WINDOWS_CROSS]
1128                 ], body)
1129             }, // getParStages
1130         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
1131         ] // one slowBuild filter configuration
1132
1133         ,[name: 'Strict C standard builds on non-Windows platforms, without distcheck and docs (allowed to fail)',
1134          disabled: dynacfgPipeline.disableSlowBuildCIBuildExperimental,
1135          branchRegexSource: ~/^(PR-.+|.*fightwarn.*)$/,
1136          branchRegexTarget: ~/fightwarn/,
1137          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
1138          'getParStages': { def dynamatrix, Closure body ->
1139             return dynamatrix.generateBuild([
1140                 requiredNodelabels: [],
1141                 excludedNodelabels: [],
1142
1143                 dynamatrixAxesVirtualLabelsMap: [
1144                     'BITS': [32, 64],
1145                     'CSTDVERSION_${KEY}': [ ['c': '99', 'cxx': '98'], ['c': '99', 'cxx': '11'], ['c': '11', 'cxx': '11'], ['c': '17', 'cxx': '17'] ],
1146                     'CSTDVARIANT': ['c'],
1147                     ],
1148                 dynamatrixAxesCommonEnv: [],
1149                 dynamatrixAxesCommonEnvCartesian: [
1150                     [ ['LANG=C','LC_ALL=C','TZ=UTC', 'BUILD_TYPE=default-all-errors'] ],
1151                     [ ['BUILD_WARNFATAL=yes','BUILD_WARNOPT=hard'], ['BUILD_WARNFATAL=no','BUILD_WARNOPT=minimal'] ]
1152                     ],
1153                 allowedFailure: [
1154                     dynacfgPipeline.axisCombos_STRICT_C,
1155                     dynacfgPipeline.axisCombos_COMPILER_GCC_TOO_OLD,
1156                     [~/BUILD_WARNOPT=hard/]
1157                     ],
1158                 runAllowedFailure: true,
1159                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
1160                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_CPPUNIT + [
1161                     dynacfgPipeline.axisCombos_GNU_C,
1162                     dynacfgPipeline.axisCombos_WINDOWS_CROSS,
1163                     dynacfgPipeline.axisCombos_WINDOWS
1164                     ]
1165                 ], body)
1166             }, // getParStages
1167         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
1168         ] // one slowBuild filter configuration
1169
1170         ,[name: 'Strict ANSI C (C89/C90) standard builds on non-Windows platforms and GCC toolkits, without distcheck and docs (allowed to fail)',
1171          //disabled: dynacfgPipeline.disableSlowBuildCIBuildExperimentalANSI,
1172          //branchRegexSource: ~/^(PR-.+|.*fightwarn.*89.*)$/,
1173          //branchRegexTarget: ~/fightwarn.*89.*/,
1174          disabled: dynacfgPipeline.disableSlowBuildCIBuildExperimental,
1175          branchRegexSource: ~/^(PR-.+|.*fightwarn.*)$/,
1176          branchRegexTarget: ~/fightwarn.*/,
1177          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
1178          'getParStages': { def dynamatrix, Closure body ->
1179             return dynamatrix.generateBuild([
1180                 requiredNodelabels: [],
1181                 excludedNodelabels: [],
1182
1183                 dynamatrixAxesVirtualLabelsMap: [
1184                     'BITS': [32, 64],
1185                     'CSTDVERSION_${KEY}': [ 'ansi' ],
1186                     'CSTDVARIANT': ['c'],
1187                     ],
1188                 dynamatrixAxesCommonEnv: [],
1189                 dynamatrixAxesCommonEnvCartesian: [
1190                     [ ['LANG=C','LC_ALL=C','TZ=UTC', 'BUILD_TYPE=default-all-errors'] ],
1191                     [ ['BUILD_WARNFATAL=yes','BUILD_WARNOPT=hard'], ['BUILD_WARNFATAL=no','BUILD_WARNOPT=minimal'] ]
1192                     ],
1193                 allowedFailure: [
1194                     dynacfgPipeline.axisCombos_STRICT_C,
1195                     dynacfgPipeline.axisCombos_COMPILER_GCC_TOO_OLD,
1196                     [~/BUILD_WARNOPT=hard/]
1197                     ],
1198                 runAllowedFailure: true,
1199                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
1200                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_CPPUNIT + [
1201                     dynacfgPipeline.axisCombos_GNU_C,
1202                     dynacfgPipeline.axisCombos_WINDOWS_CROSS,
1203                     dynacfgPipeline.axisCombos_WINDOWS
1204                     ] + [dynacfgPipeline.axisCombos_COMPILER_NOT_GCC]
1205                 ], body)
1206             }, // getParStages
1207         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
1208         ] // one slowBuild filter configuration
1209
1210         ,[name: 'Strict ANSI C (C89/C90) standard builds on non-Windows platforms and non-GCC toolkits, without distcheck and docs (allowed to fail)',
1211          disabled: dynacfgPipeline.disableSlowBuildCIBuildExperimentalANSI,
1212          branchRegexSource: ~/^(PR-.+|.*fightwarn.*89.*)$/,
1213          branchRegexTarget: ~/fightwarn.*89.*/,
1214          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
1215          'getParStages': { def dynamatrix, Closure body ->
1216             return dynamatrix.generateBuild([
1217                 requiredNodelabels: [],
1218                 excludedNodelabels: [],
1219
1220                 dynamatrixAxesVirtualLabelsMap: [
1221                     'BITS': [32, 64],
1222                     'CSTDVERSION_${KEY}': [ 'ansi' ],
1223                     'CSTDVARIANT': ['c'],
1224                     ],
1225                 dynamatrixAxesCommonEnv: [],
1226                 dynamatrixAxesCommonEnvCartesian: [
1227                     [ ['LANG=C','LC_ALL=C','TZ=UTC', 'BUILD_TYPE=default-all-errors'] ],
1228                     [ ['BUILD_WARNFATAL=yes','BUILD_WARNOPT=hard'], ['BUILD_WARNFATAL=no','BUILD_WARNOPT=minimal'] ]
1229                     ],
1230                 allowedFailure: [
1231                     dynacfgPipeline.axisCombos_STRICT_C,
1232                     dynacfgPipeline.axisCombos_COMPILER_GCC_TOO_OLD,
1233                     [~/BUILD_WARNOPT=hard/]
1234                     ],
1235                 runAllowedFailure: true,
1236                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
1237                 excludeCombos: dynacfgPipeline.excludeCombos_DEFAULT_CPPUNIT + [
1238                     dynacfgPipeline.axisCombos_GNU_C,
1239                     dynacfgPipeline.axisCombos_WINDOWS_CROSS,
1240                     dynacfgPipeline.axisCombos_WINDOWS
1241                     ] + [dynacfgPipeline.axisCombos_COMPILER_GCC]
1242                 ], body)
1243             }, // getParStages
1244         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
1245         ] // one slowBuild filter configuration
1246
1247         ,[name: 'Strict C and GNU standard builds on native-Windows platforms, without distcheck and docs (allowed to fail)',
1248          disabled: dynacfgPipeline.disableSlowBuildCIBuildExperimental,
1249          branchRegexSource: ~/^(PR-.+|.*fightwarn.*|.*Windows.*)$/,
1250          branchRegexTarget: ~/fightwarn|Windows-.*/,
1251          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
1252          'getParStages': { def dynamatrix, Closure body ->
1253             return dynamatrix.generateBuild([
1254                 requiredNodelabels: [],
1255                 excludedNodelabels: [],
1256
1257                 dynamatrixAxesVirtualLabelsMap: [
1258                     'BITS': [32, 64],
1259                     'CSTDVERSION_${KEY}': [ ['c': '99', 'cxx': '98'], ['c': '99', 'cxx': '11'], ['c': '17', 'cxx': '17'] ],
1260                     'CSTDVARIANT': ['c', 'gnu'],
1261                     ],
1262                 dynamatrixAxesCommonEnv: [
1263                     ['LANG=C','LC_ALL=C','TZ=UTC',
1264                      'BUILD_TYPE=default-all-errors',
1265                      'BUILD_WARNFATAL=yes','BUILD_WARNOPT=hard',
1266                      'CPPFLAGS=-fms-extensions'
1267                     ]
1268                     ],
1269                 allowedFailure: [
1270                     dynacfgPipeline.axisCombos_STRICT_C,
1271                     dynacfgPipeline.axisCombos_WINDOWS,
1272                     [~/BUILD_WARNOPT=hard/]
1273                     ],
1274                 runAllowedFailure: true,
1275                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
1276                 excludeCombos: [
1277                     dynacfgPipeline.axisCombos_ARCH32x64,
1278                     dynacfgPipeline.axisCombos_ARCH64x32,
1279                     dynacfgPipeline.axisCombos_COMPILER_GCC_TOO_OLD,
1280                     dynacfgPipeline.axisCombos_WINDOWS_CROSS,
1281                     dynacfgPipeline.axisCombos_NOT_WINDOWS
1282                     ]
1283                 ], body)
1284             }, // getParStages
1285         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
1286         ] // one slowBuild filter configuration
1287
1288         ,[name: (dynacfgPipeline.disableStrictCIBuild_CrossWindows ? '' : 'Strict C and ') + 'GNU standard builds on cross-Windows platforms (Linux+mingw), without distcheck and docs (allowed to fail)',
1289          disabled: dynacfgPipeline.disableSlowBuildCIBuild,
1290          //branchRegexSource: ~/^(PR-.+|.*fightwarn.*|.*Windows.*)$/,
1291          //branchRegexTarget: ~/fightwarn|Windows-.*/,
1292          appliesToChangedFilesRegex: dynacfgPipeline.appliesToChangedFilesRegex_C,
1293          'getParStages': { def dynamatrix, Closure body ->
1294             return dynamatrix.generateBuild([
1295                 commonLabelExpr: "cross-windows-nut-builder",
1296                 dynamatrixAxesLabels: ['OS_FAMILY', 'OS_DISTRO', 'ARCH${ARCH_BITS}', 'COMPILER'],
1297                 requiredNodelabels: ["NUT_BUILD_CAPS=cross-windows-mingw"],
1298                 excludedNodelabels: [],
1299
1300                 dynamatrixAxesVirtualLabelsMap: [
1301                     'BITS': [64, 32],
1302                     'CSTDVERSION_${KEY}': [ ['c': '99', 'cxx': '11'] ],
1303                     'CSTDVARIANT': ['gnu'] + (dynacfgPipeline.disableStrictCIBuild_CrossWindows ? [] : ['c']),
1304                     ],
1305                 dynamatrixAxesCommonEnv: [
1306                     ['LANG=C','LC_ALL=C','TZ=UTC',
1307                      'BUILD_TYPE=cross-windows-mingw',
1308                      // Note: warnings options are currently ignored in ci_build.sh
1309                      // for this BUILD_TYPE (technically in build-mingw-nut.sh)
1310                      'BUILD_WARNFATAL=yes','BUILD_WARNOPT=hard'
1311                     ]
1312                     ],
1313                 allowedFailure: [
1314                     dynacfgPipeline.axisCombos_STRICT_C,
1315                     //dynacfgPipeline.axisCombos_WINDOWS_CROSS,
1316                     //[~/BUILD_WARNOPT=hard/]
1317                     ],
1318                 runAllowedFailure: true,
1319                 mergeMode: [ 'excludeCombos': 'merge', 'dynamatrixAxesCommonEnv': 'replace' ], // NOTE: We might want to replace other fields, but excludeCombos must be merged to filter compiler versions vs language standards as centrally defined!
1320                 excludeCombos: [
1321                     dynacfgPipeline.axisCombos_ARCH32x64,
1322                     dynacfgPipeline.axisCombos_ARCH64x32,
1323                     dynacfgPipeline.axisCombos_COMPILER_GCC_TOO_OLD,
1324                     dynacfgPipeline.axisCombos_WINDOWS,
1325                     dynacfgPipeline.axisCombos_NOT_WINDOWS_CROSS
1326                     ]
1327                 ], body)
1328             }, // getParStages
1329         'bodyParStages': dynacfgPipeline.slowBuildDefaultBody_ci_build
1330         ] // one slowBuild filter configuration
1331     ]
1332
1333     dynacfgPipeline.notifyHandler = {
1334         def summary = null
1335         try {
1336             summary = dynamatrix.toStringStageCountDump()
1337         } catch (Throwable t) {}
1338
1339         if (summary == null || summary == "") {
1340             ircNotify (notificationStrategy:'FAILURE_AND_FIXED')
1341         } else {
1342             ircNotify (notificationStrategy:'FAILURE_AND_FIXED', customMessage: summary)
1343         }
1344     }
1345
1346 @NonCPS
1347 def stageNameFunc_ShellcheckCustom(DynamatrixSingleBuildConfig dsbc) {
1348     // NOTE: A direct Closure seems to confuse Jenkins/Groovy CPS, so using a func
1349     def labelMap = dsbc.getKVMap(false)
1350     String sn = ""
1351     if (labelMap.containsKey("OS_FAMILY"))
1352         sn += labelMap.OS_FAMILY + "-"
1353     if (labelMap.containsKey("OS_DISTRO"))
1354         sn += labelMap.OS_DISTRO + "-"
1355     return "MATRIX_TAG=\"${sn}shellcheckCustom\""
1356 }
1357 //dynacfgPipeline.shellcheck.stageNameFunc = this.&stageNameFunc_ShellcheckCustom
1358
1359 ///////////////////////////////////////////////////////////////////////////
1360
1361 // Hacky big switch for a max debug option
1362 //if (true)  // <<< (Un-)comment away in select runs/branches
1363 //if (false) // <<< (Un-)comment away in select runs/branches
1364 if ( env?.BRANCH_NAME ==~ /.*verbose.*/ )
1365 {
1366     dynamatrixGlobalState.enableDebugTrace = true
1367     dynamatrixGlobalState.enableDebugErrors = true
1368     dynamatrixGlobalState.enableDebugMilestones = true
1369     dynamatrixGlobalState.enableDebugMilestonesDetails = true
1370     dynamatrixGlobalState.enableDebugTraceGithubStatusHighlights = true
1371 }
1372
1373 //if (true)  // <<< (Un-)comment away in select runs/branches
1374 //if (false) // <<< (Un-)comment away in select runs/branches
1375 if ( env?.BRANCH_NAME ==~ /.*fightwarn.*/ )
1376 {
1377     dynamatrixGlobalState.enableDebugTraceGithubStatusHighlights = true
1378 }
1379
1380 dynamatrixPipeline(dynacfgBase, dynacfgPipeline)