script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   7 import os
   8 import tarfile
   9 import sys
  10 import time
  11 import random
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from sysconfig import get_path
  20 import platform
  21
  22 import logging
  23
  24 try:
  25     from waflib.Build import CACHE_SUFFIX
  26 except ImportError:
  27     sys.path.insert(0, "./third_party/waf")
  28     from waflib.Build import CACHE_SUFFIX
  29
  30 logging.basicConfig(format='%(asctime)s %(message)s')
  31 logger = logging.getLogger('autobuild')
  32 logger.setLevel(logging.INFO)
  33
  34 os.environ["PYTHONUNBUFFERED"] = "1"
  35
  36 # This speeds up testing remarkably.
  37 os.environ['TDB_NO_FSYNC'] = '1'
  38
  39 # allow autobuild to run within git rebase -i
  40 if "GIT_DIR" in os.environ:
  41     del os.environ["GIT_DIR"]
  42 if "GIT_WORK_TREE" in os.environ:
  43     del os.environ["GIT_WORK_TREE"]
  44
  45 def find_git_root():
  46     '''get to the top of the git repo'''
  47     p = os.getcwd()
  48     while p != '/':
  49         if os.path.exists(os.path.join(p, ".git")):
  50             return p
  51         p = os.path.abspath(os.path.join(p, '..'))
  52     return None
  53
  54
  55 gitroot = find_git_root()
  56 if gitroot is None:
  57     raise Exception("Failed to find git root")
  58
  59
  60 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  61
  62 parser = OptionParser()
  63 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  64 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  65 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  66 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  67 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  68                   default=def_testbase)
  69 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  70                   default=None)
  71 parser.add_option("--passcmd", help="command to run on success", default=None)
  72 parser.add_option("--verbose", help="show all commands as they are run",
  73                   default=False, action="store_true")
  74 parser.add_option("--rebase", help="rebase on the given tree before testing",
  75                   default=None, type='str')
  76 parser.add_option("--pushto", help="push to a git url on success",
  77                   default=None, type='str')
  78 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  79                   default=False, action="store_true")
  80 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  81                   default=False, action="store_true")
  82 parser.add_option("--retry", help="automatically retry if master changes",
  83                   default=False, action="store_true")
  84 parser.add_option("--email", help="send email to the given address on failure",
  85                   type='str', default=None)
  86 parser.add_option("--email-from", help="send email from the given address",
  87                   type='str', default="autobuild@samba.org")
  88 parser.add_option("--email-server", help="send email via the given server",
  89                   type='str', default='localhost')
  90 parser.add_option("--always-email", help="always send email, even on success",
  91                   action="store_true")
  92 parser.add_option("--daemon", help="daemonize after initial setup",
  93                   action="store_true")
  94 parser.add_option("--branch", help="the branch to work on (default=master)",
  95                   default="master", type='str')
  96 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
  97                   default=gitroot, type='str')
  98 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
  99                   default=False, action="store_true")
 100 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
 101                   default='')
 102 parser.add_option("--enable-coverage", dest='enable_coverage',
 103                   action="store_const", const='--enable-coverage', default='',
 104                   help="Add --enable-coverage option while configure")
 105
 106 (options, args) = parser.parse_args()
 107
 108 if options.retry:
 109     if options.rebase is None:
 110         raise Exception('You can only use --retry if you also rebase')
 111
 112 if options.verbose:
 113     logger.setLevel(logging.DEBUG)
 114
 115 if options.full_testbase is not None:
 116     testbase = options.full_testbase
 117 else:
 118     testbase = "%s/b%u" % (options.testbase, os.getpid())
 119 test_master = "%s/master" % testbase
 120 test_prefix = "%s/prefix" % testbase
 121 test_tmpdir = "%s/tmp" % testbase
 122 os.environ['TMPDIR'] = test_tmpdir
 123
 124 if options.enable_coverage:
 125     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 126 else:
 127     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 128
 129 if options.enable_coverage:
 130     PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
 131 else:
 132     PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
 133
 134 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 135
 136
 137 def check_symbols(sofile, expected_symbols=""):
 138     return "objdump --dynamic-syms " + sofile + " | " + \
 139            "awk \'$0 !~ /" + expected_symbols + "/ {if ($2 == \"g\" && $3 ~ /D(F|O)/ && $4 ~ /(.bss|.text)/ && $7 !~ /(__gcov_|mangle_path)/) exit 1}\'"
 140
 141 def check_versioned_symbol(sofile, symvol, version):
 142     return "objdump --dynamic-syms " + sofile + " | " + \
 143            "awk \'$7 == \"" + symvol + "\" { " + \
 144                 "if ($2 == \"g\" && $3 ~ /D(F|O)/ && $4 ~ /(.bss|.text)/ && " + \
 145                      "$6 == \"" + version + "\") print $0 }\'" + \
 146                 "| wc -l | grep -q \'^1$\'"
 147
 148 if args:
 149     # If we are only running specific test,
 150     # do not sleep randomly to wait for it to start
 151     def random_sleep(low, high):
 152         return 'sleep 1'
 153 else:
 154     def random_sleep(low, high):
 155         return 'sleep {}'.format(random.randint(low, high))
 156
 157 cleanup_list = []
 158
 159 builddirs = {
 160     "ctdb": "ctdb",
 161     "tdb": "lib/tdb",
 162     "talloc": "lib/talloc",
 163     "replace": "lib/replace",
 164     "tevent": "lib/tevent",
 165     "pidl": "pidl",
 166     "docs-xml": "docs-xml"
 167 }
 168
 169 ctdb_configure_params = " --enable-developer ${PREFIX}"
 170 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 171
 172 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 173 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 174 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 175 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 176 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 177 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!tevent,!pytevent,!popt"
 178 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 179
 180
 181 def format_option(name, value=None):
 182     """Format option as str list."""
 183     if value is None:  # boolean option
 184         return [name]
 185     if not isinstance(value, list):  # single value option
 186         value = [value]
 187     # repeatable option
 188     return ['{}={}'.format(name, item) for item in value]
 189
 190
 191 def make_test(
 192         cmd='make testonly',
 193         INJECT_SELFTEST_PREFIX=1,
 194         TESTS='',
 195         include_envs=None,
 196         exclude_envs=None):
 197
 198     test_options = []
 199     if include_envs:
 200         test_options = format_option('--include-env', include_envs)
 201     if exclude_envs:
 202         test_options = format_option('--exclude-env', exclude_envs)
 203     if test_options:
 204         # join envs options to original test options
 205         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 206
 207     _options = []
 208
 209     # Allow getting a full CI with
 210     # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0'
 211
 212     FAIL_IMMEDIATELY = os.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1")
 213
 214     if int(FAIL_IMMEDIATELY):
 215         _options.append('FAIL_IMMEDIATELY=1')
 216     if TESTS:
 217         _options.append("TESTS='{}'".format(TESTS))
 218
 219     if INJECT_SELFTEST_PREFIX:
 220         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 221         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 222
 223     return ' '.join([cmd] + _options)
 224
 225
 226 # When updating this list, also update .gitlab-ci.yml to add the job
 227 # and to make it a dependency of 'page' for the coverage report.
 228
 229 tasks = {
 230     "ctdb": {
 231         "sequence": [
 232             ("random-sleep", random_sleep(300, 900)),
 233             ("configure", "./configure " + ctdb_configure_params),
 234             ("make", "make all"),
 235             ("install", "make install"),
 236             ("test", "make autotest"),
 237             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 238             ("clean", "make clean"),
 239         ],
 240     },
 241     "docs-xml": {
 242         "sequence": [
 243             ("random-sleep", random_sleep(300, 900)),
 244             ("autoconf", "autoconf"),
 245             ("configure", "./configure"),
 246             ("make", "make html htmlman"),
 247             ("publish-docs", PUBLISH_DOCS),
 248             ("clean", "make clean"),
 249         ],
 250     },
 251
 252     "samba-def-build": {
 253         "git-clone-required": True,
 254         "sequence": [
 255             ("configure", "./configure.developer" + samba_configure_params),
 256             ("make", "make -j"),
 257             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 258             ("chmod-R-a-w", "chmod -R a-w ."),
 259         ],
 260     },
 261
 262     "samba-mit-build": {
 263         "git-clone-required": True,
 264         "sequence": [
 265             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 266             ("make", "make -j"),
 267             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 268             ("chmod-R-a-w", "chmod -R a-w ."),
 269         ],
 270     },
 271
 272     "samba-nt4-build": {
 273         "git-clone-required": True,
 274         "sequence": [
 275             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 276             ("make", "make -j"),
 277             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 278             ("chmod-R-a-w", "chmod -R a-w ."),
 279         ],
 280     },
 281
 282     "samba-h5l-build": {
 283         "git-clone-required": True,
 284         "sequence": [
 285             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 286             ("make", "make -j"),
 287             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 288             ("chmod-R-a-w", "chmod -R a-w ."),
 289         ],
 290     },
 291
 292     "samba-without-smb1-build": {
 293         "git-clone-required": True,
 294         "sequence": [
 295             ("configure", "./configure.developer --without-smb1-server --without-ad-dc" + samba_configure_params),
 296             ("make", "make -j"),
 297             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 298             ("chmod-R-a-w", "chmod -R a-w ."),
 299         ],
 300     },
 301
 302     "samba-no-opath-build": {
 303         "git-clone-required": True,
 304         "sequence": [
 305             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1 -DDISABLE_VFS_OPEN_HOW_RESOLVE_NO_SYMLINKS=1 -DDISABLE_PROC_FDS=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 306             ("make", "make -j"),
 307             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 308             ("chmod-R-a-w", "chmod -R a-w ."),
 309         ],
 310     },
 311
 312     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 313     "samba": {
 314         "sequence": [
 315             ("random-sleep", random_sleep(300, 900)),
 316             ("configure", "./configure.developer" + samba_configure_params),
 317             ("make", "make -j"),
 318             ("test", make_test(exclude_envs=[
 319             "none",
 320             "nt4_dc",
 321             "nt4_dc_smb1",
 322             "nt4_dc_smb1_done",
 323             "nt4_dc_schannel",
 324             "nt4_member",
 325             "ad_dc",
 326             "ad_dc_smb1",
 327             "ad_dc_smb1_done",
 328             "ad_dc_backup",
 329             "ad_dc_ntvfs",
 330             "ad_dc_default",
 331             "ad_dc_default_smb1",
 332             "ad_dc_slowtests",
 333             "ad_dc_no_nss",
 334             "ad_dc_no_ntlm",
 335             "fl2003dc",
 336             "fl2008dc",
 337             "fl2008r2dc",
 338             "ad_member",
 339             "ad_member_idmap_rid",
 340             "admem_idmap_autorid",
 341             "ad_member_idmap_ad",
 342             "ad_member_rfc2307",
 343             "ad_member_idmap_nss",
 344             "ad_member_oneway",
 345             "chgdcpass",
 346             "vampire_2000_dc",
 347             "fl2000dc",
 348             "fileserver",
 349             "fileserver_smb1",
 350             "fileserver_smb1_done",
 351             "maptoguest",
 352             "simpleserver",
 353             "backupfromdc",
 354             "restoredc",
 355             "renamedc",
 356             "offlinebackupdc",
 357             "labdc",
 358             "preforkrestartdc",
 359             "proclimitdc",
 360             "promoted_dc",
 361             "vampire_dc",
 362             "rodc",
 363             "ad_dc_default",
 364             "ad_dc_default_smb1",
 365             "ad_dc_default_smb1_done",
 366             "ad_dc_slowtests",
 367             "schema_pair_dc",
 368             "schema_dc",
 369             "clusteredmember",
 370             "ad_dc_fips",
 371             "ad_member_fips",
 372             ])),
 373             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 374             ("lcov", LCOV_CMD),
 375             ("install", "make install"),
 376             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 377             ("clean", "make clean"),
 378         ],
 379     },
 380
 381     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 382     "samba-mitkrb5": {
 383         "sequence": [
 384             ("random-sleep", random_sleep(300, 900)),
 385             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 386             ("make", "make -j"),
 387             ("test", make_test(exclude_envs=[
 388             "none",
 389             "nt4_dc",
 390             "nt4_dc_smb1",
 391             "nt4_dc_smb1_done",
 392             "nt4_dc_schannel",
 393             "nt4_member",
 394             "ad_dc",
 395             "ad_dc_smb1",
 396             "ad_dc_smb1_done",
 397             "ad_dc_backup",
 398             "ad_dc_ntvfs",
 399             "ad_dc_default",
 400             "ad_dc_default_smb1",
 401             "ad_dc_default_smb1_done",
 402             "ad_dc_slowtests",
 403             "ad_dc_no_nss",
 404             "ad_dc_no_ntlm",
 405             "fl2003dc",
 406             "fl2008dc",
 407             "fl2008r2dc",
 408             "ad_member",
 409             "ad_member_idmap_rid",
 410             "admem_idmap_autorid",
 411             "ad_member_idmap_ad",
 412             "ad_member_rfc2307",
 413             "ad_member_idmap_nss",
 414             "ad_member_oneway",
 415             "chgdcpass",
 416             "vampire_2000_dc",
 417             "fl2000dc",
 418             "fileserver",
 419             "fileserver_smb1",
 420             "fileserver_smb1_done",
 421             "maptoguest",
 422             "simpleserver",
 423             "backupfromdc",
 424             "restoredc",
 425             "renamedc",
 426             "offlinebackupdc",
 427             "labdc",
 428             "preforkrestartdc",
 429             "proclimitdc",
 430             "promoted_dc",
 431             "vampire_dc",
 432             "rodc",
 433             "ad_dc_default",
 434             "ad_dc_default_smb1",
 435             "ad_dc_default_smb1_done",
 436             "ad_dc_slowtests",
 437             "schema_pair_dc",
 438             "schema_dc",
 439             "clusteredmember",
 440             "ad_dc_fips",
 441             "ad_member_fips",
 442             ])),
 443             ("lcov", LCOV_CMD),
 444             ("install", "make install"),
 445             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 446             ("clean", "make clean"),
 447         ],
 448     },
 449
 450     "samba-nt4": {
 451         "dependency": "samba-nt4-build",
 452         "sequence": [
 453             ("random-sleep", random_sleep(300, 900)),
 454             ("test", make_test(include_envs=[
 455             "nt4_dc",
 456             "nt4_dc_smb1",
 457             "nt4_dc_smb1_done",
 458             "nt4_dc_schannel",
 459             "nt4_member",
 460             "simpleserver",
 461             ])),
 462             ("lcov", LCOV_CMD),
 463             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 464         ],
 465     },
 466
 467     "samba-fileserver": {
 468         "dependency": "samba-h5l-build",
 469         "sequence": [
 470             ("random-sleep", random_sleep(300, 900)),
 471             ("test", make_test(include_envs=[
 472             "fileserver",
 473             "fileserver_smb1",
 474             "fileserver_smb1_done",
 475             "maptoguest",
 476             "ktest", # ktest is also tested in samba-ktest-mit samba
 477                      # and samba-mitkrb5 but is tested here against
 478                      # a system Heimdal
 479             ])),
 480             ("lcov", LCOV_CMD),
 481             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 482         ],
 483     },
 484
 485     "samba-fileserver-without-smb1": {
 486         "dependency": "samba-without-smb1-build",
 487         "sequence": [
 488             ("random-sleep", random_sleep(300, 900)),
 489             ("test", make_test(include_envs=["fileserver"])),
 490             ("lcov", LCOV_CMD),
 491             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 492         ],
 493     },
 494
 495     # This is a full build without the AD DC so we test the build with
 496     # MIT Kerberos from the current system.  Runtime behaviour is
 497     # confirmed via the ktest (static ccache and keytab) environment
 498
 499     # This environment also used to confirm we can still build with --with-libunwind
 500     "samba-ktest-mit": {
 501         "sequence": [
 502             ("random-sleep", random_sleep(300, 900)),
 503             ("configure", "./configure.developer --without-ad-dc --with-libunwind --with-system-mitkrb5 " + samba_configure_params),
 504             ("make", "make -j"),
 505             ("test", make_test(include_envs=[
 506             "ktest", # ktest is also tested in fileserver, samba and
 507                      # samba-mitkrb5 but is tested here against a
 508                      # system MIT krb5
 509             ])),
 510             ("lcov", LCOV_CMD),
 511             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 512         ],
 513     },
 514
 515     "samba-admem": {
 516         "dependency": "samba-def-build",
 517         "sequence": [
 518             ("random-sleep", random_sleep(300, 900)),
 519             ("test", make_test(include_envs=[
 520             "ad_member",
 521             "ad_member_idmap_rid",
 522             "admem_idmap_autorid",
 523             "ad_member_idmap_ad",
 524             "ad_member_rfc2307",
 525             "ad_member_idmap_nss",
 526             "ad_member_offlogon",
 527             ])),
 528             ("lcov", LCOV_CMD),
 529             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 530         ],
 531     },
 532
 533     "samba-no-opath1": {
 534         "dependency": "samba-no-opath-build",
 535         "sequence": [
 536             ("random-sleep", random_sleep(300, 900)),
 537             ("test", make_test(
 538                 cmd="make testonly DISABLE_OPATH=1",
 539                 include_envs=[
 540                 "nt4_dc",
 541                 "nt4_dc_smb1",
 542                 "nt4_dc_smb1_done",
 543                 "nt4_dc_schannel",
 544                 "nt4_member",
 545                 "simpleserver",
 546                 ])),
 547             ("lcov", LCOV_CMD),
 548             ("check-clean-tree", "script/clean-source-tree.sh"),
 549         ],
 550     },
 551
 552     "samba-no-opath2": {
 553         "dependency": "samba-no-opath-build",
 554         "sequence": [
 555             ("random-sleep", random_sleep(300, 900)),
 556             ("test", make_test(
 557                 cmd="make testonly DISABLE_OPATH=1",
 558                 include_envs=[
 559                 "fileserver",
 560                 "fileserver_smb1",
 561                 "fileserver_smb1_done",
 562                 ])),
 563             ("lcov", LCOV_CMD),
 564             ("check-clean-tree", "script/clean-source-tree.sh"),
 565         ],
 566     },
 567
 568     "samba-ad-dc-1": {
 569         "dependency": "samba-def-build",
 570         "sequence": [
 571             ("random-sleep", random_sleep(1, 1)),
 572             ("test", make_test(include_envs=[
 573             "ad_dc",
 574             "ad_dc_smb1",
 575             "ad_dc_smb1_done",
 576             "ad_dc_no_nss",
 577             "ad_dc_no_ntlm",
 578             ])),
 579             ("lcov", LCOV_CMD),
 580             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 581         ],
 582     },
 583
 584     "samba-ad-dc-2": {
 585         "dependency": "samba-def-build",
 586         "sequence": [
 587             ("random-sleep", random_sleep(1, 1)),
 588             ("test", make_test(include_envs=[
 589             "vampire_dc",
 590             "vampire_2000_dc",
 591             "rodc",
 592             ])),
 593             ("lcov", LCOV_CMD),
 594             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 595         ],
 596     },
 597
 598     "samba-ad-dc-3": {
 599         "dependency": "samba-def-build",
 600         "sequence": [
 601             ("random-sleep", random_sleep(1, 1)),
 602             ("test", make_test(include_envs=[
 603             "promoted_dc",
 604             "chgdcpass",
 605             "preforkrestartdc",
 606             "proclimitdc",
 607             ])),
 608             ("lcov", LCOV_CMD),
 609             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 610         ],
 611     },
 612
 613     "samba-ad-dc-4a": {
 614         "dependency": "samba-def-build",
 615         "sequence": [
 616             ("random-sleep", random_sleep(1, 1)),
 617             ("test", make_test(include_envs=[
 618             "fl2000dc",
 619             "ad_member_oneway",
 620             "fl2003dc",
 621             ])),
 622             ("lcov", LCOV_CMD),
 623             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 624         ],
 625     },
 626     "samba-ad-dc-4b": {
 627         "dependency": "samba-def-build",
 628         "sequence": [
 629             ("random-sleep", random_sleep(1, 1)),
 630             ("test", make_test(include_envs=[
 631             "fl2008dc",
 632             "fl2008r2dc",
 633             ])),
 634             ("lcov", LCOV_CMD),
 635             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 636         ],
 637     },
 638
 639     "samba-ad-dc-5": {
 640         "dependency": "samba-def-build",
 641         "sequence": [
 642             ("random-sleep", random_sleep(1, 1)),
 643             ("test", make_test(include_envs=[
 644             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 645             ("lcov", LCOV_CMD),
 646             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 647         ],
 648     },
 649
 650     "samba-ad-dc-6": {
 651         "dependency": "samba-def-build",
 652         "sequence": [
 653             ("random-sleep", random_sleep(1, 1)),
 654             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 655             ("lcov", LCOV_CMD),
 656             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 657         ],
 658     },
 659
 660     "samba-schemaupgrade": {
 661         "dependency": "samba-def-build",
 662         "sequence": [
 663             ("random-sleep", random_sleep(1, 1)),
 664             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 665             ("lcov", LCOV_CMD),
 666             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 667         ],
 668     },
 669
 670     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 671     # This is currently the longest task, so we don't randomly delay it.
 672     "samba-ad-dc-ntvfs": {
 673         "dependency": "samba-def-build",
 674         "sequence": [
 675             ("random-sleep", random_sleep(1, 1)),
 676             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 677             ("lcov", LCOV_CMD),
 678             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 679         ],
 680     },
 681
 682     # Test fips compliance
 683     "samba-fips": {
 684         "dependency": "samba-mit-build",
 685         "sequence": [
 686             ("random-sleep", random_sleep(1, 1)),
 687             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 688             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 689             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 690         ],
 691     },
 692
 693     # run the backup/restore testenvs separately as they're fairly standalone
 694     # (and CI seems to max out at ~3 different DCs running at once)
 695     "samba-ad-back1": {
 696         "dependency": "samba-def-build",
 697         "sequence": [
 698             ("random-sleep", random_sleep(300, 900)),
 699             ("test", make_test(include_envs=[
 700             "backupfromdc",
 701             "restoredc",
 702             "renamedc",
 703             ])),
 704             ("lcov", LCOV_CMD),
 705             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 706         ],
 707     },
 708     "samba-ad-back2": {
 709         "dependency": "samba-def-build",
 710         "sequence": [
 711             ("random-sleep", random_sleep(300, 900)),
 712             ("test", make_test(include_envs=[
 713             "backupfromdc",
 714             "offlinebackupdc",
 715             "labdc",
 716             ])),
 717             ("lcov", LCOV_CMD),
 718             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 719         ],
 720     },
 721
 722     "samba-admem-mit": {
 723         "dependency": "samba-mit-build",
 724         "sequence": [
 725             ("random-sleep", random_sleep(1, 1)),
 726             ("test", make_test(include_envs=[
 727             "ad_member",
 728             "ad_member_idmap_rid",
 729             "admem_idmap_autorid",
 730             "ad_member_idmap_ad",
 731             "ad_member_rfc2307",
 732             "ad_member_idmap_nss",
 733             "ad_member_offlogon",
 734             ])),
 735             ("lcov", LCOV_CMD),
 736             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 737         ],
 738     },
 739
 740     "samba-addc-mit-1": {
 741         "dependency": "samba-mit-build",
 742         "sequence": [
 743             ("random-sleep", random_sleep(1, 1)),
 744             ("test", make_test(include_envs=[
 745             "ad_dc",
 746             "ad_dc_smb1",
 747             "ad_dc_smb1_done",
 748             "ad_dc_no_nss",
 749             "ad_dc_no_ntlm",
 750             ])),
 751             ("lcov", LCOV_CMD),
 752             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 753         ],
 754     },
 755
 756     "samba-addc-mit-4a": {
 757         "dependency": "samba-mit-build",
 758         "sequence": [
 759             ("random-sleep", random_sleep(1, 1)),
 760             ("test", make_test(include_envs=[
 761             "fl2000dc",
 762             "ad_member_oneway",
 763             "fl2003dc",
 764             ])),
 765             ("lcov", LCOV_CMD),
 766             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 767         ],
 768     },
 769     "samba-addc-mit-4b": {
 770         "dependency": "samba-mit-build",
 771         "sequence": [
 772             ("random-sleep", random_sleep(1, 1)),
 773             ("test", make_test(include_envs=[
 774             "fl2008dc",
 775             "fl2008r2dc",
 776             ])),
 777             ("lcov", LCOV_CMD),
 778             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 779         ],
 780     },
 781
 782     "samba-test-only": {
 783         "sequence": [
 784             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 785             ("make", "make -j"),
 786             ("test", make_test(TESTS="${TESTS}")),
 787             ("lcov", LCOV_CMD),
 788         ],
 789     },
 790
 791     # Test cross-compile infrastructure
 792     "samba-xc": {
 793         "sequence": [
 794             ("random-sleep", random_sleep(900, 1500)),
 795             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 796             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 797             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 798             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 799             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 800             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 801             ("compare-results", "script/compare_cc_results.py "
 802             "./bin/c4che/default{} "
 803             "./bin-xe/c4che/default{} "
 804             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 805             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 806             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 807             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 808             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 809             " = \"'1234'\"".format(CACHE_SUFFIX)),
 810             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 811             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 812             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 813             " ; test $? -ne 0"),
 814         ],
 815     },
 816
 817     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 818     "samba-o3": {
 819         "sequence": [
 820             ("random-sleep", random_sleep(300, 900)),
 821             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + samba_configure_params),
 822             ("make", "make -j"),
 823             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 824             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 825             ("lcov", LCOV_CMD),
 826             ("install", "make install"),
 827             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 828             ("clean", "make clean"),
 829         ],
 830     },
 831
 832     "samba-32bit": {
 833         "sequence": [
 834             ("random-sleep", random_sleep(300, 900)),
 835             ("configure", "./configure.developer --abi-check-disable --disable-warnings-as-errors" + samba_configure_params),
 836             ("make", "make -j"),
 837             ("nonetest", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 838             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 839             ("ktest", make_test(cmd='make test', include_envs=["ktest"])),
 840             ("install", "make install"),
 841             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 842             ("clean", "make clean"),
 843         ],
 844     },
 845
 846     "samba-ctdb": {
 847         "sequence": [
 848             ("random-sleep", random_sleep(900, 1500)),
 849
 850         # make sure we have tdb around:
 851             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 852             ("tdb-make", "cd lib/tdb && make"),
 853             ("tdb-install", "cd lib/tdb && make install"),
 854
 855         # build samba with cluster support (also building ctdb):
 856             ("samba-configure",
 857          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 858          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 859          "./configure.developer ${PREFIX} "
 860          "--with-selftest-prefix=./bin/ab "
 861          "--with-cluster-support "
 862          "--bundled-libraries=!tdb"),
 863             ("samba-make", "make"),
 864             ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
 865             ("samba-install", "make install"),
 866             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 867
 868             ("test", make_test(
 869                 cmd='PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH make test',
 870                 INJECT_SELFTEST_PREFIX=0,
 871                 include_envs=["clusteredmember"])
 872             ),
 873
 874         # clean up:
 875             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 876             ("clean", "make clean"),
 877             ("ctdb-clean", "cd ./ctdb && make clean"),
 878         ],
 879     },
 880
 881     "samba-libs": {
 882         "sequence": [
 883             ("random-sleep", random_sleep(300, 900)),
 884             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 885             ("talloc-make", "cd lib/talloc && make"),
 886             ("talloc-install", "cd lib/talloc && make install"),
 887             ("talloc-abi-check1",
 888                 check_versioned_symbol(
 889                     "./lib/talloc/bin/shared/libtalloc.so.2",
 890                     "talloc_named",
 891                     "TALLOC_2.0.2"
 892                 )
 893             ),
 894             ("talloc-abi-check2",
 895                 check_versioned_symbol(
 896                     "./lib/talloc/bin/shared/libtalloc.so.2",
 897                     "talloc_asprintf_addbuf",
 898                     "TALLOC_2.3.5"
 899                 )
 900             ),
 901
 902             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 903             ("tdb-make", "cd lib/tdb && make"),
 904             ("tdb-install", "cd lib/tdb && make install"),
 905             ("tdb-abi-check1",
 906                 check_versioned_symbol(
 907                     "./lib/tdb/bin/shared/libtdb.so.1",
 908                     "tdb_errorstr",
 909                     "TDB_1.2.1"
 910                 )
 911             ),
 912             ("tdb-abi-check2",
 913                 check_versioned_symbol(
 914                     "./lib/tdb/bin/shared/libtdb.so.1",
 915                     "tdb_traverse_chain",
 916                     "TDB_1.3.17"
 917                 )
 918             ),
 919
 920             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 921             ("tevent-make", "cd lib/tevent && make"),
 922             ("tevent-install", "cd lib/tevent && make install"),
 923             ("tevent-abi-check1",
 924                 check_versioned_symbol(
 925                     "./lib/tevent/bin/shared/libtevent.so.0",
 926                     "_tevent_loop_once",
 927                     "TEVENT_0.9.9"
 928                 )
 929             ),
 930             ("tevent-abi-check2",
 931                 check_versioned_symbol(
 932                     "./lib/tevent/bin/shared/libtevent.so.0",
 933                     "__tevent_req_create",
 934                     "TEVENT_0.15.0"
 935                 )
 936             ),
 937
 938             ("nondevel-configure", samba_libs_envvars + " ./configure --private-libraries='!ldb' --vendor-suffix=TEST-STRING~5.1.2 ${PREFIX}"),
 939             ("nondevel-make", "make -j"),
 940             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 941             ("nondevel-check", "./bin/smbd --version | grep -F 'TEST-STRING~5.1.2' && exit 0; exit 1"),
 942             ("nondevel-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 943             ("nondevel-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 944             ("nondevel-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 945             ("nondevel-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 946             ("nondevel-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 947             ("nondevel-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 948             ("nondevel-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 949             ("nondevel-no-public-nss_winbind",
 950                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 951             ("nondevel-no-public-nss_wins",
 952                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 953             ("nondevel-no-public-libwbclient",
 954                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 955             ("nondevel-libwbclient-wbcCtxPingDc2@WBCLIENT_0.12",
 956                 check_versioned_symbol("./bin/shared/libwbclient.so.0", "wbcCtxPingDc2", "WBCLIENT_0.12")),
 957             ("nondevel-no-public-pam_winbind",
 958                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 959             ("nondevel-no-public-winbind_krb5_locator",
 960                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 961             ("nondevel-no-public-async_dns_krb5_locator",
 962                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 963             ("nondevel-libndr-krb5pac-ndr_pull_PAC_DATA@NDR_KRB5PAC_0.0.1",
 964                 check_versioned_symbol("./bin/shared/libndr-krb5pac.so.0", "ndr_pull_PAC_DATA", "NDR_KRB5PAC_0.0.1")),
 965             ("nondevel-install", "make -j install"),
 966             ("nondevel-dist", "make dist"),
 967
 968             ("prefix-no-private-libtalloc", "find ${PREFIX_DIR} | grep -v 'libtalloc-report' | grep 'private.*libtalloc' && exit 1; exit 0"),
 969             ("prefix-no-private-libtdb", "find ${PREFIX_DIR} | grep -v 'libtdb-wrap' | grep 'private.*libtdb' && exit 1; exit 0"),
 970             ("prefix-no-private-libtevent", "find ${PREFIX_DIR} | grep -v 'libtevent-util' | grep 'private.*libtevent' && exit 1; exit 0"),
 971             ("prefix-no-private-libldb", "find ${PREFIX_DIR} | grep -v 'module' | grep -v 'libldbsamba' | grep 'private.*libldb.so' && exit 1; exit 0"),
 972             ("prefix-public-libldb", "find ${PREFIX_DIR} | grep 'lib/libldb.so' && exit 0; exit 1"),
 973             ("prefix-no-samba-nss_winbind", "ldd ${PREFIX_DIR}/lib/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 974             ("prefix-no-samba-nss_wins", "ldd ${PREFIX_DIR}/lib/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 975             ("prefix-no-samba-libwbclient", "ldd ${PREFIX_DIR}/lib/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 976             ("prefix-no-samba-pam_winbind", "ldd ${PREFIX_DIR}/lib/security/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 977             ("prefix-no-public-nss_winbind",
 978                 check_symbols("${PREFIX_DIR}/lib/libnss_winbind.so.2", "_nss_winbind_")),
 979             ("prefix-no-public-nss_wins",
 980                 check_symbols("${PREFIX_DIR}/lib/libnss_wins.so.2", "_nss_wins_")),
 981             ("prefix-no-public-libwbclient",
 982                 check_symbols("${PREFIX_DIR}/lib/libwbclient.so.0", "wbc")),
 983             ("prefix-no-public-pam_winbind",
 984                 check_symbols("${PREFIX_DIR}/lib/security/pam_winbind.so", "pam_sm_")),
 985             ("prefix-no-public-winbind_krb5_locator",
 986                 check_symbols("${PREFIX_DIR}/lib/krb5/winbind_krb5_locator.so",
 987                               "service_locator")),
 988             ("prefix-no-public-async_dns_krb5_locator",
 989                 check_symbols("${PREFIX_DIR}/lib/krb5/async_dns_krb5_locator.so",
 990                               "service_locator")),
 991
 992             # retry with all modules shared
 993             ("allshared-distclean", "make distclean"),
 994             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
 995             ("allshared-make", "make -j"),
 996             ("allshared-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 997             ("allshared-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 998             ("allshared-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 999             ("allshared-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
1000             ("allshared-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
1001             ("allshared-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
1002             ("allshared-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
1003             ("allshared-no-public-nss_winbind",
1004                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
1005             ("allshared-no-public-nss_wins",
1006                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
1007             ("allshared-no-public-libwbclient",
1008                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
1009             ("allshared-no-public-pam_winbind",
1010                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
1011             ("allshared-no-public-winbind_krb5_locator",
1012                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
1013             ("allshared-no-public-async_dns_krb5_locator",
1014                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
1015         ],
1016     },
1017
1018     "samba-fuzz": {
1019         "sequence": [
1020         # build the fuzzers (static) via the oss-fuzz script
1021             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
1022             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
1023         ],
1024     },
1025
1026     # * Test smbd and smbtorture can build semi-static
1027     #
1028     # * Test Samba without python still builds.
1029     #
1030     # When this test fails due to more use of Python, the expectations
1031     # is that the newly failing part of the code should be disabled
1032     # when --disable-python is set (rather than major work being done
1033     # to support this environment).
1034     #
1035     # The target here is for vendors shipping a minimal smbd.
1036     "samba-minimal-smbd": {
1037         "sequence": [
1038             ("random-sleep", random_sleep(300, 900)),
1039
1040         # build with all modules static
1041             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
1042             ("allstatic-make", "make -j"),
1043             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
1044             ("allstatic-lcov", LCOV_CMD),
1045             ("allstatic-def-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1046             ("allstatic-def-clean", "make clean"),
1047
1048         # force all libraries as private
1049             ("allprivate-def-distclean", "make distclean"),
1050             ("allprivate-def-configure", "./configure.developer " + samba_configure_params + " --private-libraries=ALL"),
1051             ("allprivate-def-make", "make -j"),
1052             # note wrapper libraries need to be public
1053             ("allprivate-def-no-public", "ls ./bin/shared | egrep -v '^private$|lib[nprsu][saeoi][smscd].*-wrapper.so$|pam_set_items.so|pam_matrix.so' | wc -l | grep -q '^0'"),
1054             ("allprivate-def-only-private-ext", "ls ./bin/shared/private | egrep 'private-samba' | wc -l | grep -q '^0' && exit 1; exit 0"),
1055             ("allprivate-def-no-non-private-ext", "ls ./bin/shared/private | egrep -v 'private-samba|^libpypamtest.so$' | wc -l | grep -q '^0'"),
1056             ("allprivate-def-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
1057             ("allprivate-def-lcov", LCOV_CMD),
1058             ("allprivate-def-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1059             ("allprivate-def-clean", "make clean"),
1060
1061         # force all libraries as private with a non default
1062         # extension and 2 exceptions
1063             ("allprivate-ext-distclean", "make distclean"),
1064             ("allprivate-ext-configure", "./configure.developer " + samba_configure_params + " --private-libraries=ALL --private-library-extension=private-library --private-extension-exception=pac,ndr"),
1065             ("allprivate-ext-make", "make -j"),
1066             # note wrapper libraries need to be public
1067             ("allprivate-ext-no-public", "ls ./bin/shared | egrep -v '^private$|lib[nprsu][saeoi][smscd].*-wrapper.so$|pam_set_items.so|pam_matrix.so' | wc -l | grep -q '^0'"),
1068             ("allprivate-ext-no-private-default-ext", "ls ./bin/shared/private | grep 'private-samba' | wc -l | grep -q '^0'"),
1069             ("allprivate-ext-has-private-ext", "ls ./bin/shared/private | grep 'private-library' | wc -l | grep -q '^0' && exit 1; exit 0"),
1070             ("allprivate-ext-libndr-no-private-ext", "ls ./bin/shared/private | grep -v 'private-library' | grep 'libndr' | wc -l | grep -q '^1'"),
1071             ("allprivate-ext-libpac-no-private-ext", "ls ./bin/shared/private | grep -v 'private-library' | grep 'libpac' | wc -l | grep -q '^1'"),
1072             ("allprivate-ext-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
1073             ("allprivate-ext-lcov", LCOV_CMD),
1074             ("allprivate-ext-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1075             ("allprivate-ext-clean", "make clean"),
1076
1077         # retry with nonshared smbd and smbtorture
1078             ("nonshared-distclean", "make distclean"),
1079             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
1080             ("nonshared-make", "make -j"),
1081             ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
1082             ("nonshared-lcov", LCOV_CMD),
1083             ("nonshared-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1084             ("nonshared-clean", "make clean"),
1085
1086         # retry without winbindd
1087             ("nonwinbind-distclean", "make distclean"),
1088             ("nonwinbind-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --without-winbind"),
1089             ("nonwinbind-make", "make -j"),
1090             ("nonwinbind-test", make_test(TESTS="samba3.smb2.*.simpleserver")),
1091             ("nonwinbind-lcov", LCOV_CMD),
1092             ("nonwinbind-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1093             ("nonwinbind-clean", "make clean"),
1094         ],
1095     },
1096
1097     "samba-nopython": {
1098         "sequence": [
1099             ("random-sleep", random_sleep(300, 900)),
1100
1101             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
1102             ("make", "make -j"),
1103             ("find-python", "script/find_python.sh ${PREFIX}"),
1104             ("test", "make test-nopython"),
1105             ("lcov", LCOV_CMD),
1106             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1107             ("clean", "make clean"),
1108
1109             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1110             ("talloc-make", "cd lib/talloc && make"),
1111             ("talloc-install", "cd lib/talloc && make install"),
1112
1113             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1114             ("tdb-make", "cd lib/tdb && make"),
1115             ("tdb-install", "cd lib/tdb && make install"),
1116
1117             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1118             ("tevent-make", "cd lib/tevent && make"),
1119             ("tevent-install", "cd lib/tevent && make install"),
1120
1121         # retry against installed library packages, but no required modules
1122             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
1123             ("libs-make", "make -j"),
1124             ("libs-install", "make install"),
1125             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1126             ("libs-clean", "make clean"),
1127
1128         ],
1129     },
1130
1131     "samba-codecheck": {
1132         "sequence": [
1133             ("run", "script/check-shell-scripts.sh ."),
1134             ("run", "script/codespell.sh ."),
1135         ],
1136     },
1137
1138     "tdb": {
1139         "sequence": [
1140             ("random-sleep", random_sleep(60, 600)),
1141             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1142             ("make", "make"),
1143             ("install", "make install"),
1144             ("test", "make test"),
1145             ("lcov", LCOV_CMD),
1146             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1147             ("distcheck", "make distcheck"),
1148             ("clean", "make clean"),
1149         ],
1150     },
1151
1152     "talloc": {
1153         "sequence": [
1154             ("random-sleep", random_sleep(60, 600)),
1155             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1156             ("make", "make"),
1157             ("install", "make install"),
1158             ("test", "make test"),
1159             ("lcov", LCOV_CMD),
1160             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1161             ("distcheck", "make distcheck"),
1162             ("clean", "make clean"),
1163         ],
1164     },
1165
1166     "replace": {
1167         "sequence": [
1168             ("random-sleep", random_sleep(60, 600)),
1169             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1170             ("make", "make"),
1171             ("install", "make install"),
1172             ("test", "make test"),
1173             ("lcov", LCOV_CMD),
1174             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1175             ("distcheck", "make distcheck"),
1176             ("clean", "make clean"),
1177         ],
1178     },
1179
1180     "tevent": {
1181         "sequence": [
1182             ("random-sleep", random_sleep(60, 600)),
1183             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1184             ("make", "make"),
1185             ("install", "make install"),
1186             ("test", "make test"),
1187             ("lcov", LCOV_CMD),
1188             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1189             ("distcheck", "make distcheck"),
1190             ("clean", "make clean"),
1191         ],
1192     },
1193
1194     "pidl": {
1195         "git-clone-required": True,
1196         "sequence": [
1197             ("random-sleep", random_sleep(60, 600)),
1198             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
1199             ("touch", "touch *.yp"),
1200             ("make", "make"),
1201             ("test", "make test"),
1202             ("install", "make install"),
1203             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
1204             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1205             ("clean", "make clean"),
1206         ],
1207     },
1208
1209     # these are useful for debugging autobuild
1210     "pass": {
1211         "sequence": [
1212             ("pass", 'echo passing && /bin/true'),
1213         ],
1214     },
1215     "fail": {
1216         "sequence": [
1217             ("fail", 'echo failing && /bin/false'),
1218         ],
1219     },
1220 }
1221
1222 defaulttasks = list(tasks.keys())
1223
1224 defaulttasks.remove("pass")
1225 defaulttasks.remove("fail")
1226
1227 # The build tasks will be brought in by the test tasks as needed
1228 defaulttasks.remove("samba-def-build")
1229 defaulttasks.remove("samba-nt4-build")
1230 defaulttasks.remove("samba-mit-build")
1231 defaulttasks.remove("samba-h5l-build")
1232 defaulttasks.remove("samba-no-opath-build")
1233
1234 # This is not a normal test, but a task to support manually running
1235 # one test under autobuild
1236 defaulttasks.remove("samba-test-only")
1237
1238 # Only built on GitLab CI and not in the default autobuild because it
1239 # uses too much space (4GB of semi-static binaries)
1240 defaulttasks.remove("samba-fuzz")
1241
1242 # The FIPS build runs only in GitLab CI on a current Fedora Docker
1243 # container where a simulated FIPS mode is possible.
1244 defaulttasks.remove("samba-fips")
1245
1246 # The MIT build runs on a current Fedora where an up to date MIT KDC
1247 # is already packaged.  This avoids needing to backport a current MIT
1248 # to the default Ubuntu 18.04, particularly during development, and
1249 # the need to install on the shared sn-devel-184.
1250
1251 defaulttasks.remove("samba-mitkrb5")
1252 defaulttasks.remove("samba-admem-mit")
1253 defaulttasks.remove("samba-addc-mit-1")
1254 defaulttasks.remove("samba-addc-mit-4a")
1255 defaulttasks.remove("samba-addc-mit-4b")
1256
1257 defaulttasks.remove("samba-32bit")
1258
1259 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
1260     defaulttasks.remove("samba-o3")
1261
1262
1263 def do_print(msg):
1264     logger.info(msg)
1265     sys.stdout.flush()
1266     sys.stderr.flush()
1267
1268 def do_debug(msg):
1269     logger.debug(msg)
1270     sys.stdout.flush()
1271     sys.stderr.flush()
1272
1273
1274 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
1275     if show is None:
1276         do_debug("Running: '%s' in '%s'" % (cmd, dir))
1277     elif show:
1278         do_print("Running: '%s' in '%s'" % (cmd, dir))
1279
1280     if output:
1281         out = check_output([cmd], shell=True, cwd=dir)
1282         return out.decode(encoding='utf-8', errors='backslashreplace')
1283     elif checkfail:
1284         return check_call(cmd, shell=True, cwd=dir)
1285     else:
1286         return call(cmd, shell=True, cwd=dir)
1287
1288 def rmdir_force(dirname, re_raise=True):
1289     try:
1290         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
1291                 dirname, dirname, dirname), output=True, show=True)
1292     except CalledProcessError as e:
1293         do_print("Failed: '%s'" % (str(e)))
1294         run_cmd("tree %s" % dirname, output=True, show=True)
1295         if re_raise:
1296             raise
1297         return False
1298     return True
1299
1300 class builder(object):
1301     '''handle build of one directory'''
1302
1303     def __init__(self, name, definition):
1304         self.name = name
1305         self.dir = builddirs.get(name, '.')
1306         self.tag = self.name.replace('/', '_')
1307         self.definition = definition
1308         self.sequence = definition["sequence"]
1309         self.git_clone_required = False
1310         if "git-clone-required" in definition:
1311             self.git_clone_required = bool(definition["git-clone-required"])
1312         self.proc = None
1313         self.done = False
1314         self.next = 0
1315         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1316         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1317         do_debug("stdout for %s in %s" % (self.name, self.stdout_path))
1318         do_debug("stderr for %s in %s" % (self.name, self.stderr_path))
1319         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1320         self.stdout = open(self.stdout_path, 'w')
1321         self.stderr = open(self.stderr_path, 'w')
1322         self.stdin  = open("/dev/null", 'r')
1323         self.builder_dir = "%s/%s" % (testbase, self.tag)
1324         self.test_source_dir = self.builder_dir
1325         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1326         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1327         self.prefix = "%s/%s" % (test_prefix, self.tag)
1328         self.consumers = []
1329         self.producer = None
1330
1331         if self.git_clone_required:
1332             assert "dependency" not in definition
1333
1334     def mark_existing(self):
1335         do_debug('%s: Mark as existing dependency' % self.name)
1336         self.next = len(self.sequence)
1337         self.done = True
1338
1339     def add_consumer(self, consumer):
1340         do_debug("%s: add consumer: %s" % (self.name, consumer.name))
1341         consumer.producer = self
1342         consumer.test_source_dir = self.test_source_dir
1343         self.consumers.append(consumer)
1344
1345     def start_next(self):
1346         if self.producer is not None:
1347             if not self.producer.done:
1348                 do_debug("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1349                 return
1350
1351         if self.next == 0:
1352             rmdir_force(self.builder_dir)
1353             rmdir_force(self.prefix)
1354             if self.producer is not None:
1355                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1356             elif not self.git_clone_required:
1357                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1358             else:
1359                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1360
1361         if self.next == len(self.sequence):
1362             if not self.done:
1363                 do_print('%s: Completed OK' % self.name)
1364                 self.done = True
1365             if not options.nocleanup and len(self.consumers) == 0:
1366                 do_print('%s: Cleaning up' % self.name)
1367                 rmdir_force(self.builder_dir)
1368                 rmdir_force(self.prefix)
1369             for consumer in self.consumers:
1370                 if consumer.next != 0:
1371                     continue
1372                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1373                 consumer.start_next()
1374             if self.producer is not None:
1375                 self.producer.consumers.remove(self)
1376                 assert self.producer.done
1377                 self.producer.start_next()
1378             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1379             return
1380         (self.stage, self.cmd) = self.sequence[self.next]
1381         self.cmd = self.cmd.replace("${PYTHON_PREFIX}",
1382                                     get_path(name='platlib',
1383                                              scheme="posix_prefix",
1384                                              vars={"base": self.prefix,
1385                                                    "platbase": self.prefix}))
1386         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1387         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1388         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1389         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1390         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1391         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1392         self.cmd = self.cmd.replace("${NAME}", self.name)
1393         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1394         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1395         self.proc = Popen(self.cmd, shell=True,
1396                           close_fds=True, cwd=self.cwd,
1397                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1398         self.next += 1
1399
1400 def expand_dependencies(n):
1401     deps = list()
1402     if "dependency" in tasks[n]:
1403         depname = tasks[n]["dependency"]
1404         assert depname in tasks
1405         sdeps = expand_dependencies(depname)
1406         assert n not in sdeps
1407         for sdep in sdeps:
1408             deps.append(sdep)
1409         deps.append(depname)
1410     return deps
1411
1412
1413 class buildlist(object):
1414     '''handle build of multiple directories'''
1415
1416     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1417         self.tail_proc = None
1418         self.retry = None
1419         if not tasknames:
1420             if options.restrict_tests:
1421                 tasknames = ["samba-test-only"]
1422             else:
1423                 tasknames = defaulttasks
1424
1425         given_tasknames = tasknames.copy()
1426         implicit_tasknames = []
1427         for n in given_tasknames:
1428             deps = expand_dependencies(n)
1429             for dep in deps:
1430                 if dep in given_tasknames:
1431                     continue
1432                 if dep in implicit_tasknames:
1433                     continue
1434                 implicit_tasknames.append(dep)
1435
1436         tasknames = implicit_tasknames.copy()
1437         tasknames.extend(given_tasknames)
1438         do_debug("given_tasknames: %s" % given_tasknames)
1439         do_debug("implicit_tasknames: %s" % implicit_tasknames)
1440         do_debug("tasknames: %s" % tasknames)
1441         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1442
1443         if options.retry:
1444             rebase_remote = "rebaseon"
1445             retry_task = {
1446                     "git-clone-required": True,
1447                     "sequence": [
1448                             ("retry",
1449                             '''set -e
1450                             git remote add -t %s %s %s
1451                             git fetch %s
1452                             while :; do
1453                               sleep 60
1454                               git describe %s/%s > old_remote_branch.desc
1455                               git fetch %s
1456                               git describe %s/%s > remote_branch.desc
1457                               diff old_remote_branch.desc remote_branch.desc
1458                             done
1459                            ''' % (
1460                                rebase_branch, rebase_remote, rebase_url,
1461                                rebase_remote,
1462                                rebase_remote, rebase_branch,
1463                                rebase_remote,
1464                                rebase_remote, rebase_branch
1465                             ))]}
1466
1467             self.retry = builder('retry', retry_task)
1468             self.need_retry = False
1469
1470         if options.skip_dependencies:
1471             for b in self.tlist:
1472                 if b.name in implicit_tasknames:
1473                     b.mark_existing()
1474
1475         for b in self.tlist:
1476             do_debug("b.name=%s" % b.name)
1477             if "dependency" not in b.definition:
1478                 continue
1479             depname = b.definition["dependency"]
1480             do_debug("b.name=%s: dependency:%s" % (b.name, depname))
1481             for p in self.tlist:
1482                 if p.name == depname:
1483                     p.add_consumer(b)
1484
1485     def kill_kids(self):
1486         if self.tail_proc is not None:
1487             self.tail_proc.terminate()
1488             self.tail_proc.wait()
1489             self.tail_proc = None
1490         if self.retry is not None:
1491             self.retry.proc.terminate()
1492             self.retry.proc.wait()
1493             self.retry = None
1494         for b in self.tlist:
1495             if b.proc is not None:
1496                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1497                 b.proc.terminate()
1498                 b.proc.wait()
1499                 b.proc = None
1500
1501     def wait_one(self):
1502         while True:
1503             none_running = True
1504             for b in self.tlist:
1505                 if b.proc is None:
1506                     continue
1507                 none_running = False
1508                 b.status = b.proc.poll()
1509                 if b.status is None:
1510                     continue
1511                 b.proc = None
1512                 return b
1513             if options.retry:
1514                 ret = self.retry.proc.poll()
1515                 if ret is not None:
1516                     self.need_retry = True
1517                     self.retry = None
1518                     return None
1519             if none_running:
1520                 return None
1521             time.sleep(0.1)
1522
1523     def run(self):
1524         for b in self.tlist:
1525             b.start_next()
1526         if options.retry:
1527             self.retry.start_next()
1528         while True:
1529             b = self.wait_one()
1530             if options.retry and self.need_retry:
1531                 self.kill_kids()
1532                 do_print("retry needed")
1533                 return (0, None, None, None, "retry")
1534             if b is None:
1535                 break
1536             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1537                 self.kill_kids()
1538                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1539             b.start_next()
1540         self.kill_kids()
1541         return (0, None, None, None, "All OK")
1542
1543     def write_system_info(self, filename):
1544         with open(filename, 'w') as f:
1545             for cmd in ['uname -a',
1546                         'lsb_release -a',
1547                         'free',
1548                         'mount',
1549                         'cat /proc/cpuinfo',
1550                         'cc --version',
1551                         'df -m .',
1552                         'df -m %s' % testbase]:
1553                 try:
1554                     out = run_cmd(cmd, output=True, checkfail=False)
1555                 except CalledProcessError as e:
1556                     out = "<failed: %s>" % str(e)
1557                 print('### %s' % cmd, file=f)
1558                 print(out, file=f)
1559                 print(file=f)
1560
1561     def tarlogs(self, fname):
1562         with tarfile.open(fname, "w:gz") as tar:
1563             for b in self.tlist:
1564                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1565                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1566             if os.path.exists("autobuild.log"):
1567                 tar.add("autobuild.log")
1568             filename = 'system-info.txt'
1569             self.write_system_info(filename)
1570             tar.add(filename)
1571
1572     def remove_logs(self):
1573         for b in self.tlist:
1574             os.unlink(b.stdout_path)
1575             os.unlink(b.stderr_path)
1576
1577     def start_tail(self):
1578         cmd = ["tail", "-f"]
1579         for b in self.tlist:
1580             cmd.append(b.stdout_path)
1581             cmd.append(b.stderr_path)
1582         self.tail_proc = Popen(cmd, close_fds=True)
1583
1584
1585 def cleanup(do_raise=False):
1586     if options.nocleanup:
1587         return
1588     run_cmd("stat %s || true" % test_tmpdir, show=True)
1589     run_cmd("stat %s" % testbase, show=True)
1590     do_print("Cleaning up %r" % cleanup_list)
1591     for d in cleanup_list:
1592         ok = rmdir_force(d, re_raise=False)
1593         if ok:
1594             continue
1595         if os.path.isdir(d):
1596             do_print("Killing, waiting and retry")
1597             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1598         else:
1599             do_print("Waiting and retry")
1600         time.sleep(1)
1601         rmdir_force(d, re_raise=do_raise)
1602
1603
1604 def daemonize(logfile):
1605     pid = os.fork()
1606     if pid == 0:  # Parent
1607         os.setsid()
1608         pid = os.fork()
1609         if pid != 0:  # Actual daemon
1610             os._exit(0)
1611     else:  # Grandparent
1612         os._exit(0)
1613
1614     import resource      # Resource usage information.
1615     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1616     if maxfd == resource.RLIM_INFINITY:
1617         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1618     for fd in range(0, maxfd):
1619         try:
1620             os.close(fd)
1621         except OSError:
1622             pass
1623     os.open(logfile, os.O_RDWR | os.O_CREAT)
1624     os.dup2(0, 1)
1625     os.dup2(0, 2)
1626
1627
1628 def write_pidfile(fname):
1629     '''write a pid file, cleanup on exit'''
1630     with open(fname, mode='w') as f:
1631         f.write("%u\n" % os.getpid())
1632
1633
1634 def rebase_tree(rebase_url, rebase_branch="master"):
1635     rebase_remote = "rebaseon"
1636     do_print("Rebasing on %s" % rebase_url)
1637     run_cmd("git describe HEAD", show=True, dir=test_master)
1638     run_cmd("git remote add -t %s %s %s" %
1639             (rebase_branch, rebase_remote, rebase_url),
1640             show=True, dir=test_master)
1641     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1642     if options.fix_whitespace:
1643         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1644                 (rebase_remote, rebase_branch),
1645                 show=True, dir=test_master)
1646     else:
1647         run_cmd("git rebase --force-rebase %s/%s" %
1648                 (rebase_remote, rebase_branch),
1649                 show=True, dir=test_master)
1650     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1651                    (rebase_remote, rebase_branch),
1652                    dir=test_master, output=True)
1653     if diff == '':
1654         do_print("No differences between HEAD and %s/%s - exiting" %
1655                  (rebase_remote, rebase_branch))
1656         sys.exit(0)
1657     run_cmd("git describe %s/%s" %
1658             (rebase_remote, rebase_branch),
1659             show=True, dir=test_master)
1660     run_cmd("git describe HEAD", show=True, dir=test_master)
1661     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1662             (rebase_remote, rebase_branch),
1663             show=True, dir=test_master)
1664
1665
1666 def push_to(push_url, push_branch="master"):
1667     push_remote = "pushto"
1668     do_print("Pushing to %s" % push_url)
1669     if options.mark:
1670         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1671         run_cmd("git commit --amend -c HEAD", dir=test_master)
1672         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1673         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1674     run_cmd("git remote add -t %s %s %s" %
1675             (push_branch, push_remote, push_url),
1676             show=True, dir=test_master)
1677     run_cmd("git push %s +HEAD:%s" %
1678             (push_remote, push_branch),
1679             show=True, dir=test_master)
1680
1681
1682 def send_email(subject, text, log_tar):
1683     if options.email is None:
1684         do_print("not sending email because the recipient is not set")
1685         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1686                  (subject, text))
1687         return
1688     outer = MIMEMultipart()
1689     outer['Subject'] = subject
1690     outer['To'] = options.email
1691     outer['From'] = options.email_from
1692     outer['Date'] = email.utils.formatdate(localtime=True)
1693     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1694     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1695     if options.attach_logs:
1696         with open(log_tar, 'rb') as fp:
1697             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1698         # Set the filename parameter
1699         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1700         outer.attach(msg)
1701     content = outer.as_string()
1702     s = smtplib.SMTP(options.email_server)
1703     email_user = os.getenv('SMTP_USERNAME')
1704     email_password = os.getenv('SMTP_PASSWORD')
1705     if email_user is not None:
1706         s.starttls()
1707         s.login(email_user, email_password)
1708
1709     s.sendmail(options.email_from, [options.email], content)
1710     s.set_debuglevel(1)
1711     s.quit()
1712
1713
1714 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1715                   elapsed_time, log_base=None, add_log_tail=True):
1716     '''send an email to options.email about the failure'''
1717     elapsed_minutes = elapsed_time / 60.0
1718     if log_base is None:
1719         log_base = gitroot
1720     text = '''
1721 Dear Developer,
1722
1723 Your autobuild on %s failed after %.1f minutes
1724 when trying to test %s with the following error:
1725
1726    %s
1727
1728 the autobuild has been abandoned. Please fix the error and resubmit.
1729
1730 A summary of the autobuild process is here:
1731
1732   %s/autobuild.log
1733 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1734
1735     if options.restrict_tests:
1736         text += """
1737 The build was restricted to tests matching %s\n""" % options.restrict_tests
1738
1739     if failed_task != 'rebase':
1740         text += '''
1741 You can see logs of the failed task here:
1742
1743   %s/%s.stdout
1744   %s/%s.stderr
1745
1746 or you can get full logs of all tasks in this job here:
1747
1748   %s/logs.tar.gz
1749
1750 The top commit for the tree that was built was:
1751
1752 %s
1753
1754 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1755
1756     log_stdout = "%s/%s.stdout" % (gitroot, failed_tag)
1757     if add_log_tail and os.access(log_stdout, os.R_OK):
1758         f = open(log_stdout, 'r')
1759         lines = f.readlines()
1760         log_tail = "".join(lines[-50:])
1761         num_lines = len(lines)
1762         log_stderr = "%s/%s.stderr" % (gitroot, failed_tag)
1763         if num_lines < 50 and os.access(log_stderr, os.R_OK):
1764             # Also include stderr (compile failures) if < 50 lines of stdout
1765             f = open(log_stderr, 'r')
1766             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1767
1768         text += '''
1769 The last 50 lines of log messages:
1770
1771 %s
1772     ''' % log_tail
1773         f.close()
1774
1775     logs = os.path.join(gitroot, 'logs.tar.gz')
1776     send_email('autobuild[%s] failure on %s for task %s during %s'
1777                % (options.branch, platform.node(), failed_task, failed_stage),
1778                text, logs)
1779
1780
1781 def email_success(elapsed_time, log_base=None):
1782     '''send an email to options.email about a successful build'''
1783     if log_base is None:
1784         log_base = gitroot
1785     text = '''
1786 Dear Developer,
1787
1788 Your autobuild on %s has succeeded after %.1f minutes.
1789
1790 ''' % (platform.node(), elapsed_time / 60.)
1791
1792     if options.restrict_tests:
1793         text += """
1794 The build was restricted to tests matching %s\n""" % options.restrict_tests
1795
1796     if options.keeplogs:
1797         text += '''
1798
1799 you can get full logs of all tasks in this job here:
1800
1801   %s/logs.tar.gz
1802
1803 ''' % log_base
1804
1805     text += '''
1806 The top commit for the tree that was built was:
1807
1808 %s
1809 ''' % top_commit_msg
1810
1811     logs = os.path.join(gitroot, 'logs.tar.gz')
1812     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1813                text, logs)
1814
1815
1816 # get the top commit message, for emails
1817 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1818
1819 try:
1820     if options.skip_dependencies:
1821         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1822     else:
1823         os.makedirs(testbase)
1824 except Exception as reason:
1825     raise Exception("Unable to create %s : %s" % (testbase, reason))
1826 cleanup_list.append(testbase)
1827
1828 if options.daemon:
1829     logfile = os.path.join(testbase, "log")
1830     do_print("Forking into the background, writing progress to %s" % logfile)
1831     daemonize(logfile)
1832
1833 write_pidfile(gitroot + "/autobuild.pid")
1834
1835 start_time = time.time()
1836
1837 while True:
1838     try:
1839         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1840         os.makedirs(test_tmpdir)
1841         # The waf uninstall code removes empty directories all the way
1842         # up the tree.  Creating a file in test_tmpdir stops it from
1843         # being removed.
1844         run_cmd("touch %s" % os.path.join(test_tmpdir,
1845                                           ".directory-is-not-empty"), show=True)
1846         run_cmd("stat %s" % test_tmpdir, show=True)
1847         run_cmd("stat %s" % testbase, show=True)
1848         if options.skip_dependencies:
1849             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1850         else:
1851             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1852     except Exception:
1853         cleanup()
1854         raise
1855
1856     try:
1857         if options.rebase is not None:
1858             rebase_tree(options.rebase, rebase_branch=options.branch)
1859     except Exception:
1860         cleanup_list.append(gitroot + "/autobuild.pid")
1861         cleanup()
1862         elapsed_time = time.time() - start_time
1863         email_failure(-1, 'rebase', 'rebase', 'rebase',
1864                       'rebase on %s failed' % options.branch,
1865                       elapsed_time, log_base=options.log_base)
1866         sys.exit(1)
1867
1868     try:
1869         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1870         if options.tail:
1871             blist.start_tail()
1872         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1873         if status != 0 or errstr != "retry":
1874             break
1875         cleanup(do_raise=True)
1876     except Exception:
1877         cleanup()
1878         raise
1879
1880 cleanup_list.append(gitroot + "/autobuild.pid")
1881
1882 do_print(errstr)
1883
1884 blist.kill_kids()
1885 if options.tail:
1886     do_print("waiting for tail to flush")
1887     time.sleep(1)
1888
1889 elapsed_time = time.time() - start_time
1890 if status == 0:
1891     if options.passcmd is not None:
1892         do_print("Running passcmd: %s" % options.passcmd)
1893         run_cmd(options.passcmd, dir=test_master)
1894     if options.pushto is not None:
1895         push_to(options.pushto, push_branch=options.branch)
1896     if options.keeplogs or options.attach_logs:
1897         blist.tarlogs("logs.tar.gz")
1898         do_print("Logs in logs.tar.gz")
1899     if options.always_email:
1900         email_success(elapsed_time, log_base=options.log_base)
1901     blist.remove_logs()
1902     cleanup()
1903     do_print(errstr)
1904     sys.exit(0)
1905
1906 # something failed, gather a tar of the logs
1907 blist.tarlogs("logs.tar.gz")
1908
1909 if options.email is not None:
1910     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1911                   elapsed_time, log_base=options.log_base)
1912 else:
1913     elapsed_minutes = elapsed_time / 60.0
1914     print('''
1915
1916 ####################################################################
1917
1918 AUTOBUILD FAILURE
1919
1920 Your autobuild[%s] on %s failed after %.1f minutes
1921 when trying to test %s with the following error:
1922
1923    %s
1924
1925 the autobuild has been abandoned. Please fix the error and resubmit.
1926
1927 ####################################################################
1928
1929 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1930
1931 cleanup()
1932 do_print(errstr)
1933 do_print("Logs in logs.tar.gz")
1934 sys.exit(status)