script/autobuild.py

   1 #!/usr/bin/env python3
   2 # run tests on all Samba subprojects and push to a git tree on success
   3 # Copyright Andrew Tridgell 2010
   4 # released under GNU GPL v3 or later
   5
   6 from subprocess import call, check_call, check_output, Popen, PIPE, CalledProcessError
   7 import os
   8 import tarfile
   9 import sys
  10 import time
  11 import random
  12 from optparse import OptionParser
  13 import smtplib
  14 import email
  15 from email.mime.text import MIMEText
  16 from email.mime.base import MIMEBase
  17 from email.mime.application import MIMEApplication
  18 from email.mime.multipart import MIMEMultipart
  19 from sysconfig import get_path
  20 import platform
  21 import ssl
  22 import shutil
  23
  24 def get_libc_version():
  25     import ctypes
  26     libc = ctypes.CDLL("libc.so.6")
  27     gnu_get_libc_version = libc.gnu_get_libc_version
  28     gnu_get_libc_version.restype = ctypes.c_char_p
  29     return gnu_get_libc_version().decode()
  30
  31 import logging
  32
  33 try:
  34     from waflib.Build import CACHE_SUFFIX
  35 except ImportError:
  36     sys.path.insert(0, "./third_party/waf")
  37     from waflib.Build import CACHE_SUFFIX
  38
  39 logging.basicConfig(format='%(asctime)s %(message)s')
  40 logger = logging.getLogger('autobuild')
  41 logger.setLevel(logging.INFO)
  42
  43 os.environ["PYTHONUNBUFFERED"] = "1"
  44
  45 # This speeds up testing remarkably.
  46 os.environ['TDB_NO_FSYNC'] = '1'
  47
  48 # allow autobuild to run within git rebase -i
  49 if "GIT_DIR" in os.environ:
  50     del os.environ["GIT_DIR"]
  51 if "GIT_WORK_TREE" in os.environ:
  52     del os.environ["GIT_WORK_TREE"]
  53
  54 def find_git_root():
  55     '''get to the top of the git repo'''
  56     p = os.getcwd()
  57     while p != '/':
  58         if os.path.exists(os.path.join(p, ".git")):
  59             return p
  60         p = os.path.abspath(os.path.join(p, '..'))
  61     return None
  62
  63
  64 gitroot = find_git_root()
  65 if gitroot is None:
  66     raise Exception("Failed to find git root")
  67
  68
  69 def_testbase = os.getenv("AUTOBUILD_TESTBASE", "/memdisk/%s" % os.getenv('USER'))
  70
  71 parser = OptionParser()
  72 parser.add_option("--tail", help="show output while running", default=False, action="store_true")
  73 parser.add_option("--keeplogs", help="keep logs", default=False, action="store_true")
  74 parser.add_option("--nocleanup", help="don't remove test tree", default=False, action="store_true")
  75 parser.add_option("--skip-dependencies", help="skip to run task dependency tasks", default=False, action="store_true")
  76 parser.add_option("--testbase", help="base directory to run tests in (default %s)" % def_testbase,
  77                   default=def_testbase)
  78 parser.add_option("--full-testbase", help="full base directory to run tests in (default %s/b$PID)" % def_testbase,
  79                   default=None)
  80 parser.add_option("--passcmd", help="command to run on success", default=None)
  81 parser.add_option("--verbose", help="show all commands as they are run",
  82                   default=False, action="store_true")
  83 parser.add_option("--rebase", help="rebase on the given tree before testing",
  84                   default=None, type='str')
  85 parser.add_option("--pushto", help="push to a git url on success",
  86                   default=None, type='str')
  87 parser.add_option("--mark", help="add a Tested-By signoff before pushing",
  88                   default=False, action="store_true")
  89 parser.add_option("--fix-whitespace", help="fix whitespace on rebase",
  90                   default=False, action="store_true")
  91 parser.add_option("--retry", help="automatically retry if master changes",
  92                   default=False, action="store_true")
  93 parser.add_option("--email", help="send email to the given address on failure",
  94                   type='str', default=None)
  95 parser.add_option("--email-from", help="send email from the given address",
  96                   type='str', default="autobuild@samba.org")
  97 parser.add_option("--email-server", help="send email via the given server",
  98                   type='str', default='localhost')
  99 parser.add_option("--always-email", help="always send email, even on success",
 100                   action="store_true")
 101 parser.add_option("--daemon", help="daemonize after initial setup",
 102                   action="store_true")
 103 parser.add_option("--branch", help="the branch to work on (default=master)",
 104                   default="master", type='str')
 105 parser.add_option("--log-base", help="location where the logs can be found (default=cwd)",
 106                   default=gitroot, type='str')
 107 parser.add_option("--attach-logs", help="Attach logs to mails sent on success/failure?",
 108                   default=False, action="store_true")
 109 parser.add_option("--restrict-tests", help="run as make test with this TESTS= regex",
 110                   default='')
 111 parser.add_option("--enable-coverage", dest='enable_coverage',
 112                   action="store_const", const='--enable-coverage', default='',
 113                   help="Add --enable-coverage option while configure")
 114
 115 (options, args) = parser.parse_args()
 116
 117 if options.retry:
 118     if options.rebase is None:
 119         raise Exception('You can only use --retry if you also rebase')
 120
 121 if options.verbose:
 122     logger.setLevel(logging.DEBUG)
 123
 124 if options.full_testbase is not None:
 125     testbase = options.full_testbase
 126 else:
 127     testbase = "%s/b%u" % (options.testbase, os.getpid())
 128 test_master = "%s/master" % testbase
 129 test_prefix = "%s/prefix" % testbase
 130 test_tmpdir = "%s/tmp" % testbase
 131 os.environ['TMPDIR'] = test_tmpdir
 132
 133 if options.enable_coverage:
 134     LCOV_CMD = "cd ${TEST_SOURCE_DIR} && lcov --capture --directory . --output-file ${LOG_BASE}/${NAME}.info --rc 'geninfo_adjust_src_path=${TEST_SOURCE_DIR}/'"
 135 else:
 136     LCOV_CMD = 'echo "lcov skipped since no --enable-coverage specified"'
 137
 138 if options.enable_coverage:
 139     PUBLISH_DOCS = "mkdir -p ${LOG_BASE}/public && mv output/htmldocs ${LOG_BASE}/public/htmldocs"
 140 else:
 141     PUBLISH_DOCS = 'echo "HTML documentation publishing skipped since no --enable-coverage specified"'
 142
 143 CLEAN_SOURCE_TREE_CMD = "cd ${TEST_SOURCE_DIR} && script/clean-source-tree.sh"
 144
 145
 146 def check_symbols(sofile, expected_symbols=""):
 147     return "objdump --dynamic-syms " + sofile + " | " + \
 148            "awk \'$0 !~ /" + expected_symbols + "/ {if ($2 == \"g\" && $3 ~ /D(F|O)/ && $4 ~ /(.bss|.text)/ && $7 !~ /(__gcov_|mangle_path)/) exit 1}\'"
 149
 150 def check_versioned_symbol(sofile, symvol, version):
 151     return "objdump --dynamic-syms " + sofile + " | " + \
 152            "awk \'$7 == \"" + symvol + "\" { " + \
 153                 "if ($2 == \"g\" && $3 ~ /D(F|O)/ && $4 ~ /(.bss|.text)/ && " + \
 154                      "$6 == \"" + version + "\") print $0 }\'" + \
 155                 "| wc -l | grep -q \'^1$\'"
 156
 157 if args:
 158     # If we are only running specific test,
 159     # do not sleep randomly to wait for it to start
 160     def random_sleep(low, high):
 161         return 'sleep 1'
 162 else:
 163     def random_sleep(low, high):
 164         return 'sleep {}'.format(random.randint(low, high))
 165
 166 cleanup_list = []
 167
 168 builddirs = {
 169     "ctdb": "ctdb",
 170     "tdb": "lib/tdb",
 171     "talloc": "lib/talloc",
 172     "replace": "lib/replace",
 173     "tevent": "lib/tevent",
 174     "pidl": "pidl",
 175     "docs-xml": "docs-xml"
 176 }
 177
 178 ctdb_configure_params = " --enable-developer ${PREFIX}"
 179 samba_configure_params = " ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data"
 180
 181 # We cannot configure himmelblau on old systems missing openssl 3, with glibc
 182 # older than version 2.32, or when cargo isn't available.
 183 himmelblau_configure_params = ''
 184 rust_configure_param = ''
 185 glibc_vers = float('.'.join(get_libc_version().split('.')[:2]))
 186 cargo = shutil.which('cargo')
 187 if glibc_vers >= 2.32 and cargo != None:
 188     rust_configure_param = ' --enable-rust'
 189 if ssl.OPENSSL_VERSION_INFO[0] >= 3 and rust_configure_param:
 190     himmelblau_configure_params = rust_configure_param + ' --with-himmelblau'
 191
 192 samba_libs_envvars = "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH"
 193 samba_libs_envvars += " PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig"
 194 samba_libs_envvars += " ADDITIONAL_CFLAGS='-Wmissing-prototypes'"
 195 samba_libs_configure_base = samba_libs_envvars + " ./configure --abi-check ${ENABLE_COVERAGE} --enable-debug -C ${PREFIX}"
 196 samba_libs_configure_libs = samba_libs_configure_base + " --bundled-libraries=cmocka,popt,NONE"
 197 samba_libs_configure_bundled_libs = " --bundled-libraries=!talloc,!pytalloc-util,!tdb,!pytdb,!tevent,!pytevent,!popt"
 198 samba_libs_configure_samba = samba_libs_configure_base + samba_libs_configure_bundled_libs
 199
 200
 201 def format_option(name, value=None):
 202     """Format option as str list."""
 203     if value is None:  # boolean option
 204         return [name]
 205     if not isinstance(value, list):  # single value option
 206         value = [value]
 207     # repeatable option
 208     return ['{}={}'.format(name, item) for item in value]
 209
 210
 211 def make_test(
 212         cmd='make testonly',
 213         INJECT_SELFTEST_PREFIX=1,
 214         TESTS='',
 215         include_envs=None,
 216         exclude_envs=None):
 217
 218     test_options = []
 219     if include_envs:
 220         test_options = format_option('--include-env', include_envs)
 221     if exclude_envs:
 222         test_options = format_option('--exclude-env', exclude_envs)
 223     if test_options:
 224         # join envs options to original test options
 225         TESTS = (TESTS + ' ' + ' '.join(test_options)).strip()
 226
 227     _options = []
 228
 229     # Allow getting a full CI with
 230     # git push -o ci.variable='AUTOBUILD_FAIL_IMMEDIATELY=0'
 231
 232     FAIL_IMMEDIATELY = os.getenv("AUTOBUILD_FAIL_IMMEDIATELY", "1")
 233
 234     if int(FAIL_IMMEDIATELY):
 235         _options.append('FAIL_IMMEDIATELY=1')
 236     if TESTS:
 237         _options.append("TESTS='{}'".format(TESTS))
 238
 239     if INJECT_SELFTEST_PREFIX:
 240         _options.append("TEST_OPTIONS='--with-selftest-prefix={}'".format("${SELFTEST_PREFIX}"))
 241         _options.append("--directory='{}'".format("${TEST_SOURCE_DIR}"))
 242
 243     return ' '.join([cmd] + _options)
 244
 245
 246 # When updating this list, also update .gitlab-ci.yml to add the job
 247 # and to make it a dependency of 'page' for the coverage report.
 248
 249 tasks = {
 250     "ctdb": {
 251         "sequence": [
 252             ("random-sleep", random_sleep(300, 900)),
 253             ("configure", "./configure " + ctdb_configure_params),
 254             ("make", "make all"),
 255             ("install", "make install"),
 256             ("test", "make autotest"),
 257             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 258             ("clean", "make clean"),
 259         ],
 260     },
 261     "docs-xml": {
 262         "sequence": [
 263             ("random-sleep", random_sleep(300, 900)),
 264             ("autoconf", "autoconf"),
 265             ("configure", "./configure"),
 266             ("make", "make html htmlman"),
 267             ("publish-docs", PUBLISH_DOCS),
 268             ("clean", "make clean"),
 269         ],
 270     },
 271
 272     "samba-def-build": {
 273         "git-clone-required": True,
 274         "sequence": [
 275             ("configure", "./configure.developer" + samba_configure_params),
 276             ("make", "make -j"),
 277             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 278             ("chmod-R-a-w", "chmod -R a-w ."),
 279         ],
 280     },
 281
 282     "samba-mit-build": {
 283         "git-clone-required": True,
 284         "sequence": [
 285             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 286             ("make", "make -j"),
 287             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 288             ("chmod-R-a-w", "chmod -R a-w ."),
 289         ],
 290     },
 291
 292     "samba-nt4-build": {
 293         "git-clone-required": True,
 294         "sequence": [
 295             ("configure", "./configure.developer --without-ad-dc --without-ldap --without-ads --without-json" + samba_configure_params),
 296             ("make", "make -j"),
 297             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 298             ("chmod-R-a-w", "chmod -R a-w ."),
 299         ],
 300     },
 301
 302     "samba-h5l-build": {
 303         "git-clone-required": True,
 304         "sequence": [
 305             ("configure", "./configure.developer --without-ad-dc --with-system-heimdalkrb5" + samba_configure_params),
 306             ("make", "make -j"),
 307             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 308             ("chmod-R-a-w", "chmod -R a-w ."),
 309         ],
 310     },
 311
 312     "samba-without-smb1-build": {
 313         "git-clone-required": True,
 314         "sequence": [
 315             ("configure", "./configure.developer --without-smb1-server --without-ad-dc" + samba_configure_params),
 316             ("make", "make -j"),
 317             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 318             ("chmod-R-a-w", "chmod -R a-w ."),
 319         ],
 320     },
 321
 322     "samba-no-opath-build": {
 323         "git-clone-required": True,
 324         "sequence": [
 325             ("configure", "ADDITIONAL_CFLAGS='-DDISABLE_OPATH=1 -DDISABLE_VFS_OPEN_HOW_RESOLVE_NO_SYMLINKS=1 -DDISABLE_PROC_FDS=1' ./configure.developer --without-ad-dc " + samba_configure_params),
 326             ("make", "make -j"),
 327             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 328             ("chmod-R-a-w", "chmod -R a-w ."),
 329         ],
 330     },
 331
 332     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 333     "samba": {
 334         "sequence": [
 335             ("random-sleep", random_sleep(300, 900)),
 336             ("configure", "./configure.developer" + samba_configure_params),
 337             ("make", "make -j"),
 338             ("test", make_test(exclude_envs=[
 339             "none",
 340             "nt4_dc",
 341             "nt4_dc_smb1",
 342             "nt4_dc_smb1_done",
 343             "nt4_dc_schannel",
 344             "nt4_member",
 345             "ad_dc",
 346             "ad_dc_smb1",
 347             "ad_dc_smb1_done",
 348             "ad_dc_backup",
 349             "ad_dc_ntvfs",
 350             "ad_dc_default",
 351             "ad_dc_default_smb1",
 352             "ad_dc_slowtests",
 353             "ad_dc_no_nss",
 354             "ad_dc_no_ntlm",
 355             "fl2003dc",
 356             "fl2008dc",
 357             "fl2008r2dc",
 358             "ad_member",
 359             "ad_member_idmap_rid",
 360             "admem_idmap_autorid",
 361             "ad_member_idmap_ad",
 362             "ad_member_rfc2307",
 363             "ad_member_idmap_nss",
 364             "ad_member_oneway",
 365             "chgdcpass",
 366             "vampire_2000_dc",
 367             "fl2000dc",
 368             "fileserver",
 369             "fileserver_smb1",
 370             "fileserver_smb1_done",
 371             "maptoguest",
 372             "simpleserver",
 373             "backupfromdc",
 374             "restoredc",
 375             "renamedc",
 376             "offlinebackupdc",
 377             "labdc",
 378             "preforkrestartdc",
 379             "proclimitdc",
 380             "promoted_dc",
 381             "vampire_dc",
 382             "rodc",
 383             "ad_dc_default",
 384             "ad_dc_default_smb1",
 385             "ad_dc_default_smb1_done",
 386             "ad_dc_slowtests",
 387             "schema_pair_dc",
 388             "schema_dc",
 389             "clusteredmember",
 390             "ad_dc_fips",
 391             "ad_member_fips",
 392             ])),
 393             ("test-slow-none", make_test(cmd='make test', TESTS="--include=selftest/slow-none", include_envs=["none"])),
 394             ("lcov", LCOV_CMD),
 395             ("install", "make install"),
 396             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 397             ("clean", "make clean"),
 398         ],
 399     },
 400
 401     # We have 'test' before 'install' because, 'test' should work without 'install (runs all the other envs)'
 402     "samba-mitkrb5": {
 403         "sequence": [
 404             ("random-sleep", random_sleep(300, 900)),
 405             ("configure", "./configure.developer --with-system-mitkrb5 --with-experimental-mit-ad-dc" + samba_configure_params),
 406             ("make", "make -j"),
 407             ("test", make_test(exclude_envs=[
 408             "none",
 409             "nt4_dc",
 410             "nt4_dc_smb1",
 411             "nt4_dc_smb1_done",
 412             "nt4_dc_schannel",
 413             "nt4_member",
 414             "ad_dc",
 415             "ad_dc_smb1",
 416             "ad_dc_smb1_done",
 417             "ad_dc_backup",
 418             "ad_dc_ntvfs",
 419             "ad_dc_default",
 420             "ad_dc_default_smb1",
 421             "ad_dc_default_smb1_done",
 422             "ad_dc_slowtests",
 423             "ad_dc_no_nss",
 424             "ad_dc_no_ntlm",
 425             "fl2003dc",
 426             "fl2008dc",
 427             "fl2008r2dc",
 428             "ad_member",
 429             "ad_member_idmap_rid",
 430             "admem_idmap_autorid",
 431             "ad_member_idmap_ad",
 432             "ad_member_rfc2307",
 433             "ad_member_idmap_nss",
 434             "ad_member_oneway",
 435             "chgdcpass",
 436             "vampire_2000_dc",
 437             "fl2000dc",
 438             "fileserver",
 439             "fileserver_smb1",
 440             "fileserver_smb1_done",
 441             "maptoguest",
 442             "simpleserver",
 443             "backupfromdc",
 444             "restoredc",
 445             "renamedc",
 446             "offlinebackupdc",
 447             "labdc",
 448             "preforkrestartdc",
 449             "proclimitdc",
 450             "promoted_dc",
 451             "vampire_dc",
 452             "rodc",
 453             "ad_dc_default",
 454             "ad_dc_default_smb1",
 455             "ad_dc_default_smb1_done",
 456             "ad_dc_slowtests",
 457             "schema_pair_dc",
 458             "schema_dc",
 459             "clusteredmember",
 460             "ad_dc_fips",
 461             "ad_member_fips",
 462             ])),
 463             ("lcov", LCOV_CMD),
 464             ("install", "make install"),
 465             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 466             ("clean", "make clean"),
 467         ],
 468     },
 469
 470     "samba-nt4": {
 471         "dependency": "samba-nt4-build",
 472         "sequence": [
 473             ("random-sleep", random_sleep(300, 900)),
 474             ("test", make_test(include_envs=[
 475             "nt4_dc",
 476             "nt4_dc_smb1",
 477             "nt4_dc_smb1_done",
 478             "nt4_dc_schannel",
 479             "nt4_member",
 480             "simpleserver",
 481             ])),
 482             ("lcov", LCOV_CMD),
 483             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 484         ],
 485     },
 486
 487     "samba-fileserver": {
 488         "dependency": "samba-h5l-build",
 489         "sequence": [
 490             ("random-sleep", random_sleep(300, 900)),
 491             ("test", make_test(include_envs=[
 492             "fileserver",
 493             "fileserver_smb1",
 494             "fileserver_smb1_done",
 495             "maptoguest",
 496             "ktest", # ktest is also tested in samba-ktest-mit samba
 497                      # and samba-mitkrb5 but is tested here against
 498                      # a system Heimdal
 499             ])),
 500             ("lcov", LCOV_CMD),
 501             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 502         ],
 503     },
 504
 505     "samba-fileserver-without-smb1": {
 506         "dependency": "samba-without-smb1-build",
 507         "sequence": [
 508             ("random-sleep", random_sleep(300, 900)),
 509             ("test", make_test(include_envs=["fileserver"])),
 510             ("lcov", LCOV_CMD),
 511             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 512         ],
 513     },
 514
 515     # This is a full build without the AD DC so we test the build with
 516     # MIT Kerberos from the current system.  Runtime behaviour is
 517     # confirmed via the ktest (static ccache and keytab) environment
 518
 519     # This environment also used to confirm we can still build with --with-libunwind
 520     "samba-ktest-mit": {
 521         "sequence": [
 522             ("random-sleep", random_sleep(300, 900)),
 523             ("configure", "./configure.developer --without-ad-dc --with-libunwind --with-system-mitkrb5 " + samba_configure_params),
 524             ("make", "make -j"),
 525             ("test", make_test(include_envs=[
 526             "ktest", # ktest is also tested in fileserver, samba and
 527                      # samba-mitkrb5 but is tested here against a
 528                      # system MIT krb5
 529             ])),
 530             ("lcov", LCOV_CMD),
 531             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 532         ],
 533     },
 534
 535     "samba-admem": {
 536         "dependency": "samba-def-build",
 537         "sequence": [
 538             ("random-sleep", random_sleep(300, 900)),
 539             ("test", make_test(include_envs=[
 540             "ad_member",
 541             "ad_member_idmap_rid",
 542             "admem_idmap_autorid",
 543             "ad_member_idmap_ad",
 544             "ad_member_rfc2307",
 545             "ad_member_idmap_nss",
 546             "ad_member_offlogon",
 547             ])),
 548             ("lcov", LCOV_CMD),
 549             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 550         ],
 551     },
 552
 553     "samba-no-opath1": {
 554         "dependency": "samba-no-opath-build",
 555         "sequence": [
 556             ("random-sleep", random_sleep(300, 900)),
 557             ("test", make_test(
 558                 cmd="make testonly DISABLE_OPATH=1",
 559                 include_envs=[
 560                 "nt4_dc",
 561                 "nt4_dc_smb1",
 562                 "nt4_dc_smb1_done",
 563                 "nt4_dc_schannel",
 564                 "nt4_member",
 565                 "simpleserver",
 566                 ])),
 567             ("lcov", LCOV_CMD),
 568             ("check-clean-tree", "script/clean-source-tree.sh"),
 569         ],
 570     },
 571
 572     "samba-no-opath2": {
 573         "dependency": "samba-no-opath-build",
 574         "sequence": [
 575             ("random-sleep", random_sleep(300, 900)),
 576             ("test", make_test(
 577                 cmd="make testonly DISABLE_OPATH=1",
 578                 include_envs=[
 579                 "fileserver",
 580                 "fileserver_smb1",
 581                 "fileserver_smb1_done",
 582                 ])),
 583             ("lcov", LCOV_CMD),
 584             ("check-clean-tree", "script/clean-source-tree.sh"),
 585         ],
 586     },
 587
 588     "samba-ad-dc-1": {
 589         "dependency": "samba-def-build",
 590         "sequence": [
 591             ("random-sleep", random_sleep(1, 1)),
 592             ("test", make_test(include_envs=[
 593             "ad_dc",
 594             "ad_dc_smb1",
 595             "ad_dc_smb1_done",
 596             "ad_dc_no_nss",
 597             "ad_dc_no_ntlm",
 598             ])),
 599             ("lcov", LCOV_CMD),
 600             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 601         ],
 602     },
 603
 604     "samba-ad-dc-2": {
 605         "dependency": "samba-def-build",
 606         "sequence": [
 607             ("random-sleep", random_sleep(1, 1)),
 608             ("test", make_test(include_envs=[
 609             "vampire_dc",
 610             "vampire_2000_dc",
 611             "rodc",
 612             ])),
 613             ("lcov", LCOV_CMD),
 614             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 615         ],
 616     },
 617
 618     "samba-ad-dc-3": {
 619         "dependency": "samba-def-build",
 620         "sequence": [
 621             ("random-sleep", random_sleep(1, 1)),
 622             ("test", make_test(include_envs=[
 623             "promoted_dc",
 624             "chgdcpass",
 625             "preforkrestartdc",
 626             "proclimitdc",
 627             ])),
 628             ("lcov", LCOV_CMD),
 629             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 630         ],
 631     },
 632
 633     "samba-ad-dc-4a": {
 634         "dependency": "samba-def-build",
 635         "sequence": [
 636             ("random-sleep", random_sleep(1, 1)),
 637             ("test", make_test(include_envs=[
 638             "fl2000dc",
 639             "ad_member_oneway",
 640             "fl2003dc",
 641             ])),
 642             ("lcov", LCOV_CMD),
 643             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 644         ],
 645     },
 646     "samba-ad-dc-4b": {
 647         "dependency": "samba-def-build",
 648         "sequence": [
 649             ("random-sleep", random_sleep(1, 1)),
 650             ("test", make_test(include_envs=[
 651             "fl2008dc",
 652             "fl2008r2dc",
 653             ])),
 654             ("lcov", LCOV_CMD),
 655             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 656         ],
 657     },
 658
 659     "samba-ad-dc-5": {
 660         "dependency": "samba-def-build",
 661         "sequence": [
 662             ("random-sleep", random_sleep(1, 1)),
 663             ("test", make_test(include_envs=[
 664             "ad_dc_default", "ad_dc_default_smb1", "ad_dc_default_smb1_done"])),
 665             ("lcov", LCOV_CMD),
 666             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 667         ],
 668     },
 669
 670     "samba-ad-dc-6": {
 671         "dependency": "samba-def-build",
 672         "sequence": [
 673             ("random-sleep", random_sleep(1, 1)),
 674             ("test", make_test(include_envs=["ad_dc_slowtests", "ad_dc_backup"])),
 675             ("lcov", LCOV_CMD),
 676             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 677         ],
 678     },
 679
 680     "samba-schemaupgrade": {
 681         "dependency": "samba-def-build",
 682         "sequence": [
 683             ("random-sleep", random_sleep(1, 1)),
 684             ("test", make_test(include_envs=["schema_dc", "schema_pair_dc"])),
 685             ("lcov", LCOV_CMD),
 686             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 687         ],
 688     },
 689
 690     # We split out the ad_dc_ntvfs tests (which are long) so other test do not wait
 691     # This is currently the longest task, so we don't randomly delay it.
 692     "samba-ad-dc-ntvfs": {
 693         "dependency": "samba-def-build",
 694         "sequence": [
 695             ("random-sleep", random_sleep(1, 1)),
 696             ("test", make_test(include_envs=["ad_dc_ntvfs"])),
 697             ("lcov", LCOV_CMD),
 698             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 699         ],
 700     },
 701
 702     # Test fips compliance
 703     "samba-fips": {
 704         "dependency": "samba-mit-build",
 705         "sequence": [
 706             ("random-sleep", random_sleep(1, 1)),
 707             ("test", make_test(include_envs=["ad_dc_fips", "ad_member_fips"])),
 708             # TODO: This seems to generate only an empty samba-fips.info ("lcov", LCOV_CMD),
 709             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 710         ],
 711     },
 712
 713     # run the backup/restore testenvs separately as they're fairly standalone
 714     # (and CI seems to max out at ~3 different DCs running at once)
 715     "samba-ad-back1": {
 716         "dependency": "samba-def-build",
 717         "sequence": [
 718             ("random-sleep", random_sleep(300, 900)),
 719             ("test", make_test(include_envs=[
 720             "backupfromdc",
 721             "restoredc",
 722             "renamedc",
 723             ])),
 724             ("lcov", LCOV_CMD),
 725             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 726         ],
 727     },
 728     "samba-ad-back2": {
 729         "dependency": "samba-def-build",
 730         "sequence": [
 731             ("random-sleep", random_sleep(300, 900)),
 732             ("test", make_test(include_envs=[
 733             "backupfromdc",
 734             "offlinebackupdc",
 735             "labdc",
 736             ])),
 737             ("lcov", LCOV_CMD),
 738             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 739         ],
 740     },
 741
 742     "samba-admem-mit": {
 743         "dependency": "samba-mit-build",
 744         "sequence": [
 745             ("random-sleep", random_sleep(1, 1)),
 746             ("test", make_test(include_envs=[
 747             "ad_member",
 748             "ad_member_idmap_rid",
 749             "admem_idmap_autorid",
 750             "ad_member_idmap_ad",
 751             "ad_member_rfc2307",
 752             "ad_member_idmap_nss",
 753             "ad_member_offlogon",
 754             ])),
 755             ("lcov", LCOV_CMD),
 756             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 757         ],
 758     },
 759
 760     "samba-addc-mit-1": {
 761         "dependency": "samba-mit-build",
 762         "sequence": [
 763             ("random-sleep", random_sleep(1, 1)),
 764             ("test", make_test(include_envs=[
 765             "ad_dc",
 766             "ad_dc_smb1",
 767             "ad_dc_smb1_done",
 768             "ad_dc_no_nss",
 769             "ad_dc_no_ntlm",
 770             ])),
 771             ("lcov", LCOV_CMD),
 772             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 773         ],
 774     },
 775
 776     "samba-addc-mit-4a": {
 777         "dependency": "samba-mit-build",
 778         "sequence": [
 779             ("random-sleep", random_sleep(1, 1)),
 780             ("test", make_test(include_envs=[
 781             "fl2000dc",
 782             "ad_member_oneway",
 783             "fl2003dc",
 784             ])),
 785             ("lcov", LCOV_CMD),
 786             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 787         ],
 788     },
 789     "samba-addc-mit-4b": {
 790         "dependency": "samba-mit-build",
 791         "sequence": [
 792             ("random-sleep", random_sleep(1, 1)),
 793             ("test", make_test(include_envs=[
 794             "fl2008dc",
 795             "fl2008r2dc",
 796             ])),
 797             ("lcov", LCOV_CMD),
 798             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 799         ],
 800     },
 801
 802     "samba-test-only": {
 803         "sequence": [
 804             ("configure", "./configure.developer  --abi-check-disable" + samba_configure_params),
 805             ("make", "make -j"),
 806             ("test", make_test(TESTS="${TESTS}")),
 807             ("lcov", LCOV_CMD),
 808         ],
 809     },
 810
 811     # Test cross-compile infrastructure
 812     "samba-xc": {
 813         "sequence": [
 814             ("random-sleep", random_sleep(900, 1500)),
 815             ("configure-native", "./configure.developer --with-selftest-prefix=./bin/ab" + samba_configure_params),
 816             ("configure-cross-execute", "./configure.developer --out ./bin-xe --cross-compile --cross-execute=script/identity_cc.sh" \
 817             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xe/ab" + samba_configure_params),
 818             ("verify-cross-execute-output", "grep '^Checking value of NSIG' ./bin-xe/cross-answers.txt"),
 819             ("configure-cross-answers", "./configure.developer --out ./bin-xa --cross-compile" \
 820             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa/ab" + samba_configure_params),
 821             ("compare-results", "script/compare_cc_results.py "
 822             "./bin/c4che/default{} "
 823             "./bin-xe/c4che/default{} "
 824             "./bin-xa/c4che/default{}".format(*([CACHE_SUFFIX]*3))),
 825             ("modify-cross-answers", "sed -i.bak -e 's/^\\(Checking value of NSIG:\\) .*/\\1 \"1234\"/' ./bin-xe/cross-answers.txt"),
 826             ("configure-cross-answers-modified", "./configure.developer --out ./bin-xa2 --cross-compile" \
 827             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa2/ab" + samba_configure_params),
 828             ("verify-cross-answers", "test $(sed -n -e 's/VALUEOF_NSIG = \\(.*\\)/\\1/p' ./bin-xa2/c4che/default{})" \
 829             " = \"'1234'\"".format(CACHE_SUFFIX)),
 830             ("invalidate-cross-answers", "sed -i.bak -e '/^Checking value of NSIG/d' ./bin-xe/cross-answers.txt"),
 831             ("configure-cross-answers-fail", "./configure.developer --out ./bin-xa3 --cross-compile" \
 832             " --cross-answers=./bin-xe/cross-answers.txt --with-selftest-prefix=./bin-xa3/ab" + samba_configure_params + \
 833             " ; test $? -ne 0"),
 834         ],
 835     },
 836
 837     # test build with -O3 -- catches extra warnings and bugs, tests the ad_dc environments
 838     "samba-o3": {
 839         "sequence": [
 840             ("random-sleep", random_sleep(300, 900)),
 841             ("configure", "ADDITIONAL_CFLAGS='-O3 -Wp,-D_FORTIFY_SOURCE=2' ./configure.developer --abi-check-disable" + himmelblau_configure_params + samba_configure_params),
 842             ("make", "make -j"),
 843             ("test", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 844             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 845             ("lcov", LCOV_CMD),
 846             ("install", "make install"),
 847             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 848             ("clean", "make clean"),
 849         ],
 850     },
 851
 852     "samba-32bit": {
 853         "sequence": [
 854             ("random-sleep", random_sleep(300, 900)),
 855             ("configure", "./configure.developer --abi-check-disable --disable-warnings-as-errors" + samba_configure_params),
 856             ("make", "make -j"),
 857             ("nonetest", make_test(cmd='make test', TESTS="--exclude=selftest/slow-none", include_envs=["none"])),
 858             ("quicktest", make_test(cmd='make quicktest', include_envs=["ad_dc", "ad_dc_smb1", "ad_dc_smb1_done"])),
 859             ("ktest", make_test(cmd='make test', include_envs=["ktest"])),
 860             ("install", "make install"),
 861             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 862             ("clean", "make clean"),
 863         ],
 864     },
 865
 866     "samba-ctdb": {
 867         "sequence": [
 868             ("random-sleep", random_sleep(900, 1500)),
 869
 870         # make sure we have tdb around:
 871             ("tdb-configure", "cd lib/tdb && PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${PREFIX_DIR}/lib/pkgconfig ./configure --bundled-libraries=NONE --abi-check --enable-debug -C ${PREFIX}"),
 872             ("tdb-make", "cd lib/tdb && make"),
 873             ("tdb-install", "cd lib/tdb && make install"),
 874
 875         # build samba with cluster support (also building ctdb):
 876             ("samba-configure",
 877          "PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH "
 878          "PKG_CONFIG_PATH=${PREFIX_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH} "
 879          "./configure.developer ${PREFIX} "
 880          "--with-selftest-prefix=./bin/ab "
 881          "--with-cluster-support "
 882          "--bundled-libraries=!tdb"),
 883             ("samba-make", "make"),
 884             ("samba-check", "./bin/smbd --configfile=/dev/null -b | grep CLUSTER_SUPPORT"),
 885             ("samba-install", "make install"),
 886             ("ctdb-check", "test -e ${PREFIX_DIR}/sbin/ctdbd"),
 887
 888             ("test", make_test(
 889                 cmd='PYTHONPATH=${PYTHON_PREFIX}:$PYTHONPATH make test',
 890                 INJECT_SELFTEST_PREFIX=0,
 891                 include_envs=["clusteredmember"])
 892             ),
 893
 894         # clean up:
 895             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
 896             ("clean", "make clean"),
 897             ("ctdb-clean", "cd ./ctdb && make clean"),
 898         ],
 899     },
 900
 901     "samba-libs": {
 902         "sequence": [
 903             ("random-sleep", random_sleep(300, 900)),
 904             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_libs),
 905             ("talloc-make", "cd lib/talloc && make"),
 906             ("talloc-install", "cd lib/talloc && make install"),
 907             ("talloc-abi-check1",
 908                 check_versioned_symbol(
 909                     "./lib/talloc/bin/shared/libtalloc.so.2",
 910                     "talloc_named",
 911                     "TALLOC_2.0.2"
 912                 )
 913             ),
 914             ("talloc-abi-check2",
 915                 check_versioned_symbol(
 916                     "./lib/talloc/bin/shared/libtalloc.so.2",
 917                     "talloc_asprintf_addbuf",
 918                     "TALLOC_2.3.5"
 919                 )
 920             ),
 921
 922             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_libs),
 923             ("tdb-make", "cd lib/tdb && make"),
 924             ("tdb-install", "cd lib/tdb && make install"),
 925             ("tdb-abi-check1",
 926                 check_versioned_symbol(
 927                     "./lib/tdb/bin/shared/libtdb.so.1",
 928                     "tdb_errorstr",
 929                     "TDB_1.2.1"
 930                 )
 931             ),
 932             ("tdb-abi-check2",
 933                 check_versioned_symbol(
 934                     "./lib/tdb/bin/shared/libtdb.so.1",
 935                     "tdb_traverse_chain",
 936                     "TDB_1.3.17"
 937                 )
 938             ),
 939
 940             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_libs),
 941             ("tevent-make", "cd lib/tevent && make"),
 942             ("tevent-install", "cd lib/tevent && make install"),
 943             ("tevent-abi-check1",
 944                 check_versioned_symbol(
 945                     "./lib/tevent/bin/shared/libtevent.so.0",
 946                     "_tevent_loop_once",
 947                     "TEVENT_0.9.9"
 948                 )
 949             ),
 950             ("tevent-abi-check2",
 951                 check_versioned_symbol(
 952                     "./lib/tevent/bin/shared/libtevent.so.0",
 953                     "__tevent_req_create",
 954                     "TEVENT_0.15.0"
 955                 )
 956             ),
 957
 958             ("nondevel-configure", samba_libs_envvars + " ./configure --private-libraries='!ldb' --vendor-suffix=TEST-STRING~5.1.2 ${PREFIX}"),
 959             ("nondevel-make", "make -j"),
 960             ("nondevel-check", "./bin/smbd -b | grep WITH_NTVFS_FILESERVER && exit 1; exit 0"),
 961             ("nondevel-check", "./bin/smbd --version | grep -F 'TEST-STRING~5.1.2' && exit 0; exit 1"),
 962             ("nondevel-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
 963             ("nondevel-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
 964             ("nondevel-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
 965             ("nondevel-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 966             ("nondevel-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 967             ("nondevel-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 968             ("nondevel-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 969             ("nondevel-no-public-nss_winbind",
 970                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
 971             ("nondevel-no-public-nss_wins",
 972                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
 973             ("nondevel-no-public-libwbclient",
 974                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
 975             ("nondevel-libwbclient-wbcCtxPingDc2@WBCLIENT_0.12",
 976                 check_versioned_symbol("./bin/shared/libwbclient.so.0", "wbcCtxPingDc2", "WBCLIENT_0.12")),
 977             ("nondevel-no-public-pam_winbind",
 978                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
 979             ("nondevel-no-public-winbind_krb5_locator",
 980                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
 981             ("nondevel-no-public-async_dns_krb5_locator",
 982                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
 983             ("nondevel-libndr-krb5pac-ndr_pull_PAC_DATA@NDR_KRB5PAC_0.0.1",
 984                 check_versioned_symbol("./bin/shared/libndr-krb5pac.so.0", "ndr_pull_PAC_DATA", "NDR_KRB5PAC_0.0.1")),
 985             ("nondevel-install", "make -j install"),
 986             ("nondevel-dist", "make dist"),
 987
 988             ("prefix-no-private-libtalloc", "find ${PREFIX_DIR} | grep -v 'libtalloc-report' | grep 'private.*libtalloc' && exit 1; exit 0"),
 989             ("prefix-no-private-libtdb", "find ${PREFIX_DIR} | grep -v 'libtdb-wrap' | grep 'private.*libtdb' && exit 1; exit 0"),
 990             ("prefix-no-private-libtevent", "find ${PREFIX_DIR} | grep -v 'libtevent-util' | grep 'private.*libtevent' && exit 1; exit 0"),
 991             ("prefix-no-private-libldb", "find ${PREFIX_DIR} | grep -v 'module' | grep -v 'libldbsamba' | grep 'private.*libldb.so' && exit 1; exit 0"),
 992             ("prefix-public-libldb", "find ${PREFIX_DIR} | grep 'lib/libldb.so' && exit 0; exit 1"),
 993             ("prefix-no-samba-nss_winbind", "ldd ${PREFIX_DIR}/lib/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
 994             ("prefix-no-samba-nss_wins", "ldd ${PREFIX_DIR}/lib/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
 995             ("prefix-no-samba-libwbclient", "ldd ${PREFIX_DIR}/lib/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
 996             ("prefix-no-samba-pam_winbind", "ldd ${PREFIX_DIR}/lib/security/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
 997             ("prefix-no-public-nss_winbind",
 998                 check_symbols("${PREFIX_DIR}/lib/libnss_winbind.so.2", "_nss_winbind_")),
 999             ("prefix-no-public-nss_wins",
1000                 check_symbols("${PREFIX_DIR}/lib/libnss_wins.so.2", "_nss_wins_")),
1001             ("prefix-no-public-libwbclient",
1002                 check_symbols("${PREFIX_DIR}/lib/libwbclient.so.0", "wbc")),
1003             ("prefix-no-public-pam_winbind",
1004                 check_symbols("${PREFIX_DIR}/lib/security/pam_winbind.so", "pam_sm_")),
1005             ("prefix-no-public-winbind_krb5_locator",
1006                 check_symbols("${PREFIX_DIR}/lib/krb5/winbind_krb5_locator.so",
1007                               "service_locator")),
1008             ("prefix-no-public-async_dns_krb5_locator",
1009                 check_symbols("${PREFIX_DIR}/lib/krb5/async_dns_krb5_locator.so",
1010                               "service_locator")),
1011
1012             # retry with all modules shared
1013             ("allshared-distclean", "make distclean"),
1014             ("allshared-configure", samba_libs_configure_samba + " --with-shared-modules=ALL"),
1015             ("allshared-make", "make -j"),
1016             ("allshared-no-libtalloc", "find ./bin | grep -v 'libtalloc-report' | grep 'libtalloc' && exit 1; exit 0"),
1017             ("allshared-no-libtdb", "find ./bin | grep -v 'libtdb-wrap' | grep 'libtdb' && exit 1; exit 0"),
1018             ("allshared-no-libtevent", "find ./bin | grep -v 'libtevent-util' | grep 'libtevent' && exit 1; exit 0"),
1019             ("allshared-no-samba-nss_winbind", "ldd ./bin/plugins/libnss_winbind.so.2 | grep 'samba' && exit 1; exit 0"),
1020             ("allshared-no-samba-nss_wins", "ldd ./bin/plugins/libnss_wins.so.2 | grep 'samba' && exit 1; exit 0"),
1021             ("allshared-no-samba-libwbclient", "ldd ./bin/shared/libwbclient.so.0 | grep 'samba' && exit 1; exit 0"),
1022             ("allshared-no-samba-pam_winbind", "ldd ./bin/plugins/pam_winbind.so | grep -v 'libtalloc.so.2' | grep 'samba' && exit 1; exit 0"),
1023             ("allshared-no-public-nss_winbind",
1024                 check_symbols("./bin/plugins/libnss_winbind.so.2", "_nss_winbind_")),
1025             ("allshared-no-public-nss_wins",
1026                 check_symbols("./bin/plugins/libnss_wins.so.2", "_nss_wins_")),
1027             ("allshared-no-public-libwbclient",
1028                 check_symbols("./bin/shared/libwbclient.so.0", "wbc")),
1029             ("allshared-no-public-pam_winbind",
1030                 check_symbols("./bin/plugins/pam_winbind.so", "pam_sm_")),
1031             ("allshared-no-public-winbind_krb5_locator",
1032                 check_symbols("./bin/plugins/winbind_krb5_locator.so", "service_locator")),
1033             ("allshared-no-public-async_dns_krb5_locator",
1034                 check_symbols("./bin/plugins/async_dns_krb5_locator.so", "service_locator")),
1035         ],
1036     },
1037
1038     "samba-fuzz": {
1039         "sequence": [
1040         # build the fuzzers (static) via the oss-fuzz script
1041             ("fuzzers-mkdir-prefix", "mkdir -p ${PREFIX_DIR}"),
1042             ("fuzzers-build", "OUT=${PREFIX_DIR} LIB_FUZZING_ENGINE= SANITIZER=address CXX= CFLAGS= ADDITIONAL_LDFLAGS='-fuse-ld=bfd' ./lib/fuzzing/oss-fuzz/build_samba.sh --enable-afl-fuzzer"),
1043         ],
1044     },
1045
1046     # * Test smbd and smbtorture can build semi-static
1047     #
1048     # * Test Samba without python still builds.
1049     #
1050     # When this test fails due to more use of Python, the expectations
1051     # is that the newly failing part of the code should be disabled
1052     # when --disable-python is set (rather than major work being done
1053     # to support this environment).
1054     #
1055     # The target here is for vendors shipping a minimal smbd.
1056     "samba-minimal-smbd": {
1057         "sequence": [
1058             ("random-sleep", random_sleep(300, 900)),
1059
1060         # build with all modules static
1061             ("allstatic-configure", "./configure.developer " + samba_configure_params + " --with-static-modules=ALL"),
1062             ("allstatic-make", "make -j"),
1063             ("allstatic-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
1064             ("allstatic-lcov", LCOV_CMD),
1065             ("allstatic-def-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1066             ("allstatic-def-clean", "make clean"),
1067
1068         # force all libraries as private
1069             ("allprivate-def-distclean", "make distclean"),
1070             ("allprivate-def-configure", "./configure.developer " + samba_configure_params + " --private-libraries=ALL"),
1071             ("allprivate-def-make", "make -j"),
1072             # note wrapper libraries need to be public
1073             ("allprivate-def-no-public", "ls ./bin/shared | egrep -v '^private$|lib[nprsu][saeoi][smscd].*-wrapper.so$|pam_set_items.so|pam_matrix.so' | wc -l | grep -q '^0'"),
1074             ("allprivate-def-only-private-ext", "ls ./bin/shared/private | egrep 'private-samba' | wc -l | grep -q '^0' && exit 1; exit 0"),
1075             ("allprivate-def-no-non-private-ext", "ls ./bin/shared/private | egrep -v 'private-samba|^libpypamtest.so$' | wc -l | grep -q '^0'"),
1076             ("allprivate-def-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
1077             ("allprivate-def-lcov", LCOV_CMD),
1078             ("allprivate-def-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1079             ("allprivate-def-clean", "make clean"),
1080
1081         # force all libraries as private with a non default
1082         # extension and 2 exceptions
1083             ("allprivate-ext-distclean", "make distclean"),
1084             ("allprivate-ext-configure", "./configure.developer " + samba_configure_params + " --private-libraries=ALL --private-library-extension=private-library --private-extension-exception=pac,ndr"),
1085             ("allprivate-ext-make", "make -j"),
1086             # note wrapper libraries need to be public
1087             ("allprivate-ext-no-public", "ls ./bin/shared | egrep -v '^private$|lib[nprsu][saeoi][smscd].*-wrapper.so$|pam_set_items.so|pam_matrix.so' | wc -l | grep -q '^0'"),
1088             ("allprivate-ext-no-private-default-ext", "ls ./bin/shared/private | grep 'private-samba' | wc -l | grep -q '^0'"),
1089             ("allprivate-ext-has-private-ext", "ls ./bin/shared/private | grep 'private-library' | wc -l | grep -q '^0' && exit 1; exit 0"),
1090             ("allprivate-ext-libndr-no-private-ext", "ls ./bin/shared/private | grep -v 'private-library' | grep 'libndr' | wc -l | grep -q '^1'"),
1091             ("allprivate-ext-libpac-no-private-ext", "ls ./bin/shared/private | grep -v 'private-library' | grep 'libpac' | wc -l | grep -q '^1'"),
1092             ("allprivate-ext-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
1093             ("allprivate-ext-lcov", LCOV_CMD),
1094             ("allprivate-ext-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1095             ("allprivate-ext-clean", "make clean"),
1096
1097         # retry with nonshared smbd and smbtorture
1098             ("nonshared-distclean", "make distclean"),
1099             ("nonshared-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --nonshared-binary=smbtorture,smbd/smbd"),
1100             ("nonshared-make", "make -j"),
1101             ("nonshared-test", make_test(TESTS="samba3.smb2.create.*nt4_dc")),
1102             ("nonshared-lcov", LCOV_CMD),
1103             ("nonshared-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1104             ("nonshared-clean", "make clean"),
1105
1106         # retry without winbindd
1107             ("nonwinbind-distclean", "make distclean"),
1108             ("nonwinbind-configure", "./configure.developer " + samba_configure_params + " --bundled-libraries=ALL --with-static-modules=ALL --without-winbind"),
1109             ("nonwinbind-make", "make -j"),
1110             ("nonwinbind-test", make_test(TESTS="samba3.smb2.*.simpleserver")),
1111             ("nonwinbind-lcov", LCOV_CMD),
1112             ("nonwinbind-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1113             ("nonwinbind-clean", "make clean"),
1114         ],
1115     },
1116
1117     "samba-nopython": {
1118         "sequence": [
1119             ("random-sleep", random_sleep(300, 900)),
1120
1121             ("configure", "./configure.developer ${ENABLE_COVERAGE} ${PREFIX} --with-profiling-data --disable-python --without-ad-dc"),
1122             ("make", "make -j"),
1123             ("find-python", "script/find_python.sh ${PREFIX}"),
1124             ("test", "make test-nopython"),
1125             ("lcov", LCOV_CMD),
1126             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1127             ("clean", "make clean"),
1128
1129             ("talloc-configure", "cd lib/talloc && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1130             ("talloc-make", "cd lib/talloc && make"),
1131             ("talloc-install", "cd lib/talloc && make install"),
1132
1133             ("tdb-configure", "cd lib/tdb && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1134             ("tdb-make", "cd lib/tdb && make"),
1135             ("tdb-install", "cd lib/tdb && make install"),
1136
1137             ("tevent-configure", "cd lib/tevent && " + samba_libs_configure_base + " --bundled-libraries=cmocka,NONE --disable-python"),
1138             ("tevent-make", "cd lib/tevent && make"),
1139             ("tevent-install", "cd lib/tevent && make install"),
1140
1141         # retry against installed library packages, but no required modules
1142             ("libs-configure", samba_libs_configure_base + samba_libs_configure_bundled_libs + " --disable-python --without-ad-dc  --with-static-modules=!FORCED,!DEFAULT --with-shared-modules=!FORCED,!DEFAULT"),
1143             ("libs-make", "make -j"),
1144             ("libs-install", "make install"),
1145             ("libs-check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1146             ("libs-clean", "make clean"),
1147
1148         ],
1149     },
1150
1151     "samba-codecheck": {
1152         "sequence": [
1153             ("run", "script/check-shell-scripts.sh ."),
1154             ("run", "script/codespell.sh ."),
1155         ],
1156     },
1157
1158     "tdb": {
1159         "sequence": [
1160             ("random-sleep", random_sleep(60, 600)),
1161             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1162             ("make", "make"),
1163             ("install", "make install"),
1164             ("test", "make test"),
1165             ("lcov", LCOV_CMD),
1166             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1167             ("distcheck", "make distcheck"),
1168             ("clean", "make clean"),
1169         ],
1170     },
1171
1172     "talloc": {
1173         "sequence": [
1174             ("random-sleep", random_sleep(60, 600)),
1175             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1176             ("make", "make"),
1177             ("install", "make install"),
1178             ("test", "make test"),
1179             ("lcov", LCOV_CMD),
1180             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1181             ("distcheck", "make distcheck"),
1182             ("clean", "make clean"),
1183         ],
1184     },
1185
1186     "replace": {
1187         "sequence": [
1188             ("random-sleep", random_sleep(60, 600)),
1189             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1190             ("make", "make"),
1191             ("install", "make install"),
1192             ("test", "make test"),
1193             ("lcov", LCOV_CMD),
1194             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1195             ("distcheck", "make distcheck"),
1196             ("clean", "make clean"),
1197         ],
1198     },
1199
1200     "tevent": {
1201         "sequence": [
1202             ("random-sleep", random_sleep(60, 600)),
1203             ("configure", "./configure ${ENABLE_COVERAGE} --enable-developer -C ${PREFIX}"),
1204             ("make", "make"),
1205             ("install", "make install"),
1206             ("test", "make test"),
1207             ("lcov", LCOV_CMD),
1208             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1209             ("distcheck", "make distcheck"),
1210             ("clean", "make clean"),
1211         ],
1212     },
1213
1214     "pidl": {
1215         "git-clone-required": True,
1216         "sequence": [
1217             ("random-sleep", random_sleep(60, 600)),
1218             ("configure", "perl Makefile.PL PREFIX=${PREFIX_DIR}"),
1219             ("touch", "touch *.yp"),
1220             ("make", "make"),
1221             ("test", "make test"),
1222             ("install", "make install"),
1223             ("checkout-yapp-generated", "git checkout lib/Parse/Pidl/IDL.pm lib/Parse/Pidl/Expr.pm"),
1224             ("check-clean-tree", CLEAN_SOURCE_TREE_CMD),
1225             ("clean", "make clean"),
1226         ],
1227     },
1228
1229     # these are useful for debugging autobuild
1230     "pass": {
1231         "sequence": [
1232             ("pass", 'echo passing && /bin/true'),
1233         ],
1234     },
1235     "fail": {
1236         "sequence": [
1237             ("fail", 'echo failing && /bin/false'),
1238         ],
1239     },
1240 }
1241
1242 defaulttasks = list(tasks.keys())
1243
1244 defaulttasks.remove("pass")
1245 defaulttasks.remove("fail")
1246
1247 # The build tasks will be brought in by the test tasks as needed
1248 defaulttasks.remove("samba-def-build")
1249 defaulttasks.remove("samba-nt4-build")
1250 defaulttasks.remove("samba-mit-build")
1251 defaulttasks.remove("samba-h5l-build")
1252 defaulttasks.remove("samba-no-opath-build")
1253
1254 # This is not a normal test, but a task to support manually running
1255 # one test under autobuild
1256 defaulttasks.remove("samba-test-only")
1257
1258 # Only built on GitLab CI and not in the default autobuild because it
1259 # uses too much space (4GB of semi-static binaries)
1260 defaulttasks.remove("samba-fuzz")
1261
1262 # The FIPS build runs only in GitLab CI on a current Fedora Docker
1263 # container where a simulated FIPS mode is possible.
1264 defaulttasks.remove("samba-fips")
1265
1266 # The MIT build runs on a current Fedora where an up to date MIT KDC
1267 # is already packaged.  This avoids needing to backport a current MIT
1268 # to the default Ubuntu 18.04, particularly during development, and
1269 # the need to install on the shared sn-devel-184.
1270
1271 defaulttasks.remove("samba-mitkrb5")
1272 defaulttasks.remove("samba-admem-mit")
1273 defaulttasks.remove("samba-addc-mit-1")
1274 defaulttasks.remove("samba-addc-mit-4a")
1275 defaulttasks.remove("samba-addc-mit-4b")
1276
1277 defaulttasks.remove("samba-32bit")
1278
1279 if os.environ.get("AUTOBUILD_SKIP_SAMBA_O3", "0") == "1":
1280     defaulttasks.remove("samba-o3")
1281
1282
1283 def do_print(msg):
1284     logger.info(msg)
1285     sys.stdout.flush()
1286     sys.stderr.flush()
1287
1288 def do_debug(msg):
1289     logger.debug(msg)
1290     sys.stdout.flush()
1291     sys.stderr.flush()
1292
1293
1294 def run_cmd(cmd, dir=".", show=None, output=False, checkfail=True):
1295     if show is None:
1296         do_debug("Running: '%s' in '%s'" % (cmd, dir))
1297     elif show:
1298         do_print("Running: '%s' in '%s'" % (cmd, dir))
1299
1300     if output:
1301         out = check_output([cmd], shell=True, cwd=dir)
1302         return out.decode(encoding='utf-8', errors='backslashreplace')
1303     elif checkfail:
1304         return check_call(cmd, shell=True, cwd=dir)
1305     else:
1306         return call(cmd, shell=True, cwd=dir)
1307
1308 def rmdir_force(dirname, re_raise=True):
1309     try:
1310         run_cmd("test -d %s && chmod -R +w %s; rm -rf %s" % (
1311                 dirname, dirname, dirname), output=True, show=True)
1312     except CalledProcessError as e:
1313         do_print("Failed: '%s'" % (str(e)))
1314         run_cmd("tree %s" % dirname, output=True, show=True)
1315         if re_raise:
1316             raise
1317         return False
1318     return True
1319
1320 class builder(object):
1321     '''handle build of one directory'''
1322
1323     def __init__(self, name, definition):
1324         self.name = name
1325         self.dir = builddirs.get(name, '.')
1326         self.tag = self.name.replace('/', '_')
1327         self.definition = definition
1328         self.sequence = definition["sequence"]
1329         self.git_clone_required = False
1330         if "git-clone-required" in definition:
1331             self.git_clone_required = bool(definition["git-clone-required"])
1332         self.proc = None
1333         self.done = False
1334         self.next = 0
1335         self.stdout_path = "%s/%s.stdout" % (gitroot, self.tag)
1336         self.stderr_path = "%s/%s.stderr" % (gitroot, self.tag)
1337         do_debug("stdout for %s in %s" % (self.name, self.stdout_path))
1338         do_debug("stderr for %s in %s" % (self.name, self.stderr_path))
1339         run_cmd("rm -f %s %s" % (self.stdout_path, self.stderr_path))
1340         self.stdout = open(self.stdout_path, 'w')
1341         self.stderr = open(self.stderr_path, 'w')
1342         self.stdin  = open("/dev/null", 'r')
1343         self.builder_dir = "%s/%s" % (testbase, self.tag)
1344         self.test_source_dir = self.builder_dir
1345         self.cwd = "%s/%s" % (self.builder_dir, self.dir)
1346         self.selftest_prefix = "%s/bin/ab" % (self.cwd)
1347         self.prefix = "%s/%s" % (test_prefix, self.tag)
1348         self.consumers = []
1349         self.producer = None
1350
1351         if self.git_clone_required:
1352             assert "dependency" not in definition
1353
1354     def mark_existing(self):
1355         do_debug('%s: Mark as existing dependency' % self.name)
1356         self.next = len(self.sequence)
1357         self.done = True
1358
1359     def add_consumer(self, consumer):
1360         do_debug("%s: add consumer: %s" % (self.name, consumer.name))
1361         consumer.producer = self
1362         consumer.test_source_dir = self.test_source_dir
1363         self.consumers.append(consumer)
1364
1365     def start_next(self):
1366         if self.producer is not None:
1367             if not self.producer.done:
1368                 do_debug("%s: Waiting for producer: %s" % (self.name, self.producer.name))
1369                 return
1370
1371         if self.next == 0:
1372             rmdir_force(self.builder_dir)
1373             rmdir_force(self.prefix)
1374             if self.producer is not None:
1375                 run_cmd("mkdir %s" % (self.builder_dir), dir=test_master, show=True)
1376             elif not self.git_clone_required:
1377                 run_cmd("cp -R -a -l %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1378             else:
1379                 run_cmd("git clone --recursive --shared %s %s" % (test_master, self.builder_dir), dir=test_master, show=True)
1380
1381         if self.next == len(self.sequence):
1382             if not self.done:
1383                 do_print('%s: Completed OK' % self.name)
1384                 self.done = True
1385             if not options.nocleanup and len(self.consumers) == 0:
1386                 do_print('%s: Cleaning up' % self.name)
1387                 rmdir_force(self.builder_dir)
1388                 rmdir_force(self.prefix)
1389             for consumer in self.consumers:
1390                 if consumer.next != 0:
1391                     continue
1392                 do_print('%s: Starting consumer %s' % (self.name, consumer.name))
1393                 consumer.start_next()
1394             if self.producer is not None:
1395                 self.producer.consumers.remove(self)
1396                 assert self.producer.done
1397                 self.producer.start_next()
1398             do_print('%s: Remaining consumers %u' % (self.name, len(self.consumers)))
1399             return
1400         (self.stage, self.cmd) = self.sequence[self.next]
1401         self.cmd = self.cmd.replace("${PYTHON_PREFIX}",
1402                                     get_path(name='platlib',
1403                                              scheme="posix_prefix",
1404                                              vars={"base": self.prefix,
1405                                                    "platbase": self.prefix}))
1406         self.cmd = self.cmd.replace("${PREFIX}", "--prefix=%s" % self.prefix)
1407         self.cmd = self.cmd.replace("${PREFIX_DIR}", "%s" % self.prefix)
1408         self.cmd = self.cmd.replace("${TESTS}", options.restrict_tests)
1409         self.cmd = self.cmd.replace("${TEST_SOURCE_DIR}", self.test_source_dir)
1410         self.cmd = self.cmd.replace("${SELFTEST_PREFIX}", self.selftest_prefix)
1411         self.cmd = self.cmd.replace("${LOG_BASE}", options.log_base)
1412         self.cmd = self.cmd.replace("${NAME}", self.name)
1413         self.cmd = self.cmd.replace("${ENABLE_COVERAGE}", options.enable_coverage)
1414         do_print('%s: [%s] Running %s in %r' % (self.name, self.stage, self.cmd, self.cwd))
1415         self.proc = Popen(self.cmd, shell=True,
1416                           close_fds=True, cwd=self.cwd,
1417                           stdout=self.stdout, stderr=self.stderr, stdin=self.stdin)
1418         self.next += 1
1419
1420 def expand_dependencies(n):
1421     deps = list()
1422     if "dependency" in tasks[n]:
1423         depname = tasks[n]["dependency"]
1424         assert depname in tasks
1425         sdeps = expand_dependencies(depname)
1426         assert n not in sdeps
1427         for sdep in sdeps:
1428             deps.append(sdep)
1429         deps.append(depname)
1430     return deps
1431
1432
1433 class buildlist(object):
1434     '''handle build of multiple directories'''
1435
1436     def __init__(self, tasknames, rebase_url, rebase_branch="master"):
1437         self.tail_proc = None
1438         self.retry = None
1439         if not tasknames:
1440             if options.restrict_tests:
1441                 tasknames = ["samba-test-only"]
1442             else:
1443                 tasknames = defaulttasks
1444
1445         given_tasknames = tasknames.copy()
1446         implicit_tasknames = []
1447         for n in given_tasknames:
1448             deps = expand_dependencies(n)
1449             for dep in deps:
1450                 if dep in given_tasknames:
1451                     continue
1452                 if dep in implicit_tasknames:
1453                     continue
1454                 implicit_tasknames.append(dep)
1455
1456         tasknames = implicit_tasknames.copy()
1457         tasknames.extend(given_tasknames)
1458         do_debug("given_tasknames: %s" % given_tasknames)
1459         do_debug("implicit_tasknames: %s" % implicit_tasknames)
1460         do_debug("tasknames: %s" % tasknames)
1461         self.tlist = [builder(n, tasks[n]) for n in tasknames]
1462
1463         if options.retry:
1464             rebase_remote = "rebaseon"
1465             retry_task = {
1466                     "git-clone-required": True,
1467                     "sequence": [
1468                             ("retry",
1469                             '''set -e
1470                             git remote add -t %s %s %s
1471                             git fetch %s
1472                             while :; do
1473                               sleep 60
1474                               git describe %s/%s > old_remote_branch.desc
1475                               git fetch %s
1476                               git describe %s/%s > remote_branch.desc
1477                               diff old_remote_branch.desc remote_branch.desc
1478                             done
1479                            ''' % (
1480                                rebase_branch, rebase_remote, rebase_url,
1481                                rebase_remote,
1482                                rebase_remote, rebase_branch,
1483                                rebase_remote,
1484                                rebase_remote, rebase_branch
1485                             ))]}
1486
1487             self.retry = builder('retry', retry_task)
1488             self.need_retry = False
1489
1490         if options.skip_dependencies:
1491             for b in self.tlist:
1492                 if b.name in implicit_tasknames:
1493                     b.mark_existing()
1494
1495         for b in self.tlist:
1496             do_debug("b.name=%s" % b.name)
1497             if "dependency" not in b.definition:
1498                 continue
1499             depname = b.definition["dependency"]
1500             do_debug("b.name=%s: dependency:%s" % (b.name, depname))
1501             for p in self.tlist:
1502                 if p.name == depname:
1503                     p.add_consumer(b)
1504
1505     def kill_kids(self):
1506         if self.tail_proc is not None:
1507             self.tail_proc.terminate()
1508             self.tail_proc.wait()
1509             self.tail_proc = None
1510         if self.retry is not None:
1511             self.retry.proc.terminate()
1512             self.retry.proc.wait()
1513             self.retry = None
1514         for b in self.tlist:
1515             if b.proc is not None:
1516                 run_cmd("killbysubdir %s > /dev/null 2>&1" % b.test_source_dir, checkfail=False)
1517                 b.proc.terminate()
1518                 b.proc.wait()
1519                 b.proc = None
1520
1521     def wait_one(self):
1522         while True:
1523             none_running = True
1524             for b in self.tlist:
1525                 if b.proc is None:
1526                     continue
1527                 none_running = False
1528                 b.status = b.proc.poll()
1529                 if b.status is None:
1530                     continue
1531                 b.proc = None
1532                 return b
1533             if options.retry:
1534                 ret = self.retry.proc.poll()
1535                 if ret is not None:
1536                     self.need_retry = True
1537                     self.retry = None
1538                     return None
1539             if none_running:
1540                 return None
1541             time.sleep(0.1)
1542
1543     def run(self):
1544         for b in self.tlist:
1545             b.start_next()
1546         if options.retry:
1547             self.retry.start_next()
1548         while True:
1549             b = self.wait_one()
1550             if options.retry and self.need_retry:
1551                 self.kill_kids()
1552                 do_print("retry needed")
1553                 return (0, None, None, None, "retry")
1554             if b is None:
1555                 break
1556             if os.WIFSIGNALED(b.status) or os.WEXITSTATUS(b.status) != 0:
1557                 self.kill_kids()
1558                 return (b.status, b.name, b.stage, b.tag, "%s: [%s] failed '%s' with status %d" % (b.name, b.stage, b.cmd, b.status))
1559             b.start_next()
1560         self.kill_kids()
1561         return (0, None, None, None, "All OK")
1562
1563     def write_system_info(self, filename):
1564         with open(filename, 'w') as f:
1565             for cmd in ['uname -a',
1566                         'lsb_release -a',
1567                         'free',
1568                         'mount',
1569                         'cat /proc/cpuinfo',
1570                         'cc --version',
1571                         'df -m .',
1572                         'df -m %s' % testbase]:
1573                 try:
1574                     out = run_cmd(cmd, output=True, checkfail=False)
1575                 except CalledProcessError as e:
1576                     out = "<failed: %s>" % str(e)
1577                 print('### %s' % cmd, file=f)
1578                 print(out, file=f)
1579                 print(file=f)
1580
1581     def tarlogs(self, fname):
1582         with tarfile.open(fname, "w:gz") as tar:
1583             for b in self.tlist:
1584                 tar.add(b.stdout_path, arcname="%s.stdout" % b.tag)
1585                 tar.add(b.stderr_path, arcname="%s.stderr" % b.tag)
1586             if os.path.exists("autobuild.log"):
1587                 tar.add("autobuild.log")
1588             filename = 'system-info.txt'
1589             self.write_system_info(filename)
1590             tar.add(filename)
1591
1592     def remove_logs(self):
1593         for b in self.tlist:
1594             os.unlink(b.stdout_path)
1595             os.unlink(b.stderr_path)
1596
1597     def start_tail(self):
1598         cmd = ["tail", "-f"]
1599         for b in self.tlist:
1600             cmd.append(b.stdout_path)
1601             cmd.append(b.stderr_path)
1602         self.tail_proc = Popen(cmd, close_fds=True)
1603
1604
1605 def cleanup(do_raise=False):
1606     if options.nocleanup:
1607         return
1608     run_cmd("stat %s || true" % test_tmpdir, show=True)
1609     run_cmd("stat %s" % testbase, show=True)
1610     do_print("Cleaning up %r" % cleanup_list)
1611     for d in cleanup_list:
1612         ok = rmdir_force(d, re_raise=False)
1613         if ok:
1614             continue
1615         if os.path.isdir(d):
1616             do_print("Killing, waiting and retry")
1617             run_cmd("killbysubdir %s > /dev/null 2>&1" % d, checkfail=False)
1618         else:
1619             do_print("Waiting and retry")
1620         time.sleep(1)
1621         rmdir_force(d, re_raise=do_raise)
1622
1623
1624 def daemonize(logfile):
1625     pid = os.fork()
1626     if pid == 0:  # Parent
1627         os.setsid()
1628         pid = os.fork()
1629         if pid != 0:  # Actual daemon
1630             os._exit(0)
1631     else:  # Grandparent
1632         os._exit(0)
1633
1634     import resource      # Resource usage information.
1635     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
1636     if maxfd == resource.RLIM_INFINITY:
1637         maxfd = 1024  # Rough guess at maximum number of open file descriptors.
1638     for fd in range(0, maxfd):
1639         try:
1640             os.close(fd)
1641         except OSError:
1642             pass
1643     os.open(logfile, os.O_RDWR | os.O_CREAT)
1644     os.dup2(0, 1)
1645     os.dup2(0, 2)
1646
1647
1648 def write_pidfile(fname):
1649     '''write a pid file, cleanup on exit'''
1650     with open(fname, mode='w') as f:
1651         f.write("%u\n" % os.getpid())
1652
1653
1654 def rebase_tree(rebase_url, rebase_branch="master"):
1655     rebase_remote = "rebaseon"
1656     do_print("Rebasing on %s" % rebase_url)
1657     run_cmd("git describe HEAD", show=True, dir=test_master)
1658     run_cmd("git remote add -t %s %s %s" %
1659             (rebase_branch, rebase_remote, rebase_url),
1660             show=True, dir=test_master)
1661     run_cmd("git fetch %s" % rebase_remote, show=True, dir=test_master)
1662     if options.fix_whitespace:
1663         run_cmd("git rebase --force-rebase --whitespace=fix %s/%s" %
1664                 (rebase_remote, rebase_branch),
1665                 show=True, dir=test_master)
1666     else:
1667         run_cmd("git rebase --force-rebase %s/%s" %
1668                 (rebase_remote, rebase_branch),
1669                 show=True, dir=test_master)
1670     diff = run_cmd("git --no-pager diff HEAD %s/%s" %
1671                    (rebase_remote, rebase_branch),
1672                    dir=test_master, output=True)
1673     if diff == '':
1674         do_print("No differences between HEAD and %s/%s - exiting" %
1675                  (rebase_remote, rebase_branch))
1676         sys.exit(0)
1677     run_cmd("git describe %s/%s" %
1678             (rebase_remote, rebase_branch),
1679             show=True, dir=test_master)
1680     run_cmd("git describe HEAD", show=True, dir=test_master)
1681     run_cmd("git --no-pager diff --stat HEAD %s/%s" %
1682             (rebase_remote, rebase_branch),
1683             show=True, dir=test_master)
1684
1685
1686 def push_to(push_url, push_branch="master"):
1687     push_remote = "pushto"
1688     do_print("Pushing to %s" % push_url)
1689     if options.mark:
1690         run_cmd("git config --replace-all core.editor script/commit_mark.sh", dir=test_master)
1691         run_cmd("git commit --amend -c HEAD", dir=test_master)
1692         # the notes method doesn't work yet, as metze hasn't allowed refs/notes/* in master
1693         # run_cmd("EDITOR=script/commit_mark.sh git notes edit HEAD", dir=test_master)
1694     run_cmd("git remote add -t %s %s %s" %
1695             (push_branch, push_remote, push_url),
1696             show=True, dir=test_master)
1697     run_cmd("git push %s +HEAD:%s" %
1698             (push_remote, push_branch),
1699             show=True, dir=test_master)
1700
1701
1702 def send_email(subject, text, log_tar):
1703     if options.email is None:
1704         do_print("not sending email because the recipient is not set")
1705         do_print("the text content would have been:\n\nSubject: %s\n\n%s" %
1706                  (subject, text))
1707         return
1708     outer = MIMEMultipart()
1709     outer['Subject'] = subject
1710     outer['To'] = options.email
1711     outer['From'] = options.email_from
1712     outer['Date'] = email.utils.formatdate(localtime=True)
1713     outer.preamble = 'Autobuild mails are now in MIME because we optionally attach the logs.\n'
1714     outer.attach(MIMEText(text, 'plain', 'utf-8'))
1715     if options.attach_logs:
1716         with open(log_tar, 'rb') as fp:
1717             msg = MIMEApplication(fp.read(), 'gzip', email.encoders.encode_base64)
1718         # Set the filename parameter
1719         msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(log_tar))
1720         outer.attach(msg)
1721     content = outer.as_string()
1722     s = smtplib.SMTP(options.email_server)
1723     email_user = os.getenv('SMTP_USERNAME')
1724     email_password = os.getenv('SMTP_PASSWORD')
1725     if email_user is not None:
1726         s.starttls()
1727         s.login(email_user, email_password)
1728
1729     s.sendmail(options.email_from, [options.email], content)
1730     s.set_debuglevel(1)
1731     s.quit()
1732
1733
1734 def email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1735                   elapsed_time, log_base=None, add_log_tail=True):
1736     '''send an email to options.email about the failure'''
1737     elapsed_minutes = elapsed_time / 60.0
1738     if log_base is None:
1739         log_base = gitroot
1740     text = '''
1741 Dear Developer,
1742
1743 Your autobuild on %s failed after %.1f minutes
1744 when trying to test %s with the following error:
1745
1746    %s
1747
1748 the autobuild has been abandoned. Please fix the error and resubmit.
1749
1750 A summary of the autobuild process is here:
1751
1752   %s/autobuild.log
1753 ''' % (platform.node(), elapsed_minutes, failed_task, errstr, log_base)
1754
1755     if options.restrict_tests:
1756         text += """
1757 The build was restricted to tests matching %s\n""" % options.restrict_tests
1758
1759     if failed_task != 'rebase':
1760         text += '''
1761 You can see logs of the failed task here:
1762
1763   %s/%s.stdout
1764   %s/%s.stderr
1765
1766 or you can get full logs of all tasks in this job here:
1767
1768   %s/logs.tar.gz
1769
1770 The top commit for the tree that was built was:
1771
1772 %s
1773
1774 ''' % (log_base, failed_tag, log_base, failed_tag, log_base, top_commit_msg)
1775
1776     log_stdout = "%s/%s.stdout" % (gitroot, failed_tag)
1777     if add_log_tail and os.access(log_stdout, os.R_OK):
1778         f = open(log_stdout, 'r')
1779         lines = f.readlines()
1780         log_tail = "".join(lines[-50:])
1781         num_lines = len(lines)
1782         log_stderr = "%s/%s.stderr" % (gitroot, failed_tag)
1783         if num_lines < 50 and os.access(log_stderr, os.R_OK):
1784             # Also include stderr (compile failures) if < 50 lines of stdout
1785             f = open(log_stderr, 'r')
1786             log_tail += "".join(f.readlines()[-(50 - num_lines):])
1787
1788         text += '''
1789 The last 50 lines of log messages:
1790
1791 %s
1792     ''' % log_tail
1793         f.close()
1794
1795     logs = os.path.join(gitroot, 'logs.tar.gz')
1796     send_email('autobuild[%s] failure on %s for task %s during %s'
1797                % (options.branch, platform.node(), failed_task, failed_stage),
1798                text, logs)
1799
1800
1801 def email_success(elapsed_time, log_base=None):
1802     '''send an email to options.email about a successful build'''
1803     if log_base is None:
1804         log_base = gitroot
1805     text = '''
1806 Dear Developer,
1807
1808 Your autobuild on %s has succeeded after %.1f minutes.
1809
1810 ''' % (platform.node(), elapsed_time / 60.)
1811
1812     if options.restrict_tests:
1813         text += """
1814 The build was restricted to tests matching %s\n""" % options.restrict_tests
1815
1816     if options.keeplogs:
1817         text += '''
1818
1819 you can get full logs of all tasks in this job here:
1820
1821   %s/logs.tar.gz
1822
1823 ''' % log_base
1824
1825     text += '''
1826 The top commit for the tree that was built was:
1827
1828 %s
1829 ''' % top_commit_msg
1830
1831     logs = os.path.join(gitroot, 'logs.tar.gz')
1832     send_email('autobuild[%s] success on %s' % (options.branch, platform.node()),
1833                text, logs)
1834
1835
1836 # get the top commit message, for emails
1837 top_commit_msg = run_cmd("git log -1", dir=gitroot, output=True)
1838
1839 try:
1840     if options.skip_dependencies:
1841         run_cmd("stat %s" % testbase, dir=testbase, output=True)
1842     else:
1843         os.makedirs(testbase)
1844 except Exception as reason:
1845     raise Exception("Unable to create %s : %s" % (testbase, reason))
1846 cleanup_list.append(testbase)
1847
1848 if options.daemon:
1849     logfile = os.path.join(testbase, "log")
1850     do_print("Forking into the background, writing progress to %s" % logfile)
1851     daemonize(logfile)
1852
1853 write_pidfile(gitroot + "/autobuild.pid")
1854
1855 start_time = time.time()
1856
1857 while True:
1858     try:
1859         run_cmd("rm -rf %s" % test_tmpdir, show=True)
1860         os.makedirs(test_tmpdir)
1861         # The waf uninstall code removes empty directories all the way
1862         # up the tree.  Creating a file in test_tmpdir stops it from
1863         # being removed.
1864         run_cmd("touch %s" % os.path.join(test_tmpdir,
1865                                           ".directory-is-not-empty"), show=True)
1866         run_cmd("stat %s" % test_tmpdir, show=True)
1867         run_cmd("stat %s" % testbase, show=True)
1868         if options.skip_dependencies:
1869             run_cmd("stat %s" % test_master, dir=testbase, output=True)
1870         else:
1871             run_cmd("git clone --recursive --shared %s %s" % (gitroot, test_master), show=True, dir=gitroot)
1872     except Exception:
1873         cleanup()
1874         raise
1875
1876     try:
1877         if options.rebase is not None:
1878             rebase_tree(options.rebase, rebase_branch=options.branch)
1879     except Exception:
1880         cleanup_list.append(gitroot + "/autobuild.pid")
1881         cleanup()
1882         elapsed_time = time.time() - start_time
1883         email_failure(-1, 'rebase', 'rebase', 'rebase',
1884                       'rebase on %s failed' % options.branch,
1885                       elapsed_time, log_base=options.log_base)
1886         sys.exit(1)
1887
1888     try:
1889         blist = buildlist(args, options.rebase, rebase_branch=options.branch)
1890         if options.tail:
1891             blist.start_tail()
1892         (status, failed_task, failed_stage, failed_tag, errstr) = blist.run()
1893         if status != 0 or errstr != "retry":
1894             break
1895         cleanup(do_raise=True)
1896     except Exception:
1897         cleanup()
1898         raise
1899
1900 cleanup_list.append(gitroot + "/autobuild.pid")
1901
1902 do_print(errstr)
1903
1904 blist.kill_kids()
1905 if options.tail:
1906     do_print("waiting for tail to flush")
1907     time.sleep(1)
1908
1909 elapsed_time = time.time() - start_time
1910 if status == 0:
1911     if options.passcmd is not None:
1912         do_print("Running passcmd: %s" % options.passcmd)
1913         run_cmd(options.passcmd, dir=test_master)
1914     if options.pushto is not None:
1915         push_to(options.pushto, push_branch=options.branch)
1916     if options.keeplogs or options.attach_logs:
1917         blist.tarlogs("logs.tar.gz")
1918         do_print("Logs in logs.tar.gz")
1919     if options.always_email:
1920         email_success(elapsed_time, log_base=options.log_base)
1921     blist.remove_logs()
1922     cleanup()
1923     do_print(errstr)
1924     sys.exit(0)
1925
1926 # something failed, gather a tar of the logs
1927 blist.tarlogs("logs.tar.gz")
1928
1929 if options.email is not None:
1930     email_failure(status, failed_task, failed_stage, failed_tag, errstr,
1931                   elapsed_time, log_base=options.log_base)
1932 else:
1933     elapsed_minutes = elapsed_time / 60.0
1934     print('''
1935
1936 ####################################################################
1937
1938 AUTOBUILD FAILURE
1939
1940 Your autobuild[%s] on %s failed after %.1f minutes
1941 when trying to test %s with the following error:
1942
1943    %s
1944
1945 the autobuild has been abandoned. Please fix the error and resubmit.
1946
1947 ####################################################################
1948
1949 ''' % (options.branch, platform.node(), elapsed_minutes, failed_task, errstr))
1950
1951 cleanup()
1952 do_print(errstr)
1953 do_print("Logs in logs.tar.gz")
1954 sys.exit(status)