utils/llvm-compilers-check

   1 #!/usr/bin/python3
   2 ##===- utils/llvmbuild - Build the LLVM project ----------------*-python-*-===##
   3 #
   4 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   5 # See https://llvm.org/LICENSE.txt for license information.
   6 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   7 #
   8 ##===----------------------------------------------------------------------===##
   9 #
  10 # This script builds many different flavors of the LLVM ecosystem.  It
  11 # will build LLVM, Clang and dragonegg as well as run tests on them.
  12 # This script is convenient to use to check builds and tests before
  13 # committing changes to the upstream repository
  14 #
  15 # A typical source setup uses three trees and looks like this:
  16 #
  17 # official
  18 #   dragonegg
  19 #   llvm
  20 #     tools
  21 #       clang
  22 # staging
  23 #   dragonegg
  24 #   llvm
  25 #     tools
  26 #       clang
  27 # commit
  28 #   dragonegg
  29 #   llvm
  30 #     tools
  31 #       clang
  32 #
  33 # In a typical workflow, the "official" tree always contains unchanged
  34 # sources from the main LLVM project repositories.  The "staging" tree
  35 # is where local work is done.  A set of changes resides there waiting
  36 # to be moved upstream.  The "commit" tree is where changes from
  37 # "staging" make their way upstream.  Individual incremental changes
  38 # from "staging" are applied to "commit" and committed upstream after
  39 # a successful build and test run.  A successful build is one in which
  40 # testing results in no more failures than seen in the testing of the
  41 # "official" tree.
  42 #
  43 # A build may be invoked as such:
  44 #
  45 # llvmbuild --src=~/llvm/commit --src=~/llvm/staging --src=~/llvm/official
  46 #   --build=debug --build=release --build=paranoid
  47 #   --prefix=/home/greened/install --builddir=/home/greened/build
  48 #
  49 # This will build the LLVM ecosystem, including LLVM, Clangand
  50 # dragonegg, putting build results in ~/build and installing tools in
  51 # ~/install.  llvm-compilers-check creates separate build and install
  52 # directories for each source/build flavor.  In the above example,
  53 # llvmbuild will build debug, release and paranoid (debug+checks)
  54 # flavors from each source tree (official, staging and commit) for a
  55 # total of nine builds.  All builds will be run in parallel.
  56 #
  57 # The user may control parallelism via the --jobs and --threads
  58 # switches.  --jobs tells llvm-compilers-checl the maximum total
  59 # number of builds to activate in parallel.  The user may think of it
  60 # as equivalent to the GNU make -j switch.  --threads tells
  61 # llvm-compilers-check how many worker threads to use to accomplish
  62 # those builds.  If --threads is less than --jobs, --threads workers
  63 # will be launched and each one will pick a source/flavor combination
  64 # to build.  Then llvm-compilers-check will invoke GNU make with -j
  65 # (--jobs / --threads) to use up the remaining job capacity.  Once a
  66 # worker is finished with a build, it will pick another combination
  67 # off the list and start building it.
  68 #
  69 ##===----------------------------------------------------------------------===##
  70
  71 import optparse
  72 import os
  73 import sys
  74 import threading
  75 import queue
  76 import logging
  77 import traceback
  78 import subprocess
  79 import re
  80
  81 # TODO: Use shutil.which when it is available (3.2 or later)
  82 def find_executable(executable, path=None):
  83     """Try to find 'executable' in the directories listed in 'path' (a
  84     string listing directories separated by 'os.pathsep'; defaults to
  85     os.environ['PATH']).  Returns the complete filename or None if not
  86     found
  87     """
  88     if path is None:
  89         path = os.environ['PATH']
  90     paths = path.split(os.pathsep)
  91     extlist = ['']
  92     if os.name == 'os2':
  93         (base, ext) = os.path.splitext(executable)
  94         # executable files on OS/2 can have an arbitrary extension, but
  95         # .exe is automatically appended if no dot is present in the name
  96         if not ext:
  97             executable = executable + ".exe"
  98     elif sys.platform == 'win32':
  99         pathext = os.environ['PATHEXT'].lower().split(os.pathsep)
 100         (base, ext) = os.path.splitext(executable)
 101         if ext.lower() not in pathext:
 102             extlist = pathext
 103     for ext in extlist:
 104         execname = executable + ext
 105         if os.path.isfile(execname):
 106             return execname
 107         else:
 108             for p in paths:
 109                 f = os.path.join(p, execname)
 110                 if os.path.isfile(f):
 111                     return f
 112     else:
 113         return None
 114
 115 def is_executable(fpath):
 116     return os.path.exists(fpath) and os.access(fpath, os.X_OK)
 117
 118 def add_options(parser):
 119     parser.add_option("-v", "--verbose", action="store_true",
 120                       default=False,
 121                       help=("Output informational messages"
 122                             " [default: %default]"))
 123     parser.add_option("--src", action="append",
 124                       help=("Top-level source directory [default: %default]"))
 125     parser.add_option("--build", action="append",
 126                       help=("Build types to run [default: %default]"))
 127     parser.add_option("--cc", default=find_executable("cc"),
 128                       help=("The C compiler to use [default: %default]"))
 129     parser.add_option("--cxx", default=find_executable("c++"),
 130                       help=("The C++ compiler to use [default: %default]"))
 131     parser.add_option("--threads", default=4, type="int",
 132                       help=("The number of worker threads to use "
 133                             "[default: %default]"))
 134     parser.add_option("--jobs", "-j", default=8, type="int",
 135                       help=("The number of simultaneous build jobs "
 136                             "[default: %default]"))
 137     parser.add_option("--prefix",
 138                       help=("Root install directory [default: %default]"))
 139     parser.add_option("--builddir",
 140                       help=("Root build directory [default: %default]"))
 141     parser.add_option("--extra-llvm-config-flags", default="",
 142                       help=("Extra flags to pass to llvm configure [default: %default]"))
 143     parser.add_option("--force-configure", default=False, action="store_true",
 144                       help=("Force reconfigure of all components"))
 145     parser.add_option("--no-dragonegg", default=False, action="store_true",
 146                       help=("Do not build dragonegg"))
 147     parser.add_option("--no-install", default=False, action="store_true",
 148                       help=("Do not do installs"))
 149     parser.add_option("--keep-going", default=False, action="store_true",
 150                       help=("Keep going after failures"))
 151     parser.add_option("--no-flavor-prefix", default=False, action="store_true",
 152                       help=("Do not append the build flavor to the install path"))
 153     parser.add_option("--enable-werror", default=False, action="store_true",
 154                       help=("Build with -Werror"))
 155     return
 156
 157 def check_options(parser, options, valid_builds):
 158     # See if we're building valid flavors.
 159     for build in options.build:
 160         if (build not in valid_builds):
 161             parser.error("'" + build + "' is not a valid build flavor "
 162                          + str(valid_builds))
 163
 164     # See if we can find source directories.
 165     for src in options.src:
 166         for component in components:
 167             component = component.rstrip("2")
 168             compsrc = src + "/" + component
 169             if (not os.path.isdir(compsrc)):
 170                 parser.error("'" + compsrc + "' does not exist")
 171
 172     # See if we can find the compilers
 173     options.cc = find_executable(options.cc)
 174     options.cxx = find_executable(options.cxx)
 175
 176     return
 177
 178 # Find a unique short name for the given set of paths.  This searches
 179 # back through path components until it finds unique component names
 180 # among all given paths.
 181 def get_path_abbrevs(paths):
 182     # Find the number of common starting characters in the last component
 183     # of the paths.
 184     unique_paths = list(paths)
 185
 186     class NotFoundException(Exception): pass
 187
 188     # Find a unique component of each path.
 189     unique_bases = unique_paths[:]
 190     found = 0
 191     while len(unique_paths) > 0:
 192         bases = [os.path.basename(src) for src in unique_paths]
 193         components = { c for c in bases }
 194         # Account for single entry in paths.
 195         if len(components) > 1 or len(components) == len(bases):
 196             # We found something unique.
 197             for c in components:
 198                 if bases.count(c) == 1:
 199                    index = bases.index(c)
 200                    unique_bases[index] = c
 201                    # Remove the corresponding path from the set under
 202                    # consideration.
 203                    unique_paths[index] = None
 204             unique_paths = [ p for p in unique_paths if p is not None ]
 205         unique_paths = [os.path.dirname(src) for src in unique_paths]
 206
 207     if len(unique_paths) > 0:
 208         raise NotFoundException()
 209
 210     abbrevs = dict(zip(paths, [base for base in unique_bases]))
 211
 212     return abbrevs
 213
 214 # Given a set of unique names, find a short character sequence that
 215 # uniquely identifies them.
 216 def get_short_abbrevs(unique_bases):
 217     # Find a unique start character for each path base.
 218     my_unique_bases = unique_bases[:]
 219     unique_char_starts = unique_bases[:]
 220     while len(my_unique_bases) > 0:
 221         for start, char_tuple in enumerate(zip(*[base
 222                                                  for base in my_unique_bases])):
 223             chars = { c for c in char_tuple }
 224             # Account for single path.
 225             if len(chars) > 1 or len(chars) == len(char_tuple):
 226                 # We found something unique.
 227                 for c in chars:
 228                     if char_tuple.count(c) == 1:
 229                         index = char_tuple.index(c)
 230                         unique_char_starts[index] = start
 231                         # Remove the corresponding path from the set under
 232                         # consideration.
 233                         my_unique_bases[index] = None
 234                 my_unique_bases = [ b for b in my_unique_bases
 235                                     if b is not None ]
 236                 break
 237
 238     if len(my_unique_bases) > 0:
 239         raise NotFoundException()
 240
 241     abbrevs = [abbrev[start_index:start_index+3]
 242                for abbrev, start_index
 243                in zip([base for base in unique_bases],
 244                       [index for index in unique_char_starts])]
 245
 246     abbrevs = dict(zip(unique_bases, abbrevs))
 247
 248     return abbrevs
 249
 250 class Builder(threading.Thread):
 251     class ExecutableNotFound(Exception): pass
 252     class FileNotExecutable(Exception): pass
 253
 254     def __init__(self, work_queue, jobs,
 255                  build_abbrev, source_abbrev,
 256                  options):
 257         super().__init__()
 258         self.work_queue = work_queue
 259         self.jobs = jobs
 260         self.cc = options.cc
 261         self.cxx = options.cxx
 262         self.build_abbrev = build_abbrev
 263         self.source_abbrev = source_abbrev
 264         self.build_prefix = options.builddir
 265         self.install_prefix = options.prefix
 266         self.options = options
 267         self.component_abbrev = dict(
 268             llvm="llvm",
 269             dragonegg="degg")
 270     def run(self):
 271         while True:
 272             try:
 273                 source, build = self.work_queue.get()
 274                 self.dobuild(source, build)
 275             except:
 276                 traceback.print_exc()
 277             finally:
 278                 self.work_queue.task_done()
 279
 280     def execute(self, command, execdir, env, component):
 281         prefix = self.component_abbrev[component.replace("-", "_")]
 282         pwd = os.getcwd()
 283         if not os.path.exists(execdir):
 284             os.makedirs(execdir)
 285
 286         execenv = os.environ.copy()
 287
 288         for key, value in env.items():
 289             execenv[key] = value
 290
 291         self.logger.debug("[" + prefix + "] " + "env " + str(env) + " "
 292                           + " ".join(command));
 293
 294         try:
 295             proc = subprocess.Popen(command,
 296                                     cwd=execdir,
 297                                     env=execenv,
 298                                     stdout=subprocess.PIPE,
 299                                     stderr=subprocess.STDOUT)
 300
 301             line = proc.stdout.readline()
 302             while line:
 303                 self.logger.info("[" + prefix + "] "
 304                                  + str(line, "utf-8").rstrip())
 305                 line = proc.stdout.readline()
 306
 307             (stdoutdata, stderrdata) = proc.communicate()
 308             retcode = proc.wait()
 309
 310             return retcode
 311
 312         except:
 313             traceback.print_exc()
 314
 315     # Get a list of C++ include directories to pass to clang.
 316     def get_includes(self):
 317         # Assume we're building with g++ for now.
 318         command = [self.cxx]
 319         command += ["-v", "-x", "c++", "/dev/null", "-fsyntax-only"]
 320         includes = []
 321         self.logger.debug(command)
 322         try:
 323             proc = subprocess.Popen(command,
 324                                     stdout=subprocess.PIPE,
 325                                     stderr=subprocess.STDOUT)
 326
 327             gather = False
 328             line = proc.stdout.readline()
 329             while line:
 330                 self.logger.debug(line)
 331                 if re.search("End of search list", str(line)) is not None:
 332                     self.logger.debug("Stop Gather")
 333                     gather = False
 334                 if gather:
 335                     includes.append(str(line, "utf-8").strip())
 336                 if re.search("#include <...> search starts", str(line)) is not None:
 337                     self.logger.debug("Start Gather")
 338                     gather = True
 339                 line = proc.stdout.readline()
 340
 341         except:
 342             traceback.print_exc()
 343         self.logger.debug(includes)
 344         return includes
 345
 346     def dobuild(self, source, build):
 347         build_suffix = ""
 348
 349         ssabbrev = get_short_abbrevs([ab for ab in self.source_abbrev.values()])
 350
 351         prefix = "[" + ssabbrev[self.source_abbrev[source]] + "-" + self.build_abbrev[build] + "]"
 352         if (not self.options.no_flavor_prefix):
 353             self.install_prefix += "/" + self.source_abbrev[source] + "/" + build
 354
 355         build_suffix += "/" + self.source_abbrev[source] + "/" + build
 356
 357         self.logger = logging.getLogger(prefix)
 358
 359         self.logger.debug(self.install_prefix)
 360
 361         # Assume we're building with gcc for now.
 362         cxxincludes = self.get_includes()
 363         cxxroot = os.path.dirname(cxxincludes[0]) # Remove the version
 364         cxxroot = os.path.dirname(cxxroot)        # Remove the c++
 365         cxxroot = os.path.dirname(cxxroot)        # Remove the include
 366
 367         configure_flags = dict(
 368             llvm=dict(debug=["--prefix=" + self.install_prefix,
 369                              "--enable-assertions",
 370                              "--disable-optimized",
 371                              "--with-gcc-toolchain=" + cxxroot],
 372                       release=["--prefix=" + self.install_prefix,
 373                                "--enable-optimized",
 374                                "--with-gcc-toolchain=" + cxxroot],
 375                       paranoid=["--prefix=" + self.install_prefix,
 376                                 "--enable-assertions",
 377                                 "--enable-expensive-checks",
 378                                 "--disable-optimized",
 379                                 "--with-gcc-toolchain=" + cxxroot]),
 380             dragonegg=dict(debug=[],
 381                            release=[],
 382                            paranoid=[]))
 383
 384         if (self.options.enable_werror):
 385             configure_flags["llvm"]["debug"].append("--enable-werror")
 386             configure_flags["llvm"]["release"].append("--enable-werror")
 387             configure_flags["llvm"]["paranoid"].append("--enable-werror")
 388
 389         configure_env = dict(
 390             llvm=dict(debug=dict(CC=self.cc,
 391                                  CXX=self.cxx),
 392                       release=dict(CC=self.cc,
 393                                    CXX=self.cxx),
 394                       paranoid=dict(CC=self.cc,
 395                                     CXX=self.cxx)),
 396             dragonegg=dict(debug=dict(CC=self.cc,
 397                                       CXX=self.cxx),
 398                            release=dict(CC=self.cc,
 399                                         CXX=self.cxx),
 400                            paranoid=dict(CC=self.cc,
 401                                          CXX=self.cxx)))
 402
 403         make_flags = dict(
 404             llvm=dict(debug=["-j" + str(self.jobs)],
 405                       release=["-j" + str(self.jobs)],
 406                       paranoid=["-j" + str(self.jobs)]),
 407             dragonegg=dict(debug=["-j" + str(self.jobs)],
 408                            release=["-j" + str(self.jobs)],
 409                            paranoid=["-j" + str(self.jobs)]))
 410
 411         make_env = dict(
 412             llvm=dict(debug=dict(),
 413                       release=dict(),
 414                       paranoid=dict()),
 415             dragonegg=dict(debug=dict(GCC=self.cc,
 416                                       LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"),
 417                            release=dict(GCC=self.cc,
 418                                         LLVM_CONFIG=self.install_prefix + "/bin/llvm-config"),
 419                            paranoid=dict(GCC=self.cc,
 420                                          LLVM_CONFIG=self.install_prefix + "/bin/llvm-config")))
 421
 422         make_install_flags = dict(
 423             llvm=dict(debug=["install"],
 424                       release=["install"],
 425                       paranoid=["install"]),
 426             dragonegg=dict(debug=["install"],
 427                            release=["install"],
 428                            paranoid=["install"]))
 429
 430         make_install_env = dict(
 431             llvm=dict(debug=dict(),
 432                       release=dict(),
 433                       paranoid=dict()),
 434             dragonegg=dict(debug=dict(),
 435                            release=dict(),
 436                            paranoid=dict()))
 437
 438         make_check_flags = dict(
 439             llvm=dict(debug=["check"],
 440                       release=["check"],
 441                       paranoid=["check"]),
 442             dragonegg=dict(debug=["check"],
 443                            release=["check"],
 444                            paranoid=["check"]))
 445
 446         make_check_env = dict(
 447             llvm=dict(debug=dict(),
 448                       release=dict(),
 449                       paranoid=dict()),
 450             dragonegg=dict(debug=dict(),
 451                            release=dict(),
 452                            paranoid=dict()))
 453
 454         for component in components:
 455             comp = component[:]
 456
 457             if (self.options.no_dragonegg):
 458                 if (comp == 'dragonegg'):
 459                     self.logger.info("Skipping " + component + " in "
 460                                      + builddir)
 461                     continue
 462
 463             srcdir = source + "/" + comp.rstrip("2")
 464             builddir = self.build_prefix + "/" + comp + "/" + build_suffix
 465             installdir = self.install_prefix
 466
 467             comp_key = comp.replace("-", "_")
 468
 469             config_args = configure_flags[comp_key][build][:]
 470             config_args.extend(getattr(self.options,
 471                                        "extra_" + comp_key.rstrip("2")
 472                                        + "_config_flags",
 473                                        "").split())
 474
 475             self.logger.info("Configuring " + component + " in " + builddir)
 476             configrc = self.configure(component, srcdir, builddir,
 477                                       config_args,
 478                                       configure_env[comp_key][build])
 479
 480             if (configrc == None) :
 481                 self.logger.info("[None] Failed to configure " + component + " in " + installdir)
 482
 483             if (configrc == 0 or self.options.keep_going) :
 484                 self.logger.info("Building " + component + " in " + builddir)
 485                 self.logger.info("Build: make " + str(make_flags[comp_key][build]))
 486                 buildrc = self.make(component, srcdir, builddir,
 487                                     make_flags[comp_key][build],
 488                                     make_env[comp_key][build])
 489
 490                 if (buildrc == None) :
 491                     self.logger.info("[None] Failed to build " + component + " in " + installdir)
 492
 493                 if (buildrc == 0 or self.options.keep_going) :
 494                     self.logger.info("Testing " + component + " in " + builddir)
 495                     self.logger.info("Test: make "
 496                                      + str(make_check_flags[comp_key][build]))
 497                     testrc = self.make(component, srcdir, builddir,
 498                                        make_check_flags[comp_key][build],
 499                                        make_check_env[comp_key][build])
 500
 501                     if (testrc == None) :
 502                         self.logger.info("[None] Failed to test " + component + " in " + installdir)
 503
 504                     if ((testrc == 0  or self.options.keep_going)
 505                         and not self.options.no_install):
 506                         self.logger.info("Installing " + component + " in " + installdir)
 507                         self.make(component, srcdir, builddir,
 508                                   make_install_flags[comp_key][build],
 509                                   make_install_env[comp_key][build])
 510                     else :
 511                         self.logger.info("Failed testing " + component + " in " + installdir)
 512
 513                 else :
 514                     self.logger.info("Failed to build " + component + " in " + installdir)
 515
 516             else :
 517                 self.logger.info("Failed to configure " + component + " in " + installdir)
 518
 519     def configure(self, component, srcdir, builddir, flags, env):
 520         prefix = self.component_abbrev[component.replace("-", "_")]
 521
 522         self.logger.debug("Configure " + str(flags) + " " + str(srcdir) + " -> "
 523                           + str(builddir))
 524
 525         configure_files = dict(
 526             llvm=[(srcdir + "/configure", builddir + "/Makefile")],
 527             dragonegg=[(None,None)])
 528
 529
 530         doconfig = False
 531         for conf, mf in configure_files[component.replace("-", "_")]:
 532             if conf is None:
 533                 # No configure necessary
 534                 return 0
 535
 536             if not os.path.exists(conf):
 537                 self.logger.info("[" + prefix + "] Configure failed, no configure script " + conf)
 538                 return -1
 539
 540             if os.path.exists(conf) and os.path.exists(mf):
 541                 confstat = os.stat(conf)
 542                 makestat = os.stat(mf)
 543                 if confstat.st_mtime > makestat.st_mtime:
 544                     doconfig = True
 545                     break
 546             else:
 547                 doconfig = True
 548                 break
 549
 550         if not doconfig and not self.options.force_configure:
 551             return 0
 552
 553         program = srcdir + "/configure"
 554         if not is_executable(program):
 555             self.logger.info("[" + prefix + "] Configure failed, cannot execute " + program)
 556             return -1
 557
 558         args = [program]
 559         args += ["--verbose"]
 560         args += flags
 561         return self.execute(args, builddir, env, component)
 562
 563     def make(self, component, srcdir, builddir, flags, env):
 564         program = find_executable("make")
 565         if program is None:
 566             raise ExecutableNotFound
 567
 568         if not is_executable(program):
 569             raise FileNotExecutable
 570
 571         args = [program]
 572         args += flags
 573         return self.execute(args, builddir, env, component)
 574
 575 # Global constants
 576 build_abbrev = dict(debug="dbg", release="opt", paranoid="par")
 577 components = ["llvm", "dragonegg"]
 578
 579 # Parse options
 580 parser = optparse.OptionParser(version="%prog 1.0")
 581 add_options(parser)
 582 (options, args) = parser.parse_args()
 583 check_options(parser, options, build_abbrev.keys());
 584
 585 if options.verbose:
 586     logging.basicConfig(level=logging.DEBUG,
 587                         format='%(name)-13s: %(message)s')
 588 else:
 589     logging.basicConfig(level=logging.INFO,
 590                         format='%(name)-13s: %(message)s')
 591
 592 source_abbrev = get_path_abbrevs(set(options.src))
 593
 594 work_queue = queue.Queue()
 595
 596 jobs = options.jobs // options.threads
 597 if jobs == 0:
 598     jobs = 1
 599
 600 numthreads = options.threads
 601
 602 logging.getLogger().info("Building with " + str(options.jobs) + " jobs and "
 603                          + str(numthreads) + " threads using " + str(jobs)
 604                          + " make jobs")
 605
 606 logging.getLogger().info("CC  = " + str(options.cc))
 607 logging.getLogger().info("CXX = " + str(options.cxx))
 608
 609 for t in range(numthreads):
 610     builder = Builder(work_queue, jobs,
 611                       build_abbrev, source_abbrev,
 612                       options)
 613     builder.daemon = True
 614     builder.start()
 615
 616 for build in set(options.build):
 617     for source in set(options.src):
 618         work_queue.put((source, build))
 619
 620 work_queue.join()