xapian-maintainer-tools/buildbot/xapian_factories.py

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 from buildbot.process import factory
   5 from buildbot.steps import source, shell, slave
   6 from datetime import date
   7
   8 xapian_config_arg = 'XAPIAN_CONFIG=../xapian-core/xapian-config'
   9
  10 get_tarballs_url = 'https://raw.githubusercontent.com/xapian/xapian/master/xapian-maintainer-tools/buildbot/scripts/get_tarballs.py'
  11
  12 def Bootstrap():
  13     return shell.ShellCommand(
  14         name = "bootstrap",
  15         haltOnFailure = 1,
  16         description = ["bootstrapping"],
  17         descriptionDone = ["bootstrap"],
  18         command = ["./bootstrap"],
  19     )
  20
  21 def CleanInstall():
  22     return shell.ShellCommand(
  23         name = "cleaninstall",
  24         haltOnFailure = 1,
  25         description = ["cleaninstall"],
  26         descriptionDone = ["cleaninstall"],
  27         command = ['rm', '-rf', 'install'],
  28     )
  29
  30 def Install():
  31     return shell.ShellCommand(
  32         name = "install",
  33         haltOnFailure = 1,
  34         description = ["install"],
  35         descriptionDone = ["install"],
  36         command = ['make', 'install'],
  37     )
  38
  39 def MakeWritable():
  40     """
  41     Step which ensures that the permissions are writable on all
  42     subdirectories.
  43     """
  44     return shell.ShellCommand(
  45         name = "make writable",
  46         haltOnFailure = 1,
  47         description = ["making writable"],
  48         descriptionDone = ["made writable"],
  49         command = ["chmod", "-R", "+w", "."],
  50     )
  51
  52 def core_factory(repourl, usedocs=True, configure=None, audit=False,
  53                  clean=False, nocheck = False, configure_opts=None):
  54     f = factory.BuildFactory()
  55     mode = "update"
  56     if clean:
  57         #f.addStep(MakeWritable, workdir='.')
  58         f.addStep(shell.ShellCommand(command = ["chmod", "-R", "+w", "."], workdir='.'))
  59         mode = "clobber"
  60     f.addStep(source.Git(repourl=repourl, mode=mode))
  61     if audit:
  62         f.addStep(shell.ShellCommand(command = ["python", 'audit.py'], workdir='build/xapian-maintainer-tools'))
  63         f.addStep(shell.ShellCommand(command = ["chmod", '644', 'copyright.csv', 'fixmes.csv'], workdir='build/xapian-maintainer-tools'))
  64         f.addStep(shell.ShellCommand(command = ["mv", 'copyright.csv', 'fixmes.csv', '/var/www/html/'], workdir='build/xapian-maintainer-tools'))
  65
  66     f.addStep(Bootstrap())
  67     if configure:
  68         f.addStep(shell.Configure(command=configure))
  69     else:
  70         if configure_opts is None:
  71             configure_opts = []
  72         if not usedocs:
  73             configure_opts.append("--disable-documentation")
  74         if configure_opts:
  75             f.addStep(shell.Configure(command=["sh", "configure"] + configure_opts))
  76         else:
  77             f.addStep(shell.Configure())
  78
  79     f.addStep(shell.Compile())
  80     if not nocheck:
  81         f.addStep(shell.Test(name="check", command=["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain", "VALGRIND=", "AUTOMATED_TESTING=1"]))
  82     return f
  83
  84 def gen_git_updated_factory(repourl, usedocs=True, clean=False, configure_opts=None):
  85     """
  86     Make a factory for doing build from git master, but without cleaning
  87     first.  This build is intended to catch commonly made mistakes quickly.
  88     """
  89     return core_factory(repourl=repourl, usedocs=usedocs, clean=clean, configure_opts=configure_opts)
  90
  91 def gen_git_updated_factory_llvm(repourl):
  92     """
  93     Make a factory for doing build from git master, but without cleaning
  94     first.  This build is intended to catch commonly made mistakes quickly.
  95     """
  96     return core_factory(repourl=repourl, configure_opts=["CXX=/Developer/usr/llvm-gcc-4.2/bin/llvm-g++-4.2", "CC=/Developer/usr/llvm-gcc-4.2/bin/llvm-gcc-4.2"])
  97
  98 def gen_git_updated_factory2(repourl, configure_opts=[]):
  99     """
 100     Make a factory for doing build from git master, but without cleaning
 101     first.  This build is intended to catch commonly made mistakes quickly.
 102     This factory also runs audit.py and publishes the result.
 103     """
 104     return core_factory(repourl=repourl, usedocs=True, audit=True,
 105                         configure_opts=configure_opts)
 106
 107 def gen_git_updated_factory3(repourl):
 108     """
 109     Make a factory for doing build from git master, but without cleaning
 110     first.  This build is intended to catch commonly made mistakes quickly.
 111     This build runs with --disable-documentation, so the documentation building
 112     tools aren't required.
 113     """
 114     return core_factory(repourl=repourl, usedocs=True)
 115
 116 def gen_git_gccsnapshot_updated_factory(repourl):
 117     """
 118     Make a factory for doing build from git master, but without cleaning
 119     first, using gcc snapshot.  Also uses compiles with logging and assertions.
 120     """
 121     return core_factory(repourl=repourl,
 122                         configure_opts=["--enable-assertions", "--enable-log", "CXX=/usr/lib/gcc-snapshot/bin/g++", "CC=/usr/lib/gcc-snapshot/bin/gcc",
 123         ])
 124
 125 def gen_git_debug_updated_factory(repourl, opts, nocheck=False):
 126     """
 127     Make a factory for doing a debug build from git master, but without cleaning
 128     first.  This build is intended to catch commonly made mistakes quickly.
 129     """
 130     f = factory.BuildFactory()
 131     f.addStep(source.Git(repourl=repourl, mode="update"))
 132     f.addStep(Bootstrap())
 133     f.addStep(shell.Configure(command = ["sh", "configure", ] + opts))
 134     f.addStep(shell.Compile())
 135     if not nocheck:
 136         f.addStep(shell.Test(name="check", command = ["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain", "VALGRIND=", "AUTOMATED_TESTING=1"]))
 137     return f
 138
 139 # Factory that can be temporarily set for a builder to clean it up like so:
 140 #
 141 #   'factory': rm_rf_factory(),
 142 def gen_rm_rf_factory():
 143     f = factory.BuildFactory()
 144     f.addStep(slave.RemoveDirectory('build'))
 145     return f
 146
 147 def gen_tarball_updated_factory(rooturl, nocheck=False, omega=True, bindings=True, configure_opts=[]):
 148     """
 149     Make a factory for doing builds from tarballs.
 150     """
 151     configure_cmd = ["sh", "configure", ] + configure_opts
 152     f = factory.BuildFactory()
 153     f.addStep(shell.ShellCommand(command = ["python", "-c", "try: import urllib2 as u\nexcept: import urllib.request as u\nopen('get_tarballs.py', 'wb').write(u.urlopen('%s').read())" %
 154               get_tarballs_url], workdir='.', haltOnFailure=True))
 155     f.addStep(shell.ShellCommand(command = ["python", 'get_tarballs.py', rooturl], workdir='.', haltOnFailure=True))
 156     f.addStep(shell.Configure(workdir='build/xapian-core', command=configure_cmd))
 157     f.addStep(shell.Compile(workdir='build/xapian-core'))
 158     if not nocheck:
 159         f.addStep(shell.Test(workdir='build/xapian-core', name="check", command = ["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain", "VALGRIND=", "AUTOMATED_TESTING=1"]))
 160     if omega:
 161         f.addStep(shell.Configure(workdir='build/xapian-omega', command = ["./configure", xapian_config_arg] + configure_opts))
 162         f.addStep(shell.Compile(workdir='build/xapian-omega'))
 163         if not nocheck:
 164             f.addStep(shell.Test(workdir='build/xapian-omega', name="check", command = ["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain", "VALGRIND=", "AUTOMATED_TESTING=1"]))
 165     if bindings:
 166         f.addStep(shell.Configure(workdir='build/xapian-bindings', command = ["./configure", xapian_config_arg] + configure_opts))
 167         f.addStep(shell.Compile(workdir='build/xapian-bindings', command = ["make"]))
 168         if not nocheck:
 169             f.addStep(shell.Test(workdir='build/xapian-bindings', name="check", command = ["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain", "VALGRIND=", "AUTOMATED_TESTING=1"]))
 170     # If everything passed, there's not much point keeping the build - we'd
 171     # delete the old build tree and download new tarballs next time anyway.
 172     f.addStep(slave.RemoveDirectory('build'))
 173     return f
 174
 175 def gen_git_updated_valgrind_factory(repourl, configure_opts=[]):
 176     """
 177     Factory for doing build from git master, without cleaning first, and using
 178     valgrind to check.  This one is much more expensive, so should be run with
 179     a higher stable time.
 180     """
 181     f = factory.BuildFactory()
 182     f.addStep(source.Git(repourl=repourl, mode="update"))
 183     f.addStep(Bootstrap())
 184     f.addStep(shell.Configure(command = ["sh", "configure", "CXXFLAGS=-O0 -g"] + configure_opts))
 185     f.addStep(shell.Compile())
 186
 187     f.addStep(shell.Test(name="check", command = ["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain"], workdir='build/xapian-core'))
 188
 189     return f
 190
 191 def gen_git_updated_lcov_factory(repourl, configure_opts=[]):
 192     """
 193     Factory for doing build from git master, without cleaning first, and using
 194     lcov to generate a coverage report.  This one is much more expensive, so
 195     should be run with a higher stable time.
 196     """
 197     f = factory.BuildFactory()
 198     f.addStep(source.Git(repourl=repourl, mode="update"))
 199     f.addStep(Bootstrap())
 200     f.addStep(shell.Configure(command = ["sh", "configure", "--enable-maintainer-mode", "--disable-documentation", "CXXFLAGS=-O0 --coverage", "VALGRIND="] + configure_opts, workdir="build/xapian-core"))
 201     f.addStep(shell.Compile(workdir="build/xapian-core"))
 202     f.addStep(shell.ShellCommand(command = ["make", "coverage-check", "GENHTML_ARGS=--html-gzip"], workdir="build/xapian-core", haltOnFailure=True))
 203     f.addStep(shell.ShellCommand(command = ["chmod", "-R", "a+rX", "lcov"], workdir="build/xapian-core", haltOnFailure=True))
 204     f.addStep(shell.ShellCommand(command = 'NOW=`date -u +%Y-%m-%d`; cp -a lcov/. /var/www/html/"$NOW" && ln -sfT "$NOW" /var/www/html/latest', workdir="build/xapian-core", haltOnFailure=True))
 205
 206     return f
 207
 208 #### FIXME: factories beyond here not updated
 209
 210 def gen_git_clean_dist_factory(repourl):
 211     """
 212     Factory for doing build from a clean checkout of git master.  This build also
 213     performs a "make distcheck", so should catch problems with files which have
 214     been missed from the distribution.  This one is much more expensive, so
 215     should be run with a higher stable time.
 216     """
 217     f = factory.BuildFactory()
 218     f.addStep(MakeWritable, workdir='.')
 219     f.addStep(source.Git(repourl=repourl, mode="clobber"))
 220     f.addStep(Bootstrap())
 221     f.addStep(step.Configure, command = ["xapian-maintainer-tools/buildbot/scripts/configure_with_prefix.sh"])
 222     extraargs = (
 223         "XAPIAN_TESTSUITE_OUTPUT=plain", "VALGRIND=", "AUTOMATED_TESTING=1"
 224     )
 225     f.addStep(step.Compile, command = ["make",] + extraargs)
 226     # Don't bother running check as a separate step - all the checks will be
 227     # done by distcheck, anyway.  (Running it as a separate step _does_ check
 228     # that the tests work in a non-VPATH build, but this is tested by other
 229     # factories, anyway.)
 230     #f.addStep(step.Test, name="check", command = ["make", "check"] + extraargs)
 231     f.addStep(step.Test, name="distcheck", command = ["make", "distcheck"] + extraargs, workdir='build/xapian-core')
 232     f.addStep(step.Test, name="distcheck", command = ["make", "distcheck"] + extraargs, workdir='build/xapian-applications/omega')
 233
 234     # Have to install the core for distcheck to pass on the bindings.
 235     f.addStep(step.Test, name="install", command = ["make", "install"] + extraargs, workdir='build/xapian-core')
 236     f.addStep(step.Test, name="distcheck", command = ["make", "distcheck"] + extraargs, workdir='build/xapian-bindings')
 237     return f
 238
 239 def gen_git_updated_win_factory(repourl):
 240     """
 241     Factory for doing a windows build from git, without cleaning first.
 242     """
 243     f = factory.BuildFactory()
 244     f.addStep(step.Git(repourl=repourl, mode="update"))
 245     f.addStep(step.ShellCommand, command="xapian-maintainer-tools\\buildbot\\scripts\\prepare_build.bat")
 246
 247     # Compile core: we use a .bat file to get vsvars32.bat to run before the
 248     # command.
 249     env = {}
 250     f.addStep(step.Compile, workdir="build/xapian-core/win32",
 251               command="..\\..\\xapian-maintainer-tools\\buildbot\\scripts\\compile_with_vc7.bat",
 252               env=env)
 253
 254     return f
 255
 256 def gen_tarball_updated_win_factory(rooturl):
 257     """Make a factory for doing builds from tarballs on windows.
 258
 259     """
 260     f = factory.BuildFactory()
 261     f.addStep(shell.ShellCommand(command = ["python", "-c", "try: import urllib2 as u\nexcept: import urllib.request as u\nopen('get_tarballs.py', 'wb').write(u.urlopen('%s').read())" %
 262               get_tarballs_url], workdir='.', haltOnFailure=True))
 263     f.addStep(shell.ShellCommand, command = ["python", 'get_tarballs.py', rooturl], workdir='.', haltOnFailure=True)
 264     f.addStep(shell.Compile, workdir='build/xapian-core/win32', command = ["compile_with_vc7.bat"])
 265     return f
 266
 267 all = []
 268 for gen in dir():
 269     if gen.startswith('gen_'):
 270         all.append(gen[4:])
 271         locals()[gen[4:]] = locals()[gen]