xapian-maintainer-tools/buildbot/xapian_factories.py

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 from buildbot.process import factory
   5 from buildbot.steps import source, shell, slave
   6 from datetime import date
   7
   8 xapian_config_arg = 'XAPIAN_CONFIG=../xapian-core/xapian-config'
   9
  10 get_tarballs_url = 'https://raw.githubusercontent.com/xapian/xapian/master/xapian-maintainer-tools/buildbot/scripts/get_tarballs.py'
  11
  12 def Bootstrap():
  13     return shell.ShellCommand(
  14         name = "bootstrap",
  15         haltOnFailure = 1,
  16         description = ["bootstrapping"],
  17         descriptionDone = ["bootstrap"],
  18         command = ["./bootstrap"],
  19     )
  20
  21 def CleanInstall():
  22     return shell.ShellCommand(
  23         name = "cleaninstall",
  24         haltOnFailure = 1,
  25         description = ["cleaninstall"],
  26         descriptionDone = ["cleaninstall"],
  27         command = ['rm', '-rf', 'install'],
  28     )
  29
  30 def Install():
  31     return shell.ShellCommand(
  32         name = "install",
  33         haltOnFailure = 1,
  34         description = ["install"],
  35         descriptionDone = ["install"],
  36         command = ['make', 'install'],
  37     )
  38
  39 def MakeWritable():
  40     """
  41     Step which ensures that the permissions are writable on all
  42     subdirectories.
  43     """
  44     return shell.ShellCommand(
  45         name = "make writable",
  46         haltOnFailure = 1,
  47         description = ["making writable"],
  48         descriptionDone = ["made writable"],
  49         command = ["chmod", "-R", "+w", "."],
  50     )
  51
  52 def core_factory(repourl, usedocs=False, configure=None, audit=False,
  53                  clean=False, nocheck = False, configure_opts=None):
  54     f = factory.BuildFactory()
  55     mode = "update"
  56     if clean:
  57         #f.addStep(MakeWritable, workdir='.')
  58         f.addStep(shell.ShellCommand(command = ["chmod", "-R", "+w", "."], workdir='.'))
  59         mode = "clobber"
  60     f.addStep(source.Git(repourl=repourl, mode=mode))
  61     if audit:
  62         f.addStep(shell.ShellCommand(command = ["python", 'audit.py'], workdir='build/xapian-maintainer-tools'))
  63         f.addStep(shell.ShellCommand(command = ["chmod", '644', 'copyright.csv', 'fixmes.csv'], workdir='build/xapian-maintainer-tools'))
  64         f.addStep(shell.ShellCommand(command = ["mv", 'copyright.csv', 'fixmes.csv', '/var/www/'], workdir='build/xapian-maintainer-tools'))
  65
  66     f.addStep(Bootstrap())
  67     if configure:
  68         f.addStep(shell.Configure(command=configure))
  69     else:
  70         if configure_opts is None:
  71             configure_opts = []
  72         if not usedocs:
  73             configure_opts.append("--disable-documentation")
  74         if configure_opts:
  75             f.addStep(shell.Configure(command=["sh", "configure"] + configure_opts))
  76         else:
  77             f.addStep(shell.Configure())
  78
  79     f.addStep(shell.Compile())
  80     if not nocheck:
  81         f.addStep(shell.Test(name="check", command=["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain", "VALGRIND="]))
  82     return f
  83
  84 def gen_git_updated_factory(repourl, usedocs=False, clean=False):
  85     """
  86     Make a factory for doing build from git master, but without cleaning
  87     first.  This build is intended to catch commonly made mistakes quickly.
  88     """
  89     return core_factory(repourl=repourl, usedocs=usedocs, clean=clean)
  90
  91 def gen_git_updated_factory_llvm(repourl):
  92     """
  93     Make a factory for doing build from git master, but without cleaning
  94     first.  This build is intended to catch commonly made mistakes quickly.
  95     """
  96     return core_factory(repourl=repourl, configure_opts=["CXX=/Developer/usr/llvm-gcc-4.2/bin/llvm-g++-4.2", "CC=/Developer/usr/llvm-gcc-4.2/bin/llvm-gcc-4.2"])
  97
  98 def gen_git_updated_factory2(repourl, configure_opts=[]):
  99     """
 100     Make a factory for doing build from git master, but without cleaning
 101     first.  This build is intended to catch commonly made mistakes quickly.
 102     This factory also runs audit.py and publishes the result.
 103     """
 104     return core_factory(repourl=repourl, usedocs=False, audit=True,
 105                         configure_opts=configure_opts)
 106
 107 def gen_git_updated_factory3(repourl):
 108     """
 109     Make a factory for doing build from git master, but without cleaning
 110     first.  This build is intended to catch commonly made mistakes quickly.
 111     This build runs with --disable-documentation, so the documentation building
 112     tools aren't required.
 113     """
 114     return core_factory(repourl=repourl, usedocs=False)
 115
 116 def gen_git_gccsnapshot_updated_factory(repourl):
 117     """
 118     Make a factory for doing build from git master, but without cleaning
 119     first, using gcc snapshot.  Also uses compiles with logging and assertions.
 120     """
 121     return core_factory(repourl=repourl,
 122                         configure_opts=["--enable-assertions", "--enable-log", "CXX=/usr/lib/gcc-snapshot/bin/g++", "CC=/usr/lib/gcc-snapshot/bin/gcc",
 123         ])
 124
 125 def gen_git_debug_updated_factory(repourl, opts, nocheck=False):
 126     """
 127     Make a factory for doing a debug build from git master, but without cleaning
 128     first.  This build is intended to catch commonly made mistakes quickly.
 129     """
 130     f = factory.BuildFactory()
 131     f.addStep(source.Git(repourl=repourl, mode="update"))
 132     f.addStep(Bootstrap())
 133     opts.append("--disable-documentation")
 134     f.addStep(shell.Configure(command = ["sh", "configure", ] + opts))
 135     f.addStep(shell.Compile())
 136     if not nocheck:
 137         f.addStep(shell.Test(name="check", command = ["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain", "VALGRIND="]))
 138     return f
 139
 140 def gen_tarball_updated_factory(rooturl, nocheck=False, omega=True, bindings=True, configure_opts=[]):
 141     """
 142     Make a factory for doing builds from tarballs.
 143     """
 144     configure_cmd = ["sh", "configure", ] + configure_opts
 145     f = factory.BuildFactory()
 146     f.addStep(shell.ShellCommand(command = ["python", "-c", "try: import urllib2 as u\nexcept: import urllib.request as u\nopen('get_tarballs.py', 'wb').write(u.urlopen('%s').read())" %
 147               get_tarballs_url], workdir='.', haltOnFailure=True))
 148     f.addStep(shell.ShellCommand(command = ["python", 'get_tarballs.py', rooturl], workdir='.', haltOnFailure=True))
 149     f.addStep(shell.Configure(workdir='build/xapian-core', command=configure_cmd))
 150     f.addStep(shell.Compile(workdir='build/xapian-core'))
 151     if not nocheck:
 152         f.addStep(shell.Test(workdir='build/xapian-core', name="check", command = ["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain", "VALGRIND="]))
 153     if omega:
 154         f.addStep(shell.Configure(workdir='build/xapian-omega', command = ["./configure", xapian_config_arg] + configure_opts))
 155         f.addStep(shell.Compile(workdir='build/xapian-omega'))
 156         if not nocheck:
 157             f.addStep(shell.Test(workdir='build/xapian-omega', name="check", command = ["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain", "VALGRIND="]))
 158     if bindings:
 159         f.addStep(shell.Configure(workdir='build/xapian-bindings', command = ["./configure", xapian_config_arg] + configure_opts))
 160         f.addStep(shell.Compile(workdir='build/xapian-bindings', command = ["make"]))
 161         if not nocheck:
 162             f.addStep(shell.Test(workdir='build/xapian-bindings', name="check", command = ["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain", "VALGRIND="]))
 163     # If everything passed, there's not much point keeping the build - we'd
 164     # delete the old build tree and download new tarballs next time anyway.
 165     f.addStep(slave.RemoveDirectory('build'))
 166     return f
 167
 168 def gen_git_updated_valgrind_factory(repourl, configure_opts=[]):
 169     """
 170     Factory for doing build from git master, without cleaning first, and using
 171     valgrind to check.  This one is much more expensive, so should be run with
 172     a higher stable time.
 173     """
 174     f = factory.BuildFactory()
 175     f.addStep(source.Git(repourl=repourl, mode="update"))
 176     f.addStep(Bootstrap())
 177     configure_opts.append("--disable-documentation")
 178     f.addStep(shell.Configure(command = ["sh", "configure", "CXXFLAGS=-O0 -g"] + configure_opts))
 179     f.addStep(shell.Compile())
 180
 181     f.addStep(shell.Test(name="check", command = ["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain"], workdir='build/xapian-core'))
 182
 183     return f
 184
 185 def gen_git_updated_lcov_factory(repourl, configure_opts=[]):
 186     """
 187     Factory for doing build from git master, without cleaning first, and using
 188     lcov to generate a coverage report.  This one is much more expensive, so
 189     should be run with a higher stable time.
 190     """
 191     f = factory.BuildFactory()
 192     f.addStep(source.Git(repourl=repourl, mode="update"))
 193     f.addStep(Bootstrap())
 194     f.addStep(shell.Configure(command = ["sh", "configure", "--enable-maintainer-mode", "--disable-shared", "--disable-documentation", "CXXFLAGS=-O0 --coverage", "VALGRIND=", "CCACHE_DISABLE=1"] + configure_opts, workdir="build/xapian-core"))
 195     f.addStep(shell.Compile(workdir="build/xapian-core"))
 196     f.addStep(shell.ShellCommand(command = ["make", "coverage-check", "GENHTML_ARGS=--html-gzip"], workdir="build/xapian-core", haltOnFailure=True))
 197     f.addStep(shell.ShellCommand(command = ["chmod", "-R", "a+rX", "lcov"], workdir="build/xapian-core", haltOnFailure=True))
 198     f.addStep(shell.ShellCommand(command = 'NOW=`date -u +%Y-%m-%d`; cp -a lcov/. /var/www/"$NOW" && ln -sfT "$NOW" /var/www/latest', workdir="build/xapian-core", haltOnFailure=True))
 199
 200     return f
 201
 202 #### FIXME: factories beyond here not updated
 203
 204 def gen_git_clean_dist_factory(repourl):
 205     """
 206     Factory for doing build from a clean checkout of git master.  This build also
 207     performs a "make distcheck", so should catch problems with files which have
 208     been missed from the distribution.  This one is much more expensive, so
 209     should be run with a higher stable time.
 210     """
 211     f = factory.BuildFactory()
 212     f.addStep(MakeWritable, workdir='.')
 213     f.addStep(source.Git(repourl=repourl, mode="clobber"))
 214     f.addStep(Bootstrap())
 215     f.addStep(step.Configure, command = ["xapian-maintainer-tools/buildbot/scripts/configure_with_prefix.sh"])
 216     extraargs = (
 217         "XAPIAN_TESTSUITE_OUTPUT=plain", "VALGRIND="
 218     )
 219     f.addStep(step.Compile, command = ["make",] + extraargs)
 220     # Don't bother running check as a separate step - all the checks will be
 221     # done by distcheck, anyway.  (Running it as a separate step _does_ check
 222     # that the tests work in a non-VPATH build, but this is tested by other
 223     # factories, anyway.)
 224     #f.addStep(step.Test, name="check", command = ["make", "check"] + extraargs)
 225     f.addStep(step.Test, name="distcheck", command = ["make", "distcheck"] + extraargs, workdir='build/xapian-core')
 226     f.addStep(step.Test, name="distcheck", command = ["make", "distcheck"] + extraargs, workdir='build/xapian-applications/omega')
 227
 228     # Have to install the core for distcheck to pass on the bindings.
 229     f.addStep(step.Test, name="install", command = ["make", "install"] + extraargs, workdir='build/xapian-core')
 230     f.addStep(step.Test, name="distcheck", command = ["make", "distcheck"] + extraargs, workdir='build/xapian-bindings')
 231     return f
 232
 233 def gen_git_updated_win_factory(repourl):
 234     """
 235     Factory for doing a windows build from git, without cleaning first.
 236     """
 237     f = factory.BuildFactory()
 238     f.addStep(step.Git(repourl=repourl, mode="update"))
 239     f.addStep(step.ShellCommand, command="xapian-maintainer-tools\\buildbot\\scripts\\prepare_build.bat")
 240
 241     # Compile core: we use a .bat file to get vsvars32.bat to run before the
 242     # command.
 243     env = {}
 244     f.addStep(step.Compile, workdir="build/xapian-core/win32",
 245               command="..\\..\\xapian-maintainer-tools\\buildbot\\scripts\\compile_with_vc7.bat",
 246               env=env)
 247
 248     return f
 249
 250 def gen_tarball_updated_win_factory(rooturl):
 251     """Make a factory for doing builds from tarballs on windows.
 252
 253     """
 254     f = factory.BuildFactory()
 255     f.addStep(shell.ShellCommand(command = ["python", "-c", "try: import urllib2 as u\nexcept: import urllib.request as u\nopen('get_tarballs.py', 'wb').write(u.urlopen('%s').read())" %
 256               get_tarballs_url], workdir='.', haltOnFailure=True))
 257     f.addStep(shell.ShellCommand, command = ["python", 'get_tarballs.py', rooturl], workdir='.', haltOnFailure=True)
 258     f.addStep(shell.Compile, workdir='build/xapian-core/win32', command = ["compile_with_vc7.bat"])
 259     return f
 260
 261 all = []
 262 for gen in dir():
 263     if gen.startswith('gen_'):
 264         all.append(gen[4:])
 265         locals()[gen[4:]] = locals()[gen]