build.py

   1 #!/usr/bin/env python3
   2
   3 #
   4 # Copyright (c) 2017 Vojtech Horky
   5 # All rights reserved.
   6 #
   7 # Redistribution and use in source and binary forms, with or without
   8 # modification, are permitted provided that the following conditions
   9 # are met:
  10 #
  11 # - Redistributions of source code must retain the above copyright
  12 #   notice, this list of conditions and the following disclaimer.
  13 # - Redistributions in binary form must reproduce the above copyright
  14 #   notice, this list of conditions and the following disclaimer in the
  15 #   documentation and/or other materials provided with the distribution.
  16 # - The name of the author may not be used to endorse or promote products
  17 #   derived from this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  20 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  21 # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  22 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  23 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  24 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  28 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29 #
  30
  31 import subprocess
  32 import os
  33 import time
  34 import sys
  35 import argparse
  36 import multiprocessing
  37 from hbuild.scheduler import BuildScheduler
  38 from hbuild.cvs import *
  39 from hbuild.builders.helenos import *
  40 from hbuild.builders.coastline import *
  41 from hbuild.builders.tests import *
  42 from hbuild.gendoc import BrowsableSourcesViaGnuGlobalTask, DoxygenTask
  43 from hbuild.checkers.sycek import SycekBuildTask, SycekCheckTask
  44 from hbuild.web import *
  45 from hbuild.output import ConsolePrinter
  46 from hbuild.toolchain import check_tool, check_package
  47
  48 REQUIRED_TOOLS = [
  49     'convert',
  50     'msim',
  51     'meson',
  52     'ninja',
  53     'Xvfb',
  54     'xterm',
  55     'xdotool',
  56     'timeout',
  57 ]
  58
  59 REQUIRED_PACKAGES = [
  60     'argparse',
  61     'logging',
  62     'multiprocessing',
  63     'socket',
  64     'subprocess',
  65     'yaml',
  66     'PIL',
  67 ]
  68
  69 def create_checkout_task(name, url):
  70     if url.startswith("wip://"):
  71         return RsyncCheckoutTask(name, url[6:])
  72     else:
  73         return GitCheckoutTask(name, url)
  74
  75 # Command-line options
  76 args = argparse.ArgumentParser(description='HelenOS integration build')
  77 args.add_argument('--helenos-repository', default='https://github.com/HelenOS/helenos.git', dest='helenos_repository',
  78     metavar='PATH',
  79     help='HelenOS repository path'
  80 )
  81 args.add_argument('--coastline-repository', default='https://github.com/HelenOS/harbours.git', dest='coastline_repository',
  82     metavar='PATH',
  83     help='Coastline repository path'
  84 )
  85 args.add_argument('--build-id', default='0', dest='build_id',
  86     metavar='ID',
  87     help='Build id (typically sequence number).',
  88 )
  89 args.add_argument('--rss-url', default='', dest='rss_url',
  90     metavar='URL_WITH_RSS',
  91     help='URL of RSS for latest builds.'
  92 )
  93 args.add_argument('--resource-path', default=None, dest='web_resource_path',
  94     metavar='RELATIVE_PATH',
  95     help='Path where static web resources are stored (when specified, the resources are NOT copied).'
  96 )
  97 args.add_argument('--build-directory', default=os.path.abspath('tmp'), dest='build_directory',
  98     metavar='DIR',
  99     help='Where to build (space for temporary files).',
 100 )
 101 args.add_argument('--artefact-directory', default=os.path.abspath('out/'), dest='artefact_directory',
 102     metavar='DIR',
 103     help='Where to place downloadable files and HTML report.'
 104 )
 105 args.add_argument('--platforms', default='ALL', dest='platforms',
 106     metavar='PLATFORM1[,PLATFORM2[,...]',
 107     help='Which platforms to build (defaults to all detected ones; can be either machine specific "ia64/ski" or architecture specific "ia64/*").'
 108 )
 109 args.add_argument('--harbours', default='ALL', dest='harbours',
 110     metavar='HARBOUR1[,HARBOUR2[,...]',
 111     help='Which harbours to build (defaults to all detected ones).'
 112 )
 113 args.add_argument('--tests', default='ALL', dest='tests',
 114     metavar='TEST1[,TEST2[,...]]',
 115     help='Which tests to run (shell wildcards supported).'
 116 )
 117 args.add_argument('--vm-memory-size', default=256, dest='vm_memory_size',
 118     type=int,
 119     metavar='RAM_SIZE_IN_MB',
 120     help='How much memory to give the virtual machine running the tests.'
 121 )
 122 args.add_argument('--inline-log-lines', default=10, dest='inline_log_lines',
 123     type=int,
 124     metavar='LINES',
 125     help='How many lines of log to show on the web page.'
 126 )
 127 args.add_argument('--archive-format', default='tar.xz', dest='archive_format',
 128     choices=['tar.xz', 'tar.gz'],
 129     metavar='FORMAT',
 130     help='Format of the archives (tar.gz or tar.xz).'
 131 )
 132 args.add_argument('--no-source-browser', default=True, dest='code_browser',
 133     action='store_false',
 134     help='Do not generate source code browser.'
 135 )
 136 args.add_argument('--no-style-check', default=True, dest='style_check',
 137     action='store_false',
 138     help='Do not check C style.'
 139 )
 140 args.add_argument('--no-doxygen', default=True, dest='doxygen',
 141     action='store_false',
 142     help='Do not run Doxygen.'
 143 )
 144 args.add_argument('--no-tool-check', default=True, dest='tool_check',
 145     action='store_false',
 146     help='Do not check that all tools are installed.'
 147 )
 148 args.add_argument('--jobs', default=multiprocessing.cpu_count(), dest='jobs',
 149     type=int,
 150     metavar='COUNT',
 151     help='Number of concurrent jobs.'
 152 )
 153 args.add_argument('--no-colors', default=False, dest='no_colors',
 154     action='store_true',
 155     help='Disable colorful output'
 156 )
 157 args.add_argument('--debug', default=False, dest='debug',
 158     action='store_true',
 159     help='Print debugging messages'
 160 )
 161
 162 config = args.parse_args()
 163 config.artefact_directory = os.path.abspath(config.artefact_directory)
 164 config.build_directory = os.path.abspath(config.build_directory)
 165 config.self_path = os.path.dirname(os.path.realpath(sys.argv[0]))
 166
 167 printer = ConsolePrinter(config.no_colors)
 168
 169 if config.tool_check:
 170     try:
 171         for tool in REQUIRED_TOOLS:
 172             check_tool(tool, printer)
 173         for pkg in REQUIRED_PACKAGES:
 174             check_package(pkg, printer)
 175     except NotImplementedError:
 176         sys.exit(1)
 177
 178 if config.vm_memory_size < 8:
 179     printer.print_warning("VM memory size too small, upgrading to 8MB.")
 180     config.vm_memory_size = 8
 181
 182 scheduler = BuildScheduler(
 183     max_workers=config.jobs,
 184     build=config.build_directory,
 185     artefact=config.artefact_directory,
 186     build_id=config.build_id,
 187     printer=printer,
 188     inline_log_lines=config.inline_log_lines,
 189     debug=config.debug
 190 )
 191
 192 #
 193 # Check-out both HelenOS and coastline repositories
 194 scheduler.submit("Checking-out HelenOS",
 195     "helenos-checkout",
 196     create_checkout_task("helenos", config.helenos_repository))
 197 scheduler.submit("Checking-out Coastline",
 198     "coastline-checkout",
 199     create_checkout_task("coastline", config.coastline_repository))
 200
 201 #
 202 # Build browsable sources
 203 if config.code_browser:
 204     scheduler.submit("Browsable sources",
 205         "helenos-browsable-sources",
 206         BrowsableSourcesViaGnuGlobalTask(),
 207         ["helenos-checkout"])
 208
 209 if config.doxygen:
 210     scheduler.submit("Generate Doxygen documentation",
 211         "helenos-doxygen",
 212         DoxygenTask(),
 213         ["helenos-checkout"])
 214
 215 #
 216 # C style check
 217 if config.style_check:
 218     scheduler.submit("Build Sycek C style checker",
 219         "sycek-build",
 220         SycekBuildTask(),
 221         [])
 222
 223     scheduler.submit("Check C style with Sycek",
 224         "helenos-sycek-check",
 225         SycekCheckTask(),
 226         ["helenos-checkout", "sycek-build"])
 227
 228 #
 229 # HelenOS (mainline): get list of profiles (i.e. supported architectures
 230 # and platforms) and build all of them
 231 scheduler.submit("Determininig available profiles",
 232     "helenos-get-profiles",
 233     HelenOSGetProfilesTask(config.platforms.split(',')),
 234     ["helenos-checkout"])
 235
 236
 237 scheduler.submit("Schedule HelenOS builds",
 238     "helenos-build",
 239     HelenOSScheduleBuildsTask(scheduler),
 240     ["helenos-get-profiles"])
 241
 242
 243 #
 244 # Coastline: get list of harbours (i.e. ported software) and build all of them
 245 # for all HelenOS builds
 246 scheduler.submit("Determining available harbours",
 247     "coastline-get-harbours",
 248      CoastlineGetHarboursTask(config.harbours.split(',')),
 249      ["coastline-checkout"])
 250
 251 scheduler.submit("Schedule harbour tarballs fetches",
 252     "coastline-fetch",
 253     CoastlineScheduleFetchesTask(scheduler),
 254     ["coastline-get-harbours" ])
 255
 256
 257 scheduler.submit("Schedule Coastline builds",
 258     "coastline-build",
 259     CoastlineScheduleBuildsTask(scheduler, config.archive_format),
 260     [
 261         # Data dependencies
 262         "helenos-get-profiles", "coastline-get-harbours",
 263         # Task dependencies
 264         "helenos-build", "coastline-fetch"
 265     ])
 266
 267
 268 extra_builds = HelenOSExtraBuildsManager(scheduler)
 269
 270 #
 271 # Tests
 272 scheduler.submit("Determine available test scenarios",
 273     "tests-get-list",
 274     GetTestListTask(config.self_path, config.tests.split(',')),
 275     [])
 276
 277 scheduler.submit("Schedule tests",
 278     "tests-schedule",
 279     ScheduleTestsTask(scheduler,
 280         extra_builds,
 281         config.self_path,
 282         [ "--memory={}".format(config.vm_memory_size) ]
 283     ),
 284     [
 285         "tests-get-list",
 286         "helenos-build",
 287         "coastline-build"
 288     ])
 289
 290
 291 #
 292 # Wait for all builds (and everything) to complete before creating
 293 # the web page with results.
 294 scheduler.barrier()
 295
 296 scheduler.close_report()
 297
 298 scheduler.submit("Generate HTML report",
 299     "html-report",
 300     MakeHtmlReportTask(config.self_path, config.rss_url, config.web_resource_path))
 301
 302
 303 scheduler.done()