hbuild/builders/tests.py

   1 #!/usr/bin/env python3
   2
   3 #
   4 # Copyright (c) 2017 Vojtech Horky
   5 # All rights reserved.
   6 #
   7 # Redistribution and use in source and binary forms, with or without
   8 # modification, are permitted provided that the following conditions
   9 # are met:
  10 #
  11 # - Redistributions of source code must retain the above copyright
  12 #   notice, this list of conditions and the following disclaimer.
  13 # - Redistributions in binary form must reproduce the above copyright
  14 #   notice, this list of conditions and the following disclaimer in the
  15 #   documentation and/or other materials provided with the distribution.
  16 # - The name of the author may not be used to endorse or promote products
  17 #   derived from this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  20 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  21 # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  22 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  23 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  24 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  28 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29 #
  30
  31 import os
  32 import re
  33 import fnmatch
  34
  35 from hbuild.scheduler import Task
  36 from hbuild.builders.helenos import HelenOSBuildWithHarboursTask
  37
  38 class GetTestListTask(Task):
  39     def __init__(self, root_path, test_filter):
  40         Task.__init__(self, None)
  41         self.root_path = os.path.abspath(os.path.join(root_path, 'scenarios'))
  42         self.test_filter = test_filter
  43
  44     def get_scenario_list(self, root, base):
  45         if base == 'dummy/':
  46             return []
  47
  48         tests = []
  49         for name in os.listdir(root):
  50             path = os.path.join(root, name)
  51             if os.path.isdir(path):
  52                 tmp = self.get_scenario_list(path, base + name + '/')
  53                 for t in tmp:
  54                     tests.append(t)
  55             elif os.path.isfile(path):
  56                 xxx, ext = os.path.splitext(path)
  57                 if ext == '.yml':
  58                     tests.append(base + name)
  59
  60         return tests
  61
  62     def run(self):
  63         files = self.get_scenario_list(self.root_path, '')
  64
  65         if 'ALL' in self.test_filter:
  66             self.test_filter = files
  67
  68         files_filtered = []
  69         for fn in files:
  70             for pat in self.test_filter:
  71                 if fnmatch.fnmatch(fn, pat):
  72                     files_filtered.append(fn)
  73                     break
  74
  75         self.ctl.dprint('scenarios files: %s', files_filtered)
  76         return {
  77             'scenarios' : files_filtered,
  78             'scenario_dir': self.root_path
  79         }
  80
  81
  82 class ScheduleTestsTask(Task):
  83     NEEDED_HARBOUR_PATTERN = re.compile('^[ ]*#[ ]*@needs[ ]+(?P<HARBOURS>.*)$')
  84     SPLIT_HARBOURS_PATTERN = re.compile('\w+')
  85
  86     def __init__(self, scheduler, extra_builds, base_path, extra_tester_options):
  87         self.scheduler = scheduler
  88         self.testable_profiles = [ 'ia32', 'amd64', 'arm32/integratorcp', 'ppc32' ]
  89         self.extra_builds = extra_builds
  90         self.base_path = base_path
  91         self.extra_tester_options = extra_tester_options
  92         Task.__init__(self, None)
  93
  94     def run(self):
  95         helenos_build_tasks = self.ctl.get_dependency_data('helenos_tasks')
  96         scenarios = self.ctl.get_dependency_data('scenarios')
  97         scenario_base_path = self.ctl.get_dependency_data('scenario_dir')
  98         harbour_tasks = self.ctl.get_dependency_data('harbour_tasks')
  99
 100         self.extra_builds.set_dependent_tasks(helenos_build_tasks, harbour_tasks)
 101
 102         profiles_all = helenos_build_tasks.keys()
 103         profiles = []
 104         for p in self.testable_profiles:
 105             if p in profiles_all:
 106                 profiles.append(p)
 107
 108
 109         for scenario in scenarios:
 110             for profile in profiles:
 111                 scenario_filename = os.path.join(scenario_base_path, scenario)
 112                 dep_harbours = self.get_needed_harbours(scenario_filename)
 113                 if len(dep_harbours) > 0:
 114                     helenos_task = self.extra_builds.build(profile, dep_harbours)
 115                     if helenos_task is None:
 116                         # TODO: properly handle the error
 117                         continue
 118                 else:
 119                     helenos_task = helenos_build_tasks[profile]
 120                 scenario_flat = scenario.replace('/', '-').replace('.', '-')
 121                 self.scheduler.submit("Testing {} on {}".format(scenario, profile),
 122                     'test-{}-{}'.format(profile.replace('/', '-'), scenario_flat),
 123                     TestRunTask(profile, scenario, scenario_filename,
 124                         os.path.abspath(os.path.join(self.base_path, 'vm-test.py')), self.extra_tester_options),
 125                     [ helenos_task ],
 126                     [ 'qemu-kvm' ]
 127                 )
 128
 129
 130 #         scenario_dependencies = {}
 131 #         for s in scenarios:
 132 #             scenario_dependencies[s] = {}
 133 #             harbours =
 134 #             for p in profiles:
 135 #                 scenario_dependencies[s][p] = [
 136 #                     self.extra_builds.build(p, harbours)
 137 #                 ]
 138 #
 139 #
 140 #         needed_harbours = {}
 141 #         for s in scenarios:
 142 #
 143 #             needed_harbours[ s ] = {
 144 #                 'harbours': harbours,
 145 #                 'hash': '-'.join(harbours)
 146 #             }
 147 #
 148 #         self.ctl.dprint('Harbour tasks: {}'.format(harbour_tasks))
 149 #         self.ctl.dprint('Needed harbours: {}'.format(needed_harbours))
 150 #
 151 #         for p in profiles:
 152 #             special_images_tasks = {}
 153 #             for s in scenarios:
 154 #                 self.ctl.dprint('{}/{}: special-tasks: {}'.format(p, s, special_images_tasks))
 155 #                 s_flat = s.replace('/', '-').replace('.', '-')
 156 #                 deps = [ helenos_build_tasks[p] ]
 157 #                 if len(needed_harbours[s]['harbours']) > 0:
 158 #                     if not needed_harbours[s]['hash'] in special_images_tasks:
 159 #                         extra_deps = [ helenos_build_tasks[p] ]
 160 #                         continue_outer = False
 161 #                         for h in needed_harbours[s]['harbours']:
 162 #                             if h not in harbour_tasks[p]:
 163 #                                 continue_outer = True
 164 #                                 break
 165 #                             else:
 166 #                                 extra_deps.append(harbour_tasks[p][h])
 167 #                         if continue_outer:
 168 #                             continue
 169 #                         task_name = 'extra-{}-with-{}'.format(p.replace('/', '-'), needed_harbours[s]['hash'])
 170 #                         self.scheduler.submit("Special build of {} with {}".format(p, ','.join(needed_harbours[s]['harbours'])),
 171 #                             task_name,
 172 #                             HelenOSBuildWithHarboursTask(p, needed_harbours[s]['harbours']),
 173 #                             extra_deps,
 174 #                             [ 'extras-{}'.format(p) ]
 175 #                         )
 176 #                         special_images_tasks[ needed_harbours[s]['hash'] ] = task_name
 177 #                     deps.append(special_images_tasks[ needed_harbours[s]['hash'] ])
 178 #                 self.scheduler.submit("Testing {} on {}".format(s, p),
 179 #                     'test-{}-{}'.format(p.replace('/', '-'), s_flat),
 180 #                     TestRunTask(p, s, os.path.join(scenario_base_path, s)),
 181 #                     deps,
 182 #                     [ 'qemu-kvm' ]
 183 #                 )
 184 #
 185         return True
 186
 187     def get_needed_harbours(self, scenario_filename):
 188         with open(scenario_filename) as f:
 189             try:
 190                 import yaml
 191                 scenario = yaml.load(f)
 192                 if ('meta' in scenario) and ('harbours' in scenario['meta']):
 193                     res = scenario['meta']['harbours']
 194                     res.sort()
 195                     return res
 196             except Exception as ex:
 197                 pass
 198         return []
 199
 200 class TestRunTask(Task):
 201     def __init__(self, profile, scenario_name, scenario_full_filename, test_script_filename, extra_test_script_options):
 202         self.profile = profile
 203         self.scenario_name = scenario_name
 204         self.scenario = scenario_full_filename
 205         self.tester = os.path.abspath(test_script_filename)
 206         self.tester_options = extra_test_script_options
 207         Task.__init__(self, 'test', arch=profile, scenario=scenario_name)
 208
 209     def run(self):
 210         os_image = self.ctl.get_dependency_data('image')
 211         my_dir = self.ctl.get_dependency_data('dir')
 212         if os_image is None:
 213             return False
 214         # FIXME: this is probably not the best location for the files
 215         vterm_dump = os.path.join(my_dir, 'dump.txt')
 216         screenshot = os.path.join(my_dir, 'screenshot.png')
 217         command = [
 218             self.tester,
 219             '--debug',
 220             '--headless',
 221             '--arch={}'.format(self.profile),
 222             '--image={}'.format(os_image),
 223             '--vterm-dump={}'.format(vterm_dump),
 224             '--last-screenshot={}'.format(screenshot),
 225         ]
 226         for i in self.tester_options:
 227             command.append(i)
 228         command.append('--scenario')
 229         command.append(self.scenario)
 230         res = self.ctl.run_command(command)
 231         if res['failed']:
 232             return False
 233
 234         profile_flat = self.profile.replace("/", "-")
 235         scenario_flat = self.scenario_name.replace('.yml', '').replace('/', '-').replace('.', '-')
 236         self.ctl.add_downloadable_file("Last screen", '{}/test-{}-screen.png'.format(profile_flat, scenario_flat), screenshot)
 237         self.ctl.add_downloadable_file("Terminal dump", '{}/test-{}-vterm.txt'.format(profile_flat, scenario_flat), vterm_dump)
 238
 239         return True