hbuild/builders/tests.py

   1 #!/usr/bin/env python3
   2
   3 #
   4 # Copyright (c) 2017 Vojtech Horky
   5 # All rights reserved.
   6 #
   7 # Redistribution and use in source and binary forms, with or without
   8 # modification, are permitted provided that the following conditions
   9 # are met:
  10 #
  11 # - Redistributions of source code must retain the above copyright
  12 #   notice, this list of conditions and the following disclaimer.
  13 # - Redistributions in binary form must reproduce the above copyright
  14 #   notice, this list of conditions and the following disclaimer in the
  15 #   documentation and/or other materials provided with the distribution.
  16 # - The name of the author may not be used to endorse or promote products
  17 #   derived from this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  20 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  21 # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  22 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  23 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  24 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  28 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29 #
  30
  31 import os
  32 import re
  33 import fnmatch
  34
  35 from hbuild.scheduler import Task, RunCommandException
  36 from hbuild.builders.helenos import HelenOSBuildWithHarboursTask
  37
  38 class GetTestListTask(Task):
  39     def __init__(self, root_path, test_filter):
  40         Task.__init__(self, None)
  41         self.root_path = os.path.abspath(os.path.join(root_path, 'scenarios'))
  42         self.test_filter = test_filter
  43
  44     def get_scenario_list(self, root, base):
  45         if base == 'dummy/':
  46             return []
  47
  48         tests = []
  49         for name in os.listdir(root):
  50             path = os.path.join(root, name)
  51             if os.path.isdir(path):
  52                 tmp = self.get_scenario_list(path, base + name + '/')
  53                 for t in tmp:
  54                     tests.append(t)
  55             elif os.path.isfile(path):
  56                 xxx, ext = os.path.splitext(path)
  57                 if ext == '.yml':
  58                     tests.append(base + name)
  59
  60         return tests
  61
  62     def run(self):
  63         files = self.get_scenario_list(self.root_path, '')
  64
  65         if 'ALL' in self.test_filter:
  66             self.test_filter = files
  67
  68         files_filtered = []
  69         for fn in files:
  70             for pat in self.test_filter:
  71                 if fnmatch.fnmatch(fn, pat):
  72                     files_filtered.append(fn)
  73                     break
  74
  75         self.ctl.dprint('scenarios files: %s', files_filtered)
  76         return {
  77             'scenarios' : files_filtered,
  78             'scenario_dir': self.root_path
  79         }
  80
  81
  82 class ScheduleTestsTask(Task):
  83     NEEDED_HARBOUR_PATTERN = re.compile('^[ ]*#[ ]*@needs[ ]+(?P<HARBOURS>.*)$')
  84     SPLIT_HARBOURS_PATTERN = re.compile('\w+')
  85
  86     def __init__(self, scheduler, extra_builds, base_path, extra_tester_options):
  87         self.scheduler = scheduler
  88         self.testable_profiles = [ 'ia32', 'amd64', 'arm32/integratorcp', 'ppc32' ]
  89         self.extra_builds = extra_builds
  90         self.base_path = base_path
  91         self.extra_tester_options = extra_tester_options
  92         Task.__init__(self, None)
  93
  94     def run(self):
  95         helenos_build_tasks = self.ctl.get_dependency_data('helenos_tasks')
  96         scenarios = self.ctl.get_dependency_data('scenarios')
  97         scenario_base_path = self.ctl.get_dependency_data('scenario_dir')
  98         harbour_tasks = self.ctl.get_dependency_data('harbour_tasks')
  99
 100         self.extra_builds.set_dependent_tasks(helenos_build_tasks, harbour_tasks)
 101
 102         profiles_all = helenos_build_tasks.keys()
 103         profiles = []
 104         for p in self.testable_profiles:
 105             if p in profiles_all:
 106                 profiles.append(p)
 107
 108
 109         for scenario in scenarios:
 110             for profile in profiles:
 111                 scenario_filename = os.path.join(scenario_base_path, scenario)
 112                 dep_harbours = self.get_needed_harbours(scenario_filename)
 113                 if len(dep_harbours) > 0:
 114                     helenos_task = self.extra_builds.build(profile, dep_harbours)
 115                     if helenos_task is None:
 116                         # TODO: properly handle the error
 117                         continue
 118                 else:
 119                     helenos_task = helenos_build_tasks[profile]
 120                 scenario_flat = scenario.replace('/', '-').replace('.', '-')
 121                 self.scheduler.submit("Testing {} on {}".format(scenario, profile),
 122                     'test-{}-{}'.format(profile.replace('/', '-'), scenario_flat),
 123                     TestRunTask(profile, scenario, scenario_filename,
 124                         os.path.abspath(os.path.join(self.base_path, 'test-in-vm.py')), self.extra_tester_options),
 125                     [ helenos_task ],
 126                     [ 'qemu-kvm' ]
 127                 )
 128
 129         return True
 130
 131     def get_needed_harbours(self, scenario_filename):
 132         with open(scenario_filename) as f:
 133             try:
 134                 import yaml
 135                 scenario = yaml.load(f)
 136                 if ('meta' in scenario) and ('harbours' in scenario['meta']):
 137                     res = scenario['meta']['harbours']
 138                     res.sort()
 139                     return res
 140             except Exception as ex:
 141                 pass
 142         return []
 143
 144 class TestRunTask(Task):
 145     def __init__(self, profile, scenario_name, scenario_full_filename, test_script_filename, extra_test_script_options):
 146         self.profile = profile
 147         self.scenario_name = scenario_name
 148         self.scenario = scenario_full_filename
 149         self.tester = os.path.abspath(test_script_filename)
 150         self.tester_options = extra_test_script_options
 151         Task.__init__(self, 'test',
 152             arch=profile,
 153             scenario=scenario_name,
 154             description=self.get_scenario_description(scenario_full_filename)
 155         )
 156
 157     def get_scenario_description(self, filename):
 158         with open(filename) as f:
 159             try:
 160                 import yaml
 161                 scenario = yaml.load(f)
 162                 if ('meta' in scenario) and ('name' in scenario['meta']):
 163                     return '{}'.format(scenario['meta']['name'])
 164             except Exception as ex:
 165                 pass
 166         return ""
 167
 168     def run(self):
 169         os_image = self.ctl.get_dependency_data('image')
 170         my_dir = self.ctl.get_dependency_data('dir')
 171         if os_image is None:
 172             return False
 173         # FIXME: this is probably not the best location for the files
 174         vterm_dump = os.path.join(my_dir, 'dump.txt')
 175         screenshot = os.path.join(my_dir, 'screenshot.png')
 176         command = [
 177             self.tester,
 178             '--debug',
 179             '--headless',
 180             '--arch={}'.format(self.profile),
 181             '--image={}'.format(os_image),
 182             '--vterm-dump={}'.format(vterm_dump),
 183             '--last-screenshot={}'.format(screenshot),
 184         ]
 185         for i in self.tester_options:
 186             command.append(i)
 187         command.append('--scenario')
 188         command.append(self.scenario)
 189
 190         command_exc = None
 191
 192         try:
 193             command_res = self.ctl.run_command(command)
 194         except RunCommandException as ex:
 195             command_exc = ex
 196
 197         profile_flat = self.profile.replace("/", "-")
 198         scenario_flat = self.scenario_name.replace('.yml', '').replace('/', '-').replace('.', '-')
 199         try:
 200             self.ctl.add_downloadable_file("Last screen", '{}/test-{}-screen.png'.format(profile_flat, scenario_flat), screenshot)
 201         except OSError:
 202             pass
 203         try:
 204             self.ctl.add_downloadable_file("Terminal dump", '{}/test-{}-vterm.txt'.format(profile_flat, scenario_flat), vterm_dump)
 205         except OSError:
 206             pass
 207
 208         if command_exc is not None:
 209             raise command_exc
 210
 211         return {}