hbuild/builders/coastline.py

   1 #!/usr/bin/env python3
   2
   3 #
   4 # Copyright (c) 2017 Vojtech Horky
   5 # All rights reserved.
   6 #
   7 # Redistribution and use in source and binary forms, with or without
   8 # modification, are permitted provided that the following conditions
   9 # are met:
  10 #
  11 # - Redistributions of source code must retain the above copyright
  12 #   notice, this list of conditions and the following disclaimer.
  13 # - Redistributions in binary form must reproduce the above copyright
  14 #   notice, this list of conditions and the following disclaimer in the
  15 #   documentation and/or other materials provided with the distribution.
  16 # - The name of the author may not be used to endorse or promote products
  17 #   derived from this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  20 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  21 # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  22 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  23 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  24 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  28 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29 #
  30
  31 import os
  32
  33 from hbuild.scheduler import Task, TaskException, RunCommandException
  34
  35 def sorted_dir(root):
  36     list = os.listdir(root)
  37     list.sort()
  38     return list
  39
  40 def create_hsct_conf(dir, profile, repo_dir, archive_format, sources = None):
  41     with open(dir + '/hsct.conf', 'w') as f:
  42         if profile == 'mips32/malta-be':
  43             profile = 'mips32eb/malta-be'
  44
  45         profile_parts = ( profile + '/').split('/')
  46
  47         f.write("root = %s\n" % repo_dir)
  48         f.write("arch = %s\n" % profile_parts[0])
  49         f.write("machine = %s\n" % profile_parts[1])
  50         f.write("parallel = 1\n")
  51         f.write("archive_format = %s\n" % archive_format)
  52         if not sources is None:
  53             f.write("sources = %s\n" % sources)
  54
  55
  56 class CoastlineGetHarboursTask(Task):
  57     def __init__(self, harbour_filter):
  58         self.harbour_filter = harbour_filter
  59         Task.__init__(self, None)
  60
  61     def get_harbour_dependencies(self, harbour_file):
  62         try:
  63             cmd = '. %s; echo $shiptugs;' % harbour_file
  64             res = self.ctl.run_command([ 'sh', '-c', cmd ], needs_output=True)
  65             return res['stdout'].split()
  66         except RunCommandException as e:
  67             return []
  68
  69     def run(self):
  70         root = self.ctl.get_dependency_data('dir')
  71         self.ctl.dprint("Looking into %s", root)
  72         harbours = []
  73         dependencies = {}
  74         for name in sorted_dir(root):
  75             path = os.path.join(root, name)
  76             canon = os.path.join(path, 'HARBOUR')
  77             if os.path.isdir(path) and os.path.exists(canon) and os.path.isfile(canon):
  78                 harbours.append(name)
  79                 dependencies[name] = self.get_harbour_dependencies(canon)
  80
  81         # Clean-up the dependencies
  82         for h in harbours:
  83             deps = dependencies[h]
  84             dependencies[h] = []
  85             for d in deps:
  86                 if d in harbours:
  87                     dependencies[h].append(d)
  88
  89         # Filter which harbours should be actually built
  90         if 'ALL' in self.harbour_filter:
  91             self.harbour_filter = dependencies.keys()
  92
  93         harbours_to_build = {}
  94         for h in harbours:
  95             if h in self.harbour_filter:
  96                 harbours_to_build[h] = True
  97         dependency_added = True
  98         while dependency_added:
  99             dependency_added = False
 100             so_far = list(harbours_to_build.keys())
 101             for h in so_far:
 102                 for d in dependencies[h]:
 103                     if not d in harbours_to_build:
 104                         harbours_to_build[d] = True
 105                         dependency_added = True
 106
 107         self.ctl.dprint("harbours_to_build = %s", list(harbours_to_build.keys()))
 108
 109         # Sort harbours in buildable order
 110         build_order = []
 111         remaining_dependencies = dependencies.copy()
 112         while len(remaining_dependencies) > 0:
 113             self.ctl.dprint("remaining = %s", remaining_dependencies)
 114             leafs = []
 115             for d in remaining_dependencies:
 116                 if len(remaining_dependencies[d]) == 0:
 117                     if d in harbours_to_build:
 118                         build_order.append(d)
 119                     leafs.append(d)
 120             # In every step we have to find (at least) one element to
 121             # remove
 122             if len(leafs) == 0:
 123                 raise TaskException("Circular dependency found!")
 124             for l in leafs:
 125                 del remaining_dependencies[l]
 126                 for d in remaining_dependencies:
 127                     remaining_dependencies[d] = [x for x in remaining_dependencies[d] if x != l]
 128
 129
 130         self.ctl.dprint("harbours = %s", build_order)
 131         self.ctl.dprint("deps = %s" , dependencies)
 132
 133         return {
 134             'harbours': build_order,
 135             'harbour_deps': dependencies,
 136             'coastline_root': root
 137         }
 138
 139 class CoastlineFetchTask(Task):
 140     def __init__(self, harbour, root, mirror):
 141         self.harbour = harbour
 142         self.root = root
 143         self.mirror = mirror
 144         Task.__init__(self, 'harbour-fetch', package=harbour)
 145
 146     def run(self):
 147         self.ctl.run_command([ self.root + '/hsct.sh', 'fetch', self.harbour ], cwd=self.mirror)
 148
 149
 150 class CoastlineScheduleFetchesTask(Task):
 151     def __init__(self, scheduler):
 152         self.scheduler = scheduler
 153         Task.__init__(self, None)
 154
 155     def run(self):
 156         harbours = self.ctl.get_dependency_data('harbours')
 157         coastline_root = self.ctl.make_temp_dir('repo/coastline')
 158
 159         mirror = self.ctl.make_temp_dir('mirror')
 160         create_hsct_conf(mirror, 'mirror', 'not-really-needed', '')
 161
 162         tasks = {}
 163         for h in harbours:
 164             task_name = "coastline-fetch-%s" % h
 165             self.scheduler.submit("Fetching tarballs for %s" % h,
 166                 task_name,
 167                 CoastlineFetchTask(h, coastline_root, mirror))
 168             tasks[ h ] = task_name
 169         return {
 170             'fetch_tasks': tasks
 171         }
 172
 173 class CoastlinePrebuildTask(Task):
 174     def __init__(self, profile, build_dir_basename, archive_format):
 175         self.profile = profile
 176         self.build_dir_basename = build_dir_basename
 177         self.archive_format = archive_format
 178         Task.__init__(self, None)
 179
 180     def run(self):
 181         root = self.ctl.make_temp_dir('repo/coastline')
 182
 183         my_dir = self.ctl.make_temp_dir('build/%s/coast' % self.build_dir_basename)
 184         create_hsct_conf(my_dir, self.profile,
 185             self.ctl.make_temp_dir('build/%s/helenos' % self.build_dir_basename),
 186             self.archive_format,
 187             self.ctl.make_temp_dir('mirror/sources'))
 188
 189         self.ctl.run_command([ root + '/hsct.sh', 'update' ], cwd=my_dir)
 190
 191         create_hsct_conf(my_dir, self.profile,
 192             self.ctl.make_temp_dir('build/%s/helenos-nonexistent' % self.build_dir_basename),
 193             self.archive_format,
 194             self.ctl.make_temp_dir('mirror/sources'))
 195
 196         return {
 197             'dir': my_dir
 198         }
 199
 200 class CoastlineBuildTask(Task):
 201     def __init__(self, harbour, profile, archive_format):
 202         self.harbour = harbour
 203         self.profile = profile
 204         self.archive_format = archive_format
 205         Task.__init__(self, 'harbour-build', package=harbour, arch=profile)
 206
 207     def run(self):
 208         my_dir = self.ctl.get_dependency_data('dir')
 209
 210         root = self.ctl.make_temp_dir('repo/coastline')
 211
 212         res = self.ctl.run_command([ root + '/hsct.sh', 'archive', self.harbour ], cwd=my_dir)
 213         if res['failed']:
 214             return False
 215
 216         # Add downloadable archive
 217         profile_flat = self.profile.replace("/", "-")
 218         title = "%s for %s" % ( self.harbour, self.profile )
 219         target_filename = '%s/%s-for-helenos-%s.%s' % ( profile_flat, self.harbour, profile_flat, self.archive_format )
 220         current_filename = '%s/archives/%s.%s' % ( my_dir, self.harbour, self.archive_format )
 221         self.ctl.add_downloadable_file(title, target_filename, current_filename)
 222
 223         return {
 224             'harbour-{}'.format(self.harbour): current_filename
 225         }
 226
 227
 228 class CoastlineScheduleBuildsTask(Task):
 229     def __init__(self, scheduler, archive_format):
 230         self.scheduler = scheduler
 231         self.archive_format =archive_format
 232         Task.__init__(self, None)
 233
 234     def run(self):
 235         profiles = self.ctl.get_dependency_data('profiles')
 236         harbours = self.ctl.get_dependency_data('harbours')
 237         harbour_deps = self.ctl.get_dependency_data('harbour_deps')
 238
 239         ret = {}
 240
 241         for p in profiles:
 242             if p == 'special/abs32le':
 243                 continue
 244
 245             ret[ p ] = {}
 246
 247             p_flat = p.replace("/", "-")
 248
 249             self.scheduler.submit("Preparing coastline for %s" % p,
 250                 "coastline-prepare-for-%s" % p_flat,
 251                 CoastlinePrebuildTask(p, p_flat, self.archive_format),
 252                 [ "helenos-build-%s" % p_flat ])
 253
 254             for h in harbours:
 255                 task_name = "coastline-build-%s-for-%s" % (h, p_flat)
 256                 deps = [
 257                     "coastline-prepare-for-%s" % p_flat,
 258                     "coastline-fetch-%s" % h
 259                 ]
 260                 for d in harbour_deps[ h ]:
 261                     deps.append( "coastline-build-%s-for-%s" % (d, p_flat) )
 262                 self.scheduler.submit("Building %s for %s" % (h, p),
 263                     task_name,
 264                     CoastlineBuildTask(h, p, self.archive_format),
 265                     deps)
 266
 267                 ret[p][h] = task_name
 268
 269         return {
 270             'harbour_tasks' : ret
 271         }