hbuild/builders/coastline.py

   1 #!/usr/bin/env python3
   2
   3 #
   4 # Copyright (c) 2017 Vojtech Horky
   5 # All rights reserved.
   6 #
   7 # Redistribution and use in source and binary forms, with or without
   8 # modification, are permitted provided that the following conditions
   9 # are met:
  10 #
  11 # - Redistributions of source code must retain the above copyright
  12 #   notice, this list of conditions and the following disclaimer.
  13 # - Redistributions in binary form must reproduce the above copyright
  14 #   notice, this list of conditions and the following disclaimer in the
  15 #   documentation and/or other materials provided with the distribution.
  16 # - The name of the author may not be used to endorse or promote products
  17 #   derived from this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  20 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  21 # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  22 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  23 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  24 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  28 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29 #
  30
  31 import os
  32
  33 from hbuild.scheduler import Task, TaskException, RunCommandException
  34
  35 def sorted_dir(root):
  36     list = os.listdir(root)
  37     list.sort()
  38     return list
  39
  40 class CoastlineGetHarboursTask(Task):
  41     def __init__(self, harbour_filter):
  42         self.harbour_filter = harbour_filter
  43         Task.__init__(self, None)
  44
  45     def get_harbour_dependencies(self, harbour_file):
  46         try:
  47             cmd = '. %s; echo $shiptugs;' % harbour_file
  48             res = self.ctl.run_command([ 'sh', '-c', cmd ], needs_output=True)
  49             return res['stdout'].split()
  50         except RunCommandException as e:
  51             return []
  52
  53     def run(self):
  54         root = self.ctl.get_dependency_data('dir')
  55         self.ctl.dprint("Looking into %s", root)
  56         harbours = []
  57         dependencies = {}
  58         for name in sorted_dir(root):
  59             path = os.path.join(root, name)
  60             canon = os.path.join(path, 'HARBOUR')
  61             if os.path.isdir(path) and os.path.exists(canon) and os.path.isfile(canon):
  62                 harbours.append(name)
  63                 dependencies[name] = self.get_harbour_dependencies(canon)
  64
  65         # Clean-up the dependencies
  66         for h in harbours:
  67             deps = dependencies[h]
  68             dependencies[h] = []
  69             for d in deps:
  70                 if d in harbours:
  71                     dependencies[h].append(d)
  72
  73         # Filter which harbours should be actually built
  74         if 'ALL' in self.harbour_filter:
  75             self.harbour_filter = dependencies.keys()
  76
  77         harbours_to_build = {}
  78         for h in harbours:
  79             if h in self.harbour_filter:
  80                 harbours_to_build[h] = True
  81         dependency_added = True
  82         while dependency_added:
  83             dependency_added = False
  84             so_far = list(harbours_to_build.keys())
  85             for h in so_far:
  86                 for d in dependencies[h]:
  87                     if not d in harbours_to_build:
  88                         harbours_to_build[d] = True
  89                         dependency_added = True
  90
  91         self.ctl.dprint("harbours_to_build = %s", list(harbours_to_build.keys()))
  92
  93         # Sort harbours in buildable order
  94         build_order = []
  95         remaining_dependencies = dependencies.copy()
  96         while len(remaining_dependencies) > 0:
  97             self.ctl.dprint("remaining = %s", remaining_dependencies)
  98             leafs = []
  99             for d in remaining_dependencies:
 100                 if len(remaining_dependencies[d]) == 0:
 101                     if d in harbours_to_build:
 102                         build_order.append(d)
 103                     leafs.append(d)
 104             # In every step we have to find (at least) one element to
 105             # remove
 106             if len(leafs) == 0:
 107                 raise TaskException("Circular dependency found!")
 108             for l in leafs:
 109                 del remaining_dependencies[l]
 110                 for d in remaining_dependencies:
 111                     remaining_dependencies[d] = [x for x in remaining_dependencies[d] if x != l]
 112
 113
 114         self.ctl.dprint("harbours = %s", build_order)
 115         self.ctl.dprint("deps = %s" , dependencies)
 116
 117         return {
 118             'harbours': build_order,
 119             'harbour_deps': dependencies,
 120             'coastline_root': root
 121         }
 122
 123 class CoastlineFetchTask(Task):
 124     def __init__(self, harbour, root, mirror):
 125         self.harbour = harbour
 126         self.root = root
 127         self.mirror = mirror
 128         Task.__init__(self, 'harbour-fetch', package=harbour)
 129
 130     def run(self):
 131         self.ctl.run_command([ self.root + '/hsct.sh', 'fetch', self.harbour ], cwd=self.mirror)
 132
 133
 134 class CoastlineScheduleFetchesTask(Task):
 135     def __init__(self, scheduler):
 136         self.scheduler = scheduler
 137         Task.__init__(self, None)
 138
 139     def run(self):
 140         harbours = self.ctl.get_dependency_data('harbours')
 141         coastline_root = self.ctl.make_temp_dir('repo/coastline')
 142
 143         mirror = self.ctl.make_temp_dir('mirror')
 144
 145         tasks = {}
 146         for h in harbours:
 147             task_name = "coastline-fetch-%s" % h
 148             self.scheduler.submit("Fetching tarballs for %s" % h,
 149                 task_name,
 150                 CoastlineFetchTask(h, coastline_root, mirror))
 151             tasks[ h ] = task_name
 152         return {
 153             'fetch_tasks': tasks
 154         }
 155
 156 class CoastlinePrebuildTask(Task):
 157     def __init__(self, profile, build_dir_basename, archive_format):
 158         self.profile = profile
 159         self.build_dir_basename = build_dir_basename
 160         self.archive_format = archive_format
 161         Task.__init__(self, None)
 162
 163     def run(self):
 164         root = self.ctl.make_temp_dir('repo/coastline')
 165
 166         my_dir = self.ctl.make_temp_dir('build/%s/coast' % self.build_dir_basename)
 167         hsrootdir = self.ctl.make_temp_dir('build/%s/helenos' % self.build_dir_basename)
 168         self.ctl.run_command([ root + '/hsct.sh', 'init', hsrootdir ], cwd=my_dir)
 169         with open('%s/config.sh' % my_dir, 'a') as cfg:
 170             cfg.write("HSCT_FORMAT=\"%s\"\n" % self.archive_format)
 171             cfg.write("HSCT_SOURCES_DIR=\"%s\"\n" % self.ctl.make_temp_dir('mirror/sources'))
 172             cfg.write("HSCT_PARALLELISM=1\n")
 173
 174         return {
 175             'dir': my_dir
 176         }
 177
 178 class CoastlineBuildTask(Task):
 179     def __init__(self, harbour, profile, archive_format):
 180         self.harbour = harbour
 181         self.profile = profile
 182         self.archive_format = archive_format
 183         Task.__init__(self, 'harbour-build', package=harbour, arch=profile)
 184
 185     def run(self):
 186         my_dir = self.ctl.get_dependency_data('dir')
 187
 188         root = self.ctl.make_temp_dir('repo/coastline')
 189
 190         res = self.ctl.run_command([ root + '/hsct.sh', 'archive', self.harbour ], cwd=my_dir)
 191         if res['failed']:
 192             return False
 193
 194         # Add downloadable archive
 195         profile_flat = self.profile.replace("/", "-")
 196         title = "%s for %s" % ( self.harbour, self.profile )
 197         target_filename = '%s/%s-for-helenos-%s.%s' % ( profile_flat, self.harbour, profile_flat, self.archive_format )
 198         current_filename = '%s/archives/%s.%s' % ( my_dir, self.harbour, self.archive_format )
 199         self.ctl.add_downloadable_file(title, target_filename, current_filename)
 200
 201         return {
 202             'harbour-{}'.format(self.harbour): current_filename
 203         }
 204
 205
 206 class CoastlineScheduleBuildsTask(Task):
 207     def __init__(self, scheduler, archive_format):
 208         self.scheduler = scheduler
 209         self.archive_format =archive_format
 210         Task.__init__(self, None)
 211
 212     def run(self):
 213         profiles = self.ctl.get_dependency_data('profiles')
 214         harbours = self.ctl.get_dependency_data('harbours')
 215         harbour_deps = self.ctl.get_dependency_data('harbour_deps')
 216
 217         ret = {}
 218
 219         for p in profiles:
 220             if p == 'special/abs32le':
 221                 continue
 222
 223             ret[ p ] = {}
 224
 225             p_flat = p.replace("/", "-")
 226
 227             self.scheduler.submit("Preparing coastline for %s" % p,
 228                 "coastline-prepare-for-%s" % p_flat,
 229                 CoastlinePrebuildTask(p, p_flat, self.archive_format),
 230                 [ "helenos-build-%s" % p_flat ])
 231
 232             for h in harbours:
 233                 task_name = "coastline-build-%s-for-%s" % (h, p_flat)
 234                 deps = [
 235                     "coastline-prepare-for-%s" % p_flat,
 236                     "coastline-fetch-%s" % h
 237                 ]
 238                 for d in harbour_deps[ h ]:
 239                     deps.append( "coastline-build-%s-for-%s" % (d, p_flat) )
 240                 self.scheduler.submit("Building %s for %s" % (h, p),
 241                     task_name,
 242                     CoastlineBuildTask(h, p, self.archive_format),
 243                     deps)
 244
 245                 ret[p][h] = task_name
 246
 247         return {
 248             'harbour_tasks' : ret
 249         }