tools/bass/makefiles.py

   1 #!/usr/bin/python3.5
   2 #
   3 # CDDL HEADER START
   4 #
   5 # The contents of this file are subject to the terms of the
   6 # Common Development and Distribution License (the "License").
   7 # You may not use this file except in compliance with the License.
   8 #
   9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  10 # or http://www.opensolaris.org/os/licensing.
  11 # See the License for the specific language governing permissions
  12 # and limitations under the License.
  13 #
  14 # When distributing Covered Code, include this CDDL HEADER in each
  15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  16 # If applicable, add the following below this CDDL HEADER, with the
  17 # fields enclosed by brackets "[]" replaced with your own identifying
  18 # information: Portions Copyright [yyyy] [name of copyright owner]
  19 #
  20 # CDDL HEADER END
  21 #
  22 # This is all very naive and will hurt pythonists' eyes.
  23 #
  24
  25 import json
  26 import os
  27 import re
  28 import subprocess
  29 import warnings
  30 from urllib.request import urlopen
  31
  32 from .component import Component
  33
  34
  35 class Keywords(object):
  36     def __init__(self):
  37         self.variables = {
  38             "BUILD_BITS":
  39             ["NO_ARCH",
  40              "32",
  41              "64",
  42              "32_and_64",
  43              "64_and_32"],
  44             "BUILD_STYLE":
  45                 ["ant",
  46                  "attpackagemake",
  47                  "cmake",
  48                  "configure",
  49                  "gem",
  50                  "justmake",
  51                  "makemaker",
  52                  "meson",
  53                  "ocaml",
  54                  "setup.py",
  55                  "waf"],
  56             "MK_BITS":
  57             ["NO_ARCH",
  58              "32",
  59              "64",
  60              "32_and_64"],
  61             "COMPONENT_NAME": [],
  62             "COMPONENT_VERSION": [],
  63             "COMPONENT_REVISION": [],
  64             "COMPONENT_FMRI": [],
  65             "COMPONENT_CLASSIFICATION": [],
  66             "COMPONENT_SUMMARY": [],
  67             "COMPONENT_PROJECT_URL": [],
  68             "COMPONENT_SRC": ["$(COMPONENT_NAME)-$(COMPONENT_VERSION)"],
  69             "COMPONENT_ARCHIVE": [],
  70             "COMPONENT_ARCHIVE_URL": [],
  71             "COMPONENT_ARCHIVE_HASH": [],
  72             "COMPONENT_LICENSE": [],
  73             "COMPONENT_LICENSE_FILE": []
  74         }
  75         self.targets = {
  76             "build": [ "BUILD_$(MK_BITS)"],
  77             "install": ["INSTALL_$(MK_BITS)"],
  78             "test": ["TEST_$(MK_BITS)", "NO_TESTS"],
  79             "system-test": ["SYSTEM_TEST_$(MK_BITS)", "SYSTEM_TESTS_NOT_IMPLEMENTED"]
  80         }
  81
  82     @staticmethod
  83     def assignment(name, value):
  84         return name + "=" + value + "\n"
  85
  86     @staticmethod
  87     def target_variable_assignment(name, value):
  88         return Keywords.assignment(name.upper()+"_TARGET", value)
  89
  90
  91 class Item(object):
  92     def __init__(self, line=None, content=[]):
  93         self.idx = line
  94         self.str = content
  95         for l in iter(self.str):
  96             l = l.strip()
  97
  98     def append(self, line):
  99         self.str.append(line.strip())
 100
 101     def extend(self, content):
 102         for l in content:
 103             self.append(l)
 104
 105     def line(self):
 106         return self.idx
 107
 108     def length(self):
 109         return len(self.str)
 110
 111     def value(self):
 112         return "".join(self.str).replace("\n","").strip()
 113
 114     def set_value(self, value):
 115         self.str = [ i.strip()+"\n" for i in value.split("\n") ]
 116
 117     def include_line(self):
 118         return "include "+self.str[0]
 119
 120     def variable_assignment(self, variable):
 121         if self.length() == 1:
 122             return ["{0:<24}{1}".format(variable+"=",self.str[0])]
 123         # Handle continuation lines
 124         lines = ["{0:<24}{1}".format(variable+"=",self.str[0])]
 125         for l in self.str[1:]:
 126             lines[-1] += "\\\n"
 127             lines.append("\t"+l)
 128         lines[-1] += "\n"
 129         return lines
 130
 131     def target_definition(self, target):
 132         lines = ["{0:<24}{1}".format(target+":", self.str[0])]
 133         for l in self.str[1:]:
 134             lines.append("\t"+l)
 135         return lines
 136
 137
 138 class Makefile(object):
 139     def __init__(self, path=None, debug=False):
 140         self.debug = debug
 141         self.path = path
 142         self.component = Component()
 143         self.includes = []
 144         self.variables = {}
 145         self.targets = {}
 146         makefile = os.path.join(path, 'Makefile')
 147         with open(makefile, 'r') as f:
 148             self.contents = f.readlines()
 149         self.update()
 150
 151     def update(self, contents=None):
 152         self.includes = []
 153         self.variables = {}
 154         self.targets = {}
 155         if contents is not None:
 156             self.contents = contents
 157         # Construct list of keywords
 158         kw = Keywords()
 159
 160         # Variable is set
 161         m = None
 162         # Target is set
 163         t = None
 164         # Rule definition
 165         d = None
 166         for idx, line in enumerate(self.contents):
 167             # Continuation of target line
 168             if t is not None:
 169                 r = re.match(r"^[\s]*(.*)[\s]*([\\]?)[\s]*$", line)
 170                 # Concatenate
 171                 self.targets[t].str[0] += "\\".join(r.group(1))
 172                 # Check for continuation or move to definition
 173                 if not r.group(2):
 174                     d = t
 175                     t = None
 176                 continue
 177             if d is not None:
 178                 # Concatenate
 179                 r = re.match(r"^[\t][\s]*(.*)[\s]*$", line)
 180                 # End of definition
 181                 if r is None:
 182                     d = None
 183                     continue
 184                 self.targets[d].append(r.group(1))
 185             # Continuation line of variable
 186             if m is not None:
 187                 r = re.match(r"^[\s]*(.*)[\s]*([\\]?)[\s]*$", line)
 188                 self.variables[m].append(r.group(1))
 189                 if not r.group(2):
 190                     m = None
 191                 continue
 192             if re.match(r"^#", line):
 193                 continue
 194             # Check for include
 195             r = re.match(r"^include[\s]+(.*)", line)
 196             if r is not None:
 197                self.includes.append(Item(idx, [r.group(1)]))
 198             else:
 199                 found = False
 200                 # Collect known variables
 201                 for k in list(kw.variables.keys()):
 202                     r = re.match(
 203                         r"^[\s]*("+k+r")[\s]*=[\s]*([^\\]*)[\s]*([\\]?)[\s]*$", line)
 204                     if r is not None:
 205                         found = True
 206                         v = r.group(2)
 207                         if v in self.variables.keys():
 208                             warnings.warn("Variable '"+v+"' redefined line "+idx)
 209                         self.variables[k] = Item(idx, [v])
 210                         if r.group(3):
 211                             m = k
 212                         break
 213                 if found is True:
 214                     continue
 215                 # Collect known targets
 216                 for k in list(kw.targets.keys()):
 217                     r = re.match(
 218                         "^"+k+r"[\s]*:[\s]*(.*)[\s]*([\\]?)[\s]*$", line)
 219                     if r is not None:
 220                         found = True
 221                         v = r.group(1)
 222                         if v in self.targets.keys():
 223                             warnings.warn("Target '"+v+"' redefined line "+idx)
 224                         self.targets[k] = Item(idx, [v])
 225                         if r.group(2):
 226                             t = k
 227                             d = None
 228                         else:
 229                             t = None
 230                             d = k
 231                         break
 232                 if found is True:
 233                     continue
 234
 235     def write(self):
 236         with open(os.path.join(self.path, "Makefile"), 'w') as f:
 237             for line in self.contents:
 238                 f.write(line)
 239
 240     def display(self):
 241         print(self.path)
 242         print('-' * 78)
 243         if self.includes:
 244             print("includes:")
 245             print("---------")
 246             for i in iter(self.includes):
 247                 print("{0:>3}: {1}".format(i.line(), i.value()))
 248             print("")
 249         if self.variables:
 250             print("variables:")
 251             print("----------")
 252             for k,i in iter(sorted(self.variables.items())):
 253                 print("{0:>3}: {1:<24}= {2}".format(i.line(), k, i.value()))
 254             print("")
 255         if self.targets:
 256             print("targets:")
 257             print("--------")
 258             for k,i in iter(sorted(self.targets.items())):
 259                 print("{0:>3}: {1:<24}= {2}".format(i.line(), k, i.value()))
 260             print("")
 261         print('-' * 78)
 262
 263     def run(self, targets):
 264         path = self.path
 265         result = []
 266
 267         if self.debug:
 268             logger.debug('Executing \'gmake %s\' in %s', targets, path)
 269
 270         proc = subprocess.Popen(['gmake', '-s', targets],
 271                                 stdout=subprocess.PIPE,
 272                                 stderr=subprocess.PIPE,
 273                                 cwd=path,
 274                                 universal_newlines=True)
 275         stdout, stderr = proc.communicate()
 276
 277         for out in stdout.splitlines():
 278             result.append(out.rstrip())
 279
 280         if self.debug:
 281             if proc.returncode != 0:
 282                 logger.debug('exit: %d, %s', proc.returncode, stderr)
 283
 284         return result
 285
 286     def print_value(self, name):
 287         return self.run('print-value-'+name)[0]
 288
 289     def build_style(self):
 290         return self.variables['BUILD_STYLE'].value()
 291
 292     def build_bits(self):
 293         return self.variables['BUILD_BITS'].value()
 294
 295     def has_mk_include(self, name):
 296         for i in iter(self.includes):
 297             if re.match('^.*/'+name+'.mk$', i.value()):
 298                 return True
 299         return False
 300
 301     def get_mk_include(self, name):
 302         for i in iter(self.includes):
 303             if re.match('^.*/'+name+'.mk$', i.value()):
 304                 return i
 305         return None
 306
 307     def has_variable(self, variable):
 308         return variable in self.variables
 309
 310     def variable(self, variable):
 311         return self.variables[variable]
 312
 313     def remove_variable(self, variable):
 314         idx = self.variable(variable).line()
 315         del self.contents[idx]
 316         self.update()
 317
 318     def set_variable(self, variable, value, line=None):
 319         if not self.has_variable(variable):
 320             self.variables[variable] = Item(None)
 321             self.variables[variable].set_value(str(value))
 322             if line is not None:
 323                 contents = self.contents[0:line]
 324                 contents.extend(self.variable_assignment(variable))
 325                 contents.extend(self.contents[line:])
 326                 self.update(contents)
 327             return True
 328         else:
 329             idx = self.variables[variable].line()
 330             oln = self.variables[variable].length()
 331             self.variables[variable].set_value(str(value))
 332             nln = self.variables[variable].length()
 333             if idx is not None:
 334                 if line is None:
 335                     line = idx
 336                 if line == idx and nln == 1 and oln == 1:
 337                     self.contents[idx] = self.variable_assignment(variable)[0]
 338                 elif line <= idx:
 339                     contents = self.contents[0:line]
 340                     contents.extend(self.variable_assignment(variable))
 341                     contents.extend(self.contents[line:idx])
 342                     contents.extend(self.contents[idx+oln:])
 343                     self.update(contents)
 344                 else:
 345                     contents = self.contents[0:idx]
 346                     contents.extend(self.contents[idx+oln:line])
 347                     contents.extend(self.variable_assignment(variable))
 348                     contents.extend(self.contents[line:])
 349                     self.update(contents)
 350             # Add variable at given line
 351             elif line is not None:
 352                 contents = self.contents[0:line]
 353                 contents.extend(self.variable_assignment(variable))
 354                 contents.extend(self.contents[line:])
 355                 self.update(contents)
 356
 357     def set_archive_hash(self, checksum):
 358         self.set_variable('COMPONENT_ARCHIVE_HASH', "sha256:"+str(checksum))
 359
 360     def variable_assignment(self, variable):
 361         return self.variables[variable].variable_assignment(variable)
 362
 363     def has_target(self, target):
 364         return target in self.targets
 365
 366     def target(self, target):
 367         return self.targets[target]
 368
 369     def target_definition(self, target):
 370         return self.targets[target].target_definition(target)
 371
 372     def required_packages(self):
 373         return self.run("print-value-REQUIRED_PACKAGES")[0]
 374
 375     def uses_pypi(self):
 376         # Default build style is configure
 377         if not self.has_variable('BUILD_STYLE'):
 378             return False
 379         is_py = (self.build_style() == 'setup.py')
 380         urlnone = (not self.has_variable('COMPONENT_ARCHIVE_URL'))
 381         urlpipy = urlnone or (self.variable('COMPONENT_ARCHIVE_URL').value() == '$(call pypi_url)')
 382         return is_py and urlpipy
 383
 384     def get_pypi_data(self):
 385         name = self.print_value('COMPONENT_PYPI')
 386         jsurl = "https://pypi.python.org/pypi/%s/json" % name
 387         try:
 388             f = urlopen(jsurl, data=None)
 389         except HTTPError as e:
 390             if e.getcode() == 404:
 391                 print("Unknown component '%s'" % name)
 392             else:
 393                 printIOError(e, "Can't open PyPI JSON url %s" % jsurl)
 394             return None
 395         except IOError as e:
 396             printIOError(e, "Can't open PyPI JSON url %s" % jsurl)
 397             return None
 398         content = f.read().decode("utf-8")
 399         return json.loads(content)
 400
 401     @staticmethod
 402     def value(variable):
 403         return "$("+variable+")"
 404
 405     @staticmethod
 406     def directory_variable(subdir):
 407         return Makefile.value("WS_"+subdir.upper().replace("-","_"))
 408
 409     @staticmethod
 410     def makefile_path(name):
 411         return os.path.join("$(WS_MAKE_RULES)", name+".mk")
 412
 413     @staticmethod
 414     def target_value(name, bits):
 415         return Makefile.value(name.upper()+"_"+bits)
 416