tools/bass/makefiles.py

   1 #
   2 # CDDL HEADER START
   3 #
   4 # The contents of this file are subject to the terms of the
   5 # Common Development and Distribution License (the "License").
   6 # You may not use this file except in compliance with the License.
   7 #
   8 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9 # or http://www.opensolaris.org/os/licensing.
  10 # See the License for the specific language governing permissions
  11 # and limitations under the License.
  12 #
  13 # When distributing Covered Code, include this CDDL HEADER in each
  14 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15 # If applicable, add the following below this CDDL HEADER, with the
  16 # fields enclosed by brackets "[]" replaced with your own identifying
  17 # information: Portions Copyright [yyyy] [name of copyright owner]
  18 #
  19 # CDDL HEADER END
  20 #
  21 # This is all very naive and will hurt pythonists' eyes.
  22 #
  23
  24 import json
  25 import os
  26 import re
  27 import subprocess
  28 import warnings
  29 from urllib.request import urlopen
  30
  31 from .component import Component
  32
  33
  34 class Keywords(object):
  35     def __init__(self):
  36         self.variables = {
  37             "BUILD_BITS":
  38             ["NO_ARCH",
  39              "32",
  40              "64",
  41              "32_and_64",
  42              "64_and_32"],
  43             "BUILD_STYLE":
  44                 ["ant",
  45                  "attpackagemake",
  46                  "cmake",
  47                  "configure",
  48                  "gem",
  49                  "justmake",
  50                  "makemaker",
  51                  "meson",
  52                  "ocaml",
  53                  "setup.py",
  54                  "waf"],
  55             "MK_BITS":
  56             ["NO_ARCH",
  57              "32",
  58              "64",
  59              "32_and_64"],
  60             "COMPONENT_NAME": [],
  61             "COMPONENT_VERSION": [],
  62             "COMPONENT_REVISION": [],
  63             "COMPONENT_FMRI": [],
  64             "COMPONENT_CLASSIFICATION": [],
  65             "COMPONENT_SUMMARY": [],
  66             "COMPONENT_PROJECT_URL": [],
  67             "COMPONENT_SRC": ["$(COMPONENT_NAME)-$(COMPONENT_VERSION)"],
  68             "COMPONENT_ARCHIVE": [],
  69             "COMPONENT_ARCHIVE_URL": [],
  70             "COMPONENT_ARCHIVE_HASH": [],
  71             "COMPONENT_LICENSE": [],
  72             "COMPONENT_LICENSE_FILE": []
  73         }
  74         self.targets = {
  75             "build": [ "BUILD_$(MK_BITS)"],
  76             "install": ["INSTALL_$(MK_BITS)"],
  77             "test": ["TEST_$(MK_BITS)", "NO_TESTS"],
  78             "system-test": ["SYSTEM_TEST_$(MK_BITS)", "SYSTEM_TESTS_NOT_IMPLEMENTED"]
  79         }
  80
  81     @staticmethod
  82     def assignment(name, value):
  83         return name + "=" + value + "\n"
  84
  85     @staticmethod
  86     def target_variable_assignment(name, value):
  87         return Keywords.assignment(name.upper()+"_TARGET", value)
  88
  89
  90 class Item(object):
  91     def __init__(self, line=None, content=[]):
  92         self.idx = line
  93         self.str = content
  94         for l in iter(self.str):
  95             l = l.strip()
  96
  97     def append(self, line):
  98         self.str.append(line.strip())
  99
 100     def extend(self, content):
 101         for l in content:
 102             self.append(l)
 103
 104     def line(self):
 105         return self.idx
 106
 107     def length(self):
 108         return len(self.str)
 109
 110     def value(self):
 111         return "".join(self.str).replace("\n","").strip()
 112
 113     def set_value(self, value):
 114         self.str = [ i.strip()+"\n" for i in value.split("\n") ]
 115
 116     def include_line(self):
 117         return "include "+self.str[0]
 118
 119     def variable_assignment(self, variable):
 120         if self.length() == 1:
 121             return ["{0:<24}{1}".format(variable+"=",self.str[0])]
 122         # Handle continuation lines
 123         lines = ["{0:<24}{1}".format(variable+"=",self.str[0])]
 124         for l in self.str[1:]:
 125             lines[-1] += "\\\n"
 126             lines.append("\t"+l)
 127         lines[-1] += "\n"
 128         return lines
 129
 130     def target_definition(self, target):
 131         lines = ["{0:<24}{1}".format(target+":", self.str[0])]
 132         for l in self.str[1:]:
 133             lines.append("\t"+l)
 134         return lines
 135
 136
 137 class Makefile(object):
 138     def __init__(self, path=None, debug=False):
 139         self.debug = debug
 140         self.path = path
 141         self.component = Component()
 142         self.includes = []
 143         self.variables = {}
 144         self.targets = {}
 145         makefile = os.path.join(path, 'Makefile')
 146         with open(makefile, 'r') as f:
 147             self.contents = f.readlines()
 148         self.update()
 149
 150     def update(self, contents=None):
 151         self.includes = []
 152         self.variables = {}
 153         self.targets = {}
 154         if contents is not None:
 155             self.contents = contents
 156         # Construct list of keywords
 157         kw = Keywords()
 158
 159         # Variable is set
 160         m = None
 161         # Target is set
 162         t = None
 163         # Rule definition
 164         d = None
 165         for idx, line in enumerate(self.contents):
 166             # Continuation of target line
 167             if t is not None:
 168                 r = re.match(r"^[\s]*(.*)[\s]*([\\]?)[\s]*$", line)
 169                 # Concatenate
 170                 self.targets[t].str[0] += "\\".join(r.group(1))
 171                 # Check for continuation or move to definition
 172                 if not r.group(2):
 173                     d = t
 174                     t = None
 175                 continue
 176             if d is not None:
 177                 # Concatenate
 178                 r = re.match(r"^[\t][\s]*(.*)[\s]*$", line)
 179                 # End of definition
 180                 if r is None:
 181                     d = None
 182                     continue
 183                 self.targets[d].append(r.group(1))
 184             # Continuation line of variable
 185             if m is not None:
 186                 r = re.match(r"^[\s]*(.*)[\s]*([\\]?)[\s]*$", line)
 187                 self.variables[m].append(r.group(1))
 188                 if not r.group(2):
 189                     m = None
 190                 continue
 191             if re.match(r"^#", line):
 192                 continue
 193             # Check for include
 194             r = re.match(r"^include[\s]+(.*)", line)
 195             if r is not None:
 196                self.includes.append(Item(idx, [r.group(1)]))
 197             else:
 198                 found = False
 199                 # Collect known variables
 200                 for k in list(kw.variables.keys()):
 201                     r = re.match(
 202                         r"^[\s]*("+k+r")[\s]*=[\s]*([^\\]*)[\s]*([\\]?)[\s]*$", line)
 203                     if r is not None:
 204                         found = True
 205                         v = r.group(2)
 206                         if v in self.variables.keys():
 207                             warnings.warn("Variable '"+v+"' redefined line "+idx)
 208                         self.variables[k] = Item(idx, [v])
 209                         if r.group(3):
 210                             m = k
 211                         break
 212                 if found is True:
 213                     continue
 214                 # Collect known targets
 215                 for k in list(kw.targets.keys()):
 216                     r = re.match(
 217                         "^"+k+r"[\s]*:[\s]*(.*)[\s]*([\\]?)[\s]*$", line)
 218                     if r is not None:
 219                         found = True
 220                         v = r.group(1)
 221                         if v in self.targets.keys():
 222                             warnings.warn("Target '"+v+"' redefined line "+idx)
 223                         self.targets[k] = Item(idx, [v])
 224                         if r.group(2):
 225                             t = k
 226                             d = None
 227                         else:
 228                             t = None
 229                             d = k
 230                         break
 231                 if found is True:
 232                     continue
 233
 234     def write(self):
 235         with open(os.path.join(self.path, "Makefile"), 'w') as f:
 236             for line in self.contents:
 237                 f.write(line)
 238
 239     def display(self):
 240         print(self.path)
 241         print('-' * 78)
 242         if self.includes:
 243             print("includes:")
 244             print("---------")
 245             for i in iter(self.includes):
 246                 print("{0:>3}: {1}".format(i.line(), i.value()))
 247             print("")
 248         if self.variables:
 249             print("variables:")
 250             print("----------")
 251             for k,i in iter(sorted(self.variables.items())):
 252                 print("{0:>3}: {1:<24}= {2}".format(i.line(), k, i.value()))
 253             print("")
 254         if self.targets:
 255             print("targets:")
 256             print("--------")
 257             for k,i in iter(sorted(self.targets.items())):
 258                 print("{0:>3}: {1:<24}= {2}".format(i.line(), k, i.value()))
 259             print("")
 260         print('-' * 78)
 261
 262     def run(self, targets):
 263         path = self.path
 264         result = []
 265
 266         if self.debug:
 267             logger.debug('Executing \'gmake %s\' in %s', targets, path)
 268
 269         proc = subprocess.Popen(['gmake', '-s', targets],
 270                                 stdout=subprocess.PIPE,
 271                                 stderr=subprocess.PIPE,
 272                                 cwd=path,
 273                                 universal_newlines=True)
 274         stdout, stderr = proc.communicate()
 275
 276         for out in stdout.splitlines():
 277             result.append(out.rstrip())
 278
 279         if self.debug:
 280             if proc.returncode != 0:
 281                 logger.debug('exit: %d, %s', proc.returncode, stderr)
 282
 283         return result
 284
 285     def print_value(self, name):
 286         return self.run('print-value-'+name)[0]
 287
 288     def build_style(self):
 289         return self.variables['BUILD_STYLE'].value()
 290
 291     def build_bits(self):
 292         return self.variables['BUILD_BITS'].value()
 293
 294     def has_mk_include(self, name):
 295         for i in iter(self.includes):
 296             if re.match('^.*/'+name+'.mk$', i.value()):
 297                 return True
 298         return False
 299
 300     def get_mk_include(self, name):
 301         for i in iter(self.includes):
 302             if re.match('^.*/'+name+'.mk$', i.value()):
 303                 return i
 304         return None
 305
 306     def has_variable(self, variable):
 307         return variable in self.variables
 308
 309     def variable(self, variable):
 310         return self.variables[variable]
 311
 312     def remove_variable(self, variable):
 313         idx = self.variable(variable).line()
 314         del self.contents[idx]
 315         self.update()
 316
 317     def set_variable(self, variable, value, line=None):
 318         if not self.has_variable(variable):
 319             self.variables[variable] = Item(None)
 320             self.variables[variable].set_value(str(value))
 321             if line is not None:
 322                 contents = self.contents[0:line]
 323                 contents.extend(self.variable_assignment(variable))
 324                 contents.extend(self.contents[line:])
 325                 self.update(contents)
 326             return True
 327         else:
 328             idx = self.variables[variable].line()
 329             oln = self.variables[variable].length()
 330             self.variables[variable].set_value(str(value))
 331             nln = self.variables[variable].length()
 332             if idx is not None:
 333                 if line is None:
 334                     line = idx
 335                 if line == idx and nln == 1 and oln == 1:
 336                     self.contents[idx] = self.variable_assignment(variable)[0]
 337                 elif line <= idx:
 338                     contents = self.contents[0:line]
 339                     contents.extend(self.variable_assignment(variable))
 340                     contents.extend(self.contents[line:idx])
 341                     contents.extend(self.contents[idx+oln:])
 342                     self.update(contents)
 343                 else:
 344                     contents = self.contents[0:idx]
 345                     contents.extend(self.contents[idx+oln:line])
 346                     contents.extend(self.variable_assignment(variable))
 347                     contents.extend(self.contents[line:])
 348                     self.update(contents)
 349             # Add variable at given line
 350             elif line is not None:
 351                 contents = self.contents[0:line]
 352                 contents.extend(self.variable_assignment(variable))
 353                 contents.extend(self.contents[line:])
 354                 self.update(contents)
 355
 356     def set_archive_hash(self, checksum):
 357         self.set_variable('COMPONENT_ARCHIVE_HASH', "sha256:"+str(checksum))
 358
 359     def variable_assignment(self, variable):
 360         return self.variables[variable].variable_assignment(variable)
 361
 362     def has_target(self, target):
 363         return target in self.targets
 364
 365     def target(self, target):
 366         return self.targets[target]
 367
 368     def target_definition(self, target):
 369         return self.targets[target].target_definition(target)
 370
 371     def required_packages(self):
 372         return self.run("print-value-REQUIRED_PACKAGES")[0]
 373
 374     def uses_pypi(self):
 375         # Default build style is configure
 376         if not self.has_variable('BUILD_STYLE'):
 377             return False
 378         is_py = (self.build_style() == 'setup.py')
 379         urlnone = (not self.has_variable('COMPONENT_ARCHIVE_URL'))
 380         urlpipy = urlnone or (self.variable('COMPONENT_ARCHIVE_URL').value() == '$(call pypi_url)')
 381         return is_py and urlpipy
 382
 383     def get_pypi_data(self):
 384         name = self.print_value('COMPONENT_PYPI')
 385         jsurl = "https://pypi.python.org/pypi/%s/json" % name
 386         try:
 387             f = urlopen(jsurl, data=None)
 388         except HTTPError as e:
 389             if e.getcode() == 404:
 390                 print("Unknown component '%s'" % name)
 391             else:
 392                 printIOError(e, "Can't open PyPI JSON url %s" % jsurl)
 393             return None
 394         except IOError as e:
 395             printIOError(e, "Can't open PyPI JSON url %s" % jsurl)
 396             return None
 397         content = f.read().decode("utf-8")
 398         return json.loads(content)
 399
 400     @staticmethod
 401     def value(variable):
 402         return "$("+variable+")"
 403
 404     @staticmethod
 405     def directory_variable(subdir):
 406         return Makefile.value("WS_"+subdir.upper().replace("-","_"))
 407
 408     @staticmethod
 409     def makefile_path(name):
 410         return os.path.join("$(WS_MAKE_RULES)", name+".mk")
 411
 412     @staticmethod
 413     def target_value(name, bits):
 414         return Makefile.value(name.upper()+"_"+bits)
 415