pkgs/build-support/templaterpm/nix-template-rpm.py

   1 #!/bin/env python
   2
   3 import sys
   4 import os
   5 import subprocess
   6 import argparse
   7 import re
   8 import shutil
   9 import rpm
  10 import urlparse
  11 import traceback
  12 import toposort
  13
  14
  15
  16
  17
  18 class SPECTemplate(object):
  19   def __init__(self, specFilename, outputDir, inputDir=None, buildRootInclude=None, translateTable=None, repositoryDir=None, allPackagesDir=None, maintainer="MAINTAINER"):
  20     rpm.addMacro("buildroot","$out")
  21     rpm.addMacro("_libdir","lib")
  22     rpm.addMacro("_libexecdir","libexec")
  23     rpm.addMacro("_sbindir","sbin")
  24     rpm.addMacro("_sysconfdir","etc")
  25     rpm.addMacro("_topdir","SPACER_DIR_FOR_REMOVAL")
  26     rpm.addMacro("_sourcedir","SOURCE_DIR_SPACER")
  27
  28     self.packageGroups = [ "ocaml", "python" ]
  29
  30     ts = rpm.TransactionSet()
  31
  32     self.specFilename = specFilename
  33     self.spec = ts.parseSpec(specFilename)
  34
  35     self.inputDir = inputDir
  36     self.buildRootInclude = buildRootInclude
  37     self.repositoryDir = repositoryDir
  38     self.allPackagesDir = allPackagesDir
  39     self.maintainer = maintainer
  40
  41     self.translateTable = translateTable
  42
  43     self.facts = self.getFacts()
  44     self.key = self.getSelfKey()
  45
  46     tmpDir = os.path.join(outputDir, self.rewriteName(self.spec.sourceHeader['name']))
  47     if self.translateTable is not None:
  48       self.relOutputDir = self.translateTable.path(self.key,tmpDir)
  49     else:
  50       self.relOutputDir = tmpDir
  51
  52     self.final_output_dir = os.path.normpath( self.relOutputDir )
  53
  54     if self.repositoryDir is not None:
  55       self.potential_repository_dir = os.path.normpath( os.path.join(self.repositoryDir,self.relOutputDir) )
  56
  57
  58
  59   def rewriteCommands(self, string):
  60     string = string.replace('SPACER_DIR_FOR_REMOVAL/','')
  61     string = string.replace('SPACER_DIR_FOR_REMOVAL','')
  62     string = '\n'.join(map(lambda line: ' '.join(map(lambda x: x.replace('SOURCE_DIR_SPACER/',('${./' if (self.buildRootInclude is None) else '${buildRoot}/usr/share/buildroot/SOURCES/'))+('}' if (self.buildRootInclude is None) else '') if x.startswith('SOURCE_DIR_SPACER/') else x, line.split(' '))), string.split('\n')))
  63     string = string.replace('\n','\n    ')
  64     string = string.rstrip()
  65     return string
  66
  67
  68   def rewriteName(self, string):
  69     parts = string.split('-')
  70     parts = filter(lambda x: not x == "devel", parts)
  71     parts = filter(lambda x: not x == "doc", parts)
  72     if len(parts) > 1 and parts[0] in self.packageGroups:
  73       return parts[0] + '-' + ''.join(parts[1:2] + map(lambda x: x.capitalize(), parts[2:]))
  74     else:
  75       return ''.join(parts[:1] + map(lambda x: x.capitalize(), parts[1:]))
  76
  77
  78   def rewriteInputs(self,target,inputs):
  79     camelcase = lambda l: l[:1] + map(lambda x: x.capitalize(), l[1:])
  80     filterDevel = lambda l: filter(lambda x: not x == "devel", l)
  81     filterDoc = lambda l: filter(lambda x: not x == "doc", l)
  82     rewrite = lambda l: ''.join(camelcase(filterDoc(filterDevel(l))))
  83
  84     def filterPackageGroup(target):
  85       if target is None:
  86         return [ rewrite(x.split('-')) for x in inputs if (not x.split('-')[0] in self.packageGroups) or (len(x.split('-')) == 1) ]
  87       elif target in self.packageGroups:
  88         return [ target + '_' + rewrite(x.split('-')[1:]) for x in inputs if (x.split('-')[0] == target) and (len(x.split('-')) > 1)]
  89       else:
  90         raise Exception("Unknown target")
  91         return []
  92
  93     if target is None:
  94       packages = filterPackageGroup(None)
  95       packages.sort()
  96     elif target in self.packageGroups:
  97       packages = filterPackageGroup(target)
  98       packages.sort()
  99     elif target == "ALL":
 100       packages = []
 101       for t in [None] + self.packageGroups:
 102         tmp = filterPackageGroup(t)
 103         tmp.sort()
 104         packages += tmp
 105     else:
 106       raise Exception("Unknown target")
 107       packages = []
 108
 109     return packages
 110
 111
 112   def getBuildInputs(self,target=None):
 113     inputs = self.rewriteInputs(target,self.spec.sourceHeader['requires'])
 114     if self.translateTable is not None:
 115       return map(lambda x: self.translateTable.name(x), inputs)
 116     else:
 117       return inputs
 118
 119   def getSelfKey(self):
 120     name = self.spec.sourceHeader['name']
 121     if len(name.split('-')) > 1 and name.split('-')[0] in self.packageGroups:
 122       key = self.rewriteInputs(name.split('-')[0], [self.spec.sourceHeader['name']])[0]
 123     else:
 124       key = self.rewriteInputs(None, [self.spec.sourceHeader['name']])[0]
 125     return key
 126
 127   def getSelf(self):
 128     if self.translateTable is not None:
 129       return self.translateTable.name(self.key)
 130     else:
 131       return self.key
 132
 133
 134
 135
 136   def copyPatches(self, input_dir, output_dir):
 137     patches = [source for (source, _, flag) in self.spec.sources if flag==2]
 138     for filename in patches:
 139       shutil.copyfile(os.path.join(input_dir, filename), os.path.join(output_dir, filename))
 140
 141
 142   def copySources(self, input_dir, output_dir):
 143     filenames = [source for (source, _, flag) in self.spec.sources if flag==1 if not urlparse.urlparse(source).scheme in ["http", "https"] ]
 144     for filename in filenames:
 145       shutil.copyfile(os.path.join(input_dir, filename), os.path.join(output_dir, filename))
 146
 147
 148   def getFacts(self):
 149     facts = {}
 150     facts["name"] = self.rewriteName(self.spec.sourceHeader['name'])
 151     facts["version"] = self.spec.sourceHeader['version']
 152
 153     facts["url"] = []
 154     facts["sha256"] = []
 155     sources = [source for (source, _, flag) in self.spec.sources if flag==1 if urlparse.urlparse(source).scheme in ["http", "https"] ]
 156     for url in sources:
 157       p = subprocess.Popen(['nix-prefetch-url', url], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 158       output, err = p.communicate()
 159       sha256 = output[:-1] #remove new line
 160       facts["url"].append(url)
 161       facts["sha256"].append(sha256)
 162
 163     patches = [source for (source, _, flag) in self.spec.sources if flag==2]
 164     if self.buildRootInclude is None:
 165       facts["patches"] = map(lambda x: './'+x, patches)
 166     else:
 167       facts["patches"] = map(lambda x: '"${buildRoot}/usr/share/buildroot/SOURCES/'+x+'"', reversed(patches))
 168
 169     return facts
 170
 171
 172   @property
 173   def name(self):
 174     out = '  name = "' + self.facts["name"] + '-' + self.facts["version"] + '";\n'
 175     out += '  version = "' + self.facts['version'] + '";\n'
 176     return out
 177
 178
 179   @property
 180   def src(self):
 181     sources = [source for (source, _, flag) in self.spec.sources if flag==1 if urlparse.urlparse(source).scheme in ["http", "https"] ]
 182     out = ''
 183     for (url,sha256) in zip(self.facts['url'],self.facts['sha256']):
 184       out += '  src = fetchurl {\n'
 185       out += '    url = "' + url + '";\n'
 186       out += '    sha256 = "' + sha256 + '";\n'
 187       out += '  };\n'
 188     return out
 189
 190
 191   @property
 192   def patch(self):
 193     out = '  patches = [ ' + ' '.join(self.facts['patches']) + ' ];\n'
 194     return out
 195
 196
 197   @property
 198   def buildInputs(self):
 199     out = '  buildInputs = [ '
 200     out += ' '.join(self.getBuildInputs("ALL"))
 201     out += ' ];\n'
 202     return out
 203
 204
 205   @property
 206   def configure(self):
 207     out = '  configurePhase = \'\'\n    ' + self.rewriteCommands(self.spec.prep) + '\n    \'\';\n';
 208     return out
 209
 210
 211   @property
 212   def build(self):
 213     out = '  buildPhase = \'\'\n    ' + self.rewriteCommands(self.spec.build) + '\n    \'\';\n';
 214     return out
 215
 216
 217   @property
 218   def install(self):
 219     out = '  installPhase = \'\'\n    ' + self.rewriteCommands(self.spec.install) + '\n    \'\';\n';
 220     return out
 221
 222   @property
 223   def ocamlExtra(self):
 224     if "ocaml" in self.getBuildInputs("ALL"):
 225       return '  createFindlibDestdir = true;\n'
 226     else:
 227       return ''
 228
 229
 230   @property
 231   def meta(self):
 232     out = '  meta = with lib; {\n'
 233     out += '    homepage = ' + self.spec.sourceHeader['url'] + ';\n'
 234     out += '    description = "' + self.spec.sourceHeader['summary'] + '";\n'
 235     out += '    license = lib.licenses.' + self.spec.sourceHeader['license'] + ';\n'
 236     out += '    platforms = [ "i686-linux" "x86_64-linux" ];\n'
 237     out += '    maintainers = with lib.maintainers; [ ' + self.maintainer + ' ];\n'
 238     out += '  };\n'
 239     out += '}\n'
 240     return out
 241
 242
 243   def __str__(self):
 244     head = '{lib, stdenv, fetchurl, ' + ', '.join(self.getBuildInputs("ALL")) + '}:\n\n'
 245     head += 'stdenv.mkDerivation {\n'
 246     body = [ self.name, self.src, self.patch, self.buildInputs, self.configure, self.build, self.ocamlExtra, self.install, self.meta ]
 247     return head + '\n'.join(body)
 248
 249
 250   def getTemplate(self):
 251     head = '{lib, stdenv, buildRoot, fetchurl, ' + ', '.join(self.getBuildInputs("ALL")) + '}:\n\n'
 252     head += 'let\n'
 253     head += '  buildRootInput = (import "${buildRoot}/usr/share/buildroot/buildRootInput.nix") { fetchurl=fetchurl; buildRoot=buildRoot; };\n'
 254     head += 'in\n\n'
 255     head += 'stdenv.mkDerivation {\n'
 256     head += '  inherit (buildRootInput.'+self.rewriteName(self.spec.sourceHeader['name'])+') name version src;\n'
 257     head += '  patches = buildRootInput.'+self.rewriteName(self.spec.sourceHeader['name'])+'.patches ++ [];\n\n'
 258     body = [ self.buildInputs, self.configure, self.build, self.ocamlExtra, self.install, self.meta ]
 259     return head + '\n'.join(body)
 260
 261
 262   def getInclude(self):
 263     head = self.rewriteName(self.spec.sourceHeader['name']) + ' = {\n'
 264     body = [ self.name, self.src, self.patch ]
 265     return head + '\n'.join(body) + '};\n'
 266
 267
 268   def __cmp__(self,other):
 269     if self.getSelf() in other.getBuildInputs("ALL"):
 270       return 1
 271     else:
 272       return -1
 273
 274
 275   def callPackage(self):
 276     callPackage = '  ' + self.getSelf() + ' = callPackage ' + os.path.relpath(self.final_output_dir, self.allPackagesDir) + ' {'
 277     newline = False;
 278     for target in self.packageGroups:
 279       tmp = self.getBuildInputs(target)
 280       if len(tmp) > 0:
 281         newline = True;
 282         callPackage += '\n    ' + 'inherit (' + target + 'Packages) ' + ' '.join(tmp) + ';'
 283     if newline:
 284       callPackage += '\n  };'
 285     else:
 286       callPackage += ' };'
 287     return callPackage
 288
 289
 290
 291   def generateCombined(self):
 292     if not os.path.exists(self.final_output_dir):
 293       os.makedirs(self.final_output_dir)
 294
 295     if self.inputDir is not None:
 296       self.copySources(self.inputDir, self.final_output_dir)
 297       self.copyPatches(self.inputDir, self.final_output_dir)
 298
 299     nixfile = open(os.path.join(self.final_output_dir,'default.nix'), 'w')
 300     nixfile.write(str(self))
 301     nixfile.close()
 302
 303     shutil.copyfile(self.specFilename, os.path.join(self.final_output_dir, os.path.basename(self.specFilename)))
 304
 305
 306
 307   def generateSplit(self):
 308     if not os.path.exists(self.final_output_dir):
 309       os.makedirs(self.final_output_dir)
 310
 311     nixfile = open(os.path.join(self.final_output_dir,'default.nix'), 'w')
 312     nixfile.write(self.getTemplate())
 313     nixfile.close()
 314
 315     return self.getInclude()
 316
 317
 318
 319
 320
 321
 322 class NixTemplate(object):
 323   def __init__(self, nixfile):
 324     self.nixfile = nixfile
 325     self.original = { "name":None, "version":None, "url":None, "sha256":None, "patches":None }
 326     self.update = { "name":None, "version":None, "url":None, "sha256":None, "patches":None }
 327     self.matchedLines = {}
 328
 329     if os.path.isfile(nixfile):
 330       with file(nixfile, 'r') as infile:
 331         for (n,line) in enumerate(infile):
 332           name = re.match(r'^\s*name\s*=\s*"(.*?)"\s*;\s*$', line)
 333           version = re.match(r'^\s*version\s*=\s*"(.*?)"\s*;\s*$', line)
 334           url = re.match(r'^\s*url\s*=\s*"?(.*?)"?\s*;\s*$', line)
 335           sha256 = re.match(r'^\s*sha256\s*=\s*"(.*?)"\s*;\s*$', line)
 336           patches = re.match(r'^\s*patches\s*=\s*(\[.*?\])\s*;\s*$', line)
 337           if name is not None and self.original["name"] is None:
 338               self.original["name"] = name.group(1)
 339               self.matchedLines[n] = "name"
 340           if version is not None and self.original["version"] is None:
 341               self.original["version"] = version.group(1)
 342               self.matchedLines[n] = "version"
 343           if url is not None and self.original["url"] is None:
 344               self.original["url"] = url.group(1)
 345               self.matchedLines[n] = "url"
 346           if sha256 is not None and self.original["sha256"] is None:
 347               self.original["sha256"] = sha256.group(1)
 348               self.matchedLines[n] = "sha256"
 349           if patches is not None and self.original["patches"] is None:
 350               self.original["patches"] = patches.group(1)
 351               self.matchedLines[n] = "patches"
 352
 353
 354   def generateUpdated(self, nixOut):
 355     nixTemplateFile = open(os.path.normpath(self.nixfile),'r')
 356     nixOutFile = open(os.path.normpath(nixOut),'w')
 357     for (n,line) in enumerate(nixTemplateFile):
 358       if self.matchedLines.has_key(n) and self.update[self.matchedLines[n]] is not None:
 359         nixOutFile.write(line.replace(self.original[self.matchedLines[n]], self.update[self.matchedLines[n]], 1))
 360       else:
 361         nixOutFile.write(line)
 362     nixTemplateFile.close()
 363     nixOutFile.close()
 364
 365
 366   def loadUpdate(self,orig):
 367     if orig.has_key("name") and orig.has_key("version"):
 368       self.update["name"] = orig["name"] + '-' + orig["version"]
 369       self.update["version"] = orig["version"]
 370     if orig.has_key("url") and orig.has_key("sha256") and len(orig["url"])>0:
 371       self.update["url"] = orig["url"][0]
 372       self.update["sha256"] = orig["sha256"][0]
 373       for url in orig["url"][1:-1]:
 374         sys.stderr.write("WARNING: URL has been dropped: %s\n" % url)
 375     if orig.has_key("patches"):
 376       self.update["patches"] = '[ ' + ' '.join(orig['patches']) + ' ]'
 377
 378
 379 class TranslationTable(object):
 380   def __init__(self):
 381     self.tablePath = {}
 382     self.tableName = {}
 383
 384   def update(self, key, path, name=None):
 385     self.tablePath[key] = path
 386     if name is not None:
 387       self.tableName[key] = name
 388
 389   def readTable(self, tableFile):
 390     with file(tableFile, 'r') as infile:
 391       for line in infile:
 392         match = re.match(r'^(.+?)\s+(.+?)\s+(.+?)\s*$', line)
 393         if match is not None:
 394           if not self.tablePath.has_key(match.group(1)):
 395             self.tablePath[match.group(1)] = match.group(2)
 396           if not self.tableName.has_key(match.group(1)):
 397             self.tableName[match.group(1)] = match.group(3)
 398         else:
 399           match = re.match(r'^(.+?)\s+(.+?)\s*$', line)
 400           if not self.tablePath.has_key(match.group(1)):
 401             self.tablePath[match.group(1)] = match.group(2)
 402
 403   def writeTable(self, tableFile):
 404     outFile = open(os.path.normpath(tableFile),'w')
 405     keys = self.tablePath.keys()
 406     keys.sort()
 407     for k in keys:
 408       if self.tableName.has_key(k):
 409         outFile.write( k + " " + self.tablePath[k] + " " + self.tableName[k] + "\n" )
 410       else:
 411         outFile.write( k + " " + self.tablePath[k] + "\n" )
 412     outFile.close()
 413
 414   def name(self, key):
 415    if self.tableName.has_key(key):
 416      return self.tableName[key]
 417    else:
 418      return key
 419
 420   def path(self, key, orig):
 421    if self.tablePath.has_key(key):
 422      return self.tablePath[key]
 423    else:
 424      return orig
 425
 426
 427
 428
 429
 430 if __name__ == "__main__":
 431     #Parse command line options
 432     parser = argparse.ArgumentParser(description="Generate .nix templates from RPM spec files")
 433     parser.add_argument("specs", metavar="SPEC", nargs="+", help="spec file")
 434     parser.add_argument("-o", "--output", metavar="OUT_DIR", required=True, help="output directory")
 435     parser.add_argument("-b", "--buildRoot", metavar="BUILDROOT_DIR", default=None, help="buildroot output directory")
 436     parser.add_argument("-i", "--inputSources", metavar="IN_DIR", default=None, help="sources input directory")
 437     parser.add_argument("-m", "--maintainer", metavar="MAINTAINER", default="__NIX_MAINTAINER__", help="package maintainer")
 438     parser.add_argument("-r", "--repository", metavar="REP_DIR", default=None, help="nix repository to compare output against")
 439     parser.add_argument("-t", "--translate", metavar="TRANSLATE_TABLE", default=None, help="path of translation table for name and path")
 440     parser.add_argument("-u", "--translateOut", metavar="TRANSLATE_OUT", default=None, help="output path for updated translation table")
 441     parser.add_argument("-a", "--allPackages", metavar="ALL_PACKAGES", default=None, help="top level dir to call packages from")
 442     args = parser.parse_args()
 443
 444     allPackagesDir = os.path.normpath( os.path.dirname(args.allPackages) )
 445     if not os.path.exists(allPackagesDir):
 446       os.makedirs(allPackagesDir)
 447
 448     buildRootContent = {}
 449     nameMap = {}
 450
 451     newTable = TranslationTable()
 452     if args.translate is not None:
 453       table = TranslationTable()
 454       table.readTable(args.translate)
 455       newTable.readTable(args.translate)
 456     else:
 457       table = None
 458
 459     for specPath in args.specs:
 460       try:
 461         sys.stderr.write("INFO: generate nix file from: %s\n" % specPath)
 462
 463         spec = SPECTemplate(specPath, args.output, args.inputSources, args.buildRoot, table, args.repository, allPackagesDir, args.maintainer)
 464         if args.repository is not None:
 465           if os.path.exists(os.path.join(spec.potential_repository_dir,'default.nix')):
 466             nixTemplate = NixTemplate(os.path.join(spec.potential_repository_dir,'default.nix'))
 467             nixTemplate.loadUpdate(spec.facts)
 468             if not os.path.exists(spec.final_output_dir):
 469               os.makedirs(spec.final_output_dir)
 470             nixTemplate.generateUpdated(os.path.join(spec.final_output_dir,'default.nix'))
 471           else:
 472             sys.stderr.write("WARNING: Repository does not contain template: %s\n" % os.path.join(spec.potential_repository_dir,'default.nix'))
 473             if args.buildRoot is None:
 474               spec.generateCombined()
 475             else:
 476               buildRootContent[spec.key] = spec.generateSplit()
 477         else:
 478           if args.buildRoot is None:
 479             spec.generateCombined()
 480           else:
 481             buildRootContent[spec.key] = spec.generateSplit()
 482
 483         newTable.update(spec.key,spec.relOutputDir,spec.getSelf())
 484         nameMap[spec.getSelf()] = spec
 485
 486       except Exception, e:
 487         sys.stderr.write("ERROR: %s failed with:\n%s\n%s\n" % (specPath,e.message,traceback.format_exc()))
 488
 489     if args.translateOut is not None:
 490       if not os.path.exists(os.path.dirname(os.path.normpath(args.translateOut))):
 491         os.makedirs(os.path.dirname(os.path.normpath(args.translateOut)))
 492       newTable.writeTable(args.translateOut)
 493
 494     graph = {}
 495     for k, v in nameMap.items():
 496       graph[k] = set(v.getBuildInputs("ALL"))
 497
 498     sortedSpecs = toposort.toposort_flatten(graph)
 499     sortedSpecs = filter( lambda x: x in nameMap.keys(), sortedSpecs)
 500
 501     allPackagesFile = open(os.path.normpath( args.allPackages ), 'w')
 502     allPackagesFile.write( '\n\n'.join(map(lambda x: x.callPackage(), map(lambda x: nameMap[x], sortedSpecs))) )
 503     allPackagesFile.close()
 504
 505     if args.buildRoot is not None:
 506       buildRootFilename = os.path.normpath( args.buildRoot )
 507       if not os.path.exists(os.path.dirname(buildRootFilename)):
 508         os.makedirs(os.path.dirname(buildRootFilename))
 509       buildRootFile = open(buildRootFilename, 'w')
 510       buildRootFile.write( "{ fetchurl, buildRoot }: {\n\n" )
 511       keys = buildRootContent.keys()
 512       keys.sort()
 513       for k in keys:
 514         buildRootFile.write( buildRootContent[k] + '\n' )
 515       buildRootFile.write( "}\n" )
 516       buildRootFile.close()
 517
 518