config/JarMaker.py

   1 # ***** BEGIN LICENSE BLOCK *****
   2 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   3 #
   4 # The contents of this file are subject to the Mozilla Public License Version
   5 # 1.1 (the "License"); you may not use this file except in compliance with
   6 # the License. You may obtain a copy of the License at
   7 # http://www.mozilla.org/MPL/
   8 #
   9 # Software distributed under the License is distributed on an "AS IS" basis,
  10 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  11 # for the specific language governing rights and limitations under the
  12 # License.
  13 #
  14 # The Original Code is Mozilla build system.
  15 #
  16 # The Initial Developer of the Original Code is
  17 # Mozilla Foundation.
  18 # Portions created by the Initial Developer are Copyright (C) 2008
  19 # the Initial Developer. All Rights Reserved.
  20 #
  21 # Contributor(s):
  22 #  Axel Hecht <l10n@mozilla.com>
  23 #
  24 # Alternatively, the contents of this file may be used under the terms of
  25 # either the GNU General Public License Version 2 or later (the "GPL"), or
  26 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  27 # in which case the provisions of the GPL or the LGPL are applicable instead
  28 # of those above. If you wish to allow use of your version of this file only
  29 # under the terms of either the GPL or the LGPL, and not to allow others to
  30 # use your version of this file under the terms of the MPL, indicate your
  31 # decision by deleting the provisions above and replace them with the notice
  32 # and other provisions required by the GPL or the LGPL. If you do not delete
  33 # the provisions above, a recipient may use your version of this file under
  34 # the terms of any one of the MPL, the GPL or the LGPL.
  35 #
  36 # ***** END LICENSE BLOCK *****
  37
  38 '''jarmaker.py provides a python class to package up chrome content by
  39 processing jar.mn files.
  40
  41 See the documentation for jar.mn on MDC for further details on the format.
  42 '''
  43
  44 import sys
  45 import os
  46 import os.path
  47 import re
  48 import logging
  49 from time import localtime
  50 from optparse import OptionParser
  51 from MozZipFile import ZipFile
  52 from cStringIO import StringIO
  53 from datetime import datetime
  54
  55 from utils import pushback_iter
  56 from Preprocessor import Preprocessor
  57
  58 __all__ = ['JarMaker']
  59
  60 class ZipEntry:
  61   '''Helper class for jar output.
  62
  63   This class defines a simple file-like object for a zipfile.ZipEntry
  64   so that we can consecutively write to it and then close it.
  65   This methods hooks into ZipFile.writestr on close().
  66   '''
  67   def __init__(self, name, zipfile):
  68     self._zipfile = zipfile
  69     self._name = name
  70     self._inner = StringIO()
  71
  72   def write(self, content):
  73     'Append the given content to this zip entry'
  74     self._inner.write(content)
  75     return
  76
  77   def close(self):
  78     'The close method writes the content back to the zip file.'
  79     self._zipfile.writestr(self._name, self._inner.getvalue())
  80
  81 def getModTime(aPath):
  82   if not os.path.isfile(aPath):
  83     return 0
  84   mtime = os.stat(aPath).st_mtime
  85   return localtime(mtime)
  86
  87
  88 class JarMaker(object):
  89   '''JarMaker reads jar.mn files and process those into jar files or
  90   flat directories, along with chrome.manifest files.
  91   '''
  92
  93   ignore = re.compile('\s*(\#.*)?$')
  94   jarline = re.compile('(?:(?P<jarfile>[\w\d.\-\_\\\/]+).jar\:)|(?:\s*(\#.*)?)\s*$')
  95   regline = re.compile('\%\s+(.*)$')
  96   entryre = '(?P<optPreprocess>\*)?(?P<optOverwrite>\+?)\s+'
  97   entryline = re.compile(entryre + '(?P<output>[\w\d.\-\_\\\/\+]+)\s*(\((?P<locale>\%?)(?P<source>[\w\d.\-\_\\\/]+)\))?\s*$')
  98
  99   def __init__(self, outputFormat = 'flat', useJarfileManifest = True,
 100                useChromeManifest = False):
 101     self.outputFormat = outputFormat
 102     self.useJarfileManifest = useJarfileManifest
 103     self.useChromeManifest = useChromeManifest
 104     self.pp = Preprocessor()
 105
 106   def getCommandLineParser(self):
 107     '''Get a optparse.OptionParser for jarmaker.
 108
 109     This OptionParser has the options for jarmaker as well as
 110     the options for the inner PreProcessor.
 111     '''
 112     # HACK, we need to unescape the string variables we get,
 113     # the perl versions didn't grok strings right
 114     p = self.pp.getCommandLineParser(unescapeDefines = True)
 115     p.add_option('-f', type="choice", default="jar",
 116                  choices=('jar', 'flat', 'symlink'),
 117                  help="fileformat used for output", metavar="[jar, flat, symlink]")
 118     p.add_option('-v', action="store_true", dest="verbose",
 119                  help="verbose output")
 120     p.add_option('-q', action="store_false", dest="verbose",
 121                  help="verbose output")
 122     p.add_option('-e', action="store_true",
 123                  help="create chrome.manifest instead of jarfile.manifest")
 124     p.add_option('-s', type="string", action="append", default=[],
 125                  help="source directory")
 126     p.add_option('-t', type="string",
 127                  help="top source directory")
 128     p.add_option('-c', '--l10n-src', type="string", action="append",
 129                  help="localization directory")
 130     p.add_option('--l10n-base', type="string", action="append", default=[],
 131                  help="base directory to be used for localization (multiple)")
 132     p.add_option('-j', type="string",
 133                  help="jarfile directory")
 134     # backwards compat, not needed
 135     p.add_option('-a', action="store_false", default=True,
 136                  help="NOT SUPPORTED, turn auto-registration of chrome off (installed-chrome.txt)")
 137     p.add_option('-d', type="string",
 138                  help="UNUSED, chrome directory")
 139     p.add_option('-o', help="cross compile for auto-registration, ignored")
 140     p.add_option('-l', action="store_true",
 141                  help="ignored (used to switch off locks)")
 142     p.add_option('-x', action="store_true",
 143                  help="force Unix")
 144     p.add_option('-z', help="backwards compat, ignored")
 145     p.add_option('-p', help="backwards compat, ignored")
 146     return p
 147
 148   def processIncludes(self, includes):
 149     '''Process given includes with the inner PreProcessor.
 150
 151     Only use this for #defines, the includes shouldn't generate
 152     content.
 153     '''
 154     self.pp.out = StringIO()
 155     for inc in includes:
 156       self.pp.do_include(inc)
 157     includesvalue = self.pp.out.getvalue()
 158     if includesvalue:
 159       logging.info("WARNING: Includes produce non-empty output")
 160     self.pp.out = None
 161     pass
 162
 163   def finalizeJar(self, jarPath, chromebasepath, register,
 164                    doZip=True):
 165     '''Helper method to write out the chrome registration entries to
 166     jarfile.manifest or chrome.manifest, or both.
 167
 168     The actual file processing is done in updateManifest.
 169     '''
 170     # rewrite the manifest, if entries given
 171     if not register:
 172       return
 173     if self.useJarfileManifest:
 174       self.updateManifest(jarPath + '.manifest', chromebasepath % '',
 175                           register)
 176     if self.useChromeManifest:
 177       manifestPath = os.path.join(os.path.dirname(jarPath),
 178                                   '..', 'chrome.manifest')
 179       self.updateManifest(manifestPath, chromebasepath % 'chrome/',
 180                           register)
 181
 182   def updateManifest(self, manifestPath, chromebasepath, register):
 183     '''updateManifest replaces the % in the chrome registration entries
 184     with the given chrome base path, and updates the given manifest file.
 185     '''
 186     myregister = dict.fromkeys(map(lambda s: s.replace('%', chromebasepath),
 187                                    register.iterkeys()))
 188     manifestExists = os.path.isfile(manifestPath)
 189     mode = (manifestExists and 'r+b') or 'wb'
 190     mf = open(manifestPath, mode)
 191     if manifestExists:
 192       # import previous content into hash, ignoring empty ones and comments
 193       imf = re.compile('(#.*)?$')
 194       for l in re.split('[\r\n]+', mf.read()):
 195         if imf.match(l):
 196           continue
 197         myregister[l] = None
 198       mf.seek(0)
 199     for k in myregister.iterkeys():
 200       mf.write(k + os.linesep)
 201     mf.close()
 202
 203   def makeJar(self, infile=None,
 204                jardir='',
 205                sourcedirs=[], topsourcedir='', localedirs=None):
 206     '''makeJar is the main entry point to JarMaker.
 207
 208     It takes the input file, the output directory, the source dirs and the
 209     top source dir as argument, and optionally the l10n dirs.
 210     '''
 211     if isinstance(infile, basestring):
 212       logging.info("processing " + infile)
 213     pp = self.pp.clone()
 214     pp.out = StringIO()
 215     pp.do_include(infile)
 216     lines = pushback_iter(pp.out.getvalue().splitlines())
 217     try:
 218       while True:
 219         l = lines.next()
 220         m = self.jarline.match(l)
 221         if not m:
 222           raise RuntimeError(l)
 223         if m.group('jarfile') is None:
 224           # comment
 225           continue
 226         self.processJarSection(m.group('jarfile'), lines,
 227                                jardir, sourcedirs, topsourcedir,
 228                                localedirs)
 229     except StopIteration:
 230       # we read the file
 231       pass
 232     return
 233
 234   def processJarSection(self, jarfile, lines,
 235                         jardir, sourcedirs, topsourcedir, localedirs):
 236     '''Internal method called by makeJar to actually process a section
 237     of a jar.mn file.
 238
 239     jarfile is the basename of the jarfile or the directory name for
 240     flat output, lines is a pushback_iterator of the lines of jar.mn,
 241     the remaining options are carried over from makeJar.
 242     '''
 243
 244     # chromebasepath is used for chrome registration manifests
 245     # %s is getting replaced with chrome/ for chrome.manifest, and with
 246     # an empty string for jarfile.manifest
 247     chromebasepath = '%s' + jarfile
 248     if self.outputFormat == 'jar':
 249       chromebasepath = 'jar:' + chromebasepath + '.jar!'
 250     chromebasepath += '/'
 251
 252     jarfile = os.path.join(jardir, jarfile)
 253     jf = None
 254     if self.outputFormat == 'jar':
 255       #jar
 256       jarfilepath = jarfile + '.jar'
 257       if os.path.isfile(jarfilepath) and \
 258             os.path.getsize(jarfilepath) > 0:
 259         jf = ZipFile(jarfilepath, 'a', lock = True)
 260       else:
 261         if not os.path.isdir(os.path.dirname(jarfilepath)):
 262           os.makedirs(os.path.dirname(jarfilepath))
 263         jf = ZipFile(jarfilepath, 'w', lock = True)
 264       outHelper = self.OutputHelper_jar(jf)
 265     else:
 266       outHelper = getattr(self, 'OutputHelper_' + self.outputFormat)(jarfile)
 267     register = {}
 268     # This loop exits on either
 269     # - the end of the jar.mn file
 270     # - an line in the jar.mn file that's not part of a jar section
 271     # - on an exception raised, close the jf in that case in a finally
 272     try:
 273       while True:
 274         try:
 275           l = lines.next()
 276         except StopIteration:
 277           # we're done with this jar.mn, and this jar section
 278           self.finalizeJar(jarfile, chromebasepath, register)
 279           if jf is not None:
 280             jf.close()
 281           # reraise the StopIteration for makeJar
 282           raise
 283         if self.ignore.match(l):
 284           continue
 285         m = self.regline.match(l)
 286         if  m:
 287           rline = m.group(1)
 288           register[rline] = 1
 289           continue
 290         m = self.entryline.match(l)
 291         if not m:
 292           # neither an entry line nor chrome reg, this jar section is done
 293           self.finalizeJar(jarfile, chromebasepath, register)
 294           if jf is not None:
 295             jf.close()
 296           lines.pushback(l)
 297           return
 298         self._processEntryLine(m, sourcedirs, topsourcedir, localedirs,
 299                               outHelper, jf)
 300     finally:
 301       if jf is not None:
 302         jf.close()
 303     return
 304
 305   def _processEntryLine(self, m,
 306                         sourcedirs, topsourcedir, localedirs,
 307                         outHelper, jf):
 308       out = m.group('output')
 309       src = m.group('source') or os.path.basename(out)
 310       # pick the right sourcedir -- l10n, topsrc or src
 311       if m.group('locale'):
 312         src_base = localedirs
 313       elif src.startswith('/'):
 314         # path/in/jar/file_name.xul     (/path/in/sourcetree/file_name.xul)
 315         # refers to a path relative to topsourcedir, use that as base
 316         # and strip the leading '/'
 317         src_base = [topsourcedir]
 318         src = src[1:]
 319       else:
 320         # use srcdirs and the objdir (current working dir) for relative paths
 321         src_base = sourcedirs + ['.']
 322       # check if the source file exists
 323       realsrc = None
 324       for _srcdir in src_base:
 325         if os.path.isfile(os.path.join(_srcdir, src)):
 326           realsrc = os.path.join(_srcdir, src)
 327           break
 328       if realsrc is None:
 329         if jf is not None:
 330           jf.close()
 331         raise RuntimeError("file not found: " + src)
 332       if m.group('optPreprocess'):
 333         outf = outHelper.getOutput(out)
 334         inf = open(realsrc)
 335         pp = self.pp.clone()
 336         if src[-4:] == '.css':
 337           pp.setMarker('%')
 338         pp.out = outf
 339         pp.do_include(inf)
 340         outf.close()
 341         inf.close()
 342         return
 343       # copy or symlink if newer or overwrite
 344       if (m.group('optOverwrite')
 345           or (getModTime(realsrc) >
 346               outHelper.getDestModTime(m.group('output')))):
 347         if self.outputFormat == 'symlink' and hasattr(os, 'symlink'):
 348           outHelper.symlink(realsrc, out)
 349           return
 350         outf = outHelper.getOutput(out)
 351         # open in binary mode, this can be images etc
 352         inf = open(realsrc, 'rb')
 353         outf.write(inf.read())
 354         outf.close()
 355         inf.close()
 356
 357
 358   class OutputHelper_jar(object):
 359     '''Provide getDestModTime and getOutput for a given jarfile.
 360     '''
 361     def __init__(self, jarfile):
 362       self.jarfile = jarfile
 363     def getDestModTime(self, aPath):
 364       try :
 365         info = self.jarfile.getinfo(aPath)
 366         return info.date_time
 367       except:
 368         return 0
 369     def getOutput(self, name):
 370       return ZipEntry(name, self.jarfile)
 371
 372   class OutputHelper_flat(object):
 373     '''Provide getDestModTime and getOutput for a given flat
 374     output directory. The helper method ensureDirFor is used by
 375     the symlink subclass.
 376     '''
 377     def __init__(self, basepath):
 378       self.basepath = basepath
 379     def getDestModTime(self, aPath):
 380       return getModTime(os.path.join(self.basepath, aPath))
 381     def getOutput(self, name):
 382       out = self.ensureDirFor(name)
 383       # remove previous link or file
 384       try:
 385         os.remove(out)
 386       except OSError, e:
 387         if e.errno != 2:
 388           raise
 389       return open(out, 'wb')
 390     def ensureDirFor(self, name):
 391       out = os.path.join(self.basepath, name)
 392       outdir = os.path.dirname(out)
 393       if not os.path.isdir(outdir):
 394         os.makedirs(outdir)
 395       return out
 396
 397   class OutputHelper_symlink(OutputHelper_flat):
 398     '''Subclass of OutputHelper_flat that provides a helper for
 399     creating a symlink including creating the parent directories.
 400     '''
 401     def symlink(self, src, dest):
 402       out = self.ensureDirFor(dest)
 403       # remove previous link or file
 404       try:
 405         os.remove(out)
 406       except OSError, e:
 407         if e.errno != 2:
 408           raise
 409       os.symlink(src, out)
 410
 411 def main():
 412   jm = JarMaker()
 413   p = jm.getCommandLineParser()
 414   (options, args) = p.parse_args()
 415   jm.processIncludes(options.I)
 416   jm.outputFormat = options.f
 417   if options.e:
 418     jm.useChromeManifest = True
 419     jm.useJarfileManifest = False
 420   noise = logging.INFO
 421   if options.verbose is not None:
 422     noise = (options.verbose and logging.DEBUG) or logging.WARN
 423   if sys.version_info[:2] > (2,3):
 424     logging.basicConfig(format = "%(message)s")
 425   else:
 426     logging.basicConfig()
 427   logging.getLogger().setLevel(noise)
 428   if not args:
 429     jm.makeJar(infile=sys.stdin,
 430                sourcedirs=options.s, topsourcedir=options.t,
 431                localedirs=options.l10n_src,
 432                jardir=options.j)
 433     return
 434   topsrc = options.t
 435   topsrc = os.path.normpath(os.path.abspath(topsrc))
 436   for infile in args:
 437     # guess srcdir and l10n dirs from jar.mn path and topsrcdir
 438     # srcdir is the dir of the jar.mn and
 439     # the l10n dirs are the relative path of topsrcdir to srcdir
 440     # resolved against all l10n base dirs.
 441     srcdir = os.path.normpath(os.path.abspath(os.path.dirname(infile)))
 442     l10ndir = srcdir
 443     if os.path.basename(srcdir) == 'locales':
 444       l10ndir = os.path.dirname(l10ndir)
 445     assert srcdir.startswith(topsrc), "src dir %s not in topsrcdir %s" % (srcdir, topsrc)
 446     rell10ndir = l10ndir[len(topsrc):].lstrip(os.sep)
 447     l10ndirs = map(lambda d: os.path.join(d, rell10ndir), options.l10n_base)
 448     if options.l10n_src is not None:
 449       l10ndirs += options.l10n_src
 450     srcdirs = options.s + [srcdir]
 451     jm.makeJar(infile=infile,
 452                sourcedirs=srcdirs, topsourcedir=options.t,
 453                localedirs=l10ndirs,
 454                jardir=options.j)
 455
 456 if __name__ == "__main__":
 457   main()