storage/xpi.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 #
   4 # Copyright 2004, 2005 Zuza Software Foundation
   5 #
   6 # This file is part of translate.
   7 #
   8 # translate is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # translate is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with translate; if not, write to the Free Software
  20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  21
  22 """module for accessing mozilla xpi packages"""
  23
  24 from __future__ import generators
  25 import zipfile
  26 import os.path
  27 from translate import __version__
  28 import StringIO
  29 import re
  30
  31 # we have some enhancements to zipfile in a file called zipfileext
  32 # hopefully they will be included in a future version of python
  33 from translate.misc import zipfileext
  34 ZipFileBase = zipfileext.ZipFileExt
  35
  36 from translate.misc import wStringIO
  37 # this is a fix to the StringIO in Python 2.3.3
  38 # submitted as patch 951915 on sourceforge
  39 class FixedStringIO(wStringIO.StringIO):
  40     def truncate(self, size=None):
  41         StringIO.StringIO.truncate(self, size)
  42         self.len = len(self.buf)
  43
  44 NamedStringInput = wStringIO.StringIO
  45 NamedStringOutput = wStringIO.StringIO
  46
  47 def _commonprefix(itemlist):
  48     def cp(a, b):
  49         l = min(len(a), len(b))
  50         for n in range(l):
  51             if a[n] != b[n]: return a[:n]
  52         return a[:l]
  53     if itemlist:
  54         return reduce(cp, itemlist)
  55     else:
  56         return ''
  57
  58 def rememberchanged(self, method):
  59     def changed(*args, **kwargs):
  60         self.changed = True
  61         method(*args, **kwargs)
  62     return changed
  63
  64 class CatchPotentialOutput(NamedStringInput, object):
  65     """catches output if there has been, before closing"""
  66     def __init__(self, contents, onclose):
  67         """Set up the output stream, and remember a method to call on closing"""
  68         NamedStringInput.__init__(self, contents)
  69         self.onclose = onclose
  70         self.changed = False
  71         s = super(CatchPotentialOutput, self)
  72         self.write = rememberchanged(self, s.write)
  73         self.writelines = rememberchanged(self, s.writelines)
  74         self.truncate = rememberchanged(self, s.truncate)
  75
  76     def close(self):
  77         """wrap the underlying close method, to pass the value to onclose before it goes"""
  78         if self.changed:
  79             value = self.getvalue()
  80             self.onclose(value)
  81         NamedStringInput.close(self)
  82
  83     def flush(self):
  84         """zip files call flush, not close, on file-like objects"""
  85         value = self.getvalue()
  86         self.onclose(value)
  87         NamedStringInput.flush(self)
  88
  89     def slam(self):
  90         """use this method to force the closing of the stream if it isn't closed yet"""
  91         if not self.closed:
  92             self.close()
  93
  94 class ZipFileCatcher(ZipFileBase, object):
  95     """a ZipFile that calls any methods its instructed to before closing (useful for catching stream output)"""
  96     def __init__(self, *args, **kwargs):
  97         """initialize the ZipFileCatcher"""
  98         # storing oldclose as attribute, since if close is called from __del__ it has no access to external variables
  99         self.oldclose = super(ZipFileCatcher, self).close
 100         super(ZipFileCatcher, self).__init__(*args, **kwargs)
 101
 102     def addcatcher(self, pendingsave):
 103         """remember to call the given method before closing"""
 104         if hasattr(self, "pendingsaves"):
 105             if not pendingsave in self.pendingsaves:
 106                 self.pendingsaves.append(pendingsave)
 107         else:
 108             self.pendingsaves = [pendingsave]
 109
 110     def close(self):
 111         """close the stream, remembering to call any addcatcher methods first"""
 112         if hasattr(self, "pendingsaves"):
 113             for pendingsave in self.pendingsaves:
 114                 pendingsave()
 115         # if close is called from __del__, it somehow can't see ZipFileCatcher, so we've cached oldclose...
 116         if ZipFileCatcher is None:
 117             self.oldclose()
 118         else:
 119             super(ZipFileCatcher, self).close()
 120
 121     def overwritestr(self, zinfo_or_arcname, bytes):
 122         """writes the string into the archive, overwriting the file if it exists..."""
 123         if isinstance(zinfo_or_arcname, zipfile.ZipInfo):
 124             filename = zinfo_or_arcname.filename
 125         else:
 126             filename = zinfo_or_arcname
 127         if filename in self.NameToInfo:
 128             self.delete(filename)
 129         self.writestr(zinfo_or_arcname, bytes)
 130         self.writeendrec()
 131
 132 class XpiFile(ZipFileCatcher):
 133     def __init__(self, *args, **kwargs):
 134         """sets up the xpi file"""
 135         self.includenonloc = kwargs.get("includenonloc", True)
 136         if "includenonloc" in kwargs:
 137             del kwargs["includenonloc"]
 138         if "compression" not in kwargs:
 139             kwargs["compression"] = zipfile.ZIP_DEFLATED
 140         self.locale = kwargs.pop("locale", None)
 141         self.region = kwargs.pop("region", None)
 142         super(XpiFile, self).__init__(*args, **kwargs)
 143         self.jarfiles = {}
 144         self.findlangreg()
 145         self.jarprefixes = self.findjarprefixes()
 146         self.reverseprefixes = dict([
 147             (prefix,jarfilename) for jarfilename, prefix in self.jarprefixes.iteritems() if prefix])
 148         self.reverseprefixes["package/"] = None
 149
 150     def iterjars(self):
 151         """iterate through the jar files in the xpi as ZipFile objects"""
 152         for filename in self.namelist():
 153             if filename.lower().endswith('.jar'):
 154                 if filename not in self.jarfiles:
 155                     jarstream = self.openinputstream(None, filename)
 156                     jarfile = ZipFileCatcher(jarstream, mode=self.mode)
 157                     self.jarfiles[filename] = jarfile
 158                 else:
 159                     jarfile = self.jarfiles[filename]
 160                 yield filename, jarfile
 161
 162     def islocfile(self, filename):
 163         """returns whether the given file is needed for localization (basically .dtd and .properties)"""
 164         base, ext = os.path.splitext(filename)
 165         return ext in (os.extsep + "dtd", os.extsep + "properties")
 166
 167     def findlangreg(self):
 168         """finds the common prefix of all the files stored in the jar files"""
 169         dirstructure = {}
 170         locale = self.locale
 171         region = self.region
 172         localematch = re.compile("^[a-z]{2,3}(-[a-zA-Z]{2,3}|)$")
 173         regionmatch = re.compile("^[a-zA-Z]{2,3}$")
 174         # exclude en-mac, en-win, en-unix for seamonkey
 175         osmatch = re.compile("^[a-z]{2,3}-(mac|unix|win)$")
 176         for jarfilename, jarfile in self.iterjars():
 177             jarname = "".join(jarfilename.split('/')[-1:]).replace(".jar", "", 1)
 178             if localematch.match(jarname) and not osmatch.match(jarname):
 179                 if locale is None:
 180                     locale = jarname
 181                 elif locale != jarname:
 182                     locale = 0
 183             elif regionmatch.match(jarname):
 184                 if region is None:
 185                     region = jarname
 186                 elif region != jarname:
 187                     region = 0
 188             for filename in jarfile.namelist():
 189                 if filename.endswith('/'): continue
 190                 if not self.islocfile(filename) and not self.includenonloc: continue
 191                 parts = filename.split('/')[:-1]
 192                 treepoint = dirstructure
 193                 for partnum in range(len(parts)):
 194                     part = parts[partnum]
 195                     if part in treepoint:
 196                         treepoint = treepoint[part]
 197                     else:
 198                         treepoint[part] = {}
 199                         treepoint = treepoint[part]
 200         localeentries = {}
 201         if 'locale' in dirstructure:
 202             for dirname in dirstructure['locale']:
 203                 localeentries[dirname] = 1
 204                 if localematch.match(dirname) and not osmatch.match(dirname):
 205                     if locale is None:
 206                         locale = dirname
 207                     elif locale != dirname:
 208                         print "locale dir mismatch - ", dirname, "but locale is", locale, "setting to 0"
 209                         locale = 0
 210                 elif regionmatch.match(dirname):
 211                     if region is None:
 212                         region = dirname
 213                     elif region != dirname:
 214                         region = 0
 215         if locale and locale in localeentries:
 216             del localeentries[locale]
 217         if region and region in localeentries:
 218             del localeentries[region]
 219         if locale and not region:
 220             if "-" in locale:
 221                 region = locale.split("-", 1)[1]
 222             else:
 223                 region = ""
 224         self.setlangreg(locale, region)
 225
 226     def setlangreg(self, locale, region):
 227         """set the locale and region of this xpi"""
 228         if locale == 0 or locale is None:
 229             raise ValueError("unable to determine locale")
 230         self.locale = locale
 231         self.region = region
 232         self.dirmap = {}
 233         if self.locale is not None:
 234             self.dirmap[('locale', self.locale)] = ('lang-reg',)
 235         if self.region:
 236             self.dirmap[('locale', self.region)] = ('reg',)
 237
 238     def findjarprefixes(self):
 239         """checks the uniqueness of the jar files contents"""
 240         uniquenames = {}
 241         jarprefixes = {}
 242         for jarfilename, jarfile in self.iterjars():
 243             jarprefixes[jarfilename] = ""
 244             for filename in jarfile.namelist():
 245                 if filename.endswith('/'): continue
 246                 if filename in uniquenames:
 247                     jarprefixes[jarfilename] = True
 248                     jarprefixes[uniquenames[filename]] = True
 249                 else:
 250                     uniquenames[filename] = jarfilename
 251         for jarfilename, hasconflicts in jarprefixes.items():
 252             if hasconflicts:
 253                 shortjarfilename = os.path.split(jarfilename)[1]
 254                 shortjarfilename = os.path.splitext(shortjarfilename)[0]
 255                 jarprefixes[jarfilename] = shortjarfilename+'/'
 256         # this is a clever trick that will e.g. remove zu- from zu-win, zu-mac, zu-unix
 257         commonjarprefix = _commonprefix([prefix for prefix in jarprefixes.itervalues() if prefix])
 258         if commonjarprefix:
 259             for jarfilename, prefix in jarprefixes.items():
 260                 if prefix:
 261                     jarprefixes[jarfilename] = prefix.replace(commonjarprefix, '', 1)
 262         return jarprefixes
 263
 264     def ziptoospath(self, zippath):
 265         """converts a zipfile filepath to an os-style filepath"""
 266         return os.path.join(*zippath.split('/'))
 267
 268     def ostozippath(self, ospath):
 269         """converts an os-style filepath to a zipfile filepath"""
 270         return '/'.join(ospath.split(os.sep))
 271
 272     def mapfilename(self, filename):
 273         """uses a map to simplify the directory structure"""
 274         parts = tuple(filename.split('/'))
 275         possiblematch = None
 276         for prefix, mapto in self.dirmap.iteritems():
 277             if parts[:len(prefix)] == prefix:
 278                 if possiblematch is None or len(possiblematch[0]) < len(prefix):
 279                     possiblematch = prefix, mapto
 280         if possiblematch is not None:
 281             prefix, mapto = possiblematch
 282             mapped = mapto + parts[len(prefix):]
 283             return '/'.join(mapped)
 284         return filename
 285
 286     def mapxpifilename(self, filename):
 287         """uses a map to rename files that occur straight in the xpi"""
 288         if filename.startswith('bin/chrome/') and filename.endswith(".manifest"):
 289             return 'bin/chrome/lang-reg.manifest'
 290         return filename
 291
 292     def reversemapfile(self, filename):
 293         """unmaps the filename..."""
 294         possiblematch = None
 295         parts = tuple(filename.split('/'))
 296         for prefix, mapto in self.dirmap.iteritems():
 297             if parts[:len(mapto)] == mapto:
 298                 if possiblematch is None or len(possiblematch[0]) < len(mapto):
 299                     possiblematch = (mapto, prefix)
 300         if possiblematch is None:
 301             return filename
 302         mapto, prefix = possiblematch
 303         reversemapped = prefix + parts[len(mapto):]
 304         return '/'.join(reversemapped)
 305
 306     def reversemapxpifilename(self, filename):
 307         """uses a map to rename files that occur straight in the xpi"""
 308         if filename == 'bin/chrome/lang-reg.manifest':
 309             if self.locale:
 310                 return '/'.join(('bin', 'chrome', self.locale + '.manifest'))
 311             else:
 312                 for otherfilename in self.namelist():
 313                     if otherfilename.startswith("bin/chrome/") and otherfilename.endswith(".manifest"):
 314                         return otherfilename
 315         return filename
 316
 317     def jartoospath(self, jarfilename, filename):
 318         """converts a filename from within a jarfile to an os-style filepath"""
 319         if jarfilename:
 320             jarprefix = self.jarprefixes[jarfilename]
 321             return self.ziptoospath(jarprefix+self.mapfilename(filename))
 322         else:
 323             return self.ziptoospath(os.path.join("package", self.mapxpifilename(filename)))
 324
 325     def ostojarpath(self, ospath):
 326         """converts an extracted os-style filepath to a jarfilename and filename"""
 327         zipparts = ospath.split(os.sep)
 328         prefix = zipparts[0] + '/'
 329         if prefix in self.reverseprefixes:
 330             jarfilename = self.reverseprefixes[prefix]
 331             filename = self.reversemapfile('/'.join(zipparts[1:]))
 332             if jarfilename is None:
 333                 filename = self.reversemapxpifilename(filename)
 334             return jarfilename, filename
 335         else:
 336             filename = self.ostozippath(ospath)
 337             if filename in self.namelist():
 338                 return None, filename
 339             filename = self.reversemapfile('/'.join(zipparts))
 340             possiblejarfilenames = [jarfilename for jarfilename, prefix in self.jarprefixes.iteritems() if not prefix]
 341             for jarfilename in possiblejarfilenames:
 342                 jarfile = self.jarfiles[jarfilename]
 343                 if filename in jarfile.namelist():
 344                     return jarfilename, filename
 345             raise IndexError("ospath not found in xpi file, could not guess location: %r" % ospath)
 346
 347     def jarfileexists(self, jarfilename, filename):
 348         """checks whether the given file exists inside the xpi"""
 349         if jarfilename is None:
 350             return filename in self.namelist()
 351         else:
 352             jarfile = self.jarfiles[jarfilename]
 353             return filename in jarfile.namelist()
 354
 355     def ospathexists(self, ospath):
 356         """checks whether the given file exists inside the xpi"""
 357         jarfilename, filename = self.ostojarpath(ospath)
 358         if jarfilename is None:
 359             return filename in self.namelist()
 360         else:
 361             jarfile = self.jarfiles[jarfilename]
 362             return filename in jarfile.namelist()
 363
 364     def openinputstream(self, jarfilename, filename):
 365         """opens a file (possibly inside a jarfile as a StringIO"""
 366         if jarfilename is None:
 367             contents = self.read(filename)
 368             def onclose(contents):
 369                 if contents != self.read(filename):
 370                     self.overwritestr(filename, contents)
 371             inputstream = CatchPotentialOutput(contents, onclose)
 372             self.addcatcher(inputstream.slam)
 373         else:
 374             jarfile = self.jarfiles[jarfilename]
 375             contents = jarfile.read(filename)
 376             inputstream = NamedStringInput(contents)
 377         inputstream.name = self.jartoospath(jarfilename, filename)
 378         if hasattr(self.fp, 'name'):
 379             inputstream.name = "%s:%s" % (self.fp.name, inputstream.name)
 380         return inputstream
 381
 382     def openoutputstream(self, jarfilename, filename):
 383         """opens a file for writing (possibly inside a jarfile as a StringIO"""
 384         if jarfilename is None:
 385             def onclose(contents):
 386                 self.overwritestr(filename, contents)
 387         else:
 388             if jarfilename in self.jarfiles:
 389                 jarfile = self.jarfiles[jarfilename]
 390             else:
 391                 jarstream = self.openoutputstream(None, jarfilename)
 392                 jarfile = ZipFileCatcher(jarstream, "w")
 393                 self.jarfiles[jarfilename] = jarfile
 394                 self.addcatcher(jarstream.slam)
 395             def onclose(contents):
 396                 jarfile.overwritestr(filename, contents)
 397         outputstream = wStringIO.CatchStringOutput(onclose)
 398         outputstream.name = "%s %s" % (jarfilename, filename)
 399         if jarfilename is None:
 400             self.addcatcher(outputstream.slam)
 401         else:
 402             jarfile.addcatcher(outputstream.slam)
 403         return outputstream
 404
 405     def close(self):
 406         """Close the file, and for mode "w" and "a" write the ending records."""
 407         for jarfile in self.jarfiles.itervalues():
 408             jarfile.close()
 409         super(XpiFile, self).close()
 410
 411     def testzip(self):
 412         """test the xpi zipfile and all enclosed jar files..."""
 413         for jarfile in self.jarfiles.itervalues():
 414             jarfile.testzip()
 415         super(XpiFile, self).testzip()
 416
 417     def restructurejar(self, origjarfilename, newjarfilename, otherxpi, newlang, newregion):
 418         """Create a new .jar file with the same contents as the given name, but rename directories, write to outputstream"""
 419         jarfile = self.jarfiles[origjarfilename]
 420         origlang = self.locale[:self.locale.find("-")]
 421         if newregion:
 422             newlocale = "%s-%s" % (newlang, newregion)
 423         else:
 424             newlocale = newlang
 425         for filename in jarfile.namelist():
 426             filenameparts = filename.split("/")
 427             for i in range(len(filenameparts)):
 428                 part = filenameparts[i]
 429                 if part == origlang:
 430                     filenameparts[i] = newlang
 431                 elif part == self.locale:
 432                     filenameparts[i] = newlocale
 433                 elif part == self.region:
 434                     filenameparts[i] = newregion
 435             newfilename = '/'.join(filenameparts)
 436             fileoutputstream = otherxpi.openoutputstream(newjarfilename, newfilename)
 437             fileinputstream = self.openinputstream(origjarfilename, filename)
 438             fileoutputstream.write(fileinputstream.read())
 439             fileinputstream.close()
 440             fileoutputstream.close()
 441
 442     def clone(self, newfilename, newmode=None, newlang=None, newregion=None):
 443         """Create a new .xpi file with the same contents as this one..."""
 444         other = XpiFile(newfilename, "w", locale=newlang, region=newregion)
 445         origlang = self.locale[:self.locale.find("-")]
 446         # TODO: check if this language replacement code is still neccessary
 447         if newlang is None:
 448             newlang = origlang
 449         if newregion is None:
 450             newregion = self.region
 451         if newregion:
 452             newlocale = "%s-%s" % (newlang, newregion)
 453         else:
 454             newlocale = newlang
 455         for filename in self.namelist():
 456             filenameparts = filename.split('/')
 457             basename = filenameparts[-1]
 458             if basename.startswith(self.locale):
 459                 newbasename = basename.replace(self.locale, newlocale)
 460             elif basename.startswith(origlang):
 461                 newbasename = basename.replace(origlang, newlang)
 462             elif basename.startswith(self.region):
 463                 newbasename = basename.replace(self.region, newregion)
 464             else:
 465                 newbasename = basename
 466             if newbasename != basename:
 467                 filenameparts[-1] = newbasename
 468                 renamefilename = "/".join(filenameparts)
 469                 print "cloning", filename, "and renaming to", renamefilename
 470             else:
 471                 print "cloning", filename
 472                 renamefilename = filename
 473             if filename.lower().endswith(".jar"):
 474                 self.restructurejar(filename, renamefilename, other, newlang, newregion)
 475             else:
 476                 inputstream = self.openinputstream(None, filename)
 477                 outputstream = other.openoutputstream(None, renamefilename)
 478                 outputstream.write(inputstream.read())
 479                 inputstream.close()
 480                 outputstream.close()
 481         other.close()
 482         if newmode is None: newmode = self.mode
 483         if newmode == "w": newmode = "a"
 484         other = XpiFile(newfilename, newmode)
 485         other.setlangreg(newlocale, newregion)
 486         return other
 487
 488     def iterextractnames(self, includenonjars=False, includedirs=False):
 489         """iterates through all the localization files with the common prefix stripped and a jarfile name added if neccessary"""
 490         if includenonjars:
 491             for filename in self.namelist():
 492                 if filename.endswith('/') and not includedirs: continue
 493                 if not self.islocfile(filename) and not self.includenonloc: continue
 494                 if not filename.lower().endswith(".jar"):
 495                     yield self.jartoospath(None, filename)
 496         for jarfilename, jarfile in self.iterjars():
 497             for filename in jarfile.namelist():
 498                 if filename.endswith('/'):
 499                     if not includedirs: continue
 500                 if not self.islocfile(filename) and not self.includenonloc: continue
 501                 yield self.jartoospath(jarfilename, filename)
 502
 503     # the following methods are required by translate.convert.ArchiveConvertOptionParser #
 504     def __iter__(self):
 505         """iterates through all the files. this is the method use by the converters"""
 506         for inputpath in self.iterextractnames(includenonjars=True):
 507             yield inputpath
 508
 509     def __contains__(self, fullpath):
 510         """returns whether the given pathname exists in the archive"""
 511         try:
 512             jarfilename, filename = self.ostojarpath(fullpath)
 513         except IndexError:
 514             return False
 515         return self.jarfileexists(jarfilename, filename)
 516
 517     def openinputfile(self, fullpath):
 518         """opens an input file given the full pathname"""
 519         jarfilename, filename = self.ostojarpath(fullpath)
 520         return self.openinputstream(jarfilename, filename)
 521
 522     def openoutputfile(self, fullpath):
 523         """opens an output file given the full pathname"""
 524         try:
 525             jarfilename, filename = self.ostojarpath(fullpath)
 526         except IndexError:
 527             return None
 528         return self.openoutputstream(jarfilename, filename)
 529
 530 if __name__ == '__main__':
 531     import optparse
 532     optparser = optparse.OptionParser(version="%prog "+__version__.ver)
 533     optparser.usage = "%prog [-l|-x] [options] file.xpi"
 534     optparser.add_option("-l", "--list", help="list files", \
 535         action="store_true", dest="listfiles", default=False)
 536     optparser.add_option("-p", "--prefix", help="show common prefix", \
 537         action="store_true", dest="showprefix", default=False)
 538     optparser.add_option("-x", "--extract", help="extract files", \
 539         action="store_true", dest="extractfiles", default=False)
 540     optparser.add_option("-d", "--extractdir", help="extract into EXTRACTDIR", \
 541         default=".", metavar="EXTRACTDIR")
 542     (options, args) = optparser.parse_args()
 543     if len(args) < 1:
 544         optparser.error("need at least one argument")
 545     xpifile = XpiFile(args[0])
 546     if options.showprefix:
 547         for prefix, mapto in xpifile.dirmap.iteritems():
 548             print "/".join(prefix), "->", "/".join(mapto)
 549     if options.listfiles:
 550         for name in xpifile.iterextractnames(includenonjars=True, includedirs=True):
 551             print name #, xpifile.ostojarpath(name)
 552     if options.extractfiles:
 553         if options.extractdir and not os.path.isdir(options.extractdir):
 554             os.mkdir(options.extractdir)
 555         for name in xpifile.iterextractnames(includenonjars=True, includedirs=False):
 556             abspath = os.path.join(options.extractdir, name)
 557             # check neccessary directories exist - this way we don't create empty directories
 558             currentpath = options.extractdir
 559             subparts = os.path.dirname(name).split(os.sep)
 560             for part in subparts:
 561                 currentpath = os.path.join(currentpath, part)
 562                 if not os.path.isdir(currentpath):
 563                     os.mkdir(currentpath)
 564             outputstream = open(abspath, 'w')
 565             jarfilename, filename = xpifile.ostojarpath(name)
 566             inputstream = xpifile.openinputstream(jarfilename, filename)
 567             outputstream.write(inputstream.read())
 568             outputstream.close()
 569