Lib/gettext.py

   1 """Internationalization and localization support.
   2
   3 This module provides internationalization (I18N) and localization (L10N)
   4 support for your Python programs by providing an interface to the GNU gettext
   5 message catalog library.
   6
   7 I18N refers to the operation by which a program is made aware of multiple
   8 languages.  L10N refers to the adaptation of your program, once
   9 internationalized, to the local language and cultural habits.
  10
  11 """
  12
  13 # This module represents the integration of work, contributions, feedback, and
  14 # suggestions from the following people:
  15 #
  16 # Martin von Loewis, who wrote the initial implementation of the underlying
  17 # C-based libintlmodule (later renamed _gettext), along with a skeletal
  18 # gettext.py implementation.
  19 #
  20 # Peter Funk, who wrote fintl.py, a fairly complete wrapper around intlmodule,
  21 # which also included a pure-Python implementation to read .mo files if
  22 # intlmodule wasn't available.
  23 #
  24 # James Henstridge, who also wrote a gettext.py module, which has some
  25 # interesting, but currently unsupported experimental features: the notion of
  26 # a Catalog class and instances, and the ability to add to a catalog file via
  27 # a Python API.
  28 #
  29 # Barry Warsaw integrated these modules, wrote the .install() API and code,
  30 # and conformed all C and Python code to Python's coding standards.
  31 #
  32 # Francois Pinard and Marc-Andre Lemburg also contributed valuably to this
  33 # module.
  34 #
  35 # TODO:
  36 # - Lazy loading of .mo files.  Currently the entire catalog is loaded into
  37 #   memory, but that's probably bad for large translated programs.  Instead,
  38 #   the lexical sort of original strings in GNU .mo files should be exploited
  39 #   to do binary searches and lazy initializations.  Or you might want to use
  40 #   the undocumented double-hash algorithm for .mo files with hash tables, but
  41 #   you'll need to study the GNU gettext code to do this.
  42 #
  43 # - Support Solaris .mo file formats.  Unfortunately, we've been unable to
  44 #   find this format documented anywhere.
  45
  46 import os
  47 import sys
  48 import struct
  49 import copy
  50 from errno import ENOENT
  51
  52 __all__ = ["bindtextdomain","textdomain","gettext","dgettext",
  53            "find","translation","install","Catalog"]
  54
  55 _default_localedir = os.path.join(sys.prefix, 'share', 'locale')
  56
  57
  58
  59 def _expand_lang(locale):
  60     from locale import normalize
  61     locale = normalize(locale)
  62     COMPONENT_CODESET   = 1 << 0
  63     COMPONENT_TERRITORY = 1 << 1
  64     COMPONENT_MODIFIER  = 1 << 2
  65     # split up the locale into its base components
  66     mask = 0
  67     pos = locale.find('@')
  68     if pos >= 0:
  69         modifier = locale[pos:]
  70         locale = locale[:pos]
  71         mask |= COMPONENT_MODIFIER
  72     else:
  73         modifier = ''
  74     pos = locale.find('.')
  75     if pos >= 0:
  76         codeset = locale[pos:]
  77         locale = locale[:pos]
  78         mask |= COMPONENT_CODESET
  79     else:
  80         codeset = ''
  81     pos = locale.find('_')
  82     if pos >= 0:
  83         territory = locale[pos:]
  84         locale = locale[:pos]
  85         mask |= COMPONENT_TERRITORY
  86     else:
  87         territory = ''
  88     language = locale
  89     ret = []
  90     for i in range(mask+1):
  91         if not (i & ~mask):  # if all components for this combo exist ...
  92             val = language
  93             if i & COMPONENT_TERRITORY: val += territory
  94             if i & COMPONENT_CODESET:   val += codeset
  95             if i & COMPONENT_MODIFIER:  val += modifier
  96             ret.append(val)
  97     ret.reverse()
  98     return ret
  99
 100
 101
 102 class NullTranslations:
 103     def __init__(self, fp=None):
 104         self._info = {}
 105         self._charset = None
 106         self._fallback = None
 107         if fp is not None:
 108             self._parse(fp)
 109
 110     def _parse(self, fp):
 111         pass
 112
 113     def add_fallback(self, fallback):
 114         if self._fallback:
 115             self._fallback.add_fallback(fallback)
 116         else:
 117             self._fallback = fallback
 118
 119     def gettext(self, message):
 120         if self._fallback:
 121             return self._fallback.gettext(message)
 122         return message
 123
 124     def ugettext(self, message):
 125         if self._fallback:
 126             return self._fallback.ugettext(message)
 127         return unicode(message)
 128
 129     def info(self):
 130         return self._info
 131
 132     def charset(self):
 133         return self._charset
 134
 135     def install(self, unicode=0):
 136         import __builtin__
 137         __builtin__.__dict__['_'] = unicode and self.ugettext or self.gettext
 138
 139
 140 class GNUTranslations(NullTranslations):
 141     # Magic number of .mo files
 142     LE_MAGIC = 0x950412deL
 143     BE_MAGIC = 0xde120495L
 144
 145     def _parse(self, fp):
 146         """Override this method to support alternative .mo formats."""
 147         unpack = struct.unpack
 148         filename = getattr(fp, 'name', '')
 149         # Parse the .mo file header, which consists of 5 little endian 32
 150         # bit words.
 151         self._catalog = catalog = {}
 152         buf = fp.read()
 153         buflen = len(buf)
 154         # Are we big endian or little endian?
 155         magic = unpack('<I', buf[:4])[0]
 156         if magic == self.LE_MAGIC:
 157             version, msgcount, masteridx, transidx = unpack('<4I', buf[4:20])
 158             ii = '<II'
 159         elif magic == self.BE_MAGIC:
 160             version, msgcount, masteridx, transidx = unpack('>4I', buf[4:20])
 161             ii = '>II'
 162         else:
 163             raise IOError(0, 'Bad magic number', filename)
 164         # Now put all messages from the .mo file buffer into the catalog
 165         # dictionary.
 166         for i in xrange(0, msgcount):
 167             mlen, moff = unpack(ii, buf[masteridx:masteridx+8])
 168             mend = moff + mlen
 169             tlen, toff = unpack(ii, buf[transidx:transidx+8])
 170             tend = toff + tlen
 171             if mend < buflen and tend < buflen:
 172                 tmsg = buf[toff:tend]
 173                 catalog[buf[moff:mend]] = tmsg
 174             else:
 175                 raise IOError(0, 'File is corrupt', filename)
 176             # See if we're looking at GNU .mo conventions for metadata
 177             if mlen == 0 and tmsg.lower().startswith('project-id-version:'):
 178                 # Catalog description
 179                 for item in tmsg.split('\n'):
 180                     item = item.strip()
 181                     if not item:
 182                         continue
 183                     k, v = item.split(':', 1)
 184                     k = k.strip().lower()
 185                     v = v.strip()
 186                     self._info[k] = v
 187                     if k == 'content-type':
 188                         self._charset = v.split('charset=')[1]
 189             # advance to next entry in the seek tables
 190             masteridx += 8
 191             transidx += 8
 192
 193     def gettext(self, message):
 194         try:
 195             return self._catalog[message]
 196         except KeyError:
 197             if self._fallback:
 198                 return self._fallback.gettext(message)
 199             return message
 200
 201     def ugettext(self, message):
 202         try:
 203             tmsg = self._catalog[message]
 204         except KeyError:
 205             if self._fallback:
 206                 return self._fallback.ugettext(message)
 207             tmsg = message
 208         return unicode(tmsg, self._charset)
 209
 210
 211
 212 # Locate a .mo file using the gettext strategy
 213 def find(domain, localedir=None, languages=None, all=0):
 214     # Get some reasonable defaults for arguments that were not supplied
 215     if localedir is None:
 216         localedir = _default_localedir
 217     if languages is None:
 218         languages = []
 219         for envar in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'):
 220             val = os.environ.get(envar)
 221             if val:
 222                 languages = val.split(':')
 223                 break
 224         if 'C' not in languages:
 225             languages.append('C')
 226     # now normalize and expand the languages
 227     nelangs = []
 228     for lang in languages:
 229         for nelang in _expand_lang(lang):
 230             if nelang not in nelangs:
 231                 nelangs.append(nelang)
 232     # select a language
 233     if all:
 234         result = []
 235     else:
 236         result = None
 237     for lang in nelangs:
 238         if lang == 'C':
 239             break
 240         mofile = os.path.join(localedir, lang, 'LC_MESSAGES', '%s.mo' % domain)
 241         if os.path.exists(mofile):
 242             if all:
 243                 result.append(mofile)
 244             else:
 245                 return mofile
 246     return result
 247
 248
 249
 250 # a mapping between absolute .mo file path and Translation object
 251 _translations = {}
 252
 253 def translation(domain, localedir=None, languages=None,
 254                 class_=None, fallback=0):
 255     if class_ is None:
 256         class_ = GNUTranslations
 257     mofiles = find(domain, localedir, languages, all=1)
 258     if len(mofiles)==0:
 259         if fallback:
 260             return NullTranslations()
 261         raise IOError(ENOENT, 'No translation file found for domain', domain)
 262     # TBD: do we need to worry about the file pointer getting collected?
 263     # Avoid opening, reading, and parsing the .mo file after it's been done
 264     # once.
 265     result = None
 266     for mofile in mofiles:
 267         key = os.path.abspath(mofile)
 268         t = _translations.get(key)
 269         if t is None:
 270             t = _translations.setdefault(key, class_(open(mofile, 'rb')))
 271         # Copy the translation object to allow setting fallbacks.
 272         # All other instance data is shared with the cached object.
 273         t = copy.copy(t)
 274         if result is None:
 275             result = t
 276         else:
 277             result.add_fallback(t)
 278     return result
 279
 280
 281 def install(domain, localedir=None, unicode=0):
 282     translation(domain, localedir, fallback=1).install(unicode)
 283
 284
 285
 286 # a mapping b/w domains and locale directories
 287 _localedirs = {}
 288 # current global domain, `messages' used for compatibility w/ GNU gettext
 289 _current_domain = 'messages'
 290
 291
 292 def textdomain(domain=None):
 293     global _current_domain
 294     if domain is not None:
 295         _current_domain = domain
 296     return _current_domain
 297
 298
 299 def bindtextdomain(domain, localedir=None):
 300     global _localedirs
 301     if localedir is not None:
 302         _localedirs[domain] = localedir
 303     return _localedirs.get(domain, _default_localedir)
 304
 305
 306 def dgettext(domain, message):
 307     try:
 308         t = translation(domain, _localedirs.get(domain, None))
 309     except IOError:
 310         return message
 311     return t.gettext(message)
 312
 313
 314 def gettext(message):
 315     return dgettext(_current_domain, message)
 316
 317
 318 # dcgettext() has been deemed unnecessary and is not implemented.
 319
 320 # James Henstridge's Catalog constructor from GNOME gettext.  Documented usage
 321 # was:
 322 #
 323 #    import gettext
 324 #    cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR)
 325 #    _ = cat.gettext
 326 #    print _('Hello World')
 327
 328 # The resulting catalog object currently don't support access through a
 329 # dictionary API, which was supported (but apparently unused) in GNOME
 330 # gettext.
 331
 332 Catalog = translation