Lib/gettext.py

   1 """Internationalization and localization support.
   2
   3 This module provides internationalization (I18N) and localization (L10N)
   4 support for your Python programs by providing an interface to the GNU gettext
   5 message catalog library.
   6
   7 I18N refers to the operation by which a program is made aware of multiple
   8 languages.  L10N refers to the adaptation of your program, once
   9 internationalized, to the local language and cultural habits.
  10
  11 """
  12
  13 # This module represents the integration of work, contributions, feedback, and
  14 # suggestions from the following people:
  15 #
  16 # Martin von Loewis, who wrote the initial implementation of the underlying
  17 # C-based libintlmodule (later renamed _gettext), along with a skeletal
  18 # gettext.py implementation.
  19 #
  20 # Peter Funk, who wrote fintl.py, a fairly complete wrapper around intlmodule,
  21 # which also included a pure-Python implementation to read .mo files if
  22 # intlmodule wasn't available.
  23 #
  24 # James Henstridge, who also wrote a gettext.py module, which has some
  25 # interesting, but currently unsupported experimental features: the notion of
  26 # a Catalog class and instances, and the ability to add to a catalog file via
  27 # a Python API.
  28 #
  29 # Barry Warsaw integrated these modules, wrote the .install() API and code,
  30 # and conformed all C and Python code to Python's coding standards.
  31 #
  32 # Francois Pinard and Marc-Andre Lemburg also contributed valuably to this
  33 # module.
  34 #
  35 # TODO:
  36 # - Lazy loading of .mo files.  Currently the entire catalog is loaded into
  37 #   memory, but that's probably bad for large translated programs.  Instead,
  38 #   the lexical sort of original strings in GNU .mo files should be exploited
  39 #   to do binary searches and lazy initializations.  Or you might want to use
  40 #   the undocumented double-hash algorithm for .mo files with hash tables, but
  41 #   you'll need to study the GNU gettext code to do this.
  42 #
  43 # - Support Solaris .mo file formats.  Unfortunately, we've been unable to
  44 #   find this format documented anywhere.
  45
  46 import os
  47 import sys
  48 import struct
  49 import copy
  50 from errno import ENOENT
  51
  52 __all__ = ["bindtextdomain","textdomain","gettext","dgettext",
  53            "find","translation","install","Catalog"]
  54
  55 _default_localedir = os.path.join(sys.prefix, 'share', 'locale')
  56
  57
  58
  59 def _expand_lang(locale):
  60     from locale import normalize
  61     locale = normalize(locale)
  62     COMPONENT_CODESET   = 1 << 0
  63     COMPONENT_TERRITORY = 1 << 1
  64     COMPONENT_MODIFIER  = 1 << 2
  65     # split up the locale into its base components
  66     mask = 0
  67     pos = locale.find('@')
  68     if pos >= 0:
  69         modifier = locale[pos:]
  70         locale = locale[:pos]
  71         mask |= COMPONENT_MODIFIER
  72     else:
  73         modifier = ''
  74     pos = locale.find('.')
  75     if pos >= 0:
  76         codeset = locale[pos:]
  77         locale = locale[:pos]
  78         mask |= COMPONENT_CODESET
  79     else:
  80         codeset = ''
  81     pos = locale.find('_')
  82     if pos >= 0:
  83         territory = locale[pos:]
  84         locale = locale[:pos]
  85         mask |= COMPONENT_TERRITORY
  86     else:
  87         territory = ''
  88     language = locale
  89     ret = []
  90     for i in range(mask+1):
  91         if not (i & ~mask):  # if all components for this combo exist ...
  92             val = language
  93             if i & COMPONENT_TERRITORY: val += territory
  94             if i & COMPONENT_CODESET:   val += codeset
  95             if i & COMPONENT_MODIFIER:  val += modifier
  96             ret.append(val)
  97     ret.reverse()
  98     return ret
  99
 100
 101
 102 class NullTranslations:
 103     def __init__(self, fp=None):
 104         self._info = {}
 105         self._charset = None
 106         self._fallback = None
 107         if fp is not None:
 108             self._parse(fp)
 109
 110     def _parse(self, fp):
 111         pass
 112
 113     def add_fallback(self, fallback):
 114         if self._fallback:
 115             self._fallback.add_fallback(fallback)
 116         else:
 117             self._fallback = fallback
 118
 119     def gettext(self, message):
 120         if self._fallback:
 121             return self._fallback.gettext(message)
 122         return message
 123
 124     def ugettext(self, message):
 125         if self._fallback:
 126             return self._fallback.ugettext(message)
 127         return unicode(message)
 128
 129     def info(self):
 130         return self._info
 131
 132     def charset(self):
 133         return self._charset
 134
 135     def install(self, unicode=0):
 136         import __builtin__
 137         __builtin__.__dict__['_'] = unicode and self.ugettext or self.gettext
 138
 139
 140 class GNUTranslations(NullTranslations):
 141     # Magic number of .mo files
 142     LE_MAGIC = 0x950412de
 143     BE_MAGIC = 0xde120495
 144
 145     def _parse(self, fp):
 146         """Override this method to support alternative .mo formats."""
 147         # We need to & all 32 bit unsigned integers with 0xffffffff for
 148         # portability to 64 bit machines.
 149         MASK = 0xffffffff
 150         unpack = struct.unpack
 151         filename = getattr(fp, 'name', '')
 152         # Parse the .mo file header, which consists of 5 little endian 32
 153         # bit words.
 154         self._catalog = catalog = {}
 155         buf = fp.read()
 156         buflen = len(buf)
 157         # Are we big endian or little endian?
 158         magic = unpack('<i', buf[:4])[0] & MASK
 159         if magic == self.LE_MAGIC:
 160             version, msgcount, masteridx, transidx = unpack('<4i', buf[4:20])
 161             ii = '<ii'
 162         elif magic == self.BE_MAGIC:
 163             version, msgcount, masteridx, transidx = unpack('>4i', buf[4:20])
 164             ii = '>ii'
 165         else:
 166             raise IOError(0, 'Bad magic number', filename)
 167         # more unsigned ints
 168         msgcount &= MASK
 169         masteridx &= MASK
 170         transidx &= MASK
 171         # Now put all messages from the .mo file buffer into the catalog
 172         # dictionary.
 173         for i in xrange(0, msgcount):
 174             mlen, moff = unpack(ii, buf[masteridx:masteridx+8])
 175             moff &= MASK
 176             mend = moff + (mlen & MASK)
 177             tlen, toff = unpack(ii, buf[transidx:transidx+8])
 178             toff &= MASK
 179             tend = toff + (tlen & MASK)
 180             if mend < buflen and tend < buflen:
 181                 tmsg = buf[toff:tend]
 182                 catalog[buf[moff:mend]] = tmsg
 183             else:
 184                 raise IOError(0, 'File is corrupt', filename)
 185             # See if we're looking at GNU .mo conventions for metadata
 186             if mlen == 0 and tmsg.lower().startswith('project-id-version:'):
 187                 # Catalog description
 188                 for item in tmsg.split('\n'):
 189                     item = item.strip()
 190                     if not item:
 191                         continue
 192                     k, v = item.split(':', 1)
 193                     k = k.strip().lower()
 194                     v = v.strip()
 195                     self._info[k] = v
 196                     if k == 'content-type':
 197                         self._charset = v.split('charset=')[1]
 198             # advance to next entry in the seek tables
 199             masteridx += 8
 200             transidx += 8
 201
 202     def gettext(self, message):
 203         try:
 204             return self._catalog[message]
 205         except KeyError:
 206             if self._fallback:
 207                 return self._fallback.gettext(message)
 208             return message
 209
 210     def ugettext(self, message):
 211         try:
 212             tmsg = self._catalog[message]
 213         except KeyError:
 214             if self._fallback:
 215                 return self._fallback.ugettext(message)
 216             tmsg = message
 217         return unicode(tmsg, self._charset)
 218
 219
 220
 221 # Locate a .mo file using the gettext strategy
 222 def find(domain, localedir=None, languages=None, all=0):
 223     # Get some reasonable defaults for arguments that were not supplied
 224     if localedir is None:
 225         localedir = _default_localedir
 226     if languages is None:
 227         languages = []
 228         for envar in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'):
 229             val = os.environ.get(envar)
 230             if val:
 231                 languages = val.split(':')
 232                 break
 233         if 'C' not in languages:
 234             languages.append('C')
 235     # now normalize and expand the languages
 236     nelangs = []
 237     for lang in languages:
 238         for nelang in _expand_lang(lang):
 239             if nelang not in nelangs:
 240                 nelangs.append(nelang)
 241     # select a language
 242     if all:
 243         result = []
 244     else:
 245         result = None
 246     for lang in nelangs:
 247         if lang == 'C':
 248             break
 249         mofile = os.path.join(localedir, lang, 'LC_MESSAGES', '%s.mo' % domain)
 250         if os.path.exists(mofile):
 251             if all:
 252                 result.append(mofile)
 253             else:
 254                 return mofile
 255     return result
 256
 257
 258
 259 # a mapping between absolute .mo file path and Translation object
 260 _translations = {}
 261
 262 def translation(domain, localedir=None, languages=None,
 263                 class_=None, fallback=0):
 264     if class_ is None:
 265         class_ = GNUTranslations
 266     mofiles = find(domain, localedir, languages, all=1)
 267     if len(mofiles)==0:
 268         if fallback:
 269             return NullTranslations()
 270         raise IOError(ENOENT, 'No translation file found for domain', domain)
 271     # TBD: do we need to worry about the file pointer getting collected?
 272     # Avoid opening, reading, and parsing the .mo file after it's been done
 273     # once.
 274     result = None
 275     for mofile in mofiles:
 276         key = os.path.abspath(mofile)
 277         t = _translations.get(key)
 278         if t is None:
 279             t = _translations.setdefault(key, class_(open(mofile, 'rb')))
 280         # Copy the translation object to allow setting fallbacks.
 281         # All other instance data is shared with the cached object.
 282         t = copy.copy(t)
 283         if result is None:
 284             result = t
 285         else:
 286             result.add_fallback(t)
 287     return result
 288
 289
 290 def install(domain, localedir=None, unicode=0):
 291     translation(domain, localedir, fallback=1).install(unicode)
 292
 293
 294
 295 # a mapping b/w domains and locale directories
 296 _localedirs = {}
 297 # current global domain, `messages' used for compatibility w/ GNU gettext
 298 _current_domain = 'messages'
 299
 300
 301 def textdomain(domain=None):
 302     global _current_domain
 303     if domain is not None:
 304         _current_domain = domain
 305     return _current_domain
 306
 307
 308 def bindtextdomain(domain, localedir=None):
 309     global _localedirs
 310     if localedir is not None:
 311         _localedirs[domain] = localedir
 312     return _localedirs.get(domain, _default_localedir)
 313
 314
 315 def dgettext(domain, message):
 316     try:
 317         t = translation(domain, _localedirs.get(domain, None))
 318     except IOError:
 319         return message
 320     return t.gettext(message)
 321
 322
 323 def gettext(message):
 324     return dgettext(_current_domain, message)
 325
 326
 327 # dcgettext() has been deemed unnecessary and is not implemented.
 328
 329 # James Henstridge's Catalog constructor from GNOME gettext.  Documented usage
 330 # was:
 331 #
 332 #    import gettext
 333 #    cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR)
 334 #    _ = cat.gettext
 335 #    print _('Hello World')
 336
 337 # The resulting catalog object currently don't support access through a
 338 # dictionary API, which was supported (but apparently unused) in GNOME
 339 # gettext.
 340
 341 Catalog = translation