Lib/gettext.py

   1 """Internationalization and localization support.
   2
   3 This module provides internationalization (I18N) and localization (L10N)
   4 support for your Python programs by providing an interface to the GNU gettext
   5 message catalog library.
   6
   7 I18N refers to the operation by which a program is made aware of multiple
   8 languages.  L10N refers to the adaptation of your program, once
   9 internationalized, to the local language and cultural habits.
  10
  11 """
  12
  13 # This module represents the integration of work, contributions, feedback, and
  14 # suggestions from the following people:
  15 #
  16 # Martin von Loewis, who wrote the initial implementation of the underlying
  17 # C-based libintlmodule (later renamed _gettext), along with a skeletal
  18 # gettext.py implementation.
  19 #
  20 # Peter Funk, who wrote fintl.py, a fairly complete wrapper around intlmodule,
  21 # which also included a pure-Python implementation to read .mo files if
  22 # intlmodule wasn't available.
  23 #
  24 # James Henstridge, who also wrote a gettext.py module, which has some
  25 # interesting, but currently unsupported experimental features: the notion of
  26 # a Catalog class and instances, and the ability to add to a catalog file via
  27 # a Python API.
  28 #
  29 # Barry Warsaw integrated these modules, wrote the .install() API and code,
  30 # and conformed all C and Python code to Python's coding standards.
  31 #
  32 # Francois Pinard and Marc-Andre Lemburg also contributed valuably to this
  33 # module.
  34 #
  35 # TODO:
  36 # - Lazy loading of .mo files.  Currently the entire catalog is loaded into
  37 #   memory, but that's probably bad for large translated programs.  Instead,
  38 #   the lexical sort of original strings in GNU .mo files should be exploited
  39 #   to do binary searches and lazy initializations.  Or you might want to use
  40 #   the undocumented double-hash algorithm for .mo files with hash tables, but
  41 #   you'll need to study the GNU gettext code to do this.
  42 #
  43 # - Support Solaris .mo file formats.  Unfortunately, we've been unable to
  44 #   find this format documented anywhere.
  45
  46 import os
  47 import sys
  48 import struct
  49 from errno import ENOENT
  50
  51 __all__ = ["bindtextdomain","textdomain","gettext","dgettext",
  52            "find","translation","install","Catalog"]
  53
  54 _default_localedir = os.path.join(sys.prefix, 'share', 'locale')
  55
  56
  57
  58 def _expand_lang(locale):
  59     from locale import normalize
  60     locale = normalize(locale)
  61     COMPONENT_CODESET   = 1 << 0
  62     COMPONENT_TERRITORY = 1 << 1
  63     COMPONENT_MODIFIER  = 1 << 2
  64     # split up the locale into its base components
  65     mask = 0
  66     pos = locale.find('@')
  67     if pos >= 0:
  68         modifier = locale[pos:]
  69         locale = locale[:pos]
  70         mask |= COMPONENT_MODIFIER
  71     else:
  72         modifier = ''
  73     pos = locale.find('.')
  74     if pos >= 0:
  75         codeset = locale[pos:]
  76         locale = locale[:pos]
  77         mask |= COMPONENT_CODESET
  78     else:
  79         codeset = ''
  80     pos = locale.find('_')
  81     if pos >= 0:
  82         territory = locale[pos:]
  83         locale = locale[:pos]
  84         mask |= COMPONENT_TERRITORY
  85     else:
  86         territory = ''
  87     language = locale
  88     ret = []
  89     for i in range(mask+1):
  90         if not (i & ~mask):  # if all components for this combo exist ...
  91             val = language
  92             if i & COMPONENT_TERRITORY: val += territory
  93             if i & COMPONENT_CODESET:   val += codeset
  94             if i & COMPONENT_MODIFIER:  val += modifier
  95             ret.append(val)
  96     ret.reverse()
  97     return ret
  98
  99
 100
 101 class NullTranslations:
 102     def __init__(self, fp=None):
 103         self._info = {}
 104         self._charset = None
 105         if fp:
 106             self._parse(fp)
 107
 108     def _parse(self, fp):
 109         pass
 110
 111     def gettext(self, message):
 112         return message
 113
 114     def ugettext(self, message):
 115         return unicode(message)
 116
 117     def info(self):
 118         return self._info
 119
 120     def charset(self):
 121         return self._charset
 122
 123     def install(self, unicode=0):
 124         import __builtin__
 125         __builtin__.__dict__['_'] = unicode and self.ugettext or self.gettext
 126
 127
 128 class GNUTranslations(NullTranslations):
 129     # Magic number of .mo files
 130     LE_MAGIC = 0x950412de
 131     BE_MAGIC = 0xde120495
 132
 133     def _parse(self, fp):
 134         """Override this method to support alternative .mo formats."""
 135         # We need to & all 32 bit unsigned integers with 0xffffffff for
 136         # portability to 64 bit machines.
 137         MASK = 0xffffffff
 138         unpack = struct.unpack
 139         filename = getattr(fp, 'name', '')
 140         # Parse the .mo file header, which consists of 5 little endian 32
 141         # bit words.
 142         self._catalog = catalog = {}
 143         buf = fp.read()
 144         buflen = len(buf)
 145         # Are we big endian or little endian?
 146         magic = unpack('<i', buf[:4])[0] & MASK
 147         if magic == self.LE_MAGIC:
 148             version, msgcount, masteridx, transidx = unpack('<4i', buf[4:20])
 149             ii = '<ii'
 150         elif magic == self.BE_MAGIC:
 151             version, msgcount, masteridx, transidx = unpack('>4i', buf[4:20])
 152             ii = '>ii'
 153         else:
 154             raise IOError(0, 'Bad magic number', filename)
 155         # more unsigned ints
 156         msgcount &= MASK
 157         masteridx &= MASK
 158         transidx &= MASK
 159         # Now put all messages from the .mo file buffer into the catalog
 160         # dictionary.
 161         for i in xrange(0, msgcount):
 162             mlen, moff = unpack(ii, buf[masteridx:masteridx+8])
 163             moff &= MASK
 164             mend = moff + (mlen & MASK)
 165             tlen, toff = unpack(ii, buf[transidx:transidx+8])
 166             toff &= MASK
 167             tend = toff + (tlen & MASK)
 168             if mend < buflen and tend < buflen:
 169                 tmsg = buf[toff:tend]
 170                 catalog[buf[moff:mend]] = tmsg
 171             else:
 172                 raise IOError(0, 'File is corrupt', filename)
 173             # See if we're looking at GNU .mo conventions for metadata
 174             if mlen == 0 and tmsg.lower().startswith('project-id-version:'):
 175                 # Catalog description
 176                 for item in tmsg.split('\n'):
 177                     item = item.strip()
 178                     if not item:
 179                         continue
 180                     k, v = item.split(':', 1)
 181                     k = k.strip().lower()
 182                     v = v.strip()
 183                     self._info[k] = v
 184                     if k == 'content-type':
 185                         self._charset = v.split('charset=')[1]
 186             # advance to next entry in the seek tables
 187             masteridx += 8
 188             transidx += 8
 189
 190     def gettext(self, message):
 191         return self._catalog.get(message, message)
 192
 193     def ugettext(self, message):
 194         tmsg = self._catalog.get(message, message)
 195         return unicode(tmsg, self._charset)
 196
 197
 198
 199 # Locate a .mo file using the gettext strategy
 200 def find(domain, localedir=None, languages=None):
 201     # Get some reasonable defaults for arguments that were not supplied
 202     if localedir is None:
 203         localedir = _default_localedir
 204     if languages is None:
 205         languages = []
 206         for envar in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'):
 207             val = os.environ.get(envar)
 208             if val:
 209                 languages = val.split(':')
 210                 break
 211         if 'C' not in languages:
 212             languages.append('C')
 213     # now normalize and expand the languages
 214     nelangs = []
 215     for lang in languages:
 216         for nelang in _expand_lang(lang):
 217             if nelang not in nelangs:
 218                 nelangs.append(nelang)
 219     # select a language
 220     for lang in nelangs:
 221         if lang == 'C':
 222             break
 223         mofile = os.path.join(localedir, lang, 'LC_MESSAGES', '%s.mo' % domain)
 224         if os.path.exists(mofile):
 225             return mofile
 226     return None
 227
 228
 229
 230 # a mapping between absolute .mo file path and Translation object
 231 _translations = {}
 232
 233 def translation(domain, localedir=None, languages=None, class_=None):
 234     if class_ is None:
 235         class_ = GNUTranslations
 236     mofile = find(domain, localedir, languages)
 237     if mofile is None:
 238         raise IOError(ENOENT, 'No translation file found for domain', domain)
 239     key = os.path.abspath(mofile)
 240     # TBD: do we need to worry about the file pointer getting collected?
 241     # Avoid opening, reading, and parsing the .mo file after it's been done
 242     # once.
 243     t = _translations.get(key)
 244     if t is None:
 245         t = _translations.setdefault(key, class_(open(mofile, 'rb')))
 246     return t
 247
 248
 249
 250 def install(domain, localedir=None, unicode=0):
 251     translation(domain, localedir).install(unicode)
 252
 253
 254
 255 # a mapping b/w domains and locale directories
 256 _localedirs = {}
 257 # current global domain, `messages' used for compatibility w/ GNU gettext
 258 _current_domain = 'messages'
 259
 260
 261 def textdomain(domain=None):
 262     global _current_domain
 263     if domain is not None:
 264         _current_domain = domain
 265     return _current_domain
 266
 267
 268 def bindtextdomain(domain, localedir=None):
 269     global _localedirs
 270     if localedir is not None:
 271         _localedirs[domain] = localedir
 272     return _localedirs.get(domain, _default_localedir)
 273
 274
 275 def dgettext(domain, message):
 276     try:
 277         t = translation(domain, _localedirs.get(domain, None))
 278     except IOError:
 279         return message
 280     return t.gettext(message)
 281
 282
 283 def gettext(message):
 284     return dgettext(_current_domain, message)
 285
 286
 287 # dcgettext() has been deemed unnecessary and is not implemented.
 288
 289 # James Henstridge's Catalog constructor from GNOME gettext.  Documented usage
 290 # was:
 291 #
 292 #    import gettext
 293 #    cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR)
 294 #    _ = cat.gettext
 295 #    print _('Hello World')
 296
 297 # The resulting catalog object currently don't support access through a
 298 # dictionary API, which was supported (but apparently unused) in GNOME
 299 # gettext.
 300
 301 Catalog = translation