Lib/gettext.py

   1 """Internationalization and localization support.
   2
   3 This module provides internationalization (I18N) and localization (L10N)
   4 support for your Python programs by providing an interface to the GNU gettext
   5 message catalog library.
   6
   7 I18N refers to the operation by which a program is made aware of multiple
   8 languages.  L10N refers to the adaptation of your program, once
   9 internationalized, to the local language and cultural habits.
  10
  11 """
  12
  13 # This module represents the integration of work, contributions, feedback, and
  14 # suggestions from the following people:
  15 #
  16 # Martin von Loewis, who wrote the initial implementation of the underlying
  17 # C-based libintlmodule (later renamed _gettext), along with a skeletal
  18 # gettext.py implementation.
  19 #
  20 # Peter Funk, who wrote fintl.py, a fairly complete wrapper around intlmodule,
  21 # which also included a pure-Python implementation to read .mo files if
  22 # intlmodule wasn't available.
  23 #
  24 # James Henstridge, who also wrote a gettext.py module, which has some
  25 # interesting, but currently unsupported experimental features: the notion of
  26 # a Catalog class and instances, and the ability to add to a catalog file via
  27 # a Python API.
  28 #
  29 # Barry Warsaw integrated these modules, wrote the .install() API and code,
  30 # and conformed all C and Python code to Python's coding standards.
  31 #
  32 # Francois Pinard and Marc-Andre Lemburg also contributed valuably to this
  33 # module.
  34 #
  35 # TODO:
  36 # - Lazy loading of .mo files.  Currently the entire catalog is loaded into
  37 #   memory, but that's probably bad for large translated programs.  Instead,
  38 #   the lexical sort of original strings in GNU .mo files should be exploited
  39 #   to do binary searches and lazy initializations.  Or you might want to use
  40 #   the undocumented double-hash algorithm for .mo files with hash tables, but
  41 #   you'll need to study the GNU gettext code to do this.
  42 #
  43 # - Support Solaris .mo file formats.  Unfortunately, we've been unable to
  44 #   find this format documented anywhere.
  45
  46 import os
  47 import sys
  48 import struct
  49 from errno import ENOENT
  50
  51 _default_localedir = os.path.join(sys.prefix, 'share', 'locale')
  52
  53
  54 \f
  55 def _expand_lang(locale):
  56     from locale import normalize
  57     locale = normalize(locale)
  58     COMPONENT_CODESET   = 1 << 0
  59     COMPONENT_TERRITORY = 1 << 1
  60     COMPONENT_MODIFIER  = 1 << 2
  61     # split up the locale into its base components
  62     mask = 0
  63     pos = locale.find('@')
  64     if pos >= 0:
  65         modifier = locale[pos:]
  66         locale = locale[:pos]
  67         mask |= COMPONENT_MODIFIER
  68     else:
  69         modifier = ''
  70     pos = locale.find('.')
  71     if pos >= 0:
  72         codeset = locale[pos:]
  73         locale = locale[:pos]
  74         mask |= COMPONENT_CODESET
  75     else:
  76         codeset = ''
  77     pos = locale.find('_')
  78     if pos >= 0:
  79         territory = locale[pos:]
  80         locale = locale[:pos]
  81         mask |= COMPONENT_TERRITORY
  82     else:
  83         territory = ''
  84     language = locale
  85     ret = []
  86     for i in range(mask+1):
  87         if not (i & ~mask):  # if all components for this combo exist ...
  88             val = language
  89             if i & COMPONENT_TERRITORY: val += territory
  90             if i & COMPONENT_CODESET:   val += codeset
  91             if i & COMPONENT_MODIFIER:  val += modifier
  92             ret.append(val)
  93     ret.reverse()
  94     return ret
  95
  96
  97 \f
  98 class NullTranslations:
  99     def __init__(self, fp=None):
 100         self._info = {}
 101         self._charset = None
 102         if fp:
 103             self._parse(fp)
 104
 105     def _parse(self, fp):
 106         pass
 107
 108     def gettext(self, message):
 109         return message
 110
 111     def ugettext(self, message):
 112         return unicode(message)
 113
 114     def info(self):
 115         return self._info
 116
 117     def charset(self):
 118         return self._charset
 119
 120     def install(self, unicode=0):
 121         import __builtin__
 122         __builtin__.__dict__['_'] = unicode and self.ugettext or self.gettext
 123
 124
 125 class GNUTranslations(NullTranslations):
 126     # Magic number of .mo files
 127     LE_MAGIC = 0x950412de
 128     BE_MAGIC = 0xde120495
 129
 130     def _parse(self, fp):
 131         """Override this method to support alternative .mo formats."""
 132         # We need to & all 32 bit unsigned integers with 0xffffffff for
 133         # portability to 64 bit machines.
 134         MASK = 0xffffffff
 135         unpack = struct.unpack
 136         filename = getattr(fp, 'name', '')
 137         # Parse the .mo file header, which consists of 5 little endian 32
 138         # bit words.
 139         self._catalog = catalog = {}
 140         buf = fp.read()
 141         buflen = len(buf)
 142         # Are we big endian or little endian?
 143         magic = unpack('<i', buf[:4])[0] & MASK
 144         if magic == self.LE_MAGIC:
 145             version, msgcount, masteridx, transidx = unpack('<4i', buf[4:20])
 146             ii = '<ii'
 147         elif magic == self.BE_MAGIC:
 148             version, msgcount, masteridx, transidx = unpack('>4i', buf[4:20])
 149             ii = '>ii'
 150         else:
 151             raise IOError(0, 'Bad magic number', filename)
 152         # more unsigned ints
 153         msgcount &= MASK
 154         masteridx &= MASK
 155         transidx &= MASK
 156         # Now put all messages from the .mo file buffer into the catalog
 157         # dictionary.
 158         for i in xrange(0, msgcount):
 159             mlen, moff = unpack(ii, buf[masteridx:masteridx+8])
 160             moff &= MASK
 161             mend = moff + (mlen & MASK)
 162             tlen, toff = unpack(ii, buf[transidx:transidx+8])
 163             toff &= MASK
 164             tend = toff + (tlen & MASK)
 165             if mend < buflen and tend < buflen:
 166                 tmsg = buf[toff:tend]
 167                 catalog[buf[moff:mend]] = tmsg
 168             else:
 169                 raise IOError(0, 'File is corrupt', filename)
 170             # See if we're looking at GNU .mo conventions for metadata
 171             if mlen == 0 and tmsg.lower().startswith('project-id-version:'):
 172                 # Catalog description
 173                 for item in tmsg.split('\n'):
 174                     item = item.strip()
 175                     if not item:
 176                         continue
 177                     k, v = item.split(':', 1)
 178                     k = k.strip().lower()
 179                     v = v.strip()
 180                     self._info[k] = v
 181                     if k == 'content-type':
 182                         self._charset = v.split('charset=')[1]
 183             # advance to next entry in the seek tables
 184             masteridx += 8
 185             transidx += 8
 186
 187     def gettext(self, message):
 188         return self._catalog.get(message, message)
 189
 190     def ugettext(self, message):
 191         tmsg = self._catalog.get(message, message)
 192         return unicode(tmsg, self._charset)
 193
 194
 195 \f
 196 # Locate a .mo file using the gettext strategy
 197 def find(domain, localedir=None, languages=None):
 198     # Get some reasonable defaults for arguments that were not supplied
 199     if localedir is None:
 200         localedir = _default_localedir
 201     if languages is None:
 202         languages = []
 203         for envar in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'):
 204             val = os.environ.get(envar)
 205             if val:
 206                 languages = val.split(':')
 207                 break
 208         if 'C' not in languages:
 209             languages.append('C')
 210     # now normalize and expand the languages
 211     langdict = {}
 212     for lang in languages:
 213         for nelang in _expand_lang(lang):
 214             langdict[nelang] = nelang
 215     languages = langdict.keys()
 216     # select a language
 217     for lang in languages:
 218         if lang == 'C':
 219             break
 220         mofile = os.path.join(localedir, lang, 'LC_MESSAGES', '%s.mo' % domain)
 221         if os.path.exists(mofile):
 222             return mofile
 223     return None
 224
 225
 226 \f
 227 # a mapping between absolute .mo file path and Translation object
 228 _translations = {}
 229
 230 def translation(domain, localedir=None, languages=None, class_=None):
 231     if class_ is None:
 232         class_ = GNUTranslations
 233     mofile = find(domain, localedir, languages)
 234     if mofile is None:
 235         raise IOError(ENOENT, 'No translation file found for domain', domain)
 236     key = os.path.abspath(mofile)
 237     # TBD: do we need to worry about the file pointer getting collected?
 238     t = _translations.setdefault(key, class_(open(mofile, 'rb')))
 239     return t
 240
 241
 242 \f
 243 def install(domain, localedir=None, unicode=0):
 244     translation(domain, localedir).install(unicode)
 245
 246
 247 \f
 248 # a mapping b/w domains and locale directories
 249 _localedirs = {}
 250 # current global domain, `messages' used for compatibility w/ GNU gettext
 251 _current_domain = 'messages'
 252
 253
 254 def textdomain(domain=None):
 255     global _current_domain
 256     if domain is not None:
 257         _current_domain = domain
 258     return _current_domain
 259
 260
 261 def bindtextdomain(domain, localedir=None):
 262     global _localedirs
 263     if localedir is not None:
 264         _localedirs[domain] = localedir
 265     return _localedirs.get(domain, _default_localedir)
 266
 267
 268 def dgettext(domain, message):
 269     try:
 270         t = translation(domain, _localedirs.get(domain, None))
 271     except IOError:
 272         return message
 273     return t.gettext(message)
 274
 275
 276 def gettext(message):
 277     return dgettext(_current_domain, message)
 278
 279
 280 # dcgettext() has been deemed unnecessary and is not implemented.
 281
 282 # James Henstridge's Catalog constructor from GNOME gettext.  Documented usage
 283 # was:
 284 #
 285 #    import gettext
 286 #    cat = gettext.Catalog(PACKAGE, localedir=LOCALEDIR)
 287 #    _ = cat.gettext
 288 #    print _('Hello World')
 289
 290 # The resulting catalog object currently don't support access through a
 291 # dictionary API, which was supported (but apparently unused) in GNOME
 292 # gettext.
 293
 294 Catalog = translation