Lib/email/Charset.py

   1 # Copyright (C) 2001,2002 Python Software Foundation
   2 # Author: che@debian.org (Ben Gertzfield), barry@zope.com (Barry Warsaw)
   3
   4 from types import UnicodeType
   5 from email.Encoders import encode_7or8bit
   6 import email.base64MIME
   7 import email.quopriMIME
   8
   9 def _isunicode(s):
  10     return isinstance(s, UnicodeType)
  11
  12 # Python 2.2.1 and beyond has these symbols
  13 try:
  14     True, False
  15 except NameError:
  16     True = 1
  17     False = 0
  18
  19
  20 \f
  21 # Flags for types of header encodings
  22 QP     = 1   # Quoted-Printable
  23 BASE64 = 2   # Base64
  24 SHORTEST = 3 # the shorter of QP and base64, but only for headers
  25
  26 # In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7
  27 MISC_LEN = 7
  28
  29 DEFAULT_CHARSET = 'us-ascii'
  30
  31
  32 \f
  33 # Defaults
  34 CHARSETS = {
  35     # input        header enc  body enc output conv
  36     'iso-8859-1':  (QP,        QP,      None),
  37     'iso-8859-2':  (QP,        QP,      None),
  38     'us-ascii':    (None,      None,    None),
  39     'big5':        (BASE64,    BASE64,  None),
  40     'gb2312':      (BASE64,    BASE64,  None),
  41     'euc-jp':      (BASE64,    None,    'iso-2022-jp'),
  42     'shift_jis':   (BASE64,    None,    'iso-2022-jp'),
  43     'iso-2022-jp': (BASE64,    None,    None),
  44     'koi8-r':      (BASE64,    BASE64,  None),
  45     'utf-8':       (SHORTEST,  BASE64, 'utf-8'),
  46     # We're making this one up to represent raw unencoded 8-bit
  47     '8bit':        (None,      BASE64, 'utf-8'),
  48     }
  49
  50 # Aliases for other commonly-used names for character sets.  Map
  51 # them to the real ones used in email.
  52 ALIASES = {
  53     'latin_1': 'iso-8859-1',
  54     'latin-1': 'iso-8859-1',
  55     'ascii':   'us-ascii',
  56     }
  57
  58 # Map charsets to their Unicode codec strings.  Note that Python doesn't come
  59 # with any Asian codecs by default.  Here's where to get them:
  60 #
  61 # Japanese -- http://www.asahi-net.or.jp/~rd6t-kjym/python
  62 # Korean   -- http://sf.net/projects/koco
  63 # Chinese  -- http://sf.net/projects/python-codecs
  64 #
  65 # Note that these codecs have their own lifecycle and may be in varying states
  66 # of stability and useability.
  67
  68 CODEC_MAP = {
  69     'euc-jp':      'japanese.euc-jp',
  70     'iso-2022-jp': 'japanese.iso-2022-jp',
  71     'shift_jis':   'japanese.shift_jis',
  72     'gb2132':      'eucgb2312_cn',
  73     'big5':        'big5_tw',
  74     'utf-8':       'utf-8',
  75     # Hack: We don't want *any* conversion for stuff marked us-ascii, as all
  76     # sorts of garbage might be sent to us in the guise of 7-bit us-ascii.
  77     # Let that stuff pass through without conversion to/from Unicode.
  78     'us-ascii':    None,
  79     }
  80
  81
  82 \f
  83 # Convenience functions for extending the above mappings
  84 def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
  85     """Add character set properties to the global registry.
  86
  87     charset is the input character set, and must be the canonical name of a
  88     character set.
  89
  90     Optional header_enc and body_enc is either Charset.QP for
  91     quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for
  92     the shortest of qp or base64 encoding, or None for no encoding.  SHORTEST
  93     is only valid for header_enc.  It describes how message headers and
  94     message bodies in the input charset are to be encoded.  Default is no
  95     encoding.
  96
  97     Optional output_charset is the character set that the output should be
  98     in.  Conversions will proceed from input charset, to Unicode, to the
  99     output charset when the method Charset.convert() is called.  The default
 100     is to output in the same character set as the input.
 101
 102     Both input_charset and output_charset must have Unicode codec entries in
 103     the module's charset-to-codec mapping; use add_codec(charset, codecname)
 104     to add codecs the module does not know about.  See the codecs module's
 105     documentation for more information.
 106     """
 107     if body_enc == SHORTEST:
 108         raise ValueError, 'SHORTEST not allowed for body_enc'
 109     CHARSETS[charset] = (header_enc, body_enc, output_charset)
 110
 111
 112 def add_alias(alias, canonical):
 113     """Add a character set alias.
 114
 115     alias is the alias name, e.g. latin-1
 116     canonical is the character set's canonical name, e.g. iso-8859-1
 117     """
 118     ALIASES[alias] = canonical
 119
 120
 121 def add_codec(charset, codecname):
 122     """Add a codec that map characters in the given charset to/from Unicode.
 123
 124     charset is the canonical name of a character set.  codecname is the name
 125     of a Python codec, as appropriate for the second argument to the unicode()
 126     built-in, or to the encode() method of a Unicode string.
 127     """
 128     CODEC_MAP[charset] = codecname
 129
 130
 131 \f
 132 class Charset:
 133     """Map character sets to their email properties.
 134
 135     This class provides information about the requirements imposed on email
 136     for a specific character set.  It also provides convenience routines for
 137     converting between character sets, given the availability of the
 138     applicable codecs.  Given a character set, it will do its best to provide
 139     information on how to use that character set in an email in an
 140     RFC-compliant way.
 141
 142     Certain character sets must be encoded with quoted-printable or base64
 143     when used in email headers or bodies.  Certain character sets must be
 144     converted outright, and are not allowed in email.  Instances of this
 145     module expose the following information about a character set:
 146
 147     input_charset: The initial character set specified.  Common aliases
 148                    are converted to their `official' email names (e.g. latin_1
 149                    is converted to iso-8859-1).  Defaults to 7-bit us-ascii.
 150
 151     header_encoding: If the character set must be encoded before it can be
 152                      used in an email header, this attribute will be set to
 153                      Charset.QP (for quoted-printable), Charset.BASE64 (for
 154                      base64 encoding), or Charset.SHORTEST for the shortest of
 155                      QP or BASE64 encoding.  Otherwise, it will be None.
 156
 157     body_encoding: Same as header_encoding, but describes the encoding for the
 158                    mail message's body, which indeed may be different than the
 159                    header encoding.  Charset.SHORTEST is not allowed for
 160                    body_encoding.
 161
 162     output_charset: Some character sets must be converted before the can be
 163                     used in email headers or bodies.  If the input_charset is
 164                     one of them, this attribute will contain the name of the
 165                     charset output will be converted to.  Otherwise, it will
 166                     be None.
 167
 168     input_codec: The name of the Python codec used to convert the
 169                  input_charset to Unicode.  If no conversion codec is
 170                  necessary, this attribute will be None.
 171
 172     output_codec: The name of the Python codec used to convert Unicode
 173                   to the output_charset.  If no conversion codec is necessary,
 174                   this attribute will have the same value as the input_codec.
 175     """
 176     def __init__(self, input_charset=DEFAULT_CHARSET):
 177         # RFC 2046, $4.1.2 says charsets are not case sensitive
 178         input_charset = input_charset.lower()
 179         # Set the input charset after filtering through the aliases
 180         self.input_charset = ALIASES.get(input_charset, input_charset)
 181         # We can try to guess which encoding and conversion to use by the
 182         # charset_map dictionary.  Try that first, but let the user override
 183         # it.
 184         henc, benc, conv = CHARSETS.get(self.input_charset,
 185                                         (SHORTEST, BASE64, None))
 186         # Set the attributes, allowing the arguments to override the default.
 187         self.header_encoding = henc
 188         self.body_encoding = benc
 189         self.output_charset = ALIASES.get(conv, conv)
 190         # Now set the codecs.  If one isn't defined for input_charset,
 191         # guess and try a Unicode codec with the same name as input_codec.
 192         self.input_codec = CODEC_MAP.get(self.input_charset,
 193                                          self.input_charset)
 194         self.output_codec = CODEC_MAP.get(self.output_charset,
 195                                             self.input_codec)
 196
 197     def __str__(self):
 198         return self.input_charset.lower()
 199
 200     def __eq__(self, other):
 201         return str(self) == str(other).lower()
 202
 203     def __ne__(self, other):
 204         return not self.__eq__(other)
 205
 206     def get_body_encoding(self):
 207         """Return the content-transfer-encoding used for body encoding.
 208
 209         This is either the string `quoted-printable' or `base64' depending on
 210         the encoding used, or it is a function in which case you should call
 211         the function with a single argument, the Message object being
 212         encoded.  The function should then set the Content-Transfer-Encoding
 213         header itself to whatever is appropriate.
 214
 215         Returns "quoted-printable" if self.body_encoding is QP.
 216         Returns "base64" if self.body_encoding is BASE64.
 217         Returns "7bit" otherwise.
 218         """
 219         assert self.body_encoding <> SHORTEST
 220         if self.body_encoding == QP:
 221             return 'quoted-printable'
 222         elif self.body_encoding == BASE64:
 223             return 'base64'
 224         else:
 225             return encode_7or8bit
 226
 227     def convert(self, s):
 228         """Convert a string from the input_codec to the output_codec."""
 229         if self.input_codec <> self.output_codec:
 230             return unicode(s, self.input_codec).encode(self.output_codec)
 231         else:
 232             return s
 233
 234     def to_splittable(self, s):
 235         """Convert a possibly multibyte string to a safely splittable format.
 236
 237         Uses the input_codec to try and convert the string to Unicode, so it
 238         can be safely split on character boundaries (even for multibyte
 239         characters).
 240
 241         Returns the string as-is if it isn't known how to convert it to
 242         Unicode with the input_charset.
 243
 244         Characters that could not be converted to Unicode will be replaced
 245         with the Unicode replacement character U+FFFD.
 246         """
 247         if _isunicode(s) or self.input_codec is None:
 248             return s
 249         try:
 250             return unicode(s, self.input_codec, 'replace')
 251         except LookupError:
 252             # Input codec not installed on system, so return the original
 253             # string unchanged.
 254             return s
 255
 256     def from_splittable(self, ustr, to_output=True):
 257         """Convert a splittable string back into an encoded string.
 258
 259         Uses the proper codec to try and convert the string from Unicode back
 260         into an encoded format.  Return the string as-is if it is not Unicode,
 261         or if it could not be converted from Unicode.
 262
 263         Characters that could not be converted from Unicode will be replaced
 264         with an appropriate character (usually '?').
 265
 266         If to_output is True (the default), uses output_codec to convert to an
 267         encoded format.  If to_output is False, uses input_codec.
 268         """
 269         if to_output:
 270             codec = self.output_codec
 271         else:
 272             codec = self.input_codec
 273         if not _isunicode(ustr) or codec is None:
 274             return ustr
 275         try:
 276             return ustr.encode(codec, 'replace')
 277         except LookupError:
 278             # Output codec not installed
 279             return ustr
 280
 281     def get_output_charset(self):
 282         """Return the output character set.
 283
 284         This is self.output_charset if that is not None, otherwise it is
 285         self.input_charset.
 286         """
 287         return self.output_charset or self.input_charset
 288
 289     def encoded_header_len(self, s):
 290         """Return the length of the encoded header string."""
 291         cset = self.get_output_charset()
 292         # The len(s) of a 7bit encoding is len(s)
 293         if self.header_encoding == BASE64:
 294             return email.base64MIME.base64_len(s) + len(cset) + MISC_LEN
 295         elif self.header_encoding == QP:
 296             return email.quopriMIME.header_quopri_len(s) + len(cset) + MISC_LEN
 297         elif self.header_encoding == SHORTEST:
 298             lenb64 = email.base64MIME.base64_len(s)
 299             lenqp = email.quopriMIME.header_quopri_len(s)
 300             return min(lenb64, lenqp) + len(cset) + MISC_LEN
 301         else:
 302             return len(s)
 303
 304     def header_encode(self, s, convert=False):
 305         """Header-encode a string, optionally converting it to output_charset.
 306
 307         If convert is True, the string will be converted from the input
 308         charset to the output charset automatically.  This is not useful for
 309         multibyte character sets, which have line length issues (multibyte
 310         characters must be split on a character, not a byte boundary); use the
 311         high-level Header class to deal with these issues.  convert defaults
 312         to False.
 313
 314         The type of encoding (base64 or quoted-printable) will be based on
 315         self.header_encoding.
 316         """
 317         cset = self.get_output_charset()
 318         if convert:
 319             s = self.convert(s)
 320         # 7bit/8bit encodings return the string unchanged (modulo conversions)
 321         if self.header_encoding == BASE64:
 322             return email.base64MIME.header_encode(s, cset)
 323         elif self.header_encoding == QP:
 324             return email.quopriMIME.header_encode(s, cset)
 325         elif self.header_encoding == SHORTEST:
 326             lenb64 = email.base64MIME.base64_len(s)
 327             lenqp = email.quopriMIME.header_quopri_len(s)
 328             if lenb64 < lenqp:
 329                 return email.base64MIME.header_encode(s, cset)
 330             else:
 331                 return email.quopriMIME.header_encode(s, cset)
 332         else:
 333             return s
 334
 335     def body_encode(self, s, convert=True):
 336         """Body-encode a string and convert it to output_charset.
 337
 338         If convert is True (the default), the string will be converted from
 339         the input charset to output charset automatically.  Unlike
 340         header_encode(), there are no issues with byte boundaries and
 341         multibyte charsets in email bodies, so this is usually pretty safe.
 342
 343         The type of encoding (base64 or quoted-printable) will be based on
 344         self.body_encoding.
 345         """
 346         if convert:
 347             s = self.convert(s)
 348         # 7bit/8bit encodings return the string unchanged (module conversions)
 349         if self.body_encoding is BASE64:
 350             return email.base64MIME.body_encode(s)
 351         elif self.body_encoding is QP:
 352             return email.quopriMIME.body_encode(s)
 353         else:
 354             return s