Lib/email/Charset.py

   1 # Copyright (C) 2001-2004 Python Software Foundation
   2 # Author: che@debian.org (Ben Gertzfield), barry@python.org (Barry Warsaw)
   3
   4 # XXX The following information needs updating.
   5
   6 # Python 2.3 doesn't come with any Asian codecs by default.  Two packages are
   7 # currently available and supported as of this writing (30-Dec-2003):
   8 #
   9 # CJKCodecs
  10 # http://cjkpython.i18n.org
  11 # This package contains Chinese, Japanese, and Korean codecs
  12
  13 # JapaneseCodecs
  14 # http://www.asahi-net.or.jp/~rd6t-kjym/python
  15 # Some Japanese users prefer this codec package
  16
  17 import email.base64MIME
  18 import email.quopriMIME
  19 from email.Encoders import encode_7or8bit
  20
  21
  22 \f
  23 # Flags for types of header encodings
  24 QP     = 1   # Quoted-Printable
  25 BASE64 = 2   # Base64
  26 SHORTEST = 3 # the shorter of QP and base64, but only for headers
  27
  28 # In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7
  29 MISC_LEN = 7
  30
  31 DEFAULT_CHARSET = 'us-ascii'
  32
  33
  34 \f
  35 # Defaults
  36 CHARSETS = {
  37     # input        header enc  body enc output conv
  38     'iso-8859-1':  (QP,        QP,      None),
  39     'iso-8859-2':  (QP,        QP,      None),
  40     'iso-8859-3':  (QP,        QP,      None),
  41     'iso-8859-4':  (QP,        QP,      None),
  42     # iso-8859-5 is Cyrillic, and not especially used
  43     # iso-8859-6 is Arabic, also not particularly used
  44     # iso-8859-7 is Greek, QP will not make it readable
  45     # iso-8859-8 is Hebrew, QP will not make it readable
  46     'iso-8859-9':  (QP,        QP,      None),
  47     'iso-8859-10': (QP,        QP,      None),
  48     # iso-8859-11 is Thai, QP will not make it readable
  49     'iso-8859-13': (QP,        QP,      None),
  50     'iso-8859-14': (QP,        QP,      None),
  51     'iso-8859-15': (QP,        QP,      None),
  52     'windows-1252':(QP,        QP,      None),
  53     'viscii':      (QP,        QP,      None),
  54     'us-ascii':    (None,      None,    None),
  55     'big5':        (BASE64,    BASE64,  None),
  56     'gb2312':      (BASE64,    BASE64,  None),
  57     'euc-jp':      (BASE64,    None,    'iso-2022-jp'),
  58     'shift_jis':   (BASE64,    None,    'iso-2022-jp'),
  59     'iso-2022-jp': (BASE64,    None,    None),
  60     'koi8-r':      (BASE64,    BASE64,  None),
  61     'utf-8':       (SHORTEST,  BASE64, 'utf-8'),
  62     # We're making this one up to represent raw unencoded 8-bit
  63     '8bit':        (None,      BASE64, 'utf-8'),
  64     }
  65
  66 # Aliases for other commonly-used names for character sets.  Map
  67 # them to the real ones used in email.
  68 ALIASES = {
  69     'latin_1': 'iso-8859-1',
  70     'latin-1': 'iso-8859-1',
  71     'latin_2': 'iso-8859-2',
  72     'latin-2': 'iso-8859-2',
  73     'latin_3': 'iso-8859-3',
  74     'latin-3': 'iso-8859-3',
  75     'latin_4': 'iso-8859-4',
  76     'latin-4': 'iso-8859-4',
  77     'latin_5': 'iso-8859-9',
  78     'latin-5': 'iso-8859-9',
  79     'latin_6': 'iso-8859-10',
  80     'latin-6': 'iso-8859-10',
  81     'latin_7': 'iso-8859-13',
  82     'latin-7': 'iso-8859-13',
  83     'latin_8': 'iso-8859-14',
  84     'latin-8': 'iso-8859-14',
  85     'latin_9': 'iso-8859-15',
  86     'latin-9': 'iso-8859-15',
  87     'cp949':   'ks_c_5601-1987',
  88     'euc_jp':  'euc-jp',
  89     'euc_kr':  'euc-kr',
  90     'ascii':   'us-ascii',
  91     }
  92
  93
  94 # Map charsets to their Unicode codec strings.
  95 CODEC_MAP = {
  96     'gb2312':      'eucgb2312_cn',
  97     'big5':        'big5_tw',
  98     # Hack: We don't want *any* conversion for stuff marked us-ascii, as all
  99     # sorts of garbage might be sent to us in the guise of 7-bit us-ascii.
 100     # Let that stuff pass through without conversion to/from Unicode.
 101     'us-ascii':    None,
 102     }
 103
 104
 105 \f
 106 # Convenience functions for extending the above mappings
 107 def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
 108     """Add character set properties to the global registry.
 109
 110     charset is the input character set, and must be the canonical name of a
 111     character set.
 112
 113     Optional header_enc and body_enc is either Charset.QP for
 114     quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for
 115     the shortest of qp or base64 encoding, or None for no encoding.  SHORTEST
 116     is only valid for header_enc.  It describes how message headers and
 117     message bodies in the input charset are to be encoded.  Default is no
 118     encoding.
 119
 120     Optional output_charset is the character set that the output should be
 121     in.  Conversions will proceed from input charset, to Unicode, to the
 122     output charset when the method Charset.convert() is called.  The default
 123     is to output in the same character set as the input.
 124
 125     Both input_charset and output_charset must have Unicode codec entries in
 126     the module's charset-to-codec mapping; use add_codec(charset, codecname)
 127     to add codecs the module does not know about.  See the codecs module's
 128     documentation for more information.
 129     """
 130     if body_enc == SHORTEST:
 131         raise ValueError, 'SHORTEST not allowed for body_enc'
 132     CHARSETS[charset] = (header_enc, body_enc, output_charset)
 133
 134
 135 def add_alias(alias, canonical):
 136     """Add a character set alias.
 137
 138     alias is the alias name, e.g. latin-1
 139     canonical is the character set's canonical name, e.g. iso-8859-1
 140     """
 141     ALIASES[alias] = canonical
 142
 143
 144 def add_codec(charset, codecname):
 145     """Add a codec that map characters in the given charset to/from Unicode.
 146
 147     charset is the canonical name of a character set.  codecname is the name
 148     of a Python codec, as appropriate for the second argument to the unicode()
 149     built-in, or to the encode() method of a Unicode string.
 150     """
 151     CODEC_MAP[charset] = codecname
 152
 153
 154 \f
 155 class Charset:
 156     """Map character sets to their email properties.
 157
 158     This class provides information about the requirements imposed on email
 159     for a specific character set.  It also provides convenience routines for
 160     converting between character sets, given the availability of the
 161     applicable codecs.  Given a character set, it will do its best to provide
 162     information on how to use that character set in an email in an
 163     RFC-compliant way.
 164
 165     Certain character sets must be encoded with quoted-printable or base64
 166     when used in email headers or bodies.  Certain character sets must be
 167     converted outright, and are not allowed in email.  Instances of this
 168     module expose the following information about a character set:
 169
 170     input_charset: The initial character set specified.  Common aliases
 171                    are converted to their `official' email names (e.g. latin_1
 172                    is converted to iso-8859-1).  Defaults to 7-bit us-ascii.
 173
 174     header_encoding: If the character set must be encoded before it can be
 175                      used in an email header, this attribute will be set to
 176                      Charset.QP (for quoted-printable), Charset.BASE64 (for
 177                      base64 encoding), or Charset.SHORTEST for the shortest of
 178                      QP or BASE64 encoding.  Otherwise, it will be None.
 179
 180     body_encoding: Same as header_encoding, but describes the encoding for the
 181                    mail message's body, which indeed may be different than the
 182                    header encoding.  Charset.SHORTEST is not allowed for
 183                    body_encoding.
 184
 185     output_charset: Some character sets must be converted before the can be
 186                     used in email headers or bodies.  If the input_charset is
 187                     one of them, this attribute will contain the name of the
 188                     charset output will be converted to.  Otherwise, it will
 189                     be None.
 190
 191     input_codec: The name of the Python codec used to convert the
 192                  input_charset to Unicode.  If no conversion codec is
 193                  necessary, this attribute will be None.
 194
 195     output_codec: The name of the Python codec used to convert Unicode
 196                   to the output_charset.  If no conversion codec is necessary,
 197                   this attribute will have the same value as the input_codec.
 198     """
 199     def __init__(self, input_charset=DEFAULT_CHARSET):
 200         # RFC 2046, $4.1.2 says charsets are not case sensitive
 201         input_charset = input_charset.lower()
 202         # Set the input charset after filtering through the aliases
 203         self.input_charset = ALIASES.get(input_charset, input_charset)
 204         # We can try to guess which encoding and conversion to use by the
 205         # charset_map dictionary.  Try that first, but let the user override
 206         # it.
 207         henc, benc, conv = CHARSETS.get(self.input_charset,
 208                                         (SHORTEST, BASE64, None))
 209         if not conv:
 210             conv = self.input_charset
 211         # Set the attributes, allowing the arguments to override the default.
 212         self.header_encoding = henc
 213         self.body_encoding = benc
 214         self.output_charset = ALIASES.get(conv, conv)
 215         # Now set the codecs.  If one isn't defined for input_charset,
 216         # guess and try a Unicode codec with the same name as input_codec.
 217         self.input_codec = CODEC_MAP.get(self.input_charset,
 218                                          self.input_charset)
 219         self.output_codec = CODEC_MAP.get(self.output_charset,
 220                                             self.output_charset)
 221
 222     def __str__(self):
 223         return self.input_charset.lower()
 224
 225     __repr__ = __str__
 226
 227     def __eq__(self, other):
 228         return str(self) == str(other).lower()
 229
 230     def __ne__(self, other):
 231         return not self.__eq__(other)
 232
 233     def get_body_encoding(self):
 234         """Return the content-transfer-encoding used for body encoding.
 235
 236         This is either the string `quoted-printable' or `base64' depending on
 237         the encoding used, or it is a function in which case you should call
 238         the function with a single argument, the Message object being
 239         encoded.  The function should then set the Content-Transfer-Encoding
 240         header itself to whatever is appropriate.
 241
 242         Returns "quoted-printable" if self.body_encoding is QP.
 243         Returns "base64" if self.body_encoding is BASE64.
 244         Returns "7bit" otherwise.
 245         """
 246         assert self.body_encoding <> SHORTEST
 247         if self.body_encoding == QP:
 248             return 'quoted-printable'
 249         elif self.body_encoding == BASE64:
 250             return 'base64'
 251         else:
 252             return encode_7or8bit
 253
 254     def convert(self, s):
 255         """Convert a string from the input_codec to the output_codec."""
 256         if self.input_codec <> self.output_codec:
 257             return unicode(s, self.input_codec).encode(self.output_codec)
 258         else:
 259             return s
 260
 261     def to_splittable(self, s):
 262         """Convert a possibly multibyte string to a safely splittable format.
 263
 264         Uses the input_codec to try and convert the string to Unicode, so it
 265         can be safely split on character boundaries (even for multibyte
 266         characters).
 267
 268         Returns the string as-is if it isn't known how to convert it to
 269         Unicode with the input_charset.
 270
 271         Characters that could not be converted to Unicode will be replaced
 272         with the Unicode replacement character U+FFFD.
 273         """
 274         if isinstance(s, unicode) or self.input_codec is None:
 275             return s
 276         try:
 277             return unicode(s, self.input_codec, 'replace')
 278         except LookupError:
 279             # Input codec not installed on system, so return the original
 280             # string unchanged.
 281             return s
 282
 283     def from_splittable(self, ustr, to_output=True):
 284         """Convert a splittable string back into an encoded string.
 285
 286         Uses the proper codec to try and convert the string from Unicode back
 287         into an encoded format.  Return the string as-is if it is not Unicode,
 288         or if it could not be converted from Unicode.
 289
 290         Characters that could not be converted from Unicode will be replaced
 291         with an appropriate character (usually '?').
 292
 293         If to_output is True (the default), uses output_codec to convert to an
 294         encoded format.  If to_output is False, uses input_codec.
 295         """
 296         if to_output:
 297             codec = self.output_codec
 298         else:
 299             codec = self.input_codec
 300         if not isinstance(ustr, unicode) or codec is None:
 301             return ustr
 302         try:
 303             return ustr.encode(codec, 'replace')
 304         except LookupError:
 305             # Output codec not installed
 306             return ustr
 307
 308     def get_output_charset(self):
 309         """Return the output character set.
 310
 311         This is self.output_charset if that is not None, otherwise it is
 312         self.input_charset.
 313         """
 314         return self.output_charset or self.input_charset
 315
 316     def encoded_header_len(self, s):
 317         """Return the length of the encoded header string."""
 318         cset = self.get_output_charset()
 319         # The len(s) of a 7bit encoding is len(s)
 320         if self.header_encoding == BASE64:
 321             return email.base64MIME.base64_len(s) + len(cset) + MISC_LEN
 322         elif self.header_encoding == QP:
 323             return email.quopriMIME.header_quopri_len(s) + len(cset) + MISC_LEN
 324         elif self.header_encoding == SHORTEST:
 325             lenb64 = email.base64MIME.base64_len(s)
 326             lenqp = email.quopriMIME.header_quopri_len(s)
 327             return min(lenb64, lenqp) + len(cset) + MISC_LEN
 328         else:
 329             return len(s)
 330
 331     def header_encode(self, s, convert=False):
 332         """Header-encode a string, optionally converting it to output_charset.
 333
 334         If convert is True, the string will be converted from the input
 335         charset to the output charset automatically.  This is not useful for
 336         multibyte character sets, which have line length issues (multibyte
 337         characters must be split on a character, not a byte boundary); use the
 338         high-level Header class to deal with these issues.  convert defaults
 339         to False.
 340
 341         The type of encoding (base64 or quoted-printable) will be based on
 342         self.header_encoding.
 343         """
 344         cset = self.get_output_charset()
 345         if convert:
 346             s = self.convert(s)
 347         # 7bit/8bit encodings return the string unchanged (modulo conversions)
 348         if self.header_encoding == BASE64:
 349             return email.base64MIME.header_encode(s, cset)
 350         elif self.header_encoding == QP:
 351             return email.quopriMIME.header_encode(s, cset, maxlinelen=None)
 352         elif self.header_encoding == SHORTEST:
 353             lenb64 = email.base64MIME.base64_len(s)
 354             lenqp = email.quopriMIME.header_quopri_len(s)
 355             if lenb64 < lenqp:
 356                 return email.base64MIME.header_encode(s, cset)
 357             else:
 358                 return email.quopriMIME.header_encode(s, cset, maxlinelen=None)
 359         else:
 360             return s
 361
 362     def body_encode(self, s, convert=True):
 363         """Body-encode a string and convert it to output_charset.
 364
 365         If convert is True (the default), the string will be converted from
 366         the input charset to output charset automatically.  Unlike
 367         header_encode(), there are no issues with byte boundaries and
 368         multibyte charsets in email bodies, so this is usually pretty safe.
 369
 370         The type of encoding (base64 or quoted-printable) will be based on
 371         self.body_encoding.
 372         """
 373         if convert:
 374             s = self.convert(s)
 375         # 7bit/8bit encodings return the string unchanged (module conversions)
 376         if self.body_encoding is BASE64:
 377             return email.base64MIME.body_encode(s)
 378         elif self.body_encoding is QP:
 379             return email.quopriMIME.body_encode(s)
 380         else:
 381             return s