external_libraries/simplejson-2.3.2/decoder.py

   1 """Implementation of JSONDecoder
   2 """
   3 import re
   4 import sys
   5 import struct
   6
   7 from .scanner import make_scanner
   8 def _import_c_scanstring():
   9     try:
  10         from ._speedups import scanstring
  11         return scanstring
  12     except ImportError:
  13         return None
  14 c_scanstring = _import_c_scanstring()
  15
  16 __all__ = ['JSONDecoder']
  17
  18 FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
  19
  20 def _floatconstants():
  21     _BYTES = '7FF80000000000007FF0000000000000'.decode('hex')
  22     # The struct module in Python 2.4 would get frexp() out of range here
  23     # when an endian is specified in the format string. Fixed in Python 2.5+
  24     if sys.byteorder != 'big':
  25         _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1]
  26     nan, inf = struct.unpack('dd', _BYTES)
  27     return nan, inf, -inf
  28
  29 NaN, PosInf, NegInf = _floatconstants()
  30
  31
  32 class JSONDecodeError(ValueError):
  33     """Subclass of ValueError with the following additional properties:
  34
  35     msg: The unformatted error message
  36     doc: The JSON document being parsed
  37     pos: The start index of doc where parsing failed
  38     end: The end index of doc where parsing failed (may be None)
  39     lineno: The line corresponding to pos
  40     colno: The column corresponding to pos
  41     endlineno: The line corresponding to end (may be None)
  42     endcolno: The column corresponding to end (may be None)
  43
  44     """
  45     def __init__(self, msg, doc, pos, end=None):
  46         ValueError.__init__(self, errmsg(msg, doc, pos, end=end))
  47         self.msg = msg
  48         self.doc = doc
  49         self.pos = pos
  50         self.end = end
  51         self.lineno, self.colno = linecol(doc, pos)
  52         if end is not None:
  53             self.endlineno, self.endcolno = linecol(doc, end)
  54         else:
  55             self.endlineno, self.endcolno = None, None
  56
  57
  58 def linecol(doc, pos):
  59     lineno = doc.count('\n', 0, pos) + 1
  60     if lineno == 1:
  61         colno = pos
  62     else:
  63         colno = pos - doc.rindex('\n', 0, pos)
  64     return lineno, colno
  65
  66
  67 def errmsg(msg, doc, pos, end=None):
  68     # Note that this function is called from _speedups
  69     lineno, colno = linecol(doc, pos)
  70     if end is None:
  71         #fmt = '{0}: line {1} column {2} (char {3})'
  72         #return fmt.format(msg, lineno, colno, pos)
  73         fmt = '%s: line %d column %d (char %d)'
  74         return fmt % (msg, lineno, colno, pos)
  75     endlineno, endcolno = linecol(doc, end)
  76     #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})'
  77     #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end)
  78     fmt = '%s: line %d column %d - line %d column %d (char %d - %d)'
  79     return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end)
  80
  81
  82 _CONSTANTS = {
  83     '-Infinity': NegInf,
  84     'Infinity': PosInf,
  85     'NaN': NaN,
  86 }
  87
  88 STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
  89 BACKSLASH = {
  90     '"': u'"', '\\': u'\\', '/': u'/',
  91     'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
  92 }
  93
  94 DEFAULT_ENCODING = "utf-8"
  95
  96 def py_scanstring(s, end, encoding=None, strict=True,
  97         _b=BACKSLASH, _m=STRINGCHUNK.match):
  98     """Scan the string s for a JSON string. End is the index of the
  99     character in s after the quote that started the JSON string.
 100     Unescapes all valid JSON string escape sequences and raises ValueError
 101     on attempt to decode an invalid string. If strict is False then literal
 102     control characters are allowed in the string.
 103
 104     Returns a tuple of the decoded string and the index of the character in s
 105     after the end quote."""
 106     if encoding is None:
 107         encoding = DEFAULT_ENCODING
 108     chunks = []
 109     _append = chunks.append
 110     begin = end - 1
 111     while 1:
 112         chunk = _m(s, end)
 113         if chunk is None:
 114             raise JSONDecodeError(
 115                 "Unterminated string starting at", s, begin)
 116         end = chunk.end()
 117         content, terminator = chunk.groups()
 118         # Content is contains zero or more unescaped string characters
 119         if content:
 120             if not isinstance(content, unicode):
 121                 content = unicode(content, encoding)
 122             _append(content)
 123         # Terminator is the end of string, a literal control character,
 124         # or a backslash denoting that an escape sequence follows
 125         if terminator == '"':
 126             break
 127         elif terminator != '\\':
 128             if strict:
 129                 msg = "Invalid control character %r at" % (terminator,)
 130                 #msg = "Invalid control character {0!r} at".format(terminator)
 131                 raise JSONDecodeError(msg, s, end)
 132             else:
 133                 _append(terminator)
 134                 continue
 135         try:
 136             esc = s[end]
 137         except IndexError:
 138             raise JSONDecodeError(
 139                 "Unterminated string starting at", s, begin)
 140         # If not a unicode escape sequence, must be in the lookup table
 141         if esc != 'u':
 142             try:
 143                 char = _b[esc]
 144             except KeyError:
 145                 msg = "Invalid \\escape: " + repr(esc)
 146                 raise JSONDecodeError(msg, s, end)
 147             end += 1
 148         else:
 149             # Unicode escape sequence
 150             esc = s[end + 1:end + 5]
 151             next_end = end + 5
 152             if len(esc) != 4:
 153                 msg = "Invalid \\uXXXX escape"
 154                 raise JSONDecodeError(msg, s, end)
 155             uni = int(esc, 16)
 156             # Check for surrogate pair on UCS-4 systems
 157             if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
 158                 msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
 159                 if not s[end + 5:end + 7] == '\\u':
 160                     raise JSONDecodeError(msg, s, end)
 161                 esc2 = s[end + 7:end + 11]
 162                 if len(esc2) != 4:
 163                     raise JSONDecodeError(msg, s, end)
 164                 uni2 = int(esc2, 16)
 165                 uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
 166                 next_end += 6
 167             char = unichr(uni)
 168             end = next_end
 169         # Append the unescaped character
 170         _append(char)
 171     return u''.join(chunks), end
 172
 173
 174 # Use speedup if available
 175 scanstring = c_scanstring or py_scanstring
 176
 177 WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS)
 178 WHITESPACE_STR = ' \t\n\r'
 179
 180 def JSONObject((s, end), encoding, strict, scan_once, object_hook,
 181         object_pairs_hook, memo=None,
 182         _w=WHITESPACE.match, _ws=WHITESPACE_STR):
 183     # Backwards compatibility
 184     if memo is None:
 185         memo = {}
 186     memo_get = memo.setdefault
 187     pairs = []
 188     # Use a slice to prevent IndexError from being raised, the following
 189     # check will raise a more specific ValueError if the string is empty
 190     nextchar = s[end:end + 1]
 191     # Normally we expect nextchar == '"'
 192     if nextchar != '"':
 193         if nextchar in _ws:
 194             end = _w(s, end).end()
 195             nextchar = s[end:end + 1]
 196         # Trivial empty object
 197         if nextchar == '}':
 198             if object_pairs_hook is not None:
 199                 result = object_pairs_hook(pairs)
 200                 return result, end + 1
 201             pairs = {}
 202             if object_hook is not None:
 203                 pairs = object_hook(pairs)
 204             return pairs, end + 1
 205         elif nextchar != '"':
 206             raise JSONDecodeError("Expecting property name", s, end)
 207     end += 1
 208     while True:
 209         key, end = scanstring(s, end, encoding, strict)
 210         key = memo_get(key, key)
 211
 212         # To skip some function call overhead we optimize the fast paths where
 213         # the JSON key separator is ": " or just ":".
 214         if s[end:end + 1] != ':':
 215             end = _w(s, end).end()
 216             if s[end:end + 1] != ':':
 217                 raise JSONDecodeError("Expecting : delimiter", s, end)
 218
 219         end += 1
 220
 221         try:
 222             if s[end] in _ws:
 223                 end += 1
 224                 if s[end] in _ws:
 225                     end = _w(s, end + 1).end()
 226         except IndexError:
 227             pass
 228
 229         try:
 230             value, end = scan_once(s, end)
 231         except StopIteration:
 232             raise JSONDecodeError("Expecting object", s, end)
 233         pairs.append((key, value))
 234
 235         try:
 236             nextchar = s[end]
 237             if nextchar in _ws:
 238                 end = _w(s, end + 1).end()
 239                 nextchar = s[end]
 240         except IndexError:
 241             nextchar = ''
 242         end += 1
 243
 244         if nextchar == '}':
 245             break
 246         elif nextchar != ',':
 247             raise JSONDecodeError("Expecting , delimiter", s, end - 1)
 248
 249         try:
 250             nextchar = s[end]
 251             if nextchar in _ws:
 252                 end += 1
 253                 nextchar = s[end]
 254                 if nextchar in _ws:
 255                     end = _w(s, end + 1).end()
 256                     nextchar = s[end]
 257         except IndexError:
 258             nextchar = ''
 259
 260         end += 1
 261         if nextchar != '"':
 262             raise JSONDecodeError("Expecting property name", s, end - 1)
 263
 264     if object_pairs_hook is not None:
 265         result = object_pairs_hook(pairs)
 266         return result, end
 267     pairs = dict(pairs)
 268     if object_hook is not None:
 269         pairs = object_hook(pairs)
 270     return pairs, end
 271
 272 def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
 273     values = []
 274     nextchar = s[end:end + 1]
 275     if nextchar in _ws:
 276         end = _w(s, end + 1).end()
 277         nextchar = s[end:end + 1]
 278     # Look-ahead for trivial empty array
 279     if nextchar == ']':
 280         return values, end + 1
 281     _append = values.append
 282     while True:
 283         try:
 284             value, end = scan_once(s, end)
 285         except StopIteration:
 286             raise JSONDecodeError("Expecting object", s, end)
 287         _append(value)
 288         nextchar = s[end:end + 1]
 289         if nextchar in _ws:
 290             end = _w(s, end + 1).end()
 291             nextchar = s[end:end + 1]
 292         end += 1
 293         if nextchar == ']':
 294             break
 295         elif nextchar != ',':
 296             raise JSONDecodeError("Expecting , delimiter", s, end)
 297
 298         try:
 299             if s[end] in _ws:
 300                 end += 1
 301                 if s[end] in _ws:
 302                     end = _w(s, end + 1).end()
 303         except IndexError:
 304             pass
 305
 306     return values, end
 307
 308 class JSONDecoder(object):
 309     """Simple JSON <http://json.org> decoder
 310
 311     Performs the following translations in decoding by default:
 312
 313     +---------------+-------------------+
 314     | JSON          | Python            |
 315     +===============+===================+
 316     | object        | dict              |
 317     +---------------+-------------------+
 318     | array         | list              |
 319     +---------------+-------------------+
 320     | string        | unicode           |
 321     +---------------+-------------------+
 322     | number (int)  | int, long         |
 323     +---------------+-------------------+
 324     | number (real) | float             |
 325     +---------------+-------------------+
 326     | true          | True              |
 327     +---------------+-------------------+
 328     | false         | False             |
 329     +---------------+-------------------+
 330     | null          | None              |
 331     +---------------+-------------------+
 332
 333     It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
 334     their corresponding ``float`` values, which is outside the JSON spec.
 335
 336     """
 337
 338     def __init__(self, encoding=None, object_hook=None, parse_float=None,
 339             parse_int=None, parse_constant=None, strict=True,
 340             object_pairs_hook=None):
 341         """
 342         *encoding* determines the encoding used to interpret any
 343         :class:`str` objects decoded by this instance (``'utf-8'`` by
 344         default).  It has no effect when decoding :class:`unicode` objects.
 345
 346         Note that currently only encodings that are a superset of ASCII work,
 347         strings of other encodings should be passed in as :class:`unicode`.
 348
 349         *object_hook*, if specified, will be called with the result of every
 350         JSON object decoded and its return value will be used in place of the
 351         given :class:`dict`.  This can be used to provide custom
 352         deserializations (e.g. to support JSON-RPC class hinting).
 353
 354         *object_pairs_hook* is an optional function that will be called with
 355         the result of any object literal decode with an ordered list of pairs.
 356         The return value of *object_pairs_hook* will be used instead of the
 357         :class:`dict`.  This feature can be used to implement custom decoders
 358         that rely on the order that the key and value pairs are decoded (for
 359         example, :func:`collections.OrderedDict` will remember the order of
 360         insertion). If *object_hook* is also defined, the *object_pairs_hook*
 361         takes priority.
 362
 363         *parse_float*, if specified, will be called with the string of every
 364         JSON float to be decoded.  By default, this is equivalent to
 365         ``float(num_str)``. This can be used to use another datatype or parser
 366         for JSON floats (e.g. :class:`decimal.Decimal`).
 367
 368         *parse_int*, if specified, will be called with the string of every
 369         JSON int to be decoded.  By default, this is equivalent to
 370         ``int(num_str)``.  This can be used to use another datatype or parser
 371         for JSON integers (e.g. :class:`float`).
 372
 373         *parse_constant*, if specified, will be called with one of the
 374         following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``.  This
 375         can be used to raise an exception if invalid JSON numbers are
 376         encountered.
 377
 378         *strict* controls the parser's behavior when it encounters an
 379         invalid control character in a string. The default setting of
 380         ``True`` means that unescaped control characters are parse errors, if
 381         ``False`` then control characters will be allowed in strings.
 382
 383         """
 384         self.encoding = encoding
 385         self.object_hook = object_hook
 386         self.object_pairs_hook = object_pairs_hook
 387         self.parse_float = parse_float or float
 388         self.parse_int = parse_int or int
 389         self.parse_constant = parse_constant or _CONSTANTS.__getitem__
 390         self.strict = strict
 391         self.parse_object = JSONObject
 392         self.parse_array = JSONArray
 393         self.parse_string = scanstring
 394         self.memo = {}
 395         self.scan_once = make_scanner(self)
 396
 397     def decode(self, s, _w=WHITESPACE.match):
 398         """Return the Python representation of ``s`` (a ``str`` or ``unicode``
 399         instance containing a JSON document)
 400
 401         """
 402         obj, end = self.raw_decode(s, idx=_w(s, 0).end())
 403         end = _w(s, end).end()
 404         if end != len(s):
 405             raise JSONDecodeError("Extra data", s, end, len(s))
 406         return obj
 407
 408     def raw_decode(self, s, idx=0):
 409         """Decode a JSON document from ``s`` (a ``str`` or ``unicode``
 410         beginning with a JSON document) and return a 2-tuple of the Python
 411         representation and the index in ``s`` where the document ended.
 412
 413         This can be used to decode a JSON document from a string that may
 414         have extraneous data at the end.
 415
 416         """
 417         try:
 418             obj, end = self.scan_once(s, idx)
 419         except StopIteration:
 420             raise JSONDecodeError("No JSON object could be decoded", s, idx)
 421         return obj, end