third_party/simplejson/decoder.py

   1 """Implementation of JSONDecoder
   2 """
   3 import re
   4 import sys
   5 import struct
   6
   7 from simplejson.scanner import make_scanner
   8 def _import_c_scanstring():
   9     try:
  10         from simplejson._speedups import scanstring
  11         return scanstring
  12     except ImportError:
  13         return None
  14 c_scanstring = _import_c_scanstring()
  15
  16 __all__ = ['JSONDecoder']
  17
  18 FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
  19
  20 def _floatconstants():
  21     _BYTES = '7FF80000000000007FF0000000000000'.decode('hex')
  22     # The struct module in Python 2.4 would get frexp() out of range here
  23     # when an endian is specified in the format string. Fixed in Python 2.5+
  24     if sys.byteorder != 'big':
  25         _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1]
  26     nan, inf = struct.unpack('dd', _BYTES)
  27     return nan, inf, -inf
  28
  29 NaN, PosInf, NegInf = _floatconstants()
  30
  31
  32 class JSONDecodeError(ValueError):
  33     """Subclass of ValueError with the following additional properties:
  34
  35     msg: The unformatted error message
  36     doc: The JSON document being parsed
  37     pos: The start index of doc where parsing failed
  38     end: The end index of doc where parsing failed (may be None)
  39     lineno: The line corresponding to pos
  40     colno: The column corresponding to pos
  41     endlineno: The line corresponding to end (may be None)
  42     endcolno: The column corresponding to end (may be None)
  43
  44     """
  45     def __init__(self, msg, doc, pos, end=None):
  46         ValueError.__init__(self, errmsg(msg, doc, pos, end=end))
  47         self.msg = msg
  48         self.doc = doc
  49         self.pos = pos
  50         self.end = end
  51         self.lineno, self.colno = linecol(doc, pos)
  52         if end is not None:
  53             self.endlineno, self.endcolno = linecol(doc, end)
  54         else:
  55             self.endlineno, self.endcolno = None, None
  56
  57
  58 def linecol(doc, pos):
  59     lineno = doc.count('\n', 0, pos) + 1
  60     if lineno == 1:
  61         colno = pos
  62     else:
  63         colno = pos - doc.rindex('\n', 0, pos)
  64     return lineno, colno
  65
  66
  67 def errmsg(msg, doc, pos, end=None):
  68     # Note that this function is called from _speedups
  69     lineno, colno = linecol(doc, pos)
  70     if end is None:
  71         #fmt = '{0}: line {1} column {2} (char {3})'
  72         #return fmt.format(msg, lineno, colno, pos)
  73         fmt = '%s: line %d column %d (char %d)'
  74         return fmt % (msg, lineno, colno, pos)
  75     endlineno, endcolno = linecol(doc, end)
  76     #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})'
  77     #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end)
  78     fmt = '%s: line %d column %d - line %d column %d (char %d - %d)'
  79     return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end)
  80
  81
  82 _CONSTANTS = {
  83     '-Infinity': NegInf,
  84     'Infinity': PosInf,
  85     'NaN': NaN,
  86 }
  87
  88 STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
  89 BACKSLASH = {
  90     '"': u'"', '\\': u'\\', '/': u'/',
  91     'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
  92 }
  93
  94 DEFAULT_ENCODING = "utf-8"
  95
  96 def py_scanstring(s, end, encoding=None, strict=True,
  97         _b=BACKSLASH, _m=STRINGCHUNK.match):
  98     """Scan the string s for a JSON string. End is the index of the
  99     character in s after the quote that started the JSON string.
 100     Unescapes all valid JSON string escape sequences and raises ValueError
 101     on attempt to decode an invalid string. If strict is False then literal
 102     control characters are allowed in the string.
 103
 104     Returns a tuple of the decoded string and the index of the character in s
 105     after the end quote."""
 106     if encoding is None:
 107         encoding = DEFAULT_ENCODING
 108     chunks = []
 109     _append = chunks.append
 110     begin = end - 1
 111     while 1:
 112         chunk = _m(s, end)
 113         if chunk is None:
 114             raise JSONDecodeError(
 115                 "Unterminated string starting at", s, begin)
 116         end = chunk.end()
 117         content, terminator = chunk.groups()
 118         # Content is contains zero or more unescaped string characters
 119         if content:
 120             if not isinstance(content, unicode):
 121                 content = unicode(content, encoding)
 122             _append(content)
 123         # Terminator is the end of string, a literal control character,
 124         # or a backslash denoting that an escape sequence follows
 125         if terminator == '"':
 126             break
 127         elif terminator != '\\':
 128             if strict:
 129                 msg = "Invalid control character %r at" % (terminator,)
 130                 #msg = "Invalid control character {0!r} at".format(terminator)
 131                 raise JSONDecodeError(msg, s, end)
 132             else:
 133                 _append(terminator)
 134                 continue
 135         try:
 136             esc = s[end]
 137         except IndexError:
 138             raise JSONDecodeError(
 139                 "Unterminated string starting at", s, begin)
 140         # If not a unicode escape sequence, must be in the lookup table
 141         if esc != 'u':
 142             try:
 143                 char = _b[esc]
 144             except KeyError:
 145                 msg = "Invalid \\escape: " + repr(esc)
 146                 raise JSONDecodeError(msg, s, end)
 147             end += 1
 148         else:
 149             # Unicode escape sequence
 150             esc = s[end + 1:end + 5]
 151             next_end = end + 5
 152             if len(esc) != 4:
 153                 msg = "Invalid \\uXXXX escape"
 154                 raise JSONDecodeError(msg, s, end)
 155             uni = int(esc, 16)
 156             # Check for surrogate pair on UCS-4 systems
 157             if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
 158                 msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
 159                 if not s[end + 5:end + 7] == '\\u':
 160                     raise JSONDecodeError(msg, s, end)
 161                 esc2 = s[end + 7:end + 11]
 162                 if len(esc2) != 4:
 163                     raise JSONDecodeError(msg, s, end)
 164                 uni2 = int(esc2, 16)
 165                 uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
 166                 next_end += 6
 167             char = unichr(uni)
 168             end = next_end
 169         # Append the unescaped character
 170         _append(char)
 171     return u''.join(chunks), end
 172
 173
 174 # Use speedup if available
 175 scanstring = c_scanstring or py_scanstring
 176
 177 WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS)
 178 WHITESPACE_STR = ' \t\n\r'
 179
 180 def JSONObject((s, end), encoding, strict, scan_once, object_hook,
 181         object_pairs_hook, memo=None,
 182         _w=WHITESPACE.match, _ws=WHITESPACE_STR):
 183     # Backwards compatibility
 184     if memo is None:
 185         memo = {}
 186     memo_get = memo.setdefault
 187     pairs = []
 188     # Use a slice to prevent IndexError from being raised, the following
 189     # check will raise a more specific ValueError if the string is empty
 190     nextchar = s[end:end + 1]
 191     # Normally we expect nextchar == '"'
 192     if nextchar != '"':
 193         if nextchar in _ws:
 194             end = _w(s, end).end()
 195             nextchar = s[end:end + 1]
 196         # Trivial empty object
 197         if nextchar == '}':
 198             if object_pairs_hook is not None:
 199                 result = object_pairs_hook(pairs)
 200                 return result, end + 1
 201             pairs = {}
 202             if object_hook is not None:
 203                 pairs = object_hook(pairs)
 204             return pairs, end + 1
 205         elif nextchar != '"':
 206             raise JSONDecodeError(
 207                 "Expecting property name enclosed in double quotes",
 208                 s, end)
 209     end += 1
 210     while True:
 211         key, end = scanstring(s, end, encoding, strict)
 212         key = memo_get(key, key)
 213
 214         # To skip some function call overhead we optimize the fast paths where
 215         # the JSON key separator is ": " or just ":".
 216         if s[end:end + 1] != ':':
 217             end = _w(s, end).end()
 218             if s[end:end + 1] != ':':
 219                 raise JSONDecodeError("Expecting ':' delimiter", s, end)
 220
 221         end += 1
 222
 223         try:
 224             if s[end] in _ws:
 225                 end += 1
 226                 if s[end] in _ws:
 227                     end = _w(s, end + 1).end()
 228         except IndexError:
 229             pass
 230
 231         try:
 232             value, end = scan_once(s, end)
 233         except StopIteration:
 234             raise JSONDecodeError("Expecting object", s, end)
 235         pairs.append((key, value))
 236
 237         try:
 238             nextchar = s[end]
 239             if nextchar in _ws:
 240                 end = _w(s, end + 1).end()
 241                 nextchar = s[end]
 242         except IndexError:
 243             nextchar = ''
 244         end += 1
 245
 246         if nextchar == '}':
 247             break
 248         elif nextchar != ',':
 249             raise JSONDecodeError("Expecting ',' delimiter", s, end - 1)
 250
 251         try:
 252             nextchar = s[end]
 253             if nextchar in _ws:
 254                 end += 1
 255                 nextchar = s[end]
 256                 if nextchar in _ws:
 257                     end = _w(s, end + 1).end()
 258                     nextchar = s[end]
 259         except IndexError:
 260             nextchar = ''
 261
 262         end += 1
 263         if nextchar != '"':
 264             raise JSONDecodeError(
 265                 "Expecting property name enclosed in double quotes",
 266                 s, end - 1)
 267
 268     if object_pairs_hook is not None:
 269         result = object_pairs_hook(pairs)
 270         return result, end
 271     pairs = dict(pairs)
 272     if object_hook is not None:
 273         pairs = object_hook(pairs)
 274     return pairs, end
 275
 276 def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
 277     values = []
 278     nextchar = s[end:end + 1]
 279     if nextchar in _ws:
 280         end = _w(s, end + 1).end()
 281         nextchar = s[end:end + 1]
 282     # Look-ahead for trivial empty array
 283     if nextchar == ']':
 284         return values, end + 1
 285     _append = values.append
 286     while True:
 287         try:
 288             value, end = scan_once(s, end)
 289         except StopIteration:
 290             raise JSONDecodeError("Expecting object", s, end)
 291         _append(value)
 292         nextchar = s[end:end + 1]
 293         if nextchar in _ws:
 294             end = _w(s, end + 1).end()
 295             nextchar = s[end:end + 1]
 296         end += 1
 297         if nextchar == ']':
 298             break
 299         elif nextchar != ',':
 300             raise JSONDecodeError("Expecting ',' delimiter", s, end)
 301
 302         try:
 303             if s[end] in _ws:
 304                 end += 1
 305                 if s[end] in _ws:
 306                     end = _w(s, end + 1).end()
 307         except IndexError:
 308             pass
 309
 310     return values, end
 311
 312 class JSONDecoder(object):
 313     """Simple JSON <http://json.org> decoder
 314
 315     Performs the following translations in decoding by default:
 316
 317     +---------------+-------------------+
 318     | JSON          | Python            |
 319     +===============+===================+
 320     | object        | dict              |
 321     +---------------+-------------------+
 322     | array         | list              |
 323     +---------------+-------------------+
 324     | string        | unicode           |
 325     +---------------+-------------------+
 326     | number (int)  | int, long         |
 327     +---------------+-------------------+
 328     | number (real) | float             |
 329     +---------------+-------------------+
 330     | true          | True              |
 331     +---------------+-------------------+
 332     | false         | False             |
 333     +---------------+-------------------+
 334     | null          | None              |
 335     +---------------+-------------------+
 336
 337     It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
 338     their corresponding ``float`` values, which is outside the JSON spec.
 339
 340     """
 341
 342     def __init__(self, encoding=None, object_hook=None, parse_float=None,
 343             parse_int=None, parse_constant=None, strict=True,
 344             object_pairs_hook=None):
 345         """
 346         *encoding* determines the encoding used to interpret any
 347         :class:`str` objects decoded by this instance (``'utf-8'`` by
 348         default).  It has no effect when decoding :class:`unicode` objects.
 349
 350         Note that currently only encodings that are a superset of ASCII work,
 351         strings of other encodings should be passed in as :class:`unicode`.
 352
 353         *object_hook*, if specified, will be called with the result of every
 354         JSON object decoded and its return value will be used in place of the
 355         given :class:`dict`.  This can be used to provide custom
 356         deserializations (e.g. to support JSON-RPC class hinting).
 357
 358         *object_pairs_hook* is an optional function that will be called with
 359         the result of any object literal decode with an ordered list of pairs.
 360         The return value of *object_pairs_hook* will be used instead of the
 361         :class:`dict`.  This feature can be used to implement custom decoders
 362         that rely on the order that the key and value pairs are decoded (for
 363         example, :func:`collections.OrderedDict` will remember the order of
 364         insertion). If *object_hook* is also defined, the *object_pairs_hook*
 365         takes priority.
 366
 367         *parse_float*, if specified, will be called with the string of every
 368         JSON float to be decoded.  By default, this is equivalent to
 369         ``float(num_str)``. This can be used to use another datatype or parser
 370         for JSON floats (e.g. :class:`decimal.Decimal`).
 371
 372         *parse_int*, if specified, will be called with the string of every
 373         JSON int to be decoded.  By default, this is equivalent to
 374         ``int(num_str)``.  This can be used to use another datatype or parser
 375         for JSON integers (e.g. :class:`float`).
 376
 377         *parse_constant*, if specified, will be called with one of the
 378         following strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``.  This
 379         can be used to raise an exception if invalid JSON numbers are
 380         encountered.
 381
 382         *strict* controls the parser's behavior when it encounters an
 383         invalid control character in a string. The default setting of
 384         ``True`` means that unescaped control characters are parse errors, if
 385         ``False`` then control characters will be allowed in strings.
 386
 387         """
 388         self.encoding = encoding
 389         self.object_hook = object_hook
 390         self.object_pairs_hook = object_pairs_hook
 391         self.parse_float = parse_float or float
 392         self.parse_int = parse_int or int
 393         self.parse_constant = parse_constant or _CONSTANTS.__getitem__
 394         self.strict = strict
 395         self.parse_object = JSONObject
 396         self.parse_array = JSONArray
 397         self.parse_string = scanstring
 398         self.memo = {}
 399         self.scan_once = make_scanner(self)
 400
 401     def decode(self, s, _w=WHITESPACE.match):
 402         """Return the Python representation of ``s`` (a ``str`` or ``unicode``
 403         instance containing a JSON document)
 404
 405         """
 406         obj, end = self.raw_decode(s)
 407         end = _w(s, end).end()
 408         if end != len(s):
 409             raise JSONDecodeError("Extra data", s, end, len(s))
 410         return obj
 411
 412     def raw_decode(self, s, idx=0, _w=WHITESPACE.match):
 413         """Decode a JSON document from ``s`` (a ``str`` or ``unicode``
 414         beginning with a JSON document) and return a 2-tuple of the Python
 415         representation and the index in ``s`` where the document ended.
 416         Optionally, ``idx`` can be used to specify an offset in ``s`` where
 417         the JSON document begins.
 418
 419         This can be used to decode a JSON document from a string that may
 420         have extraneous data at the end.
 421
 422         """
 423         try:
 424             obj, end = self.scan_once(s, idx=_w(s, idx).end())
 425         except StopIteration:
 426             raise JSONDecodeError("No JSON object could be decoded", s, idx)
 427         return obj, end