base/json/json_parser.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/json/json_parser.h"
   6
   7 #include "base/float_util.h"
   8 #include "base/logging.h"
   9 #include "base/memory/scoped_ptr.h"
  10 #include "base/string_number_conversions.h"
  11 #include "base/string_piece.h"
  12 #include "base/string_util.h"
  13 #include "base/stringprintf.h"
  14 #include "base/third_party/icu/icu_utf.h"
  15 #include "base/utf_string_conversion_utils.h"
  16 #include "base/utf_string_conversions.h"
  17 #include "base/values.h"
  18
  19 namespace base {
  20 namespace internal {
  21
  22 namespace {
  23
  24 const int kStackMaxDepth = 100;
  25
  26 const int32 kExtendedASCIIStart = 0x80;
  27
  28 // This and the class below are used to own the JSON input string for when
  29 // string tokens are stored as StringPiece instead of std::string. This
  30 // optimization avoids about 2/3rds of string memory copies. The constructor
  31 // takes ownership of the input string. The real root value is Swap()ed into
  32 // the new instance.
  33 class DictionaryHiddenRootValue : public base::DictionaryValue {
  34  public:
  35   DictionaryHiddenRootValue(std::string* json, Value* root) : json_(json) {
  36     DCHECK(root->IsType(Value::TYPE_DICTIONARY));
  37     DictionaryValue::Swap(static_cast<DictionaryValue*>(root));
  38   }
  39
  40   virtual void Swap(DictionaryValue* other) OVERRIDE {
  41     DVLOG(1) << "Swap()ing a DictionaryValue inefficiently.";
  42
  43     // First deep copy to convert JSONStringValue to std::string and swap that
  44     // copy with |other|, which contains the new contents of |this|.
  45     scoped_ptr<base::DictionaryValue> copy(DeepCopy());
  46     copy->Swap(other);
  47
  48     // Then erase the contents of the current dictionary and swap in the
  49     // new contents, originally from |other|.
  50     Clear();
  51     json_.reset();
  52     DictionaryValue::Swap(copy.get());
  53   }
  54
  55   // Not overriding DictionaryValue::Remove because it just calls through to
  56   // the method below.
  57
  58   virtual bool RemoveWithoutPathExpansion(const std::string& key,
  59                                           Value** out) OVERRIDE {
  60     // If the caller won't take ownership of the removed value, just call up.
  61     if (!out)
  62       return DictionaryValue::RemoveWithoutPathExpansion(key, out);
  63
  64     DVLOG(1) << "Remove()ing from a DictionaryValue inefficiently.";
  65
  66     // Otherwise, remove the value while its still "owned" by this and copy it
  67     // to convert any JSONStringValues to std::string.
  68     Value* out_owned = NULL;
  69     if (!DictionaryValue::RemoveWithoutPathExpansion(key, &out_owned))
  70       return false;
  71
  72     *out = out_owned->DeepCopy();
  73     delete out_owned;
  74
  75     return true;
  76   }
  77
  78  private:
  79   scoped_ptr<std::string> json_;
  80
  81   DISALLOW_COPY_AND_ASSIGN(DictionaryHiddenRootValue);
  82 };
  83
  84 class ListHiddenRootValue : public base::ListValue {
  85  public:
  86   ListHiddenRootValue(std::string* json, Value* root) : json_(json) {
  87     DCHECK(root->IsType(Value::TYPE_LIST));
  88     ListValue::Swap(static_cast<ListValue*>(root));
  89   }
  90
  91   virtual void Swap(ListValue* other) OVERRIDE {
  92     DVLOG(1) << "Swap()ing a ListValue inefficiently.";
  93
  94     // First deep copy to convert JSONStringValue to std::string and swap that
  95     // copy with |other|, which contains the new contents of |this|.
  96     scoped_ptr<base::ListValue> copy(DeepCopy());
  97     copy->Swap(other);
  98
  99     // Then erase the contents of the current list and swap in the new contents,
 100     // originally from |other|.
 101     Clear();
 102     json_.reset();
 103     ListValue::Swap(copy.get());
 104   }
 105
 106   virtual bool Remove(size_t index, Value** out) OVERRIDE {
 107     // If the caller won't take ownership of the removed value, just call up.
 108     if (!out)
 109       return ListValue::Remove(index, out);
 110
 111     DVLOG(1) << "Remove()ing from a ListValue inefficiently.";
 112
 113     // Otherwise, remove the value while its still "owned" by this and copy it
 114     // to convert any JSONStringValues to std::string.
 115     Value* out_owned = NULL;
 116     if (!ListValue::Remove(index, &out_owned))
 117       return false;
 118
 119     *out = out_owned->DeepCopy();
 120     delete out_owned;
 121
 122     return true;
 123   }
 124
 125  private:
 126   scoped_ptr<std::string> json_;
 127
 128   DISALLOW_COPY_AND_ASSIGN(ListHiddenRootValue);
 129 };
 130
 131 // A variant on StringValue that uses StringPiece instead of copying the string
 132 // into the Value. This can only be stored in a child of hidden root (above),
 133 // otherwise the referenced string will not be guaranteed to outlive it.
 134 class JSONStringValue : public base::Value {
 135  public:
 136   explicit JSONStringValue(const base::StringPiece& piece)
 137       : Value(TYPE_STRING),
 138         string_piece_(piece) {
 139   }
 140
 141   // Overridden from base::Value:
 142   virtual bool GetAsString(std::string* out_value) const OVERRIDE {
 143     string_piece_.CopyToString(out_value);
 144     return true;
 145   }
 146   virtual bool GetAsString(string16* out_value) const OVERRIDE {
 147     *out_value = UTF8ToUTF16(string_piece_);
 148     return true;
 149   }
 150   virtual Value* DeepCopy() const OVERRIDE {
 151     return new StringValue(string_piece_.as_string());
 152   }
 153   virtual bool Equals(const Value* other) const OVERRIDE {
 154     std::string other_string;
 155     return other->IsType(TYPE_STRING) && other->GetAsString(&other_string) &&
 156         StringPiece(other_string) == string_piece_;
 157   }
 158
 159  private:
 160   // The location in the original input stream.
 161   base::StringPiece string_piece_;
 162
 163   DISALLOW_COPY_AND_ASSIGN(JSONStringValue);
 164 };
 165
 166 // Simple class that checks for maximum recursion/"stack overflow."
 167 class StackMarker {
 168  public:
 169   explicit StackMarker(int* depth) : depth_(depth) {
 170     ++(*depth_);
 171     DCHECK_LE(*depth_, kStackMaxDepth);
 172   }
 173   ~StackMarker() {
 174     --(*depth_);
 175   }
 176
 177   bool IsTooDeep() const {
 178     return *depth_ >= kStackMaxDepth;
 179   }
 180
 181  private:
 182   int* const depth_;
 183
 184   DISALLOW_COPY_AND_ASSIGN(StackMarker);
 185 };
 186
 187 }  // namespace
 188
 189 JSONParser::JSONParser(int options)
 190     : options_(options),
 191       start_pos_(NULL),
 192       pos_(NULL),
 193       end_pos_(NULL),
 194       index_(0),
 195       stack_depth_(0),
 196       line_number_(0),
 197       index_last_line_(0),
 198       error_code_(JSONReader::JSON_NO_ERROR),
 199       error_line_(0),
 200       error_column_(0) {
 201 }
 202
 203 JSONParser::~JSONParser() {
 204 }
 205
 206 Value* JSONParser::Parse(const StringPiece& input) {
 207   scoped_ptr<std::string> input_copy;
 208   // If the children of a JSON root can be detached, then hidden roots cannot
 209   // be used, so do not bother copying the input because StringPiece will not
 210   // be used anywhere.
 211   if (!(options_ & JSON_DETACHABLE_CHILDREN)) {
 212     input_copy.reset(new std::string(input.as_string()));
 213     start_pos_ = input_copy->data();
 214   } else {
 215     start_pos_ = input.data();
 216   }
 217   pos_ = start_pos_;
 218   end_pos_ = start_pos_ + input.length();
 219   index_ = 0;
 220   line_number_ = 1;
 221   index_last_line_ = 0;
 222
 223   error_code_ = JSONReader::JSON_NO_ERROR;
 224   error_line_ = 0;
 225   error_column_ = 0;
 226
 227   // When the input JSON string starts with a UTF-8 Byte-Order-Mark
 228   // <0xEF 0xBB 0xBF>, advance the start position to avoid the
 229   // ParseNextToken function mis-treating a Unicode BOM as an invalid
 230   // character and returning NULL.
 231   if (CanConsume(3) && static_cast<uint8>(*pos_) == 0xEF &&
 232       static_cast<uint8>(*(pos_ + 1)) == 0xBB &&
 233       static_cast<uint8>(*(pos_ + 2)) == 0xBF) {
 234     NextNChars(3);
 235   }
 236
 237   // Parse the first and any nested tokens.
 238   scoped_ptr<Value> root(ParseNextToken());
 239   if (!root.get())
 240     return NULL;
 241
 242   // Make sure the input stream is at an end.
 243   if (GetNextToken() != T_END_OF_INPUT) {
 244     if (!CanConsume(1) || (NextChar() && GetNextToken() != T_END_OF_INPUT)) {
 245       ReportError(JSONReader::JSON_UNEXPECTED_DATA_AFTER_ROOT, 1);
 246       return NULL;
 247     }
 248   }
 249
 250   // Dictionaries and lists can contain JSONStringValues, so wrap them in a
 251   // hidden root.
 252   if (!(options_ & JSON_DETACHABLE_CHILDREN)) {
 253     if (root->IsType(Value::TYPE_DICTIONARY)) {
 254       return new DictionaryHiddenRootValue(input_copy.release(), root.get());
 255     } else if (root->IsType(Value::TYPE_LIST)) {
 256       return new ListHiddenRootValue(input_copy.release(), root.get());
 257     } else if (root->IsType(Value::TYPE_STRING)) {
 258       // A string type could be a JSONStringValue, but because there's no
 259       // corresponding HiddenRootValue, the memory will be lost. Deep copy to
 260       // preserve it.
 261       return root->DeepCopy();
 262     }
 263   }
 264
 265   // All other values can be returned directly.
 266   return root.release();
 267 }
 268
 269 JSONReader::JsonParseError JSONParser::error_code() const {
 270   return error_code_;
 271 }
 272
 273 std::string JSONParser::GetErrorMessage() const {
 274   return FormatErrorMessage(error_line_, error_column_,
 275       JSONReader::ErrorCodeToString(error_code_));
 276 }
 277
 278 // StringBuilder ///////////////////////////////////////////////////////////////
 279
 280 JSONParser::StringBuilder::StringBuilder()
 281     : pos_(NULL),
 282       length_(0),
 283       string_(NULL) {
 284 }
 285
 286 JSONParser::StringBuilder::StringBuilder(const char* pos)
 287     : pos_(pos),
 288       length_(0),
 289       string_(NULL) {
 290 }
 291
 292 void JSONParser::StringBuilder::Swap(StringBuilder* other) {
 293   std::swap(other->string_, string_);
 294   std::swap(other->pos_, pos_);
 295   std::swap(other->length_, length_);
 296 }
 297
 298 JSONParser::StringBuilder::~StringBuilder() {
 299   delete string_;
 300 }
 301
 302 void JSONParser::StringBuilder::Append(const char& c) {
 303   DCHECK_GE(c, 0);
 304   DCHECK_LT(c, 128);
 305
 306   if (string_)
 307     string_->push_back(c);
 308   else
 309     ++length_;
 310 }
 311
 312 void JSONParser::StringBuilder::AppendString(const std::string& str) {
 313   DCHECK(string_);
 314   string_->append(str);
 315 }
 316
 317 void JSONParser::StringBuilder::Convert() {
 318   if (string_)
 319     return;
 320   string_  = new std::string(pos_, length_);
 321 }
 322
 323 bool JSONParser::StringBuilder::CanBeStringPiece() const {
 324   return !string_;
 325 }
 326
 327 StringPiece JSONParser::StringBuilder::AsStringPiece() {
 328   if (string_)
 329     return StringPiece();
 330   return StringPiece(pos_, length_);
 331 }
 332
 333 const std::string& JSONParser::StringBuilder::AsString() {
 334   if (!string_)
 335     Convert();
 336   return *string_;
 337 }
 338
 339 // JSONParser private //////////////////////////////////////////////////////////
 340
 341 inline bool JSONParser::CanConsume(int length) {
 342   return pos_ + length <= end_pos_;
 343 }
 344
 345 const char* JSONParser::NextChar() {
 346   DCHECK(CanConsume(1));
 347   ++index_;
 348   ++pos_;
 349   return pos_;
 350 }
 351
 352 void JSONParser::NextNChars(int n) {
 353   DCHECK(CanConsume(n));
 354   index_ += n;
 355   pos_ += n;
 356 }
 357
 358 JSONParser::Token JSONParser::GetNextToken() {
 359   EatWhitespaceAndComments();
 360   if (!CanConsume(1))
 361     return T_END_OF_INPUT;
 362
 363   switch (*pos_) {
 364     case '{':
 365       return T_OBJECT_BEGIN;
 366     case '}':
 367       return T_OBJECT_END;
 368     case '[':
 369       return T_ARRAY_BEGIN;
 370     case ']':
 371       return T_ARRAY_END;
 372     case '"':
 373       return T_STRING;
 374     case '0':
 375     case '1':
 376     case '2':
 377     case '3':
 378     case '4':
 379     case '5':
 380     case '6':
 381     case '7':
 382     case '8':
 383     case '9':
 384     case '-':
 385       return T_NUMBER;
 386     case 't':
 387       return T_BOOL_TRUE;
 388     case 'f':
 389       return T_BOOL_FALSE;
 390     case 'n':
 391       return T_NULL;
 392     case ',':
 393       return T_LIST_SEPARATOR;
 394     case ':':
 395       return T_OBJECT_PAIR_SEPARATOR;
 396     default:
 397       return T_INVALID_TOKEN;
 398   }
 399 }
 400
 401 void JSONParser::EatWhitespaceAndComments() {
 402   while (pos_ < end_pos_) {
 403     switch (*pos_) {
 404       case '\r':
 405       case '\n':
 406         index_last_line_ = index_;
 407         ++line_number_;
 408         // Fall through.
 409       case ' ':
 410       case '\t':
 411         NextChar();
 412         break;
 413       case '/':
 414         if (!EatComment())
 415           return;
 416         break;
 417       default:
 418         return;
 419     }
 420   }
 421 }
 422
 423 bool JSONParser::EatComment() {
 424   if (*pos_ != '/' || !CanConsume(1))
 425     return false;
 426
 427   char next_char = *NextChar();
 428   if (next_char == '/') {
 429     // Single line comment, read to newline.
 430     while (CanConsume(1)) {
 431       char next_char = *NextChar();
 432       if (next_char == '\n' || next_char == '\r')
 433         return true;
 434     }
 435   } else if (next_char == '*') {
 436     // Block comment, read until end marker.
 437     while (CanConsume(2)) {
 438       if (*NextChar() == '*' && *NextChar() == '/') {
 439         // EatWhitespaceAndComments will inspect pos_, which will still be on
 440         // the last / of the comment, so advance once more (which may also be
 441         // end of input).
 442         NextChar();
 443         return true;
 444       }
 445     }
 446
 447     // If the comment is unterminated, GetNextToken will report T_END_OF_INPUT.
 448   }
 449
 450   return false;
 451 }
 452
 453 Value* JSONParser::ParseNextToken() {
 454   return ParseToken(GetNextToken());
 455 }
 456
 457 Value* JSONParser::ParseToken(Token token) {
 458   switch (token) {
 459     case T_OBJECT_BEGIN:
 460       return ConsumeDictionary();
 461     case T_ARRAY_BEGIN:
 462       return ConsumeList();
 463     case T_STRING:
 464       return ConsumeString();
 465     case T_NUMBER:
 466       return ConsumeNumber();
 467     case T_BOOL_TRUE:
 468     case T_BOOL_FALSE:
 469     case T_NULL:
 470       return ConsumeLiteral();
 471     default:
 472       ReportError(JSONReader::JSON_UNEXPECTED_TOKEN, 1);
 473       return NULL;
 474   }
 475 }
 476
 477 Value* JSONParser::ConsumeDictionary() {
 478   if (*pos_ != '{') {
 479     ReportError(JSONReader::JSON_UNEXPECTED_TOKEN, 1);
 480     return NULL;
 481   }
 482
 483   StackMarker depth_check(&stack_depth_);
 484   if (depth_check.IsTooDeep()) {
 485     ReportError(JSONReader::JSON_TOO_MUCH_NESTING, 1);
 486     return NULL;
 487   }
 488
 489   scoped_ptr<DictionaryValue> dict(new DictionaryValue);
 490
 491   NextChar();
 492   Token token = GetNextToken();
 493   while (token != T_OBJECT_END) {
 494     if (token != T_STRING) {
 495       ReportError(JSONReader::JSON_UNQUOTED_DICTIONARY_KEY, 1);
 496       return NULL;
 497     }
 498
 499     // First consume the key.
 500     StringBuilder key;
 501     if (!ConsumeStringRaw(&key)) {
 502       return NULL;
 503     }
 504
 505     // Read the separator.
 506     NextChar();
 507     token = GetNextToken();
 508     if (token != T_OBJECT_PAIR_SEPARATOR) {
 509       ReportError(JSONReader::JSON_SYNTAX_ERROR, 1);
 510       return NULL;
 511     }
 512
 513     // The next token is the value. Ownership transfers to |dict|.
 514     NextChar();
 515     Value* value = ParseNextToken();
 516     if (!value) {
 517       // ReportError from deeper level.
 518       return NULL;
 519     }
 520
 521     dict->SetWithoutPathExpansion(key.AsString(), value);
 522
 523     NextChar();
 524     token = GetNextToken();
 525     if (token == T_LIST_SEPARATOR) {
 526       NextChar();
 527       token = GetNextToken();
 528       if (token == T_OBJECT_END && !(options_ & JSON_ALLOW_TRAILING_COMMAS)) {
 529         ReportError(JSONReader::JSON_TRAILING_COMMA, 1);
 530         return NULL;
 531       }
 532     } else if (token != T_OBJECT_END) {
 533       ReportError(JSONReader::JSON_SYNTAX_ERROR, 0);
 534       return NULL;
 535     }
 536   }
 537
 538   return dict.release();
 539 }
 540
 541 Value* JSONParser::ConsumeList() {
 542   if (*pos_ != '[') {
 543     ReportError(JSONReader::JSON_UNEXPECTED_TOKEN, 1);
 544     return NULL;
 545   }
 546
 547   StackMarker depth_check(&stack_depth_);
 548   if (depth_check.IsTooDeep()) {
 549     ReportError(JSONReader::JSON_TOO_MUCH_NESTING, 1);
 550     return NULL;
 551   }
 552
 553   scoped_ptr<ListValue> list(new ListValue);
 554
 555   NextChar();
 556   Token token = GetNextToken();
 557   while (token != T_ARRAY_END) {
 558     Value* item = ParseToken(token);
 559     if (!item) {
 560       // ReportError from deeper level.
 561       return NULL;
 562     }
 563
 564     list->Append(item);
 565
 566     NextChar();
 567     token = GetNextToken();
 568     if (token == T_LIST_SEPARATOR) {
 569       NextChar();
 570       token = GetNextToken();
 571       if (token == T_ARRAY_END && !(options_ & JSON_ALLOW_TRAILING_COMMAS)) {
 572         ReportError(JSONReader::JSON_TRAILING_COMMA, 1);
 573         return NULL;
 574       }
 575     } else if (token != T_ARRAY_END) {
 576       ReportError(JSONReader::JSON_SYNTAX_ERROR, 1);
 577       return NULL;
 578     }
 579   }
 580
 581   return list.release();
 582 }
 583
 584 Value* JSONParser::ConsumeString() {
 585   StringBuilder string;
 586   if (!ConsumeStringRaw(&string))
 587     return NULL;
 588
 589   // Create the Value representation, using a hidden root, if configured
 590   // to do so, and if the string can be represented by StringPiece.
 591   if (string.CanBeStringPiece() && !(options_ & JSON_DETACHABLE_CHILDREN)) {
 592     return new JSONStringValue(string.AsStringPiece());
 593   } else {
 594     if (string.CanBeStringPiece())
 595       string.Convert();
 596     return new StringValue(string.AsString());
 597   }
 598 }
 599
 600 bool JSONParser::ConsumeStringRaw(StringBuilder* out) {
 601   if (*pos_ != '"') {
 602     ReportError(JSONReader::JSON_UNEXPECTED_TOKEN, 1);
 603     return false;
 604   }
 605
 606   // StringBuilder will internally build a StringPiece unless a UTF-16
 607   // conversion occurs, at which point it will perform a copy into a
 608   // std::string.
 609   StringBuilder string(NextChar());
 610
 611   int length = end_pos_ - start_pos_;
 612   int32 next_char = 0;
 613
 614   while (CanConsume(1)) {
 615     pos_ = start_pos_ + index_;  // CBU8_NEXT is postcrement.
 616     CBU8_NEXT(start_pos_, index_, length, next_char);
 617     if (next_char < 0 || !IsValidCharacter(next_char)) {
 618       ReportError(JSONReader::JSON_UNSUPPORTED_ENCODING, 1);
 619       return false;
 620     }
 621
 622     // If this character is an escape sequence...
 623     if (next_char == '\\') {
 624       // The input string will be adjusted (either by combining the two
 625       // characters of an encoded escape sequence, or with a UTF conversion),
 626       // so using StringPiece isn't possible -- force a conversion.
 627       string.Convert();
 628
 629       if (!CanConsume(1)) {
 630         ReportError(JSONReader::JSON_INVALID_ESCAPE, 0);
 631         return false;
 632       }
 633
 634       switch (*NextChar()) {
 635         // Allowed esape sequences:
 636         case 'x': {  // UTF-8 sequence.
 637           // UTF-8 \x escape sequences are not allowed in the spec, but they
 638           // are supported here for backwards-compatiblity with the old parser.
 639           if (!CanConsume(2)) {
 640             ReportError(JSONReader::JSON_INVALID_ESCAPE, 1);
 641             return false;
 642           }
 643
 644           int hex_digit = 0;
 645           if (!HexStringToInt(StringPiece(NextChar(), 2), &hex_digit)) {
 646             ReportError(JSONReader::JSON_INVALID_ESCAPE, -1);
 647             return false;
 648           }
 649           NextChar();
 650
 651           if (hex_digit < kExtendedASCIIStart)
 652             string.Append(hex_digit);
 653           else
 654             DecodeUTF8(hex_digit, &string);
 655           break;
 656         }
 657         case 'u': {  // UTF-16 sequence.
 658           // UTF units are of the form \uXXXX.
 659           if (!CanConsume(5)) {  // 5 being 'u' and four HEX digits.
 660             ReportError(JSONReader::JSON_INVALID_ESCAPE, 0);
 661             return false;
 662           }
 663
 664           // Skip the 'u'.
 665           NextChar();
 666
 667           std::string utf8_units;
 668           if (!DecodeUTF16(&utf8_units)) {
 669             ReportError(JSONReader::JSON_INVALID_ESCAPE, -1);
 670             return false;
 671           }
 672
 673           string.AppendString(utf8_units);
 674           break;
 675         }
 676         case '"':
 677           string.Append('"');
 678           break;
 679         case '\\':
 680           string.Append('\\');
 681           break;
 682         case '/':
 683           string.Append('/');
 684           break;
 685         case 'b':
 686           string.Append('\b');
 687           break;
 688         case 'f':
 689           string.Append('\f');
 690           break;
 691         case 'n':
 692           string.Append('\n');
 693           break;
 694         case 'r':
 695           string.Append('\r');
 696           break;
 697         case 't':
 698           string.Append('\t');
 699           break;
 700         case 'v':  // Not listed as valid escape sequence in the RFC.
 701           string.Append('\v');
 702           break;
 703         // All other escape squences are illegal.
 704         default:
 705           ReportError(JSONReader::JSON_INVALID_ESCAPE, 0);
 706           return false;
 707       }
 708     } else if (next_char == '"') {
 709       --index_;  // Rewind by one because of CBU8_NEXT.
 710       out->Swap(&string);
 711       return true;
 712     } else {
 713       if (next_char < kExtendedASCIIStart)
 714         string.Append(next_char);
 715       else
 716         DecodeUTF8(next_char, &string);
 717     }
 718   }
 719
 720   ReportError(JSONReader::JSON_SYNTAX_ERROR, 0);
 721   return false;
 722 }
 723
 724 // Entry is at the first X in \uXXXX.
 725 bool JSONParser::DecodeUTF16(std::string* dest_string) {
 726   if (!CanConsume(4))
 727     return false;
 728
 729   // This is a 32-bit field because the shift operations in the
 730   // conversion process below cause MSVC to error about "data loss."
 731   // This only stores UTF-16 code units, though.
 732   // Consume the UTF-16 code unit, which may be a high surrogate.
 733   int code_unit16_high = 0;
 734   if (!HexStringToInt(StringPiece(pos_, 4), &code_unit16_high))
 735     return false;
 736
 737   // Only add 3, not 4, because at the end of this iteration, the parser has
 738   // finished working with the last digit of the UTF sequence, meaning that
 739   // the next iteration will advance to the next byte.
 740   NextNChars(3);
 741
 742   // Used to convert the UTF-16 code units to a code point and then to a UTF-8
 743   // code unit sequence.
 744   char code_unit8[8] = { 0 };
 745   size_t offset = 0;
 746
 747   // If this is a high surrogate, consume the next code unit to get the
 748   // low surrogate.
 749   if (CBU16_IS_SURROGATE(code_unit16_high)) {
 750     // Make sure this is the high surrogate. If not, it's an encoding
 751     // error.
 752     if (!CBU16_IS_SURROGATE_LEAD(code_unit16_high))
 753       return false;
 754
 755     // Make sure that the token has more characters to consume the
 756     // lower surrogate.
 757     if (!CanConsume(6))  // 6 being '\' 'u' and four HEX digits.
 758       return false;
 759     if (*NextChar() != '\\' || *NextChar() != 'u')
 760       return false;
 761
 762     NextChar();  // Read past 'u'.
 763     int code_unit16_low = 0;
 764     if (!HexStringToInt(StringPiece(pos_, 4), &code_unit16_low))
 765       return false;
 766
 767     NextNChars(3);
 768
 769     if (!CBU16_IS_TRAIL(code_unit16_low)) {
 770       return false;
 771     }
 772
 773     uint32 code_point = CBU16_GET_SUPPLEMENTARY(code_unit16_high,
 774                                                 code_unit16_low);
 775     offset = 0;
 776     CBU8_APPEND_UNSAFE(code_unit8, offset, code_point);
 777   } else {
 778     // Not a surrogate.
 779     DCHECK(CBU16_IS_SINGLE(code_unit16_high));
 780     CBU8_APPEND_UNSAFE(code_unit8, offset, code_unit16_high);
 781   }
 782
 783   dest_string->append(code_unit8);
 784   return true;
 785 }
 786
 787 void JSONParser::DecodeUTF8(const int32& point, StringBuilder* dest) {
 788   // Anything outside of the basic ASCII plane will need to be decoded from
 789   // int32 to a multi-byte sequence.
 790   if (point < kExtendedASCIIStart) {
 791     dest->Append(point);
 792   } else {
 793     char utf8_units[4] = { 0 };
 794     int offset = 0;
 795     CBU8_APPEND_UNSAFE(utf8_units, offset, point);
 796     dest->Convert();
 797     // CBU8_APPEND_UNSAFE can overwrite up to 4 bytes, so utf8_units may not be
 798     // zero terminated at this point.  |offset| contains the correct length.
 799     dest->AppendString(std::string(utf8_units, offset));
 800   }
 801 }
 802
 803 Value* JSONParser::ConsumeNumber() {
 804   const char* num_start = pos_;
 805   const int start_index = index_;
 806   int end_index = start_index;
 807
 808   if (*pos_ == '-')
 809     NextChar();
 810
 811   if (!ReadInt(false)) {
 812     ReportError(JSONReader::JSON_SYNTAX_ERROR, 1);
 813     return NULL;
 814   }
 815   end_index = index_;
 816
 817   // The optional fraction part.
 818   if (*pos_ == '.') {
 819     if (!CanConsume(1)) {
 820       ReportError(JSONReader::JSON_SYNTAX_ERROR, 1);
 821       return NULL;
 822     }
 823     NextChar();
 824     if (!ReadInt(true)) {
 825       ReportError(JSONReader::JSON_SYNTAX_ERROR, 1);
 826       return NULL;
 827     }
 828     end_index = index_;
 829   }
 830
 831   // Optional exponent part.
 832   if (*pos_ == 'e' || *pos_ == 'E') {
 833     NextChar();
 834     if (*pos_ == '-' || *pos_ == '+')
 835       NextChar();
 836     if (!ReadInt(true)) {
 837       ReportError(JSONReader::JSON_SYNTAX_ERROR, 1);
 838       return NULL;
 839     }
 840     end_index = index_;
 841   }
 842
 843   // ReadInt is greedy because numbers have no easily detectable sentinel,
 844   // so save off where the parser should be on exit (see Consume invariant at
 845   // the top of the header), then make sure the next token is one which is
 846   // valid.
 847   const char* exit_pos = pos_ - 1;
 848   int exit_index = index_ - 1;
 849
 850   switch (GetNextToken()) {
 851     case T_OBJECT_END:
 852     case T_ARRAY_END:
 853     case T_LIST_SEPARATOR:
 854     case T_END_OF_INPUT:
 855       break;
 856     default:
 857       ReportError(JSONReader::JSON_SYNTAX_ERROR, 1);
 858       return NULL;
 859   }
 860
 861   pos_ = exit_pos;
 862   index_ = exit_index;
 863
 864   StringPiece num_string(num_start, end_index - start_index);
 865
 866   int num_int;
 867   if (StringToInt(num_string, &num_int))
 868     return new FundamentalValue(num_int);
 869
 870   double num_double;
 871   if (base::StringToDouble(num_string.as_string(), &num_double) &&
 872       IsFinite(num_double)) {
 873     return new FundamentalValue(num_double);
 874   }
 875
 876   return NULL;
 877 }
 878
 879 bool JSONParser::ReadInt(bool allow_leading_zeros) {
 880   char first = *pos_;
 881   int len = 0;
 882
 883   char c = first;
 884   while (CanConsume(1) && IsAsciiDigit(c)) {
 885     c = *NextChar();
 886     ++len;
 887   }
 888
 889   if (len == 0)
 890     return false;
 891
 892   if (!allow_leading_zeros && len > 1 && first == '0')
 893     return false;
 894
 895   return true;
 896 }
 897
 898 Value* JSONParser::ConsumeLiteral() {
 899   switch (*pos_) {
 900     case 't': {
 901       const char* kTrueLiteral = "true";
 902       const int kTrueLen = static_cast<int>(strlen(kTrueLiteral));
 903       if (!CanConsume(kTrueLen - 1) ||
 904           !StringsAreEqual(pos_, kTrueLiteral, kTrueLen)) {
 905         ReportError(JSONReader::JSON_SYNTAX_ERROR, 1);
 906         return NULL;
 907       }
 908       NextNChars(kTrueLen - 1);
 909       return new FundamentalValue(true);
 910     }
 911     case 'f': {
 912       const char* kFalseLiteral = "false";
 913       const int kFalseLen = static_cast<int>(strlen(kFalseLiteral));
 914       if (!CanConsume(kFalseLen - 1) ||
 915           !StringsAreEqual(pos_, kFalseLiteral, kFalseLen)) {
 916         ReportError(JSONReader::JSON_SYNTAX_ERROR, 1);
 917         return NULL;
 918       }
 919       NextNChars(kFalseLen - 1);
 920       return new FundamentalValue(false);
 921     }
 922     case 'n': {
 923       const char* kNullLiteral = "null";
 924       const int kNullLen = static_cast<int>(strlen(kNullLiteral));
 925       if (!CanConsume(kNullLen - 1) ||
 926           !StringsAreEqual(pos_, kNullLiteral, kNullLen)) {
 927         ReportError(JSONReader::JSON_SYNTAX_ERROR, 1);
 928         return NULL;
 929       }
 930       NextNChars(kNullLen - 1);
 931       return Value::CreateNullValue();
 932     }
 933     default:
 934       ReportError(JSONReader::JSON_UNEXPECTED_TOKEN, 1);
 935       return NULL;
 936   }
 937 }
 938
 939 // static
 940 bool JSONParser::StringsAreEqual(const char* one, const char* two, size_t len) {
 941   return strncmp(one, two, len) == 0;
 942 }
 943
 944 void JSONParser::ReportError(JSONReader::JsonParseError code,
 945                              int column_adjust) {
 946   error_code_ = code;
 947   error_line_ = line_number_;
 948   error_column_ = index_ - index_last_line_ + column_adjust;
 949 }
 950
 951 // static
 952 std::string JSONParser::FormatErrorMessage(int line, int column,
 953                                            const std::string& description) {
 954   if (line || column) {
 955     return StringPrintf("Line: %i, column: %i, %s",
 956         line, column, description.c_str());
 957   }
 958   return description;
 959 }
 960
 961 }  // namespace internal
 962 }  // namespace base