src/google/protobuf/compiler/parser.cc

   1 // Protocol Buffers - Google's data interchange format
   2 // Copyright 2008 Google Inc.
   3 // http://code.google.com/p/protobuf/
   4 //
   5 // Licensed under the Apache License, Version 2.0 (the "License");
   6 // you may not use this file except in compliance with the License.
   7 // You may obtain a copy of the License at
   8 //
   9 //      http://www.apache.org/licenses/LICENSE-2.0
  10 //
  11 // Unless required by applicable law or agreed to in writing, software
  12 // distributed under the License is distributed on an "AS IS" BASIS,
  13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 // See the License for the specific language governing permissions and
  15 // limitations under the License.
  16
  17 // Author: kenton@google.com (Kenton Varda)
  18 //  Based on original Protocol Buffers design by
  19 //  Sanjay Ghemawat, Jeff Dean, and others.
  20 //
  21 // Recursive descent FTW.
  22
  23 #include <google/protobuf/stubs/hash.h>
  24 #include <float.h>
  25
  26
  27 #include <google/protobuf/compiler/parser.h>
  28 #include <google/protobuf/descriptor.h>
  29 #include <google/protobuf/descriptor.pb.h>
  30 #include <google/protobuf/wire_format.h>
  31 #include <google/protobuf/io/tokenizer.h>
  32 #include <google/protobuf/stubs/common.h>
  33 #include <google/protobuf/stubs/strutil.h>
  34 #include <google/protobuf/stubs/map-util.h>
  35
  36 namespace google {
  37 namespace protobuf {
  38 namespace compiler {
  39
  40 using internal::WireFormat;
  41
  42 namespace {
  43
  44 typedef hash_map<string, FieldDescriptorProto::Type> TypeNameMap;
  45
  46 TypeNameMap MakeTypeNameTable() {
  47   TypeNameMap result;
  48
  49   result["double"  ] = FieldDescriptorProto::TYPE_DOUBLE;
  50   result["float"   ] = FieldDescriptorProto::TYPE_FLOAT;
  51   result["uint64"  ] = FieldDescriptorProto::TYPE_UINT64;
  52   result["fixed64" ] = FieldDescriptorProto::TYPE_FIXED64;
  53   result["fixed32" ] = FieldDescriptorProto::TYPE_FIXED32;
  54   result["bool"    ] = FieldDescriptorProto::TYPE_BOOL;
  55   result["string"  ] = FieldDescriptorProto::TYPE_STRING;
  56   result["group"   ] = FieldDescriptorProto::TYPE_GROUP;
  57
  58   result["bytes"   ] = FieldDescriptorProto::TYPE_BYTES;
  59   result["uint32"  ] = FieldDescriptorProto::TYPE_UINT32;
  60   result["sfixed32"] = FieldDescriptorProto::TYPE_SFIXED32;
  61   result["sfixed64"] = FieldDescriptorProto::TYPE_SFIXED64;
  62   result["int32"   ] = FieldDescriptorProto::TYPE_INT32;
  63   result["int64"   ] = FieldDescriptorProto::TYPE_INT64;
  64   result["sint32"  ] = FieldDescriptorProto::TYPE_SINT32;
  65   result["sint64"  ] = FieldDescriptorProto::TYPE_SINT64;
  66
  67   return result;
  68 }
  69
  70 const TypeNameMap kTypeNames = MakeTypeNameTable();
  71
  72 }  // anonymous namespace
  73
  74 // Makes code slightly more readable.  The meaning of "DO(foo)" is
  75 // "Execute foo and fail if it fails.", where failure is indicated by
  76 // returning false.
  77 #define DO(STATEMENT) if (STATEMENT) {} else return false
  78
  79 // ===================================================================
  80
  81 Parser::Parser()
  82   : input_(NULL),
  83     error_collector_(NULL),
  84     source_location_table_(NULL),
  85     had_errors_(false),
  86     require_syntax_identifier_(false) {
  87 }
  88
  89 Parser::~Parser() {
  90 }
  91
  92 // ===================================================================
  93
  94 inline bool Parser::LookingAt(const char* text) {
  95   return input_->current().text == text;
  96 }
  97
  98 inline bool Parser::LookingAtType(io::Tokenizer::TokenType token_type) {
  99   return input_->current().type == token_type;
 100 }
 101
 102 inline bool Parser::AtEnd() {
 103   return LookingAtType(io::Tokenizer::TYPE_END);
 104 }
 105
 106 bool Parser::TryConsume(const char* text) {
 107   if (LookingAt(text)) {
 108     input_->Next();
 109     return true;
 110   } else {
 111     return false;
 112   }
 113 }
 114
 115 bool Parser::Consume(const char* text, const char* error) {
 116   if (TryConsume(text)) {
 117     return true;
 118   } else {
 119     AddError(error);
 120     return false;
 121   }
 122 }
 123
 124 bool Parser::Consume(const char* text) {
 125   if (TryConsume(text)) {
 126     return true;
 127   } else {
 128     AddError("Expected \"" + string(text) + "\".");
 129     return false;
 130   }
 131 }
 132
 133 bool Parser::ConsumeIdentifier(string* output, const char* error) {
 134   if (LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) {
 135     *output = input_->current().text;
 136     input_->Next();
 137     return true;
 138   } else {
 139     AddError(error);
 140     return false;
 141   }
 142 }
 143
 144 bool Parser::ConsumeInteger(int* output, const char* error) {
 145   if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) {
 146     uint64 value = 0;
 147     if (!io::Tokenizer::ParseInteger(input_->current().text,
 148                                      kint32max, &value)) {
 149       AddError("Integer out of range.");
 150       // We still return true because we did, in fact, parse an integer.
 151     }
 152     *output = value;
 153     input_->Next();
 154     return true;
 155   } else {
 156     AddError(error);
 157     return false;
 158   }
 159 }
 160
 161 bool Parser::ConsumeInteger64(uint64 max_value, uint64* output,
 162                               const char* error) {
 163   if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) {
 164     if (!io::Tokenizer::ParseInteger(input_->current().text, max_value,
 165                                      output)) {
 166       AddError("Integer out of range.");
 167       // We still return true because we did, in fact, parse an integer.
 168       *output = 0;
 169     }
 170     input_->Next();
 171     return true;
 172   } else {
 173     AddError(error);
 174     return false;
 175   }
 176 }
 177
 178 bool Parser::ConsumeNumber(double* output, const char* error) {
 179   if (LookingAtType(io::Tokenizer::TYPE_FLOAT)) {
 180     *output = io::Tokenizer::ParseFloat(input_->current().text);
 181     input_->Next();
 182     return true;
 183   } else if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) {
 184     // Also accept integers.
 185     uint64 value = 0;
 186     if (!io::Tokenizer::ParseInteger(input_->current().text,
 187                                      kuint64max, &value)) {
 188       AddError("Integer out of range.");
 189       // We still return true because we did, in fact, parse a number.
 190     }
 191     *output = value;
 192     input_->Next();
 193     return true;
 194   } else {
 195     AddError(error);
 196     return false;
 197   }
 198 }
 199
 200 bool Parser::ConsumeString(string* output, const char* error) {
 201   if (LookingAtType(io::Tokenizer::TYPE_STRING)) {
 202     io::Tokenizer::ParseString(input_->current().text, output);
 203     input_->Next();
 204     return true;
 205   } else {
 206     AddError(error);
 207     return false;
 208   }
 209 }
 210
 211 // -------------------------------------------------------------------
 212
 213 void Parser::AddError(int line, int column, const string& error) {
 214   if (error_collector_ != NULL) {
 215     error_collector_->AddError(line, column, error);
 216   }
 217   had_errors_ = true;
 218 }
 219
 220 void Parser::AddError(const string& error) {
 221   AddError(input_->current().line, input_->current().column, error);
 222 }
 223
 224 void Parser::RecordLocation(
 225     const Message* descriptor,
 226     DescriptorPool::ErrorCollector::ErrorLocation location,
 227     int line, int column) {
 228   if (source_location_table_ != NULL) {
 229     source_location_table_->Add(descriptor, location, line, column);
 230   }
 231 }
 232
 233 void Parser::RecordLocation(
 234     const Message* descriptor,
 235     DescriptorPool::ErrorCollector::ErrorLocation location) {
 236   RecordLocation(descriptor, location,
 237                  input_->current().line, input_->current().column);
 238 }
 239
 240 // -------------------------------------------------------------------
 241
 242 void Parser::SkipStatement() {
 243   while (true) {
 244     if (AtEnd()) {
 245       return;
 246     } else if (LookingAtType(io::Tokenizer::TYPE_SYMBOL)) {
 247       if (TryConsume(";")) {
 248         return;
 249       } else if (TryConsume("{")) {
 250         SkipRestOfBlock();
 251         return;
 252       } else if (LookingAt("}")) {
 253         return;
 254       }
 255     }
 256     input_->Next();
 257   }
 258 }
 259
 260 void Parser::SkipRestOfBlock() {
 261   while (true) {
 262     if (AtEnd()) {
 263       return;
 264     } else if (LookingAtType(io::Tokenizer::TYPE_SYMBOL)) {
 265       if (TryConsume("}")) {
 266         return;
 267       } else if (TryConsume("{")) {
 268         SkipRestOfBlock();
 269       }
 270     }
 271     input_->Next();
 272   }
 273 }
 274
 275 // ===================================================================
 276
 277 bool Parser::Parse(io::Tokenizer* input, FileDescriptorProto* file) {
 278   input_ = input;
 279   had_errors_ = false;
 280   syntax_identifier_.clear();
 281
 282   if (LookingAtType(io::Tokenizer::TYPE_START)) {
 283     // Advance to first token.
 284     input_->Next();
 285   }
 286
 287   if (require_syntax_identifier_ || LookingAt("syntax")) {
 288     if (!ParseSyntaxIdentifier()) {
 289       // Don't attempt to parse the file if we didn't recognize the syntax
 290       // identifier.
 291       return false;
 292     }
 293   } else {
 294     syntax_identifier_ = "proto2";
 295   }
 296
 297   // Repeatedly parse statemetns until we reach the end of the file.
 298   while (!AtEnd()) {
 299     if (!ParseTopLevelStatement(file)) {
 300       // This statement failed to parse.  Skip it, but keep looping to parse
 301       // other statements.
 302       SkipStatement();
 303
 304       if (LookingAt("}")) {
 305         AddError("Unmatched \"}\".");
 306         input_->Next();
 307       }
 308     }
 309   }
 310
 311   input_ = NULL;
 312   return !had_errors_;
 313 }
 314
 315 bool Parser::ParseSyntaxIdentifier() {
 316   DO(Consume("syntax", "File must begin with 'syntax = \"proto2\";'."));
 317   DO(Consume("="));
 318   io::Tokenizer::Token syntax_token = input_->current();
 319   string syntax;
 320   DO(ConsumeString(&syntax, "Expected syntax identifier."));
 321   DO(Consume(";"));
 322
 323   syntax_identifier_ = syntax;
 324
 325   if (syntax != "proto2") {
 326     AddError(syntax_token.line, syntax_token.column,
 327       "Unrecognized syntax identifier \"" + syntax + "\".  This parser "
 328       "only recognizes \"proto2\".");
 329     return false;
 330   }
 331
 332   return true;
 333 }
 334
 335 bool Parser::ParseTopLevelStatement(FileDescriptorProto* file) {
 336   if (TryConsume(";")) {
 337     // empty statement; ignore
 338     return true;
 339   } else if (LookingAt("message")) {
 340     return ParseMessageDefinition(file->add_message_type());
 341   } else if (LookingAt("enum")) {
 342     return ParseEnumDefinition(file->add_enum_type());
 343   } else if (LookingAt("service")) {
 344     return ParseServiceDefinition(file->add_service());
 345   } else if (LookingAt("extend")) {
 346     return ParseExtend(file->mutable_extension(),
 347                        file->mutable_message_type());
 348   } else if (LookingAt("import")) {
 349     return ParseImport(file->add_dependency());
 350   } else if (LookingAt("package")) {
 351     return ParsePackage(file);
 352   } else if (LookingAt("option")) {
 353     return ParseOption(file->mutable_options());
 354   } else {
 355     AddError("Expected top-level statement (e.g. \"message\").");
 356     return false;
 357   }
 358 }
 359
 360 // -------------------------------------------------------------------
 361 // Messages
 362
 363 bool Parser::ParseMessageDefinition(DescriptorProto* message) {
 364   DO(Consume("message"));
 365   RecordLocation(message, DescriptorPool::ErrorCollector::NAME);
 366   DO(ConsumeIdentifier(message->mutable_name(), "Expected message name."));
 367   DO(ParseMessageBlock(message));
 368   return true;
 369 }
 370
 371 bool Parser::ParseMessageBlock(DescriptorProto* message) {
 372   DO(Consume("{"));
 373
 374   while (!TryConsume("}")) {
 375     if (AtEnd()) {
 376       AddError("Reached end of input in message definition (missing '}').");
 377       return false;
 378     }
 379
 380     if (!ParseMessageStatement(message)) {
 381       // This statement failed to parse.  Skip it, but keep looping to parse
 382       // other statements.
 383       SkipStatement();
 384     }
 385   }
 386
 387   return true;
 388 }
 389
 390 bool Parser::ParseMessageStatement(DescriptorProto* message) {
 391   if (TryConsume(";")) {
 392     // empty statement; ignore
 393     return true;
 394   } else if (LookingAt("message")) {
 395     return ParseMessageDefinition(message->add_nested_type());
 396   } else if (LookingAt("enum")) {
 397     return ParseEnumDefinition(message->add_enum_type());
 398   } else if (LookingAt("extensions")) {
 399     return ParseExtensions(message);
 400   } else if (LookingAt("extend")) {
 401     return ParseExtend(message->mutable_extension(),
 402                        message->mutable_nested_type());
 403   } else if (LookingAt("option")) {
 404     return ParseOption(message->mutable_options());
 405   } else {
 406     return ParseMessageField(message->add_field(),
 407                              message->mutable_nested_type());
 408   }
 409 }
 410
 411 bool Parser::ParseMessageField(FieldDescriptorProto* field,
 412                                RepeatedPtrField<DescriptorProto>* messages) {
 413   // Parse label and type.
 414   FieldDescriptorProto::Label label;
 415   DO(ParseLabel(&label));
 416   field->set_label(label);
 417
 418   RecordLocation(field, DescriptorPool::ErrorCollector::TYPE);
 419   FieldDescriptorProto::Type type = FieldDescriptorProto::TYPE_INT32;
 420   string type_name;
 421   DO(ParseType(&type, &type_name));
 422   if (type_name.empty()) {
 423     field->set_type(type);
 424   } else {
 425     field->set_type_name(type_name);
 426   }
 427
 428   // Parse name and '='.
 429   RecordLocation(field, DescriptorPool::ErrorCollector::NAME);
 430   io::Tokenizer::Token name_token = input_->current();
 431   DO(ConsumeIdentifier(field->mutable_name(), "Expected field name."));
 432   DO(Consume("=", "Missing field number."));
 433
 434   // Parse field number.
 435   RecordLocation(field, DescriptorPool::ErrorCollector::NUMBER);
 436   int number;
 437   DO(ConsumeInteger(&number, "Expected field number."));
 438   field->set_number(number);
 439
 440   // Parse options.
 441   DO(ParseFieldOptions(field));
 442
 443   // Deal with groups.
 444   if (type_name.empty() && type == FieldDescriptorProto::TYPE_GROUP) {
 445     DescriptorProto* group = messages->Add();
 446     group->set_name(field->name());
 447     // Record name location to match the field name's location.
 448     RecordLocation(group, DescriptorPool::ErrorCollector::NAME,
 449                    name_token.line, name_token.column);
 450
 451     // As a hack for backwards-compatibility, we force the group name to start
 452     // with a capital letter and lower-case the field name.  New code should
 453     // not use groups; it should use nested messages.
 454     if (group->name()[0] < 'A' || 'Z' < group->name()[0]) {
 455       AddError(name_token.line, name_token.column,
 456         "Group names must start with a capital letter.");
 457     }
 458     LowerString(field->mutable_name());
 459
 460     field->set_type_name(group->name());
 461     if (LookingAt("{")) {
 462       DO(ParseMessageBlock(group));
 463     } else {
 464       AddError("Missing group body.");
 465       return false;
 466     }
 467   } else {
 468     DO(Consume(";"));
 469   }
 470
 471   return true;
 472 }
 473
 474 bool Parser::ParseFieldOptions(FieldDescriptorProto* field) {
 475   if (!TryConsume("[")) return true;
 476
 477   // Parse field options.
 478   do {
 479     if (LookingAt("default")) {
 480       DO(ParseDefaultAssignment(field));
 481     } else {
 482       DO(ParseOptionAssignment(field->mutable_options()));
 483     }
 484   } while (TryConsume(","));
 485
 486   DO(Consume("]"));
 487   return true;
 488 }
 489
 490 bool Parser::ParseDefaultAssignment(FieldDescriptorProto* field) {
 491   if (field->has_default_value()) {
 492     AddError("Already set option \"default\".");
 493     field->clear_default_value();
 494   }
 495
 496   DO(Consume("default"));
 497   DO(Consume("="));
 498
 499   RecordLocation(field, DescriptorPool::ErrorCollector::DEFAULT_VALUE);
 500   string* default_value = field->mutable_default_value();
 501
 502   if (!field->has_type()) {
 503     // The field has a type name, but we don't know if it is a message or an
 504     // enum yet.  Assume an enum for now.
 505     DO(ConsumeIdentifier(default_value, "Expected identifier."));
 506     return true;
 507   }
 508
 509   switch (field->type()) {
 510     case FieldDescriptorProto::TYPE_INT32:
 511     case FieldDescriptorProto::TYPE_INT64:
 512     case FieldDescriptorProto::TYPE_SINT32:
 513     case FieldDescriptorProto::TYPE_SINT64:
 514     case FieldDescriptorProto::TYPE_SFIXED32:
 515     case FieldDescriptorProto::TYPE_SFIXED64: {
 516       uint64 max_value = kint64max;
 517       if (field->type() == FieldDescriptorProto::TYPE_INT32 ||
 518           field->type() == FieldDescriptorProto::TYPE_SINT32 ||
 519           field->type() == FieldDescriptorProto::TYPE_SFIXED32) {
 520         max_value = kint32max;
 521       }
 522
 523       // These types can be negative.
 524       if (TryConsume("-")) {
 525         default_value->append("-");
 526         // Two's complement always has one more negative value than positive.
 527         ++max_value;
 528       }
 529       // Parse the integer to verify that it is not out-of-range.
 530       uint64 value;
 531       DO(ConsumeInteger64(max_value, &value, "Expected integer."));
 532       // And stringify it again.
 533       default_value->append(SimpleItoa(value));
 534       break;
 535     }
 536
 537     case FieldDescriptorProto::TYPE_UINT32:
 538     case FieldDescriptorProto::TYPE_UINT64:
 539     case FieldDescriptorProto::TYPE_FIXED32:
 540     case FieldDescriptorProto::TYPE_FIXED64: {
 541       uint64 max_value = kuint64max;
 542       if (field->type() == FieldDescriptorProto::TYPE_UINT32 ||
 543           field->type() == FieldDescriptorProto::TYPE_FIXED32) {
 544         max_value = kuint32max;
 545       }
 546
 547       // Numeric, not negative.
 548       if (TryConsume("-")) {
 549         AddError("Unsigned field can't have negative default value.");
 550       }
 551       // Parse the integer to verify that it is not out-of-range.
 552       uint64 value;
 553       DO(ConsumeInteger64(max_value, &value, "Expected integer."));
 554       // And stringify it again.
 555       default_value->append(SimpleItoa(value));
 556       break;
 557     }
 558
 559     case FieldDescriptorProto::TYPE_FLOAT:
 560     case FieldDescriptorProto::TYPE_DOUBLE:
 561       // These types can be negative.
 562       if (TryConsume("-")) {
 563         default_value->append("-");
 564       }
 565       // Parse the integer because we have to convert hex integers to decimal
 566       // floats.
 567       double value;
 568       DO(ConsumeNumber(&value, "Expected number."));
 569       // And stringify it again.
 570       default_value->append(SimpleDtoa(value));
 571       break;
 572
 573     case FieldDescriptorProto::TYPE_BOOL:
 574       if (TryConsume("true")) {
 575         default_value->assign("true");
 576       } else if (TryConsume("false")) {
 577         default_value->assign("false");
 578       } else {
 579         AddError("Expected \"true\" or \"false\".");
 580         return false;
 581       }
 582       break;
 583
 584     case FieldDescriptorProto::TYPE_STRING:
 585       DO(ConsumeString(default_value, "Expected string."));
 586       break;
 587
 588     case FieldDescriptorProto::TYPE_BYTES:
 589       DO(ConsumeString(default_value, "Expected string."));
 590       *default_value = CEscape(*default_value);
 591       break;
 592
 593     case FieldDescriptorProto::TYPE_ENUM:
 594       DO(ConsumeIdentifier(default_value, "Expected identifier."));
 595       break;
 596
 597     case FieldDescriptorProto::TYPE_MESSAGE:
 598     case FieldDescriptorProto::TYPE_GROUP:
 599       AddError("Messages can't have default values.");
 600       return false;
 601   }
 602
 603   return true;
 604 }
 605
 606 bool Parser::ParseOptionAssignment(Message* options) {
 607   Message::Reflection* reflection = options->GetReflection();
 608   const Descriptor* descriptor = options->GetDescriptor();
 609
 610   // Parse name.
 611   string name;
 612   int line = input_->current().line;
 613   int column = input_->current().column;
 614   DO(ConsumeIdentifier(&name, "Expected option name."));
 615
 616   // Is it valid?
 617   const FieldDescriptor* field = descriptor->FindFieldByName(name);
 618   if (field == NULL) {
 619     AddError(line, column, "Unknown option: " + name);
 620     return false;
 621   }
 622   if (field->is_repeated()) {
 623     AddError(line, column, "Not implemented: repeated options.");
 624     return false;
 625   }
 626   if (reflection->HasField(field)) {
 627     AddError(line, column, "Option \"" + name + "\" was already set.");
 628     return false;
 629   }
 630
 631   // Are we trying to assign a member of a message?
 632   if (LookingAt(".")) {
 633     if (field->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) {
 634       AddError("Option \"" + name + "\" is an atomic type, not a message.");
 635       return false;
 636     }
 637     DO(Consume("."));
 638
 639     // This field is a message/group.  The user must identify a field within
 640     // it to set.
 641     return ParseOptionAssignment(reflection->MutableMessage(field));
 642   }
 643
 644   DO(Consume("="));
 645
 646   // Parse the option value.
 647   switch (field->cpp_type()) {
 648     case FieldDescriptor::CPPTYPE_INT32: {
 649       uint64 value;
 650       bool is_negative = TryConsume("-");
 651       uint64 max_value = kint32max;
 652       if (is_negative) ++max_value;
 653       DO(ConsumeInteger64(max_value, &value, "Expected integer."));
 654       reflection->SetInt32(field, is_negative ? -value : value);
 655       break;
 656     }
 657
 658     case FieldDescriptor::CPPTYPE_INT64: {
 659       uint64 value;
 660       bool is_negative = TryConsume("-");
 661       uint64 max_value = kint64max;
 662       if (is_negative) ++max_value;
 663       DO(ConsumeInteger64(max_value, &value, "Expected integer."));
 664       reflection->SetInt64(field, is_negative ? -value : value);
 665       break;
 666     }
 667
 668     case FieldDescriptor::CPPTYPE_UINT32: {
 669       uint64 value;
 670       DO(ConsumeInteger64(kuint32max, &value, "Expected integer."));
 671       reflection->SetUInt32(field, value);
 672       break;
 673     }
 674
 675     case FieldDescriptor::CPPTYPE_UINT64: {
 676       uint64 value;
 677       DO(ConsumeInteger64(kuint64max, &value, "Expected integer."));
 678       reflection->SetUInt64(field, value);
 679       break;
 680     }
 681
 682     case FieldDescriptor::CPPTYPE_DOUBLE: {
 683       double value;
 684       bool is_negative = TryConsume("-");
 685       DO(ConsumeNumber(&value, "Expected number."));
 686       reflection->SetDouble(field, is_negative ? -value : value);
 687       break;
 688     }
 689
 690     case FieldDescriptor::CPPTYPE_FLOAT: {
 691       double value;
 692       bool is_negative = TryConsume("-");
 693       DO(ConsumeNumber(&value, "Expected number."));
 694       reflection->SetFloat(field, is_negative ? -value : value);
 695       break;
 696     }
 697
 698     case FieldDescriptor::CPPTYPE_BOOL:
 699       if (TryConsume("true")) {
 700         reflection->SetBool(field, true);
 701       } else if (TryConsume("false")) {
 702         reflection->SetBool(field, false);
 703       } else {
 704         AddError("Expected \"true\" or \"false\".");
 705         return false;
 706       }
 707       break;
 708
 709     case FieldDescriptor::CPPTYPE_ENUM: {
 710       string value_name;
 711       int value_line = input_->current().line;
 712       int value_column = input_->current().column;
 713       DO(ConsumeIdentifier(&value_name, "Expected enum value."));
 714       const EnumValueDescriptor* value =
 715         field->enum_type()->FindValueByName(value_name);
 716       if (value == NULL) {
 717         AddError(value_line, value_column,
 718           "Enum type \"" + field->enum_type()->full_name() + "\" has no value "
 719           "named \"" + value_name + "\".");
 720         return false;
 721       }
 722       reflection->SetEnum(field, value);
 723       break;
 724     }
 725
 726     case FieldDescriptor::CPPTYPE_STRING: {
 727       string value;
 728       DO(ConsumeString(&value, "Expected string."));
 729       reflection->SetString(field, value);
 730       break;
 731     }
 732
 733     case FieldDescriptor::CPPTYPE_MESSAGE: {
 734       // TODO(kenton):  Allow use of protocol buffer text format here?
 735       AddError("\"" + name + "\" is a message.  To set fields within it, use "
 736                "syntax like \"" + name + ".foo = value\".");
 737       return false;
 738       break;
 739     }
 740   }
 741
 742   return true;
 743 }
 744
 745 bool Parser::ParseExtensions(DescriptorProto* message) {
 746   // Parse the declaration.
 747   DO(Consume("extensions"));
 748
 749   do {
 750     DescriptorProto::ExtensionRange* range = message->add_extension_range();
 751     RecordLocation(range, DescriptorPool::ErrorCollector::NUMBER);
 752
 753     int start, end;
 754     DO(ConsumeInteger(&start, "Expected field number range."));
 755
 756     if (TryConsume("to")) {
 757       if (TryConsume("max")) {
 758         end = FieldDescriptor::kMaxNumber;
 759       } else {
 760         DO(ConsumeInteger(&end, "Expected integer."));
 761       }
 762     } else {
 763       end = start;
 764     }
 765
 766     // Users like to specify inclusive ranges, but in code we like the end
 767     // number to be exclusive.
 768     ++end;
 769
 770     range->set_start(start);
 771     range->set_end(end);
 772   } while (TryConsume(","));
 773
 774   DO(Consume(";"));
 775   return true;
 776 }
 777
 778 bool Parser::ParseExtend(RepeatedPtrField<FieldDescriptorProto>* extensions,
 779                          RepeatedPtrField<DescriptorProto>* messages) {
 780   DO(Consume("extend"));
 781
 782   // We expect to see at least one extension field defined in the extend block.
 783   // We need to create it now so we can record the extendee's location.
 784   FieldDescriptorProto* first_field = extensions->Add();
 785
 786   // Parse the extendee type.
 787   RecordLocation(first_field, DescriptorPool::ErrorCollector::EXTENDEE);
 788   DO(ParseUserDefinedType(first_field->mutable_extendee()));
 789
 790   // Parse the block.
 791   DO(Consume("{"));
 792
 793   bool is_first = true;
 794
 795   do {
 796     if (AtEnd()) {
 797       AddError("Reached end of input in extend definition (missing '}').");
 798       return false;
 799     }
 800
 801     FieldDescriptorProto* field;
 802     if (is_first) {
 803       field = first_field;
 804       is_first = false;
 805     } else {
 806       field = extensions->Add();
 807       field->set_extendee(first_field->extendee());
 808     }
 809
 810     if (!ParseMessageField(field, messages)) {
 811       // This statement failed to parse.  Skip it, but keep looping to parse
 812       // other statements.
 813       SkipStatement();
 814     }
 815   } while(!TryConsume("}"));
 816
 817   return true;
 818 }
 819
 820 // -------------------------------------------------------------------
 821 // Enums
 822
 823 bool Parser::ParseEnumDefinition(EnumDescriptorProto* enum_type) {
 824   DO(Consume("enum"));
 825   RecordLocation(enum_type, DescriptorPool::ErrorCollector::NAME);
 826   DO(ConsumeIdentifier(enum_type->mutable_name(), "Expected enum name."));
 827   DO(ParseEnumBlock(enum_type));
 828   return true;
 829 }
 830
 831 bool Parser::ParseEnumBlock(EnumDescriptorProto* enum_type) {
 832   DO(Consume("{"));
 833
 834   while (!TryConsume("}")) {
 835     if (AtEnd()) {
 836       AddError("Reached end of input in enum definition (missing '}').");
 837       return false;
 838     }
 839
 840     if (!ParseEnumStatement(enum_type)) {
 841       // This statement failed to parse.  Skip it, but keep looping to parse
 842       // other statements.
 843       SkipStatement();
 844     }
 845   }
 846
 847   return true;
 848 }
 849
 850 bool Parser::ParseEnumStatement(EnumDescriptorProto* enum_type) {
 851   if (TryConsume(";")) {
 852     // empty statement; ignore
 853     return true;
 854   } else if (LookingAt("option")) {
 855     return ParseOption(enum_type->mutable_options());
 856   } else {
 857     return ParseEnumConstant(enum_type->add_value());
 858   }
 859 }
 860
 861 bool Parser::ParseEnumConstant(EnumValueDescriptorProto* enum_value) {
 862   RecordLocation(enum_value, DescriptorPool::ErrorCollector::NAME);
 863   DO(ConsumeIdentifier(enum_value->mutable_name(),
 864                        "Expected enum constant name."));
 865   DO(Consume("=", "Missing numeric value for enum constant."));
 866
 867   bool is_negative = TryConsume("-");
 868   int number;
 869   DO(ConsumeInteger(&number, "Expected integer."));
 870   if (is_negative) number *= -1;
 871   enum_value->set_number(number);
 872
 873   // TODO(kenton):  Options for enum values?
 874
 875   DO(Consume(";"));
 876
 877   return true;
 878 }
 879
 880 // -------------------------------------------------------------------
 881 // Services
 882
 883 bool Parser::ParseServiceDefinition(ServiceDescriptorProto* service) {
 884   DO(Consume("service"));
 885   RecordLocation(service, DescriptorPool::ErrorCollector::NAME);
 886   DO(ConsumeIdentifier(service->mutable_name(), "Expected service name."));
 887   DO(ParseServiceBlock(service));
 888   return true;
 889 }
 890
 891 bool Parser::ParseServiceBlock(ServiceDescriptorProto* service) {
 892   DO(Consume("{"));
 893
 894   while (!TryConsume("}")) {
 895     if (AtEnd()) {
 896       AddError("Reached end of input in service definition (missing '}').");
 897       return false;
 898     }
 899
 900     if (!ParseServiceStatement(service)) {
 901       // This statement failed to parse.  Skip it, but keep looping to parse
 902       // other statements.
 903       SkipStatement();
 904     }
 905   }
 906
 907   return true;
 908 }
 909
 910 bool Parser::ParseServiceStatement(ServiceDescriptorProto* service) {
 911   if (TryConsume(";")) {
 912     // empty statement; ignore
 913     return true;
 914   } else if (LookingAt("option")) {
 915     return ParseOption(service->mutable_options());
 916   } else {
 917     return ParseServiceMethod(service->add_method());
 918   }
 919 }
 920
 921 bool Parser::ParseServiceMethod(MethodDescriptorProto* method) {
 922   DO(Consume("rpc"));
 923   RecordLocation(method, DescriptorPool::ErrorCollector::NAME);
 924   DO(ConsumeIdentifier(method->mutable_name(), "Expected method name."));
 925
 926   // Parse input type.
 927   DO(Consume("("));
 928   RecordLocation(method, DescriptorPool::ErrorCollector::INPUT_TYPE);
 929   DO(ParseUserDefinedType(method->mutable_input_type()));
 930   DO(Consume(")"));
 931
 932   // Parse output type.
 933   DO(Consume("returns"));
 934   DO(Consume("("));
 935   RecordLocation(method, DescriptorPool::ErrorCollector::OUTPUT_TYPE);
 936   DO(ParseUserDefinedType(method->mutable_output_type()));
 937   DO(Consume(")"));
 938
 939   if (TryConsume("{")) {
 940     // Options!
 941     while (!TryConsume("}")) {
 942       if (AtEnd()) {
 943         AddError("Reached end of input in method options (missing '}').");
 944         return false;
 945       }
 946
 947       if (TryConsume(";")) {
 948         // empty statement; ignore
 949       } else {
 950         if (!ParseOption(method->mutable_options())) {
 951           // This statement failed to parse.  Skip it, but keep looping to
 952           // parse other statements.
 953           SkipStatement();
 954         }
 955       }
 956     }
 957   } else {
 958     DO(Consume(";"));
 959   }
 960
 961   return true;
 962 }
 963
 964 // -------------------------------------------------------------------
 965
 966 bool Parser::ParseLabel(FieldDescriptorProto::Label* label) {
 967   if (TryConsume("optional")) {
 968     *label = FieldDescriptorProto::LABEL_OPTIONAL;
 969     return true;
 970   } else if (TryConsume("repeated")) {
 971     *label = FieldDescriptorProto::LABEL_REPEATED;
 972     return true;
 973   } else if (TryConsume("required")) {
 974     *label = FieldDescriptorProto::LABEL_REQUIRED;
 975     return true;
 976   } else {
 977     AddError("Expected \"required\", \"optional\", or \"repeated\".");
 978     // We can actually reasonably recover here by just assuming the user
 979     // forgot the label altogether.
 980     *label = FieldDescriptorProto::LABEL_OPTIONAL;
 981     return true;
 982   }
 983 }
 984
 985 bool Parser::ParseType(FieldDescriptorProto::Type* type,
 986                        string* type_name) {
 987   TypeNameMap::const_iterator iter = kTypeNames.find(input_->current().text);
 988   if (iter != kTypeNames.end()) {
 989     *type = iter->second;
 990     input_->Next();
 991   } else {
 992     DO(ParseUserDefinedType(type_name));
 993   }
 994   return true;
 995 }
 996
 997 bool Parser::ParseUserDefinedType(string* type_name) {
 998   type_name->clear();
 999
1000   TypeNameMap::const_iterator iter = kTypeNames.find(input_->current().text);
1001   if (iter != kTypeNames.end()) {
1002     // Note:  The only place enum types are allowed is for field types, but
1003     //   if we are parsing a field type then we would not get here because
1004     //   primitives are allowed there as well.  So this error message doesn't
1005     //   need to account for enums.
1006     AddError("Expected message type.");
1007
1008     // Pretend to accept this type so that we can go on parsing.
1009     *type_name = input_->current().text;
1010     input_->Next();
1011     return true;
1012   }
1013
1014   // A leading "." means the name is fully-qualified.
1015   if (TryConsume(".")) type_name->append(".");
1016
1017   // Consume the first part of the name.
1018   string identifier;
1019   DO(ConsumeIdentifier(&identifier, "Expected type name."));
1020   type_name->append(identifier);
1021
1022   // Consume more parts.
1023   while (TryConsume(".")) {
1024     type_name->append(".");
1025     DO(ConsumeIdentifier(&identifier, "Expected identifier."));
1026     type_name->append(identifier);
1027   }
1028
1029   return true;
1030 }
1031
1032 // ===================================================================
1033
1034 bool Parser::ParsePackage(FileDescriptorProto* file) {
1035   if (file->has_package()) {
1036     AddError("Multiple package definitions.");
1037   }
1038
1039   DO(Consume("package"));
1040
1041   RecordLocation(file, DescriptorPool::ErrorCollector::NAME);
1042
1043   while (true) {
1044     string identifier;
1045     DO(ConsumeIdentifier(&identifier, "Expected identifier."));
1046     file->mutable_package()->append(identifier);
1047     if (!TryConsume(".")) break;
1048     file->mutable_package()->append(".");
1049   }
1050
1051   DO(Consume(";"));
1052   return true;
1053 }
1054
1055 bool Parser::ParseImport(string* import_filename) {
1056   DO(Consume("import"));
1057   DO(ConsumeString(import_filename,
1058     "Expected a string naming the file to import."));
1059   DO(Consume(";"));
1060   return true;
1061 }
1062
1063 bool Parser::ParseOption(Message* options) {
1064   DO(Consume("option"));
1065   DO(ParseOptionAssignment(options));
1066   DO(Consume(";"));
1067   return true;
1068 }
1069
1070 // ===================================================================
1071
1072 SourceLocationTable::SourceLocationTable() {}
1073 SourceLocationTable::~SourceLocationTable() {}
1074
1075 bool SourceLocationTable::Find(
1076     const Message* descriptor,
1077     DescriptorPool::ErrorCollector::ErrorLocation location,
1078     int* line, int* column) const {
1079   const pair<int, int>* result =
1080     FindOrNull(location_map_, make_pair(descriptor, location));
1081   if (result == NULL) {
1082     *line   = -1;
1083     *column = 0;
1084     return false;
1085   } else {
1086     *line   = result->first;
1087     *column = result->second;
1088     return true;
1089   }
1090 }
1091
1092 void SourceLocationTable::Add(
1093     const Message* descriptor,
1094     DescriptorPool::ErrorCollector::ErrorLocation location,
1095     int line, int column) {
1096   location_map_[make_pair(descriptor, location)] = make_pair(line, column);
1097 }
1098
1099 void SourceLocationTable::Clear() {
1100   location_map_.clear();
1101 }
1102
1103 }  // namespace compiler
1104 }  // namespace protobuf
1105 }  // namespace google