clang/lib/AST/CommentParser.cpp

   1 //===--- CommentParser.cpp - Doxygen comment parser -----------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "clang/AST/CommentParser.h"
  10 #include "clang/AST/CommentCommandTraits.h"
  11 #include "clang/AST/CommentDiagnostic.h"
  12 #include "clang/AST/CommentSema.h"
  13 #include "clang/Basic/CharInfo.h"
  14 #include "clang/Basic/SourceManager.h"
  15 #include "llvm/Support/ErrorHandling.h"
  16
  17 namespace clang {
  18
  19 static inline bool isWhitespace(llvm::StringRef S) {
  20   for (StringRef::const_iterator I = S.begin(), E = S.end(); I != E; ++I) {
  21     if (!isWhitespace(*I))
  22       return false;
  23   }
  24   return true;
  25 }
  26
  27 namespace comments {
  28
  29 /// Re-lexes a sequence of tok::text tokens.
  30 class TextTokenRetokenizer {
  31   llvm::BumpPtrAllocator &Allocator;
  32   Parser &P;
  33
  34   /// This flag is set when there are no more tokens we can fetch from lexer.
  35   bool NoMoreInterestingTokens;
  36
  37   /// Token buffer: tokens we have processed and lookahead.
  38   SmallVector<Token, 16> Toks;
  39
  40   /// A position in \c Toks.
  41   struct Position {
  42     const char *BufferStart;
  43     const char *BufferEnd;
  44     const char *BufferPtr;
  45     SourceLocation BufferStartLoc;
  46     unsigned CurToken;
  47   };
  48
  49   /// Current position in Toks.
  50   Position Pos;
  51
  52   bool isEnd() const {
  53     return Pos.CurToken >= Toks.size();
  54   }
  55
  56   /// Sets up the buffer pointers to point to current token.
  57   void setupBuffer() {
  58     assert(!isEnd());
  59     const Token &Tok = Toks[Pos.CurToken];
  60
  61     Pos.BufferStart = Tok.getText().begin();
  62     Pos.BufferEnd = Tok.getText().end();
  63     Pos.BufferPtr = Pos.BufferStart;
  64     Pos.BufferStartLoc = Tok.getLocation();
  65   }
  66
  67   SourceLocation getSourceLocation() const {
  68     const unsigned CharNo = Pos.BufferPtr - Pos.BufferStart;
  69     return Pos.BufferStartLoc.getLocWithOffset(CharNo);
  70   }
  71
  72   char peek() const {
  73     assert(!isEnd());
  74     assert(Pos.BufferPtr != Pos.BufferEnd);
  75     return *Pos.BufferPtr;
  76   }
  77
  78   void consumeChar() {
  79     assert(!isEnd());
  80     assert(Pos.BufferPtr != Pos.BufferEnd);
  81     Pos.BufferPtr++;
  82     if (Pos.BufferPtr == Pos.BufferEnd) {
  83       Pos.CurToken++;
  84       if (isEnd() && !addToken())
  85         return;
  86
  87       assert(!isEnd());
  88       setupBuffer();
  89     }
  90   }
  91
  92   /// Add a token.
  93   /// Returns true on success, false if there are no interesting tokens to
  94   /// fetch from lexer.
  95   bool addToken() {
  96     if (NoMoreInterestingTokens)
  97       return false;
  98
  99     if (P.Tok.is(tok::newline)) {
 100       // If we see a single newline token between text tokens, skip it.
 101       Token Newline = P.Tok;
 102       P.consumeToken();
 103       if (P.Tok.isNot(tok::text)) {
 104         P.putBack(Newline);
 105         NoMoreInterestingTokens = true;
 106         return false;
 107       }
 108     }
 109     if (P.Tok.isNot(tok::text)) {
 110       NoMoreInterestingTokens = true;
 111       return false;
 112     }
 113
 114     Toks.push_back(P.Tok);
 115     P.consumeToken();
 116     if (Toks.size() == 1)
 117       setupBuffer();
 118     return true;
 119   }
 120
 121   void consumeWhitespace() {
 122     while (!isEnd()) {
 123       if (isWhitespace(peek()))
 124         consumeChar();
 125       else
 126         break;
 127     }
 128   }
 129
 130   void formTokenWithChars(Token &Result,
 131                           SourceLocation Loc,
 132                           const char *TokBegin,
 133                           unsigned TokLength,
 134                           StringRef Text) {
 135     Result.setLocation(Loc);
 136     Result.setKind(tok::text);
 137     Result.setLength(TokLength);
 138 #ifndef NDEBUG
 139     Result.TextPtr = "<UNSET>";
 140     Result.IntVal = 7;
 141 #endif
 142     Result.setText(Text);
 143   }
 144
 145 public:
 146   TextTokenRetokenizer(llvm::BumpPtrAllocator &Allocator, Parser &P):
 147       Allocator(Allocator), P(P), NoMoreInterestingTokens(false) {
 148     Pos.CurToken = 0;
 149     addToken();
 150   }
 151
 152   /// Extract a word -- sequence of non-whitespace characters.
 153   bool lexWord(Token &Tok) {
 154     if (isEnd())
 155       return false;
 156
 157     Position SavedPos = Pos;
 158
 159     consumeWhitespace();
 160     SmallString<32> WordText;
 161     const char *WordBegin = Pos.BufferPtr;
 162     SourceLocation Loc = getSourceLocation();
 163     while (!isEnd()) {
 164       const char C = peek();
 165       if (!isWhitespace(C)) {
 166         WordText.push_back(C);
 167         consumeChar();
 168       } else
 169         break;
 170     }
 171     const unsigned Length = WordText.size();
 172     if (Length == 0) {
 173       Pos = SavedPos;
 174       return false;
 175     }
 176
 177     char *TextPtr = Allocator.Allocate<char>(Length + 1);
 178
 179     memcpy(TextPtr, WordText.c_str(), Length + 1);
 180     StringRef Text = StringRef(TextPtr, Length);
 181
 182     formTokenWithChars(Tok, Loc, WordBegin, Length, Text);
 183     return true;
 184   }
 185
 186   bool lexDelimitedSeq(Token &Tok, char OpenDelim, char CloseDelim) {
 187     if (isEnd())
 188       return false;
 189
 190     Position SavedPos = Pos;
 191
 192     consumeWhitespace();
 193     SmallString<32> WordText;
 194     const char *WordBegin = Pos.BufferPtr;
 195     SourceLocation Loc = getSourceLocation();
 196     bool Error = false;
 197     if (!isEnd()) {
 198       const char C = peek();
 199       if (C == OpenDelim) {
 200         WordText.push_back(C);
 201         consumeChar();
 202       } else
 203         Error = true;
 204     }
 205     char C = '\0';
 206     while (!Error && !isEnd()) {
 207       C = peek();
 208       WordText.push_back(C);
 209       consumeChar();
 210       if (C == CloseDelim)
 211         break;
 212     }
 213     if (!Error && C != CloseDelim)
 214       Error = true;
 215
 216     if (Error) {
 217       Pos = SavedPos;
 218       return false;
 219     }
 220
 221     const unsigned Length = WordText.size();
 222     char *TextPtr = Allocator.Allocate<char>(Length + 1);
 223
 224     memcpy(TextPtr, WordText.c_str(), Length + 1);
 225     StringRef Text = StringRef(TextPtr, Length);
 226
 227     formTokenWithChars(Tok, Loc, WordBegin,
 228                        Pos.BufferPtr - WordBegin, Text);
 229     return true;
 230   }
 231
 232   /// Put back tokens that we didn't consume.
 233   void putBackLeftoverTokens() {
 234     if (isEnd())
 235       return;
 236
 237     bool HavePartialTok = false;
 238     Token PartialTok;
 239     if (Pos.BufferPtr != Pos.BufferStart) {
 240       formTokenWithChars(PartialTok, getSourceLocation(),
 241                          Pos.BufferPtr, Pos.BufferEnd - Pos.BufferPtr,
 242                          StringRef(Pos.BufferPtr,
 243                                    Pos.BufferEnd - Pos.BufferPtr));
 244       HavePartialTok = true;
 245       Pos.CurToken++;
 246     }
 247
 248     P.putBack(llvm::ArrayRef(Toks.begin() + Pos.CurToken, Toks.end()));
 249     Pos.CurToken = Toks.size();
 250
 251     if (HavePartialTok)
 252       P.putBack(PartialTok);
 253   }
 254 };
 255
 256 Parser::Parser(Lexer &L, Sema &S, llvm::BumpPtrAllocator &Allocator,
 257                const SourceManager &SourceMgr, DiagnosticsEngine &Diags,
 258                const CommandTraits &Traits):
 259     L(L), S(S), Allocator(Allocator), SourceMgr(SourceMgr), Diags(Diags),
 260     Traits(Traits) {
 261   consumeToken();
 262 }
 263
 264 void Parser::parseParamCommandArgs(ParamCommandComment *PC,
 265                                    TextTokenRetokenizer &Retokenizer) {
 266   Token Arg;
 267   // Check if argument looks like direction specification: [dir]
 268   // e.g., [in], [out], [in,out]
 269   if (Retokenizer.lexDelimitedSeq(Arg, '[', ']'))
 270     S.actOnParamCommandDirectionArg(PC,
 271                                     Arg.getLocation(),
 272                                     Arg.getEndLocation(),
 273                                     Arg.getText());
 274
 275   if (Retokenizer.lexWord(Arg))
 276     S.actOnParamCommandParamNameArg(PC,
 277                                     Arg.getLocation(),
 278                                     Arg.getEndLocation(),
 279                                     Arg.getText());
 280 }
 281
 282 void Parser::parseTParamCommandArgs(TParamCommandComment *TPC,
 283                                     TextTokenRetokenizer &Retokenizer) {
 284   Token Arg;
 285   if (Retokenizer.lexWord(Arg))
 286     S.actOnTParamCommandParamNameArg(TPC,
 287                                      Arg.getLocation(),
 288                                      Arg.getEndLocation(),
 289                                      Arg.getText());
 290 }
 291
 292 ArrayRef<Comment::Argument>
 293 Parser::parseCommandArgs(TextTokenRetokenizer &Retokenizer, unsigned NumArgs) {
 294   auto *Args = new (Allocator.Allocate<Comment::Argument>(NumArgs))
 295       Comment::Argument[NumArgs];
 296   unsigned ParsedArgs = 0;
 297   Token Arg;
 298   while (ParsedArgs < NumArgs && Retokenizer.lexWord(Arg)) {
 299     Args[ParsedArgs] = Comment::Argument{
 300         SourceRange(Arg.getLocation(), Arg.getEndLocation()), Arg.getText()};
 301     ParsedArgs++;
 302   }
 303
 304   return llvm::ArrayRef(Args, ParsedArgs);
 305 }
 306
 307 BlockCommandComment *Parser::parseBlockCommand() {
 308   assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));
 309
 310   ParamCommandComment *PC = nullptr;
 311   TParamCommandComment *TPC = nullptr;
 312   BlockCommandComment *BC = nullptr;
 313   const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
 314   CommandMarkerKind CommandMarker =
 315       Tok.is(tok::backslash_command) ? CMK_Backslash : CMK_At;
 316   if (Info->IsParamCommand) {
 317     PC = S.actOnParamCommandStart(Tok.getLocation(),
 318                                   Tok.getEndLocation(),
 319                                   Tok.getCommandID(),
 320                                   CommandMarker);
 321   } else if (Info->IsTParamCommand) {
 322     TPC = S.actOnTParamCommandStart(Tok.getLocation(),
 323                                     Tok.getEndLocation(),
 324                                     Tok.getCommandID(),
 325                                     CommandMarker);
 326   } else {
 327     BC = S.actOnBlockCommandStart(Tok.getLocation(),
 328                                   Tok.getEndLocation(),
 329                                   Tok.getCommandID(),
 330                                   CommandMarker);
 331   }
 332   consumeToken();
 333
 334   if (isTokBlockCommand()) {
 335     // Block command ahead.  We can't nest block commands, so pretend that this
 336     // command has an empty argument.
 337     ParagraphComment *Paragraph = S.actOnParagraphComment(std::nullopt);
 338     if (PC) {
 339       S.actOnParamCommandFinish(PC, Paragraph);
 340       return PC;
 341     } else if (TPC) {
 342       S.actOnTParamCommandFinish(TPC, Paragraph);
 343       return TPC;
 344     } else {
 345       S.actOnBlockCommandFinish(BC, Paragraph);
 346       return BC;
 347     }
 348   }
 349
 350   if (PC || TPC || Info->NumArgs > 0) {
 351     // In order to parse command arguments we need to retokenize a few
 352     // following text tokens.
 353     TextTokenRetokenizer Retokenizer(Allocator, *this);
 354
 355     if (PC)
 356       parseParamCommandArgs(PC, Retokenizer);
 357     else if (TPC)
 358       parseTParamCommandArgs(TPC, Retokenizer);
 359     else
 360       S.actOnBlockCommandArgs(BC, parseCommandArgs(Retokenizer, Info->NumArgs));
 361
 362     Retokenizer.putBackLeftoverTokens();
 363   }
 364
 365   // If there's a block command ahead, we will attach an empty paragraph to
 366   // this command.
 367   bool EmptyParagraph = false;
 368   if (isTokBlockCommand())
 369     EmptyParagraph = true;
 370   else if (Tok.is(tok::newline)) {
 371     Token PrevTok = Tok;
 372     consumeToken();
 373     EmptyParagraph = isTokBlockCommand();
 374     putBack(PrevTok);
 375   }
 376
 377   ParagraphComment *Paragraph;
 378   if (EmptyParagraph)
 379     Paragraph = S.actOnParagraphComment(std::nullopt);
 380   else {
 381     BlockContentComment *Block = parseParagraphOrBlockCommand();
 382     // Since we have checked for a block command, we should have parsed a
 383     // paragraph.
 384     Paragraph = cast<ParagraphComment>(Block);
 385   }
 386
 387   if (PC) {
 388     S.actOnParamCommandFinish(PC, Paragraph);
 389     return PC;
 390   } else if (TPC) {
 391     S.actOnTParamCommandFinish(TPC, Paragraph);
 392     return TPC;
 393   } else {
 394     S.actOnBlockCommandFinish(BC, Paragraph);
 395     return BC;
 396   }
 397 }
 398
 399 InlineCommandComment *Parser::parseInlineCommand() {
 400   assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));
 401   const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
 402
 403   const Token CommandTok = Tok;
 404   consumeToken();
 405
 406   TextTokenRetokenizer Retokenizer(Allocator, *this);
 407   ArrayRef<Comment::Argument> Args =
 408       parseCommandArgs(Retokenizer, Info->NumArgs);
 409
 410   InlineCommandComment *IC = S.actOnInlineCommand(
 411       CommandTok.getLocation(), CommandTok.getEndLocation(),
 412       CommandTok.getCommandID(), Args);
 413
 414   if (Args.size() < Info->NumArgs) {
 415     Diag(CommandTok.getEndLocation().getLocWithOffset(1),
 416          diag::warn_doc_inline_command_not_enough_arguments)
 417         << CommandTok.is(tok::at_command) << Info->Name << Args.size()
 418         << Info->NumArgs
 419         << SourceRange(CommandTok.getLocation(), CommandTok.getEndLocation());
 420   }
 421
 422   Retokenizer.putBackLeftoverTokens();
 423
 424   return IC;
 425 }
 426
 427 HTMLStartTagComment *Parser::parseHTMLStartTag() {
 428   assert(Tok.is(tok::html_start_tag));
 429   HTMLStartTagComment *HST =
 430       S.actOnHTMLStartTagStart(Tok.getLocation(),
 431                                Tok.getHTMLTagStartName());
 432   consumeToken();
 433
 434   SmallVector<HTMLStartTagComment::Attribute, 2> Attrs;
 435   while (true) {
 436     switch (Tok.getKind()) {
 437     case tok::html_ident: {
 438       Token Ident = Tok;
 439       consumeToken();
 440       if (Tok.isNot(tok::html_equals)) {
 441         Attrs.push_back(HTMLStartTagComment::Attribute(Ident.getLocation(),
 442                                                        Ident.getHTMLIdent()));
 443         continue;
 444       }
 445       Token Equals = Tok;
 446       consumeToken();
 447       if (Tok.isNot(tok::html_quoted_string)) {
 448         Diag(Tok.getLocation(),
 449              diag::warn_doc_html_start_tag_expected_quoted_string)
 450           << SourceRange(Equals.getLocation());
 451         Attrs.push_back(HTMLStartTagComment::Attribute(Ident.getLocation(),
 452                                                        Ident.getHTMLIdent()));
 453         while (Tok.is(tok::html_equals) ||
 454                Tok.is(tok::html_quoted_string))
 455           consumeToken();
 456         continue;
 457       }
 458       Attrs.push_back(HTMLStartTagComment::Attribute(
 459                               Ident.getLocation(),
 460                               Ident.getHTMLIdent(),
 461                               Equals.getLocation(),
 462                               SourceRange(Tok.getLocation(),
 463                                           Tok.getEndLocation()),
 464                               Tok.getHTMLQuotedString()));
 465       consumeToken();
 466       continue;
 467     }
 468
 469     case tok::html_greater:
 470       S.actOnHTMLStartTagFinish(HST, S.copyArray(llvm::ArrayRef(Attrs)),
 471                                 Tok.getLocation(),
 472                                 /* IsSelfClosing = */ false);
 473       consumeToken();
 474       return HST;
 475
 476     case tok::html_slash_greater:
 477       S.actOnHTMLStartTagFinish(HST, S.copyArray(llvm::ArrayRef(Attrs)),
 478                                 Tok.getLocation(),
 479                                 /* IsSelfClosing = */ true);
 480       consumeToken();
 481       return HST;
 482
 483     case tok::html_equals:
 484     case tok::html_quoted_string:
 485       Diag(Tok.getLocation(),
 486            diag::warn_doc_html_start_tag_expected_ident_or_greater);
 487       while (Tok.is(tok::html_equals) ||
 488              Tok.is(tok::html_quoted_string))
 489         consumeToken();
 490       if (Tok.is(tok::html_ident) ||
 491           Tok.is(tok::html_greater) ||
 492           Tok.is(tok::html_slash_greater))
 493         continue;
 494
 495       S.actOnHTMLStartTagFinish(HST, S.copyArray(llvm::ArrayRef(Attrs)),
 496                                 SourceLocation(),
 497                                 /* IsSelfClosing = */ false);
 498       return HST;
 499
 500     default:
 501       // Not a token from an HTML start tag.  Thus HTML tag prematurely ended.
 502       S.actOnHTMLStartTagFinish(HST, S.copyArray(llvm::ArrayRef(Attrs)),
 503                                 SourceLocation(),
 504                                 /* IsSelfClosing = */ false);
 505       bool StartLineInvalid;
 506       const unsigned StartLine = SourceMgr.getPresumedLineNumber(
 507                                                   HST->getLocation(),
 508                                                   &StartLineInvalid);
 509       bool EndLineInvalid;
 510       const unsigned EndLine = SourceMgr.getPresumedLineNumber(
 511                                                   Tok.getLocation(),
 512                                                   &EndLineInvalid);
 513       if (StartLineInvalid || EndLineInvalid || StartLine == EndLine)
 514         Diag(Tok.getLocation(),
 515              diag::warn_doc_html_start_tag_expected_ident_or_greater)
 516           << HST->getSourceRange();
 517       else {
 518         Diag(Tok.getLocation(),
 519              diag::warn_doc_html_start_tag_expected_ident_or_greater);
 520         Diag(HST->getLocation(), diag::note_doc_html_tag_started_here)
 521           << HST->getSourceRange();
 522       }
 523       return HST;
 524     }
 525   }
 526 }
 527
 528 HTMLEndTagComment *Parser::parseHTMLEndTag() {
 529   assert(Tok.is(tok::html_end_tag));
 530   Token TokEndTag = Tok;
 531   consumeToken();
 532   SourceLocation Loc;
 533   if (Tok.is(tok::html_greater)) {
 534     Loc = Tok.getLocation();
 535     consumeToken();
 536   }
 537
 538   return S.actOnHTMLEndTag(TokEndTag.getLocation(),
 539                            Loc,
 540                            TokEndTag.getHTMLTagEndName());
 541 }
 542
 543 BlockContentComment *Parser::parseParagraphOrBlockCommand() {
 544   SmallVector<InlineContentComment *, 8> Content;
 545
 546   while (true) {
 547     switch (Tok.getKind()) {
 548     case tok::verbatim_block_begin:
 549     case tok::verbatim_line_name:
 550     case tok::eof:
 551       break; // Block content or EOF ahead, finish this parapgaph.
 552
 553     case tok::unknown_command:
 554       Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
 555                                               Tok.getEndLocation(),
 556                                               Tok.getUnknownCommandName()));
 557       consumeToken();
 558       continue;
 559
 560     case tok::backslash_command:
 561     case tok::at_command: {
 562       const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
 563       if (Info->IsBlockCommand) {
 564         if (Content.size() == 0)
 565           return parseBlockCommand();
 566         break; // Block command ahead, finish this parapgaph.
 567       }
 568       if (Info->IsVerbatimBlockEndCommand) {
 569         Diag(Tok.getLocation(),
 570              diag::warn_verbatim_block_end_without_start)
 571           << Tok.is(tok::at_command)
 572           << Info->Name
 573           << SourceRange(Tok.getLocation(), Tok.getEndLocation());
 574         consumeToken();
 575         continue;
 576       }
 577       if (Info->IsUnknownCommand) {
 578         Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
 579                                                 Tok.getEndLocation(),
 580                                                 Info->getID()));
 581         consumeToken();
 582         continue;
 583       }
 584       assert(Info->IsInlineCommand);
 585       Content.push_back(parseInlineCommand());
 586       continue;
 587     }
 588
 589     case tok::newline: {
 590       consumeToken();
 591       if (Tok.is(tok::newline) || Tok.is(tok::eof)) {
 592         consumeToken();
 593         break; // Two newlines -- end of paragraph.
 594       }
 595       // Also allow [tok::newline, tok::text, tok::newline] if the middle
 596       // tok::text is just whitespace.
 597       if (Tok.is(tok::text) && isWhitespace(Tok.getText())) {
 598         Token WhitespaceTok = Tok;
 599         consumeToken();
 600         if (Tok.is(tok::newline) || Tok.is(tok::eof)) {
 601           consumeToken();
 602           break;
 603         }
 604         // We have [tok::newline, tok::text, non-newline].  Put back tok::text.
 605         putBack(WhitespaceTok);
 606       }
 607       if (Content.size() > 0)
 608         Content.back()->addTrailingNewline();
 609       continue;
 610     }
 611
 612     // Don't deal with HTML tag soup now.
 613     case tok::html_start_tag:
 614       Content.push_back(parseHTMLStartTag());
 615       continue;
 616
 617     case tok::html_end_tag:
 618       Content.push_back(parseHTMLEndTag());
 619       continue;
 620
 621     case tok::text:
 622       Content.push_back(S.actOnText(Tok.getLocation(),
 623                                     Tok.getEndLocation(),
 624                                     Tok.getText()));
 625       consumeToken();
 626       continue;
 627
 628     case tok::verbatim_block_line:
 629     case tok::verbatim_block_end:
 630     case tok::verbatim_line_text:
 631     case tok::html_ident:
 632     case tok::html_equals:
 633     case tok::html_quoted_string:
 634     case tok::html_greater:
 635     case tok::html_slash_greater:
 636       llvm_unreachable("should not see this token");
 637     }
 638     break;
 639   }
 640
 641   return S.actOnParagraphComment(S.copyArray(llvm::ArrayRef(Content)));
 642 }
 643
 644 VerbatimBlockComment *Parser::parseVerbatimBlock() {
 645   assert(Tok.is(tok::verbatim_block_begin));
 646
 647   VerbatimBlockComment *VB =
 648       S.actOnVerbatimBlockStart(Tok.getLocation(),
 649                                 Tok.getVerbatimBlockID());
 650   consumeToken();
 651
 652   // Don't create an empty line if verbatim opening command is followed
 653   // by a newline.
 654   if (Tok.is(tok::newline))
 655     consumeToken();
 656
 657   SmallVector<VerbatimBlockLineComment *, 8> Lines;
 658   while (Tok.is(tok::verbatim_block_line) ||
 659          Tok.is(tok::newline)) {
 660     VerbatimBlockLineComment *Line;
 661     if (Tok.is(tok::verbatim_block_line)) {
 662       Line = S.actOnVerbatimBlockLine(Tok.getLocation(),
 663                                       Tok.getVerbatimBlockText());
 664       consumeToken();
 665       if (Tok.is(tok::newline)) {
 666         consumeToken();
 667       }
 668     } else {
 669       // Empty line, just a tok::newline.
 670       Line = S.actOnVerbatimBlockLine(Tok.getLocation(), "");
 671       consumeToken();
 672     }
 673     Lines.push_back(Line);
 674   }
 675
 676   if (Tok.is(tok::verbatim_block_end)) {
 677     const CommandInfo *Info = Traits.getCommandInfo(Tok.getVerbatimBlockID());
 678     S.actOnVerbatimBlockFinish(VB, Tok.getLocation(), Info->Name,
 679                                S.copyArray(llvm::ArrayRef(Lines)));
 680     consumeToken();
 681   } else {
 682     // Unterminated \\verbatim block
 683     S.actOnVerbatimBlockFinish(VB, SourceLocation(), "",
 684                                S.copyArray(llvm::ArrayRef(Lines)));
 685   }
 686
 687   return VB;
 688 }
 689
 690 VerbatimLineComment *Parser::parseVerbatimLine() {
 691   assert(Tok.is(tok::verbatim_line_name));
 692
 693   Token NameTok = Tok;
 694   consumeToken();
 695
 696   SourceLocation TextBegin;
 697   StringRef Text;
 698   // Next token might not be a tok::verbatim_line_text if verbatim line
 699   // starting command comes just before a newline or comment end.
 700   if (Tok.is(tok::verbatim_line_text)) {
 701     TextBegin = Tok.getLocation();
 702     Text = Tok.getVerbatimLineText();
 703   } else {
 704     TextBegin = NameTok.getEndLocation();
 705     Text = "";
 706   }
 707
 708   VerbatimLineComment *VL = S.actOnVerbatimLine(NameTok.getLocation(),
 709                                                 NameTok.getVerbatimLineID(),
 710                                                 TextBegin,
 711                                                 Text);
 712   consumeToken();
 713   return VL;
 714 }
 715
 716 BlockContentComment *Parser::parseBlockContent() {
 717   switch (Tok.getKind()) {
 718   case tok::text:
 719   case tok::unknown_command:
 720   case tok::backslash_command:
 721   case tok::at_command:
 722   case tok::html_start_tag:
 723   case tok::html_end_tag:
 724     return parseParagraphOrBlockCommand();
 725
 726   case tok::verbatim_block_begin:
 727     return parseVerbatimBlock();
 728
 729   case tok::verbatim_line_name:
 730     return parseVerbatimLine();
 731
 732   case tok::eof:
 733   case tok::newline:
 734   case tok::verbatim_block_line:
 735   case tok::verbatim_block_end:
 736   case tok::verbatim_line_text:
 737   case tok::html_ident:
 738   case tok::html_equals:
 739   case tok::html_quoted_string:
 740   case tok::html_greater:
 741   case tok::html_slash_greater:
 742     llvm_unreachable("should not see this token");
 743   }
 744   llvm_unreachable("bogus token kind");
 745 }
 746
 747 FullComment *Parser::parseFullComment() {
 748   // Skip newlines at the beginning of the comment.
 749   while (Tok.is(tok::newline))
 750     consumeToken();
 751
 752   SmallVector<BlockContentComment *, 8> Blocks;
 753   while (Tok.isNot(tok::eof)) {
 754     Blocks.push_back(parseBlockContent());
 755
 756     // Skip extra newlines after paragraph end.
 757     while (Tok.is(tok::newline))
 758       consumeToken();
 759   }
 760   return S.actOnFullComment(S.copyArray(llvm::ArrayRef(Blocks)));
 761 }
 762
 763 } // end namespace comments
 764 } // end namespace clang