clang/unittests/Format/TestLexer.h

   1 //===--- TestLexer.h - Format C++ code --------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 ///
   9 /// \file
  10 /// This file contains a TestLexer to create FormatTokens from strings.
  11 ///
  12 //===----------------------------------------------------------------------===//
  13
  14 #ifndef CLANG_UNITTESTS_FORMAT_TESTLEXER_H
  15 #define CLANG_UNITTESTS_FORMAT_TESTLEXER_H
  16
  17 #include "../../lib/Format/FormatTokenLexer.h"
  18 #include "../../lib/Format/TokenAnalyzer.h"
  19 #include "../../lib/Format/TokenAnnotator.h"
  20 #include "../../lib/Format/UnwrappedLineParser.h"
  21
  22 #include "clang/Basic/FileManager.h"
  23 #include "clang/Basic/SourceManager.h"
  24
  25 #include <numeric>
  26 #include <ostream>
  27
  28 namespace clang {
  29 namespace format {
  30
  31 typedef SmallVector<FormatToken *, 8> TokenList;
  32
  33 inline std::ostream &operator<<(std::ostream &Stream, const FormatToken &Tok) {
  34   Stream << "(" << Tok.Tok.getName() << ", \"" << Tok.TokenText.str() << "\" , "
  35          << getTokenTypeName(Tok.getType()) << ")";
  36   return Stream;
  37 }
  38 inline std::ostream &operator<<(std::ostream &Stream, const TokenList &Tokens) {
  39   Stream << "{";
  40   for (size_t I = 0, E = Tokens.size(); I != E; ++I)
  41     Stream << (I > 0 ? ", " : "") << *Tokens[I];
  42   Stream << "} (" << Tokens.size() << " tokens)";
  43   return Stream;
  44 }
  45
  46 inline TokenList uneof(const TokenList &Tokens) {
  47   assert(!Tokens.empty() && Tokens.back()->is(tok::eof));
  48   return TokenList(Tokens.begin(), std::prev(Tokens.end()));
  49 }
  50
  51 inline std::string text(ArrayRef<FormatToken *> Tokens) {
  52   return std::accumulate(Tokens.begin(), Tokens.end(), std::string(),
  53                          [](const std::string &R, FormatToken *Tok) {
  54                            return (R + Tok->TokenText).str();
  55                          });
  56 }
  57
  58 class TestLexer : public UnwrappedLineConsumer {
  59 public:
  60   TestLexer(llvm::SpecificBumpPtrAllocator<FormatToken> &Allocator,
  61             std::vector<std::unique_ptr<llvm::MemoryBuffer>> &Buffers,
  62             FormatStyle Style = getLLVMStyle())
  63       : Allocator(Allocator), Buffers(Buffers), Style(Style),
  64         SourceMgr("test.cpp", ""), IdentTable(getFormattingLangOpts(Style)) {}
  65
  66   TokenList lex(StringRef Code) {
  67     FormatTokenLexer Lex = getNewLexer(Code);
  68     ArrayRef<FormatToken *> Result = Lex.lex();
  69     return TokenList(Result.begin(), Result.end());
  70   }
  71
  72   TokenList annotate(StringRef Code) {
  73     FormatTokenLexer Lex = getNewLexer(Code);
  74     auto Tokens = Lex.lex();
  75     UnwrappedLineParser Parser(SourceMgr.get(), Style, Lex.getKeywords(), 0,
  76                                Tokens, *this, Allocator, IdentTable);
  77     Parser.parse();
  78     TokenAnnotator Annotator(Style, Lex.getKeywords());
  79     for (auto &Line : UnwrappedLines) {
  80       AnnotatedLine Annotated(Line);
  81       Annotator.annotate(Annotated);
  82       Annotator.calculateFormattingInformation(Annotated);
  83     }
  84     UnwrappedLines.clear();
  85     return TokenList(Tokens.begin(), Tokens.end());
  86   }
  87
  88   FormatToken *id(StringRef Code) {
  89     auto Result = uneof(lex(Code));
  90     assert(Result.size() == 1U && "Code must expand to 1 token.");
  91     return Result[0];
  92   }
  93
  94 protected:
  95   void consumeUnwrappedLine(const UnwrappedLine &TheLine) override {
  96     UnwrappedLines.push_back(TheLine);
  97   }
  98   void finishRun() override {}
  99
 100   FormatTokenLexer getNewLexer(StringRef Code) {
 101     Buffers.push_back(
 102         llvm::MemoryBuffer::getMemBufferCopy(Code, "<scratch space>"));
 103     FileID FID =
 104         SourceMgr.get().createFileID(Buffers.back()->getMemBufferRef());
 105     return FormatTokenLexer(SourceMgr.get(), FID, 0, Style, Encoding, Allocator,
 106                             IdentTable);
 107   }
 108
 109 public:
 110   llvm::SpecificBumpPtrAllocator<FormatToken> &Allocator;
 111   std::vector<std::unique_ptr<llvm::MemoryBuffer>> &Buffers;
 112   FormatStyle Style;
 113   encoding::Encoding Encoding = encoding::Encoding_UTF8;
 114   SourceManagerForFile SourceMgr;
 115   IdentifierTable IdentTable;
 116   SmallVector<UnwrappedLine, 16> UnwrappedLines;
 117 };
 118
 119 } // namespace format
 120 } // namespace clang
 121
 122 #endif // LLVM_CLANG_UNITTESTS_FORMAT_TEST_LEXER_H