clang/unittests/Format/FormatTokenSourceTest.cpp

   1 //===- unittest/Format/FormatTokenSourceTest.cpp --------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "../../lib/Format/FormatTokenSource.h"
  10 #include "TestLexer.h"
  11 #include "clang/Basic/TokenKinds.h"
  12 #include "gtest/gtest.h"
  13
  14 namespace clang {
  15 namespace format {
  16 namespace {
  17
  18 class IndexedTokenSourceTest : public testing::Test {
  19 protected:
  20   TokenList lex(StringRef Code, const FormatStyle &Style = getLLVMStyle()) {
  21     return TestLexer(Allocator, Buffers, Style).lex(Code);
  22   }
  23   llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
  24   std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers;
  25 };
  26
  27 #define EXPECT_TOKEN_KIND(FormatTok, Kind)                                     \
  28   do {                                                                         \
  29     FormatToken *Tok = FormatTok;                                              \
  30     EXPECT_EQ(Tok->Tok.getKind(), Kind) << *Tok;                               \
  31   } while (false);
  32 #define EXPECT_TOKEN_ID(FormatTok, Name)                                       \
  33   do {                                                                         \
  34     FormatToken *Tok = FormatTok;                                              \
  35     EXPECT_EQ(Tok->Tok.getKind(), tok::identifier) << *Tok;                    \
  36     EXPECT_EQ(Tok->TokenText, Name) << *Tok;                                   \
  37   } while (false);
  38
  39 TEST_F(IndexedTokenSourceTest, EmptyInput) {
  40   IndexedTokenSource Source(lex(""));
  41   EXPECT_FALSE(Source.isEOF());
  42   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof);
  43   EXPECT_TRUE(Source.isEOF());
  44   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof);
  45   EXPECT_TRUE(Source.isEOF());
  46   EXPECT_TOKEN_KIND(Source.peekNextToken(/*SkipComment=*/false), tok::eof);
  47   EXPECT_TOKEN_KIND(Source.peekNextToken(/*SkipComment=*/true), tok::eof);
  48   EXPECT_EQ(Source.getPreviousToken(), nullptr);
  49   EXPECT_TRUE(Source.isEOF());
  50 }
  51
  52 TEST_F(IndexedTokenSourceTest, NavigateTokenStream) {
  53   IndexedTokenSource Source(lex("int a;"));
  54   EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::kw_int);
  55   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::kw_int);
  56   EXPECT_EQ(Source.getPreviousToken(), nullptr);
  57   EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::identifier);
  58   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::identifier);
  59   EXPECT_TOKEN_KIND(Source.getPreviousToken(), tok::kw_int);
  60   EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::semi);
  61   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::semi);
  62   EXPECT_TOKEN_KIND(Source.getPreviousToken(), tok::identifier);
  63   EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::eof);
  64   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof);
  65   EXPECT_TOKEN_KIND(Source.getPreviousToken(), tok::semi);
  66   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof);
  67   EXPECT_TOKEN_KIND(Source.getPreviousToken(), tok::semi);
  68 }
  69
  70 TEST_F(IndexedTokenSourceTest, ResetPosition) {
  71   IndexedTokenSource Source(lex("int a;"));
  72   Source.getNextToken();
  73   unsigned Position = Source.getPosition();
  74   Source.getNextToken();
  75   Source.getNextToken();
  76   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof);
  77   EXPECT_TOKEN_KIND(Source.setPosition(Position), tok::kw_int);
  78 }
  79
  80 TEST_F(IndexedTokenSourceTest, InsertTokens) {
  81   IndexedTokenSource Source(lex("A1 A2"));
  82   EXPECT_TOKEN_ID(Source.getNextToken(), "A1");
  83   EXPECT_TOKEN_ID(Source.insertTokens(lex("B1 B2")), "B1");
  84   EXPECT_TOKEN_ID(Source.getNextToken(), "B2");
  85   EXPECT_TOKEN_ID(Source.getNextToken(), "A1");
  86   EXPECT_TOKEN_ID(Source.getNextToken(), "A2");
  87 }
  88
  89 TEST_F(IndexedTokenSourceTest, InsertTokensAtEOF) {
  90   IndexedTokenSource Source(lex("A1"));
  91   EXPECT_TOKEN_ID(Source.getNextToken(), "A1");
  92   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof);
  93   EXPECT_TOKEN_ID(Source.insertTokens(lex("B1 B2")), "B1");
  94   EXPECT_TOKEN_ID(Source.getNextToken(), "B2");
  95   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof);
  96 }
  97
  98 TEST_F(IndexedTokenSourceTest, InsertTokensRecursive) {
  99   IndexedTokenSource Source(lex("A1"));
 100   EXPECT_TOKEN_ID(Source.getNextToken(), "A1");
 101   // A1
 102   EXPECT_TOKEN_ID(Source.insertTokens(lex("B1")), "B1");
 103   // B1 A1
 104   EXPECT_TOKEN_ID(Source.insertTokens(lex("C1")), "C1");
 105   // C1 B1 A1
 106   EXPECT_TOKEN_ID(Source.insertTokens(lex("D1")), "D1");
 107   // D1 C1 B1 A1
 108   EXPECT_TOKEN_ID(Source.getNextToken(), "C1");
 109   EXPECT_TOKEN_ID(Source.getNextToken(), "B1");
 110   EXPECT_TOKEN_ID(Source.getNextToken(), "A1");
 111 }
 112
 113 TEST_F(IndexedTokenSourceTest, InsertTokensRecursiveAtEndOfSequence) {
 114   IndexedTokenSource Source(lex("A1"));
 115   EXPECT_TOKEN_ID(Source.getNextToken(), "A1");
 116   EXPECT_TOKEN_ID(Source.insertTokens(lex("B1")), "B1");
 117   EXPECT_TOKEN_ID(Source.getNextToken(), "A1");
 118   EXPECT_TOKEN_ID(Source.insertTokens(lex("C1")), "C1");
 119   EXPECT_TOKEN_ID(Source.getNextToken(), "A1");
 120   EXPECT_TOKEN_ID(Source.insertTokens(lex("D1")), "D1");
 121   EXPECT_TOKEN_ID(Source.getNextToken(), "A1");
 122 }
 123
 124 } // namespace
 125 } // namespace format
 126 } // namespace clang