clang/unittests/Format/FormatTokenSourceTest.cpp

   1 //===- unittest/Format/FormatTokenSourceTest.cpp --------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "../../lib/Format/FormatTokenSource.h"
  10 #include "TestLexer.h"
  11 #include "clang/Basic/TokenKinds.h"
  12 #include "gtest/gtest.h"
  13
  14 namespace clang {
  15 namespace format {
  16 namespace {
  17
  18 class IndexedTokenSourceTest : public ::testing::Test {
  19 protected:
  20   TokenList lex(llvm::StringRef Code,
  21                 const FormatStyle &Style = getLLVMStyle()) {
  22     return TestLexer(Allocator, Buffers, Style).lex(Code);
  23   }
  24   llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
  25   std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers;
  26 };
  27
  28 #define EXPECT_TOKEN_KIND(FormatTok, Kind)                                     \
  29   do {                                                                         \
  30     FormatToken *Tok = FormatTok;                                              \
  31     EXPECT_EQ(Tok->Tok.getKind(), Kind) << *Tok;                               \
  32   } while (false);
  33 #define EXPECT_TOKEN_ID(FormatTok, Name)                                       \
  34   do {                                                                         \
  35     FormatToken *Tok = FormatTok;                                              \
  36     EXPECT_EQ(Tok->Tok.getKind(), tok::identifier) << *Tok;                    \
  37     EXPECT_EQ(Tok->TokenText, Name) << *Tok;                                   \
  38   } while (false);
  39
  40 TEST_F(IndexedTokenSourceTest, EmptyInput) {
  41   IndexedTokenSource Source(lex(""));
  42   EXPECT_FALSE(Source.isEOF());
  43   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof);
  44   EXPECT_TRUE(Source.isEOF());
  45   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof);
  46   EXPECT_TRUE(Source.isEOF());
  47   EXPECT_TOKEN_KIND(Source.peekNextToken(/*SkipComment=*/false), tok::eof);
  48   EXPECT_TOKEN_KIND(Source.peekNextToken(/*SkipComment=*/true), tok::eof);
  49   EXPECT_EQ(Source.getPreviousToken(), nullptr);
  50   EXPECT_TRUE(Source.isEOF());
  51 }
  52
  53 TEST_F(IndexedTokenSourceTest, NavigateTokenStream) {
  54   IndexedTokenSource Source(lex("int a;"));
  55   EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::kw_int);
  56   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::kw_int);
  57   EXPECT_EQ(Source.getPreviousToken(), nullptr);
  58   EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::identifier);
  59   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::identifier);
  60   EXPECT_TOKEN_KIND(Source.getPreviousToken(), tok::kw_int);
  61   EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::semi);
  62   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::semi);
  63   EXPECT_TOKEN_KIND(Source.getPreviousToken(), tok::identifier);
  64   EXPECT_TOKEN_KIND(Source.peekNextToken(), tok::eof);
  65   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof);
  66   EXPECT_TOKEN_KIND(Source.getPreviousToken(), tok::semi);
  67   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof);
  68   EXPECT_TOKEN_KIND(Source.getPreviousToken(), tok::semi);
  69 }
  70
  71 TEST_F(IndexedTokenSourceTest, ResetPosition) {
  72   IndexedTokenSource Source(lex("int a;"));
  73   Source.getNextToken();
  74   unsigned Position = Source.getPosition();
  75   Source.getNextToken();
  76   Source.getNextToken();
  77   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof);
  78   EXPECT_TOKEN_KIND(Source.setPosition(Position), tok::kw_int);
  79 }
  80
  81 TEST_F(IndexedTokenSourceTest, InsertTokens) {
  82   IndexedTokenSource Source(lex("A1 A2"));
  83   EXPECT_TOKEN_ID(Source.getNextToken(), "A1");
  84   EXPECT_TOKEN_ID(Source.insertTokens(lex("B1 B2")), "B1");
  85   EXPECT_TOKEN_ID(Source.getNextToken(), "B2");
  86   EXPECT_TOKEN_ID(Source.getNextToken(), "A1");
  87   EXPECT_TOKEN_ID(Source.getNextToken(), "A2");
  88 }
  89
  90 TEST_F(IndexedTokenSourceTest, InsertTokensAtEOF) {
  91   IndexedTokenSource Source(lex("A1"));
  92   EXPECT_TOKEN_ID(Source.getNextToken(), "A1");
  93   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof);
  94   EXPECT_TOKEN_ID(Source.insertTokens(lex("B1 B2")), "B1");
  95   EXPECT_TOKEN_ID(Source.getNextToken(), "B2");
  96   EXPECT_TOKEN_KIND(Source.getNextToken(), tok::eof);
  97 }
  98
  99 TEST_F(IndexedTokenSourceTest, InsertTokensRecursive) {
 100   IndexedTokenSource Source(lex("A1"));
 101   EXPECT_TOKEN_ID(Source.getNextToken(), "A1");
 102   // A1
 103   EXPECT_TOKEN_ID(Source.insertTokens(lex("B1")), "B1");
 104   // B1 A1
 105   EXPECT_TOKEN_ID(Source.insertTokens(lex("C1")), "C1");
 106   // C1 B1 A1
 107   EXPECT_TOKEN_ID(Source.insertTokens(lex("D1")), "D1");
 108   // D1 C1 B1 A1
 109   EXPECT_TOKEN_ID(Source.getNextToken(), "C1");
 110   EXPECT_TOKEN_ID(Source.getNextToken(), "B1");
 111   EXPECT_TOKEN_ID(Source.getNextToken(), "A1");
 112 }
 113
 114 TEST_F(IndexedTokenSourceTest, InsertTokensRecursiveAtEndOfSequence) {
 115   IndexedTokenSource Source(lex("A1"));
 116   EXPECT_TOKEN_ID(Source.getNextToken(), "A1");
 117   EXPECT_TOKEN_ID(Source.insertTokens(lex("B1")), "B1");
 118   EXPECT_TOKEN_ID(Source.getNextToken(), "A1");
 119   EXPECT_TOKEN_ID(Source.insertTokens(lex("C1")), "C1");
 120   EXPECT_TOKEN_ID(Source.getNextToken(), "A1");
 121   EXPECT_TOKEN_ID(Source.insertTokens(lex("D1")), "D1");
 122   EXPECT_TOKEN_ID(Source.getNextToken(), "A1");
 123 }
 124
 125 } // namespace
 126 } // namespace format
 127 } // namespace clang