clang-tools-extra/pseudo/unittests/TokenTest.cpp

   1 //===--- TokenTest.cpp ----------------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "clang-pseudo/Token.h"
  10 #include "clang/Basic/LangOptions.h"
  11 #include "clang/Basic/TokenKinds.h"
  12 #include "gmock/gmock.h"
  13 #include "gtest/gtest.h"
  14
  15 namespace clang {
  16 namespace pseudo {
  17 namespace {
  18
  19 using testing::AllOf;
  20 using testing::ElementsAre;
  21 using testing::ElementsAreArray;
  22 using testing::Not;
  23
  24 MATCHER_P2(token, Text, Kind, "") {
  25   return arg.Kind == Kind && arg.text() == Text;
  26 }
  27
  28 MATCHER_P(hasFlag, Flag, "") { return arg.flag(Flag); }
  29
  30 MATCHER_P2(lineIndent, Line, Indent, "") {
  31   return arg.Line == (unsigned)Line && arg.Indent == (unsigned)Indent;
  32 }
  33
  34 MATCHER_P(originalIndex, index, "") {
  35   return arg.OriginalIndex == (Token::Index)index;
  36 }
  37
  38 TEST(TokenTest, Lex) {
  39   LangOptions Opts;
  40   std::string Code = R"cpp(
  41     #include <stdio.h>
  42     int main() {
  43       return 42; // the answer
  44     }
  45   )cpp";
  46   TokenStream Raw = lex(Code, Opts);
  47   ASSERT_TRUE(Raw.isFinalized());
  48   EXPECT_THAT(Raw.tokens(),
  49               ElementsAreArray({
  50                   // Lexing of directives is weird, especially <angled> strings.
  51                   token("#", tok::hash),
  52                   token("include", tok::raw_identifier),
  53                   token("<", tok::less),
  54                   token("stdio", tok::raw_identifier),
  55                   token(".", tok::period),
  56                   token("h", tok::raw_identifier),
  57                   token(">", tok::greater),
  58
  59                   token("int", tok::raw_identifier),
  60                   token("main", tok::raw_identifier),
  61                   token("(", tok::l_paren),
  62                   token(")", tok::r_paren),
  63                   token("{", tok::l_brace),
  64                   token("return", tok::raw_identifier),
  65                   token("42", tok::numeric_constant),
  66                   token(";", tok::semi),
  67                   token("// the answer", tok::comment),
  68                   token("}", tok::r_brace),
  69               }));
  70
  71   TokenStream Cooked = cook(Raw, Opts);
  72   ASSERT_TRUE(Cooked.isFinalized());
  73   EXPECT_THAT(Cooked.tokens(),
  74               ElementsAreArray({
  75                   // Cooked identifier types in directives are not meaningful.
  76                   token("#", tok::hash),
  77                   token("include", tok::identifier),
  78                   token("<", tok::less),
  79                   token("stdio", tok::identifier),
  80                   token(".", tok::period),
  81                   token("h", tok::identifier),
  82                   token(">", tok::greater),
  83
  84                   token("int", tok::kw_int),
  85                   token("main", tok::identifier),
  86                   token("(", tok::l_paren),
  87                   token(")", tok::r_paren),
  88                   token("{", tok::l_brace),
  89                   token("return", tok::kw_return),
  90                   token("42", tok::numeric_constant),
  91                   token(";", tok::semi),
  92                   token("// the answer", tok::comment),
  93                   token("}", tok::r_brace),
  94               }));
  95   // Check raw tokens point back into original source code.
  96   EXPECT_EQ(Raw.tokens().front().text().begin(), &Code[Code.find('#')]);
  97 }
  98
  99 TEST(TokenTest, LineContinuation) {
 100   LangOptions Opts;
 101   std::string Code = R"cpp(
 102 one_\
 103 token
 104 two \
 105 tokens
 106   )cpp";
 107   TokenStream Raw = lex(Code, Opts);
 108   EXPECT_THAT(
 109       Raw.tokens(),
 110       ElementsAre(AllOf(token("one_\\\ntoken", tok::raw_identifier),
 111                         hasFlag(LexFlags::StartsPPLine),
 112                         hasFlag(LexFlags::NeedsCleaning), lineIndent(1, 0),
 113                         originalIndex(0)),
 114                   AllOf(token("two", tok::raw_identifier),
 115                         hasFlag(LexFlags::StartsPPLine),
 116                         Not(hasFlag(LexFlags::NeedsCleaning)),
 117                         originalIndex(1)),
 118                   AllOf(token("\\\ntokens", tok::raw_identifier),
 119                         Not(hasFlag(LexFlags::StartsPPLine)),
 120                         hasFlag(LexFlags::NeedsCleaning), originalIndex(2))));
 121
 122   TokenStream Cooked = cook(Raw, Opts);
 123   EXPECT_THAT(
 124       Cooked.tokens(),
 125       ElementsAre(AllOf(token("one_token", tok::identifier), lineIndent(1, 0),
 126                         originalIndex(0)),
 127                   AllOf(token("two", tok::identifier), originalIndex(1)),
 128                   AllOf(token("tokens", tok::identifier), originalIndex(2))));
 129 }
 130
 131 TEST(TokenTest, EncodedCharacters) {
 132   LangOptions Opts;
 133   Opts.Trigraphs = true;
 134   Opts.Digraphs = true;
 135   Opts.C99 = true; // UCNs
 136   Opts.CXXOperatorNames = true;
 137   std::string Code = R"(and <: ??! '??=' \u00E9)";
 138   TokenStream Raw = lex(Code, Opts);
 139   EXPECT_THAT(
 140       Raw.tokens(),
 141       ElementsAre( // and is not recognized as && until cook().
 142           AllOf(token("and", tok::raw_identifier),
 143                 Not(hasFlag(LexFlags::NeedsCleaning))),
 144           // Digraphs are just different spellings of tokens.
 145           AllOf(token("<:", tok::l_square),
 146                 Not(hasFlag(LexFlags::NeedsCleaning))),
 147           // Trigraps are interpreted, still need text cleaning.
 148           AllOf(token(R"(??!)", tok::pipe), hasFlag(LexFlags::NeedsCleaning)),
 149           // Trigraphs must be substituted inside constants too.
 150           AllOf(token(R"('??=')", tok::char_constant),
 151                 hasFlag(LexFlags::NeedsCleaning)),
 152           // UCNs need substitution.
 153           AllOf(token(R"(\u00E9)", tok::raw_identifier),
 154                 hasFlag(LexFlags::NeedsCleaning))));
 155
 156   TokenStream Cooked = cook(Raw, Opts);
 157   EXPECT_THAT(
 158       Cooked.tokens(),
 159       ElementsAre(token("and", tok::ampamp), // alternate spelling recognized
 160                   token("<:", tok::l_square),
 161                   token("|", tok::pipe),            // trigraph substituted
 162                   token("'#'", tok::char_constant), // trigraph substituted
 163                   token("é", tok::identifier)));    // UCN substituted
 164 }
 165
 166 TEST(TokenTest, Indentation) {
 167   LangOptions Opts;
 168   std::string Code = R"cpp(   hello world
 169 no_indent \
 170   line_was_continued
 171 )cpp";
 172   TokenStream Raw = lex(Code, Opts);
 173   EXPECT_THAT(Raw.tokens(), ElementsAreArray({
 174                                 lineIndent(0, 3), // hello
 175                                 lineIndent(0, 3), // world
 176                                 lineIndent(1, 0), // no_indent
 177                                 lineIndent(2, 2), // line_was_continued
 178                             }));
 179 }
 180
 181 TEST(TokenTest, SplitGreaterGreater) {
 182   LangOptions Opts;
 183   std::string Code = R"cpp(
 184 >> // split
 185 // >> with an escaped newline in the middle, split
 186 >\
 187 >
 188 >>= // not split
 189 )cpp";
 190   TokenStream Cook = cook(lex(Code, Opts), Opts);
 191   TokenStream Split = stripComments(Cook);
 192   EXPECT_THAT(Split.tokens(),
 193               ElementsAre(AllOf(token(">", tok::greater), originalIndex(0)),
 194                           AllOf(token(">", tok::greater), originalIndex(0)),
 195                           // Token 1 and 2 are comments.
 196                           AllOf(token(">", tok::greater), originalIndex(3)),
 197                           AllOf(token(">", tok::greater), originalIndex(3)),
 198                           AllOf(token(">>=", tok::greatergreaterequal),
 199                                 originalIndex(4))));
 200 }
 201
 202 TEST(TokenTest, DropComments) {
 203   LangOptions Opts;
 204   std::string Code = R"cpp(
 205   // comment
 206   int /*abc*/;
 207 )cpp";
 208   TokenStream Raw = cook(lex(Code, Opts), Opts);
 209   TokenStream Stripped = stripComments(Raw);
 210   EXPECT_THAT(
 211       Raw.tokens(),
 212       ElementsAre(AllOf(token("// comment", tok::comment), originalIndex(0)),
 213                   AllOf(token("int", tok::kw_int), originalIndex(1)),
 214                   AllOf(token("/*abc*/", tok::comment), originalIndex(2)),
 215                   AllOf(token(";", tok::semi), originalIndex(3))));
 216
 217   EXPECT_THAT(Stripped.tokens(),
 218               ElementsAre(AllOf(token("int", tok::kw_int), originalIndex(1)),
 219                           AllOf(token(";", tok::semi), originalIndex(3))));
 220 }
 221
 222 } // namespace
 223 } // namespace pseudo
 224 } // namespace clang