tools/gn/tokenizer_unittest.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "testing/gtest/include/gtest/gtest.h"
   6 #include "tools/gn/input_file.h"
   7 #include "tools/gn/token.h"
   8 #include "tools/gn/tokenizer.h"
   9
  10 namespace {
  11
  12 struct TokenExpectation {
  13   Token::Type type;
  14   const char* value;
  15 };
  16
  17 template<size_t len>
  18 bool CheckTokenizer(const char* input, const TokenExpectation (&expect)[len]) {
  19   InputFile input_file(SourceFile("/test"));
  20   input_file.SetContents(input);
  21
  22   Err err;
  23   std::vector<Token> results = Tokenizer::Tokenize(&input_file, &err);
  24
  25   if (results.size() != len)
  26     return false;
  27   for (size_t i = 0; i < len; i++) {
  28     if (expect[i].type != results[i].type())
  29       return false;
  30     if (expect[i].value != results[i].value())
  31       return false;
  32   }
  33   return true;
  34 }
  35
  36 }  // namespace
  37
  38 TEST(Tokenizer, Empty) {
  39   InputFile empty_string_input(SourceFile("/test"));
  40   empty_string_input.SetContents("");
  41
  42   Err err;
  43   std::vector<Token> results = Tokenizer::Tokenize(&empty_string_input, &err);
  44   EXPECT_TRUE(results.empty());
  45
  46   InputFile whitespace_input(SourceFile("/test"));
  47   whitespace_input.SetContents("  \r \n \r\n");
  48
  49   results = Tokenizer::Tokenize(&whitespace_input, &err);
  50   EXPECT_TRUE(results.empty());
  51 }
  52
  53 TEST(Tokenizer, Identifier) {
  54   TokenExpectation one_ident[] = {
  55     { Token::IDENTIFIER, "foo" }
  56   };
  57   EXPECT_TRUE(CheckTokenizer("  foo ", one_ident));
  58 }
  59
  60 TEST(Tokenizer, Integer) {
  61   TokenExpectation integers[] = {
  62     { Token::INTEGER, "123" },
  63     { Token::INTEGER, "-123" }
  64   };
  65   EXPECT_TRUE(CheckTokenizer("  123 -123 ", integers));
  66 }
  67
  68 TEST(Tokenizer, IntegerNoSpace) {
  69   TokenExpectation integers[] = {
  70     { Token::INTEGER, "123" },
  71     { Token::INTEGER, "-123" }
  72   };
  73   EXPECT_TRUE(CheckTokenizer("  123-123 ", integers));
  74 }
  75
  76 TEST(Tokenizer, String) {
  77   TokenExpectation strings[] = {
  78     { Token::STRING, "\"foo\"" },
  79     { Token::STRING, "\"bar\\\"baz\"" },
  80     { Token::STRING, "\"asdf\\\\\"" }
  81   };
  82   EXPECT_TRUE(CheckTokenizer("  \"foo\" \"bar\\\"baz\" \"asdf\\\\\" ",
  83               strings));
  84 }
  85
  86 TEST(Tokenizer, Operator) {
  87   TokenExpectation operators[] = {
  88     { Token::MINUS, "-" },
  89     { Token::PLUS, "+" },
  90     { Token::EQUAL, "=" },
  91     { Token::PLUS_EQUALS, "+=" },
  92     { Token::MINUS_EQUALS, "-=" },
  93     { Token::NOT_EQUAL, "!=" },
  94     { Token::EQUAL_EQUAL, "==" },
  95     { Token::LESS_THAN, "<" },
  96     { Token::GREATER_THAN, ">" },
  97     { Token::LESS_EQUAL, "<=" },
  98     { Token::GREATER_EQUAL, ">=" },
  99     { Token::BANG, "!" },
 100     { Token::BOOLEAN_OR, "||" },
 101     { Token::BOOLEAN_AND, "&&" },
 102     { Token::DOT, "." },
 103     { Token::COMMA, "," },
 104   };
 105   EXPECT_TRUE(CheckTokenizer("- + = += -= != ==  < > <= >= ! || && . ,",
 106               operators));
 107 }
 108
 109 TEST(Tokenizer, Scoper) {
 110   TokenExpectation scopers[] = {
 111     { Token::LEFT_BRACE, "{" },
 112     { Token::LEFT_BRACKET, "[" },
 113     { Token::RIGHT_BRACKET, "]" },
 114     { Token::RIGHT_BRACE, "}" },
 115     { Token::LEFT_PAREN, "(" },
 116     { Token::RIGHT_PAREN, ")" },
 117   };
 118   EXPECT_TRUE(CheckTokenizer("{[ ]} ()", scopers));
 119 }
 120
 121 TEST(Tokenizer, FunctionCall) {
 122   TokenExpectation fn[] = {
 123     { Token::IDENTIFIER, "fun" },
 124     { Token::LEFT_PAREN, "(" },
 125     { Token::STRING, "\"foo\"" },
 126     { Token::RIGHT_PAREN, ")" },
 127     { Token::LEFT_BRACE, "{" },
 128     { Token::IDENTIFIER, "foo" },
 129     { Token::EQUAL, "=" },
 130     { Token::INTEGER, "12" },
 131     { Token::RIGHT_BRACE, "}" },
 132   };
 133   EXPECT_TRUE(CheckTokenizer("fun(\"foo\") {\nfoo = 12}", fn));
 134 }
 135
 136 TEST(Tokenizer, Locations) {
 137   InputFile input(SourceFile("/test"));
 138   input.SetContents("1 2 \"three\"\n  4");
 139   Err err;
 140   std::vector<Token> results = Tokenizer::Tokenize(&input, &err);
 141
 142   ASSERT_EQ(4u, results.size());
 143   ASSERT_TRUE(results[0].location() == Location(&input, 1, 1, 1));
 144   ASSERT_TRUE(results[1].location() == Location(&input, 1, 3, 3));
 145   ASSERT_TRUE(results[2].location() == Location(&input, 1, 5, 5));
 146   ASSERT_TRUE(results[3].location() == Location(&input, 2, 3, 8));
 147 }
 148
 149 TEST(Tokenizer, ByteOffsetOfNthLine) {
 150   EXPECT_EQ(0u, Tokenizer::ByteOffsetOfNthLine("foo", 1));
 151
 152   // Windows and Posix have different line endings, so check the byte at the
 153   // location rather than the offset.
 154   char input1[] = "aaa\nxaa\n\nya";
 155   EXPECT_EQ('x', input1[Tokenizer::ByteOffsetOfNthLine(input1, 2)]);
 156   EXPECT_EQ('y', input1[Tokenizer::ByteOffsetOfNthLine(input1, 4)]);
 157
 158   char input2[3];
 159   input2[0] = 'a';
 160   input2[1] = '\n';  // Manually set to avoid Windows double-byte endings.
 161   input2[2] = 0;
 162   EXPECT_EQ(0u, Tokenizer::ByteOffsetOfNthLine(input2, 1));
 163   EXPECT_EQ(2u, Tokenizer::ByteOffsetOfNthLine(input2, 2));
 164 }
 165
 166 TEST(Tokenizer, Comments) {
 167   TokenExpectation fn[] = {
 168     { Token::LINE_COMMENT, "# Stuff" },
 169     { Token::IDENTIFIER, "fun" },
 170     { Token::LEFT_PAREN, "(" },
 171     { Token::STRING, "\"foo\"" },
 172     { Token::RIGHT_PAREN, ")" },
 173     { Token::LEFT_BRACE, "{" },
 174     { Token::SUFFIX_COMMENT, "# Things" },
 175     { Token::LINE_COMMENT, "#Wee" },
 176     { Token::IDENTIFIER, "foo" },
 177     { Token::EQUAL, "=" },
 178     { Token::INTEGER, "12" },
 179     { Token::SUFFIX_COMMENT, "#Zip" },
 180     { Token::RIGHT_BRACE, "}" },
 181   };
 182   EXPECT_TRUE(CheckTokenizer(
 183       "# Stuff\n"
 184       "fun(\"foo\") {  # Things\n"
 185       "#Wee\n"
 186       "foo = 12 #Zip\n"
 187       "}",
 188       fn));
 189 }
 190
 191 TEST(Tokenizer, CommentsContinued) {
 192   // In the first test, the comments aren't horizontally aligned, so they're
 193   // considered separate. In the second test, they are, so "B" is a
 194   // continuation of "A" (another SUFFIX comment).
 195   TokenExpectation fn1[] = {
 196     { Token::IDENTIFIER, "fun" },
 197     { Token::LEFT_PAREN, "(" },
 198     { Token::STRING, "\"foo\"" },
 199     { Token::RIGHT_PAREN, ")" },
 200     { Token::LEFT_BRACE, "{" },
 201     { Token::SUFFIX_COMMENT, "# A" },
 202     { Token::LINE_COMMENT, "# B" },
 203     { Token::RIGHT_BRACE, "}" },
 204   };
 205   EXPECT_TRUE(CheckTokenizer(
 206       "fun(\"foo\") {  # A\n"
 207       "  # B\n"
 208       "}",
 209       fn1));
 210
 211   TokenExpectation fn2[] = {
 212     { Token::IDENTIFIER, "fun" },
 213     { Token::LEFT_PAREN, "(" },
 214     { Token::STRING, "\"foo\"" },
 215     { Token::RIGHT_PAREN, ")" },
 216     { Token::LEFT_BRACE, "{" },
 217     { Token::SUFFIX_COMMENT, "# A" },
 218     { Token::SUFFIX_COMMENT, "# B" },
 219     { Token::RIGHT_BRACE, "}" },
 220   };
 221   EXPECT_TRUE(CheckTokenizer(
 222       "fun(\"foo\") {  # A\n"
 223       "              # B\n"  // Note that these are aligned, the \"s move A out.
 224       "}",
 225       fn2));
 226 }