base/string_tokenizer_unittest.cc

   1 // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/string_tokenizer.h"
   6
   7 #include "testing/gtest/include/gtest/gtest.h"
   8
   9 using std::string;
  10
  11 namespace {
  12
  13 TEST(StringTokenizerTest, Simple) {
  14   string input = "this is a test";
  15   StringTokenizer t(input, " ");
  16
  17   EXPECT_TRUE(t.GetNext());
  18   EXPECT_EQ(string("this"), t.token());
  19
  20   EXPECT_TRUE(t.GetNext());
  21   EXPECT_EQ(string("is"), t.token());
  22
  23   EXPECT_TRUE(t.GetNext());
  24   EXPECT_EQ(string("a"), t.token());
  25
  26   EXPECT_TRUE(t.GetNext());
  27   EXPECT_EQ(string("test"), t.token());
  28
  29   EXPECT_FALSE(t.GetNext());
  30 }
  31
  32 TEST(StringTokenizerTest, Reset) {
  33   string input = "this is a test";
  34   StringTokenizer t(input, " ");
  35
  36   for (int i = 0; i < 2; ++i) {
  37     EXPECT_TRUE(t.GetNext());
  38     EXPECT_EQ(string("this"), t.token());
  39
  40     EXPECT_TRUE(t.GetNext());
  41     EXPECT_EQ(string("is"), t.token());
  42
  43     EXPECT_TRUE(t.GetNext());
  44     EXPECT_EQ(string("a"), t.token());
  45
  46     EXPECT_TRUE(t.GetNext());
  47     EXPECT_EQ(string("test"), t.token());
  48
  49     EXPECT_FALSE(t.GetNext());
  50     t.Reset();
  51   }
  52 }
  53
  54 TEST(StringTokenizerTest, RetDelims) {
  55   string input = "this is a test";
  56   StringTokenizer t(input, " ");
  57   t.set_options(StringTokenizer::RETURN_DELIMS);
  58
  59   EXPECT_TRUE(t.GetNext());
  60   EXPECT_EQ(string("this"), t.token());
  61
  62   EXPECT_TRUE(t.GetNext());
  63   EXPECT_EQ(string(" "), t.token());
  64
  65   EXPECT_TRUE(t.GetNext());
  66   EXPECT_EQ(string("is"), t.token());
  67
  68   EXPECT_TRUE(t.GetNext());
  69   EXPECT_EQ(string(" "), t.token());
  70
  71   EXPECT_TRUE(t.GetNext());
  72   EXPECT_EQ(string("a"), t.token());
  73
  74   EXPECT_TRUE(t.GetNext());
  75   EXPECT_EQ(string(" "), t.token());
  76
  77   EXPECT_TRUE(t.GetNext());
  78   EXPECT_EQ(string("test"), t.token());
  79
  80   EXPECT_FALSE(t.GetNext());
  81 }
  82
  83 TEST(StringTokenizerTest, ManyDelims) {
  84   string input = "this: is, a-test";
  85   StringTokenizer t(input, ": ,-");
  86
  87   EXPECT_TRUE(t.GetNext());
  88   EXPECT_EQ(string("this"), t.token());
  89
  90   EXPECT_TRUE(t.GetNext());
  91   EXPECT_EQ(string("is"), t.token());
  92
  93   EXPECT_TRUE(t.GetNext());
  94   EXPECT_EQ(string("a"), t.token());
  95
  96   EXPECT_TRUE(t.GetNext());
  97   EXPECT_EQ(string("test"), t.token());
  98
  99   EXPECT_FALSE(t.GetNext());
 100 }
 101
 102 TEST(StringTokenizerTest, ParseHeader) {
 103   string input = "Content-Type: text/html ; charset=UTF-8";
 104   StringTokenizer t(input, ": ;=");
 105   t.set_options(StringTokenizer::RETURN_DELIMS);
 106
 107   EXPECT_TRUE(t.GetNext());
 108   EXPECT_FALSE(t.token_is_delim());
 109   EXPECT_EQ(string("Content-Type"), t.token());
 110
 111   EXPECT_TRUE(t.GetNext());
 112   EXPECT_TRUE(t.token_is_delim());
 113   EXPECT_EQ(string(":"), t.token());
 114
 115   EXPECT_TRUE(t.GetNext());
 116   EXPECT_TRUE(t.token_is_delim());
 117   EXPECT_EQ(string(" "), t.token());
 118
 119   EXPECT_TRUE(t.GetNext());
 120   EXPECT_FALSE(t.token_is_delim());
 121   EXPECT_EQ(string("text/html"), t.token());
 122
 123   EXPECT_TRUE(t.GetNext());
 124   EXPECT_TRUE(t.token_is_delim());
 125   EXPECT_EQ(string(" "), t.token());
 126
 127   EXPECT_TRUE(t.GetNext());
 128   EXPECT_TRUE(t.token_is_delim());
 129   EXPECT_EQ(string(";"), t.token());
 130
 131   EXPECT_TRUE(t.GetNext());
 132   EXPECT_TRUE(t.token_is_delim());
 133   EXPECT_EQ(string(" "), t.token());
 134
 135   EXPECT_TRUE(t.GetNext());
 136   EXPECT_FALSE(t.token_is_delim());
 137   EXPECT_EQ(string("charset"), t.token());
 138
 139   EXPECT_TRUE(t.GetNext());
 140   EXPECT_TRUE(t.token_is_delim());
 141   EXPECT_EQ(string("="), t.token());
 142
 143   EXPECT_TRUE(t.GetNext());
 144   EXPECT_FALSE(t.token_is_delim());
 145   EXPECT_EQ(string("UTF-8"), t.token());
 146
 147   EXPECT_FALSE(t.GetNext());
 148   EXPECT_FALSE(t.token_is_delim());
 149 }
 150
 151 TEST(StringTokenizerTest, ParseQuotedString) {
 152   string input = "foo bar 'hello world' baz";
 153   StringTokenizer t(input, " ");
 154   t.set_quote_chars("'");
 155
 156   EXPECT_TRUE(t.GetNext());
 157   EXPECT_EQ(string("foo"), t.token());
 158
 159   EXPECT_TRUE(t.GetNext());
 160   EXPECT_EQ(string("bar"), t.token());
 161
 162   EXPECT_TRUE(t.GetNext());
 163   EXPECT_EQ(string("'hello world'"), t.token());
 164
 165   EXPECT_TRUE(t.GetNext());
 166   EXPECT_EQ(string("baz"), t.token());
 167
 168   EXPECT_FALSE(t.GetNext());
 169 }
 170
 171 TEST(StringTokenizerTest, ParseQuotedString_Malformed) {
 172   string input = "bar 'hello wo";
 173   StringTokenizer t(input, " ");
 174   t.set_quote_chars("'");
 175
 176   EXPECT_TRUE(t.GetNext());
 177   EXPECT_EQ(string("bar"), t.token());
 178
 179   EXPECT_TRUE(t.GetNext());
 180   EXPECT_EQ(string("'hello wo"), t.token());
 181
 182   EXPECT_FALSE(t.GetNext());
 183 }
 184
 185 TEST(StringTokenizerTest, ParseQuotedString_Multiple) {
 186   string input = "bar 'hel\"lo\" wo' baz\"";
 187   StringTokenizer t(input, " ");
 188   t.set_quote_chars("'\"");
 189
 190   EXPECT_TRUE(t.GetNext());
 191   EXPECT_EQ(string("bar"), t.token());
 192
 193   EXPECT_TRUE(t.GetNext());
 194   EXPECT_EQ(string("'hel\"lo\" wo'"), t.token());
 195
 196   EXPECT_TRUE(t.GetNext());
 197   EXPECT_EQ(string("baz\""), t.token());
 198
 199   EXPECT_FALSE(t.GetNext());
 200 }
 201
 202 TEST(StringTokenizerTest, ParseQuotedString_EscapedQuotes) {
 203   string input = "foo 'don\\'t do that'";
 204   StringTokenizer t(input, " ");
 205   t.set_quote_chars("'");
 206
 207   EXPECT_TRUE(t.GetNext());
 208   EXPECT_EQ(string("foo"), t.token());
 209
 210   EXPECT_TRUE(t.GetNext());
 211   EXPECT_EQ(string("'don\\'t do that'"), t.token());
 212
 213   EXPECT_FALSE(t.GetNext());
 214 }
 215
 216 TEST(StringTokenizerTest, ParseQuotedString_EscapedQuotes2) {
 217   string input = "foo='a, b', bar";
 218   StringTokenizer t(input, ", ");
 219   t.set_quote_chars("'");
 220
 221   EXPECT_TRUE(t.GetNext());
 222   EXPECT_EQ(string("foo='a, b'"), t.token());
 223
 224   EXPECT_TRUE(t.GetNext());
 225   EXPECT_EQ(string("bar"), t.token());
 226
 227   EXPECT_FALSE(t.GetNext());
 228 }
 229
 230 }  // namespace