base/strings/string_tokenizer_unittest.cc

   1 // Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/strings/string_tokenizer.h"
   6
   7 #include "testing/gtest/include/gtest/gtest.h"
   8
   9 using std::string;
  10
  11 namespace base {
  12
  13 namespace {
  14
  15 TEST(StringTokenizerTest, Simple) {
  16   string input = "this is a test";
  17   StringTokenizer t(input, " ");
  18
  19   EXPECT_TRUE(t.GetNext());
  20   EXPECT_EQ(string("this"), t.token());
  21
  22   EXPECT_TRUE(t.GetNext());
  23   EXPECT_EQ(string("is"), t.token());
  24
  25   EXPECT_TRUE(t.GetNext());
  26   EXPECT_EQ(string("a"), t.token());
  27
  28   EXPECT_TRUE(t.GetNext());
  29   EXPECT_EQ(string("test"), t.token());
  30
  31   EXPECT_FALSE(t.GetNext());
  32 }
  33
  34 TEST(StringTokenizerTest, Reset) {
  35   string input = "this is a test";
  36   StringTokenizer t(input, " ");
  37
  38   for (int i = 0; i < 2; ++i) {
  39     EXPECT_TRUE(t.GetNext());
  40     EXPECT_EQ(string("this"), t.token());
  41
  42     EXPECT_TRUE(t.GetNext());
  43     EXPECT_EQ(string("is"), t.token());
  44
  45     EXPECT_TRUE(t.GetNext());
  46     EXPECT_EQ(string("a"), t.token());
  47
  48     EXPECT_TRUE(t.GetNext());
  49     EXPECT_EQ(string("test"), t.token());
  50
  51     EXPECT_FALSE(t.GetNext());
  52     t.Reset();
  53   }
  54 }
  55
  56 TEST(StringTokenizerTest, RetDelims) {
  57   string input = "this is a test";
  58   StringTokenizer t(input, " ");
  59   t.set_options(StringTokenizer::RETURN_DELIMS);
  60
  61   EXPECT_TRUE(t.GetNext());
  62   EXPECT_EQ(string("this"), t.token());
  63
  64   EXPECT_TRUE(t.GetNext());
  65   EXPECT_EQ(string(" "), t.token());
  66
  67   EXPECT_TRUE(t.GetNext());
  68   EXPECT_EQ(string("is"), t.token());
  69
  70   EXPECT_TRUE(t.GetNext());
  71   EXPECT_EQ(string(" "), t.token());
  72
  73   EXPECT_TRUE(t.GetNext());
  74   EXPECT_EQ(string("a"), t.token());
  75
  76   EXPECT_TRUE(t.GetNext());
  77   EXPECT_EQ(string(" "), t.token());
  78
  79   EXPECT_TRUE(t.GetNext());
  80   EXPECT_EQ(string("test"), t.token());
  81
  82   EXPECT_FALSE(t.GetNext());
  83 }
  84
  85 TEST(StringTokenizerTest, ManyDelims) {
  86   string input = "this: is, a-test";
  87   StringTokenizer t(input, ": ,-");
  88
  89   EXPECT_TRUE(t.GetNext());
  90   EXPECT_EQ(string("this"), t.token());
  91
  92   EXPECT_TRUE(t.GetNext());
  93   EXPECT_EQ(string("is"), t.token());
  94
  95   EXPECT_TRUE(t.GetNext());
  96   EXPECT_EQ(string("a"), t.token());
  97
  98   EXPECT_TRUE(t.GetNext());
  99   EXPECT_EQ(string("test"), t.token());
 100
 101   EXPECT_FALSE(t.GetNext());
 102 }
 103
 104 TEST(StringTokenizerTest, ParseHeader) {
 105   string input = "Content-Type: text/html ; charset=UTF-8";
 106   StringTokenizer t(input, ": ;=");
 107   t.set_options(StringTokenizer::RETURN_DELIMS);
 108
 109   EXPECT_TRUE(t.GetNext());
 110   EXPECT_FALSE(t.token_is_delim());
 111   EXPECT_EQ(string("Content-Type"), t.token());
 112
 113   EXPECT_TRUE(t.GetNext());
 114   EXPECT_TRUE(t.token_is_delim());
 115   EXPECT_EQ(string(":"), t.token());
 116
 117   EXPECT_TRUE(t.GetNext());
 118   EXPECT_TRUE(t.token_is_delim());
 119   EXPECT_EQ(string(" "), t.token());
 120
 121   EXPECT_TRUE(t.GetNext());
 122   EXPECT_FALSE(t.token_is_delim());
 123   EXPECT_EQ(string("text/html"), t.token());
 124
 125   EXPECT_TRUE(t.GetNext());
 126   EXPECT_TRUE(t.token_is_delim());
 127   EXPECT_EQ(string(" "), t.token());
 128
 129   EXPECT_TRUE(t.GetNext());
 130   EXPECT_TRUE(t.token_is_delim());
 131   EXPECT_EQ(string(";"), t.token());
 132
 133   EXPECT_TRUE(t.GetNext());
 134   EXPECT_TRUE(t.token_is_delim());
 135   EXPECT_EQ(string(" "), t.token());
 136
 137   EXPECT_TRUE(t.GetNext());
 138   EXPECT_FALSE(t.token_is_delim());
 139   EXPECT_EQ(string("charset"), t.token());
 140
 141   EXPECT_TRUE(t.GetNext());
 142   EXPECT_TRUE(t.token_is_delim());
 143   EXPECT_EQ(string("="), t.token());
 144
 145   EXPECT_TRUE(t.GetNext());
 146   EXPECT_FALSE(t.token_is_delim());
 147   EXPECT_EQ(string("UTF-8"), t.token());
 148
 149   EXPECT_FALSE(t.GetNext());
 150   EXPECT_FALSE(t.token_is_delim());
 151 }
 152
 153 TEST(StringTokenizerTest, ParseQuotedString) {
 154   string input = "foo bar 'hello world' baz";
 155   StringTokenizer t(input, " ");
 156   t.set_quote_chars("'");
 157
 158   EXPECT_TRUE(t.GetNext());
 159   EXPECT_EQ(string("foo"), t.token());
 160
 161   EXPECT_TRUE(t.GetNext());
 162   EXPECT_EQ(string("bar"), t.token());
 163
 164   EXPECT_TRUE(t.GetNext());
 165   EXPECT_EQ(string("'hello world'"), t.token());
 166
 167   EXPECT_TRUE(t.GetNext());
 168   EXPECT_EQ(string("baz"), t.token());
 169
 170   EXPECT_FALSE(t.GetNext());
 171 }
 172
 173 TEST(StringTokenizerTest, ParseQuotedString_Malformed) {
 174   string input = "bar 'hello wo";
 175   StringTokenizer t(input, " ");
 176   t.set_quote_chars("'");
 177
 178   EXPECT_TRUE(t.GetNext());
 179   EXPECT_EQ(string("bar"), t.token());
 180
 181   EXPECT_TRUE(t.GetNext());
 182   EXPECT_EQ(string("'hello wo"), t.token());
 183
 184   EXPECT_FALSE(t.GetNext());
 185 }
 186
 187 TEST(StringTokenizerTest, ParseQuotedString_Multiple) {
 188   string input = "bar 'hel\"lo\" wo' baz\"";
 189   StringTokenizer t(input, " ");
 190   t.set_quote_chars("'\"");
 191
 192   EXPECT_TRUE(t.GetNext());
 193   EXPECT_EQ(string("bar"), t.token());
 194
 195   EXPECT_TRUE(t.GetNext());
 196   EXPECT_EQ(string("'hel\"lo\" wo'"), t.token());
 197
 198   EXPECT_TRUE(t.GetNext());
 199   EXPECT_EQ(string("baz\""), t.token());
 200
 201   EXPECT_FALSE(t.GetNext());
 202 }
 203
 204 TEST(StringTokenizerTest, ParseQuotedString_EscapedQuotes) {
 205   string input = "foo 'don\\'t do that'";
 206   StringTokenizer t(input, " ");
 207   t.set_quote_chars("'");
 208
 209   EXPECT_TRUE(t.GetNext());
 210   EXPECT_EQ(string("foo"), t.token());
 211
 212   EXPECT_TRUE(t.GetNext());
 213   EXPECT_EQ(string("'don\\'t do that'"), t.token());
 214
 215   EXPECT_FALSE(t.GetNext());
 216 }
 217
 218 TEST(StringTokenizerTest, ParseQuotedString_EscapedQuotes2) {
 219   string input = "foo='a, b', bar";
 220   StringTokenizer t(input, ", ");
 221   t.set_quote_chars("'");
 222
 223   EXPECT_TRUE(t.GetNext());
 224   EXPECT_EQ(string("foo='a, b'"), t.token());
 225
 226   EXPECT_TRUE(t.GetNext());
 227   EXPECT_EQ(string("bar"), t.token());
 228
 229   EXPECT_FALSE(t.GetNext());
 230 }
 231
 232 }  // namespace
 233
 234 }  // namespace base