base/i18n/streaming_utf8_validator_unittest.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/i18n/streaming_utf8_validator.h"
   6
   7 #include <stdio.h>
   8 #include <string.h>
   9
  10 #include <string>
  11
  12 #include "base/strings/string_piece.h"
  13 #include "testing/gtest/include/gtest/gtest.h"
  14
  15 // Define BASE_I18N_UTF8_VALIDATOR_THOROUGH_TEST to verify that this class
  16 // accepts exactly the same set of 4-byte strings as ICU-based validation. This
  17 // tests every possible 4-byte string, so it is too slow to run routinely on
  18 // low-powered machines.
  19 //
  20 // #define BASE_I18N_UTF8_VALIDATOR_THOROUGH_TEST
  21
  22 #ifdef BASE_I18N_UTF8_VALIDATOR_THOROUGH_TEST
  23
  24 #include "base/basictypes.h"
  25 #include "base/bind.h"
  26 #include "base/location.h"
  27 #include "base/logging.h"
  28 #include "base/memory/ref_counted.h"
  29 #include "base/strings/string_util.h"
  30 #include "base/strings/stringprintf.h"
  31 #include "base/strings/utf_string_conversion_utils.h"
  32 #include "base/synchronization/condition_variable.h"
  33 #include "base/synchronization/lock.h"
  34 #include "base/threading/sequenced_worker_pool.h"
  35 #include "third_party/icu/source/common/unicode/utf8.h"
  36
  37 #endif  // BASE_I18N_UTF8_VALIDATOR_THOROUGH_TEST
  38
  39 namespace base {
  40 namespace {
  41
  42 // Avoid having to qualify the enum values in the tests.
  43 const StreamingUtf8Validator::State VALID_ENDPOINT =
  44     StreamingUtf8Validator::VALID_ENDPOINT;
  45 const StreamingUtf8Validator::State VALID_MIDPOINT =
  46     StreamingUtf8Validator::VALID_MIDPOINT;
  47 const StreamingUtf8Validator::State INVALID = StreamingUtf8Validator::INVALID;
  48
  49 #ifdef BASE_I18N_UTF8_VALIDATOR_THOROUGH_TEST
  50
  51 const uint32 kThoroughTestChunkSize = 1 << 24;
  52
  53 class StreamingUtf8ValidatorThoroughTest : public ::testing::Test {
  54  protected:
  55   StreamingUtf8ValidatorThoroughTest()
  56       : all_done_(&lock_), tasks_dispatched_(0), tasks_finished_(0) {}
  57
  58   // This uses the same logic as base::IsStringUTF8 except it considers
  59   // non-characters valid (and doesn't require a string as input).
  60   static bool IsStringUtf8(const char* src, int32 src_len) {
  61     int32 char_index = 0;
  62
  63     while (char_index < src_len) {
  64       int32 code_point;
  65       U8_NEXT(src, char_index, src_len, code_point);
  66       if (!base::IsValidCodepoint(code_point))
  67         return false;
  68     }
  69     return true;
  70   }
  71
  72   // Converts the passed-in integer to a 4 byte string and then
  73   // verifies that IsStringUtf8 and StreamingUtf8Validator agree on
  74   // whether it is valid UTF-8 or not.
  75   void TestNumber(uint32 n) const {
  76     char test[sizeof n];
  77     memcpy(test, &n, sizeof n);
  78     StreamingUtf8Validator validator;
  79     EXPECT_EQ(IsStringUtf8(test, sizeof n),
  80               validator.AddBytes(test, sizeof n) == VALID_ENDPOINT)
  81         << "Difference of opinion for \""
  82         << base::StringPrintf("\\x%02X\\x%02X\\x%02X\\x%02X",
  83                               test[0] & 0xFF,
  84                               test[1] & 0xFF,
  85                               test[2] & 0xFF,
  86                               test[3] & 0xFF) << "\"";
  87   }
  88
  89  public:
  90   // Tests the 4-byte sequences corresponding to the |size| integers
  91   // starting at |begin|. This is intended to be run from a worker
  92   // pool. Signals |all_done_| at the end if it thinks all tasks are
  93   // finished.
  94   void TestRange(uint32 begin, uint32 size) {
  95     for (uint32 i = 0; i < size; ++i) {
  96       TestNumber(begin + i);
  97     }
  98     base::AutoLock al(lock_);
  99     ++tasks_finished_;
 100     LOG(INFO) << tasks_finished_ << " / " << tasks_dispatched_
 101               << " tasks done\n";
 102     if (tasks_finished_ >= tasks_dispatched_) {
 103       all_done_.Signal();
 104     }
 105   }
 106
 107  protected:
 108   base::Lock lock_;
 109   base::ConditionVariable all_done_;
 110   int tasks_dispatched_;
 111   int tasks_finished_;
 112 };
 113
 114 TEST_F(StreamingUtf8ValidatorThoroughTest, TestEverything) {
 115   scoped_refptr<base::SequencedWorkerPool> pool =
 116       new base::SequencedWorkerPool(32, "TestEverything");
 117   base::AutoLock al(lock_);
 118   uint32 begin = 0;
 119   do {
 120     pool->PostWorkerTask(
 121         FROM_HERE,
 122         base::Bind(&StreamingUtf8ValidatorThoroughTest::TestRange,
 123                    base::Unretained(this),
 124                    begin,
 125                    kThoroughTestChunkSize));
 126     ++tasks_dispatched_;
 127     begin += kThoroughTestChunkSize;
 128   } while (begin != 0);
 129   while (tasks_finished_ < tasks_dispatched_)
 130     all_done_.Wait();
 131 }
 132
 133 #endif  // BASE_I18N_UTF8_VALIDATOR_THOROUGH_TEST
 134
 135 // These valid and invalid UTF-8 sequences are based on the tests from
 136 // base/strings/string_util_unittest.cc
 137
 138 // All of the strings in |valid| must represent a single codepoint, because
 139 // partial sequences are constructed by taking non-empty prefixes of these
 140 // strings.
 141 const char* const valid[] = {"\r",           "\n",           "a",
 142                              "\xc2\x81",     "\xe1\x80\xbf", "\xf1\x80\xa0\xbf",
 143                              "\xef\xbb\xbf",  // UTF-8 BOM
 144 };
 145
 146 const char* const* const valid_end = valid + arraysize(valid);
 147
 148 const char* const invalid[] = {
 149     // always invalid bytes
 150     "\xc0", "\xc1",
 151     "\xf5", "\xf6", "\xf7",
 152     "\xf8", "\xf9", "\xfa", "\xfb", "\xfc", "\xfd", "\xfe", "\xff",
 153     // surrogate code points
 154     "\xed\xa0\x80", "\xed\x0a\x8f", "\xed\xbf\xbf",
 155     //
 156     // overlong sequences
 157     "\xc0\x80"               // U+0000
 158     "\xc1\x80",              // "A"
 159     "\xc1\x81",              // "B"
 160     "\xe0\x80\x80",          // U+0000
 161     "\xe0\x82\x80",          // U+0080
 162     "\xe0\x9f\xbf",          // U+07ff
 163     "\xf0\x80\x80\x8D",      // U+000D
 164     "\xf0\x80\x82\x91",      // U+0091
 165     "\xf0\x80\xa0\x80",      // U+0800
 166     "\xf0\x8f\xbb\xbf",      // U+FEFF (BOM)
 167     "\xf8\x80\x80\x80\xbf",  // U+003F
 168     "\xfc\x80\x80\x80\xa0\xa5",
 169     //
 170     // Beyond U+10FFFF
 171     "\xf4\x90\x80\x80",          // U+110000
 172     "\xf8\xa0\xbf\x80\xbf",      // 5 bytes
 173     "\xfc\x9c\xbf\x80\xbf\x80",  // 6 bytes
 174     //
 175     // BOMs in UTF-16(BE|LE)
 176     "\xfe\xff", "\xff\xfe",
 177 };
 178
 179 const char* const* const invalid_end = invalid + arraysize(invalid);
 180
 181 // A ForwardIterator which returns all the non-empty prefixes of the elements of
 182 // "valid".
 183 class PartialIterator {
 184  public:
 185   // The constructor returns the first iterator, ie. it is equivalent to
 186   // begin().
 187   PartialIterator() : index_(0), prefix_length_(0) { Advance(); }
 188   // The trivial destructor left intentionally undefined.
 189   // This is a value type; the default copy constructor and assignment operator
 190   // generated by the compiler are used.
 191
 192   static PartialIterator end() { return PartialIterator(arraysize(valid), 1); }
 193
 194   PartialIterator& operator++() {
 195     Advance();
 196     return *this;
 197   }
 198
 199   base::StringPiece operator*() const {
 200     return base::StringPiece(valid[index_], prefix_length_);
 201   }
 202
 203   bool operator==(const PartialIterator& rhs) const {
 204     return index_ == rhs.index_ && prefix_length_ == rhs.prefix_length_;
 205   }
 206
 207   bool operator!=(const PartialIterator& rhs) const { return !(rhs == *this); }
 208
 209  private:
 210   // This constructor is used by the end() method.
 211   PartialIterator(size_t index, size_t prefix_length)
 212       : index_(index), prefix_length_(prefix_length) {}
 213
 214   void Advance() {
 215     if (index_ < arraysize(valid) && prefix_length_ < strlen(valid[index_]))
 216       ++prefix_length_;
 217     while (index_ < arraysize(valid) &&
 218            prefix_length_ == strlen(valid[index_])) {
 219       ++index_;
 220       prefix_length_ = 1;
 221     }
 222   }
 223
 224   // The UTF-8 sequence, as an offset into the |valid| array.
 225   size_t index_;
 226   size_t prefix_length_;
 227 };
 228
 229 // A test fixture for tests which test one UTF-8 sequence (or invalid
 230 // byte sequence) at a time.
 231 class StreamingUtf8ValidatorSingleSequenceTest : public ::testing::Test {
 232  protected:
 233   // Iterator must be convertible when de-referenced to StringPiece.
 234   template <typename Iterator>
 235   void CheckRange(Iterator begin,
 236                   Iterator end,
 237                   StreamingUtf8Validator::State expected) {
 238     for (Iterator it = begin; it != end; ++it) {
 239       StreamingUtf8Validator validator;
 240       base::StringPiece sequence = *it;
 241       EXPECT_EQ(expected,
 242                 validator.AddBytes(sequence.data(), sequence.size()))
 243           << "Failed for \"" << sequence << "\"";
 244     }
 245   }
 246
 247   // Adding input a byte at a time should make absolutely no difference.
 248   template <typename Iterator>
 249   void CheckRangeByteAtATime(Iterator begin,
 250                              Iterator end,
 251                              StreamingUtf8Validator::State expected) {
 252     for (Iterator it = begin; it != end; ++it) {
 253       StreamingUtf8Validator validator;
 254       base::StringPiece sequence = *it;
 255       StreamingUtf8Validator::State state = VALID_ENDPOINT;
 256       for (base::StringPiece::const_iterator cit = sequence.begin();
 257            cit != sequence.end();
 258            ++cit) {
 259         state = validator.AddBytes(&*cit, 1);
 260       }
 261       EXPECT_EQ(expected, state) << "Failed for \"" << sequence << "\"";
 262     }
 263   }
 264 };
 265
 266 // A test fixture for tests which test the concatenation of byte sequences.
 267 class StreamingUtf8ValidatorDoubleSequenceTest : public ::testing::Test {
 268  protected:
 269   // Check every possible concatenation of byte sequences from two
 270   // ranges, and verify that the combination matches the expected
 271   // state.
 272   template <typename Iterator1, typename Iterator2>
 273   void CheckCombinations(Iterator1 begin1,
 274                          Iterator1 end1,
 275                          Iterator2 begin2,
 276                          Iterator2 end2,
 277                          StreamingUtf8Validator::State expected) {
 278     StreamingUtf8Validator validator;
 279     for (Iterator1 it1 = begin1; it1 != end1; ++it1) {
 280       base::StringPiece c1 = *it1;
 281       for (Iterator2 it2 = begin2; it2 != end2; ++it2) {
 282         base::StringPiece c2 = *it2;
 283         validator.AddBytes(c1.data(), c1.size());
 284         EXPECT_EQ(expected, validator.AddBytes(c2.data(), c2.size()))
 285             << "Failed for \"" << c1 << c2 << "\"";
 286         validator.Reset();
 287       }
 288     }
 289   }
 290 };
 291
 292 TEST(StreamingUtf8ValidatorTest, NothingIsValid) {
 293   static const char kNothing[] = "";
 294   EXPECT_EQ(VALID_ENDPOINT, StreamingUtf8Validator().AddBytes(kNothing, 0));
 295 }
 296
 297 // Because the members of the |valid| array need to be non-zero length
 298 // sequences and are measured with strlen(), |valid| cannot be used it
 299 // to test the NUL character '\0', so the NUL character gets its own
 300 // test.
 301 TEST(StreamingUtf8ValidatorTest, NulIsValid) {
 302   static const char kNul[] = "\x00";
 303   EXPECT_EQ(VALID_ENDPOINT, StreamingUtf8Validator().AddBytes(kNul, 1));
 304 }
 305
 306 // Just a basic sanity test before we start getting fancy.
 307 TEST(StreamingUtf8ValidatorTest, HelloWorld) {
 308   static const char kHelloWorld[] = "Hello, World!";
 309   EXPECT_EQ(
 310       VALID_ENDPOINT,
 311       StreamingUtf8Validator().AddBytes(kHelloWorld, strlen(kHelloWorld)));
 312 }
 313
 314 // Check that the Reset() method works.
 315 TEST(StreamingUtf8ValidatorTest, ResetWorks) {
 316   StreamingUtf8Validator validator;
 317   EXPECT_EQ(INVALID, validator.AddBytes("\xC0", 1));
 318   EXPECT_EQ(INVALID, validator.AddBytes("a", 1));
 319   validator.Reset();
 320   EXPECT_EQ(VALID_ENDPOINT, validator.AddBytes("a", 1));
 321 }
 322
 323 TEST_F(StreamingUtf8ValidatorSingleSequenceTest, Valid) {
 324   CheckRange(valid, valid_end, VALID_ENDPOINT);
 325 }
 326
 327 TEST_F(StreamingUtf8ValidatorSingleSequenceTest, Partial) {
 328   CheckRange(PartialIterator(), PartialIterator::end(), VALID_MIDPOINT);
 329 }
 330
 331 TEST_F(StreamingUtf8ValidatorSingleSequenceTest, Invalid) {
 332   CheckRange(invalid, invalid_end, INVALID);
 333 }
 334
 335 TEST_F(StreamingUtf8ValidatorSingleSequenceTest, ValidByByte) {
 336   CheckRangeByteAtATime(valid, valid_end, VALID_ENDPOINT);
 337 }
 338
 339 TEST_F(StreamingUtf8ValidatorSingleSequenceTest, PartialByByte) {
 340   CheckRangeByteAtATime(
 341       PartialIterator(), PartialIterator::end(), VALID_MIDPOINT);
 342 }
 343
 344 TEST_F(StreamingUtf8ValidatorSingleSequenceTest, InvalidByByte) {
 345   CheckRangeByteAtATime(invalid, invalid_end, INVALID);
 346 }
 347
 348 TEST_F(StreamingUtf8ValidatorDoubleSequenceTest, ValidPlusValidIsValid) {
 349   CheckCombinations(valid, valid_end, valid, valid_end, VALID_ENDPOINT);
 350 }
 351
 352 TEST_F(StreamingUtf8ValidatorDoubleSequenceTest, ValidPlusPartialIsPartial) {
 353   CheckCombinations(valid,
 354                     valid_end,
 355                     PartialIterator(),
 356                     PartialIterator::end(),
 357                     VALID_MIDPOINT);
 358 }
 359
 360 TEST_F(StreamingUtf8ValidatorDoubleSequenceTest, PartialPlusValidIsInvalid) {
 361   CheckCombinations(
 362       PartialIterator(), PartialIterator::end(), valid, valid_end, INVALID);
 363 }
 364
 365 TEST_F(StreamingUtf8ValidatorDoubleSequenceTest, PartialPlusPartialIsInvalid) {
 366   CheckCombinations(PartialIterator(),
 367                     PartialIterator::end(),
 368                     PartialIterator(),
 369                     PartialIterator::end(),
 370                     INVALID);
 371 }
 372
 373 TEST_F(StreamingUtf8ValidatorDoubleSequenceTest, ValidPlusInvalidIsInvalid) {
 374   CheckCombinations(valid, valid_end, invalid, invalid_end, INVALID);
 375 }
 376
 377 TEST_F(StreamingUtf8ValidatorDoubleSequenceTest, InvalidPlusValidIsInvalid) {
 378   CheckCombinations(invalid, invalid_end, valid, valid_end, INVALID);
 379 }
 380
 381 TEST_F(StreamingUtf8ValidatorDoubleSequenceTest, InvalidPlusInvalidIsInvalid) {
 382   CheckCombinations(invalid, invalid_end, invalid, invalid_end, INVALID);
 383 }
 384
 385 TEST_F(StreamingUtf8ValidatorDoubleSequenceTest, InvalidPlusPartialIsInvalid) {
 386   CheckCombinations(
 387       invalid, invalid_end, PartialIterator(), PartialIterator::end(), INVALID);
 388 }
 389
 390 TEST_F(StreamingUtf8ValidatorDoubleSequenceTest, PartialPlusInvalidIsInvalid) {
 391   CheckCombinations(
 392       PartialIterator(), PartialIterator::end(), invalid, invalid_end, INVALID);
 393 }
 394
 395 TEST(StreamingUtf8ValidatorValidateTest, EmptyIsValid) {
 396   EXPECT_TRUE(StreamingUtf8Validator::Validate(std::string()));
 397 }
 398
 399 TEST(StreamingUtf8ValidatorValidateTest, SimpleValidCase) {
 400   EXPECT_TRUE(StreamingUtf8Validator::Validate("\xc2\x81"));
 401 }
 402
 403 TEST(StreamingUtf8ValidatorValidateTest, SimpleInvalidCase) {
 404   EXPECT_FALSE(StreamingUtf8Validator::Validate("\xc0\x80"));
 405 }
 406
 407 TEST(StreamingUtf8ValidatorValidateTest, TruncatedIsInvalid) {
 408   EXPECT_FALSE(StreamingUtf8Validator::Validate("\xc2"));
 409 }
 410
 411 }  // namespace
 412 }  // namespace base