base/i18n/icu_string_conversions_unittest.cc

   1 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <math.h>
   6 #include <stdarg.h>
   7
   8 #include <limits>
   9 #include <sstream>
  10
  11 #include "base/basictypes.h"
  12 #include "base/format_macros.h"
  13 #include "base/i18n/icu_string_conversions.h"
  14 #include "base/logging.h"
  15 #include "base/strings/string_piece.h"
  16 #include "base/strings/stringprintf.h"
  17 #include "base/strings/utf_string_conversions.h"
  18 #include "testing/gtest/include/gtest/gtest.h"
  19
  20 namespace base {
  21
  22 namespace {
  23
  24 // Given a null-terminated string of wchar_t with each wchar_t representing
  25 // a UTF-16 code unit, returns a string16 made up of wchar_t's in the input.
  26 // Each wchar_t should be <= 0xFFFF and a non-BMP character (> U+FFFF)
  27 // should be represented as a surrogate pair (two UTF-16 units)
  28 // *even* where wchar_t is 32-bit (Linux and Mac).
  29 //
  30 // This is to help write tests for functions with string16 params until
  31 // the C++ 0x UTF-16 literal is well-supported by compilers.
  32 string16 BuildString16(const wchar_t* s) {
  33 #if defined(WCHAR_T_IS_UTF16)
  34   return string16(s);
  35 #elif defined(WCHAR_T_IS_UTF32)
  36   string16 u16;
  37   while (*s != 0) {
  38     DCHECK_LE(static_cast<unsigned int>(*s), 0xFFFFu);
  39     u16.push_back(*s++);
  40   }
  41   return u16;
  42 #endif
  43 }
  44
  45 const wchar_t* const kConvertRoundtripCases[] = {
  46   L"Google Video",
  47   // "网页 图片 资讯更多 »"
  48   L"\x7f51\x9875\x0020\x56fe\x7247\x0020\x8d44\x8baf\x66f4\x591a\x0020\x00bb",
  49   //  "Παγκόσμιος Ιστός"
  50   L"\x03a0\x03b1\x03b3\x03ba\x03cc\x03c3\x03bc\x03b9"
  51   L"\x03bf\x03c2\x0020\x0399\x03c3\x03c4\x03cc\x03c2",
  52   // "Поиск страниц на русском"
  53   L"\x041f\x043e\x0438\x0441\x043a\x0020\x0441\x0442"
  54   L"\x0440\x0430\x043d\x0438\x0446\x0020\x043d\x0430"
  55   L"\x0020\x0440\x0443\x0441\x0441\x043a\x043e\x043c",
  56   // "전체서비스"
  57   L"\xc804\xccb4\xc11c\xbe44\xc2a4",
  58
  59   // Test characters that take more than 16 bits. This will depend on whether
  60   // wchar_t is 16 or 32 bits.
  61 #if defined(WCHAR_T_IS_UTF16)
  62   L"\xd800\xdf00",
  63   // ?????  (Mathematical Alphanumeric Symbols (U+011d40 - U+011d44 : A,B,C,D,E)
  64   L"\xd807\xdd40\xd807\xdd41\xd807\xdd42\xd807\xdd43\xd807\xdd44",
  65 #elif defined(WCHAR_T_IS_UTF32)
  66   L"\x10300",
  67   // ?????  (Mathematical Alphanumeric Symbols (U+011d40 - U+011d44 : A,B,C,D,E)
  68   L"\x11d40\x11d41\x11d42\x11d43\x11d44",
  69 #endif
  70 };
  71
  72 }  // namespace
  73
  74 TEST(ICUStringConversionsTest, ConvertCodepageUTF8) {
  75   // Make sure WideToCodepage works like WideToUTF8.
  76   for (size_t i = 0; i < arraysize(kConvertRoundtripCases); ++i) {
  77     SCOPED_TRACE(base::StringPrintf("Test[%" PRIuS "]: %ls",
  78                                     i, kConvertRoundtripCases[i]));
  79
  80     std::string expected(WideToUTF8(kConvertRoundtripCases[i]));
  81     std::string utf8;
  82     EXPECT_TRUE(WideToCodepage(kConvertRoundtripCases[i], kCodepageUTF8,
  83                                OnStringConversionError::SKIP, &utf8));
  84     EXPECT_EQ(expected, utf8);
  85   }
  86 }
  87
  88 // kConverterCodepageCases is not comprehensive. There are a number of cases
  89 // to add if we really want to have a comprehensive coverage of various
  90 // codepages and their 'idiosyncrasies'. Currently, the only implementation
  91 // for CodepageTo* and *ToCodepage uses ICU, which has a very extensive
  92 // set of tests for the charset conversion. So, we can get away with a
  93 // relatively small number of cases listed below.
  94 //
  95 // Note about |u16_wide| in the following struct.
  96 // On Windows, the field is always identical to |wide|. On Mac and Linux,
  97 // it's identical as long as there's no character outside the
  98 // BMP (<= U+FFFF). When there is, it is different from |wide| and
  99 // is not a real wide string (UTF-32 string) in that each wchar_t in
 100 // the string is a UTF-16 code unit zero-extended to be 32-bit
 101 // even when the code unit belongs to a surrogate pair.
 102 // For instance, a Unicode string (U+0041 U+010000) is represented as
 103 // L"\x0041\xD800\xDC00" instead of L"\x0041\x10000".
 104 // To avoid the clutter, |u16_wide| will be set to NULL
 105 // if it's identical to |wide| on *all* platforms.
 106
 107 static const struct {
 108   const char* codepage_name;
 109   const char* encoded;
 110   OnStringConversionError::Type on_error;
 111   bool success;
 112   const wchar_t* wide;
 113   const wchar_t* u16_wide;
 114 } kConvertCodepageCases[] = {
 115   // Test a case where the input cannot be decoded, using SKIP, FAIL
 116   // and SUBSTITUTE error handling rules. "A7 41" is valid, but "A6" isn't.
 117   {"big5",
 118    "\xA7\x41\xA6",
 119    OnStringConversionError::FAIL,
 120    false,
 121    L"",
 122    NULL},
 123   {"big5",
 124    "\xA7\x41\xA6",
 125    OnStringConversionError::SKIP,
 126    true,
 127    L"\x4F60",
 128    NULL},
 129   {"big5",
 130    "\xA7\x41\xA6",
 131    OnStringConversionError::SUBSTITUTE,
 132    true,
 133    L"\x4F60\xFFFD",
 134    NULL},
 135   // Arabic (ISO-8859)
 136   {"iso-8859-6",
 137    "\xC7\xEE\xE4\xD3\xF1\xEE\xE4\xC7\xE5\xEF" " "
 138    "\xD9\xEE\xE4\xEE\xEA\xF2\xE3\xEF\xE5\xF2",
 139    OnStringConversionError::FAIL,
 140    true,
 141    L"\x0627\x064E\x0644\x0633\x0651\x064E\x0644\x0627\x0645\x064F" L" "
 142    L"\x0639\x064E\x0644\x064E\x064A\x0652\x0643\x064F\x0645\x0652",
 143    NULL},
 144   // Chinese Simplified (GB2312)
 145   {"gb2312",
 146    "\xC4\xE3\xBA\xC3",
 147    OnStringConversionError::FAIL,
 148    true,
 149    L"\x4F60\x597D",
 150    NULL},
 151   // Chinese (GB18030) : 4 byte sequences mapped to BMP characters
 152   {"gb18030",
 153    "\x81\x30\x84\x36\xA1\xA7",
 154    OnStringConversionError::FAIL,
 155    true,
 156    L"\x00A5\x00A8",
 157    NULL},
 158   // Chinese (GB18030) : A 4 byte sequence mapped to plane 2 (U+20000)
 159   {"gb18030",
 160    "\x95\x32\x82\x36\xD2\xBB",
 161    OnStringConversionError::FAIL,
 162    true,
 163 #if defined(WCHAR_T_IS_UTF16)
 164    L"\xD840\xDC00\x4E00",
 165 #elif defined(WCHAR_T_IS_UTF32)
 166    L"\x20000\x4E00",
 167 #endif
 168    L"\xD840\xDC00\x4E00"},
 169   {"big5",
 170    "\xA7\x41\xA6\x6E",
 171    OnStringConversionError::FAIL,
 172    true,
 173    L"\x4F60\x597D",
 174    NULL},
 175   // Greek (ISO-8859)
 176   {"iso-8859-7",
 177    "\xE3\xE5\xE9\xDC" " " "\xF3\xEF\xF5",
 178    OnStringConversionError::FAIL,
 179    true,
 180    L"\x03B3\x03B5\x03B9\x03AC" L" " L"\x03C3\x03BF\x03C5",
 181    NULL},
 182   // Hebrew (Windows)
 183   {"windows-1255",
 184    "\xF9\xD1\xC8\xEC\xE5\xC9\xED",
 185    OnStringConversionError::FAIL,
 186    true,
 187    L"\x05E9\x05C1\x05B8\x05DC\x05D5\x05B9\x05DD",
 188    NULL},
 189   // Hindi Devanagari (ISCII)
 190   {"iscii-dev",
 191    "\xEF\x42" "\xC6\xCC\xD7\xE8\xB3\xDA\xCF",
 192    OnStringConversionError::FAIL,
 193    true,
 194    L"\x0928\x092E\x0938\x094D\x0915\x093E\x0930",
 195    NULL},
 196   // Korean (EUC)
 197   {"euc-kr",
 198    "\xBE\xC8\xB3\xE7\xC7\xCF\xBC\xBC\xBF\xE4",
 199    OnStringConversionError::FAIL,
 200    true,
 201    L"\xC548\xB155\xD558\xC138\xC694",
 202    NULL},
 203   // Japanese (EUC)
 204   {"euc-jp",
 205    "\xA4\xB3\xA4\xF3\xA4\xCB\xA4\xC1\xA4\xCF\xB0\xEC\x8F\xB0\xA1\x8E\xA6",
 206    OnStringConversionError::FAIL,
 207    true,
 208    L"\x3053\x3093\x306B\x3061\x306F\x4E00\x4E02\xFF66",
 209    NULL},
 210   // Japanese (ISO-2022)
 211   {"iso-2022-jp",
 212    "\x1B$B" "\x24\x33\x24\x73\x24\x4B\x24\x41\x24\x4F\x30\x6C" "\x1B(B"
 213    "ab" "\x1B(J" "\x5C\x7E#$" "\x1B(B",
 214    OnStringConversionError::FAIL,
 215    true,
 216    L"\x3053\x3093\x306B\x3061\x306F\x4E00" L"ab\x00A5\x203E#$",
 217    NULL},
 218   // Japanese (Shift-JIS)
 219   {"sjis",
 220    "\x82\xB1\x82\xF1\x82\xC9\x82\xBF\x82\xCD\x88\xEA\xA6",
 221    OnStringConversionError::FAIL,
 222    true,
 223    L"\x3053\x3093\x306B\x3061\x306F\x4E00\xFF66",
 224    NULL},
 225   // Russian (KOI8)
 226   {"koi8-r",
 227    "\xDA\xC4\xD2\xC1\xD7\xD3\xD4\xD7\xD5\xCA\xD4\xC5",
 228    OnStringConversionError::FAIL,
 229    true,
 230    L"\x0437\x0434\x0440\x0430\x0432\x0441\x0442\x0432"
 231    L"\x0443\x0439\x0442\x0435",
 232    NULL},
 233   // Thai (windows-874)
 234   {"windows-874",
 235    "\xCA\xC7\xD1\xCA\xB4\xD5" "\xA4\xC3\xD1\xBA",
 236    OnStringConversionError::FAIL,
 237    true,
 238    L"\x0E2A\x0E27\x0E31\x0E2A\x0E14\x0E35"
 239    L"\x0E04\x0E23\x0e31\x0E1A",
 240    NULL},
 241   // Empty text
 242   {"iscii-dev",
 243    "",
 244    OnStringConversionError::FAIL,
 245    true,
 246    L"",
 247    NULL},
 248 };
 249
 250 TEST(ICUStringConversionsTest, ConvertBetweenCodepageAndWide) {
 251   for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kConvertCodepageCases); ++i) {
 252     SCOPED_TRACE(base::StringPrintf(
 253                      "Test[%" PRIuS "]: <encoded: %s> <codepage: %s>", i,
 254                      kConvertCodepageCases[i].encoded,
 255                      kConvertCodepageCases[i].codepage_name));
 256
 257     std::wstring wide;
 258     bool success = CodepageToWide(kConvertCodepageCases[i].encoded,
 259                                   kConvertCodepageCases[i].codepage_name,
 260                                   kConvertCodepageCases[i].on_error,
 261                                   &wide);
 262     EXPECT_EQ(kConvertCodepageCases[i].success, success);
 263     EXPECT_EQ(kConvertCodepageCases[i].wide, wide);
 264
 265     // When decoding was successful and nothing was skipped, we also check the
 266     // reverse conversion. Not all conversions are round-trippable, but
 267     // kConverterCodepageCases does not have any one-way conversion at the
 268     // moment.
 269     if (success &&
 270         kConvertCodepageCases[i].on_error ==
 271             OnStringConversionError::FAIL) {
 272       std::string encoded;
 273       success = WideToCodepage(wide, kConvertCodepageCases[i].codepage_name,
 274                                kConvertCodepageCases[i].on_error, &encoded);
 275       EXPECT_EQ(kConvertCodepageCases[i].success, success);
 276       EXPECT_EQ(kConvertCodepageCases[i].encoded, encoded);
 277     }
 278   }
 279
 280   // The above cases handled codepage->wide errors, but not wide->codepage.
 281   // Test that here.
 282   std::string encoded("Temp data");  // Make sure the string gets cleared.
 283
 284   // First test going to an encoding that can not represent that character.
 285   EXPECT_FALSE(WideToCodepage(L"Chinese\xff27", "iso-8859-1",
 286                               OnStringConversionError::FAIL, &encoded));
 287   EXPECT_TRUE(encoded.empty());
 288   EXPECT_TRUE(WideToCodepage(L"Chinese\xff27", "iso-8859-1",
 289                              OnStringConversionError::SKIP, &encoded));
 290   EXPECT_STREQ("Chinese", encoded.c_str());
 291   // From Unicode, SUBSTITUTE is the same as SKIP for now.
 292   EXPECT_TRUE(WideToCodepage(L"Chinese\xff27", "iso-8859-1",
 293                              OnStringConversionError::SUBSTITUTE,
 294                              &encoded));
 295   EXPECT_STREQ("Chinese", encoded.c_str());
 296
 297 #if defined(WCHAR_T_IS_UTF16)
 298   // When we're in UTF-16 mode, test an invalid UTF-16 character in the input.
 299   EXPECT_FALSE(WideToCodepage(L"a\xd800z", "iso-8859-1",
 300                               OnStringConversionError::FAIL, &encoded));
 301   EXPECT_TRUE(encoded.empty());
 302   EXPECT_TRUE(WideToCodepage(L"a\xd800z", "iso-8859-1",
 303                              OnStringConversionError::SKIP, &encoded));
 304   EXPECT_STREQ("az", encoded.c_str());
 305 #endif  // WCHAR_T_IS_UTF16
 306
 307   // Invalid characters should fail.
 308   EXPECT_TRUE(WideToCodepage(L"a\xffffz", "iso-8859-1",
 309                              OnStringConversionError::SKIP, &encoded));
 310   EXPECT_STREQ("az", encoded.c_str());
 311
 312   // Invalid codepages should fail.
 313   EXPECT_FALSE(WideToCodepage(L"Hello, world", "awesome-8571-2",
 314                               OnStringConversionError::SKIP, &encoded));
 315 }
 316
 317 TEST(ICUStringConversionsTest, ConvertBetweenCodepageAndUTF16) {
 318   for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kConvertCodepageCases); ++i) {
 319     SCOPED_TRACE(base::StringPrintf(
 320                      "Test[%" PRIuS "]: <encoded: %s> <codepage: %s>", i,
 321                      kConvertCodepageCases[i].encoded,
 322                      kConvertCodepageCases[i].codepage_name));
 323
 324     string16 utf16;
 325     bool success = CodepageToUTF16(kConvertCodepageCases[i].encoded,
 326                                    kConvertCodepageCases[i].codepage_name,
 327                                    kConvertCodepageCases[i].on_error,
 328                                    &utf16);
 329     string16 utf16_expected;
 330     if (kConvertCodepageCases[i].u16_wide == NULL)
 331       utf16_expected = BuildString16(kConvertCodepageCases[i].wide);
 332     else
 333       utf16_expected = BuildString16(kConvertCodepageCases[i].u16_wide);
 334     EXPECT_EQ(kConvertCodepageCases[i].success, success);
 335     EXPECT_EQ(utf16_expected, utf16);
 336
 337     // When decoding was successful and nothing was skipped, we also check the
 338     // reverse conversion. See also the corresponding comment in
 339     // ConvertBetweenCodepageAndWide.
 340     if (success &&
 341         kConvertCodepageCases[i].on_error == OnStringConversionError::FAIL) {
 342       std::string encoded;
 343       success = UTF16ToCodepage(utf16, kConvertCodepageCases[i].codepage_name,
 344                                 kConvertCodepageCases[i].on_error, &encoded);
 345       EXPECT_EQ(kConvertCodepageCases[i].success, success);
 346       EXPECT_EQ(kConvertCodepageCases[i].encoded, encoded);
 347     }
 348   }
 349 }
 350
 351 static const struct {
 352   const char* encoded;
 353   const char* codepage_name;
 354   bool expected_success;
 355   const char* expected_value;
 356 } kConvertAndNormalizeCases[] = {
 357   {"foo-\xe4.html", "iso-8859-1", true, "foo-\xc3\xa4.html"},
 358   {"foo-\xe4.html", "iso-8859-7", true, "foo-\xce\xb4.html"},
 359   {"foo-\xe4.html", "foo-bar", false, ""},
 360   {"foo-\xff.html", "ascii", false, ""},
 361   {"foo.html", "ascii", true, "foo.html"},
 362   {"foo-a\xcc\x88.html", "utf-8", true, "foo-\xc3\xa4.html"},
 363   {"\x95\x32\x82\x36\xD2\xBB", "gb18030", true, "\xF0\xA0\x80\x80\xE4\xB8\x80"},
 364   {"\xA7\x41\xA6\x6E", "big5", true, "\xE4\xBD\xA0\xE5\xA5\xBD"},
 365   // Windows-1258 does have a combining character at xD2 (which is U+0309).
 366   // The sequence of (U+00E2, U+0309) is also encoded as U+1EA9.
 367   {"foo\xE2\xD2", "windows-1258", true, "foo\xE1\xBA\xA9"},
 368   {"", "iso-8859-1", true, ""},
 369 };
 370 TEST(ICUStringConversionsTest, ConvertToUtf8AndNormalize) {
 371   std::string result;
 372   for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kConvertAndNormalizeCases); ++i) {
 373     SCOPED_TRACE(base::StringPrintf(
 374                      "Test[%" PRIuS "]: <encoded: %s> <codepage: %s>", i,
 375                      kConvertAndNormalizeCases[i].encoded,
 376                      kConvertAndNormalizeCases[i].codepage_name));
 377
 378     bool success = ConvertToUtf8AndNormalize(
 379         kConvertAndNormalizeCases[i].encoded,
 380         kConvertAndNormalizeCases[i].codepage_name, &result);
 381     EXPECT_EQ(kConvertAndNormalizeCases[i].expected_success, success);
 382     EXPECT_EQ(kConvertAndNormalizeCases[i].expected_value, result);
 383   }
 384 }
 385
 386 }  // namespace base