base/i18n/icu_string_conversions.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/i18n/icu_string_conversions.h"
   6
   7 #include <vector>
   8
   9 #include "base/basictypes.h"
  10 #include "base/logging.h"
  11 #include "base/memory/scoped_ptr.h"
  12 #include "base/strings/string_util.h"
  13 #include "base/strings/utf_string_conversions.h"
  14 #include "third_party/icu/source/common/unicode/ucnv.h"
  15 #include "third_party/icu/source/common/unicode/ucnv_cb.h"
  16 #include "third_party/icu/source/common/unicode/ucnv_err.h"
  17 #include "third_party/icu/source/common/unicode/unorm.h"
  18 #include "third_party/icu/source/common/unicode/ustring.h"
  19
  20 namespace base {
  21
  22 namespace {
  23 // ToUnicodeCallbackSubstitute() is based on UCNV_TO_U_CALLBACK_SUBSTITUTE
  24 // in source/common/ucnv_err.c.
  25
  26 // Copyright (c) 1995-2006 International Business Machines Corporation
  27 // and others
  28 //
  29 // All rights reserved.
  30 //
  31
  32 // Permission is hereby granted, free of charge, to any person obtaining a
  33 // copy of this software and associated documentation files (the "Software"),
  34 // to deal in the Software without restriction, including without limitation
  35 // the rights to use, copy, modify, merge, publish, distribute, and/or
  36 // sell copies of the Software, and to permit persons to whom the Software
  37 // is furnished to do so, provided that the above copyright notice(s) and
  38 // this permission notice appear in all copies of the Software and that
  39 // both the above copyright notice(s) and this permission notice appear in
  40 // supporting documentation.
  41 //
  42 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  43 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  44 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
  45 // OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
  46 // INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT
  47 // OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
  48 // OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
  49 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE
  50 // OR PERFORMANCE OF THIS SOFTWARE.
  51 //
  52 // Except as contained in this notice, the name of a copyright holder
  53 // shall not be used in advertising or otherwise to promote the sale, use
  54 // or other dealings in this Software without prior written authorization
  55 // of the copyright holder.
  56
  57 //  ___________________________________________________________________________
  58 //
  59 // All trademarks and registered trademarks mentioned herein are the property
  60 // of their respective owners.
  61
  62 void ToUnicodeCallbackSubstitute(const void* context,
  63                                  UConverterToUnicodeArgs *to_args,
  64                                  const char* code_units,
  65                                  int32_t length,
  66                                  UConverterCallbackReason reason,
  67                                  UErrorCode * err) {
  68   static const UChar kReplacementChar = 0xFFFD;
  69   if (reason <= UCNV_IRREGULAR) {
  70       if (context == NULL ||
  71           (*(reinterpret_cast<const char*>(context)) == 'i' &&
  72            reason == UCNV_UNASSIGNED)) {
  73         *err = U_ZERO_ERROR;
  74         ucnv_cbToUWriteUChars(to_args, &kReplacementChar, 1, 0, err);
  75       }
  76       // else the caller must have set the error code accordingly.
  77   }
  78   // else ignore the reset, close and clone calls.
  79 }
  80
  81 bool ConvertFromUTF16(UConverter* converter, const UChar* uchar_src,
  82                       int uchar_len, OnStringConversionError::Type on_error,
  83                       std::string* encoded) {
  84   int encoded_max_length = UCNV_GET_MAX_BYTES_FOR_STRING(uchar_len,
  85       ucnv_getMaxCharSize(converter));
  86   encoded->resize(encoded_max_length);
  87
  88   UErrorCode status = U_ZERO_ERROR;
  89
  90   // Setup our error handler.
  91   switch (on_error) {
  92     case OnStringConversionError::FAIL:
  93       ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_STOP, 0,
  94                             NULL, NULL, &status);
  95       break;
  96     case OnStringConversionError::SKIP:
  97     case OnStringConversionError::SUBSTITUTE:
  98       ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_SKIP, 0,
  99                             NULL, NULL, &status);
 100       break;
 101     default:
 102       NOTREACHED();
 103   }
 104
 105   // ucnv_fromUChars returns size not including terminating null
 106   int actual_size = ucnv_fromUChars(converter, &(*encoded)[0],
 107       encoded_max_length, uchar_src, uchar_len, &status);
 108   encoded->resize(actual_size);
 109   ucnv_close(converter);
 110   if (U_SUCCESS(status))
 111     return true;
 112   encoded->clear();  // Make sure the output is empty on error.
 113   return false;
 114 }
 115
 116 // Set up our error handler for ToUTF-16 converters
 117 void SetUpErrorHandlerForToUChars(OnStringConversionError::Type on_error,
 118                                   UConverter* converter, UErrorCode* status) {
 119   switch (on_error) {
 120     case OnStringConversionError::FAIL:
 121       ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_STOP, 0,
 122                           NULL, NULL, status);
 123       break;
 124     case OnStringConversionError::SKIP:
 125       ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_SKIP, 0,
 126                           NULL, NULL, status);
 127       break;
 128     case OnStringConversionError::SUBSTITUTE:
 129       ucnv_setToUCallBack(converter, ToUnicodeCallbackSubstitute, 0,
 130                           NULL, NULL, status);
 131       break;
 132     default:
 133       NOTREACHED();
 134   }
 135 }
 136
 137 }  // namespace
 138
 139 // Codepage <-> Wide/UTF-16  ---------------------------------------------------
 140
 141 bool UTF16ToCodepage(const string16& utf16,
 142                      const char* codepage_name,
 143                      OnStringConversionError::Type on_error,
 144                      std::string* encoded) {
 145   encoded->clear();
 146
 147   UErrorCode status = U_ZERO_ERROR;
 148   UConverter* converter = ucnv_open(codepage_name, &status);
 149   if (!U_SUCCESS(status))
 150     return false;
 151
 152   return ConvertFromUTF16(converter, utf16.c_str(),
 153                           static_cast<int>(utf16.length()), on_error, encoded);
 154 }
 155
 156 bool CodepageToUTF16(const std::string& encoded,
 157                      const char* codepage_name,
 158                      OnStringConversionError::Type on_error,
 159                      string16* utf16) {
 160   utf16->clear();
 161
 162   UErrorCode status = U_ZERO_ERROR;
 163   UConverter* converter = ucnv_open(codepage_name, &status);
 164   if (!U_SUCCESS(status))
 165     return false;
 166
 167   // Even in the worst case, the maximum length in 2-byte units of UTF-16
 168   // output would be at most the same as the number of bytes in input. There
 169   // is no single-byte encoding in which a character is mapped to a
 170   // non-BMP character requiring two 2-byte units.
 171   //
 172   // Moreover, non-BMP characters in legacy multibyte encodings
 173   // (e.g. EUC-JP, GB18030) take at least 2 bytes. The only exceptions are
 174   // BOCU and SCSU, but we don't care about them.
 175   size_t uchar_max_length = encoded.length() + 1;
 176
 177   SetUpErrorHandlerForToUChars(on_error, converter, &status);
 178   scoped_ptr<char16[]> buffer(new char16[uchar_max_length]);
 179   int actual_size = ucnv_toUChars(converter, buffer.get(),
 180       static_cast<int>(uchar_max_length), encoded.data(),
 181       static_cast<int>(encoded.length()), &status);
 182   ucnv_close(converter);
 183   if (!U_SUCCESS(status)) {
 184     utf16->clear();  // Make sure the output is empty on error.
 185     return false;
 186   }
 187
 188   utf16->assign(buffer.get(), actual_size);
 189   return true;
 190 }
 191
 192 bool ConvertToUtf8AndNormalize(const std::string& text,
 193                                const std::string& charset,
 194                                std::string* result) {
 195   result->clear();
 196   string16 utf16;
 197   if (!CodepageToUTF16(
 198       text, charset.c_str(), OnStringConversionError::FAIL, &utf16))
 199     return false;
 200
 201   UErrorCode status = U_ZERO_ERROR;
 202   size_t max_length = utf16.length() + 1;
 203   string16 normalized_utf16;
 204   scoped_ptr<char16[]> buffer(new char16[max_length]);
 205   int actual_length = unorm_normalize(
 206       utf16.c_str(), utf16.length(), UNORM_NFC, 0,
 207       buffer.get(), static_cast<int>(max_length), &status);
 208   if (!U_SUCCESS(status))
 209     return false;
 210   normalized_utf16.assign(buffer.get(), actual_length);
 211
 212   return UTF16ToUTF8(normalized_utf16.data(),
 213                      normalized_utf16.length(), result);
 214 }
 215
 216 }  // namespace base