third_party/WebKit/Source/wtf/text/TextCodecICU.cpp

   1 /*
   2  * Copyright (C) 2004, 2006, 2007, 2008, 2011 Apple Inc. All rights reserved.
   3  * Copyright (C) 2006 Alexey Proskuryakov <ap@nypop.com>
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
  15  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE COMPUTER, INC. OR
  18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  21  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  22  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  24  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 #include "config.h"
  28 #include "wtf/text/TextCodecICU.h"
  29
  30 #include "wtf/Assertions.h"
  31 #include "wtf/StringExtras.h"
  32 #include "wtf/Threading.h"
  33 #include "wtf/WTFThreadData.h"
  34 #include "wtf/text/CString.h"
  35 #include "wtf/text/CharacterNames.h"
  36 #include "wtf/text/StringBuilder.h"
  37 #include <unicode/ucnv.h>
  38 #include <unicode/ucnv_cb.h>
  39
  40 using std::min;
  41
  42 namespace WTF {
  43
  44 const size_t ConversionBufferSize = 16384;
  45
  46 ICUConverterWrapper::~ICUConverterWrapper()
  47 {
  48     if (converter)
  49         ucnv_close(converter);
  50 }
  51
  52 static UConverter*& cachedConverterICU()
  53 {
  54     return wtfThreadData().cachedConverterICU().converter;
  55 }
  56
  57 PassOwnPtr<TextCodec> TextCodecICU::create(const TextEncoding& encoding, const void*)
  58 {
  59     return adoptPtr(new TextCodecICU(encoding));
  60 }
  61
  62 void TextCodecICU::registerEncodingNames(EncodingNameRegistrar registrar)
  63 {
  64     // We register Hebrew with logical ordering using a separate name.
  65     // Otherwise, this would share the same canonical name as the
  66     // visual ordering case, and then TextEncoding could not tell them
  67     // apart; ICU treats these names as synonyms.
  68     registrar("ISO-8859-8-I", "ISO-8859-8-I");
  69
  70     int32_t numEncodings = ucnv_countAvailable();
  71     for (int32_t i = 0; i < numEncodings; ++i) {
  72         const char* name = ucnv_getAvailableName(i);
  73         UErrorCode error = U_ZERO_ERROR;
  74 #if !defined(USING_SYSTEM_ICU)
  75         const char* primaryStandard = "HTML";
  76         const char* secondaryStandard = "MIME";
  77 #else
  78         const char* primaryStandard = "MIME";
  79         const char* secondaryStandard = "IANA";
  80 #endif
  81         const char* standardName = ucnv_getStandardName(name, primaryStandard, &error);
  82         if (U_FAILURE(error) || !standardName) {
  83             error = U_ZERO_ERROR;
  84             // Try IANA to pick up 'windows-12xx' and other names
  85             // which are not preferred MIME names but are widely used.
  86             standardName = ucnv_getStandardName(name, secondaryStandard, &error);
  87             if (U_FAILURE(error) || !standardName)
  88                 continue;
  89         }
  90
  91         // A number of these aliases are handled in Chrome's copy of ICU, but
  92         // Chromium can be compiled with the system ICU.
  93
  94         // 1. Treat GB2312 encoding as GBK (its more modern superset), to match other browsers.
  95         // 2. On the Web, GB2312 is encoded as EUC-CN or HZ, while ICU provides a native encoding
  96         //    for encoding GB_2312-80 and several others. So, we need to override this behavior, too.
  97 #if defined(USING_SYSTEM_ICU)
  98         if (!strcmp(standardName, "GB2312") || !strcmp(standardName, "GB_2312-80"))
  99             standardName = "GBK";
 100         // Similarly, EUC-KR encodings all map to an extended version, but
 101         // per HTML5, the canonical name still should be EUC-KR.
 102         else if (!strcmp(standardName, "EUC-KR") || !strcmp(standardName, "KSC_5601") || !strcmp(standardName, "cp1363"))
 103             standardName = "EUC-KR";
 104         // And so on.
 105         else if (!strcasecmp(standardName, "iso-8859-9")) // This name is returned in different case by ICU 3.2 and 3.6.
 106             standardName = "windows-1254";
 107         else if (!strcmp(standardName, "TIS-620"))
 108             standardName = "windows-874";
 109 #endif
 110
 111         registrar(standardName, standardName);
 112
 113         uint16_t numAliases = ucnv_countAliases(name, &error);
 114         ASSERT(U_SUCCESS(error));
 115         if (U_SUCCESS(error))
 116             for (uint16_t j = 0; j < numAliases; ++j) {
 117                 error = U_ZERO_ERROR;
 118                 const char* alias = ucnv_getAlias(name, j, &error);
 119                 ASSERT(U_SUCCESS(error));
 120                 if (U_SUCCESS(error) && alias != standardName)
 121                     registrar(alias, standardName);
 122             }
 123     }
 124
 125     // These two entries have to be added here because ICU's converter table
 126     // cannot have both ISO-8859-8-I and ISO-8859-8.
 127     registrar("csISO88598I", "ISO-8859-8-I");
 128     registrar("logical", "ISO-8859-8-I");
 129
 130 #if defined(USING_SYSTEM_ICU)
 131     // Additional alias for MacCyrillic not present in ICU.
 132     registrar("maccyrillic", "x-mac-cyrillic");
 133
 134     // Additional aliases that historically were present in the encoding
 135     // table in WebKit on Macintosh that don't seem to be present in ICU.
 136     // Perhaps we can prove these are not used on the web and remove them.
 137     // Or perhaps we can get them added to ICU.
 138     registrar("x-mac-roman", "macintosh");
 139     registrar("x-mac-ukrainian", "x-mac-cyrillic");
 140     registrar("cn-big5", "Big5");
 141     registrar("x-x-big5", "Big5");
 142     registrar("cn-gb", "GBK");
 143     registrar("csgb231280", "GBK");
 144     registrar("x-euc-cn", "GBK");
 145     registrar("x-gbk", "GBK");
 146     registrar("koi", "KOI8-R");
 147     registrar("visual", "ISO-8859-8");
 148     registrar("winarabic", "windows-1256");
 149     registrar("winbaltic", "windows-1257");
 150     registrar("wincyrillic", "windows-1251");
 151     registrar("iso-8859-11", "windows-874");
 152     registrar("iso8859-11", "windows-874");
 153     registrar("dos-874", "windows-874");
 154     registrar("wingreek", "windows-1253");
 155     registrar("winhebrew", "windows-1255");
 156     registrar("winlatin2", "windows-1250");
 157     registrar("winturkish", "windows-1254");
 158     registrar("winvietnamese", "windows-1258");
 159     registrar("x-cp1250", "windows-1250");
 160     registrar("x-cp1251", "windows-1251");
 161     registrar("x-euc", "EUC-JP");
 162     registrar("x-windows-949", "EUC-KR");
 163     registrar("KSC5601", "EUC-KR");
 164     registrar("x-uhc", "EUC-KR");
 165     registrar("shift-jis", "Shift_JIS");
 166
 167     // Alternative spelling of ISO encoding names.
 168     registrar("ISO8859-1", "ISO-8859-1");
 169     registrar("ISO8859-2", "ISO-8859-2");
 170     registrar("ISO8859-3", "ISO-8859-3");
 171     registrar("ISO8859-4", "ISO-8859-4");
 172     registrar("ISO8859-5", "ISO-8859-5");
 173     registrar("ISO8859-6", "ISO-8859-6");
 174     registrar("ISO8859-7", "ISO-8859-7");
 175     registrar("ISO8859-8", "ISO-8859-8");
 176     registrar("ISO8859-8-I", "ISO-8859-8-I");
 177     registrar("ISO8859-9", "ISO-8859-9");
 178     registrar("ISO8859-10", "ISO-8859-10");
 179     registrar("ISO8859-13", "ISO-8859-13");
 180     registrar("ISO8859-14", "ISO-8859-14");
 181     registrar("ISO8859-15", "ISO-8859-15");
 182     // No need to have an entry for ISO8859-16. ISO-8859-16 has just one label
 183     // listed in WHATWG Encoding Living Standard (http://encoding.spec.whatwg.org/ ).
 184
 185     // Additional aliases present in the WHATWG Encoding Standard
 186     // and Firefox (as of Oct 2014), but not in the upstream ICU.
 187     // Three entries for windows-1252 need not be listed here because
 188     // TextCodecLatin1 registers them.
 189     registrar("csiso58gb231280", "GBK");
 190     registrar("csiso88596e", "ISO-8859-6");
 191     registrar("csiso88596i", "ISO-8859-6");
 192     registrar("csiso88598e", "ISO-8859-8");
 193     registrar("gb_2312", "GBK");
 194     registrar("iso88592", "ISO-8859-2");
 195     registrar("iso88593", "ISO-8859-3");
 196     registrar("iso88594", "ISO-8859-4");
 197     registrar("iso88595", "ISO-8859-5");
 198     registrar("iso88596", "ISO-8859-6");
 199     registrar("iso88597", "ISO-8859-7");
 200     registrar("iso88598", "ISO-8859-8");
 201     registrar("iso88599", "windows-1254");
 202     registrar("iso885910", "ISO-8859-10");
 203     registrar("iso885911", "windows-874");
 204     registrar("iso885913", "ISO-8859-13");
 205     registrar("iso885914", "ISO-8859-14");
 206     registrar("iso885915", "ISO-8859-15");
 207     registrar("iso_8859-2", "ISO-8859-2");
 208     registrar("iso_8859-3", "ISO-8859-3");
 209     registrar("iso_8859-4", "ISO-8859-4");
 210     registrar("iso_8859-5", "ISO-8859-5");
 211     registrar("iso_8859-6", "ISO-8859-6");
 212     registrar("iso_8859-7", "ISO-8859-7");
 213     registrar("iso_8859-8", "ISO-8859-8");
 214     registrar("iso_8859-9", "windows-1254");
 215     registrar("iso_8859-15", "ISO-8859-15");
 216     registrar("koi8_r", "KOI8-R");
 217     registrar("x-cp1253", "windows-1253");
 218     registrar("x-cp1254", "windows-1254");
 219     registrar("x-cp1255", "windows-1255");
 220     registrar("x-cp1256", "windows-1256");
 221     registrar("x-cp1257", "windows-1257");
 222     registrar("x-cp1258", "windows-1258");
 223 #endif
 224 }
 225
 226 void TextCodecICU::registerCodecs(TextCodecRegistrar registrar)
 227 {
 228     // See comment above in registerEncodingNames.
 229     registrar("ISO-8859-8-I", create, 0);
 230
 231     int32_t numEncodings = ucnv_countAvailable();
 232     for (int32_t i = 0; i < numEncodings; ++i) {
 233         const char* name = ucnv_getAvailableName(i);
 234         UErrorCode error = U_ZERO_ERROR;
 235         const char* standardName = ucnv_getStandardName(name, "MIME", &error);
 236         if (!U_SUCCESS(error) || !standardName) {
 237             error = U_ZERO_ERROR;
 238             standardName = ucnv_getStandardName(name, "IANA", &error);
 239             if (!U_SUCCESS(error) || !standardName)
 240                 continue;
 241         }
 242         registrar(standardName, create, 0);
 243     }
 244 }
 245
 246 TextCodecICU::TextCodecICU(const TextEncoding& encoding)
 247     : m_encoding(encoding)
 248     , m_converterICU(0)
 249 #if defined(USING_SYSTEM_ICU)
 250     , m_needsGBKFallbacks(false)
 251 #endif
 252 {
 253 }
 254
 255 TextCodecICU::~TextCodecICU()
 256 {
 257     releaseICUConverter();
 258 }
 259
 260 void TextCodecICU::releaseICUConverter() const
 261 {
 262     if (m_converterICU) {
 263         UConverter*& cachedConverter = cachedConverterICU();
 264         if (cachedConverter)
 265             ucnv_close(cachedConverter);
 266         cachedConverter = m_converterICU;
 267         m_converterICU = 0;
 268     }
 269 }
 270
 271 void TextCodecICU::createICUConverter() const
 272 {
 273     ASSERT(!m_converterICU);
 274
 275 #if defined(USING_SYSTEM_ICU)
 276     const char* name = m_encoding.name();
 277     m_needsGBKFallbacks = name[0] == 'G' && name[1] == 'B' && name[2] == 'K' && !name[3];
 278 #endif
 279
 280     UErrorCode err;
 281
 282     UConverter*& cachedConverter = cachedConverterICU();
 283     if (cachedConverter) {
 284         err = U_ZERO_ERROR;
 285         const char* cachedName = ucnv_getName(cachedConverter, &err);
 286         if (U_SUCCESS(err) && m_encoding == cachedName) {
 287             m_converterICU = cachedConverter;
 288             cachedConverter = 0;
 289             return;
 290         }
 291     }
 292
 293     err = U_ZERO_ERROR;
 294     m_converterICU = ucnv_open(m_encoding.name(), &err);
 295 #if !LOG_DISABLED
 296     if (err == U_AMBIGUOUS_ALIAS_WARNING)
 297         WTF_LOG_ERROR("ICU ambiguous alias warning for encoding: %s", m_encoding.name());
 298 #endif
 299     if (m_converterICU)
 300         ucnv_setFallback(m_converterICU, TRUE);
 301 }
 302
 303 int TextCodecICU::decodeToBuffer(UChar* target, UChar* targetLimit, const char*& source, const char* sourceLimit, int32_t* offsets, bool flush, UErrorCode& err)
 304 {
 305     UChar* targetStart = target;
 306     err = U_ZERO_ERROR;
 307     ucnv_toUnicode(m_converterICU, &target, targetLimit, &source, sourceLimit, offsets, flush, &err);
 308     return target - targetStart;
 309 }
 310
 311 class ErrorCallbackSetter {
 312 public:
 313     ErrorCallbackSetter(UConverter* converter, bool stopOnError)
 314         : m_converter(converter)
 315         , m_shouldStopOnEncodingErrors(stopOnError)
 316     {
 317         if (m_shouldStopOnEncodingErrors) {
 318             UErrorCode err = U_ZERO_ERROR;
 319             ucnv_setToUCallBack(m_converter, UCNV_TO_U_CALLBACK_SUBSTITUTE,
 320                            UCNV_SUB_STOP_ON_ILLEGAL, &m_savedAction,
 321                            &m_savedContext, &err);
 322             ASSERT(err == U_ZERO_ERROR);
 323         }
 324     }
 325     ~ErrorCallbackSetter()
 326     {
 327         if (m_shouldStopOnEncodingErrors) {
 328             UErrorCode err = U_ZERO_ERROR;
 329             const void* oldContext;
 330             UConverterToUCallback oldAction;
 331             ucnv_setToUCallBack(m_converter, m_savedAction,
 332                    m_savedContext, &oldAction,
 333                    &oldContext, &err);
 334             ASSERT(oldAction == UCNV_TO_U_CALLBACK_SUBSTITUTE);
 335             ASSERT(!strcmp(static_cast<const char*>(oldContext), UCNV_SUB_STOP_ON_ILLEGAL));
 336             ASSERT(err == U_ZERO_ERROR);
 337         }
 338     }
 339
 340 private:
 341     UConverter* m_converter;
 342     bool m_shouldStopOnEncodingErrors;
 343     const void* m_savedContext;
 344     UConverterToUCallback m_savedAction;
 345 };
 346
 347 String TextCodecICU::decode(const char* bytes, size_t length, FlushBehavior flush, bool stopOnError, bool& sawError)
 348 {
 349     // Get a converter for the passed-in encoding.
 350     if (!m_converterICU) {
 351         createICUConverter();
 352         ASSERT(m_converterICU);
 353         if (!m_converterICU) {
 354             WTF_LOG_ERROR("error creating ICU encoder even though encoding was in table");
 355             return String();
 356         }
 357     }
 358
 359     ErrorCallbackSetter callbackSetter(m_converterICU, stopOnError);
 360
 361     StringBuilder result;
 362
 363     UChar buffer[ConversionBufferSize];
 364     UChar* bufferLimit = buffer + ConversionBufferSize;
 365     const char* source = reinterpret_cast<const char*>(bytes);
 366     const char* sourceLimit = source + length;
 367     int32_t* offsets = NULL;
 368     UErrorCode err = U_ZERO_ERROR;
 369
 370     do {
 371         int ucharsDecoded = decodeToBuffer(buffer, bufferLimit, source, sourceLimit, offsets, flush != DoNotFlush, err);
 372         result.append(buffer, ucharsDecoded);
 373     } while (err == U_BUFFER_OVERFLOW_ERROR);
 374
 375     if (U_FAILURE(err)) {
 376         // flush the converter so it can be reused, and not be bothered by this error.
 377         do {
 378             decodeToBuffer(buffer, bufferLimit, source, sourceLimit, offsets, true, err);
 379         } while (source < sourceLimit);
 380         sawError = true;
 381     }
 382
 383 #if !defined(USING_SYSTEM_ICU)
 384     // Chrome's copy of ICU does not have the issue described below.
 385     return result.toString();
 386 #else
 387     String resultString = result.toString();
 388
 389     // <http://bugs.webkit.org/show_bug.cgi?id=17014>
 390     // Simplified Chinese pages use the code A3A0 to mean "full-width space", but ICU decodes it as U+E5E5.
 391     if (!strcmp(m_encoding.name(), "GBK")) {
 392         if (!strcasecmp(m_encoding.name(), "gb18030"))
 393             resultString.replace(0xE5E5, ideographicSpaceCharacter);
 394         // Make GBK compliant to the encoding spec and align with GB18030
 395         resultString.replace(0x01F9, 0xE7C8);
 396         // FIXME: Once https://www.w3.org/Bugs/Public/show_bug.cgi?id=28740#c3
 397         // is resolved, add U+1E3F => 0xE7C7.
 398     }
 399
 400     return resultString;
 401 #endif
 402 }
 403
 404 #if defined(USING_SYSTEM_ICU)
 405 // U+01F9 and U+1E3F have to be mapped to xA8xBF and xA8xBC per the encoding
 406 // spec, but ICU converter does not have them.
 407 static UChar fallbackForGBK(UChar32 character)
 408 {
 409     switch (character) {
 410     case 0x01F9:
 411         return 0xE7C8; // mapped to xA8xBF by ICU.
 412     case 0x1E3F:
 413         return 0xE7C7; // mapped to xA8xBC by ICU.
 414     }
 415     return 0;
 416 }
 417 #endif
 418
 419 // Invalid character handler when writing escaped entities for unrepresentable
 420 // characters. See the declaration of TextCodec::encode for more.
 421 static void urlEscapedEntityCallback(const void* context, UConverterFromUnicodeArgs* fromUArgs, const UChar* codeUnits, int32_t length,
 422     UChar32 codePoint, UConverterCallbackReason reason, UErrorCode* err)
 423 {
 424     if (reason == UCNV_UNASSIGNED) {
 425         *err = U_ZERO_ERROR;
 426
 427         UnencodableReplacementArray entity;
 428         int entityLen = TextCodec::getUnencodableReplacement(codePoint, URLEncodedEntitiesForUnencodables, entity);
 429         ucnv_cbFromUWriteBytes(fromUArgs, entity, entityLen, 0, err);
 430     } else
 431         UCNV_FROM_U_CALLBACK_ESCAPE(context, fromUArgs, codeUnits, length, codePoint, reason, err);
 432 }
 433
 434 #if defined(USING_SYSTEM_ICU)
 435 // Substitutes special GBK characters, escaping all other unassigned entities.
 436 static void gbkCallbackEscape(const void* context, UConverterFromUnicodeArgs* fromUArgs, const UChar* codeUnits, int32_t length,
 437     UChar32 codePoint, UConverterCallbackReason reason, UErrorCode* err)
 438 {
 439     UChar outChar;
 440     if (reason == UCNV_UNASSIGNED && (outChar = fallbackForGBK(codePoint))) {
 441         const UChar* source = &outChar;
 442         *err = U_ZERO_ERROR;
 443         ucnv_cbFromUWriteUChars(fromUArgs, &source, source + 1, 0, err);
 444         return;
 445     }
 446     UCNV_FROM_U_CALLBACK_ESCAPE(context, fromUArgs, codeUnits, length, codePoint, reason, err);
 447 }
 448
 449 // Combines both gbkUrlEscapedEntityCallback and GBK character substitution.
 450 static void gbkUrlEscapedEntityCallack(const void* context, UConverterFromUnicodeArgs* fromUArgs, const UChar* codeUnits, int32_t length,
 451     UChar32 codePoint, UConverterCallbackReason reason, UErrorCode* err)
 452 {
 453     if (reason == UCNV_UNASSIGNED) {
 454         if (UChar outChar = fallbackForGBK(codePoint)) {
 455             const UChar* source = &outChar;
 456             *err = U_ZERO_ERROR;
 457             ucnv_cbFromUWriteUChars(fromUArgs, &source, source + 1, 0, err);
 458             return;
 459         }
 460         urlEscapedEntityCallback(context, fromUArgs, codeUnits, length, codePoint, reason, err);
 461         return;
 462     }
 463     UCNV_FROM_U_CALLBACK_ESCAPE(context, fromUArgs, codeUnits, length, codePoint, reason, err);
 464 }
 465
 466 static void gbkCallbackSubstitute(const void* context, UConverterFromUnicodeArgs* fromUArgs, const UChar* codeUnits, int32_t length,
 467     UChar32 codePoint, UConverterCallbackReason reason, UErrorCode* err)
 468 {
 469     UChar outChar;
 470     if (reason == UCNV_UNASSIGNED && (outChar = fallbackForGBK(codePoint))) {
 471         const UChar* source = &outChar;
 472         *err = U_ZERO_ERROR;
 473         ucnv_cbFromUWriteUChars(fromUArgs, &source, source + 1, 0, err);
 474         return;
 475     }
 476     UCNV_FROM_U_CALLBACK_SUBSTITUTE(context, fromUArgs, codeUnits, length, codePoint, reason, err);
 477 }
 478 #endif // USING_SYSTEM_ICU
 479
 480 class TextCodecInput {
 481 public:
 482     TextCodecInput(const TextEncoding& encoding, const UChar* characters, size_t length)
 483         : m_begin(characters)
 484         , m_end(characters + length)
 485     { }
 486
 487     TextCodecInput(const TextEncoding& encoding, const LChar* characters, size_t length)
 488     {
 489         m_buffer.reserveInitialCapacity(length);
 490         for (size_t i = 0; i < length; ++i)
 491             m_buffer.append(characters[i]);
 492         m_begin = m_buffer.data();
 493         m_end = m_begin + m_buffer.size();
 494     }
 495
 496     const UChar* begin() const { return m_begin; }
 497     const UChar* end() const { return m_end; }
 498
 499 private:
 500     const UChar* m_begin;
 501     const UChar* m_end;
 502     Vector<UChar> m_buffer;
 503 };
 504
 505 CString TextCodecICU::encodeInternal(const TextCodecInput& input, UnencodableHandling handling)
 506 {
 507     const UChar* source = input.begin();
 508     const UChar* end = input.end();
 509
 510     UErrorCode err = U_ZERO_ERROR;
 511
 512     switch (handling) {
 513         case QuestionMarksForUnencodables:
 514             ucnv_setSubstChars(m_converterICU, "?", 1, &err);
 515 #if !defined(USING_SYSTEM_ICU)
 516             ucnv_setFromUCallBack(m_converterICU, UCNV_FROM_U_CALLBACK_SUBSTITUTE, 0, 0, 0, &err);
 517 #else
 518             ucnv_setFromUCallBack(m_converterICU, m_needsGBKFallbacks ? gbkCallbackSubstitute : UCNV_FROM_U_CALLBACK_SUBSTITUTE, 0, 0, 0, &err);
 519 #endif
 520             break;
 521         case EntitiesForUnencodables:
 522 #if !defined(USING_SYSTEM_ICU)
 523             ucnv_setFromUCallBack(m_converterICU, UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_XML_DEC, 0, 0, &err);
 524 #else
 525             ucnv_setFromUCallBack(m_converterICU, m_needsGBKFallbacks ? gbkCallbackEscape : UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_ESCAPE_XML_DEC, 0, 0, &err);
 526 #endif
 527             break;
 528         case URLEncodedEntitiesForUnencodables:
 529 #if !defined(USING_SYSTEM_ICU)
 530             ucnv_setFromUCallBack(m_converterICU, urlEscapedEntityCallback, 0, 0, 0, &err);
 531 #else
 532             ucnv_setFromUCallBack(m_converterICU, m_needsGBKFallbacks ? gbkUrlEscapedEntityCallack : urlEscapedEntityCallback, 0, 0, 0, &err);
 533 #endif
 534             break;
 535     }
 536
 537     ASSERT(U_SUCCESS(err));
 538     if (U_FAILURE(err))
 539         return CString();
 540
 541     Vector<char> result;
 542     size_t size = 0;
 543     do {
 544         char buffer[ConversionBufferSize];
 545         char* target = buffer;
 546         char* targetLimit = target + ConversionBufferSize;
 547         err = U_ZERO_ERROR;
 548         ucnv_fromUnicode(m_converterICU, &target, targetLimit, &source, end, 0, true, &err);
 549         size_t count = target - buffer;
 550         result.grow(size + count);
 551         memcpy(result.data() + size, buffer, count);
 552         size += count;
 553     } while (err == U_BUFFER_OVERFLOW_ERROR);
 554
 555     return CString(result.data(), size);
 556 }
 557
 558 template<typename CharType>
 559 CString TextCodecICU::encodeCommon(const CharType* characters, size_t length, UnencodableHandling handling)
 560 {
 561     if (!length)
 562         return "";
 563
 564     if (!m_converterICU)
 565         createICUConverter();
 566     if (!m_converterICU)
 567         return CString();
 568
 569     TextCodecInput input(m_encoding, characters, length);
 570     return encodeInternal(input, handling);
 571 }
 572
 573 CString TextCodecICU::encode(const UChar* characters, size_t length, UnencodableHandling handling)
 574 {
 575     return encodeCommon(characters, length, handling);
 576 }
 577
 578 CString TextCodecICU::encode(const LChar* characters, size_t length, UnencodableHandling handling)
 579 {
 580     return encodeCommon(characters, length, handling);
 581 }
 582
 583 } // namespace WTF