src/corelib/codecs/qiconvcodec.cpp

   1 /****************************************************************************
   2 **
   3 ** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).
   4 ** All rights reserved.
   5 ** Contact: Nokia Corporation (qt-info@nokia.com)
   6 **
   7 ** This file is part of the QtCore module of the Qt Toolkit.
   8 **
   9 ** $QT_BEGIN_LICENSE:LGPL$
  10 ** No Commercial Usage
  11 ** This file contains pre-release code and may not be distributed.
  12 ** You may use this file in accordance with the terms and conditions
  13 ** contained in the Technology Preview License Agreement accompanying
  14 ** this package.
  15 **
  16 ** GNU Lesser General Public License Usage
  17 ** Alternatively, this file may be used under the terms of the GNU Lesser
  18 ** General Public License version 2.1 as published by the Free Software
  19 ** Foundation and appearing in the file LICENSE.LGPL included in the
  20 ** packaging of this file.  Please review the following information to
  21 ** ensure the GNU Lesser General Public License version 2.1 requirements
  22 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
  23 **
  24 ** In addition, as a special exception, Nokia gives you certain additional
  25 ** rights.  These rights are described in the Nokia Qt LGPL Exception
  26 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
  27 **
  28 ** If you have questions regarding the use of this file, please contact
  29 ** Nokia at qt-info@nokia.com.
  30 **
  31 **
  32 **
  33 **
  34 **
  35 **
  36 **
  37 **
  38 ** $QT_END_LICENSE$
  39 **
  40 ****************************************************************************/
  41
  42 #include "qiconvcodec_p.h"
  43 #include "qtextcodec_p.h"
  44 #include <qlibrary.h>
  45 #include <qdebug.h>
  46 #include <qthreadstorage.h>
  47
  48 #include <errno.h>
  49 #include <locale.h>
  50 #include <stdio.h>
  51 #include <dlfcn.h>
  52
  53 // unistd.h is needed for the _XOPEN_UNIX macro
  54 #include <unistd.h>
  55 #if defined(_XOPEN_UNIX) && !defined(Q_OS_QNX) && !defined(Q_OS_OSF)
  56 #  include <langinfo.h>
  57 #endif
  58
  59 #if defined(Q_OS_HPUX)
  60 #  define NO_BOM
  61 #  define UTF16 "ucs2"
  62 #elif defined(Q_OS_AIX)
  63 #  define NO_BOM
  64 #  define UTF16 "UCS-2"
  65 #elif defined(Q_OS_FREEBSD) || defined(Q_OS_MAC)
  66 #  define NO_BOM
  67 #  if Q_BYTE_ORDER == Q_BIG_ENDIAN
  68 #    define UTF16 "UTF-16BE"
  69 #  else
  70 #    define UTF16 "UTF-16LE"
  71 #  endif
  72 #else
  73 #  define UTF16 "UTF-16"
  74 #endif
  75
  76 #if defined(Q_OS_MAC)
  77 #ifndef GNU_LIBICONV
  78 #define GNU_LIBICONV
  79 #endif
  80 typedef iconv_t (*Ptr_iconv_open) (const char*, const char*);
  81 typedef size_t (*Ptr_iconv) (iconv_t, const char **, size_t *, char **, size_t *);
  82 typedef int (*Ptr_iconv_close) (iconv_t);
  83
  84 static Ptr_iconv_open ptr_iconv_open = 0;
  85 static Ptr_iconv ptr_iconv = 0;
  86 static Ptr_iconv_close ptr_iconv_close = 0;
  87 #endif
  88
  89 QT_BEGIN_NAMESPACE
  90
  91 extern bool qt_locale_initialized;
  92
  93 QIconvCodec::QIconvCodec()
  94     : utf16Codec(0)
  95 {
  96     utf16Codec = QTextCodec::codecForMib(1015);
  97     Q_ASSERT_X(utf16Codec != 0,
  98                "QIconvCodec::convertToUnicode",
  99                "internal error, UTF-16 codec not found");
 100     if (!utf16Codec) {
 101         fprintf(stderr, "QIconvCodec::convertToUnicode: internal error, UTF-16 codec not found\n");
 102         utf16Codec = reinterpret_cast<QTextCodec *>(~0);
 103     }
 104 #if defined(Q_OS_MAC)
 105     if (ptr_iconv_open == 0) {
 106         QLibrary libiconv(QLatin1String("/usr/lib/libiconv"));
 107         libiconv.setLoadHints(QLibrary::ExportExternalSymbolsHint);
 108
 109         ptr_iconv_open = reinterpret_cast<Ptr_iconv_open>(libiconv.resolve("libiconv_open"));
 110         if (!ptr_iconv_open)
 111             ptr_iconv_open = reinterpret_cast<Ptr_iconv_open>(libiconv.resolve("iconv_open"));
 112         ptr_iconv = reinterpret_cast<Ptr_iconv>(libiconv.resolve("libiconv"));
 113         if (!ptr_iconv)
 114             ptr_iconv = reinterpret_cast<Ptr_iconv>(libiconv.resolve("iconv"));
 115         ptr_iconv_close = reinterpret_cast<Ptr_iconv_close>(libiconv.resolve("libiconv_close"));
 116         if (!ptr_iconv_close)
 117             ptr_iconv_close = reinterpret_cast<Ptr_iconv_close>(libiconv.resolve("iconv_close"));
 118
 119         Q_ASSERT_X(ptr_iconv_open && ptr_iconv && ptr_iconv_close,
 120         "QIconvCodec::QIconvCodec()",
 121         "internal error, could not resolve the iconv functions");
 122
 123 #       undef iconv_open
 124 #       define iconv_open ptr_iconv_open
 125 #       undef iconv
 126 #       define iconv ptr_iconv
 127 #       undef iconv_close
 128 #       define iconv_close ptr_iconv_close
 129     }
 130 #endif
 131 }
 132
 133 QIconvCodec::~QIconvCodec()
 134 {
 135 }
 136
 137 QIconvCodec::IconvState::IconvState(iconv_t x)
 138     : buffer(array), bufferLen(sizeof array), cd(x)
 139 {
 140 }
 141
 142 QIconvCodec::IconvState::~IconvState()
 143 {
 144     if (cd != reinterpret_cast<iconv_t>(-1))
 145         iconv_close(cd);
 146     if (buffer != array)
 147         delete[] buffer;
 148 }
 149
 150 void QIconvCodec::IconvState::saveChars(const char *c, int count)
 151 {
 152     if (count > bufferLen) {
 153         if (buffer != array)
 154             delete[] buffer;
 155         buffer = new char[bufferLen = count];
 156     }
 157
 158     memcpy(buffer, c, count);
 159 }
 160
 161 static void qIconvCodecStateFree(QTextCodec::ConverterState *state)
 162 {
 163     delete reinterpret_cast<QIconvCodec::IconvState *>(state->d);
 164 }
 165
 166 Q_GLOBAL_STATIC(QThreadStorage<QIconvCodec::IconvState *>, toUnicodeState)
 167
 168 QString QIconvCodec::convertToUnicode(const char* chars, int len, ConverterState *convState) const
 169 {
 170     if (utf16Codec == reinterpret_cast<QTextCodec *>(~0))
 171         return QString::fromAscii(chars, len);
 172
 173     int invalidCount = 0;
 174     int remainingCount = 0;
 175     char *remainingBuffer = 0;
 176     IconvState **pstate;
 177
 178     if (convState) {
 179         // stateful conversion
 180         pstate = reinterpret_cast<IconvState **>(&convState->d);
 181         if (convState->d) {
 182             // restore state
 183             remainingCount = convState->remainingChars;
 184             remainingBuffer = (*pstate)->buffer;
 185         } else {
 186             // first time
 187             convState->flags |= FreeFunction;
 188             QTextCodecUnalignedPointer::encode(convState->state_data, qIconvCodecStateFree);
 189         }
 190     } else {
 191         QThreadStorage<QIconvCodec::IconvState *> *ts = toUnicodeState();
 192         if (!qt_locale_initialized || !ts) {
 193             // we're running after the Q_GLOBAL_STATIC has been deleted
 194             // or before the QCoreApplication initialization
 195             // bad programmer, no cookie for you
 196             return QString::fromLatin1(chars, len);
 197         }
 198
 199         // stateless conversion -- use thread-local data
 200         pstate = &toUnicodeState()->localData();
 201     }
 202
 203     if (!*pstate) {
 204         // first time, create the state
 205         iconv_t cd = QIconvCodec::createIconv_t(UTF16, 0);
 206         if (cd == reinterpret_cast<iconv_t>(-1)) {
 207             static int reported = 0;
 208             if (!reported++) {
 209                 fprintf(stderr,
 210                         "QIconvCodec::convertToUnicode: using ASCII for conversion, iconv_open failed\n");
 211             }
 212             return QString::fromAscii(chars, len);
 213         }
 214
 215         *pstate = new IconvState(cd);
 216     }
 217
 218     IconvState *state = *pstate;
 219     size_t inBytesLeft = len;
 220     // best case assumption, each byte is converted into one UTF-16 character, plus 2 bytes for the BOM
 221 #ifdef GNU_LIBICONV
 222     // GNU doesn't disagree with POSIX :/
 223     const char *inBytes = chars;
 224 #else
 225     char *inBytes = const_cast<char *>(chars);
 226 #endif
 227
 228     QByteArray in;
 229     if (remainingCount) {
 230         // we have to prepend the remaining bytes from the previous conversion
 231         inBytesLeft += remainingCount;
 232         in.resize(inBytesLeft);
 233         inBytes = in.data();
 234
 235         memcpy(in.data(), remainingBuffer, remainingCount);
 236         memcpy(in.data() + remainingCount, chars, len);
 237
 238         remainingCount = 0;
 239     }
 240
 241     size_t outBytesLeft = len * 2 + 2;
 242     QByteArray ba(outBytesLeft, Qt::Uninitialized);
 243     char *outBytes = ba.data();
 244     do {
 245         size_t ret = iconv(state->cd, &inBytes, &inBytesLeft, &outBytes, &outBytesLeft);
 246         if (ret == (size_t) -1) {
 247             if (errno == E2BIG) {
 248                 int offset = ba.size() - outBytesLeft;
 249                 ba.resize(ba.size() * 2);
 250                 outBytes = ba.data() + offset;
 251                 outBytesLeft = ba.size() - offset;
 252
 253                 continue;
 254             }
 255
 256             if (errno == EILSEQ) {
 257                 // conversion stopped because of an invalid character in the sequence
 258                 ++invalidCount;
 259             } else if (errno == EINVAL && convState) {
 260                 // conversion stopped because the remaining inBytesLeft make up
 261                 // an incomplete multi-byte sequence; save them for later
 262                 state->saveChars(inBytes, inBytesLeft);
 263                 remainingCount = inBytesLeft;
 264                 break;
 265             }
 266
 267             if (errno == EILSEQ || errno == EINVAL) {
 268                 // skip the next character
 269                 ++inBytes;
 270                 --inBytesLeft;
 271                 continue;
 272             }
 273
 274             // some other error
 275             // note, cannot use qWarning() since we are implementing the codecForLocale :)
 276             perror("QIconvCodec::convertToUnicode: using ASCII for conversion, iconv failed");
 277
 278             if (!convState) {
 279                 // reset state
 280                 iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
 281             }
 282
 283             return QString::fromAscii(chars, len);
 284         }
 285     } while (inBytesLeft != 0);
 286
 287     QString s = utf16Codec->toUnicode(ba.constData(), ba.size() - outBytesLeft);
 288
 289     if (convState) {
 290         convState->invalidChars = invalidCount;
 291         convState->remainingChars = remainingCount;
 292     } else {
 293         // reset state
 294         iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
 295     }
 296
 297     return s;
 298 }
 299
 300 Q_GLOBAL_STATIC(QThreadStorage<QIconvCodec::IconvState *>, fromUnicodeState)
 301
 302 QByteArray QIconvCodec::convertFromUnicode(const QChar *uc, int len, ConverterState *convState) const
 303 {
 304     char *inBytes;
 305     char *outBytes;
 306     size_t inBytesLeft;
 307
 308 #if defined(GNU_LIBICONV)
 309     const char **inBytesPtr = const_cast<const char **>(&inBytes);
 310 #else
 311     char **inBytesPtr = &inBytes;
 312 #endif
 313
 314     QThreadStorage<QIconvCodec::IconvState *> *ts = fromUnicodeState();
 315     if (!qt_locale_initialized || !ts) {
 316         // we're running after the Q_GLOBAL_STATIC has been deleted
 317         // or before the QCoreApplication initialization
 318         // bad programmer, no cookie for you
 319         if (!len)
 320             // this is a special case - zero-sized string should be
 321             // translated to empty but not-null QByteArray.
 322             return QByteArray("");
 323         return QString::fromRawData(uc, len).toLatin1();
 324     }
 325     IconvState *&state = ts->localData();
 326     if (!state) {
 327         state = new IconvState(QIconvCodec::createIconv_t(0, UTF16));
 328         if (state->cd != reinterpret_cast<iconv_t>(-1)) {
 329             size_t outBytesLeft = len + 3; // +3 for the BOM
 330             QByteArray ba(outBytesLeft, Qt::Uninitialized);
 331             outBytes = ba.data();
 332
 333 #if !defined(NO_BOM)
 334             // give iconv() a BOM
 335             QChar bom[] = { QChar(QChar::ByteOrderMark) };
 336             inBytes = reinterpret_cast<char *>(bom);
 337             inBytesLeft = sizeof(bom);
 338             if (iconv(state->cd, inBytesPtr, &inBytesLeft, &outBytes, &outBytesLeft) == (size_t) -1) {
 339                 perror("QIconvCodec::convertFromUnicode: using ASCII for conversion, iconv failed for BOM");
 340
 341                 iconv_close(state->cd);
 342                 state->cd = reinterpret_cast<iconv_t>(-1);
 343
 344                 return QString(uc, len).toAscii();
 345             }
 346 #endif // NO_BOM
 347         }
 348     }
 349     if (state->cd == reinterpret_cast<iconv_t>(-1)) {
 350         static int reported = 0;
 351         if (!reported++) {
 352             fprintf(stderr,
 353                     "QIconvCodec::convertFromUnicode: using ASCII for conversion, iconv_open failed\n");
 354         }
 355         return QString(uc, len).toAscii();
 356     }
 357
 358     size_t outBytesLeft = len;
 359     QByteArray ba(outBytesLeft, Qt::Uninitialized);
 360     outBytes = ba.data();
 361
 362     // now feed iconv() the real data
 363     inBytes = const_cast<char *>(reinterpret_cast<const char *>(uc));
 364     inBytesLeft = len * sizeof(QChar);
 365
 366     QByteArray in;
 367     if (convState && convState->remainingChars) {
 368         // we have one surrogate char to be prepended
 369         in.resize(sizeof(QChar) + len);
 370         inBytes = in.data();
 371
 372         QChar remaining = convState->state_data[0];
 373         memcpy(in.data(), &remaining, sizeof(QChar));
 374         memcpy(in.data() + sizeof(QChar), uc, inBytesLeft);
 375
 376         inBytesLeft += sizeof(QChar);
 377         convState->remainingChars = 0;
 378     }
 379
 380     int invalidCount = 0;
 381     while (inBytesLeft != 0) {
 382         if (iconv(state->cd, inBytesPtr, &inBytesLeft, &outBytes, &outBytesLeft) == (size_t) -1) {
 383             if (errno == EINVAL && convState) {
 384                 // buffer ends in a surrogate
 385                 Q_ASSERT(inBytesLeft == 2);
 386                 convState->remainingChars = 1;
 387                 convState->state_data[0] = uc[len - 1].unicode();
 388                 break;
 389             }
 390
 391             switch (errno) {
 392             case EILSEQ:
 393                 ++invalidCount;
 394                 // fall through
 395             case EINVAL:
 396                 {
 397                     inBytes += sizeof(QChar);
 398                     inBytesLeft -= sizeof(QChar);
 399                     break;
 400                 }
 401             case E2BIG:
 402                 {
 403                     int offset = ba.size() - outBytesLeft;
 404                     ba.resize(ba.size() * 2);
 405                     outBytes = ba.data() + offset;
 406                     outBytesLeft = ba.size() - offset;
 407                     break;
 408                 }
 409             default:
 410                 {
 411                     // note, cannot use qWarning() since we are implementing the codecForLocale :)
 412                     perror("QIconvCodec::convertFromUnicode: using ASCII for conversion, iconv failed");
 413
 414                     // reset to initial state
 415                     iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
 416
 417                     return QString(uc, len).toAscii();
 418                 }
 419             }
 420         }
 421     }
 422
 423     // reset to initial state
 424     iconv(state->cd, 0, &inBytesLeft, 0, &outBytesLeft);
 425
 426     ba.resize(ba.size() - outBytesLeft);
 427
 428     if (convState)
 429         convState->invalidChars = invalidCount;
 430
 431     return ba;
 432 }
 433
 434 QByteArray QIconvCodec::name() const
 435 {
 436     return "System";
 437 }
 438
 439 int QIconvCodec::mibEnum() const
 440 {
 441     return 0;
 442 }
 443
 444 iconv_t QIconvCodec::createIconv_t(const char *to, const char *from)
 445 {
 446     Q_ASSERT((to == 0 && from != 0) || (to != 0 && from == 0));
 447
 448     iconv_t cd = (iconv_t) -1;
 449 #if defined(__GLIBC__) || defined(GNU_LIBICONV)
 450     // both GLIBC and libgnuiconv will use the locale's encoding if from or to is an empty string
 451     static const char empty_codeset[] = "";
 452     const char *codeset = empty_codeset;
 453     cd = iconv_open(to ? to : codeset, from ? from : codeset);
 454 #else
 455     char *codeset = 0;
 456 #endif
 457
 458 #if defined(_XOPEN_UNIX) && !defined(Q_OS_QNX) && !defined(Q_OS_OSF)
 459     if (cd == (iconv_t) -1) {
 460         codeset = nl_langinfo(CODESET);
 461         if (codeset)
 462             cd = iconv_open(to ? to : codeset, from ? from : codeset);
 463     }
 464 #endif
 465
 466     if (cd == (iconv_t) -1) {
 467         // Very poorly defined and followed standards causes lots of
 468         // code to try to get all the cases... This logic is
 469         // duplicated in QTextCodec, so if you change it here, change
 470         // it there too.
 471
 472         // Try to determine locale codeset from locale name assigned to
 473         // LC_CTYPE category.
 474
 475         // First part is getting that locale name.  First try setlocale() which
 476         // definitely knows it, but since we cannot fully trust it, get ready
 477         // to fall back to environment variables.
 478         char * ctype = qstrdup(setlocale(LC_CTYPE, 0));
 479
 480         // Get the first nonempty value from $LC_ALL, $LC_CTYPE, and $LANG
 481         // environment variables.
 482         char * lang = qstrdup(qgetenv("LC_ALL").constData());
 483         if (!lang || lang[0] == 0 || strcmp(lang, "C") == 0) {
 484             if (lang) delete [] lang;
 485             lang = qstrdup(qgetenv("LC_CTYPE").constData());
 486         }
 487         if (!lang || lang[0] == 0 || strcmp(lang, "C") == 0) {
 488             if (lang) delete [] lang;
 489             lang = qstrdup(qgetenv("LANG").constData());
 490         }
 491
 492         // Now try these in order:
 493         // 1. CODESET from ctype if it contains a .CODESET part (e.g. en_US.ISO8859-15)
 494         // 2. CODESET from lang if it contains a .CODESET part
 495         // 3. ctype (maybe the locale is named "ISO-8859-1" or something)
 496         // 4. locale (ditto)
 497         // 5. check for "@euro"
 498
 499         // 1. CODESET from ctype if it contains a .CODESET part (e.g. en_US.ISO8859-15)
 500         codeset = ctype ? strchr(ctype, '.') : 0;
 501         if (codeset && *codeset == '.') {
 502             ++codeset;
 503             cd = iconv_open(to ? to : codeset, from ? from : codeset);
 504         }
 505
 506         // 2. CODESET from lang if it contains a .CODESET part
 507         codeset = lang ? strchr(lang, '.') : 0;
 508         if (cd == (iconv_t) -1 && codeset && *codeset == '.') {
 509             ++codeset;
 510             cd = iconv_open(to ? to : codeset, from ? from : codeset);
 511         }
 512
 513         // 3. ctype (maybe the locale is named "ISO-8859-1" or something)
 514         if (cd == (iconv_t) -1 && ctype && *ctype != 0 && strcmp (ctype, "C") != 0)
 515             cd = iconv_open(to ? to : ctype, from ? from : ctype);
 516
 517
 518         // 4. locale (ditto)
 519         if (cd == (iconv_t) -1 && lang && *lang != 0)
 520             cd = iconv_open(to ? to : lang, from ? from : lang);
 521
 522         // 5. "@euro"
 523         if ((cd == (iconv_t) -1 && ctype && strstr(ctype, "@euro")) || (lang && strstr(lang, "@euro")))
 524             cd = iconv_open(to ? to : "ISO8859-15", from ? from : "ISO8859-15");
 525
 526         delete [] ctype;
 527         delete [] lang;
 528     }
 529
 530     return cd;
 531 }
 532
 533 QT_END_NAMESPACE