sal/textenc/convertisciidevangari.cxx

   1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
   2 /*
   3  * This file is part of the LibreOffice project.
   4  *
   5  * This Source Code Form is subject to the terms of the Mozilla Public
   6  * License, v. 2.0. If a copy of the MPL was not distributed with this
   7  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
   8  */
   9
  10 #include "converter.hxx"
  11 #include "convertisciidevangari.hxx"
  12 #include "convertsinglebytetobmpunicode.hxx"
  13 #include <rtl/textcvt.h>
  14
  15 using namespace sal::detail::textenc;
  16 using namespace rtl::textenc;
  17
  18 struct IsciiDevanagariToUnicode
  19 {
  20     sal_uInt8 m_cPrevChar;
  21     IsciiDevanagariToUnicode()
  22         : m_cPrevChar(0)
  23     {
  24     }
  25     void reset()
  26     {
  27         m_cPrevChar = 0;
  28     }
  29     sal_Size convert(char const* pSrcBuf, sal_Size nSrcBytes,
  30         sal_Unicode* pDestBuf, sal_Size nDestChars, sal_uInt32 nFlags,
  31         sal_uInt32* pInfo, sal_Size* pSrcCvtBytes);
  32 };
  33
  34 struct UnicodeToIsciiDevanagari
  35 {
  36     sal_Unicode m_cPrevChar;
  37     sal_Unicode m_cHighSurrogate;
  38     UnicodeToIsciiDevanagari()
  39         : m_cPrevChar(0)
  40         , m_cHighSurrogate(0)
  41     {
  42     }
  43     void reset()
  44     {
  45         m_cPrevChar = 0;
  46         m_cHighSurrogate = 0;
  47     }
  48     sal_Size convert(sal_Unicode const* pSrcBuf, sal_Size nSrcChars,
  49         char* pDestBuf, sal_Size nDestBytes, sal_uInt32 nFlags,
  50         sal_uInt32 * pInfo, sal_Size * pSrcCvtChars);
  51 };
  52
  53 static const sal_Unicode IsciiDevanagariMap[256] =
  54 {
  55     0x0000,0x0001,0x0002,0x0003,0x0004,0x0005,0x0006,0x0007,
  56     0x0008,0x0009,0x000A,0x000B,0x000C,0x000D,0x000E,0x000F,
  57     0x0010,0x0011,0x0012,0x0013,0x0014,0x0015,0x0016,0x0017,
  58     0x0018,0x0019,0x001A,0x001B,0x001C,0x001D,0x001E,0x001F,
  59     0x0020,0x0021,0x0022,0x0023,0x0024,0x0025,0x0026,0x0027,
  60     0x0028,0x0029,0x002A,0x002B,0x002C,0x002D,0x002E,0x002F,
  61     0x0030,0x0031,0x0032,0x0033,0x0034,0x0035,0x0036,0x0037,
  62     0x0038,0x0039,0x003A,0x003B,0x003C,0x003D,0x003E,0x003F,
  63     0x0040,0x0041,0x0042,0x0043,0x0044,0x0045,0x0046,0x0047,
  64     0x0048,0x0049,0x004A,0x004B,0x004C,0x004D,0x004E,0x004F,
  65     0x0050,0x0051,0x0052,0x0053,0x0054,0x0055,0x0056,0x0057,
  66     0x0058,0x0059,0x005A,0x005B,0x005C,0x005D,0x005E,0x005F,
  67     0x0060,0x0061,0x0062,0x0063,0x0064,0x0065,0x0066,0x0067,
  68     0x0068,0x0069,0x006A,0x006B,0x006C,0x006D,0x006E,0x006F,
  69     0x0070,0x0071,0x0072,0x0073,0x0074,0x0075,0x0076,0x0077,
  70     0x0078,0x0079,0x007A,0x007B,0x007C,0x007D,0x007E,0x007F,
  71     0xFFFF,0xFFFF,0xFFFF,0xFFFF,0xFFFF,0xFFFF,0xFFFF,0xFFFF,
  72     0xFFFF,0xFFFF,0xFFFF,0xFFFF,0xFFFF,0xFFFF,0xFFFF,0xFFFF,
  73     0xFFFF,0xFFFF,0xFFFF,0xFFFF,0xFFFF,0xFFFF,0xFFFF,0xFFFF,
  74     0xFFFF,0xFFFF,0xFFFF,0xFFFF,0xFFFF,0xFFFF,0xFFFF,0xFFFF,
  75     0xFFFF,0x0901,0x0902,0x0903,0x0905,0x0906,0x0907,0x0908,
  76     0x0909,0x090A,0x090B,0x090E,0x090F,0x0910,0x090D,0x0912,
  77     0x0913,0x0914,0x0911,0x0915,0x0916,0x0917,0x0918,0x0919,
  78     0x091A,0x091B,0x091C,0x091D,0x091E,0x091F,0x0920,0x0921,
  79     0x0922,0x0923,0x0924,0x0925,0x0926,0x0927,0x0928,0x0929,
  80     0x092A,0x092B,0x092C,0x092D,0x092E,0x092F,0x095F,0x0930,
  81     0x0931,0x0932,0x0933,0x0934,0x0935,0x0936,0x0937,0x0938,
  82     0x0939,0xFFFF,0x093E,0x093F,0x0940,0x0941,0x0942,0x0943,
  83     0x0946,0x0947,0x0948,0x0945,0x094A,0x094B,0x094C,0x0949,
  84     0x094D,0x093C,0x0964,0xFFFF,0xFFFF,0xFFFF,0xFFFF,0xFFFF,
  85     0xFFFF,0x0966,0x0967,0x0968,0x0969,0x096A,0x096B,0x096C,
  86     0x096D,0x096E,0x096F,0xFFFF,0xFFFF,0xFFFF,0xFFFF,0xFFFF
  87 };
  88
  89 sal_Size IsciiDevanagariToUnicode::convert(
  90     char const* pSrcBuf, sal_Size nSrcBytes,
  91     sal_Unicode* pDestBuf, sal_Size nDestChars, sal_uInt32 nFlags,
  92     sal_uInt32* pInfo, sal_Size* pSrcCvtBytes)
  93 {
  94     sal_uInt32 nInfo = 0;
  95     sal_Size nConverted = 0;
  96     sal_Unicode* pDestBufPtr = pDestBuf;
  97     sal_Unicode* pDestBufEnd = pDestBuf + nDestChars;
  98
  99     while (nConverted < nSrcBytes)
 100     {
 101         if (pDestBufPtr == pDestBufEnd)
 102         {
 103             nInfo |= RTL_TEXTTOUNICODE_INFO_DESTBUFFERTOSMALL;
 104             break;
 105         }
 106
 107         sal_Unicode cChar = sal_Unicode();
 108         sal_uInt8 nIn = static_cast<sal_uInt8>(pSrcBuf[nConverted]);
 109         sal_uInt8 nNext = nConverted < nSrcBytes + 1 ? static_cast<sal_uInt8>(pSrcBuf[nConverted+1]) : 0;
 110         bool bNormal = true;
 111         bool bDouble = false;
 112         //halant + halant     E8 E8  -> halant + ZWNJ   094D 200C
 113         //halant + nukta    E8 E9   halant + ZWJ    094D 200D
 114         if (m_cPrevChar == 0xE8 && nIn == 0xE8)
 115         {
 116             cChar = 0x200C;
 117             bNormal = false;
 118         }
 119         else if (m_cPrevChar == 0xE8 && nIn == 0xE9)
 120         {
 121             cChar = 0x200D;
 122             bNormal = false;
 123         }
 124         else if (nNext == 0xE9)
 125         {
 126             bNormal = false;
 127             bDouble = true;
 128             switch(nIn)
 129             {
 130                 case 0xA1:
 131                     cChar = 0x0950;
 132                     break;
 133                 case 0xA6:
 134                     cChar = 0x090C;
 135                     break;
 136                 case 0xA7:
 137                     cChar = 0x0961;
 138                     break;
 139                 case 0xAA:
 140                     cChar = 0x0960;
 141                     break;
 142                 case 0xB3:
 143                     cChar = 0x0958;
 144                     break;
 145                 case 0xB4:
 146                     cChar = 0x0959;
 147                     break;
 148                 case 0xB5:
 149                     cChar = 0x095A;
 150                     break;
 151                 case 0xBA:
 152                     cChar = 0x095B;
 153                     break;
 154                 case 0xBF:
 155                     cChar = 0x095C;
 156                     break;
 157                 case 0xC0:
 158                     cChar = 0x095D;
 159                     break;
 160                 case 0xC9:
 161                     cChar = 0x095E;
 162                     break;
 163                 case 0xDB:
 164                     cChar = 0x0962;
 165                     break;
 166                 case 0xDC:
 167                     cChar = 0x0963;
 168                     break;
 169                 case 0xDF:
 170                     cChar = 0x0944;
 171                     break;
 172                 case 0xEA:
 173                     cChar = 0x093D;
 174                     break;
 175                 default:
 176                     bNormal = true;
 177                     bDouble = false;
 178                     break;
 179             };
 180         }
 181
 182         if (bNormal)
 183             cChar = IsciiDevanagariMap[nIn];
 184
 185         bool bUndefined = cChar == 0xffff;
 186
 187         if (bUndefined)
 188         {
 189             BadInputConversionAction eAction = handleBadInputTextToUnicodeConversion(
 190                         bUndefined, true, 0, nFlags, &pDestBufPtr, pDestBufEnd,
 191                         &nInfo);
 192             if (eAction == BAD_INPUT_CONTINUE)
 193                 continue;
 194             if (eAction == BAD_INPUT_STOP)
 195                 break;
 196             else if (eAction == BAD_INPUT_NO_OUTPUT)
 197             {
 198                 nInfo |= RTL_TEXTTOUNICODE_INFO_DESTBUFFERTOSMALL;
 199                 break;
 200             }
 201         }
 202         ++nConverted;
 203         if (bDouble)
 204             ++nConverted;
 205
 206         *pDestBufPtr++ = cChar;
 207         m_cPrevChar = bNormal ? nIn : 0;
 208     }
 209
 210     if (pInfo)
 211         *pInfo = nInfo;
 212     if (pSrcCvtBytes)
 213         *pSrcCvtBytes = nConverted;
 214
 215     return pDestBufPtr - pDestBuf;
 216 }
 217
 218 BmpUnicodeToSingleByteRange const unicodeToISCIIEncoding[] =
 219 {
 220     { 0x0000, 0x007F - 0x0000, 0x00 }, { 0x0901, 0x0903 - 0x0901, 0xA1 },
 221     { 0x0905, 0x090B - 0x0905, 0xA4 }, { 0x090D, 0x090D - 0x090D, 0xAE },
 222     { 0x090E, 0x0910 - 0x090E, 0xAB }, { 0x0911, 0x0911 - 0x0911, 0xB2 },
 223     { 0x0912, 0x0914 - 0x0912, 0xAF }, { 0x0915, 0x092F - 0x0915, 0xB3 },
 224     { 0x0930, 0x0939 - 0x0930, 0xCF }, { 0x093C, 0x093C - 0x093C, 0xE9 },
 225     { 0x093E, 0x0943 - 0x093E, 0xDA }, { 0x0945, 0x0945 - 0x0945, 0xE3 },
 226     { 0x0946, 0x0948 - 0x0946, 0xE0 }, { 0x0949, 0x0949 - 0x0949, 0xE7 },
 227     { 0x094A, 0x094C - 0x094A, 0xE4 }, { 0x094D, 0x094D - 0x094D, 0xE8 },
 228     { 0x095F, 0x095F - 0x095F, 0xCE }, { 0x0964, 0x0964 - 0x0964, 0xEA },
 229     { 0x0966, 0x096F - 0x0966, 0xF1 }
 230 };
 231
 232 sal_Size UnicodeToIsciiDevanagari::convert(sal_Unicode const* pSrcBuf, sal_Size nSrcChars,
 233     char* pDestBuf, sal_Size nDestBytes, sal_uInt32 nFlags,
 234     sal_uInt32 * pInfo, sal_Size* pSrcCvtChars)
 235 {
 236     size_t entries = SAL_N_ELEMENTS(unicodeToISCIIEncoding);
 237     BmpUnicodeToSingleByteRange const * ranges = unicodeToISCIIEncoding;
 238
 239     sal_Unicode cHighSurrogate = m_cHighSurrogate;
 240     sal_uInt32 nInfo = 0;
 241     sal_Size nConverted = 0;
 242     sal_Char* pDestBufPtr = pDestBuf;
 243     sal_Char* pDestBufEnd = pDestBuf + nDestBytes;
 244     for (; nConverted < nSrcChars; ++nConverted)
 245     {
 246         bool bUndefined = true;
 247         sal_uInt32 c = *pSrcBuf++;
 248         sal_Char cSpecialChar = 0;
 249         if (cHighSurrogate == 0)
 250         {
 251             if (ImplIsHighSurrogate(c))
 252             {
 253                 cHighSurrogate = static_cast< sal_Unicode >(c);
 254                 continue;
 255             }
 256         }
 257         else if (ImplIsLowSurrogate(c))
 258         {
 259             c = ImplCombineSurrogates(cHighSurrogate, c);
 260         }
 261         else
 262         {
 263             bUndefined = false;
 264             goto bad_input;
 265         }
 266         if (ImplIsLowSurrogate(c) || ImplIsNoncharacter(c))
 267         {
 268             bUndefined = false;
 269             goto bad_input;
 270         }
 271
 272         //halant + halant     E8 E8  -> halant + ZWNJ   094D 200C
 273         //halant + nukta    E8 E9   halant + ZWJ    094D 200D
 274         if (m_cPrevChar == 0x094D && c == 0x200C)
 275             cSpecialChar = '\xE8';
 276         else if (m_cPrevChar == 0x094D && c == 0x200D)
 277             cSpecialChar = '\xE9';
 278         if (cSpecialChar)
 279         {
 280             if (pDestBufEnd - pDestBufPtr < 1)
 281             {
 282                 nInfo |= RTL_UNICODETOTEXT_INFO_DESTBUFFERTOSMALL;
 283                 break;
 284             }
 285             *pDestBufPtr++ = cSpecialChar;
 286             m_cPrevChar = 0;
 287             goto done;
 288         }
 289         switch (c)
 290         {
 291             case 0x0950:
 292                 cSpecialChar = '\xA1';
 293                 break;
 294             case 0x090C:
 295                 cSpecialChar = '\xA6';
 296                 break;
 297             case 0x0961:
 298                 cSpecialChar = '\xA7';
 299                 break;
 300             case 0x0960:
 301                 cSpecialChar = '\xAA';
 302                 break;
 303             case 0x0958:
 304                 cSpecialChar = '\xB3';
 305                 break;
 306             case 0x0959:
 307                 cSpecialChar = '\xB4';
 308                 break;
 309             case 0x095A:
 310                 cSpecialChar = '\xB5';
 311                 break;
 312             case 0x095B:
 313                 cSpecialChar = '\xBA';
 314                 break;
 315             case 0x095C:
 316                 cSpecialChar = '\xBF';
 317                 break;
 318             case 0x095D:
 319                 cSpecialChar = '\xC0';
 320                 break;
 321             case 0x095E:
 322                 cSpecialChar = '\xC9';
 323                 break;
 324             case 0x0962:
 325                 cSpecialChar = '\xDB';
 326                 break;
 327             case 0x0963:
 328                 cSpecialChar = '\xDC';
 329                 break;
 330             case 0x0944:
 331                 cSpecialChar = '\xDF';
 332                 break;
 333             case 0x093D:
 334                 cSpecialChar = '\xEA';
 335                 break;
 336             default:
 337                 break;
 338         }
 339         if (cSpecialChar)
 340         {
 341             if (pDestBufEnd - pDestBufPtr < 2)
 342             {
 343                 nInfo |= RTL_UNICODETOTEXT_INFO_DESTBUFFERTOSMALL;
 344                 break;
 345             }
 346             *pDestBufPtr++ = cSpecialChar;
 347             *pDestBufPtr++ = '\xE9';
 348             m_cPrevChar = 0;
 349             goto done;
 350         }
 351
 352         // Linearly searching through the ranges if probably fastest, assuming
 353         // that most converted characters belong to the ASCII subset:
 354         for (size_t i = 0; i < entries; ++i)
 355         {
 356             if (c < ranges[i].unicode)
 357             {
 358                 break;
 359             }
 360             else if (c <= sal::static_int_cast< sal_uInt32 >(
 361                            ranges[i].unicode + ranges[i].range))
 362             {
 363                 if (pDestBufEnd - pDestBufPtr < 1)
 364                 {
 365                     goto no_output;
 366                 }
 367                 *pDestBufPtr++ = static_cast< sal_Char >(
 368                     ranges[i].byte + (c - ranges[i].unicode));
 369                 m_cPrevChar = c;
 370                 goto done;
 371             }
 372         }
 373         goto bad_input;
 374     done:
 375         cHighSurrogate = 0;
 376         continue;
 377     bad_input:
 378         switch (sal::detail::textenc::handleBadInputUnicodeToTextConversion(
 379                     bUndefined, c, nFlags, &pDestBufPtr, pDestBufEnd, &nInfo, 0,
 380                     0, 0))
 381         {
 382         case sal::detail::textenc::BAD_INPUT_STOP:
 383             cHighSurrogate = 0;
 384             break;
 385
 386         case sal::detail::textenc::BAD_INPUT_CONTINUE:
 387             cHighSurrogate = 0;
 388             continue;
 389
 390         case sal::detail::textenc::BAD_INPUT_NO_OUTPUT:
 391             goto no_output;
 392         }
 393         break;
 394     no_output:
 395         --pSrcBuf;
 396         nInfo |= RTL_UNICODETOTEXT_INFO_DESTBUFFERTOSMALL;
 397         break;
 398     }
 399
 400     if (cHighSurrogate != 0
 401         && ((nInfo
 402              & (RTL_UNICODETOTEXT_INFO_ERROR
 403                 | RTL_UNICODETOTEXT_INFO_DESTBUFFERTOSMALL))
 404             == 0))
 405     {
 406         if ((nFlags & RTL_UNICODETOTEXT_FLAGS_FLUSH) != 0)
 407         {
 408             nInfo |= RTL_UNICODETOTEXT_INFO_SRCBUFFERTOSMALL;
 409         }
 410         else
 411         {
 412             switch (sal::detail::textenc::handleBadInputUnicodeToTextConversion(
 413                         false, 0, nFlags, &pDestBufPtr, pDestBufEnd, &nInfo, 0,
 414                         0, 0))
 415             {
 416             case sal::detail::textenc::BAD_INPUT_STOP:
 417             case sal::detail::textenc::BAD_INPUT_CONTINUE:
 418                 cHighSurrogate = 0;
 419                 break;
 420
 421             case sal::detail::textenc::BAD_INPUT_NO_OUTPUT:
 422                 nInfo |= RTL_UNICODETOTEXT_INFO_DESTBUFFERTOSMALL;
 423                 break;
 424             }
 425         }
 426     }
 427     m_cHighSurrogate = cHighSurrogate;
 428     if (pInfo)
 429         *pInfo = nInfo;
 430     if (pSrcCvtChars)
 431         *pSrcCvtChars = nConverted;
 432
 433     return pDestBufPtr - pDestBuf;
 434 }
 435
 436 sal_Size ImplConvertIsciiDevanagariToUnicode(void const*,
 437     void* pContext, char const* pSrcBuf, sal_Size nSrcBytes,
 438     sal_Unicode* pDestBuf, sal_Size nDestChars, sal_uInt32 nFlags,
 439     sal_uInt32* pInfo, sal_Size* pSrcCvtBytes)
 440 {
 441     IsciiDevanagariToUnicode *pCtx =
 442         static_cast<IsciiDevanagariToUnicode*>(pContext);
 443     return pCtx->convert(pSrcBuf, nSrcBytes, pDestBuf, nDestChars, nFlags,
 444         pInfo, pSrcCvtBytes);
 445 }
 446
 447 sal_Size ImplConvertUnicodeToIsciiDevanagari(void const*,
 448     void * pContext, sal_Unicode const * pSrcBuf, sal_Size nSrcChars,
 449     char * pDestBuf, sal_Size nDestBytes, sal_uInt32 nFlags,
 450     sal_uInt32 * pInfo, sal_Size * pSrcCvtChars)
 451 {
 452     UnicodeToIsciiDevanagari *pCtx =
 453         static_cast<UnicodeToIsciiDevanagari*>(pContext);
 454     return pCtx->convert(pSrcBuf, nSrcChars,
 455         pDestBuf, nDestBytes, nFlags, pInfo, pSrcCvtChars);
 456 }
 457
 458 void *ImplCreateIsciiDevanagariToUnicodeContext()
 459 {
 460     return new IsciiDevanagariToUnicode;
 461 }
 462
 463 void ImplDestroyIsciiDevanagariToUnicodeContext(void * pContext)
 464 {
 465     IsciiDevanagariToUnicode *pCtx =
 466         static_cast<IsciiDevanagariToUnicode*>(pContext);
 467     delete pCtx;
 468 }
 469
 470 void ImplResetIsciiDevanagariToUnicodeContext(void * pContext)
 471 {
 472     IsciiDevanagariToUnicode *pCtx =
 473         static_cast<IsciiDevanagariToUnicode*>(pContext);
 474     pCtx->reset();
 475 }
 476
 477 void *ImplCreateUnicodeToIsciiDevanagariContext()
 478 {
 479     return new UnicodeToIsciiDevanagari;
 480 }
 481
 482 void ImplResetUnicodeToIsciiDevanagariContext(void * pContext)
 483 {
 484     UnicodeToIsciiDevanagari *pCtx =
 485         static_cast<UnicodeToIsciiDevanagari*>(pContext);
 486     pCtx->reset();
 487 }
 488
 489 void ImplDestroyUnicodeToIsciiDevanagariContext(void * pContext)
 490 {
 491     UnicodeToIsciiDevanagari *pCtx =
 492         static_cast<UnicodeToIsciiDevanagari*>(pContext);
 493     delete pCtx;
 494 }
 495
 496 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */