lib/iconv.c

   1 /*
   2  * Copyright (C) 1999-2008, 2011, 2016, 2018, 2020, 2022-2023 Free Software Foundation, Inc.
   3  * This file is part of the GNU LIBICONV Library.
   4  *
   5  * The GNU LIBICONV Library is free software; you can redistribute it
   6  * and/or modify it under the terms of the GNU Lesser General Public
   7  * License as published by the Free Software Foundation; either version 2.1
   8  * of the License, or (at your option) any later version.
   9  *
  10  * The GNU LIBICONV Library is distributed in the hope that it will be
  11  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * Lesser General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU Lesser General Public
  16  * License along with the GNU LIBICONV Library; see the file COPYING.LIB.
  17  * If not, see <https://www.gnu.org/licenses/>.
  18  */
  19
  20 #include <iconv.h>
  21
  22 #include <limits.h>
  23 #include <stdlib.h>
  24 #include <string.h>
  25 #include "config.h"
  26 #include "localcharset.h"
  27
  28 #ifdef __CYGWIN__
  29 #include <cygwin/version.h>
  30 #endif
  31
  32 #if ENABLE_EXTRA
  33 /*
  34  * Consider all system dependent encodings, for any system,
  35  * and the extra encodings.
  36  */
  37 #define USE_AIX
  38 #define USE_OSF1
  39 #define USE_DOS
  40 #define USE_ZOS
  41 #define USE_EXTRA
  42 #else
  43 /*
  44  * Consider those system dependent encodings that are needed for the
  45  * current system.
  46  */
  47 #ifdef _AIX
  48 #define USE_AIX
  49 #endif
  50 #if defined(__osf__) || defined(VMS)
  51 #define USE_OSF1
  52 #endif
  53 #if defined(__DJGPP__) || (defined(_WIN32) && (defined(_MSC_VER) || defined(__MINGW32__)))
  54 #define USE_DOS
  55 #endif
  56 /* Enable the EBCDIC encodings not only on z/OS but also on Linux/s390, for
  57    easier interoperability between z/OS and Linux/s390.  */
  58 #if defined(__MVS__) || (defined(__linux__) && (defined(__s390__) || defined(__s390x__)))
  59 #define USE_ZOS
  60 #endif
  61 #endif
  62
  63 /*
  64  * Data type for general conversion loop.
  65  */
  66 struct loop_funcs {
  67   size_t (*loop_convert) (iconv_t icd,
  68                           const char* * inbuf, size_t *inbytesleft,
  69                           char* * outbuf, size_t *outbytesleft);
  70   size_t (*loop_reset) (iconv_t icd,
  71                         char* * outbuf, size_t *outbytesleft);
  72 };
  73
  74 /*
  75  * Converters.
  76  */
  77 #include "converters.h"
  78
  79 /*
  80  * Transliteration tables.
  81  */
  82 #include "cjk_variants.h"
  83 #include "translit.h"
  84
  85 /*
  86  * Table of all supported encodings.
  87  */
  88 struct encoding {
  89   struct mbtowc_funcs ifuncs; /* conversion multibyte -> unicode */
  90   struct wctomb_funcs ofuncs; /* conversion unicode -> multibyte */
  91   int oflags;                 /* flags for unicode -> multibyte conversion */
  92 };
  93 #define DEFALIAS(xxx_alias,xxx) /* nothing */
  94 enum {
  95 #define DEFENCODING(xxx_names,xxx,xxx_ifuncs1,xxx_ifuncs2,xxx_ofuncs1,xxx_ofuncs2) \
  96   ei_##xxx ,
  97 #include "encodings.def"
  98 #ifdef USE_AIX
  99 # include "encodings_aix.def"
 100 #endif
 101 #ifdef USE_OSF1
 102 # include "encodings_osf1.def"
 103 #endif
 104 #ifdef USE_DOS
 105 # include "encodings_dos.def"
 106 #endif
 107 #ifdef USE_ZOS
 108 # include "encodings_zos.def"
 109 #endif
 110 #ifdef USE_EXTRA
 111 # include "encodings_extra.def"
 112 #endif
 113 #include "encodings_local.def"
 114 #undef DEFENCODING
 115 ei_for_broken_compilers_that_dont_like_trailing_commas
 116 };
 117 #include "flags.h"
 118 static struct encoding const all_encodings[] = {
 119 #define DEFENCODING(xxx_names,xxx,xxx_ifuncs1,xxx_ifuncs2,xxx_ofuncs1,xxx_ofuncs2) \
 120   { xxx_ifuncs1,xxx_ifuncs2, xxx_ofuncs1,xxx_ofuncs2, ei_##xxx##_oflags },
 121 #include "encodings.def"
 122 #ifdef USE_AIX
 123 # include "encodings_aix.def"
 124 #endif
 125 #ifdef USE_OSF1
 126 # include "encodings_osf1.def"
 127 #endif
 128 #ifdef USE_DOS
 129 # include "encodings_dos.def"
 130 #endif
 131 #ifdef USE_ZOS
 132 # include "encodings_zos.def"
 133 #endif
 134 #ifdef USE_EXTRA
 135 # include "encodings_extra.def"
 136 #endif
 137 #undef DEFENCODING
 138 #define DEFENCODING(xxx_names,xxx,xxx_ifuncs1,xxx_ifuncs2,xxx_ofuncs1,xxx_ofuncs2) \
 139   { xxx_ifuncs1,xxx_ifuncs2, xxx_ofuncs1,xxx_ofuncs2, 0 },
 140 #include "encodings_local.def"
 141 #undef DEFENCODING
 142 };
 143 #undef DEFALIAS
 144
 145 /*
 146  * Conversion loops.
 147  */
 148 #include "loops.h"
 149
 150 /*
 151  * Alias lookup function.
 152  * Defines
 153  *   struct alias { int name; unsigned int encoding_index; };
 154  *   const struct alias * aliases_lookup (const char *str, unsigned int len);
 155  *   #define MAX_WORD_LENGTH ...
 156  */
 157 #if defined _AIX
 158 # include "aliases_sysaix.h"
 159 #elif defined hpux || defined __hpux
 160 # include "aliases_syshpux.h"
 161 #elif defined __osf__
 162 # include "aliases_sysosf1.h"
 163 #elif defined __sun
 164 # include "aliases_syssolaris.h"
 165 #else
 166 # include "aliases.h"
 167 #endif
 168
 169 /*
 170  * System dependent alias lookup function.
 171  * Defines
 172  *   const struct alias * aliases2_lookup (const char *str);
 173  */
 174 #if defined(USE_AIX) || defined(USE_OSF1) || defined(USE_DOS) || defined(USE_ZOS) || defined(USE_EXTRA) /* || ... */
 175 struct stringpool2_t {
 176 #define S(tag,name,encoding_index) char stringpool_##tag[sizeof(name)];
 177 #include "aliases2.h"
 178 #undef S
 179 };
 180 static const struct stringpool2_t stringpool2_contents = {
 181 #define S(tag,name,encoding_index) name,
 182 #include "aliases2.h"
 183 #undef S
 184 };
 185 #define stringpool2 ((const char *) &stringpool2_contents)
 186 static const struct alias sysdep_aliases[] = {
 187 #define S(tag,name,encoding_index) { (int)(long)&((struct stringpool2_t *)0)->stringpool_##tag, encoding_index },
 188 #include "aliases2.h"
 189 #undef S
 190 };
 191 #ifdef __GNUC__
 192 __inline
 193 #else
 194 #ifdef __cplusplus
 195 inline
 196 #endif
 197 #endif
 198 static const struct alias *
 199 aliases2_lookup (register const char *str)
 200 {
 201   const struct alias * ptr;
 202   unsigned int count;
 203   for (ptr = sysdep_aliases, count = sizeof(sysdep_aliases)/sizeof(sysdep_aliases[0]); count > 0; ptr++, count--)
 204     if (!strcmp(str, stringpool2 + ptr->name))
 205       return ptr;
 206   return NULL;
 207 }
 208 #else
 209 #define aliases2_lookup(str)  NULL
 210 #define stringpool2  NULL
 211 #endif
 212
 213 #if 0
 214 /* Like !strcasecmp, except that the both strings can be assumed to be ASCII
 215    and the first string can be assumed to be in uppercase. */
 216 static int strequal (const char* str1, const char* str2)
 217 {
 218   unsigned char c1;
 219   unsigned char c2;
 220   for (;;) {
 221     c1 = * (unsigned char *) str1++;
 222     c2 = * (unsigned char *) str2++;
 223     if (c1 == 0)
 224       break;
 225     if (c2 >= 'a' && c2 <= 'z')
 226       c2 -= 'a'-'A';
 227     if (c1 != c2)
 228       break;
 229   }
 230   return (c1 == c2);
 231 }
 232 #endif
 233
 234 iconv_t iconv_open (const char* tocode, const char* fromcode)
 235 {
 236   struct conv_struct * cd;
 237   unsigned int from_index;
 238   int from_wchar;
 239   unsigned int from_surface;
 240   unsigned int to_index;
 241   int to_wchar;
 242   unsigned int to_surface;
 243   int transliterate;
 244   int discard_ilseq;
 245
 246 #include "iconv_open1.h"
 247
 248   cd = (struct conv_struct *) malloc(from_wchar != to_wchar
 249                                      ? sizeof(struct wchar_conv_struct)
 250                                      : sizeof(struct conv_struct));
 251   if (cd == NULL) {
 252     errno = ENOMEM;
 253     return (iconv_t)(-1);
 254   }
 255
 256 #include "iconv_open2.h"
 257
 258   return (iconv_t)cd;
 259 invalid:
 260   errno = EINVAL;
 261   return (iconv_t)(-1);
 262 }
 263
 264 size_t iconv (iconv_t icd,
 265               ICONV_CONST char* * inbuf, size_t *inbytesleft,
 266               char* * outbuf, size_t *outbytesleft)
 267 {
 268   conv_t cd = (conv_t) icd;
 269   if (inbuf == NULL || *inbuf == NULL)
 270     return cd->lfuncs.loop_reset(icd,outbuf,outbytesleft);
 271   else
 272     return cd->lfuncs.loop_convert(icd,
 273                                    (const char* *)inbuf,inbytesleft,
 274                                    outbuf,outbytesleft);
 275 }
 276
 277 int iconv_close (iconv_t icd)
 278 {
 279   conv_t cd = (conv_t) icd;
 280   free(cd);
 281   return 0;
 282 }
 283
 284 #ifndef LIBICONV_PLUG
 285
 286 /*
 287  * Verify that a 'struct conv_struct' and a 'struct wchar_conv_struct' each
 288  * fit in an iconv_allocation_t.
 289  * If this verification fails, iconv_allocation_t must be made larger and
 290  * the major version in LIBICONV_VERSION_INFO must be bumped.
 291  * Currently 'struct conv_struct' has 23 integer/pointer fields, and
 292  * 'struct wchar_conv_struct' additionally has an 'mbstate_t' field.
 293  */
 294 typedef int verify_size_1[2 * (sizeof (struct conv_struct) <= sizeof (iconv_allocation_t)) - 1];
 295 typedef int verify_size_2[2 * (sizeof (struct wchar_conv_struct) <= sizeof (iconv_allocation_t)) - 1];
 296
 297 int iconv_open_into (const char* tocode, const char* fromcode,
 298                      iconv_allocation_t* resultp)
 299 {
 300   struct conv_struct * cd;
 301   unsigned int from_index;
 302   int from_wchar;
 303   unsigned int from_surface;
 304   unsigned int to_index;
 305   int to_wchar;
 306   unsigned int to_surface;
 307   int transliterate;
 308   int discard_ilseq;
 309
 310 #include "iconv_open1.h"
 311
 312   cd = (struct conv_struct *) resultp;
 313
 314 #include "iconv_open2.h"
 315
 316   return 0;
 317 invalid:
 318   errno = EINVAL;
 319   return -1;
 320 }
 321
 322 /* Bit mask of all valid surfaces. */
 323 #define ALL_SURFACES (ICONV_SURFACE_EBCDIC_ZOS_UNIX)
 324
 325 int iconvctl (iconv_t icd, int request, void* argument)
 326 {
 327   conv_t cd = (conv_t) icd;
 328   switch (request) {
 329     case ICONV_TRIVIALP:
 330       *(int *)argument =
 331         ((cd->lfuncs.loop_convert == unicode_loop_convert
 332           && cd->iindex == cd->oindex
 333           && cd->isurface == cd->osurface)
 334          || cd->lfuncs.loop_convert == wchar_id_loop_convert
 335          ? 1 : 0);
 336       return 0;
 337     case ICONV_GET_TRANSLITERATE:
 338       *(int *)argument = cd->transliterate;
 339       return 0;
 340     case ICONV_SET_TRANSLITERATE:
 341       cd->transliterate = (*(const int *)argument ? 1 : 0);
 342       return 0;
 343     case ICONV_GET_DISCARD_ILSEQ:
 344       *(int *)argument = cd->discard_ilseq;
 345       return 0;
 346     case ICONV_SET_DISCARD_ILSEQ:
 347       cd->discard_ilseq = (*(const int *)argument ? 1 : 0);
 348       return 0;
 349     case ICONV_SET_HOOKS:
 350       if (argument != NULL) {
 351         cd->hooks = *(const struct iconv_hooks *)argument;
 352       } else {
 353         cd->hooks.uc_hook = NULL;
 354         cd->hooks.wc_hook = NULL;
 355         cd->hooks.data = NULL;
 356       }
 357       return 0;
 358     case ICONV_SET_FALLBACKS:
 359       if (argument != NULL) {
 360         cd->fallbacks = *(const struct iconv_fallbacks *)argument;
 361       } else {
 362         cd->fallbacks.mb_to_uc_fallback = NULL;
 363         cd->fallbacks.uc_to_mb_fallback = NULL;
 364         cd->fallbacks.mb_to_wc_fallback = NULL;
 365         cd->fallbacks.wc_to_mb_fallback = NULL;
 366         cd->fallbacks.data = NULL;
 367       }
 368       return 0;
 369     case ICONV_GET_FROM_SURFACE:
 370       *(unsigned int *)argument = cd->isurface;
 371       return 0;
 372     case ICONV_SET_FROM_SURFACE:
 373       if ((*(const unsigned int *)argument & ~ALL_SURFACES) == 0) {
 374         cd->isurface = *(const unsigned int *)argument;
 375         return 0;
 376       } else {
 377         errno = EINVAL;
 378         return -1;
 379       }
 380     case ICONV_GET_TO_SURFACE:
 381       *(unsigned int *)argument = cd->osurface;
 382       return 0;
 383     case ICONV_SET_TO_SURFACE:
 384       if ((*(const unsigned int *)argument & ~ALL_SURFACES) == 0) {
 385         cd->osurface = *(const unsigned int *)argument;
 386         return 0;
 387       } else {
 388         errno = EINVAL;
 389         return -1;
 390       }
 391     default:
 392       errno = EINVAL;
 393       return -1;
 394   }
 395 }
 396
 397 /* An alias after its name has been converted from 'int' to 'const char*'. */
 398 struct nalias { const char* name; unsigned int encoding_index; };
 399
 400 static int compare_by_index (const void * arg1, const void * arg2)
 401 {
 402   const struct nalias * alias1 = (const struct nalias *) arg1;
 403   const struct nalias * alias2 = (const struct nalias *) arg2;
 404   return (int)alias1->encoding_index - (int)alias2->encoding_index;
 405 }
 406
 407 static int compare_by_name (const void * arg1, const void * arg2)
 408 {
 409   const char * name1 = *(const char * const *)arg1;
 410   const char * name2 = *(const char * const *)arg2;
 411   /* Compare alphabetically, but put "CS" names at the end. */
 412   int sign = strcmp(name1,name2);
 413   if (sign != 0) {
 414     sign = ((name1[0]=='C' && name1[1]=='S') - (name2[0]=='C' && name2[1]=='S'))
 415            * 4 + (sign >= 0 ? 1 : -1);
 416   }
 417   return sign;
 418 }
 419
 420 void iconvlist (int (*do_one) (unsigned int namescount,
 421                                const char * const * names,
 422                                void* data),
 423                 void* data)
 424 {
 425 #define aliascount1  sizeof(aliases)/sizeof(aliases[0])
 426 #ifndef aliases2_lookup
 427 #define aliascount2  sizeof(sysdep_aliases)/sizeof(sysdep_aliases[0])
 428 #else
 429 #define aliascount2  0
 430 #endif
 431 #define aliascount  (aliascount1+aliascount2)
 432   struct nalias aliasbuf[aliascount];
 433   const char * namesbuf[aliascount];
 434   size_t num_aliases;
 435   {
 436     /* Put all existing aliases into a buffer. */
 437     size_t i;
 438     size_t j;
 439     j = 0;
 440     for (i = 0; i < aliascount1; i++) {
 441       const struct alias * p = &aliases[i];
 442       if (p->name >= 0
 443           && p->encoding_index != ei_local_char
 444           && p->encoding_index != ei_local_wchar_t) {
 445         aliasbuf[j].name = stringpool + p->name;
 446         aliasbuf[j].encoding_index = p->encoding_index;
 447         j++;
 448       }
 449     }
 450 #ifndef aliases2_lookup
 451     for (i = 0; i < aliascount2; i++) {
 452       aliasbuf[j].name = stringpool2 + sysdep_aliases[i].name;
 453       aliasbuf[j].encoding_index = sysdep_aliases[i].encoding_index;
 454       j++;
 455     }
 456 #endif
 457     num_aliases = j;
 458   }
 459   /* Sort by encoding_index. */
 460   if (num_aliases > 1)
 461     qsort(aliasbuf, num_aliases, sizeof(struct nalias), compare_by_index);
 462   {
 463     /* Process all aliases with the same encoding_index together. */
 464     size_t j;
 465     j = 0;
 466     while (j < num_aliases) {
 467       unsigned int ei = aliasbuf[j].encoding_index;
 468       size_t i = 0;
 469       do
 470         namesbuf[i++] = aliasbuf[j++].name;
 471       while (j < num_aliases && aliasbuf[j].encoding_index == ei);
 472       if (i > 1)
 473         qsort(namesbuf, i, sizeof(const char *), compare_by_name);
 474       /* Call the callback. */
 475       if (do_one(i,namesbuf,data))
 476         break;
 477     }
 478   }
 479 #undef aliascount
 480 #undef aliascount2
 481 #undef aliascount1
 482 }
 483
 484 /*
 485  * Table of canonical names of encodings.
 486  * Instead of strings, it contains offsets into stringpool and stringpool2.
 487  */
 488 static const unsigned short all_canonical[] = {
 489 #if defined _AIX
 490 # include "canonical_sysaix.h"
 491 #elif defined hpux || defined __hpux
 492 # include "canonical_syshpux.h"
 493 #elif defined __osf__
 494 # include "canonical_sysosf1.h"
 495 #elif defined __sun
 496 # include "canonical_syssolaris.h"
 497 #else
 498 # include "canonical.h"
 499 #endif
 500 #ifdef USE_AIX
 501 # if defined _AIX
 502 #  include "canonical_aix_sysaix.h"
 503 # else
 504 #  include "canonical_aix.h"
 505 # endif
 506 #endif
 507 #ifdef USE_OSF1
 508 # if defined __osf__
 509 #  include "canonical_osf1_sysosf1.h"
 510 # else
 511 #  include "canonical_osf1.h"
 512 # endif
 513 #endif
 514 #ifdef USE_DOS
 515 # include "canonical_dos.h"
 516 #endif
 517 #ifdef USE_ZOS
 518 # include "canonical_zos.h"
 519 #endif
 520 #ifdef USE_EXTRA
 521 # include "canonical_extra.h"
 522 #endif
 523 #if defined _AIX
 524 # include "canonical_local_sysaix.h"
 525 #elif defined hpux || defined __hpux
 526 # include "canonical_local_syshpux.h"
 527 #elif defined __osf__
 528 # include "canonical_local_sysosf1.h"
 529 #elif defined __sun
 530 # include "canonical_local_syssolaris.h"
 531 #else
 532 # include "canonical_local.h"
 533 #endif
 534 };
 535
 536 const char * iconv_canonicalize (const char * name)
 537 {
 538   const char* code;
 539   char buf[MAX_WORD_LENGTH+10+1];
 540   const char* cp;
 541   char* bp;
 542   const struct alias * ap;
 543   unsigned int count;
 544   unsigned int index;
 545   const char* pool;
 546
 547   /* Before calling aliases_lookup, convert the input string to upper case,
 548    * and check whether it's entirely ASCII (we call gperf with option "-7"
 549    * to achieve a smaller table) and non-empty. If it's not entirely ASCII,
 550    * or if it's too long, it is not a valid encoding name.
 551    */
 552   for (code = name;;) {
 553     /* Search code in the table. */
 554     for (cp = code, bp = buf, count = MAX_WORD_LENGTH+10+1; ; cp++, bp++) {
 555       unsigned char c = (unsigned char) *cp;
 556       if (c >= 0x80)
 557         goto invalid;
 558       if (c >= 'a' && c <= 'z')
 559         c -= 'a'-'A';
 560       *bp = c;
 561       if (c == '\0')
 562         break;
 563       if (--count == 0)
 564         goto invalid;
 565     }
 566     for (;;) {
 567       if (bp-buf >= 10 && memcmp(bp-10,"//TRANSLIT",10)==0) {
 568         bp -= 10;
 569         *bp = '\0';
 570         continue;
 571       }
 572       if (bp-buf >= 8 && memcmp(bp-8,"//IGNORE",8)==0) {
 573         bp -= 8;
 574         *bp = '\0';
 575         continue;
 576       }
 577       break;
 578     }
 579     if (buf[0] == '\0') {
 580       code = locale_charset();
 581       /* Avoid an endless loop that could occur when using an older version
 582          of localcharset.c. */
 583       if (code[0] == '\0')
 584         goto invalid;
 585       continue;
 586     }
 587     pool = stringpool;
 588     ap = aliases_lookup(buf,bp-buf);
 589     if (ap == NULL) {
 590       pool = stringpool2;
 591       ap = aliases2_lookup(buf);
 592       if (ap == NULL)
 593         goto invalid;
 594     }
 595     if (ap->encoding_index == ei_local_char) {
 596       code = locale_charset();
 597       /* Avoid an endless loop that could occur when using an older version
 598          of localcharset.c. */
 599       if (code[0] == '\0')
 600         goto invalid;
 601       continue;
 602     }
 603     if (ap->encoding_index == ei_local_wchar_t) {
 604       /* On systems which define __STDC_ISO_10646__, wchar_t is Unicode.
 605          This is also the case on native Woe32 systems and Cygwin >= 1.7, where
 606          we know that it is UTF-16.  */
 607 #if (defined _WIN32 && !defined __CYGWIN__) || (defined __CYGWIN__ && CYGWIN_VERSION_DLL_MAJOR >= 1007)
 608       if (sizeof(wchar_t) == 4) {
 609         index = ei_ucs4internal;
 610         break;
 611       }
 612       if (sizeof(wchar_t) == 2) {
 613 # if WORDS_LITTLEENDIAN
 614         index = ei_utf16le;
 615 # else
 616         index = ei_utf16be;
 617 # endif
 618         break;
 619       }
 620 #elif __STDC_ISO_10646__
 621       if (sizeof(wchar_t) == 4) {
 622         index = ei_ucs4internal;
 623         break;
 624       }
 625       if (sizeof(wchar_t) == 2) {
 626         index = ei_ucs2internal;
 627         break;
 628       }
 629       if (sizeof(wchar_t) == 1) {
 630         index = ei_iso8859_1;
 631         break;
 632       }
 633 #endif
 634     }
 635     index = ap->encoding_index;
 636     break;
 637   }
 638   return all_canonical[index] + pool;
 639  invalid:
 640   return name;
 641 }
 642
 643 int _libiconv_version = _LIBICONV_VERSION;
 644
 645 #if defined __FreeBSD__ && !defined __gnu_freebsd__
 646 /* GNU libiconv is the native FreeBSD iconv implementation since 2002.
 647    It wants to define the symbols 'iconv_open', 'iconv', 'iconv_close'.  */
 648 #define strong_alias(name, aliasname) _strong_alias(name, aliasname)
 649 #define _strong_alias(name, aliasname) \
 650   extern __typeof (name) aliasname __attribute__ ((alias (#name)));
 651 #undef iconv_open
 652 #undef iconv
 653 #undef iconv_close
 654 strong_alias (libiconv_open, iconv_open)
 655 strong_alias (libiconv, iconv)
 656 strong_alias (libiconv_close, iconv_close)
 657 #endif
 658
 659 #endif