lib/converters.h

   1 /*
   2  * Copyright (C) 1999-2002, 2004-2011, 2016, 2022-2023 Free Software Foundation, Inc.
   3  * This file is part of the GNU LIBICONV Library.
   4  *
   5  * The GNU LIBICONV Library is free software; you can redistribute it
   6  * and/or modify it under the terms of the GNU Lesser General Public
   7  * License as published by the Free Software Foundation; either version 2.1
   8  * of the License, or (at your option) any later version.
   9  *
  10  * The GNU LIBICONV Library is distributed in the hope that it will be
  11  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * Lesser General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU Lesser General Public
  16  * License along with the GNU LIBICONV Library; see the file COPYING.LIB.
  17  * If not, see <https://www.gnu.org/licenses/>.
  18  */
  19
  20 /* This file defines all the converters. */
  21
  22
  23 /* Our own notion of wide character, as UCS-4, according to ISO-10646-1. */
  24 typedef unsigned int ucs4_t;
  25
  26 /* State used by a conversion. 0 denotes the initial state. */
  27 typedef unsigned int state_t;
  28
  29 /* iconv_t is an opaque type. This is the real iconv_t type. */
  30 typedef struct conv_struct * conv_t;
  31
  32 /*
  33  * Data type for conversion multibyte -> unicode
  34  */
  35 struct mbtowc_funcs {
  36   int (*xxx_mbtowc) (conv_t conv, ucs4_t *pwc, unsigned char const *s, size_t n);
  37   /*
  38    * int xxx_mbtowc (conv_t conv, ucs4_t *pwc, unsigned char const *s, size_t n)
  39    * converts the byte sequence starting at s to a wide character. Up to n bytes
  40    * are available at s. n is >= 1.
  41    * Result is number of bytes consumed (if a wide character was read),
  42    * or -1 if invalid, or -2 if n too small,
  43    * or RET_SHIFT_ILSEQ(number of bytes consumed) if invalid input after a shift
  44    * sequence was read,
  45    * or RET_TOOFEW(number of bytes consumed) if only a shift sequence was read.
  46    */
  47   int (*xxx_flushwc) (conv_t conv, ucs4_t *pwc);
  48   /*
  49    * int xxx_flushwc (conv_t conv, ucs4_t *pwc)
  50    * returns to the initial state and stores the pending wide character, if any.
  51    * Result is 1 (if a wide character was read) or 0 if none was pending.
  52    */
  53 };
  54
  55 /* Return code if invalid input after a shift sequence of n bytes was read.
  56    (xxx_mbtowc) */
  57 #define RET_SHIFT_ILSEQ(n)  (-1-2*(n))
  58 /* Return code if invalid. (xxx_mbtowc) */
  59 #define RET_ILSEQ           RET_SHIFT_ILSEQ(0)
  60 /* Return code if only a shift sequence of n bytes was read. (xxx_mbtowc) */
  61 #define RET_TOOFEW(n)       (-2-2*(n))
  62 /* Retrieve the n from the encoded RET_... value. */
  63 #define DECODE_SHIFT_ILSEQ(r)  ((unsigned int)(RET_SHIFT_ILSEQ(0) - (r)) / 2)
  64 #define DECODE_TOOFEW(r)       ((unsigned int)(RET_TOOFEW(0) - (r)) / 2)
  65 /* Maximum value of n that may be used as argument to RET_SHIFT_ILSEQ or RET_TOOFEW. */
  66 #define RET_COUNT_MAX       ((INT_MAX / 2) - 1)
  67
  68 /*
  69  * Data type for conversion unicode -> multibyte
  70  */
  71 struct wctomb_funcs {
  72   int (*xxx_wctomb) (conv_t conv, unsigned char *r, ucs4_t wc, size_t n);
  73   /*
  74    * int xxx_wctomb (conv_t conv, unsigned char *r, ucs4_t wc, size_t n)
  75    * converts the wide character wc to the character set xxx, and stores the
  76    * result beginning at r. Up to n bytes may be written at r. n is >= 1.
  77    * Result is number of bytes written, or -1 if invalid, or -2 if n too small.
  78    */
  79   int (*xxx_reset) (conv_t conv, unsigned char *r, size_t n);
  80   /*
  81    * int xxx_reset (conv_t conv, unsigned char *r, size_t n)
  82    * stores a shift sequences returning to the initial state beginning at r.
  83    * Up to n bytes may be written at r. n is >= 0.
  84    * Result is number of bytes written, or -2 if n too small.
  85    */
  86 };
  87
  88 /* Return code if invalid. (xxx_wctomb) */
  89 #define RET_ILUNI      -1
  90 /* Return code if output buffer is too small. (xxx_wctomb, xxx_reset) */
  91 #define RET_TOOSMALL   -2
  92
  93 /*
  94  * Contents of a conversion descriptor.
  95  */
  96 struct conv_struct {
  97   struct loop_funcs lfuncs;
  98   /* Input (conversion multibyte -> unicode) */
  99   int iindex;
 100   struct mbtowc_funcs ifuncs;
 101   unsigned int isurface;
 102   state_t istate;
 103   /* Output (conversion unicode -> multibyte) */
 104   int oindex;
 105   struct wctomb_funcs ofuncs;
 106   int oflags;
 107   unsigned int osurface;
 108   state_t ostate;
 109   /* Operation flags */
 110   int transliterate;
 111   int discard_ilseq;
 112   #ifndef LIBICONV_PLUG
 113   struct iconv_fallbacks fallbacks;
 114   struct iconv_hooks hooks;
 115   #endif
 116 };
 117
 118 /*
 119  * Include all the converters.
 120  */
 121
 122 #include "ascii.h"
 123
 124 /* General multi-byte encodings */
 125 #include "utf8.h"
 126 #include "ucs2.h"
 127 #include "ucs2be.h"
 128 #include "ucs2le.h"
 129 #include "ucs4.h"
 130 #include "ucs4be.h"
 131 #include "ucs4le.h"
 132 #include "utf16.h"
 133 #include "utf16be.h"
 134 #include "utf16le.h"
 135 #include "utf32.h"
 136 #include "utf32be.h"
 137 #include "utf32le.h"
 138 #include "utf7.h"
 139 #include "ucs2internal.h"
 140 #include "ucs2swapped.h"
 141 #include "ucs4internal.h"
 142 #include "ucs4swapped.h"
 143 #include "c99.h"
 144 #include "java.h"
 145
 146 /* 8-bit encodings */
 147 #include "iso8859_1.h"
 148 #include "iso8859_2.h"
 149 #include "iso8859_3.h"
 150 #include "iso8859_4.h"
 151 #include "iso8859_5.h"
 152 #include "iso8859_6.h"
 153 #include "iso8859_7.h"
 154 #include "iso8859_8.h"
 155 #include "iso8859_9.h"
 156 #include "iso8859_10.h"
 157 #include "iso8859_11.h"
 158 #include "iso8859_13.h"
 159 #include "iso8859_14.h"
 160 #include "iso8859_15.h"
 161 #include "iso8859_16.h"
 162 #include "koi8_r.h"
 163 #include "koi8_u.h"
 164 #include "koi8_ru.h"
 165 #include "cp1250.h"
 166 #include "cp1251.h"
 167 #include "cp1252.h"
 168 #include "cp1253.h"
 169 #include "cp1254.h"
 170 #include "cp1255.h"
 171 #include "cp1256.h"
 172 #include "cp1257.h"
 173 #include "cp1258.h"
 174 #include "cp850.h"
 175 #include "cp862.h"
 176 #include "cp866.h"
 177 #include "cp1131.h"
 178 #include "mac_roman.h"
 179 #include "mac_centraleurope.h"
 180 #include "mac_iceland.h"
 181 #include "mac_croatian.h"
 182 #include "mac_romania.h"
 183 #include "mac_cyrillic.h"
 184 #include "mac_ukraine.h"
 185 #include "mac_greek.h"
 186 #include "mac_turkish.h"
 187 #include "mac_hebrew.h"
 188 #include "mac_arabic.h"
 189 #include "mac_thai.h"
 190 #include "hp_roman8.h"
 191 #include "nextstep.h"
 192 #include "armscii_8.h"
 193 #include "georgian_academy.h"
 194 #include "georgian_ps.h"
 195 #include "koi8_t.h"
 196 #include "pt154.h"
 197 #include "rk1048.h"
 198 #include "mulelao.h"
 199 #include "cp1133.h"
 200 #include "tis620.h"
 201 #include "cp874.h"
 202 #include "viscii.h"
 203 #include "tcvn.h"
 204
 205 /* CJK character sets [CCS = coded character set] [CJKV.INF chapter 3] */
 206
 207 typedef struct {
 208   unsigned short indx; /* index into big table */
 209   unsigned short used; /* bitmask of used entries */
 210 } Summary16;
 211
 212 #include "iso646_jp.h"
 213 #include "jisx0201.h"
 214 #include "jisx0208.h"
 215 #include "jisx0212.h"
 216
 217 #include "iso646_cn.h"
 218 #include "gb2312.h"
 219 #include "isoir165.h"
 220 /*#include "gb12345.h"*/
 221 #include "gbk.h"
 222 #include "cns11643.h"
 223 #include "big5.h"
 224
 225 #include "ksc5601.h"
 226 #include "johab_hangul.h"
 227
 228 /* CJK encodings [CES = character encoding scheme] [CJKV.INF chapter 4] */
 229
 230 #include "euc_jp.h"
 231 #include "sjis.h"
 232 #include "cp932.h"
 233 #include "iso2022_jp.h"
 234 #include "iso2022_jp1.h"
 235 #include "iso2022_jp2.h"
 236 #include "iso2022_jpms.h"
 237
 238 #include "euc_cn.h"
 239 #include "ces_gbk.h"
 240 #include "cp936.h"
 241 #include "gb18030_2005.h"
 242 #include "gb18030_2022.h"
 243 #include "iso2022_cn.h"
 244 #include "iso2022_cnext.h"
 245 #include "hz.h"
 246 #include "euc_tw.h"
 247 #include "ces_big5.h"
 248 #include "cp950.h"
 249 #include "big5hkscs1999.h"
 250 #include "big5hkscs2001.h"
 251 #include "big5hkscs2004.h"
 252 #include "big5hkscs2008.h"
 253
 254 #include "euc_kr.h"
 255 #include "cp949.h"
 256 #include "johab.h"
 257 #include "iso2022_kr.h"
 258
 259 /* Encodings used by system dependent locales. */
 260
 261 #ifdef USE_AIX
 262 #include "cp856.h"
 263 #include "cp922.h"
 264 #include "cp943.h"
 265 #include "cp1046.h"
 266 #include "cp1124.h"
 267 #include "cp1129.h"
 268 #include "cp1161.h"
 269 #include "cp1162.h"
 270 #include "cp1163.h"
 271 #endif
 272
 273 #ifdef USE_OSF1
 274 #include "dec_kanji.h"
 275 #include "dec_hanyu.h"
 276 #endif
 277
 278 #ifdef USE_DOS
 279 #include "cp437.h"
 280 #include "cp737.h"
 281 #include "cp775.h"
 282 #include "cp852.h"
 283 #include "cp853.h"
 284 #include "cp855.h"
 285 #include "cp857.h"
 286 #include "cp858.h"
 287 #include "cp860.h"
 288 #include "cp861.h"
 289 #include "cp863.h"
 290 #include "cp864.h"
 291 #include "cp865.h"
 292 #include "cp869.h"
 293 #include "cp1125.h"
 294 #endif
 295
 296 #ifdef USE_ZOS
 297
 298 #define DEDUPLICATE_TABLES 1
 299
 300 /* Swaps the values 0x15 and 0x25.
 301    Both gcc and clang compile this expression to something that involves as few
 302    conditional branching instructions as possible. */
 303 #define swap_x15_x25_a(x) ((x) == 0x15 ? 0x25 : (x) == 0x25 ? 0x15 : (x))
 304 #define swap_x15_x25_b(x) ((x) ^ ((x) == 0x15 || (x) == 0x25 ? 0x30 : 0))
 305 #define swap_x15_x25_c(x) ((x) ^ ((((x) - 0x15) & ~0x10) == 0 ? 0x30 : 0))
 306 /* Number of conditional branches (with "gcc -O2", as of 2023):
 307                    a    b    c
 308                  ---------------
 309    aarch64         1    0    0
 310    alpha           0    0    0
 311    arm             1    0    0
 312    hppa            1    1    1
 313    i686            1    0    0
 314    m68k            2    1    1
 315    mips            2    1    0
 316    mips64          2    1    0
 317    powerpc         2    1    1
 318    powerpc64       2    1    1
 319    powerpc64le     2    1    1
 320    riscv64         2    1    1
 321    s390x           1    1    1
 322    sh4             2    1    1
 323    x86_64          1    0    0
 324 */
 325 #define swap_x15_x25 swap_x15_x25_c
 326
 327 #include "ebcdic037.h"
 328 #include "ebcdic273.h"
 329 #include "ebcdic277.h"
 330 #include "ebcdic278.h"
 331 #include "ebcdic280.h"
 332 #include "ebcdic282.h"
 333 #include "ebcdic284.h"
 334 #include "ebcdic285.h"
 335 #include "ebcdic297.h"
 336 #include "ebcdic423.h"
 337 #include "ebcdic424.h"
 338 #include "ebcdic425.h"
 339 #include "ebcdic500.h"
 340 #include "ebcdic838.h"
 341 #include "ebcdic870.h"
 342 #include "ebcdic871.h"
 343 #include "ebcdic875.h"
 344 #include "ebcdic880.h"
 345 #include "ebcdic905.h"
 346 #include "ebcdic924.h"
 347 #include "ebcdic1025.h"
 348 #include "ebcdic1026.h"
 349 #include "ebcdic1047.h"
 350 #include "ebcdic1097.h"
 351 #include "ebcdic1112.h"
 352 #include "ebcdic1122.h"
 353 #include "ebcdic1123.h"
 354 #include "ebcdic1130.h"
 355 #include "ebcdic1132.h"
 356 #include "ebcdic1137.h"
 357 #include "ebcdic1140.h"
 358 #include "ebcdic1141.h"
 359 #include "ebcdic1142.h"
 360 #include "ebcdic1143.h"
 361 #include "ebcdic1144.h"
 362 #include "ebcdic1145.h"
 363 #include "ebcdic1146.h"
 364 #include "ebcdic1147.h"
 365 #include "ebcdic1148.h"
 366 #include "ebcdic1149.h"
 367 #include "ebcdic1153.h"
 368 #include "ebcdic1154.h"
 369 #include "ebcdic1155.h"
 370 #include "ebcdic1156.h"
 371 #include "ebcdic1157.h"
 372 #include "ebcdic1158.h"
 373 #include "ebcdic1160.h"
 374 #include "ebcdic1164.h"
 375 #include "ebcdic1165.h"
 376 #include "ebcdic1166.h"
 377 #include "ebcdic4971.h"
 378 #include "ebcdic12712.h"
 379 #include "ebcdic16804.h"
 380 #endif
 381
 382 #ifdef USE_EXTRA
 383 #include "euc_jisx0213.h"
 384 #include "shift_jisx0213.h"
 385 #include "iso2022_jp3.h"
 386 #include "big5_2003.h"
 387 #include "tds565.h"
 388 #include "atarist.h"
 389 #include "riscos1.h"
 390 #endif
 391