lib/gb18030_2005.h

   1 /*
   2  * Copyright (C) 1999-2001, 2005, 2012, 2016, 2023 Free Software Foundation, Inc.
   3  * This file is part of the GNU LIBICONV Library.
   4  *
   5  * The GNU LIBICONV Library is free software; you can redistribute it
   6  * and/or modify it under the terms of the GNU Lesser General Public
   7  * License as published by the Free Software Foundation; either version 2.1
   8  * of the License, or (at your option) any later version.
   9  *
  10  * The GNU LIBICONV Library is distributed in the hope that it will be
  11  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * Lesser General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU Lesser General Public
  16  * License along with the GNU LIBICONV Library; see the file COPYING.LIB.
  17  * If not, see <https://www.gnu.org/licenses/>.
  18  */
  19
  20 /*
  21  * GB18030:2005
  22  */
  23
  24 /*
  25  * GB18030, as specified in the GB18030 standard, is an extension of GBK.
  26  *
  27  * In what follows, page numbers refer to the GB18030 standard (second
  28  * printing) from 2005.
  29  *
  30  *
  31  * It consists of the following parts:
  32  *
  33  * One-byte range:
  34  *   ASCII        p. 2         0x{00..7F}
  35  *
  36  * Two-byte range:
  37  *   GBK part 1   p. 10..12    0x{A1..A9}{A1..FE}
  38  *   GBK part 2   p. 13..36    0x{B0..F7}{A1..FE}
  39  *   GBK part 3   p. 37..52    0x{81..A0}{40..7E,80..FE}
  40  *   GBK part 4   p. 53..81    0x{AA..FE}{40..7E,80..A0}
  41  *   GBK part 5   p. 82        0x{A8..A9}{40..7E,80..A0}
  42  *   UDA part 1   p. 83..84    0x{AA..AF}{A1..FE}              U+E000..U+E233
  43  *   UDA part 2   p. 85..87    0x{F8..FE}{A1..FE}              U+E234..U+E4C5
  44  *   UDA part 3   p. 88..90    0x{A1..A7}{40..7E,80..A0}       U+E4C6..U+E765
  45  *
  46  * Four-byte range:
  47  *   BMP rest     p. 94..283   0x{81..84}{30..39}{81..FE}{30..39}
  48  *                                                     rest of U+0080..U+FFFF
  49  *   Planes 1-16  p. 5         0x{90..FE}{30..39}{81..FE}{30..39}
  50  *                                                            U+10000..U+10FFFF
  51  *
  52  * To GBK part 1 were added:
  53  * 1. 0xA2E3, 0xA8BF.
  54  * 2. Characters mapped to the Unicode PUA
  55  *      0xA2AB..0xA2B0                      U+E766..U+E76B
  56  *      0xA2E4                              U+E76D
  57  *      0xA2EF..0xA2F0                      U+E76E..U+E76F
  58  *      0xA2FD..0xA2FE                      U+E770..U+E771
  59  *      0xA4F4..0xA4FE                      U+E772..U+E77C
  60  *      0xA5F7..0xA5FE                      U+E77D..U+E784
  61  *      0xA6B9..0xA6C0                      U+E785..U+E78C
  62  *      0xA6D9..0xA6DF [glyphs here!!]      U+E78D..U+E793
  63  *      0xA6EC..0xA6ED [glyphs here!!]      U+E794..U+E795
  64  *      0xA6F3         [glyphs here!!]      U+E796
  65  *      0xA6F6..0xA6FE                      U+E797..U+E79F
  66  *      0xA7C2..0xA7D0                      U+E7A0..U+E7AE
  67  *      0xA7F2..0xA7FE                      U+E7AF..U+E7BB
  68  *      0xA8BC         [glyphs here!!]      U+E7C7
  69  *      0xA8C1..0xA8C4                      U+E7C9..U+E7CC
  70  *      0xA8EA..0xA8FE                      U+E7CD..U+E7E1
  71  *      0xA9A1..0xA9A3                      U+E7FE..U+E800
  72  *      0xA9F0..0xA9FE                      U+E801..U+E80F
  73  *
  74  * To GBK part 2 were added:
  75  * 3. Characters mapped to the Unicode PUA
  76  *      0xD7FA..0xD7FE                      U+E810..0xE814
  77  *
  78  * To GBK part 3 nothing was added.
  79  *
  80  * To GBK part 4 were added:
  81  * 4. 0xFE{50,54..58,5A..60,62..65,68..6B,6E..75,77..7D,80..8F,92..9F}.
  82  * 5. Characters mapped to the Unicode PUA
  83  *      0xFE51..0xFE53 [glyphs here!!]      U+E816..U+E818
  84  *      0xFE59         [glyphs here!!]      U+E81E
  85  *      0xFE61         [glyphs here!!]      U+E826
  86  *      0xFE66..0xFE67 [glyphs here!!]      U+E82B..U+E82C
  87  *      0xFE6C..0xFE6D [glyphs here!!]      U+E831..U+E832
  88  *      0xFE76         [glyphs here!!]      U+E83B
  89  *      0xFE7E         [glyphs here!!]      U+E843
  90  *      0xFE90..0xFE91 [glyphs here!!]      U+E854..U+E855
  91  *      0xFEA0         [glyphs here!!]      U+E864
  92  *
  93  * To GBK part 5 were added:
  94  * 6. 0xA98A..0xA995.
  95  * 7. Characters mapped to the Unicode PUA
  96  *      0xA896..0xA8A0                      U+E7BC..U+E7C6
  97  *      0xA958                              U+E7E2
  98  *      0xA95B                              U+E7E3
  99  *      0xA95D..0xA95F                      U+E7E4..U+E7E6
 100  *      0xA997..0xA9A0                      U+E7F4..U+E7FD
 101  *
 102  * UDA part 1 contains the user-defined characters, mapped to the Unicode PUA
 103  * U+E000..U+E233 in ascending order.
 104  *
 105  * UDA part 2 contains the user-defined characters, mapped to the Unicode PUA
 106  * U+E234..U+E4C5 in ascending order.
 107  *
 108  * UDA part 3 contains the user-defined characters, mapped to the Unicode PUA
 109  * U+E4C6..U+E765 in ascending order.
 110  *
 111  * The four-byte range 0x{81..84}{30..39}{81..FE}{30..39}
 112  * contains the rest of the Unicode BMP in ascending order.
 113  *    Start: 0x81308130 = 0x0080
 114  *    End:   0x8431A439 = 0xFFFF
 115  *
 116  * The four-byte range 0x{90..E3}{30..39}{81..FE}{30..39}
 117  * contains the remaining 16 Unicode planes in Unicode order.
 118  *    Start: 0x90308130 = 0x010000
 119  *    End:   0xE3329A35 = 0x10FFFF
 120  *
 121  *
 122  * Unassigned Unicode characters are mapped. For example,
 123  *   U+173F = 0x8134BF35   (p. 120)
 124  *   U+2EFF = 0x81398B31   (p. 148)
 125  *   U+FFFE = 0x8431A438   (p. 283)
 126  *
 127  *
 128  * The Unicode PUA (U+E000..U+F8FF) is mapped as follows:
 129  *   p. 83..84    0x{AA..AF}{A1..FE}                  U+E000..U+E233
 130  *   p. 85..87    0x{F8..FE}{A1..FE}                  U+E234..U+E4C5
 131  *   p. 88..90    0x{A1..A7}{40..7E,80..A0}           U+E4C6..U+E765
 132  *   p. 10        0xA2AB..0xA2B0                      U+E766..U+E76B
 133  *   p. 255       0x8336C739                          U+E76C
 134  *   p. 10        0xA2E4                              U+E76D
 135  *   p. 10        0xA2EF..0xA2F0                      U+E76E..U+E76F
 136  *   p. 10        0xA2FD..0xA2FE                      U+E770..U+E771
 137  *   p. 11        0xA4F4..0xA4FE                      U+E772..U+E77C
 138  *   p. 11        0xA5F7..0xA5FE                      U+E77D..U+E784
 139  *   p. 11        0xA6B9..0xA6C0                      U+E785..U+E78C
 140  *   p. 11        0xA6D9..0xA6DF [glyphs here!!]      U+E78D..U+E793
 141  *   p. 11        0xA6EC..0xA6ED [glyphs here!!]      U+E794..U+E795
 142  *   p. 11        0xA6F3 [glyphs here!!]              U+E796
 143  *   p. 11        0xA6F6..0xA6FE                      U+E797..U+E79F
 144  *   p. 12        0xA7C2..0xA7D0                      U+E7A0..U+E7AE
 145  *   p. 12        0xA7F2..0xA7FE                      U+E7AF..U+E7BB
 146  *   p. 82        0xA896..0xA8A0                      U+E7BC..U+E7C6
 147  *   p. 12        0x8135F437                          U+E7C7
 148  *   p. 255       0x8336C830                          U+E7C8
 149  *   p. 12        0xA8C1..0xA8C4                      U+E7C9..U+E7CC
 150  *   p. 12        0xA8EA..0xA8FE                      U+E7CD..U+E7E1
 151  *   p. 82        0xA958                              U+E7E2
 152  *   p. 82        0xA95B                              U+E7E3
 153  *   p. 82        0xA95D..0xA95F                      U+E7E4..U+E7E6
 154  *   p. 255       0x8336C831..0x8336C933              U+E7E7..U+E7F3
 155  *   p. 82        0xA997..0xA9A0                      U+E7F4..U+E7FD
 156  *   p. 12        0xA9A1..0xA9A3                      U+E7FE..U+E800
 157  *   p. 12        0xA9F0..0xA9FE                      U+E801..U+E80F
 158  *   p. 26        0xD7FA..0xD7FE                      U+E810..0xE814
 159  *   p. 255       0x8336C934                          U+E815
 160  *   p. 81        0xFE51..0xFE53 [glyphs here!!]      U+E816..U+E818
 161  *   p. 255       0x8336C935..0x8336C939              U+E819..U+E81D
 162  *   p. 81        0xFE59 [glyphs here!!]              U+E81E
 163  *   p. 255       0x8336CA30..0x8336CA36              U+E81F..U+E825
 164  *   p. 81        0xFE61 [glyphs here!!]              U+E826
 165  *   p. 255       0x8336CA37..0x8336CB30              U+E827..U+E82A
 166  *   p. 81        0xFE66..0xFE67 [glyphs here!!]      U+E82B..U+E82C
 167  *   p. 255       0x8336CB31..0x8336CB34              U+E82D..U+E830
 168  *   p. 81        0xFE6C..0xFE6D [glyphs here!!]      U+E831..U+E832
 169  *   p. 255       0x8336CB35..0x8336CC32              U+E833..U+E83A
 170  *   p. 81        0xFE76 [glyphs here!!]              U+E83B
 171  *   p. 255       0x8336CC33..0x8336CC39              U+E83C..U+E842
 172  *   p. 81        0xFE7E [glyphs here!!]              U+E843
 173  *   p. 255       0x8336CD30..0x8336CE35              U+E844..U+E853
 174  *   p. 81        0xFE90..0xFE91 [glyphs here!!]      U+E854..U+E855
 175  *   p. 255       0x8336CE36..0x8336CF39              U+E856..U+E863
 176  *   p. 81        0xFEA0 [glyphs here!!]              U+E864
 177  *   p. 255..276  0x8336D030..0x84308130              U+E865..U+F8FF
 178  *
 179  *
 180  * The Unicode surrogate area (U+D800..U+DFFF) is not mapped. (p. 255)
 181  *
 182  */
 183
 184 #include "gb18030ext.h"
 185 #include "gb18030uni.h"
 186
 187 static int
 188 gb18030_2005_mbtowc (conv_t conv, ucs4_t *pwc, const unsigned char *s, size_t n)
 189 {
 190   int ret;
 191
 192   /* Code set 0 (ASCII) */
 193   if (*s < 0x80)
 194     return ascii_mbtowc(conv,pwc,s,n);
 195
 196   /* Code set 1 (GBK extended) */
 197   ret = gbk_mbtowc(conv,pwc,s,n);
 198   if (ret != RET_ILSEQ)
 199     return ret;
 200
 201   ret = gb18030_2005_ext_mbtowc(conv,pwc,s,n);
 202   if (ret != RET_ILSEQ)
 203     return ret;
 204
 205   /* Code set 2 (remainder of Unicode U+0000..U+FFFF), including
 206      User-defined characters, two-byte part of range U+E766..U+E864 */
 207   ret = gb18030_2005_uni_mbtowc(conv,pwc,s,n);
 208   if (ret != RET_ILSEQ)
 209     return ret;
 210   /* User-defined characters range U+E000..U+E765 */
 211   {
 212     unsigned char c1 = s[0];
 213     if ((c1 >= 0xaa && c1 <= 0xaf) || (c1 >= 0xf8 && c1 <= 0xfe)) {
 214       if (n >= 2) {
 215         unsigned char c2 = s[1];
 216         if (c2 >= 0xa1 && c2 <= 0xfe) {
 217           *pwc = 0xe000 + 94 * (c1 >= 0xf8 ? c1 - 0xf2 : c1 - 0xaa) + (c2 - 0xa1);
 218           return 2;
 219         }
 220       } else
 221         return RET_TOOFEW(0);
 222     } else if (c1 >= 0xa1 && c1 <= 0xa7) {
 223       if (n >= 2) {
 224         unsigned char c2 = s[1];
 225         if (c2 >= 0x40 && c2 <= 0xa1 && c2 != 0x7f) {
 226           *pwc = 0xe4c6 + 96 * (c1 - 0xa1) + c2 - (c2 >= 0x80 ? 0x41 : 0x40);
 227           return 2;
 228         }
 229       } else
 230         return RET_TOOFEW(0);
 231     }
 232   }
 233
 234   /* Code set 3 (Unicode U+10000..U+10FFFF) */
 235   {
 236     unsigned char c1 = s[0];
 237     if (c1 >= 0x90 && c1 <= 0xe3) {
 238       if (n >= 2) {
 239         unsigned char c2 = s[1];
 240         if (c2 >= 0x30 && c2 <= 0x39) {
 241           if (n >= 3) {
 242             unsigned char c3 = s[2];
 243             if (c3 >= 0x81 && c3 <= 0xfe) {
 244               if (n >= 4) {
 245                 unsigned char c4 = s[3];
 246                 if (c4 >= 0x30 && c4 <= 0x39) {
 247                   unsigned int i = (((c1 - 0x90) * 10 + (c2 - 0x30)) * 126 + (c3 - 0x81)) * 10 + (c4 - 0x30);
 248                   if (i >= 0 && i < 0x100000) {
 249                     *pwc = (ucs4_t) (0x10000 + i);
 250                     return 4;
 251                   }
 252                 }
 253                 return RET_ILSEQ;
 254               }
 255               return RET_TOOFEW(0);
 256             }
 257             return RET_ILSEQ;
 258           }
 259           return RET_TOOFEW(0);
 260         }
 261         return RET_ILSEQ;
 262       }
 263       return RET_TOOFEW(0);
 264     }
 265     return RET_ILSEQ;
 266   }
 267 }
 268
 269 static const struct { unsigned short uni[2]; unsigned int charset; } gb18030_2005_pua2charset[35] = {
 270 /*    Unicode range      GB18030 range */
 271   { { 0xe766, 0xe76b },  0xa2ab /*.. 0xa2b0, */ },
 272   { { 0xe76d, 0xe76d },  0xa2e4                 },
 273   { { 0xe76e, 0xe76f },  0xa2ef /*.. 0xa2f0, */ },
 274   { { 0xe770, 0xe771 },  0xa2fd /*.. 0xa2fe, */ },
 275   { { 0xe772, 0xe77c },  0xa4f4 /*.. 0xa4fe, */ },
 276   { { 0xe77d, 0xe784 },  0xa5f7 /*.. 0xa5fe, */ },
 277   { { 0xe785, 0xe78c },  0xa6b9 /*.. 0xa6c0, */ },
 278   { { 0xe78d, 0xe793 },  0xa6d9 /*.. 0xa6df, */ },
 279   { { 0xe794, 0xe795 },  0xa6ec /*.. 0xa6ed, */ },
 280   { { 0xe796, 0xe796 },  0xa6f3                 },
 281   { { 0xe797, 0xe79f },  0xa6f6 /*.. 0xa6fe, */ },
 282   { { 0xe7a0, 0xe7ae },  0xa7c2 /*.. 0xa7d0, */ },
 283   { { 0xe7af, 0xe7bb },  0xa7f2 /*.. 0xa7fe, */ },
 284   { { 0xe7bc, 0xe7c6 },  0xa896 /*.. 0xa8a0, */ },
 285   { { 0xe7c9, 0xe7cc },  0xa8c1 /*.. 0xa8c4, */ },
 286   { { 0xe7cd, 0xe7e1 },  0xa8ea /*.. 0xa8fe, */ },
 287   { { 0xe7e2, 0xe7e2 },  0xa958                 },
 288   { { 0xe7e3, 0xe7e3 },  0xa95b                 },
 289   { { 0xe7e4, 0xe7e6 },  0xa95d /*.. 0xa95f, */ },
 290   { { 0xe7f4, 0xe800 },  0xa997 /*.. 0xa9a3, */ },
 291   { { 0xe801, 0xe80f },  0xa9f0 /*.. 0xa9fe, */ },
 292   { { 0xe810, 0xe814 },  0xd7fa /*.. 0xd7fe, */ },
 293   { { 0xe816, 0xe816 },  0x95329031             },
 294   { { 0xe817, 0xe817 },  0x95329033             },
 295   { { 0xe818, 0xe818 },  0x95329730             },
 296   { { 0xe81e, 0xe81e },  0xfe59                 },
 297   { { 0xe826, 0xe826 },  0xfe61                 },
 298   { { 0xe82b, 0xe82c },  0xfe66 /*.. 0xfe67, */ },
 299   { { 0xe831, 0xe831 },  0x9536b937             },
 300   { { 0xe832, 0xe832 },  0xfe6d                 },
 301   { { 0xe83b, 0xe83b },  0x9630ba35             },
 302   { { 0xe843, 0xe843 },  0xfe7e                 },
 303   { { 0xe854, 0xe854 },  0xfe90                 },
 304   { { 0xe855, 0xe855 },  0x9635b630             },
 305   { { 0xe864, 0xe864 },  0xfea0                 },
 306 };
 307
 308 static int
 309 gb18030_2005_wctomb (conv_t conv, unsigned char *r, ucs4_t wc, size_t n)
 310 {
 311   int ret;
 312
 313   /* Code set 0 (ASCII) */
 314   ret = ascii_wctomb(conv,r,wc,n);
 315   if (ret != RET_ILUNI)
 316     return ret;
 317
 318   /* Code set 1 (GBK extended) */
 319   ret = gbk_wctomb(conv,r,wc,n);
 320   if (ret != RET_ILUNI)
 321     return ret;
 322
 323   ret = gb18030ext_wctomb(conv,r,wc,n);
 324   if (ret != RET_ILUNI)
 325     return ret;
 326
 327   /* Code set 2 (remainder of Unicode U+0000..U+FFFF) */
 328   if (wc >= 0xe000 && wc <= 0xe864) {
 329     if (n >= 2) {
 330       if (wc < 0xe766) {
 331         /* User-defined characters range U+E000..U+E765 */
 332         if (wc < 0xe4c6) {
 333           unsigned int i = wc - 0xe000;
 334           r[1] = (i % 94) + 0xa1; i = i / 94;
 335           r[0] = (i < 6 ? i + 0xaa : i + 0xf2);
 336           return 2;
 337         } else {
 338           unsigned int i = wc - 0xe4c6;
 339           r[0] = (i / 96) + 0xa1; i = i % 96;
 340           r[1] = i + (i >= 0x3f ? 0x41 : 0x40);
 341           return 2;
 342         }
 343       } else {
 344         /* User-defined characters, two-byte part and 6 four-byte mappings in
 345            range U+E766..U+E864 */
 346         unsigned int k1 = 0;
 347         unsigned int k2 = 35;
 348         /* Invariant: We know that if wc occurs in Unicode interval in
 349            gb18030_2005_pua2charset, it does so at a k with  k1 <= k < k2. */
 350         while (k1 < k2) {
 351           unsigned int k = (k1 + k2) / 2;
 352           if (wc < gb18030_2005_pua2charset[k].uni[0])
 353             k2 = k;
 354           else if (wc > gb18030_2005_pua2charset[k].uni[1])
 355             k1 = k + 1;
 356           else {
 357             unsigned int c =
 358               gb18030_2005_pua2charset[k].charset + (wc - gb18030_2005_pua2charset[k].uni[0]);
 359             if (c < 0x10000) {
 360               r[0] = (c >> 8);
 361               r[1] = c & 0xff;
 362               return 2;
 363             } else {
 364               r[0] = (c >> 24);
 365               r[1] = (c >> 16) & 0xff;
 366               r[2] = (c >> 8) & 0xff;
 367               r[3] = c & 0xff;
 368               return 4;
 369             }
 370           }
 371         }
 372       }
 373     } else
 374       return RET_TOOSMALL;
 375   }
 376   ret = gb18030_2005_uni_wctomb(conv,r,wc,n);
 377   if (ret != RET_ILUNI)
 378     return ret;
 379
 380   /* Code set 3 (Unicode U+10000..U+10FFFF) */
 381   if (n >= 4) {
 382     if (wc >= 0x10000 && wc < 0x110000) {
 383       unsigned int i = wc - 0x10000;
 384       r[3] = (i % 10) + 0x30; i = i / 10;
 385       r[2] = (i % 126) + 0x81; i = i / 126;
 386       r[1] = (i % 10) + 0x30; i = i / 10;
 387       r[0] = i + 0x90;
 388       return 4;
 389     }
 390     return RET_ILUNI;
 391   }
 392   return RET_TOOSMALL;
 393 }