Modules/expat/xmltok.c

   1 /*
   2 Copyright (c) 1998, 1999 Thai Open Source Software Center Ltd
   3 See the file COPYING for copying permission.
   4 */
   5
   6 #ifdef COMPILED_FROM_DSP
   7 #  include "winconfig.h"
   8 #else
   9 #ifdef HAVE_CONFIG_H
  10 #  include <config.h>
  11 #endif
  12 #endif /* ndef COMPILED_FROM_DSP */
  13
  14 #include "xmltok.h"
  15 #include "nametab.h"
  16
  17 #ifdef XML_DTD
  18 #define IGNORE_SECTION_TOK_VTABLE , PREFIX(ignoreSectionTok)
  19 #else
  20 #define IGNORE_SECTION_TOK_VTABLE /* as nothing */
  21 #endif
  22
  23 #define VTABLE1 \
  24   { PREFIX(prologTok), PREFIX(contentTok), \
  25     PREFIX(cdataSectionTok) IGNORE_SECTION_TOK_VTABLE }, \
  26   { PREFIX(attributeValueTok), PREFIX(entityValueTok) }, \
  27   PREFIX(sameName), \
  28   PREFIX(nameMatchesAscii), \
  29   PREFIX(nameLength), \
  30   PREFIX(skipS), \
  31   PREFIX(getAtts), \
  32   PREFIX(charRefNumber), \
  33   PREFIX(predefinedEntityName), \
  34   PREFIX(updatePosition), \
  35   PREFIX(isPublicId)
  36
  37 #define VTABLE VTABLE1, PREFIX(toUtf8), PREFIX(toUtf16)
  38
  39 #define UCS2_GET_NAMING(pages, hi, lo) \
  40    (namingBitmap[(pages[hi] << 3) + ((lo) >> 5)] & (1 << ((lo) & 0x1F)))
  41
  42 /* A 2 byte UTF-8 representation splits the characters 11 bits
  43 between the bottom 5 and 6 bits of the bytes.
  44 We need 8 bits to index into pages, 3 bits to add to that index and
  45 5 bits to generate the mask. */
  46 #define UTF8_GET_NAMING2(pages, byte) \
  47     (namingBitmap[((pages)[(((byte)[0]) >> 2) & 7] << 3) \
  48                       + ((((byte)[0]) & 3) << 1) \
  49                       + ((((byte)[1]) >> 5) & 1)] \
  50          & (1 << (((byte)[1]) & 0x1F)))
  51
  52 /* A 3 byte UTF-8 representation splits the characters 16 bits
  53 between the bottom 4, 6 and 6 bits of the bytes.
  54 We need 8 bits to index into pages, 3 bits to add to that index and
  55 5 bits to generate the mask. */
  56 #define UTF8_GET_NAMING3(pages, byte) \
  57   (namingBitmap[((pages)[((((byte)[0]) & 0xF) << 4) \
  58                              + ((((byte)[1]) >> 2) & 0xF)] \
  59                        << 3) \
  60                       + ((((byte)[1]) & 3) << 1) \
  61                       + ((((byte)[2]) >> 5) & 1)] \
  62          & (1 << (((byte)[2]) & 0x1F)))
  63
  64 #define UTF8_GET_NAMING(pages, p, n) \
  65   ((n) == 2 \
  66   ? UTF8_GET_NAMING2(pages, (const unsigned char *)(p)) \
  67   : ((n) == 3 \
  68      ? UTF8_GET_NAMING3(pages, (const unsigned char *)(p)) \
  69      : 0))
  70
  71 #define UTF8_INVALID3(p) \
  72   ((*p) == 0xED \
  73   ? (((p)[1] & 0x20) != 0) \
  74   : ((*p) == 0xEF \
  75      ? ((p)[1] == 0xBF && ((p)[2] == 0xBF || (p)[2] == 0xBE)) \
  76      : 0))
  77
  78 #define UTF8_INVALID4(p) ((*p) == 0xF4 && ((p)[1] & 0x30) != 0)
  79
  80 static
  81 int isNever(const ENCODING *enc, const char *p)
  82 {
  83   return 0;
  84 }
  85
  86 static
  87 int utf8_isName2(const ENCODING *enc, const char *p)
  88 {
  89   return UTF8_GET_NAMING2(namePages, (const unsigned char *)p);
  90 }
  91
  92 static
  93 int utf8_isName3(const ENCODING *enc, const char *p)
  94 {
  95   return UTF8_GET_NAMING3(namePages, (const unsigned char *)p);
  96 }
  97
  98 #define utf8_isName4 isNever
  99
 100 static
 101 int utf8_isNmstrt2(const ENCODING *enc, const char *p)
 102 {
 103   return UTF8_GET_NAMING2(nmstrtPages, (const unsigned char *)p);
 104 }
 105
 106 static
 107 int utf8_isNmstrt3(const ENCODING *enc, const char *p)
 108 {
 109   return UTF8_GET_NAMING3(nmstrtPages, (const unsigned char *)p);
 110 }
 111
 112 #define utf8_isNmstrt4 isNever
 113
 114 #define utf8_isInvalid2 isNever
 115
 116 static
 117 int utf8_isInvalid3(const ENCODING *enc, const char *p)
 118 {
 119   return UTF8_INVALID3((const unsigned char *)p);
 120 }
 121
 122 static
 123 int utf8_isInvalid4(const ENCODING *enc, const char *p)
 124 {
 125   return UTF8_INVALID4((const unsigned char *)p);
 126 }
 127
 128 struct normal_encoding {
 129   ENCODING enc;
 130   unsigned char type[256];
 131 #ifdef XML_MIN_SIZE
 132   int (*byteType)(const ENCODING *, const char *);
 133   int (*isNameMin)(const ENCODING *, const char *);
 134   int (*isNmstrtMin)(const ENCODING *, const char *);
 135   int (*byteToAscii)(const ENCODING *, const char *);
 136   int (*charMatches)(const ENCODING *, const char *, int);
 137 #endif /* XML_MIN_SIZE */
 138   int (*isName2)(const ENCODING *, const char *);
 139   int (*isName3)(const ENCODING *, const char *);
 140   int (*isName4)(const ENCODING *, const char *);
 141   int (*isNmstrt2)(const ENCODING *, const char *);
 142   int (*isNmstrt3)(const ENCODING *, const char *);
 143   int (*isNmstrt4)(const ENCODING *, const char *);
 144   int (*isInvalid2)(const ENCODING *, const char *);
 145   int (*isInvalid3)(const ENCODING *, const char *);
 146   int (*isInvalid4)(const ENCODING *, const char *);
 147 };
 148
 149 #ifdef XML_MIN_SIZE
 150
 151 #define STANDARD_VTABLE(E) \
 152  E ## byteType, \
 153  E ## isNameMin, \
 154  E ## isNmstrtMin, \
 155  E ## byteToAscii, \
 156  E ## charMatches,
 157
 158 #else
 159
 160 #define STANDARD_VTABLE(E) /* as nothing */
 161
 162 #endif
 163
 164 #define NORMAL_VTABLE(E) \
 165  E ## isName2, \
 166  E ## isName3, \
 167  E ## isName4, \
 168  E ## isNmstrt2, \
 169  E ## isNmstrt3, \
 170  E ## isNmstrt4, \
 171  E ## isInvalid2, \
 172  E ## isInvalid3, \
 173  E ## isInvalid4
 174
 175 static int checkCharRefNumber(int);
 176
 177 #include "xmltok_impl.h"
 178 #include "ascii.h"
 179
 180 #ifdef XML_MIN_SIZE
 181 #define sb_isNameMin isNever
 182 #define sb_isNmstrtMin isNever
 183 #endif
 184
 185 #ifdef XML_MIN_SIZE
 186 #define MINBPC(enc) ((enc)->minBytesPerChar)
 187 #else
 188 /* minimum bytes per character */
 189 #define MINBPC(enc) 1
 190 #endif
 191
 192 #define SB_BYTE_TYPE(enc, p) \
 193   (((struct normal_encoding *)(enc))->type[(unsigned char)*(p)])
 194
 195 #ifdef XML_MIN_SIZE
 196 static
 197 int sb_byteType(const ENCODING *enc, const char *p)
 198 {
 199   return SB_BYTE_TYPE(enc, p);
 200 }
 201 #define BYTE_TYPE(enc, p) \
 202  (((const struct normal_encoding *)(enc))->byteType(enc, p))
 203 #else
 204 #define BYTE_TYPE(enc, p) SB_BYTE_TYPE(enc, p)
 205 #endif
 206
 207 #ifdef XML_MIN_SIZE
 208 #define BYTE_TO_ASCII(enc, p) \
 209  (((const struct normal_encoding *)(enc))->byteToAscii(enc, p))
 210 static
 211 int sb_byteToAscii(const ENCODING *enc, const char *p)
 212 {
 213   return *p;
 214 }
 215 #else
 216 #define BYTE_TO_ASCII(enc, p) (*(p))
 217 #endif
 218
 219 #define IS_NAME_CHAR(enc, p, n) \
 220  (((const struct normal_encoding *)(enc))->isName ## n(enc, p))
 221 #define IS_NMSTRT_CHAR(enc, p, n) \
 222  (((const struct normal_encoding *)(enc))->isNmstrt ## n(enc, p))
 223 #define IS_INVALID_CHAR(enc, p, n) \
 224  (((const struct normal_encoding *)(enc))->isInvalid ## n(enc, p))
 225
 226 #ifdef XML_MIN_SIZE
 227 #define IS_NAME_CHAR_MINBPC(enc, p) \
 228  (((const struct normal_encoding *)(enc))->isNameMin(enc, p))
 229 #define IS_NMSTRT_CHAR_MINBPC(enc, p) \
 230  (((const struct normal_encoding *)(enc))->isNmstrtMin(enc, p))
 231 #else
 232 #define IS_NAME_CHAR_MINBPC(enc, p) (0)
 233 #define IS_NMSTRT_CHAR_MINBPC(enc, p) (0)
 234 #endif
 235
 236 #ifdef XML_MIN_SIZE
 237 #define CHAR_MATCHES(enc, p, c) \
 238  (((const struct normal_encoding *)(enc))->charMatches(enc, p, c))
 239 static
 240 int sb_charMatches(const ENCODING *enc, const char *p, int c)
 241 {
 242   return *p == c;
 243 }
 244 #else
 245 /* c is an ASCII character */
 246 #define CHAR_MATCHES(enc, p, c) (*(p) == c)
 247 #endif
 248
 249 #define PREFIX(ident) normal_ ## ident
 250 #include "xmltok_impl.c"
 251
 252 #undef MINBPC
 253 #undef BYTE_TYPE
 254 #undef BYTE_TO_ASCII
 255 #undef CHAR_MATCHES
 256 #undef IS_NAME_CHAR
 257 #undef IS_NAME_CHAR_MINBPC
 258 #undef IS_NMSTRT_CHAR
 259 #undef IS_NMSTRT_CHAR_MINBPC
 260 #undef IS_INVALID_CHAR
 261
 262 enum {  /* UTF8_cvalN is value of masked first byte of N byte sequence */
 263   UTF8_cval1 = 0x00,
 264   UTF8_cval2 = 0xc0,
 265   UTF8_cval3 = 0xe0,
 266   UTF8_cval4 = 0xf0
 267 };
 268
 269 static
 270 void utf8_toUtf8(const ENCODING *enc,
 271                  const char **fromP, const char *fromLim,
 272                  char **toP, const char *toLim)
 273 {
 274   char *to;
 275   const char *from;
 276   if (fromLim - *fromP > toLim - *toP) {
 277     /* Avoid copying partial characters. */
 278     for (fromLim = *fromP + (toLim - *toP); fromLim > *fromP; fromLim--)
 279       if (((unsigned char)fromLim[-1] & 0xc0) != 0x80)
 280         break;
 281   }
 282   for (to = *toP, from = *fromP; from != fromLim; from++, to++)
 283     *to = *from;
 284   *fromP = from;
 285   *toP = to;
 286 }
 287
 288 static
 289 void utf8_toUtf16(const ENCODING *enc,
 290                   const char **fromP, const char *fromLim,
 291                   unsigned short **toP, const unsigned short *toLim)
 292 {
 293   unsigned short *to = *toP;
 294   const char *from = *fromP;
 295   while (from != fromLim && to != toLim) {
 296     switch (((struct normal_encoding *)enc)->type[(unsigned char)*from]) {
 297     case BT_LEAD2:
 298       *to++ = ((from[0] & 0x1f) << 6) | (from[1] & 0x3f);
 299       from += 2;
 300       break;
 301     case BT_LEAD3:
 302       *to++ = ((from[0] & 0xf) << 12) | ((from[1] & 0x3f) << 6) | (from[2] & 0x3f);
 303       from += 3;
 304       break;
 305     case BT_LEAD4:
 306       {
 307         unsigned long n;
 308         if (to + 1 == toLim)
 309           break;
 310         n = ((from[0] & 0x7) << 18) | ((from[1] & 0x3f) << 12) | ((from[2] & 0x3f) << 6) | (from[3] & 0x3f);
 311         n -= 0x10000;
 312         to[0] = (unsigned short)((n >> 10) | 0xD800);
 313         to[1] = (unsigned short)((n & 0x3FF) | 0xDC00);
 314         to += 2;
 315         from += 4;
 316       }
 317       break;
 318     default:
 319       *to++ = *from++;
 320       break;
 321     }
 322   }
 323   *fromP = from;
 324   *toP = to;
 325 }
 326
 327 #ifdef XML_NS
 328 static const struct normal_encoding utf8_encoding_ns = {
 329   { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
 330   {
 331 #include "asciitab.h"
 332 #include "utf8tab.h"
 333   },
 334   STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
 335 };
 336 #endif
 337
 338 static const struct normal_encoding utf8_encoding = {
 339   { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
 340   {
 341 #define BT_COLON BT_NMSTRT
 342 #include "asciitab.h"
 343 #undef BT_COLON
 344 #include "utf8tab.h"
 345   },
 346   STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
 347 };
 348
 349 #ifdef XML_NS
 350
 351 static const struct normal_encoding internal_utf8_encoding_ns = {
 352   { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
 353   {
 354 #include "iasciitab.h"
 355 #include "utf8tab.h"
 356   },
 357   STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
 358 };
 359
 360 #endif
 361
 362 static const struct normal_encoding internal_utf8_encoding = {
 363   { VTABLE1, utf8_toUtf8, utf8_toUtf16, 1, 1, 0 },
 364   {
 365 #define BT_COLON BT_NMSTRT
 366 #include "iasciitab.h"
 367 #undef BT_COLON
 368 #include "utf8tab.h"
 369   },
 370   STANDARD_VTABLE(sb_) NORMAL_VTABLE(utf8_)
 371 };
 372
 373 static
 374 void latin1_toUtf8(const ENCODING *enc,
 375                    const char **fromP, const char *fromLim,
 376                    char **toP, const char *toLim)
 377 {
 378   for (;;) {
 379     unsigned char c;
 380     if (*fromP == fromLim)
 381       break;
 382     c = (unsigned char)**fromP;
 383     if (c & 0x80) {
 384       if (toLim - *toP < 2)
 385         break;
 386       *(*toP)++ = ((c >> 6) | UTF8_cval2);
 387       *(*toP)++ = ((c & 0x3f) | 0x80);
 388       (*fromP)++;
 389     }
 390     else {
 391       if (*toP == toLim)
 392         break;
 393       *(*toP)++ = *(*fromP)++;
 394     }
 395   }
 396 }
 397
 398 static
 399 void latin1_toUtf16(const ENCODING *enc,
 400                     const char **fromP, const char *fromLim,
 401                     unsigned short **toP, const unsigned short *toLim)
 402 {
 403   while (*fromP != fromLim && *toP != toLim)
 404     *(*toP)++ = (unsigned char)*(*fromP)++;
 405 }
 406
 407 #ifdef XML_NS
 408
 409 static const struct normal_encoding latin1_encoding_ns = {
 410   { VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0 },
 411   {
 412 #include "asciitab.h"
 413 #include "latin1tab.h"
 414   },
 415   STANDARD_VTABLE(sb_)
 416 };
 417
 418 #endif
 419
 420 static const struct normal_encoding latin1_encoding = {
 421   { VTABLE1, latin1_toUtf8, latin1_toUtf16, 1, 0, 0 },
 422   {
 423 #define BT_COLON BT_NMSTRT
 424 #include "asciitab.h"
 425 #undef BT_COLON
 426 #include "latin1tab.h"
 427   },
 428   STANDARD_VTABLE(sb_)
 429 };
 430
 431 static
 432 void ascii_toUtf8(const ENCODING *enc,
 433                   const char **fromP, const char *fromLim,
 434                   char **toP, const char *toLim)
 435 {
 436   while (*fromP != fromLim && *toP != toLim)
 437     *(*toP)++ = *(*fromP)++;
 438 }
 439
 440 #ifdef XML_NS
 441
 442 static const struct normal_encoding ascii_encoding_ns = {
 443   { VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0 },
 444   {
 445 #include "asciitab.h"
 446 /* BT_NONXML == 0 */
 447   },
 448   STANDARD_VTABLE(sb_)
 449 };
 450
 451 #endif
 452
 453 static const struct normal_encoding ascii_encoding = {
 454   { VTABLE1, ascii_toUtf8, latin1_toUtf16, 1, 1, 0 },
 455   {
 456 #define BT_COLON BT_NMSTRT
 457 #include "asciitab.h"
 458 #undef BT_COLON
 459 /* BT_NONXML == 0 */
 460   },
 461   STANDARD_VTABLE(sb_)
 462 };
 463
 464 static int unicode_byte_type(char hi, char lo)
 465 {
 466   switch ((unsigned char)hi) {
 467   case 0xD8: case 0xD9: case 0xDA: case 0xDB:
 468     return BT_LEAD4;
 469   case 0xDC: case 0xDD: case 0xDE: case 0xDF:
 470     return BT_TRAIL;
 471   case 0xFF:
 472     switch ((unsigned char)lo) {
 473     case 0xFF:
 474     case 0xFE:
 475       return BT_NONXML;
 476     }
 477     break;
 478   }
 479   return BT_NONASCII;
 480 }
 481
 482 #define DEFINE_UTF16_TO_UTF8(E) \
 483 static \
 484 void E ## toUtf8(const ENCODING *enc, \
 485                  const char **fromP, const char *fromLim, \
 486                  char **toP, const char *toLim) \
 487 { \
 488   const char *from; \
 489   for (from = *fromP; from != fromLim; from += 2) { \
 490     int plane; \
 491     unsigned char lo2; \
 492     unsigned char lo = GET_LO(from); \
 493     unsigned char hi = GET_HI(from); \
 494     switch (hi) { \
 495     case 0: \
 496       if (lo < 0x80) { \
 497         if (*toP == toLim) { \
 498           *fromP = from; \
 499           return; \
 500         } \
 501         *(*toP)++ = lo; \
 502         break; \
 503       } \
 504       /* fall through */ \
 505     case 0x1: case 0x2: case 0x3: \
 506     case 0x4: case 0x5: case 0x6: case 0x7: \
 507       if (toLim -  *toP < 2) { \
 508         *fromP = from; \
 509         return; \
 510       } \
 511       *(*toP)++ = ((lo >> 6) | (hi << 2) |  UTF8_cval2); \
 512       *(*toP)++ = ((lo & 0x3f) | 0x80); \
 513       break; \
 514     default: \
 515       if (toLim -  *toP < 3)  { \
 516         *fromP = from; \
 517         return; \
 518       } \
 519       /* 16 bits divided 4, 6, 6 amongst 3 bytes */ \
 520       *(*toP)++ = ((hi >> 4) | UTF8_cval3); \
 521       *(*toP)++ = (((hi & 0xf) << 2) | (lo >> 6) | 0x80); \
 522       *(*toP)++ = ((lo & 0x3f) | 0x80); \
 523       break; \
 524     case 0xD8: case 0xD9: case 0xDA: case 0xDB: \
 525       if (toLim -  *toP < 4) { \
 526         *fromP = from; \
 527         return; \
 528       } \
 529       plane = (((hi & 0x3) << 2) | ((lo >> 6) & 0x3)) + 1; \
 530       *(*toP)++ = ((plane >> 2) | UTF8_cval4); \
 531       *(*toP)++ = (((lo >> 2) & 0xF) | ((plane & 0x3) << 4) | 0x80); \
 532       from += 2; \
 533       lo2 = GET_LO(from); \
 534       *(*toP)++ = (((lo & 0x3) << 4) \
 535                    | ((GET_HI(from) & 0x3) << 2) \
 536                    | (lo2 >> 6) \
 537                    | 0x80); \
 538       *(*toP)++ = ((lo2 & 0x3f) | 0x80); \
 539       break; \
 540     } \
 541   } \
 542   *fromP = from; \
 543 }
 544
 545 #define DEFINE_UTF16_TO_UTF16(E) \
 546 static \
 547 void E ## toUtf16(const ENCODING *enc, \
 548                   const char **fromP, const char *fromLim, \
 549                   unsigned short **toP, const unsigned short *toLim) \
 550 { \
 551   /* Avoid copying first half only of surrogate */ \
 552   if (fromLim - *fromP > ((toLim - *toP) << 1) \
 553       && (GET_HI(fromLim - 2) & 0xF8) == 0xD8) \
 554     fromLim -= 2; \
 555   for (; *fromP != fromLim && *toP != toLim; *fromP += 2) \
 556     *(*toP)++ = (GET_HI(*fromP) << 8) | GET_LO(*fromP); \
 557 }
 558
 559 #define SET2(ptr, ch) \
 560   (((ptr)[0] = ((ch) & 0xff)), ((ptr)[1] = ((ch) >> 8)))
 561 #define GET_LO(ptr) ((unsigned char)(ptr)[0])
 562 #define GET_HI(ptr) ((unsigned char)(ptr)[1])
 563
 564 DEFINE_UTF16_TO_UTF8(little2_)
 565 DEFINE_UTF16_TO_UTF16(little2_)
 566
 567 #undef SET2
 568 #undef GET_LO
 569 #undef GET_HI
 570
 571 #define SET2(ptr, ch) \
 572   (((ptr)[0] = ((ch) >> 8)), ((ptr)[1] = ((ch) & 0xFF)))
 573 #define GET_LO(ptr) ((unsigned char)(ptr)[1])
 574 #define GET_HI(ptr) ((unsigned char)(ptr)[0])
 575
 576 DEFINE_UTF16_TO_UTF8(big2_)
 577 DEFINE_UTF16_TO_UTF16(big2_)
 578
 579 #undef SET2
 580 #undef GET_LO
 581 #undef GET_HI
 582
 583 #define LITTLE2_BYTE_TYPE(enc, p) \
 584  ((p)[1] == 0 \
 585   ? ((struct normal_encoding *)(enc))->type[(unsigned char)*(p)] \
 586   : unicode_byte_type((p)[1], (p)[0]))
 587 #define LITTLE2_BYTE_TO_ASCII(enc, p) ((p)[1] == 0 ? (p)[0] : -1)
 588 #define LITTLE2_CHAR_MATCHES(enc, p, c) ((p)[1] == 0 && (p)[0] == c)
 589 #define LITTLE2_IS_NAME_CHAR_MINBPC(enc, p) \
 590   UCS2_GET_NAMING(namePages, (unsigned char)p[1], (unsigned char)p[0])
 591 #define LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p) \
 592   UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[1], (unsigned char)p[0])
 593
 594 #ifdef XML_MIN_SIZE
 595
 596 static
 597 int little2_byteType(const ENCODING *enc, const char *p)
 598 {
 599   return LITTLE2_BYTE_TYPE(enc, p);
 600 }
 601
 602 static
 603 int little2_byteToAscii(const ENCODING *enc, const char *p)
 604 {
 605   return LITTLE2_BYTE_TO_ASCII(enc, p);
 606 }
 607
 608 static
 609 int little2_charMatches(const ENCODING *enc, const char *p, int c)
 610 {
 611   return LITTLE2_CHAR_MATCHES(enc, p, c);
 612 }
 613
 614 static
 615 int little2_isNameMin(const ENCODING *enc, const char *p)
 616 {
 617   return LITTLE2_IS_NAME_CHAR_MINBPC(enc, p);
 618 }
 619
 620 static
 621 int little2_isNmstrtMin(const ENCODING *enc, const char *p)
 622 {
 623   return LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p);
 624 }
 625
 626 #undef VTABLE
 627 #define VTABLE VTABLE1, little2_toUtf8, little2_toUtf16
 628
 629 #else /* not XML_MIN_SIZE */
 630
 631 #undef PREFIX
 632 #define PREFIX(ident) little2_ ## ident
 633 #define MINBPC(enc) 2
 634 /* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */
 635 #define BYTE_TYPE(enc, p) LITTLE2_BYTE_TYPE(enc, p)
 636 #define BYTE_TO_ASCII(enc, p) LITTLE2_BYTE_TO_ASCII(enc, p)
 637 #define CHAR_MATCHES(enc, p, c) LITTLE2_CHAR_MATCHES(enc, p, c)
 638 #define IS_NAME_CHAR(enc, p, n) 0
 639 #define IS_NAME_CHAR_MINBPC(enc, p) LITTLE2_IS_NAME_CHAR_MINBPC(enc, p)
 640 #define IS_NMSTRT_CHAR(enc, p, n) (0)
 641 #define IS_NMSTRT_CHAR_MINBPC(enc, p) LITTLE2_IS_NMSTRT_CHAR_MINBPC(enc, p)
 642
 643 #include "xmltok_impl.c"
 644
 645 #undef MINBPC
 646 #undef BYTE_TYPE
 647 #undef BYTE_TO_ASCII
 648 #undef CHAR_MATCHES
 649 #undef IS_NAME_CHAR
 650 #undef IS_NAME_CHAR_MINBPC
 651 #undef IS_NMSTRT_CHAR
 652 #undef IS_NMSTRT_CHAR_MINBPC
 653 #undef IS_INVALID_CHAR
 654
 655 #endif /* not XML_MIN_SIZE */
 656
 657 #ifdef XML_NS
 658
 659 static const struct normal_encoding little2_encoding_ns = {
 660   { VTABLE, 2, 0,
 661 #if XML_BYTE_ORDER == 12
 662     1
 663 #else
 664     0
 665 #endif
 666   },
 667   {
 668 #include "asciitab.h"
 669 #include "latin1tab.h"
 670   },
 671   STANDARD_VTABLE(little2_)
 672 };
 673
 674 #endif
 675
 676 static const struct normal_encoding little2_encoding = {
 677   { VTABLE, 2, 0,
 678 #if XML_BYTE_ORDER == 12
 679     1
 680 #else
 681     0
 682 #endif
 683   },
 684   {
 685 #define BT_COLON BT_NMSTRT
 686 #include "asciitab.h"
 687 #undef BT_COLON
 688 #include "latin1tab.h"
 689   },
 690   STANDARD_VTABLE(little2_)
 691 };
 692
 693 #if XML_BYTE_ORDER != 21
 694
 695 #ifdef XML_NS
 696
 697 static const struct normal_encoding internal_little2_encoding_ns = {
 698   { VTABLE, 2, 0, 1 },
 699   {
 700 #include "iasciitab.h"
 701 #include "latin1tab.h"
 702   },
 703   STANDARD_VTABLE(little2_)
 704 };
 705
 706 #endif
 707
 708 static const struct normal_encoding internal_little2_encoding = {
 709   { VTABLE, 2, 0, 1 },
 710   {
 711 #define BT_COLON BT_NMSTRT
 712 #include "iasciitab.h"
 713 #undef BT_COLON
 714 #include "latin1tab.h"
 715   },
 716   STANDARD_VTABLE(little2_)
 717 };
 718
 719 #endif
 720
 721
 722 #define BIG2_BYTE_TYPE(enc, p) \
 723  ((p)[0] == 0 \
 724   ? ((struct normal_encoding *)(enc))->type[(unsigned char)(p)[1]] \
 725   : unicode_byte_type((p)[0], (p)[1]))
 726 #define BIG2_BYTE_TO_ASCII(enc, p) ((p)[0] == 0 ? (p)[1] : -1)
 727 #define BIG2_CHAR_MATCHES(enc, p, c) ((p)[0] == 0 && (p)[1] == c)
 728 #define BIG2_IS_NAME_CHAR_MINBPC(enc, p) \
 729   UCS2_GET_NAMING(namePages, (unsigned char)p[0], (unsigned char)p[1])
 730 #define BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p) \
 731   UCS2_GET_NAMING(nmstrtPages, (unsigned char)p[0], (unsigned char)p[1])
 732
 733 #ifdef XML_MIN_SIZE
 734
 735 static
 736 int big2_byteType(const ENCODING *enc, const char *p)
 737 {
 738   return BIG2_BYTE_TYPE(enc, p);
 739 }
 740
 741 static
 742 int big2_byteToAscii(const ENCODING *enc, const char *p)
 743 {
 744   return BIG2_BYTE_TO_ASCII(enc, p);
 745 }
 746
 747 static
 748 int big2_charMatches(const ENCODING *enc, const char *p, int c)
 749 {
 750   return BIG2_CHAR_MATCHES(enc, p, c);
 751 }
 752
 753 static
 754 int big2_isNameMin(const ENCODING *enc, const char *p)
 755 {
 756   return BIG2_IS_NAME_CHAR_MINBPC(enc, p);
 757 }
 758
 759 static
 760 int big2_isNmstrtMin(const ENCODING *enc, const char *p)
 761 {
 762   return BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p);
 763 }
 764
 765 #undef VTABLE
 766 #define VTABLE VTABLE1, big2_toUtf8, big2_toUtf16
 767
 768 #else /* not XML_MIN_SIZE */
 769
 770 #undef PREFIX
 771 #define PREFIX(ident) big2_ ## ident
 772 #define MINBPC(enc) 2
 773 /* CHAR_MATCHES is guaranteed to have MINBPC bytes available. */
 774 #define BYTE_TYPE(enc, p) BIG2_BYTE_TYPE(enc, p)
 775 #define BYTE_TO_ASCII(enc, p) BIG2_BYTE_TO_ASCII(enc, p)
 776 #define CHAR_MATCHES(enc, p, c) BIG2_CHAR_MATCHES(enc, p, c)
 777 #define IS_NAME_CHAR(enc, p, n) 0
 778 #define IS_NAME_CHAR_MINBPC(enc, p) BIG2_IS_NAME_CHAR_MINBPC(enc, p)
 779 #define IS_NMSTRT_CHAR(enc, p, n) (0)
 780 #define IS_NMSTRT_CHAR_MINBPC(enc, p) BIG2_IS_NMSTRT_CHAR_MINBPC(enc, p)
 781
 782 #include "xmltok_impl.c"
 783
 784 #undef MINBPC
 785 #undef BYTE_TYPE
 786 #undef BYTE_TO_ASCII
 787 #undef CHAR_MATCHES
 788 #undef IS_NAME_CHAR
 789 #undef IS_NAME_CHAR_MINBPC
 790 #undef IS_NMSTRT_CHAR
 791 #undef IS_NMSTRT_CHAR_MINBPC
 792 #undef IS_INVALID_CHAR
 793
 794 #endif /* not XML_MIN_SIZE */
 795
 796 #ifdef XML_NS
 797
 798 static const struct normal_encoding big2_encoding_ns = {
 799   { VTABLE, 2, 0,
 800 #if XML_BYTE_ORDER == 21
 801   1
 802 #else
 803   0
 804 #endif
 805   },
 806   {
 807 #include "asciitab.h"
 808 #include "latin1tab.h"
 809   },
 810   STANDARD_VTABLE(big2_)
 811 };
 812
 813 #endif
 814
 815 static const struct normal_encoding big2_encoding = {
 816   { VTABLE, 2, 0,
 817 #if XML_BYTE_ORDER == 21
 818   1
 819 #else
 820   0
 821 #endif
 822   },
 823   {
 824 #define BT_COLON BT_NMSTRT
 825 #include "asciitab.h"
 826 #undef BT_COLON
 827 #include "latin1tab.h"
 828   },
 829   STANDARD_VTABLE(big2_)
 830 };
 831
 832 #if XML_BYTE_ORDER != 12
 833
 834 #ifdef XML_NS
 835
 836 static const struct normal_encoding internal_big2_encoding_ns = {
 837   { VTABLE, 2, 0, 1 },
 838   {
 839 #include "iasciitab.h"
 840 #include "latin1tab.h"
 841   },
 842   STANDARD_VTABLE(big2_)
 843 };
 844
 845 #endif
 846
 847 static const struct normal_encoding internal_big2_encoding = {
 848   { VTABLE, 2, 0, 1 },
 849   {
 850 #define BT_COLON BT_NMSTRT
 851 #include "iasciitab.h"
 852 #undef BT_COLON
 853 #include "latin1tab.h"
 854   },
 855   STANDARD_VTABLE(big2_)
 856 };
 857
 858 #endif
 859
 860 #undef PREFIX
 861
 862 static
 863 int streqci(const char *s1, const char *s2)
 864 {
 865   for (;;) {
 866     char c1 = *s1++;
 867     char c2 = *s2++;
 868     if (ASCII_a <= c1 && c1 <= ASCII_z)
 869       c1 += ASCII_A - ASCII_a;
 870     if (ASCII_a <= c2 && c2 <= ASCII_z)
 871       c2 += ASCII_A - ASCII_a;
 872     if (c1 != c2)
 873       return 0;
 874     if (!c1)
 875       break;
 876   }
 877   return 1;
 878 }
 879
 880 static
 881 void initUpdatePosition(const ENCODING *enc, const char *ptr,
 882                         const char *end, POSITION *pos)
 883 {
 884   normal_updatePosition(&utf8_encoding.enc, ptr, end, pos);
 885 }
 886
 887 static
 888 int toAscii(const ENCODING *enc, const char *ptr, const char *end)
 889 {
 890   char buf[1];
 891   char *p = buf;
 892   XmlUtf8Convert(enc, &ptr, end, &p, p + 1);
 893   if (p == buf)
 894     return -1;
 895   else
 896     return buf[0];
 897 }
 898
 899 static
 900 int isSpace(int c)
 901 {
 902   switch (c) {
 903   case 0x20:
 904   case 0xD:
 905   case 0xA:
 906   case 0x9:
 907     return 1;
 908   }
 909   return 0;
 910 }
 911
 912 /* Return 1 if there's just optional white space
 913 or there's an S followed by name=val. */
 914 static
 915 int parsePseudoAttribute(const ENCODING *enc,
 916                          const char *ptr,
 917                          const char *end,
 918                          const char **namePtr,
 919                          const char **nameEndPtr,
 920                          const char **valPtr,
 921                          const char **nextTokPtr)
 922 {
 923   int c;
 924   char open;
 925   if (ptr == end) {
 926     *namePtr = 0;
 927     return 1;
 928   }
 929   if (!isSpace(toAscii(enc, ptr, end))) {
 930     *nextTokPtr = ptr;
 931     return 0;
 932   }
 933   do {
 934     ptr += enc->minBytesPerChar;
 935   } while (isSpace(toAscii(enc, ptr, end)));
 936   if (ptr == end) {
 937     *namePtr = 0;
 938     return 1;
 939   }
 940   *namePtr = ptr;
 941   for (;;) {
 942     c = toAscii(enc, ptr, end);
 943     if (c == -1) {
 944       *nextTokPtr = ptr;
 945       return 0;
 946     }
 947     if (c == ASCII_EQUALS) {
 948       *nameEndPtr = ptr;
 949       break;
 950     }
 951     if (isSpace(c)) {
 952       *nameEndPtr = ptr;
 953       do {
 954         ptr += enc->minBytesPerChar;
 955       } while (isSpace(c = toAscii(enc, ptr, end)));
 956       if (c != ASCII_EQUALS) {
 957         *nextTokPtr = ptr;
 958         return 0;
 959       }
 960       break;
 961     }
 962     ptr += enc->minBytesPerChar;
 963   }
 964   if (ptr == *namePtr) {
 965     *nextTokPtr = ptr;
 966     return 0;
 967   }
 968   ptr += enc->minBytesPerChar;
 969   c = toAscii(enc, ptr, end);
 970   while (isSpace(c)) {
 971     ptr += enc->minBytesPerChar;
 972     c = toAscii(enc, ptr, end);
 973   }
 974   if (c != ASCII_QUOT && c != ASCII_APOS) {
 975     *nextTokPtr = ptr;
 976     return 0;
 977   }
 978   open = c;
 979   ptr += enc->minBytesPerChar;
 980   *valPtr = ptr;
 981   for (;; ptr += enc->minBytesPerChar) {
 982     c = toAscii(enc, ptr, end);
 983     if (c == open)
 984       break;
 985     if (!(ASCII_a <= c && c <= ASCII_z)
 986         && !(ASCII_A <= c && c <= ASCII_Z)
 987         && !(ASCII_0 <= c && c <= ASCII_9)
 988         && c != ASCII_PERIOD
 989         && c != ASCII_MINUS
 990         && c != ASCII_UNDERSCORE) {
 991       *nextTokPtr = ptr;
 992       return 0;
 993     }
 994   }
 995   *nextTokPtr = ptr + enc->minBytesPerChar;
 996   return 1;
 997 }
 998
 999 static const char KW_version[] = {
1000   ASCII_v, ASCII_e, ASCII_r, ASCII_s, ASCII_i, ASCII_o, ASCII_n, '\0'
1001 };
1002
1003 static const char KW_encoding[] = {
1004   ASCII_e, ASCII_n, ASCII_c, ASCII_o, ASCII_d, ASCII_i, ASCII_n, ASCII_g, '\0'
1005 };
1006
1007 static const char KW_standalone[] = {
1008   ASCII_s, ASCII_t, ASCII_a, ASCII_n, ASCII_d, ASCII_a, ASCII_l, ASCII_o, ASCII_n, ASCII_e, '\0'
1009 };
1010
1011 static const char KW_yes[] = {
1012   ASCII_y, ASCII_e, ASCII_s,  '\0'
1013 };
1014
1015 static const char KW_no[] = {
1016   ASCII_n, ASCII_o,  '\0'
1017 };
1018
1019 static
1020 int doParseXmlDecl(const ENCODING *(*encodingFinder)(const ENCODING *,
1021                                                      const char *,
1022                                                      const char *),
1023                    int isGeneralTextEntity,
1024                    const ENCODING *enc,
1025                    const char *ptr,
1026                    const char *end,
1027                    const char **badPtr,
1028                    const char **versionPtr,
1029                    const char **versionEndPtr,
1030                    const char **encodingName,
1031                    const ENCODING **encoding,
1032                    int *standalone)
1033 {
1034   const char *val = 0;
1035   const char *name = 0;
1036   const char *nameEnd = 0;
1037   ptr += 5 * enc->minBytesPerChar;
1038   end -= 2 * enc->minBytesPerChar;
1039   if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr) || !name) {
1040     *badPtr = ptr;
1041     return 0;
1042   }
1043   if (!XmlNameMatchesAscii(enc, name, nameEnd, KW_version)) {
1044     if (!isGeneralTextEntity) {
1045       *badPtr = name;
1046       return 0;
1047     }
1048   }
1049   else {
1050     if (versionPtr)
1051       *versionPtr = val;
1052     if (versionEndPtr)
1053       *versionEndPtr = ptr;
1054     if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
1055       *badPtr = ptr;
1056       return 0;
1057     }
1058     if (!name) {
1059       if (isGeneralTextEntity) {
1060         /* a TextDecl must have an EncodingDecl */
1061         *badPtr = ptr;
1062         return 0;
1063       }
1064       return 1;
1065     }
1066   }
1067   if (XmlNameMatchesAscii(enc, name, nameEnd, KW_encoding)) {
1068     int c = toAscii(enc, val, end);
1069     if (!(ASCII_a <= c && c <= ASCII_z) && !(ASCII_A <= c && c <= ASCII_Z)) {
1070       *badPtr = val;
1071       return 0;
1072     }
1073     if (encodingName)
1074       *encodingName = val;
1075     if (encoding)
1076       *encoding = encodingFinder(enc, val, ptr - enc->minBytesPerChar);
1077     if (!parsePseudoAttribute(enc, ptr, end, &name, &nameEnd, &val, &ptr)) {
1078       *badPtr = ptr;
1079       return 0;
1080     }
1081     if (!name)
1082       return 1;
1083   }
1084   if (!XmlNameMatchesAscii(enc, name, nameEnd, KW_standalone) || isGeneralTextEntity) {
1085     *badPtr = name;
1086     return 0;
1087   }
1088   if (XmlNameMatchesAscii(enc, val, ptr - enc->minBytesPerChar, KW_yes)) {
1089     if (standalone)
1090       *standalone = 1;
1091   }
1092   else if (XmlNameMatchesAscii(enc, val, ptr - enc->minBytesPerChar, KW_no)) {
1093     if (standalone)
1094       *standalone = 0;
1095   }
1096   else {
1097     *badPtr = val;
1098     return 0;
1099   }
1100   while (isSpace(toAscii(enc, ptr, end)))
1101     ptr += enc->minBytesPerChar;
1102   if (ptr != end) {
1103     *badPtr = ptr;
1104     return 0;
1105   }
1106   return 1;
1107 }
1108
1109 static
1110 int checkCharRefNumber(int result)
1111 {
1112   switch (result >> 8) {
1113   case 0xD8: case 0xD9: case 0xDA: case 0xDB:
1114   case 0xDC: case 0xDD: case 0xDE: case 0xDF:
1115     return -1;
1116   case 0:
1117     if (latin1_encoding.type[result] == BT_NONXML)
1118       return -1;
1119     break;
1120   case 0xFF:
1121     if (result == 0xFFFE || result == 0xFFFF)
1122       return -1;
1123     break;
1124   }
1125   return result;
1126 }
1127
1128 int XmlUtf8Encode(int c, char *buf)
1129 {
1130   enum {
1131     /* minN is minimum legal resulting value for N byte sequence */
1132     min2 = 0x80,
1133     min3 = 0x800,
1134     min4 = 0x10000
1135   };
1136
1137   if (c < 0)
1138     return 0;
1139   if (c < min2) {
1140     buf[0] = (c | UTF8_cval1);
1141     return 1;
1142   }
1143   if (c < min3) {
1144     buf[0] = ((c >> 6) | UTF8_cval2);
1145     buf[1] = ((c & 0x3f) | 0x80);
1146     return 2;
1147   }
1148   if (c < min4) {
1149     buf[0] = ((c >> 12) | UTF8_cval3);
1150     buf[1] = (((c >> 6) & 0x3f) | 0x80);
1151     buf[2] = ((c & 0x3f) | 0x80);
1152     return 3;
1153   }
1154   if (c < 0x110000) {
1155     buf[0] = ((c >> 18) | UTF8_cval4);
1156     buf[1] = (((c >> 12) & 0x3f) | 0x80);
1157     buf[2] = (((c >> 6) & 0x3f) | 0x80);
1158     buf[3] = ((c & 0x3f) | 0x80);
1159     return 4;
1160   }
1161   return 0;
1162 }
1163
1164 int XmlUtf16Encode(int charNum, unsigned short *buf)
1165 {
1166   if (charNum < 0)
1167     return 0;
1168   if (charNum < 0x10000) {
1169     buf[0] = charNum;
1170     return 1;
1171   }
1172   if (charNum < 0x110000) {
1173     charNum -= 0x10000;
1174     buf[0] = (charNum >> 10) + 0xD800;
1175     buf[1] = (charNum & 0x3FF) + 0xDC00;
1176     return 2;
1177   }
1178   return 0;
1179 }
1180
1181 struct unknown_encoding {
1182   struct normal_encoding normal;
1183   int (*convert)(void *userData, const char *p);
1184   void *userData;
1185   unsigned short utf16[256];
1186   char utf8[256][4];
1187 };
1188
1189 int XmlSizeOfUnknownEncoding(void)
1190 {
1191   return sizeof(struct unknown_encoding);
1192 }
1193
1194 static
1195 int unknown_isName(const ENCODING *enc, const char *p)
1196 {
1197   int c = ((const struct unknown_encoding *)enc)
1198           ->convert(((const struct unknown_encoding *)enc)->userData, p);
1199   if (c & ~0xFFFF)
1200     return 0;
1201   return UCS2_GET_NAMING(namePages, c >> 8, c & 0xFF);
1202 }
1203
1204 static
1205 int unknown_isNmstrt(const ENCODING *enc, const char *p)
1206 {
1207   int c = ((const struct unknown_encoding *)enc)
1208           ->convert(((const struct unknown_encoding *)enc)->userData, p);
1209   if (c & ~0xFFFF)
1210     return 0;
1211   return UCS2_GET_NAMING(nmstrtPages, c >> 8, c & 0xFF);
1212 }
1213
1214 static
1215 int unknown_isInvalid(const ENCODING *enc, const char *p)
1216 {
1217   int c = ((const struct unknown_encoding *)enc)
1218            ->convert(((const struct unknown_encoding *)enc)->userData, p);
1219   return (c & ~0xFFFF) || checkCharRefNumber(c) < 0;
1220 }
1221
1222 static
1223 void unknown_toUtf8(const ENCODING *enc,
1224                     const char **fromP, const char *fromLim,
1225                     char **toP, const char *toLim)
1226 {
1227   char buf[XML_UTF8_ENCODE_MAX];
1228   for (;;) {
1229     const char *utf8;
1230     int n;
1231     if (*fromP == fromLim)
1232       break;
1233     utf8 = ((const struct unknown_encoding *)enc)->utf8[(unsigned char)**fromP];
1234     n = *utf8++;
1235     if (n == 0) {
1236       int c = ((const struct unknown_encoding *)enc)
1237               ->convert(((const struct unknown_encoding *)enc)->userData, *fromP);
1238       n = XmlUtf8Encode(c, buf);
1239       if (n > toLim - *toP)
1240         break;
1241       utf8 = buf;
1242       *fromP += ((const struct normal_encoding *)enc)->type[(unsigned char)**fromP]
1243                  - (BT_LEAD2 - 2);
1244     }
1245     else {
1246       if (n > toLim - *toP)
1247         break;
1248       (*fromP)++;
1249     }
1250     do {
1251       *(*toP)++ = *utf8++;
1252     } while (--n != 0);
1253   }
1254 }
1255
1256 static
1257 void unknown_toUtf16(const ENCODING *enc,
1258                      const char **fromP, const char *fromLim,
1259                      unsigned short **toP, const unsigned short *toLim)
1260 {
1261   while (*fromP != fromLim && *toP != toLim) {
1262     unsigned short c
1263       = ((const struct unknown_encoding *)enc)->utf16[(unsigned char)**fromP];
1264     if (c == 0) {
1265       c = (unsigned short)((const struct unknown_encoding *)enc)
1266            ->convert(((const struct unknown_encoding *)enc)->userData, *fromP);
1267       *fromP += ((const struct normal_encoding *)enc)->type[(unsigned char)**fromP]
1268                  - (BT_LEAD2 - 2);
1269     }
1270     else
1271       (*fromP)++;
1272     *(*toP)++ = c;
1273   }
1274 }
1275
1276 ENCODING *
1277 XmlInitUnknownEncoding(void *mem,
1278                        int *table,
1279                        int (*convert)(void *userData, const char *p),
1280                        void *userData)
1281 {
1282   int i;
1283   struct unknown_encoding *e = mem;
1284   for (i = 0; i < (int)sizeof(struct normal_encoding); i++)
1285     ((char *)mem)[i] = ((char *)&latin1_encoding)[i];
1286   for (i = 0; i < 128; i++)
1287     if (latin1_encoding.type[i] != BT_OTHER
1288         && latin1_encoding.type[i] != BT_NONXML
1289         && table[i] != i)
1290       return 0;
1291   for (i = 0; i < 256; i++) {
1292     int c = table[i];
1293     if (c == -1) {
1294       e->normal.type[i] = BT_MALFORM;
1295       /* This shouldn't really get used. */
1296       e->utf16[i] = 0xFFFF;
1297       e->utf8[i][0] = 1;
1298       e->utf8[i][1] = 0;
1299     }
1300     else if (c < 0) {
1301       if (c < -4)
1302         return 0;
1303       e->normal.type[i] = BT_LEAD2 - (c + 2);
1304       e->utf8[i][0] = 0;
1305       e->utf16[i] = 0;
1306     }
1307     else if (c < 0x80) {
1308       if (latin1_encoding.type[c] != BT_OTHER
1309           && latin1_encoding.type[c] != BT_NONXML
1310           && c != i)
1311         return 0;
1312       e->normal.type[i] = latin1_encoding.type[c];
1313       e->utf8[i][0] = 1;
1314       e->utf8[i][1] = (char)c;
1315       e->utf16[i] = c == 0 ? 0xFFFF : c;
1316     }
1317     else if (checkCharRefNumber(c) < 0) {
1318       e->normal.type[i] = BT_NONXML;
1319       /* This shouldn't really get used. */
1320       e->utf16[i] = 0xFFFF;
1321       e->utf8[i][0] = 1;
1322       e->utf8[i][1] = 0;
1323     }
1324     else {
1325       if (c > 0xFFFF)
1326         return 0;
1327       if (UCS2_GET_NAMING(nmstrtPages, c >> 8, c & 0xff))
1328         e->normal.type[i] = BT_NMSTRT;
1329       else if (UCS2_GET_NAMING(namePages, c >> 8, c & 0xff))
1330         e->normal.type[i] = BT_NAME;
1331       else
1332         e->normal.type[i] = BT_OTHER;
1333       e->utf8[i][0] = (char)XmlUtf8Encode(c, e->utf8[i] + 1);
1334       e->utf16[i] = c;
1335     }
1336   }
1337   e->userData = userData;
1338   e->convert = convert;
1339   if (convert) {
1340     e->normal.isName2 = unknown_isName;
1341     e->normal.isName3 = unknown_isName;
1342     e->normal.isName4 = unknown_isName;
1343     e->normal.isNmstrt2 = unknown_isNmstrt;
1344     e->normal.isNmstrt3 = unknown_isNmstrt;
1345     e->normal.isNmstrt4 = unknown_isNmstrt;
1346     e->normal.isInvalid2 = unknown_isInvalid;
1347     e->normal.isInvalid3 = unknown_isInvalid;
1348     e->normal.isInvalid4 = unknown_isInvalid;
1349   }
1350   e->normal.enc.utf8Convert = unknown_toUtf8;
1351   e->normal.enc.utf16Convert = unknown_toUtf16;
1352   return &(e->normal.enc);
1353 }
1354
1355 /* If this enumeration is changed, getEncodingIndex and encodings
1356 must also be changed. */
1357 enum {
1358   UNKNOWN_ENC = -1,
1359   ISO_8859_1_ENC = 0,
1360   US_ASCII_ENC,
1361   UTF_8_ENC,
1362   UTF_16_ENC,
1363   UTF_16BE_ENC,
1364   UTF_16LE_ENC,
1365   /* must match encodingNames up to here */
1366   NO_ENC
1367 };
1368
1369 static const char KW_ISO_8859_1[] = {
1370   ASCII_I, ASCII_S, ASCII_O, ASCII_MINUS, ASCII_8, ASCII_8, ASCII_5, ASCII_9, ASCII_MINUS, ASCII_1, '\0'
1371 };
1372 static const char KW_US_ASCII[] = {
1373   ASCII_U, ASCII_S, ASCII_MINUS, ASCII_A, ASCII_S, ASCII_C, ASCII_I, ASCII_I, '\0'
1374 };
1375 static const char KW_UTF_8[] =  {
1376   ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_8, '\0'
1377 };
1378 static const char KW_UTF_16[] = {
1379   ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, '\0'
1380 };
1381 static const char KW_UTF_16BE[] = {
1382   ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, ASCII_B, ASCII_E, '\0'
1383 };
1384 static const char KW_UTF_16LE[] = {
1385   ASCII_U, ASCII_T, ASCII_F, ASCII_MINUS, ASCII_1, ASCII_6, ASCII_L, ASCII_E, '\0'
1386 };
1387
1388 static
1389 int getEncodingIndex(const char *name)
1390 {
1391   static const char *encodingNames[] = {
1392     KW_ISO_8859_1,
1393     KW_US_ASCII,
1394     KW_UTF_8,
1395     KW_UTF_16,
1396     KW_UTF_16BE,
1397     KW_UTF_16LE,
1398   };
1399   int i;
1400   if (name == 0)
1401     return NO_ENC;
1402   for (i = 0; i < (int)(sizeof(encodingNames)/sizeof(encodingNames[0])); i++)
1403     if (streqci(name, encodingNames[i]))
1404       return i;
1405   return UNKNOWN_ENC;
1406 }
1407
1408 /* For binary compatibility, we store the index of the encoding specified
1409 at initialization in the isUtf16 member. */
1410
1411 #define INIT_ENC_INDEX(enc) ((int)(enc)->initEnc.isUtf16)
1412 #define SET_INIT_ENC_INDEX(enc, i) ((enc)->initEnc.isUtf16 = (char)i)
1413
1414 /* This is what detects the encoding.
1415 encodingTable maps from encoding indices to encodings;
1416 INIT_ENC_INDEX(enc) is the index of the external (protocol) specified encoding;
1417 state is XML_CONTENT_STATE if we're parsing an external text entity,
1418 and XML_PROLOG_STATE otherwise.
1419 */
1420
1421
1422 static
1423 int initScan(const ENCODING **encodingTable,
1424              const INIT_ENCODING *enc,
1425              int state,
1426              const char *ptr,
1427              const char *end,
1428              const char **nextTokPtr)
1429 {
1430   const ENCODING **encPtr;
1431
1432   if (ptr == end)
1433     return XML_TOK_NONE;
1434   encPtr = enc->encPtr;
1435   if (ptr + 1 == end) {
1436     /* only a single byte available for auto-detection */
1437 #ifndef XML_DTD /* FIXME */
1438     /* a well-formed document entity must have more than one byte */
1439     if (state != XML_CONTENT_STATE)
1440       return XML_TOK_PARTIAL;
1441 #endif
1442     /* so we're parsing an external text entity... */
1443     /* if UTF-16 was externally specified, then we need at least 2 bytes */
1444     switch (INIT_ENC_INDEX(enc)) {
1445     case UTF_16_ENC:
1446     case UTF_16LE_ENC:
1447     case UTF_16BE_ENC:
1448       return XML_TOK_PARTIAL;
1449     }
1450     switch ((unsigned char)*ptr) {
1451     case 0xFE:
1452     case 0xFF:
1453     case 0xEF: /* possibly first byte of UTF-8 BOM */
1454       if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
1455           && state == XML_CONTENT_STATE)
1456         break;
1457       /* fall through */
1458     case 0x00:
1459     case 0x3C:
1460       return XML_TOK_PARTIAL;
1461     }
1462   }
1463   else {
1464     switch (((unsigned char)ptr[0] << 8) | (unsigned char)ptr[1]) {
1465     case 0xFEFF:
1466       if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
1467           && state == XML_CONTENT_STATE)
1468         break;
1469       *nextTokPtr = ptr + 2;
1470       *encPtr = encodingTable[UTF_16BE_ENC];
1471       return XML_TOK_BOM;
1472     /* 00 3C is handled in the default case */
1473     case 0x3C00:
1474       if ((INIT_ENC_INDEX(enc) == UTF_16BE_ENC
1475            || INIT_ENC_INDEX(enc) == UTF_16_ENC)
1476           && state == XML_CONTENT_STATE)
1477         break;
1478       *encPtr = encodingTable[UTF_16LE_ENC];
1479       return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
1480     case 0xFFFE:
1481       if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC
1482           && state == XML_CONTENT_STATE)
1483         break;
1484       *nextTokPtr = ptr + 2;
1485       *encPtr = encodingTable[UTF_16LE_ENC];
1486       return XML_TOK_BOM;
1487     case 0xEFBB:
1488       /* Maybe a UTF-8 BOM (EF BB BF) */
1489       /* If there's an explicitly specified (external) encoding
1490          of ISO-8859-1 or some flavour of UTF-16
1491          and this is an external text entity,
1492          don't look for the BOM,
1493          because it might be a legal data. */
1494       if (state == XML_CONTENT_STATE) {
1495         int e = INIT_ENC_INDEX(enc);
1496         if (e == ISO_8859_1_ENC || e == UTF_16BE_ENC || e == UTF_16LE_ENC || e == UTF_16_ENC)
1497           break;
1498       }
1499       if (ptr + 2 == end)
1500         return XML_TOK_PARTIAL;
1501       if ((unsigned char)ptr[2] == 0xBF) {
1502         *nextTokPtr = ptr + 3;
1503         *encPtr = encodingTable[UTF_8_ENC];
1504         return XML_TOK_BOM;
1505       }
1506       break;
1507     default:
1508       if (ptr[0] == '\0') {
1509         /* 0 isn't a legal data character. Furthermore a document entity can only
1510            start with ASCII characters.  So the only way this can fail to be big-endian
1511            UTF-16 if it it's an external parsed general entity that's labelled as
1512            UTF-16LE. */
1513         if (state == XML_CONTENT_STATE && INIT_ENC_INDEX(enc) == UTF_16LE_ENC)
1514           break;
1515         *encPtr = encodingTable[UTF_16BE_ENC];
1516         return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
1517       }
1518       else if (ptr[1] == '\0') {
1519         /* We could recover here in the case:
1520             - parsing an external entity
1521             - second byte is 0
1522             - no externally specified encoding
1523             - no encoding declaration
1524            by assuming UTF-16LE.  But we don't, because this would mean when
1525            presented just with a single byte, we couldn't reliably determine
1526            whether we needed further bytes. */
1527         if (state == XML_CONTENT_STATE)
1528           break;
1529         *encPtr = encodingTable[UTF_16LE_ENC];
1530         return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
1531       }
1532       break;
1533     }
1534   }
1535   *encPtr = encodingTable[INIT_ENC_INDEX(enc)];
1536   return XmlTok(*encPtr, state, ptr, end, nextTokPtr);
1537 }
1538
1539
1540 #define NS(x) x
1541 #define ns(x) x
1542 #include "xmltok_ns.c"
1543 #undef NS
1544 #undef ns
1545
1546 #ifdef XML_NS
1547
1548 #define NS(x) x ## NS
1549 #define ns(x) x ## _ns
1550
1551 #include "xmltok_ns.c"
1552
1553 #undef NS
1554 #undef ns
1555
1556 ENCODING *
1557 XmlInitUnknownEncodingNS(void *mem,
1558                          int *table,
1559                          int (*convert)(void *userData, const char *p),
1560                          void *userData)
1561 {
1562   ENCODING *enc = XmlInitUnknownEncoding(mem, table, convert, userData);
1563   if (enc)
1564     ((struct normal_encoding *)enc)->type[ASCII_COLON] = BT_COLON;
1565   return enc;
1566 }
1567
1568 #endif /* XML_NS */