base/third_party/icu/icu_utf.cc

   1 /*
   2 ******************************************************************************
   3 *
   4 *   Copyright (C) 1999-2006, International Business Machines
   5 *   Corporation and others.  All Rights Reserved.
   6 *
   7 ******************************************************************************
   8 *   file name:  utf_impl.c
   9 *   encoding:   US-ASCII
  10 *   tab size:   8 (not used)
  11 *   indentation:4
  12 *
  13 *   created on: 1999sep13
  14 *   created by: Markus W. Scherer
  15 *
  16 *   This file provides implementation functions for macros in the utfXX.h
  17 *   that would otherwise be too long as macros.
  18 */
  19
  20 #include "base/third_party/icu/icu_utf.h"
  21
  22 namespace base_icu {
  23
  24 /**
  25  * UTF8_ERROR_VALUE_1 and UTF8_ERROR_VALUE_2 are special error values for UTF-8,
  26  * which need 1 or 2 bytes in UTF-8:
  27  * \code
  28  * U+0015 = NAK = Negative Acknowledge, C0 control character
  29  * U+009f = highest C1 control character
  30  * \endcode
  31  *
  32  * These are used by UTF8_..._SAFE macros so that they can return an error value
  33  * that needs the same number of code units (bytes) as were seen by
  34  * a macro. They should be tested with UTF_IS_ERROR() or UTF_IS_VALID().
  35  *
  36  * @deprecated ICU 2.4. Obsolete, see utf_old.h.
  37  */
  38 #define CBUTF8_ERROR_VALUE_1 0x15
  39
  40 /**
  41  * See documentation on UTF8_ERROR_VALUE_1 for details.
  42  *
  43  * @deprecated ICU 2.4. Obsolete, see utf_old.h.
  44  */
  45 #define CBUTF8_ERROR_VALUE_2 0x9f
  46
  47
  48 /**
  49  * Error value for all UTFs. This code point value will be set by macros with e>
  50  * checking if an error is detected.
  51  *
  52  * @deprecated ICU 2.4. Obsolete, see utf_old.h.
  53  */
  54 #define CBUTF_ERROR_VALUE 0xffff
  55
  56 /*
  57  * This table could be replaced on many machines by
  58  * a few lines of assembler code using an
  59  * "index of first 0-bit from msb" instruction and
  60  * one or two more integer instructions.
  61  *
  62  * For example, on an i386, do something like
  63  * - MOV AL, leadByte
  64  * - NOT AL         (8-bit, leave b15..b8==0..0, reverse only b7..b0)
  65  * - MOV AH, 0
  66  * - BSR BX, AX     (16-bit)
  67  * - MOV AX, 6      (result)
  68  * - JZ finish      (ZF==1 if leadByte==0xff)
  69  * - SUB AX, BX (result)
  70  * -finish:
  71  * (BSR: Bit Scan Reverse, scans for a 1-bit, starting from the MSB)
  72  *
  73  * In Unicode, all UTF-8 byte sequences with more than 4 bytes are illegal;
  74  * lead bytes above 0xf4 are illegal.
  75  * We keep them in this table for skipping long ISO 10646-UTF-8 sequences.
  76  */
  77 const uint8
  78 utf8_countTrailBytes[256]={
  79     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  80     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  81     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  82     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  83
  84     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  85     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  86     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  87     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  88
  89     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  90     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  91     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  92     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  93
  94     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  95     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  96
  97     2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  98     3, 3, 3, 3, 3,
  99     3, 3, 3,    /* illegal in Unicode */
 100     4, 4, 4, 4, /* illegal in Unicode */
 101     5, 5,       /* illegal in Unicode */
 102     0, 0        /* illegal bytes 0xfe and 0xff */
 103 };
 104
 105 static const UChar32
 106 utf8_minLegal[4]={ 0, 0x80, 0x800, 0x10000 };
 107
 108 static const UChar32
 109 utf8_errorValue[6]={
 110     CBUTF8_ERROR_VALUE_1, CBUTF8_ERROR_VALUE_2, CBUTF_ERROR_VALUE, 0x10ffff,
 111     0x3ffffff, 0x7fffffff
 112 };
 113
 114 /*
 115  * Handle the non-inline part of the U8_NEXT() macro and its obsolete sibling
 116  * UTF8_NEXT_CHAR_SAFE().
 117  *
 118  * The "strict" parameter controls the error behavior:
 119  * <0  "Safe" behavior of U8_NEXT(): All illegal byte sequences yield a negative
 120  *     code point result.
 121  *  0  Obsolete "safe" behavior of UTF8_NEXT_CHAR_SAFE(..., FALSE):
 122  *     All illegal byte sequences yield a positive code point such that this
 123  *     result code point would be encoded with the same number of bytes as
 124  *     the illegal sequence.
 125  * >0  Obsolete "strict" behavior of UTF8_NEXT_CHAR_SAFE(..., TRUE):
 126  *     Same as the obsolete "safe" behavior, but non-characters are also treated
 127  *     like illegal sequences.
 128  *
 129  * The special negative (<0) value -2 is used for lenient treatment of surrogate
 130  * code points as legal. Some implementations use this for roundtripping of
 131  * Unicode 16-bit strings that are not well-formed UTF-16, that is, they
 132  * contain unpaired surrogates.
 133  *
 134  * Note that a UBool is the same as an int8_t.
 135  */
 136 UChar32
 137 utf8_nextCharSafeBody(const uint8 *s, int32 *pi, int32 length, UChar32 c, UBool strict) {
 138     int32 i=*pi;
 139     uint8 count=CBU8_COUNT_TRAIL_BYTES(c);
 140     if((i)+count<=(length)) {
 141         uint8 trail, illegal=0;
 142
 143         CBU8_MASK_LEAD_BYTE((c), count);
 144         /* count==0 for illegally leading trail bytes and the illegal bytes 0xfe and 0xff */
 145         switch(count) {
 146         /* each branch falls through to the next one */
 147         case 5:
 148         case 4:
 149             /* count>=4 is always illegal: no more than 3 trail bytes in Unicode's UTF-8 */
 150             illegal=1;
 151             break;
 152         case 3:
 153             trail=s[(i)++];
 154             (c)=((c)<<6)|(trail&0x3f);
 155             if(c<0x110) {
 156                 illegal|=(trail&0xc0)^0x80;
 157             } else {
 158                 /* code point>0x10ffff, outside Unicode */
 159                 illegal=1;
 160                 break;
 161             }
 162         case 2:
 163             trail=s[(i)++];
 164             (c)=((c)<<6)|(trail&0x3f);
 165             illegal|=(trail&0xc0)^0x80;
 166         case 1:
 167             trail=s[(i)++];
 168             (c)=((c)<<6)|(trail&0x3f);
 169             illegal|=(trail&0xc0)^0x80;
 170             break;
 171         case 0:
 172             if(strict>=0) {
 173                 return CBUTF8_ERROR_VALUE_1;
 174             } else {
 175                 return CBU_SENTINEL;
 176             }
 177         /* no default branch to optimize switch()  - all values are covered */
 178         }
 179
 180         /*
 181          * All the error handling should return a value
 182          * that needs count bytes so that UTF8_GET_CHAR_SAFE() works right.
 183          *
 184          * Starting with Unicode 3.0.1, non-shortest forms are illegal.
 185          * Starting with Unicode 3.2, surrogate code points must not be
 186          * encoded in UTF-8, and there are no irregular sequences any more.
 187          *
 188          * U8_ macros (new in ICU 2.4) return negative values for error conditions.
 189          */
 190
 191         /* correct sequence - all trail bytes have (b7..b6)==(10)? */
 192         /* illegal is also set if count>=4 */
 193         if(illegal || (c)<utf8_minLegal[count] || (CBU_IS_SURROGATE(c) && strict!=-2)) {
 194             /* error handling */
 195             uint8 errorCount=count;
 196             /* don't go beyond this sequence */
 197             i=*pi;
 198             while(count>0 && CBU8_IS_TRAIL(s[i])) {
 199                 ++(i);
 200                 --count;
 201             }
 202             if(strict>=0) {
 203                 c=utf8_errorValue[errorCount-count];
 204             } else {
 205                 c=CBU_SENTINEL;
 206             }
 207         } else if((strict)>0 && CBU_IS_UNICODE_NONCHAR(c)) {
 208             /* strict: forbid non-characters like U+fffe */
 209             c=utf8_errorValue[count];
 210         }
 211     } else /* too few bytes left */ {
 212         /* error handling */
 213         int32 i0=i;
 214         /* don't just set (i)=(length) in case there is an illegal sequence */
 215         while((i)<(length) && CBU8_IS_TRAIL(s[i])) {
 216             ++(i);
 217         }
 218         if(strict>=0) {
 219             c=utf8_errorValue[i-i0];
 220         } else {
 221             c=CBU_SENTINEL;
 222         }
 223     }
 224     *pi=i;
 225     return c;
 226 }
 227
 228 }  // namespace base_icu