enc/utf_16le.c

   1 /**********************************************************************
   2   utf_16le.c -  Oniguruma (regular expression library)
   3 **********************************************************************/
   4 /*-
   5  * Copyright (c) 2002-2007  K.Kosako  <sndgk393 AT ybb DOT ne DOT jp>
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include "regenc.h"
  31
  32 #define UTF16_IS_SURROGATE_FIRST(c)    (((c) & 0xfc) == 0xd8)
  33 #define UTF16_IS_SURROGATE_SECOND(c)   (((c) & 0xfc) == 0xdc)
  34 #define UTF16_IS_SURROGATE(c)          (((c) & 0xf8) == 0xd8)
  35
  36 static const int EncLen_UTF16[] = {
  37   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  38   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  39   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  40   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  41   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  42   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  43   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  44   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  45   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  46   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  47   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  48   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  49   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  50   2, 2, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 2, 2, 2, 2,
  51   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
  52   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
  53 };
  54
  55 static int
  56 utf16le_mbc_enc_len(const UChar* p, const OnigUChar* e,
  57                     OnigEncoding enc ARG_UNUSED)
  58 {
  59   int len = e-p, byte;
  60   if (len < 2)
  61     return ONIGENC_CONSTRUCT_MBCLEN_NEEDMORE(1);
  62   byte = p[1];
  63   if (!UTF16_IS_SURROGATE(byte)) {
  64     return ONIGENC_CONSTRUCT_MBCLEN_CHARFOUND(2);
  65   }
  66   if (UTF16_IS_SURROGATE_FIRST(byte)) {
  67     if (len < 4)
  68       return ONIGENC_CONSTRUCT_MBCLEN_NEEDMORE(4-len);
  69     if (UTF16_IS_SURROGATE_SECOND(p[3]))
  70       return ONIGENC_CONSTRUCT_MBCLEN_CHARFOUND(4);
  71   }
  72   return ONIGENC_CONSTRUCT_MBCLEN_INVALID();
  73 }
  74
  75 static int
  76 utf16le_is_mbc_newline(const UChar* p, const UChar* end,
  77                        OnigEncoding enc ARG_UNUSED)
  78 {
  79   if (p + 1 < end) {
  80     if (*p == 0x0a && *(p+1) == 0x00)
  81       return 1;
  82 #ifdef USE_UNICODE_ALL_LINE_TERMINATORS
  83     if ((
  84 #ifndef USE_CRNL_AS_LINE_TERMINATOR
  85          *p == 0x0d ||
  86 #endif
  87          *p == 0x85) && *(p+1) == 0x00)
  88       return 1;
  89     if (*(p+1) == 0x20 && (*p == 0x29 || *p == 0x28))
  90       return 1;
  91 #endif
  92   }
  93   return 0;
  94 }
  95
  96 static OnigCodePoint
  97 utf16le_mbc_to_code(const UChar* p, const UChar* end ARG_UNUSED,
  98                     OnigEncoding enc ARG_UNUSED)
  99 {
 100   OnigCodePoint code;
 101   UChar c0 = *p;
 102   UChar c1 = *(p+1);
 103
 104   if (UTF16_IS_SURROGATE_FIRST(c1)) {
 105     code = ((((c1 - 0xd8) << 2) + ((c0  & 0xc0) >> 6) + 1) << 16)
 106          + ((((c0 & 0x3f) << 2) + (p[3] - 0xdc)) << 8)
 107          + p[2];
 108   }
 109   else {
 110     code = c1 * 256 + p[0];
 111   }
 112   return code;
 113 }
 114
 115 static int
 116 utf16le_code_to_mbclen(OnigCodePoint code,
 117                        OnigEncoding enc ARG_UNUSED)
 118 {
 119   return (code > 0xffff ? 4 : 2);
 120 }
 121
 122 static int
 123 utf16le_code_to_mbc(OnigCodePoint code, UChar *buf,
 124                     OnigEncoding enc ARG_UNUSED)
 125 {
 126   UChar* p = buf;
 127
 128   if (code > 0xffff) {
 129     unsigned int high = (code >> 10) + 0xD7C0;
 130     unsigned int low = (code & 0x3FF) + 0xDC00;
 131     *p++ = high & 0xFF;
 132     *p++ = (high >> 8) & 0xFF;
 133     *p++ = low & 0xFF;
 134     *p++ = (low >> 8) & 0xFF;
 135     return 4;
 136   }
 137   else {
 138     *p++ = (UChar )(code & 0xff);
 139     *p++ = (UChar )((code & 0xff00) >> 8);
 140     return 2;
 141   }
 142 }
 143
 144 static int
 145 utf16le_mbc_case_fold(OnigCaseFoldType flag,
 146                       const UChar** pp, const UChar* end, UChar* fold,
 147                       OnigEncoding enc)
 148 {
 149   const UChar* p = *pp;
 150
 151   if (ONIGENC_IS_ASCII_CODE(*p) && *(p+1) == 0) {
 152 #ifdef USE_UNICODE_CASE_FOLD_TURKISH_AZERI
 153     if ((flag & ONIGENC_CASE_FOLD_TURKISH_AZERI) != 0) {
 154       if (*p == 0x49) {
 155         *fold++ = 0x31;
 156         *fold   = 0x01;
 157         (*pp) += 2;
 158         return 2;
 159       }
 160     }
 161 #endif
 162
 163     *fold++ = ONIGENC_ASCII_CODE_TO_LOWER_CASE(*p);
 164     *fold   = 0;
 165     *pp += 2;
 166     return 2;
 167   }
 168   else
 169     return onigenc_unicode_mbc_case_fold(enc, flag, pp,
 170                                          end, fold);
 171 }
 172
 173 #if 0
 174 static int
 175 utf16le_is_mbc_ambiguous(OnigCaseFoldType flag, const UChar** pp,
 176                          const UChar* end)
 177 {
 178   const UChar* p = *pp;
 179
 180   (*pp) += EncLen_UTF16[*(p+1)];
 181
 182   if (*(p+1) == 0) {
 183     int c, v;
 184
 185     if (*p == 0xdf && (flag & INTERNAL_ONIGENC_CASE_FOLD_MULTI_CHAR) != 0) {
 186       return TRUE;
 187     }
 188
 189     c = *p;
 190     v = ONIGENC_IS_UNICODE_ISO_8859_1_BIT_CTYPE(c,
 191                        (BIT_CTYPE_UPPER | BIT_CTYPE_LOWER));
 192     if ((v | BIT_CTYPE_LOWER) != 0) {
 193       /* 0xaa, 0xb5, 0xba are lower case letter, but can't convert. */
 194       if (c >= 0xaa && c <= 0xba)
 195         return FALSE;
 196       else
 197         return TRUE;
 198     }
 199     return (v != 0 ? TRUE : FALSE);
 200   }
 201
 202   return FALSE;
 203 }
 204 #endif
 205
 206 static UChar*
 207 utf16le_left_adjust_char_head(const UChar* start, const UChar* s,
 208                               OnigEncoding enc ARG_UNUSED)
 209 {
 210   if (s <= start) return (UChar* )s;
 211
 212   if ((s - start) % 2 == 1) {
 213     s--;
 214   }
 215
 216   if (UTF16_IS_SURROGATE_SECOND(*(s+1)) && s > start + 1)
 217     s -= 2;
 218
 219   return (UChar* )s;
 220 }
 221
 222 static int
 223 utf16le_get_case_fold_codes_by_str(OnigCaseFoldType flag,
 224                                    const OnigUChar* p, const OnigUChar* end,
 225                                    OnigCaseFoldCodeItem items[],
 226                                    OnigEncoding enc)
 227 {
 228   return onigenc_unicode_get_case_fold_codes_by_str(enc,
 229                                                     flag, p, end, items);
 230 }
 231
 232 OnigEncodingDefine(utf_16le, UTF_16LE) = {
 233   utf16le_mbc_enc_len,
 234   "UTF-16LE",   /* name */
 235   4,            /* max byte length */
 236   2,            /* min byte length */
 237   utf16le_is_mbc_newline,
 238   utf16le_mbc_to_code,
 239   utf16le_code_to_mbclen,
 240   utf16le_code_to_mbc,
 241   utf16le_mbc_case_fold,
 242   onigenc_unicode_apply_all_case_fold,
 243   utf16le_get_case_fold_codes_by_str,
 244   onigenc_unicode_property_name_to_ctype,
 245   onigenc_unicode_is_code_ctype,
 246   onigenc_utf16_32_get_ctype_code_range,
 247   utf16le_left_adjust_char_head,
 248   onigenc_always_false_is_allowed_reverse_match
 249 };