firmware/common/unicode.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (c) 2004,2005 by Marcoen Hirschberg
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21 /*   Some conversion functions for handling UTF-8
  22  *
  23  *   I got all the info from:
  24  *   http://www.cl.cam.ac.uk/~mgk25/unicode.html#utf-8
  25  *   and
  26  *   http://en.wikipedia.org/wiki/Unicode
  27  */
  28
  29 #include <stdio.h>
  30 #include "file.h"
  31 #include "debug.h"
  32 #include "rbunicode.h"
  33 #include "config.h"
  34
  35 #ifndef O_BINARY
  36 #define O_BINARY 0
  37 #endif
  38
  39 #define CODEPAGE_DIR    ROCKBOX_DIR"/codepages"
  40 static int default_codepage = 0;
  41 static int loaded_cp_table = 0;
  42
  43 #ifdef HAVE_LCD_BITMAP
  44
  45 #define MAX_CP_TABLE_SIZE  32768
  46 #define NUM_TABLES             5
  47
  48 static const char *filename[NUM_TABLES] =
  49 {
  50     CODEPAGE_DIR"/iso.cp",
  51     CODEPAGE_DIR"/932.cp",  /* SJIS    */
  52     CODEPAGE_DIR"/936.cp",  /* GB2312  */
  53     CODEPAGE_DIR"/949.cp",  /* KSX1001 */
  54     CODEPAGE_DIR"/950.cp"   /* BIG5    */
  55 };
  56
  57 static const char cp_2_table[NUM_CODEPAGES] =
  58 {
  59     0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 4, 5, 0
  60 };
  61
  62 static const char *name_codepages[NUM_CODEPAGES+1] =
  63 {
  64     "ISO-8859-1",
  65     "ISO-8859-7",
  66     "ISO-8859-8",
  67     "CP1251",
  68     "ISO-8859-11",
  69     "CP1256",
  70     "ISO-8859-9",
  71     "ISO-8859-2",
  72     "CP1250",
  73     "SJIS",
  74     "GB-2312",
  75     "KSX-1001",
  76     "BIG5",
  77     "UTF-8",
  78     "unknown"
  79 };
  80
  81 #else /* !HAVE_LCD_BITMAP, reduced support */
  82
  83 #define MAX_CP_TABLE_SIZE  640
  84 #define NUM_TABLES           1
  85
  86 static const char *filename[NUM_TABLES] = {
  87     CODEPAGE_DIR"/isomini.cp"
  88 };
  89
  90 static const char cp_2_table[NUM_CODEPAGES] =
  91 {
  92     0, 1, 1, 1, 1, 1, 0
  93 };
  94
  95 static const char *name_codepages[NUM_CODEPAGES+1] =
  96 {
  97     "ISO-8859-1",
  98     "ISO-8859-7",
  99     "CP1251",
 100     "ISO-8859-9",
 101     "ISO-8859-2",
 102     "CP1250",
 103     "UTF-8",
 104     "unknown"
 105 };
 106
 107 #endif
 108
 109 static unsigned short codepage_table[MAX_CP_TABLE_SIZE];
 110
 111 static const unsigned char utf8comp[6] =
 112 {
 113     0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
 114 };
 115
 116 /* Load codepage file into memory */
 117 static int load_cp_table(int cp)
 118 {
 119     int i=0;
 120     int table = cp_2_table[cp];
 121     int file, tablesize;
 122     unsigned char tmp[2];
 123
 124     if (table == 0 || table == loaded_cp_table)
 125         return 1;
 126
 127     file = open(filename[table-1], O_RDONLY|O_BINARY);
 128
 129     if (file < 0) {
 130         DEBUGF("Can't open codepage file: %s.cp\n", filename[table-1]);
 131         return 0;
 132     }
 133
 134     tablesize = filesize(file) / 2;
 135
 136     if (tablesize > MAX_CP_TABLE_SIZE) {
 137         DEBUGF("Invalid codepage file: %s.cp\n", filename[table-1]);
 138         close(file);
 139         return 0;
 140     }
 141
 142     while (i < tablesize) {
 143         if (!read(file, tmp, 2)) {
 144             DEBUGF("Can't read from codepage file: %s.cp\n",
 145                     filename[table-1]);
 146             loaded_cp_table = 0;
 147             return 0;
 148         }
 149         codepage_table[i++] = (tmp[1] << 8) | tmp[0];
 150     }
 151
 152     loaded_cp_table = table;
 153     close(file);
 154     return 1;
 155 }
 156
 157 /* Encode a UCS value as UTF-8 and return a pointer after this UTF-8 char. */
 158 unsigned char* utf8encode(unsigned long ucs, unsigned char *utf8)
 159 {
 160     int tail = 0;
 161
 162     if (ucs > 0x7F)
 163         while (ucs >> (5*tail + 6))
 164             tail++;
 165
 166     *utf8++ = (ucs >> (6*tail)) | utf8comp[tail];
 167     while (tail--)
 168         *utf8++ = ((ucs >> (6*tail)) & (MASK ^ 0xFF)) | COMP;
 169
 170     return utf8;
 171 }
 172
 173 /* Recode an iso encoded string to UTF-8 */
 174 unsigned char* iso_decode(const unsigned char *iso, unsigned char *utf8,
 175                           int cp, int count)
 176 {
 177     unsigned short ucs, tmp;
 178
 179     if (cp == -1) /* use default codepage */
 180         cp = default_codepage;
 181
 182     if (!load_cp_table(cp)) cp = 0;
 183
 184     while (count--) {
 185         if (*iso < 128 || cp == UTF_8) /* Already UTF-8 */
 186             *utf8++ = *iso++;
 187
 188         else {
 189
 190             /* cp tells us which codepage to convert from */
 191             switch (cp) {
 192                 case ISO_8859_7:  /* Greek */
 193                 case WIN_1251:    /* Cyrillic */
 194                 case ISO_8859_9:  /* Turkish */
 195                 case ISO_8859_2:  /* Latin Extended */
 196                 case WIN_1250:    /* Central European */
 197 #ifdef HAVE_LCD_BITMAP
 198                 case ISO_8859_8:  /* Hebrew */
 199                 case ISO_8859_11: /* Thai */
 200                 case WIN_1256:    /* Arabic */
 201 #endif
 202                     tmp = ((cp-1)*128) + (*iso++ - 128);
 203                     ucs = codepage_table[tmp];
 204                     break;
 205
 206 #ifdef HAVE_LCD_BITMAP
 207                 case SJIS: /* Japanese */
 208                     if (*iso > 0xA0 && *iso < 0xE0) {
 209                         tmp = *iso++ | (0xA100 - 0x8000);
 210                         ucs = codepage_table[tmp];
 211                         break;
 212                     }
 213
 214                 case GB_2312:  /* Simplified Chinese */
 215                 case KSX_1001: /* Korean */
 216                 case BIG_5:    /* Traditional Chinese */
 217                     if (count < 1 || !iso[1]) {
 218                         ucs = *iso++;
 219                         break;
 220                     }
 221
 222                     /* we assume all cjk strings are written
 223                        in big endian order */
 224                     tmp = *iso++ << 8;
 225                     tmp |= *iso++;
 226                     tmp -= 0x8000;
 227                     ucs = codepage_table[tmp];
 228                     count--;
 229                     break;
 230 #endif /* HAVE_LCD_BITMAP */
 231
 232                 default:
 233                     ucs = *iso++;
 234                     break;
 235             }
 236
 237             if (ucs == 0) /* unknown char, use replacement char */
 238                 ucs = 0xfffd;
 239             utf8 = utf8encode(ucs, utf8);
 240         }
 241     }
 242     return utf8;
 243 }
 244
 245 /* Recode a UTF-16 string with little-endian byte ordering to UTF-8 */
 246 unsigned char* utf16LEdecode(const unsigned char *utf16, unsigned char *utf8,
 247         int count)
 248 {
 249     unsigned long ucs;
 250
 251     while (count > 0) {
 252         /* Check for a surrogate pair */
 253         if (utf16[1] >= 0xD8 && utf16[1] < 0xE0) {
 254             ucs = 0x10000 + ((utf16[0] << 10) | ((utf16[1] - 0xD8) << 18)
 255                     | utf16[2] | ((utf16[3] - 0xDC) << 8));
 256             utf16 += 4;
 257             count -= 2;
 258         } else {
 259             ucs = (utf16[0] | (utf16[1] << 8));
 260             utf16 += 2;
 261             count -= 1;
 262         }
 263         utf8 = utf8encode(ucs, utf8);
 264     }
 265     return utf8;
 266 }
 267
 268 /* Recode a UTF-16 string with big-endian byte ordering to UTF-8 */
 269 unsigned char* utf16BEdecode(const unsigned char *utf16, unsigned char *utf8,
 270         int count)
 271 {
 272     unsigned long ucs;
 273
 274     while (count > 0) {
 275         if (*utf16 >= 0xD8 && *utf16 < 0xE0) { /* Check for a surrogate pair */
 276             ucs = 0x10000 + (((utf16[0] - 0xD8) << 18) | (utf16[1] << 10)
 277                     | ((utf16[2] - 0xDC) << 8) | utf16[3]);
 278             utf16 += 4;
 279             count -= 2;
 280         } else {
 281             ucs = (utf16[0] << 8) | utf16[1];
 282             utf16 += 2;
 283             count -= 1;
 284         }
 285         utf8 = utf8encode(ucs, utf8);
 286     }
 287     return utf8;
 288 }
 289
 290 #if 0 /* currently unused */
 291 /* Recode any UTF-16 string to UTF-8 */
 292 unsigned char* utf16decode(const unsigned char *utf16, unsigned char *utf8,
 293         unsigned int count)
 294 {
 295     unsigned long ucs;
 296
 297     ucs = *(utf16++) << 8;
 298     ucs |= *(utf16++);
 299
 300     if (ucs == 0xFEFF) /* Check for BOM */
 301         return utf16BEdecode(utf16, utf8, count-1);
 302     else if (ucs == 0xFFFE)
 303         return utf16LEdecode(utf16, utf8, count-1);
 304     else { /* ADDME: Should default be LE or BE? */
 305         utf16 -= 2;
 306         return utf16BEdecode(utf16, utf8, count);
 307     }
 308 }
 309 #endif
 310
 311 /* Return the number of UTF-8 chars in a string */
 312 unsigned long utf8length(const unsigned char *utf8)
 313 {
 314     unsigned long l = 0;
 315
 316     while (*utf8 != 0)
 317         if ((*utf8++ & MASK) != COMP)
 318             l++;
 319
 320     return l;
 321 }
 322
 323 /* Decode 1 UTF-8 char and return a pointer to the next char. */
 324 const unsigned char* utf8decode(const unsigned char *utf8, unsigned short *ucs)
 325 {
 326     unsigned char c = *utf8++;
 327     unsigned long code;
 328     int tail = 0;
 329
 330     if ((c <= 0x7f) || (c >= 0xc2)) {
 331         /* Start of new character. */
 332         if (c < 0x80) {        /* U-00000000 - U-0000007F, 1 byte */
 333             code = c;
 334         } else if (c < 0xe0) { /* U-00000080 - U-000007FF, 2 bytes */
 335             tail = 1;
 336             code = c & 0x1f;
 337         } else if (c < 0xf0) { /* U-00000800 - U-0000FFFF, 3 bytes */
 338             tail = 2;
 339             code = c & 0x0f;
 340         } else if (c < 0xf5) { /* U-00010000 - U-001FFFFF, 4 bytes */
 341             tail = 3;
 342             code = c & 0x07;
 343         } else {
 344             /* Invalid size. */
 345             code = 0xfffd;
 346         }
 347
 348         while (tail-- && ((c = *utf8++) != 0)) {
 349             if ((c & 0xc0) == 0x80) {
 350                 /* Valid continuation character. */
 351                 code = (code << 6) | (c & 0x3f);
 352
 353             } else {
 354                 /* Invalid continuation char */
 355                 code = 0xfffd;
 356                 utf8--;
 357                 break;
 358             }
 359         }
 360     } else {
 361         /* Invalid UTF-8 char */
 362         code = 0xfffd;
 363     }
 364     /* currently we don't support chars above U-FFFF */
 365     *ucs = (code < 0x10000) ? code : 0xfffd;
 366     return utf8;
 367 }
 368
 369 void set_codepage(int cp)
 370 {
 371     default_codepage = cp;
 372     return;
 373 }
 374
 375 /* seek to a given char in a utf8 string and
 376    return its start position in the string */
 377 int utf8seek(const unsigned char* utf8, int offset)
 378 {
 379     int pos = 0;
 380
 381     while (offset--) {
 382         pos++;
 383         while ((utf8[pos] & MASK) == COMP)
 384             pos++;
 385     }
 386     return pos;
 387 }
 388
 389 const char* get_codepage_name(int cp)
 390 {
 391     if (cp < 0 || cp>= NUM_CODEPAGES)
 392         return name_codepages[NUM_CODEPAGES];
 393     return name_codepages[cp];
 394 }