transcode.c

   1 /**********************************************************************
   2
   3   transcode.c -
   4
   5   $Author$
   6   created at: Tue Oct 30 16:10:22 JST 2007
   7
   8   Copyright (C) 2007 Martin Duerst
   9
  10 **********************************************************************/
  11
  12 #include "ruby/ruby.h"
  13 #include "ruby/encoding.h"
  14 #define PType (int)
  15 #include "transcode_data.h"
  16 #include <ctype.h>
  17
  18 static VALUE sym_invalid, sym_undef, sym_ignore, sym_replace;
  19 #define INVALID_IGNORE 0x1
  20 #define INVALID_REPLACE 0x2
  21 #define UNDEF_IGNORE 0x10
  22 #define UNDEF_REPLACE 0x20
  23
  24 /*
  25  *  Dispatch data and logic
  26  */
  27
  28 static st_table *transcoder_table, *transcoder_lib_table;
  29
  30 #define TRANSCODER_INTERNAL_SEPARATOR '\t'
  31
  32 static char *
  33 transcoder_key(const char *from_e, const char *to_e)
  34 {
  35     int to_len = strlen(to_e);
  36     int from_len = strlen(from_e);
  37     char *const key = xmalloc(to_len + from_len + 2);
  38
  39     memcpy(key, to_e, to_len);
  40     memcpy(key + to_len + 1, from_e, from_len + 1);
  41     key[to_len] = TRANSCODER_INTERNAL_SEPARATOR;
  42     return key;
  43 }
  44
  45 void
  46 rb_register_transcoder(const rb_transcoder *tr)
  47 {
  48     st_data_t k, val = 0;
  49     const char *const from_e = tr->from_encoding;
  50     const char *const to_e = tr->to_encoding;
  51     char *const key = transcoder_key(from_e, to_e);
  52
  53     if (st_lookup(transcoder_table, (st_data_t)key, &val)) {
  54         xfree(key);
  55         rb_raise(rb_eArgError, "transcoder from %s to %s has been already registered",
  56                  from_e, to_e);
  57     }
  58     k = (st_data_t)key;
  59     if (st_delete(transcoder_lib_table, &k, &val)) {
  60         xfree((char *)k);
  61     }
  62     st_insert(transcoder_table, (st_data_t)key, (st_data_t)tr);
  63 }
  64
  65 static void
  66 declare_transcoder(const char *to, const char *from, const char *lib)
  67 {
  68     const char *const key = transcoder_key(to, from);
  69     st_data_t k = (st_data_t)key, val;
  70
  71     if (st_delete(transcoder_lib_table, &k, &val)) {
  72         xfree((char *)k);
  73     }
  74     st_insert(transcoder_lib_table, (st_data_t)key, (st_data_t)lib);
  75 }
  76
  77 #define MAX_TRANSCODER_LIBNAME_LEN 64
  78 static const char transcoder_lib_prefix[] = "enc/trans/";
  79
  80 void
  81 rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib)
  82 {
  83     if (!lib || strlen(lib) > MAX_TRANSCODER_LIBNAME_LEN) {
  84         rb_raise(rb_eArgError, "invalid library name - %s",
  85                  lib ? lib : "(null)");
  86     }
  87     declare_transcoder(enc1, enc2, lib);
  88     declare_transcoder(enc2, enc1, lib);
  89 }
  90
  91 #define encoding_equal(enc1, enc2) (STRCASECMP(enc1, enc2) == 0)
  92
  93 static const rb_transcoder *
  94 transcode_dispatch(const char *from_encoding, const char *to_encoding)
  95 {
  96     char *const key = transcoder_key(from_encoding, to_encoding);
  97     st_data_t k, val = 0;
  98
  99     while (!st_lookup(transcoder_table, (k = (st_data_t)key), &val) &&
 100            st_delete(transcoder_lib_table, &k, &val)) {
 101         const char *const lib = (const char *)val;
 102         int len = strlen(lib);
 103         char path[sizeof(transcoder_lib_prefix) + MAX_TRANSCODER_LIBNAME_LEN];
 104
 105         xfree((char *)k);
 106         if (len > MAX_TRANSCODER_LIBNAME_LEN) return NULL;
 107         memcpy(path, transcoder_lib_prefix, sizeof(transcoder_lib_prefix) - 1);
 108         memcpy(path + sizeof(transcoder_lib_prefix) - 1, lib, len + 1);
 109         if (!rb_require(path)) return NULL;
 110     }
 111     if (!val) {
 112         if (!st_lookup(transcoder_table, (st_data_t)key, &val)) {
 113             xfree(key);
 114             /* multistep logic, via UTF-8 */
 115             if (!encoding_equal(from_encoding, "UTF-8") &&
 116                 !encoding_equal(to_encoding, "UTF-8") &&
 117                 transcode_dispatch("UTF-8", to_encoding)) {  /* check that we have a second step */
 118                 return transcode_dispatch(from_encoding, "UTF-8"); /* return first step */
 119             }
 120             return NULL;
 121         }
 122     }
 123     xfree(key);
 124     return (rb_transcoder *)val;
 125 }
 126
 127 static void
 128 output_replacement_character(unsigned char **out_pp, rb_encoding *enc)
 129 {
 130     unsigned char *out_p = *out_pp;
 131     static rb_encoding *utf16be_encoding, *utf16le_encoding;
 132     static rb_encoding *utf32be_encoding, *utf32le_encoding;
 133     if (!utf16be_encoding) {
 134         utf16be_encoding = rb_enc_find("UTF-16BE");
 135         utf16le_encoding = rb_enc_find("UTF-16LE");
 136         utf32be_encoding = rb_enc_find("UTF-32BE");
 137         utf32le_encoding = rb_enc_find("UTF-32LE");
 138     }
 139     if (rb_utf8_encoding() == enc) {
 140         *out_p++ = 0xEF;
 141         *out_p++ = 0xBF;
 142         *out_p++ = 0xBD;
 143     }
 144     else if (utf16be_encoding == enc) {
 145         *out_p++ = 0xFF;
 146         *out_p++ = 0xFD;
 147     }
 148     else if (utf16le_encoding == enc) {
 149         *out_p++ = 0xFD;
 150         *out_p++ = 0xFF;
 151     }
 152     else if (utf32be_encoding == enc) {
 153         *out_p++ = 0x00;
 154         *out_p++ = 0x00;
 155         *out_p++ = 0xFF;
 156         *out_p++ = 0xFD;
 157     }
 158     else if (utf32le_encoding == enc) {
 159         *out_p++ = 0xFD;
 160         *out_p++ = 0xFF;
 161         *out_p++ = 0x00;
 162         *out_p++ = 0x00;
 163     }
 164     else {
 165         *out_p++ = '?';
 166     }
 167     *out_pp = out_p;
 168     return;
 169 }
 170
 171 /*
 172  *  Transcoding engine logic
 173  */
 174 static void
 175 transcode_loop(const unsigned char **in_pos, unsigned char **out_pos,
 176                const unsigned char *in_stop, unsigned char *out_stop,
 177                const rb_transcoder *my_transcoder,
 178                rb_transcoding *my_transcoding,
 179                const int opt)
 180 {
 181     const unsigned char *in_p = *in_pos;
 182     unsigned char *out_p = *out_pos;
 183     const BYTE_LOOKUP *conv_tree_start = my_transcoder->conv_tree_start;
 184     const BYTE_LOOKUP *next_table;
 185     const unsigned char *char_start;
 186     unsigned int next_offset;
 187     VALUE next_info;
 188     unsigned char next_byte;
 189     int from_utf8 = my_transcoder->from_utf8;
 190     unsigned char *out_s = out_stop - my_transcoder->max_output + 1;
 191     rb_encoding *from_encoding = rb_enc_find(my_transcoder->from_encoding);
 192     rb_encoding *to_encoding = rb_enc_find(my_transcoder->to_encoding);
 193
 194     while (in_p < in_stop) {
 195         char_start = in_p;
 196         next_table = conv_tree_start;
 197         if (out_p >= out_s) {
 198             int len = (out_p - *out_pos);
 199             int new_len = (len + my_transcoder->max_output) * 2;
 200             *out_pos = (*my_transcoding->flush_func)(my_transcoding, len, new_len);
 201             out_p = *out_pos + len;
 202             out_s = *out_pos + new_len - my_transcoder->max_output;
 203         }
 204         next_byte = (unsigned char)*in_p++;
 205       follow_byte:
 206         next_offset = next_table->base[next_byte];
 207         next_info = (VALUE)next_table->info[next_offset];
 208       follow_info:
 209         switch (next_info & 0x1F) {
 210           case NOMAP:
 211             *out_p++ = next_byte;
 212             continue;
 213           case 0x00: case 0x04: case 0x08: case 0x0C:
 214           case 0x10: case 0x14: case 0x18: case 0x1C:
 215             if (in_p >= in_stop) {
 216                 /* todo: deal with the case of backtracking */
 217                 /* todo: deal with incomplete input (streaming) */
 218                 goto invalid;
 219             }
 220             next_byte = (unsigned char)*in_p++;
 221             if (from_utf8) {
 222                 if ((next_byte&0xC0) == 0x80)
 223                     next_byte -= 0x80;
 224                 else {
 225                     in_p--; /* may need to add more code later to revert other things */
 226                     goto invalid;
 227                 }
 228             }
 229             next_table = (const BYTE_LOOKUP *)next_info;
 230             goto follow_byte;
 231             /* maybe rewrite the following cases to use fallthrough???? */
 232           case ZERObt: /* drop input */
 233             continue;
 234           case ONEbt:
 235             *out_p++ = getBT1(next_info);
 236             continue;
 237           case TWObt:
 238             *out_p++ = getBT1(next_info);
 239             *out_p++ = getBT2(next_info);
 240             continue;
 241           case FOURbt:
 242             *out_p++ = getBT0(next_info);
 243           case THREEbt: /* fall through */
 244             *out_p++ = getBT1(next_info);
 245             *out_p++ = getBT2(next_info);
 246             *out_p++ = getBT3(next_info);
 247             continue;
 248           case FUNii:
 249             next_info = (VALUE)(*my_transcoder->func_ii)(next_info);
 250             goto follow_info;
 251           case FUNsi:
 252             next_info = (VALUE)(*my_transcoder->func_si)(char_start);
 253             goto follow_info;
 254             break;
 255           case FUNio:
 256             out_p += (VALUE)(*my_transcoder->func_io)(next_info, out_p);
 257             break;
 258           case FUNso:
 259             out_p += (VALUE)(*my_transcoder->func_so)(char_start, out_p);
 260             break;
 261           case INVALID:
 262             goto invalid;
 263           case UNDEF:
 264             goto undef;
 265         }
 266         continue;
 267       invalid:
 268         /* deal with invalid byte sequence */
 269         /* todo: add more alternative behaviors */
 270         if (opt&INVALID_IGNORE) {
 271             continue;
 272         }
 273         else if (opt&INVALID_REPLACE) {
 274             output_replacement_character(&out_p, to_encoding);
 275             continue;
 276         }
 277         rb_raise(TRANSCODE_ERROR, "invalid byte sequence");
 278         continue;
 279       undef:
 280         /* valid character in from encoding
 281          * but no related character(s) in to encoding */
 282         /* todo: add more alternative behaviors */
 283         {
 284             int len = rb_enc_mbclen((const char *)char_start, (const char *)in_stop, from_encoding);
 285             while (in_p < char_start + len) in_p++;
 286         }
 287         if (opt&UNDEF_IGNORE) {
 288             continue;
 289         }
 290         else if (opt&UNDEF_REPLACE) {
 291             output_replacement_character(&out_p, to_encoding);
 292             continue;
 293         }
 294         rb_raise(TRANSCODE_ERROR, "conversion undefined for byte sequence (maybe invalid byte sequence)");
 295         continue;
 296     }
 297     /* cleanup */
 298     *in_pos  = in_p;
 299     *out_pos = out_p;
 300 }
 301
 302
 303 /*
 304  *  String-specific code
 305  */
 306
 307 static unsigned char *
 308 str_transcoding_resize(rb_transcoding *my_transcoding, int len, int new_len)
 309 {
 310     VALUE dest_string = my_transcoding->ruby_string_dest;
 311     rb_str_resize(dest_string, new_len);
 312     return (unsigned char *)RSTRING_PTR(dest_string);
 313 }
 314
 315 static int
 316 str_transcode(int argc, VALUE *argv, VALUE *self)
 317 {
 318     VALUE dest;
 319     VALUE str = *self;
 320     long blen, slen;
 321     unsigned char *buf, *bp, *sp;
 322     const unsigned char *fromp;
 323     rb_encoding *from_enc, *to_enc;
 324     const char *from_e, *to_e;
 325     int from_encidx, to_encidx;
 326     VALUE from_encval, to_encval;
 327     const rb_transcoder *my_transcoder;
 328     rb_transcoding my_transcoding;
 329     int final_encoding = 0;
 330     VALUE opt;
 331     int options = 0;
 332
 333     opt = rb_check_convert_type(argv[argc-1], T_HASH, "Hash", "to_hash");
 334     if (!NIL_P(opt)) {
 335         VALUE v;
 336
 337         argc--;
 338         v = rb_hash_aref(opt, sym_invalid);
 339         if (NIL_P(v)) {
 340         }
 341         else if (v==sym_ignore) {
 342             options |= INVALID_IGNORE;
 343         }
 344         else if (v==sym_replace) {
 345             options |= INVALID_REPLACE;
 346             v = rb_hash_aref(opt, sym_replace);
 347         }
 348         else {
 349             rb_raise(rb_eArgError, "unknown value for invalid: setting");
 350         }
 351         v = rb_hash_aref(opt, sym_undef);
 352         if (NIL_P(v)) {
 353         }
 354         else if (v==sym_ignore) {
 355             options |= UNDEF_IGNORE;
 356         }
 357         else if (v==sym_replace) {
 358             options |= UNDEF_REPLACE;
 359         }
 360         else {
 361             rb_raise(rb_eArgError, "unknown value for undef: setting");
 362         }
 363     }
 364     if (argc < 1 || argc > 2) {
 365         rb_raise(rb_eArgError, "wrong number of arguments (%d for 1..2)", argc);
 366     }
 367     if ((to_encidx = rb_to_encoding_index(to_encval = argv[0])) < 0) {
 368         to_enc = 0;
 369         to_encidx = 0;
 370         to_e = StringValueCStr(to_encval);
 371     }
 372     else {
 373         to_enc = rb_enc_from_index(to_encidx);
 374         to_e = rb_enc_name(to_enc);
 375     }
 376     if (argc==1) {
 377         from_encidx = rb_enc_get_index(str);
 378         from_enc = rb_enc_from_index(from_encidx);
 379         from_e = rb_enc_name(from_enc);
 380     }
 381     else if ((from_encidx = rb_to_encoding_index(from_encval = argv[1])) < 0) {
 382         from_enc = 0;
 383         from_e = StringValueCStr(from_encval);
 384     }
 385     else {
 386         from_enc = rb_enc_from_index(from_encidx);
 387         from_e = rb_enc_name(from_enc);
 388     }
 389
 390     if (from_enc && from_enc == to_enc) {
 391         return -1;
 392     }
 393     if (from_enc && to_enc && rb_enc_asciicompat(from_enc) && rb_enc_asciicompat(to_enc)) {
 394         if (ENC_CODERANGE(str) == ENC_CODERANGE_7BIT) {
 395             return to_encidx;
 396         }
 397     }
 398     if (encoding_equal(from_e, to_e)) {
 399         return -1;
 400     }
 401
 402     do { /* loop for multistep transcoding */
 403         /* later, maybe use smaller intermediate strings for very long strings */
 404         if (!(my_transcoder = transcode_dispatch(from_e, to_e))) {
 405             rb_raise(rb_eArgError, "transcoding not supported (from %s to %s)", from_e, to_e);
 406         }
 407
 408         my_transcoding.transcoder = my_transcoder;
 409
 410         if (my_transcoder->preprocessor) {
 411             fromp = sp = (unsigned char *)RSTRING_PTR(str);
 412             slen = RSTRING_LEN(str);
 413             blen = slen + 30; /* len + margin */
 414             dest = rb_str_tmp_new(blen);
 415             bp = (unsigned char *)RSTRING_PTR(dest);
 416             my_transcoding.ruby_string_dest = dest;
 417             (*my_transcoder->preprocessor)(&fromp, &bp, (sp+slen), (bp+blen), &my_transcoding);
 418             if (fromp != sp+slen) {
 419                 rb_raise(rb_eArgError, "not fully converted, %"PRIdPTRDIFF" bytes left", sp+slen-fromp);
 420             }
 421             rb_str_set_len(dest, (char *)bp - RSTRING_PTR(dest));
 422             str = dest;
 423         }
 424         fromp = sp = (unsigned char *)RSTRING_PTR(str);
 425         slen = RSTRING_LEN(str);
 426         blen = slen + 30; /* len + margin */
 427         dest = rb_str_tmp_new(blen);
 428         bp = (unsigned char *)RSTRING_PTR(dest);
 429         my_transcoding.ruby_string_dest = dest;
 430         my_transcoding.flush_func = str_transcoding_resize;
 431
 432         transcode_loop(&fromp, &bp, (sp+slen), (bp+blen), my_transcoder, &my_transcoding, options);
 433         if (fromp != sp+slen) {
 434             rb_raise(rb_eArgError, "not fully converted, %"PRIdPTRDIFF" bytes left", sp+slen-fromp);
 435         }
 436         buf = (unsigned char *)RSTRING_PTR(dest);
 437         *bp = '\0';
 438         rb_str_set_len(dest, bp - buf);
 439         if (my_transcoder->postprocessor) {
 440             str = dest;
 441             fromp = sp = (unsigned char *)RSTRING_PTR(str);
 442             slen = RSTRING_LEN(str);
 443             blen = slen + 30; /* len + margin */
 444             dest = rb_str_tmp_new(blen);
 445             bp = (unsigned char *)RSTRING_PTR(dest);
 446             my_transcoding.ruby_string_dest = dest;
 447             (*my_transcoder->postprocessor)(&fromp, &bp, (sp+slen), (bp+blen), &my_transcoding);
 448             if (fromp != sp+slen) {
 449                 rb_raise(rb_eArgError, "not fully converted, %"PRIdPTRDIFF" bytes left", sp+slen-fromp);
 450             }
 451             buf = (unsigned char *)RSTRING_PTR(dest);
 452             rb_str_set_len(dest, bp - buf);
 453         }
 454
 455         if (encoding_equal(my_transcoder->to_encoding, to_e)) {
 456             final_encoding = 1;
 457         }
 458         else {
 459             from_e = my_transcoder->to_encoding;
 460             str = dest;
 461         }
 462     } while (!final_encoding);
 463     /* set encoding */
 464     if (!to_enc) {
 465         to_encidx = rb_define_dummy_encoding(to_e);
 466     }
 467     *self = dest;
 468
 469     return to_encidx;
 470 }
 471
 472 static inline VALUE
 473 str_encode_associate(VALUE str, int encidx)
 474 {
 475     int cr = 0;
 476
 477     rb_enc_associate_index(str, encidx);
 478
 479     /* transcoded string never be broken. */
 480     if (rb_enc_asciicompat(rb_enc_from_index(encidx))) {
 481         rb_str_coderange_scan_restartable(RSTRING_PTR(str), RSTRING_END(str), 0, &cr);
 482     }
 483     else {
 484         cr = ENC_CODERANGE_VALID;
 485     }
 486     ENC_CODERANGE_SET(str, cr);
 487     return str;
 488 }
 489
 490 /*
 491  *  call-seq:
 492  *     str.encode!(encoding [, options] )   => str
 493  *     str.encode!(to_encoding, from_encoding [, options] )   => str
 494  *
 495  *  The first form transcodes the contents of <i>str</i> from
 496  *  str.encoding to +encoding+.
 497  *  The second form transcodes the contents of <i>str</i> from
 498  *  from_encoding to to_encoding.
 499  *  The options Hash gives details for conversion. See String#encode
 500  *  for details.
 501  *  Returns the string even if no changes were made.
 502  */
 503
 504 static VALUE
 505 str_encode_bang(int argc, VALUE *argv, VALUE str)
 506 {
 507     VALUE newstr = str;
 508     int encidx = str_transcode(argc, argv, &newstr);
 509
 510     if (encidx < 0) return str;
 511     rb_str_shared_replace(str, newstr);
 512     return str_encode_associate(str, encidx);
 513 }
 514
 515 /*
 516  *  call-seq:
 517  *     str.encode(encoding [, options] )   => str
 518  *     str.encode(to_encoding, from_encoding [, options] )   => str
 519  *
 520  *  The first form returns a copy of <i>str</i> transcoded
 521  *  to encoding +encoding+.
 522  *  The second form returns a copy of <i>str</i> transcoded
 523  *  from from_encoding to to_encoding.
 524  *  The options Hash gives details for conversion. Details
 525  *  to be added.
 526  */
 527
 528 static VALUE
 529 str_encode(int argc, VALUE *argv, VALUE str)
 530 {
 531     VALUE newstr = str;
 532     int encidx = str_transcode(argc, argv, &newstr);
 533
 534     if (encidx < 0) return rb_str_dup(str);
 535     RBASIC(newstr)->klass = rb_obj_class(str);
 536     return str_encode_associate(newstr, encidx);
 537 }
 538
 539 VALUE
 540 rb_str_transcode(VALUE str, VALUE to)
 541 {
 542     return str_encode(1, &to, str);
 543 }
 544
 545 void
 546 Init_transcode(void)
 547 {
 548     transcoder_table = st_init_strcasetable();
 549     transcoder_lib_table = st_init_strcasetable();
 550
 551     sym_invalid = ID2SYM(rb_intern("invalid"));
 552     sym_undef = ID2SYM(rb_intern("undef"));
 553     sym_ignore = ID2SYM(rb_intern("ignore"));
 554     sym_replace = ID2SYM(rb_intern("replace"));
 555
 556     rb_define_method(rb_cString, "encode", str_encode, -1);
 557     rb_define_method(rb_cString, "encode!", str_encode_bang, -1);
 558 }