encoding.c

   1 /**********************************************************************
   2
   3   encoding.c -
   4
   5   $Author$
   6   created at: Thu May 24 17:23:27 JST 2007
   7
   8   Copyright (C) 2007 Yukihiro Matsumoto
   9
  10 **********************************************************************/
  11
  12 #include "ruby/ruby.h"
  13 #include "ruby/encoding.h"
  14 #include "regenc.h"
  15 #include <ctype.h>
  16 #ifdef HAVE_LANGINFO_H
  17 #include <langinfo.h>
  18 #endif
  19
  20 static ID id_encoding, id_base_encoding;
  21 VALUE rb_cEncoding;
  22 static VALUE rb_encoding_list;
  23
  24 struct rb_encoding_entry {
  25     const char *name;
  26     rb_encoding *enc;
  27     rb_encoding *base;
  28 };
  29
  30 static struct {
  31     struct rb_encoding_entry *list;
  32     int count;
  33     int size;
  34     st_table *names;
  35 } enc_table;
  36
  37 void rb_enc_init(void);
  38
  39 #define ENCODING_COUNT ENCINDEX_BUILTIN_MAX
  40
  41 #define enc_autoload_p(enc) (!rb_enc_mbmaxlen(enc))
  42
  43 static int load_encoding(const char *name);
  44 static VALUE enc_base_encoding(VALUE self);
  45
  46 static void
  47 enc_mark(void *ptr)
  48 {
  49 }
  50
  51 static VALUE
  52 enc_new(rb_encoding *encoding)
  53 {
  54     return Data_Wrap_Struct(rb_cEncoding, enc_mark, 0, encoding);
  55 }
  56
  57 VALUE
  58 rb_enc_from_encoding(rb_encoding *encoding)
  59 {
  60     VALUE list, enc;
  61     int idx;
  62
  63     if (!encoding) return Qnil;
  64     idx = ENC_TO_ENCINDEX(encoding);
  65     if (!(list = rb_encoding_list)) {
  66         rb_bug("rb_enc_from_encoding(%d\"%s\"): no rb_encoding_list",
  67                idx, rb_enc_name(encoding));
  68     }
  69     enc = rb_ary_entry(list, idx);
  70     if (NIL_P(enc)) {
  71         rb_bug("rb_enc_from_encoding(%d\"%s\"): not created yet",
  72                idx, rb_enc_name(encoding));
  73     }
  74     return enc;
  75 }
  76
  77 static int enc_autoload(rb_encoding *);
  78
  79 static int
  80 check_encoding(rb_encoding *enc)
  81 {
  82     int index = rb_enc_to_index(enc);
  83     if (rb_enc_from_index(index) != enc)
  84         return -1;
  85     if (enc_autoload_p(enc)) {
  86         index = enc_autoload(enc);
  87     }
  88     return index;
  89 }
  90
  91 static int
  92 enc_check_encoding(VALUE obj)
  93 {
  94     if (SPECIAL_CONST_P(obj) || BUILTIN_TYPE(obj) != T_DATA ||
  95         RDATA(obj)->dmark != enc_mark) {
  96         return -1;
  97     }
  98     return check_encoding(RDATA(obj)->data);
  99 }
 100
 101 static int
 102 must_encoding(VALUE enc)
 103 {
 104     int index = enc_check_encoding(enc);
 105     if (index < 0) {
 106         rb_raise(rb_eTypeError, "wrong argument type %s (expected Encoding)",
 107                  rb_obj_classname(enc));
 108     }
 109     return index;
 110 }
 111
 112 int
 113 rb_to_encoding_index(VALUE enc)
 114 {
 115     int idx;
 116
 117     idx = enc_check_encoding(enc);
 118     if (idx >= 0) {
 119         return idx;
 120     }
 121     else if (NIL_P(enc = rb_check_string_type(enc))) {
 122         return -1;
 123     }
 124     if (!rb_enc_asciicompat(rb_enc_get(enc))) {
 125         return -1;
 126     }
 127     return rb_enc_find_index(StringValueCStr(enc));
 128 }
 129
 130 static rb_encoding *
 131 to_encoding(VALUE enc)
 132 {
 133     int idx;
 134
 135     StringValue(enc);
 136     if (!rb_enc_asciicompat(rb_enc_get(enc))) {
 137         rb_raise(rb_eArgError, "invalid name encoding (non ASCII)");
 138     }
 139     idx = rb_enc_find_index(StringValueCStr(enc));
 140     if (idx < 0) {
 141         rb_raise(rb_eArgError, "unknown encoding name - %s", RSTRING_PTR(enc));
 142     }
 143     return rb_enc_from_index(idx);
 144 }
 145
 146 rb_encoding *
 147 rb_to_encoding(VALUE enc)
 148 {
 149     if (enc_check_encoding(enc) >= 0) return RDATA(enc)->data;
 150     return to_encoding(enc);
 151 }
 152
 153 void
 154 rb_gc_mark_encodings(void)
 155 {
 156 }
 157
 158 static int
 159 enc_table_expand(int newsize)
 160 {
 161     struct rb_encoding_entry *ent;
 162     int count = newsize;
 163
 164     if (enc_table.size >= newsize) return newsize;
 165     newsize = (newsize + 7) / 8 * 8;
 166     ent = realloc(enc_table.list, sizeof(*enc_table.list) * newsize);
 167     if (!ent) return -1;
 168     memset(ent + enc_table.size, 0, sizeof(*ent)*(newsize - enc_table.size));
 169     enc_table.list = ent;
 170     enc_table.size = newsize;
 171     return count;
 172 }
 173
 174 static int
 175 enc_register_at(int index, const char *name, rb_encoding *encoding)
 176 {
 177     struct rb_encoding_entry *ent = &enc_table.list[index];
 178     VALUE list;
 179
 180     if (!ent->name) {
 181         ent->name = name = strdup(name);
 182     }
 183     else if (STRCASECMP(name, ent->name)) {
 184         return -1;
 185     }
 186     if (!ent->enc) {
 187         ent->enc = xmalloc(sizeof(rb_encoding));
 188     }
 189     if (encoding) {
 190         *ent->enc = *encoding;
 191     }
 192     else {
 193         memset(ent->enc, 0, sizeof(*ent->enc));
 194     }
 195     encoding = ent->enc;
 196     encoding->name = name;
 197     encoding->ruby_encoding_index = index;
 198     st_insert(enc_table.names, (st_data_t)name, (st_data_t)index);
 199     list = rb_encoding_list;
 200     if (list && NIL_P(rb_ary_entry(list, index))) {
 201         /* initialize encoding data */
 202         rb_ary_store(list, index, enc_new(encoding));
 203     }
 204     return index;
 205 }
 206
 207 static int
 208 enc_register(const char *name, rb_encoding *encoding)
 209 {
 210     int index = enc_table.count;
 211
 212     if ((index = enc_table_expand(index + 1)) < 0) return -1;
 213     enc_table.count = index;
 214     return enc_register_at(index - 1, name, encoding);
 215 }
 216
 217 static void set_encoding_const(const char *, rb_encoding *);
 218 int rb_enc_registered(const char *name);
 219
 220 int
 221 rb_enc_register(const char *name, rb_encoding *encoding)
 222 {
 223     int index = rb_enc_registered(name);
 224
 225     if (index >= 0) {
 226         rb_encoding *oldenc = rb_enc_from_index(index);
 227         if (STRCASECMP(name, rb_enc_name(oldenc))) {
 228             index = enc_register(name, encoding);
 229         }
 230         else if (enc_autoload_p(oldenc) || !ENC_DUMMY_P(oldenc)) {
 231             enc_register_at(index, name, encoding);
 232         }
 233         else {
 234             rb_raise(rb_eArgError, "encoding %s is already registered", name);
 235         }
 236     }
 237     else {
 238         index = enc_register(name, encoding);
 239         set_encoding_const(name, rb_enc_from_index(index));
 240     }
 241     return index;
 242 }
 243
 244 void
 245 rb_encdb_declare(const char *name)
 246 {
 247     int idx = rb_enc_registered(name);
 248     if (idx < 0) {
 249         idx = enc_register(name, 0);
 250     }
 251     set_encoding_const(name, rb_enc_from_index(idx));
 252 }
 253
 254 static void
 255 enc_check_duplication(const char *name)
 256 {
 257     if (rb_enc_registered(name) >= 0) {
 258         rb_raise(rb_eArgError, "encoding %s is already registered", name);
 259     }
 260 }
 261
 262 static rb_encoding*
 263 set_base_encoding(int index, rb_encoding *base)
 264 {
 265     rb_encoding *enc = enc_table.list[index].enc;
 266
 267     enc_table.list[index].base = base;
 268     if (rb_enc_dummy_p(base)) ENC_SET_DUMMY(enc);
 269     return enc;
 270 }
 271
 272 int
 273 rb_enc_replicate(const char *name, rb_encoding *encoding)
 274 {
 275     int idx;
 276
 277     enc_check_duplication(name);
 278     idx = enc_register(name, encoding);
 279     set_base_encoding(idx, encoding);
 280     set_encoding_const(name, rb_enc_from_index(idx));
 281     return idx;
 282 }
 283
 284 static int
 285 enc_replicate(int idx, const char *name, rb_encoding *origenc)
 286 {
 287     if (idx < 0) {
 288         idx = enc_register(name, origenc);
 289     }
 290     else {
 291         idx = enc_register_at(idx, name, origenc);
 292     }
 293     if (idx >= 0) {
 294         set_base_encoding(idx, origenc);
 295         set_encoding_const(name, rb_enc_from_index(idx));
 296     }
 297     return idx;
 298 }
 299
 300 int
 301 rb_encdb_replicate(const char *name, const char *orig)
 302 {
 303     int origidx = rb_enc_registered(orig);
 304     int idx = rb_enc_registered(name);
 305
 306     if (origidx < 0) {
 307         origidx = enc_register(orig, 0);
 308     }
 309     return enc_replicate(idx, name, rb_enc_from_index(origidx));
 310 }
 311
 312 int
 313 rb_define_dummy_encoding(const char *name)
 314 {
 315     int index = rb_enc_replicate(name, rb_ascii8bit_encoding());
 316     rb_encoding *enc = enc_table.list[index].enc;
 317
 318     ENC_SET_DUMMY(enc);
 319     return index;
 320 }
 321
 322 int
 323 rb_encdb_dummy(const char *name)
 324 {
 325     int index = enc_replicate(rb_enc_registered(name), name,
 326                               rb_ascii8bit_encoding());
 327     rb_encoding *enc = enc_table.list[index].enc;
 328
 329     ENC_SET_DUMMY(enc);
 330     return index;
 331 }
 332
 333 /*
 334  * call-seq:
 335  *   enc.dummy? => true or false
 336  *
 337  * Returns true for dummy encodings.
 338  * A dummy encoding is an encoding for which character handling is not properly
 339  * implemented.
 340  * It is used for stateful encodings.
 341  *
 342  *   Encoding::ISO_2022_JP.dummy?       #=> true
 343  *   Encoding::UTF_8.dummy?             #=> false
 344  *
 345  */
 346 static VALUE
 347 enc_dummy_p(VALUE enc)
 348 {
 349     return ENC_DUMMY_P(enc_table.list[must_encoding(enc)].enc) ? Qtrue : Qfalse;
 350 }
 351
 352 static int
 353 enc_alias(const char *alias, int idx)
 354 {
 355     alias = strdup(alias);
 356     st_insert(enc_table.names, (st_data_t)alias, (st_data_t)idx);
 357     set_encoding_const(alias, rb_enc_from_index(idx));
 358     return idx;
 359 }
 360
 361 int
 362 rb_enc_alias(const char *alias, const char *orig)
 363 {
 364     int idx;
 365
 366     enc_check_duplication(alias);
 367     if (!enc_table.list) {
 368         rb_enc_init();
 369     }
 370     if ((idx = rb_enc_find_index(orig)) < 0) {
 371         return -1;
 372     }
 373     return enc_alias(alias, idx);
 374 }
 375
 376 int
 377 rb_encdb_alias(const char *alias, const char *orig)
 378 {
 379     int idx = rb_enc_registered(orig);
 380
 381     if (idx < 0) {
 382         idx = enc_register(orig, 0);
 383     }
 384     return enc_alias(alias, idx);
 385 }
 386
 387 enum {
 388     ENCINDEX_ASCII,
 389     ENCINDEX_UTF_8,
 390     ENCINDEX_US_ASCII,
 391     ENCINDEX_BUILTIN_MAX
 392 };
 393
 394 extern rb_encoding OnigEncodingUTF_8;
 395 extern rb_encoding OnigEncodingUS_ASCII;
 396
 397 void
 398 rb_enc_init(void)
 399 {
 400     enc_table_expand(ENCODING_COUNT + 1);
 401     if (!enc_table.names) {
 402         enc_table.names = st_init_strcasetable();
 403     }
 404 #define ENC_REGISTER(enc) enc_register_at(ENCINDEX_##enc, rb_enc_name(&OnigEncoding##enc), &OnigEncoding##enc)
 405     ENC_REGISTER(ASCII);
 406     ENC_REGISTER(UTF_8);
 407     ENC_REGISTER(US_ASCII);
 408 #undef ENC_REGISTER
 409     enc_table.count = ENCINDEX_BUILTIN_MAX;
 410 }
 411
 412 rb_encoding *
 413 rb_enc_from_index(int index)
 414 {
 415     if (!enc_table.list) {
 416         rb_enc_init();
 417     }
 418     if (index < 0 || enc_table.count <= index) {
 419         return 0;
 420     }
 421     return enc_table.list[index].enc;
 422 }
 423
 424 int
 425 rb_enc_registered(const char *name)
 426 {
 427     st_data_t idx = 0;
 428
 429     if (!name) return -1;
 430     if (!enc_table.list) return -1;
 431     if (st_lookup(enc_table.names, (st_data_t)name, &idx)) {
 432         return (int)idx;
 433     }
 434     return -1;
 435 }
 436
 437 static VALUE
 438 require_enc(VALUE enclib)
 439 {
 440     return rb_require_safe(enclib, rb_safe_level());
 441 }
 442
 443 static int
 444 load_encoding(const char *name)
 445 {
 446     VALUE enclib = rb_sprintf("enc/%s", name);
 447     VALUE verbose = ruby_verbose;
 448     VALUE debug = ruby_debug;
 449     VALUE loaded;
 450     char *s = RSTRING_PTR(enclib) + 4, *e = RSTRING_END(enclib);
 451     int idx;
 452
 453     while (s < e) {
 454         if (!ISALNUM(*s)) *s = '_';
 455         else if (ISUPPER(*s)) *s = TOLOWER(*s);
 456         ++s;
 457     }
 458     OBJ_FREEZE(enclib);
 459     ruby_verbose = Qfalse;
 460     ruby_debug = Qfalse;
 461     loaded = rb_protect(require_enc, enclib, 0);
 462     ruby_verbose = verbose;
 463     ruby_debug = debug;
 464     rb_set_errinfo(Qnil);
 465     if (NIL_P(loaded)) return -1;
 466     if ((idx = rb_enc_registered(name)) < 0) return -1;
 467     if (enc_autoload_p(enc_table.list[idx].enc)) return -1;
 468     return idx;
 469 }
 470
 471 static int
 472 enc_autoload(rb_encoding *enc)
 473 {
 474     int i;
 475     rb_encoding *base = enc_table.list[ENC_TO_ENCINDEX(enc)].base;
 476
 477     if (base) {
 478         i = 0;
 479         do {
 480             if (i >= enc_table.count) return -1;
 481         } while (enc_table.list[i].enc != base && (++i, 1));
 482         if (enc_autoload_p(base)) {
 483             if (enc_autoload(base) < 0) return -1;
 484         }
 485         i = ENC_TO_ENCINDEX(enc);
 486         enc_register_at(i, rb_enc_name(enc), base);
 487     }
 488     else {
 489         i = load_encoding(rb_enc_name(enc));
 490     }
 491     return i;
 492 }
 493
 494 int
 495 rb_enc_find_index(const char *name)
 496 {
 497     int i = rb_enc_registered(name);
 498     rb_encoding *enc;
 499
 500     if (i < 0) {
 501         i = load_encoding(name);
 502     }
 503     else if (enc_autoload_p(enc = rb_enc_from_index(i))) {
 504         if (enc_autoload(enc) < 0) {
 505             rb_warn("failed to load encoding (%s); use ASCII-8BIT instead",
 506                     name);
 507             return 0;
 508         }
 509     }
 510     return i;
 511 }
 512
 513 rb_encoding *
 514 rb_enc_find(const char *name)
 515 {
 516     int idx = rb_enc_find_index(name);
 517     if (idx < 0) idx = 0;
 518     return rb_enc_from_index(idx);
 519 }
 520
 521 static inline int
 522 enc_capable(VALUE obj)
 523 {
 524     if (SPECIAL_CONST_P(obj)) return Qfalse;
 525     switch (BUILTIN_TYPE(obj)) {
 526       case T_STRING:
 527       case T_REGEXP:
 528       case T_FILE:
 529         return Qtrue;
 530       case T_DATA:
 531         if (RDATA(obj)->dmark == enc_mark) return Qtrue;
 532       default:
 533         return Qfalse;
 534     }
 535 }
 536
 537 ID
 538 rb_id_encoding(void)
 539 {
 540     CONST_ID(id_encoding, "encoding");
 541     return id_encoding;
 542 }
 543
 544 int
 545 rb_enc_get_index(VALUE obj)
 546 {
 547     int i;
 548
 549     i = ENCODING_GET_INLINED(obj);
 550     if (i == ENCODING_INLINE_MAX) {
 551         VALUE iv;
 552
 553         iv = rb_ivar_get(obj, rb_id_encoding());
 554         i = NUM2INT(iv);
 555     }
 556     return i;
 557 }
 558
 559 void
 560 rb_enc_set_index(VALUE obj, int idx)
 561 {
 562     if (idx < ENCODING_INLINE_MAX) {
 563         ENCODING_SET_INLINED(obj, idx);
 564         return;
 565     }
 566     ENCODING_SET_INLINED(obj, ENCODING_INLINE_MAX);
 567     rb_ivar_set(obj, rb_id_encoding(), INT2NUM(idx));
 568     return;
 569 }
 570
 571 VALUE
 572 rb_enc_associate_index(VALUE obj, int idx)
 573 {
 574 /*    enc_check_capable(obj);*/
 575     if (rb_enc_get_index(obj) == idx)
 576         return obj;
 577     if (!ENC_CODERANGE_ASCIIONLY(obj) ||
 578         !rb_enc_asciicompat(rb_enc_from_index(idx))) {
 579         ENC_CODERANGE_CLEAR(obj);
 580     }
 581     rb_enc_set_index(obj, idx);
 582     return obj;
 583 }
 584
 585 VALUE
 586 rb_enc_associate(VALUE obj, rb_encoding *enc)
 587 {
 588     return rb_enc_associate_index(obj, rb_enc_to_index(enc));
 589 }
 590
 591 rb_encoding*
 592 rb_enc_get(VALUE obj)
 593 {
 594     return rb_enc_from_index(rb_enc_get_index(obj));
 595 }
 596
 597 rb_encoding*
 598 rb_enc_check(VALUE str1, VALUE str2)
 599 {
 600     rb_encoding *enc = rb_enc_compatible(str1, str2);
 601     if (!enc)
 602         rb_raise(rb_eArgError, "character encodings differ: %s and %s",
 603                  rb_enc_name(rb_enc_get(str1)),
 604                  rb_enc_name(rb_enc_get(str2)));
 605     return enc;
 606 }
 607
 608 rb_encoding*
 609 rb_enc_compatible(VALUE str1, VALUE str2)
 610 {
 611     int idx1, idx2;
 612     rb_encoding *enc1, *enc2;
 613
 614     idx1 = rb_enc_get_index(str1);
 615     idx2 = rb_enc_get_index(str2);
 616
 617     if (idx1 < 0 || idx2 < 0)
 618         return 0;
 619
 620     if (idx1 == idx2) {
 621         return rb_enc_from_index(idx1);
 622     }
 623     enc1 = rb_enc_from_index(idx1);
 624     enc2 = rb_enc_from_index(idx2);
 625
 626     if (TYPE(str2) == T_STRING && RSTRING_LEN(str2) == 0)
 627         return enc1;
 628     if (TYPE(str1) == T_STRING && RSTRING_LEN(str1) == 0)
 629         return enc2;
 630     if (!rb_enc_asciicompat(enc1) || !rb_enc_asciicompat(enc2)) {
 631         return 0;
 632     }
 633
 634     if (BUILTIN_TYPE(str1) != T_STRING) {
 635         VALUE tmp = str1;
 636         int idx0 = idx1;
 637         str1 = str2;
 638         str2 = tmp;
 639         idx1 = idx2;
 640         idx2 = idx0;
 641     }
 642     if (BUILTIN_TYPE(str1) == T_STRING) {
 643         int cr1, cr2;
 644
 645         cr1 = rb_enc_str_coderange(str1);
 646         if (BUILTIN_TYPE(str2) == T_STRING) {
 647             cr2 = rb_enc_str_coderange(str2);
 648             if (cr1 != cr2) {
 649                 /* may need to handle ENC_CODERANGE_BROKEN */
 650                 if (cr1 == ENC_CODERANGE_7BIT) return enc2;
 651                 if (cr2 == ENC_CODERANGE_7BIT) return enc1;
 652             }
 653             if (cr2 == ENC_CODERANGE_7BIT) {
 654                 if (idx1 == 0) return enc2;
 655                 return enc1;
 656             }
 657         }
 658         if (cr1 == ENC_CODERANGE_7BIT)
 659             return enc2;
 660     }
 661     return 0;
 662 }
 663
 664 void
 665 rb_enc_copy(VALUE obj1, VALUE obj2)
 666 {
 667     rb_enc_associate_index(obj1, rb_enc_get_index(obj2));
 668 }
 669
 670
 671 /*
 672  *  call-seq:
 673  *     obj.encoding   => encoding
 674  *
 675  *  Returns the Encoding object that represents the encoding of obj.
 676  */
 677
 678 VALUE
 679 rb_obj_encoding(VALUE obj)
 680 {
 681     rb_encoding *enc = rb_enc_get(obj);
 682     if (!enc) {
 683         rb_raise(rb_eTypeError, "unknown encoding");
 684     }
 685     return rb_enc_from_encoding(enc);
 686 }
 687
 688 int
 689 rb_enc_mbclen(const char *p, const char *e, rb_encoding *enc)
 690 {
 691     int n = ONIGENC_PRECISE_MBC_ENC_LEN(enc, (UChar*)p, (UChar*)e);
 692     if (MBCLEN_CHARFOUND_P(n) && MBCLEN_CHARFOUND_LEN(n) <= e-p)
 693         return MBCLEN_CHARFOUND_LEN(n);
 694     else {
 695         int min = rb_enc_mbminlen(enc);
 696         return min <= e-p ? min : e-p;
 697     }
 698 }
 699
 700 int
 701 rb_enc_precise_mbclen(const char *p, const char *e, rb_encoding *enc)
 702 {
 703     int n;
 704     if (e <= p)
 705         return ONIGENC_CONSTRUCT_MBCLEN_NEEDMORE(1);
 706     n = ONIGENC_PRECISE_MBC_ENC_LEN(enc, (UChar*)p, (UChar*)e);
 707     if (e-p < n)
 708         return ONIGENC_CONSTRUCT_MBCLEN_NEEDMORE(n-(e-p));
 709     return n;
 710 }
 711
 712 int
 713 rb_enc_ascget(const char *p, const char *e, int *len, rb_encoding *enc)
 714 {
 715     int c, l;
 716     if (e <= p)
 717         return -1;
 718     if (rb_enc_asciicompat(enc)) {
 719         c = (unsigned char)*p;
 720         if (!ISASCII(c))
 721             return -1;
 722         if (len) *len = 1;
 723         return c;
 724     }
 725     l = rb_enc_precise_mbclen(p, e, enc);
 726     if (!MBCLEN_CHARFOUND_P(l))
 727         return -1;
 728     c = rb_enc_mbc_to_codepoint(p, e, enc);
 729     if (!rb_enc_isascii(c, enc))
 730         return -1;
 731     if (len) *len = l;
 732     return c;
 733 }
 734
 735 int
 736 rb_enc_codepoint(const char *p, const char *e, rb_encoding *enc)
 737 {
 738     int r;
 739     if (e <= p)
 740         rb_raise(rb_eArgError, "empty string");
 741     r = rb_enc_precise_mbclen(p, e, enc);
 742     if (MBCLEN_CHARFOUND_P(r))
 743         return rb_enc_mbc_to_codepoint(p, e, enc);
 744     else
 745         rb_raise(rb_eArgError, "invalid mbstring sequence");
 746 }
 747
 748 int
 749 rb_enc_codelen(int c, rb_encoding *enc)
 750 {
 751     int n = ONIGENC_CODE_TO_MBCLEN(enc,c);
 752     if (n == 0) {
 753         rb_raise(rb_eArgError, "invalid codepoint 0x%x", c);
 754     }
 755     return n;
 756 }
 757
 758 int
 759 rb_enc_toupper(int c, rb_encoding *enc)
 760 {
 761     return (ONIGENC_IS_ASCII_CODE(c)?ONIGENC_ASCII_CODE_TO_UPPER_CASE(c):(c));
 762 }
 763
 764 int
 765 rb_enc_tolower(int c, rb_encoding *enc)
 766 {
 767     return (ONIGENC_IS_ASCII_CODE(c)?ONIGENC_ASCII_CODE_TO_LOWER_CASE(c):(c));
 768 }
 769
 770 /*
 771  * call-seq:
 772  *   enc.inspect => string
 773  *
 774  * Returns a string which represents the encoding for programmers.
 775  *
 776  *   Encoding::UTF_8.inspect       #=> "#<Encoding:UTF-8>"
 777  *   Encoding::ISO_2022_JP.inspect #=> "#<Encoding:ISO-2022-JP (dummy)>"
 778  */
 779 static VALUE
 780 enc_inspect(VALUE self)
 781 {
 782     VALUE str = rb_sprintf("#<%s:%s%s>", rb_obj_classname(self),
 783                       rb_enc_name((rb_encoding*)DATA_PTR(self)),
 784                       (enc_dummy_p(self) ? " (dummy)" : ""));
 785     ENCODING_CODERANGE_SET(str, rb_usascii_encindex(), ENC_CODERANGE_7BIT);
 786     return str;
 787 }
 788
 789 /*
 790  * call-seq:
 791  *   enc.name => string
 792  *
 793  * Returns the name of the encoding.
 794  *
 795  *   Encoding::UTF_8.name       => "UTF-8"
 796  */
 797 static VALUE
 798 enc_name(VALUE self)
 799 {
 800     return rb_usascii_str_new2(rb_enc_name((rb_encoding*)DATA_PTR(self)));
 801 }
 802
 803 static VALUE
 804 enc_base_encoding(VALUE self)
 805 {
 806     rb_encoding *base = enc_table.list[must_encoding(self)].base;
 807     if (!base) return Qnil;
 808     return ENC_FROM_ENCODING(base);
 809 }
 810
 811 /*
 812  * call-seq:
 813  *   Encoding.list => [enc1, enc2, ...]
 814  *
 815  * Returns the list of loaded encodings.
 816  *
 817  *   Encoding.list
 818  *   => [#<Encoding:ASCII-8BIT>, #<Encoding:UTF-8>,
 819  *       #<Encoding:ISO-2022-JP (dummy)>]
 820  *
 821  *   Encoding.find("US-ASCII")
 822  *   => #<Encoding:US-ASCII>
 823  *
 824  *   Encoding.list
 825  *   => [#<Encoding:ASCII-8BIT>, #<Encoding:UTF-8>,
 826  *       #<Encoding:US-ASCII>, #<Encoding:ISO-2022-JP (dummy)>]
 827  *
 828  */
 829 static VALUE
 830 enc_list(VALUE klass)
 831 {
 832     VALUE ary = rb_ary_new2(0);
 833     rb_ary_replace(ary, rb_encoding_list);
 834     return ary;
 835 }
 836
 837 /*
 838  * call-seq:
 839  *   Encoding.find(string) => enc
 840  *   Encoding.find(symbol) => enc
 841  *
 842  * Search the encoding with specified <i>name</i>.
 843  * <i>name</i> should be a string or symbol.
 844  *
 845  *   Encoding.find("US-ASCII")  => #<Encoding:US-ASCII>
 846  *   Encoding.find(:Shift_JIS)  => #<Encoding:Shift_JIS>
 847  *
 848  */
 849 static VALUE
 850 enc_find(VALUE klass, VALUE enc)
 851 {
 852     return rb_enc_from_encoding(to_encoding(enc));
 853 }
 854
 855 /*
 856  * call-seq:
 857  *   Encoding.compatible?(str1, str2) => enc or nil
 858  *
 859  * Checks the compatibility of two strings.
 860  * If they are compatible, means concatenatable,
 861  * returns an encoding which the concatinated string will be.
 862  * If they are not compatible, nil is returned.
 863  *
 864  *   Encoding.compatible?("\xa1".force_encoding("iso-8859-1"), "b")
 865  *   => #<Encoding:ISO-8859-1>
 866  *
 867  *   Encoding.compatible?(
 868  *     "\xa1".force_encoding("iso-8859-1"),
 869  *     "\xa1\xa1".force_encoding("euc-jp"))
 870  *   => nil
 871  *
 872  */
 873 static VALUE
 874 enc_compatible_p(VALUE klass, VALUE str1, VALUE str2)
 875 {
 876     rb_encoding *enc;
 877
 878     if (!enc_capable(str1)) return Qnil;
 879     if (!enc_capable(str2)) return Qnil;
 880     enc = rb_enc_compatible(str1, str2);
 881     if (!enc) return Qnil;
 882     return rb_enc_from_encoding(enc);
 883 }
 884
 885 /* :nodoc: */
 886 static VALUE
 887 enc_dump(int argc, VALUE *argv, VALUE self)
 888 {
 889     rb_scan_args(argc, argv, "01", 0);
 890     return enc_name(self);
 891 }
 892
 893 /* :nodoc: */
 894 static VALUE
 895 enc_load(VALUE klass, VALUE str)
 896 {
 897     return enc_find(klass, str);
 898 }
 899
 900 rb_encoding *
 901 rb_ascii8bit_encoding(void)
 902 {
 903     if (!enc_table.list) {
 904         rb_enc_init();
 905     }
 906     return enc_table.list[ENCINDEX_ASCII].enc;
 907 }
 908
 909 int
 910 rb_ascii8bit_encindex(void)
 911 {
 912     return ENCINDEX_ASCII;
 913 }
 914
 915 rb_encoding *
 916 rb_utf8_encoding(void)
 917 {
 918     if (!enc_table.list) {
 919         rb_enc_init();
 920     }
 921     return enc_table.list[ENCINDEX_UTF_8].enc;
 922 }
 923
 924 int
 925 rb_utf8_encindex(void)
 926 {
 927     return ENCINDEX_UTF_8;
 928 }
 929
 930 rb_encoding *
 931 rb_usascii_encoding(void)
 932 {
 933     if (!enc_table.list) {
 934         rb_enc_init();
 935     }
 936     return enc_table.list[ENCINDEX_US_ASCII].enc;
 937 }
 938
 939 int
 940 rb_usascii_encindex(void)
 941 {
 942     return ENCINDEX_US_ASCII;
 943 }
 944
 945 rb_encoding *
 946 rb_locale_encoding(void)
 947 {
 948     VALUE charmap = rb_locale_charmap(rb_cEncoding);
 949     int idx;
 950
 951     if (NIL_P(charmap))
 952         return rb_usascii_encoding();
 953     else
 954         idx = rb_enc_find_index(StringValueCStr(charmap));
 955     if (idx < 0)
 956         return rb_ascii8bit_encoding();
 957
 958     return rb_enc_from_index(idx);
 959 }
 960
 961 rb_encoding *
 962 rb_filesystem_encoding(void)
 963 {
 964     static rb_encoding *enc;
 965     if (!enc) {
 966 #if defined _WIN32
 967         enc = rb_locale_encoding();
 968 #elif defined __APPLE__
 969         enc = rb_enc_find("UTF8-MAC");
 970 #else
 971         enc = rb_locale_encoding();
 972 #endif
 973     }
 974     return enc;
 975 }
 976
 977 static int default_external_index;
 978
 979 rb_encoding *
 980 rb_default_external_encoding(void)
 981 {
 982     return rb_enc_from_index(default_external_index);
 983 }
 984
 985 VALUE
 986 rb_enc_default_external(void)
 987 {
 988     return rb_enc_from_encoding(rb_default_external_encoding());
 989 }
 990
 991 /*
 992  * call-seq:
 993  *   Encoding.default_external => enc
 994  *
 995  * Returns default external encoding.
 996  *
 997  * It is initialized by the locale or -E option.
 998  */
 999 static VALUE
1000 get_default_external(VALUE klass)
1001 {
1002     return rb_enc_default_external();
1003 }
1004
1005 void
1006 rb_enc_set_default_external(VALUE encoding)
1007 {
1008     default_external_index = rb_enc_to_index(rb_to_encoding(encoding));
1009 }
1010
1011 /*
1012  * call-seq:
1013  *   Encoding.locale_charmap => string
1014  *
1015  * Returns the locale charmap name.
1016  *
1017  *   Debian GNU/Linux
1018  *     LANG=C
1019  *       Encoding.locale_charmap  => "ANSI_X3.4-1968"
1020  *     LANG=ja_JP.EUC-JP
1021  *       Encoding.locale_charmap  => "EUC-JP"
1022  *
1023  *   SunOS 5
1024  *     LANG=C
1025  *       Encoding.locale_charmap  => "646"
1026  *     LANG=ja
1027  *       Encoding.locale_charmap  => "eucJP"
1028  *
1029  */
1030 VALUE
1031 rb_locale_charmap(VALUE klass)
1032 {
1033 #if defined NO_LOCALE_CHARMAP
1034     return rb_usascii_str_new2("ASCII-8BIT");
1035 #elif defined HAVE_LANGINFO_H
1036     char *codeset;
1037     codeset = nl_langinfo(CODESET);
1038     return rb_usascii_str_new2(codeset);
1039 #elif defined _WIN32
1040     return rb_sprintf("CP%d", GetACP());
1041 #else
1042     return Qnil;
1043 #endif
1044 }
1045
1046 static void
1047 set_encoding_const(const char *name, rb_encoding *enc)
1048 {
1049     VALUE encoding = rb_enc_from_encoding(enc);
1050     char *s = (char *)name;
1051     int haslower = 0, hasupper = 0, valid = 0;
1052
1053     if (ISDIGIT(*s)) return;
1054     if (ISUPPER(*s)) {
1055         hasupper = 1;
1056         while (*++s && (ISALNUM(*s) || *s == '_')) {
1057             if (ISLOWER(*s)) haslower = 1;
1058         }
1059     }
1060     if (!*s) {
1061         valid = 1;
1062         rb_define_const(rb_cEncoding, name, encoding);
1063     }
1064     if (!valid || haslower) {
1065         int len = strlen(name) + 1;
1066         if (!haslower || !hasupper) {
1067             do {
1068                 if (ISLOWER(*s)) haslower = 1;
1069                 if (ISUPPER(*s)) hasupper = 1;
1070             } while (*++s && (!haslower || !hasupper));
1071         }
1072         MEMCPY(s = ALLOCA_N(char, len), name, char, len);
1073         name = s;
1074         if (!valid) {
1075             if (ISLOWER(*s)) *s = ONIGENC_ASCII_CODE_TO_UPPER_CASE((int)*s);
1076             for (; *s; ++s) {
1077                 if (!ISALNUM(*s)) *s = '_';
1078             }
1079             if (hasupper) {
1080                 rb_define_const(rb_cEncoding, name, encoding);
1081             }
1082         }
1083         if (haslower) {
1084             for (s = (char *)name; *s; ++s) {
1085                 if (ISLOWER(*s)) *s = ONIGENC_ASCII_CODE_TO_UPPER_CASE((int)*s);
1086             }
1087             rb_define_const(rb_cEncoding, name, encoding);
1088         }
1089     }
1090 }
1091
1092 static int
1093 rb_enc_name_list_i(st_data_t name, st_data_t idx, st_data_t arg)
1094 {
1095     VALUE ary = (VALUE)arg;
1096     VALUE str = rb_usascii_str_new2((char *)name);
1097     OBJ_FREEZE(str);
1098     rb_ary_push(ary, str);
1099     return ST_CONTINUE;
1100 }
1101
1102 /*
1103  * call-seq:
1104  *   Encoding.name_list => ["enc1", "enc2", ...]
1105  *
1106  * Returns the list of available encoding names.
1107  *
1108  *   Encoding.name_list
1109  *   => ["US-ASCII", "ASCII-8BIT", "UTF-8",
1110  *       "ISO-8859-1", "Shift_JIS", "EUC-JP",
1111  *       "Windows-31J",
1112  *       "BINARY", "CP932", "eucJP"]
1113  *
1114  * This list doesn't include dummy encodings.
1115  *
1116  */
1117
1118 static VALUE
1119 rb_enc_name_list(VALUE klass)
1120 {
1121     VALUE ary = rb_ary_new2(enc_table.names->num_entries);
1122     st_foreach(enc_table.names, rb_enc_name_list_i, (st_data_t)ary);
1123     return ary;
1124 }
1125
1126 static int
1127 rb_enc_aliases_enc_i(st_data_t name, st_data_t orig, st_data_t arg)
1128 {
1129     VALUE *p = (VALUE *)arg;
1130     VALUE aliases = p[0], ary = p[1];
1131     int idx = (int)orig;
1132     VALUE key, str = rb_ary_entry(ary, idx);
1133
1134     if (NIL_P(str)) {
1135         rb_encoding *enc = rb_enc_from_index(idx);
1136
1137         if (STRCASECMP((char*)name, rb_enc_name(enc)) == 0) {
1138             return ST_CONTINUE;
1139         }
1140         str = rb_usascii_str_new2(rb_enc_name(enc));
1141         OBJ_FREEZE(str);
1142         rb_ary_store(ary, idx, str);
1143     }
1144     key = rb_usascii_str_new2((char *)name);
1145     OBJ_FREEZE(key);
1146     rb_hash_aset(aliases, key, str);
1147     return ST_CONTINUE;
1148 }
1149
1150 /*
1151  * call-seq:
1152  *   Encoding.aliases => {"alias1" => "orig1", "alias2" => "orig2", ...}
1153  *
1154  * Returns the hash of available encoding alias and original encoding name.
1155  *
1156  *   Encoding.aliases
1157  *   => {"BINARY"=>"ASCII-8BIT", "ASCII"=>"US-ASCII", "ANSI_X3.4-1986"=>"US-ASCII",
1158  *       "SJIS"=>"Shift_JIS", "eucJP"=>"EUC-JP", "CP932"=>"Windows-31J"}
1159  *
1160  */
1161
1162 static VALUE
1163 rb_enc_aliases(VALUE klass)
1164 {
1165     VALUE aliases[2];
1166     aliases[0] = rb_hash_new();
1167     aliases[1] = rb_ary_new();
1168     st_foreach(enc_table.names, rb_enc_aliases_enc_i, (st_data_t)aliases);
1169     return aliases[0];
1170 }
1171
1172 void
1173 Init_Encoding(void)
1174 {
1175 #undef rb_intern
1176     VALUE list;
1177     int i;
1178
1179     id_base_encoding = rb_intern("#base_encoding");
1180
1181     rb_cEncoding = rb_define_class("Encoding", rb_cObject);
1182     rb_undef_alloc_func(rb_cEncoding);
1183     rb_define_method(rb_cEncoding, "to_s", enc_name, 0);
1184     rb_define_method(rb_cEncoding, "inspect", enc_inspect, 0);
1185     rb_define_method(rb_cEncoding, "name", enc_name, 0);
1186     rb_define_method(rb_cEncoding, "base_encoding", enc_base_encoding, 0);
1187     rb_define_method(rb_cEncoding, "dummy?", enc_dummy_p, 0);
1188     rb_define_singleton_method(rb_cEncoding, "list", enc_list, 0);
1189     rb_define_singleton_method(rb_cEncoding, "name_list", rb_enc_name_list, 0);
1190     rb_define_singleton_method(rb_cEncoding, "aliases", rb_enc_aliases, 0);
1191     rb_define_singleton_method(rb_cEncoding, "find", enc_find, 1);
1192     rb_define_singleton_method(rb_cEncoding, "compatible?", enc_compatible_p, 2);
1193
1194     rb_define_method(rb_cEncoding, "_dump", enc_dump, -1);
1195     rb_define_singleton_method(rb_cEncoding, "_load", enc_load, 1);
1196
1197     rb_define_singleton_method(rb_cEncoding, "default_external", get_default_external, 0);
1198     rb_define_singleton_method(rb_cEncoding, "locale_charmap", rb_locale_charmap, 0);
1199
1200     rb_gc_register_address(&rb_encoding_list);
1201     list = rb_ary_new2(enc_table.count);
1202     RBASIC(list)->klass = 0;
1203     rb_encoding_list = list;
1204     for (i = 0; i < enc_table.count; ++i) {
1205         rb_ary_push(list, enc_new(enc_table.list[i].enc));
1206     }
1207 }
1208
1209 /* locale insensitive functions */
1210
1211 #define ctype_test(c, ctype) \
1212     (rb_isascii(c) && ONIGENC_IS_ASCII_CODE_CTYPE((c), ctype))
1213
1214 int rb_isalnum(int c) { return ctype_test(c, ONIGENC_CTYPE_ALNUM); }
1215 int rb_isalpha(int c) { return ctype_test(c, ONIGENC_CTYPE_ALPHA); }
1216 int rb_isblank(int c) { return ctype_test(c, ONIGENC_CTYPE_BLANK); }
1217 int rb_iscntrl(int c) { return ctype_test(c, ONIGENC_CTYPE_CNTRL); }
1218 int rb_isdigit(int c) { return ctype_test(c, ONIGENC_CTYPE_DIGIT); }
1219 int rb_isgraph(int c) { return ctype_test(c, ONIGENC_CTYPE_GRAPH); }
1220 int rb_islower(int c) { return ctype_test(c, ONIGENC_CTYPE_LOWER); }
1221 int rb_isprint(int c) { return ctype_test(c, ONIGENC_CTYPE_PRINT); }
1222 int rb_ispunct(int c) { return ctype_test(c, ONIGENC_CTYPE_PUNCT); }
1223 int rb_isspace(int c) { return ctype_test(c, ONIGENC_CTYPE_SPACE); }
1224 int rb_isupper(int c) { return ctype_test(c, ONIGENC_CTYPE_UPPER); }
1225 int rb_isxdigit(int c) { return ctype_test(c, ONIGENC_CTYPE_XDIGIT); }
1226
1227 int
1228 rb_tolower(int c)
1229 {
1230     return rb_isascii(c) ? ONIGENC_ASCII_CODE_TO_LOWER_CASE(c) : c;
1231 }
1232
1233 int
1234 rb_toupper(int c)
1235 {
1236     return rb_isascii(c) ? ONIGENC_ASCII_CODE_TO_UPPER_CASE(c) : c;
1237 }
1238