?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
* gc.c (set_heaps_increment): fix memory allocation strategy by
[ruby-svn.git] / encoding.c
blobe243451569ca8745148573823a776ca6bba1cd13
1 /**********************************************************************
2
3 encoding.c -
4
5 $Author$
6 created at: Thu May 24 17:23:27 JST 2007
7
8 Copyright (C) 2007 Yukihiro Matsumoto
9
10 **********************************************************************/
11
12 #include "ruby/ruby.h"
13 #include "ruby/encoding.h"
14 #include "regenc.h"
15 #include <ctype.h>
16 #ifdef HAVE_LANGINFO_H
17 #include <langinfo.h>
18 #endif
19
20 static ID id_encoding, id_base_encoding;
21 static VALUE rb_cEncoding;
22
23 struct rb_encoding_entry {
24 const char *name;
25 rb_encoding *enc;
26 };
27
28 static struct {
29 struct rb_encoding_entry *list;
30 int count;
31 int size;
32 st_table *names;
33 } enc_table;
34
35 void rb_enc_init(void);
36
37 #define ENCODING_COUNT ENCINDEX_BUILTIN_MAX
38
39 #define enc_autoload_p(enc) (!rb_enc_mbmaxlen(enc))
40
41 #define ENC_UNINITIALIZED (&rb_cEncoding)
42 #define enc_initialized_p(enc) ((enc)->auxiliary_data != &rb_cEncoding)
43 #define ENC_FROM_ENCODING(enc) ((VALUE)(enc)->auxiliary_data)
44
45 #define ENC_DUMMY_FLAG FL_USER2
46 #define ENC_DUMMY_P(enc) (RBASIC(enc)->flags & ENC_DUMMY_FLAG)
47 #define ENC_SET_DUMMY(enc) (RBASIC(enc)->flags |= ENC_DUMMY_FLAG)
48
49 static int load_encoding(const char *name);
50 static VALUE enc_base_encoding(VALUE self);
51
52 static void
53 enc_mark(void *ptr)
54 {
55 }
56
57 static VALUE
58 enc_new(rb_encoding *encoding)
59 {
60 VALUE enc = Data_Wrap_Struct(rb_cEncoding, enc_mark, 0, encoding);
61 encoding->auxiliary_data = (void *)enc;
62 return enc;
63 }
64
65 VALUE
66 rb_enc_from_encoding(rb_encoding *encoding)
67 {
68 if (!encoding) return Qnil;
69 if (enc_initialized_p(encoding))
70 return ENC_FROM_ENCODING(encoding);
71 return enc_new(encoding);
72 }
73
74 static int
75 enc_check_encoding(VALUE obj)
76 {
77 int index;
78 rb_encoding *enc;
79
80 if (SPECIAL_CONST_P(obj) || BUILTIN_TYPE(obj) != T_DATA ||
81 RDATA(obj)->dmark != enc_mark) {
82 return -1;
83 }
84 enc = (rb_encoding*)RDATA(obj)->data;
85 index = rb_enc_to_index(enc);
86 if (rb_enc_from_index(index) != enc)
87 return -1;
88 if (enc_autoload_p(enc)) {
89 index = rb_enc_find_index(enc->name);
90 }
91 return index;
92 }
93
94 int
95 rb_to_encoding_index(VALUE enc)
96 {
97 int idx;
98
99 idx = enc_check_encoding(enc);
100 if (idx >= 0) {
101 return idx;
102 }
103 else if (NIL_P(enc = rb_check_string_type(enc))) {
104 return -1;
105 }
106 else {
107 return rb_enc_find_index(StringValueCStr(enc));
108 }
109 }
110
111 rb_encoding *
112 rb_to_encoding(VALUE enc)
113 {
114 int idx;
115
116 idx = enc_check_encoding(enc);
117 if (idx >= 0) return RDATA(enc)->data;
118 if ((idx = rb_enc_find_index(StringValueCStr(enc))) < 0) {
119 rb_raise(rb_eArgError, "unknown encoding name - %s", RSTRING_PTR(enc));
120 }
121 return rb_enc_from_index(idx);
122 }
123
124 void
125 rb_gc_mark_encodings(void)
126 {
127 int i;
128 for (i = 0; i < enc_table.count; ++i) {
129 rb_encoding *enc = enc_table.list[i].enc;
130 if (enc && enc_initialized_p(enc)) {
131 rb_gc_mark(ENC_FROM_ENCODING(enc));
132 }
133 }
134 }
135
136 static int
137 enc_table_expand(int newsize)
138 {
139 struct rb_encoding_entry *ent;
140 int count = newsize;
141
142 if (enc_table.size >= newsize) return newsize;
143 newsize = (newsize + 7) / 8 * 8;
144 ent = realloc(enc_table.list, sizeof(*enc_table.list) * newsize);
145 if (!ent) return -1;
146 memset(ent + enc_table.size, 0, sizeof(*ent)*(newsize - enc_table.size));
147 enc_table.list = ent;
148 enc_table.size = newsize;
149 return count;
150 }
151
152 static int
153 enc_register_at(int index, const char *name, rb_encoding *encoding)
154 {
155 struct rb_encoding_entry *ent = &enc_table.list[index];
156 void *obj = ENC_UNINITIALIZED;
157
158 if (!ent->name) {
159 ent->name = name = strdup(name);
160 }
161 else if (STRCASECMP(name, ent->name)) {
162 return -1;
163 }
164 if (!ent->enc) {
165 ent->enc = malloc(sizeof(rb_encoding));
166 }
167 else {
168 obj = ent->enc->auxiliary_data;
169 }
170 if (encoding) {
171 *ent->enc = *encoding;
172 }
173 else {
174 memset(ent->enc, 0, sizeof(*ent->enc));
175 }
176 encoding = ent->enc;
177 encoding->name = name;
178 encoding->ruby_encoding_index = index;
179 st_insert(enc_table.names, (st_data_t)name, (st_data_t)index);
180 if (obj != ENC_UNINITIALIZED) {
181 encoding->auxiliary_data = obj;
182 }
183 else if (rb_cEncoding) {
184 /* initialize encoding data */
185 enc_new(encoding);
186 }
187 else {
188 encoding->auxiliary_data = ENC_UNINITIALIZED;
189 }
190 return index;
191 }
192
193 static int
194 enc_register(const char *name, rb_encoding *encoding)
195 {
196 int index = enc_table.count;
197
198 if ((index = enc_table_expand(index + 1)) < 0) return -1;
199 enc_table.count = index;
200 return enc_register_at(index - 1, name, encoding);
201 }
202
203 static void set_encoding_const(const char *, rb_encoding *);
204 int rb_enc_registered(const char *name);
205
206 int
207 rb_enc_register(const char *name, rb_encoding *encoding)
208 {
209 int index = rb_enc_registered(name);
210
211 if (index >= 0) {
212 rb_encoding *oldenc = rb_enc_from_index(index);
213 if (STRCASECMP(name, rb_enc_name(oldenc))) {
214 index = enc_register(name, encoding);
215 }
216 else if (!enc_autoload_p(oldenc) ||
217 (enc_initialized_p(oldenc) &&
218 !ENC_DUMMY_P(ENC_FROM_ENCODING(oldenc)))) {
219 enc_register_at(index, name, encoding);
220 }
221 else {
222 rb_raise(rb_eArgError, "encoding %s is already registered", name);
223 }
224 }
225 else {
226 index = enc_register(name, encoding);
227 set_encoding_const(name, rb_enc_from_index(index));
228 }
229 return index;
230 }
231
232 void
233 rb_encdb_declare(const char *name)
234 {
235 int idx = rb_enc_registered(name);
236 if (idx < 0) {
237 idx = enc_register(name, 0);
238 }
239 set_encoding_const(name, rb_enc_from_index(idx));
240 }
241
242 static void
243 enc_check_duplication(const char *name)
244 {
245 if (rb_enc_registered(name) >= 0) {
246 rb_raise(rb_eArgError, "encoding %s is already registered", name);
247 }
248 }
249
250 static VALUE
251 set_base_encoding(int index, rb_encoding *base)
252 {
253 VALUE enc = rb_enc_from_encoding(enc_table.list[index].enc);
254
255 rb_ivar_set(enc, id_base_encoding, rb_enc_from_encoding(base));
256 if (rb_enc_dummy_p(base)) ENC_SET_DUMMY(enc);
257 return enc;
258 }
259
260 int
261 rb_enc_replicate(const char *name, rb_encoding *encoding)
262 {
263 int idx;
264
265 enc_check_duplication(name);
266 idx = enc_register(name, encoding);
267 set_base_encoding(idx, encoding);
268 set_encoding_const(name, rb_enc_from_index(idx));
269 return idx;
270 }
271
272 static int
273 enc_replicate(int idx, const char *name, rb_encoding *origenc)
274 {
275 if (idx < 0) {
276 idx = enc_register(name, origenc);
277 }
278 else {
279 idx = enc_register_at(idx, name, origenc);
280 }
281 if (idx >= 0) {
282 set_base_encoding(idx, origenc);
283 set_encoding_const(name, rb_enc_from_index(idx));
284 }
285 return idx;
286 }
287
288 int
289 rb_encdb_replicate(const char *name, const char *orig)
290 {
291 int origidx = rb_enc_registered(orig);
292 int idx = rb_enc_registered(name);
293
294 if (origidx < 0) {
295 origidx = enc_register(orig, 0);
296 }
297 return enc_replicate(idx, name, rb_enc_from_index(origidx));
298 }
299
300 int
301 rb_define_dummy_encoding(const char *name)
302 {
303 int index = rb_enc_replicate(name, rb_ascii8bit_encoding());
304 VALUE enc = rb_enc_from_encoding(enc_table.list[index].enc);
305
306 ENC_SET_DUMMY(enc);
307 return index;
308 }
309
310 int
311 rb_encdb_dummy(const char *name)
312 {
313 int index = enc_replicate(rb_enc_registered(name), name,
314 rb_ascii8bit_encoding());
315 VALUE enc = rb_enc_from_encoding(enc_table.list[index].enc);
316
317 ENC_SET_DUMMY(enc);
318 return index;
319 }
320
321 int
322 rb_enc_dummy_p(rb_encoding *enc)
323 {
324 VALUE encoding;
325 if (!enc_initialized_p(enc)) return Qfalse;
326 encoding = rb_enc_from_encoding(enc);
327 return ENC_DUMMY_P(encoding);
328 }
329
330 /*
331 * call-seq:
332 * enc.dummy? => true or false
333 *
334 * Returns true for dummy encodings.
335 * A dummy encoding is an encoding for which character handling is not properly
336 * implemented.
337 * It is used for stateful encodings.
338 *
339 * Encoding::ISO_2022_JP.dummy? #=> true
340 * Encoding::UTF_8.dummy? #=> false
341 *
342 */
343 static VALUE
344 enc_dummy_p(VALUE enc)
345 {
346 return rb_enc_dummy_p(rb_to_encoding(enc)) ? Qtrue : Qfalse;
347 }
348
349 static int
350 enc_alias(const char *alias, int idx)
351 {
352 alias = strdup(alias);
353 st_insert(enc_table.names, (st_data_t)alias, (st_data_t)idx);
354 set_encoding_const(alias, rb_enc_from_index(idx));
355 return idx;
356 }
357
358 int
359 rb_enc_alias(const char *alias, const char *orig)
360 {
361 int idx;
362
363 enc_check_duplication(alias);
364 if (!enc_table.list) {
365 rb_enc_init();
366 }
367 if ((idx = rb_enc_find_index(orig)) < 0) {
368 return -1;
369 }
370 return enc_alias(alias, idx);
371 }
372
373 int
374 rb_encdb_alias(const char *alias, const char *orig)
375 {
376 int idx = rb_enc_registered(orig);
377
378 if (idx < 0) {
379 idx = enc_register(orig, 0);
380 }
381 return enc_alias(alias, idx);
382 }
383
384 enum {
385 ENCINDEX_ASCII,
386 ENCINDEX_UTF_8,
387 ENCINDEX_US_ASCII,
388 ENCINDEX_BUILTIN_MAX
389 };
390
391 extern rb_encoding OnigEncodingUTF_8;
392 extern rb_encoding OnigEncodingUS_ASCII;
393
394 void
395 rb_enc_init(void)
396 {
397 enc_table_expand(ENCODING_COUNT + 1);
398 if (!enc_table.names) {
399 enc_table.names = st_init_strcasetable();
400 }
401 #define ENC_REGISTER(enc) enc_register_at(ENCINDEX_##enc, rb_enc_name(&OnigEncoding##enc), &OnigEncoding##enc)
402 ENC_REGISTER(ASCII);
403 ENC_REGISTER(UTF_8);
404 ENC_REGISTER(US_ASCII);
405 #undef ENC_REGISTER
406 enc_table.count = ENCINDEX_BUILTIN_MAX;
407 }
408
409 rb_encoding *
410 rb_enc_from_index(int index)
411 {
412 if (!enc_table.list) {
413 rb_enc_init();
414 }
415 if (index < 0 || enc_table.count <= index) {
416 return 0;
417 }
418 return enc_table.list[index].enc;
419 }
420
421 int
422 rb_enc_registered(const char *name)
423 {
424 st_data_t idx = 0;
425
426 if (!name) return -1;
427 if (!enc_table.list) return -1;
428 if (st_lookup(enc_table.names, (st_data_t)name, &idx)) {
429 return (int)idx;
430 }
431 return -1;
432 }
433
434 static VALUE
435 require_enc(VALUE enclib)
436 {
437 return rb_require_safe(enclib, rb_safe_level());
438 }
439
440 static int
441 load_encoding(const char *name)
442 {
443 VALUE enclib = rb_sprintf("enc/%s", name);
444 VALUE verbose = ruby_verbose;
445 VALUE debug = ruby_debug;
446 VALUE loaded;
447 char *s = RSTRING_PTR(enclib) + 4, *e = RSTRING_END(enclib);
448 int idx;
449
450 while (s < e) {
451 if (!ISALNUM(*s)) *s = '_';
452 else if (ISUPPER(*s)) *s = TOLOWER(*s);
453 ++s;
454 }
455 OBJ_FREEZE(enclib);
456 ruby_verbose = Qfalse;
457 ruby_debug = Qfalse;
458 loaded = rb_protect(require_enc, enclib, 0);
459 ruby_verbose = verbose;
460 ruby_debug = debug;
461 rb_set_errinfo(Qnil);
462 if (NIL_P(loaded)) return -1;
463 if ((idx = rb_enc_registered(name)) < 0) return -1;
464 if (enc_autoload_p(enc_table.list[idx].enc)) return -1;
465 return idx;
466 }
467
468 int
469 rb_enc_find_index(const char *name)
470 {
471 int i = rb_enc_registered(name), b;
472 rb_encoding *enc;
473 VALUE base;
474
475 if (i < 0) {
476 i = load_encoding(name);
477 }
478 else if (enc_autoload_p(enc = rb_enc_from_index(i))) {
479 if (enc_initialized_p(enc) &&
480 (base = enc_base_encoding(ENC_FROM_ENCODING(enc)), !NIL_P(base))) {
481 if ((b = enc_check_encoding(base)) < 0) {
482 goto failed;
483 }
484 enc_register_at(i, rb_enc_name(enc), rb_enc_from_index(b));
485 }
486 else {
487 i = load_encoding(rb_enc_name(enc));
488 if (i < 0) {
489 failed:
490 rb_warn("failed to load encoding (%s); use ASCII-8BIT instead",
491 name);
492 return 0;
493 }
494 }
495 }
496 return i;
497 }
498
499 rb_encoding *
500 rb_enc_find(const char *name)
501 {
502 int idx = rb_enc_find_index(name);
503 if (idx < 0) idx = 0;
504 return rb_enc_from_index(idx);
505 }
506
507 static inline int
508 enc_capable(VALUE obj)
509 {
510 if (SPECIAL_CONST_P(obj)) return Qfalse;
511 switch (BUILTIN_TYPE(obj)) {
512 case T_STRING:
513 case T_REGEXP:
514 case T_FILE:
515 return Qtrue;
516 case T_DATA:
517 if (RDATA(obj)->dmark == enc_mark) return Qtrue;
518 default:
519 return Qfalse;
520 }
521 }
522
523 static void
524 enc_check_capable(VALUE x)
525 {
526 if (!enc_capable(x)) {
527 const char *etype;
528
529 if (NIL_P(x)) {
530 etype = "nil";
531 }
532 else if (FIXNUM_P(x)) {
533 etype = "Fixnum";
534 }
535 else if (SYMBOL_P(x)) {
536 etype = "Symbol";
537 }
538 else if (rb_special_const_p(x)) {
539 etype = RSTRING_PTR(rb_obj_as_string(x));
540 }
541 else {
542 etype = rb_obj_classname(x);
543 }
544 rb_raise(rb_eTypeError, "wrong argument type %s (not encode capable)", etype);
545 }
546 }
547
548 ID
549 rb_id_encoding(void)
550 {
551 if (!id_encoding) {
552 id_encoding = rb_intern("encoding");
553 }
554 return id_encoding;
555 }
556
557 int
558 rb_enc_internal_get_index(VALUE obj)
559 {
560 int i;
561
562 i = ENCODING_GET_INLINED(obj);
563 if (i == ENCODING_INLINE_MAX) {
564 VALUE iv;
565
566 iv = rb_ivar_get(obj, rb_id_encoding());
567 i = NUM2INT(iv);
568 }
569 return i;
570 }
571
572 void
573 rb_enc_internal_set_index(VALUE obj, int idx)
574 {
575 if (idx < ENCODING_INLINE_MAX) {
576 ENCODING_SET_INLINED(obj, idx);
577 return;
578 }
579 ENCODING_SET_INLINED(obj, ENCODING_INLINE_MAX);
580 rb_ivar_set(obj, rb_id_encoding(), INT2NUM(idx));
581 return;
582 }
583
584 void
585 rb_enc_associate_index(VALUE obj, int idx)
586 {
587 enc_check_capable(obj);
588 if (rb_enc_internal_get_index(obj) == idx)
589 return;
590 if (!ENC_CODERANGE_ASCIIONLY(obj) ||
591 !rb_enc_asciicompat(rb_enc_from_index(idx))) {
592 ENC_CODERANGE_CLEAR(obj);
593 }
594 rb_enc_internal_set_index(obj, idx);
595 }
596
597 void
598 rb_enc_associate(VALUE obj, rb_encoding *enc)
599 {
600 rb_enc_associate_index(obj, rb_enc_to_index(enc));
601 }
602
603 int
604 rb_enc_get_index(VALUE obj)
605 {
606 if (!enc_capable(obj)) return -1;
607 return rb_enc_internal_get_index(obj);
608 }
609
610 rb_encoding*
611 rb_enc_get(VALUE obj)
612 {
613 return rb_enc_from_index(rb_enc_get_index(obj));
614 }
615
616 rb_encoding*
617 rb_enc_check(VALUE str1, VALUE str2)
618 {
619 rb_encoding *enc = rb_enc_compatible(str1, str2);
620 if (!enc)
621 rb_raise(rb_eArgError, "character encodings differ: %s and %s",
622 rb_enc_name(rb_enc_get(str1)),
623 rb_enc_name(rb_enc_get(str2)));
624 return enc;
625 }
626
627 rb_encoding*
628 rb_enc_compatible(VALUE str1, VALUE str2)
629 {
630 int idx1, idx2;
631 rb_encoding *enc1, *enc2;
632
633 idx1 = rb_enc_get_index(str1);
634 idx2 = rb_enc_get_index(str2);
635
636 if (idx1 < 0 || idx2 < 0)
637 return 0;
638
639 if (idx1 == idx2) {
640 return rb_enc_from_index(idx1);
641 }
642 enc1 = rb_enc_from_index(idx1);
643 enc2 = rb_enc_from_index(idx2);
644
645 if (TYPE(str2) == T_STRING && RSTRING_LEN(str2) == 0)
646 return enc1;
647 if (TYPE(str1) == T_STRING && RSTRING_LEN(str1) == 0)
648 return enc2;
649 if (!rb_enc_asciicompat(enc1) || !rb_enc_asciicompat(enc2)) {
650 return 0;
651 }
652
653 if (BUILTIN_TYPE(str1) != T_STRING) {
654 VALUE tmp = str1;
655 int idx0 = idx1;
656 str1 = str2;
657 str2 = tmp;
658 idx1 = idx2;
659 idx2 = idx0;
660 }
661 if (BUILTIN_TYPE(str1) == T_STRING) {
662 int cr1, cr2;
663
664 cr1 = rb_enc_str_coderange(str1);
665 if (BUILTIN_TYPE(str2) == T_STRING) {
666 cr2 = rb_enc_str_coderange(str2);
667 if (cr1 != cr2) {
668 /* may need to handle ENC_CODERANGE_BROKEN */
669 if (cr1 == ENC_CODERANGE_7BIT) return enc2;
670 if (cr2 == ENC_CODERANGE_7BIT) return enc1;
671 }
672 if (cr2 == ENC_CODERANGE_7BIT) {
673 if (idx1 == 0) return enc2;
674 return enc1;
675 }
676 }
677 if (cr1 == ENC_CODERANGE_7BIT)
678 return enc2;
679 }
680 return 0;
681 }
682
683 void
684 rb_enc_copy(VALUE obj1, VALUE obj2)
685 {
686 rb_enc_associate_index(obj1, rb_enc_get_index(obj2));
687 }
688
689
690 /*
691 * call-seq:
692 * obj.encoding => encoding
693 *
694 * Returns the Encoding object that represents the encoding of obj.
695 */
696
697 VALUE
698 rb_obj_encoding(VALUE obj)
699 {
700 rb_encoding *enc = rb_enc_get(obj);
701 if (!enc) {
702 rb_raise(rb_eTypeError, "unknown encoding");
703 }
704 return rb_enc_from_encoding(enc);
705 }
706
707 int
708 rb_enc_mbclen(const char *p, const char *e, rb_encoding *enc)
709 {
710 int n = ONIGENC_PRECISE_MBC_ENC_LEN(enc, (UChar*)p, (UChar*)e);
711 if (MBCLEN_CHARFOUND_P(n) && MBCLEN_CHARFOUND_LEN(n) <= e-p)
712 return MBCLEN_CHARFOUND_LEN(n);
713 else {
714 int min = rb_enc_mbminlen(enc);
715 return min <= e-p ? min : e-p;
716 }
717 }
718
719 int
720 rb_enc_precise_mbclen(const char *p, const char *e, rb_encoding *enc)
721 {
722 int n;
723 if (e <= p)
724 return ONIGENC_CONSTRUCT_MBCLEN_NEEDMORE(1);
725 n = ONIGENC_PRECISE_MBC_ENC_LEN(enc, (UChar*)p, (UChar*)e);
726 if (e-p < n)
727 return ONIGENC_CONSTRUCT_MBCLEN_NEEDMORE(n-(e-p));
728 return n;
729 }
730
731 int
732 rb_enc_ascget(const char *p, const char *e, int *len, rb_encoding *enc)
733 {
734 int c, l;
735 if (e <= p)
736 return -1;
737 if (rb_enc_asciicompat(enc)) {
738 c = (unsigned char)*p;
739 if (!ISASCII(c))
740 return -1;
741 if (len) *len = 1;
742 return c;
743 }
744 l = rb_enc_precise_mbclen(p, e, enc);
745 if (!MBCLEN_CHARFOUND_P(l))
746 return -1;
747 c = rb_enc_mbc_to_codepoint(p, e, enc);
748 if (!rb_enc_isascii(c, enc))
749 return -1;
750 if (len) *len = l;
751 return c;
752 }
753
754 int
755 rb_enc_codepoint(const char *p, const char *e, rb_encoding *enc)
756 {
757 int r;
758 if (e <= p)
759 rb_raise(rb_eArgError, "empty string");
760 r = rb_enc_precise_mbclen(p, e, enc);
761 if (MBCLEN_CHARFOUND_P(r))
762 return rb_enc_mbc_to_codepoint(p, e, enc);
763 else
764 rb_raise(rb_eArgError, "invalid mbstring sequence");
765 }
766
767 int
768 rb_enc_codelen(int c, rb_encoding *enc)
769 {
770 int n = ONIGENC_CODE_TO_MBCLEN(enc,c);
771 if (n == 0) {
772 rb_raise(rb_eArgError, "invalid codepoint 0x%x", c);
773 }
774 return n;
775 }
776
777 int
778 rb_enc_toupper(int c, rb_encoding *enc)
779 {
780 return (ONIGENC_IS_ASCII_CODE(c)?ONIGENC_ASCII_CODE_TO_UPPER_CASE(c):(c));
781 }
782
783 int
784 rb_enc_tolower(int c, rb_encoding *enc)
785 {
786 return (ONIGENC_IS_ASCII_CODE(c)?ONIGENC_ASCII_CODE_TO_LOWER_CASE(c):(c));
787 }
788
789 /*
790 * call-seq:
791 * enc.inspect => string
792 *
793 * Returns a string which represents the encoding for programmers.
794 *
795 * Encoding::UTF_8.inspect #=> "#<Encoding:UTF-8>"
796 * Encoding::ISO_2022_JP.inspect #=> "#<Encoding:ISO-2022-JP (dummy)>"
797 */
798 static VALUE
799 enc_inspect(VALUE self)
800 {
801 VALUE str = rb_sprintf("#<%s:%s%s>", rb_obj_classname(self),
802 rb_enc_name((rb_encoding*)DATA_PTR(self)),
803 (ENC_DUMMY_P(self) ? " (dummy)" : ""));
804 ENCODING_CODERANGE_SET(str, rb_usascii_encindex(), ENC_CODERANGE_7BIT);
805 return str;
806 }
807
808 /*
809 * call-seq:
810 * enc.name => string
811 *
812 * Returns the name of the encoding.
813 *
814 * Encoding::UTF_8.name => "UTF-8"
815 */
816 static VALUE
817 enc_name(VALUE self)
818 {
819 return rb_usascii_str_new2(rb_enc_name((rb_encoding*)DATA_PTR(self)));
820 }
821
822 static VALUE
823 enc_base_encoding(VALUE self)
824 {
825 return rb_attr_get(self, id_base_encoding);
826 }
827
828 /*
829 * call-seq:
830 * Encoding.list => [enc1, enc2, ...]
831 *
832 * Returns the list of loaded encodings.
833 *
834 * Encoding.list
835 * => [#<Encoding:ASCII-8BIT>, #<Encoding:UTF-8>,
836 * #<Encoding:ISO-2022-JP (dummy)>]
837 *
838 * Encoding.find("US-ASCII")
839 * => #<Encoding:US-ASCII>
840 *
841 * Encoding.list
842 * => [#<Encoding:ASCII-8BIT>, #<Encoding:UTF-8>,
843 * #<Encoding:US-ASCII>, #<Encoding:ISO-2022-JP (dummy)>]
844 *
845 */
846 static VALUE
847 enc_list(VALUE klass)
848 {
849 VALUE ary = rb_ary_new2(enc_table.count);
850 int i;
851 for (i = 0; i < enc_table.count; ++i) {
852 rb_encoding *enc = enc_table.list[i].enc;
853 if (enc) {
854 rb_ary_push(ary, rb_enc_from_encoding(enc));
855 }
856 }
857 return ary;
858 }
859
860 /*
861 * call-seq:
862 * Encoding.find(string) => enc
863 * Encoding.find(symbol) => enc
864 *
865 * Search the encoding with specified <i>name</i>.
866 * <i>name</i> should be a string or symbol.
867 *
868 * Encoding.find("US-ASCII") => #<Encoding:US-ASCII>
869 * Encoding.find(:Shift_JIS) => #<Encoding:Shift_JIS>
870 *
871 */
872 static VALUE
873 enc_find(VALUE klass, VALUE enc)
874 {
875 int idx;
876
877 StringValue(enc);
878 if (!rb_enc_asciicompat(rb_enc_get(enc))) {
879 rb_raise(rb_eArgError, "invalid name encoding (non ASCII)");
880 }
881 idx = rb_enc_find_index(StringValueCStr(enc));
882 if (idx < 0) {
883 rb_raise(rb_eArgError, "unknown encoding name - %s", RSTRING_PTR(enc));
884 }
885 return rb_enc_from_encoding(rb_enc_from_index(idx));
886 }
887
888 /*
889 * call-seq:
890 * Encoding.compatible?(str1, str2) => enc or nil
891 *
892 * Checks the compatibility of two strings.
893 * If they are compatible, means concatenatable,
894 * returns an encoding which the concatinated string will be.
895 * If they are not compatible, nil is returned.
896 *
897 * Encoding.compatible?("\xa1".force_encoding("iso-8859-1"), "b")
898 * => #<Encoding:ISO-8859-1>
899 *
900 * Encoding.compatible?(
901 * "\xa1".force_encoding("iso-8859-1"),
902 * "\xa1\xa1".force_encoding("euc-jp"))
903 * => nil
904 *
905 */
906 static VALUE
907 enc_compatible_p(VALUE klass, VALUE str1, VALUE str2)
908 {
909 rb_encoding *enc = rb_enc_compatible(str1, str2);
910 VALUE encoding = Qnil;
911 if (!enc || !(encoding = rb_enc_from_encoding(enc)))
912 encoding = Qnil;
913 return encoding;
914 }
915
916 /* :nodoc: */
917 static VALUE
918 enc_dump(int argc, VALUE *argv, VALUE self)
919 {
920 rb_scan_args(argc, argv, "01", 0);
921 return enc_name(self);
922 }
923
924 /* :nodoc: */
925 static VALUE
926 enc_load(VALUE klass, VALUE str)
927 {
928 return enc_find(klass, str);
929 }
930
931 rb_encoding *
932 rb_ascii8bit_encoding(void)
933 {
934 if (!enc_table.list) {
935 rb_enc_init();
936 }
937 return enc_table.list[0].enc;
938 }
939
940 rb_encoding *
941 rb_utf8_encoding(void)
942 {
943 if (!enc_table.list) {
944 rb_enc_init();
945 }
946 return enc_table.list[ENCINDEX_UTF_8].enc;
947 }
948
949 rb_encoding *
950 rb_usascii_encoding(void)
951 {
952 if (!enc_table.list) {
953 rb_enc_init();
954 }
955 return enc_table.list[ENCINDEX_US_ASCII].enc;
956 }
957
958 int
959 rb_usascii_encindex(void)
960 {
961 return ENCINDEX_US_ASCII;
962 }
963
964 rb_encoding *
965 rb_locale_encoding(void)
966 {
967 VALUE charmap = rb_locale_charmap(rb_cEncoding);
968 int idx;
969
970 if (NIL_P(charmap))
971 idx = rb_enc_find_index("US-ASCII");
972 else
973 idx = rb_enc_find_index(StringValueCStr(charmap));
974 if (idx < 0)
975 return rb_ascii8bit_encoding();
976
977 return rb_enc_from_index(idx);
978 }
979
980 static int default_external_index;
981
982 rb_encoding *
983 rb_default_external_encoding(void)
984 {
985 return rb_enc_from_index(default_external_index);
986 }
987
988 VALUE
989 rb_enc_default_external(void)
990 {
991 return rb_enc_from_encoding(rb_default_external_encoding());
992 }
993
994 /*
995 * call-seq:
996 * Encoding.default_external => enc
997 *
998 * Returns default external encoding.
999 *
1000 * It is initialized by the locale or -E option.
1001 */
1002 static VALUE
1003 get_default_external(VALUE klass)
1004 {
1005 return rb_enc_default_external();
1006 }
1007
1008 void
1009 rb_enc_set_default_external(VALUE encoding)
1010 {
1011 default_external_index = rb_enc_to_index(rb_to_encoding(encoding));
1012 }
1013
1014 /*
1015 * call-seq:
1016 * Encoding.locale_charmap => string
1017 *
1018 * Returns the locale charmap name.
1019 *
1020 * Debian GNU/Linux
1021 * LANG=C
1022 * Encoding.locale_charmap => "ANSI_X3.4-1968"
1023 * LANG=ja_JP.EUC-JP
1024 * Encoding.locale_charmap => "EUC-JP"
1025 *
1026 * SunOS 5
1027 * LANG=C
1028 * Encoding.locale_charmap => "646"
1029 * LANG=ja
1030 * Encoding.locale_charmap => "eucJP"
1031 *
1032 */
1033 VALUE
1034 rb_locale_charmap(VALUE klass)
1035 {
1036 #if defined NO_LOCALE_CHARMAP
1037 return rb_usascii_str_new2("ASCII-8BIT");
1038 #elif defined HAVE_LANGINFO_H
1039 char *codeset;
1040 codeset = nl_langinfo(CODESET);
1041 return rb_usascii_str_new2(codeset);
1042 #elif defined _WIN32
1043 return rb_sprintf("CP%d", GetACP());
1044 #else
1045 return Qnil;
1046 #endif
1047 }
1048
1049 static void
1050 set_encoding_const(const char *name, rb_encoding *enc)
1051 {
1052 VALUE encoding = rb_enc_from_encoding(enc);
1053 char *s = (char *)name;
1054 int haslower = 0, hasupper = 0, valid = 0;
1055
1056 if (ISDIGIT(*s)) return;
1057 if (ISUPPER(*s)) {
1058 hasupper = 1;
1059 while (*++s && (ISALNUM(*s) || *s == '_')) {
1060 if (ISLOWER(*s)) haslower = 1;
1061 }
1062 }
1063 if (!*s) {
1064 valid = 1;
1065 rb_define_const(rb_cEncoding, name, encoding);
1066 }
1067 if (!valid || haslower) {
1068 int len = strlen(name) + 1;
1069 if (!haslower || !hasupper) {
1070 do {
1071 if (ISLOWER(*s)) haslower = 1;
1072 if (ISUPPER(*s)) hasupper = 1;
1073 } while (*++s && (!haslower || !hasupper));
1074 }
1075 MEMCPY(s = ALLOCA_N(char, len), name, char, len);
1076 name = s;
1077 if (!valid) {
1078 if (ISLOWER(*s)) *s = ONIGENC_ASCII_CODE_TO_UPPER_CASE((int)*s);
1079 for (; *s; ++s) {
1080 if (!ISALNUM(*s)) *s = '_';
1081 }
1082 if (hasupper) {
1083 rb_define_const(rb_cEncoding, name, encoding);
1084 }
1085 }
1086 if (haslower) {
1087 for (s = (char *)name; *s; ++s) {
1088 if (ISLOWER(*s)) *s = ONIGENC_ASCII_CODE_TO_UPPER_CASE((int)*s);
1089 }
1090 rb_define_const(rb_cEncoding, name, encoding);
1091 }
1092 }
1093 }
1094
1095 static int
1096 rb_enc_name_list_i(st_data_t name, st_data_t idx, st_data_t arg)
1097 {
1098 VALUE ary = (VALUE)arg;
1099 VALUE str = rb_usascii_str_new2((char *)name);
1100 OBJ_FREEZE(str);
1101 rb_ary_push(ary, str);
1102 return ST_CONTINUE;
1103 }
1104
1105 /*
1106 * call-seq:
1107 * Encoding.name_list => ["enc1", "enc2", ...]
1108 *
1109 * Returns the list of available encoding names.
1110 *
1111 * Encoding.name_list
1112 * => ["US-ASCII", "ASCII-8BIT", "UTF-8",
1113 * "ISO-8859-1", "Shift_JIS", "EUC-JP",
1114 * "Windows-31J",
1115 * "BINARY", "CP932", "eucJP"]
1116 *
1117 * This list doesn't include dummy encodings.
1118 *
1119 */
1120
1121 static VALUE
1122 rb_enc_name_list(VALUE klass)
1123 {
1124 VALUE ary = rb_ary_new2(enc_table.names->num_entries);
1125 st_foreach(enc_table.names, rb_enc_name_list_i, (st_data_t)ary);
1126 return ary;
1127 }
1128
1129 static int
1130 rb_enc_aliases_enc_i(st_data_t name, st_data_t orig, st_data_t arg)
1131 {
1132 VALUE *p = (VALUE *)arg;
1133 VALUE aliases = p[0], ary = p[1];
1134 int idx = (int)orig;
1135 VALUE key, str = rb_ary_entry(ary, idx);
1136
1137 if (NIL_P(str)) {
1138 rb_encoding *enc = rb_enc_from_index(idx);
1139
1140 if (STRCASECMP((char*)name, rb_enc_name(enc)) == 0) {
1141 return ST_CONTINUE;
1142 }
1143 str = rb_usascii_str_new2(rb_enc_name(enc));
1144 OBJ_FREEZE(str);
1145 rb_ary_store(ary, idx, str);
1146 }
1147 key = rb_usascii_str_new2((char *)name);
1148 OBJ_FREEZE(key);
1149 rb_hash_aset(aliases, key, str);
1150 return ST_CONTINUE;
1151 }
1152
1153 /*
1154 * call-seq:
1155 * Encoding.aliases => {"alias1" => "orig1", "alias2" => "orig2", ...}
1156 *
1157 * Returns the hash of available encoding alias and original encoding name.
1158 *
1159 * Encoding.aliases
1160 * => {"BINARY"=>"ASCII-8BIT", "ASCII"=>"US-ASCII", "ANSI_X3.4-1986"=>"US-ASCII",
1161 * "SJIS"=>"Shift_JIS", "eucJP"=>"EUC-JP", "CP932"=>"Windows-31J"}
1162 *
1163 */
1164
1165 static VALUE
1166 rb_enc_aliases(VALUE klass)
1167 {
1168 VALUE aliases[2];
1169 aliases[0] = rb_hash_new();
1170 aliases[1] = rb_ary_new();
1171 st_foreach(enc_table.names, rb_enc_aliases_enc_i, (st_data_t)aliases);
1172 return aliases[0];
1173 }
1174
1175 void
1176 Init_Encoding(void)
1177 {
1178 id_base_encoding = rb_intern("#base_encoding");
1179
1180 rb_cEncoding = rb_define_class("Encoding", rb_cObject);
1181 rb_undef_alloc_func(rb_cEncoding);
1182 rb_define_method(rb_cEncoding, "to_s", enc_name, 0);
1183 rb_define_method(rb_cEncoding, "inspect", enc_inspect, 0);
1184 rb_define_method(rb_cEncoding, "name", enc_name, 0);
1185 rb_define_method(rb_cEncoding, "base_encoding", enc_base_encoding, 0);
1186 rb_define_method(rb_cEncoding, "dummy?", enc_dummy_p, 0);
1187 rb_define_singleton_method(rb_cEncoding, "list", enc_list, 0);
1188 rb_define_singleton_method(rb_cEncoding, "name_list", rb_enc_name_list, 0);
1189 rb_define_singleton_method(rb_cEncoding, "aliases", rb_enc_aliases, 0);
1190 rb_define_singleton_method(rb_cEncoding, "find", enc_find, 1);
1191 rb_define_singleton_method(rb_cEncoding, "compatible?", enc_compatible_p, 2);
1192
1193 rb_define_method(rb_cEncoding, "_dump", enc_dump, -1);
1194 rb_define_singleton_method(rb_cEncoding, "_load", enc_load, 1);
1195
1196 rb_define_singleton_method(rb_cEncoding, "default_external", get_default_external, 0);
1197 rb_define_singleton_method(rb_cEncoding, "locale_charmap", rb_locale_charmap, 0);
1198 }
1199
1200 /* locale insensitive functions */
1201
1202 #define ctype_test(c, ctype) \
1203 (rb_isascii(c) && ONIGENC_IS_ASCII_CODE_CTYPE((c), ctype))
1204
1205 int rb_isalnum(int c) { return ctype_test(c, ONIGENC_CTYPE_ALNUM); }
1206 int rb_isalpha(int c) { return ctype_test(c, ONIGENC_CTYPE_ALPHA); }
1207 int rb_isblank(int c) { return ctype_test(c, ONIGENC_CTYPE_BLANK); }
1208 int rb_iscntrl(int c) { return ctype_test(c, ONIGENC_CTYPE_CNTRL); }
1209 int rb_isdigit(int c) { return ctype_test(c, ONIGENC_CTYPE_DIGIT); }
1210 int rb_isgraph(int c) { return ctype_test(c, ONIGENC_CTYPE_GRAPH); }
1211 int rb_islower(int c) { return ctype_test(c, ONIGENC_CTYPE_LOWER); }
1212 int rb_isprint(int c) { return ctype_test(c, ONIGENC_CTYPE_PRINT); }
1213 int rb_ispunct(int c) { return ctype_test(c, ONIGENC_CTYPE_PUNCT); }
1214 int rb_isspace(int c) { return ctype_test(c, ONIGENC_CTYPE_SPACE); }
1215 int rb_isupper(int c) { return ctype_test(c, ONIGENC_CTYPE_UPPER); }
1216 int rb_isxdigit(int c) { return ctype_test(c, ONIGENC_CTYPE_XDIGIT); }
1217
1218 int
1219 rb_tolower(int c)
1220 {
1221 return rb_isascii(c) ? ONIGENC_ASCII_CODE_TO_LOWER_CASE(c) : c;
1222 }
1223
1224 int
1225 rb_toupper(int c)
1226 {
1227 return rb_isascii(c) ? ONIGENC_ASCII_CODE_TO_UPPER_CASE(c) : c;
1228 }
1229
ruby svn mirror