test/ruby/test_econv.rb

   1 require 'test/unit'
   2
   3 class TestEncodingConverter < Test::Unit::TestCase
   4   def check_ec(edst, esrc, eres, dst, src, ec, off, len, flags=0)
   5     res = ec.primitive_convert(src, dst, off, len, flags)
   6     assert_equal([edst.dup.force_encoding("ASCII-8BIT"),
   7                   esrc.dup.force_encoding("ASCII-8BIT"),
   8                   eres],
   9                  [dst.dup.force_encoding("ASCII-8BIT"),
  10                   src.dup.force_encoding("ASCII-8BIT"),
  11                   res])
  12   end
  13
  14   def assert_econv(converted, eres, obuf_bytesize, ec, consumed, rest, flags=0)
  15     ec = Encoding::Converter.new(*ec) if Array === ec
  16     i = consumed + rest
  17     o = ""
  18     ret = ec.primitive_convert(i, o, 0, obuf_bytesize, flags)
  19     assert_equal([converted,    eres,       rest],
  20                  [o,            ret,           i])
  21   end
  22
  23   def test_new
  24     assert_kind_of(Encoding::Converter, Encoding::Converter.new("UTF-8", "EUC-JP"))
  25     assert_kind_of(Encoding::Converter, Encoding::Converter.new(Encoding::UTF_8, Encoding::EUC_JP))
  26   end
  27
  28   def test_new_fail
  29     name1 = "encoding-which-is-not-exist-1"
  30     name2 = "encoding-which-is-not-exist-2"
  31
  32     assert_raise(ArgumentError) {
  33       Encoding::Converter.new(name1, name2)
  34     }
  35
  36     encoding_list = Encoding.list.map {|e| e.name }
  37     assert(!encoding_list.include?(name1))
  38     assert(!encoding_list.include?(name2))
  39   end
  40
  41   def test_get_encoding
  42     ec = Encoding::Converter.new("UTF-8", "EUC-JP")
  43     assert_equal(Encoding::UTF_8, ec.source_encoding)
  44     assert_equal(Encoding::EUC_JP, ec.destination_encoding)
  45   end
  46
  47   def test_result_encoding
  48     ec = Encoding::Converter.new("UTF-8", "EUC-JP")
  49     dst = "".force_encoding("ASCII-8BIT")
  50     assert_equal(Encoding::ASCII_8BIT, dst.encoding)
  51     ec.primitive_convert("\u{3042}", dst, nil, 10)
  52     assert_equal(Encoding::EUC_JP, dst.encoding)
  53   end
  54
  55   def test_output_region
  56     ec = Encoding::Converter.new("UTF-8", "EUC-JP")
  57     ec.primitive_convert(src="a", dst="b", nil, 1, Encoding::Converter::PARTIAL_INPUT)
  58     assert_equal("ba", dst)
  59     ec.primitive_convert(src="a", dst="b", 0, 1, Encoding::Converter::PARTIAL_INPUT)
  60     assert_equal("a", dst)
  61     ec.primitive_convert(src="a", dst="b", 1, 1, Encoding::Converter::PARTIAL_INPUT)
  62     assert_equal("ba", dst)
  63     assert_raise(ArgumentError) {
  64       ec.primitive_convert(src="a", dst="b", 2, 1, Encoding::Converter::PARTIAL_INPUT)
  65     }
  66     assert_raise(ArgumentError) {
  67       ec.primitive_convert(src="a", dst="b", -1, 1, Encoding::Converter::PARTIAL_INPUT)
  68     }
  69     assert_raise(ArgumentError) {
  70       ec.primitive_convert(src="a", dst="b", 1, -1, Encoding::Converter::PARTIAL_INPUT)
  71     }
  72   end
  73
  74   def test_partial_input
  75     ec = Encoding::Converter.new("UTF-8", "EUC-JP")
  76     ret = ec.primitive_convert(src="", dst="", nil, 10, Encoding::Converter::PARTIAL_INPUT)
  77     assert_equal(:source_buffer_empty, ret)
  78     ret = ec.primitive_convert(src="", dst="", nil, 10)
  79     assert_equal(:finished, ret)
  80   end
  81
  82   def test_accumulate_dst1
  83     ec = Encoding::Converter.new("UTF-8", "EUC-JP")
  84     a =     ["", "abc\u{3042}def", ec, nil, 1]
  85     check_ec("a",  "c\u{3042}def", :destination_buffer_full, *a)
  86     check_ec("ab",  "\u{3042}def", :destination_buffer_full, *a)
  87     check_ec("abc",         "def", :destination_buffer_full, *a)
  88     check_ec("abc\xA4",     "def", :destination_buffer_full, *a)
  89     check_ec("abc\xA4\xA2",  "ef", :destination_buffer_full, *a)
  90     check_ec("abc\xA4\xA2d",  "f", :destination_buffer_full, *a)
  91     check_ec("abc\xA4\xA2de",  "", :destination_buffer_full, *a)
  92     check_ec("abc\xA4\xA2def", "", :finished,  *a)
  93   end
  94
  95   def test_accumulate_dst2
  96     ec = Encoding::Converter.new("UTF-8", "EUC-JP")
  97     a =     ["", "abc\u{3042}def", ec, nil, 2]
  98     check_ec("ab",  "\u{3042}def", :destination_buffer_full, *a)
  99     check_ec("abc\xA4",     "def", :destination_buffer_full, *a)
 100     check_ec("abc\xA4\xA2d",  "f", :destination_buffer_full, *a)
 101     check_ec("abc\xA4\xA2def", "", :finished,  *a)
 102   end
 103
 104   def test_eucjp_to_utf8
 105     assert_econv("", :finished, 100, ["UTF-8", "EUC-JP"], "", "")
 106     assert_econv("a", :finished, 100, ["UTF-8", "EUC-JP"], "a", "")
 107   end
 108
 109   def test_iso2022jp
 110     assert_econv("", :finished, 100, ["Shift_JIS", "ISO-2022-JP"], "", "")
 111   end
 112
 113   def test_iso2022jp_outstream
 114     ec = Encoding::Converter.new("EUC-JP", "ISO-2022-JP")
 115     a = ["", src="", ec, nil, 50, Encoding::Converter::PARTIAL_INPUT]
 116     src << "a";        check_ec("a",                           "", :source_buffer_empty, *a)
 117     src << "\xA2";     check_ec("a",                           "", :source_buffer_empty, *a)
 118     src << "\xA4";     check_ec("a\e$B\"$",                    "", :source_buffer_empty, *a)
 119     src << "\xA1";     check_ec("a\e$B\"$",                    "", :source_buffer_empty, *a)
 120     src << "\xA2";     check_ec("a\e$B\"$!\"",                 "", :source_buffer_empty, *a)
 121     src << "b";        check_ec("a\e$B\"$!\"\e(Bb",            "", :source_buffer_empty, *a)
 122     src << "\xA2\xA6"; check_ec("a\e$B\"$!\"\e(Bb\e$B\"&",     "", :source_buffer_empty, *a)
 123     a[-1] = 0;         check_ec("a\e$B\"$!\"\e(Bb\e$B\"&\e(B", "", :finished, *a)
 124   end
 125
 126   def test_invalid
 127     assert_econv("", :invalid_byte_sequence,    100, ["UTF-8", "EUC-JP"], "\x80", "")
 128     assert_econv("a", :invalid_byte_sequence,   100, ["UTF-8", "EUC-JP"], "a\x80", "")
 129     assert_econv("a", :invalid_byte_sequence,   100, ["UTF-8", "EUC-JP"], "a\x80", "\x80")
 130     assert_econv("abc", :invalid_byte_sequence, 100, ["UTF-8", "EUC-JP"], "abc\xFF", "def")
 131     assert_econv("abc", :invalid_byte_sequence, 100, ["Shift_JIS", "EUC-JP"], "abc\xFF", "def")
 132     assert_econv("abc", :invalid_byte_sequence, 100, ["ISO-2022-JP", "EUC-JP"], "abc\xFF", "def")
 133   end
 134
 135   def test_invalid2
 136     ec = Encoding::Converter.new("Shift_JIS", "EUC-JP")
 137     a =     ["", "abc\xFFdef", ec, nil, 1]
 138     check_ec("a",  "c\xFFdef", :destination_buffer_full, *a)
 139     check_ec("ab",  "\xFFdef", :destination_buffer_full, *a)
 140     check_ec("abc",     "def", :invalid_byte_sequence, *a)
 141     check_ec("abcd",      "f", :destination_buffer_full, *a)
 142     check_ec("abcde",      "", :destination_buffer_full, *a)
 143     check_ec("abcdef",     "", :finished, *a)
 144   end
 145
 146   def test_invalid3
 147     ec = Encoding::Converter.new("Shift_JIS", "EUC-JP")
 148     a =     ["", "abc\xFFdef", ec, nil, 10]
 149     check_ec("abc",     "def", :invalid_byte_sequence, *a)
 150     check_ec("abcdef",     "", :finished, *a)
 151   end
 152
 153   def test_invalid4
 154     ec = Encoding::Converter.new("Shift_JIS", "EUC-JP")
 155     a =     ["", "abc\xFFdef", ec, nil, 10, Encoding::Converter::OUTPUT_FOLLOWED_BY_INPUT]
 156     check_ec("a", "bc\xFFdef", :output_followed_by_input, *a)
 157     check_ec("ab", "c\xFFdef", :output_followed_by_input, *a)
 158     check_ec("abc", "\xFFdef", :output_followed_by_input, *a)
 159     check_ec("abc",     "def", :invalid_byte_sequence, *a)
 160     check_ec("abcd",     "ef", :output_followed_by_input, *a)
 161     check_ec("abcde",     "f", :output_followed_by_input, *a)
 162     check_ec("abcdef",     "", :output_followed_by_input, *a)
 163     check_ec("abcdef",     "", :finished, *a)
 164   end
 165
 166   def test_invalid_utf16le
 167     ec = Encoding::Converter.new("UTF-16LE", "UTF-8")
 168     a = ["", src="", ec, nil, 50, Encoding::Converter::PARTIAL_INPUT]
 169     src << "A";         check_ec("",                            "", :source_buffer_empty, *a)
 170     src << "\x00";      check_ec("A",                           "", :source_buffer_empty, *a)
 171     src << "\x00";      check_ec("A",                           "", :source_buffer_empty, *a)
 172     src << "\xd8";      check_ec("A",                           "", :source_buffer_empty, *a)
 173     src << "\x01";      check_ec("A",                           "", :source_buffer_empty, *a)
 174     src << "\x02";      check_ec("A",                           "", :invalid_byte_sequence, *a)
 175     src << "\x03";      check_ec("A\u{0201}",                   "", :source_buffer_empty, *a)
 176     src << "\x04";      check_ec("A\u{0201}\u{0403}",           "", :source_buffer_empty, *a)
 177     src << "\x00";      check_ec("A\u{0201}\u{0403}",           "", :source_buffer_empty, *a)
 178     src << "\xd8";      check_ec("A\u{0201}\u{0403}",           "", :source_buffer_empty, *a)
 179     src << "\x00";      check_ec("A\u{0201}\u{0403}",           "", :source_buffer_empty, *a)
 180     src << "\xd8";      check_ec("A\u{0201}\u{0403}",           "", :invalid_byte_sequence, *a)
 181     src << "\x00";      check_ec("A\u{0201}\u{0403}",           "", :source_buffer_empty, *a)
 182     src << "\xdc";      check_ec("A\u{0201}\u{0403}\u{10000}",  "", :source_buffer_empty, *a)
 183   end
 184
 185   def test_invalid_utf16be
 186     ec = Encoding::Converter.new("UTF-16BE", "UTF-8")
 187     a = ["", src="", ec, nil, 50, Encoding::Converter::PARTIAL_INPUT]
 188     src << "\x00";      check_ec("",                            "", :source_buffer_empty, *a)
 189     src << "A";         check_ec("A",                           "", :source_buffer_empty, *a)
 190     src << "\xd8";      check_ec("A",                           "", :source_buffer_empty, *a)
 191     src << "\x00";      check_ec("A",                           "", :source_buffer_empty, *a)
 192     src << "\x02";      check_ec("A",                           "", :invalid_byte_sequence, *a)
 193     src << "\x01";      check_ec("A\u{0201}",                   "", :source_buffer_empty, *a)
 194     src << "\x04";      check_ec("A\u{0201}",                   "", :source_buffer_empty, *a)
 195     src << "\x03";      check_ec("A\u{0201}\u{0403}",           "", :source_buffer_empty, *a)
 196     src << "\xd8";      check_ec("A\u{0201}\u{0403}",           "", :source_buffer_empty, *a)
 197     src << "\x00";      check_ec("A\u{0201}\u{0403}",           "", :source_buffer_empty, *a)
 198     src << "\xd8";      check_ec("A\u{0201}\u{0403}",           "", :invalid_byte_sequence, *a)
 199     src << "\x00";      check_ec("A\u{0201}\u{0403}",           "", :source_buffer_empty, *a)
 200     src << "\xdc";      check_ec("A\u{0201}\u{0403}",           "", :source_buffer_empty, *a)
 201     src << "\x00";      check_ec("A\u{0201}\u{0403}\u{10000}",  "", :source_buffer_empty, *a)
 202   end
 203
 204   def test_errors
 205     ec = Encoding::Converter.new("UTF-16BE", "EUC-JP")
 206     a =     ["", "\xFF\xFE\x00A\xDC\x00\x00B", ec, nil, 10]
 207     check_ec("",         "\x00A\xDC\x00\x00B", :undefined_conversion, *a)
 208     check_ec("A",                     "\x00B", :invalid_byte_sequence, *a) # \xDC\x00 is invalid as UTF-16BE
 209     check_ec("AB",                         "", :finished, *a)
 210   end
 211
 212   def test_errors2
 213     ec = Encoding::Converter.new("UTF-16BE", "EUC-JP")
 214     a =     ["", "\xFF\xFE\x00A\xDC\x00\x00B", ec, nil, 10, Encoding::Converter::OUTPUT_FOLLOWED_BY_INPUT]
 215     check_ec("",         "\x00A\xDC\x00\x00B", :undefined_conversion, *a)
 216     check_ec("A",             "\xDC\x00\x00B", :output_followed_by_input, *a)
 217     check_ec("A",                     "\x00B", :invalid_byte_sequence, *a)
 218     check_ec("AB",                         "", :output_followed_by_input, *a)
 219     check_ec("AB",                         "", :finished, *a)
 220   end
 221
 222   def test_universal_newline
 223     ec = Encoding::Converter.new("UTF-8", "EUC-JP", Encoding::Converter::UNIVERSAL_NEWLINE_DECODER)
 224     a = ["", src="", ec, nil, 50, Encoding::Converter::PARTIAL_INPUT]
 225     src << "abc\r\ndef"; check_ec("abc\ndef",                             "", :source_buffer_empty, *a)
 226     src << "ghi\njkl";   check_ec("abc\ndefghi\njkl",                     "", :source_buffer_empty, *a)
 227     src << "mno\rpqr";   check_ec("abc\ndefghi\njklmno\npqr",             "", :source_buffer_empty, *a)
 228     src << "stu\r";      check_ec("abc\ndefghi\njklmno\npqrstu\n",        "", :source_buffer_empty, *a)
 229     src << "\nvwx";      check_ec("abc\ndefghi\njklmno\npqrstu\nvwx",     "", :source_buffer_empty, *a)
 230     src << "\nyz";       check_ec("abc\ndefghi\njklmno\npqrstu\nvwx\nyz", "", :source_buffer_empty, *a)
 231   end
 232
 233   def test_crlf_newline
 234     ec = Encoding::Converter.new("UTF-8", "EUC-JP", Encoding::Converter::CRLF_NEWLINE_ENCODER)
 235     assert_econv("abc\r\ndef", :finished, 50, ec, "abc\ndef", "")
 236   end
 237
 238   def test_cr_newline
 239     ec = Encoding::Converter.new("UTF-8", "EUC-JP", Encoding::Converter::CR_NEWLINE_ENCODER)
 240     assert_econv("abc\rdef", :finished, 50, ec, "abc\ndef", "")
 241   end
 242
 243   def test_output_followed_by_input
 244     ec = Encoding::Converter.new("UTF-8", "EUC-JP")
 245     a =     ["",  "abc\u{3042}def", ec, nil, 100, Encoding::Converter::OUTPUT_FOLLOWED_BY_INPUT]
 246     check_ec("a",  "bc\u{3042}def", :output_followed_by_input, *a)
 247     check_ec("ab",  "c\u{3042}def", :output_followed_by_input, *a)
 248     check_ec("abc",  "\u{3042}def", :output_followed_by_input, *a)
 249     check_ec("abc\xA4\xA2",  "def", :output_followed_by_input, *a)
 250     check_ec("abc\xA4\xA2d",  "ef", :output_followed_by_input, *a)
 251     check_ec("abc\xA4\xA2de",  "f", :output_followed_by_input, *a)
 252     check_ec("abc\xA4\xA2def",  "", :output_followed_by_input, *a)
 253     check_ec("abc\xA4\xA2def",  "", :finished, *a)
 254   end
 255 end