encoding.d

   1 /* Invisible Vector Library
   2  * coded by Ketmar // Invisible Vector <ketmar@ketmar.no-ip.org>
   3  * Understanding is not required. Only obedience.
   4  *
   5  * This program is free software: you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation, version 3 of the License ONLY.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  16  */
  17 // 866,1251,koi8u
  18 module iv.encoding /*is aliced*/;
  19
  20 import std.encoding;
  21 import iv.alice;
  22 import iv.strex;
  23
  24
  25 // ////////////////////////////////////////////////////////////////////////// //
  26 // utils
  27 // `ch`: utf8 start
  28 // -1: invalid utf8
  29 byte utf8CodeLen (char ch) {
  30   //pragma(inline, true);
  31   if (ch < 0x80) return 1;
  32   if ((ch&0b1111_1110) == 0b1111_1100) return 6;
  33   if ((ch&0b1111_1100) == 0b1111_1000) return 5;
  34   if ((ch&0b1111_1000) == 0b1111_0000) return 4;
  35   if ((ch&0b1111_0000) == 0b1110_0000) return 3;
  36   if ((ch&0b1110_0000) == 0b1100_0000) return 2;
  37   return -1; // invalid
  38 }
  39
  40
  41 bool utf8Valid (const(char)[] buf) {
  42   auto bp = buf.ptr;
  43   auto left = buf.length;
  44   while (left-- > 0) {
  45     auto len = utf8CodeLen(*bp++)-1;
  46     if (len < 0 || len > left) return false;
  47     left -= len;
  48     while (len-- > 0) if (((*bp++)&0b1100_0000) != 0b1000_0000) return false;
  49   }
  50   return true;
  51 }
  52
  53
  54 // ////////////////////////////////////////////////////////////////////////// //
  55 immutable wchar[128] charMap1251 = [
  56   '\u0402','\u0403','\u201A','\u0453','\u201E','\u2026','\u2020','\u2021','\u20AC','\u2030','\u0409','\u2039','\u040A','\u040C','\u040B','\u040F',
  57   '\u0452','\u2018','\u2019','\u201C','\u201D','\u2022','\u2013','\u2014','\u003F','\u2122','\u0459','\u203A','\u045A','\u045C','\u045B','\u045F',
  58   '\u00A0','\u040E','\u045E','\u0408','\u00A4','\u0490','\u00A6','\u00A7','\u0401','\u00A9','\u0404','\u00AB','\u00AC','\u00AD','\u00AE','\u0407',
  59   '\u00B0','\u00B1','\u0406','\u0456','\u0491','\u00B5','\u00B6','\u00B7','\u0451','\u2116','\u0454','\u00BB','\u0458','\u0405','\u0455','\u0457',
  60   '\u0410','\u0411','\u0412','\u0413','\u0414','\u0415','\u0416','\u0417','\u0418','\u0419','\u041A','\u041B','\u041C','\u041D','\u041E','\u041F',
  61   '\u0420','\u0421','\u0422','\u0423','\u0424','\u0425','\u0426','\u0427','\u0428','\u0429','\u042A','\u042B','\u042C','\u042D','\u042E','\u042F',
  62   '\u0430','\u0431','\u0432','\u0433','\u0434','\u0435','\u0436','\u0437','\u0438','\u0439','\u043A','\u043B','\u043C','\u043D','\u043E','\u043F',
  63   '\u0440','\u0441','\u0442','\u0443','\u0444','\u0445','\u0446','\u0447','\u0448','\u0449','\u044A','\u044B','\u044C','\u044D','\u044E','\u044F',
  64 ];
  65
  66
  67 immutable wchar[128] charMap866 = [
  68   '\u0410','\u0411','\u0412','\u0413','\u0414','\u0415','\u0416','\u0417','\u0418','\u0419','\u041A','\u041B','\u041C','\u041D','\u041E','\u041F',
  69   '\u0420','\u0421','\u0422','\u0423','\u0424','\u0425','\u0426','\u0427','\u0428','\u0429','\u042A','\u042B','\u042C','\u042D','\u042E','\u042F',
  70   '\u0430','\u0431','\u0432','\u0433','\u0434','\u0435','\u0436','\u0437','\u0438','\u0439','\u043A','\u043B','\u043C','\u043D','\u043E','\u043F',
  71   '\u2591','\u2592','\u2593','\u2502','\u2524','\u2561','\u2562','\u2556','\u2555','\u2563','\u2551','\u2557','\u255D','\u255C','\u255B','\u2510',
  72   '\u2514','\u2534','\u252C','\u251C','\u2500','\u253C','\u255E','\u255F','\u255A','\u2554','\u2569','\u2566','\u2560','\u2550','\u256C','\u2567',
  73   '\u2568','\u2564','\u2565','\u2559','\u2558','\u2552','\u2553','\u256B','\u256A','\u2518','\u250C','\u2588','\u2584','\u258C','\u2590','\u2580',
  74   '\u0440','\u0441','\u0442','\u0443','\u0444','\u0445','\u0446','\u0447','\u0448','\u0449','\u044A','\u044B','\u044C','\u044D','\u044E','\u044F',
  75   '\u0401','\u0451','\u0404','\u0454','\u0407','\u0457','\u040E','\u045E','\u00B0','\u2219','\u00B7','\u221A','\u2116','\u00A4','\u25A0','\u00A0',
  76 ];
  77
  78
  79 immutable wchar[128] charMapKOI8 = [
  80   '\u2500','\u2502','\u250C','\u2510','\u2514','\u2518','\u251C','\u2524','\u252C','\u2534','\u253C','\u2580','\u2584','\u2588','\u258C','\u2590',
  81   '\u2591','\u2592','\u2593','\u2320','\u25A0','\u2219','\u221A','\u2248','\u2264','\u2265','\u00A0','\u2321','\u00B0','\u00B2','\u00B7','\u00F7',
  82   '\u2550','\u2551','\u2552','\u0451','\u0454','\u2554','\u0456','\u0457','\u2557','\u2558','\u2559','\u255A','\u255B','\u0491','\u255D','\u255E',
  83   '\u255F','\u2560','\u2561','\u0401','\u0404','\u2563','\u0406','\u0407','\u2566','\u2567','\u2568','\u2569','\u256A','\u0490','\u256C','\u00A9',
  84   '\u044E','\u0430','\u0431','\u0446','\u0434','\u0435','\u0444','\u0433','\u0445','\u0438','\u0439','\u043A','\u043B','\u043C','\u043D','\u043E',
  85   '\u043F','\u044F','\u0440','\u0441','\u0442','\u0443','\u0436','\u0432','\u044C','\u044B','\u0437','\u0448','\u044D','\u0449','\u0447','\u044A',
  86   '\u042E','\u0410','\u0411','\u0426','\u0414','\u0415','\u0424','\u0413','\u0425','\u0418','\u0419','\u041A','\u041B','\u041C','\u041D','\u041E',
  87   '\u041F','\u042F','\u0420','\u0421','\u0422','\u0423','\u0416','\u0412','\u042C','\u042B','\u0417','\u0428','\u042D','\u0429','\u0427','\u042A',
  88 ];
  89
  90
  91
  92 // ////////////////////////////////////////////////////////////////////////// //
  93 class K8ByteEncoding(alias charMap) : EncodingScheme {
  94   override bool canEncode (dchar c) const {
  95     if (c < 0x80) return true;
  96     foreach (wchar d; charMap) if (c == d) return true;
  97     return false;
  98   }
  99
 100   override usize encodedLength (dchar c) const
 101   in {
 102     assert(canEncode(c));
 103   }
 104   body {
 105     return 1;
 106   }
 107
 108   override usize encode (dchar c, ubyte[] buffer) const {
 109     if (c < 0x80) {
 110       buffer[0] = c&0xff;
 111     } else {
 112       buffer[0] = '?';
 113       foreach (immutable i, wchar d; charMap) {
 114         if (c == d) {
 115           buffer[0] = (i+128)&0xff;
 116           break;
 117         }
 118       }
 119     }
 120     return 1;
 121   }
 122
 123   override dchar decode (ref const(ubyte)[] s) const {
 124     ubyte bc = s[0];
 125     s = s[1..$];
 126     if (bc < 0x80) return cast(dchar)bc;
 127     return cast(dchar)charMap.ptr[bc-128];
 128   }
 129
 130   override dchar safeDecode (ref const(ubyte)[] s) const {
 131     ubyte bc = s[0];
 132     s = s[1..$];
 133     if (bc < 0x80) return cast(dchar)bc;
 134     auto res = cast(dchar)charMap.ptr[bc-128];
 135     if (res < 0x80) return INVALID_SEQUENCE;
 136     return res;
 137   }
 138
 139   override @property immutable(ubyte)[] replacementSequence () const { return ['?']; }
 140 }
 141
 142
 143 // ////////////////////////////////////////////////////////////////////////// //
 144 class KBBCWindows1251 : K8ByteEncoding!charMap1251 {
 145   shared static this () { EncodingScheme.register!KBBCWindows1251; }
 146   override string toString () const @safe pure nothrow @nogc { return "windows-1251"; }
 147   override string[] names () const @safe pure nothrow { return ["windows-1251", "cp-1251", "cp1251"]; }
 148 }
 149
 150
 151 class KBBCP866 : K8ByteEncoding!charMap866 {
 152   shared static this () { EncodingScheme.register!KBBCP866; }
 153   override string toString () const @safe pure nothrow @nogc { return "windows-866"; }
 154   override string[] names () const @safe pure nothrow { return ["windows-866", "cp-866", "cp866"]; }
 155 }
 156
 157
 158 class KBBCKOI8 : K8ByteEncoding!charMapKOI8 {
 159   shared static this () { EncodingScheme.register!KBBCKOI8; }
 160   override string toString () const @safe pure nothrow @nogc { return "koi8-u"; }
 161   override string[] names () const @safe pure nothrow { return ["koi8-u", "koi8-r", "koi8", "koi8u", "koi8r"]; }
 162 }
 163
 164
 165 // ////////////////////////////////////////////////////////////////////////// //
 166 public immutable char[256] koi8from866Table = [
 167   '\x00','\x01','\x02','\x03','\x04','\x05','\x06','\x07','\x08','\x09','\x0a','\x0b','\x0c','\x0d','\x0e','\x0f',
 168   '\x10','\x11','\x12','\x13','\x14','\x15','\x16','\x17','\x18','\x19','\x1a','\x1b','\x1c','\x1d','\x1e','\x1f',
 169   '\x20','\x21','\x22','\x23','\x24','\x25','\x26','\x27','\x28','\x29','\x2a','\x2b','\x2c','\x2d','\x2e','\x2f',
 170   '\x30','\x31','\x32','\x33','\x34','\x35','\x36','\x37','\x38','\x39','\x3a','\x3b','\x3c','\x3d','\x3e','\x3f',
 171   '\x40','\x41','\x42','\x43','\x44','\x45','\x46','\x47','\x48','\x49','\x4a','\x4b','\x4c','\x4d','\x4e','\x4f',
 172   '\x50','\x51','\x52','\x53','\x54','\x55','\x56','\x57','\x58','\x59','\x5a','\x5b','\x5c','\x5d','\x5e','\x5f',
 173   '\x60','\x61','\x62','\x63','\x64','\x65','\x66','\x67','\x68','\x69','\x6a','\x6b','\x6c','\x6d','\x6e','\x6f',
 174   '\x70','\x71','\x72','\x73','\x74','\x75','\x76','\x77','\x78','\x79','\x7a','\x7b','\x7c','\x7d','\x7e','\x7f',
 175   '\xe1','\xe2','\xf7','\xe7','\xe4','\xe5','\xf6','\xfa','\xe9','\xea','\xeb','\xec','\xed','\xee','\xef','\xf0',
 176   '\xf2','\xf3','\xf4','\xf5','\xe6','\xe8','\xe3','\xfe','\xfb','\xfd','\xff','\xf9','\xf8','\xfc','\xe0','\xf1',
 177   '\xc1','\xc2','\xd7','\xc7','\xc4','\xc5','\xd6','\xda','\xc9','\xca','\xcb','\xcc','\xcd','\xce','\xcf','\xd0',
 178   '\x90','\x91','\x92','\x81','\x87','\xb2','\x3f','\x3f','\x3f','\xb5','\xa1','\xa8','\xae','\x3f','\xac','\x83',
 179   '\x84','\x89','\x88','\x86','\x80','\x8a','\xaf','\xb0','\xab','\xa5','\xbb','\xb8','\xb1','\xa0','\xbe','\xb9',
 180   '\xba','\x3f','\x3f','\xaa','\xa9','\xa2','\x3f','\x3f','\xbc','\x85','\x82','\x8d','\x8c','\x8e','\x8f','\x8b',
 181   '\xd2','\xd3','\xd4','\xd5','\xc6','\xc8','\xc3','\xde','\xdb','\xdd','\xdf','\xd9','\xd8','\xdc','\xc0','\xd1',
 182   '\xb3','\xa3','\xb4','\xa4','\xb7','\xa7','\x3f','\x3f','\x9c','\x95','\x9e','\x96','\x3f','\x3f','\x94','\x9a',
 183 ];
 184
 185 public immutable char[256] koi8from1251Table = [
 186   '\x00','\x01','\x02','\x03','\x04','\x05','\x06','\x07','\x08','\x09','\x0a','\x0b','\x0c','\x0d','\x0e','\x0f',
 187   '\x10','\x11','\x12','\x13','\x14','\x15','\x16','\x17','\x18','\x19','\x1a','\x1b','\x1c','\x1d','\x1e','\x1f',
 188   '\x20','\x21','\x22','\x23','\x24','\x25','\x26','\x27','\x28','\x29','\x2a','\x2b','\x2c','\x2d','\x2e','\x2f',
 189   '\x30','\x31','\x32','\x33','\x34','\x35','\x36','\x37','\x38','\x39','\x3a','\x3b','\x3c','\x3d','\x3e','\x3f',
 190   '\x40','\x41','\x42','\x43','\x44','\x45','\x46','\x47','\x48','\x49','\x4a','\x4b','\x4c','\x4d','\x4e','\x4f',
 191   '\x50','\x51','\x52','\x53','\x54','\x55','\x56','\x57','\x58','\x59','\x5a','\x5b','\x5c','\x5d','\x5e','\x5f',
 192   '\x60','\x61','\x62','\x63','\x64','\x65','\x66','\x67','\x68','\x69','\x6a','\x6b','\x6c','\x6d','\x6e','\x6f',
 193   '\x70','\x71','\x72','\x73','\x74','\x75','\x76','\x77','\x78','\x79','\x7a','\x7b','\x7c','\x7d','\x7e','\x7f',
 194   '\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f',
 195   '\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f','\x3f',
 196   '\x9a','\x3f','\x3f','\x3f','\x3f','\xbd','\x3f','\x3f','\xb3','\xbf','\xb4','\x3f','\x3f','\x3f','\x3f','\xb7',
 197   '\x9c','\x3f','\xb6','\xa6','\xad','\x3f','\x3f','\x9e','\xa3','\x3f','\xa4','\x3f','\x3f','\x3f','\x3f','\xa7',
 198   '\xe1','\xe2','\xf7','\xe7','\xe4','\xe5','\xf6','\xfa','\xe9','\xea','\xeb','\xec','\xed','\xee','\xef','\xf0',
 199   '\xf2','\xf3','\xf4','\xf5','\xe6','\xe8','\xe3','\xfe','\xfb','\xfd','\xff','\xf9','\xf8','\xfc','\xe0','\xf1',
 200   '\xc1','\xc2','\xd7','\xc7','\xc4','\xc5','\xd6','\xda','\xc9','\xca','\xcb','\xcc','\xcd','\xce','\xcf','\xd0',
 201   '\xd2','\xd3','\xd4','\xd5','\xc6','\xc8','\xc3','\xde','\xdb','\xdd','\xdf','\xd9','\xd8','\xdc','\xc0','\xd1',
 202 ];
 203
 204 // char toupper/tolower, koi8
 205 immutable char[256] koi8tolowerTable = [
 206   '\x00','\x01','\x02','\x03','\x04','\x05','\x06','\x07','\x08','\x09','\x0a','\x0b','\x0c','\x0d','\x0e','\x0f',
 207   '\x10','\x11','\x12','\x13','\x14','\x15','\x16','\x17','\x18','\x19','\x1a','\x1b','\x1c','\x1d','\x1e','\x1f',
 208   '\x20','\x21','\x22','\x23','\x24','\x25','\x26','\x27','\x28','\x29','\x2a','\x2b','\x2c','\x2d','\x2e','\x2f',
 209   '\x30','\x31','\x32','\x33','\x34','\x35','\x36','\x37','\x38','\x39','\x3a','\x3b','\x3c','\x3d','\x3e','\x3f',
 210   '\x40','\x61','\x62','\x63','\x64','\x65','\x66','\x67','\x68','\x69','\x6a','\x6b','\x6c','\x6d','\x6e','\x6f',
 211   '\x70','\x71','\x72','\x73','\x74','\x75','\x76','\x77','\x78','\x79','\x7a','\x5b','\x5c','\x5d','\x5e','\x5f',
 212   '\x60','\x61','\x62','\x63','\x64','\x65','\x66','\x67','\x68','\x69','\x6a','\x6b','\x6c','\x6d','\x6e','\x6f',
 213   '\x70','\x71','\x72','\x73','\x74','\x75','\x76','\x77','\x78','\x79','\x7a','\x7b','\x7c','\x7d','\x7e','\x7f',
 214   '\x80','\x81','\x82','\x83','\x84','\x85','\x86','\x87','\x88','\x89','\x8a','\x8b','\x8c','\x8d','\x8e','\x8f',
 215   '\x90','\x91','\x92','\x93','\x94','\x95','\x96','\x97','\x98','\x99','\x9a','\x9b','\x9c','\x9d','\x9e','\x9f',
 216   '\xa0','\xa1','\xa2','\xa3','\xa4','\xa5','\xa6','\xa7','\xa8','\xa9','\xaa','\xab','\xac','\xad','\xae','\xaf',
 217   '\xb0','\xb1','\xb2','\xa3','\xa4','\xb5','\xa6','\xa7','\xb8','\xb9','\xba','\xbb','\xbc','\xad','\xbe','\xbf',
 218   '\xc0','\xc1','\xc2','\xc3','\xc4','\xc5','\xc6','\xc7','\xc8','\xc9','\xca','\xcb','\xcc','\xcd','\xce','\xcf',
 219   '\xd0','\xd1','\xd2','\xd3','\xd4','\xd5','\xd6','\xd7','\xd8','\xd9','\xda','\xdb','\xdc','\xdd','\xde','\xdf',
 220   '\xc0','\xc1','\xc2','\xc3','\xc4','\xc5','\xc6','\xc7','\xc8','\xc9','\xca','\xcb','\xcc','\xcd','\xce','\xcf',
 221   '\xd0','\xd1','\xd2','\xd3','\xd4','\xd5','\xd6','\xd7','\xd8','\xd9','\xda','\xdb','\xdc','\xdd','\xde','\xdf',
 222 ];
 223
 224 immutable char[256] koi8toupperTable = [
 225   '\x00','\x01','\x02','\x03','\x04','\x05','\x06','\x07','\x08','\x09','\x0a','\x0b','\x0c','\x0d','\x0e','\x0f',
 226   '\x10','\x11','\x12','\x13','\x14','\x15','\x16','\x17','\x18','\x19','\x1a','\x1b','\x1c','\x1d','\x1e','\x1f',
 227   '\x20','\x21','\x22','\x23','\x24','\x25','\x26','\x27','\x28','\x29','\x2a','\x2b','\x2c','\x2d','\x2e','\x2f',
 228   '\x30','\x31','\x32','\x33','\x34','\x35','\x36','\x37','\x38','\x39','\x3a','\x3b','\x3c','\x3d','\x3e','\x3f',
 229   '\x40','\x41','\x42','\x43','\x44','\x45','\x46','\x47','\x48','\x49','\x4a','\x4b','\x4c','\x4d','\x4e','\x4f',
 230   '\x50','\x51','\x52','\x53','\x54','\x55','\x56','\x57','\x58','\x59','\x5a','\x5b','\x5c','\x5d','\x5e','\x5f',
 231   '\x60','\x41','\x42','\x43','\x44','\x45','\x46','\x47','\x48','\x49','\x4a','\x4b','\x4c','\x4d','\x4e','\x4f',
 232   '\x50','\x51','\x52','\x53','\x54','\x55','\x56','\x57','\x58','\x59','\x5a','\x7b','\x7c','\x7d','\x7e','\x7f',
 233   '\x80','\x81','\x82','\x83','\x84','\x85','\x86','\x87','\x88','\x89','\x8a','\x8b','\x8c','\x8d','\x8e','\x8f',
 234   '\x90','\x91','\x92','\x93','\x94','\x95','\x96','\x97','\x98','\x99','\x9a','\x9b','\x9c','\x9d','\x9e','\x9f',
 235   '\xa0','\xa1','\xa2','\xb3','\xb4','\xa5','\xb6','\xb7','\xa8','\xa9','\xaa','\xab','\xac','\xbd','\xae','\xaf',
 236   '\xb0','\xb1','\xb2','\xb3','\xb4','\xb5','\xb6','\xb7','\xb8','\xb9','\xba','\xbb','\xbc','\xbd','\xbe','\xbf',
 237   '\xe0','\xe1','\xe2','\xe3','\xe4','\xe5','\xe6','\xe7','\xe8','\xe9','\xea','\xeb','\xec','\xed','\xee','\xef',
 238   '\xf0','\xf1','\xf2','\xf3','\xf4','\xf5','\xf6','\xf7','\xf8','\xf9','\xfa','\xfb','\xfc','\xfd','\xfe','\xff',
 239   '\xe0','\xe1','\xe2','\xe3','\xe4','\xe5','\xe6','\xe7','\xe8','\xe9','\xea','\xeb','\xec','\xed','\xee','\xef',
 240   '\xf0','\xf1','\xf2','\xf3','\xf4','\xf5','\xf6','\xf7','\xf8','\xf9','\xfa','\xfb','\xfc','\xfd','\xfe','\xff',
 241 ];
 242
 243 immutable ubyte[32] koi8alphaTable = [
 244   0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xfe,0xff,0xff,0x07,0xfe,0xff,0xff,0x07,
 245   0x00,0x00,0x00,0x00,0xd8,0x20,0xd8,0x20,0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff,
 246 ];
 247
 248 char koi8lower (char ch) pure nothrow @trusted @nogc {
 249   pragma(inline, true);
 250   return koi8tolowerTable.ptr[cast(int)ch];
 251 }
 252
 253 char koi8upper (char ch) pure nothrow @trusted @nogc {
 254   pragma(inline, true);
 255   return koi8toupperTable.ptr[cast(int)ch];
 256 }
 257
 258 bool koi8isAlpha (char ch) pure nothrow @trusted @nogc {
 259   pragma(inline, true);
 260   return ((koi8alphaTable.ptr[ch/8]&(1<<(ch%8))) != 0);
 261 }
 262
 263
 264 // ////////////////////////////////////////////////////////////////////////// //
 265 import std.traits;
 266
 267
 268 // SLOOOOOW. but who cares?
 269 T recode(T) (T s, const(char)[] to, const(char)[] from) if (isSomeString!T) {
 270   ubyte[] res;
 271   ubyte[16] buf;
 272   if (strEquCI(from, to)) return s;
 273   // check if we need to do anything
 274   bool needWork = true;
 275   foreach (immutable char ch; s) if (ch >= 128) { needWork = true; break; }
 276   if (!needWork) return s;
 277   auto efrom = (strEquCI(from, "utf-8") ? new EncodingSchemeUtf8() : EncodingScheme.create(cast(string)from)); // sorry, phobos API is fubared
 278   auto eto = (strEquCI(to, "utf-8") ? new EncodingSchemeUtf8() : EncodingScheme.create(cast(string)to)); // sorry, phobos API is fubared
 279   auto ub = cast(const(ubyte)[])s;
 280   while (ub.length > 0) {
 281     dchar dc = efrom.safeDecode(ub);
 282     if (dc == INVALID_SEQUENCE) dc = '?';
 283     auto len = eto.encode(dc, buf);
 284     res ~= buf[0..len];
 285   }
 286   //import std.stdio; writefln("<%s>", cast(string)res);
 287   return cast(T)res; // it is safe to cast here
 288 }
 289
 290
 291 T recodeToKOI8(T) (T s, string from=null) if (isSomeString!T) {
 292   ubyte[] res;
 293   ubyte[1] buf;
 294   if (strEquCI(from, "koi8-u") || strEquCI(from, "koi8-r") || strEquCI(from, "koi8")) return s;
 295   // check if we need to do anything
 296   bool needWork = true;
 297   foreach (immutable char ch; s) if (ch >= 128) { needWork = true; break; }
 298   if (!needWork) return s;
 299   auto eto = EncodingScheme.create("koi8-u");
 300   if (from.length == 0) {
 301     import std.utf : count;
 302     // from utf-8
 303     auto len = s.count;
 304     res = new ubyte[](len);
 305     usize idx = 0;
 306     foreach (dchar dc; s) {
 307       eto.encode(dc, buf);
 308       if (idx >= res.length) {
 309         //import std.conv : to;
 310         //assert(0, "pos: "~idx.to!string~"; len="~res.length.to!string~"; slen="~s.length.to!string~"; string: '"~s.to!string);
 311         res ~= buf[0];
 312       } else {
 313         res[idx++] = buf[0];
 314       }
 315     }
 316   } else {
 317     auto efrom = EncodingScheme.create(from);
 318     auto ub = cast(const(ubyte)[])s;
 319     while (ub.length > 0) {
 320       dchar dc = efrom.safeDecode(ub);
 321       if (dc == INVALID_SEQUENCE) dc = '?';
 322       eto.encode(dc, buf);
 323       res ~= buf[0];
 324     }
 325   }
 326   return cast(T)res; // it is safe to cast here
 327 }
 328
 329
 330 // ////////////////////////////////////////////////////////////////////////// //
 331 //public import std.conv;
 332
 333 T to(string enc, T) (T s)
 334 if (isSomeString!T && (enc == "koi8" || enc == "koi8-r" || enc == "koi8-u" || enc == "koi8r" || enc == "koi8u"))
 335 {
 336   return recodeToKOI8(s);
 337 }