transcode_data.h

   1 #ifndef RUBY_TRANSCODE_DATA_H
   2 #define RUBY_TRANSCODE_DATA_H 1
   3 /**********************************************************************
   4
   5   transcode_data.h -
   6
   7   $Author$
   8   created at: Mon 10 Dec 2007 14:01:47 JST 2007
   9
  10   Copyright (C) 2007 Martin Duerst
  11
  12 **********************************************************************/
  13
  14 #include "ruby/ruby.h"
  15
  16 RUBY_SYMBOL_EXPORT_BEGIN
  17
  18 #define WORDINDEX_SHIFT_BITS 2
  19 #define WORDINDEX2INFO(widx)      ((widx) << WORDINDEX_SHIFT_BITS)
  20 #define INFO2WORDINDEX(info)      ((info) >> WORDINDEX_SHIFT_BITS)
  21 #define BYTE_LOOKUP_BASE(bl) ((bl)[0])
  22 #define BYTE_LOOKUP_INFO(bl) ((bl)[1])
  23
  24 #define PType (unsigned int)
  25
  26 #define NOMAP   (PType 0x01)    /* direct map */
  27 #define ONEbt   (0x02)          /* one byte payload */
  28 #define TWObt   (0x03)          /* two bytes payload */
  29 #define THREEbt (0x05)          /* three bytes payload */
  30 #define FOURbt  (0x06)          /* four bytes payload, UTF-8 only, macros start at getBT0 */
  31 #define INVALID (PType 0x07)    /* invalid byte sequence */
  32 #define UNDEF   (PType 0x09)    /* legal but undefined */
  33 #define ZERObt  (PType 0x0A)    /* zero bytes of payload, i.e. remove */
  34 #define FUNii   (PType 0x0B)    /* function from info to info */
  35 #define FUNsi   (PType 0x0D)    /* function from start to info */
  36 #define FUNio   (PType 0x0E)    /* function from info to output */
  37 #define FUNso   (PType 0x0F)    /* function from start to output */
  38 #define STR1    (PType 0x11)    /* string 4 <= len <= 259 bytes: 1byte length + content */
  39 #define GB4bt   (PType 0x12)    /* GB18030 four bytes payload */
  40 #define FUNsio  (PType 0x13)    /* function from start and info to output */
  41
  42 #define STR1_LENGTH(byte_addr) (unsigned int)(*(byte_addr) + 4)
  43 #define STR1_BYTEINDEX(w) ((w) >> 6)
  44 #define makeSTR1(bi) (((bi) << 6) | STR1)
  45 #define makeSTR1LEN(len) ((len)-4)
  46
  47 #define o1(b1)          (PType((((unsigned char)(b1))<<8)|ONEbt))
  48 #define o2(b1,b2)       (PType((((unsigned char)(b1))<<8)|\
  49                                (((unsigned char)(b2))<<16)|\
  50                                TWObt))
  51 #define o3(b1,b2,b3)    (PType(((((unsigned char)(b1))<<8)|\
  52                                 (((unsigned char)(b2))<<16)|\
  53                                 (((unsigned int)(unsigned char)(b3))<<24)|\
  54                                 THREEbt)&\
  55                                0xffffffffU))
  56 #define o4(b0,b1,b2,b3) (PType(((((unsigned char)(b1))<<8)|\
  57                                 (((unsigned char)(b2))<<16)|\
  58                                 (((unsigned int)(unsigned char)(b3))<<24)|\
  59                                 ((((unsigned char)(b0))&0x07)<<5)|\
  60                                 FOURbt)&\
  61                                0xffffffffU))
  62 #define g4(b0,b1,b2,b3) (PType(((((unsigned char)(b0))<<8)|\
  63                                 (((unsigned char)(b2))<<16)|\
  64                                 ((((unsigned char)(b1))&0x0f)<<24)|\
  65                                 ((((unsigned int)(unsigned char)(b3))&0x0f)<<28)|\
  66                                 GB4bt)&\
  67                                0xffffffffU))
  68 #define funsio(diff)    (PType((((unsigned int)(diff))<<8)|FUNsio))
  69
  70 #define getBT1(a)       ((unsigned char)((a)>> 8))
  71 #define getBT2(a)       ((unsigned char)((a)>>16))
  72 #define getBT3(a)       ((unsigned char)((a)>>24))
  73 #define getBT0(a)       (((unsigned char)((a)>> 5)&0x07)|0xF0)   /* for UTF-8 only!!! */
  74
  75 #define getGB4bt0(a)    ((unsigned char)((a)>> 8))
  76 #define getGB4bt1(a)    (((unsigned char)((a)>>24)&0x0F)|0x30)
  77 #define getGB4bt2(a)    ((unsigned char)((a)>>16))
  78 #define getGB4bt3(a)    (((unsigned char)((a)>>28)&0x0F)|0x30)
  79
  80 #define o2FUNii(b1,b2)  (PType((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|FUNii))
  81
  82 /* do we need these??? maybe not, can be done with simple tables */
  83 #define ONETRAIL       /* legal but undefined if one more trailing UTF-8 */
  84 #define TWOTRAIL       /* legal but undefined if two more trailing UTF-8 */
  85 #define THREETRAIL     /* legal but undefined if three more trailing UTF-8 */
  86
  87 typedef enum {
  88     asciicompat_converter,        /* ASCII-compatible -> ASCII-compatible */
  89     asciicompat_decoder,          /* ASCII-incompatible -> ASCII-compatible */
  90     asciicompat_encoder           /* ASCII-compatible -> ASCII-incompatible */
  91   /* ASCII-incompatible -> ASCII-incompatible is intentionally omitted. */
  92 } rb_transcoder_asciicompat_type_t;
  93
  94 typedef struct rb_transcoder rb_transcoder;
  95
  96 /* static structure, one per supported encoding pair */
  97 struct rb_transcoder {
  98     const char *src_encoding;
  99     const char *dst_encoding;
 100     unsigned int conv_tree_start;
 101     const unsigned char *byte_array;
 102     unsigned int byte_array_length;
 103     const unsigned int *word_array;
 104     unsigned int word_array_length;
 105     int word_size;
 106     int input_unit_length;
 107     int max_input;
 108     int max_output;
 109     rb_transcoder_asciicompat_type_t asciicompat_type;
 110     size_t state_size;
 111     int (*state_init_func)(void*); /* ret==0:success ret!=0:failure(errno) */
 112     int (*state_fini_func)(void*); /* ret==0:success ret!=0:failure(errno) */
 113     VALUE (*func_ii)(void*, VALUE); /* info  -> info   */
 114     VALUE (*func_si)(void*, const unsigned char*, size_t); /* start -> info   */
 115     ssize_t (*func_io)(void*, VALUE, const unsigned char*, size_t); /* info  -> output */
 116     ssize_t (*func_so)(void*, const unsigned char*, size_t, unsigned char*, size_t); /* start -> output */
 117     ssize_t (*finish_func)(void*, unsigned char*, size_t); /* -> output */
 118     ssize_t (*resetsize_func)(void*); /* -> len */
 119     ssize_t (*resetstate_func)(void*, unsigned char*, size_t); /* -> output */
 120     ssize_t (*func_sio)(void*, const unsigned char*, size_t, VALUE, unsigned char*, size_t); /* start -> output */
 121 };
 122
 123 void rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib);
 124 void rb_register_transcoder(const rb_transcoder *);
 125
 126 /*
 127  * To get rid of collision of initializer symbols in statically-linked encodings
 128  * and transcoders
 129  */
 130 #if defined(EXTSTATIC) && EXTSTATIC
 131 # define TRANS_INIT(name) void Init_trans_ ## name(void)
 132 #else
 133 # define TRANS_INIT(name) void Init_ ## name(void)
 134 #endif
 135
 136 RUBY_SYMBOL_EXPORT_END
 137
 138 #endif /* RUBY_TRANSCODE_DATA_H */