transcode_data.h

   1 /**********************************************************************
   2
   3   transcode_data.h -
   4
   5   $Author$
   6   created at: Mon 10 Dec 2007 14:01:47 JST 2007
   7
   8   Copyright (C) 2007 Martin Duerst
   9
  10 **********************************************************************/
  11
  12 #include "ruby/ruby.h"
  13
  14 #ifndef RUBY_TRANSCODE_DATA_H
  15 #define RUBY_TRANSCODE_DATA_H 1
  16
  17 typedef unsigned char base_element;
  18
  19 typedef struct byte_lookup {
  20     const base_element *base;
  21     const struct byte_lookup *const *info;
  22 } BYTE_LOOKUP;
  23
  24 #ifndef PType
  25 /* data file needs to treat this as a pointer, to remove warnings */
  26 #define PType (const BYTE_LOOKUP *)
  27 #endif
  28
  29 #define NOMAP   (PType 0x01)    /* single byte direct map */
  30 #define ONEbt   (0x02)          /* one byte payload */
  31 #define TWObt   (0x03)          /* two bytes payload */
  32 #define THREEbt (0x05)          /* three bytes payload */
  33 #define FOURbt  (0x06)          /* four bytes payload, UTF-8 only, macros start at getBT0 */
  34 #define INVALID (PType 0x07)    /* invalid byte sequence */
  35 #define UNDEF   (PType 0x09)    /* legal but undefined */
  36 #define ZERObt  (PType 0x0A)    /* zero bytes of payload, i.e. remove */
  37 #define FUNii   (PType 0x0B)    /* function from info to info */
  38 #define FUNsi   (PType 0x0D)    /* function from start to info */
  39 #define FUNio   (PType 0x0E)    /* function from info to output */
  40 #define FUNso   (PType 0x0F)    /* function from start to output */
  41
  42 #define o1(b1)          (PType((((unsigned char)(b1))<<8)|ONEbt))
  43 #define o2(b1,b2)       (PType((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|TWObt))
  44 #define o3(b1,b2,b3)    (PType((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|(((unsigned char)(b3))<<24)|THREEbt))
  45 #define o4(b0,b1,b2,b3) (PType((((unsigned char)(b1))<< 8)|(((unsigned char)(b2))<<16)|(((unsigned char)(b3))<<24)|((((unsigned char)(b0))&0x07)<<5)|FOURbt))
  46
  47 #define getBT1(a)       (((a)>> 8)&0xFF)
  48 #define getBT2(a)       (((a)>>16)&0xFF)
  49 #define getBT3(a)       (((a)>>24)&0xFF)
  50 #define getBT0(a)       ((((a)>> 5)&0x07)|0xF0)   /* for UTF-8 only!!! */
  51
  52 #define o2FUNii(b1,b2)  (PType((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|FUNii))
  53
  54 /* do we need these??? maybe not, can be done with simple tables */
  55 #define ONETRAIL       /* legal but undefined if one more trailing UTF-8 */
  56 #define TWOTRAIL       /* legal but undefined if two more trailing UTF-8 */
  57 #define THREETRAIL     /* legal but undefined if three more trailing UTF-8 */
  58
  59 typedef struct rb_transcoder rb_transcoder;
  60
  61 /* dynamic structure, one per conversion (similar to iconv_t) */
  62 /* may carry conversion state (e.g. for iso-2022-jp) */
  63 typedef struct rb_transcoding {
  64     const rb_transcoder *transcoder;
  65
  66     int flags;
  67
  68     int resume_position;
  69     const BYTE_LOOKUP *next_table;
  70     VALUE next_info;
  71     unsigned char next_byte;
  72
  73     int recognized_len; /* already interpreted */
  74     int readagain_len; /* not yet interpreted */
  75     union {
  76         unsigned char ary[8]; /* max_input <= sizeof(ary) */
  77         unsigned char *ptr; /* length: max_input */
  78     } readbuf; /* recognized_len + readagain_len used */
  79
  80     int writebuf_off;
  81     int writebuf_len;
  82     union {
  83         unsigned char ary[8]; /* max_output <= sizeof(ary) */
  84         unsigned char *ptr; /* length: max_output */
  85     } writebuf;
  86
  87     unsigned char stateful[256]; /* opaque data for stateful encoding */
  88 } rb_transcoding;
  89 #define TRANSCODING_READBUF(tc) \
  90     ((tc)->transcoder->max_input <= sizeof((tc)->readbuf.ary) ? \
  91      (tc)->readbuf.ary : \
  92      (tc)->readbuf.ptr)
  93 #define TRANSCODING_WRITEBUF(tc) \
  94     ((tc)->transcoder->max_output <= sizeof((tc)->writebuf.ary) ? \
  95      (tc)->writebuf.ary : \
  96      (tc)->writebuf.ptr)
  97
  98 /* static structure, one per supported encoding pair */
  99 struct rb_transcoder {
 100     const char *from_encoding;
 101     const char *to_encoding;
 102     const BYTE_LOOKUP *conv_tree_start;
 103     int input_unit_length;
 104     int max_input;
 105     int max_output;
 106     VALUE (*func_ii)(rb_transcoding*, VALUE); /* info  -> info   */
 107     VALUE (*func_si)(rb_transcoding*, const unsigned char*, size_t); /* start -> info   */
 108     int (*func_io)(rb_transcoding*, VALUE, const unsigned char*); /* info  -> output */
 109     int (*func_so)(rb_transcoding*, const unsigned char*, size_t, unsigned char*); /* start -> output */
 110     int (*resetstate_func)(rb_transcoding*, unsigned char*); /* -> output */
 111     int (*finish_func)(rb_transcoding*, unsigned char*); /* -> output */
 112 };
 113
 114 typedef enum {
 115     transcode_invalid_byte_sequence,
 116     transcode_undefined_conversion,
 117     transcode_destination_buffer_full,
 118     transcode_source_buffer_empty,
 119     transcode_finished,
 120     transcode_output_followed_by_input,
 121 } rb_econv_result_t;
 122
 123 typedef struct {
 124     const char *from;
 125     const char *to;
 126     rb_transcoding *tc;
 127     unsigned char *out_buf_start;
 128     unsigned char *out_data_start;
 129     unsigned char *out_data_end;
 130     unsigned char *out_buf_end;
 131     rb_econv_result_t last_result;
 132 } rb_econv_elem_t;
 133
 134 typedef struct {
 135     rb_econv_elem_t *elems;
 136     int num_trans;
 137     int num_finished;
 138     rb_transcoding *last_tc;
 139 } rb_econv_t;
 140
 141 void rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib);
 142 void rb_register_transcoder(const rb_transcoder *);
 143
 144 #endif /* RUBY_TRANSCODE_DATA_H */