transcode_data.h

   1 /**********************************************************************
   2
   3   transcode_data.h -
   4
   5   $Author$
   6   created at: Mon 10 Dec 2007 14:01:47 JST 2007
   7
   8   Copyright (C) 2007 Martin Duerst
   9
  10 **********************************************************************/
  11
  12 #include "ruby/ruby.h"
  13
  14 #ifndef RUBY_TRANSCODE_DATA_H
  15 #define RUBY_TRANSCODE_DATA_H 1
  16
  17 typedef unsigned char base_element;
  18
  19 typedef struct byte_lookup {
  20     const base_element *base;
  21     const struct byte_lookup *const *info;
  22 } BYTE_LOOKUP;
  23
  24 #ifndef PType
  25 /* data file needs to treat this as a pointer, to remove warnings */
  26 #define PType (const BYTE_LOOKUP *)
  27 #endif
  28
  29 #define NOMAP   (PType 0x01)    /* single byte direct map */
  30 #define ONEbt   (0x02)          /* one byte payload */
  31 #define TWObt   (0x03)          /* two bytes payload */
  32 #define THREEbt (0x05)          /* three bytes payload */
  33 #define FOURbt  (0x06)          /* four bytes payload, UTF-8 only, macros start at getBT0 */
  34 #define INVALID (PType 0x07)    /* invalid byte sequence */
  35 #define UNDEF   (PType 0x09)    /* legal but undefined */
  36 #define ZERObt  (PType 0x0A)    /* zero bytes of payload, i.e. remove */
  37 #define FUNii   (PType 0x0B)    /* function from info to info */
  38 #define FUNsi   (PType 0x0D)    /* function from start to info */
  39 #define FUNio   (PType 0x0E)    /* function from info to output */
  40 #define FUNso   (PType 0x0F)    /* function from start to output */
  41
  42 #define o1(b1)          (PType((((unsigned char)(b1))<<8)|ONEbt))
  43 #define o2(b1,b2)       (PType((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|TWObt))
  44 #define o3(b1,b2,b3)    (PType((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|(((unsigned char)(b3))<<24)|THREEbt))
  45 #define o4(b0,b1,b2,b3) (PType((((unsigned char)(b1))<< 8)|(((unsigned char)(b2))<<16)|(((unsigned char)(b3))<<24)|((((unsigned char)(b0))&0x07)<<5)|FOURbt))
  46
  47 #define getBT1(a)       (((a)>> 8)&0xFF)
  48 #define getBT2(a)       (((a)>>16)&0xFF)
  49 #define getBT3(a)       (((a)>>24)&0xFF)
  50 #define getBT0(a)       ((((a)>> 5)&0x07)|0xF0)   /* for UTF-8 only!!! */
  51
  52 #define o2FUNii(b1,b2)  (PType((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|FUNii))
  53
  54 /* do we need these??? maybe not, can be done with simple tables */
  55 #define ONETRAIL       /* legal but undefined if one more trailing UTF-8 */
  56 #define TWOTRAIL       /* legal but undefined if two more trailing UTF-8 */
  57 #define THREETRAIL     /* legal but undefined if three more trailing UTF-8 */
  58
  59 typedef enum {
  60   stateless_converter,  /* stateless -> stateless */
  61   stateful_decoder,     /* stateful -> stateless */
  62   stateful_encoder      /* stateless -> stateful */
  63   /* stateful -> stateful is intentionally ommitted. */
  64 } rb_transcoder_stateful_type_t;
  65
  66 typedef struct rb_transcoder rb_transcoder;
  67
  68 /* dynamic structure, one per conversion (similar to iconv_t) */
  69 /* may carry conversion state (e.g. for iso-2022-jp) */
  70 typedef struct rb_transcoding {
  71     const rb_transcoder *transcoder;
  72
  73     int flags;
  74
  75     int resume_position;
  76     const BYTE_LOOKUP *next_table;
  77     VALUE next_info;
  78     unsigned char next_byte;
  79
  80     int recognized_len; /* already interpreted */
  81     int readagain_len; /* not yet interpreted */
  82     union {
  83         unsigned char ary[8]; /* max_input <= sizeof(ary) */
  84         unsigned char *ptr; /* length: max_input */
  85     } readbuf; /* recognized_len + readagain_len used */
  86
  87     int writebuf_off;
  88     int writebuf_len;
  89     union {
  90         unsigned char ary[8]; /* max_output <= sizeof(ary) */
  91         unsigned char *ptr; /* length: max_output */
  92     } writebuf;
  93
  94     unsigned char stateful[256]; /* opaque data for stateful encoding */
  95 } rb_transcoding;
  96 #define TRANSCODING_READBUF(tc) \
  97     ((tc)->transcoder->max_input <= sizeof((tc)->readbuf.ary) ? \
  98      (tc)->readbuf.ary : \
  99      (tc)->readbuf.ptr)
 100 #define TRANSCODING_WRITEBUF(tc) \
 101     ((tc)->transcoder->max_output <= sizeof((tc)->writebuf.ary) ? \
 102      (tc)->writebuf.ary : \
 103      (tc)->writebuf.ptr)
 104
 105 /* static structure, one per supported encoding pair */
 106 struct rb_transcoder {
 107     const char *from_encoding;
 108     const char *to_encoding;
 109     const BYTE_LOOKUP *conv_tree_start;
 110     int input_unit_length;
 111     int max_input;
 112     int max_output;
 113     rb_transcoder_stateful_type_t stateful_type;
 114     VALUE (*func_ii)(rb_transcoding*, VALUE); /* info  -> info   */
 115     VALUE (*func_si)(rb_transcoding*, const unsigned char*, size_t); /* start -> info   */
 116     int (*func_io)(rb_transcoding*, VALUE, const unsigned char*); /* info  -> output */
 117     int (*func_so)(rb_transcoding*, const unsigned char*, size_t, unsigned char*); /* start -> output */
 118     int (*finish_func)(rb_transcoding*, unsigned char*); /* -> output */
 119     int (*resetsize_func)(rb_transcoding*); /* -> len */
 120     int (*resetstate_func)(rb_transcoding*, unsigned char*); /* -> output */
 121 };
 122
 123 void rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib);
 124 void rb_register_transcoder(const rb_transcoder *);
 125
 126 #endif /* RUBY_TRANSCODE_DATA_H */