Ruby  2.7.1p83(2020-03-31revisiona0c7c23c9cec0d0ffcba012279cd652d28ad5bf3)
transcode_data.h
Go to the documentation of this file.
1 /**********************************************************************
2 
3  transcode_data.h -
4 
5  $Author$
6  created at: Mon 10 Dec 2007 14:01:47 JST 2007
7 
8  Copyright (C) 2007 Martin Duerst
9 
10 **********************************************************************/
11 
12 #include "ruby/ruby.h"
13 
14 #ifndef RUBY_TRANSCODE_DATA_H
15 #define RUBY_TRANSCODE_DATA_H 1
16 
18 
19 #define WORDINDEX_SHIFT_BITS 2
20 #define WORDINDEX2INFO(widx) ((widx) << WORDINDEX_SHIFT_BITS)
21 #define INFO2WORDINDEX(info) ((info) >> WORDINDEX_SHIFT_BITS)
22 #define BYTE_LOOKUP_BASE(bl) ((bl)[0])
23 #define BYTE_LOOKUP_INFO(bl) ((bl)[1])
24 
25 #define PType (unsigned int)
26 
27 #define NOMAP (PType 0x01) /* direct map */
28 #define ONEbt (0x02) /* one byte payload */
29 #define TWObt (0x03) /* two bytes payload */
30 #define THREEbt (0x05) /* three bytes payload */
31 #define FOURbt (0x06) /* four bytes payload, UTF-8 only, macros start at getBT0 */
32 #define INVALID (PType 0x07) /* invalid byte sequence */
33 #define UNDEF (PType 0x09) /* legal but undefined */
34 #define ZERObt (PType 0x0A) /* zero bytes of payload, i.e. remove */
35 #define FUNii (PType 0x0B) /* function from info to info */
36 #define FUNsi (PType 0x0D) /* function from start to info */
37 #define FUNio (PType 0x0E) /* function from info to output */
38 #define FUNso (PType 0x0F) /* function from start to output */
39 #define STR1 (PType 0x11) /* string 4 <= len <= 259 bytes: 1byte length + content */
40 #define GB4bt (PType 0x12) /* GB18030 four bytes payload */
41 #define FUNsio (PType 0x13) /* function from start and info to output */
42 
43 #define STR1_LENGTH(byte_addr) (unsigned int)(*(byte_addr) + 4)
44 #define STR1_BYTEINDEX(w) ((w) >> 6)
45 #define makeSTR1(bi) (((bi) << 6) | STR1)
46 #define makeSTR1LEN(len) ((len)-4)
47 
48 #define o1(b1) (PType((((unsigned char)(b1))<<8)|ONEbt))
49 #define o2(b1,b2) (PType((((unsigned char)(b1))<<8)|\
50  (((unsigned char)(b2))<<16)|\
51  TWObt))
52 #define o3(b1,b2,b3) (PType(((((unsigned char)(b1))<<8)|\
53  (((unsigned char)(b2))<<16)|\
54  (((unsigned int)(unsigned char)(b3))<<24)|\
55  THREEbt)&\
56  0xffffffffU))
57 #define o4(b0,b1,b2,b3) (PType(((((unsigned char)(b1))<<8)|\
58  (((unsigned char)(b2))<<16)|\
59  (((unsigned int)(unsigned char)(b3))<<24)|\
60  ((((unsigned char)(b0))&0x07)<<5)|\
61  FOURbt)&\
62  0xffffffffU))
63 #define g4(b0,b1,b2,b3) (PType(((((unsigned char)(b0))<<8)|\
64  (((unsigned char)(b2))<<16)|\
65  ((((unsigned char)(b1))&0x0f)<<24)|\
66  ((((unsigned int)(unsigned char)(b3))&0x0f)<<28)|\
67  GB4bt)&\
68  0xffffffffU))
69 #define funsio(diff) (PType((((unsigned int)(diff))<<8)|FUNsio))
70 
71 #define getBT1(a) ((unsigned char)((a)>> 8))
72 #define getBT2(a) ((unsigned char)((a)>>16))
73 #define getBT3(a) ((unsigned char)((a)>>24))
74 #define getBT0(a) (((unsigned char)((a)>> 5)&0x07)|0xF0) /* for UTF-8 only!!! */
75 
76 #define getGB4bt0(a) ((unsigned char)((a)>> 8))
77 #define getGB4bt1(a) (((unsigned char)((a)>>24)&0x0F)|0x30)
78 #define getGB4bt2(a) ((unsigned char)((a)>>16))
79 #define getGB4bt3(a) (((unsigned char)((a)>>28)&0x0F)|0x30)
80 
81 #define o2FUNii(b1,b2) (PType((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|FUNii))
82 
83 /* do we need these??? maybe not, can be done with simple tables */
84 #define ONETRAIL /* legal but undefined if one more trailing UTF-8 */
85 #define TWOTRAIL /* legal but undefined if two more trailing UTF-8 */
86 #define THREETRAIL /* legal but undefined if three more trailing UTF-8 */
87 
88 typedef enum {
89  asciicompat_converter, /* ASCII-compatible -> ASCII-compatible */
90  asciicompat_decoder, /* ASCII-incompatible -> ASCII-compatible */
91  asciicompat_encoder /* ASCII-compatible -> ASCII-incompatible */
92  /* ASCII-incompatible -> ASCII-incompatible is intentionally omitted. */
94 
96 
97 /* static structure, one per supported encoding pair */
98 struct rb_transcoder {
99  const char *src_encoding;
100  const char *dst_encoding;
101  unsigned int conv_tree_start;
102  const unsigned char *byte_array;
103  unsigned int byte_array_length;
104  const unsigned int *word_array;
105  unsigned int word_array_length;
111  size_t state_size;
112  int (*state_init_func)(void*); /* ret==0:success ret!=0:failure(errno) */
113  int (*state_fini_func)(void*); /* ret==0:success ret!=0:failure(errno) */
114  VALUE (*func_ii)(void*, VALUE); /* info -> info */
115  VALUE (*func_si)(void*, const unsigned char*, size_t); /* start -> info */
116  ssize_t (*func_io)(void*, VALUE, const unsigned char*, size_t); /* info -> output */
117  ssize_t (*func_so)(void*, const unsigned char*, size_t, unsigned char*, size_t); /* start -> output */
118  ssize_t (*finish_func)(void*, unsigned char*, size_t); /* -> output */
119  ssize_t (*resetsize_func)(void*); /* -> len */
120  ssize_t (*resetstate_func)(void*, unsigned char*, size_t); /* -> output */
121  ssize_t (*func_sio)(void*, const unsigned char*, size_t, VALUE, unsigned char*, size_t); /* start -> output */
122 };
123 
124 void rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib);
126 
127 /*
128  * To get rid of collision of initializer symbols in statically-linked encodings
129  * and transcoders
130  */
131 #if defined(EXTSTATIC) && EXTSTATIC
132 # define TRANS_INIT(name) void Init_trans_ ## name(void)
133 #else
134 # define TRANS_INIT(name) void Init_ ## name(void)
135 #endif
136 
138 
139 #endif /* RUBY_TRANSCODE_DATA_H */
RUBY_SYMBOL_EXPORT_END
#define RUBY_SYMBOL_EXPORT_END
Definition: missing.h:49
rb_transcoder::finish_func
ssize_t(* finish_func)(void *, unsigned char *, size_t)
Definition: transcode_data.h:118
rb_transcoder::word_size
int word_size
Definition: transcode_data.h:106
rb_transcoder::func_sio
ssize_t(* func_sio)(void *, const unsigned char *, size_t, VALUE, unsigned char *, size_t)
Definition: transcode_data.h:121
VALUE
unsigned long VALUE
Definition: ruby.h:102
rb_transcoder::func_ii
VALUE(* func_ii)(void *, VALUE)
Definition: transcode_data.h:114
rb_transcoder::word_array_length
unsigned int word_array_length
Definition: transcode_data.h:105
rb_transcoder::src_encoding
const char * src_encoding
Definition: transcode_data.h:99
rb_declare_transcoder
void rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib)
Definition: transcode.c:233
asciicompat_decoder
@ asciicompat_decoder
Definition: transcode_data.h:90
ruby.h
rb_register_transcoder
void rb_register_transcoder(const rb_transcoder *)
Definition: transcode.c:205
rb_transcoder::func_si
VALUE(* func_si)(void *, const unsigned char *, size_t)
Definition: transcode_data.h:115
rb_transcoder::func_io
ssize_t(* func_io)(void *, VALUE, const unsigned char *, size_t)
Definition: transcode_data.h:116
rb_transcoder::max_output
int max_output
Definition: transcode_data.h:109
rb_transcoder
Definition: transcode_data.h:98
rb_transcoder::resetstate_func
ssize_t(* resetstate_func)(void *, unsigned char *, size_t)
Definition: transcode_data.h:120
rb_transcoder::conv_tree_start
unsigned int conv_tree_start
Definition: transcode_data.h:101
rb_transcoder::state_size
size_t state_size
Definition: transcode_data.h:111
size_t
unsigned int size_t
Definition: rb_mjit_min_header-2.7.1.h:660
asciicompat_converter
@ asciicompat_converter
Definition: transcode_data.h:89
rb_transcoder_asciicompat_type_t
rb_transcoder_asciicompat_type_t
Definition: transcode_data.h:88
rb_transcoder::word_array
const unsigned int * word_array
Definition: transcode_data.h:104
rb_transcoder::func_so
ssize_t(* func_so)(void *, const unsigned char *, size_t, unsigned char *, size_t)
Definition: transcode_data.h:117
rb_transcoder::max_input
int max_input
Definition: transcode_data.h:108
rb_transcoder::byte_array_length
unsigned int byte_array_length
Definition: transcode_data.h:103
RUBY_SYMBOL_EXPORT_BEGIN
#define RUBY_SYMBOL_EXPORT_BEGIN
Definition: missing.h:48
rb_transcoder::state_init_func
int(* state_init_func)(void *)
Definition: transcode_data.h:112
ssize_t
_ssize_t ssize_t
Definition: rb_mjit_min_header-2.7.1.h:1324
int
__inline__ int
Definition: rb_mjit_min_header-2.7.1.h:2807
rb_transcoder::byte_array
const unsigned char * byte_array
Definition: transcode_data.h:102
rb_transcoder::asciicompat_type
rb_transcoder_asciicompat_type_t asciicompat_type
Definition: transcode_data.h:110
rb_transcoder::state_fini_func
int(* state_fini_func)(void *)
Definition: transcode_data.h:113
rb_transcoder::resetsize_func
ssize_t(* resetsize_func)(void *)
Definition: transcode_data.h:119
asciicompat_encoder
@ asciicompat_encoder
Definition: transcode_data.h:91
rb_transcoder::input_unit_length
int input_unit_length
Definition: transcode_data.h:107
rb_transcoder::dst_encoding
const char * dst_encoding
Definition: transcode_data.h:100