Ruby 2.7.0p0(2019-12-25revision647ee6f091eafcce70ffb75ddf7e121e192ab217)

transcode_data.h
Go to the documentation of this file.
1 /**********************************************************************
2
3 transcode_data.h -
4
5 $Author$
6 created at: Mon 10 Dec 2007 14:01:47 JST 2007
7
8 Copyright (C) 2007 Martin Duerst
9
10 **********************************************************************/
11
12 #include "ruby/ruby.h"
13
14 #ifndef RUBY_TRANSCODE_DATA_H
15 #define RUBY_TRANSCODE_DATA_H 1
16
17 RUBY_SYMBOL_EXPORT_BEGIN
18
19 #define WORDINDEX_SHIFT_BITS 2
20 #define WORDINDEX2INFO(widx) ((widx) << WORDINDEX_SHIFT_BITS)
21 #define INFO2WORDINDEX(info) ((info) >> WORDINDEX_SHIFT_BITS)
22 #define BYTE_LOOKUP_BASE(bl) ((bl)[0])
23 #define BYTE_LOOKUP_INFO(bl) ((bl)[1])
24
25 #define PType (unsigned int)
26
27 #define NOMAP (PType 0x01) /* direct map */
28 #define ONEbt (0x02) /* one byte payload */
29 #define TWObt (0x03) /* two bytes payload */
30 #define THREEbt (0x05) /* three bytes payload */
31 #define FOURbt (0x06) /* four bytes payload, UTF-8 only, macros start at getBT0 */
32 #define INVALID (PType 0x07) /* invalid byte sequence */
33 #define UNDEF (PType 0x09) /* legal but undefined */
34 #define ZERObt (PType 0x0A) /* zero bytes of payload, i.e. remove */
35 #define FUNii (PType 0x0B) /* function from info to info */
36 #define FUNsi (PType 0x0D) /* function from start to info */
37 #define FUNio (PType 0x0E) /* function from info to output */
38 #define FUNso (PType 0x0F) /* function from start to output */
39 #define STR1 (PType 0x11) /* string 4 <= len <= 259 bytes: 1byte length + content */
40 #define GB4bt (PType 0x12) /* GB18030 four bytes payload */
41 #define FUNsio (PType 0x13) /* function from start and info to output */
42
43 #define STR1_LENGTH(byte_addr) (unsigned int)(*(byte_addr) + 4)
44 #define STR1_BYTEINDEX(w) ((w) >> 6)
45 #define makeSTR1(bi) (((bi) << 6) | STR1)
46 #define makeSTR1LEN(len) ((len)-4)
47
48 #define o1(b1) (PType((((unsigned char)(b1))<<8)|ONEbt))
49 #define o2(b1,b2) (PType((((unsigned char)(b1))<<8)|\
50 (((unsigned char)(b2))<<16)|\
51 TWObt))
52 #define o3(b1,b2,b3) (PType(((((unsigned char)(b1))<<8)|\
53 (((unsigned char)(b2))<<16)|\
54 (((unsigned int)(unsigned char)(b3))<<24)|\
55 THREEbt)&\
56 0xffffffffU))
57 #define o4(b0,b1,b2,b3) (PType(((((unsigned char)(b1))<<8)|\
58 (((unsigned char)(b2))<<16)|\
59 (((unsigned int)(unsigned char)(b3))<<24)|\
60 ((((unsigned char)(b0))&0x07)<<5)|\
61 FOURbt)&\
62 0xffffffffU))
63 #define g4(b0,b1,b2,b3) (PType(((((unsigned char)(b0))<<8)|\
64 (((unsigned char)(b2))<<16)|\
65 ((((unsigned char)(b1))&0x0f)<<24)|\
66 ((((unsigned int)(unsigned char)(b3))&0x0f)<<28)|\
67 GB4bt)&\
68 0xffffffffU))
69 #define funsio(diff) (PType((((unsigned int)(diff))<<8)|FUNsio))
70
71 #define getBT1(a) ((unsigned char)((a)>> 8))
72 #define getBT2(a) ((unsigned char)((a)>>16))
73 #define getBT3(a) ((unsigned char)((a)>>24))
74 #define getBT0(a) (((unsigned char)((a)>> 5)&0x07)|0xF0) /* for UTF-8 only!!! */
75
76 #define getGB4bt0(a) ((unsigned char)((a)>> 8))
77 #define getGB4bt1(a) (((unsigned char)((a)>>24)&0x0F)|0x30)
78 #define getGB4bt2(a) ((unsigned char)((a)>>16))
79 #define getGB4bt3(a) (((unsigned char)((a)>>28)&0x0F)|0x30)
80
81 #define o2FUNii(b1,b2) (PType((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|FUNii))
82
83 /* do we need these??? maybe not, can be done with simple tables */
84 #define ONETRAIL /* legal but undefined if one more trailing UTF-8 */
85 #define TWOTRAIL /* legal but undefined if two more trailing UTF-8 */
86 #define THREETRAIL /* legal but undefined if three more trailing UTF-8 */
87
88 typedef enum {
89 asciicompat_converter, /* ASCII-compatible -> ASCII-compatible */
90 asciicompat_decoder, /* ASCII-incompatible -> ASCII-compatible */
91 asciicompat_encoder /* ASCII-compatible -> ASCII-incompatible */
92 /* ASCII-incompatible -> ASCII-incompatible is intentionally omitted. */
93 } rb_transcoder_asciicompat_type_t;
94
95 typedef struct rb_transcoder rb_transcoder;
96
97 /* static structure, one per supported encoding pair */
98 struct rb_transcoder {
99 const char *src_encoding;
100 const char *dst_encoding;
101 unsigned int conv_tree_start;
102 const unsigned char *byte_array;
103 unsigned int byte_array_length;
104 const unsigned int *word_array;
105 unsigned int word_array_length;
106 int word_size;
107 int input_unit_length;
108 int max_input;
109 int max_output;
110 rb_transcoder_asciicompat_type_t asciicompat_type;
111 size_t state_size;
112 int (*state_init_func)(void*); /* ret==0:success ret!=0:failure(errno) */
113 int (*state_fini_func)(void*); /* ret==0:success ret!=0:failure(errno) */
114 VALUE (*func_ii)(void*, VALUE); /* info -> info */
115 VALUE (*func_si)(void*, const unsigned char*, size_t); /* start -> info */
116 ssize_t (*func_io)(void*, VALUE, const unsigned char*, size_t); /* info -> output */
117 ssize_t (*func_so)(void*, const unsigned char*, size_t, unsigned char*, size_t); /* start -> output */
118 ssize_t (*finish_func)(void*, unsigned char*, size_t); /* -> output */
119 ssize_t (*resetsize_func)(void*); /* -> len */
120 ssize_t (*resetstate_func)(void*, unsigned char*, size_t); /* -> output */
121 ssize_t (*func_sio)(void*, const unsigned char*, size_t, VALUE, unsigned char*, size_t); /* start -> output */
122 };
123
124 void rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib);
125 void rb_register_transcoder(const rb_transcoder *);
126
127 /*
128 * To get rid of collision of initializer symbols in statically-linked encodings
129 * and transcoders
130 */
131 #if defined(EXTSTATIC) && EXTSTATIC
132 # define TRANS_INIT(name) void Init_trans_ ## name(void)
133 #else
134 # define TRANS_INIT(name) void Init_ ## name(void)
135 #endif
136
137 RUBY_SYMBOL_EXPORT_END
138
139 #endif /* RUBY_TRANSCODE_DATA_H */
RUBY_SYMBOL_EXPORT_END
#define RUBY_SYMBOL_EXPORT_END
Definition: missing.h:49
rb_transcoder::finish_func
ssize_t(* finish_func)(void *, unsigned char *, size_t)
Definition: transcode_data.h:118
rb_transcoder::word_size
int word_size
Definition: transcode_data.h:106
size_t
long unsigned int size_t
Definition: rb_mjit_min_header-2.7.0.h:666
rb_transcoder::func_sio
ssize_t(* func_sio)(void *, const unsigned char *, size_t, VALUE, unsigned char *, size_t)
Definition: transcode_data.h:121
VALUE
unsigned long VALUE
Definition: ruby.h:102
rb_transcoder::func_ii
VALUE(* func_ii)(void *, VALUE)
Definition: transcode_data.h:114
rb_transcoder::word_array_length
unsigned int word_array_length
Definition: transcode_data.h:105
rb_transcoder::src_encoding
const char * src_encoding
Definition: transcode_data.h:99
int
__inline__ int
Definition: rb_mjit_min_header-2.7.0.h:2839
rb_declare_transcoder
void rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib)
Definition: transcode.c:232
asciicompat_decoder
@ asciicompat_decoder
Definition: transcode_data.h:90
ssize_t
_ssize_t ssize_t
Definition: rb_mjit_min_header-2.7.0.h:1329
ruby.h
rb_register_transcoder
void rb_register_transcoder(const rb_transcoder *)
Definition: transcode.c:204
rb_transcoder::func_si
VALUE(* func_si)(void *, const unsigned char *, size_t)
Definition: transcode_data.h:115
rb_transcoder::func_io
ssize_t(* func_io)(void *, VALUE, const unsigned char *, size_t)
Definition: transcode_data.h:116
rb_transcoder::max_output
int max_output
Definition: transcode_data.h:109
rb_transcoder
Definition: transcode_data.h:98
rb_transcoder::resetstate_func
ssize_t(* resetstate_func)(void *, unsigned char *, size_t)
Definition: transcode_data.h:120
rb_transcoder::conv_tree_start
unsigned int conv_tree_start
Definition: transcode_data.h:101
rb_transcoder::state_size
size_t state_size
Definition: transcode_data.h:111
asciicompat_converter
@ asciicompat_converter
Definition: transcode_data.h:89
rb_transcoder_asciicompat_type_t
rb_transcoder_asciicompat_type_t
Definition: transcode_data.h:88
rb_transcoder::word_array
const unsigned int * word_array
Definition: transcode_data.h:104
rb_transcoder::func_so
ssize_t(* func_so)(void *, const unsigned char *, size_t, unsigned char *, size_t)
Definition: transcode_data.h:117
rb_transcoder::max_input
int max_input
Definition: transcode_data.h:108
rb_transcoder::byte_array_length
unsigned int byte_array_length
Definition: transcode_data.h:103
RUBY_SYMBOL_EXPORT_BEGIN
#define RUBY_SYMBOL_EXPORT_BEGIN
Definition: missing.h:48
rb_transcoder::state_init_func
int(* state_init_func)(void *)
Definition: transcode_data.h:112
rb_transcoder::byte_array
const unsigned char * byte_array
Definition: transcode_data.h:102
rb_transcoder::asciicompat_type
rb_transcoder_asciicompat_type_t asciicompat_type
Definition: transcode_data.h:110
rb_transcoder::state_fini_func
int(* state_fini_func)(void *)
Definition: transcode_data.h:113
rb_transcoder::resetsize_func
ssize_t(* resetsize_func)(void *)
Definition: transcode_data.h:119
asciicompat_encoder
@ asciicompat_encoder
Definition: transcode_data.h:91
rb_transcoder::input_unit_length
int input_unit_length
Definition: transcode_data.h:107
rb_transcoder::dst_encoding
const char * dst_encoding
Definition: transcode_data.h:100