diff options
| -rw-r--r-- | ChangeLog | 10 | ||||
| -rw-r--r-- | enc/trans/gb18030.trans | 110 | ||||
| -rw-r--r-- | test/ruby/test_transcode.rb | 6 | ||||
| -rwxr-xr-x | tool/transcode-tblgen.rb | 2 | ||||
| -rw-r--r-- | transcode.c | 26 | ||||
| -rw-r--r-- | transcode_data.h | 3 |
6 files changed, 149 insertions, 8 deletions
@@ -1,3 +1,13 @@ +Thu Dec 10 20:55:55 2009 Martin Duerst <duerst@it.aoyama.ac.jp> + + * transcode_data.h, transcode.c, tool/transcode-tblgen.rb: Added + support for new transcoding instruction FUNsio (with Tatsuya Mizuno) + + * enc/trans/gb18030.trans: Significantly reduced GB18030 conversion + table footprint using FUNsio and differences (with Tatsuya Mizuno) + + * test/ruby/test_transcode.rb: Minor name fix (from Tatsuya Mizuno) + Thu Dec 10 17:22:36 2009 Shugo Maeda <shugo@ruby-lang.org> * vm_eval.c (yield_under): yields self the same as 1.8. diff --git a/enc/trans/gb18030.trans b/enc/trans/gb18030.trans index 700522c9f0..c7da4c64b4 100644 --- a/enc/trans/gb18030.trans +++ b/enc/trans/gb18030.trans @@ -2,16 +2,64 @@ <% require "gb18030-tbl" + + def linear(code) + bytes = [code].pack('H8').unpack 'C4' + ((bytes[0]*10+bytes[1])*126+bytes[2])*10+bytes[3] + end + + def calculate_differences_gb_utf(table) + table.collect do |code| + code = code.dup + if code[0].length == 4 + if code[1] < 0x800 # GB-18030: 2 bytes, UTF-8: 2 bytes + # do nothing + else # GB-18030: 2 bytes, UTF-8: 3 bytes + gb_linear2b = code[0].to_i(16) + diff2b = gb_linear2b - code[1] + 24055 + code[1] = "funsio(#{diff2b})" + end + else + if code[1] < 0x800 # GB-18030: 4 bytes, UTF-8: 2 bytes + # do nothing + else # GB-18030: 4 bytes, UTF-8: 3 bytes + gb_linear4b = linear(code[0]) + diff4b = gb_linear4b - code[1] - 0x170000 + code[1] = "funsio(#{diff4b})" + end + end + code + end + end + + def calculate_differences_utf_gb(table) + table_rev = table.map{|a,b| [b,a]} + table_rev.collect do |code| + code = code.dup + if code[0] >= 0x800 + if code[1].length == 4 #GB18030 2byte UTF-8 3byte + gb_linear2b = code[1].to_i(16) + diff2b = gb_linear2b - code[0] + 24055 + code[1] = "funsio(#{diff2b})" + else # GB-18030: 4 bytes, UTF-8: 3 bytes + gb_linear4b = linear(code[1]) + diff4b = gb_linear4b - code[0] - 0x170000 + code[1] = "funsio(#{diff4b})" + end + end + code + end + end transcode_tbl_only "GB18030", "UTF-8", [["{00-7f}", :nomap]] + - GB18030_TO_UCS_TBL + [ + calculate_differences_gb_utf(GB18030_TO_UCS_TBL) + [ ["{90-e2}{30-39}{81-fe}{30-39}", :func_so], ["e3{30-31}{81-fe}{30-39}", :func_so], ["e332{81-99}{30-39}", :func_so], ["e3329a{30-35}", :func_so], # "E3329A35" is U+10FFFF ] transcode_tbl_only "UTF-8", "GB18030", [["{00-7f}", :nomap]] + - GB18030_TO_UCS_TBL.map {|a,b| [b,a] } + [ + calculate_differences_utf_gb(GB18030_TO_UCS_TBL) + [ ["f0{90-bf}{80-bf}{80-bf}", :func_so], ["{f1-f3}{80-bf}{80-bf}{80-bf}", :func_so], ["f4{80-8f}{80-bf}{80-bf}", :func_so] @@ -20,6 +68,7 @@ <%= transcode_generated_code %> +/* GB18030 4byte, UTF-8 4byte*/ static ssize_t fun_so_from_gb18030(void *statep, const unsigned char *s, size_t l, unsigned char *o, size_t osize) { @@ -33,6 +82,7 @@ fun_so_from_gb18030(void *statep, const unsigned char *s, size_t l, unsigned cha return 4; } +/* GB18030 4byte, UTF-8 4byte*/ static ssize_t fun_so_to_gb18030(void *statep, const unsigned char *s, size_t l, unsigned char *o, size_t osize) { @@ -49,6 +99,56 @@ fun_so_to_gb18030(void *statep, const unsigned char *s, size_t l, unsigned char return 4; } +/* GB18030 2byte, UTF-8 3byte and GB18030 4byte, UTF-8 3byte*/ +static ssize_t +fun_sio_from_gb18030(void *statep, const unsigned char *s, size_t l, VALUE info, unsigned char *o, size_t osize) +{ + unsigned int diff = (unsigned int)(info >> 8); + unsigned int u; /* Unicode Scalar Value */ + if (diff & 0x20000) { /* GB18030 4 bytes */ + u = ((s[0]*10+s[1])*126+s[2])*10+s[3] - diff - 0x170000; + } + else { /* GB18030 2 bytes */ + u = s[0]*256 + s[1] + 24055 - diff; + } + o[0] = 0xE0 | (u>>12); + o[1] = 0x80 | ((u>>6)&0x3F); + o[2] = 0x80 | (u&0x3F); + return 3; +} + +/* GB18030 2byte, UTF-8 3byte and GB18030 4byte, UTF-8 3byte*/ +static ssize_t +fun_sio_to_gb18030(void *statep, const unsigned char *s, size_t l, VALUE info, unsigned char *o, size_t osize) +{ + unsigned int diff = (unsigned int)(info >> 8); + unsigned int u; /* Unicode Scalar Value */ + + u = ((s[0]&0x0F)<<12) | ((s[1]&0x3F)<<6) | (s[2]&0x3F); + + if (diff & 0x20000) { /* GB18030 4 bytes */ + u += (diff + 0x170000); + u -= 1688980; + u += 0x2; + o[3] = 0x30 + u%10; + u /= 10; + u += 0x32; + o[2] = 0x81 + u%126; + u /= 126; + u += 0x1; + o[1] = 0x30 + u%10; + u /= 10; + o[0] = 0x81 + u; + return 4; + } + else { /* GB18030 2 bytes */ + u += (diff - 24055); + o[1] = u%256; + o[0] = u/256; + return 2; + } +} + static const rb_transcoder rb_from_GB18030 = { @@ -56,11 +156,11 @@ rb_from_GB18030 = { TRANSCODE_TABLE_INFO, 1, /* input_unit_length */ 4, /* max_input */ - 3, /* max_output */ + 4, /* max_output */ asciicompat_converter, /* asciicompat_type */ 0, NULL, NULL, /* state_size, state_init, state_fini */ NULL, NULL, NULL, fun_so_from_gb18030, - NULL, NULL, NULL + NULL, NULL, NULL, fun_sio_from_gb18030 }; static const rb_transcoder rb_to_GB18030 = { @@ -72,7 +172,7 @@ rb_to_GB18030 = { asciicompat_converter, /* asciicompat_type */ 0, NULL, NULL, /* state_size, state_init, state_fini */ NULL, NULL, NULL, fun_so_to_gb18030, - NULL, NULL, NULL + NULL, NULL, NULL, fun_sio_to_gb18030 }; diff --git a/test/ruby/test_transcode.rb b/test/ruby/test_transcode.rb index bec153b491..9d4305876f 100644 --- a/test/ruby/test_transcode.rb +++ b/test/ruby/test_transcode.rb @@ -1579,10 +1579,10 @@ class TestTranscode < Test::Unit::TestCase def test_gb18030 # overall roundtrip test - all_gb18030 = (0x0..0xD7FF).to_a.pack 'U*' #追加 - all_gb18030 << (0xE000..0xFFFF).to_a.pack("U*") #追加 + all_unicode = (0x0..0xD7FF).to_a.pack 'U*' #追加 + all_unicode << (0xE000..0xFFFF).to_a.pack("U*") #追加 - assert_equal(all_gb18030, all_gb18030.encode("gb18030").encode("UTF-8")) #追加 + assert_equal(all_unicode, all_unicode.encode("gb18030").encode("UTF-8")) #追加 # tests from GBK check_both_ways("\u4E02", "\x81\x40", 'GB18030') # diff --git a/tool/transcode-tblgen.rb b/tool/transcode-tblgen.rb index 19d4fe0927..4f505379b7 100755 --- a/tool/transcode-tblgen.rb +++ b/tool/transcode-tblgen.rb @@ -339,6 +339,8 @@ class ActionMap "o2(0x#$1,0x#$2)" when /\A([0-9a-f][0-9a-f])([0-9a-f][0-9a-f])([0-9a-f][0-9a-f])\z/i "o3(0x#$1,0x#$2,0x#$3)" + when /funsio\((\d+)\)/ + "funsio(#{$1})" when /\A([0-9a-f][0-9a-f])(3[0-9])([0-9a-f][0-9a-f])(3[0-9])\z/i "g4(0x#$1,0x#$2,0x#$3,0x#$4)" when /\A(f[0-7])([0-9a-f][0-9a-f])([0-9a-f][0-9a-f])([0-9a-f][0-9a-f])\z/i diff --git a/transcode.c b/transcode.c index b45d2397ab..d511547916 100644 --- a/transcode.c +++ b/transcode.c @@ -505,6 +505,8 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, case 30: goto resume_label30; case 31: goto resume_label31; case 32: goto resume_label32; + case 33: goto resume_label33; + case 34: goto resume_label34; } while (1) { @@ -649,6 +651,30 @@ transcode_restartable0(const unsigned char **in_pos, unsigned char **out_pos, } break; } + case FUNsio: + { + const unsigned char *char_start; + size_t char_len; + SUSPEND_OBUF(33); + if (tr->max_output <= out_stop - out_p) { + char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len); + out_p += tr->func_sio(TRANSCODING_STATE(tc), + char_start, (size_t)char_len, next_info, + out_p, out_stop - out_p); + } + else { + char_start = transcode_char_start(tc, *in_pos, inchar_start, in_p, &char_len); + writebuf_len = tr->func_sio(TRANSCODING_STATE(tc), + char_start, (size_t)char_len, next_info, + TRANSCODING_WRITEBUF(tc), TRANSCODING_WRITEBUF_SIZE(tc)); + writebuf_off = 0; + while (writebuf_off < writebuf_len) { + SUSPEND_OBUF(34); + *out_p++ = TRANSCODING_WRITEBUF(tc)[writebuf_off++]; + } + } + break; + } case INVALID: if (tc->recognized_len + (in_p - inchar_start) <= unitlen) { if (tc->recognized_len + (in_p - inchar_start) < unitlen) diff --git a/transcode_data.h b/transcode_data.h index 7362275cbe..61146d2c83 100644 --- a/transcode_data.h +++ b/transcode_data.h @@ -36,6 +36,7 @@ #define FUNso (PType 0x0F) /* function from start to output */ #define STR1 (PType 0x11) /* string 4 <= len <= 259 bytes: 1byte length + content */ #define GB4bt (PType 0x12) /* GB18030 four bytes payload */ +#define FUNsio (PType 0x13) /* function from start and info to output */ #define STR1_LENGTH(byte_addr) (unsigned int)(*(byte_addr) + 4) #define STR1_BYTEINDEX(w) ((w) >> 6) @@ -47,6 +48,7 @@ #define o3(b1,b2,b3) (PType(((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|(((unsigned int)(unsigned char)(b3))<<24)|THREEbt)&0xffffffffU)) #define o4(b0,b1,b2,b3) (PType(((((unsigned char)(b1))<<8)|(((unsigned char)(b2))<<16)|(((unsigned char)(b3))<<24)|((((unsigned char)(b0))&0x07)<<5)|FOURbt)&0xffffffffU)) #define g4(b0,b1,b2,b3) (PType(((((unsigned char)(b0))<<8)|(((unsigned char)(b2))<<16)|((((unsigned char)(b1))&0x0f)<<24)|((((unsigned int)(unsigned char)(b3))&0x0f)<<28)|GB4bt)&0xffffffffU)) +#define funsio(diff) (PType((((unsigned int)(diff))<<8)|FUNsio)) #define getBT1(a) ((unsigned char)((a)>> 8)) #define getBT2(a) ((unsigned char)((a)>>16)) @@ -98,6 +100,7 @@ struct rb_transcoder { ssize_t (*finish_func)(void*, unsigned char*, size_t); /* -> output */ ssize_t (*resetsize_func)(void*); /* -> len */ ssize_t (*resetstate_func)(void*, unsigned char*, size_t); /* -> output */ + ssize_t (*func_sio)(void*, const unsigned char*, size_t, VALUE, unsigned char*, size_t); /* start -> output */ }; void rb_declare_transcoder(const char *enc1, const char *enc2, const char *lib); |