* transcode_data.h, transcode.c, tool/transcode-tblgen.rb: Added

  support for new transcoding instruction FUNsio (with Tatsuya Mizuno)

* enc/trans/gb18030.trans: Significantly reduced GB18030 conversion
  table footprint using FUNsio and differences (with Tatsuya Mizuno)

* test/ruby/test_transcode.rb: Minor name fix (from Tatsuya Mizuno)


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@26065 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 105 insertions, 5 deletions

diff --git a/enc/trans/gb18030.trans b/enc/trans/gb18030.trans
index 700522c9f0..c7da4c64b4 100644
--- a/enc/trans/gb18030.trans
+++ b/enc/trans/gb18030.trans
@@ -2,16 +2,64 @@
 
 <%
   require "gb18030-tbl"
+  
+  def linear(code)
+    bytes = [code].pack('H8').unpack 'C4' 
+    ((bytes[0]*10+bytes[1])*126+bytes[2])*10+bytes[3]
+  end
+
+  def calculate_differences_gb_utf(table)
+    table.collect do |code|
+      code = code.dup
+      if code[0].length == 4
+        if code[1] < 0x800 # GB-18030: 2 bytes, UTF-8: 2 bytes
+          # do nothing
+        else # GB-18030: 2 bytes, UTF-8: 3 bytes
+          gb_linear2b = code[0].to_i(16)
+          diff2b = gb_linear2b - code[1] + 24055
+          code[1] = "funsio(#{diff2b})"
+        end
+      else
+        if code[1] < 0x800 # GB-18030: 4 bytes, UTF-8: 2 bytes
+          # do nothing
+        else # GB-18030: 4 bytes, UTF-8: 3 bytes
+          gb_linear4b = linear(code[0])
+          diff4b = gb_linear4b - code[1] - 0x170000
+          code[1] = "funsio(#{diff4b})"
+        end
+      end
+      code
+    end
+  end
+  
+  def calculate_differences_utf_gb(table)
+    table_rev = table.map{|a,b| [b,a]}
+    table_rev.collect do |code|
+      code = code.dup
+      if code[0] >= 0x800
+        if code[1].length == 4 #GB18030 2byte UTF-8 3byte
+          gb_linear2b = code[1].to_i(16)
+          diff2b = gb_linear2b - code[0] + 24055
+          code[1] = "funsio(#{diff2b})"
+        else # GB-18030: 4 bytes, UTF-8: 3 bytes
+          gb_linear4b = linear(code[1])
+          diff4b = gb_linear4b - code[0] - 0x170000
+          code[1] = "funsio(#{diff4b})"
+        end
+      end
+      code
+    end
+  end
 
   transcode_tbl_only "GB18030", "UTF-8", [["{00-7f}", :nomap]] +
-                                         GB18030_TO_UCS_TBL + [
+                                         calculate_differences_gb_utf(GB18030_TO_UCS_TBL) + [
                                         ["{90-e2}{30-39}{81-fe}{30-39}", :func_so],
                                         ["e3{30-31}{81-fe}{30-39}", :func_so],
                                         ["e332{81-99}{30-39}", :func_so],
                                         ["e3329a{30-35}", :func_so],   #  "E3329A35" is U+10FFFF
                                        ]
   transcode_tbl_only "UTF-8", "GB18030", [["{00-7f}", :nomap]] +
-                                         GB18030_TO_UCS_TBL.map {|a,b| [b,a] } + [
+                                        calculate_differences_utf_gb(GB18030_TO_UCS_TBL) + [
                                         ["f0{90-bf}{80-bf}{80-bf}", :func_so],
                                         ["{f1-f3}{80-bf}{80-bf}{80-bf}", :func_so],
                                         ["f4{80-8f}{80-bf}{80-bf}", :func_so]
@@ -20,6 +68,7 @@
 
 <%= transcode_generated_code %>
 
+/* GB18030 4byte, UTF-8 4byte*/
 static ssize_t
 fun_so_from_gb18030(void *statep, const unsigned char *s, size_t l, unsigned char *o, size_t osize)
 {
@@ -33,6 +82,7 @@ fun_so_from_gb18030(void *statep, const unsigned char *s, size_t l, unsigned cha
     return 4;
 }
 
+/* GB18030 4byte, UTF-8 4byte*/
 static ssize_t
 fun_so_to_gb18030(void *statep, const unsigned char *s, size_t l, unsigned char *o, size_t osize)
 {
@@ -49,6 +99,56 @@ fun_so_to_gb18030(void *statep, const unsigned char *s, size_t l, unsigned char
     return 4;
 }
 
+/* GB18030 2byte, UTF-8 3byte and GB18030 4byte, UTF-8 3byte*/
+static ssize_t
+fun_sio_from_gb18030(void *statep, const unsigned char *s, size_t l, VALUE info, unsigned char *o, size_t osize)
+{
+    unsigned int diff = (unsigned int)(info >> 8);
+    unsigned int u;    /* Unicode Scalar Value */
+    if (diff & 0x20000) { /* GB18030 4 bytes */
+        u = ((s[0]*10+s[1])*126+s[2])*10+s[3] - diff - 0x170000;
+    }
+    else { /* GB18030 2 bytes */
+        u = s[0]*256 + s[1] + 24055 - diff;
+    }
+    o[0] = 0xE0 | (u>>12);
+    o[1] = 0x80 | ((u>>6)&0x3F);
+    o[2] = 0x80 | (u&0x3F);
+    return 3;
+}
+
+/* GB18030 2byte, UTF-8 3byte and GB18030 4byte, UTF-8 3byte*/
+static ssize_t
+fun_sio_to_gb18030(void *statep, const unsigned char *s, size_t l, VALUE info, unsigned char *o, size_t osize)
+{
+    unsigned int diff = (unsigned int)(info >> 8);
+    unsigned int u;    /* Unicode Scalar Value */
+    
+    u = ((s[0]&0x0F)<<12) | ((s[1]&0x3F)<<6) | (s[2]&0x3F);
+
+    if (diff & 0x20000) { /* GB18030 4 bytes */
+        u += (diff + 0x170000);
+	u -= 1688980;
+	u += 0x2;
+        o[3] = 0x30 + u%10;
+        u /= 10;
+        u += 0x32;
+        o[2] = 0x81 + u%126;
+        u /= 126;
+        u += 0x1;
+        o[1] = 0x30 + u%10;
+        u /= 10;
+        o[0] = 0x81 + u;
+	return 4;
+    }
+    else { /* GB18030 2 bytes */
+        u += (diff - 24055);
+	o[1] = u%256;
+	o[0] = u/256;
+	return 2;
+    }
+}
+
 
 static const rb_transcoder
 rb_from_GB18030 = {
@@ -56,11 +156,11 @@ rb_from_GB18030 = {
     TRANSCODE_TABLE_INFO,
     1, /* input_unit_length */
     4, /* max_input */
-    3, /* max_output */
+    4, /* max_output */
     asciicompat_converter, /* asciicompat_type */
     0, NULL, NULL, /* state_size, state_init, state_fini */
     NULL, NULL, NULL, fun_so_from_gb18030,
-    NULL, NULL, NULL
+    NULL, NULL, NULL, fun_sio_from_gb18030
 };
 static const rb_transcoder
 rb_to_GB18030 = {
@@ -72,7 +172,7 @@ rb_to_GB18030 = {
     asciicompat_converter, /* asciicompat_type */
     0, NULL, NULL, /* state_size, state_init, state_fini */
     NULL, NULL, NULL, fun_so_to_gb18030,
-    NULL, NULL, NULL
+    NULL, NULL, NULL, fun_sio_to_gb18030
 };