* original nkf.c rev:1.40

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/branches/ruby_1_8@7329 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 515 insertions, 168 deletions

diff --git a/ext/nkf/nkf-utf8/nkf.c b/ext/nkf/nkf-utf8/nkf.c
index aa7c459b83..d30008d6c0 100644
--- a/ext/nkf/nkf-utf8/nkf.c
+++ b/ext/nkf/nkf-utf8/nkf.c
@@ -39,14 +39,14 @@
 **        E-Mail: furukawa@tcp-ip.or.jp
 **    まで御連絡をお願いします。
 ***********************************************************************/
+/* $Id$ */
+#define NKF_VERSION "2.0.4"
+#define NKF_RELEASE_DATE "2004-11-15"
 #include "config.h"
 
 static char *CopyRight =
       "Copyright (C) 1987, FUJITSU LTD. (I.Ichikawa),2000 S. Kono, COW, 2002-2004 Kono, Furukawa";
-static char *Version =
-      "2.0";
-static char *Patchlevel =
-      "4/0401/Shinji Kono";
+
 
 /*
 **
@@ -110,6 +110,8 @@ static char *Patchlevel =
 #include <stdio.h>
 #endif
 
+#include <stdlib.h>
+
 #if defined(MSDOS) || defined(__OS2__) 
 #include <fcntl.h>
 #include <io.h>
@@ -142,7 +144,6 @@ static char *Patchlevel =
 
 #ifdef OVERWRITE
 /* added by satoru@isoternet.org */
-#include <stdlib.h>
 #include <string.h>
 #include <sys/stat.h>
 #ifndef MSDOS /* UNIX, OS/2 */
@@ -198,7 +199,7 @@ static char *Patchlevel =
 
 #define		UTF8           12
 #define		UTF8_INPUT     13
-#define		UTF16_INPUT    14
+#define		UTF16LE_INPUT  14
 #define		UTF16BE_INPUT  15
 
 #define         WISH_TRUE      15
@@ -206,6 +207,7 @@ static char *Patchlevel =
 /* ASCII CODE */
 
 #define         BS      0x08
+#define         TAB     0x09
 #define         NL      0x0a
 #define         CR      0x0d
 #define         ESC     0x1b
@@ -216,6 +218,7 @@ static char *Patchlevel =
 #define         SI      0x0f
 #define         SO      0x0e
 #define         SSO     0x8e
+#define         SS3     0x8f
 
 #define		is_alnum(c)  \
             (('a'<=c && c<='z')||('A'<= c && c<='Z')||('0'<=c && c<='9'))
@@ -376,8 +379,9 @@ static int             x0201_f = NO_X0201;     /* Assume NO JISX0201 */
 #endif
 static int             iso2022jp_f = FALSE;    /* convert ISO-2022-JP */
 #ifdef UTF8_OUTPUT_ENABLE
-static int             w_oconv16_begin_f= 0;   /* utf-16 header */
+static int             unicode_bom_f= 0;   /* Output Unicode BOM */
 static int             w_oconv16_LE = 0;   /* utf-16 little endian */
+static int             ms_ucs_map_f = FALSE;   /* Microsoft UCS Mapping Compatible */
 #endif
 
 
@@ -443,7 +447,7 @@ STATIC void s_status PROTO((struct input_code *, int));
 #ifdef UTF8_INPUT_ENABLE
 STATIC void w_status PROTO((struct input_code *, int));
 STATIC void w16_status PROTO((struct input_code *, int));
-static int             utf16_mode = UTF16_INPUT;
+static int             utf16_mode = UTF16LE_INPUT;
 #endif
 
 struct input_code input_code_list[] = {
@@ -467,8 +471,8 @@ static int             fold_f  = FALSE;
 static int             fold_len  = 0;
 
 /* options */
-static unsigned char   kanji_intro = DEFAULT_J,
-                       ascii_intro = DEFAULT_R;
+static unsigned char   kanji_intro = DEFAULT_J;
+static unsigned char   ascii_intro = DEFAULT_R;
 
 /* Folding */
 
@@ -892,6 +896,7 @@ struct {
 #ifdef UTF8_OUTPUT_ENABLE
     {"utf8", "w"},
     {"utf16", "w16"},
+    {"ms-ucs-map", ""},
 #endif
 #ifdef UTF8_INPUT_ENABLE
     {"utf8-input", "W"},
@@ -922,7 +927,7 @@ struct {
     {"prefix=", ""},
 };
 
-static int option_mode;
+static int option_mode = 0;
 
 void
 options(cp) 
@@ -936,6 +941,10 @@ options(cp)
     if (*cp++ != '-') 
 	return;
     while (*cp) {
+	if (p && !*cp) {
+	    cp = p;
+	    p = 0;
+	}
         switch (*cp++) {
         case '-':  /* literal options */
 	    if (!*cp) {        /* ignore the rest of arguments */
@@ -946,9 +955,9 @@ options(cp)
 		int j;
                 p = (unsigned char *)long_option[i].name;
                 for (j=0;*p && (*p != '=') && *p == cp[j];p++, j++);
-		if (*p == cp[j]){
-		  p = &cp[j];
-		  break;
+		if (!*p || *p == cp[j]){
+		    p = &cp[j];
+		    break;
 		}
 		p = 0;
             }
@@ -1008,6 +1017,12 @@ options(cp)
                       return;
                   }
 #endif
+#ifdef UTF8_OUTPUT_ENABLE
+                if (strcmp(long_option[i].name, "ms-ucs-map") == 0){
+                    ms_ucs_map_f = TRUE;
+                    continue;
+                }
+#endif
                 if (strcmp(long_option[i].name, "prefix=") == 0){
                     if (*p == '=' && ' ' < p[1] && p[1] < 128){
                         for (i = 2; ' ' < p[i] && p[i] < 128; i++){
@@ -1082,17 +1097,23 @@ options(cp)
             if ('1'== cp[0] && '6'==cp[1]) {
 		output_conv = w_oconv16; cp+=2;
 		if (cp[0]=='L') {
-		    w_oconv16_begin_f=2; cp++;
+		    unicode_bom_f=2; cp++;
 		    w_oconv16_LE = 1;
                     if (cp[0] == '0'){
-                        w_oconv16_begin_f=1; cp++;
+                        unicode_bom_f=1; cp++;
                     }
 		} else if (cp[0] == 'B') {
-		    w_oconv16_begin_f=2; cp++;
+		    unicode_bom_f=2; cp++;
                     if (cp[0] == '0'){
-                        w_oconv16_begin_f=1; cp++;
+                        unicode_bom_f=1; cp++;
                     }
-                }
+                } 
+	    } else if (cp[0] == '8') {
+		output_conv = w_oconv; cp++;
+		unicode_bom_f=2;
+		if (cp[0] == '0'){
+		    unicode_bom_f=1; cp++;
+		}
 	    } else
                 output_conv = w_oconv;
             continue;
@@ -1100,7 +1121,16 @@ options(cp)
 #ifdef UTF8_INPUT_ENABLE
         case 'W':           /* UTF-8 input */
             if ('1'== cp[0] && '6'==cp[1]) {
-		input_f = UTF16_INPUT;
+		input_f = UTF16LE_INPUT;
+		if (cp[0]=='L') {
+		    cp++;
+		} else if (cp[0] == 'B') {
+		    cp++;
+		    input_f = UTF16BE_INPUT;
+		}
+	    } else if (cp[0] == '8') {
+		cp++;
+		input_f = UTF8_INPUT;
 	    } else
                 input_f = UTF8_INPUT;
             continue;
@@ -1760,7 +1790,7 @@ module_connection()
 #ifdef UTF8_INPUT_ENABLE
     } else if (input_f == UTF8_INPUT) {
         set_iconv(-TRUE, w_iconv);
-    } else if (input_f == UTF16_INPUT) {
+    } else if (input_f == UTF16LE_INPUT) {
         set_iconv(-TRUE, w_iconv16);
 #endif
     } else {
@@ -2039,6 +2069,32 @@ kanji_convert(f)
             } else if ((c1 == NL || c1 == CR) && broken_f&4) {
                 input_mode = ASCII; set_iconv(FALSE, 0);
                 SEND;
+	    } else if (c1 == NL && mime_f && !mime_decode_mode ) {
+		if ((c1=(*i_getc)(f))!=EOF && c1 == SPACE) {
+		    i_ungetc(SPACE,f);
+		    continue;
+		} else {
+		    i_ungetc(c1,f);
+		}
+		c1 = NL;
+		SEND;
+	    } else if (c1 == CR && mime_f && !mime_decode_mode ) {
+		if ((c1=(*i_getc)(f))!=EOF) {
+		    if (c1==SPACE) {
+			i_ungetc(SPACE,f);
+			continue;
+		    } else if (c1 == NL && (c1=(*i_getc)(f))!=EOF && c1 == SPACE) {
+			i_ungetc(SPACE,f);
+			continue;
+		    } else {
+			i_ungetc(c1,f);
+		    }
+		    i_ungetc(NL,f);
+		} else {
+		    i_ungetc(c1,f);
+		}
+		c1 = CR;
+		SEND;
 	    } else 
                 SEND;
         }
@@ -2364,7 +2420,7 @@ w_iconv16(c2, c1, c0)
     int ret;
 
     if (c2==0376 && c1==0377){
-	utf16_mode = UTF16_INPUT;
+	utf16_mode = UTF16LE_INPUT;
 	return 0;    
     } else if (c2==0377 && c1==0376){
 	utf16_mode = UTF16BE_INPUT;
@@ -2424,6 +2480,7 @@ e2w_conv(c2, c1)
 {
     extern unsigned short euc_to_utf8_1byte[];
     extern unsigned short * euc_to_utf8_2bytes[];
+    extern unsigned short * euc_to_utf8_2bytes_ms[];
     unsigned short *p;
 
     if (c2 == X0201) {
@@ -2432,7 +2489,7 @@ e2w_conv(c2, c1)
         c2 &= 0x7f;
         c2 = (c2&0x7f) - 0x21;
         if (0<=c2 && c2<sizeof_euc_to_utf8_2bytes)
-	    p = euc_to_utf8_2bytes[c2];
+            p = ms_ucs_map_f ? euc_to_utf8_2bytes_ms[c2] : euc_to_utf8_2bytes[c2];
 	else
 	    return 0;
     }
@@ -2462,7 +2519,16 @@ w_oconv(c2, c1)
     if (c2 == EOF) {
         (*o_putc)(EOF);
         return;
-    } else if (c2 == 0) { 
+    }
+
+    if (unicode_bom_f==2) {
+	(*o_putc)('\357');
+	(*o_putc)('\273');
+	(*o_putc)('\277');
+	unicode_bom_f=1;
+    }
+
+    if (c2 == 0) { 
 	output_mode = ASCII;
         (*o_putc)(c1);
     } else if (c2 == ISO8859_1) {
@@ -2489,7 +2555,7 @@ w_oconv16(c2, c1)
         return;
     }    
 
-    if (w_oconv16_begin_f==2) {
+    if (unicode_bom_f==2) {
         if (w_oconv16_LE){
             (*o_putc)((unsigned char)'\377');
             (*o_putc)('\376');
@@ -2497,7 +2563,7 @@ w_oconv16(c2, c1)
             (*o_putc)('\376');
             (*o_putc)((unsigned char)'\377');
         }
-	w_oconv16_begin_f=1;
+	unicode_bom_f=1;
     }
 
     if (c2 == ISO8859_1) {
@@ -2679,11 +2745,12 @@ base64_conv(c2, c1)
                     c1;
 {
     if (base64_count>50 && !mimeout_mode && c2==0 && c1==SPACE) {
+	(*o_putc)(EOF);
 	(*o_putc)(NL);
     } else if (base64_count>66 && mimeout_mode) {
 	(*o_base64conv)(EOF,0);
-	(*o_putc)(NL);
-	(*o_putc)('\t'); base64_count += 7;
+	(*o_base64conv)(NL,0);
+	(*o_base64conv)(SPACE,0);
     }
     (*o_base64conv)(c2,c1);
 }
@@ -3530,6 +3597,10 @@ FILE *f;
 {
     int c1, c2, c3, c4, cc;
     int t1, t2, t3, t4, mode, exit_mode;
+    int lwsp_count;
+    char *lwsp_buf;
+    char *lwsp_buf_new;
+    int lwsp_size = 128;
 
     if (mime_top != mime_last) {  /* Something is in FIFO */
         return  Fifo(mime_top++);
@@ -3558,8 +3629,69 @@ restart_mime_q:
         if (c1=='?'&&c2=='=' && mimebuf_f != FIXED_MIME) {
             /* end Q encoding */
             input_mode = exit_mode;
-            while((c1=(*i_getc)(f))!=EOF && c1==SPACE 
-                        /* && (c1==NL||c1==TAB||c1=='\r') */ ) ;
+	    lwsp_count = 0;
+	    lwsp_buf = malloc((lwsp_size+5)*sizeof(char));
+	    if (lwsp_buf==NULL) {
+		perror("can't malloc");
+		return -1;
+	    }
+	    while ((c1=(*i_getc)(f))!=EOF) {
+		switch (c1) {
+		case NL:
+		case CR:
+		    if (c1==NL) {
+			if ((c1=(*i_getc)(f))!=EOF && (c1==SPACE||c1==TAB)) {
+			    i_ungetc(SPACE,f);
+			    continue;
+			} else {
+			    i_ungetc(c1,f);
+			}
+			c1 = NL;
+		    } else {
+			if ((c1=(*i_getc)(f))!=EOF && c1 == NL) {
+			    if ((c1=(*i_getc)(f))!=EOF && (c1==SPACE||c1==TAB)) {
+				i_ungetc(SPACE,f);
+				continue;
+			    } else {
+				i_ungetc(c1,f);
+			    }
+			    i_ungetc(NL,f);
+			} else {
+			    i_ungetc(c1,f);
+			}
+			c1 = CR;
+		    }
+		    break;
+		case SPACE:
+		case TAB:
+		    lwsp_buf[lwsp_count] = c1;
+		    if (lwsp_count++>lwsp_size){
+			lwsp_size *= 2;
+			lwsp_buf_new = realloc(lwsp_buf, (lwsp_size+5)*sizeof(char));
+			if (lwsp_buf_new==NULL) {
+			    free(lwsp_buf);
+			    lwsp_buf = NULL;
+			    perror("can't realloc");
+			    return -1;
+			}
+			lwsp_buf = lwsp_buf_new;
+		    }
+		    continue;
+		}
+		break;
+	    }
+	    if (lwsp_count > 0) {
+		if (c1=='=' && (lwsp_buf[lwsp_count-1]==SPACE||lwsp_buf[lwsp_count-1]==TAB)) {
+		    lwsp_count = 0;
+		} else {
+		    i_ungetc(c1,f);
+		    for(lwsp_count--;lwsp_count>0;lwsp_count--)
+			i_ungetc(lwsp_buf[lwsp_count],f);
+		    c1 = lwsp_buf[0];
+		}
+	    }
+	    free(lwsp_buf);
+	    lwsp_buf = NULL;
             return c1;
         }
         if (c1=='='&&c2<' ') { /* this is soft wrap */
@@ -3613,8 +3745,72 @@ mime_c2_retry:
     }
     if ((c1 == '?') && (c2 == '=')) {
         input_mode = ASCII;
-        while((c1=(*i_getc)(f))!=EOF && c1==SPACE 
-                    /* && (c1==NL||c1==TAB||c1=='\r') */ ) ;
+	lwsp_count = 0;
+	lwsp_buf = malloc((lwsp_size+5)*sizeof(char));
+	if (lwsp_buf==NULL) {
+	    perror("can't malloc");
+	    return -1;
+	}
+	while ((c1=(*i_getc)(f))!=EOF) {
+	    switch (c1) {
+	    case NL:
+	    case CR:
+		if (c1==NL) {
+		    if ((c1=(*i_getc)(f))!=EOF && (c1==SPACE||c1==TAB)) {
+			i_ungetc(SPACE,f);
+			continue;
+		    } else {
+			i_ungetc(c1,f);
+		    }
+		    c1 = NL;
+		} else {
+		    if ((c1=(*i_getc)(f))!=EOF) {
+			if (c1==SPACE) {
+			    i_ungetc(SPACE,f);
+			    continue;
+			} else if ((c1=(*i_getc)(f))!=EOF && (c1==SPACE||c1==TAB)) {
+			    i_ungetc(SPACE,f);
+			    continue;
+			} else {
+			    i_ungetc(c1,f);
+			}
+			i_ungetc(NL,f);
+		    } else {
+			i_ungetc(c1,f);
+		    }
+		    c1 = CR;
+		}
+		break;
+	    case SPACE:
+	    case TAB:
+		lwsp_buf[lwsp_count] = c1;
+		if (lwsp_count++>lwsp_size){
+		    lwsp_size *= 2;
+		    lwsp_buf_new = realloc(lwsp_buf, (lwsp_size+5)*sizeof(char));
+		    if (lwsp_buf_new==NULL) {
+			free(lwsp_buf);
+			lwsp_buf = NULL;
+			perror("can't realloc");
+			return -1;
+		    }
+		    lwsp_buf = lwsp_buf_new;
+		}
+		continue;
+	    }
+	    break;
+	}
+	if (lwsp_count > 0) {
+	    if (c1=='=' && (lwsp_buf[lwsp_count-1]==SPACE||lwsp_buf[lwsp_count-1]==TAB)) {
+		lwsp_count = 0;
+	    } else {
+		i_ungetc(c1,f);
+		for(lwsp_count--;lwsp_count>0;lwsp_count--)
+		    i_ungetc(lwsp_buf[lwsp_count],f);
+		c1 = lwsp_buf[0];
+	    }
+	}
+	free(lwsp_buf);
+	lwsp_buf = NULL;
         return c1;
     }
 mime_c3_retry:
@@ -3732,6 +3928,11 @@ static char basis_64[] =
    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 
 static int b64c;
+#define MIMEOUT_BUF_LENGTH (60)
+char mimeout_buf[MIMEOUT_BUF_LENGTH+1];
+int mimeout_buf_count = 0;
+int mimeout_preserve_space = 0;
+#define itoh4(c)   (c>=10?c+'A'-10:c+'0')
 
 void
 open_mime(mode)
@@ -3739,20 +3940,49 @@ int mode;
 {
     unsigned char *p;
     int i;
+    int j;
     p  = mime_pattern[0];
     for(i=0;mime_encode[i];i++) {
 	if (mode == mime_encode[i]) {
 	    p = mime_pattern[i];
-		break;
+	    break;
 	}
     }
     mimeout_mode = mime_encode_method[i];
-	    
-    /* (*o_mputc)(' '); */
+    
+    i = 0;
+    if (base64_count>45) {
+	(*o_mputc)(NL);
+	(*o_mputc)(SPACE);
+	base64_count = 1;
+	if (!mimeout_preserve_space && mimeout_buf_count>0
+	    && (mimeout_buf[i]==SPACE || mimeout_buf[i]==TAB
+	    	|| mimeout_buf[i]==CR || mimeout_buf[i]==NL )) {
+	    i++;
+	}
+    }
+    if (!mimeout_preserve_space) {
+	for (;i<mimeout_buf_count;i++) {
+	    if (mimeout_buf[i]==SPACE || mimeout_buf[i]==TAB
+		|| mimeout_buf[i]==CR || mimeout_buf[i]==NL ) {
+		(*o_mputc)(mimeout_buf[i]);
+		base64_count ++;
+	    } else {
+		break;
+	    }
+	}
+    }
+    mimeout_preserve_space = FALSE;
+    
     while(*p) {
         (*o_mputc)(*p++);
         base64_count ++;
     }
+    j = mimeout_buf_count;
+    mimeout_buf_count = 0;
+    for (;i<j;i++) {
+	mime_putc(mimeout_buf[i]);
+    }
 }
 
 void
@@ -3760,80 +3990,51 @@ close_mime()
 {
     (*o_mputc)('?');
     (*o_mputc)('=');
-    (*o_mputc)(' ');
-    base64_count += 3;
+    base64_count += 2;
     mimeout_mode = 0;
 }
 
-#define itoh4(c)   (c>=10?c+'A'-10:c+'0')
+void
+eof_mime()
+{
+    switch(mimeout_mode) {
+    case 'Q':
+    case 'B':
+	break;
+    case 2:
+	(*o_mputc)(basis_64[((b64c & 0x3)<< 4)]);
+	(*o_mputc)('=');
+	(*o_mputc)('=');
+	base64_count += 3;
+	break;
+    case 1:
+	(*o_mputc)(basis_64[((b64c & 0xF) << 2)]);
+	(*o_mputc)('=');
+	base64_count += 2;
+	break;
+    }
+    if (mimeout_mode) {
+	if (mimeout_f!=FIXED_MIME) {
+	    close_mime(); 
+	} else if (mimeout_mode != 'Q')
+	    mimeout_mode = 'B';
+    }
+}
 
 void
-mime_putc(c)
+mimeout_addchar(c)
     int            c;
 {
-    if (mimeout_f==FIXED_MIME) {
-	if (base64_count>71) {
-            (*o_mputc)('\n');
-            base64_count=0;
-        }
-    } else if (c==NL) {
-	base64_count=0;
-    } 
-    if (c!=EOF) {
-        if ( c<=DEL &&(output_mode==ASCII ||output_mode == ISO8859_1 )
-		&& mimeout_f!=FIXED_MIME) {
-	    if (mimeout_mode=='Q') {
-		if (c<=SPACE) {
-		    close_mime();
-		}
-		(*o_mputc)(c);
-		return;
-	    }
-            if (mimeout_mode!='B' || c!=SPACE) {
-		if (mimeout_mode) {
-		    mime_putc(EOF);
-		    mimeout_mode=0;
-		}
-		(*o_mputc)(c);
-		base64_count ++;
-		return;
-	    }
-        } else if (!mimeout_mode && mimeout_f!=FIXED_MIME) {
-	    open_mime(output_mode);
-        }
-    } else { /* c==EOF */
-        switch(mimeout_mode) {
-        case 'Q':
-        case 'B':
-            break;
-        case 2:
-            (*o_mputc)(basis_64[((b64c & 0x3)<< 4)]);
-            (*o_mputc)('=');
-            (*o_mputc)('=');
-            base64_count += 3;
-            break;
-        case 1:
-            (*o_mputc)(basis_64[((b64c & 0xF) << 2)]);
-            (*o_mputc)('=');
-            base64_count += 2;
-            break;
-        }
-        if (mimeout_mode) {
-            if (mimeout_f!=FIXED_MIME) {
-                close_mime(); 
-            } else if (mimeout_mode != 'Q')
-                mimeout_mode = 'B';
-        }
-        return;
-    }
     switch(mimeout_mode) {
     case 'Q':
 	if(c>=DEL) {
 	    (*o_mputc)('=');
 	    (*o_mputc)(itoh4(((c>>4)&0xf)));
 	    (*o_mputc)(itoh4((c&0xf)));
+	    base64_count += 3;
 	} else {
 	    (*o_mputc)(c);
+	    base64_count++;
 	}
         break;
     case 'B':
@@ -3857,102 +4058,228 @@ mime_putc(c)
     }
 }
 
+void
+mime_putc(c)
+    int            c;
+{
+    int i = 0;
+    int j = 0;
+    
+    if (mimeout_f==FIXED_MIME && base64_count>50) {
+	eof_mime();
+	(*o_mputc)(NL);
+	base64_count=0;
+    } else if (c==CR||c==NL) {
+	base64_count=0;
+    }
+    if (c!=EOF && mimeout_f!=FIXED_MIME) {
+        if ( c<=DEL &&(output_mode==ASCII ||output_mode == ISO8859_1 ) ) {
+	    if (mimeout_mode=='Q') {
+		if (c<=SPACE) {
+		    close_mime();
+		    (*o_mputc)(SPACE);
+		    base64_count++;
+		}
+		(*o_mputc)(c);
+		base64_count++;
+		return;
+	    } else if (mimeout_mode) {
+		if (base64_count>63) {
+		    eof_mime();
+		    (*o_mputc)(NL);
+		    (*o_mputc)(SPACE);
+		    base64_count=1;
+		    mimeout_preserve_space = TRUE;
+		}
+		if (c==SPACE || c==TAB || c==CR || c==NL) {
+		    for (i=0;i<mimeout_buf_count;i++) {
+			if (SPACE<mimeout_buf[i] && mimeout_buf[i]<DEL) {
+			    eof_mime();
+			    for (i=0;i<mimeout_buf_count;i++) {
+				(*o_mputc)(mimeout_buf[i]);
+				base64_count++;
+			    }
+			    mimeout_buf_count = 0;
+			}
+		    }
+		    mimeout_buf[mimeout_buf_count++] = c;
+		    if (mimeout_buf_count>MIMEOUT_BUF_LENGTH) {
+		    	eof_mime();
+			base64_count = 0;
+			for (i=0;i<mimeout_buf_count;i++) {
+			    (*o_mputc)(mimeout_buf[i]);
+			    base64_count++;
+			}
+		    }
+		    return;
+		}
+		if (mimeout_buf_count>0 && SPACE<c) {
+		    mimeout_buf[mimeout_buf_count++] = c;
+		    if (mimeout_buf_count>MIMEOUT_BUF_LENGTH) {
+		    } else {
+			return;
+		    }
+		}
+	    } else if (!mimeout_mode) {
+		if (c==SPACE || c==TAB || c==CR || c==NL) {
+		    if ((c==CR || c==NL)
+			&&(mimeout_buf[mimeout_buf_count-1]==SPACE
+			   || mimeout_buf[mimeout_buf_count-1]==TAB)) {
+			mimeout_buf_count--;
+		    }
+		    for (i=0;i<mimeout_buf_count;i++) {
+			(*o_mputc)(mimeout_buf[i]);
+			base64_count++;
+		    }
+		    mimeout_buf_count = 0;
+		}
+		mimeout_buf[mimeout_buf_count++] = c;
+		if (mimeout_buf_count>75) {
+		    open_mime(output_mode);
+		}
+		return;
+	    }
+        } else if (!mimeout_mode) {
+	    if (mimeout_buf_count>0 && mimeout_buf[mimeout_buf_count-1]==SPACE) {
+		for (i=0;i<mimeout_buf_count-1;i++) {
+		    (*o_mputc)(mimeout_buf[i]);
+		    base64_count++;
+		}
+		mimeout_buf[0] = SPACE;
+		mimeout_buf_count = 1;
+	    }
+	    open_mime(output_mode);
+        }
+    } else { /* c==EOF */
+	j = mimeout_buf_count;
+	i = 0;
+	for (;i<j;i++) {
+	    if (mimeout_buf[i]==SPACE || mimeout_buf[i]==TAB
+		|| mimeout_buf[i]==CR || mimeout_buf[i]==NL)
+		break;
+	    (*mime_putc)(mimeout_buf[i]);
+	}
+        eof_mime();
+	for (;i<j;i++) {
+	    (*o_mputc)(mimeout_buf[i]);
+	    base64_count++;
+	}
+        return;
+    }
+    
+    if (mimeout_buf_count>0) {
+	j = mimeout_buf_count;
+	mimeout_buf_count = 0;
+	for (i=0;i<j;i++) {
+	    mimeout_addchar(mimeout_buf[i]);
+	}
+    }
+    mimeout_addchar(c);
+}
+
 
 #ifdef PERL_XS
 void 
 reinit()
 {
+    {
+        struct input_code *p = input_code_list;
+        while (p->name){
+            status_reinit(p++);
+        }
+    }
     unbuf_f = FALSE;
     estab_f = FALSE;
     nop_f = FALSE;
-    binmode_f = TRUE;       
-    rot_f = FALSE;         
-    hira_f = FALSE;         
-    input_f = FALSE;      
-    alpha_f = FALSE;     
-    mime_f = STRICT_MIME; 
-    mimebuf_f = FALSE; 
-    broken_f = FALSE;  
-    iso8859_f = FALSE; 
-#if defined(MSDOS) || defined(__OS2__) 
-     x0201_f = TRUE;   
+    binmode_f = TRUE;
+    rot_f = FALSE;
+    hira_f = FALSE;
+    input_f = FALSE;
+    alpha_f = FALSE;
+    mime_f = STRICT_MIME;
+    mimebuf_f = FALSE;
+    broken_f = FALSE;
+    iso8859_f = FALSE;
+    mimeout_f = FALSE;
+#if defined(MSDOS) || defined(__OS2__)
+     x0201_f = TRUE;
 #else
      x0201_f = NO_X0201;
 #endif
     iso2022jp_f = FALSE;
-
+#ifdef UTF8_OUTPUT_ENABLE
+    unicode_bom_f = 0;
+    w_oconv16_LE = 0;
+    ms_ucs_map_f = FALSE;
+#endif
+#ifdef INPUT_OPTION
+    cap_f = FALSE;
+    url_f = FALSE;
+    numchar_f = FALSE;
+#endif
+#ifdef CHECK_OPTION
+    noout_f = FALSE;
+    debug_f = FALSE;
+#endif
+    guess_f = FALSE;
+    is_inputcode_mixed = FALSE;
+    is_inputcode_set   = FALSE;
+#ifdef EXEC_IO
+    exec_f = 0;
+#endif
+#ifdef SHIFTJIS_CP932
+    cp932_f = TRUE;
+    cp932inv_f = FALSE;
+#endif
+    {
+        int i;
+        for (i = 0; i < 256; i++){
+            prefix_table[i] = 0;
+        }
+    }
+#ifdef UTF8_INPUT_ENABLE
+    utf16_mode = UTF16LE_INPUT;
+#endif
+    mimeout_mode = 0;
+    base64_count = 0;
+    f_line = 0;
+    f_prev = 0;
+    fold_preserve_f = FALSE;
+    fold_f = FALSE;
+    fold_len = 0;
     kanji_intro = DEFAULT_J;
     ascii_intro = DEFAULT_R;
-
-    output_conv = DEFAULT_CONV; 
-    oconv = DEFAULT_CONV; 
-
-    i_mgetc  = std_getc; 
-    i_mungetc  = std_ungetc;
-    i_mgetc_buf = std_getc; 
-    i_mungetc_buf = std_ungetc;
-
-    i_getc= std_getc; 
-    i_ungetc=std_ungetc;
-
-    i_bgetc= std_getc;
-    i_bungetc= std_ungetc;
-
-    o_putc = std_putc;
-    o_mputc = std_putc;
-    o_crconv = no_connection; 
-    o_rot_conv = no_connection; 
-    o_iso2022jp_check_conv = no_connection;
-    o_hira_conv = no_connection; 
-    o_fconv = no_connection; 
+    fold_margin  = FOLD_MARGIN;
+    output_conv = DEFAULT_CONV;
+    oconv = DEFAULT_CONV;
     o_zconv = no_connection;
-
+    o_fconv = no_connection;
+    o_crconv = no_connection;
+    o_rot_conv = no_connection;
+    o_hira_conv = no_connection;
+    o_base64conv = no_connection;
+    o_iso2022jp_check_conv = no_connection;
+    o_putc = std_putc;
     i_getc = std_getc;
     i_ungetc = std_ungetc;
-    i_mgetc = std_getc; 
-    i_mungetc = std_ungetc; 
-
+    i_bgetc = std_getc;
+    i_bungetc = std_ungetc;
+    o_mputc = std_putc;
+    i_mgetc = std_getc;
+    i_mungetc  = std_ungetc;
+    i_mgetc_buf = std_getc;
+    i_mungetc_buf = std_ungetc;
     output_mode = ASCII;
     input_mode =  ASCII;
     shift_mode =  FALSE;
-    mime_decode_mode =   FALSE;
+    mime_decode_mode = FALSE;
     file_out = FALSE;
-    mimeout_mode = 0;
-    mimeout_f = FALSE;
-    base64_count = 0;
-    option_mode = 0;
     crmode_f = 0;
-
-    {
-        struct input_code *p = input_code_list;
-        while (p->name){
-            status_reinit(p++);
-        }
-    }
-#ifdef UTF8_OUTPUT_ENABLE
-    if (w_oconv16_begin_f) {
-	w_oconv16_begin_f = 2;
-    }
-#endif
-    f_line = 0;    
-    f_prev = 0;
-    fold_preserve_f = FALSE; 
-    fold_f  = FALSE;
-    fold_len  = 0;
-    fold_margin  = FOLD_MARGIN;
+    option_mode = 0;
     broken_counter = 0;
     broken_last = 0;
     z_prev2=0,z_prev1=0;
 
-    {
-        int i;
-        for (i = 0; i < 256; i++){
-            prefix_table[i] = 0;
-        }
-    }
-    input_codename = "";
-    is_inputcode_mixed = FALSE;
-    is_inputcode_set   = FALSE;
 }
 #endif
 
@@ -3990,7 +4317,13 @@ usage()
 #ifdef DEFAULT_CODE_UTF8
     fprintf(stderr,"j,s,e,w  Outout code is JIS 7 bit, Shift JIS, AT&T JIS (EUC), UTF-8 (DEFAULT)\n");
 #endif
+#ifdef UTF8_OUTPUT_ENABLE
+    fprintf(stderr,"         After 'w' you can add more options. (80?|16((B|L)0?)?) \n");
+#endif
     fprintf(stderr,"J,S,E,W  Input assumption is JIS 7 bit , Shift JIS, AT&T JIS (EUC), UTF-8\n");
+#ifdef UTF8_INPUT_ENABLE
+    fprintf(stderr,"         After 'W' you can add more options. (8|16(B|L)?) \n");
+#endif
     fprintf(stderr,"t        no conversion\n");
     fprintf(stderr,"i_/o_    Output sequence to designate JIS-kanji/ASCII (DEFAULT B)\n");
     fprintf(stderr,"r        {de/en}crypt ROT13/47\n");
@@ -4012,12 +4345,26 @@ usage()
     fprintf(stderr,"I        Convert non ISO-2022-JP charactor to GETA\n");
     fprintf(stderr,"-L[uwm]  line mode u:LF w:CRLF m:CR (DEFAULT noconversion)\n");
     fprintf(stderr,"long name options\n");
-    fprintf(stderr," --fj,--unix,--mac,--windows                convert for the system\n");
+    fprintf(stderr," --fj,--unix,--mac,--windows                        convert for the system\n");
     fprintf(stderr," --jis,--euc,--sjis,--utf8,--utf16,--mime,--base64  convert for the code\n");
+    fprintf(stderr," --hiragana, --katakana    Hiragana/Katakana Conversion\n");
+#ifdef INPUT_OPTION
+    fprintf(stderr," --cap-input, --url-input  Convert hex after ':' or '%'\n");
+#endif
+#ifdef NUMCHAR_OPTION
+    fprintf(stderr," --numchar-input   Convert Unicode Character Reference\n");
+#endif
+#ifdef SHIFTJIS_CP932
+    fprintf(stderr," --no-cp932        Don't convert Shift_JIS FAxx-FCxx to equivalnet CP932\n");
+    fprintf(stderr," --cp932inv        convert Shift_JIS EDxx-EFxx to equivalnet CP932 FAxx-FCxx\n");
+#endif
+#ifdef UTF8_OUTPUT_ENABLE
+    fprintf(stderr," --ms-ucs-map      Microsoft UCS Mapping Compatible\n");
+#endif
 #ifdef OVERWRITE
-    fprintf(stderr," --overwrite          Overwrite original listed files by filtered result\n");
+    fprintf(stderr," --overwrite       Overwrite original listed files by filtered result\n");
 #endif
-    fprintf(stderr," -g, --guess          Guess the input code\n");
+    fprintf(stderr," -g, --guess       Guess the input code\n");
     fprintf(stderr," --help,--version\n");
     version();
 }
@@ -4038,7 +4385,7 @@ version()
 #ifdef __OS2__
                   "for OS/2"
 #endif
-                  ,Version,Patchlevel);
+                  ,NKF_VERSION,NKF_RELEASE_DATE);
     fprintf(stderr,"\n%s\n",CopyRight);
 }
 #endif