diff options
Diffstat (limited to 'ruby_1_8_5/ext/nkf/nkf.c')
| -rw-r--r-- | ruby_1_8_5/ext/nkf/nkf.c | 654 |
1 files changed, 0 insertions, 654 deletions
diff --git a/ruby_1_8_5/ext/nkf/nkf.c b/ruby_1_8_5/ext/nkf/nkf.c deleted file mode 100644 index 8bcb6d15f7..0000000000 --- a/ruby_1_8_5/ext/nkf/nkf.c +++ /dev/null @@ -1,654 +0,0 @@ -/* - * NKF - Ruby extension for Network Kanji Filter - * - * original nkf2.x is maintained at http://sourceforge.jp/projects/nkf/ - * - * $Id: nkf.c,v 1.9.2.4 2006/03/27 13:28:14 naruse Exp $ - * - */ - -#define RUBY_NKF_REVISION "$Revision: 1.9.2.4 $" -#define RUBY_NKF_VERSION NKF_VERSION " (" NKF_RELEASE_DATE ")" - -#include "ruby.h" - -/* Encoding Constants */ -#define _AUTO 0 -#define _JIS 1 -#define _EUC 2 -#define _SJIS 3 -#define _BINARY 4 -#define _NOCONV 4 -#define _ASCII 5 -/* 0b011x is reserved for UTF-8 Family */ -#define _UTF8 6 -/* 0b10xx is reserved for UTF-16 Family */ -#define _UTF16 8 -/* 0b11xx is reserved for UTF-32 Family */ -#define _UTF32 12 -#define _OTHER 16 -#define _UNKNOWN _AUTO - -/* Replace nkf's getchar/putchar for variable modification */ -/* we never use getc, ungetc */ - -#undef getc -#undef ungetc -#define getc(f) (input_ctr>=i_len?-1:input[input_ctr++]) -#define ungetc(c,f) input_ctr-- - -#define INCSIZE 32 -#undef putchar -#undef TRUE -#undef FALSE -#define putchar(c) rb_nkf_putchar(c) - -/* Input/Output pointers */ - -static unsigned char *output; -static unsigned char *input; -static int input_ctr; -static int i_len; -static int output_ctr; -static int o_len; -static int incsize; - -static VALUE result; - -static int -rb_nkf_putchar(c) - unsigned int c; -{ - if (output_ctr >= o_len) { - o_len += incsize; - rb_str_resize(result, o_len); - incsize *= 2; - output = RSTRING(result)->ptr; - } - output[output_ctr++] = c; - - return c; -} - -/* Include kanji filter main part */ -/* getchar and putchar will be replaced during inclusion */ - -#define PERL_XS 1 -#include "nkf-utf8/config.h" -#include "nkf-utf8/utf8tbl.c" -#include "nkf-utf8/nkf.c" - -int nkf_split_options(arg) - const char* arg; -{ - int count = 0; - char option[256]; - int i = 0, j = 0; - int is_escaped = FALSE; - int is_single_quoted = FALSE; - int is_double_quoted = FALSE; - for(i = 0; arg[i]; i++){ - if(j == 255){ - return -1; - }else if(is_single_quoted){ - if(arg[i] == '\''){ - is_single_quoted = FALSE; - }else{ - option[j++] = arg[i]; - } - }else if(is_escaped){ - is_escaped = FALSE; - option[j++] = arg[i]; - }else if(arg[i] == '\\'){ - is_escaped = TRUE; - }else if(is_double_quoted){ - if(arg[i] == '"'){ - is_double_quoted = FALSE; - }else{ - option[j++] = arg[i]; - } - }else if(arg[i] == '\''){ - is_single_quoted = TRUE; - }else if(arg[i] == '"'){ - is_double_quoted = TRUE; - }else if(arg[i] == ' '){ - option[j] = '\0'; - options(option); - j = 0; - }else{ - option[j++] = arg[i]; - } - } - if(j){ - option[j] = '\0'; - options(option); - } - return count; -} - -/* - * call-seq: - * NKF.nkf(opt, str) -> string - * - * Convert _str_ and return converted result. - * Conversion details are specified by _opt_ as String. - * - * require 'nkf' - * output = NKF.nkf("-s", input) - * - * *Note* - * By default, nkf decodes MIME encoded string. - * If you want not to decode input, use NKF.nkf with <b>-m0</b> flag. - */ - -static VALUE -rb_nkf_kconv(obj, opt, src) - VALUE obj, opt, src; -{ - char *opt_ptr, *opt_end; - volatile VALUE v; - - reinit(); - StringValue(opt); - opt_ptr = RSTRING(opt)->ptr; - opt_end = opt_ptr + RSTRING(opt)->len; - nkf_split_options(opt_ptr); - - incsize = INCSIZE; - - input_ctr = 0; - StringValue(src); - input = RSTRING(src)->ptr; - i_len = RSTRING(src)->len; - result = rb_str_new(0, i_len*3 + 10); - v = result; - - output_ctr = 0; - output = RSTRING(result)->ptr; - o_len = RSTRING(result)->len; - *output = '\0'; - - if(x0201_f == WISH_TRUE) - x0201_f = ((!iso2022jp_f)? TRUE : NO_X0201); - - kanji_convert(NULL); - RSTRING(result)->ptr[output_ctr] = '\0'; - RSTRING(result)->len = output_ctr; - OBJ_INFECT(result, src); - - return result; -} - - -/* - * call-seq: - * NKF.guess1(str) -> integer - * - * Returns guessed encoding of _str_ as integer. - * - * Algorithm described in: - * Ken Lunde. `Understanding Japanese Information Processing' - * Sebastopol, CA: O'Reilly & Associates. - * - * case NKF.guess1(input) - * when NKF::JIS - * "ISO-2022-JP" - * when NKF::SJIS - * "Shift_JIS" - * when NKF::EUC - * "EUC-JP" - * when NKF::UNKNOWN - * "UNKNOWN(ASCII)" - * when NKF::BINARY - * "BINARY" - * end - */ - -static VALUE -rb_nkf_guess1(obj, src) - VALUE obj, src; -{ - unsigned char *p; - unsigned char *pend; - int sequence_counter = 0; - - StringValue(src); - p = RSTRING(src)->ptr; - pend = p + RSTRING(src)->len; - if (p == pend) return INT2FIX(_UNKNOWN); - -#define INCR do {\ - p++;\ - if (p==pend) return INT2FIX(_UNKNOWN);\ - sequence_counter++;\ - if (sequence_counter % 2 == 1 && *p != 0xa4)\ - sequence_counter = 0;\ - if (6 <= sequence_counter) {\ - sequence_counter = 0;\ - return INT2FIX(_EUC);\ - }\ - } while (0) - - if (*p == 0xa4) - sequence_counter = 1; - - while (p<pend) { - if (*p == '\033') { - return INT2FIX(_JIS); - } - if (*p < '\006' || *p == 0x7f || *p == 0xff) { - return INT2FIX(_BINARY); - } - if (0x81 <= *p && *p <= 0x8d) { - return INT2FIX(_SJIS); - } - if (0x8f <= *p && *p <= 0x9f) { - return INT2FIX(_SJIS); - } - if (*p == 0x8e) { /* SS2 */ - INCR; - if ((0x40 <= *p && *p <= 0x7e) || - (0x80 <= *p && *p <= 0xa0) || - (0xe0 <= *p && *p <= 0xfc)) - return INT2FIX(_SJIS); - } - else if (0xa1 <= *p && *p <= 0xdf) { - INCR; - if (0xf0 <= *p && *p <= 0xfe) - return INT2FIX(_EUC); - if (0xe0 <= *p && *p <= 0xef) { - while (p < pend && *p >= 0x40) { - if (*p >= 0x81) { - if (*p <= 0x8d || (0x8f <= *p && *p <= 0x9f)) { - return INT2FIX(_SJIS); - } - else if (0xfd <= *p && *p <= 0xfe) { - return INT2FIX(_EUC); - } - } - INCR; - } - } - else if (*p <= 0x9f) { - return INT2FIX(_SJIS); - } - } - else if (0xf0 <= *p && *p <= 0xfe) { - return INT2FIX(_EUC); - } - else if (0xe0 <= *p && *p <= 0xef) { - INCR; - if ((0x40 <= *p && *p <= 0x7e) || - (0x80 <= *p && *p <= 0xa0)) { - return INT2FIX(_SJIS); - } - if (0xfd <= *p && *p <= 0xfe) { - return INT2FIX(_EUC); - } - } - INCR; - } - return INT2FIX(_UNKNOWN); -} - - -/* - * call-seq: - * NKF.guess2(str) -> integer - * - * Returns guessed encoding of _str_ as integer by nkf routine. - * - * case NKF.guess(input) - * when NKF::ASCII - * "ASCII" - * when NKF::JIS - * "ISO-2022-JP" - * when NKF::SJIS - * "Shift_JIS" - * when NKF::EUC - * "EUC-JP" - * when NKF::UTF8 - * "UTF-8" - * when NKF::UTF16 - * "UTF-16" - * when NKF::UNKNOWN - * "UNKNOWN" - * when NKF::BINARY - * "BINARY" - * end - */ - -static VALUE -rb_nkf_guess2(obj, src) - VALUE obj, src; -{ - int code = _BINARY; - - reinit(); - - input_ctr = 0; - StringValue(src); - input = RSTRING(src)->ptr; - i_len = RSTRING(src)->len; - - if(x0201_f == WISH_TRUE) - x0201_f = ((!iso2022jp_f)? TRUE : NO_X0201); - - guess_f = TRUE; - kanji_convert( NULL ); - guess_f = FALSE; - - if (!is_inputcode_mixed) { - if (strcmp(input_codename, "") == 0) { - code = _ASCII; - } else if (strcmp(input_codename, "ISO-2022-JP") == 0) { - code = _JIS; - } else if (strcmp(input_codename, "EUC-JP") == 0) { - code = _EUC; - } else if (strcmp(input_codename, "Shift_JIS") == 0) { - code = _SJIS; - } else if (strcmp(input_codename, "UTF-8") == 0) { - code = _UTF8; - } else if (strcmp(input_codename, "UTF-16") == 0) { - code = _UTF16; - } else if (strlen(input_codename) > 0) { - code = _UNKNOWN; - } - } - - return INT2FIX( code ); -} - - -/* - * NKF - Ruby extension for Network Kanji Filter - * - * == Description - * - * This is a Ruby Extension version of nkf (Netowrk Kanji Filter). - * It converts the first argument and return converted result. Conversion - * details are specified by flags as the first argument. - * - * *Nkf* is a yet another kanji code converter among networks, hosts and terminals. - * It converts input kanji code to designated kanji code - * such as ISO-2022-JP, Shift_JIS, EUC-JP, UTF-8 or UTF-16. - * - * One of the most unique faculty of *nkf* is the guess of the input kanji encodings. - * It currently recognizes ISO-2022-JP, Shift_JIS, EUC-JP, UTF-8 and UTF-16. - * So users needn't set the input kanji code explicitly. - * - * By default, X0201 kana is converted into X0208 kana. - * For X0201 kana, SO/SI, SSO and ESC-(-I methods are supported. - * For automatic code detection, nkf assumes no X0201 kana in Shift_JIS. - * To accept X0201 in Shift_JIS, use <b>-X</b>, <b>-x</b> or <b>-S</b>. - * - * == Flags - * - * === -b -u - * - * Output is buffered (DEFAULT), Output is unbuffered. - * - * === -j -s -e -w -w16 - * - * Output code is ISO-2022-JP (7bit JIS), Shift_JIS, EUC-JP, - * UTF-8N, UTF-16BE. - * Without this option and compile option, ISO-2022-JP is assumed. - * - * === -J -S -E -W -W16 - * - * Input assumption is JIS 7 bit, Shift_JIS, EUC-JP, - * UTF-8, UTF-16LE. - * - * ==== -J - * - * Assume JIS input. It also accepts EUC-JP. - * This is the default. This flag does not exclude Shift_JIS. - * - * ==== -S - * - * Assume Shift_JIS and X0201 kana input. It also accepts JIS. - * EUC-JP is recognized as X0201 kana. Without <b>-x</b> flag, - * X0201 kana (halfwidth kana) is converted into X0208. - * - * ==== -E - * - * Assume EUC-JP input. It also accepts JIS. - * Same as -J. - * - * === -t - * - * No conversion. - * - * === -i_ - * - * Output sequence to designate JIS-kanji. (DEFAULT B) - * - * === -o_ - * - * Output sequence to designate ASCII. (DEFAULT B) - * - * === -r - * - * {de/en}crypt ROT13/47 - * - * === -h[123] --hiragana --katakana --katakana-hiragana - * - * [-h1 --hiragana] Katakana to Hiragana conversion. - * - * [-h2 --katakana] Hiragana to Katakana conversion. - * - * [-h3 --katakana-hiragana] Katakana to Hiragana and Hiragana to Katakana conversion. - * - * === -T - * - * Text mode output (MS-DOS) - * - * === -l - * - * ISO8859-1 (Latin-1) support - * - * === -f[<code>m</code> [- <code>n</code>]] - * - * Folding on <code>m</code> length with <code>n</code> margin in a line. - * Without this option, fold length is 60 and fold margin is 10. - * - * === -F - * - * New line preserving line folding. - * - * === -Z[0-3] - * - * Convert X0208 alphabet (Fullwidth Alphabets) to ASCII. - * - * [-Z -Z0] Convert X0208 alphabet to ASCII. - * - * [-Z1] Converts X0208 kankaku to single ASCII space. - * - * [-Z2] Converts X0208 kankaku to double ASCII spaces. - * - * [-Z3] Replacing Fullwidth >, <, ", & into '>', '<', '"', '&' as in HTML. - * - * === -X -x - * - * Assume X0201 kana in MS-Kanji. - * With <b>-X</b> or without this option, X0201 is converted into X0208 Kana. - * With <b>-x</b>, try to preserve X0208 kana and do not convert X0201 kana to X0208. - * In JIS output, ESC-(-I is used. In EUC output, SSO is used. - * - * === -B[0-2] - * - * Assume broken JIS-Kanji input, which lost ESC. - * Useful when your site is using old B-News Nihongo patch. - * - * [-B1] allows any char after ESC-( or ESC-$. - * - * [-B2] forces ASCII after NL. - * - * === -I - * - * Replacing non iso-2022-jp char into a geta character - * (substitute character in Japanese). - * - * === -d -c - * - * Delete \r in line feed, Add \r in line feed. - * - * === -m[BQN0] - * - * MIME ISO-2022-JP/ISO8859-1 decode. (DEFAULT) - * To see ISO8859-1 (Latin-1) -l is necessary. - * - * [-mB] Decode MIME base64 encoded stream. Remove header or other part before - * conversion. - * - * [-mQ] Decode MIME quoted stream. '_' in quoted stream is converted to space. - * - * [-mN] Non-strict decoding. - * It allows line break in the middle of the base64 encoding. - * - * [-m0] No MIME decode. - * - * === -M - * - * MIME encode. Header style. All ASCII code and control characters are intact. - * Kanji conversion is performed before encoding, so this cannot be used as a picture encoder. - * - * [-MB] MIME encode Base64 stream. - * - * [-MQ] Perfome quoted encoding. - * - * === -l - * - * Input and output code is ISO8859-1 (Latin-1) and ISO-2022-JP. - * <b>-s</b>, <b>-e</b> and <b>-x</b> are not compatible with this option. - * - * === -L[uwm] - * - * new line mode - * Without this option, nkf doesn't convert line breaks. - * - * [-Lu] unix (LF) - * - * [-Lw] windows (CRLF) - * - * [-Lm] mac (CR) - * - * === --fj --unix --mac --msdos --windows - * - * convert for these system - * - * === --jis --euc --sjis --mime --base64 - * - * convert for named code - * - * === --jis-input --euc-input --sjis-input --mime-input --base64-input - * - * assume input system - * - * === --ic=<code>input codeset</code> --oc=<code>output codeset</code> - * - * Set the input or output codeset. - * NKF supports following codesets and those codeset name are case insensitive. - * - * [ISO-2022-JP] a.k.a. RFC1468, 7bit JIS, JUNET - * - * [EUC-JP (eucJP-nkf)] a.k.a. AT&T JIS, Japanese EUC, UJIS - * - * [eucJP-ascii] a.k.a. x-eucjp-open-19970715-ascii - * - * [eucJP-ms] a.k.a. x-eucjp-open-19970715-ms - * - * [CP51932] Microsoft Version of EUC-JP. - * - * [Shift_JIS] SJIS, MS-Kanji - * - * [CP932] a.k.a. Windows-31J - * - * [UTF-8] same as UTF-8N - * - * [UTF-8N] UTF-8 without BOM - * - * [UTF-8-BOM] UTF-8 with BOM - * - * [UTF-16] same as UTF-16BE - * - * [UTF-16BE] UTF-16 Big Endian without BOM - * - * [UTF-16BE-BOM] UTF-16 Big Endian with BOM - * - * [UTF-16LE] UTF-16 Little Endian without BOM - * - * [UTF-16LE-BOM] UTF-16 Little Endian with BOM - * - * [UTF8-MAC] NKDed UTF-8, a.k.a. UTF8-NFD (input only) - * - * === --fb-{skip, html, xml, perl, java, subchar} - * - * Specify the way that nkf handles unassigned characters. - * Without this option, --fb-skip is assumed. - * - * === --prefix= <code>escape character</code> <code>target character</code> .. - * - * When nkf converts to Shift_JIS, - * nkf adds a specified escape character to specified 2nd byte of Shift_JIS characters. - * 1st byte of argument is the escape character and following bytes are target characters. - * - * === --disable-cp932ext - * - * Handle the characters extended in CP932 as unassigned characters. - * - * === --cap-input - * - * Decode hex encoded characters. - * - * === --url-input - * - * Unescape percent escaped characters. - * - * === -- - * - * Ignore rest of -option. - */ - -void -Init_nkf() -{ - /* hoge */ - VALUE mKconv = rb_define_module("NKF"); - /* hoge */ - - rb_define_module_function(mKconv, "nkf", rb_nkf_kconv, 2); - rb_define_module_function(mKconv, "guess1", rb_nkf_guess1, 1); - rb_define_module_function(mKconv, "guess2", rb_nkf_guess2, 1); - rb_define_alias(mKconv, "guess", "guess2"); - rb_define_alias(rb_singleton_class(mKconv), "guess", "guess2"); - - /* Auto-Detect */ - rb_define_const(mKconv, "AUTO", INT2FIX(_AUTO)); - /* ISO-2022-JP */ - rb_define_const(mKconv, "JIS", INT2FIX(_JIS)); - /* EUC-JP */ - rb_define_const(mKconv, "EUC", INT2FIX(_EUC)); - /* Shift_JIS */ - rb_define_const(mKconv, "SJIS", INT2FIX(_SJIS)); - /* BINARY */ - rb_define_const(mKconv, "BINARY", INT2FIX(_BINARY)); - /* No conversion */ - rb_define_const(mKconv, "NOCONV", INT2FIX(_NOCONV)); - /* ASCII */ - rb_define_const(mKconv, "ASCII", INT2FIX(_ASCII)); - /* UTF-8 */ - rb_define_const(mKconv, "UTF8", INT2FIX(_UTF8)); - /* UTF-16 */ - rb_define_const(mKconv, "UTF16", INT2FIX(_UTF16)); - /* UTF-32 */ - rb_define_const(mKconv, "UTF32", INT2FIX(_UTF32)); - /* UNKNOWN */ - rb_define_const(mKconv, "UNKNOWN", INT2FIX(_UNKNOWN)); - /* Full version string of nkf */ - rb_define_const(mKconv, "VERSION", rb_str_new2(RUBY_NKF_VERSION)); - /* Version of nkf */ - rb_define_const(mKconv, "NKF_VERSION", rb_str_new2(NKF_VERSION)); - /* Release date of nkf */ - rb_define_const(mKconv, "NKF_RELEASE_DATE", rb_str_new2(NKF_RELEASE_DATE)); -} |