summaryrefslogtreecommitdiff
path: root/ext/nkf/nkf.c
diff options
context:
space:
mode:
Diffstat (limited to 'ext/nkf/nkf.c')
-rw-r--r--ext/nkf/nkf.c430
1 files changed, 396 insertions, 34 deletions
diff --git a/ext/nkf/nkf.c b/ext/nkf/nkf.c
index 65442a130c..8eb92b6cd6 100644
--- a/ext/nkf/nkf.c
+++ b/ext/nkf/nkf.c
@@ -1,9 +1,9 @@
/*
- * NKF Module for Ruby base on nkf 2.x
+ * NKF - Ruby extension for Network Kanji Filter
*
- * original nkf2.x is maintained at http://sourceforge.jp/projects/nkf/
+ * original nkf2.x is maintained at http://sourceforge.jp/projects/nkf/
*
- * $Id$
+ * $Id$
*
*/
@@ -78,6 +78,69 @@ rb_nkf_putchar(c)
#include "nkf-utf8/utf8tbl.c"
#include "nkf-utf8/nkf.c"
+int nkf_split_options(arg)
+ const char* arg;
+{
+ int count = 0;
+ char option[256];
+ int i = 0, j = 0;
+ int is_escaped = FALSE;
+ int is_single_quoted = FALSE;
+ int is_double_quoted = FALSE;
+ for(i = 0; arg[i]; i++){
+ if(j == 255){
+ return -1;
+ }else if(is_single_quoted){
+ if(arg[i] == '\''){
+ is_single_quoted = FALSE;
+ }else{
+ option[j++] = arg[i];
+ }
+ }else if(is_escaped){
+ is_escaped = FALSE;
+ option[j++] = arg[i];
+ }else if(arg[i] == '\\'){
+ is_escaped = TRUE;
+ }else if(is_double_quoted){
+ if(arg[i] == '"'){
+ is_double_quoted = FALSE;
+ }else{
+ option[j++] = arg[i];
+ }
+ }else if(arg[i] == '\''){
+ is_single_quoted = TRUE;
+ }else if(arg[i] == '"'){
+ is_double_quoted = TRUE;
+ }else if(arg[i] == ' '){
+ option[j] = '\0';
+ options(option);
+ j = 0;
+ }else{
+ option[j++] = arg[i];
+ }
+ }
+ if(j){
+ option[j] = '\0';
+ options(option);
+ }
+ return count;
+}
+
+/*
+ * call-seq:
+ * NKF.nkf(opt, str) -> string
+ *
+ * Convert _str_ and return converted result.
+ * Conversion details are specified by _opt_ as String.
+ *
+ * require 'nkf'
+ * output = NKF.nkf("-s", input)
+ *
+ * *Note*
+ * By default, nkf decodes MIME encoded string.
+ * If you want not to decode input, use NKF.nkf with <b>-m0</b> flag.
+ */
+
static VALUE
rb_nkf_kconv(obj, opt, src)
VALUE obj, opt, src;
@@ -89,7 +152,7 @@ rb_nkf_kconv(obj, opt, src)
StringValue(opt);
opt_ptr = RSTRING(opt)->ptr;
opt_end = opt_ptr + RSTRING(opt)->len;
- options(opt_ptr);
+ nkf_split_options(opt_ptr);
incsize = INCSIZE;
@@ -118,11 +181,27 @@ rb_nkf_kconv(obj, opt, src)
/*
- * NKF.guess1
+ * call-seq:
+ * NKF.guess1(str) -> integer
+ *
+ * Returns guessed encoding of _str_ as integer.
+ *
+ * Algorithm described in:
+ * Ken Lunde. `Understanding Japanese Information Processing'
+ * Sebastopol, CA: O'Reilly & Associates.
*
- * Character code detection - Algorithm described in:
- * Ken Lunde. `Understanding Japanese Information Processing'
- * Sebastopol, CA: O'Reilly & Associates.
+ * case NKF.guess1(input)
+ * when NKF::JIS
+ * "ISO-2022-JP"
+ * when NKF::SJIS
+ * "Shift_JIS"
+ * when NKF::EUC
+ * "EUC-JP"
+ * when NKF::UNKNOWN
+ * "UNKNOWN(ASCII)"
+ * when NKF::BINARY
+ * "BINARY"
+ * end
*/
static VALUE
@@ -214,9 +293,29 @@ rb_nkf_guess1(obj, src)
/*
- * NKF.guess2
+ * call-seq:
+ * NKF.guess2(str) -> integer
*
- * Guess Encoding By NKF2.0 Routine
+ * Returns guessed encoding of _str_ as integer by nkf routine.
+ *
+ * case NKF.guess(input)
+ * when NKF::ASCII
+ * "ASCII"
+ * when NKF::JIS
+ * "ISO-2022-JP"
+ * when NKF::SJIS
+ * "Shift_JIS"
+ * when NKF::EUC
+ * "EUC-JP"
+ * when NKF::UTF8
+ * "UTF-8"
+ * when NKF::UTF16
+ * "UTF-16"
+ * when NKF::UNKNOWN
+ * "UNKNOWN"
+ * when NKF::BINARY
+ * "BINARY"
+ * end
*/
static VALUE
@@ -261,32 +360,295 @@ rb_nkf_guess2(obj, src)
}
-/* Initialize NKF Module */
+/*
+ * NKF - Ruby extension for Network Kanji Filter
+ *
+ * == Description
+ *
+ * This is a Ruby Extension version of nkf (Netowrk Kanji Filter).
+ * It converts the first argument and return converted result. Conversion
+ * details are specified by flags as the first argument.
+ *
+ * *Nkf* is a yet another kanji code converter among networks, hosts and terminals.
+ * It converts input kanji code to designated kanji code
+ * such as ISO-2022-JP, Shift_JIS, EUC-JP, UTF-8 or UTF-16.
+ *
+ * One of the most unique faculty of *nkf* is the guess of the input kanji encodings.
+ * It currently recognizes ISO-2022-JP, Shift_JIS, EUC-JP, UTF-8 and UTF-16.
+ * So users needn't set the input kanji code explicitly.
+ *
+ * By default, X0201 kana is converted into X0208 kana.
+ * For X0201 kana, SO/SI, SSO and ESC-(-I methods are supported.
+ * For automatic code detection, nkf assumes no X0201 kana in Shift_JIS.
+ * To accept X0201 in Shift_JIS, use <b>-X</b>, <b>-x</b> or <b>-S</b>.
+ *
+ * == Flags
+ *
+ * === -b -u
+ *
+ * Output is buffered (DEFAULT), Output is unbuffered.
+ *
+ * === -j -s -e -w -w16
+ *
+ * Output code is ISO-2022-JP (7bit JIS), Shift_JIS, EUC-JP,
+ * UTF-8N, UTF-16BE.
+ * Without this option and compile option, ISO-2022-JP is assumed.
+ *
+ * === -J -S -E -W -W16
+ *
+ * Input assumption is JIS 7 bit, Shift_JIS, EUC-JP,
+ * UTF-8, UTF-16LE.
+ *
+ * ==== -J
+ *
+ * Assume JIS input. It also accepts EUC-JP.
+ * This is the default. This flag does not exclude Shift_JIS.
+ *
+ * ==== -S
+ *
+ * Assume Shift_JIS and X0201 kana input. It also accepts JIS.
+ * EUC-JP is recognized as X0201 kana. Without <b>-x</b> flag,
+ * X0201 kana (halfwidth kana) is converted into X0208.
+ *
+ * ==== -E
+ *
+ * Assume EUC-JP input. It also accepts JIS.
+ * Same as -J.
+ *
+ * === -t
+ *
+ * No conversion.
+ *
+ * === -i_
+ *
+ * Output sequence to designate JIS-kanji. (DEFAULT B)
+ *
+ * === -o_
+ *
+ * Output sequence to designate ASCII. (DEFAULT B)
+ *
+ * === -r
+ *
+ * {de/en}crypt ROT13/47
+ *
+ * === -h[123] --hiragana --katakana --katakana-hiragana
+ *
+ * [-h1 --hiragana] Katakana to Hiragana conversion.
+ *
+ * [-h2 --katakana] Hiragana to Katakana conversion.
+ *
+ * [-h3 --katakana-hiragana] Katakana to Hiragana and Hiragana to Katakana conversion.
+ *
+ * === -T
+ *
+ * Text mode output (MS-DOS)
+ *
+ * === -l
+ *
+ * ISO8859-1 (Latin-1) support
+ *
+ * === -f[<code>m</code> [- <code>n</code>]]
+ *
+ * Folding on <code>m</code> length with <code>n</code> margin in a line.
+ * Without this option, fold length is 60 and fold margin is 10.
+ *
+ * === -F
+ *
+ * New line preserving line folding.
+ *
+ * === -Z[0-3]
+ *
+ * Convert X0208 alphabet (Fullwidth Alphabets) to ASCII.
+ *
+ * [-Z -Z0] Convert X0208 alphabet to ASCII.
+ *
+ * [-Z1] Converts X0208 kankaku to single ASCII space.
+ *
+ * [-Z2] Converts X0208 kankaku to double ASCII spaces.
+ *
+ * [-Z3] Replacing Fullwidth >, <, ", & into '&gt;', '&lt;', '&quot;', '&amp;' as in HTML.
+ *
+ * === -X -x
+ *
+ * Assume X0201 kana in MS-Kanji.
+ * With <b>-X</b> or without this option, X0201 is converted into X0208 Kana.
+ * With <b>-x</b>, try to preserve X0208 kana and do not convert X0201 kana to X0208.
+ * In JIS output, ESC-(-I is used. In EUC output, SSO is used.
+ *
+ * === -B[0-2]
+ *
+ * Assume broken JIS-Kanji input, which lost ESC.
+ * Useful when your site is using old B-News Nihongo patch.
+ *
+ * [-B1] allows any char after ESC-( or ESC-$.
+ *
+ * [-B2] forces ASCII after NL.
+ *
+ * === -I
+ *
+ * Replacing non iso-2022-jp char into a geta character
+ * (substitute character in Japanese).
+ *
+ * === -d -c
+ *
+ * Delete \r in line feed, Add \r in line feed.
+ *
+ * === -m[BQN0]
+ *
+ * MIME ISO-2022-JP/ISO8859-1 decode. (DEFAULT)
+ * To see ISO8859-1 (Latin-1) -l is necessary.
+ *
+ * [-mB] Decode MIME base64 encoded stream. Remove header or other part before
+ * conversion.
+ *
+ * [-mQ] Decode MIME quoted stream. '_' in quoted stream is converted to space.
+ *
+ * [-mN] Non-strict decoding.
+ * It allows line break in the middle of the base64 encoding.
+ *
+ * [-m0] No MIME decode.
+ *
+ * === -M
+ *
+ * MIME encode. Header style. All ASCII code and control characters are intact.
+ * Kanji conversion is performed before encoding, so this cannot be used as a picture encoder.
+ *
+ * [-MB] MIME encode Base64 stream.
+ *
+ * [-MQ] Perfome quoted encoding.
+ *
+ * === -l
+ *
+ * Input and output code is ISO8859-1 (Latin-1) and ISO-2022-JP.
+ * <b>-s</b>, <b>-e</b> and <b>-x</b> are not compatible with this option.
+ *
+ * === -L[uwm]
+ *
+ * new line mode
+ * Without this option, nkf doesn't convert line breaks.
+ *
+ * [-Lu] unix (LF)
+ *
+ * [-Lw] windows (CRLF)
+ *
+ * [-Lm] mac (CR)
+ *
+ * === --fj --unix --mac --msdos --windows
+ *
+ * convert for these system
+ *
+ * === --jis --euc --sjis --mime --base64
+ *
+ * convert for named code
+ *
+ * === --jis-input --euc-input --sjis-input --mime-input --base64-input
+ *
+ * assume input system
+ *
+ * === --ic=<code>input codeset</code> --oc=<code>output codeset</code>
+ *
+ * Set the input or output codeset.
+ * NKF supports following codesets and those codeset name are case insensitive.
+ *
+ * [ISO-2022-JP] a.k.a. RFC1468, 7bit JIS, JUNET
+ *
+ * [EUC-JP (eucJP-nkf)] a.k.a. AT&T JIS, Japanese EUC, UJIS
+ *
+ * [eucJP-ascii] a.k.a. x-eucjp-open-19970715-ascii
+ *
+ * [eucJP-ms] a.k.a. x-eucjp-open-19970715-ms
+ *
+ * [CP51932] Microsoft Version of EUC-JP.
+ *
+ * [Shift_JIS] SJIS, MS-Kanji
+ *
+ * [CP932] a.k.a. Windows-31J
+ *
+ * [UTF-8] same as UTF-8N
+ *
+ * [UTF-8N] UTF-8 without BOM
+ *
+ * [UTF-8-BOM] UTF-8 with BOM
+ *
+ * [UTF-16] same as UTF-16BE
+ *
+ * [UTF-16BE] UTF-16 Big Endian without BOM
+ *
+ * [UTF-16BE-BOM] UTF-16 Big Endian with BOM
+ *
+ * [UTF-16LE] UTF-16 Little Endian without BOM
+ *
+ * [UTF-16LE-BOM] UTF-16 Little Endian with BOM
+ *
+ * [UTF8-MAC] NKDed UTF-8, a.k.a. UTF8-NFD (input only)
+ *
+ * === --fb-{skip, html, xml, perl, java, subchar}
+ *
+ * Specify the way that nkf handles unassigned characters.
+ * Without this option, --fb-skip is assumed.
+ *
+ * === --prefix= <code>escape character</code> <code>target character</code> ..
+ *
+ * When nkf converts to Shift_JIS,
+ * nkf adds a specified escape character to specified 2nd byte of Shift_JIS characters.
+ * 1st byte of argument is the escape character and following bytes are target characters.
+ *
+ * === --disable-cp932ext
+ *
+ * Handle the characters extended in CP932 as unassigned characters.
+ *
+ * === --cap-input
+ *
+ * Decode hex encoded characters.
+ *
+ * === --url-input
+ *
+ * Unescape percent escaped characters.
+ *
+ * === --
+ *
+ * Ignore rest of -option.
+ */
void
Init_nkf()
{
- VALUE mKconv = rb_define_module("NKF");
-
- rb_define_module_function(mKconv, "nkf", rb_nkf_kconv, 2);
- rb_define_module_function(mKconv, "guess", rb_nkf_guess2, 1);
- rb_define_module_function(mKconv, "guess1", rb_nkf_guess1, 1);
- rb_define_module_function(mKconv, "guess2", rb_nkf_guess2, 1);
-
- rb_define_const(mKconv, "AUTO", INT2FIX(_AUTO));
- rb_define_const(mKconv, "JIS", INT2FIX(_JIS));
- rb_define_const(mKconv, "EUC", INT2FIX(_EUC));
- rb_define_const(mKconv, "SJIS", INT2FIX(_SJIS));
- rb_define_const(mKconv, "BINARY", INT2FIX(_BINARY));
- rb_define_const(mKconv, "NOCONV", INT2FIX(_NOCONV));
- rb_define_const(mKconv, "ASCII", INT2FIX(_ASCII));
- rb_define_const(mKconv, "UTF8", INT2FIX(_UTF8));
- rb_define_const(mKconv, "UTF16", INT2FIX(_UTF16));
- rb_define_const(mKconv, "UTF32", INT2FIX(_UTF32));
- rb_define_const(mKconv, "UNKNOWN", INT2FIX(_UNKNOWN));
- rb_define_const(mKconv, "VERSION", rb_str_new2(RUBY_NKF_VERSION));
- /* for debug */
- rb_define_const(mKconv, "NKF_VERSION", rb_str_new2(NKF_VERSION));
- rb_define_const(mKconv, "NKF_RELEASE_DATE", rb_str_new2(NKF_RELEASE_DATE));
- rb_define_const(mKconv, "REVISION", rb_str_new2(RUBY_NKF_REVISION));
+ /* hoge */
+ VALUE mKconv = rb_define_module("NKF");
+ /* hoge */
+
+ rb_define_module_function(mKconv, "nkf", rb_nkf_kconv, 2);
+ rb_define_module_function(mKconv, "guess1", rb_nkf_guess1, 1);
+ rb_define_module_function(mKconv, "guess2", rb_nkf_guess2, 1);
+ rb_define_alias(mKconv, "guess", "guess2");
+ rb_define_alias(rb_singleton_class(mKconv), "guess", "guess2");
+
+ /* Auto-Detect */
+ rb_define_const(mKconv, "AUTO", INT2FIX(_AUTO));
+ /* ISO-2022-JP */
+ rb_define_const(mKconv, "JIS", INT2FIX(_JIS));
+ /* EUC-JP */
+ rb_define_const(mKconv, "EUC", INT2FIX(_EUC));
+ /* Shift_JIS */
+ rb_define_const(mKconv, "SJIS", INT2FIX(_SJIS));
+ /* BINARY */
+ rb_define_const(mKconv, "BINARY", INT2FIX(_BINARY));
+ /* No conversion */
+ rb_define_const(mKconv, "NOCONV", INT2FIX(_NOCONV));
+ /* ASCII */
+ rb_define_const(mKconv, "ASCII", INT2FIX(_ASCII));
+ /* UTF-8 */
+ rb_define_const(mKconv, "UTF8", INT2FIX(_UTF8));
+ /* UTF-16 */
+ rb_define_const(mKconv, "UTF16", INT2FIX(_UTF16));
+ /* UTF-32 */
+ rb_define_const(mKconv, "UTF32", INT2FIX(_UTF32));
+ /* UNKNOWN */
+ rb_define_const(mKconv, "UNKNOWN", INT2FIX(_UNKNOWN));
+ /* Full version string of nkf */
+ rb_define_const(mKconv, "VERSION", rb_str_new2(RUBY_NKF_VERSION));
+ /* Version of nkf */
+ rb_define_const(mKconv, "NKF_VERSION", rb_str_new2(NKF_VERSION));
+ /* Release date of nkf */
+ rb_define_const(mKconv, "NKF_RELEASE_DATE", rb_str_new2(NKF_RELEASE_DATE));
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-04-16 11:04:47 +0000