* encoding.c: provide basic features for M17N.

* parse.y: encoding aware parsing.

* parse.y (pragma_encoding): encoding specification pragma.

* parse.y (rb_intern3): encoding specified symbols.

* string.c (rb_str_length): length based on characters.  
  for older behavior, bytesize method added.

* string.c (rb_str_index_m): index based on characters.  rindex as
  well.

* string.c (succ_char): encoding aware succeeding string.

* string.c (rb_str_reverse): reverse based on characters.

* string.c (rb_str_inspect): encoding aware string description.

* string.c (rb_str_upcase_bang): encoding aware case conversion.
  downcase, capitalize, swapcase as well.

* string.c (rb_str_tr_bang): tr based on characters.  delete,
  squeeze, tr_s, count as well.

* string.c (rb_str_split_m): split based on characters.

* string.c (rb_str_each_line): encoding aware each_line.

* string.c (rb_str_each_char): added.  iteration based on
  characters.

* string.c (rb_str_strip_bang): encoding aware whitespace
  stripping.  lstrip, rstrip as well.

* string.c (rb_str_justify): encoding aware justifying (ljust,
  rjust, center).

* string.c (str_encoding): get encoding attribute from a string. 

* re.c (rb_reg_initialize): encoding aware regular expression

* sprintf.c (rb_str_format): formatting (i.e. length count) based
  on characters.

* io.c (rb_io_getc): getc to return one-character string.
  for older behavior, getbyte method added.

* ext/stringio/stringio.c (strio_getc): ditto.

* io.c (rb_io_ungetc): allow pushing arbitrary string at the
  current reading point.

* ext/stringio/stringio.c (strio_ungetc): ditto.

* ext/strscan/strscan.c: encoding support.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@13261 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

8 files changed, 43 insertions, 14 deletions

diff --git a/include/ruby/intern.h b/include/ruby/intern.h
index aefee7f674..ae231c7d91 100644
--- a/include/ruby/intern.h
+++ b/include/ruby/intern.h
@@ -6,7 +6,7 @@
   $Date$
   created at: Thu Jun 10 14:22:17 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
   Copyright (C) 2000  Network Applied Communication Laboratory, Inc.
   Copyright (C) 2000  Information-technology Promotion Agency, Japan
 
@@ -337,7 +337,7 @@ RUBY_EXTERN VALUE rb_default_rs;
 RUBY_EXTERN VALUE rb_output_rs;
 VALUE rb_io_write(VALUE, VALUE);
 VALUE rb_io_gets(VALUE);
-VALUE rb_io_getc(VALUE);
+VALUE rb_io_getbyte(VALUE);
 VALUE rb_io_ungetc(VALUE, VALUE);
 VALUE rb_io_close(VALUE);
 VALUE rb_io_flush(VALUE);
@@ -444,7 +444,7 @@ VALUE rb_reg_last_match(VALUE);
 VALUE rb_reg_match_pre(VALUE);
 VALUE rb_reg_match_post(VALUE);
 VALUE rb_reg_match_last(VALUE);
-VALUE rb_reg_new(const char*, long, int);
+VALUE rb_reg_new(VALUE, int);
 VALUE rb_reg_match(VALUE, VALUE);
 VALUE rb_reg_match2(VALUE);
 int rb_reg_options(VALUE);
@@ -498,6 +498,7 @@ VALUE rb_str_unlocktmp(VALUE);
 VALUE rb_str_dup_frozen(VALUE);
 VALUE rb_str_plus(VALUE, VALUE);
 VALUE rb_str_times(VALUE, VALUE);
+int rb_str_sublen(VALUE, int);
 VALUE rb_str_substr(VALUE, long, long);
 void rb_str_modify(VALUE);
 VALUE rb_str_freeze(VALUE);
diff --git a/include/ruby/io.h b/include/ruby/io.h
index 6d22de8df1..becf262eca 100644
--- a/include/ruby/io.h
+++ b/include/ruby/io.h
@@ -6,7 +6,7 @@
   $Date$
   created at: Fri Nov 12 16:47:09 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 
@@ -22,6 +22,7 @@ extern "C" {
 
 #include <stdio.h>
 #include <errno.h>
+#include "ruby/encoding.h"
 
 #if defined(HAVE_STDIO_EXT_H)
 #include <stdio_ext.h>
@@ -44,6 +45,7 @@ typedef struct rb_io_t {
     int rbuf_off;
     int rbuf_len;
     int rbuf_capa;
+    rb_encoding *enc;
 } rb_io_t;
 
 #define HAVE_RB_IO_T 1
diff --git a/include/ruby/node.h b/include/ruby/node.h
index e3722e0c41..4d6d73c7fd 100644
--- a/include/ruby/node.h
+++ b/include/ruby/node.h
@@ -6,7 +6,7 @@
   $Date$
   created at: Fri May 28 15:14:02 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 
diff --git a/include/ruby/re.h b/include/ruby/re.h
index ae6e0357b6..5b0cc24e9a 100644
--- a/include/ruby/re.h
+++ b/include/ruby/re.h
@@ -6,7 +6,7 @@
   $Date$
   created at: Thu Sep 30 14:18:32 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 
diff --git a/include/ruby/regex.h b/include/ruby/regex.h
index 118c37c480..ad736775fe 100644
--- a/include/ruby/regex.h
+++ b/include/ruby/regex.h
@@ -5,7 +5,7 @@
   $Author$
   $Date$
 
-  Copyright (C) 1993-2005 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 
@@ -29,10 +29,8 @@ extern "C" {
 
 ONIG_EXTERN OnigEncoding    OnigEncDefaultCharEncoding;
 
-#undef ismbchar
-#define ismbchar(c) (mbclen((c)) != 1)
-#define mbclen(c)  \
-  ONIGENC_MBC_ENC_LEN(OnigEncDefaultCharEncoding, (UChar* )(&c))
+#define ismbchar(p, enc) (mbclen((p),(enc)) != 1)
+#define mbclen(p,enc)  rb_enc_mbclen((p), (enc))
 
 #endif /* ifndef ONIG_RUBY_M17N */
 
diff --git a/include/ruby/ruby.h b/include/ruby/ruby.h
index ea3265505b..2dbf29956e 100644
--- a/include/ruby/ruby.h
+++ b/include/ruby/ruby.h
@@ -5,7 +5,7 @@
   $Author$
   created at: Thu Jun 10 14:26:32 JST 1993
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
   Copyright (C) 2000  Network Applied Communication Laboratory, Inc.
   Copyright (C) 2000  Information-technology Promotion Agency, Japan
 
@@ -455,6 +455,7 @@ struct RString {
     (!(RBASIC(str)->flags & RSTRING_NOEMBED) ? \
      RSTRING(str)->as.ary : \
      RSTRING(str)->as.heap.ptr)
+#define RSTRING_END(str) (RSTRING_PTR(str)+RSTRING_LEN(str))
 
 struct RArray {
     struct RBasic basic;
@@ -598,6 +599,32 @@ enum ruby_value_flags {
 #define FL_USER6     RUBY_FL_USER6
     RUBY_FL_USER7     = (1<<(FL_USHIFT+7)),
 #define FL_USER7     RUBY_FL_USER7
+    RUBY_FL_USER8     = (1<<(FL_USHIFT+8)),
+#define FL_USER8     RUBY_FL_USER8
+    RUBY_FL_USER9     = (1<<(FL_USHIFT+9)),
+#define FL_USER9     RUBY_FL_USER9
+    RUBY_FL_USER10     = (1<<(FL_USHIFT+10)),
+#define FL_USER10     RUBY_FL_USER10
+    RUBY_FL_USER11     = (1<<(FL_USHIFT+11)),
+#define FL_USER11     RUBY_FL_USER11
+    RUBY_FL_USER12     = (1<<(FL_USHIFT+12)),
+#define FL_USER12     RUBY_FL_USER12
+    RUBY_FL_USER13     = (1<<(FL_USHIFT+13)),
+#define FL_USER13     RUBY_FL_USER13
+    RUBY_FL_USER14     = (1<<(FL_USHIFT+14)),
+#define FL_USER14     RUBY_FL_USER14
+    RUBY_FL_USER15     = (1<<(FL_USHIFT+15)),
+#define FL_USER15     RUBY_FL_USER15
+    RUBY_FL_USER16     = (1<<(FL_USHIFT+16)),
+#define FL_USER16     RUBY_FL_USER16
+    RUBY_FL_USER17     = (1<<(FL_USHIFT+17)),
+#define FL_USER17     RUBY_FL_USER17
+    RUBY_FL_USER18     = (1<<(FL_USHIFT+18)),
+#define FL_USER18     RUBY_FL_USER18
+    RUBY_FL_USER19     = (1<<(FL_USHIFT+19)),
+#define FL_USER19     RUBY_FL_USER19
+    RUBY_FL_USER20     = (1<<(FL_USHIFT+20)),
+#define FL_USER20     RUBY_FL_USER20
 };
 
 #define SPECIAL_CONST_P(x) (IMMEDIATE_P(x) || !RTEST(x))
@@ -667,6 +694,7 @@ void rb_gc_unregister_address(VALUE*);
 
 ID rb_intern(const char*);
 ID rb_intern2(const char*, long);
+ID rb_intern_str(VALUE str);
 const char *rb_id2name(ID);
 ID rb_to_id(VALUE);
 VALUE rb_id2str(ID);
diff --git a/include/ruby/signal.h b/include/ruby/signal.h
index 29ffcd9f11..23db123d92 100644
--- a/include/ruby/signal.h
+++ b/include/ruby/signal.h
@@ -6,7 +6,7 @@
   $Date$
   created at: Wed Aug 16 01:15:38 JST 1995
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/
 
diff --git a/include/ruby/util.h b/include/ruby/util.h
index 8437872479..f9ce983269 100644
--- a/include/ruby/util.h
+++ b/include/ruby/util.h
@@ -6,7 +6,7 @@
   $Date$
   created at: Thu Mar  9 11:55:53 JST 1995
 
-  Copyright (C) 1993-2003 Yukihiro Matsumoto
+  Copyright (C) 1993-2007 Yukihiro Matsumoto
 
 **********************************************************************/