* encoding.c (rb_enc_mbclen): make it never fail.

  (rb_enc_nth): don't check the return value of rb_enc_mbclen.
  (rb_enc_strlen): ditto.
  (rb_enc_precise_mbclen): return needmore(1) if e <= p.
  (rb_enc_get_ascii): new function for extracting ASCII character.

* include/ruby/encoding.h (rb_enc_get_ascii): declared.

* include/ruby/regex.h (ismbchar): removed.

* re.c (rb_reg_expr_str): use rb_enc_get_ascii.
  (unescape_escaped_nonascii): use rb_enc_precise_mbclen to determine
  the termination of escaped non-ASCII character.
  (unescape_nonascii): use rb_enc_precise_mbclen.
  (rb_reg_quote): use rb_enc_get_ascii.
  (rb_reg_regsub): use rb_enc_get_ascii.

* string.c (rb_str_reverse) don't check the return value of
  rb_enc_mbclen.
  (rb_str_split_m): don't call rb_enc_mbclen with e <= p.

* parse.y (is_identchar): use ISASCII.
  (parser_ismbchar): removed.
  (parser_precise_mbclen): new macro.
  (parser_isascii): new macro.
  (parser_tokadd_mbchar): use parser_precise_mbclen to check invalid
  character precisely.
  (parser_tokadd_string): use parser_isascii.
  (parser_yylex): ditto.
  (is_special_global_name): don't call is_identchar with e <= p.
  (rb_enc_symname_p): ditto.

  [ruby-dev:32455]

* ext/tk/sample/tkextlib/vu/canvSticker2.rb: remove coding cookie
  because the encoding is not UTF-8.  [ruby-dev:32475]


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@14131 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 12 insertions, 10 deletions

diff --git a/parse.y b/parse.y
index 78cae5a367..0becb9280d 100644
--- a/parse.y
+++ b/parse.y
@@ -4583,10 +4583,12 @@ ripper_dispatch_delayed_token(struct parser_params *parser, int t)
 #endif
 
 #define parser_mbclen()  mbclen((lex_p-1),lex_pend,parser->enc)
-#define is_identchar(p,e,enc) (rb_enc_isalnum(*p,enc) || (*p) == '_' || ismbchar(p,e,enc))
-#define parser_ismbchar() ismbchar((lex_p-1), lex_pend, parser->enc)
+#define parser_precise_mbclen()  rb_enc_precise_mbclen((lex_p-1),lex_pend,parser->enc)
+#define is_identchar(p,e,enc) (rb_enc_isalnum(*p,enc) || (*p) == '_' || !ISASCII(*p))
 #define parser_is_identchar() (!parser->eofp && is_identchar((lex_p-1),lex_pend,parser->enc))
 
+#define parser_isascii() ISASCII(*(lex_p-1))
+
 static int
 parser_yyerror(struct parser_params *parser, const char *msg)
 {
@@ -5305,8 +5307,8 @@ dispose_string(VALUE str)
 static int
 parser_tokadd_mbchar(struct parser_params *parser, int c)
 {
-    int len = parser_mbclen();
-    if (len <= 0 || lex_p + len - 1 > lex_pend) {
+    int len = parser_precise_mbclen();
+    if (!MBCLEN_CHARFOUND(len)) {
 	compile_error(PARSER_ARG "illegal multibyte char");
 	return -1;
     }
@@ -5414,7 +5416,7 @@ parser_tokadd_string(struct parser_params *parser,
 		}
 	    }
 	}
-	else if (parser_ismbchar()) {
+	else if (!parser_isascii()) {
 	    has_nonascii = 1;
 	    if (enc != *encp) {
 		mixed_error(enc, *encp);
@@ -6306,7 +6308,7 @@ parser_yylex(struct parser_params *parser)
 	}
 	newtok();
 	enc = parser->enc;
-	if (parser_ismbchar()) {
+	if (!parser_isascii()) {
 	    if (tokadd_mbchar(c) == -1) return 0;
 	}
 	else if ((rb_enc_isalnum(c, parser->enc) || c == '_') &&
@@ -6889,7 +6891,7 @@ parser_yylex(struct parser_params *parser)
 	    }
 	    else {
 		term = nextc();
-		if (rb_enc_isalnum(term, parser->enc) || parser_ismbchar()) {
+		if (rb_enc_isalnum(term, parser->enc) || !parser_isascii()) {
 		    yyerror("unknown type of %string");
 		    return 0;
 		}
@@ -8693,7 +8695,7 @@ is_special_global_name(const char *m, const char *e, rb_encoding *enc)
 	break;
       case '-':
 	++m;
-	if (is_identchar(m, e, enc)) {
+	if (m < e && is_identchar(m, e, enc)) {
 	    if (!ISASCII(*m)) mb = 1;
 	    m += rb_enc_mbclen(m, e, enc);
 	}
@@ -8776,9 +8778,9 @@ rb_enc_symname_p(const char *name, rb_encoding *enc)
       default:
 	localid = !rb_enc_isupper(*m, enc);
       id:
-	if (*m != '_' && !rb_enc_isalpha(*m, enc) && !ismbchar(m, e, enc))
+	if (m >= e || (*m != '_' && !rb_enc_isalpha(*m, enc) && ISASCII(*m)))
 		  return Qfalse;
-	while (is_identchar(m, e, enc)) m += rb_enc_mbclen(m, e, enc);
+	while (m < e && is_identchar(m, e, enc)) m += rb_enc_mbclen(m, e, enc);
 	if (localid) {
 	    switch (*m) {
 	      case '!': case '?': case '=': ++m;