merge revision(s) r42230,r42231: [Backport #9651]

	parse.y, vm_eval.c: file encoding in eval

	* parse.y (yycompile): store file name as String to keep the encoding.

	* parse.y (rb_parser_compile_string_path, rb_parser_compile_file_path):
  new functions to pass file name as a String.

	* parse.y (gettable_gen): return a copy of the original file name, not
  a copy in filesystem encoding.

	* vm_eval.c (eval_string_with_cref): use Qundef instead of "(eval)".
	* parse.y (yycompile): store file name as String to keep the encoding.

	* parse.y (rb_parser_compile_string_path, rb_parser_compile_file_path):
	  new functions to pass file name as a String.

	* parse.y (gettable_gen): return a copy of the original file name, not
	  a copy in filesystem encoding.

	* vm_eval.c (eval_string_with_cref): use Qundef instead of "(eval)".


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/branches/ruby_2_0_0@45473 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

7 files changed, 91 insertions, 66 deletions

diff --git a/ChangeLog b/ChangeLog
index f5909be542..d4c13f5582 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,15 @@
+Mon Mar 31 00:02:04 2014  Nobuyoshi Nakada  <nobu@ruby-lang.org>
+
+	* parse.y (yycompile): store file name as String to keep the encoding.
+
+	* parse.y (rb_parser_compile_string_path, rb_parser_compile_file_path):
+	  new functions to pass file name as a String.
+
+	* parse.y (gettable_gen): return a copy of the original file name, not
+	  a copy in filesystem encoding.
+
+	* vm_eval.c (eval_string_with_cref): use Qundef instead of "(eval)".
+
 Sun Mar 30 23:49:21 2014  Nobuyoshi Nakada  <nobu@ruby-lang.org>
 
 	* ext/openssl/ossl.c (ossl_make_error): check NULL for unknown
diff --git a/iseq.c b/iseq.c
index a52fd40d72..43e4c8273e 100644
--- a/iseq.c
+++ b/iseq.c
@@ -581,18 +581,6 @@ rb_iseq_load(VALUE data, VALUE parent, VALUE opt)
     return iseq_load(rb_cISeq, data, parent, opt);
 }
 
-static NODE *
-parse_string(VALUE str, const char *file, int line)
-{
-    VALUE parser = rb_parser_new();
-    NODE *node = rb_parser_compile_string(parser, file, str, line);
-
-    if (!node) {
-	rb_exc_raise(GET_THREAD()->errinfo);	/* TODO: check err */
-    }
-    return node;
-}
-
 VALUE
 rb_iseq_compile_with_option(VALUE src, VALUE file, VALUE absolute_path, VALUE line, rb_block_t *base_block, VALUE opt)
 {
@@ -605,17 +593,25 @@ rb_iseq_compile_with_option(VALUE src, VALUE file, VALUE absolute_path, VALUE li
 
     TH_PUSH_TAG(th);
     if ((state = EXEC_TAG()) == 0) {
+	VALUE parser;
 	int ln = NUM2INT(line);
-	const char *fn = StringValueCStr(file);
 	NODE *node;
 	rb_compile_option_t option;
 
+	StringValueCStr(file);
 	make_compile_option(&option, opt);
 
+	parser = rb_parser_new();
+
 	if (RB_TYPE_P((src), T_FILE))
-	    node = rb_compile_file(fn, src, ln);
-	else
-	    node = parse_string(StringValue(src), fn, ln);
+	    node = rb_parser_compile_file_path(parser, file, src, ln);
+	else {
+	    node = rb_parser_compile_string_path(parser, file, src, ln);
+
+	    if (!node) {
+		rb_exc_raise(GET_THREAD()->errinfo);	/* TODO: check err */
+	    }
+	}
 
 	if (base_block && base_block->iseq) {
 	    iseqval = rb_iseq_new_with_opt(node, base_block->iseq->location.label,
diff --git a/node.h b/node.h
index 0fa72545c2..d294fcc492 100644
--- a/node.h
+++ b/node.h
@@ -482,6 +482,8 @@ NODE *rb_parser_while_loop(VALUE, NODE *, int, int);
 NODE *rb_parser_compile_cstr(volatile VALUE, const char*, const char*, int, int);
 NODE *rb_parser_compile_string(volatile VALUE, const char*, VALUE, int);
 NODE *rb_parser_compile_file(volatile VALUE, const char*, VALUE, int);
+NODE *rb_parser_compile_string_path(volatile VALUE vparser, VALUE fname, VALUE src, int line);
+NODE *rb_parser_compile_file_path(volatile VALUE vparser, VALUE fname, VALUE input, int line);
 
 NODE *rb_compile_cstr(const char*, const char*, int, int);
 NODE *rb_compile_string(const char*, VALUE, int);
diff --git a/parse.y b/parse.y
index da512a6243..c7ac527210 100644
--- a/parse.y
+++ b/parse.y
@@ -268,6 +268,7 @@ struct parser_params {
     int has_shebang;
     char *parser_ruby_sourcefile; /* current source file */
     int parser_ruby_sourceline;	/* current line no. */
+    VALUE parser_ruby_sourcefile_string;
     rb_encoding *enc;
 
     int parser_yydebug;
@@ -284,7 +285,6 @@ struct parser_params {
     token_info *parser_token_info;
 #else
     /* Ripper only */
-    VALUE parser_ruby_sourcefile_string;
     const char *tokp;
     VALUE delayed;
     int delayed_line;
@@ -339,6 +339,7 @@ static int parser_yyerror(struct parser_params*, const char*);
 #define ruby__end__seen		(parser->parser_ruby__end__seen)
 #define ruby_sourceline		(parser->parser_ruby_sourceline)
 #define ruby_sourcefile		(parser->parser_ruby_sourcefile)
+#define ruby_sourcefile_string	(parser->parser_ruby_sourcefile_string)
 #define current_enc		(parser->enc)
 #define yydebug			(parser->parser_yydebug)
 #ifdef RIPPER
@@ -5262,14 +5263,13 @@ static void parser_prepare(struct parser_params *parser);
 
 #ifndef RIPPER
 static VALUE
-debug_lines(const char *f)
+debug_lines(VALUE fname)
 {
     ID script_lines;
     CONST_ID(script_lines, "SCRIPT_LINES__");
     if (rb_const_defined_at(rb_cObject, script_lines)) {
 	VALUE hash = rb_const_get_at(rb_cObject, script_lines);
 	if (RB_TYPE_P(hash, T_HASH)) {
-	    VALUE fname = rb_external_str_new_with_enc(f, strlen(f), rb_filesystem_encoding());
 	    VALUE lines = rb_ary_new();
 	    rb_hash_aset(hash, fname, lines);
 	    return lines;
@@ -5279,11 +5279,10 @@ debug_lines(const char *f)
 }
 
 static VALUE
-coverage(const char *f, int n)
+coverage(VALUE fname, int n)
 {
     VALUE coverages = rb_get_coverages();
     if (RTEST(coverages) && RBASIC(coverages)->klass == 0) {
-	VALUE fname = rb_external_str_new_with_enc(f, strlen(f), rb_filesystem_encoding());
 	VALUE lines = rb_ary_new2(n);
 	int i;
 	RBASIC(lines)->klass = 0;
@@ -5309,7 +5308,7 @@ yycompile0(VALUE arg)
     struct parser_params *parser = (struct parser_params *)arg;
 
     if (!compile_for_eval && rb_safe_level() == 0) {
-	ruby_debug_lines = debug_lines(ruby_sourcefile);
+	ruby_debug_lines = debug_lines(ruby_sourcefile_string);
 	if (ruby_debug_lines && ruby_sourceline > 0) {
 	    VALUE str = STR_NEW0();
 	    n = ruby_sourceline;
@@ -5319,7 +5318,7 @@ yycompile0(VALUE arg)
 	}
 
 	if (!e_option_supplied(parser)) {
-	    ruby_coverage = coverage(ruby_sourcefile, ruby_sourceline);
+	    ruby_coverage = coverage(ruby_sourcefile_string, ruby_sourceline);
 	}
     }
 
@@ -5362,9 +5361,10 @@ yycompile0(VALUE arg)
 }
 
 static NODE*
-yycompile(struct parser_params *parser, const char *f, int line)
+yycompile(struct parser_params *parser, VALUE fname, int line)
 {
-    ruby_sourcefile = ruby_strdup(f);
+    ruby_sourcefile_string = rb_str_new_frozen(fname);
+    ruby_sourcefile = RSTRING_PTR(fname);
     ruby_sourceline = line - 1;
     return (NODE *)rb_suppress_tracing(yycompile0, (VALUE)parser);
 }
@@ -5424,7 +5424,7 @@ static rb_data_type_t parser_data_type;
 static const rb_data_type_t parser_data_type;
 
 static NODE*
-parser_compile_string(volatile VALUE vparser, const char *f, VALUE s, int line)
+parser_compile_string(volatile VALUE vparser, VALUE fname, VALUE s, int line)
 {
     struct parser_params *parser;
     NODE *node;
@@ -5436,7 +5436,7 @@ parser_compile_string(volatile VALUE vparser, const char *f, VALUE s, int line)
     lex_pbeg = lex_p = lex_pend = 0;
     compile_for_eval = rb_parse_in_eval();
 
-    node = yycompile(parser, f, line);
+    node = yycompile(parser, fname, line);
     RB_GC_GUARD(vparser); /* prohibit tail call optimization */
 
     return node;
@@ -5446,12 +5446,18 @@ NODE*
 rb_compile_string(const char *f, VALUE s, int line)
 {
     must_be_ascii_compatible(s);
-    return parser_compile_string(rb_parser_new(), f, s, line);
+    return parser_compile_string(rb_parser_new(), rb_filesystem_str_new_cstr(f), s, line);
 }
 
 NODE*
 rb_parser_compile_string(volatile VALUE vparser, const char *f, VALUE s, int line)
 {
+    return rb_parser_compile_string_path(vparser, rb_filesystem_str_new_cstr(f), s, line);
+}
+
+NODE*
+rb_parser_compile_string_path(volatile VALUE vparser, VALUE f, VALUE s, int line)
+{
     must_be_ascii_compatible(s);
     return parser_compile_string(vparser, f, s, line);
 }
@@ -5460,14 +5466,14 @@ NODE*
 rb_compile_cstr(const char *f, const char *s, int len, int line)
 {
     VALUE str = rb_str_new(s, len);
-    return parser_compile_string(rb_parser_new(), f, str, line);
+    return parser_compile_string(rb_parser_new(), rb_filesystem_str_new_cstr(f), str, line);
 }
 
 NODE*
 rb_parser_compile_cstr(volatile VALUE vparser, const char *f, const char *s, int len, int line)
 {
     VALUE str = rb_str_new(s, len);
-    return parser_compile_string(vparser, f, str, line);
+    return parser_compile_string(vparser, rb_filesystem_str_new_cstr(f), str, line);
 }
 
 static VALUE
@@ -5487,6 +5493,12 @@ rb_compile_file(const char *f, VALUE file, int start)
 NODE*
 rb_parser_compile_file(volatile VALUE vparser, const char *f, VALUE file, int start)
 {
+    return rb_parser_compile_file_path(vparser, rb_filesystem_str_new_cstr(f), file, start);
+}
+
+NODE*
+rb_parser_compile_file_path(volatile VALUE vparser, VALUE fname, VALUE file, int start)
+{
     struct parser_params *parser;
     NODE *node;
 
@@ -5496,7 +5508,7 @@ rb_parser_compile_file(volatile VALUE vparser, const char *f, VALUE file, int st
     lex_pbeg = lex_p = lex_pend = 0;
     compile_for_eval = rb_parse_in_eval();
 
-    node = yycompile(parser, f, start);
+    node = yycompile(parser, fname, start);
     RB_GC_GUARD(vparser); /* prohibit tail call optimization */
 
     return node;
@@ -8467,8 +8479,7 @@ gettable_gen(struct parser_params *parser, ID id)
       case keyword_false:
 	return NEW_FALSE();
       case keyword__FILE__:
-	return NEW_STR(rb_external_str_new_with_enc(ruby_sourcefile, strlen(ruby_sourcefile),
-						    rb_filesystem_encoding()));
+	return NEW_STR(rb_str_dup(ruby_sourcefile_string));
       case keyword__LINE__:
 	return NEW_LIT(INT2FIX(tokline));
       case keyword__ENCODING__:
@@ -10722,13 +10733,13 @@ parser_initialize(struct parser_params *parser)
     parser->parser_lvtbl = 0;
     parser->parser_ruby__end__seen = 0;
     parser->parser_ruby_sourcefile = 0;
+    parser->parser_ruby_sourcefile_string = Qnil;
 #ifndef RIPPER
     parser->is_ripper = 0;
     parser->parser_eval_tree_begin = 0;
     parser->parser_eval_tree = 0;
 #else
     parser->is_ripper = 1;
-    parser->parser_ruby_sourcefile_string = Qnil;
     parser->delayed = Qnil;
 
     parser->result = Qnil;
@@ -10756,12 +10767,12 @@ parser_mark(void *ptr)
     rb_gc_mark(p->parser_lex_input);
     rb_gc_mark(p->parser_lex_lastline);
     rb_gc_mark(p->parser_lex_nextline);
+    rb_gc_mark(p->parser_ruby_sourcefile_string);
 #ifndef RIPPER
     rb_gc_mark((VALUE)p->parser_eval_tree_begin) ;
     rb_gc_mark((VALUE)p->parser_eval_tree) ;
     rb_gc_mark(p->debug_lines);
 #else
-    rb_gc_mark(p->parser_ruby_sourcefile_string);
     rb_gc_mark(p->delayed);
     rb_gc_mark(p->value);
     rb_gc_mark(p->result);
@@ -10786,9 +10797,6 @@ parser_free(void *ptr)
 	prev = local->prev;
 	xfree(local);
     }
-#ifndef RIPPER
-    xfree(p->parser_ruby_sourcefile);
-#endif
     xfree(p);
 }
 
@@ -10805,11 +10813,6 @@ parser_memsize(const void *ptr)
 	size += sizeof(*local);
 	if (local->vars) size += local->vars->capa * sizeof(ID);
     }
-#ifndef RIPPER
-    if (p->parser_ruby_sourcefile) {
-	size += strlen(p->parser_ruby_sourcefile) + 1;
-    }
-#endif
     return size;
 }
 
diff --git a/test/ruby/test_eval.rb b/test/ruby/test_eval.rb
index a97b982246..8e7416b333 100644
--- a/test/ruby/test_eval.rb
+++ b/test/ruby/test_eval.rb
@@ -478,4 +478,9 @@ class TestEval < Test::Unit::TestCase
     result = foo.instance_eval(&foo_pr)
     assert_equal(1, result, 'Bug #3786, Bug #3860, [ruby-core:32501]')
   end
+
+  def test_file_encoding
+    fname = "\u{3042}".encode("euc-jp")
+    assert_equal(fname, eval("__FILE__", nil, fname, 1))
+  end
 end
diff --git a/version.h b/version.h
index db47584d4e..fb3fb916f1 100644
--- a/version.h
+++ b/version.h
@@ -1,10 +1,10 @@
 #define RUBY_VERSION "2.0.0"
-#define RUBY_RELEASE_DATE "2014-03-30"
-#define RUBY_PATCHLEVEL 463
+#define RUBY_RELEASE_DATE "2014-03-31"
+#define RUBY_PATCHLEVEL 464
 
 #define RUBY_RELEASE_YEAR 2014
 #define RUBY_RELEASE_MONTH 3
-#define RUBY_RELEASE_DAY 30
+#define RUBY_RELEASE_DAY 31
 
 #include "ruby/version.h"
 
diff --git a/vm_eval.c b/vm_eval.c
index 7644aff9c1..6095e1d84c 100644
--- a/vm_eval.c
+++ b/vm_eval.c
@@ -1168,7 +1168,7 @@ rb_each(VALUE obj)
 }
 
 static VALUE
-eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, const char *volatile file, volatile int line)
+eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, volatile VALUE file, volatile int line)
 {
     int state;
     VALUE result = Qundef;
@@ -1180,7 +1180,7 @@ eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, const char
     volatile int mild_compile_error;
 
     if (file == 0) {
-	file = rb_sourcefile();
+	file = rb_sourcefilename();
 	line = rb_sourceline();
     }
 
@@ -1192,16 +1192,17 @@ eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, const char
 	rb_iseq_t *iseq;
 	volatile VALUE iseqval;
 	VALUE absolute_path = Qnil;
+	VALUE fname;
 
 	if (scope != Qnil) {
 	    if (rb_obj_is_kind_of(scope, rb_cBinding)) {
 		GetBindingPtr(scope, bind);
 		envval = bind->env;
-		if (strcmp(file, "(eval)") != 0) {
-		    absolute_path = rb_str_new_cstr(file);
+		if (file != Qundef) {
+		    absolute_path = file;
 		}
-		else if (bind->path != Qnil) {
-		    file = RSTRING_PTR(bind->path);
+		else if (!NIL_P(bind->path)) {
+		    file = bind->path;
 		    line = bind->first_lineno;
 		    absolute_path = rb_current_realfilepath();
 		}
@@ -1228,10 +1229,14 @@ eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, const char
 	    }
 	}
 
+	if ((fname = file) == Qundef) {
+	    fname = rb_usascii_str_new_cstr("(eval)");
+	}
+
 	/* make eval iseq */
 	th->parse_in_eval++;
 	th->mild_compile_error++;
-	iseqval = rb_iseq_compile_with_option(src, rb_str_new2(file), absolute_path, INT2FIX(line), base_block, Qnil);
+	iseqval = rb_iseq_compile_with_option(src, fname, absolute_path, INT2FIX(line), base_block, Qnil);
 	th->mild_compile_error--;
 	th->parse_in_eval--;
 
@@ -1259,7 +1264,7 @@ eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, const char
     if (state) {
 	if (state == TAG_RAISE) {
 	    VALUE errinfo = th->errinfo;
-	    if (strcmp(file, "(eval)") == 0) {
+	    if (file == Qundef) {
 		VALUE mesg, errat, bt2;
 		ID id_mesg;
 
@@ -1289,7 +1294,7 @@ eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, const char
 }
 
 static VALUE
-eval_string(VALUE self, VALUE src, VALUE scope, const char *file, int line)
+eval_string(VALUE self, VALUE src, VALUE scope, VALUE file, int line)
 {
     return eval_string_with_cref(self, src, scope, 0, file, line);
 }
@@ -1316,7 +1321,7 @@ VALUE
 rb_f_eval(int argc, VALUE *argv, VALUE self)
 {
     VALUE src, scope, vfile, vline;
-    const char *file = "(eval)";
+    VALUE file = Qundef;
     int line = 1;
 
     rb_scan_args(argc, argv, "13", &src, &scope, &vfile, &vline);
@@ -1338,7 +1343,7 @@ rb_f_eval(int argc, VALUE *argv, VALUE self)
     }
 
     if (!NIL_P(vfile))
-	file = RSTRING_PTR(vfile);
+	file = vfile;
     return eval_string(self, src, scope, file, line);
 }
 
@@ -1346,27 +1351,28 @@ rb_f_eval(int argc, VALUE *argv, VALUE self)
 VALUE
 ruby_eval_string_from_file(const char *str, const char *filename)
 {
-    return eval_string(rb_vm_top_self(), rb_str_new2(str), Qnil, filename, 1);
+    VALUE file = filename ? rb_str_new_cstr(filename) : 0;
+    return eval_string(rb_vm_top_self(), rb_str_new2(str), Qnil, file, 1);
 }
 
 struct eval_string_from_file_arg {
-    const char *str;
-    const char *filename;
+    VALUE str;
+    VALUE filename;
 };
 
 static VALUE
 eval_string_from_file_helper(void *data)
 {
     const struct eval_string_from_file_arg *const arg = (struct eval_string_from_file_arg*)data;
-    return eval_string(rb_vm_top_self(), rb_str_new2(arg->str), Qnil, arg->filename, 1);
+    return eval_string(rb_vm_top_self(), arg->str, Qnil, arg->filename, 1);
 }
 
 VALUE
 ruby_eval_string_from_file_protect(const char *str, const char *filename, int *state)
 {
     struct eval_string_from_file_arg arg;
-    arg.str = str;
-    arg.filename = filename;
+    arg.str = rb_str_new_cstr(str);
+    arg.filename = filename ? rb_str_new_cstr(filename) : 0;
     return rb_protect((VALUE (*)(VALUE))eval_string_from_file_helper, (VALUE)&arg, state);
 }
 
@@ -1526,7 +1532,7 @@ rb_yield_refine_block(VALUE refinement, VALUE refinements)
 
 /* string eval under the class/module context */
 static VALUE
-eval_under(VALUE under, VALUE self, VALUE src, const char *file, int line)
+eval_under(VALUE under, VALUE self, VALUE src, VALUE file, int line)
 {
     NODE *cref = vm_cref_push(GET_THREAD(), under, NOEX_PUBLIC, NULL);
 
@@ -1551,7 +1557,7 @@ specific_eval(int argc, VALUE *argv, VALUE klass, VALUE self)
 	return yield_under(klass, self, Qundef);
     }
     else {
-	const char *file = "(eval)";
+	VALUE file = Qundef;
 	int line = 1;
 
 	rb_check_arity(argc, 1, 3);
@@ -1564,7 +1570,8 @@ specific_eval(int argc, VALUE *argv, VALUE klass, VALUE self)
 	if (argc > 2)
 	    line = NUM2INT(argv[2]);
 	if (argc > 1) {
-	    file = StringValuePtr(argv[1]);
+	    file = argv[1];
+	    if (!NIL_P(file)) StringValue(file);
 	}
 	return eval_under(klass, self, argv[0], file, line);
     }