parse.y, vm_eval.c: file encoding in eval

* parse.y (yycompile): store file name as String to keep the encoding.

* parse.y (rb_parser_compile_string_path, rb_parser_compile_file_path):
  new functions to pass file name as a String.

* parse.y (gettable_gen): return a copy of the original file name, not
  a copy in filesystem encoding.

* vm_eval.c (eval_string_with_cref): use Qundef instead of "(eval)".

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@42230 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

5 files changed, 76 insertions, 63 deletions

diff --git a/iseq.c b/iseq.c
index 5d8d7ada1f..198295a479 100644
--- a/iseq.c
+++ b/iseq.c
@@ -574,18 +574,6 @@ rb_iseq_load(VALUE data, VALUE parent, VALUE opt)
     return iseq_load(rb_cISeq, data, parent, opt);
 }
 
-static NODE *
-parse_string(VALUE str, const char *file, int line)
-{
-    VALUE parser = rb_parser_new();
-    NODE *node = rb_parser_compile_string(parser, file, str, line);
-
-    if (!node) {
-	rb_exc_raise(GET_THREAD()->errinfo);	/* TODO: check err */
-    }
-    return node;
-}
-
 VALUE
 rb_iseq_compile_with_option(VALUE src, VALUE file, VALUE absolute_path, VALUE line, rb_block_t *base_block, VALUE opt)
 {
@@ -598,17 +586,25 @@ rb_iseq_compile_with_option(VALUE src, VALUE file, VALUE absolute_path, VALUE li
 
     TH_PUSH_TAG(th);
     if ((state = EXEC_TAG()) == 0) {
+	VALUE parser;
 	int ln = NUM2INT(line);
-	const char *fn = StringValueCStr(file);
 	NODE *node;
 	rb_compile_option_t option;
 
+	StringValueCStr(file);
 	make_compile_option(&option, opt);
 
+	parser = rb_parser_new();
+
 	if (RB_TYPE_P((src), T_FILE))
-	    node = rb_compile_file(fn, src, ln);
-	else
-	    node = parse_string(StringValue(src), fn, ln);
+	    node = rb_parser_compile_file_path(parser, file, src, ln);
+	else {
+	    node = rb_parser_compile_string_path(parser, file, src, ln);
+
+	    if (!node) {
+		rb_exc_raise(GET_THREAD()->errinfo);	/* TODO: check err */
+	    }
+	}
 
 	if (base_block && base_block->iseq) {
 	    iseqval = rb_iseq_new_with_opt(node, base_block->iseq->location.label,
diff --git a/node.h b/node.h
index 57b1bc775a..97c3214d87 100644
--- a/node.h
+++ b/node.h
@@ -486,6 +486,8 @@ NODE *rb_parser_while_loop(VALUE, NODE *, int, int);
 NODE *rb_parser_compile_cstr(volatile VALUE, const char*, const char*, int, int);
 NODE *rb_parser_compile_string(volatile VALUE, const char*, VALUE, int);
 NODE *rb_parser_compile_file(volatile VALUE, const char*, VALUE, int);
+NODE *rb_parser_compile_string_path(volatile VALUE vparser, VALUE fname, VALUE src, int line);
+NODE *rb_parser_compile_file_path(volatile VALUE vparser, VALUE fname, VALUE input, int line);
 
 NODE *rb_compile_cstr(const char*, const char*, int, int);
 NODE *rb_compile_string(const char*, VALUE, int);
diff --git a/parse.y b/parse.y
index d4e73e0072..7dc4b2e9e4 100644
--- a/parse.y
+++ b/parse.y
@@ -265,6 +265,7 @@ struct parser_params {
     int has_shebang;
     char *parser_ruby_sourcefile; /* current source file */
     int parser_ruby_sourceline;	/* current line no. */
+    VALUE parser_ruby_sourcefile_string;
     rb_encoding *enc;
 
     int parser_yydebug;
@@ -281,7 +282,6 @@ struct parser_params {
     token_info *parser_token_info;
 #else
     /* Ripper only */
-    VALUE parser_ruby_sourcefile_string;
     const char *tokp;
     VALUE delayed;
     int delayed_line;
@@ -338,6 +338,7 @@ static int parser_yyerror(struct parser_params*, const char*);
 #define ruby__end__seen		(parser->parser_ruby__end__seen)
 #define ruby_sourceline		(parser->parser_ruby_sourceline)
 #define ruby_sourcefile		(parser->parser_ruby_sourcefile)
+#define ruby_sourcefile_string	(parser->parser_ruby_sourcefile_string)
 #define current_enc		(parser->enc)
 #define yydebug			(parser->parser_yydebug)
 #ifdef RIPPER
@@ -5272,14 +5273,13 @@ static void parser_prepare(struct parser_params *parser);
 
 #ifndef RIPPER
 static VALUE
-debug_lines(const char *f)
+debug_lines(VALUE fname)
 {
     ID script_lines;
     CONST_ID(script_lines, "SCRIPT_LINES__");
     if (rb_const_defined_at(rb_cObject, script_lines)) {
 	VALUE hash = rb_const_get_at(rb_cObject, script_lines);
 	if (RB_TYPE_P(hash, T_HASH)) {
-	    VALUE fname = rb_external_str_new_with_enc(f, strlen(f), rb_filesystem_encoding());
 	    VALUE lines = rb_ary_new();
 	    rb_hash_aset(hash, fname, lines);
 	    return lines;
@@ -5289,11 +5289,10 @@ debug_lines(const char *f)
 }
 
 static VALUE
-coverage(const char *f, int n)
+coverage(VALUE fname, int n)
 {
     VALUE coverages = rb_get_coverages();
     if (RTEST(coverages) && RBASIC(coverages)->klass == 0) {
-	VALUE fname = rb_external_str_new_with_enc(f, strlen(f), rb_filesystem_encoding());
 	VALUE lines = rb_ary_new2(n);
 	int i;
 	RBASIC_CLEAR_CLASS(lines);
@@ -5319,7 +5318,7 @@ yycompile0(VALUE arg)
     struct parser_params *parser = (struct parser_params *)arg;
 
     if (!compile_for_eval && rb_safe_level() == 0) {
-	ruby_debug_lines = debug_lines(ruby_sourcefile);
+	ruby_debug_lines = debug_lines(ruby_sourcefile_string);
 	if (ruby_debug_lines && ruby_sourceline > 0) {
 	    VALUE str = STR_NEW0();
 	    n = ruby_sourceline;
@@ -5329,7 +5328,7 @@ yycompile0(VALUE arg)
 	}
 
 	if (!e_option_supplied(parser)) {
-	    ruby_coverage = coverage(ruby_sourcefile, ruby_sourceline);
+	    ruby_coverage = coverage(ruby_sourcefile_string, ruby_sourceline);
 	}
     }
 
@@ -5372,9 +5371,10 @@ yycompile0(VALUE arg)
 }
 
 static NODE*
-yycompile(struct parser_params *parser, const char *f, int line)
+yycompile(struct parser_params *parser, VALUE fname, int line)
 {
-    ruby_sourcefile = ruby_strdup(f);
+    ruby_sourcefile_string = rb_str_new_frozen(fname);
+    ruby_sourcefile = RSTRING_PTR(fname);
     ruby_sourceline = line - 1;
     return (NODE *)rb_suppress_tracing(yycompile0, (VALUE)parser);
 }
@@ -5434,7 +5434,7 @@ static rb_data_type_t parser_data_type;
 static const rb_data_type_t parser_data_type;
 
 static NODE*
-parser_compile_string(volatile VALUE vparser, const char *f, VALUE s, int line)
+parser_compile_string(volatile VALUE vparser, VALUE fname, VALUE s, int line)
 {
     struct parser_params *parser;
     NODE *node;
@@ -5446,7 +5446,7 @@ parser_compile_string(volatile VALUE vparser, const char *f, VALUE s, int line)
     lex_pbeg = lex_p = lex_pend = 0;
     compile_for_eval = rb_parse_in_eval();
 
-    node = yycompile(parser, f, line);
+    node = yycompile(parser, fname, line);
     RB_GC_GUARD(vparser); /* prohibit tail call optimization */
 
     return node;
@@ -5456,12 +5456,18 @@ NODE*
 rb_compile_string(const char *f, VALUE s, int line)
 {
     must_be_ascii_compatible(s);
-    return parser_compile_string(rb_parser_new(), f, s, line);
+    return parser_compile_string(rb_parser_new(), rb_filesystem_str_new_cstr(f), s, line);
 }
 
 NODE*
 rb_parser_compile_string(volatile VALUE vparser, const char *f, VALUE s, int line)
 {
+    return rb_parser_compile_string_path(vparser, rb_filesystem_str_new_cstr(f), s, line);
+}
+
+NODE*
+rb_parser_compile_string_path(volatile VALUE vparser, VALUE f, VALUE s, int line)
+{
     must_be_ascii_compatible(s);
     return parser_compile_string(vparser, f, s, line);
 }
@@ -5470,14 +5476,14 @@ NODE*
 rb_compile_cstr(const char *f, const char *s, int len, int line)
 {
     VALUE str = rb_str_new(s, len);
-    return parser_compile_string(rb_parser_new(), f, str, line);
+    return parser_compile_string(rb_parser_new(), rb_filesystem_str_new_cstr(f), str, line);
 }
 
 NODE*
 rb_parser_compile_cstr(volatile VALUE vparser, const char *f, const char *s, int len, int line)
 {
     VALUE str = rb_str_new(s, len);
-    return parser_compile_string(vparser, f, str, line);
+    return parser_compile_string(vparser, rb_filesystem_str_new_cstr(f), str, line);
 }
 
 static VALUE
@@ -5497,6 +5503,12 @@ rb_compile_file(const char *f, VALUE file, int start)
 NODE*
 rb_parser_compile_file(volatile VALUE vparser, const char *f, VALUE file, int start)
 {
+    return rb_parser_compile_file_path(vparser, rb_filesystem_str_new_cstr(f), file, start);
+}
+
+NODE*
+rb_parser_compile_file_path(volatile VALUE vparser, VALUE fname, VALUE file, int start)
+{
     struct parser_params *parser;
     NODE *node;
 
@@ -5506,7 +5518,7 @@ rb_parser_compile_file(volatile VALUE vparser, const char *f, VALUE file, int st
     lex_pbeg = lex_p = lex_pend = 0;
     compile_for_eval = rb_parse_in_eval();
 
-    node = yycompile(parser, f, start);
+    node = yycompile(parser, fname, start);
     RB_GC_GUARD(vparser); /* prohibit tail call optimization */
 
     return node;
@@ -8480,8 +8492,7 @@ gettable_gen(struct parser_params *parser, ID id)
       case keyword_false:
 	return NEW_FALSE();
       case keyword__FILE__:
-	return NEW_STR(rb_external_str_new_with_enc(ruby_sourcefile, strlen(ruby_sourcefile),
-						    rb_filesystem_encoding()));
+	return NEW_STR(rb_str_dup(ruby_sourcefile_string));
       case keyword__LINE__:
 	return NEW_LIT(INT2FIX(tokline));
       case keyword__ENCODING__:
@@ -10694,13 +10705,13 @@ parser_initialize(struct parser_params *parser)
     parser->parser_lvtbl = 0;
     parser->parser_ruby__end__seen = 0;
     parser->parser_ruby_sourcefile = 0;
+    parser->parser_ruby_sourcefile_string = Qnil;
 #ifndef RIPPER
     parser->is_ripper = 0;
     parser->parser_eval_tree_begin = 0;
     parser->parser_eval_tree = 0;
 #else
     parser->is_ripper = 1;
-    parser->parser_ruby_sourcefile_string = Qnil;
     parser->delayed = Qnil;
 
     parser->result = Qnil;
@@ -10728,12 +10739,12 @@ parser_mark(void *ptr)
     rb_gc_mark(p->parser_lex_input);
     rb_gc_mark(p->parser_lex_lastline);
     rb_gc_mark(p->parser_lex_nextline);
+    rb_gc_mark(p->parser_ruby_sourcefile_string);
 #ifndef RIPPER
     rb_gc_mark((VALUE)p->parser_eval_tree_begin) ;
     rb_gc_mark((VALUE)p->parser_eval_tree) ;
     rb_gc_mark(p->debug_lines);
 #else
-    rb_gc_mark(p->parser_ruby_sourcefile_string);
     rb_gc_mark(p->delayed);
     rb_gc_mark(p->value);
     rb_gc_mark(p->result);
@@ -10758,9 +10769,6 @@ parser_free(void *ptr)
 	prev = local->prev;
 	xfree(local);
     }
-#ifndef RIPPER
-    xfree(p->parser_ruby_sourcefile);
-#endif
     xfree(p);
 }
 
@@ -10777,11 +10785,6 @@ parser_memsize(const void *ptr)
 	size += sizeof(*local);
 	if (local->vars) size += local->vars->capa * sizeof(ID);
     }
-#ifndef RIPPER
-    if (p->parser_ruby_sourcefile) {
-	size += strlen(p->parser_ruby_sourcefile) + 1;
-    }
-#endif
     return size;
 }
 
diff --git a/test/ruby/test_eval.rb b/test/ruby/test_eval.rb
index 790e2887d5..edbda38818 100644
--- a/test/ruby/test_eval.rb
+++ b/test/ruby/test_eval.rb
@@ -468,4 +468,9 @@ class TestEval < Test::Unit::TestCase
     result = foo.instance_eval(&foo_pr)
     assert_equal(1, result, 'Bug #3786, Bug #3860, [ruby-core:32501]')
   end
+
+  def test_file_encoding
+    fname = "\u{3042}".encode("euc-jp")
+    assert_equal(fname, eval("__FILE__", nil, fname, 1))
+  end
 end
diff --git a/vm_eval.c b/vm_eval.c
index b5748233f8..ce61849f2c 100644
--- a/vm_eval.c
+++ b/vm_eval.c
@@ -1176,7 +1176,7 @@ rb_each(VALUE obj)
 }
 
 static VALUE
-eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, const char *volatile file, volatile int line)
+eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, volatile VALUE file, volatile int line)
 {
     int state;
     VALUE result = Qundef;
@@ -1188,7 +1188,7 @@ eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, const char
     volatile int mild_compile_error;
 
     if (file == 0) {
-	file = rb_sourcefile();
+	file = rb_sourcefilename();
 	line = rb_sourceline();
     }
 
@@ -1200,16 +1200,17 @@ eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, const char
 	rb_iseq_t *iseq;
 	volatile VALUE iseqval;
 	VALUE absolute_path = Qnil;
+	VALUE fname;
 
 	if (scope != Qnil) {
 	    bind = Check_TypedStruct(scope, &ruby_binding_data_type);
 	    {
 		envval = bind->env;
-		if (strcmp(file, "(eval)") != 0) {
-		    absolute_path = rb_str_new_cstr(file);
+		if (file != Qundef) {
+		    absolute_path = file;
 		}
-		else if (bind->path != Qnil) {
-		    file = RSTRING_PTR(bind->path);
+		else if (!NIL_P(bind->path)) {
+		    file = bind->path;
 		    line = bind->first_lineno;
 		    absolute_path = rb_current_realfilepath();
 		}
@@ -1231,10 +1232,14 @@ eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, const char
 	    }
 	}
 
+	if ((fname = file) == Qundef) {
+	    fname = rb_usascii_str_new_cstr("(eval)");
+	}
+
 	/* make eval iseq */
 	th->parse_in_eval++;
 	th->mild_compile_error++;
-	iseqval = rb_iseq_compile_with_option(src, rb_str_new2(file), absolute_path, INT2FIX(line), base_block, Qnil);
+	iseqval = rb_iseq_compile_with_option(src, fname, absolute_path, INT2FIX(line), base_block, Qnil);
 	th->mild_compile_error--;
 	th->parse_in_eval--;
 
@@ -1262,7 +1267,7 @@ eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, const char
     if (state) {
 	if (state == TAG_RAISE) {
 	    VALUE errinfo = th->errinfo;
-	    if (strcmp(file, "(eval)") == 0) {
+	    if (file == Qundef) {
 		VALUE mesg, errat, bt2;
 		ID id_mesg;
 
@@ -1292,7 +1297,7 @@ eval_string_with_cref(VALUE self, VALUE src, VALUE scope, NODE *cref, const char
 }
 
 static VALUE
-eval_string(VALUE self, VALUE src, VALUE scope, const char *file, int line)
+eval_string(VALUE self, VALUE src, VALUE scope, VALUE file, int line)
 {
     return eval_string_with_cref(self, src, scope, 0, file, line);
 }
@@ -1319,7 +1324,7 @@ VALUE
 rb_f_eval(int argc, VALUE *argv, VALUE self)
 {
     VALUE src, scope, vfile, vline;
-    const char *file = "(eval)";
+    VALUE file = Qundef;
     int line = 1;
 
     rb_scan_args(argc, argv, "13", &src, &scope, &vfile, &vline);
@@ -1341,7 +1346,7 @@ rb_f_eval(int argc, VALUE *argv, VALUE self)
     }
 
     if (!NIL_P(vfile))
-	file = RSTRING_PTR(vfile);
+	file = vfile;
     return eval_string(self, src, scope, file, line);
 }
 
@@ -1349,27 +1354,28 @@ rb_f_eval(int argc, VALUE *argv, VALUE self)
 VALUE
 ruby_eval_string_from_file(const char *str, const char *filename)
 {
-    return eval_string(rb_vm_top_self(), rb_str_new2(str), Qnil, filename, 1);
+    VALUE file = filename ? rb_str_new_cstr(filename) : 0;
+    return eval_string(rb_vm_top_self(), rb_str_new2(str), Qnil, file, 1);
 }
 
 struct eval_string_from_file_arg {
-    const char *str;
-    const char *filename;
+    VALUE str;
+    VALUE filename;
 };
 
 static VALUE
 eval_string_from_file_helper(VALUE data)
 {
     const struct eval_string_from_file_arg *const arg = (struct eval_string_from_file_arg*)data;
-    return eval_string(rb_vm_top_self(), rb_str_new2(arg->str), Qnil, arg->filename, 1);
+    return eval_string(rb_vm_top_self(), arg->str, Qnil, arg->filename, 1);
 }
 
 VALUE
 ruby_eval_string_from_file_protect(const char *str, const char *filename, int *state)
 {
     struct eval_string_from_file_arg arg;
-    arg.str = str;
-    arg.filename = filename;
+    arg.str = rb_str_new_cstr(str);
+    arg.filename = filename ? rb_str_new_cstr(filename) : 0;
     return rb_protect(eval_string_from_file_helper, (VALUE)&arg, state);
 }
 
@@ -1529,7 +1535,7 @@ rb_yield_refine_block(VALUE refinement, VALUE refinements)
 
 /* string eval under the class/module context */
 static VALUE
-eval_under(VALUE under, VALUE self, VALUE src, const char *file, int line)
+eval_under(VALUE under, VALUE self, VALUE src, VALUE file, int line)
 {
     NODE *cref = vm_cref_push(GET_THREAD(), under, NOEX_PUBLIC, NULL);
 
@@ -1554,7 +1560,7 @@ specific_eval(int argc, VALUE *argv, VALUE klass, VALUE self)
 	return yield_under(klass, self, Qundef);
     }
     else {
-	const char *file = "(eval)";
+	VALUE file = Qundef;
 	int line = 1;
 
 	rb_check_arity(argc, 1, 3);
@@ -1567,7 +1573,8 @@ specific_eval(int argc, VALUE *argv, VALUE klass, VALUE self)
 	if (argc > 2)
 	    line = NUM2INT(argv[2]);
 	if (argc > 1) {
-	    file = StringValuePtr(argv[1]);
+	    file = argv[1];
+	    if (!NIL_P(file)) StringValue(file);
 	}
 	return eval_under(klass, self, argv[0], file, line);
     }