merge revision(s) 56894: [Backport #12974]

	marshal.c: fix infinite recursion

	* marshal.c (check_userdump_arg): marshal_dump should not return
  an instance of the same class, otherwise it causes infinite
  recursion.  [ruby-core:78289] [Bug #12974]

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/branches/ruby_2_2@57211 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 38 insertions, 33 deletions

diff --git a/marshal.c b/marshal.c
index fe286e8ed5..d322e3a5ba 100644
--- a/marshal.c
+++ b/marshal.c
@@ -165,15 +165,33 @@ struct dump_call_arg {
     int limit;
 };
 
-static void
-check_dump_arg(struct dump_arg *arg, const char *name)
+static VALUE
+check_dump_arg(VALUE ret, struct dump_arg *arg, const char *name)
 {
     if (!arg->symbols) {
         rb_raise(rb_eRuntimeError, "Marshal.dump reentered at %s",
 		 name);
     }
+    return ret;
+}
+
+static VALUE
+check_userdump_arg(VALUE obj, ID sym, int argc, const VALUE *argv,
+		   struct dump_arg *arg, const char *name)
+{
+    VALUE ret = rb_funcallv(obj, sym, argc, argv);
+    VALUE klass = CLASS_OF(obj);
+    if (CLASS_OF(ret) == klass) {
+        rb_raise(rb_eRuntimeError, "%"PRIsVALUE"#%s returned same class instance",
+		 klass, name);
+    }
+    return check_dump_arg(ret, arg, name);
 }
-#define check_dump_arg(arg, sym) check_dump_arg(arg, name_##sym)
+
+#define dump_funcall(arg, obj, sym, argc, argv) \
+    check_userdump_arg(obj, sym, argc, argv, arg, name_##sym)
+#define dump_check_funcall(arg, obj, sym, argc, argv) \
+    check_dump_arg(rb_check_funcall(obj, sym, argc, argv), arg, name_##sym)
 
 static void clear_dump_arg(struct dump_arg *arg);
 
@@ -706,8 +724,7 @@ w_object(VALUE obj, struct dump_arg *arg, int limit)
 	if (rb_obj_respond_to(obj, s_mdump, TRUE)) {
 	    st_add_direct(arg->data, obj, arg->data->num_entries);
 
-	    v = rb_funcall2(obj, s_mdump, 0, 0);
-	    check_dump_arg(arg, s_mdump);
+	    v = dump_funcall(arg, obj, s_mdump, 0, 0);
 	    w_class(TYPE_USRMARSHAL, obj, arg, FALSE);
 	    w_object(v, arg, limit);
 	    return;
@@ -718,8 +735,7 @@ w_object(VALUE obj, struct dump_arg *arg, int limit)
 	    VALUE encname2;
 
 	    v = INT2NUM(limit);
-	    v = rb_funcall2(obj, s_dump, 1, &v);
-	    check_dump_arg(arg, s_dump);
+	    v = dump_funcall(arg, obj, s_dump, 1, &v);
 	    if (!RB_TYPE_P(v, T_STRING)) {
 		rb_raise(rb_eTypeError, "_dump() must return string");
 	    }
@@ -901,8 +917,7 @@ w_object(VALUE obj, struct dump_arg *arg, int limit)
 			     "no _dump_data is defined for class %s",
 			     rb_obj_classname(obj));
 		}
-		v = rb_funcall2(obj, s_dump_data, 0, 0);
-		check_dump_arg(arg, s_dump_data);
+		v = dump_funcall(arg, obj, s_dump_data, 0, 0);
 		w_class(TYPE_DATA, obj, arg, TRUE);
 		w_object(v, arg, limit);
 	    }
@@ -1012,9 +1027,7 @@ marshal_dump(int argc, VALUE *argv)
 	    io_needed();
 	}
 	arg->dest = port;
-	if (rb_check_funcall(port, s_binmode, 0, 0) != Qundef) {
-	    check_dump_arg(arg, s_binmode);
-	}
+	dump_check_funcall(arg, port, s_binmode, 0, 0);
     }
     else {
 	port = arg->str;
@@ -1047,15 +1060,17 @@ struct load_arg {
     int infection;
 };
 
-static void
-check_load_arg(struct load_arg *arg, const char *name)
+static VALUE
+check_load_arg(VALUE ret, struct load_arg *arg, const char *name)
 {
     if (!arg->symbols) {
         rb_raise(rb_eRuntimeError, "Marshal.load reentered at %s",
 		 name);
     }
+    return ret;
 }
-#define check_load_arg(arg, sym) check_load_arg(arg, name_##sym)
+#define load_funcall(arg, obj, sym, argc, argv) \
+    check_load_arg(rb_funcallv(obj, sym, argc, argv), arg, name_##sym)
 
 static void clear_load_arg(struct load_arg *arg);
 
@@ -1118,9 +1133,7 @@ r_byte1_buffered(struct load_arg *arg)
 	long readable = arg->readable < BUFSIZ ? arg->readable : BUFSIZ;
 	VALUE str, n = LONG2NUM(readable);
 
-	str = rb_funcall2(arg->src, s_read, 1, &n);
-
-	check_load_arg(arg, s_read);
+	str = load_funcall(arg, arg->src, s_read, 1, &n);
 	if (NIL_P(str)) too_short();
 	StringValue(str);
 	arg->infection |= (int)FL_TEST(str, MARSHAL_INFECTION);
@@ -1150,8 +1163,7 @@ r_byte(struct load_arg *arg)
 	    c = r_byte1_buffered(arg);
 	}
 	else {
-	    VALUE v = rb_funcall2(arg->src, s_getbyte, 0, 0);
-	    check_load_arg(arg, s_getbyte);
+	    VALUE v = load_funcall(arg, arg->src, s_getbyte, 0, 0);
 	    if (NIL_P(v)) rb_eof_error();
 	    c = (unsigned char)NUM2CHR(v);
 	}
@@ -1212,8 +1224,7 @@ r_bytes1(long len, struct load_arg *arg)
 {
     VALUE str, n = LONG2NUM(len);
 
-    str = rb_funcall2(arg->src, s_read, 1, &n);
-    check_load_arg(arg, s_read);
+    str = load_funcall(arg, arg->src, s_read, 1, &n);
     if (NIL_P(str)) too_short();
     StringValue(str);
     if (RSTRING_LEN(str) != len) too_short();
@@ -1241,9 +1252,7 @@ r_bytes1_buffered(long len, struct load_arg *arg)
 	readable = readable < BUFSIZ ? readable : BUFSIZ;
 	read_len = need_len > readable ? need_len : readable;
 	n = LONG2NUM(read_len);
-	tmp = rb_funcall2(arg->src, s_read, 1, &n);
-
-	check_load_arg(arg, s_read);
+	tmp = load_funcall(arg, arg->src, s_read, 1, &n);
 	if (NIL_P(tmp)) too_short();
 	StringValue(tmp);
 
@@ -1428,8 +1437,7 @@ static VALUE
 r_post_proc(VALUE v, struct load_arg *arg)
 {
     if (arg->proc) {
-	v = rb_funcall(arg->proc, s_call, 1, v);
-	check_load_arg(arg, s_call);
+	v = load_funcall(arg, arg->proc, s_call, 1, &v);
     }
     return v;
 }
@@ -1833,8 +1841,7 @@ r_object0(struct load_arg *arg, int *ivp, VALUE extmod)
 		r_ivar(data, NULL, arg);
 		*ivp = FALSE;
 	    }
-	    v = rb_funcall2(klass, s_load, 1, &data);
-	    check_load_arg(arg, s_load);
+	    v = load_funcall(arg, klass, s_load, 1, &data);
 	    v = r_entry(v, arg);
 	    if (st_lookup(compat_allocator_tbl, (st_data_t)rb_get_alloc_func(klass), &d)) {
 		marshal_compat_t *compat = (marshal_compat_t*)d;
@@ -1861,8 +1868,7 @@ r_object0(struct load_arg *arg, int *ivp, VALUE extmod)
 	    }
 	    v = r_entry(v, arg);
 	    data = r_object(arg);
-	    rb_funcall2(v, s_mload, 1, &data);
-	    check_load_arg(arg, s_mload);
+	    load_funcall(arg, v, s_mload, 1, &data);
 	    v = r_fixup_compat(v, arg);
 	    v = r_copy_ivar(v, data);
 	    v = r_post_proc(v, arg);
@@ -1903,8 +1909,7 @@ r_object0(struct load_arg *arg, int *ivp, VALUE extmod)
 			 rb_class2name(klass));
 	    }
 	    r = r_object0(arg, 0, extmod);
-	    rb_funcall2(v, s_load_data, 1, &r);
-	    check_load_arg(arg, s_load_data);
+	    load_funcall(arg, v, s_load_data, 1, &r);
 	    v = r_leave(v, arg);
 	}
 	break;