marshal.c: fix infinite recursion

* marshal.c (check_userdump_arg): marshal_dump should not return
  an instance of the same class, otherwise it causes infinite
  recursion.  [ruby-core:78289] [Bug #12974]

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@56894 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 15 insertions, 1 deletions

diff --git a/marshal.c b/marshal.c
index 152b29671a..a9926acf56 100644
--- a/marshal.c
+++ b/marshal.c
@@ -178,8 +178,22 @@ check_dump_arg(VALUE ret, struct dump_arg *arg, const char *name)
     }
     return ret;
 }
+
+static VALUE
+check_userdump_arg(VALUE obj, ID sym, int argc, const VALUE *argv,
+		   struct dump_arg *arg, const char *name)
+{
+    VALUE ret = rb_funcallv(obj, sym, argc, argv);
+    VALUE klass = CLASS_OF(obj);
+    if (CLASS_OF(ret) == klass) {
+        rb_raise(rb_eRuntimeError, "%"PRIsVALUE"#%s returned same class instance",
+		 klass, name);
+    }
+    return check_dump_arg(ret, arg, name);
+}
+
 #define dump_funcall(arg, obj, sym, argc, argv) \
-    check_dump_arg(rb_funcallv(obj, sym, argc, argv), arg, name_##sym)
+    check_userdump_arg(obj, sym, argc, argv, arg, name_##sym)
 #define dump_check_funcall(arg, obj, sym, argc, argv) \
     check_dump_arg(rb_check_funcall(obj, sym, argc, argv), arg, name_##sym)