marshal.c: fix infinite recursion

* marshal.c (check_userdump_arg): marshal_dump should not return
  an instance of the same class, otherwise it causes infinite
  recursion.  [ruby-core:78289] [Bug #12974]

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@56894 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

2 files changed, 27 insertions, 1 deletions

diff --git a/marshal.c b/marshal.c
index 152b29671a..a9926acf56 100644
--- a/marshal.c
+++ b/marshal.c
@@ -178,8 +178,22 @@ check_dump_arg(VALUE ret, struct dump_arg *arg, const char *name)
     }
     return ret;
 }
+
+static VALUE
+check_userdump_arg(VALUE obj, ID sym, int argc, const VALUE *argv,
+		   struct dump_arg *arg, const char *name)
+{
+    VALUE ret = rb_funcallv(obj, sym, argc, argv);
+    VALUE klass = CLASS_OF(obj);
+    if (CLASS_OF(ret) == klass) {
+        rb_raise(rb_eRuntimeError, "%"PRIsVALUE"#%s returned same class instance",
+		 klass, name);
+    }
+    return check_dump_arg(ret, arg, name);
+}
+
 #define dump_funcall(arg, obj, sym, argc, argv) \
-    check_dump_arg(rb_funcallv(obj, sym, argc, argv), arg, name_##sym)
+    check_userdump_arg(obj, sym, argc, argv, arg, name_##sym)
 #define dump_check_funcall(arg, obj, sym, argc, argv) \
     check_dump_arg(rb_check_funcall(obj, sym, argc, argv), arg, name_##sym)
 
diff --git a/test/ruby/test_marshal.rb b/test/ruby/test_marshal.rb
index d21ab66ffd..bc22b5fd3a 100644
--- a/test/ruby/test_marshal.rb
+++ b/test/ruby/test_marshal.rb
@@ -757,4 +757,16 @@ class TestMarshal < Test::Unit::TestCase
     end
     assert_equal(obj, Marshal.load(dump))
   end
+
+  class Bug12974
+    def marshal_dump
+      dup
+    end
+  end
+
+  def test_marshal_dump_recursion
+    assert_raise_with_message(RuntimeError, /same class instance/) do
+      Marshal.dump(Bug12974.new)
+    end
+  end
 end