Do not use VM stack for splat arg on cfunc

On the cfunc methods, if a splat argument is given, all array elements
are expanded on the VM stack and it can cause SystemStackError.
The idea to avoid it is making a hidden array to contain all parameters
and use this array as an argv.

This patch is reviesed version of https://github.com/ruby/ruby/pull/6816
The main change is all changes are closed around calling cfunc logic.

Fixes [Bug #4040]

Co-authored-by: Jeremy Evans <code@jeremyevans.net>

1 files changed, 84 insertions, 0 deletions

diff --git a/test/ruby/test_call.rb b/test/ruby/test_call.rb
index 88e2ddcb7b..dbdb0752be 100644
--- a/test/ruby/test_call.rb
+++ b/test/ruby/test_call.rb
@@ -115,4 +115,88 @@ class TestCall < Test::Unit::TestCase
     ary = [1, 2, b]
     assert_equal([0, 1, 2, b], aaa(0, *ary, &ary.pop), bug16504)
   end
+
+  def test_call_cfunc_splat_large_array_bug_4040
+    a = 1380.times.to_a # Greater than VM_ARGC_STACK_MAX
+
+    assert_equal(a, [].push(*a))
+    assert_equal(a, [].push(a[0], *a[1..]))
+    assert_equal(a, [].push(a[0], a[1], *a[2..]))
+    assert_equal(a, [].push(*a[0..1], *a[2..]))
+    assert_equal(a, [].push(*a[...-1], a[-1]))
+    assert_equal(a, [].push(a[0], *a[1...-1], a[-1]))
+    assert_equal(a, [].push(a[0], a[1], *a[2...-1], a[-1]))
+    assert_equal(a, [].push(*a[0..1], *a[2...-1], a[-1]))
+    assert_equal(a, [].push(*a[...-2], a[-2], a[-1]))
+    assert_equal(a, [].push(a[0], *a[1...-2], a[-2], a[-1]))
+    assert_equal(a, [].push(a[0], a[1], *a[2...-2], a[-2], a[-1]))
+    assert_equal(a, [].push(*a[0..1], *a[2...-2], a[-2], a[-1]))
+
+    kw = {x: 1}
+    a_kw = a + [kw]
+
+    assert_equal(a_kw, [].push(*a, **kw))
+    assert_equal(a_kw, [].push(a[0], *a[1..], **kw))
+    assert_equal(a_kw, [].push(a[0], a[1], *a[2..], **kw))
+    assert_equal(a_kw, [].push(*a[0..1], *a[2..], **kw))
+    assert_equal(a_kw, [].push(*a[...-1], a[-1], **kw))
+    assert_equal(a_kw, [].push(a[0], *a[1...-1], a[-1], **kw))
+    assert_equal(a_kw, [].push(a[0], a[1], *a[2...-1], a[-1], **kw))
+    assert_equal(a_kw, [].push(*a[0..1], *a[2...-1], a[-1], **kw))
+    assert_equal(a_kw, [].push(*a[...-2], a[-2], a[-1], **kw))
+    assert_equal(a_kw, [].push(a[0], *a[1...-2], a[-2], a[-1], **kw))
+    assert_equal(a_kw, [].push(a[0], a[1], *a[2...-2], a[-2], a[-1], **kw))
+    assert_equal(a_kw, [].push(*a[0..1], *a[2...-2], a[-2], a[-1], **kw))
+
+    assert_equal(a_kw, [].push(*a, x: 1))
+    assert_equal(a_kw, [].push(a[0], *a[1..], x: 1))
+    assert_equal(a_kw, [].push(a[0], a[1], *a[2..], x: 1))
+    assert_equal(a_kw, [].push(*a[0..1], *a[2..], x: 1))
+    assert_equal(a_kw, [].push(*a[...-1], a[-1], x: 1))
+    assert_equal(a_kw, [].push(a[0], *a[1...-1], a[-1], x: 1))
+    assert_equal(a_kw, [].push(a[0], a[1], *a[2...-1], a[-1], x: 1))
+    assert_equal(a_kw, [].push(*a[0..1], *a[2...-1], a[-1], x: 1))
+    assert_equal(a_kw, [].push(*a[...-2], a[-2], a[-1], x: 1))
+    assert_equal(a_kw, [].push(a[0], *a[1...-2], a[-2], a[-1], x: 1))
+    assert_equal(a_kw, [].push(a[0], a[1], *a[2...-2], a[-2], a[-1], x: 1))
+    assert_equal(a_kw, [].push(*a[0..1], *a[2...-2], a[-2], a[-1], x: 1))
+
+    a_kw[-1][:y] = 2
+    kw = {y: 2}
+
+    assert_equal(a_kw, [].push(*a, x: 1, **kw))
+    assert_equal(a_kw, [].push(a[0], *a[1..], x: 1, **kw))
+    assert_equal(a_kw, [].push(a[0], a[1], *a[2..], x: 1, **kw))
+    assert_equal(a_kw, [].push(*a[0..1], *a[2..], x: 1, **kw))
+    assert_equal(a_kw, [].push(*a[...-1], a[-1], x: 1, **kw))
+    assert_equal(a_kw, [].push(a[0], *a[1...-1], a[-1], x: 1, **kw))
+    assert_equal(a_kw, [].push(a[0], a[1], *a[2...-1], a[-1], x: 1, **kw))
+    assert_equal(a_kw, [].push(*a[0..1], *a[2...-1], a[-1], x: 1, **kw))
+    assert_equal(a_kw, [].push(*a[...-2], a[-2], a[-1], x: 1, **kw))
+    assert_equal(a_kw, [].push(a[0], *a[1...-2], a[-2], a[-1], x: 1, **kw))
+    assert_equal(a_kw, [].push(a[0], a[1], *a[2...-2], a[-2], a[-1], x: 1, **kw))
+    assert_equal(a_kw, [].push(*a[0..1], *a[2...-2], a[-2], a[-1], x: 1, **kw))
+
+    kw = {}
+
+    assert_equal(a, [].push(*a, **kw))
+    assert_equal(a, [].push(a[0], *a[1..], **kw))
+    assert_equal(a, [].push(a[0], a[1], *a[2..], **kw))
+    assert_equal(a, [].push(*a[0..1], *a[2..], **kw))
+    assert_equal(a, [].push(*a[...-1], a[-1], **kw))
+    assert_equal(a, [].push(a[0], *a[1...-1], a[-1], **kw))
+    assert_equal(a, [].push(a[0], a[1], *a[2...-1], a[-1], **kw))
+    assert_equal(a, [].push(*a[0..1], *a[2...-1], a[-1], **kw))
+    assert_equal(a, [].push(*a[...-2], a[-2], a[-1], **kw))
+    assert_equal(a, [].push(a[0], *a[1...-2], a[-2], a[-1], **kw))
+    assert_equal(a, [].push(a[0], a[1], *a[2...-2], a[-2], a[-1], **kw))
+    assert_equal(a, [].push(*a[0..1], *a[2...-2], a[-2], a[-1], **kw))
+
+    a_kw = a + [Hash.ruby2_keywords_hash({})]
+    assert_equal(a, [].push(*a_kw))
+
+    # Single test with value that would cause SystemStackError.
+    # Not all tests use such a large array to reduce testing time.
+    assert_equal(1380888, [].push(*1380888.times.to_a).size)
+  end
 end