Do not allow array modifications after freeze inside sort!

If freezing an array inside sort!, previously the array could be
modified after the freeze.  This checks whether the receiver is
frozen after every yield and potential call to #> or #<,
preventing modifications if the receiver is frozen inside the
block or by the #> or #< call.

Fixes [Bug #17739]

2 files changed, 43 insertions, 2 deletions

diff --git a/array.c b/array.c
index 881270b915..83e4d186bd 100644
--- a/array.c
+++ b/array.c
@@ -3213,6 +3213,7 @@ rb_ary_rotate_m(int argc, VALUE *argv, VALUE ary)
 
 struct ary_sort_data {
     VALUE ary;
+    VALUE receiver;
     struct cmp_opt_data cmp_opt;
 };
 
@@ -3225,6 +3226,15 @@ sort_reentered(VALUE ary)
     return Qnil;
 }
 
+static void
+sort_returned(struct ary_sort_data *data)
+{
+    if (rb_obj_frozen_p(data->receiver)) {
+        rb_raise(rb_eFrozenError, "array frozen during sort");
+    }
+    sort_reentered(data->ary);
+}
+
 static int
 sort_1(const void *ap, const void *bp, void *dummy)
 {
@@ -3238,7 +3248,7 @@ sort_1(const void *ap, const void *bp, void *dummy)
     args[1] = b;
     retval = rb_yield_values2(2, args);
     n = rb_cmpint(retval, a, b);
-    sort_reentered(data->ary);
+    sort_returned(data);
     return n;
 }
 
@@ -3264,7 +3274,7 @@ sort_2(const void *ap, const void *bp, void *dummy)
 
     retval = rb_funcallv(a, id_cmp, 1, &b);
     n = rb_cmpint(retval, a, b);
-    sort_reentered(data->ary);
+    sort_returned(data);
 
     return n;
 }
@@ -3316,6 +3326,7 @@ rb_ary_sort_bang(VALUE ary)
 	long len = RARRAY_LEN(ary);
 	RBASIC_CLEAR_CLASS(tmp);
 	data.ary = tmp;
+        data.receiver = ary;
 	data.cmp_opt.opt_methods = 0;
 	data.cmp_opt.opt_inited = 0;
 	RARRAY_PTR_USE(tmp, ptr, {
diff --git a/test/ruby/test_array.rb b/test/ruby/test_array.rb
index fb50f581fe..0a9ba90564 100644
--- a/test/ruby/test_array.rb
+++ b/test/ruby/test_array.rb
@@ -1663,6 +1663,36 @@ class TestArray < Test::Unit::TestCase
     assert_equal([1, 2, 3, 4], a)
   end
 
+  def test_freeze_inside_sort!
+    array = [1, 2, 3, 4, 5]
+    frozen_array = nil
+    assert_raise(FrozenError) do
+      array.sort! do |a, b|
+        array.freeze if a == 3
+        frozen_array ||= array.map.to_a if array.frozen?
+        1
+      end
+    end
+    assert_equal(frozen_array, array)
+
+    object = Object.new
+    array = [1, 2, 3, 4, 5]
+    object.define_singleton_method(:>){|_| array.freeze; true}
+    assert_raise(FrozenError) do
+      array.sort! do |a, b|
+        object
+      end
+    end
+
+    object = Object.new
+    array = [object, object]
+    object.define_singleton_method(:>){|_| array.freeze; true}
+    object.define_singleton_method(:<=>){|o| object}
+    assert_raise(FrozenError) do
+      array.sort!
+    end
+  end
+
   def test_sort_with_callcc
     need_continuation
     n = 1000