Fix memory leak when evacuating generic ivars

The lookup in the table is using the wrong key when converting generic
instance variables to too complex, which means that it never looks up
the entry which leaks memory when the entry is overwritten.

2 files changed, 25 insertions, 1 deletions

diff --git a/test/ruby/test_shapes.rb b/test/ruby/test_shapes.rb
index 6efcbbf101..57c97d9031 100644
--- a/test/ruby/test_shapes.rb
+++ b/test/ruby/test_shapes.rb
@@ -517,6 +517,30 @@ class TestShapes < Test::Unit::TestCase
     end;
   end
 
+  def test_evacuate_generic_ivar_memory_leak
+    assert_no_memory_leak([], "#{<<~'begin;'}", "#{<<~'end;'}", rss: true)
+      o = []
+      o.instance_variable_set(:@a, 1)
+
+      i = 0
+      o = Object.new
+      while RubyVM::Shape.shapes_available > 0
+        o.instance_variable_set(:"@i#{i}", 1)
+        i += 1
+      end
+
+      ary = 1_000_000.times.map { [] }
+    begin;
+      ary.each do |o|
+        o.instance_variable_set(:@a, 1)
+        o.instance_variable_set(:@b, 1)
+      end
+      ary.clear
+      ary = nil
+      GC.start
+    end;
+  end
+
   def test_use_all_shapes_module
     assert_separately([], "#{<<~"begin;"}\n#{<<~'end;'}")
     begin;
diff --git a/variable.c b/variable.c
index b6c9cc3e0d..f155f7e712 100644
--- a/variable.c
+++ b/variable.c
@@ -1379,7 +1379,7 @@ rb_obj_convert_to_too_complex(VALUE obj, st_table *table)
         RB_VM_LOCK_ENTER();
         {
             struct st_table *gen_ivs = generic_ivtbl_no_ractor_check(obj);
-            st_lookup(gen_ivs, (st_data_t)&obj, (st_data_t *)&old_ivptr);
+            st_lookup(gen_ivs, (st_data_t)obj, (st_data_t *)&old_ivptr);
 
             struct gen_ivtbl *ivtbl = xmalloc(sizeof(struct gen_ivtbl));
             ivtbl->as.complex.table = table;