Implement Objects on VWA

This commit implements Objects on Variable Width Allocation. This allows
Objects with more ivars to be embedded (i.e. contents directly follow the
object header) which improves performance through better cache locality.

14 files changed, 216 insertions, 74 deletions

diff --git a/debug.c b/debug.c
index 6b5684efb2..d927f72231 100644
--- a/debug.c
+++ b/debug.c
@@ -53,7 +53,9 @@ const union {
     rb_econv_result_t           econv_result;
     enum ruby_preserved_encindex encoding_index;
     enum ruby_robject_flags     robject_flags;
+#if !USE_RVARGC
     enum ruby_robject_consts    robject_consts;
+#endif
     enum ruby_rmodule_flags     rmodule_flags;
     enum ruby_rstring_flags     rstring_flags;
 #if !USE_RVARGC
diff --git a/gc.c b/gc.c
index 39d82a6a77..e92a576c29 100644
--- a/gc.c
+++ b/gc.c
@@ -2865,18 +2865,65 @@ rb_newobj(void)
     return newobj_of(0, T_NONE, 0, 0, 0, FALSE, sizeof(RVALUE));
 }
 
-VALUE
-rb_newobj_of(VALUE klass, VALUE flags)
+static size_t
+rb_obj_embedded_size(uint32_t numiv)
 {
-    if ((flags & RUBY_T_MASK) == T_OBJECT) {
-        st_table *index_tbl = RCLASS_IV_INDEX_TBL(klass);
+    return offsetof(struct RObject, as.ary) + (sizeof(VALUE) * numiv);
+}
+
+static VALUE
+rb_class_instance_allocate_internal(VALUE klass, VALUE flags, bool wb_protected)
+{
+    GC_ASSERT((flags & RUBY_T_MASK) == T_OBJECT);
+    GC_ASSERT(flags & ROBJECT_EMBED);
+
+    st_table *index_tbl = RCLASS_IV_INDEX_TBL(klass);
+    uint32_t index_tbl_num_entries = index_tbl == NULL ? 0 : (uint32_t)index_tbl->num_entries;
+
+    size_t size;
+    bool embed = true;
+#if USE_RVARGC
+    size = rb_obj_embedded_size(index_tbl_num_entries);
+    if (!rb_gc_size_allocatable_p(size)) {
+        size = sizeof(struct RObject);
+        embed = false;
+    }
+#else
+    size = sizeof(struct RObject);
+    if (index_tbl_num_entries > ROBJECT_EMBED_LEN_MAX) {
+        embed = false;
+    }
+#endif
+
+#if USE_RVARGC
+    VALUE obj = newobj_of(klass, flags, 0, 0, 0, wb_protected, size);
+#else
+    VALUE obj = newobj_of(klass, flags, Qundef, Qundef, Qundef, wb_protected, size);
+#endif
 
-        VALUE obj = newobj_of(klass, (flags | ROBJECT_EMBED) & ~FL_WB_PROTECTED , Qundef, Qundef, Qundef, flags & FL_WB_PROTECTED, sizeof(RVALUE));
+    if (embed) {
+#if USE_RVARGC
+        uint32_t capa = (uint32_t)((rb_gc_obj_slot_size(obj) - offsetof(struct RObject, as.ary)) / sizeof(VALUE));
+        GC_ASSERT(capa >= index_tbl_num_entries);
 
-        if (index_tbl && index_tbl->num_entries > ROBJECT_EMBED_LEN_MAX) {
-            rb_init_iv_list(obj);
+        ROBJECT(obj)->numiv = capa;
+        for (size_t i = 0; i < capa; i++) {
+            ROBJECT(obj)->as.ary[i] = Qundef;
         }
-        return obj;
+#endif
+    }
+    else {
+        rb_init_iv_list(obj);
+    }
+
+    return obj;
+}
+
+VALUE
+rb_newobj_of(VALUE klass, VALUE flags)
+{
+    if ((flags & RUBY_T_MASK) == T_OBJECT) {
+        return rb_class_instance_allocate_internal(klass, (flags | ROBJECT_EMBED) & ~FL_WB_PROTECTED, flags & FL_WB_PROTECTED);
     }
     else {
         return newobj_of(klass, flags & ~FL_WB_PROTECTED, 0, 0, 0, flags & FL_WB_PROTECTED, sizeof(RVALUE));
@@ -2989,17 +3036,7 @@ rb_imemo_new_debug(enum imemo_type type, VALUE v1, VALUE v2, VALUE v3, VALUE v0,
 VALUE
 rb_class_allocate_instance(VALUE klass)
 {
-    st_table *index_tbl = RCLASS_IV_INDEX_TBL(klass);
-
-    VALUE flags = T_OBJECT | ROBJECT_EMBED;
-
-    VALUE obj = newobj_of(klass, flags, Qundef, Qundef, Qundef, RGENGC_WB_PROTECTED_OBJECT, sizeof(RVALUE));
-
-    if (index_tbl && index_tbl->num_entries > ROBJECT_EMBED_LEN_MAX) {
-        rb_init_iv_list(obj);
-    }
-
-    return obj;
+    return rb_class_instance_allocate_internal(klass, T_OBJECT | ROBJECT_EMBED, RGENGC_WB_PROTECTED_OBJECT);
 }
 
 static inline void
@@ -8322,17 +8359,23 @@ gc_compact_destination_pool(rb_objspace_t *objspace, rb_size_pool_t *src_pool, V
     size_t obj_size;
 
     switch (BUILTIN_TYPE(src)) {
-        case T_STRING:
-            obj_size = rb_str_size_as_embedded(src);
-            break;
         case T_ARRAY:
             obj_size = rb_ary_size_as_embedded(src);
             break;
+
+        case T_OBJECT:
+            obj_size = rb_obj_embedded_size(ROBJECT_NUMIV(src));
+            break;
+
+        case T_STRING:
+            obj_size = rb_str_size_as_embedded(src);
+            break;
+
         default:
             return src_pool;
     }
 
-    if (rb_gc_size_allocatable_p(obj_size)) {
+    if (rb_gc_size_allocatable_p(obj_size)){
         return &size_pools[size_pool_idx_for_size(obj_size)];
     }
     else {
@@ -9896,12 +9939,37 @@ gc_ref_update_array(rb_objspace_t * objspace, VALUE v)
 }
 
 static void
-gc_ref_update_object(rb_objspace_t * objspace, VALUE v)
+gc_ref_update_object(rb_objspace_t *objspace, VALUE v)
 {
     VALUE *ptr = ROBJECT_IVPTR(v);
+    uint32_t numiv = ROBJECT_NUMIV(v);
+
+#if USE_RVARGC
+    size_t slot_size = rb_gc_obj_slot_size(v);
+    size_t embed_size = rb_obj_embedded_size(numiv);
+    if (slot_size >= embed_size && !RB_FL_TEST_RAW(v, ROBJECT_EMBED)) {
+        // Object can be re-embedded
+        memcpy(ROBJECT(v)->as.ary, ptr, sizeof(VALUE) * numiv);
+        RB_FL_SET_RAW(v, ROBJECT_EMBED);
+        if (ROBJ_TRANSIENT_P(v)) {
+            ROBJ_TRANSIENT_UNSET(v);
+        }
+        else {
+            xfree(ptr);
+        }
+        ptr = ROBJECT(v)->as.ary;
+
+        uint32_t capa = (uint32_t)((slot_size - offsetof(struct RObject, as.ary)) / sizeof(VALUE));
+        ROBJECT(v)->numiv = capa;
+
+        // Fill end with Qundef
+        for (uint32_t i = numiv; i < capa; i++) {
+            ptr[i] = Qundef;
+        }
+    }
+#endif
 
-    uint32_t i, len = ROBJECT_NUMIV(v);
-    for (i = 0; i < len; i++) {
+    for (uint32_t i = 0; i < numiv; i++) {
         UPDATE_IF_MOVED(objspace, ptr[i]);
     }
 }
diff --git a/include/ruby/internal/abi.h b/include/ruby/internal/abi.h
index ed779f3558..e42a1777ff 100644
--- a/include/ruby/internal/abi.h
+++ b/include/ruby/internal/abi.h
@@ -22,7 +22,7 @@
  * In released versions of Ruby, this number should not be changed since teeny
  * versions of Ruby should guarantee ABI compatibility.
  */
-#define RUBY_ABI_VERSION 1
+#define RUBY_ABI_VERSION 2
 
 /* Windows does not support weak symbols so ruby_abi_version will not exist
  * in the shared library. */
diff --git a/include/ruby/internal/core/robject.h b/include/ruby/internal/core/robject.h
index f2028063a6..7823061d8f 100644
--- a/include/ruby/internal/core/robject.h
+++ b/include/ruby/internal/core/robject.h
@@ -75,6 +75,7 @@ enum ruby_robject_flags {
     ROBJECT_EMBED = RUBY_FL_USER1
 };
 
+#if !USE_RVARGC
 /**
  * This is an enum because GDB wants it (rather than a macro).  People need not
  * bother.
@@ -83,6 +84,7 @@ enum ruby_robject_consts {
     /** Max possible number of instance variables that can be embedded. */
     ROBJECT_EMBED_LEN_MAX = RBIMPL_EMBED_LEN_MAX_OF(VALUE)
 };
+#endif
 
 struct st_table;
 
@@ -95,6 +97,14 @@ struct RObject {
     /** Basic part, including flags and class. */
     struct RBasic basic;
 
+#if USE_RVARGC
+    /**
+    * Number of instance variables.  This is per object; objects might
+    * differ in this field even if they have the identical classes.
+    */
+    uint32_t numiv;
+#endif
+
     /** Object's specific fields. */
     union {
 
@@ -103,12 +113,13 @@ struct RObject {
          * this pattern.
          */
         struct {
-
+#if !USE_RVARGC
             /**
              * Number of instance variables.  This is per object; objects might
              * differ in this field even if they have the identical classes.
              */
             uint32_t numiv;
+#endif
 
             /** Pointer to a C array that holds instance variables. */
             VALUE *ivptr;
@@ -124,14 +135,38 @@ struct RObject {
             struct st_table *iv_index_tbl;
         } heap;
 
-        /**
-         * Embedded instance  variables.  When  an object  is small  enough, it
+#if USE_RVARGC
+        /* Embedded instance variables. When an object is small enough, it
          * uses this area to store the instance variables.
+         *
+         * This is a length 1 array because:
+         *   1. GCC has a bug that does not optimize C flexible array members
+         *      (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102452)
+         *   2. Zero length arrays are not supported by all compilers
          */
+        VALUE ary[1];
+#else
+        /**
+        * Embedded instance  variables.  When  an object  is small  enough, it
+        * uses this area to store the instance variables.
+        */
         VALUE ary[ROBJECT_EMBED_LEN_MAX];
+#endif
     } as;
 };
 
+/* Offsets for YJIT */
+#ifndef __cplusplus
+# if USE_RVARGC
+static const int32_t ROBJECT_OFFSET_NUMIV = offsetof(struct RObject, numiv);
+# else
+static const int32_t ROBJECT_OFFSET_NUMIV = offsetof(struct RObject, as.heap.numiv);
+# endif
+static const int32_t ROBJECT_OFFSET_AS_HEAP_IVPTR = offsetof(struct RObject, as.heap.ivptr);
+static const int32_t ROBJECT_OFFSET_AS_HEAP_IV_INDEX_TBL = offsetof(struct RObject, as.heap.iv_index_tbl);
+static const int32_t ROBJECT_OFFSET_AS_ARY = offsetof(struct RObject, as.ary);
+#endif
+
 RBIMPL_ATTR_PURE_UNLESS_DEBUG()
 RBIMPL_ATTR_ARTIFICIAL()
 /**
@@ -146,12 +181,16 @@ ROBJECT_NUMIV(VALUE obj)
 {
     RBIMPL_ASSERT_TYPE(obj, RUBY_T_OBJECT);
 
+#if USE_RVARGC
+    return ROBJECT(obj)->numiv;
+#else
     if (RB_FL_ANY_RAW(obj, ROBJECT_EMBED)) {
         return ROBJECT_EMBED_LEN_MAX;
     }
     else {
         return ROBJECT(obj)->as.heap.numiv;
     }
+#endif
 }
 
 RBIMPL_ATTR_PURE_UNLESS_DEBUG()
diff --git a/mjit_compile.c b/mjit_compile.c
index 66deaa9a65..2c7996c258 100644
--- a/mjit_compile.c
+++ b/mjit_compile.c
@@ -353,12 +353,16 @@ mjit_compile_body(FILE *f, const rb_iseq_t *iseq, struct compile_status *status)
     // Generate merged ivar guards first if needed
     if (!status->compile_info->disable_ivar_cache && status->merge_ivar_guards_p) {
         fprintf(f, "    if (UNLIKELY(!(RB_TYPE_P(GET_SELF(), T_OBJECT) && (rb_serial_t)%"PRI_SERIALT_PREFIX"u == RCLASS_SERIAL(RBASIC(GET_SELF())->klass) &&", status->ivar_serial);
+#if USE_RVARGC
+        fprintf(f, "%"PRIuSIZE" < ROBJECT_NUMIV(GET_SELF())", status->max_ivar_index); // index < ROBJECT_NUMIV(obj)
+#else
         if (status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX) {
             fprintf(f, "%"PRIuSIZE" < ROBJECT_NUMIV(GET_SELF())", status->max_ivar_index); // index < ROBJECT_NUMIV(obj) && !RB_FL_ANY_RAW(obj, ROBJECT_EMBED)
         }
         else {
             fprintf(f, "ROBJECT_EMBED_LEN_MAX == ROBJECT_NUMIV(GET_SELF())"); // index < ROBJECT_NUMIV(obj) && RB_FL_ANY_RAW(obj, ROBJECT_EMBED)
         }
+#endif
         fprintf(f, "))) {\n");
         fprintf(f, "        goto ivar_cancel;\n");
         fprintf(f, "    }\n");
diff --git a/object.c b/object.c
index 6a7b2cbf23..4baf7ffc56 100644
--- a/object.c
+++ b/object.c
@@ -267,34 +267,13 @@ rb_obj_singleton_class(VALUE obj)
 MJIT_FUNC_EXPORTED void
 rb_obj_copy_ivar(VALUE dest, VALUE obj)
 {
-    VALUE *dst_buf = 0;
-    VALUE *src_buf = 0;
-    uint32_t len = ROBJECT_EMBED_LEN_MAX;
+    VALUE *dest_buf = ROBJECT_IVPTR(dest);
+    VALUE *src_buf = ROBJECT_IVPTR(obj);
+    uint32_t dest_len = ROBJECT_NUMIV(dest);
+    uint32_t src_len = ROBJECT_NUMIV(obj);
+    uint32_t len = dest_len < src_len ? dest_len : src_len;
 
-    if (RBASIC(obj)->flags & ROBJECT_EMBED) {
-        src_buf = ROBJECT(obj)->as.ary;
-
-        // embedded -> embedded
-        if (RBASIC(dest)->flags & ROBJECT_EMBED) {
-            dst_buf = ROBJECT(dest)->as.ary;
-        }
-        // embedded -> extended
-        else {
-            dst_buf = ROBJECT(dest)->as.heap.ivptr;
-        }
-    }
-    // extended -> extended
-    else {
-        RUBY_ASSERT(!(RBASIC(dest)->flags & ROBJECT_EMBED));
-        uint32_t src_len = ROBJECT(obj)->as.heap.numiv;
-        uint32_t dst_len = ROBJECT(dest)->as.heap.numiv;
-
-        len = src_len < dst_len ? src_len : dst_len;
-        dst_buf = ROBJECT(dest)->as.heap.ivptr;
-        src_buf = ROBJECT(obj)->as.heap.ivptr;
-    }
-
-    MEMCPY(dst_buf, src_buf, VALUE, len);
+    MEMCPY(dest_buf, src_buf, VALUE, len);
 }
 
 static void
diff --git a/test/ruby/test_gc_compact.rb b/test/ruby/test_gc_compact.rb
index 7d72f3573b..92a2be1174 100644
--- a/test/ruby/test_gc_compact.rb
+++ b/test/ruby/test_gc_compact.rb
@@ -249,6 +249,30 @@ class TestGCCompact < Test::Unit::TestCase
     end;
   end
 
+  def test_moving_objects_between_size_pools
+    assert_separately([], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10, signal: :SEGV)
+    begin;
+      class Foo
+        def add_ivars
+          10.times do |i|
+            instance_variable_set("@foo" + i.to_s, 0)
+          end
+        end
+      end
+
+      OBJ_COUNT = 500
+
+      GC.verify_compaction_references(expand_heap: true, toward: :empty)
+
+      ary = OBJ_COUNT.times.map { Foo.new }
+      ary.each(&:add_ivars)
+
+      stats = GC.verify_compaction_references(expand_heap: true, toward: :empty)
+
+      assert_operator(stats[:moved_up][:T_OBJECT], :>=, OBJ_COUNT)
+    end;
+  end
+
   def test_moving_strings_up_size_pools
     omit if !GC.using_rvargc?
     assert_separately([], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10, signal: :SEGV)
diff --git a/test/ruby/test_mjit.rb b/test/ruby/test_mjit.rb
index fd7a4e270a..02be88aa32 100644
--- a/test/ruby/test_mjit.rb
+++ b/test/ruby/test_mjit.rb
@@ -969,23 +969,24 @@ class TestMJIT < Test::Unit::TestCase
   end
 
   def test_heap_promotion_of_ivar_in_the_middle_of_jit
+    omit if GC.using_rvargc?
+
     assert_eval_with_jit("#{<<~"begin;"}\n#{<<~"end;"}", stdout: "true\ntrue\n", success_count: 2, min_calls: 2)
     begin;
       class A
         def initialize
           @iv0 = nil
           @iv1 = []
-          @iv2 = nil
         end
 
         def test(add)
           @iv0.nil?
-          @iv2.nil?
           add_ivar if add
           @iv1.empty?
         end
 
         def add_ivar
+          @iv2 = nil
           @iv3 = nil
         end
       end
diff --git a/tool/ruby_vm/views/_mjit_compile_ivar.erb b/tool/ruby_vm/views/_mjit_compile_ivar.erb
index 5105584ba3..1425b3b055 100644
--- a/tool/ruby_vm/views/_mjit_compile_ivar.erb
+++ b/tool/ruby_vm/views/_mjit_compile_ivar.erb
@@ -30,15 +30,24 @@
             fprintf(f, "    VM_ASSERT((rb_serial_t)%"PRI_SERIALT_PREFIX"u == RCLASS_SERIAL(RBASIC(obj)->klass));\n", ic_copy->entry->class_serial);
             fprintf(f, "    VM_ASSERT(index < ROBJECT_NUMIV(obj));\n");
 % if insn.name == 'setinstancevariable'
+#if USE_RVARGC
+            fprintf(f, "    if (LIKELY(!RB_OBJ_FROZEN_RAW(obj) && index < ROBJECT_NUMIV(obj))) {\n");
+            fprintf(f, "        RB_OBJ_WRITE(obj, &ROBJECT_IVPTR(obj)[index], stack[%d]);\n", b->stack_size - 1);
+#else
             fprintf(f, "    if (LIKELY(!RB_OBJ_FROZEN_RAW(obj) && %s)) {\n", status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "true" : "RB_FL_ANY_RAW(obj, ROBJECT_EMBED)");
             fprintf(f, "        RB_OBJ_WRITE(obj, &ROBJECT(obj)->as.%s, stack[%d]);\n",
                     status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "heap.ivptr[index]" : "ary[index]", b->stack_size - 1);
+#endif
             fprintf(f, "    }\n");
 % else
             fprintf(f, "    VALUE val;\n");
+#if USE_RVARGC
+            fprintf(f, "    if (LIKELY(index < ROBJECT_NUMIV(obj) && (val = ROBJECT_IVPTR(obj)[index]) != Qundef)) {\n");
+#else
             fprintf(f, "    if (LIKELY(%s && (val = ROBJECT(obj)->as.%s) != Qundef)) {\n",
                     status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "true" : "RB_FL_ANY_RAW(obj, ROBJECT_EMBED)",
                     status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "heap.ivptr[index]" : "ary[index]");
+#endif
             fprintf(f, "        stack[%d] = val;\n", b->stack_size);
             fprintf(f, "    }\n");
 %end
diff --git a/variable.c b/variable.c
index def997d201..81258cb47b 100644
--- a/variable.c
+++ b/variable.c
@@ -1467,6 +1467,8 @@ void
 rb_obj_transient_heap_evacuate(VALUE obj, int promote)
 {
     if (ROBJ_TRANSIENT_P(obj)) {
+        assert(!RB_FL_TEST_RAW(obj, ROBJECT_EMBED));
+
         uint32_t len = ROBJECT_NUMIV(obj);
         const VALUE *old_ptr = ROBJECT_IVPTR(obj);
         VALUE *new_ptr;
@@ -1493,7 +1495,7 @@ init_iv_list(VALUE obj, uint32_t len, uint32_t newsize, st_table *index_tbl)
     if (RBASIC(obj)->flags & ROBJECT_EMBED) {
         newptr = obj_ivar_heap_alloc(obj, newsize);
         MEMCPY(newptr, ptr, VALUE, len);
-        RBASIC(obj)->flags &= ~ROBJECT_EMBED;
+        RB_FL_UNSET_RAW(obj, ROBJECT_EMBED);
         ROBJECT(obj)->as.heap.ivptr = newptr;
     }
     else {
@@ -1503,7 +1505,11 @@ init_iv_list(VALUE obj, uint32_t len, uint32_t newsize, st_table *index_tbl)
     for (; len < newsize; len++) {
         newptr[len] = Qundef;
     }
+#if USE_RVARGC
+    ROBJECT(obj)->numiv = newsize;
+#else
     ROBJECT(obj)->as.heap.numiv = newsize;
+#endif
     ROBJECT(obj)->as.heap.iv_index_tbl = index_tbl;
 }
 
diff --git a/yjit/bindgen/src/main.rs b/yjit/bindgen/src/main.rs
index 5d30d7b288..f54addc795 100644
--- a/yjit/bindgen/src/main.rs
+++ b/yjit/bindgen/src/main.rs
@@ -62,6 +62,9 @@ fn main() {
         // Import YARV bytecode instruction constants
         .allowlist_type("ruby_vminsn_type")
 
+        // From include/ruby/internal/config.h
+        .allowlist_var("USE_RVARGC")
+
         // From include/ruby/internal/intern/string.h
         .allowlist_function("rb_utf8_str_new")
         .allowlist_function("rb_str_buf_append")
@@ -135,7 +138,8 @@ fn main() {
 
         // From include/ruby/internal/core/robject.h
         .allowlist_type("ruby_robject_flags")
-        .allowlist_type("ruby_robject_consts")
+        // .allowlist_type("ruby_robject_consts") // Removed when USE_RVARGC
+        .allowlist_var("ROBJECT_OFFSET_.*")
 
         // From include/ruby/internal/core/rarray.h
         .allowlist_type("ruby_rarray_flags")
diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs
index 84f4ff4897..9cd69cd340 100644
--- a/yjit/src/codegen.rs
+++ b/yjit/src/codegen.rs
@@ -1966,9 +1966,17 @@ fn gen_get_ivar(
         ctx.stack_pop(1);
     }
 
+    if USE_RVARGC != 0 {
+        // Check that the ivar table is big enough
+        // Check that the slot is inside the ivar table (num_slots > index)
+        let num_slots = mem_opnd(32, REG0, ROBJECT_OFFSET_NUMIV);
+        cmp(cb, num_slots, uimm_opnd(ivar_index as u64));
+        jle_ptr(cb, counted_exit!(ocb, side_exit, getivar_idx_out_of_range));
+    }
+
     // Compile time self is embedded and the ivar index lands within the object
     let test_result = unsafe { FL_TEST_RAW(comptime_receiver, VALUE(ROBJECT_EMBED.as_usize())) != VALUE(0) };
-    if test_result && ivar_index < (ROBJECT_EMBED_LEN_MAX.as_usize()) {
+    if test_result {
         // See ROBJECT_IVPTR() from include/ruby/internal/core/robject.h
 
         // Guard that self is embedded
@@ -1988,7 +1996,7 @@ fn gen_get_ivar(
         );
 
         // Load the variable
-        let offs = RUBY_OFFSET_ROBJECT_AS_ARY + (ivar_index * SIZEOF_VALUE) as i32;
+        let offs = ROBJECT_OFFSET_AS_ARY + (ivar_index * SIZEOF_VALUE) as i32;
         let ivar_opnd = mem_opnd(64, REG0, offs);
         mov(cb, REG1, ivar_opnd);
 
@@ -2019,17 +2027,16 @@ fn gen_get_ivar(
             side_exit,
         );
 
-        // Check that the extended table is big enough
-        if ivar_index > (ROBJECT_EMBED_LEN_MAX.as_usize()) {
+        if USE_RVARGC == 0 {
+            // Check that the extended table is big enough
             // Check that the slot is inside the extended table (num_slots > index)
-            let num_slots = mem_opnd(32, REG0, RUBY_OFFSET_ROBJECT_AS_HEAP_NUMIV);
-
+            let num_slots = mem_opnd(32, REG0, ROBJECT_OFFSET_NUMIV);
             cmp(cb, num_slots, uimm_opnd(ivar_index as u64));
             jle_ptr(cb, counted_exit!(ocb, side_exit, getivar_idx_out_of_range));
         }
 
         // Get a pointer to the extended table
-        let tbl_opnd = mem_opnd(64, REG0, RUBY_OFFSET_ROBJECT_AS_HEAP_IVPTR);
+        let tbl_opnd = mem_opnd(64, REG0, ROBJECT_OFFSET_AS_HEAP_IVPTR);
         mov(cb, REG0, tbl_opnd);
 
         // Read the ivar from the extended table
diff --git a/yjit/src/cruby.rs b/yjit/src/cruby.rs
index 8cebfd58a5..7c21bd962b 100644
--- a/yjit/src/cruby.rs
+++ b/yjit/src/cruby.rs
@@ -739,10 +739,6 @@ mod manual_defs {
     pub const RUBY_OFFSET_RSTRUCT_AS_HEAP_PTR: i32 = 24; // struct RStruct, subfield "as.heap.ptr"
     pub const RUBY_OFFSET_RSTRUCT_AS_ARY: i32 = 16; // struct RStruct, subfield "as.ary"
 
-    pub const RUBY_OFFSET_ROBJECT_AS_ARY: i32 = 16; // struct RObject, subfield "as.ary"
-    pub const RUBY_OFFSET_ROBJECT_AS_HEAP_NUMIV: i32 = 16; // struct RObject, subfield "as.heap.numiv"
-    pub const RUBY_OFFSET_ROBJECT_AS_HEAP_IVPTR: i32 = 24; // struct RObject, subfield "as.heap.ivptr"
-
     // Constants from rb_control_frame_t vm_core.h
     pub const RUBY_OFFSET_CFP_PC: i32 = 0;
     pub const RUBY_OFFSET_CFP_SP: i32 = 8;
diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs
index 88ad8e563c..b9c239cd23 100644
--- a/yjit/src/cruby_bindings.inc.rs
+++ b/yjit/src/cruby_bindings.inc.rs
@@ -1,5 +1,6 @@
 /* automatically generated by rust-bindgen 0.59.2 */
 
+pub const USE_RVARGC: u32 = 1;
 pub const INTEGER_REDEFINED_OP_FLAG: u32 = 1;
 pub const FLOAT_REDEFINED_OP_FLAG: u32 = 2;
 pub const STRING_REDEFINED_OP_FLAG: u32 = 4;
@@ -110,8 +111,10 @@ extern "C" {
 }
 pub const ROBJECT_EMBED: ruby_robject_flags = 8192;
 pub type ruby_robject_flags = u32;
-pub const ROBJECT_EMBED_LEN_MAX: ruby_robject_consts = 3;
-pub type ruby_robject_consts = u32;
+pub const ROBJECT_OFFSET_NUMIV: i32 = 16;
+pub const ROBJECT_OFFSET_AS_HEAP_IVPTR: i32 = 24;
+pub const ROBJECT_OFFSET_AS_HEAP_IV_INDEX_TBL: i32 = 32;
+pub const ROBJECT_OFFSET_AS_ARY: i32 = 24;
 extern "C" {
     pub static mut rb_mKernel: VALUE;
 }