diff options
| -rw-r--r-- | class.c | 21 | ||||
| -rw-r--r-- | gc.c | 23 | ||||
| -rw-r--r-- | internal/class.h | 4 | ||||
| -rw-r--r-- | internal/variable.h | 2 | ||||
| -rw-r--r-- | marshal.c | 2 | ||||
| -rw-r--r-- | object.c | 18 | ||||
| -rw-r--r-- | shape.c | 9 | ||||
| -rw-r--r-- | shape.h | 24 | ||||
| -rw-r--r-- | variable.c | 336 | ||||
| -rw-r--r-- | vm_insnhelper.c | 28 |
10 files changed, 265 insertions, 202 deletions
@@ -213,7 +213,6 @@ class_alloc(VALUE flags, VALUE klass) #endif /* ZALLOC - RCLASS_IV_TBL(obj) = 0; RCLASS_CONST_TBL(obj) = 0; RCLASS_M_TBL(obj) = 0; RCLASS_IV_INDEX_TBL(obj) = 0; @@ -402,23 +401,19 @@ class_init_copy_check(VALUE clone, VALUE orig) static void copy_tables(VALUE clone, VALUE orig) { - if (RCLASS_IV_TBL(clone)) { - st_free_table(RCLASS_IV_TBL(clone)); - RCLASS_IV_TBL(clone) = 0; - } if (RCLASS_CONST_TBL(clone)) { rb_free_const_table(RCLASS_CONST_TBL(clone)); RCLASS_CONST_TBL(clone) = 0; } RCLASS_M_TBL(clone) = 0; - if (RCLASS_IV_TBL(orig)) { + if (!RB_TYPE_P(clone, T_ICLASS)) { st_data_t id; rb_iv_tbl_copy(clone, orig); CONST_ID(id, "__tmp_classpath__"); - st_delete(RCLASS_IV_TBL(clone), &id, 0); + rb_attr_delete(clone, id); CONST_ID(id, "__classpath__"); - st_delete(RCLASS_IV_TBL(clone), &id, 0); + rb_attr_delete(clone, id); } if (RCLASS_CONST_TBL(orig)) { struct clone_const_arg arg; @@ -520,7 +515,6 @@ rb_mod_init_copy(VALUE clone, VALUE orig) prev_clone_p = clone_p; RCLASS_M_TBL(clone_p) = RCLASS_M_TBL(p); RCLASS_CONST_TBL(clone_p) = RCLASS_CONST_TBL(p); - RCLASS_IV_TBL(clone_p) = RCLASS_IV_TBL(p); RCLASS_ALLOCATOR(clone_p) = RCLASS_ALLOCATOR(p); if (RB_TYPE_P(clone, T_CLASS)) { RCLASS_SET_INCLUDER(clone_p, clone); @@ -607,9 +601,7 @@ rb_singleton_class_clone_and_attach(VALUE obj, VALUE attach) RCLASS_SET_SUPER(clone, RCLASS_SUPER(klass)); RCLASS_ALLOCATOR(clone) = RCLASS_ALLOCATOR(klass); - if (RCLASS_IV_TBL(klass)) { - rb_iv_tbl_copy(clone, klass); - } + rb_iv_tbl_copy(clone, klass); if (RCLASS_CONST_TBL(klass)) { struct clone_const_arg arg; arg.tbl = RCLASS_CONST_TBL(clone) = rb_id_table_create(0); @@ -1062,13 +1054,10 @@ rb_include_class_new(VALUE module, VALUE super) module = METACLASS_OF(module); } RUBY_ASSERT(!RB_TYPE_P(module, T_ICLASS)); - if (!RCLASS_IV_TBL(module)) { - RCLASS_IV_TBL(module) = st_init_numtable(); - } if (!RCLASS_CONST_TBL(module)) { RCLASS_CONST_TBL(module) = rb_id_table_create(0); } - RCLASS_IV_TBL(klass) = RCLASS_IV_TBL(module); + RCLASS_CVC_TBL(klass) = RCLASS_CVC_TBL(module); RCLASS_CONST_TBL(klass) = RCLASS_CONST_TBL(module); @@ -3447,8 +3447,8 @@ obj_free(rb_objspace_t *objspace, VALUE obj) case T_CLASS: rb_id_table_free(RCLASS_M_TBL(obj)); cc_table_free(objspace, obj, FALSE); - if (RCLASS_IV_TBL(obj)) { - st_free_table(RCLASS_IV_TBL(obj)); + if (RCLASS_IVPTR(obj)) { + xfree(RCLASS_IVPTR(obj)); } if (RCLASS_CONST_TBL(obj)) { rb_free_const_table(RCLASS_CONST_TBL(obj)); @@ -4865,15 +4865,11 @@ obj_memsize_of(VALUE obj, int use_all_types) if (RCLASS_M_TBL(obj)) { size += rb_id_table_memsize(RCLASS_M_TBL(obj)); } - if (RCLASS_IV_TBL(obj)) { - size += st_memsize(RCLASS_IV_TBL(obj)); - } + // class IV sizes are allocated as powers of two + size += SIZEOF_VALUE << bit_length(RCLASS_IV_COUNT(obj)); if (RCLASS_CVC_TBL(obj)) { size += rb_id_table_memsize(RCLASS_CVC_TBL(obj)); } - if (RCLASS_EXT(obj)->iv_tbl) { - size += st_memsize(RCLASS_EXT(obj)->iv_tbl); - } if (RCLASS_EXT(obj)->const_tbl) { size += rb_id_table_memsize(RCLASS_EXT(obj)->const_tbl); } @@ -7212,7 +7208,9 @@ gc_mark_children(rb_objspace_t *objspace, VALUE obj) mark_m_tbl(objspace, RCLASS_M_TBL(obj)); cc_table_mark(objspace, obj); - mark_tbl_no_pin(objspace, RCLASS_IV_TBL(obj)); + for (attr_index_t i = 0; i < RCLASS_IV_COUNT(obj); i++) { + gc_mark(objspace, RCLASS_IVPTR(obj)[i]); + } mark_const_tbl(objspace, RCLASS_CONST_TBL(obj)); break; @@ -10439,7 +10437,9 @@ gc_update_object_references(rb_objspace_t *objspace, VALUE obj) update_cvc_tbl(objspace, obj); update_superclasses(objspace, obj); - gc_update_tbl_refs(objspace, RCLASS_IV_TBL(obj)); + for (attr_index_t i = 0; i < RCLASS_IV_COUNT(obj); i++) { + UPDATE_IF_MOVED(objspace, RCLASS_IVPTR(obj)[i]); + } update_class_ext(objspace, RCLASS_EXT(obj)); update_const_tbl(objspace, RCLASS_CONST_TBL(obj)); @@ -10454,9 +10454,6 @@ gc_update_object_references(rb_objspace_t *objspace, VALUE obj) UPDATE_IF_MOVED(objspace, RCLASS(obj)->super); } if (!RCLASS_EXT(obj)) break; - if (RCLASS_IV_TBL(obj)) { - gc_update_tbl_refs(objspace, RCLASS_IV_TBL(obj)); - } update_class_ext(objspace, RCLASS_EXT(obj)); update_m_tbl(objspace, RCLASS_CALLABLE_M_TBL(obj)); update_cc_tbl(objspace, obj); diff --git a/internal/class.h b/internal/class.h index 63fe84c6ab..784d508e20 100644 --- a/internal/class.h +++ b/internal/class.h @@ -33,7 +33,7 @@ struct rb_cvar_class_tbl_entry { }; struct rb_classext_struct { - struct st_table *iv_tbl; + VALUE *iv_ptr; struct rb_id_table *const_tbl; struct rb_id_table *callable_m_tbl; struct rb_id_table *cc_tbl; /* ID -> [[ci, cc1], cc2, ...] */ @@ -75,9 +75,9 @@ typedef struct rb_classext_struct rb_classext_t; #else # define RCLASS_EXT(c) (RCLASS(c)->ptr) #endif -#define RCLASS_IV_TBL(c) (RCLASS_EXT(c)->iv_tbl) #define RCLASS_CONST_TBL(c) (RCLASS_EXT(c)->const_tbl) #define RCLASS_M_TBL(c) (RCLASS(c)->m_tbl) +#define RCLASS_IVPTR(c) (RCLASS_EXT(c)->iv_ptr) #define RCLASS_CALLABLE_M_TBL(c) (RCLASS_EXT(c)->callable_m_tbl) #define RCLASS_CC_TBL(c) (RCLASS_EXT(c)->cc_tbl) #define RCLASS_CVC_TBL(c) (RCLASS_EXT(c)->cvc_tbl) diff --git a/internal/variable.h b/internal/variable.h index bb24f5129f..734884a5f6 100644 --- a/internal/variable.h +++ b/internal/variable.h @@ -24,7 +24,6 @@ void rb_gc_update_global_tbl(void); size_t rb_generic_ivar_memsize(VALUE); VALUE rb_search_class_path(VALUE); VALUE rb_attr_delete(VALUE, ID); -VALUE rb_ivar_lookup(VALUE obj, ID id, VALUE undef); void rb_autoload_str(VALUE mod, ID id, VALUE file); VALUE rb_autoload_at_p(VALUE, ID, int); NORETURN(VALUE rb_mod_const_missing(VALUE,VALUE)); @@ -49,6 +48,7 @@ void rb_iv_tbl_copy(VALUE dst, VALUE src); RUBY_SYMBOL_EXPORT_END MJIT_SYMBOL_EXPORT_BEGIN +VALUE rb_ivar_lookup(VALUE obj, ID id, VALUE undef); VALUE rb_gvar_get(ID); VALUE rb_gvar_set(ID, VALUE); VALUE rb_gvar_defined(ID); @@ -523,7 +523,7 @@ hash_each(VALUE key, VALUE value, VALUE v) #define SINGLETON_DUMP_UNABLE_P(klass) \ (rb_id_table_size(RCLASS_M_TBL(klass)) > 0 || \ - (RCLASS_IV_TBL(klass) && RCLASS_IV_TBL(klass)->num_entries > 1)) + rb_ivar_count(klass) > 1) static void w_extended(VALUE klass, struct dump_arg *arg, int check) @@ -298,20 +298,22 @@ init_copy(VALUE dest, VALUE obj) rb_copy_generic_ivar(dest, obj); rb_gc_copy_finalizer(dest, obj); - rb_shape_t *shape_to_set = rb_shape_get_shape(obj); + if (!RB_TYPE_P(obj, T_CLASS) && !RB_TYPE_P(obj, T_MODULE)) { + rb_shape_t *shape_to_set = rb_shape_get_shape(obj); - // If the object is frozen, the "dup"'d object will *not* be frozen, - // so we need to copy the frozen shape's parent to the new object. - if (rb_shape_frozen_shape_p(shape_to_set)) { - shape_to_set = rb_shape_get_shape_by_id(shape_to_set->parent_id); + // If the object is frozen, the "dup"'d object will *not* be frozen, + // so we need to copy the frozen shape's parent to the new object. + if (rb_shape_frozen_shape_p(shape_to_set)) { + shape_to_set = rb_shape_get_shape_by_id(shape_to_set->parent_id); + } + + // shape ids are different + rb_shape_set_shape(dest, shape_to_set); } if (RB_TYPE_P(obj, T_OBJECT)) { rb_obj_copy_ivar(dest, obj); } - - // shape ids are different - rb_shape_set_shape(dest, shape_to_set); } static VALUE immutable_obj_clone(VALUE obj, VALUE kwfreeze); @@ -54,9 +54,10 @@ rb_shape_get_shape_by_id_without_assertion(shape_id_t shape_id) } #if !SHAPE_IN_BASIC_FLAGS -static inline shape_id_t -RCLASS_SHAPE_ID(VALUE obj) +shape_id_t +rb_rclass_shape_id(VALUE obj) { + RUBY_ASSERT(RB_TYPE_P(obj, T_CLASS) || RB_TYPE_P(obj, T_MODULE)); return RCLASS_EXT(obj)->shape_id; } @@ -115,7 +116,9 @@ static rb_shape_t* get_next_shape_internal(rb_shape_t* shape, ID id, VALUE obj, enum shape_type shape_type) { rb_shape_t *res = NULL; - RUBY_ASSERT(SHAPE_FROZEN != (enum shape_type)shape->type); + + RUBY_ASSERT(SHAPE_FROZEN != (enum shape_type)shape->type || RB_TYPE_P(obj, T_MODULE) || RB_TYPE_P(obj, T_CLASS)); + RB_VM_LOCK_ENTER(); { if (rb_shape_lookup_id(shape, id, shape_type)) { @@ -90,6 +90,13 @@ ROBJECT_SET_SHAPE_ID(VALUE obj, shape_id_t shape_id) RBASIC_SET_SHAPE_ID(obj, shape_id); } +static inline shape_id_t +RCLASS_SHAPE_ID(VALUE obj) +{ + RUBY_ASSERT(RB_TYPE_P(obj, T_CLASS) || RB_TYPE_P(obj, T_MODULE)); + return RBASIC_SHAPE_ID(obj); +} + #else static inline shape_id_t @@ -105,6 +112,15 @@ ROBJECT_SET_SHAPE_ID(VALUE obj, shape_id_t shape_id) RBASIC(obj)->flags &= SHAPE_FLAG_MASK; RBASIC(obj)->flags |= ((VALUE)(shape_id) << SHAPE_FLAG_SHIFT); } + +MJIT_SYMBOL_EXPORT_BEGIN +shape_id_t rb_rclass_shape_id(VALUE obj); +MJIT_SYMBOL_EXPORT_END + +static inline shape_id_t RCLASS_SHAPE_ID(VALUE obj) { + return rb_rclass_shape_id(obj); +} + #endif bool rb_shape_root_shape_p(rb_shape_t* shape); @@ -134,6 +150,14 @@ ROBJECT_IV_COUNT(VALUE obj) return ivc; } +static inline uint32_t +RCLASS_IV_COUNT(VALUE obj) +{ + RUBY_ASSERT(RB_TYPE_P(obj, RUBY_T_CLASS) || RB_TYPE_P(obj, RUBY_T_MODULE)); + uint32_t ivc = rb_shape_get_shape_by_id(RCLASS_SHAPE_ID(obj))->next_iv_index; + return ivc; +} + rb_shape_t * rb_shape_alloc(ID edge_name, rb_shape_t * parent); rb_shape_t * rb_shape_alloc_with_parent_id(ID edge_name, shape_id_t parent_id); diff --git a/variable.c b/variable.c index 4c5ae2c112..0f71b27418 100644 --- a/variable.c +++ b/variable.c @@ -111,18 +111,16 @@ rb_namespace_p(VALUE obj) static VALUE classname(VALUE klass, int *permanent) { - st_table *ivtbl; - st_data_t n; - *permanent = 0; if (!RCLASS_EXT(klass)) return Qnil; - if (!(ivtbl = RCLASS_IV_TBL(klass))) return Qnil; - if (st_lookup(ivtbl, (st_data_t)classpath, &n)) { + + VALUE classpathv = rb_ivar_lookup(klass, classpath, Qnil); + if (RTEST(classpathv)) { *permanent = 1; - return (VALUE)n; + return classpathv; } - if (st_lookup(ivtbl, (st_data_t)tmp_classpath, &n)) return (VALUE)n; - return Qnil; + + return rb_ivar_lookup(klass, tmp_classpath, Qnil);; } /* @@ -946,6 +944,8 @@ gen_ivtbl_get_unlocked(VALUE obj, ID id, struct gen_ivtbl **ivtbl) MJIT_FUNC_EXPORTED int rb_gen_ivtbl_get(VALUE obj, ID id, struct gen_ivtbl **ivtbl) { + RUBY_ASSERT(!RB_TYPE_P(obj, T_ICLASS)); + st_data_t data; int r = 0; @@ -1116,54 +1116,6 @@ gen_ivtbl_count(const struct gen_ivtbl *ivtbl) return n; } -static int -lock_st_lookup(st_table *tab, st_data_t key, st_data_t *value) -{ - int r; - RB_VM_LOCK_ENTER(); - { - r = st_lookup(tab, key, value); - } - RB_VM_LOCK_LEAVE(); - return r; -} - -static int -lock_st_delete(st_table *tab, st_data_t *key, st_data_t *value) -{ - int r; - RB_VM_LOCK_ENTER(); - { - r = st_delete(tab, key, value); - } - RB_VM_LOCK_LEAVE(); - return r; -} - -static int -lock_st_is_member(st_table *tab, st_data_t key) -{ - int r; - RB_VM_LOCK_ENTER(); - { - r = st_is_member(tab, key); - } - RB_VM_LOCK_LEAVE(); - return r; -} - -static int -lock_st_insert(st_table *tab, st_data_t key, st_data_t value) -{ - int r; - RB_VM_LOCK_ENTER(); - { - r = st_insert(tab, key, value); - } - RB_VM_LOCK_LEAVE(); - return r; -} - VALUE rb_ivar_lookup(VALUE obj, ID id, VALUE undef) { @@ -1181,21 +1133,39 @@ rb_ivar_lookup(VALUE obj, ID id, VALUE undef) case T_CLASS: case T_MODULE: { - st_data_t val; + bool found; + VALUE val; - if (RCLASS_IV_TBL(obj) && - lock_st_lookup(RCLASS_IV_TBL(obj), (st_data_t)id, &val)) { - if (rb_is_instance_id(id) && - UNLIKELY(!rb_ractor_main_p()) && - !rb_ractor_shareable_p(val)) { - rb_raise(rb_eRactorIsolationError, - "can not get unshareable values from instance variables of classes/modules from non-main Ractors"); + RB_VM_LOCK_ENTER(); + { +#if !SHAPE_IN_BASIC_FLAGS + shape_id = RCLASS_SHAPE_ID(obj); +#endif + + attr_index_t index = 0; + shape = rb_shape_get_shape_by_id(shape_id); + found = rb_shape_get_iv_index(shape, id, &index); + + if (found) { + ivar_list = RCLASS_IVPTR(obj); + RUBY_ASSERT(ivar_list); + + val = ivar_list[index]; + } + else { + val = undef; } - return val; } - else { - return undef; + RB_VM_LOCK_LEAVE(); + + if (found && + rb_is_instance_id(id) && + UNLIKELY(!rb_ractor_main_p()) && + !rb_ractor_shareable_p(val)) { + rb_raise(rb_eRactorIsolationError, + "can not get unshareable values from instance variables of classes/modules from non-main Ractors"); } + return val; } case T_OBJECT: { @@ -1247,19 +1217,25 @@ rb_ivar_delete(VALUE obj, ID id, VALUE undef) { rb_check_frozen(obj); - VALUE val = Qnil; + VALUE val = undef; attr_index_t index; switch (BUILTIN_TYPE(obj)) { case T_CLASS: case T_MODULE: IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(id); - if (RCLASS_IV_TBL(obj)) { - st_data_t id_data = (st_data_t)id, val; - if (lock_st_delete(RCLASS_IV_TBL(obj), &id_data, &val)) { - return (VALUE)val; + + RB_VM_LOCK_ENTER(); + { + rb_shape_t * shape = rb_shape_get_shape(obj); + if (rb_shape_get_iv_index(shape, id, &index)) { + rb_shape_transition_shape_remove_ivar(obj, id, shape); + val = RCLASS_IVPTR(obj)[index]; + RCLASS_IVPTR(obj)[index] = Qundef; } } + RB_VM_LOCK_LEAVE(); + break; case T_OBJECT: { rb_shape_t * shape = rb_shape_get_shape(obj); @@ -1267,7 +1243,6 @@ rb_ivar_delete(VALUE obj, ID id, VALUE undef) rb_shape_transition_shape_remove_ivar(obj, id, shape); val = ROBJECT_IVPTR(obj)[index]; ROBJECT_IVPTR(obj)[index] = Qundef; - return val; } break; @@ -1281,14 +1256,13 @@ rb_ivar_delete(VALUE obj, ID id, VALUE undef) rb_gen_ivtbl_get(obj, id, &ivtbl); val = ivtbl->ivptr[index]; ivtbl->ivptr[index] = Qundef; - return val; } break; } } - return undef; + return val; } VALUE @@ -1553,10 +1527,9 @@ ivar_set(VALUE obj, ID id, VALUE val) } case T_CLASS: case T_MODULE: - // TODO: Transition shapes on classes - //rb_shape_transition_shape(obj, id, rb_shape_get_shape_by_id(RCLASS_SHAPE_ID(obj))); IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(id); rb_class_ivar_set(obj, id, val); + break; default: generic_ivar_set(obj, id, val); @@ -1587,13 +1560,7 @@ rb_ivar_defined(VALUE obj, ID id) attr_index_t index; if (SPECIAL_CONST_P(obj)) return Qfalse; - switch (BUILTIN_TYPE(obj)) { - case T_CLASS: - case T_MODULE: - return RBOOL(RCLASS_IV_TBL(obj) && lock_st_is_member(RCLASS_IV_TBL(obj), (st_data_t)id)); - default: - return RBOOL(rb_shape_get_iv_index(rb_shape_get_shape(obj), id, &index)); - } + return RBOOL(rb_shape_get_iv_index(rb_shape_get_shape(obj), id, &index)); } typedef int rb_ivar_foreach_callback_func(ID key, VALUE val, st_data_t arg); @@ -1636,6 +1603,15 @@ gen_ivar_each(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg) iterate_over_shapes_with_callback(shape, ivtbl->ivptr, func, arg); } +static void +class_ivar_each(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg) +{ + RUBY_ASSERT(RB_TYPE_P(obj, T_CLASS) || RB_TYPE_P(obj, T_MODULE)); + + rb_shape_t* shape = rb_shape_get_shape(obj); + iterate_over_shapes_with_callback(shape, RCLASS_IVPTR(obj), func, arg); +} + void rb_copy_generic_ivar(VALUE clone, VALUE obj) { @@ -1720,13 +1696,11 @@ rb_ivar_foreach(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg) case T_CLASS: case T_MODULE: IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(0); - if (RCLASS_IV_TBL(obj)) { - RB_VM_LOCK_ENTER(); - { - st_foreach_safe(RCLASS_IV_TBL(obj), func, arg); - } - RB_VM_LOCK_LEAVE(); + RB_VM_LOCK_ENTER(); + { + class_ivar_each(obj, func, arg); } + RB_VM_LOCK_LEAVE(); break; default: if (FL_TEST(obj, FL_EXIVAR)) { @@ -1739,8 +1713,6 @@ rb_ivar_foreach(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg) st_index_t rb_ivar_count(VALUE obj) { - st_table *tbl; - if (SPECIAL_CONST_P(obj)) return 0; switch (BUILTIN_TYPE(obj)) { @@ -1758,8 +1730,22 @@ rb_ivar_count(VALUE obj) break; case T_CLASS: case T_MODULE: - if ((tbl = RCLASS_IV_TBL(obj)) != 0) { - return tbl->num_entries; + if (rb_shape_get_shape(obj)->next_iv_index > 0) { + st_index_t count = 0; + + RB_VM_LOCK_ENTER(); + { + st_index_t i, num = rb_shape_get_shape(obj)->next_iv_index; + const VALUE *const ivptr = RCLASS_IVPTR(obj); + for (i = count = 0; i < num; ++i) { + if (ivptr[i] != Qundef) { + count++; + } + } + } + RB_VM_LOCK_LEAVE(); + + return count; } break; default: @@ -1879,11 +1865,12 @@ rb_obj_remove_instance_variable(VALUE obj, VALUE name) case T_CLASS: case T_MODULE: IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(id); - if (RCLASS_IV_TBL(obj)) { - st_data_t id_data = (st_data_t)id, val; - if (lock_st_delete(RCLASS_IV_TBL(obj), &id_data, &val)) { - return (VALUE)val; - } + rb_shape_t * shape = rb_shape_get_shape(obj); + if (rb_shape_get_iv_index(shape, id, &index)) { + rb_shape_transition_shape_remove_ivar(obj, id, shape); + val = RCLASS_IVPTR(obj)[index]; + RCLASS_IVPTR(obj)[index] = Qundef; + return val; } break; case T_OBJECT: { @@ -2030,8 +2017,22 @@ autoload_data(VALUE mod, ID id) struct st_table *tbl; st_data_t val; + // If we are called with a non-origin ICLASS, fetch the autoload data from + // the original module. + if (RB_TYPE_P(mod, T_ICLASS)) { + if (FL_TEST_RAW(mod, RICLASS_IS_ORIGIN)) { + return 0; + } else { + mod = RBASIC(mod)->klass; + } + } + + RUBY_ASSERT(RB_TYPE_P(mod, T_CLASS) || RB_TYPE_P(mod, T_MODULE)); + // Look up the instance variable table for `autoload`, then index into that table with the given constant name `id`. - if (!st_lookup(RCLASS_IV_TBL(mod), autoload, &val) || !(tbl = check_autoload_table((VALUE)val)) || !st_lookup(tbl, (st_data_t)id, &val)) { + + VALUE tbl_value = rb_ivar_lookup(mod, autoload, 0); + if (!tbl_value || !(tbl = check_autoload_table(tbl_value)) || !st_lookup(tbl, (st_data_t)id, &val)) { return 0; } @@ -2229,23 +2230,14 @@ autoload_feature_lookup_or_create(VALUE feature, struct autoload_data **autoload static struct st_table * autoload_table_lookup_or_create(VALUE module) { - // Get or create an autoload table in the class instance variables: - struct st_table *table = RCLASS_IV_TBL(module); - VALUE autoload_table_value; - - if (table && st_lookup(table, (st_data_t)autoload, &autoload_table_value)) { - return check_autoload_table((VALUE)autoload_table_value); - } - - if (!table) { - table = RCLASS_IV_TBL(module) = st_init_numtable(); + VALUE autoload_table_value = rb_ivar_lookup(module, autoload, 0); + if (autoload_table_value) { + return check_autoload_table(autoload_table_value); + } else { + autoload_table_value = TypedData_Wrap_Struct(0, &autoload_table_type, 0); + rb_class_ivar_set(module, autoload, autoload_table_value); + return (DATA_PTR(autoload_table_value) = st_init_numtable()); } - - autoload_table_value = TypedData_Wrap_Struct(0, &autoload_table_type, 0); - st_add_direct(table, (st_data_t)autoload, (st_data_t)autoload_table_value); - - RB_OBJ_WRITTEN(module, Qnil, autoload_table_value); - return (DATA_PTR(autoload_table_value) = st_init_numtable()); } static VALUE @@ -2314,10 +2306,13 @@ autoload_delete(VALUE module, ID name) { RUBY_ASSERT_CRITICAL_SECTION_ENTER(); - st_data_t value, load = 0, key = name; + st_data_t load = 0, key = name; - if (st_lookup(RCLASS_IV_TBL(module), (st_data_t)autoload, &value)) { - struct st_table *table = check_autoload_table((VALUE)value); + RUBY_ASSERT(RB_TYPE_P(module, T_CLASS) || RB_TYPE_P(module, T_MODULE)); + + VALUE table_value = rb_ivar_lookup(module, autoload, 0); + if (table_value) { + struct st_table *table = check_autoload_table(table_value); st_delete(table, &key, &load); @@ -2342,8 +2337,7 @@ autoload_delete(VALUE module, ID name) // If the autoload table is empty, we can delete it. if (table->num_entries == 0) { - name = autoload; - st_delete(RCLASS_IV_TBL(module), &name, &value); + rb_attr_delete(module, autoload); } } } @@ -3129,10 +3123,7 @@ set_namespace_path_i(ID id, VALUE v, void *payload) return ID_TABLE_CONTINUE; } set_namespace_path(value, build_const_path(parental_path, id)); - if (RCLASS_IV_TBL(value)) { - st_data_t tmp = tmp_classpath; - st_delete(RCLASS_IV_TBL(value), &tmp, 0); - } + rb_attr_delete(value, tmp_classpath); return ID_TABLE_CONTINUE; } @@ -3469,8 +3460,20 @@ original_module(VALUE c) static int cvar_lookup_at(VALUE klass, ID id, st_data_t *v) { - if (!RCLASS_IV_TBL(klass)) return 0; - return st_lookup(RCLASS_IV_TBL(klass), (st_data_t)id, v); + if (RB_TYPE_P(klass, T_ICLASS)) { + if (FL_TEST_RAW(klass, RICLASS_IS_ORIGIN)) { + return 0; + } else { + // check the original module + klass = RBASIC(klass)->klass; + } + } + + VALUE n = rb_ivar_lookup(klass, id, Qundef); + if (n == Qundef) return 0; + + if (v) *v = n; + return 1; } static VALUE @@ -3489,8 +3492,6 @@ static void cvar_overtaken(VALUE front, VALUE target, ID id) { if (front && target != front) { - st_data_t did = (st_data_t)id; - if (original_module(front) != original_module(target)) { rb_raise(rb_eRuntimeError, "class variable % "PRIsVALUE" of %"PRIsVALUE" is overtaken by %"PRIsVALUE"", @@ -3498,7 +3499,7 @@ cvar_overtaken(VALUE front, VALUE target, ID id) rb_class_name(original_module(target))); } if (BUILTIN_TYPE(front) == T_CLASS) { - st_delete(RCLASS_IV_TBL(front), &did, 0); + rb_ivar_delete(front, id, Qundef); } } } @@ -3543,9 +3544,8 @@ find_cvar(VALUE klass, VALUE * front, VALUE * target, ID id) static void check_for_cvar_table(VALUE subclass, VALUE key) { - st_table *tbl = RCLASS_IV_TBL(subclass); - - if (tbl && st_lookup(tbl, key, NULL)) { + // Must not check ivar on ICLASS + if (!RB_TYPE_P(subclass, T_ICLASS) && RTEST(rb_ivar_defined(subclass, key))) { RB_DEBUG_COUNTER_INC(cvar_class_invalidate); ruby_vm_global_cvar_state++; return; @@ -3688,9 +3688,9 @@ mod_cvar_at(VALUE mod, void *data) if (!tbl) { tbl = st_init_numtable(); } - if (RCLASS_IV_TBL(mod)) { - st_foreach_safe(RCLASS_IV_TBL(mod), cv_i, (st_data_t)tbl); - } + mod = original_module(mod); + + rb_ivar_foreach(mod, cv_i, (st_data_t)tbl); return tbl; } @@ -3793,13 +3793,14 @@ VALUE rb_mod_remove_cvar(VALUE mod, VALUE name) { const ID id = id_for_var_message(mod, name, class, "wrong class variable name %1$s"); - st_data_t val, n = id; + st_data_t val; if (!id) { goto not_defined; } rb_check_frozen(mod); - if (RCLASS_IV_TBL(mod) && st_delete(RCLASS_IV_TBL(mod), &n, &val)) { + val = rb_ivar_delete(mod, id, Qundef); + if (val != Qundef) { return (VALUE)val; } if (rb_cvar_defined(mod, id)) { @@ -3834,30 +3835,63 @@ rb_iv_set(VALUE obj, const char *name, VALUE val) int rb_class_ivar_set(VALUE obj, ID key, VALUE value) { - if (!RCLASS_IV_TBL(obj)) { - RCLASS_IV_TBL(obj) = st_init_numtable(); + RUBY_ASSERT(RB_TYPE_P(obj, T_CLASS) || RB_TYPE_P(obj, T_MODULE)); + int found; + + RB_VM_LOCK_ENTER(); + { + rb_shape_t * shape = rb_shape_get_shape(obj); + attr_index_t idx; + found = rb_shape_get_iv_index(shape, key, &idx); + + if (found) { + // Changing an existing instance variable + RUBY_ASSERT(RCLASS_IVPTR(obj)); + + RCLASS_IVPTR(obj)[idx] = value; + RB_OBJ_WRITTEN(obj, Qundef, value); + } else { + // Creating and setting a new instance variable + + // Move to a shape which fits the new ivar + idx = shape->next_iv_index; + shape = rb_shape_get_next(shape, obj, key); + + // We always allocate a power of two sized IV array. This way we + // only need to realloc when we expand into a new power of two size + if ((idx & (idx - 1)) == 0) { + size_t newsize = idx ? idx * 2 : 1; + REALLOC_N(RCLASS_IVPTR(obj), VALUE, newsize); + } + + RUBY_ASSERT(RCLASS_IVPTR(obj)); + + RB_OBJ_WRITE(obj, &RCLASS_IVPTR(obj)[idx], value); + rb_shape_set_shape(obj, shape); + } } + RB_VM_LOCK_LEAVE(); - st_table *tbl = RCLASS_IV_TBL(obj); - int result = lock_st_insert(tbl, (st_data_t)key, (st_data_t)value); - RB_OBJ_WRITTEN(obj, Qundef, value); - return result; + return found; } static int -tbl_copy_i(st_data_t key, st_data_t value, st_data_t data) -{ - RB_OBJ_WRITTEN((VALUE)data, Qundef, (VALUE)value); +tbl_copy_i(st_data_t key, st_data_t val, st_data_t dest) { + rb_class_ivar_set(dest, key, val); + return ST_CONTINUE; } void rb_iv_tbl_copy(VALUE dst, VALUE src) { - st_table *orig_tbl = RCLASS_IV_TBL(src); - st_table *new_tbl = st_copy(orig_tbl); - st_foreach(new_tbl, tbl_copy_i, (st_data_t)dst); - RCLASS_IV_TBL(dst) = new_tbl; + RUBY_ASSERT(rb_type(dst) == rb_type(src)); + RUBY_ASSERT(RB_TYPE_P(dst, T_CLASS) || RB_TYPE_P(dst, T_MODULE)); + + RUBY_ASSERT(RCLASS_SHAPE_ID(dst) == ROOT_SHAPE_ID); + RUBY_ASSERT(!RCLASS_IVPTR(dst)); + + rb_ivar_foreach(src, tbl_copy_i, dst); } MJIT_FUNC_EXPORTED rb_const_entry_t * diff --git a/vm_insnhelper.c b/vm_insnhelper.c index 2434c5c3c6..cff4b9138a 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -1169,7 +1169,23 @@ vm_getivar(VALUE obj, ID id, const rb_iseq_t *iseq, IVC ic, const struct rb_call case T_CLASS: case T_MODULE: { - goto general_path; + if (UNLIKELY(!rb_ractor_main_p())) { + // For two reasons we can only use the fast path on the main + // ractor. + // First, only the main ractor is allowed to set ivars on classes + // and modules. So we can skip locking. + // Second, other ractors need to check the shareability of the + // values returned from the class ivars. + goto general_path; + } + + ivar_list = RCLASS_IVPTR(obj); + +#if !SHAPE_IN_BASIC_FLAGS + shape_id = RCLASS_SHAPE_ID(obj); +#endif + + break; } default: if (FL_TEST_RAW(obj, FL_EXIVAR)) { @@ -1507,15 +1523,13 @@ vm_getclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *reg_cfp, ID { const rb_cref_t *cref; - if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE()) { - VALUE v = Qundef; + if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE() && LIKELY(rb_ractor_main_p())) { RB_DEBUG_COUNTER_INC(cvar_read_inline_hit); - if (st_lookup(RCLASS_IV_TBL(ic->entry->class_value), (st_data_t)id, &v) && - LIKELY(rb_ractor_main_p())) { + VALUE v = rb_ivar_lookup(ic->entry->class_value, id, Qundef); + RUBY_ASSERT(v != Qundef); - return v; - } + return v; } cref = vm_get_cref(GET_EP()); |