diff options
Diffstat (limited to 'variable.c')
| -rw-r--r-- | variable.c | 1447 |
1 files changed, 689 insertions, 758 deletions
diff --git a/variable.c b/variable.c index 1cd1c604c3..687fa03631 100644 --- a/variable.c +++ b/variable.c @@ -20,12 +20,12 @@ #include "id.h" #include "id_table.h" #include "internal.h" +#include "internal/box.h" #include "internal/class.h" #include "internal/compilers.h" #include "internal/error.h" #include "internal/eval.h" #include "internal/hash.h" -#include "internal/namespace.h" #include "internal/object.h" #include "internal/gc.h" #include "internal/re.h" @@ -63,7 +63,7 @@ static VALUE autoload_mutex; static void check_before_mod_set(VALUE, ID, VALUE, const char *); static void setup_const_entry(rb_const_entry_t *, VALUE, VALUE, rb_const_flag_t); -static VALUE rb_const_search(VALUE klass, ID id, int exclude, int recurse, int visibility); +static VALUE rb_const_search(VALUE klass, ID id, int exclude, int recurse, int visibility, VALUE *found_in); static st_table *generic_fields_tbl_; typedef int rb_ivar_foreach_callback_func(ID key, VALUE val, st_data_t arg); @@ -279,7 +279,7 @@ set_sub_temporary_name(VALUE mod, VALUE name) * m.name #=> nil * * c = Class.new - * c.set_temporary_name("MyClass(with description)") + * c.set_temporary_name("MyClass(with description)") # => MyClass(with description) * * c.new # => #<MyClass(with description):0x0....> * @@ -321,6 +321,7 @@ rb_mod_set_temporary_name(VALUE mod, VALUE name) } name = rb_str_new_frozen(name); + RB_OBJ_SET_SHAREABLE(name); // Set the temporary classpath to the given name: RB_VM_LOCKING() { @@ -432,6 +433,7 @@ rb_set_class_path_string(VALUE klass, VALUE under, VALUE name) str = build_const_pathname(str, name); } + RB_OBJ_SET_SHAREABLE(str); RCLASS_SET_CLASSPATH(klass, str, permanent); } @@ -471,7 +473,7 @@ rb_path_to_class(VALUE pathname) if (!id) { goto undefined_class; } - c = rb_const_search(c, id, TRUE, FALSE, FALSE); + c = rb_const_search(c, id, TRUE, FALSE, FALSE, NULL); if (UNDEF_P(c)) goto undefined_class; if (!rb_namespace_p(c)) { rb_raise(rb_eTypeError, "%"PRIsVALUE" does not refer to class/module", @@ -531,7 +533,8 @@ struct rb_global_variable { rb_gvar_marker_t *marker; rb_gvar_compact_t *compactor; struct trace_var *trace; - bool namespace_ready; + bool box_ready; + bool box_dynamic; }; struct rb_global_entry { @@ -548,10 +551,10 @@ free_global_variable(struct rb_global_variable *var) struct trace_var *trace = var->trace; while (trace) { struct trace_var *next = trace->next; - xfree(trace); + SIZED_FREE(trace); trace = next; } - xfree(var); + SIZED_FREE(var); } static enum rb_id_table_iterator_result @@ -562,7 +565,7 @@ free_global_entry_i(VALUE val, void *arg) if (entry->var->counter == 0) { free_global_variable(entry->var); } - ruby_xfree(entry); + SIZED_FREE(entry); return ID_TABLE_DELETE; } @@ -610,10 +613,17 @@ rb_gvar_ractor_local(const char *name) } void -rb_gvar_namespace_ready(const char *name) +rb_gvar_box_ready(const char *name) { struct rb_global_entry *entry = rb_find_global_entry(rb_intern(name)); - entry->var->namespace_ready = true; + entry->var->box_ready = true; +} + +void +rb_gvar_box_dynamic(const char *name) +{ + struct rb_global_entry *entry = rb_find_global_entry(rb_intern(name)); + entry->var->box_dynamic = true; } static void @@ -643,7 +653,8 @@ rb_global_entry(ID id) var->block_trace = 0; var->trace = 0; - var->namespace_ready = false; + var->box_ready = false; + var->box_dynamic = false; rb_id_table_insert(rb_global_tbl, id, (VALUE)entry); } } @@ -860,7 +871,7 @@ rb_define_virtual_variable( static void rb_trace_eval(VALUE cmd, VALUE val) { - rb_eval_cmd_kw(cmd, rb_ary_new3(1, val), RB_NO_KEYWORDS); + rb_eval_cmd_call_kw(cmd, 1, &val, RB_NO_KEYWORDS); } VALUE @@ -900,7 +911,7 @@ remove_trace(struct rb_global_variable *var) next = trace->next; if (next->removed) { trace->next = next->next; - xfree(next); + SIZED_FREE(next); } else { trace = next; @@ -998,28 +1009,35 @@ rb_gvar_set_entry(struct rb_global_entry *entry, VALUE val) return val; } -#define USE_NAMESPACE_GVAR_TBL(ns,entry) \ - (NAMESPACE_OPTIONAL_P(ns) && \ - (!entry || !entry->var->namespace_ready || entry->var->setter != rb_gvar_readonly_setter)) +static inline bool +gvar_use_box_tbl(const rb_box_t *box, const struct rb_global_entry *entry) +{ + return BOX_USER_P(box) && + !entry->var->box_dynamic && + (!entry->var->box_ready || entry->var->setter != rb_gvar_readonly_setter); +} VALUE rb_gvar_set(ID id, VALUE val) { VALUE retval; - struct rb_global_entry *entry; - const rb_namespace_t *ns = rb_current_namespace(); + struct rb_global_entry *entry = NULL; + const rb_box_t *box = rb_current_box(); + bool use_box_tbl = false; RB_VM_LOCKING() { entry = rb_global_entry(id); - if (USE_NAMESPACE_GVAR_TBL(ns, entry)) { - rb_hash_aset(ns->gvar_tbl, rb_id2sym(entry->id), val); + if (gvar_use_box_tbl(box, entry)) { + use_box_tbl = true; + rb_hash_aset(box->gvar_tbl, rb_id2sym(entry->id), val); retval = val; // TODO: think about trace } - else { - retval = rb_gvar_set_entry(entry, val); - } + } + + if (!use_box_tbl) { + retval = rb_gvar_set_entry(entry, val); } return retval; } @@ -1034,29 +1052,37 @@ VALUE rb_gvar_get(ID id) { VALUE retval, gvars, key; - const rb_namespace_t *ns = rb_current_namespace(); + const rb_box_t *box = rb_current_box(); + bool use_box_tbl = false; + struct rb_global_entry *entry = NULL; + struct rb_global_variable *var = NULL; // TODO: use lock-free rb_id_table when it's available for use (doesn't yet exist) RB_VM_LOCKING() { - struct rb_global_entry *entry = rb_global_entry(id); - struct rb_global_variable *var = entry->var; + entry = rb_global_entry(id); + var = entry->var; - if (USE_NAMESPACE_GVAR_TBL(ns, entry)) { - gvars = ns->gvar_tbl; + if (gvar_use_box_tbl(box, entry)) { + use_box_tbl = true; + gvars = box->gvar_tbl; key = rb_id2sym(entry->id); if (RTEST(rb_hash_has_key(gvars, key))) { // this gvar is already cached retval = rb_hash_aref(gvars, key); } else { - retval = (*var->getter)(entry->id, var->data); - if (rb_obj_respond_to(retval, rb_intern("clone"), 1)) { - retval = rb_funcall(retval, rb_intern("clone"), 0); + RB_VM_UNLOCK(); + { + retval = (*var->getter)(entry->id, var->data); + if (rb_obj_respond_to(retval, rb_intern("clone"), 1)) { + retval = rb_funcall(retval, rb_intern("clone"), 0); + } } + RB_VM_LOCK(); rb_hash_aset(gvars, key, retval); } } - else { - retval = (*var->getter)(entry->id, var->data); - } + } + if (!use_box_tbl) { + retval = (*var->getter)(entry->id, var->data); } return retval; } @@ -1112,7 +1138,7 @@ rb_f_global_variables(void) if (!rb_ractor_main_p()) { rb_raise(rb_eRactorIsolationError, "can not access global variables from non-main Ractors"); } - /* gvar access (get/set) in namespaces creates gvar entries globally */ + /* gvar access (get/set) in boxes creates gvar entries globally */ rb_id_table_foreach(rb_global_tbl, gvar_i, (void *)ary); if (!NIL_P(backref)) { @@ -1150,13 +1176,14 @@ rb_alias_variable(ID name1, ID name2) RB_VM_LOCKING() { entry2 = rb_global_entry(name2); if (!rb_id_table_lookup(gtbl, name1, &data1)) { - entry1 = ALLOC(struct rb_global_entry); + entry1 = ZALLOC(struct rb_global_entry); entry1->id = name1; rb_id_table_insert(gtbl, name1, (VALUE)entry1); } else if ((entry1 = (struct rb_global_entry *)data1)->var != entry2->var) { struct rb_global_variable *var = entry1->var; if (var->block_trace) { + RB_VM_UNLOCK(); rb_raise(rb_eRuntimeError, "can't alias in tracer"); } var->counter--; @@ -1164,7 +1191,7 @@ rb_alias_variable(ID name1, ID name2) free_global_variable(var); } } - if (entry1) { + if (entry1->var != entry2->var) { entry2->var->counter++; entry1->var = entry2->var; } @@ -1181,10 +1208,23 @@ IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(ID id) } } -#define CVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR() \ - if (UNLIKELY(!rb_ractor_main_p())) { \ - rb_raise(rb_eRactorIsolationError, "can not access class variables from non-main Ractors"); \ - } +static void +CVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(VALUE klass, ID id) +{ + if (UNLIKELY(!rb_ractor_main_p())) { + rb_raise(rb_eRactorIsolationError, "can not set class variables from non-main Ractors (%"PRIsVALUE" from %"PRIsVALUE")", rb_id2str(id), klass); + } +} + +static void +cvar_read_ractor_check(VALUE klass, ID id, VALUE val) +{ + if (UNLIKELY(!rb_ractor_main_p()) && !rb_ractor_shareable_p(val)) { + rb_raise(rb_eRactorIsolationError, + "can not read non-shareable class variable %"PRIsVALUE" from non-main Ractors (%"PRIsVALUE")", + rb_id2str(id), klass); + } +} static inline void ivar_ractor_check(VALUE obj, ID id) @@ -1223,20 +1263,29 @@ rb_mark_generic_ivar(VALUE obj) } VALUE +rb_obj_fields_generic_uncached(VALUE obj) +{ + VALUE fields_obj = 0; + RB_VM_LOCKING() { + if (!st_lookup(generic_fields_tbl_, (st_data_t)obj, (st_data_t *)&fields_obj)) { + rb_bug("Object is missing entry in generic_fields_tbl"); + } + } + return fields_obj; +} + +VALUE rb_obj_fields(VALUE obj, ID field_name) { RUBY_ASSERT(!RB_TYPE_P(obj, T_IMEMO)); ivar_ractor_check(obj, field_name); VALUE fields_obj = 0; - if (rb_shape_obj_has_fields(obj)) { + if (rb_obj_shape_has_fields(obj)) { switch (BUILTIN_TYPE(obj)) { case T_DATA: - if (LIKELY(RTYPEDDATA_P(obj))) { - fields_obj = RTYPEDDATA(obj)->fields_obj; - break; - } - goto generic_fields; + fields_obj = RTYPEDDATA(obj)->fields_obj; + break; case T_STRUCT: if (LIKELY(!FL_TEST_RAW(obj, RSTRUCT_GEN_FIELDS))) { fields_obj = RSTRUCT_FIELDS_OBJ(obj); @@ -1247,15 +1296,12 @@ rb_obj_fields(VALUE obj, ID field_name) generic_fields: { rb_execution_context_t *ec = GET_EC(); - if (ec->gen_fields_cache.obj == obj && rb_imemo_fields_owner(ec->gen_fields_cache.fields_obj) == obj) { + if (ec->gen_fields_cache.obj == obj && !UNDEF_P(ec->gen_fields_cache.fields_obj) && rb_imemo_fields_owner(ec->gen_fields_cache.fields_obj) == obj) { fields_obj = ec->gen_fields_cache.fields_obj; + RUBY_ASSERT(fields_obj == rb_obj_fields_generic_uncached(obj)); } else { - RB_VM_LOCKING() { - if (!st_lookup(generic_fields_tbl_, (st_data_t)obj, (st_data_t *)&fields_obj)) { - rb_bug("Object is missing entry in generic_fields_tbl"); - } - } + fields_obj = rb_obj_fields_generic_uncached(obj); ec->gen_fields_cache.fields_obj = fields_obj; ec->gen_fields_cache.obj = obj; } @@ -1268,15 +1314,12 @@ rb_obj_fields(VALUE obj, ID field_name) void rb_free_generic_ivar(VALUE obj) { - if (rb_obj_exivar_p(obj)) { + if (rb_obj_gen_fields_p(obj)) { st_data_t key = (st_data_t)obj, value; switch (BUILTIN_TYPE(obj)) { case T_DATA: - if (LIKELY(RTYPEDDATA_P(obj))) { - RB_OBJ_WRITE(obj, &RTYPEDDATA(obj)->fields_obj, 0); - break; - } - goto generic_fields; + RB_OBJ_WRITE(obj, &RTYPEDDATA(obj)->fields_obj, 0); + break; case T_STRUCT: if (LIKELY(!FL_TEST_RAW(obj, RSTRUCT_GEN_FIELDS))) { RSTRUCT_SET_FIELDS_OBJ(obj, 0); @@ -1286,36 +1329,44 @@ rb_free_generic_ivar(VALUE obj) default: generic_fields: { + // Other EC may have stale caches, so fields_obj should be + // invalidated and the GC will replace with Qundef rb_execution_context_t *ec = GET_EC(); if (ec->gen_fields_cache.obj == obj) { ec->gen_fields_cache.obj = Qundef; ec->gen_fields_cache.fields_obj = Qundef; } RB_VM_LOCKING() { - st_delete(generic_fields_tbl_no_ractor_check(), &key, &value); + if (!st_delete(generic_fields_tbl_no_ractor_check(), &key, &value)) { + rb_bug("Object is missing entry in generic_fields_tbl"); + } } } } - RBASIC_SET_SHAPE_ID(obj, ROOT_SHAPE_ID); + RBASIC_SET_SHAPE_ID(obj, rb_shape_layout(RBASIC_SHAPE_ID(obj)) | ROOT_SHAPE_ID); } } -void +static void rb_obj_set_fields(VALUE obj, VALUE fields_obj, ID field_name, VALUE original_fields_obj) { ivar_ractor_check(obj, field_name); + if (!fields_obj) { + RUBY_ASSERT(original_fields_obj); + rb_free_generic_ivar(obj); + rb_imemo_fields_clear(original_fields_obj); + return; + } + RUBY_ASSERT(IMEMO_TYPE_P(fields_obj, imemo_fields)); RUBY_ASSERT(!original_fields_obj || IMEMO_TYPE_P(original_fields_obj, imemo_fields)); if (fields_obj != original_fields_obj) { switch (BUILTIN_TYPE(obj)) { case T_DATA: - if (LIKELY(RTYPEDDATA_P(obj))) { - RB_OBJ_WRITE(obj, &RTYPEDDATA(obj)->fields_obj, fields_obj); - break; - } - goto generic_fields; + RB_OBJ_WRITE(obj, &RTYPEDDATA(obj)->fields_obj, fields_obj); + break; case T_STRUCT: if (LIKELY(!FL_TEST_RAW(obj, RSTRUCT_GEN_FIELDS))) { RSTRUCT_SET_FIELDS_OBJ(obj, fields_obj); @@ -1344,7 +1395,7 @@ rb_obj_set_fields(VALUE obj, VALUE fields_obj, ID field_name, VALUE original_fie } } - RBASIC_SET_SHAPE_ID(obj, RBASIC_SHAPE_ID(fields_obj)); + RBASIC_SET_SHAPE_ID(obj, rb_shape_layout(RBASIC_SHAPE_ID(obj)) | RBASIC_SHAPE_ID(fields_obj)); } void @@ -1362,47 +1413,35 @@ rb_obj_field_get(VALUE obj, shape_id_t target_shape_id) RUBY_ASSERT(!SPECIAL_CONST_P(obj)); RUBY_ASSERT(RSHAPE_TYPE_P(target_shape_id, SHAPE_IVAR) || RSHAPE_TYPE_P(target_shape_id, SHAPE_OBJ_ID)); - if (rb_shape_too_complex_p(target_shape_id)) { - st_table *fields_hash; - switch (BUILTIN_TYPE(obj)) { - case T_CLASS: - case T_MODULE: - fields_hash = rb_imemo_fields_complex_tbl(RCLASS_WRITABLE_FIELDS_OBJ(obj)); - break; - case T_OBJECT: - fields_hash = ROBJECT_FIELDS_HASH(obj); - break; - case T_IMEMO: - fields_hash = rb_imemo_fields_complex_tbl(obj); - break; - default: - fields_hash = rb_imemo_fields_complex_tbl(rb_obj_fields(obj, RSHAPE_EDGE_NAME(target_shape_id))); - break; - } - VALUE value = Qundef; - st_lookup(fields_hash, RSHAPE_EDGE_NAME(target_shape_id), &value); - RUBY_ASSERT(!UNDEF_P(value)); - return value; - } + VALUE fields_obj; - attr_index_t attr_index = RSHAPE_INDEX(target_shape_id); - VALUE *fields; switch (BUILTIN_TYPE(obj)) { case T_CLASS: case T_MODULE: - fields = rb_imemo_fields_ptr(RCLASS_WRITABLE_FIELDS_OBJ(obj)); + fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj); break; case T_OBJECT: - fields = ROBJECT_FIELDS(obj); + fields_obj = obj; break; case T_IMEMO: - fields = rb_imemo_fields_ptr(obj); + RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_fields)); + fields_obj = obj; break; default: - fields = rb_imemo_fields_ptr(rb_obj_fields(obj, RSHAPE_EDGE_NAME(target_shape_id))); + fields_obj = rb_obj_fields(obj, RSHAPE_EDGE_NAME(target_shape_id)); break; } - return fields[attr_index]; + + if (UNLIKELY(rb_shape_complex_p(target_shape_id))) { + st_table *fields_hash = rb_imemo_fields_complex_tbl(fields_obj); + VALUE value = Qundef; + st_lookup(fields_hash, RSHAPE_EDGE_NAME(target_shape_id), &value); + RUBY_ASSERT(!UNDEF_P(value)); + return value; + } + + attr_index_t index = RSHAPE_INDEX(target_shape_id); + return rb_imemo_fields_ptr(fields_obj)[index]; } VALUE @@ -1410,93 +1449,54 @@ rb_ivar_lookup(VALUE obj, ID id, VALUE undef) { if (SPECIAL_CONST_P(obj)) return undef; - shape_id_t shape_id; - VALUE *ivar_list; + VALUE fields_obj; switch (BUILTIN_TYPE(obj)) { case T_CLASS: case T_MODULE: { - VALUE val = undef; - VALUE fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj); - if (fields_obj) { - val = rb_ivar_lookup(fields_obj, id, undef); - } - + VALUE val = rb_ivar_lookup(RCLASS_WRITABLE_FIELDS_OBJ(obj), id, undef); if (val != undef && rb_is_instance_id(id) && UNLIKELY(!rb_ractor_main_p()) && !rb_ractor_shareable_p(val)) { rb_raise(rb_eRactorIsolationError, - "can not get unshareable values from instance variables of classes/modules from non-main Ractors"); + "can not get unshareable values from instance variables of classes/modules from non-main Ractors (%"PRIsVALUE" from %"PRIsVALUE")", + rb_id2str(id), obj); } return val; } case T_IMEMO: // Handled like T_OBJECT - { - RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_fields)); - shape_id = RBASIC_SHAPE_ID(obj); + RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_fields)); + fields_obj = obj; + break; + case T_OBJECT: + fields_obj = obj; + break; + default: + fields_obj = rb_obj_fields(obj, id); + break; + } - if (rb_shape_too_complex_p(shape_id)) { - st_table *iv_table = rb_imemo_fields_complex_tbl(obj); - VALUE val; - if (rb_st_lookup(iv_table, (st_data_t)id, (st_data_t *)&val)) { - return val; - } - else { - return undef; - } - } + if (!fields_obj) { + return undef; + } - RUBY_ASSERT(!rb_shape_obj_too_complex_p(obj)); - ivar_list = rb_imemo_fields_ptr(obj); - break; - } - case T_OBJECT: - { - shape_id = RBASIC_SHAPE_ID(obj); - if (rb_shape_too_complex_p(shape_id)) { - st_table *iv_table = ROBJECT_FIELDS_HASH(obj); - VALUE val; - if (rb_st_lookup(iv_table, (st_data_t)id, (st_data_t *)&val)) { - return val; - } - else { - return undef; - } - } + shape_id_t shape_id = RBASIC_SHAPE_ID(fields_obj); - RUBY_ASSERT(!rb_shape_obj_too_complex_p(obj)); - ivar_list = ROBJECT_FIELDS(obj); - break; - } - default: - { - shape_id = RBASIC_SHAPE_ID(obj); - VALUE fields_obj = rb_obj_fields(obj, id); - if (fields_obj) { - if (rb_shape_obj_too_complex_p(fields_obj)) { - VALUE val; - if (rb_st_lookup(rb_imemo_fields_complex_tbl(fields_obj), (st_data_t)id, (st_data_t *)&val)) { - return val; - } - else { - return undef; - } - } - ivar_list = rb_imemo_fields_ptr(fields_obj); - } - else { - return undef; - } - break; + if (UNLIKELY(rb_shape_complex_p(shape_id))) { + st_table *iv_table = rb_imemo_fields_complex_tbl(fields_obj); + VALUE val; + if (rb_st_lookup(iv_table, (st_data_t)id, (st_data_t *)&val)) { + return val; } + return undef; } attr_index_t index = 0; if (rb_shape_get_iv_index(shape_id, id, &index)) { - return ivar_list[index]; + return rb_imemo_fields_ptr(fields_obj)[index]; } return undef; @@ -1511,175 +1511,97 @@ rb_ivar_get(VALUE obj, ID id) } VALUE -rb_attr_get(VALUE obj, ID id) +rb_ivar_get_at(VALUE obj, attr_index_t index, ID id) { - return rb_ivar_lookup(obj, id, Qnil); -} + RUBY_ASSERT(rb_is_instance_id(id)); + // Used by JITs, but never for T_OBJECT. -static VALUE -rb_ivar_delete(VALUE obj, ID id, VALUE undef) -{ - rb_check_frozen(obj); - - VALUE val = undef; - if (BUILTIN_TYPE(obj) == T_CLASS || BUILTIN_TYPE(obj) == T_MODULE) { - IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(id); + switch (BUILTIN_TYPE(obj)) { + case T_OBJECT: + UNREACHABLE_RETURN(Qundef); + case T_CLASS: + case T_MODULE: + { + VALUE fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj); + VALUE val = rb_imemo_fields_ptr(fields_obj)[index]; - VALUE fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj); - if (fields_obj) { - if (rb_multi_ractor_p()) { - fields_obj = rb_imemo_fields_clone(fields_obj); - val = rb_ivar_delete(fields_obj, id, undef); - RCLASS_WRITABLE_SET_FIELDS_OBJ(obj, fields_obj); - } - else { - val = rb_ivar_delete(fields_obj, id, undef); + if (UNLIKELY(!rb_ractor_main_p()) && !rb_ractor_shareable_p(val)) { + rb_raise(rb_eRactorIsolationError, + "can not get unshareable values from instance variables of classes/modules from non-main Ractors"); } - } - return val; - } - - shape_id_t old_shape_id = rb_obj_shape_id(obj); - if (rb_shape_too_complex_p(old_shape_id)) { - goto too_complex; - } - - shape_id_t removed_shape_id = 0; - shape_id_t next_shape_id = rb_shape_transition_remove_ivar(obj, id, &removed_shape_id); - - if (next_shape_id == old_shape_id) { - return undef; - } - if (UNLIKELY(rb_shape_too_complex_p(next_shape_id))) { - rb_evict_fields_to_hash(obj); - goto too_complex; + return val; + } + default: + { + VALUE fields_obj = rb_obj_fields(obj, id); + return rb_imemo_fields_ptr(fields_obj)[index]; + } } +} - RUBY_ASSERT(RSHAPE_LEN(next_shape_id) == RSHAPE_LEN(old_shape_id) - 1); +VALUE +rb_ivar_get_at_no_ractor_check(VALUE obj, attr_index_t index) +{ + // Used by JITs, but never for T_OBJECT. - VALUE *fields; - switch(BUILTIN_TYPE(obj)) { + VALUE fields_obj; + switch (BUILTIN_TYPE(obj)) { + case T_OBJECT: + UNREACHABLE_RETURN(Qundef); case T_CLASS: case T_MODULE: - rb_bug("Unreachable"); - break; - case T_IMEMO: - RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_fields)); - fields = rb_imemo_fields_ptr(obj); - break; - case T_OBJECT: - fields = ROBJECT_FIELDS(obj); + fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj); break; - default: { - VALUE fields_obj = rb_obj_fields(obj, id); - fields = rb_imemo_fields_ptr(fields_obj); + default: + fields_obj = rb_obj_fields_no_ractor_check(obj); break; - } - } - - RUBY_ASSERT(removed_shape_id != INVALID_SHAPE_ID); - - attr_index_t removed_index = RSHAPE_INDEX(removed_shape_id); - val = fields[removed_index]; - - attr_index_t new_fields_count = RSHAPE_LEN(next_shape_id); - if (new_fields_count) { - size_t trailing_fields = new_fields_count - removed_index; - - MEMMOVE(&fields[removed_index], &fields[removed_index + 1], VALUE, trailing_fields); - } - else { - rb_free_generic_ivar(obj); } - - if (RB_TYPE_P(obj, T_OBJECT) && - !RB_FL_TEST_RAW(obj, ROBJECT_EMBED) && - rb_obj_embedded_size(new_fields_count) <= rb_gc_obj_slot_size(obj)) { - // Re-embed objects when instances become small enough - // This is necessary because YJIT assumes that objects with the same shape - // have the same embeddedness for efficiency (avoid extra checks) - RB_FL_SET_RAW(obj, ROBJECT_EMBED); - MEMCPY(ROBJECT_FIELDS(obj), fields, VALUE, new_fields_count); - xfree(fields); - } - rb_obj_set_shape_id(obj, next_shape_id); - - return val; - -too_complex: - { - st_table *table = NULL; - switch (BUILTIN_TYPE(obj)) { - case T_CLASS: - case T_MODULE: - rb_bug("Unreachable"); - break; - - case T_IMEMO: - RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_fields)); - table = rb_imemo_fields_complex_tbl(obj); - break; - - case T_OBJECT: - table = ROBJECT_FIELDS_HASH(obj); - break; - - default: { - VALUE fields_obj = rb_obj_fields(obj, id); - table = rb_imemo_fields_complex_tbl(fields_obj); - break; - } - } - - if (table) { - if (!st_delete(table, (st_data_t *)&id, (st_data_t *)&val)) { - val = undef; - } - } - } - - return val; + return rb_imemo_fields_ptr(fields_obj)[index]; } VALUE -rb_attr_delete(VALUE obj, ID id) +rb_attr_get(VALUE obj, ID id) { - return rb_ivar_delete(obj, id, Qnil); + return rb_ivar_lookup(obj, id, Qnil); } +void rb_obj_copy_fields_to_hash_table(VALUE obj, st_table *table); +static VALUE imemo_fields_complex_from_obj(VALUE owner, VALUE source_fields_obj, shape_id_t shape_id); + static shape_id_t -obj_transition_too_complex(VALUE obj, st_table *table) +obj_transition_complex(VALUE obj, st_table *table) { - if (BUILTIN_TYPE(obj) == T_CLASS || BUILTIN_TYPE(obj) == T_MODULE) { - return obj_transition_too_complex(RCLASS_WRITABLE_ENSURE_FIELDS_OBJ(obj), table); - } - - RUBY_ASSERT(!rb_shape_obj_too_complex_p(obj)); - shape_id_t shape_id = rb_shape_transition_complex(obj); + RUBY_ASSERT(!rb_obj_shape_complex_p(obj)); + shape_id_t shape_id = rb_obj_shape_transition_complex(obj); switch (BUILTIN_TYPE(obj)) { case T_OBJECT: { VALUE *old_fields = NULL; - if (!(RBASIC(obj)->flags & ROBJECT_EMBED)) { + uint32_t old_fields_len = 0; + if (FL_TEST_RAW(obj, ROBJECT_HEAP)) { old_fields = ROBJECT_FIELDS(obj); + old_fields_len = ROBJECT_FIELDS_CAPACITY(obj); + } + else { + FL_SET_RAW(obj, ROBJECT_HEAP); } RBASIC_SET_SHAPE_ID(obj, shape_id); ROBJECT_SET_FIELDS_HASH(obj, table); if (old_fields) { - xfree(old_fields); + SIZED_FREE_N(old_fields, old_fields_len); } } break; case T_CLASS: case T_MODULE: - rb_bug("Unreachable"); + case T_IMEMO: + UNREACHABLE; break; default: { - VALUE fields_obj = rb_imemo_fields_new_complex_tbl(rb_obj_class(obj), table); - RBASIC_SET_SHAPE_ID(fields_obj, shape_id); + VALUE fields_obj = rb_imemo_fields_new_complex_tbl(obj, shape_id, table, RB_OBJ_SHAREABLE_P(obj)); rb_obj_replace_fields(obj, fields_obj); } } @@ -1687,194 +1609,214 @@ obj_transition_too_complex(VALUE obj, st_table *table) return shape_id; } -void -rb_obj_init_too_complex(VALUE obj, st_table *table) -{ - // This method is meant to be called on newly allocated object. - RUBY_ASSERT(!rb_shape_obj_too_complex_p(obj)); - RUBY_ASSERT(rb_shape_canonical_p(RBASIC_SHAPE_ID(obj))); - RUBY_ASSERT(RSHAPE_LEN(RBASIC_SHAPE_ID(obj)) == 0); - - obj_transition_too_complex(obj, table); -} - -static int -imemo_fields_complex_from_obj_i(ID key, VALUE val, st_data_t arg) -{ - VALUE fields = (VALUE)arg; - st_table *table = rb_imemo_fields_complex_tbl(fields); - - RUBY_ASSERT(!st_lookup(table, (st_data_t)key, NULL)); - st_add_direct(table, (st_data_t)key, (st_data_t)val); - RB_OBJ_WRITTEN(fields, Qundef, val); - - return ST_CONTINUE; -} - -static VALUE -imemo_fields_complex_from_obj(VALUE owner, VALUE source_fields_obj, shape_id_t shape_id) -{ - attr_index_t len = source_fields_obj ? RSHAPE_LEN(RBASIC_SHAPE_ID(source_fields_obj)) : 0; - VALUE fields_obj = rb_imemo_fields_new_complex(owner, len + 1); - - rb_field_foreach(source_fields_obj, imemo_fields_complex_from_obj_i, (st_data_t)fields_obj, false); - RBASIC_SET_SHAPE_ID(fields_obj, shape_id); - - return fields_obj; -} - -static VALUE -imemo_fields_copy_capa(VALUE owner, VALUE source_fields_obj, attr_index_t new_size) -{ - VALUE fields_obj = rb_imemo_fields_new(owner, new_size); - if (source_fields_obj) { - attr_index_t fields_count = RSHAPE_LEN(RBASIC_SHAPE_ID(source_fields_obj)); - VALUE *fields = rb_imemo_fields_ptr(fields_obj); - MEMCPY(fields, rb_imemo_fields_ptr(source_fields_obj), VALUE, fields_count); - RBASIC_SET_SHAPE_ID(fields_obj, RBASIC_SHAPE_ID(source_fields_obj)); - for (attr_index_t i = 0; i < fields_count; i++) { - RB_OBJ_WRITTEN(fields_obj, Qundef, fields[i]); - } - } - return fields_obj; -} - -void rb_obj_copy_fields_to_hash_table(VALUE obj, st_table *table); - // Copy all object fields, including ivars and internal object_id, etc -shape_id_t +static shape_id_t rb_evict_fields_to_hash(VALUE obj) { - RUBY_ASSERT(!rb_shape_obj_too_complex_p(obj)); + RUBY_ASSERT(!rb_obj_shape_complex_p(obj)); st_table *table = st_init_numtable_with_size(RSHAPE_LEN(RBASIC_SHAPE_ID(obj))); rb_obj_copy_fields_to_hash_table(obj, table); - shape_id_t new_shape_id = obj_transition_too_complex(obj, table); + shape_id_t new_shape_id = obj_transition_complex(obj, table); - RUBY_ASSERT(rb_shape_obj_too_complex_p(obj)); + RUBY_ASSERT(rb_obj_shape_complex_p(obj)); return new_shape_id; } void rb_evict_ivars_to_hash(VALUE obj) { - RUBY_ASSERT(!rb_shape_obj_too_complex_p(obj)); + RUBY_ASSERT(!rb_obj_shape_complex_p(obj)); st_table *table = st_init_numtable_with_size(rb_ivar_count(obj)); // Evacuate all previous values from shape into id_table rb_obj_copy_ivs_to_hash_table(obj, table); - obj_transition_too_complex(obj, table); + obj_transition_complex(obj, table); - RUBY_ASSERT(rb_shape_obj_too_complex_p(obj)); + RUBY_ASSERT(rb_obj_shape_complex_p(obj)); } -struct general_ivar_set_result { - attr_index_t index; - bool existing; -}; +static VALUE +rb_ivar_delete(VALUE obj, ID id, VALUE undef) +{ + rb_check_frozen(obj); -static struct general_ivar_set_result -general_ivar_set(VALUE obj, ID id, VALUE val, void *data, - VALUE *(*shape_fields_func)(VALUE, void *), - void (*shape_resize_fields_func)(VALUE, attr_index_t, attr_index_t, void *), - void (*set_shape_id_func)(VALUE, shape_id_t, void *), - shape_id_t (*transition_too_complex_func)(VALUE, void *), - st_table *(*too_complex_table_func)(VALUE, void *)) -{ - struct general_ivar_set_result result = { - .index = 0, - .existing = true - }; + VALUE val = undef; + VALUE fields_obj; + bool concurrent = false; + int type = BUILTIN_TYPE(obj); - shape_id_t current_shape_id = RBASIC_SHAPE_ID(obj); + switch(type) { + case T_CLASS: + case T_MODULE: + IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(id); + + fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj); + if (rb_multi_ractor_p()) { + concurrent = true; + } + break; + case T_OBJECT: + fields_obj = obj; + break; + default: { + fields_obj = rb_obj_fields(obj, id); + break; + } + } - if (UNLIKELY(rb_shape_too_complex_p(current_shape_id))) { - goto too_complex; + if (!fields_obj) { + return undef; } - attr_index_t index; - if (!rb_shape_get_iv_index(current_shape_id, id, &index)) { - result.existing = false; + const VALUE original_fields_obj = fields_obj; + if (concurrent) { + fields_obj = rb_imemo_fields_clone(fields_obj); + } + + shape_id_t old_shape_id = RBASIC_SHAPE_ID(fields_obj); + shape_id_t removed_shape_id; + shape_id_t next_shape_id = rb_obj_shape_transition_remove_ivar(fields_obj, id, &removed_shape_id); - index = RSHAPE_LEN(current_shape_id); - if (index >= SHAPE_MAX_FIELDS) { - rb_raise(rb_eArgError, "too many instance variables"); + if (UNLIKELY(rb_shape_complex_p(next_shape_id))) { + if (UNLIKELY(!rb_shape_complex_p(old_shape_id))) { + if (type == T_OBJECT) { + rb_evict_fields_to_hash(obj); + } + else { + fields_obj = imemo_fields_complex_from_obj(obj, fields_obj, next_shape_id); + } + } + st_data_t key = id; + if (!st_delete(rb_imemo_fields_complex_tbl(fields_obj), &key, (st_data_t *)&val)) { + val = undef; + } + } + else { + if (next_shape_id == old_shape_id) { + return undef; } - shape_id_t next_shape_id = rb_shape_transition_add_ivar(obj, id); - if (UNLIKELY(rb_shape_too_complex_p(next_shape_id))) { - current_shape_id = transition_too_complex_func(obj, data); - goto too_complex; + RUBY_ASSERT(removed_shape_id != INVALID_SHAPE_ID); + RUBY_ASSERT(RSHAPE_LEN(next_shape_id) == RSHAPE_LEN(old_shape_id) - 1); + + VALUE *fields = rb_imemo_fields_ptr(fields_obj); + attr_index_t removed_index = RSHAPE_INDEX(removed_shape_id); + val = fields[removed_index]; + + attr_index_t new_fields_count = RSHAPE_LEN(next_shape_id); + if (new_fields_count) { + size_t trailing_fields = new_fields_count - removed_index; + + MEMMOVE(&fields[removed_index], &fields[removed_index + 1], VALUE, trailing_fields); + RUBY_ASSERT(rb_shape_layout(next_shape_id) == SHAPE_ID_LAYOUT_ROBJECT); + RBASIC_SET_SHAPE_ID(fields_obj, next_shape_id); + + if (FL_TEST_RAW(fields_obj, OBJ_FIELD_HEAP)) { + if (rb_obj_embedded_size(new_fields_count) <= rb_gc_obj_slot_size(fields_obj)) { + // Re-embed objects when instances become small enough + // This is necessary because YJIT assumes that objects with the same shape + // have the same embeddedness for efficiency (avoid extra checks) + FL_UNSET_RAW(fields_obj, ROBJECT_HEAP); + MEMCPY(rb_imemo_fields_ptr(fields_obj), fields, VALUE, new_fields_count); + SIZED_FREE_N(fields, RSHAPE_CAPACITY(old_shape_id)); + } + else if (RSHAPE_CAPACITY(old_shape_id) != RSHAPE_CAPACITY(next_shape_id)) { + IMEMO_OBJ_FIELDS(fields_obj)->as.external.ptr = ruby_xrealloc_sized(fields, RSHAPE_CAPACITY(next_shape_id) * sizeof(VALUE), RSHAPE_CAPACITY(old_shape_id) * sizeof(VALUE)); + } + } } - else if (UNLIKELY(RSHAPE_CAPACITY(next_shape_id) != RSHAPE_CAPACITY(current_shape_id))) { - RUBY_ASSERT(RSHAPE_CAPACITY(next_shape_id) > RSHAPE_CAPACITY(current_shape_id)); - shape_resize_fields_func(obj, RSHAPE_CAPACITY(current_shape_id), RSHAPE_CAPACITY(next_shape_id), data); + else { + fields_obj = 0; + rb_free_generic_ivar(obj); } + } - RUBY_ASSERT(RSHAPE_TYPE_P(next_shape_id, SHAPE_IVAR), - "next_shape_id: 0x%" PRIx32 " RSHAPE_TYPE(next_shape_id): %d", - next_shape_id, (int)RSHAPE_TYPE(next_shape_id)); - RUBY_ASSERT(index == (RSHAPE_INDEX(next_shape_id))); - set_shape_id_func(obj, next_shape_id, data); + RBASIC_SET_SHAPE_ID(obj, rb_shape_layout(RBASIC_SHAPE_ID(obj)) | next_shape_id); + if (fields_obj != original_fields_obj) { + switch (type) { + case T_OBJECT: + break; + case T_CLASS: + case T_MODULE: + RCLASS_WRITABLE_SET_FIELDS_OBJ(obj, fields_obj); + break; + default: + rb_obj_set_fields(obj, fields_obj, id, original_fields_obj); + break; + } } - VALUE *table = shape_fields_func(obj, data); - RB_OBJ_WRITE(obj, &table[index], val); + return val; +} - result.index = index; - return result; +VALUE +rb_attr_delete(VALUE obj, ID id) +{ + return rb_ivar_delete(obj, id, Qnil); +} -too_complex: - { - RUBY_ASSERT(rb_shape_obj_too_complex_p(obj)); +void +rb_obj_init_complex(VALUE obj, st_table *table) +{ + // This method is meant to be called on newly allocated object. + RUBY_ASSERT(rb_shape_canonical_p(RBASIC_SHAPE_ID(obj))); + RUBY_ASSERT(RSHAPE_LEN(RBASIC_SHAPE_ID(obj)) == 0); - st_table *table = too_complex_table_func(obj, data); - result.existing = st_insert(table, (st_data_t)id, (st_data_t)val); - result.index = 0; - RB_OBJ_WRITTEN(obj, Qundef, val); + if (rb_obj_shape_complex_p(obj)) { + st_table *old_table = ROBJECT_FIELDS_HASH(obj); + ROBJECT_SET_FIELDS_HASH(obj, table); + if (old_table) st_free_table(old_table); + } + else { + obj_transition_complex(obj, table); } - return result; } -static void -general_field_set(VALUE obj, shape_id_t target_shape_id, VALUE val, void *data, - VALUE *(*shape_fields_func)(VALUE, void *), - void (*shape_resize_fields_func)(VALUE, attr_index_t, attr_index_t, void *), - void (*set_shape_id_func)(VALUE, shape_id_t, void *), - shape_id_t (*transition_too_complex_func)(VALUE, void *), - st_table *(*too_complex_table_func)(VALUE, void *)) +static int +imemo_fields_complex_from_obj_i(ID key, VALUE val, st_data_t arg) { - shape_id_t current_shape_id = RBASIC_SHAPE_ID(obj); + VALUE fields = (VALUE)arg; + st_table *table = rb_imemo_fields_complex_tbl(fields); - if (UNLIKELY(rb_shape_too_complex_p(target_shape_id))) { - if (UNLIKELY(!rb_shape_too_complex_p(current_shape_id))) { - current_shape_id = transition_too_complex_func(obj, data); - } + RUBY_ASSERT(!st_lookup(table, (st_data_t)key, NULL)); + st_add_direct(table, (st_data_t)key, (st_data_t)val); + RB_OBJ_WRITTEN(fields, Qundef, val); - st_table *table = too_complex_table_func(obj, data); + return ST_CONTINUE; +} - if (RSHAPE_LEN(target_shape_id) > RSHAPE_LEN(current_shape_id)) { - RBASIC_SET_SHAPE_ID(obj, target_shape_id); - } +static VALUE +imemo_fields_complex_from_obj(VALUE owner, VALUE source_fields_obj, shape_id_t shape_id) +{ + attr_index_t len = source_fields_obj ? RSHAPE_LEN(RBASIC_SHAPE_ID(source_fields_obj)) : 0; + VALUE fields_obj = rb_imemo_fields_new_complex(owner, shape_id, len + 1, RB_OBJ_SHAREABLE_P(owner)); - RUBY_ASSERT(RSHAPE_EDGE_NAME(target_shape_id)); - st_insert(table, (st_data_t)RSHAPE_EDGE_NAME(target_shape_id), (st_data_t)val); - RB_OBJ_WRITTEN(obj, Qundef, val); - } - else { - attr_index_t index = RSHAPE_INDEX(target_shape_id); - if (index >= RSHAPE_CAPACITY(current_shape_id)) { - shape_resize_fields_func(obj, RSHAPE_CAPACITY(current_shape_id), RSHAPE_CAPACITY(target_shape_id), data); - } + rb_field_foreach(source_fields_obj, imemo_fields_complex_from_obj_i, (st_data_t)fields_obj, false); - if (RSHAPE_LEN(target_shape_id) > RSHAPE_LEN(current_shape_id)) { - set_shape_id_func(obj, target_shape_id, data); - } + return fields_obj; +} - VALUE *table = shape_fields_func(obj, data); - RB_OBJ_WRITE(obj, &table[index], val); +static VALUE +imemo_fields_copy_append(VALUE owner, VALUE source_fields_obj, shape_id_t current_shape_id, shape_id_t target_shape_id, VALUE val) +{ + attr_index_t fields_count = RSHAPE_LEN(current_shape_id); + + VALUE fields_obj = rb_imemo_fields_new(owner, target_shape_id, RB_OBJ_SHAREABLE_P(owner)); + + VALUE *fields = rb_imemo_fields_ptr(fields_obj); + + if (source_fields_obj) { + MEMCPY(fields, rb_imemo_fields_ptr(source_fields_obj), VALUE, fields_count); + for (attr_index_t i = 0; i < fields_count; i++) { + RB_OBJ_WRITTEN(fields_obj, Qundef, fields[i]); + } } + + RB_OBJ_WRITE(fields_obj, &fields[fields_count], val); + + return fields_obj; } static VALUE @@ -1883,8 +1825,8 @@ imemo_fields_set(VALUE owner, VALUE fields_obj, shape_id_t target_shape_id, ID f const VALUE original_fields_obj = fields_obj; shape_id_t current_shape_id = fields_obj ? RBASIC_SHAPE_ID(fields_obj) : ROOT_SHAPE_ID; - if (UNLIKELY(rb_shape_too_complex_p(target_shape_id))) { - if (rb_shape_too_complex_p(current_shape_id)) { + if (UNLIKELY(rb_shape_complex_p(target_shape_id))) { + if (rb_shape_complex_p(current_shape_id)) { if (concurrent) { // In multi-ractor case, we must always work on a copy because // even if the field already exist, inserting in a st_table may @@ -1902,26 +1844,26 @@ imemo_fields_set(VALUE owner, VALUE fields_obj, shape_id_t target_shape_id, ID f RUBY_ASSERT(field_name); st_insert(table, (st_data_t)field_name, (st_data_t)val); RB_OBJ_WRITTEN(fields_obj, Qundef, val); - RBASIC_SET_SHAPE_ID(fields_obj, target_shape_id); + RBASIC_SET_SHAPE_ID(fields_obj, rb_shape_id_with_robject_layout(target_shape_id)); } else { attr_index_t index = RSHAPE_INDEX(target_shape_id); if (concurrent || index >= RSHAPE_CAPACITY(current_shape_id)) { - fields_obj = imemo_fields_copy_capa(owner, original_fields_obj, RSHAPE_CAPACITY(target_shape_id)); + return imemo_fields_copy_append(owner, original_fields_obj, current_shape_id, target_shape_id, val); } VALUE *table = rb_imemo_fields_ptr(fields_obj); RB_OBJ_WRITE(fields_obj, &table[index], val); - if (RSHAPE_LEN(target_shape_id) > RSHAPE_LEN(current_shape_id)) { - RBASIC_SET_SHAPE_ID(fields_obj, target_shape_id); + if (index >= RSHAPE_LEN(current_shape_id)) { + RBASIC_SET_SHAPE_ID(fields_obj, rb_shape_id_with_robject_layout(target_shape_id)); } } return fields_obj; } -static void +static attr_index_t generic_field_set(VALUE obj, shape_id_t target_shape_id, ID field_name, VALUE val) { if (!field_name) { @@ -1933,6 +1875,7 @@ generic_field_set(VALUE obj, shape_id_t target_shape_id, ID field_name, VALUE va VALUE fields_obj = imemo_fields_set(obj, original_fields_obj, target_shape_id, field_name, val, false); rb_obj_set_fields(obj, fields_obj, field_name, original_fields_obj); + return rb_shape_complex_p(target_shape_id) ? ATTR_INDEX_NOT_SET : RSHAPE_INDEX(target_shape_id); } static shape_id_t @@ -1942,14 +1885,10 @@ generic_shape_ivar(VALUE obj, ID id, bool *new_ivar_out) shape_id_t current_shape_id = RBASIC_SHAPE_ID(obj); shape_id_t target_shape_id = current_shape_id; - if (!rb_shape_too_complex_p(current_shape_id)) { + if (!rb_shape_complex_p(current_shape_id)) { if (!rb_shape_find_ivar(current_shape_id, id, &target_shape_id)) { - if (RSHAPE_LEN(current_shape_id) >= SHAPE_MAX_FIELDS) { - rb_raise(rb_eArgError, "too many instance variables"); - } - new_ivar = true; - target_shape_id = rb_shape_transition_add_ivar(obj, id); + target_shape_id = rb_obj_shape_transition_add_ivar(obj, id); } } @@ -1957,29 +1896,29 @@ generic_shape_ivar(VALUE obj, ID id, bool *new_ivar_out) return target_shape_id; } -static void +static attr_index_t generic_ivar_set(VALUE obj, ID id, VALUE val) { bool dontcare; shape_id_t target_shape_id = generic_shape_ivar(obj, id, &dontcare); - generic_field_set(obj, target_shape_id, id, val); + return generic_field_set(obj, target_shape_id, id, val); } void rb_ensure_iv_list_size(VALUE obj, uint32_t current_len, uint32_t new_capacity) { - RUBY_ASSERT(!rb_shape_obj_too_complex_p(obj)); + RUBY_ASSERT(!rb_obj_shape_complex_p(obj)); - if (RBASIC(obj)->flags & ROBJECT_EMBED) { + if (FL_TEST_RAW(obj, ROBJECT_HEAP)) { + SIZED_REALLOC_N(ROBJECT(obj)->as.heap.fields, VALUE, new_capacity, current_len); + } + else { VALUE *ptr = ROBJECT_FIELDS(obj); VALUE *newptr = ALLOC_N(VALUE, new_capacity); MEMCPY(newptr, ptr, VALUE, current_len); - RB_FL_UNSET_RAW(obj, ROBJECT_EMBED); + FL_SET_RAW(obj, ROBJECT_HEAP); ROBJECT(obj)->as.heap.fields = newptr; } - else { - REALLOC_N(ROBJECT(obj)->as.heap.fields, VALUE, new_capacity); - } } static int @@ -2003,60 +1942,52 @@ rb_obj_copy_fields_to_hash_table(VALUE obj, st_table *table) rb_field_foreach(obj, rb_obj_copy_ivs_to_hash_table_i, (st_data_t)table, false); } -static VALUE * -obj_ivar_set_shape_fields(VALUE obj, void *_data) +static attr_index_t +obj_field_set(VALUE obj, shape_id_t target_shape_id, ID field_name, VALUE val) { - RUBY_ASSERT(!rb_shape_obj_too_complex_p(obj)); + shape_id_t current_shape_id = RBASIC_SHAPE_ID(obj); - return ROBJECT_FIELDS(obj); -} + if (UNLIKELY(rb_shape_complex_p(target_shape_id))) { + if (UNLIKELY(!rb_shape_complex_p(current_shape_id))) { + current_shape_id = rb_evict_fields_to_hash(obj); + } -static void -obj_ivar_set_shape_resize_fields(VALUE obj, attr_index_t old_capa, attr_index_t new_capa, void *_data) -{ - rb_ensure_iv_list_size(obj, old_capa, new_capa); -} + if (RSHAPE_LEN(target_shape_id) > RSHAPE_LEN(current_shape_id)) { + RBASIC_SET_SHAPE_ID(obj, target_shape_id); + } -static void -obj_ivar_set_set_shape_id(VALUE obj, shape_id_t shape_id, void *_data) -{ - rb_obj_set_shape_id(obj, shape_id); -} + if (!field_name) { + field_name = RSHAPE_EDGE_NAME(target_shape_id); + RUBY_ASSERT(field_name); + } -static shape_id_t -obj_ivar_set_transition_too_complex(VALUE obj, void *_data) -{ - return rb_evict_fields_to_hash(obj); -} + st_insert(ROBJECT_FIELDS_HASH(obj), (st_data_t)field_name, (st_data_t)val); + RB_OBJ_WRITTEN(obj, Qundef, val); -static st_table * -obj_ivar_set_too_complex_table(VALUE obj, void *_data) -{ - RUBY_ASSERT(rb_shape_obj_too_complex_p(obj)); + return ATTR_INDEX_NOT_SET; + } + else { + attr_index_t index = RSHAPE_INDEX(target_shape_id); - return ROBJECT_FIELDS_HASH(obj); -} + if (index >= RSHAPE_LEN(current_shape_id)) { + if (UNLIKELY(index >= RSHAPE_CAPACITY(current_shape_id))) { + rb_ensure_iv_list_size(obj, RSHAPE_CAPACITY(current_shape_id), RSHAPE_CAPACITY(target_shape_id)); + } + RBASIC_SET_SHAPE_ID(obj, target_shape_id); + } -attr_index_t -rb_obj_ivar_set(VALUE obj, ID id, VALUE val) -{ - return general_ivar_set(obj, id, val, NULL, - obj_ivar_set_shape_fields, - obj_ivar_set_shape_resize_fields, - obj_ivar_set_set_shape_id, - obj_ivar_set_transition_too_complex, - obj_ivar_set_too_complex_table).index; + RB_OBJ_WRITE(obj, &ROBJECT_FIELDS(obj)[index], val); + + return index; + } } -static void -obj_field_set(VALUE obj, shape_id_t target_shape_id, VALUE val) +static attr_index_t +obj_ivar_set(VALUE obj, ID id, VALUE val) { - general_field_set(obj, target_shape_id, val, NULL, - obj_ivar_set_shape_fields, - obj_ivar_set_shape_resize_fields, - obj_ivar_set_set_shape_id, - obj_ivar_set_transition_too_complex, - obj_ivar_set_too_complex_table); + bool dontcare; + shape_id_t target_shape_id = generic_shape_ivar(obj, id, &dontcare); + return obj_field_set(obj, target_shape_id, id, val); } /* Set the instance variable +val+ on object +obj+ at ivar name +id+. @@ -2067,23 +1998,12 @@ VALUE rb_vm_set_ivar_id(VALUE obj, ID id, VALUE val) { rb_check_frozen(obj); - rb_obj_ivar_set(obj, id, val); + obj_ivar_set(obj, id, val); return val; } -bool -rb_obj_set_shape_id(VALUE obj, shape_id_t shape_id) -{ - shape_id_t old_shape_id = rb_obj_shape_id(obj); - if (old_shape_id == shape_id) { - return false; - } - - RB_SET_SHAPE_ID(obj, shape_id); - return true; -} - -void rb_obj_freeze_inline(VALUE x) +void +rb_obj_freeze_inline(VALUE x) { if (RB_FL_ABLE(x)) { RB_FL_SET_RAW(x, RUBY_FL_FREEZE); @@ -2091,34 +2011,45 @@ void rb_obj_freeze_inline(VALUE x) RB_FL_UNSET_RAW(x, FL_USER2 | FL_USER3); // STR_CHILLED } - RB_SET_SHAPE_ID(x, rb_shape_transition_frozen(x)); + // rb_obj_freeze_inline(String) + shape_id_t shape_id = rb_obj_shape_transition_frozen(x); + switch (BUILTIN_TYPE(x)) { + case T_CLASS: + case T_MODULE: + rb_obj_freeze_inline(RCLASS_WRITABLE_ENSURE_FIELDS_OBJ(x)); + // FIXME: How to do multi-shape? + RBASIC_SET_SHAPE_ID(x, shape_id); + break; + default: + RBASIC_SET_SHAPE_ID(x, shape_id); + break; + } - if (RBASIC_CLASS(x)) { + if (RBASIC_CLASS(x) && RCLASS_SINGLETON_P(RBASIC_CLASS(x))) { rb_freeze_singleton_class(x); } } } -static void +static attr_index_t class_ivar_set(VALUE obj, ID id, VALUE val, bool *new_ivar); + +static attr_index_t ivar_set(VALUE obj, ID id, VALUE val) { RB_DEBUG_COUNTER_INC(ivar_set_base); switch (BUILTIN_TYPE(obj)) { case T_OBJECT: - { - rb_obj_ivar_set(obj, id, val); - break; - } + return obj_ivar_set(obj, id, val); case T_CLASS: case T_MODULE: - IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(id); - rb_class_ivar_set(obj, id, val); - - break; + { + IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(id); + bool dontcare; + return class_ivar_set(obj, id, val, &dontcare); + } default: - generic_ivar_set(obj, id, val); - break; + return generic_ivar_set(obj, id, val); } } @@ -2130,6 +2061,12 @@ rb_ivar_set(VALUE obj, ID id, VALUE val) return val; } +attr_index_t +rb_ivar_set_index(VALUE obj, ID id, VALUE val) +{ + return ivar_set(obj, id, val); +} + void rb_ivar_set_internal(VALUE obj, ID id, VALUE val) { @@ -2139,21 +2076,19 @@ rb_ivar_set_internal(VALUE obj, ID id, VALUE val) ivar_set(obj, id, val); } -void +attr_index_t rb_obj_field_set(VALUE obj, shape_id_t target_shape_id, ID field_name, VALUE val) { switch (BUILTIN_TYPE(obj)) { case T_OBJECT: - obj_field_set(obj, target_shape_id, val); - break; + return obj_field_set(obj, target_shape_id, field_name, val); case T_CLASS: case T_MODULE: // The only field is object_id and T_CLASS handle it differently. rb_bug("Unreachable"); break; default: - generic_field_set(obj, target_shape_id, field_name, val); - break; + return generic_field_set(obj, target_shape_id, field_name, val); } } @@ -2162,7 +2097,7 @@ ivar_defined0(VALUE obj, ID id) { attr_index_t index; - if (rb_shape_obj_too_complex_p(obj)) { + if (rb_obj_shape_complex_p(obj)) { VALUE idx; st_table *table = NULL; switch (BUILTIN_TYPE(obj)) { @@ -2226,6 +2161,7 @@ struct iv_itr_data { st_data_t arg; rb_ivar_foreach_callback_func *func; VALUE *fields; + shape_id_t shape_id; bool ivar_only; }; @@ -2241,12 +2177,12 @@ iterate_over_shapes_callback(shape_id_t shape_id, void *data) VALUE *fields; switch (BUILTIN_TYPE(itr_data->obj)) { case T_OBJECT: - RUBY_ASSERT(!rb_shape_obj_too_complex_p(itr_data->obj)); + RUBY_ASSERT(!rb_obj_shape_complex_p(itr_data->obj)); fields = ROBJECT_FIELDS(itr_data->obj); break; case T_IMEMO: RUBY_ASSERT(IMEMO_TYPE_P(itr_data->obj, imemo_fields)); - RUBY_ASSERT(!rb_shape_obj_too_complex_p(itr_data->obj)); + RUBY_ASSERT(!rb_obj_shape_complex_p(itr_data->obj)); fields = rb_imemo_fields_ptr(itr_data->obj); break; @@ -2254,8 +2190,14 @@ iterate_over_shapes_callback(shape_id_t shape_id, void *data) rb_bug("Unreachable"); } + RUBY_ASSERT(itr_data->shape_id == RBASIC_SHAPE_ID(itr_data->obj)); + VALUE val = fields[RSHAPE_INDEX(shape_id)]; - return itr_data->func(RSHAPE_EDGE_NAME(shape_id), val, itr_data->arg); + int ret = itr_data->func(RSHAPE_EDGE_NAME(shape_id), val, itr_data->arg); + + RUBY_ASSERT(itr_data->shape_id == RBASIC_SHAPE_ID(itr_data->obj)); + + return ret; } /* @@ -2289,11 +2231,12 @@ obj_fields_each(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg, b }; shape_id_t shape_id = RBASIC_SHAPE_ID(obj); - if (rb_shape_too_complex_p(shape_id)) { - rb_st_foreach(ROBJECT_FIELDS_HASH(obj), each_hash_iv, (st_data_t)&itr_data); + if (rb_shape_complex_p(shape_id)) { + st_foreach_safe(ROBJECT_FIELDS_HASH(obj), each_hash_iv, (st_data_t)&itr_data); } else { itr_data.fields = ROBJECT_FIELDS(obj); + itr_data.shape_id = shape_id; iterate_over_shapes(shape_id, func, &itr_data); } } @@ -2311,11 +2254,12 @@ imemo_fields_each(VALUE fields_obj, rb_ivar_foreach_callback_func *func, st_data }; shape_id_t shape_id = RBASIC_SHAPE_ID(fields_obj); - if (rb_shape_too_complex_p(shape_id)) { + if (rb_shape_complex_p(shape_id)) { rb_st_foreach(rb_imemo_fields_complex_tbl(fields_obj), each_hash_iv, (st_data_t)&itr_data); } else { itr_data.fields = rb_imemo_fields_ptr(fields_obj); + itr_data.shape_id = shape_id; iterate_over_shapes(shape_id, func, &itr_data); } } @@ -2327,7 +2271,7 @@ rb_copy_generic_ivar(VALUE dest, VALUE obj) rb_check_frozen(dest); - if (!rb_obj_exivar_p(obj)) { + if (!rb_obj_gen_fields_p(obj)) { return; } @@ -2340,7 +2284,7 @@ rb_copy_generic_ivar(VALUE dest, VALUE obj) goto clear; } - if (rb_shape_too_complex_p(src_shape_id)) { + if (rb_shape_complex_p(src_shape_id)) { rb_shape_copy_complex_ivars(dest, obj, src_shape_id, rb_imemo_fields_complex_tbl(fields_obj)); return; } @@ -2352,24 +2296,23 @@ rb_copy_generic_ivar(VALUE dest, VALUE obj) RUBY_ASSERT(RSHAPE_TYPE_P(initial_shape_id, SHAPE_ROOT)); dest_shape_id = rb_shape_rebuild(initial_shape_id, src_shape_id); - if (UNLIKELY(rb_shape_too_complex_p(dest_shape_id))) { + if (UNLIKELY(rb_shape_complex_p(dest_shape_id))) { st_table *table = rb_st_init_numtable_with_size(src_num_ivs); rb_obj_copy_ivs_to_hash_table(obj, table); - rb_obj_init_too_complex(dest, table); + rb_obj_init_complex(dest, table); return; } } if (!RSHAPE_LEN(dest_shape_id)) { - rb_obj_set_shape_id(dest, dest_shape_id); + RBASIC_SET_SHAPE_ID(dest, rb_shape_layout(RBASIC_SHAPE_ID(dest)) | dest_shape_id); return; } - new_fields_obj = rb_imemo_fields_new(dest, RSHAPE_CAPACITY(dest_shape_id)); + new_fields_obj = rb_imemo_fields_new(dest, dest_shape_id, RB_OBJ_SHAREABLE_P(dest)); VALUE *src_buf = rb_imemo_fields_ptr(fields_obj); VALUE *dest_buf = rb_imemo_fields_ptr(new_fields_obj); rb_shape_copy_fields(new_fields_obj, dest_buf, dest_shape_id, src_buf, src_shape_id); - RBASIC_SET_SHAPE_ID(new_fields_obj, dest_shape_id); rb_obj_replace_fields(dest, new_fields_obj); } @@ -2428,12 +2371,47 @@ rb_field_foreach(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg, } } +struct ivar_buf_entry { + ID name; + VALUE val; +}; + +static int +collect_ivar_i(ID id, VALUE val, st_data_t arg) +{ + struct ivar_buf_entry **pos = (struct ivar_buf_entry **)arg; + (*pos)->name = id; + (*pos)->val = val; + (*pos)++; + return ST_CONTINUE; +} + void rb_ivar_foreach(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg) { rb_field_foreach(obj, func, arg, true); } +void +rb_ivar_foreach_buffered(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg) +{ + st_index_t count = rb_ivar_count(obj); + if (count == 0) return; + + VALUE tmpbuf; + struct ivar_buf_entry *buf = ALLOCV_N(struct ivar_buf_entry, tmpbuf, count); + struct ivar_buf_entry *pos = buf; + + rb_field_foreach(obj, collect_ivar_i, (st_data_t)&pos, true); + RUBY_ASSERT((st_index_t)(pos - buf) == count); + + for (st_index_t i = 0; i < count; i++) { + if (func(buf[i].name, buf[i].val, arg) == ST_STOP) break; + } + + ALLOCV_END(tmpbuf); +} + st_index_t rb_ivar_count(VALUE obj) { @@ -2452,7 +2430,7 @@ rb_ivar_count(VALUE obj) if (!fields_obj) { return 0; } - if (rb_shape_obj_too_complex_p(fields_obj)) { + if (rb_obj_shape_complex_p(fields_obj)) { iv_count = rb_st_table_size(rb_imemo_fields_complex_tbl(fields_obj)); } else { @@ -2464,7 +2442,7 @@ rb_ivar_count(VALUE obj) case T_IMEMO: RUBY_ASSERT(IMEMO_TYPE_P(obj, imemo_fields)); - if (rb_shape_obj_too_complex_p(obj)) { + if (rb_obj_shape_complex_p(obj)) { iv_count = rb_st_table_size(rb_imemo_fields_complex_tbl(obj)); } else { @@ -2476,8 +2454,8 @@ rb_ivar_count(VALUE obj) { VALUE fields_obj = rb_obj_fields_no_ractor_check(obj); if (fields_obj) { - if (rb_shape_obj_too_complex_p(fields_obj)) { - rb_st_table_size(rb_imemo_fields_complex_tbl(fields_obj)); + if (rb_obj_shape_complex_p(fields_obj)) { + iv_count = rb_st_table_size(rb_imemo_fields_complex_tbl(fields_obj)); } else { iv_count = RBASIC_FIELDS_COUNT(obj); @@ -2487,7 +2465,7 @@ rb_ivar_count(VALUE obj) break; } - if (rb_shape_obj_has_id(obj)) { + if (rb_obj_shape_has_id(obj)) { iv_count--; } @@ -2525,9 +2503,7 @@ ivar_i(ID key, VALUE v, st_data_t a) VALUE rb_obj_instance_variables(VALUE obj) { - VALUE ary; - - ary = rb_ary_new(); + VALUE ary = rb_ary_new_capa(rb_ivar_count(obj)); rb_ivar_foreach(obj, ivar_i, ary); return ary; } @@ -2743,9 +2719,9 @@ struct autoload_const { // The shared "autoload_data" if multiple constants are defined from the same feature. VALUE autoload_data_value; - // The namespace object when the autoload is called in a user namespace - // Otherwise, Qnil means the builtin namespace, Qfalse means unspecified. - VALUE namespace; + // The box object when the autoload is called in a user box + // Otherwise, Qnil means the root box + VALUE box_value; // The module we are loading a constant into. VALUE module; @@ -2798,7 +2774,7 @@ autoload_data_free(void *ptr) ccan_list_del_init(&autoload_const->cnode); } - ruby_xfree(p); + SIZED_FREE(p); } static size_t @@ -2822,7 +2798,7 @@ autoload_const_mark_and_move(void *ptr) rb_gc_mark_and_move(&ac->autoload_data_value); rb_gc_mark_and_move(&ac->value); rb_gc_mark_and_move(&ac->file); - rb_gc_mark_and_move(&ac->namespace); + rb_gc_mark_and_move(&ac->box_value); } static size_t @@ -2837,13 +2813,13 @@ autoload_const_free(void *ptr) struct autoload_const *autoload_const = ptr; ccan_list_del(&autoload_const->cnode); - ruby_xfree(ptr); + SIZED_FREE(autoload_const); } static const rb_data_type_t autoload_const_type = { "autoload_const", {autoload_const_mark_and_move, autoload_const_free, autoload_const_memsize, autoload_const_mark_and_move,}, - 0, 0, RUBY_TYPED_FREE_IMMEDIATELY + 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED }; static struct autoload_data * @@ -2868,17 +2844,17 @@ get_autoload_data(VALUE autoload_const_value, struct autoload_const **autoload_c struct autoload_copy_table_data { VALUE dst_tbl_value; struct st_table *dst_tbl; - const rb_namespace_t *ns; + const rb_box_t *box; }; static int -autoload_copy_table_for_namespace_i(st_data_t key, st_data_t value, st_data_t arg) +autoload_copy_table_for_box_i(st_data_t key, st_data_t value, st_data_t arg) { struct autoload_const *autoload_const; struct autoload_copy_table_data *data = (struct autoload_copy_table_data *)arg; struct st_table *tbl = data->dst_tbl; VALUE tbl_value = data->dst_tbl_value; - const rb_namespace_t *ns = data->ns; + const rb_box_t *box = data->box; VALUE src_value = (VALUE)value; struct autoload_const *src_const = rb_check_typeddata(src_value, &autoload_const_type); @@ -2887,12 +2863,12 @@ autoload_copy_table_for_namespace_i(st_data_t key, st_data_t value, st_data_t ar struct autoload_data *autoload_data = rb_check_typeddata(autoload_data_value, &autoload_data_type); VALUE new_value = TypedData_Make_Struct(0, struct autoload_const, &autoload_const_type, autoload_const); - autoload_const->namespace = rb_get_namespace_object((rb_namespace_t *)ns); - autoload_const->module = src_const->module; + RB_OBJ_WRITE(new_value, &autoload_const->box_value, rb_get_box_object((rb_box_t *)box)); + RB_OBJ_WRITE(new_value, &autoload_const->module, src_const->module); autoload_const->name = src_const->name; - autoload_const->value = src_const->value; + RB_OBJ_WRITE(new_value, &autoload_const->value, src_const->value); autoload_const->flag = src_const->flag; - autoload_const->autoload_data_value = autoload_data_value; + RB_OBJ_WRITE(new_value, &autoload_const->autoload_data_value, autoload_data_value); ccan_list_add_tail(&autoload_data->constants, &autoload_const->cnode); st_insert(tbl, (st_data_t)autoload_const->name, (st_data_t)new_value); @@ -2902,7 +2878,7 @@ autoload_copy_table_for_namespace_i(st_data_t key, st_data_t value, st_data_t ar } void -rb_autoload_copy_table_for_namespace(st_table *iv_ptr, const rb_namespace_t *ns) +rb_autoload_copy_table_for_box(st_table *iv_ptr, const rb_box_t *box) { struct st_table *src_tbl, *dst_tbl; VALUE src_tbl_value, dst_tbl_value; @@ -2922,10 +2898,10 @@ rb_autoload_copy_table_for_namespace(st_table *iv_ptr, const rb_namespace_t *ns) struct autoload_copy_table_data data = { .dst_tbl_value = dst_tbl_value, .dst_tbl = dst_tbl, - .ns = ns, + .box = box, }; - st_foreach(src_tbl, autoload_copy_table_for_namespace_i, (st_data_t)&data); + st_foreach(src_tbl, autoload_copy_table_for_box_i, (st_data_t)&data); st_insert(iv_ptr, (st_data_t)autoload, (st_data_t)dst_tbl_value); } @@ -2946,7 +2922,7 @@ struct autoload_arguments { VALUE module; ID name; VALUE feature; - VALUE namespace; + VALUE box_value; }; static VALUE @@ -3016,12 +2992,12 @@ autoload_synchronized(VALUE _arguments) { struct autoload_const *autoload_const; VALUE autoload_const_value = TypedData_Make_Struct(0, struct autoload_const, &autoload_const_type, autoload_const); - autoload_const->namespace = arguments->namespace; - autoload_const->module = arguments->module; + RB_OBJ_WRITE(autoload_const_value, &autoload_const->box_value, arguments->box_value); + RB_OBJ_WRITE(autoload_const_value, &autoload_const->module, arguments->module); autoload_const->name = arguments->name; autoload_const->value = Qundef; autoload_const->flag = CONST_PUBLIC; - autoload_const->autoload_data_value = autoload_data_value; + RB_OBJ_WRITE(autoload_const_value, &autoload_const->autoload_data_value, autoload_data_value); ccan_list_add_tail(&autoload_data->constants, &autoload_const->cnode); st_insert(autoload_table, (st_data_t)arguments->name, (st_data_t)autoload_const_value); RB_OBJ_WRITTEN(autoload_table_value, Qundef, autoload_const_value); @@ -3033,8 +3009,8 @@ autoload_synchronized(VALUE _arguments) void rb_autoload_str(VALUE module, ID name, VALUE feature) { - const rb_namespace_t *ns = rb_current_namespace(); - VALUE current_namespace = rb_get_namespace_object((rb_namespace_t *)ns); + const rb_box_t *box = rb_current_box(); + VALUE current_box_value = rb_get_box_object((rb_box_t *)box); if (!rb_is_const_id(name)) { rb_raise(rb_eNameError, "autoload must be constant name: %"PRIsVALUE"", QUOTE_ID(name)); @@ -3049,7 +3025,7 @@ rb_autoload_str(VALUE module, ID name, VALUE feature) .module = module, .name = name, .feature = feature, - .namespace = current_namespace, + .box_value = current_box_value, }; VALUE result = rb_mutex_synchronize(autoload_mutex, autoload_synchronized, (VALUE)&arguments); @@ -3308,43 +3284,6 @@ autoload_apply_constants(VALUE _arguments) return Qtrue; } -struct autoload_feature_require_data { - struct autoload_load_arguments *arguments; - VALUE receiver; - VALUE feature; -}; - -static VALUE -autoload_feature_require_in_builtin(VALUE arg) -{ - struct autoload_feature_require_data *data = (struct autoload_feature_require_data *)arg; - - VALUE result = rb_funcall(data->receiver, rb_intern("require"), 1, data->feature); - if (RTEST(result)) { - return rb_mutex_synchronize(autoload_mutex, autoload_apply_constants, (VALUE)data->arguments); - } - return Qnil; -} - -static VALUE -autoload_feature_require_ensure_in_builtin(VALUE _arg) -{ - /* - * The gccct should be cleared again after the rb_funcall() to remove - * the inconsistent cache entry against the current namespace. - */ - rb_gccct_clear_table(Qnil); - rb_namespace_disable_builtin(); - return Qnil; -} - -static VALUE -autoload_feature_require_in_builtin_wrap(VALUE arg) -{ - return rb_ensure(autoload_feature_require_in_builtin, arg, - autoload_feature_require_ensure_in_builtin, Qnil); -} - static VALUE autoload_feature_require(VALUE _arguments) { @@ -3353,31 +3292,21 @@ autoload_feature_require(VALUE _arguments) struct autoload_load_arguments *arguments = (struct autoload_load_arguments*)_arguments; struct autoload_const *autoload_const = arguments->autoload_const; - VALUE autoload_namespace = autoload_const->namespace; + VALUE autoload_box_value = autoload_const->box_value; // We save this for later use in autoload_apply_constants: arguments->autoload_data = rb_check_typeddata(autoload_const->autoload_data_value, &autoload_data_type); - if (NIL_P(autoload_namespace)) { - rb_namespace_enable_builtin(); - /* - * Clear the global cc cache table because the require method can be different from the current - * namespace's one and it may cause inconsistent cc-cme states. - * For example, the assertion below may fail in gccct_method_search(); - * VM_ASSERT(vm_cc_check_cme(cc, rb_callable_method_entry(klass, mid))) - */ - rb_gccct_clear_table(Qnil); - struct autoload_feature_require_data data = { - .arguments = arguments, - .receiver = receiver, - .feature = arguments->autoload_data->feature, - }; - return rb_namespace_exec(rb_builtin_namespace(), autoload_feature_require_in_builtin_wrap, (VALUE)&data); - } + if (rb_box_available() && BOX_OBJ_P(autoload_box_value)) + receiver = autoload_box_value; - if (RTEST(autoload_namespace) && NAMESPACE_OPTIONAL_P(rb_get_namespace_t(autoload_namespace))) { - receiver = autoload_namespace; - } + /* + * Clear the global cc cache table because the require method can be different from the current + * box's one and it may cause inconsistent cc-cme states. + * For example, the assertion below may fail in gccct_method_search(); + * VM_ASSERT(vm_cc_check_cme(cc, rb_callable_method_entry(klass, mid))) + */ + rb_gccct_clear_table(); VALUE result = rb_funcall(receiver, rb_intern("require"), 1, arguments->autoload_data->feature); @@ -3504,11 +3433,12 @@ rb_const_warn_if_deprecated(const rb_const_entry_t *ce, VALUE klass, ID id) static VALUE rb_const_get_0(VALUE klass, ID id, int exclude, int recurse, int visibility) { - VALUE c = rb_const_search(klass, id, exclude, recurse, visibility); + VALUE found_in; + VALUE c = rb_const_search(klass, id, exclude, recurse, visibility, &found_in); if (!UNDEF_P(c)) { if (UNLIKELY(!rb_ractor_main_p())) { if (!rb_ractor_shareable_p(c)) { - rb_raise(rb_eRactorIsolationError, "can not access non-shareable objects in constant %"PRIsVALUE"::%s by non-main Ractor.", rb_class_path(klass), rb_id2name(id)); + rb_raise(rb_eRactorIsolationError, "can not access non-shareable objects in constant %"PRIsVALUE"::%"PRIsVALUE" by non-main Ractor.", rb_class_path(found_in), rb_id2str(id)); } } return c; @@ -3517,7 +3447,7 @@ rb_const_get_0(VALUE klass, ID id, int exclude, int recurse, int visibility) } static VALUE -rb_const_search_from(VALUE klass, ID id, int exclude, int recurse, int visibility) +rb_const_search_from(VALUE klass, ID id, int exclude, int recurse, int visibility, VALUE *found_in) { VALUE value, current; bool first_iteration = true; @@ -3554,13 +3484,17 @@ rb_const_search_from(VALUE klass, ID id, int exclude, int recurse, int visibilit if (am == tmp) break; am = tmp; ac = autoloading_const_entry(tmp, id); - if (ac) return ac->value; + if (ac) { + if (found_in) { *found_in = tmp; } + return ac->value; + } rb_autoload_load(tmp, id); continue; } if (exclude && tmp == rb_cObject) { goto not_found; } + if (found_in) { *found_in = tmp; } return value; } if (!recurse) break; @@ -3572,17 +3506,17 @@ rb_const_search_from(VALUE klass, ID id, int exclude, int recurse, int visibilit } static VALUE -rb_const_search(VALUE klass, ID id, int exclude, int recurse, int visibility) +rb_const_search(VALUE klass, ID id, int exclude, int recurse, int visibility, VALUE *found_in) { VALUE value; if (klass == rb_cObject) exclude = FALSE; - value = rb_const_search_from(klass, id, exclude, recurse, visibility); + value = rb_const_search_from(klass, id, exclude, recurse, visibility, found_in); if (!UNDEF_P(value)) return value; if (exclude) return value; if (BUILTIN_TYPE(klass) != T_MODULE) return value; /* search global const too, if klass is a module */ - return rb_const_search_from(rb_cObject, id, FALSE, recurse, visibility); + return rb_const_search_from(rb_cObject, id, FALSE, recurse, visibility, found_in); } VALUE @@ -3718,7 +3652,8 @@ rb_const_remove(VALUE mod, ID id) rb_check_frozen(mod); ce = rb_const_lookup(mod, id); - if (!ce || !rb_id_table_delete(RCLASS_WRITABLE_CONST_TBL(mod), id)) { + + if (!ce) { if (rb_const_defined_at(mod, id)) { rb_name_err_raise("cannot remove %2$s::%1$s", mod, ID2SYM(id)); } @@ -3726,6 +3661,14 @@ rb_const_remove(VALUE mod, ID id) undefined_constant(mod, ID2SYM(id)); } + VALUE writable_ce = 0; + if (rb_id_table_lookup(RCLASS_WRITABLE_CONST_TBL(mod), id, &writable_ce)) { + rb_id_table_delete(RCLASS_WRITABLE_CONST_TBL(mod), id); + if ((rb_const_entry_t *)writable_ce != ce) { + SIZED_FREE((rb_const_entry_t *)writable_ce); + } + } + rb_const_warn_if_deprecated(ce, mod, id); rb_clear_constant_cache_for_id(id); @@ -3737,7 +3680,7 @@ rb_const_remove(VALUE mod, ID id) } if (ce != const_lookup(RCLASS_PRIME_CONST_TBL(mod), id)) { - ruby_xfree(ce); + SIZED_FREE(ce); } // else - skip free'ing the ce because it still exists in the prime classext @@ -3973,6 +3916,7 @@ static void set_namespace_path(VALUE named_namespace, VALUE namespace_path) { struct rb_id_table *const_table = RCLASS_CONST_TBL(named_namespace); + RB_OBJ_SET_SHAREABLE(namespace_path); RB_VM_LOCKING() { RCLASS_WRITE_CLASSPATH(named_namespace, namespace_path, true); @@ -4051,7 +3995,8 @@ const_set(VALUE klass, ID id, VALUE val) set_namespace_path(val, build_const_path(parental_path, id)); } else if (!parental_path_permanent && NIL_P(val_path)) { - RCLASS_SET_CLASSPATH(val, build_const_path(parental_path, id), false); + VALUE path = build_const_path(parental_path, id); + RCLASS_SET_CLASSPATH(val, path, false); } } } @@ -4065,21 +4010,21 @@ rb_const_set(VALUE klass, ID id, VALUE val) const_added(klass, id); } -static struct autoload_data * -autoload_data_for_named_constant(VALUE module, ID name, struct autoload_const **autoload_const_pointer) +static VALUE +autoload_const_value_for_named_constant(VALUE module, ID name, struct autoload_const **autoload_const_pointer) { - VALUE autoload_data_value = autoload_data(module, name); - if (!autoload_data_value) return 0; + VALUE autoload_const_value = autoload_data(module, name); + if (!autoload_const_value) return Qfalse; - struct autoload_data *autoload_data = get_autoload_data(autoload_data_value, autoload_const_pointer); - if (!autoload_data) return 0; + struct autoload_data *autoload_data = get_autoload_data(autoload_const_value, autoload_const_pointer); + if (!autoload_data) return Qfalse; /* for autoloading thread, keep the defined value to autoloading storage */ if (autoload_by_current(autoload_data)) { - return autoload_data; + return autoload_const_value; } - return 0; + return Qfalse; } static void @@ -4099,13 +4044,13 @@ const_tbl_update(struct autoload_const *ac, int autoload_force) RUBY_ASSERT_CRITICAL_SECTION_ENTER(); VALUE file = ac->file; int line = ac->line; - struct autoload_data *ele = autoload_data_for_named_constant(klass, id, &ac); + VALUE autoload_const_value = autoload_const_value_for_named_constant(klass, id, &ac); - if (!autoload_force && ele) { + if (!autoload_force && autoload_const_value) { rb_clear_constant_cache_for_id(id); - ac->value = val; /* autoload_data is non-WB-protected */ - ac->file = rb_source_location(&ac->line); + RB_OBJ_WRITE(autoload_const_value, &ac->value, val); + RB_OBJ_WRITE(autoload_const_value, &ac->file, rb_source_location(&ac->line)); } else { /* otherwise autoloaded constant, allow to override */ @@ -4121,15 +4066,17 @@ const_tbl_update(struct autoload_const *ac, int autoload_force) else { VALUE name = QUOTE_ID(id); visibility = ce->flag; - if (klass == rb_cObject) - rb_warn("already initialized constant %"PRIsVALUE"", name); - else - rb_warn("already initialized constant %"PRIsVALUE"::%"PRIsVALUE"", - rb_class_name(klass), name); + + VALUE previous = Qnil; if (!NIL_P(ce->file) && ce->line) { - rb_compile_warn(RSTRING_PTR(ce->file), ce->line, - "previous definition of %"PRIsVALUE" was here", name); + previous = rb_sprintf("\n%"PRIsVALUE":%d: warning: previous definition of %"PRIsVALUE" was here", ce->file, ce->line, name); } + + if (klass == rb_cObject) + rb_warn("already initialized constant %"PRIsVALUE"%"PRIsVALUE"", name, previous); + else + rb_warn("already initialized constant %"PRIsVALUE"::%"PRIsVALUE"%"PRIsVALUE"", + rb_class_name(klass), name, previous); } rb_clear_constant_cache_for_id(id); setup_const_entry(ce, klass, val, visibility); @@ -4199,10 +4146,7 @@ set_const_visibility(VALUE mod, int argc, const VALUE *argv, ce->flag &= ~mask; ce->flag |= flag; if (UNDEF_P(ce->value)) { - struct autoload_data *ele; - - ele = autoload_data_for_named_constant(mod, id, &ac); - if (ele) { + if (autoload_const_value_for_named_constant(mod, id, &ac)) { ac->flag &= ~mask; ac->flag |= flag; } @@ -4350,7 +4294,6 @@ cvar_overtaken(VALUE front, VALUE target, ID id) } #define CVAR_LOOKUP(v,r) do {\ - CVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(); \ if (cvar_lookup_at(klass, id, (v))) {r;}\ CVAR_FOREACH_ANCESTORS(klass, v, r);\ } while(0) @@ -4369,22 +4312,11 @@ find_cvar(VALUE klass, VALUE * front, VALUE * target, ID id) return v; } -static void -check_for_cvar_table(VALUE subclass, VALUE key) -{ - // Must not check ivar on ICLASS - if (!RB_TYPE_P(subclass, T_ICLASS) && RTEST(rb_ivar_defined(subclass, key))) { - RB_DEBUG_COUNTER_INC(cvar_class_invalidate); - ruby_vm_global_cvar_state++; - return; - } - - rb_class_foreach_subclass(subclass, check_for_cvar_table, key); -} - void rb_cvar_set(VALUE klass, ID id, VALUE val) { + CVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(klass, id); + VALUE tmp, front = 0, target = 0; tmp = klass; @@ -4403,22 +4335,29 @@ rb_cvar_set(VALUE klass, ID id, VALUE val) bool new_cvar = rb_class_ivar_set(target, id, val); - struct rb_id_table *rb_cvc_tbl = RCLASS_WRITABLE_CVC_TBL(target); - - if (!rb_cvc_tbl) { - rb_cvc_tbl = rb_id_table_create(2); - RCLASS_WRITE_CVC_TBL(target, rb_cvc_tbl); - } + VALUE cvc_tbl = RCLASS_WRITABLE_CVC_TBL(target); struct rb_cvar_class_tbl_entry *ent; VALUE ent_data; - if (!rb_id_table_lookup(rb_cvc_tbl, id, &ent_data)) { - ent = ALLOC(struct rb_cvar_class_tbl_entry); - ent->class_value = target; + if (!cvc_tbl || !rb_marked_id_table_lookup(cvc_tbl, id, &ent_data)) { + ent = (struct rb_cvar_class_tbl_entry *)SHAREABLE_IMEMO_NEW(struct rb_cvar_class_tbl_entry, imemo_cvar_entry, 0); + RB_OBJ_WRITE((VALUE)ent, &ent->class_value, target); + RB_OBJ_WRITE((VALUE)ent, &ent->cref, 0); ent->global_cvar_state = GET_GLOBAL_CVAR_STATE(); - ent->cref = 0; - rb_id_table_insert(rb_cvc_tbl, id, (VALUE)ent); + + VALUE new_cvc_tbl = cvc_tbl; + if (!new_cvc_tbl) { + new_cvc_tbl = rb_marked_id_table_new(2); + } + else if (rb_multi_ractor_p()) { + new_cvc_tbl = rb_marked_id_table_dup(cvc_tbl); + } + + rb_marked_id_table_insert(new_cvc_tbl, id, (VALUE)ent); + if (new_cvc_tbl != cvc_tbl) { + RCLASS_WRITE_CVC_TBL(target, new_cvc_tbl); + } RB_DEBUG_COUNTER_INC(cvar_inline_miss); } else { @@ -4426,15 +4365,11 @@ rb_cvar_set(VALUE klass, ID id, VALUE val) ent->global_cvar_state = GET_GLOBAL_CVAR_STATE(); } - // Break the cvar cache if this is a new class variable - // and target is a module or a subclass with the same - // cvar in this lookup. + // Break the cvar cache if this is a new class variable. + // Existing caches may have resolved this name to a different + // location in the hierarchy, so we must invalidate globally. if (new_cvar) { - if (RB_TYPE_P(target, T_CLASS)) { - if (RCLASS_SUBCLASSES_FIRST(target)) { - rb_class_foreach_subclass(target, check_for_cvar_table, id); - } - } + ruby_vm_global_cvar_state++; } } @@ -4450,6 +4385,7 @@ rb_cvar_find(VALUE klass, ID id, VALUE *front) klass, ID2SYM(id)); } cvar_overtaken(*front, target, id); + cvar_read_ractor_check(klass, id, value); return (VALUE)value; } @@ -4660,70 +4596,58 @@ rb_iv_set(VALUE obj, const char *name, VALUE val) return rb_ivar_set(obj, id, val); } -static bool -class_fields_ivar_set(VALUE klass, VALUE fields_obj, ID id, VALUE val, bool concurrent, VALUE *new_fields_obj) +static attr_index_t +class_fields_ivar_set(VALUE klass, VALUE fields_obj, ID id, VALUE val, bool concurrent, VALUE *new_fields_obj, bool *new_ivar_out) { - bool existing = true; const VALUE original_fields_obj = fields_obj; - fields_obj = original_fields_obj ? original_fields_obj : rb_imemo_fields_new(klass, 1); + fields_obj = original_fields_obj ? original_fields_obj : rb_imemo_fields_new(klass, ROOT_SHAPE_ID, true); shape_id_t current_shape_id = RBASIC_SHAPE_ID(fields_obj); - shape_id_t next_shape_id = current_shape_id; - - if (UNLIKELY(rb_shape_too_complex_p(current_shape_id))) { - goto too_complex; + shape_id_t next_shape_id = current_shape_id; // for complex + if (UNLIKELY(rb_shape_complex_p(current_shape_id))) { + goto complex; } - attr_index_t index; - if (!rb_shape_get_iv_index(current_shape_id, id, &index)) { - existing = false; - - index = RSHAPE_LEN(current_shape_id); - if (index >= SHAPE_MAX_FIELDS) { - rb_raise(rb_eArgError, "too many instance variables"); - } - - next_shape_id = rb_shape_transition_add_ivar(fields_obj, id); - if (UNLIKELY(rb_shape_too_complex_p(next_shape_id))) { - fields_obj = imemo_fields_complex_from_obj(klass, fields_obj, next_shape_id); - goto too_complex; - } - - attr_index_t next_capacity = RSHAPE_CAPACITY(next_shape_id); - attr_index_t current_capacity = RSHAPE_CAPACITY(current_shape_id); + bool new_ivar; + next_shape_id = generic_shape_ivar(fields_obj, id, &new_ivar); - if (next_capacity > current_capacity) { - // We allocate a new fields_obj even when concurrency isn't a concern - // so that we're embedded as long as possible. - fields_obj = imemo_fields_copy_capa(klass, fields_obj, next_capacity); - } - - RUBY_ASSERT(RSHAPE(next_shape_id)->type == SHAPE_IVAR); - RUBY_ASSERT(index == (RSHAPE_LEN(next_shape_id) - 1)); + if (UNLIKELY(rb_shape_complex_p(next_shape_id))) { + fields_obj = imemo_fields_complex_from_obj(klass, fields_obj, next_shape_id); + goto complex; } - VALUE *fields = rb_imemo_fields_ptr(fields_obj); - - if (concurrent && original_fields_obj == fields_obj) { - // In the concurrent case, if we're mutating the existing - // fields_obj, we must use an atomic write, because if we're - // adding a new field, the shape_id must be written after the field - // and if we're updating an existing field, we at least need a relaxed - // write to avoid reaping. - RB_OBJ_ATOMIC_WRITE(fields_obj, &fields[index], val); + attr_index_t index = RSHAPE_INDEX(next_shape_id); + if (new_ivar && index >= RSHAPE_CAPACITY(current_shape_id)) { + // We allocate a new fields_obj even when concurrency isn't a concern + // so that we're embedded as long as possible. + fields_obj = imemo_fields_copy_append(klass, fields_obj, current_shape_id, next_shape_id, val); } else { - RB_OBJ_WRITE(fields_obj, &fields[index], val); - } + VALUE *fields = rb_imemo_fields_ptr(fields_obj); + + if (concurrent && original_fields_obj == fields_obj) { + // In the concurrent case, if we're mutating the existing + // fields_obj, we must use an atomic write, because if we're + // adding a new field, the shape_id must be written after the field + // and if we're updating an existing field, we at least need a relaxed + // write to avoid reaping. + RB_OBJ_ATOMIC_WRITE(fields_obj, &fields[index], val); + } + else { + RB_OBJ_WRITE(fields_obj, &fields[index], val); + } - if (!existing) { - RBASIC_SET_SHAPE_ID(fields_obj, next_shape_id); + if (new_ivar) { + RUBY_ASSERT(rb_shape_layout(next_shape_id) == SHAPE_ID_LAYOUT_ROBJECT); + RBASIC_SET_SHAPE_ID(fields_obj, next_shape_id); + } } *new_fields_obj = fields_obj; - return existing; + *new_ivar_out = new_ivar; + return index; -too_complex: +complex: { if (concurrent && fields_obj == original_fields_obj) { // In multi-ractor case, we must always work on a copy because @@ -4733,30 +4657,29 @@ too_complex: } st_table *table = rb_imemo_fields_complex_tbl(fields_obj); - existing = st_insert(table, (st_data_t)id, (st_data_t)val); + new_ivar = !st_insert(table, (st_data_t)id, (st_data_t)val); RB_OBJ_WRITTEN(fields_obj, Qundef, val); if (fields_obj != original_fields_obj) { + RUBY_ASSERT(rb_shape_layout(next_shape_id) == SHAPE_ID_LAYOUT_ROBJECT); RBASIC_SET_SHAPE_ID(fields_obj, next_shape_id); } } *new_fields_obj = fields_obj; - return existing; + *new_ivar_out = new_ivar; + return ATTR_INDEX_NOT_SET; } -bool -rb_class_ivar_set(VALUE obj, ID id, VALUE val) +static attr_index_t +class_ivar_set(VALUE obj, ID id, VALUE val, bool *new_ivar) { - RUBY_ASSERT(RB_TYPE_P(obj, T_CLASS) || RB_TYPE_P(obj, T_MODULE)); - rb_check_frozen(obj); - rb_class_ensure_writable(obj); const VALUE original_fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj); VALUE new_fields_obj = 0; - bool existing = class_fields_ivar_set(obj, original_fields_obj, id, val, rb_multi_ractor_p(), &new_fields_obj); + attr_index_t index = class_fields_ivar_set(obj, original_fields_obj, id, val, rb_multi_ractor_p(), &new_fields_obj, new_ivar); if (new_fields_obj != original_fields_obj) { RCLASS_WRITABLE_SET_FIELDS_OBJ(obj, new_fields_obj); @@ -4764,11 +4687,20 @@ rb_class_ivar_set(VALUE obj, ID id, VALUE val) // TODO: What should we set as the T_CLASS shape_id? // In most case we can replicate the single `fields_obj` shape - // but in namespaced case? - // Perhaps INVALID_SHAPE_ID? - RBASIC_SET_SHAPE_ID(obj, RBASIC_SHAPE_ID(new_fields_obj)); + // but in namespaced case? Perhaps INVALID_SHAPE_ID? + RBASIC_SET_SHAPE_ID(obj, rb_shape_layout(RBASIC_SHAPE_ID(obj)) | RBASIC_SHAPE_ID(new_fields_obj)); + return index; +} - return !existing; +bool +rb_class_ivar_set(VALUE obj, ID id, VALUE val) +{ + RUBY_ASSERT(RB_TYPE_P(obj, T_CLASS) || RB_TYPE_P(obj, T_MODULE)); + rb_check_frozen(obj); + + bool new_ivar; + class_ivar_set(obj, id, val, &new_ivar); + return new_ivar; } void @@ -4781,7 +4713,7 @@ rb_fields_tbl_copy(VALUE dst, VALUE src) VALUE fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(src); if (fields_obj) { RCLASS_WRITABLE_SET_FIELDS_OBJ(dst, rb_imemo_fields_clone(fields_obj)); - RBASIC_SET_SHAPE_ID(dst, RBASIC_SHAPE_ID(src)); + RBASIC_SET_SHAPE_ID(dst, rb_shape_layout(RBASIC_SHAPE_ID(dst)) | RBASIC_SHAPE_ID(src)); } } @@ -4805,4 +4737,3 @@ rb_const_lookup(VALUE klass, ID id) { return const_lookup(RCLASS_CONST_TBL(klass), id); } - |