diff options
Diffstat (limited to 'variable.c')
| -rw-r--r-- | variable.c | 2861 |
1 files changed, 1299 insertions, 1562 deletions
diff --git a/variable.c b/variable.c index c30971bd08..74d5b699b8 100644 --- a/variable.c +++ b/variable.c @@ -33,8 +33,7 @@ #include "ruby/encoding.h" #include "ruby/st.h" #include "ruby/util.h" -#include "shape.h" -#include "symbol.h" +#include "transient_heap.h" #include "variable.h" #include "vm_core.h" #include "ractor_core.h" @@ -46,17 +45,8 @@ RUBY_EXTERN rb_serial_t ruby_vm_global_cvar_state; typedef void rb_gvar_compact_t(void *var); static struct rb_id_table *rb_global_tbl; -static ID autoload; - -// This hash table maps file paths to loadable features. We use this to track -// autoload state until it's no longer needed. -// feature (file path) => struct autoload_data -static VALUE autoload_features; - -// This mutex is used to protect autoloading state. We use a global mutex which -// is held until a per-feature mutex can be created. This ensures there are no -// race conditions relating to autoload state. -static VALUE autoload_mutex; +static ID autoload, classpath, tmp_classpath; +static VALUE autoload_featuremap; /* feature => autoload_i */ static void check_before_mod_set(VALUE, ID, VALUE, const char *); static void setup_const_entry(rb_const_entry_t *, VALUE, VALUE, rb_const_flag_t); @@ -64,12 +54,12 @@ static VALUE rb_const_search(VALUE klass, ID id, int exclude, int recurse, int v static st_table *generic_iv_tbl_; struct ivar_update { - struct gen_ivtbl *ivtbl; - uint32_t iv_index; - uint32_t max_index; -#if !SHAPE_IN_BASIC_FLAGS - rb_shape_t *shape; -#endif + union { + st_table *iv_index_tbl; + struct gen_ivtbl *ivtbl; + } u; + st_data_t index; + int iv_extended; }; void @@ -78,14 +68,10 @@ Init_var_tables(void) rb_global_tbl = rb_id_table_create(0); generic_iv_tbl_ = st_init_numtable(); autoload = rb_intern_const("__autoload__"); - - autoload_mutex = rb_mutex_new(); - rb_obj_hide(autoload_mutex); - rb_gc_register_mark_object(autoload_mutex); - - autoload_features = rb_ident_hash_new(); - rb_obj_hide(autoload_features); - rb_gc_register_mark_object(autoload_features); + /* __classpath__: fully qualified class path */ + classpath = rb_intern_const("__classpath__"); + /* __tmp_classpath__: temporary class path which contains anonymous names */ + tmp_classpath = rb_intern_const("__tmp_classpath__"); } static inline bool @@ -108,16 +94,20 @@ rb_namespace_p(VALUE obj) * Ruby level APIs that can change a permanent +classpath+. */ static VALUE -classname(VALUE klass, bool *permanent) +classname(VALUE klass, int *permanent) { - *permanent = false; - - VALUE classpath = RCLASS_EXT(klass)->classpath; - if (classpath == 0) return Qnil; + st_table *ivtbl; + st_data_t n; - *permanent = RCLASS_EXT(klass)->permanent_classpath; - - return classpath; + *permanent = 0; + if (!RCLASS_EXT(klass)) return Qnil; + if (!(ivtbl = RCLASS_IV_TBL(klass))) return Qnil; + if (st_lookup(ivtbl, (st_data_t)classpath, &n)) { + *permanent = 1; + return (VALUE)n; + } + if (st_lookup(ivtbl, (st_data_t)tmp_classpath, &n)) return (VALUE)n; + return Qnil; } /* @@ -130,129 +120,24 @@ classname(VALUE klass, bool *permanent) VALUE rb_mod_name(VALUE mod) { - bool permanent; + int permanent; return classname(mod, &permanent); } -// Similar to logic in rb_mod_const_get() -static bool -is_constant_path(VALUE name) -{ - const char *path = RSTRING_PTR(name); - const char *pend = RSTRING_END(name); - rb_encoding *enc = rb_enc_get(name); - - const char *p = path; - - if (p >= pend || !*p) { - return false; - } - - while (p < pend) { - if (p + 2 <= pend && p[0] == ':' && p[1] == ':') { - p += 2; - } - - const char *pbeg = p; - while (p < pend && *p != ':') p++; - - if (pbeg == p) return false; - - if (rb_enc_symname_type(pbeg, p - pbeg, enc, 0) != ID_CONST) { - return false; - } - } - - return true; -} - -/* - * call-seq: - * mod.set_temporary_name(string) -> self - * mod.set_temporary_name(nil) -> self - * - * Sets the temporary name of the module +mod+. This name is used as a prefix - * for the names of constants declared in +mod+. If the module is assigned a - * permanent name, the temporary name is discarded. - * - * After a permanent name is assigned, a temporary name can no longer be set, - * and this method raises a RuntimeError. - * - * If the name given is not a string or is a zero length string, this method - * raises an ArgumentError. - * - * The temporary name must not be a valid constant name, to avoid confusion - * with actual constants. If you attempt to set a temporary name that is a - * a valid constant name, this method raises an ArgumentError. - * - * If the given name is +nil+, the module becomes anonymous. - * - * Example: - * - * m = Module.new # => #<Module:0x0000000102c68f38> - * m.name #=> nil - * - * m.set_temporary_name("fake_name") # => fake_name - * m.name #=> "fake_name" - * - * m.set_temporary_name(nil) # => #<Module:0x0000000102c68f38> - * m.name #=> nil - * - * n = Module.new - * n.set_temporary_name("fake_name") - * - * n::M = m - * n::M.name #=> "fake_name::M" - * N = n - * - * N.name #=> "nested_fake_name" - * N::M.name #=> "N::M" - */ - -VALUE -rb_mod_set_temporary_name(VALUE mod, VALUE name) -{ - // We don't allow setting the name if the classpath is already permanent: - if (RCLASS_EXT(mod)->permanent_classpath) { - rb_raise(rb_eRuntimeError, "can't change permanent name"); - } - - if (NIL_P(name)) { - // Set the temporary classpath to NULL (anonymous): - RCLASS_SET_CLASSPATH(mod, 0, FALSE); - } else { - // Ensure the name is a string: - StringValue(name); - - if (RSTRING_LEN(name) == 0) { - rb_raise(rb_eArgError, "empty class/module name"); - } - - if (is_constant_path(name)) { - rb_raise(rb_eArgError, "the temporary name must not be a constant path to avoid confusion"); - } - - // Set the temporary classpath to the given name: - RCLASS_SET_CLASSPATH(mod, name, FALSE); - } - - return mod; -} - static VALUE make_temporary_path(VALUE obj, VALUE klass) { VALUE path; switch (klass) { case Qnil: - path = rb_sprintf("#<Class:%p>", (void*)obj); - break; + path = rb_sprintf("#<Class:%p>", (void*)obj); + break; case Qfalse: - path = rb_sprintf("#<Module:%p>", (void*)obj); - break; + path = rb_sprintf("#<Module:%p>", (void*)obj); + break; default: - path = rb_sprintf("#<%"PRIsVALUE":%p>", klass, (void*)obj); - break; + path = rb_sprintf("#<%"PRIsVALUE":%p>", klass, (void*)obj); + break; } OBJ_FREEZE(path); return path; @@ -261,32 +146,32 @@ make_temporary_path(VALUE obj, VALUE klass) typedef VALUE (*fallback_func)(VALUE obj, VALUE name); static VALUE -rb_tmp_class_path(VALUE klass, bool *permanent, fallback_func fallback) +rb_tmp_class_path(VALUE klass, int *permanent, fallback_func fallback) { VALUE path = classname(klass, permanent); if (!NIL_P(path)) { - return path; + return path; } else { - if (RB_TYPE_P(klass, T_MODULE)) { - if (rb_obj_class(klass) == rb_cModule) { - path = Qfalse; - } - else { - bool perm; - path = rb_tmp_class_path(RBASIC(klass)->klass, &perm, fallback); - } - } - *permanent = false; - return fallback(klass, path); + if (RB_TYPE_P(klass, T_MODULE)) { + if (rb_obj_class(klass) == rb_cModule) { + path = Qfalse; + } + else { + int perm; + path = rb_tmp_class_path(RBASIC(klass)->klass, &perm, fallback); + } + } + *permanent = 0; + return fallback(klass, path); } } VALUE rb_class_path(VALUE klass) { - bool permanent; + int permanent; VALUE path = rb_tmp_class_path(klass, &permanent, make_temporary_path); if (!NIL_P(path)) path = rb_str_dup(path); return path; @@ -307,7 +192,7 @@ no_fallback(VALUE obj, VALUE name) VALUE rb_search_class_path(VALUE klass) { - bool permanent; + int permanent; return rb_tmp_class_path(klass, &permanent, no_fallback); } @@ -329,18 +214,21 @@ build_const_path(VALUE head, ID tail) void rb_set_class_path_string(VALUE klass, VALUE under, VALUE name) { - bool permanent = true; - VALUE str; + ID pathid = classpath; + if (under == rb_cObject) { - str = rb_str_new_frozen(name); + str = rb_str_new_frozen(name); } else { + int permanent; str = rb_tmp_class_path(under, &permanent, make_temporary_path); str = build_const_pathname(str, name); + if (!permanent) { + pathid = tmp_classpath; + } } - - RCLASS_SET_CLASSPATH(klass, str, permanent); + rb_ivar_set(klass, pathid, str); } void @@ -360,31 +248,31 @@ rb_path_to_class(VALUE pathname) VALUE c = rb_cObject; if (!rb_enc_asciicompat(enc)) { - rb_raise(rb_eArgError, "invalid class path encoding (non ASCII)"); + rb_raise(rb_eArgError, "invalid class path encoding (non ASCII)"); } pbeg = p = path; pend = path + RSTRING_LEN(pathname); if (path == pend || path[0] == '#') { - rb_raise(rb_eArgError, "can't retrieve anonymous class %"PRIsVALUE, - QUOTE(pathname)); + rb_raise(rb_eArgError, "can't retrieve anonymous class %"PRIsVALUE, + QUOTE(pathname)); } while (p < pend) { - while (p < pend && *p != ':') p++; - id = rb_check_id_cstr(pbeg, p-pbeg, enc); - if (p < pend && p[0] == ':') { - if ((size_t)(pend - p) < 2 || p[1] != ':') goto undefined_class; - p += 2; - pbeg = p; - } - if (!id) { + while (p < pend && *p != ':') p++; + id = rb_check_id_cstr(pbeg, p-pbeg, enc); + if (p < pend && p[0] == ':') { + if ((size_t)(pend - p) < 2 || p[1] != ':') goto undefined_class; + p += 2; + pbeg = p; + } + if (!id) { goto undefined_class; - } - c = rb_const_search(c, id, TRUE, FALSE, FALSE); - if (UNDEF_P(c)) goto undefined_class; + } + c = rb_const_search(c, id, TRUE, FALSE, FALSE); + if (c == Qundef) goto undefined_class; if (!rb_namespace_p(c)) { - rb_raise(rb_eTypeError, "%"PRIsVALUE" does not refer to class/module", - pathname); - } + rb_raise(rb_eTypeError, "%"PRIsVALUE" does not refer to class/module", + pathname); + } } RB_GC_GUARD(pathname); @@ -411,7 +299,7 @@ rb_class_name(VALUE klass) const char * rb_class2name(VALUE klass) { - bool permanent; + int permanent; VALUE path = rb_tmp_class_path(rb_class_real(klass), &permanent, make_temporary_path); if (NIL_P(path)) return NULL; return RSTRING_PTR(path); @@ -485,22 +373,22 @@ rb_global_entry(ID id) { struct rb_global_entry *entry = rb_find_global_entry(id); if (!entry) { - struct rb_global_variable *var; - entry = ALLOC(struct rb_global_entry); - var = ALLOC(struct rb_global_variable); - entry->id = id; - entry->var = var; + struct rb_global_variable *var; + entry = ALLOC(struct rb_global_entry); + var = ALLOC(struct rb_global_variable); + entry->id = id; + entry->var = var; entry->ractor_local = false; - var->counter = 1; - var->data = 0; - var->getter = rb_gvar_undef_getter; - var->setter = rb_gvar_undef_setter; - var->marker = rb_gvar_undef_marker; - var->compactor = rb_gvar_undef_compactor; + var->counter = 1; + var->data = 0; + var->getter = rb_gvar_undef_getter; + var->setter = rb_gvar_undef_setter; + var->marker = rb_gvar_undef_marker; + var->compactor = rb_gvar_undef_compactor; - var->block_trace = 0; - var->trace = 0; - rb_id_table_insert(rb_global_tbl, id, (VALUE)entry); + var->block_trace = 0; + var->trace = 0; + rb_id_table_insert(rb_global_tbl, id, (VALUE)entry); } return entry; } @@ -600,19 +488,18 @@ mark_global_entry(VALUE v, void *ignored) (*var->marker)(var->data); trace = var->trace; while (trace) { - if (trace->data) rb_gc_mark_maybe(trace->data); - trace = trace->next; + if (trace->data) rb_gc_mark_maybe(trace->data); + trace = trace->next; } return ID_TABLE_CONTINUE; } -#define gc_mark_table(task) \ - if (rb_global_tbl) { rb_id_table_foreach_values(rb_global_tbl, task##_global_entry, 0); } - void rb_gc_mark_global_tbl(void) { - gc_mark_table(mark); + if (rb_global_tbl) { + rb_id_table_foreach_values(rb_global_tbl, mark_global_entry, 0); + } } static enum rb_id_table_iterator_result @@ -628,7 +515,9 @@ update_global_entry(VALUE v, void *ignored) void rb_gc_update_global_tbl(void) { - gc_mark_table(update); + if (rb_global_tbl) { + rb_id_table_foreach_values(rb_global_tbl, update_global_entry, 0); + } } static ID @@ -638,12 +527,12 @@ global_id(const char *name) if (name[0] == '$') id = rb_intern(name); else { - size_t len = strlen(name); + size_t len = strlen(name); VALUE vbuf = 0; char *buf = ALLOCV_N(char, vbuf, len+1); - buf[0] = '$'; - memcpy(buf+1, name, len); - id = rb_intern2(buf, len+1); + buf[0] = '$'; + memcpy(buf+1, name, len); + id = rb_intern2(buf, len+1); ALLOCV_END(vbuf); } return id; @@ -726,10 +615,10 @@ rb_f_trace_var(int argc, const VALUE *argv) struct trace_var *trace; if (rb_scan_args(argc, argv, "11", &var, &cmd) == 1) { - cmd = rb_block_proc(); + cmd = rb_block_proc(); } if (NIL_P(cmd)) { - return rb_f_untrace_var(argc, argv); + return rb_f_untrace_var(argc, argv); } entry = rb_global_entry(rb_to_id(var)); trace = ALLOC(struct trace_var); @@ -752,14 +641,14 @@ remove_trace(struct rb_global_variable *var) t.next = trace; trace = &t; while (trace->next) { - next = trace->next; - if (next->removed) { - trace->next = next->next; - xfree(next); - } - else { - trace = next; - } + next = trace->next; + if (next->removed) { + trace->next = next->next; + xfree(next); + } + else { + trace = next; + } } var->trace = t.next; } @@ -775,35 +664,35 @@ rb_f_untrace_var(int argc, const VALUE *argv) rb_scan_args(argc, argv, "11", &var, &cmd); id = rb_check_id(&var); if (!id) { - rb_name_error_str(var, "undefined global variable %"PRIsVALUE"", QUOTE(var)); + rb_name_error_str(var, "undefined global variable %"PRIsVALUE"", QUOTE(var)); } if ((entry = rb_find_global_entry(id)) == NULL) { - rb_name_error(id, "undefined global variable %"PRIsVALUE"", QUOTE_ID(id)); + rb_name_error(id, "undefined global variable %"PRIsVALUE"", QUOTE_ID(id)); } trace = entry->var->trace; if (NIL_P(cmd)) { - VALUE ary = rb_ary_new(); + VALUE ary = rb_ary_new(); - while (trace) { - struct trace_var *next = trace->next; - rb_ary_push(ary, (VALUE)trace->data); - trace->removed = 1; - trace = next; - } + while (trace) { + struct trace_var *next = trace->next; + rb_ary_push(ary, (VALUE)trace->data); + trace->removed = 1; + trace = next; + } - if (!entry->var->block_trace) remove_trace(entry->var); - return ary; + if (!entry->var->block_trace) remove_trace(entry->var); + return ary; } else { - while (trace) { - if (trace->data == cmd) { - trace->removed = 1; - if (!entry->var->block_trace) remove_trace(entry->var); - return rb_ary_new3(1, cmd); - } - trace = trace->next; - } + while (trace) { + if (trace->data == cmd) { + trace->removed = 1; + if (!entry->var->block_trace) remove_trace(entry->var); + return rb_ary_new3(1, cmd); + } + trace = trace->next; + } } return Qnil; } @@ -820,8 +709,8 @@ trace_ev(VALUE v) struct trace_var *trace = data->trace; while (trace) { - (*trace->func)(trace->data, data->val); - trace = trace->next; + (*trace->func)(trace->data, data->val); + trace = trace->next; } return Qnil; @@ -845,10 +734,10 @@ rb_gvar_set_entry(struct rb_global_entry *entry, VALUE val) (*var->setter)(val, entry->id, var->data); if (var->trace && !var->block_trace) { - var->block_trace = 1; - trace.trace = var->trace; - trace.val = val; - rb_ensure(trace_ev, (VALUE)&trace, trace_en, (VALUE)var); + var->block_trace = 1; + trace.trace = var->trace; + trace.val = val; + rb_ensure(trace_ev, (VALUE)&trace, trace_en, (VALUE)var); } return val; } @@ -889,7 +778,7 @@ rb_gv_get(const char *name) return rb_gvar_get(id); } -VALUE +MJIT_FUNC_EXPORTED VALUE rb_gvar_defined(ID id) { struct rb_global_entry *entry = rb_global_entry(id); @@ -930,22 +819,22 @@ rb_f_global_variables(void) rb_id_table_foreach(rb_global_tbl, gvar_i, (void *)ary); if (!NIL_P(backref)) { - char buf[2]; - int i, nmatch = rb_match_count(backref); - buf[0] = '$'; - for (i = 1; i <= nmatch; ++i) { - if (!RTEST(rb_reg_nth_defined(i, backref))) continue; - if (i < 10) { - /* probably reused, make static ID */ - buf[1] = (char)(i + '0'); - sym = ID2SYM(rb_intern2(buf, 2)); - } - else { - /* dynamic symbol */ - sym = rb_str_intern(rb_sprintf("$%d", i)); - } - rb_ary_push(ary, sym); - } + char buf[2]; + int i, nmatch = rb_match_count(backref); + buf[0] = '$'; + for (i = 1; i <= nmatch; ++i) { + if (!rb_match_nth_defined(i, backref)) continue; + if (i < 10) { + /* probably reused, make static ID */ + buf[1] = (char)(i + '0'); + sym = ID2SYM(rb_intern2(buf, 2)); + } + else { + /* dynamic symbol */ + sym = rb_str_intern(rb_sprintf("$%d", i)); + } + rb_ary_push(ary, sym); + } } return ary; } @@ -963,33 +852,57 @@ rb_alias_variable(ID name1, ID name2) entry2 = rb_global_entry(name2); if (!rb_id_table_lookup(gtbl, name1, &data1)) { - entry1 = ALLOC(struct rb_global_entry); - entry1->id = name1; - rb_id_table_insert(gtbl, name1, (VALUE)entry1); + entry1 = ALLOC(struct rb_global_entry); + entry1->id = name1; + rb_id_table_insert(gtbl, name1, (VALUE)entry1); } else if ((entry1 = (struct rb_global_entry *)data1)->var != entry2->var) { - struct rb_global_variable *var = entry1->var; - if (var->block_trace) { - rb_raise(rb_eRuntimeError, "can't alias in tracer"); - } - var->counter--; - if (var->counter == 0) { - struct trace_var *trace = var->trace; - while (trace) { - struct trace_var *next = trace->next; - xfree(trace); - trace = next; - } - xfree(var); - } + struct rb_global_variable *var = entry1->var; + if (var->block_trace) { + rb_raise(rb_eRuntimeError, "can't alias in tracer"); + } + var->counter--; + if (var->counter == 0) { + struct trace_var *trace = var->trace; + while (trace) { + struct trace_var *next = trace->next; + xfree(trace); + trace = next; + } + xfree(var); + } } else { - return; + return; } entry2->var->counter++; entry1->var = entry2->var; } +static bool +iv_index_tbl_lookup(struct st_table *tbl, ID id, uint32_t *indexp) +{ + st_data_t ent_data; + int r; + + if (tbl == NULL) return false; + + RB_VM_LOCK_ENTER(); + { + r = st_lookup(tbl, (st_data_t)id, &ent_data); + } + RB_VM_LOCK_LEAVE(); + + if (r) { + struct rb_iv_index_tbl_entry *ent = (void *)ent_data; + *indexp = ent->index; + return true; + } + else { + return false; + } +} + static void IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(ID id) { @@ -1027,24 +940,9 @@ generic_ivtbl_no_ractor_check(VALUE obj) } static int -gen_ivtbl_get_unlocked(VALUE obj, ID id, struct gen_ivtbl **ivtbl) +gen_ivtbl_get(VALUE obj, ID id, struct gen_ivtbl **ivtbl) { st_data_t data; - - if (st_lookup(generic_ivtbl(obj, id, false), (st_data_t)obj, &data)) { - *ivtbl = (struct gen_ivtbl *)data; - return 1; - } - - return 0; -} - -int -rb_gen_ivtbl_get(VALUE obj, ID id, struct gen_ivtbl **ivtbl) -{ - RUBY_ASSERT(!RB_TYPE_P(obj, T_ICLASS)); - - st_data_t data; int r = 0; RB_VM_LOCK_ENTER(); @@ -1059,10 +957,66 @@ rb_gen_ivtbl_get(VALUE obj, ID id, struct gen_ivtbl **ivtbl) return r; } -int +MJIT_FUNC_EXPORTED int rb_ivar_generic_ivtbl_lookup(VALUE obj, struct gen_ivtbl **ivtbl) { - return rb_gen_ivtbl_get(obj, 0, ivtbl); + return gen_ivtbl_get(obj, 0, ivtbl); +} + +MJIT_FUNC_EXPORTED VALUE +rb_ivar_generic_lookup_with_index(VALUE obj, ID id, uint32_t index) +{ + struct gen_ivtbl *ivtbl; + + if (gen_ivtbl_get(obj, id, &ivtbl)) { + if (LIKELY(index < ivtbl->numiv)) { + VALUE val = ivtbl->ivptr[index]; + return val; + } + } + + return Qundef; +} + +static VALUE +generic_ivar_delete(VALUE obj, ID id, VALUE undef) +{ + struct gen_ivtbl *ivtbl; + + if (gen_ivtbl_get(obj, id, &ivtbl)) { + st_table *iv_index_tbl = RCLASS_IV_INDEX_TBL(rb_obj_class(obj)); + uint32_t index; + + if (iv_index_tbl && iv_index_tbl_lookup(iv_index_tbl, id, &index)) { + if (index < ivtbl->numiv) { + VALUE ret = ivtbl->ivptr[index]; + + ivtbl->ivptr[index] = Qundef; + return ret == Qundef ? undef : ret; + } + } + } + return undef; +} + +static VALUE +generic_ivar_get(VALUE obj, ID id, VALUE undef) +{ + struct gen_ivtbl *ivtbl; + + if (gen_ivtbl_get(obj, id, &ivtbl)) { + st_table *iv_index_tbl = RCLASS_IV_INDEX_TBL(rb_obj_class(obj)); + uint32_t index; + + if (iv_index_tbl && iv_index_tbl_lookup(iv_index_tbl, id, &index)) { + if (index < ivtbl->numiv) { + VALUE ret = ivtbl->ivptr[index]; + + return ret == Qundef ? undef : ret; + } + } + } + return undef; } static size_t @@ -1074,14 +1028,12 @@ gen_ivtbl_bytes(size_t n) static struct gen_ivtbl * gen_ivtbl_resize(struct gen_ivtbl *old, uint32_t n) { - RUBY_ASSERT(n > 0); - uint32_t len = old ? old->numiv : 0; struct gen_ivtbl *ivtbl = xrealloc(old, gen_ivtbl_bytes(n)); ivtbl->numiv = n; for (; len < n; len++) { - ivtbl->ivptr[len] = Qundef; + ivtbl->ivptr[len] = Qundef; } return ivtbl; @@ -1100,6 +1052,18 @@ gen_ivtbl_dup(const struct gen_ivtbl *orig) } #endif +static uint32_t +iv_index_tbl_newsize(struct ivar_update *ivup) +{ + if (!ivup->iv_extended) { + return (uint32_t)ivup->u.iv_index_tbl->num_entries; + } + else { + uint32_t index = (uint32_t)ivup->index; /* should not overflow */ + return (index+1) + (index+1)/4; /* (index+1)*1.25 */ + } +} + static int generic_ivar_update(st_data_t *k, st_data_t *v, st_data_t u, int existing) { @@ -1109,40 +1073,71 @@ generic_ivar_update(st_data_t *k, st_data_t *v, st_data_t u, int existing) struct gen_ivtbl *ivtbl = 0; if (existing) { - ivtbl = (struct gen_ivtbl *)*v; - if (ivup->iv_index < ivtbl->numiv) { - ivup->ivtbl = ivtbl; + ivtbl = (struct gen_ivtbl *)*v; + if (ivup->index < ivtbl->numiv) { + ivup->u.ivtbl = ivtbl; return ST_STOP; } } FL_SET((VALUE)*k, FL_EXIVAR); - ivtbl = gen_ivtbl_resize(ivtbl, ivup->max_index); - // Reinsert in to the hash table because ivtbl might be a newly resized chunk of memory + uint32_t newsize = iv_index_tbl_newsize(ivup); + ivtbl = gen_ivtbl_resize(ivtbl, newsize); *v = (st_data_t)ivtbl; - ivup->ivtbl = ivtbl; -#if !SHAPE_IN_BASIC_FLAGS - ivtbl->shape_id = rb_shape_id(ivup->shape); -#endif + ivup->u.ivtbl = ivtbl; return ST_CONTINUE; } +static VALUE +generic_ivar_defined(VALUE obj, ID id) +{ + struct gen_ivtbl *ivtbl; + st_table *iv_index_tbl = RCLASS_IV_INDEX_TBL(rb_obj_class(obj)); + uint32_t index; + + if (!iv_index_tbl_lookup(iv_index_tbl, id, &index)) return Qfalse; + if (!gen_ivtbl_get(obj, id, &ivtbl)) return Qfalse; + + return RBOOL((index < ivtbl->numiv) && (ivtbl->ivptr[index] != Qundef)); +} + +static int +generic_ivar_remove(VALUE obj, ID id, VALUE *valp) +{ + struct gen_ivtbl *ivtbl; + uint32_t index; + st_table *iv_index_tbl = RCLASS_IV_INDEX_TBL(rb_obj_class(obj)); + + if (!iv_index_tbl) return 0; + if (!iv_index_tbl_lookup(iv_index_tbl, id, &index)) return 0; + if (!gen_ivtbl_get(obj, id, &ivtbl)) return 0; + + if (index < ivtbl->numiv) { + if (ivtbl->ivptr[index] != Qundef) { + *valp = ivtbl->ivptr[index]; + ivtbl->ivptr[index] = Qundef; + return 1; + } + } + return 0; +} + static void -gen_ivtbl_mark_and_update(struct gen_ivtbl *ivtbl) +gen_ivtbl_mark(const struct gen_ivtbl *ivtbl) { uint32_t i; for (i = 0; i < ivtbl->numiv; i++) { - rb_gc_mark_and_move(&ivtbl->ivptr[i]); + rb_gc_mark(ivtbl->ivptr[i]); } } void -rb_mark_and_update_generic_ivar(VALUE obj) +rb_mark_generic_ivar(VALUE obj) { struct gen_ivtbl *ivtbl; - if (rb_gen_ivtbl_get(obj, 0, &ivtbl)) { - gen_ivtbl_mark_and_update(ivtbl); + if (gen_ivtbl_get(obj, 0, &ivtbl)) { + gen_ivtbl_mark(ivtbl); } } @@ -1162,7 +1157,7 @@ rb_free_generic_ivar(VALUE obj) st_data_t key = (st_data_t)obj, ivtbl; if (st_delete(generic_ivtbl_no_ractor_check(obj), &key, &ivtbl)) - xfree((struct gen_ivtbl *)ivtbl); + xfree((struct gen_ivtbl *)ivtbl); } RUBY_FUNC_EXPORTED size_t @@ -1170,142 +1165,119 @@ rb_generic_ivar_memsize(VALUE obj) { struct gen_ivtbl *ivtbl; - if (rb_gen_ivtbl_get(obj, 0, &ivtbl)) - return gen_ivtbl_bytes(ivtbl->numiv); + if (gen_ivtbl_get(obj, 0, &ivtbl)) + return gen_ivtbl_bytes(ivtbl->numiv); return 0; } -#if !SHAPE_IN_BASIC_FLAGS -shape_id_t -rb_generic_shape_id(VALUE obj) +static size_t +gen_ivtbl_count(const struct gen_ivtbl *ivtbl) { - struct gen_ivtbl *ivtbl = 0; - shape_id_t shape_id = 0; + uint32_t i; + size_t n = 0; + + for (i = 0; i < ivtbl->numiv; i++) { + if (ivtbl->ivptr[i] != Qundef) { + n++; + } + } + return n; +} + +static int +lock_st_lookup(st_table *tab, st_data_t key, st_data_t *value) +{ + int r; RB_VM_LOCK_ENTER(); { - st_table* global_iv_table = generic_ivtbl(obj, 0, false); - - if (global_iv_table && st_lookup(global_iv_table, obj, (st_data_t *)&ivtbl)) { - shape_id = ivtbl->shape_id; - } - else if (OBJ_FROZEN(obj)) { - shape_id = SPECIAL_CONST_SHAPE_ID; - } + r = st_lookup(tab, key, value); } RB_VM_LOCK_LEAVE(); - - return shape_id; + return r; } -#endif -static size_t -gen_ivtbl_count(const struct gen_ivtbl *ivtbl) +static int +lock_st_delete(st_table *tab, st_data_t *key, st_data_t *value) { - uint32_t i; - size_t n = 0; + int r; + RB_VM_LOCK_ENTER(); + { + r = st_delete(tab, key, value); + } + RB_VM_LOCK_LEAVE(); + return r; +} - for (i = 0; i < ivtbl->numiv; i++) { - if (!UNDEF_P(ivtbl->ivptr[i])) { - n++; - } +static int +lock_st_is_member(st_table *tab, st_data_t key) +{ + int r; + RB_VM_LOCK_ENTER(); + { + r = st_is_member(tab, key); } + RB_VM_LOCK_LEAVE(); + return r; +} - return n; +static int +lock_st_insert(st_table *tab, st_data_t key, st_data_t value) +{ + int r; + RB_VM_LOCK_ENTER(); + { + r = st_insert(tab, key, value); + } + RB_VM_LOCK_LEAVE(); + return r; } VALUE rb_ivar_lookup(VALUE obj, ID id, VALUE undef) { if (SPECIAL_CONST_P(obj)) return undef; - - shape_id_t shape_id; - VALUE * ivar_list; - rb_shape_t * shape; - -#if SHAPE_IN_BASIC_FLAGS - shape_id = RBASIC_SHAPE_ID(obj); -#endif - switch (BUILTIN_TYPE(obj)) { - case T_CLASS: - case T_MODULE: + case T_OBJECT: { - bool found; + uint32_t index; + uint32_t len = ROBJECT_NUMIV(obj); + VALUE *ptr = ROBJECT_IVPTR(obj); VALUE val; - RB_VM_LOCK_ENTER(); - { -#if !SHAPE_IN_BASIC_FLAGS - shape_id = RCLASS_SHAPE_ID(obj); -#endif - - attr_index_t index = 0; - shape = rb_shape_get_shape_by_id(shape_id); - found = rb_shape_get_iv_index(shape, id, &index); - - if (found) { - ivar_list = RCLASS_IVPTR(obj); - RUBY_ASSERT(ivar_list); - - val = ivar_list[index]; - } - else { - val = undef; - } + if (iv_index_tbl_lookup(ROBJECT_IV_INDEX_TBL(obj), id, &index) && + index < len && + (val = ptr[index]) != Qundef) { + return val; } - RB_VM_LOCK_LEAVE(); - - if (found && - rb_is_instance_id(id) && - UNLIKELY(!rb_ractor_main_p()) && - !rb_ractor_shareable_p(val)) { - rb_raise(rb_eRactorIsolationError, - "can not get unshareable values from instance variables of classes/modules from non-main Ractors"); + else { + break; } - return val; } - case T_OBJECT: + case T_CLASS: + case T_MODULE: { -#if !SHAPE_IN_BASIC_FLAGS - shape_id = ROBJECT_SHAPE_ID(obj); -#endif - if (rb_shape_obj_too_complex(obj)) { - st_table * iv_table = ROBJECT_IV_HASH(obj); - VALUE val; - if (rb_st_lookup(iv_table, (st_data_t)id, (st_data_t *)&val)) { - return val; - } - else { - return undef; + st_data_t val; + + if (RCLASS_IV_TBL(obj) && + lock_st_lookup(RCLASS_IV_TBL(obj), (st_data_t)id, &val)) { + if (rb_is_instance_id(id) && + UNLIKELY(!rb_ractor_main_p()) && + !rb_ractor_shareable_p(val)) { + rb_raise(rb_eRactorIsolationError, + "can not get unshareable values from instance variables of classes/modules from non-main Ractors"); } + return val; + } + else { + break; } - - RUBY_ASSERT(!rb_shape_obj_too_complex(obj)); - ivar_list = ROBJECT_IVPTR(obj); - break; } default: - if (FL_TEST_RAW(obj, FL_EXIVAR)) { - struct gen_ivtbl *ivtbl; - rb_gen_ivtbl_get(obj, id, &ivtbl); -#if !SHAPE_IN_BASIC_FLAGS - shape_id = ivtbl->shape_id; -#endif - ivar_list = ivtbl->ivptr; - } - else { - return undef; - } - break; - } - - attr_index_t index = 0; - shape = rb_shape_get_shape_by_id(shape_id); - if (rb_shape_get_iv_index(shape, id, &index)) { - return ivar_list[index]; + if (FL_TEST(obj, FL_EXIVAR)) + return generic_ivar_get(obj, id, undef); + break; } - return undef; } @@ -1326,31 +1298,42 @@ rb_attr_get(VALUE obj, ID id) static VALUE rb_ivar_delete(VALUE obj, ID id, VALUE undef) { - rb_check_frozen(obj); - - VALUE val = undef; - rb_shape_t * shape = rb_shape_get_shape(obj); + VALUE *ptr; + struct st_table *iv_index_tbl; + uint32_t len, index; + rb_check_frozen(obj); switch (BUILTIN_TYPE(obj)) { + case T_OBJECT: + len = ROBJECT_NUMIV(obj); + ptr = ROBJECT_IVPTR(obj); + iv_index_tbl = ROBJECT_IV_INDEX_TBL(obj); + if (iv_index_tbl_lookup(iv_index_tbl, id, &index) && + index < len) { + VALUE val = ptr[index]; + ptr[index] = Qundef; + + if (val != Qundef) { + return val; + } + } + break; case T_CLASS: case T_MODULE: IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(id); - - RB_VM_LOCK_ENTER(); - { - rb_shape_transition_shape_remove_ivar(obj, id, shape, &val); + if (RCLASS_IV_TBL(obj)) { + st_data_t id_data = (st_data_t)id, val; + if (lock_st_delete(RCLASS_IV_TBL(obj), &id_data, &val)) { + return (VALUE)val; + } } - RB_VM_LOCK_LEAVE(); - - break; - default: { - rb_shape_transition_shape_remove_ivar(obj, id, shape, &val); - - break; - } + break; + default: + if (FL_TEST(obj, FL_EXIVAR)) + return generic_ivar_delete(obj, id, undef); + break; } - - return val; + return undef; } VALUE @@ -1359,246 +1342,213 @@ rb_attr_delete(VALUE obj, ID id) return rb_ivar_delete(obj, id, Qnil); } -static void -generic_ivar_set(VALUE obj, ID id, VALUE val) +static st_table * +iv_index_tbl_make(VALUE obj, VALUE klass) { - struct ivar_update ivup; + st_table *iv_index_tbl; - attr_index_t index; - // The returned shape will have `id` in its iv_table - rb_shape_t *shape = rb_shape_get_shape(obj); - bool found = rb_shape_get_iv_index(shape, id, &index); - if (!found) { - index = shape->next_iv_index; - shape = rb_shape_get_next(shape, obj, id); - RUBY_ASSERT(index == (shape->next_iv_index - 1)); + if (UNLIKELY(!klass)) { + rb_raise(rb_eTypeError, "hidden object cannot have instance variables"); } - ivup.max_index = shape->next_iv_index; -#if !SHAPE_IN_BASIC_FLAGS - ivup.shape = shape; -#endif - - RB_VM_LOCK_ENTER(); - { - ivup.iv_index = (uint32_t)index; - - st_update(generic_ivtbl(obj, id, false), (st_data_t)obj, generic_ivar_update, (st_data_t)&ivup); + if ((iv_index_tbl = RCLASS_IV_INDEX_TBL(klass)) == NULL) { + RB_VM_LOCK_ENTER(); + if ((iv_index_tbl = RCLASS_IV_INDEX_TBL(klass)) == NULL) { + iv_index_tbl = RCLASS_IV_INDEX_TBL(klass) = st_init_numtable(); + } + RB_VM_LOCK_LEAVE(); } - RB_VM_LOCK_LEAVE(); - - ivup.ivtbl->ivptr[ivup.iv_index] = val; - RB_OBJ_WRITTEN(obj, Qundef, val); - if (!found) { - rb_shape_set_shape(obj, shape); - } + return iv_index_tbl; } -void -rb_ensure_iv_list_size(VALUE obj, uint32_t current_capacity, uint32_t new_capacity) +static void +iv_index_tbl_extend(struct ivar_update *ivup, ID id, VALUE klass) { - RUBY_ASSERT(!rb_shape_obj_too_complex(obj)); + ASSERT_vm_locking(); + st_data_t ent_data; + struct rb_iv_index_tbl_entry *ent; - if (RBASIC(obj)->flags & ROBJECT_EMBED) { - VALUE *ptr = ROBJECT_IVPTR(obj); - VALUE *newptr = ALLOC_N(VALUE, new_capacity); - MEMCPY(newptr, ptr, VALUE, current_capacity); - RB_FL_UNSET_RAW(obj, ROBJECT_EMBED); - ROBJECT(obj)->as.heap.ivptr = newptr; + if (st_lookup(ivup->u.iv_index_tbl, (st_data_t)id, &ent_data)) { + ent = (void *)ent_data; + ivup->index = ent->index; + return; } - else { - REALLOC_N(ROBJECT(obj)->as.heap.ivptr, VALUE, new_capacity); + if (ivup->u.iv_index_tbl->num_entries >= INT_MAX) { + rb_raise(rb_eArgError, "too many instance variables"); } + ent = ALLOC(struct rb_iv_index_tbl_entry); + ent->index = ivup->index = (uint32_t)ivup->u.iv_index_tbl->num_entries; + ent->class_value = klass; + ent->class_serial = RCLASS_SERIAL(klass); + st_add_direct(ivup->u.iv_index_tbl, (st_data_t)id, (st_data_t)ent); + ivup->iv_extended = 1; } -struct gen_ivtbl * -rb_ensure_generic_iv_list_size(VALUE obj, rb_shape_t *shape, uint32_t newsize) +static void +generic_ivar_set(VALUE obj, ID id, VALUE val) { - struct gen_ivtbl * ivtbl = 0; + VALUE klass = rb_obj_class(obj); + struct ivar_update ivup; + ivup.iv_extended = 0; + ivup.u.iv_index_tbl = iv_index_tbl_make(obj, klass); RB_VM_LOCK_ENTER(); { - if (UNLIKELY(!gen_ivtbl_get_unlocked(obj, 0, &ivtbl) || newsize > ivtbl->numiv)) { - struct ivar_update ivup = { - .iv_index = newsize - 1, - .max_index = newsize, -#if !SHAPE_IN_BASIC_FLAGS - .shape = shape -#endif - }; - st_update(generic_ivtbl_no_ractor_check(obj), (st_data_t)obj, generic_ivar_update, (st_data_t)&ivup); - ivtbl = ivup.ivtbl; - FL_SET_RAW(obj, FL_EXIVAR); - } + iv_index_tbl_extend(&ivup, id, klass); + st_update(generic_ivtbl(obj, id, false), (st_data_t)obj, generic_ivar_update, + (st_data_t)&ivup); } RB_VM_LOCK_LEAVE(); - RUBY_ASSERT(ivtbl); + ivup.u.ivtbl->ivptr[ivup.index] = val; - return ivtbl; + RB_OBJ_WRITTEN(obj, Qundef, val); } -// @note May raise when there are too many instance variables. -rb_shape_t * -rb_grow_iv_list(VALUE obj) +static VALUE * +obj_ivar_heap_alloc(VALUE obj, size_t newsize) { - rb_shape_t * initial_shape = rb_shape_get_shape(obj); - uint32_t len = initial_shape->capacity; - RUBY_ASSERT(len > 0); - uint32_t newsize = (uint32_t)(len * 2); - - rb_shape_t * res = rb_shape_transition_shape_capa(initial_shape, newsize); - - rb_ensure_iv_list_size(obj, len, newsize); + VALUE *newptr = rb_transient_heap_alloc(obj, sizeof(VALUE) * newsize); - rb_shape_set_shape(obj, res); - - return res; -} - -int -rb_obj_evacuate_ivs_to_hash_table(ID key, VALUE val, st_data_t arg) -{ - st_insert((st_table *)arg, (st_data_t)key, (st_data_t)val); - return ST_CONTINUE; + if (newptr != NULL) { + ROBJ_TRANSIENT_SET(obj); + } + else { + ROBJ_TRANSIENT_UNSET(obj); + newptr = ALLOC_N(VALUE, newsize); + } + return newptr; } -attr_index_t -rb_obj_ivar_set(VALUE obj, ID id, VALUE val) +static VALUE * +obj_ivar_heap_realloc(VALUE obj, int32_t len, size_t newsize) { - attr_index_t index; - - rb_shape_t *shape = rb_shape_get_shape(obj); - uint32_t num_iv = shape->capacity; - - if (rb_shape_obj_too_complex(obj)) { - st_table * table = ROBJECT_IV_HASH(obj); - st_insert(table, (st_data_t)id, (st_data_t)val); - RB_OBJ_WRITTEN(obj, Qundef, val); - return 0; - } - - if (!rb_shape_get_iv_index(shape, id, &index)) { - index = shape->next_iv_index; - if (index >= MAX_IVARS) { - rb_raise(rb_eArgError, "too many instance variables"); - } - - RUBY_ASSERT(!rb_shape_obj_too_complex(obj)); + VALUE *newptr; + int i; - if (UNLIKELY(shape->next_iv_index >= num_iv)) { - RUBY_ASSERT(shape->next_iv_index == num_iv); + if (ROBJ_TRANSIENT_P(obj)) { + const VALUE *orig_ptr = ROBJECT(obj)->as.heap.ivptr; + newptr = obj_ivar_heap_alloc(obj, newsize); - shape = rb_grow_iv_list(obj); - RUBY_ASSERT(shape->type == SHAPE_CAPACITY_CHANGE); + assert(newptr); + ROBJECT(obj)->as.heap.ivptr = newptr; + for (i=0; i<(int)len; i++) { + newptr[i] = orig_ptr[i]; } + } + else { + REALLOC_N(ROBJECT(obj)->as.heap.ivptr, VALUE, newsize); + newptr = ROBJECT(obj)->as.heap.ivptr; + } - rb_shape_t *next_shape = rb_shape_get_next(shape, obj, id); - - if (next_shape->type == SHAPE_OBJ_TOO_COMPLEX) { - st_table * table = st_init_numtable_with_size(shape->next_iv_index); - - // Evacuate all previous values from shape into id_table - rb_ivar_foreach(obj, rb_obj_evacuate_ivs_to_hash_table, (st_data_t)table); - - // Insert new value too - st_insert(table, (st_data_t)id, (st_data_t)val); - RB_OBJ_WRITTEN(obj, Qundef, val); - - rb_shape_set_too_complex(obj); - RUBY_ASSERT(rb_shape_obj_too_complex(obj)); - - if (!(RBASIC(obj)->flags & ROBJECT_EMBED)) { - xfree(ROBJECT(obj)->as.heap.ivptr); - } + return newptr; +} - ROBJECT(obj)->as.heap.ivptr = (VALUE *)table; +#if USE_TRANSIENT_HEAP +void +rb_obj_transient_heap_evacuate(VALUE obj, int promote) +{ + if (ROBJ_TRANSIENT_P(obj)) { + uint32_t len = ROBJECT_NUMIV(obj); + const VALUE *old_ptr = ROBJECT_IVPTR(obj); + VALUE *new_ptr; - return 0; + if (promote) { + new_ptr = ALLOC_N(VALUE, len); + ROBJ_TRANSIENT_UNSET(obj); } else { - rb_shape_set_shape(obj, next_shape); - RUBY_ASSERT(next_shape->type == SHAPE_IVAR); - RUBY_ASSERT(index == (next_shape->next_iv_index - 1)); + new_ptr = obj_ivar_heap_alloc(obj, len); } + MEMCPY(new_ptr, old_ptr, VALUE, len); + ROBJECT(obj)->as.heap.ivptr = new_ptr; } +} +#endif - RUBY_ASSERT(!rb_shape_obj_too_complex(obj)); - RB_OBJ_WRITE(obj, &ROBJECT_IVPTR(obj)[index], val); +static void +init_iv_list(VALUE obj, uint32_t len, uint32_t newsize, st_table *index_tbl) +{ + VALUE *ptr = ROBJECT_IVPTR(obj); + VALUE *newptr; + + if (RBASIC(obj)->flags & ROBJECT_EMBED) { + newptr = obj_ivar_heap_alloc(obj, newsize); + MEMCPY(newptr, ptr, VALUE, len); + RBASIC(obj)->flags &= ~ROBJECT_EMBED; + ROBJECT(obj)->as.heap.ivptr = newptr; + } + else { + newptr = obj_ivar_heap_realloc(obj, len, newsize); + } - return index; + for (; len < newsize; len++) { + newptr[len] = Qundef; + } + ROBJECT(obj)->as.heap.numiv = newsize; + ROBJECT(obj)->as.heap.iv_index_tbl = index_tbl; } -/* Set the instance variable +val+ on object +obj+ at ivar name +id+. - * This function only works with T_OBJECT objects, so make sure - * +obj+ is of type T_OBJECT before using this function. - */ -VALUE -rb_vm_set_ivar_id(VALUE obj, ID id, VALUE val) +void +rb_init_iv_list(VALUE obj) { - rb_check_frozen_internal(obj); - rb_obj_ivar_set(obj, id, val); - return val; + st_table *index_tbl = ROBJECT_IV_INDEX_TBL(obj); + uint32_t newsize = (uint32_t)index_tbl->num_entries; + uint32_t len = ROBJECT_NUMIV(obj); + init_iv_list(obj, len, newsize, index_tbl); } -bool -rb_shape_set_shape_id(VALUE obj, shape_id_t shape_id) +// Retrieve or create the id-to-index mapping for a given object and an +// instance variable name. +static struct ivar_update +obj_ensure_iv_index_mapping(VALUE obj, ID id) { - if (rb_shape_get_shape_id(obj) == shape_id) { - return false; - } - -#if SHAPE_IN_BASIC_FLAGS - RBASIC_SET_SHAPE_ID(obj, shape_id); -#else - switch (BUILTIN_TYPE(obj)) { - case T_OBJECT: - ROBJECT_SET_SHAPE_ID(obj, shape_id); - break; - case T_CLASS: - case T_MODULE: - RCLASS_SET_SHAPE_ID(obj, shape_id); - break; - default: - if (shape_id != SPECIAL_CONST_SHAPE_ID) { - struct gen_ivtbl *ivtbl = 0; - RB_VM_LOCK_ENTER(); - { - st_table* global_iv_table = generic_ivtbl(obj, 0, false); + VALUE klass = rb_obj_class(obj); + struct ivar_update ivup; + ivup.iv_extended = 0; + ivup.u.iv_index_tbl = iv_index_tbl_make(obj, klass); - if (st_lookup(global_iv_table, obj, (st_data_t *)&ivtbl)) { - ivtbl->shape_id = shape_id; - } - else { - rb_bug("Expected shape_id entry in global iv table"); - } - } - RB_VM_LOCK_LEAVE(); - } + RB_VM_LOCK_ENTER(); + { + iv_index_tbl_extend(&ivup, id, klass); } -#endif + RB_VM_LOCK_LEAVE(); - return true; + return ivup; } -/** - * Prevents further modifications to the given object. ::rb_eFrozenError shall - * be raised if modification is attempted. - * - * @param[out] x Object in question. - */ -void rb_obj_freeze_inline(VALUE x) +// Return the instance variable index for a given name and T_OBJECT object. The +// mapping between name and index lives on `rb_obj_class(obj)` and is created +// if not already present. +// +// @note May raise when there are too many instance variables. +// @note YJIT uses this function at compile time to simplify the work needed to +// access the variable at runtime. +uint32_t +rb_obj_ensure_iv_index_mapping(VALUE obj, ID id) { - if (RB_FL_ABLE(x)) { - RB_OBJ_FREEZE_RAW(x); + RUBY_ASSERT(RB_TYPE_P(obj, T_OBJECT)); + // This uint32_t cast shouldn't lose information as it's checked in + // iv_index_tbl_extend(). The index is stored as an uint32_t in + // struct rb_iv_index_tbl_entry. + return (uint32_t)obj_ensure_iv_index_mapping(obj, id).index; +} - rb_shape_transition_shape_frozen(x); +static VALUE +obj_ivar_set(VALUE obj, ID id, VALUE val) +{ + uint32_t len; + struct ivar_update ivup = obj_ensure_iv_index_mapping(obj, id); - if (RBASIC_CLASS(x) && !(RBASIC(x)->flags & RUBY_FL_SINGLETON)) { - rb_freeze_singleton_class(x); - } + len = ROBJECT_NUMIV(obj); + if (len <= ivup.index) { + uint32_t newsize = iv_index_tbl_newsize(&ivup); + init_iv_list(obj, len, newsize, ivup.u.iv_index_tbl); } + RB_OBJ_WRITE(obj, &ROBJECT_IVPTR(obj)[ivup.index], val); + + return val; } static void @@ -1608,15 +1558,12 @@ ivar_set(VALUE obj, ID id, VALUE val) switch (BUILTIN_TYPE(obj)) { case T_OBJECT: - { - rb_obj_ivar_set(obj, id, val); - break; - } + obj_ivar_set(obj, id, val); + break; case T_CLASS: case T_MODULE: IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(id); rb_class_ivar_set(obj, id, val); - break; default: generic_ivar_set(obj, id, val); @@ -1644,175 +1591,171 @@ rb_ivar_set_internal(VALUE obj, ID id, VALUE val) VALUE rb_ivar_defined(VALUE obj, ID id) { - attr_index_t index; + VALUE val; + struct st_table *iv_index_tbl; + uint32_t index; if (SPECIAL_CONST_P(obj)) return Qfalse; - if (rb_shape_obj_too_complex(obj)) { - VALUE idx; - if (!rb_st_lookup(ROBJECT_IV_HASH(obj), id, &idx)) { - return Qfalse; + switch (BUILTIN_TYPE(obj)) { + case T_OBJECT: + iv_index_tbl = ROBJECT_IV_INDEX_TBL(obj); + if (iv_index_tbl_lookup(iv_index_tbl, id, &index) && + index < ROBJECT_NUMIV(obj) && + (val = ROBJECT_IVPTR(obj)[index]) != Qundef) { + return Qtrue; } - - return Qtrue; - } - else { - return RBOOL(rb_shape_get_iv_index(rb_shape_get_shape(obj), id, &index)); + break; + case T_CLASS: + case T_MODULE: + if (RCLASS_IV_TBL(obj) && lock_st_is_member(RCLASS_IV_TBL(obj), (st_data_t)id)) + return Qtrue; + break; + default: + if (FL_TEST(obj, FL_EXIVAR)) + return generic_ivar_defined(obj, id); + break; } + return Qfalse; } typedef int rb_ivar_foreach_callback_func(ID key, VALUE val, st_data_t arg); st_data_t rb_st_nth_key(st_table *tab, st_index_t index); -struct iv_itr_data { - VALUE obj; - struct gen_ivtbl * ivtbl; - st_data_t arg; - rb_ivar_foreach_callback_func *func; -}; +static ID +iv_index_tbl_nth_id(st_table *iv_index_tbl, uint32_t index) +{ + st_data_t key; + RB_VM_LOCK_ENTER(); + { + key = rb_st_nth_key(iv_index_tbl, index); + } + RB_VM_LOCK_LEAVE(); + return (ID)key; +} -/* - * Returns a flag to stop iterating depending on the result of +callback+. - */ -static bool -iterate_over_shapes_with_callback(rb_shape_t *shape, rb_ivar_foreach_callback_func *callback, struct iv_itr_data * itr_data) +static inline bool +ivar_each_i(st_table *iv_index_tbl, VALUE val, uint32_t i, rb_ivar_foreach_callback_func *func, st_data_t arg) { - switch ((enum shape_type)shape->type) { - case SHAPE_ROOT: - return false; - case SHAPE_IVAR: - ASSUME(callback); - if (iterate_over_shapes_with_callback(rb_shape_get_parent(shape), callback, itr_data)) - return true; - VALUE * iv_list; - switch (BUILTIN_TYPE(itr_data->obj)) { - case T_OBJECT: - RUBY_ASSERT(!rb_shape_obj_too_complex(itr_data->obj)); - iv_list = ROBJECT_IVPTR(itr_data->obj); - break; - case T_CLASS: - case T_MODULE: - iv_list = RCLASS_IVPTR(itr_data->obj); + if (val != Qundef) { + ID id = iv_index_tbl_nth_id(iv_index_tbl, i); + switch (func(id, val, arg)) { + case ST_CHECK: + case ST_CONTINUE: break; + case ST_STOP: + return true; default: - iv_list = itr_data->ivtbl->ivptr; - break; - } - VALUE val = iv_list[shape->next_iv_index - 1]; - if (!UNDEF_P(val)) { - switch (callback(shape->edge_name, val, itr_data->arg)) { - case ST_CHECK: - case ST_CONTINUE: - break; - case ST_STOP: - return true; - default: - rb_bug("unreachable"); - } + rb_bug("unreachable"); } - return false; - case SHAPE_INITIAL_CAPACITY: - case SHAPE_CAPACITY_CHANGE: - case SHAPE_FROZEN: - case SHAPE_T_OBJECT: - return iterate_over_shapes_with_callback(rb_shape_get_parent(shape), callback, itr_data); - case SHAPE_OBJ_TOO_COMPLEX: - default: - rb_bug("Unreachable"); } -} - -static int -each_hash_iv(st_data_t id, st_data_t val, st_data_t data) -{ - struct iv_itr_data * itr_data = (struct iv_itr_data *)data; - rb_ivar_foreach_callback_func *callback = itr_data->func; - return callback((ID)id, (VALUE)val, itr_data->arg); + return false; } static void obj_ivar_each(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg) { - rb_shape_t* shape = rb_shape_get_shape(obj); - struct iv_itr_data itr_data; - itr_data.obj = obj; - itr_data.arg = arg; - itr_data.func = func; - if (rb_shape_obj_too_complex(obj)) { - rb_st_foreach(ROBJECT_IV_HASH(obj), each_hash_iv, (st_data_t)&itr_data); - } - else { - iterate_over_shapes_with_callback(shape, func, &itr_data); + st_table *iv_index_tbl = ROBJECT_IV_INDEX_TBL(obj); + if (!iv_index_tbl) return; + uint32_t i=0; + + for (i=0; i < ROBJECT_NUMIV(obj); i++) { + VALUE val = ROBJECT_IVPTR(obj)[i]; + if (ivar_each_i(iv_index_tbl, val, i, func, arg)) { + return; + } } } static void gen_ivar_each(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg) { - rb_shape_t *shape = rb_shape_get_shape(obj); struct gen_ivtbl *ivtbl; - if (!rb_gen_ivtbl_get(obj, 0, &ivtbl)) return; + st_table *iv_index_tbl = RCLASS_IV_INDEX_TBL(rb_obj_class(obj)); + if (!iv_index_tbl) return; + if (!gen_ivtbl_get(obj, 0, &ivtbl)) return; - struct iv_itr_data itr_data; - itr_data.obj = obj; - itr_data.ivtbl = ivtbl; - itr_data.arg = arg; - iterate_over_shapes_with_callback(shape, func, &itr_data); + for (uint32_t i=0; i<ivtbl->numiv; i++) { + VALUE val = ivtbl->ivptr[i]; + if (ivar_each_i(iv_index_tbl, val, i, func, arg)) { + return; + } + } } -static void -class_ivar_each(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg) +struct givar_copy { + VALUE obj; + VALUE klass; + st_table *iv_index_tbl; + struct gen_ivtbl *ivtbl; +}; + +static int +gen_ivar_copy(ID id, VALUE val, st_data_t arg) { - RUBY_ASSERT(RB_TYPE_P(obj, T_CLASS) || RB_TYPE_P(obj, T_MODULE)); + struct givar_copy *c = (struct givar_copy *)arg; + struct ivar_update ivup; + + ivup.iv_extended = 0; + ivup.u.iv_index_tbl = c->iv_index_tbl; + + RB_VM_LOCK_ENTER(); + { + iv_index_tbl_extend(&ivup, id, c->klass); + } + RB_VM_LOCK_LEAVE(); + + if (ivup.index >= c->ivtbl->numiv) { + uint32_t newsize = iv_index_tbl_newsize(&ivup); + c->ivtbl = gen_ivtbl_resize(c->ivtbl, newsize); + } + c->ivtbl->ivptr[ivup.index] = val; - rb_shape_t* shape = rb_shape_get_shape(obj); - struct iv_itr_data itr_data; - itr_data.obj = obj; - itr_data.arg = arg; - iterate_over_shapes_with_callback(shape, func, &itr_data); + RB_OBJ_WRITTEN(c->obj, Qundef, val); + + return ST_CONTINUE; } void rb_copy_generic_ivar(VALUE clone, VALUE obj) { - struct gen_ivtbl *obj_ivtbl; - struct gen_ivtbl *new_ivtbl; + struct gen_ivtbl *ivtbl; rb_check_frozen(clone); if (!FL_TEST(obj, FL_EXIVAR)) { goto clear; } - - if (rb_gen_ivtbl_get(obj, 0, &obj_ivtbl)) { - if (gen_ivtbl_count(obj_ivtbl) == 0) - goto clear; - - new_ivtbl = gen_ivtbl_resize(0, obj_ivtbl->numiv); - FL_SET(clone, FL_EXIVAR); - - for (uint32_t i=0; i<obj_ivtbl->numiv; i++) { - new_ivtbl->ivptr[i] = obj_ivtbl->ivptr[i]; - RB_OBJ_WRITTEN(clone, Qundef, &new_ivtbl[i]); - } - - /* - * c.ivtbl may change in gen_ivar_copy due to realloc, - * no need to free - */ + if (gen_ivtbl_get(obj, 0, &ivtbl)) { + struct givar_copy c; + uint32_t i; + + if (gen_ivtbl_count(ivtbl) == 0) + goto clear; + + if (gen_ivtbl_get(clone, 0, &c.ivtbl)) { + for (i = 0; i < c.ivtbl->numiv; i++) + c.ivtbl->ivptr[i] = Qundef; + } + else { + c.ivtbl = gen_ivtbl_resize(0, ivtbl->numiv); + FL_SET(clone, FL_EXIVAR); + } + + VALUE klass = rb_obj_class(clone); + c.iv_index_tbl = iv_index_tbl_make(clone, klass); + c.obj = clone; + c.klass = klass; + gen_ivar_each(obj, gen_ivar_copy, (st_data_t)&c); + /* + * c.ivtbl may change in gen_ivar_copy due to realloc, + * no need to free + */ RB_VM_LOCK_ENTER(); { generic_ivtbl_no_ractor_check(clone); - st_insert(generic_ivtbl_no_ractor_check(obj), (st_data_t)clone, (st_data_t)new_ivtbl); + st_insert(generic_ivtbl_no_ractor_check(obj), (st_data_t)clone, (st_data_t)c.ivtbl); } RB_VM_LOCK_LEAVE(); - - rb_shape_t * obj_shape = rb_shape_get_shape(obj); - if (rb_shape_frozen_shape_p(obj_shape)) { - rb_shape_set_shape_id(clone, obj_shape->parent_id); - } - else { - rb_shape_set_shape(clone, obj_shape); - } } return; @@ -1851,75 +1794,61 @@ rb_ivar_foreach(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg) switch (BUILTIN_TYPE(obj)) { case T_OBJECT: obj_ivar_each(obj, func, arg); - break; + break; case T_CLASS: case T_MODULE: IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(0); - RB_VM_LOCK_ENTER(); - { - class_ivar_each(obj, func, arg); - } - RB_VM_LOCK_LEAVE(); - break; + if (RCLASS_IV_TBL(obj)) { + RB_VM_LOCK_ENTER(); + { + st_foreach_safe(RCLASS_IV_TBL(obj), func, arg); + } + RB_VM_LOCK_LEAVE(); + } + break; default: - if (FL_TEST(obj, FL_EXIVAR)) { - gen_ivar_each(obj, func, arg); - } - break; + if (FL_TEST(obj, FL_EXIVAR)) { + gen_ivar_each(obj, func, arg); + } + break; } } st_index_t rb_ivar_count(VALUE obj) { + st_table *tbl; + if (SPECIAL_CONST_P(obj)) return 0; switch (BUILTIN_TYPE(obj)) { case T_OBJECT: - if (rb_shape_obj_too_complex(obj)) { - return ROBJECT_IV_COUNT(obj); - } - - if (rb_shape_get_shape(obj)->next_iv_index > 0) { - st_index_t i, count, num = ROBJECT_IV_COUNT(obj); - const VALUE *const ivptr = ROBJECT_IVPTR(obj); - for (i = count = 0; i < num; ++i) { - if (!UNDEF_P(ivptr[i])) { - count++; - } - } - return count; - } - break; + if (ROBJECT_IV_INDEX_TBL(obj) != 0) { + st_index_t i, count, num = ROBJECT_NUMIV(obj); + const VALUE *const ivptr = ROBJECT_IVPTR(obj); + for (i = count = 0; i < num; ++i) { + if (ivptr[i] != Qundef) { + count++; + } + } + return count; + } + break; case T_CLASS: case T_MODULE: - if (rb_shape_get_shape(obj)->next_iv_index > 0) { - st_index_t count = 0; - - RB_VM_LOCK_ENTER(); - { - st_index_t i, num = rb_shape_get_shape(obj)->next_iv_index; - const VALUE *const ivptr = RCLASS_IVPTR(obj); - for (i = count = 0; i < num; ++i) { - if (!UNDEF_P(ivptr[i])) { - count++; - } - } - } - RB_VM_LOCK_LEAVE(); - - return count; - } - break; + if ((tbl = RCLASS_IV_TBL(obj)) != 0) { + return tbl->num_entries; + } + break; default: - if (FL_TEST(obj, FL_EXIVAR)) { - struct gen_ivtbl *ivtbl; + if (FL_TEST(obj, FL_EXIVAR)) { + struct gen_ivtbl *ivtbl; - if (rb_gen_ivtbl_get(obj, 0, &ivtbl)) { - return gen_ivtbl_count(ivtbl); - } - } - break; + if (gen_ivtbl_get(obj, 0, &ivtbl)) { + return gen_ivtbl_count(ivtbl); + } + } + break; } return 0; } @@ -1931,7 +1860,7 @@ ivar_i(st_data_t k, st_data_t v, st_data_t a) VALUE ary = (VALUE)a; if (rb_is_instance_id(key)) { - rb_ary_push(ary, ID2SYM(key)); + rb_ary_push(ary, ID2SYM(key)); } return ST_CONTINUE; } @@ -1971,15 +1900,15 @@ rb_obj_instance_variables(VALUE obj) check_id_type(obj, &(name), rb_is_##type##_id, rb_is_##type##_name, message, strlen(message)) static ID check_id_type(VALUE obj, VALUE *pname, - int (*valid_id_p)(ID), int (*valid_name_p)(VALUE), - const char *message, size_t message_len) + int (*valid_id_p)(ID), int (*valid_name_p)(VALUE), + const char *message, size_t message_len) { ID id = rb_check_id(pname); VALUE name = *pname; if (id ? !valid_id_p(id) : !valid_name_p(name)) { - rb_name_err_raise_str(rb_fstring_new(message, message_len), - obj, name); + rb_name_err_raise_str(rb_fstring_new(message, message_len), + obj, name); } return id; } @@ -2011,49 +1940,47 @@ check_id_type(VALUE obj, VALUE *pname, VALUE rb_obj_remove_instance_variable(VALUE obj, VALUE name) { - VALUE val = Qundef; + VALUE val = Qnil; const ID id = id_for_var(obj, name, an, instance); + st_data_t n, v; + struct st_table *iv_index_tbl; + uint32_t index; - // Frozen check comes here because it's expected that we raise a - // NameError (from the id_for_var check) before we raise a FrozenError rb_check_frozen(obj); - if (!id) { - goto not_defined; + goto not_defined; } - rb_shape_t * shape = rb_shape_get_shape(obj); - switch (BUILTIN_TYPE(obj)) { + case T_OBJECT: + iv_index_tbl = ROBJECT_IV_INDEX_TBL(obj); + if (iv_index_tbl_lookup(iv_index_tbl, id, &index) && + index < ROBJECT_NUMIV(obj) && + (val = ROBJECT_IVPTR(obj)[index]) != Qundef) { + ROBJECT_IVPTR(obj)[index] = Qundef; + return val; + } + break; case T_CLASS: case T_MODULE: IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(id); - rb_shape_transition_shape_remove_ivar(obj, id, shape, &val); - break; - case T_OBJECT: { - if (rb_shape_obj_too_complex(obj)) { - if (rb_st_lookup(ROBJECT_IV_HASH(obj), (st_data_t)id, (st_data_t *)&val)) { - rb_st_delete(ROBJECT_IV_HASH(obj), (st_data_t *)&id, 0); - } - } - else { - rb_shape_transition_shape_remove_ivar(obj, id, shape, &val); - } - break; - } - default: { - rb_shape_transition_shape_remove_ivar(obj, id, shape, &val); - break; - } - } - - if (val != Qundef) { - return val; + n = id; + if (RCLASS_IV_TBL(obj) && lock_st_delete(RCLASS_IV_TBL(obj), &n, &v)) { + return (VALUE)v; + } + break; + default: + if (FL_TEST(obj, FL_EXIVAR)) { + if (generic_ivar_remove(obj, id, &val)) { + return val; + } + } + break; } not_defined: rb_name_err_raise("instance variable %1$s not defined", - obj, name); + obj, name); UNREACHABLE_RETURN(Qnil); } @@ -2062,11 +1989,11 @@ static void uninitialized_constant(VALUE klass, VALUE name) { if (klass && rb_class_real(klass) != rb_cObject) - rb_name_err_raise("uninitialized constant %2$s::%1$s", - klass, name); + rb_name_err_raise("uninitialized constant %2$s::%1$s", + klass, name); else - rb_name_err_raise("uninitialized constant %1$s", - klass, name); + rb_name_err_raise("uninitialized constant %1$s", + klass, name); } VALUE @@ -2117,12 +2044,11 @@ rb_const_missing(VALUE klass, VALUE name) VALUE rb_mod_const_missing(VALUE klass, VALUE name) { - rb_execution_context_t *ec = GET_EC(); - VALUE ref = ec->private_const_reference; + VALUE ref = GET_EC()->private_const_reference; rb_vm_pop_cfunc_frame(); if (ref) { - ec->private_const_reference = 0; - rb_name_err_raise("private constant %2$s::%1$s referenced", ref, name); + rb_name_err_raise("private constant %2$s::%1$s referenced", + ref, name); } uninitialized_constant(klass, name); @@ -2130,38 +2056,38 @@ rb_mod_const_missing(VALUE klass, VALUE name) } static void -autoload_table_mark(void *ptr) +autoload_mark(void *ptr) { rb_mark_tbl_no_pin((st_table *)ptr); } static void -autoload_table_free(void *ptr) +autoload_free(void *ptr) { st_free_table((st_table *)ptr); } static size_t -autoload_table_memsize(const void *ptr) +autoload_memsize(const void *ptr) { const st_table *tbl = ptr; return st_memsize(tbl); } static void -autoload_table_compact(void *ptr) +autoload_compact(void *ptr) { rb_gc_update_tbl_refs((st_table *)ptr); } -static const rb_data_type_t autoload_table_type = { - "autoload_table", - {autoload_table_mark, autoload_table_free, autoload_table_memsize, autoload_table_compact,}, - 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED +static const rb_data_type_t autoload_data_type = { + "autoload", + {autoload_mark, autoload_free, autoload_memsize, autoload_compact,}, + 0, 0, RUBY_TYPED_FREE_IMMEDIATELY }; #define check_autoload_table(av) \ - (struct st_table *)rb_check_typeddata((av), &autoload_table_type) + (struct st_table *)rb_check_typeddata((av), &autoload_data_type) static VALUE autoload_data(VALUE mod, ID id) @@ -2169,359 +2095,270 @@ autoload_data(VALUE mod, ID id) struct st_table *tbl; st_data_t val; - // If we are called with a non-origin ICLASS, fetch the autoload data from - // the original module. - if (RB_TYPE_P(mod, T_ICLASS)) { - if (FL_TEST_RAW(mod, RICLASS_IS_ORIGIN)) { - return 0; - } - else { - mod = RBASIC(mod)->klass; - } - } - - RUBY_ASSERT(RB_TYPE_P(mod, T_CLASS) || RB_TYPE_P(mod, T_MODULE)); - - // Look up the instance variable table for `autoload`, then index into that table with the given constant name `id`. - - VALUE tbl_value = rb_ivar_lookup(mod, autoload, Qfalse); - if (!RTEST(tbl_value) || !(tbl = check_autoload_table(tbl_value)) || !st_lookup(tbl, (st_data_t)id, &val)) { - return 0; + if (!st_lookup(RCLASS_IV_TBL(mod), autoload, &val) || + !(tbl = check_autoload_table((VALUE)val)) || + !st_lookup(tbl, (st_data_t)id, &val)) { + return 0; } - return (VALUE)val; } -// Every autoload constant has exactly one instance of autoload_const, stored in `autoload_features`. Since multiple autoload constants can refer to the same file, every `autoload_const` refers to a de-duplicated `autoload_data`. struct autoload_const { - // The linked list node of all constants which are loaded by the related autoload feature. - struct ccan_list_node cnode; /* <=> autoload_data.constants */ - - // The shared "autoload_data" if multiple constants are defined from the same feature. - VALUE autoload_data_value; - - // The module we are loading a constant into. - VALUE module; - - // The name of the constant we are loading. - ID name; - - // The value of the constant (after it's loaded). + struct list_node cnode; /* <=> autoload_data_i.constants */ + VALUE mod; + VALUE ad; /* autoload_data_i */ VALUE value; - - // The constant entry flags which need to be re-applied after autoloading the feature. - rb_const_flag_t flag; - - // The source file and line number that defined this constant (different from feature path). VALUE file; + ID id; + rb_const_flag_t flag; int line; }; -// Each `autoload_data` uniquely represents a specific feature which can be loaded, and a list of constants which it is able to define. We use a mutex to coordinate multiple threads trying to load the same feature. -struct autoload_data { - // The feature path to require to load this constant. - VALUE feature; - - // The mutex which is protecting autoloading this feature. - VALUE mutex; +/* always on stack, no need to mark */ +struct autoload_state { + struct autoload_const *ac; + VALUE result; + VALUE thread; + struct list_head waitq; +}; - // The process fork serial number since the autoload mutex will become invalid on fork. +struct autoload_data_i { + VALUE feature; + struct autoload_state *state; /* points to on-stack struct */ rb_serial_t fork_gen; - - // The linked list of all constants that are going to be loaded by this autoload. - struct ccan_list_head constants; /* <=> autoload_const.cnode */ + struct list_head constants; /* <=> autoload_const.cnode */ }; static void -autoload_data_compact(void *ptr) +autoload_i_compact(void *ptr) { - struct autoload_data *p = ptr; - + struct autoload_data_i *p = ptr; p->feature = rb_gc_location(p->feature); - p->mutex = rb_gc_location(p->mutex); } static void -autoload_data_mark(void *ptr) +autoload_i_mark(void *ptr) { - struct autoload_data *p = ptr; + struct autoload_data_i *p = ptr; rb_gc_mark_movable(p->feature); - rb_gc_mark_movable(p->mutex); + + /* allow GC to free us if no modules refer to this via autoload_const.ad */ + if (list_empty(&p->constants)) { + rb_hash_delete(autoload_featuremap, p->feature); + } } static void -autoload_data_free(void *ptr) +autoload_i_free(void *ptr) { - struct autoload_data *p = ptr; + struct autoload_data_i *p = ptr; - // We may leak some memory at VM shutdown time, no big deal...? - if (ccan_list_empty(&p->constants)) { - ruby_xfree(p); + /* we may leak some memory at VM shutdown time, no big deal */ + if (list_empty(&p->constants)) { + xfree(p); } } static size_t -autoload_data_memsize(const void *ptr) +autoload_i_memsize(const void *ptr) { - return sizeof(struct autoload_data); + return sizeof(struct autoload_data_i); } -static const rb_data_type_t autoload_data_type = { - "autoload_data", - {autoload_data_mark, autoload_data_free, autoload_data_memsize, autoload_data_compact}, - 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED +static const rb_data_type_t autoload_data_i_type = { + "autoload_i", + {autoload_i_mark, autoload_i_free, autoload_i_memsize, autoload_i_compact}, + 0, 0, RUBY_TYPED_FREE_IMMEDIATELY }; static void -autoload_const_compact(void *ptr) +autoload_c_compact(void *ptr) { struct autoload_const *ac = ptr; - ac->module = rb_gc_location(ac->module); - ac->autoload_data_value = rb_gc_location(ac->autoload_data_value); + ac->mod = rb_gc_location(ac->mod); + ac->ad = rb_gc_location(ac->ad); ac->value = rb_gc_location(ac->value); ac->file = rb_gc_location(ac->file); } static void -autoload_const_mark(void *ptr) +autoload_c_mark(void *ptr) { struct autoload_const *ac = ptr; - rb_gc_mark_movable(ac->module); - rb_gc_mark_movable(ac->autoload_data_value); + rb_gc_mark_movable(ac->mod); + rb_gc_mark_movable(ac->ad); rb_gc_mark_movable(ac->value); rb_gc_mark_movable(ac->file); } -static size_t -autoload_const_memsize(const void *ptr) +static void +autoload_c_free(void *ptr) { - return sizeof(struct autoload_const); + struct autoload_const *ac = ptr; + list_del(&ac->cnode); + xfree(ac); } -static void -autoload_const_free(void *ptr) +static size_t +autoload_c_memsize(const void *ptr) { - struct autoload_const *autoload_const = ptr; - - ccan_list_del(&autoload_const->cnode); - ruby_xfree(ptr); + return sizeof(struct autoload_const); } static const rb_data_type_t autoload_const_type = { "autoload_const", - {autoload_const_mark, autoload_const_free, autoload_const_memsize, autoload_const_compact,}, + {autoload_c_mark, autoload_c_free, autoload_c_memsize, autoload_c_compact,}, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY }; -static struct autoload_data * -get_autoload_data(VALUE autoload_const_value, struct autoload_const **autoload_const_pointer) +static struct autoload_data_i * +get_autoload_data(VALUE acv, struct autoload_const **acp) { - struct autoload_const *autoload_const = rb_check_typeddata(autoload_const_value, &autoload_const_type); - - VALUE autoload_data_value = autoload_const->autoload_data_value; - struct autoload_data *autoload_data = rb_check_typeddata(autoload_data_value, &autoload_data_type); + struct autoload_const *ac = rb_check_typeddata(acv, &autoload_const_type); + struct autoload_data_i *ele; + ele = rb_check_typeddata(ac->ad, &autoload_data_i_type); /* do not reach across stack for ->state after forking: */ - if (autoload_data && autoload_data->fork_gen != GET_VM()->fork_gen) { - RB_OBJ_WRITE(autoload_data_value, &autoload_data->mutex, Qnil); - autoload_data->fork_gen = 0; + if (ele && ele->state && ele->fork_gen != GET_VM()->fork_gen) { + ele->state = 0; + ele->fork_gen = 0; } - - if (autoload_const_pointer) *autoload_const_pointer = autoload_const; - - return autoload_data; + if (acp) *acp = ac; + return ele; } RUBY_FUNC_EXPORTED void -rb_autoload(VALUE module, ID name, const char *feature) +rb_autoload(VALUE mod, ID id, const char *file) { - if (!feature || !*feature) { - rb_raise(rb_eArgError, "empty feature name"); + if (!file || !*file) { + rb_raise(rb_eArgError, "empty file name"); } - - rb_autoload_str(module, name, rb_fstring_cstr(feature)); + rb_autoload_str(mod, id, rb_fstring_cstr(file)); } -static void const_set(VALUE klass, ID id, VALUE val); -static void const_added(VALUE klass, ID const_name); - -struct autoload_arguments { - VALUE module; - ID name; - VALUE feature; -}; - -static VALUE -autoload_feature_lookup_or_create(VALUE feature, struct autoload_data **autoload_data_pointer) +void +rb_autoload_str(VALUE mod, ID id, VALUE file) { - RUBY_ASSERT_MUTEX_OWNED(autoload_mutex); - RUBY_ASSERT_CRITICAL_SECTION_ENTER(); - - VALUE autoload_data_value = rb_hash_aref(autoload_features, feature); - struct autoload_data *autoload_data; - - if (NIL_P(autoload_data_value)) { - autoload_data_value = TypedData_Make_Struct(0, struct autoload_data, &autoload_data_type, autoload_data); - RB_OBJ_WRITE(autoload_data_value, &autoload_data->feature, feature); - RB_OBJ_WRITE(autoload_data_value, &autoload_data->mutex, Qnil); - ccan_list_head_init(&autoload_data->constants); - - if (autoload_data_pointer) *autoload_data_pointer = autoload_data; + st_data_t av; + VALUE ad; + struct st_table *tbl; + struct autoload_data_i *ele; + rb_const_entry_t *ce; - rb_hash_aset(autoload_features, feature, autoload_data_value); - } - else if (autoload_data_pointer) { - *autoload_data_pointer = rb_check_typeddata(autoload_data_value, &autoload_data_type); + if (!rb_is_const_id(id)) { + rb_raise(rb_eNameError, "autoload must be constant name: %"PRIsVALUE"", + QUOTE_ID(id)); } - RUBY_ASSERT_CRITICAL_SECTION_LEAVE(); - return autoload_data_value; -} - -static VALUE -autoload_table_lookup_or_create(VALUE module) -{ - VALUE autoload_table_value = rb_ivar_lookup(module, autoload, Qfalse); - if (RTEST(autoload_table_value)) { - return autoload_table_value; + Check_Type(file, T_STRING); + if (!RSTRING_LEN(file)) { + rb_raise(rb_eArgError, "empty file name"); } - else { - autoload_table_value = TypedData_Wrap_Struct(0, &autoload_table_type, NULL); - rb_class_ivar_set(module, autoload, autoload_table_value); - RTYPEDDATA_DATA(autoload_table_value) = st_init_numtable(); - return autoload_table_value; - } -} -static VALUE -autoload_synchronized(VALUE _arguments) -{ - struct autoload_arguments *arguments = (struct autoload_arguments *)_arguments; - - rb_const_entry_t *constant_entry = rb_const_lookup(arguments->module, arguments->name); - if (constant_entry && !UNDEF_P(constant_entry->value)) { - return Qfalse; + ce = rb_const_lookup(mod, id); + if (ce && ce->value != Qundef) { + return; } - // Reset any state associated with any previous constant: - const_set(arguments->module, arguments->name, Qundef); - - VALUE autoload_table_value = autoload_table_lookup_or_create(arguments->module); - struct st_table *autoload_table = check_autoload_table(autoload_table_value); - - // Ensure the string is uniqued since we use an identity lookup: - VALUE feature = rb_fstring(arguments->feature); - - struct autoload_data *autoload_data; - VALUE autoload_data_value = autoload_feature_lookup_or_create(feature, &autoload_data); - - { - struct autoload_const *autoload_const; - VALUE autoload_const_value = TypedData_Make_Struct(0, struct autoload_const, &autoload_const_type, autoload_const); - autoload_const->module = arguments->module; - autoload_const->name = arguments->name; - autoload_const->value = Qundef; - autoload_const->flag = CONST_PUBLIC; - autoload_const->autoload_data_value = autoload_data_value; - ccan_list_add_tail(&autoload_data->constants, &autoload_const->cnode); - st_insert(autoload_table, (st_data_t)arguments->name, (st_data_t)autoload_const_value); - RB_OBJ_WRITTEN(autoload_table_value, Qundef, autoload_const_value); + rb_const_set(mod, id, Qundef); + tbl = RCLASS_IV_TBL(mod); + if (tbl && st_lookup(tbl, (st_data_t)autoload, &av)) { + tbl = check_autoload_table((VALUE)av); } - - return Qtrue; -} - -void -rb_autoload_str(VALUE module, ID name, VALUE feature) -{ - if (!rb_is_const_id(name)) { - rb_raise(rb_eNameError, "autoload must be constant name: %"PRIsVALUE"", QUOTE_ID(name)); + else { + if (!tbl) tbl = RCLASS_IV_TBL(mod) = st_init_numtable(); + av = (st_data_t)TypedData_Wrap_Struct(0, &autoload_data_type, 0); + st_add_direct(tbl, (st_data_t)autoload, av); + RB_OBJ_WRITTEN(mod, Qnil, av); + DATA_PTR(av) = tbl = st_init_numtable(); + } + + file = rb_fstring(file); + if (!autoload_featuremap) { + autoload_featuremap = rb_ident_hash_new(); + rb_obj_hide(autoload_featuremap); + rb_gc_register_mark_object(autoload_featuremap); + } + ad = rb_hash_aref(autoload_featuremap, file); + if (NIL_P(ad)) { + ad = TypedData_Make_Struct(0, struct autoload_data_i, + &autoload_data_i_type, ele); + ele->feature = file; + ele->state = 0; + list_head_init(&ele->constants); + rb_hash_aset(autoload_featuremap, file, ad); } - - Check_Type(feature, T_STRING); - if (!RSTRING_LEN(feature)) { - rb_raise(rb_eArgError, "empty feature name"); + else { + ele = rb_check_typeddata(ad, &autoload_data_i_type); } - - struct autoload_arguments arguments = { - .module = module, - .name = name, - .feature = feature, - }; - - VALUE result = rb_mutex_synchronize(autoload_mutex, autoload_synchronized, (VALUE)&arguments); - - if (result == Qtrue) { - const_added(module, name); + { + VALUE acv; + struct autoload_const *ac; + acv = TypedData_Make_Struct(0, struct autoload_const, + &autoload_const_type, ac); + ac->mod = mod; + ac->id = id; + ac->value = Qundef; + ac->flag = CONST_PUBLIC; + ac->ad = ad; + list_add_tail(&ele->constants, &ac->cnode); + st_insert(tbl, (st_data_t)id, (st_data_t)acv); } } static void -autoload_delete(VALUE module, ID name) +autoload_delete(VALUE mod, ID id) { - RUBY_ASSERT_CRITICAL_SECTION_ENTER(); - - st_data_t load = 0, key = name; + st_data_t val, load = 0, n = id; - RUBY_ASSERT(RB_TYPE_P(module, T_CLASS) || RB_TYPE_P(module, T_MODULE)); - - VALUE table_value = rb_ivar_lookup(module, autoload, Qfalse); - if (RTEST(table_value)) { - struct st_table *table = check_autoload_table(table_value); - - st_delete(table, &key, &load); - RB_OBJ_WRITTEN(table_value, load, Qundef); + if (st_lookup(RCLASS_IV_TBL(mod), (st_data_t)autoload, &val)) { + struct st_table *tbl = check_autoload_table((VALUE)val); + struct autoload_data_i *ele; + struct autoload_const *ac; + st_delete(tbl, &n, &load); /* Qfalse can indicate already deleted */ if (load != Qfalse) { - struct autoload_const *autoload_const; - struct autoload_data *autoload_data = get_autoload_data((VALUE)load, &autoload_const); - - VM_ASSERT(autoload_data); - VM_ASSERT(!ccan_list_empty(&autoload_data->constants)); + ele = get_autoload_data((VALUE)load, &ac); + VM_ASSERT(ele); + if (ele) { + VM_ASSERT(!list_empty(&ele->constants)); + } /* * we must delete here to avoid "already initialized" warnings * with parallel autoload. Using list_del_init here so list_del - * works in autoload_const_free + * works in autoload_c_free */ - ccan_list_del_init(&autoload_const->cnode); - - if (ccan_list_empty(&autoload_data->constants)) { - rb_hash_delete(autoload_features, autoload_data->feature); - } + list_del_init(&ac->cnode); - // If the autoload table is empty, we can delete it. - if (table->num_entries == 0) { - rb_attr_delete(module, autoload); + if (tbl->num_entries == 0) { + n = autoload; + st_delete(RCLASS_IV_TBL(mod), &n, &val); } } } - - RUBY_ASSERT_CRITICAL_SECTION_LEAVE(); -} - -static int -autoload_by_someone_else(struct autoload_data *ele) -{ - return ele->mutex != Qnil && !rb_mutex_owned_p(ele->mutex); } static VALUE check_autoload_required(VALUE mod, ID id, const char **loadingpath) { - VALUE autoload_const_value = autoload_data(mod, id); - struct autoload_data *autoload_data; + VALUE file; + VALUE load = autoload_data(mod, id); + struct autoload_data_i *ele; const char *loading; - if (!autoload_const_value || !(autoload_data = get_autoload_data(autoload_const_value, 0))) { - return 0; + if (!load || !(ele = get_autoload_data(load, 0))) { + return 0; + } + file = ele->feature; + Check_Type(file, T_STRING); + if (!RSTRING_LEN(file) || !*RSTRING_PTR(file)) { + rb_raise(rb_eArgError, "empty file name"); } - - VALUE feature = autoload_data->feature; /* * if somebody else is autoloading, we MUST wait for them, since @@ -2529,27 +2366,24 @@ check_autoload_required(VALUE mod, ID id, const char **loadingpath) * completes. We must wait until autoload_const_set finishes in * the other thread. */ - if (autoload_by_someone_else(autoload_data)) { - return autoload_const_value; + if (ele->state && ele->state->thread != rb_thread_current()) { + return load; } - loading = RSTRING_PTR(feature); - + loading = RSTRING_PTR(file); if (!rb_feature_provided(loading, &loading)) { - return autoload_const_value; + return load; } - if (loadingpath && loading) { - *loadingpath = loading; - return autoload_const_value; + *loadingpath = loading; + return load; } - return 0; } static struct autoload_const *autoloading_const_entry(VALUE mod, ID id); -int +MJIT_FUNC_EXPORTED int rb_autoloading_value(VALUE mod, ID id, VALUE* value, rb_const_flag_t *flag) { struct autoload_const *ac = autoloading_const_entry(mod, id); @@ -2558,43 +2392,28 @@ rb_autoloading_value(VALUE mod, ID id, VALUE* value, rb_const_flag_t *flag) if (value) { *value = ac->value; } - if (flag) { *flag = ac->flag; } - return TRUE; } -static int -autoload_by_current(struct autoload_data *ele) -{ - return ele->mutex != Qnil && rb_mutex_owned_p(ele->mutex); -} - -// If there is an autoloading constant and it has been set by the current -// execution context, return it. This allows threads which are loading code to -// refer to their own autoloaded constants. struct autoload_const * autoloading_const_entry(VALUE mod, ID id) { VALUE load = autoload_data(mod, id); - struct autoload_data *ele; + struct autoload_data_i *ele; struct autoload_const *ac; - // Find the autoloading state: if (!load || !(ele = get_autoload_data(load, &ac))) { - // Couldn't be found: return 0; } - // Check if it's being loaded by the current thread/fiber: - if (autoload_by_current(ele)) { - if (!UNDEF_P(ac->value)) { + if (ele->state && ele->state->thread == rb_thread_current()) { + if (ac->value != Qundef) { return ac; - } + } } - return 0; } @@ -2603,206 +2422,183 @@ autoload_defined_p(VALUE mod, ID id) { rb_const_entry_t *ce = rb_const_lookup(mod, id); - // If there is no constant or the constant is not undefined (special marker for autoloading): - if (!ce || !UNDEF_P(ce->value)) { - // We are not autoloading: - return 0; + if (!ce || ce->value != Qundef) { + return 0; } - - // Otherwise check if there is an autoload in flight right now: return !rb_autoloading_value(mod, id, NULL, NULL); } -static void const_tbl_update(struct autoload_const *, int); - -struct autoload_load_arguments { - VALUE module; - ID name; - int flag; - - VALUE mutex; - - // The specific constant which triggered the autoload code to fire: - struct autoload_const *autoload_const; - - // The parent autoload data which is shared between multiple constants: - struct autoload_data *autoload_data; -}; +static void const_tbl_update(struct autoload_const *); static VALUE autoload_const_set(struct autoload_const *ac) { - check_before_mod_set(ac->module, ac->name, ac->value, "constant"); + VALUE klass = ac->mod; + ID id = ac->id; + check_before_mod_set(klass, id, ac->value, "constant"); RB_VM_LOCK_ENTER(); { - const_tbl_update(ac, true); + const_tbl_update(ac); } RB_VM_LOCK_LEAVE(); - return 0; /* ignored */ + return 0; /* ignored */ } static VALUE -autoload_load_needed(VALUE _arguments) +autoload_require(VALUE arg) { - struct autoload_load_arguments *arguments = (struct autoload_load_arguments*)_arguments; + struct autoload_state *state = (struct autoload_state *)arg; + struct autoload_const *ac = state->ac; + struct autoload_data_i *ele; - const char *loading = 0, *src; - - if (!autoload_defined_p(arguments->module, arguments->name)) { - return Qfalse; - } - - VALUE autoload_const_value = check_autoload_required(arguments->module, arguments->name, &loading); - if (!autoload_const_value) { - return Qfalse; - } - - src = rb_sourcefile(); - if (src && loading && strcmp(src, loading) == 0) { - return Qfalse; - } - - struct autoload_const *autoload_const; - struct autoload_data *autoload_data; - if (!(autoload_data = get_autoload_data(autoload_const_value, &autoload_const))) { - return Qfalse; - } - - if (NIL_P(autoload_data->mutex)) { - RB_OBJ_WRITE(autoload_const->autoload_data_value, &autoload_data->mutex, rb_mutex_new()); - autoload_data->fork_gen = GET_VM()->fork_gen; - } - else if (rb_mutex_owned_p(autoload_data->mutex)) { - return Qfalse; - } + ele = rb_check_typeddata(ac->ad, &autoload_data_i_type); + /* this may release GVL and switch threads: */ + state->result = rb_funcall(rb_vm_top_self(), rb_intern("require"), 1, + ele->feature); - arguments->mutex = autoload_data->mutex; - arguments->autoload_const = autoload_const; - - return autoload_const_value; + return state->result; } static VALUE -autoload_apply_constants(VALUE _arguments) +autoload_reset(VALUE arg) { - RUBY_ASSERT_CRITICAL_SECTION_ENTER(); - - struct autoload_load_arguments *arguments = (struct autoload_load_arguments*)_arguments; + struct autoload_state *state = (struct autoload_state *)arg; + int need_wakeups = 0; + struct autoload_const *ac = state->ac; + struct autoload_data_i *ele; - struct autoload_const *autoload_const = 0; // for ccan_container_off_var() - struct autoload_const *next; + ele = rb_check_typeddata(ac->ad, &autoload_data_i_type); + if (ele->state == state) { + need_wakeups = 1; + ele->state = 0; + ele->fork_gen = 0; + } - // We use safe iteration here because `autoload_const_set` will eventually invoke - // `autoload_delete` which will remove the constant from the linked list. In theory, once - // the `autoload_data->constants` linked list is empty, we can remove it. + /* At the last, move a value defined in autoload to constant table */ + if (RTEST(state->result)) { + struct autoload_const *next; - // Iterate over all constants and assign them: - ccan_list_for_each_safe(&arguments->autoload_data->constants, autoload_const, next, cnode) { - if (!UNDEF_P(autoload_const->value)) { - autoload_const_set(autoload_const); + list_for_each_safe(&ele->constants, ac, next, cnode) { + if (ac->value != Qundef) { + autoload_const_set(ac); + } } } - RUBY_ASSERT_CRITICAL_SECTION_LEAVE(); - - return Qtrue; -} - -static VALUE -autoload_feature_require(VALUE _arguments) -{ - struct autoload_load_arguments *arguments = (struct autoload_load_arguments*)_arguments; - - struct autoload_const *autoload_const = arguments->autoload_const; + /* wakeup any waiters we had */ + if (need_wakeups) { + struct autoload_state *cur = 0, *nxt; - // We save this for later use in autoload_apply_constants: - arguments->autoload_data = rb_check_typeddata(autoload_const->autoload_data_value, &autoload_data_type); + list_for_each_safe(&state->waitq, cur, nxt, waitq.n) { + VALUE th = cur->thread; - VALUE result = rb_funcall(rb_vm_top_self(), rb_intern("require"), 1, arguments->autoload_data->feature); + cur->thread = Qfalse; + list_del_init(&cur->waitq.n); /* idempotent */ - if (RTEST(result)) { - return rb_mutex_synchronize(autoload_mutex, autoload_apply_constants, _arguments); + /* + * cur is stored on the stack of cur->waiting_th, + * do not touch cur after waking up waiting_th + */ + rb_thread_wakeup_alive(th); + } } - return result; + return 0; /* ignored */ } static VALUE -autoload_try_load(VALUE _arguments) +autoload_sleep(VALUE arg) { - struct autoload_load_arguments *arguments = (struct autoload_load_arguments*)_arguments; - - VALUE result = autoload_feature_require(_arguments); + struct autoload_state *state = (struct autoload_state *)arg; - // After we loaded the feature, if the constant is not defined, we remove it completely: - rb_const_entry_t *ce = rb_const_lookup(arguments->module, arguments->name); + /* + * autoload_reset in other thread will resume us and remove us + * from the waitq list + */ + do { + rb_thread_sleep_deadly(); + } while (state->thread != Qfalse); - if (!ce || UNDEF_P(ce->value)) { - result = Qfalse; + return Qfalse; +} - rb_const_remove(arguments->module, arguments->name); +static VALUE +autoload_sleep_done(VALUE arg) +{ + struct autoload_state *state = (struct autoload_state *)arg; - if (arguments->module == rb_cObject) { - rb_warning( - "Expected %"PRIsVALUE" to define %"PRIsVALUE" but it didn't", - arguments->autoload_data->feature, - ID2SYM(arguments->name) - ); - } - else { - rb_warning( - "Expected %"PRIsVALUE" to define %"PRIsVALUE"::%"PRIsVALUE" but it didn't", - arguments->autoload_data->feature, - arguments->module, - ID2SYM(arguments->name) - ); - } - } - else { - // Otherwise, it was loaded, copy the flags from the autoload constant: - ce->flag |= arguments->flag; + if (state->thread != Qfalse && rb_thread_to_be_killed(state->thread)) { + list_del(&state->waitq.n); /* idempotent after list_del_init */ } - return result; + return Qfalse; } VALUE -rb_autoload_load(VALUE module, ID name) +rb_autoload_load(VALUE mod, ID id) { - rb_const_entry_t *ce = rb_const_lookup(module, name); + VALUE load, result; + const char *loading = 0, *src; + struct autoload_data_i *ele; + struct autoload_const *ac; + struct autoload_state state; + int flag = -1; + rb_const_entry_t *ce; - // We bail out as early as possible without any synchronisation: - if (!ce || !UNDEF_P(ce->value)) { - return Qfalse; - } + if (!autoload_defined_p(mod, id)) return Qfalse; + load = check_autoload_required(mod, id, &loading); + if (!load) return Qfalse; + src = rb_sourcefile(); + if (src && loading && strcmp(src, loading) == 0) return Qfalse; - // At this point, we assume there might be autoloading, so fail if it's ractor: if (UNLIKELY(!rb_ractor_main_p())) { - rb_raise(rb_eRactorUnsafeError, "require by autoload on non-main Ractor is not supported (%s)", rb_id2name(name)); + rb_raise(rb_eRactorUnsafeError, "require by autoload on non-main Ractor is not supported (%s)", rb_id2name(id)); } - // This state is stored on the stack and is used during the autoload process. - struct autoload_load_arguments arguments = {.module = module, .name = name, .mutex = Qnil}; - - // Figure out whether we can autoload the named constant: - VALUE autoload_const_value = rb_mutex_synchronize(autoload_mutex, autoload_load_needed, (VALUE)&arguments); + if ((ce = rb_const_lookup(mod, id))) { + flag = ce->flag & (CONST_DEPRECATED | CONST_VISIBILITY_MASK); + } - // This confirms whether autoloading is required or not: - if (autoload_const_value == Qfalse) return autoload_const_value; + /* set ele->state for a marker of autoloading thread */ + if (!(ele = get_autoload_data(load, &ac))) { + return Qfalse; + } + state.ac = ac; + state.thread = rb_thread_current(); + if (!ele->state) { + ele->state = &state; + ele->fork_gen = GET_VM()->fork_gen; - arguments.flag = ce->flag & (CONST_DEPRECATED | CONST_VISIBILITY_MASK); + /* + * autoload_reset will wake up any threads added to this + * if and only if the GVL is released during autoload_require + */ + list_head_init(&state.waitq); + } + else if (state.thread == ele->state->thread) { + return Qfalse; + } + else { + list_add_tail(&ele->state->waitq, &state.waitq.n); - // Only one thread will enter here at a time: - VALUE result = rb_mutex_synchronize(arguments.mutex, autoload_try_load, (VALUE)&arguments); + rb_ensure(autoload_sleep, (VALUE)&state, + autoload_sleep_done, (VALUE)&state); + } - // If you don't guard this value, it's possible for the autoload constant to - // be freed by another thread which loads multiple constants, one of which - // resolves to the constant this thread is trying to load, so proteect this - // so that it is not freed until we are done with it in `autoload_try_load`: - RB_GC_GUARD(autoload_const_value); + /* autoload_data_i can be deleted by another thread while require */ + state.result = Qfalse; + result = rb_ensure(autoload_require, (VALUE)&state, + autoload_reset, (VALUE)&state); + if (!(ce = rb_const_lookup(mod, id)) || ce->value == Qundef) { + rb_const_remove(mod, id); + } + else if (flag > 0) { + ce->flag |= flag; + } + RB_GC_GUARD(load); return result; } @@ -2816,30 +2612,30 @@ VALUE rb_autoload_at_p(VALUE mod, ID id, int recur) { VALUE load; - struct autoload_data *ele; + struct autoload_data_i *ele; while (!autoload_defined_p(mod, id)) { if (!recur) return Qnil; - mod = RCLASS_SUPER(mod); - if (!mod) return Qnil; + mod = RCLASS_SUPER(mod); + if (!mod) return Qnil; } load = check_autoload_required(mod, id, 0); if (!load) return Qnil; return (ele = get_autoload_data(load, 0)) ? ele->feature : Qnil; } -void +MJIT_FUNC_EXPORTED void rb_const_warn_if_deprecated(const rb_const_entry_t *ce, VALUE klass, ID id) { if (RB_CONST_DEPRECATED_P(ce) && rb_warning_category_enabled_p(RB_WARN_CATEGORY_DEPRECATED)) { - if (klass == rb_cObject) { + if (klass == rb_cObject) { rb_category_warn(RB_WARN_CATEGORY_DEPRECATED, "constant ::%"PRIsVALUE" is deprecated", QUOTE_ID(id)); - } - else { + } + else { rb_category_warn(RB_WARN_CATEGORY_DEPRECATED, "constant %"PRIsVALUE"::%"PRIsVALUE" is deprecated", - rb_class_name(klass), QUOTE_ID(id)); - } + rb_class_name(klass), QUOTE_ID(id)); + } } } @@ -2847,7 +2643,7 @@ static VALUE rb_const_get_0(VALUE klass, ID id, int exclude, int recurse, int visibility) { VALUE c = rb_const_search(klass, id, exclude, recurse, visibility); - if (!UNDEF_P(c)) { + if (c != Qundef) { if (UNLIKELY(!rb_ractor_main_p())) { if (!rb_ractor_shareable_p(c)) { rb_raise(rb_eRactorIsolationError, "can not access non-shareable objects in constant %"PRIsVALUE"::%s by non-main Ractor.", rb_class_path(klass), rb_id2name(id)); @@ -2868,8 +2664,8 @@ rb_const_search_from(VALUE klass, ID id, int exclude, int recurse, int visibilit RTEST(current); current = RCLASS_SUPER(current), first_iteration = false) { VALUE tmp; - VALUE am = 0; - rb_const_entry_t *ce; + VALUE am = 0; + rb_const_entry_t *ce; if (!first_iteration && RCLASS_ORIGIN(current) != current) { // This item in the super chain has an origin iclass @@ -2884,28 +2680,28 @@ rb_const_search_from(VALUE klass, ID id, int exclude, int recurse, int visibilit if (BUILTIN_TYPE(tmp) == T_ICLASS) tmp = RBASIC(tmp)->klass; // Do the lookup. Loop in case of autoload. - while ((ce = rb_const_lookup(tmp, id))) { - if (visibility && RB_CONST_PRIVATE_P(ce)) { - GET_EC()->private_const_reference = tmp; - return Qundef; - } - rb_const_warn_if_deprecated(ce, tmp, id); - value = ce->value; - if (UNDEF_P(value)) { + while ((ce = rb_const_lookup(tmp, id))) { + if (visibility && RB_CONST_PRIVATE_P(ce)) { + GET_EC()->private_const_reference = tmp; + return Qundef; + } + rb_const_warn_if_deprecated(ce, tmp, id); + value = ce->value; + if (value == Qundef) { struct autoload_const *ac; - if (am == tmp) break; - am = tmp; + if (am == tmp) break; + am = tmp; ac = autoloading_const_entry(tmp, id); if (ac) return ac->value; - rb_autoload_load(tmp, id); - continue; - } + rb_autoload_load(tmp, id); + continue; + } if (exclude && tmp == rb_cObject) { - goto not_found; - } - return value; - } - if (!recurse) break; + goto not_found; + } + return value; + } + if (!recurse) break; } not_found: @@ -2920,7 +2716,7 @@ rb_const_search(VALUE klass, ID id, int exclude, int recurse, int visibility) if (klass == rb_cObject) exclude = FALSE; value = rb_const_search_from(klass, id, exclude, recurse, visibility); - if (!UNDEF_P(value)) return value; + if (value != Qundef) return value; if (exclude) return value; if (BUILTIN_TYPE(klass) != T_MODULE) return value; /* search global const too, if klass is a module */ @@ -2945,13 +2741,13 @@ rb_const_get_at(VALUE klass, ID id) return rb_const_get_0(klass, id, TRUE, FALSE, FALSE); } -VALUE +MJIT_FUNC_EXPORTED VALUE rb_public_const_get_from(VALUE klass, ID id) { return rb_const_get_0(klass, id, TRUE, TRUE, TRUE); } -VALUE +MJIT_FUNC_EXPORTED VALUE rb_public_const_get_at(VALUE klass, ID id) { return rb_const_get_0(klass, id, TRUE, FALSE, TRUE); @@ -3009,7 +2805,7 @@ rb_const_source_location(VALUE klass, ID id) return rb_const_location(klass, id, FALSE, TRUE, FALSE); } -VALUE +MJIT_FUNC_EXPORTED VALUE rb_const_source_location_at(VALUE klass, ID id) { return rb_const_location(klass, id, TRUE, FALSE, FALSE); @@ -3043,27 +2839,23 @@ rb_const_remove(VALUE mod, ID id) rb_const_entry_t *ce; rb_check_frozen(mod); - ce = rb_const_lookup(mod, id); if (!ce || !rb_id_table_delete(RCLASS_CONST_TBL(mod), id)) { - if (rb_const_defined_at(mod, id)) { - rb_name_err_raise("cannot remove %2$s::%1$s", mod, ID2SYM(id)); - } - + if (rb_const_defined_at(mod, id)) { + rb_name_err_raise("cannot remove %2$s::%1$s", + mod, ID2SYM(id)); + } undefined_constant(mod, ID2SYM(id)); } - rb_clear_constant_cache_for_id(id); + rb_clear_constant_cache(); val = ce->value; - - if (UNDEF_P(val)) { - autoload_delete(mod, id); - val = Qnil; + if (val == Qundef) { + autoload_delete(mod, id); + val = Qnil; } - - ruby_xfree(ce); - + xfree(ce); return val; } @@ -3082,7 +2874,7 @@ sv_i(ID key, VALUE v, void *a) st_table *tbl = a; if (rb_is_const_id(key)) { - st_update(tbl, (st_data_t)key, cv_i_update, (st_data_t)ce); + st_update(tbl, (st_data_t)key, cv_i_update, (st_data_t)ce); } return ID_TABLE_CONTINUE; } @@ -3091,7 +2883,7 @@ static enum rb_id_table_iterator_result rb_local_constants_i(ID const_name, VALUE const_value, void *ary) { if (rb_is_const_id(const_name) && !RB_CONST_PRIVATE_P((rb_const_entry_t *)const_value)) { - rb_ary_push((VALUE)ary, ID2SYM(const_name)); + rb_ary_push((VALUE)ary, ID2SYM(const_name)); } return ID_TABLE_CONTINUE; } @@ -3119,7 +2911,7 @@ rb_mod_const_at(VALUE mod, void *data) { st_table *tbl = data; if (!tbl) { - tbl = st_init_numtable(); + tbl = st_init_numtable(); } if (RCLASS_CONST_TBL(mod)) { RB_VM_LOCK_ENTER(); @@ -3136,10 +2928,10 @@ rb_mod_const_of(VALUE mod, void *data) { VALUE tmp = mod; for (;;) { - data = rb_mod_const_at(tmp, data); - tmp = RCLASS_SUPER(tmp); - if (!tmp) break; - if (tmp == rb_cObject && mod != rb_cObject) break; + data = rb_mod_const_at(tmp, data); + tmp = RCLASS_SUPER(tmp); + if (!tmp) break; + if (tmp == rb_cObject && mod != rb_cObject) break; } return data; } @@ -3193,10 +2985,10 @@ rb_mod_constants(int argc, const VALUE *argv, VALUE mod) if (rb_check_arity(argc, 0, 1)) inherit = RTEST(argv[0]); if (inherit) { - return rb_const_list(rb_mod_const_of(mod, 0)); + return rb_const_list(rb_mod_const_of(mod, 0)); } else { - return rb_local_constants(mod); + return rb_local_constants(mod); } } @@ -3210,27 +3002,27 @@ rb_const_defined_0(VALUE klass, ID id, int exclude, int recurse, int visibility) tmp = klass; retry: while (tmp) { - if ((ce = rb_const_lookup(tmp, id))) { - if (visibility && RB_CONST_PRIVATE_P(ce)) { - return (int)Qfalse; - } - if (UNDEF_P(ce->value) && !check_autoload_required(tmp, id, 0) && - !rb_autoloading_value(tmp, id, NULL, NULL)) - return (int)Qfalse; - - if (exclude && tmp == rb_cObject && klass != rb_cObject) { - return (int)Qfalse; - } - - return (int)Qtrue; - } - if (!recurse) break; - tmp = RCLASS_SUPER(tmp); + if ((ce = rb_const_lookup(tmp, id))) { + if (visibility && RB_CONST_PRIVATE_P(ce)) { + return (int)Qfalse; + } + if (ce->value == Qundef && !check_autoload_required(tmp, id, 0) && + !rb_autoloading_value(tmp, id, NULL, NULL)) + return (int)Qfalse; + + if (exclude && tmp == rb_cObject && klass != rb_cObject) { + return (int)Qfalse; + } + + return (int)Qtrue; + } + if (!recurse) break; + tmp = RCLASS_SUPER(tmp); } if (!exclude && !mod_retry && BUILTIN_TYPE(klass) == T_MODULE) { - mod_retry = 1; - tmp = rb_cObject; - goto retry; + mod_retry = 1; + tmp = rb_cObject; + goto retry; } return (int)Qfalse; } @@ -3253,7 +3045,7 @@ rb_const_defined_at(VALUE klass, ID id) return rb_const_defined_0(klass, id, TRUE, FALSE, FALSE); } -int +MJIT_FUNC_EXPORTED int rb_public_const_defined_from(VALUE klass, ID id) { return rb_const_defined_0(klass, id, TRUE, TRUE, TRUE); @@ -3272,20 +3064,19 @@ set_namespace_path_i(ID id, VALUE v, void *payload) { rb_const_entry_t *ce = (rb_const_entry_t *)v; VALUE value = ce->value; + int has_permanent_classpath; VALUE parental_path = *((VALUE *) payload); if (!rb_is_const_id(id) || !rb_namespace_p(value)) { return ID_TABLE_CONTINUE; } - - bool has_permanent_classpath; classname(value, &has_permanent_classpath); if (has_permanent_classpath) { return ID_TABLE_CONTINUE; } set_namespace_path(value, build_const_path(parental_path, id)); - - if (!RCLASS_EXT(value)->permanent_classpath) { - RCLASS_SET_CLASSPATH(value, 0, false); + if (RCLASS_IV_TBL(value)) { + st_data_t tmp = tmp_classpath; + st_delete(RCLASS_IV_TBL(value), &tmp, 0); } return ID_TABLE_CONTINUE; @@ -3303,8 +3094,7 @@ set_namespace_path(VALUE named_namespace, VALUE namespace_path) RB_VM_LOCK_ENTER(); { - RCLASS_SET_CLASSPATH(named_namespace, namespace_path, true); - + rb_class_ivar_set(named_namespace, classpath, namespace_path); if (const_table) { rb_id_table_foreach(const_table, set_namespace_path_i, &namespace_path); } @@ -3312,23 +3102,14 @@ set_namespace_path(VALUE named_namespace, VALUE namespace_path) RB_VM_LOCK_LEAVE(); } -static void -const_added(VALUE klass, ID const_name) -{ - if (GET_VM()->running) { - VALUE name = ID2SYM(const_name); - rb_funcallv(klass, idConst_added, 1, &name); - } -} - -static void -const_set(VALUE klass, ID id, VALUE val) +void +rb_const_set(VALUE klass, ID id, VALUE val) { rb_const_entry_t *ce; if (NIL_P(klass)) { - rb_raise(rb_eTypeError, "no class/module to define constant %"PRIsVALUE"", - QUOTE_ID(id)); + rb_raise(rb_eTypeError, "no class/module to define constant %"PRIsVALUE"", + QUOTE_ID(id)); } if (!rb_ractor_main_p() && !rb_ractor_shareable_p(val)) { @@ -3342,19 +3123,18 @@ const_set(VALUE klass, ID id, VALUE val) struct rb_id_table *tbl = RCLASS_CONST_TBL(klass); if (!tbl) { RCLASS_CONST_TBL(klass) = tbl = rb_id_table_create(0); - rb_clear_constant_cache_for_id(id); + rb_clear_constant_cache(); ce = ZALLOC(rb_const_entry_t); rb_id_table_insert(tbl, id, (VALUE)ce); setup_const_entry(ce, klass, val, CONST_PUBLIC); } else { struct autoload_const ac = { - .module = klass, .name = id, + .mod = klass, .id = id, .value = val, .flag = CONST_PUBLIC, /* fill the rest with 0 */ }; - ac.file = rb_source_location(&ac.line); - const_tbl_update(&ac, false); + const_tbl_update(&ac); } } RB_VM_LOCK_LEAVE(); @@ -3364,118 +3144,105 @@ const_set(VALUE klass, ID id, VALUE val) * and avoid order-dependency on const_tbl */ if (rb_cObject && rb_namespace_p(val)) { - bool val_path_permanent; + int val_path_permanent; VALUE val_path = classname(val, &val_path_permanent); if (NIL_P(val_path) || !val_path_permanent) { - if (klass == rb_cObject) { + if (klass == rb_cObject) { set_namespace_path(val, rb_id2str(id)); - } - else { - bool parental_path_permanent; + } + else { + int parental_path_permanent; VALUE parental_path = classname(klass, &parental_path_permanent); if (NIL_P(parental_path)) { - bool throwaway; + int throwaway; parental_path = rb_tmp_class_path(klass, &throwaway, make_temporary_path); } if (parental_path_permanent && !val_path_permanent) { set_namespace_path(val, build_const_path(parental_path, id)); } else if (!parental_path_permanent && NIL_P(val_path)) { - RCLASS_SET_CLASSPATH(val, build_const_path(parental_path, id), false); + ivar_set(val, tmp_classpath, build_const_path(parental_path, id)); } - } - } + } + } } } -void -rb_const_set(VALUE klass, ID id, VALUE val) -{ - const_set(klass, id, val); - const_added(klass, id); -} - -static struct autoload_data * -autoload_data_for_named_constant(VALUE module, ID name, struct autoload_const **autoload_const_pointer) +static struct autoload_data_i * +current_autoload_data(VALUE mod, ID id, struct autoload_const **acp) { - VALUE autoload_data_value = autoload_data(module, name); - if (!autoload_data_value) return 0; - - struct autoload_data *autoload_data = get_autoload_data(autoload_data_value, autoload_const_pointer); - if (!autoload_data) return 0; - + struct autoload_data_i *ele; + VALUE load = autoload_data(mod, id); + if (!load) return 0; + ele = get_autoload_data(load, acp); + if (!ele) return 0; /* for autoloading thread, keep the defined value to autoloading storage */ - if (autoload_by_current(autoload_data)) { - return autoload_data; + if (ele->state && (ele->state->thread == rb_thread_current())) { + return ele; } - return 0; } static void -const_tbl_update(struct autoload_const *ac, int autoload_force) +const_tbl_update(struct autoload_const *ac) { VALUE value; - VALUE klass = ac->module; + VALUE klass = ac->mod; VALUE val = ac->value; - ID id = ac->name; + ID id = ac->id; struct rb_id_table *tbl = RCLASS_CONST_TBL(klass); rb_const_flag_t visibility = ac->flag; rb_const_entry_t *ce; if (rb_id_table_lookup(tbl, id, &value)) { - ce = (rb_const_entry_t *)value; - if (UNDEF_P(ce->value)) { - RUBY_ASSERT_CRITICAL_SECTION_ENTER(); - VALUE file = ac->file; - int line = ac->line; - struct autoload_data *ele = autoload_data_for_named_constant(klass, id, &ac); + ce = (rb_const_entry_t *)value; + if (ce->value == Qundef) { + struct autoload_data_i *ele = current_autoload_data(klass, id, &ac); - if (!autoload_force && ele) { - rb_clear_constant_cache_for_id(id); + if (ele) { + rb_clear_constant_cache(); - ac->value = val; /* autoload_data is non-WB-protected */ + ac->value = val; /* autoload_i is non-WB-protected */ ac->file = rb_source_location(&ac->line); - } + } else { /* otherwise autoloaded constant, allow to override */ autoload_delete(klass, id); ce->flag = visibility; RB_OBJ_WRITE(klass, &ce->value, val); - RB_OBJ_WRITE(klass, &ce->file, file); - ce->line = line; + RB_OBJ_WRITE(klass, &ce->file, ac->file); + ce->line = ac->line; } - RUBY_ASSERT_CRITICAL_SECTION_LEAVE(); return; - } - else { - VALUE name = QUOTE_ID(id); - visibility = ce->flag; - if (klass == rb_cObject) - rb_warn("already initialized constant %"PRIsVALUE"", name); - else - rb_warn("already initialized constant %"PRIsVALUE"::%"PRIsVALUE"", - rb_class_name(klass), name); - if (!NIL_P(ce->file) && ce->line) { - rb_compile_warn(RSTRING_PTR(ce->file), ce->line, - "previous definition of %"PRIsVALUE" was here", name); - } - } - rb_clear_constant_cache_for_id(id); - setup_const_entry(ce, klass, val, visibility); + } + else { + VALUE name = QUOTE_ID(id); + visibility = ce->flag; + if (klass == rb_cObject) + rb_warn("already initialized constant %"PRIsVALUE"", name); + else + rb_warn("already initialized constant %"PRIsVALUE"::%"PRIsVALUE"", + rb_class_name(klass), name); + if (!NIL_P(ce->file) && ce->line) { + rb_compile_warn(RSTRING_PTR(ce->file), ce->line, + "previous definition of %"PRIsVALUE" was here", name); + } + } + rb_clear_constant_cache(); + setup_const_entry(ce, klass, val, visibility); } else { - rb_clear_constant_cache_for_id(id); + rb_clear_constant_cache(); - ce = ZALLOC(rb_const_entry_t); - rb_id_table_insert(tbl, id, (VALUE)ce); - setup_const_entry(ce, klass, val, visibility); + ce = ZALLOC(rb_const_entry_t); + rb_id_table_insert(tbl, id, (VALUE)ce); + setup_const_entry(ce, klass, val, visibility); } } static void setup_const_entry(rb_const_entry_t *ce, VALUE klass, VALUE val, - rb_const_flag_t visibility) + rb_const_flag_t visibility) { ce->flag = visibility; RB_OBJ_WRITE(klass, &ce->value, val); @@ -3488,7 +3255,7 @@ rb_define_const(VALUE klass, const char *name, VALUE val) ID id = rb_intern(name); if (!rb_is_const_id(id)) { - rb_warn("rb_define_const: invalid name `%s' for constant", name); + rb_warn("rb_define_const: invalid name `%s' for constant", name); } rb_gc_register_mark_object(val); rb_const_set(klass, id, val); @@ -3502,7 +3269,7 @@ rb_define_global_const(const char *name, VALUE val) static void set_const_visibility(VALUE mod, int argc, const VALUE *argv, - rb_const_flag_t flag, rb_const_flag_t mask) + rb_const_flag_t flag, rb_const_flag_t mask) { int i; rb_const_entry_t *ce; @@ -3510,36 +3277,43 @@ set_const_visibility(VALUE mod, int argc, const VALUE *argv, rb_class_modify_check(mod); if (argc == 0) { - rb_warning("%"PRIsVALUE" with no argument is just ignored", - QUOTE_ID(rb_frame_callee())); - return; + rb_warning("%"PRIsVALUE" with no argument is just ignored", + QUOTE_ID(rb_frame_callee())); + return; } for (i = 0; i < argc; i++) { - struct autoload_const *ac; - VALUE val = argv[i]; - id = rb_check_id(&val); - if (!id) { + struct autoload_const *ac; + VALUE val = argv[i]; + id = rb_check_id(&val); + if (!id) { + if (i > 0) { + rb_clear_constant_cache(); + } + undefined_constant(mod, val); - } - if ((ce = rb_const_lookup(mod, id))) { - ce->flag &= ~mask; - ce->flag |= flag; - if (UNDEF_P(ce->value)) { - struct autoload_data *ele; - - ele = autoload_data_for_named_constant(mod, id, &ac); - if (ele) { - ac->flag &= ~mask; - ac->flag |= flag; - } - } - rb_clear_constant_cache_for_id(id); - } - else { + } + if ((ce = rb_const_lookup(mod, id))) { + ce->flag &= ~mask; + ce->flag |= flag; + if (ce->value == Qundef) { + struct autoload_data_i *ele; + + ele = current_autoload_data(mod, id, &ac); + if (ele) { + ac->flag &= ~mask; + ac->flag |= flag; + } + } + } + else { + if (i > 0) { + rb_clear_constant_cache(); + } undefined_constant(mod, ID2SYM(id)); - } + } } + rb_clear_constant_cache(); } void @@ -3617,38 +3391,25 @@ static VALUE original_module(VALUE c) { if (RB_TYPE_P(c, T_ICLASS)) - return RBASIC(c)->klass; + return RBASIC(c)->klass; return c; } static int cvar_lookup_at(VALUE klass, ID id, st_data_t *v) { - if (RB_TYPE_P(klass, T_ICLASS)) { - if (FL_TEST_RAW(klass, RICLASS_IS_ORIGIN)) { - return 0; - } - else { - // check the original module - klass = RBASIC(klass)->klass; - } - } - - VALUE n = rb_ivar_lookup(klass, id, Qundef); - if (UNDEF_P(n)) return 0; - - if (v) *v = n; - return 1; + if (!RCLASS_IV_TBL(klass)) return 0; + return st_lookup(RCLASS_IV_TBL(klass), (st_data_t)id, v); } static VALUE cvar_front_klass(VALUE klass) { if (FL_TEST(klass, FL_SINGLETON)) { - VALUE obj = RCLASS_ATTACHED_OBJECT(klass); + VALUE obj = rb_ivar_get(klass, id__attached__); if (rb_namespace_p(obj)) { - return obj; - } + return obj; + } } return RCLASS_SUPER(klass); } @@ -3657,50 +3418,63 @@ static void cvar_overtaken(VALUE front, VALUE target, ID id) { if (front && target != front) { + st_data_t did = (st_data_t)id; + if (original_module(front) != original_module(target)) { rb_raise(rb_eRuntimeError, "class variable % "PRIsVALUE" of %"PRIsVALUE" is overtaken by %"PRIsVALUE"", - ID2SYM(id), rb_class_name(original_module(front)), - rb_class_name(original_module(target))); - } - if (BUILTIN_TYPE(front) == T_CLASS) { - rb_ivar_delete(front, id, Qundef); - } + ID2SYM(id), rb_class_name(original_module(front)), + rb_class_name(original_module(target))); + } + if (BUILTIN_TYPE(front) == T_CLASS) { + st_delete(RCLASS_IV_TBL(front), &did, 0); + } } } -#define CVAR_FOREACH_ANCESTORS(klass, v, r) \ - for (klass = cvar_front_klass(klass); klass; klass = RCLASS_SUPER(klass)) { \ - if (cvar_lookup_at(klass, id, (v))) { \ - r; \ - } \ - } - -#define CVAR_LOOKUP(v,r) do {\ - CVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(); \ - if (cvar_lookup_at(klass, id, (v))) {r;}\ - CVAR_FOREACH_ANCESTORS(klass, v, r);\ -} while(0) - static VALUE find_cvar(VALUE klass, VALUE * front, VALUE * target, ID id) { VALUE v = Qundef; - CVAR_LOOKUP(&v, { + CVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(); + if (cvar_lookup_at(klass, id, (&v))) { if (!*front) { *front = klass; } *target = klass; - }); + } + + for (klass = cvar_front_klass(klass); klass; klass = RCLASS_SUPER(klass)) { + if (cvar_lookup_at(klass, id, (&v))) { + if (!*front) { + *front = klass; + } + *target = klass; + } + } return v; } +#define CVAR_FOREACH_ANCESTORS(klass, v, r) \ + for (klass = cvar_front_klass(klass); klass; klass = RCLASS_SUPER(klass)) { \ + if (cvar_lookup_at(klass, id, (v))) { \ + r; \ + } \ + } + +#define CVAR_LOOKUP(v,r) do {\ + CVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(); \ + if (cvar_lookup_at(klass, id, (v))) {r;}\ + CVAR_FOREACH_ANCESTORS(klass, v, r);\ +} while(0) + static void check_for_cvar_table(VALUE subclass, VALUE key) { - // Must not check ivar on ICLASS - if (!RB_TYPE_P(subclass, T_ICLASS) && RTEST(rb_ivar_defined(subclass, key))) { + st_table *tbl = RCLASS_IV_TBL(subclass); + + if (tbl && st_lookup(tbl, key, NULL)) { RB_DEBUG_COUNTER_INC(cvar_class_invalidate); ruby_vm_global_cvar_state++; return; @@ -3717,10 +3491,10 @@ rb_cvar_set(VALUE klass, ID id, VALUE val) tmp = klass; CVAR_LOOKUP(0, {if (!front) front = klass; target = klass;}); if (target) { - cvar_overtaken(front, target, id); + cvar_overtaken(front, target, id); } else { - target = tmp; + target = tmp; } if (RB_TYPE_P(target, T_ICLASS)) { @@ -3772,8 +3546,8 @@ rb_cvar_find(VALUE klass, ID id, VALUE *front) value = find_cvar(klass, front, &target, id); if (!target) { - rb_name_err_raise("uninitialized class variable %1$s in %2$s", - klass, ID2SYM(id)); + rb_name_err_raise("uninitialized class variable %1$s in %2$s", + klass, ID2SYM(id)); } cvar_overtaken(*front, target, id); return (VALUE)value; @@ -3799,8 +3573,8 @@ cv_intern(VALUE klass, const char *name) { ID id = rb_intern(name); if (!rb_is_class_id(id)) { - rb_name_err_raise("wrong class variable name %1$s", - klass, rb_str_new_cstr(name)); + rb_name_err_raise("wrong class variable name %1$s", + klass, rb_str_new_cstr(name)); } return id; } @@ -3832,7 +3606,7 @@ cv_i(st_data_t k, st_data_t v, st_data_t a) st_table *tbl = (st_table *)a; if (rb_is_class_id(key)) { - st_update(tbl, (st_data_t)key, cv_i_update, 0); + st_update(tbl, (st_data_t)key, cv_i_update, 0); } return ST_CONTINUE; } @@ -3842,11 +3616,11 @@ mod_cvar_at(VALUE mod, void *data) { st_table *tbl = data; if (!tbl) { - tbl = st_init_numtable(); + tbl = st_init_numtable(); + } + if (RCLASS_IV_TBL(mod)) { + st_foreach_safe(RCLASS_IV_TBL(mod), cv_i, (st_data_t)tbl); } - mod = original_module(mod); - - rb_ivar_foreach(mod, cv_i, (st_data_t)tbl); return tbl; } @@ -3855,15 +3629,15 @@ mod_cvar_of(VALUE mod, void *data) { VALUE tmp = mod; if (FL_TEST(mod, FL_SINGLETON)) { - if (rb_namespace_p(RCLASS_ATTACHED_OBJECT(mod))) { + if (rb_namespace_p(rb_ivar_get(mod, id__attached__))) { data = mod_cvar_at(tmp, data); tmp = cvar_front_klass(tmp); } } for (;;) { - data = mod_cvar_at(tmp, data); - tmp = RCLASS_SUPER(tmp); - if (!tmp) break; + data = mod_cvar_at(tmp, data); + tmp = RCLASS_SUPER(tmp); + if (!tmp) break; } return data; } @@ -3918,10 +3692,10 @@ rb_mod_class_variables(int argc, const VALUE *argv, VALUE mod) if (rb_check_arity(argc, 0, 1)) inherit = RTEST(argv[0]); if (inherit) { - tbl = mod_cvar_of(mod, 0); + tbl = mod_cvar_of(mod, 0); } else { - tbl = mod_cvar_at(mod, 0); + tbl = mod_cvar_at(mod, 0); } return cvar_list(tbl); } @@ -3949,18 +3723,17 @@ VALUE rb_mod_remove_cvar(VALUE mod, VALUE name) { const ID id = id_for_var_message(mod, name, class, "wrong class variable name %1$s"); - st_data_t val; + st_data_t val, n = id; if (!id) { goto not_defined; } rb_check_frozen(mod); - val = rb_ivar_delete(mod, id, Qundef); - if (!UNDEF_P(val)) { - return (VALUE)val; + if (RCLASS_IV_TBL(mod) && st_delete(RCLASS_IV_TBL(mod), &n, &val)) { + return (VALUE)val; } if (rb_cvar_defined(mod, id)) { - rb_name_err_raise("cannot remove %1$s for %2$s", mod, ID2SYM(id)); + rb_name_err_raise("cannot remove %1$s for %2$s", mod, ID2SYM(id)); } not_defined: rb_name_err_raise("class variable %1$s not defined for %2$s", @@ -3991,69 +3764,33 @@ rb_iv_set(VALUE obj, const char *name, VALUE val) int rb_class_ivar_set(VALUE obj, ID key, VALUE value) { - RUBY_ASSERT(RB_TYPE_P(obj, T_CLASS) || RB_TYPE_P(obj, T_MODULE)); - int found; - rb_check_frozen(obj); - - RB_VM_LOCK_ENTER(); - { - rb_shape_t * shape = rb_shape_get_shape(obj); - attr_index_t idx; - found = rb_shape_get_iv_index(shape, key, &idx); - - if (found) { - // Changing an existing instance variable - RUBY_ASSERT(RCLASS_IVPTR(obj)); - - RCLASS_IVPTR(obj)[idx] = value; - RB_OBJ_WRITTEN(obj, Qundef, value); - } - else { - // Creating and setting a new instance variable - - // Move to a shape which fits the new ivar - idx = shape->next_iv_index; - shape = rb_shape_get_next(shape, obj, key); - - // We always allocate a power of two sized IV array. This way we - // only need to realloc when we expand into a new power of two size - if ((idx & (idx - 1)) == 0) { - size_t newsize = idx ? idx * 2 : 1; - REALLOC_N(RCLASS_IVPTR(obj), VALUE, newsize); - } - - RUBY_ASSERT(RCLASS_IVPTR(obj)); - - RB_OBJ_WRITE(obj, &RCLASS_IVPTR(obj)[idx], value); - rb_shape_set_shape(obj, shape); - } + if (!RCLASS_IV_TBL(obj)) { + RCLASS_IV_TBL(obj) = st_init_numtable(); } - RB_VM_LOCK_LEAVE(); - return found; + st_table *tbl = RCLASS_IV_TBL(obj); + int result = lock_st_insert(tbl, (st_data_t)key, (st_data_t)value); + RB_OBJ_WRITTEN(obj, Qundef, value); + return result; } static int -tbl_copy_i(st_data_t key, st_data_t val, st_data_t dest) +tbl_copy_i(st_data_t key, st_data_t value, st_data_t data) { - rb_class_ivar_set(dest, key, val); - + RB_OBJ_WRITTEN((VALUE)data, Qundef, (VALUE)value); return ST_CONTINUE; } void rb_iv_tbl_copy(VALUE dst, VALUE src) { - RUBY_ASSERT(rb_type(dst) == rb_type(src)); - RUBY_ASSERT(RB_TYPE_P(dst, T_CLASS) || RB_TYPE_P(dst, T_MODULE)); - - RUBY_ASSERT(RCLASS_SHAPE_ID(dst) == ROOT_SHAPE_ID || rb_shape_get_shape_by_id(RCLASS_SHAPE_ID(dst))->type == SHAPE_INITIAL_CAPACITY); - RUBY_ASSERT(!RCLASS_IVPTR(dst)); - - rb_ivar_foreach(src, tbl_copy_i, dst); + st_table *orig_tbl = RCLASS_IV_TBL(src); + st_table *new_tbl = st_copy(orig_tbl); + st_foreach(new_tbl, tbl_copy_i, (st_data_t)dst); + RCLASS_IV_TBL(dst) = new_tbl; } -rb_const_entry_t * +MJIT_FUNC_EXPORTED rb_const_entry_t * rb_const_lookup(VALUE klass, ID id) { struct rb_id_table *tbl = RCLASS_CONST_TBL(klass); |