diff options
Diffstat (limited to 'vm.c')
| -rw-r--r-- | vm.c | 898 |
1 files changed, 614 insertions, 284 deletions
@@ -12,6 +12,7 @@ #include "eval_intern.h" #include "internal.h" +#include "internal/box.h" #include "internal/class.h" #include "internal/compile.h" #include "internal/cont.h" @@ -25,6 +26,7 @@ #include "internal/proc.h" #include "internal/re.h" #include "internal/ruby_parser.h" +#include "internal/st.h" #include "internal/symbol.h" #include "internal/thread.h" #include "internal/transcode.h" @@ -34,6 +36,8 @@ #include "iseq.h" #include "symbol.h" // This includes a macro for a more performant rb_id2sym. #include "yjit.h" +#include "insns.inc" +#include "zjit.h" #include "ruby/st.h" #include "ruby/vm.h" #include "vm_core.h" @@ -56,6 +60,8 @@ int ruby_assert_critical_section_entered = 0; static void *native_main_thread_stack_top; +bool ruby_vm_during_cleanup = false; + VALUE rb_str_concat_literals(size_t, const VALUE*); VALUE vm_exec(rb_execution_context_t *); @@ -92,6 +98,81 @@ rb_vm_search_cf_from_ep(const rb_execution_context_t *ec, const rb_control_frame } } +#if VM_CHECK_MODE > 0 +// ruby_box_crashed defined in internal/box.h +#define VM_BOX_CRASHED() {ruby_box_crashed = true;} +#define VM_BOX_ASSERT(expr, msg) \ + if (!(expr)) { ruby_box_crashed = true; rb_bug(msg); } +#else +#define VM_BOX_CRASHED() {} +#define VM_BOX_ASSERT(expr, msg) ((void)0) +#endif + +static const VALUE * +VM_EP_RUBY_LEP(const rb_execution_context_t *ec, const rb_control_frame_t *current_cfp) +{ + // rb_vmdebug_box_env_dump_raw() simulates this function + const VALUE *ep = current_cfp->ep; + const rb_control_frame_t * const eocfp = RUBY_VM_END_CONTROL_FRAME(ec); /* end of control frame pointer */ + const rb_control_frame_t *cfp = current_cfp; + + if (VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_IFUNC)) { + ep = VM_EP_LEP(current_cfp->ep); + /** + * Returns CFUNC frame only in this case. + * + * Usually CFUNC frame doesn't represent the current box and it should operate + * the caller box. See the example: + * + * # in the main box + * module Kernel + * def foo = "foo" + * module_function :foo + * end + * + * In the case above, `module_function` is defined in the root box. + * If `module_function` worked in the root box, `Kernel#foo` is invisible + * from it and it causes NameError: undefined method `foo` for module `Kernel`. + * + * But in cases of IFUNC (blocks written in C), IFUNC doesn't have its own box + * and its local env frame will be CFUNC frame. + * For example, `Enumerator#chunk` calls IFUNC blocks, written as `chunk_i` function. + * + * [1].chunk{ it.even? }.each{ ... } + * + * Before calling the Ruby block `{ it.even? }`, `#chunk` calls `chunk_i` as IFUNC + * to iterate the array's members (it's just like `#each`). + * We expect that `chunk_i` works as expected by the implementation of `#chunk` + * without any overwritten definitions from boxes. + * So the definitions on IFUNC frames should be equal to the caller CFUNC. + */ + VM_ASSERT(VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_CFUNC)); + return ep; + } + + while (VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_CFUNC)) { + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + + VM_BOX_ASSERT(cfp, "CFUNC should have a valid previous control frame"); + VM_BOX_ASSERT(cfp < eocfp, "CFUNC should have a valid caller frame"); + if (!cfp || cfp >= eocfp) { + return NULL; + } + + VM_BOX_ASSERT(cfp->ep, "CFUNC should have a valid caller frame with env"); + ep = cfp->ep; + if (!ep) { + return NULL; + } + } + + while (!VM_ENV_LOCAL_P(ep)) { + ep = VM_ENV_PREV_EP(ep); + } + + return ep; +} + const VALUE * rb_vm_ep_local_ep(const VALUE *ep) { @@ -115,7 +196,15 @@ PUREFUNC(static inline VALUE VM_CF_BLOCK_HANDLER(const rb_control_frame_t * cons static inline VALUE VM_CF_BLOCK_HANDLER(const rb_control_frame_t * const cfp) { - const VALUE *ep = VM_CF_LEP(cfp); + const VALUE *ep; + if (VM_ENV_BOXED_P(cfp->ep)) { + VM_ASSERT(VM_ENV_LOCAL_P(cfp->ep)); + /* Never set black_handler for VM_FRAME_MAGIC_TOP or VM_FRAME_MAGIC_CLASS + * and the specval is used for boxes (rb_box_t) in these case + */ + return VM_BLOCK_HANDLER_NONE; + } + ep = VM_CF_LEP(cfp); return VM_ENV_BLOCK_HANDLER(ep); } @@ -230,13 +319,9 @@ vm_cref_new0(VALUE klass, rb_method_visibility_t visi, int module_func, rb_cref_ int omod_shared = FALSE; /* scope */ - union { - rb_scope_visibility_t visi; - VALUE value; - } scope_visi; - - scope_visi.visi.method_visi = visi; - scope_visi.visi.module_func = module_func; + rb_scope_visibility_t scope_visi; + scope_visi.method_visi = visi; + scope_visi.module_func = module_func; /* refinements */ if (prev_cref != NULL && prev_cref != (void *)1 /* TODO: why CREF_NEXT(cref) is 1? */) { @@ -250,10 +335,10 @@ vm_cref_new0(VALUE klass, rb_method_visibility_t visi, int module_func, rb_cref_ VM_ASSERT(singleton || klass); - rb_cref_t *cref = IMEMO_NEW(rb_cref_t, imemo_cref, refinements); + rb_cref_t *cref = SHAREABLE_IMEMO_NEW(rb_cref_t, imemo_cref, refinements); cref->klass_or_self = klass; cref->next = use_prev_prev ? CREF_NEXT(prev_cref) : prev_cref; - *((rb_scope_visibility_t *)&cref->scope_visi) = scope_visi.visi; + *((rb_scope_visibility_t *)&cref->scope_visi) = scope_visi; if (pushed_by_eval) CREF_PUSHED_BY_EVAL_SET(cref); if (omod_shared) CREF_OMOD_SHARED_SET(cref); @@ -268,12 +353,6 @@ vm_cref_new(VALUE klass, rb_method_visibility_t visi, int module_func, rb_cref_t return vm_cref_new0(klass, visi, module_func, prev_cref, pushed_by_eval, FALSE, singleton); } -static rb_cref_t * -vm_cref_new_use_prev(VALUE klass, rb_method_visibility_t visi, int module_func, rb_cref_t *prev_cref, int pushed_by_eval) -{ - return vm_cref_new0(klass, visi, module_func, prev_cref, pushed_by_eval, TRUE, FALSE); -} - static int ref_delete_symkey(VALUE key, VALUE value, VALUE unused) { @@ -429,13 +508,13 @@ rb_yjit_threshold_hit(const rb_iseq_t *iseq, uint64_t entry_calls) // The current frame doesn't have VM_FRAME_FLAG_FINISH. The current // vm_exec does NOT stop whether JIT code returns Qundef or not. static inline rb_jit_func_t -jit_compile(rb_execution_context_t *ec) +yjit_compile(rb_execution_context_t *ec) { - const rb_iseq_t *iseq = ec->cfp->iseq; + const rb_iseq_t *iseq = CFP_ISEQ(ec->cfp); struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); // Increment the ISEQ's call counter and trigger JIT compilation if not compiled - if (body->jit_entry == NULL && rb_yjit_enabled_p) { + if (body->jit_entry == NULL) { body->jit_entry_calls++; if (rb_yjit_threshold_hit(iseq, body->jit_entry_calls)) { rb_yjit_compile_iseq(iseq, ec, false); @@ -443,26 +522,65 @@ jit_compile(rb_execution_context_t *ec) } return body->jit_entry; } +#else +# define yjit_compile(ec) ((rb_jit_func_t)0) +#endif -// Execute JIT code compiled by jit_compile() -static inline VALUE -jit_exec(rb_execution_context_t *ec) +#if USE_ZJIT +static inline rb_jit_func_t +zjit_compile(rb_execution_context_t *ec) { - rb_jit_func_t func = jit_compile(ec); - if (func) { - // Call the JIT code - return func(ec, ec->cfp); - } - else { - return Qundef; + const rb_iseq_t *iseq = CFP_ISEQ(ec->cfp); + struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); + + if (body->jit_entry == NULL) { + body->jit_entry_calls++; + + // At profile-threshold, rewrite some of the YARV instructions + // to zjit_* instructions to profile these instructions. + if (body->jit_entry_calls == rb_zjit_profile_threshold) { + rb_zjit_profile_enable(iseq); + } + + // At call-threshold, compile the ISEQ with ZJIT. + if (body->jit_entry_calls == rb_zjit_call_threshold) { + rb_zjit_compile_iseq(iseq, ec, false); + } } + return body->jit_entry; } #else -# define jit_compile(ec) ((rb_jit_func_t)0) -# define jit_exec(ec) Qundef +# define zjit_compile(ec) ((rb_jit_func_t)0) #endif +#if USE_YJIT || USE_ZJIT +// Execute JIT code compiled by yjit_compile() or zjit_compile() +static inline VALUE +jit_exec(rb_execution_context_t *ec) +{ #if USE_YJIT + if (rb_yjit_enabled_p) { + rb_jit_func_t func = yjit_compile(ec); + if (func) { + return func(ec, ec->cfp); + } + return Qundef; + } +#endif + +#if USE_ZJIT + void *zjit_entry = rb_zjit_entry; + if (zjit_entry) { + rb_jit_func_t func = zjit_compile(ec); + if (func) { + VALUE result = ((rb_zjit_func_t)zjit_entry)(ec, ec->cfp, func); + return result; + } + } +#endif + return Qundef; +} + // Generate JIT code that supports the following kind of ISEQ entry: // * The first ISEQ pushed by vm_exec_handle_exception. The frame would // point to a location specified by a catch table, and it doesn't have @@ -472,9 +590,27 @@ jit_exec(rb_execution_context_t *ec) static inline rb_jit_func_t jit_compile_exception(rb_execution_context_t *ec) { - const rb_iseq_t *iseq = ec->cfp->iseq; + const rb_iseq_t *iseq = CFP_ISEQ(ec->cfp); struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); +#if USE_ZJIT + if (body->jit_exception == NULL && rb_zjit_enabled_p) { + body->jit_exception_calls++; + + // At profile-threshold, rewrite some of the YARV instructions + // to zjit_* instructions to profile these instructions. + if (body->jit_exception_calls == rb_zjit_profile_threshold) { + rb_zjit_profile_enable(iseq); + } + + // At call-threshold, compile the ISEQ with ZJIT. + if (body->jit_exception_calls == rb_zjit_call_threshold) { + rb_zjit_compile_iseq(iseq, ec, true); + } + } +#endif + +#if USE_YJIT // Increment the ISEQ's call counter and trigger JIT compilation if not compiled if (body->jit_exception == NULL && rb_yjit_enabled_p) { body->jit_exception_calls++; @@ -482,6 +618,7 @@ jit_compile_exception(rb_execution_context_t *ec) rb_yjit_compile_iseq(iseq, ec, true); } } +#endif return body->jit_exception; } @@ -500,6 +637,7 @@ jit_exec_exception(rb_execution_context_t *ec) } #else # define jit_compile_exception(ec) ((rb_jit_func_t)0) +# define jit_exec(ec) Qundef # define jit_exec_exception(ec) Qundef #endif @@ -557,7 +695,7 @@ rb_current_ec_set(rb_execution_context_t *ec) } -#if defined(__arm64__) || defined(__aarch64__) +#ifdef RB_THREAD_CURRENT_EC_NOINLINE rb_execution_context_t * rb_current_ec(void) { @@ -577,9 +715,10 @@ rb_current_ec_noinline(void) #endif -rb_event_flag_t ruby_vm_event_flags; -rb_event_flag_t ruby_vm_event_enabled_global_flags; -unsigned int ruby_vm_event_local_num; +rb_event_flag_t ruby_vm_event_flags = 0; +rb_event_flag_t ruby_vm_event_enabled_global_flags = 0; +unsigned int ruby_vm_c_events_enabled = 0; +unsigned int ruby_vm_iseq_events_enabled = 0; rb_serial_t ruby_vm_constant_cache_invalidations = 0; rb_serial_t ruby_vm_constant_cache_misses = 0; @@ -587,7 +726,7 @@ rb_serial_t ruby_vm_global_cvar_state = 1; static const struct rb_callcache vm_empty_cc = { .flags = T_IMEMO | (imemo_callcache << FL_USHIFT) | VM_CALLCACHE_UNMARKABLE, - .klass = Qfalse, + .klass = Qundef, .cme_ = NULL, .call_ = vm_call_general, .aux_ = { @@ -597,7 +736,7 @@ static const struct rb_callcache vm_empty_cc = { static const struct rb_callcache vm_empty_cc_for_super = { .flags = T_IMEMO | (imemo_callcache << FL_USHIFT) | VM_CALLCACHE_UNMARKABLE, - .klass = Qfalse, + .klass = Qundef, .cme_ = NULL, .call_ = vm_call_super_method, .aux_ = { @@ -712,8 +851,8 @@ vm_stat(int argc, VALUE *argv, VALUE self) SET(constant_cache_invalidations, ruby_vm_constant_cache_invalidations); SET(constant_cache_misses, ruby_vm_constant_cache_misses); SET(global_cvar_state, ruby_vm_global_cvar_state); - SET(next_shape_id, (rb_serial_t)GET_SHAPE_TREE()->next_shape_id); - SET(shape_cache_size, (rb_serial_t)GET_SHAPE_TREE()->cache_size); + SET(next_shape_id, (rb_serial_t)rb_shapes_count()); + SET(shape_cache_size, (rb_serial_t)rb_shapes_cache_size()); #undef SET #if USE_DEBUG_COUNTER @@ -741,15 +880,16 @@ vm_stat(int argc, VALUE *argv, VALUE self) /* control stack frame */ static void -vm_set_top_stack(rb_execution_context_t *ec, const rb_iseq_t *iseq) +vm_set_top_stack(rb_execution_context_t *ec, const rb_iseq_t *iseq, const rb_box_t *box) { if (ISEQ_BODY(iseq)->type != ISEQ_TYPE_TOP) { rb_raise(rb_eTypeError, "Not a toplevel InstructionSequence"); } /* for return */ - vm_push_frame(ec, iseq, VM_FRAME_MAGIC_TOP | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH, rb_ec_thread_ptr(ec)->top_self, - VM_BLOCK_HANDLER_NONE, + vm_push_frame(ec, iseq, VM_FRAME_MAGIC_TOP | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH, + rb_ec_thread_ptr(ec)->top_self, + GC_GUARDED_PTR(box), (VALUE)vm_cref_new_toplevel(ec), /* cref or me */ ISEQ_BODY(iseq)->iseq_encoded, ec->cfp->sp, ISEQ_BODY(iseq)->local_table_size, ISEQ_BODY(iseq)->stack_max); @@ -787,7 +927,7 @@ rb_control_frame_t * rb_vm_get_binding_creatable_next_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) { - if (cfp->iseq) { + if (CFP_ISEQ(cfp)) { return (rb_control_frame_t *)cfp; } cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); @@ -956,6 +1096,7 @@ vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *co } } else { + VM_ASSERT(VM_ENV_LOCAL_P(ep)); VALUE block_handler = VM_ENV_BLOCK_HANDLER(ep); if (block_handler != VM_BLOCK_HANDLER_NONE) { @@ -964,13 +1105,14 @@ vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *co } } + const rb_iseq_t *iseq = CFP_ISEQ(cfp); if (!VM_FRAME_RUBYFRAME_P(cfp)) { local_size = VM_ENV_DATA_SIZE; } else { - local_size = ISEQ_BODY(cfp->iseq)->local_table_size; - if (ISEQ_BODY(cfp->iseq)->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) { - int ci_offset = local_size - ISEQ_BODY(cfp->iseq)->param.size + VM_ENV_DATA_SIZE; + local_size = ISEQ_BODY(iseq)->local_table_size; + if (ISEQ_BODY(iseq)->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) { + int ci_offset = local_size - ISEQ_BODY(iseq)->param.size + VM_ENV_DATA_SIZE; CALL_INFO ci = (CALL_INFO)VM_CF_LEP(cfp)[-ci_offset]; local_size += vm_ci_argc(ci); @@ -978,6 +1120,14 @@ vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *co local_size += VM_ENV_DATA_SIZE; } + // Invalidate JIT code that assumes cfp->ep == vm_base_ptr(cfp). + // This is done before creating the imemo_env because VM_STACK_ENV_WRITE + // below leaves the on-stack ep in a state that is unsafe to GC. + if (VM_FRAME_RUBYFRAME_P(cfp)) { + rb_yjit_invalidate_ep_is_bp(iseq); + rb_zjit_invalidate_no_ep_escape(iseq); + } + /* * # local variables on a stack frame (N == local_size) * [lvar1, lvar2, ..., lvarN, SPECVAL] @@ -1003,7 +1153,7 @@ vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *co env_ep = &env_body[local_size - 1 /* specval */]; env_ep[VM_ENV_DATA_INDEX_ENV] = (VALUE)env; - env->iseq = (rb_iseq_t *)(VM_FRAME_RUBYFRAME_P(cfp) ? cfp->iseq : NULL); + env->iseq = (rb_iseq_t *)(VM_FRAME_RUBYFRAME_P(cfp) ? iseq : NULL); env->ep = env_ep; env->env = env_body; env->env_size = env_size; @@ -1021,11 +1171,6 @@ vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *co } #endif - // Invalidate JIT code that assumes cfp->ep == vm_base_ptr(cfp). - if (env->iseq) { - rb_yjit_invalidate_ep_is_bp(env->iseq); - } - return (VALUE)env; } @@ -1209,7 +1354,7 @@ rb_proc_dup(VALUE self) break; } - if (RB_OBJ_SHAREABLE_P(self)) FL_SET_RAW(procval, RUBY_FL_SHAREABLE); + if (RB_OBJ_SHAREABLE_P(self)) RB_OBJ_SET_SHAREABLE(procval); RB_GC_GUARD(self); /* for: body = rb_proc_dup(body) */ return procval; } @@ -1273,7 +1418,19 @@ env_copy(const VALUE *src_ep, VALUE read_only_variables) const rb_env_t *copied_env = vm_env_new(ep, env_body, src_env->env_size, src_env->iseq); // Copy after allocations above, since they can move objects in src_ep. - RB_OBJ_WRITE(copied_env, &ep[VM_ENV_DATA_INDEX_ME_CREF], src_ep[VM_ENV_DATA_INDEX_ME_CREF]); + VALUE svar_val = src_ep[VM_ENV_DATA_INDEX_ME_CREF]; + if (imemo_type_p(svar_val, imemo_svar)) { + const struct vm_svar *svar = (struct vm_svar *)svar_val; + + if (svar->cref_or_me) { + svar_val = svar->cref_or_me; + } + else { + svar_val = Qfalse; + } + } + RB_OBJ_WRITE(copied_env, &ep[VM_ENV_DATA_INDEX_ME_CREF], svar_val); + ep[VM_ENV_DATA_INDEX_FLAGS] = src_ep[VM_ENV_DATA_INDEX_FLAGS] | VM_ENV_FLAG_ISOLATED; if (!VM_ENV_LOCAL_P(src_ep)) { VM_ENV_FLAGS_SET(ep, VM_ENV_FLAG_LOCAL); @@ -1283,12 +1440,22 @@ env_copy(const VALUE *src_ep, VALUE read_only_variables) for (int i=RARRAY_LENINT(read_only_variables)-1; i>=0; i--) { ID id = NUM2ID(RARRAY_AREF(read_only_variables, i)); - for (unsigned int j=0; j<ISEQ_BODY(src_env->iseq)->local_table_size; j++) { - if (id == ISEQ_BODY(src_env->iseq)->local_table[j]) { + const struct rb_iseq_constant_body *body = ISEQ_BODY(src_env->iseq); + for (unsigned int j=0; j<body->local_table_size; j++) { + if (id == body->local_table[j]) { + // check reassignment + if (body->lvar_states[j] == lvar_reassigned) { + VALUE name = rb_id2str(id); + VALUE msg = rb_sprintf("cannot make a shareable Proc because " + "the outer variable '%" PRIsVALUE "' may be reassigned.", name); + rb_exc_raise(rb_exc_new_str(rb_eRactorIsolationError, msg)); + } + + // check shareable VALUE v = src_env->env[j]; if (!rb_ractor_shareable_p(v)) { VALUE name = rb_id2str(id); - VALUE msg = rb_sprintf("can not make shareable Proc because it can refer" + VALUE msg = rb_sprintf("cannot make a shareable Proc because it can refer" " unshareable object %+" PRIsVALUE " from ", v); if (name) rb_str_catf(msg, "variable '%" PRIsVALUE "'", name); @@ -1315,6 +1482,7 @@ env_copy(const VALUE *src_ep, VALUE read_only_variables) ep[VM_ENV_DATA_INDEX_SPECVAL] = VM_BLOCK_HANDLER_NONE; } + RB_OBJ_SET_SHAREABLE((VALUE)copied_env); return copied_env; } @@ -1351,22 +1519,28 @@ proc_shared_outer_variables(struct rb_id_table *outer_variables, bool isolate, c } if (*sep == ',') rb_str_cat_cstr(str, ")"); rb_str_cat_cstr(str, data.yield ? " and uses 'yield'." : "."); - rb_exc_raise(rb_exc_new_str(rb_eArgError, str)); + rb_exc_raise(rb_exc_new_str(rb_eRactorIsolationError, str)); } else if (data.yield) { - rb_raise(rb_eArgError, "can not %s because it uses 'yield'.", message); + rb_raise(rb_eRactorIsolationError, "can not %s because it uses 'yield'.", message); } return data.read_only; } VALUE -rb_proc_isolate_bang(VALUE self) +rb_proc_isolate_bang(VALUE self, VALUE replace_self) { const rb_iseq_t *iseq = vm_proc_iseq(self); if (iseq) { rb_proc_t *proc = (rb_proc_t *)RTYPEDDATA_DATA(self); + + if (!UNDEF_P(replace_self)) { + VM_ASSERT(rb_ractor_shareable_p(replace_self)); + RB_OBJ_WRITE(self, &proc->block.as.captured.self, replace_self); + } + if (proc->block.type != block_type_iseq) rb_raise(rb_eRuntimeError, "not supported yet"); if (ISEQ_BODY(iseq)->outer_variables) { @@ -1375,9 +1549,10 @@ rb_proc_isolate_bang(VALUE self) proc_isolate_env(self, proc, Qfalse); proc->is_isolated = TRUE; + RB_OBJ_WRITE(self, &proc->block.as.captured.self, Qnil); } - FL_SET_RAW(self, RUBY_FL_SHAREABLE); + RB_OBJ_SET_SHAREABLE(self); return self; } @@ -1385,17 +1560,22 @@ VALUE rb_proc_isolate(VALUE self) { VALUE dst = rb_proc_dup(self); - rb_proc_isolate_bang(dst); + rb_proc_isolate_bang(dst, Qundef); return dst; } VALUE -rb_proc_ractor_make_shareable(VALUE self) +rb_proc_ractor_make_shareable(VALUE self, VALUE replace_self) { const rb_iseq_t *iseq = vm_proc_iseq(self); if (iseq) { rb_proc_t *proc = (rb_proc_t *)RTYPEDDATA_DATA(self); + + if (!UNDEF_P(replace_self)) { + RB_OBJ_WRITE(self, &proc->block.as.captured.self, replace_self); + } + if (proc->block.type != block_type_iseq) rb_raise(rb_eRuntimeError, "not supported yet"); if (!rb_ractor_shareable_p(vm_block_self(&proc->block))) { @@ -1414,8 +1594,19 @@ rb_proc_ractor_make_shareable(VALUE self) proc_isolate_env(self, proc, read_only_variables); proc->is_isolated = TRUE; } + else { + const struct rb_block *block = vm_proc_block(self); + if (block->type != block_type_symbol) rb_raise(rb_eRuntimeError, "not supported yet"); + + VALUE proc_self = vm_block_self(block); + if (!rb_ractor_shareable_p(proc_self)) { + rb_raise(rb_eRactorIsolationError, + "Proc's self is not shareable: %" PRIsVALUE, + self); + } + } - FL_SET_RAW(self, RUBY_FL_SHAREABLE); + RB_OBJ_SET_FROZEN_SHAREABLE(self); return self; } @@ -1484,8 +1675,8 @@ rb_vm_make_binding(const rb_execution_context_t *ec, const rb_control_frame_t *s GetBindingPtr(bindval, bind); vm_bind_update_env(bindval, bind, envval); RB_OBJ_WRITE(bindval, &bind->block.as.captured.self, cfp->self); - RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, cfp->iseq); - RB_OBJ_WRITE(bindval, &bind->pathobj, ISEQ_BODY(ruby_level_cfp->iseq)->location.pathobj); + RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, CFP_ISEQ(cfp)); + RB_OBJ_WRITE(bindval, &bind->pathobj, ISEQ_BODY(CFP_ISEQ(ruby_level_cfp))->location.pathobj); bind->first_lineno = rb_vm_get_sourceline(ruby_level_cfp); return bindval; @@ -1516,6 +1707,7 @@ rb_binding_add_dynavars(VALUE bindval, rb_binding_t *bind, int dyncount, const I rb_node_init(RNODE(&tmp_node), NODE_SCOPE); tmp_node.nd_tbl = dyns; tmp_node.nd_body = 0; + tmp_node.nd_parent = NULL; tmp_node.nd_args = 0; VALUE ast_value = rb_ruby_ast_new(RNODE(&tmp_node)); @@ -1791,9 +1983,9 @@ rb_vm_invoke_proc_with_self(rb_execution_context_t *ec, rb_proc_t *proc, VALUE s VALUE * rb_vm_svar_lep(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { - while (cfp->pc == 0 || cfp->iseq == 0) { + while (!CFP_PC(cfp) || !CFP_ISEQ(cfp)) { if (VM_FRAME_TYPE(cfp) == VM_FRAME_MAGIC_IFUNC) { - struct vm_ifunc *ifunc = (struct vm_ifunc *)cfp->iseq; + struct vm_ifunc *ifunc = (struct vm_ifunc *)CFP_ISEQ(cfp); return ifunc->svar_lep; } else { @@ -1876,7 +2068,7 @@ rb_sourcefile(void) const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp); if (cfp) { - return RSTRING_PTR(rb_iseq_path(cfp->iseq)); + return RSTRING_PTR(rb_iseq_path(CFP_ISEQ(cfp))); } else { return 0; @@ -1905,7 +2097,7 @@ rb_source_location(int *pline) if (cfp && VM_FRAME_RUBYFRAME_P(cfp)) { if (pline) *pline = rb_vm_get_sourceline(cfp); - return rb_iseq_path(cfp->iseq); + return rb_iseq_path(CFP_ISEQ(cfp)); } else { if (pline) *pline = 0; @@ -2160,7 +2352,7 @@ static void rb_vm_check_redefinition_opt_method(const rb_method_entry_t *me, VALUE klass) { st_data_t bop; - if (RB_TYPE_P(klass, T_ICLASS) && FL_TEST(klass, RICLASS_IS_ORIGIN) && + if (RB_TYPE_P(klass, T_ICLASS) && RICLASS_IS_ORIGIN_P(klass) && RB_TYPE_P(RBASIC_CLASS(klass), T_CLASS)) { klass = RBASIC_CLASS(klass); } @@ -2175,6 +2367,7 @@ rb_vm_check_redefinition_opt_method(const rb_method_entry_t *me, VALUE klass) rb_id2name(me->called_id) ); rb_yjit_bop_redefined(flag, (enum ruby_basic_operators)bop); + rb_zjit_bop_redefined(flag, (enum ruby_basic_operators)bop); ruby_vm_redefined_flag[bop] |= flag; } } @@ -2255,6 +2448,7 @@ vm_init_redefined_flag(void) OP(GT, GT), (C(Integer), C(Float)); OP(GE, GE), (C(Integer), C(Float)); OP(LTLT, LTLT), (C(String), C(Array)); + OP(GTGT, GTGT), (C(Integer)); OP(AREF, AREF), (C(Array), C(Hash), C(Integer)); OP(ASET, ASET), (C(Array), C(Hash)); OP(Length, LENGTH), (C(Array), C(String), C(Hash)); @@ -2385,8 +2579,12 @@ hook_before_rewind(rb_execution_context_t *ec, bool cfp_returning_with_value, in return; } else { - const rb_iseq_t *iseq = ec->cfp->iseq; - rb_hook_list_t *local_hooks = iseq->aux.exec.local_hooks; + const rb_iseq_t *iseq = CFP_ISEQ(ec->cfp); + rb_hook_list_t *local_hooks = NULL; + unsigned int local_hooks_cnt = iseq->aux.exec.local_hooks_cnt; + if (RB_UNLIKELY(local_hooks_cnt > 0)) { + local_hooks = rb_iseq_local_hooks(iseq, rb_ec_ractor_ptr(ec), false); + } switch (VM_FRAME_TYPE(ec->cfp)) { case VM_FRAME_MAGIC_METHOD: @@ -2424,15 +2622,18 @@ hook_before_rewind(rb_execution_context_t *ec, bool cfp_returning_with_value, in bmethod_return_value); VM_ASSERT(me->def->type == VM_METHOD_TYPE_BMETHOD); - local_hooks = me->def->body.bmethod.hooks; - - if (UNLIKELY(local_hooks && local_hooks->events & RUBY_EVENT_RETURN)) { - rb_exec_event_hook_orig(ec, local_hooks, RUBY_EVENT_RETURN, ec->cfp->self, - rb_vm_frame_method_entry(ec->cfp)->def->original_id, - rb_vm_frame_method_entry(ec->cfp)->called_id, - rb_vm_frame_method_entry(ec->cfp)->owner, - bmethod_return_value, TRUE); + unsigned int local_hooks_cnt = me->def->body.bmethod.local_hooks_cnt; + if (UNLIKELY(local_hooks_cnt > 0)) { + local_hooks = rb_method_def_local_hooks(me->def, rb_ec_ractor_ptr(ec), false); + if (local_hooks && local_hooks->events & RUBY_EVENT_RETURN) { + rb_exec_event_hook_orig(ec, local_hooks, RUBY_EVENT_RETURN, ec->cfp->self, + rb_vm_frame_method_entry(ec->cfp)->def->original_id, + rb_vm_frame_method_entry(ec->cfp)->called_id, + rb_vm_frame_method_entry(ec->cfp)->owner, + bmethod_return_value, TRUE); + } } + THROW_DATA_CONSUMED_SET(err); } else { @@ -2631,6 +2832,31 @@ vm_exec_loop(rb_execution_context_t *ec, enum ruby_tag_type state, return result; } +#if USE_ZJIT +// Materialize JITFrame-enabled CFP into interpreter-compatible CFP +void +rb_zjit_materialize_frames(const rb_execution_context_t *ec, rb_control_frame_t *cfp) +{ + if (!rb_zjit_enabled_p) return; + const rb_control_frame_t *end_cfp = ec->tag->cfp; + VM_ASSERT(cfp <= end_cfp); + + while (true) { + if (CFP_ZJIT_FRAME_P(cfp)) { + const zjit_jit_frame_t *jit_frame = CFP_ZJIT_FRAME(cfp); + cfp->pc = jit_frame->pc; + cfp->_iseq = (rb_iseq_t *)jit_frame->iseq; + if (jit_frame->materialize_block_code) { + cfp->block_code = NULL; + } + cfp->jit_return = 0; + } + if (end_cfp == cfp) break; + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + } +} +#endif + static inline VALUE vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, VALUE errinfo) { @@ -2648,7 +2874,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V cont_pc = cont_sp = 0; catch_iseq = NULL; - while (ec->cfp->pc == 0 || ec->cfp->iseq == 0) { + while (CFP_PC(ec->cfp) == 0 || CFP_ISEQ(ec->cfp) == 0) { if (UNLIKELY(VM_FRAME_TYPE(ec->cfp) == VM_FRAME_MAGIC_CFUNC)) { EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_C_RETURN, ec->cfp->self, rb_vm_frame_method_entry(ec->cfp)->def->original_id, @@ -2662,7 +2888,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } rb_control_frame_t *const cfp = ec->cfp; - epc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded; + epc = CFP_PC(cfp) - ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded; escape_cfp = NULL; if (state == TAG_BREAK || state == TAG_RETURN) { @@ -2675,7 +2901,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V THROW_DATA_STATE_SET(err, state = TAG_BREAK); } else { - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -2702,13 +2928,14 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V /* TAG_BREAK */ *cfp->sp++ = THROW_DATA_VAL(err); ec->errinfo = Qnil; + rb_zjit_materialize_frames(ec, cfp); return Qundef; } } } if (state == TAG_RAISE) { - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -2724,7 +2951,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } } else if (state == TAG_RETRY) { - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -2739,7 +2966,8 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V const rb_control_frame_t *escape_cfp; escape_cfp = THROW_DATA_CATCH_FRAME(err); if (cfp == escape_cfp) { - cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + entry->cont; + rb_zjit_materialize_frames(ec, cfp); + cfp->pc = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded + entry->cont; ec->errinfo = Qnil; return Qundef; } @@ -2757,7 +2985,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V /* otherwise = dontcare */ }[state]; - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); @@ -2769,7 +2997,8 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V break; } else if (entry->type == type) { - cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + entry->cont; + rb_zjit_materialize_frames(ec, cfp); + cfp->pc = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded + entry->cont; cfp->sp = vm_base_ptr(cfp) + entry->sp; if (state != TAG_REDO) { @@ -2783,7 +3012,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } } else { - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -2803,8 +3032,9 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V const int arg_size = 1; rb_iseq_check(catch_iseq); + rb_zjit_materialize_frames(ec, cfp); // vm_base_ptr looks at cfp->_iseq cfp->sp = vm_base_ptr(cfp) + cont_sp; - cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + cont_pc; + cfp->pc = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded + cont_pc; /* push block frame */ cfp->sp[0] = (VALUE)err; @@ -2829,6 +3059,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V if (VM_FRAME_FINISHED_P(ec->cfp)) { rb_vm_pop_frame(ec); ec->errinfo = (VALUE)err; + rb_vm_tag_jmpbuf_deinit(&ec->tag->buf); ec->tag = ec->tag->prev; EC_JUMP_TAG(ec, state); } @@ -2842,11 +3073,11 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V /* misc */ VALUE -rb_iseq_eval(const rb_iseq_t *iseq) +rb_iseq_eval(const rb_iseq_t *iseq, const rb_box_t *box) { rb_execution_context_t *ec = GET_EC(); VALUE val; - vm_set_top_stack(ec, iseq); + vm_set_top_stack(ec, iseq, box); val = vm_exec(ec); return val; } @@ -2856,7 +3087,6 @@ rb_iseq_eval_main(const rb_iseq_t *iseq) { rb_execution_context_t *ec = GET_EC(); VALUE val; - vm_set_main_stack(ec, iseq); val = vm_exec(ec); return val; @@ -2897,10 +3127,11 @@ rb_vm_call_cfunc(VALUE recv, VALUE (*func)(VALUE), VALUE arg, rb_execution_context_t *ec = GET_EC(); const rb_control_frame_t *reg_cfp = ec->cfp; const rb_iseq_t *iseq = rb_iseq_new(Qnil, filename, filename, Qnil, 0, ISEQ_TYPE_TOP); + const rb_box_t *box = rb_current_box(); VALUE val; vm_push_frame(ec, iseq, VM_FRAME_MAGIC_TOP | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH, - recv, block_handler, + recv, GC_GUARDED_PTR(box), (VALUE)vm_cref_new_toplevel(ec), /* cref or me */ 0, reg_cfp->sp, 0, 0); @@ -2910,6 +3141,117 @@ rb_vm_call_cfunc(VALUE recv, VALUE (*func)(VALUE), VALUE arg, return val; } +/* Ruby::Box */ + +VALUE +rb_vm_call_cfunc_in_box(VALUE recv, VALUE (*func)(VALUE, VALUE), VALUE arg1, VALUE arg2, + VALUE filename, const rb_box_t *box) +{ + rb_execution_context_t *ec = GET_EC(); + const rb_control_frame_t *reg_cfp = ec->cfp; + const rb_iseq_t *iseq = rb_iseq_new(Qnil, filename, filename, Qnil, 0, ISEQ_TYPE_TOP); + VALUE val; + + vm_push_frame(ec, iseq, VM_FRAME_MAGIC_TOP | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH, + recv, GC_GUARDED_PTR(box), + (VALUE)vm_cref_new_toplevel(ec), /* cref or me */ + 0, reg_cfp->sp, 0, 0); + + val = (*func)(arg1, arg2); + + rb_vm_pop_frame(ec); + return val; +} + +void +rb_vm_frame_flag_set_box_require(const rb_execution_context_t *ec) +{ + VM_ASSERT(rb_box_available()); + VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_BOX_REQUIRE); +} + +static const rb_box_t * +current_box_on_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) +{ + rb_callable_method_entry_t *cme; + const rb_box_t *box; + const VALUE *lep = VM_EP_RUBY_LEP(ec, cfp); + VM_BOX_ASSERT(lep, "lep should be valid"); + VM_BOX_ASSERT(rb_box_available(), "box should be available here"); + + if (VM_ENV_FRAME_TYPE_P(lep, VM_FRAME_MAGIC_METHOD) || VM_ENV_FRAME_TYPE_P(lep, VM_FRAME_MAGIC_CFUNC)) { + cme = check_method_entry(lep[VM_ENV_DATA_INDEX_ME_CREF], TRUE); + VM_BOX_ASSERT(cme, "cme should be valid"); + VM_BOX_ASSERT(cme->def, "cme->def shold be valid"); + return cme->def->box; + } + else if (VM_ENV_FRAME_TYPE_P(lep, VM_FRAME_MAGIC_TOP) || VM_ENV_FRAME_TYPE_P(lep, VM_FRAME_MAGIC_CLASS)) { + VM_BOX_ASSERT(VM_ENV_LOCAL_P(lep), "lep should be local on MAGIC_TOP or MAGIC_CLASS frames"); + return VM_ENV_BOX(lep); + } + else if (VM_ENV_FRAME_TYPE_P(lep, VM_FRAME_MAGIC_DUMMY)) { + // No valid local ep found (just after process boot?) + // return the root box (the only valid box) until the main is initialized + box = rb_main_box(); + if (box) + return box; + return rb_root_box(); + } + else { + VM_BOX_CRASHED(); + rb_bug("BUG: Local ep without cme/box, flags: %08lX", (unsigned long)lep[VM_ENV_DATA_INDEX_FLAGS]); + } + UNREACHABLE_RETURN(0); +} + +const rb_box_t * +rb_vm_current_box(const rb_execution_context_t *ec) +{ + return current_box_on_cfp(ec, ec->cfp); +} + +static const rb_control_frame_t * +find_loader_control_frame(const rb_execution_context_t *ec, const rb_control_frame_t *cfp, const rb_control_frame_t *end_cfp) +{ + while (RUBY_VM_VALID_CONTROL_FRAME_P(cfp, end_cfp)) { + if (!VM_ENV_FRAME_TYPE_P(cfp->ep, VM_FRAME_MAGIC_CFUNC)) + break; + if (!BOX_MASTER_P(current_box_on_cfp(ec, cfp))) + break; + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + } + VM_ASSERT(RUBY_VM_VALID_CONTROL_FRAME_P(cfp, end_cfp)); + return cfp; +} + +const rb_box_t * +rb_vm_loading_box(const rb_execution_context_t *ec) +{ + const rb_control_frame_t *cfp, *current_cfp, *end_cfp; + + if (!rb_box_available() || !ec) + return rb_root_box(); + + cfp = ec->cfp; + current_cfp = cfp; + end_cfp = RUBY_VM_END_CONTROL_FRAME(ec); + + while (RUBY_VM_VALID_CONTROL_FRAME_P(cfp, end_cfp)) { + if (VM_ENV_FLAGS(cfp->ep, VM_FRAME_FLAG_BOX_REQUIRE)) { + if (RTEST(cfp->self) && BOX_OBJ_P(cfp->self)) { + // Box#require, #require_relative, #load + return rb_get_box_t(cfp->self); + } + // Kernel#require, #require_relative, #load + cfp = find_loader_control_frame(ec, cfp, end_cfp); + return current_box_on_cfp(ec, cfp); + } + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + } + // no require/load with explicit boxes. + return current_box_on_cfp(ec, current_cfp); +} + /* vm */ void @@ -2918,21 +3260,15 @@ rb_vm_update_references(void *ptr) if (ptr) { rb_vm_t *vm = ptr; + vm->self = rb_gc_location(vm->self); vm->mark_object_ary = rb_gc_location(vm->mark_object_ary); - vm->load_path = rb_gc_location(vm->load_path); - vm->load_path_snapshot = rb_gc_location(vm->load_path_snapshot); - - if (vm->load_path_check_cache) { - vm->load_path_check_cache = rb_gc_location(vm->load_path_check_cache); - } - - vm->expanded_load_path = rb_gc_location(vm->expanded_load_path); - vm->loaded_features = rb_gc_location(vm->loaded_features); - vm->loaded_features_snapshot = rb_gc_location(vm->loaded_features_snapshot); - vm->loaded_features_realpaths = rb_gc_location(vm->loaded_features_realpaths); - vm->loaded_features_realpath_map = rb_gc_location(vm->loaded_features_realpath_map); - vm->top_self = rb_gc_location(vm->top_self); vm->orig_progname = rb_gc_location(vm->orig_progname); + vm->cc_refinement_set = rb_gc_location(vm->cc_refinement_set); + + if (vm->root_box) + rb_box_gc_update_references(vm->root_box); + if (vm->main_box) + rb_box_gc_update_references(vm->main_box); rb_gc_update_values(RUBY_NSIG, vm->trap_list.cmd); @@ -2999,32 +3335,31 @@ rb_vm_mark(void *ptr) rb_gc_mark(rb_ractor_self(r)); } - for (struct global_object_list *list = vm->global_object_list; list; list = list->next) { - rb_gc_mark_maybe(*list->varptr); + for (size_t index = 0; index < vm->global_object_list_size; index++) { + rb_gc_mark_maybe(*vm->global_object_list[index]); + } + + rb_gc_mark_movable(vm->self); + + if (vm->root_box) { + rb_box_entry_mark(vm->root_box); + } + if (vm->main_box) { + rb_box_entry_mark(vm->main_box); } rb_gc_mark_movable(vm->mark_object_ary); - rb_gc_mark_movable(vm->load_path); - rb_gc_mark_movable(vm->load_path_snapshot); - rb_gc_mark_movable(vm->load_path_check_cache); - rb_gc_mark_movable(vm->expanded_load_path); - rb_gc_mark_movable(vm->loaded_features); - rb_gc_mark_movable(vm->loaded_features_snapshot); - rb_gc_mark_movable(vm->loaded_features_realpaths); - rb_gc_mark_movable(vm->loaded_features_realpath_map); - rb_gc_mark_movable(vm->top_self); rb_gc_mark_movable(vm->orig_progname); rb_gc_mark_movable(vm->coverages); rb_gc_mark_movable(vm->me2counter); - - if (vm->loading_table) { - rb_mark_tbl(vm->loading_table); - } + rb_gc_mark_movable(vm->cc_refinement_set); rb_gc_mark_values(RUBY_NSIG, vm->trap_list.cmd); - rb_id_table_foreach_values(vm->negative_cme_table, vm_mark_negative_cme, NULL); - rb_mark_tbl_no_pin(vm->overloaded_cme_table); + rb_hook_list_mark(&vm->global_hooks); + + rb_id_table_foreach_values(&vm->negative_cme_table, vm_mark_negative_cme, NULL); + rb_mark_tbl_no_pin(&vm->overloaded_cme_table); for (i=0; i<VM_GLOBAL_CC_CACHE_TABLE_SIZE; i++) { const struct rb_callcache *cc = vm->global_cc_cache_table[i]; @@ -3055,62 +3390,41 @@ rb_vm_register_special_exception_str(enum ruby_special_exceptions sp, VALUE cls, rb_vm_register_global_object(exc); } -static int -free_loading_table_entry(st_data_t key, st_data_t value, st_data_t arg) -{ - xfree((char *)key); - return ST_DELETE; -} - -void rb_free_loaded_features_index(rb_vm_t *vm); void rb_objspace_free_objects(void *objspace); int ruby_vm_destruct(rb_vm_t *vm) { RUBY_FREE_ENTER("vm"); + ruby_vm_during_cleanup = true; if (vm) { rb_thread_t *th = vm->ractor.main_thread; - VALUE *stack = th->ec->vm_stack; + if (rb_free_at_exit) { rb_free_encoded_insn_data(); rb_free_global_enc_table(); rb_free_loaded_builtin_table(); + rb_free_global_symbol_table(); rb_free_shared_fiber_pool(); - rb_free_static_symid_str(); rb_free_transcoder_table(); rb_free_vm_opt_tables(); rb_free_warning(); rb_free_rb_global_tbl(); - rb_free_loaded_features_index(vm); - rb_id_table_free(vm->negative_cme_table); - st_free_table(vm->overloaded_cme_table); + rb_id_table_free_items(&vm->negative_cme_table); + st_free_embedded_table(&vm->overloaded_cme_table); - rb_id_table_free(RCLASS(rb_mRubyVMFrozenCore)->m_tbl); + // TODO: Is this ignorable for classext->m_tbl ? + // rb_id_table_free(RCLASS(rb_mRubyVMFrozenCore)->m_tbl); - rb_shape_t *cursor = rb_shape_get_root_shape(); - rb_shape_t *end = rb_shape_get_shape_by_id(GET_SHAPE_TREE()->next_shape_id); - while (cursor < end) { - // 0x1 == SINGLE_CHILD_P - if (cursor->edges && !(((uintptr_t)cursor->edges) & 0x1)) - rb_id_table_free(cursor->edges); - cursor += 1; - } - - xfree(GET_SHAPE_TREE()); - - st_free_table(vm->static_ext_inits); - - rb_vm_postponed_job_free(); + st_free_embedded_table(&vm->static_ext_inits); - rb_id_table_free(vm->constant_cache); - st_free_table(vm->unused_block_warning_table); + rb_id_table_free_items(&vm->constant_cache); + set_free_embedded_table(&vm->unused_block_warning_table); - xfree(th->nt); - th->nt = NULL; + rb_thread_free_native_thread(th); #ifndef HAVE_SETPROCTITLE ruby_free_proctitle(); @@ -3125,56 +3439,33 @@ ruby_vm_destruct(rb_vm_t *vm) rb_vm_living_threads_init(vm); ruby_vm_run_at_exit_hooks(vm); - if (vm->loading_table) { - st_foreach(vm->loading_table, free_loading_table_entry, 0); - st_free_table(vm->loading_table); - vm->loading_table = 0; - } - if (vm->ci_table) { - st_free_table(vm->ci_table); - vm->ci_table = NULL; - } - if (vm->frozen_strings) { - st_free_table(vm->frozen_strings); - vm->frozen_strings = 0; - } + st_free_embedded_table(&vm->ci_table); RB_ALTSTACK_FREE(vm->main_altstack); - struct global_object_list *next; - for (struct global_object_list *list = vm->global_object_list; list; list = next) { - next = list->next; - xfree(list); - } + SIZED_FREE_N(vm->global_object_list, vm->global_object_list_capa); if (objspace) { if (rb_free_at_exit) { rb_objspace_free_objects(objspace); - rb_free_generic_iv_tbl_(); + rb_free_generic_fields_tbl_(); rb_free_default_rand_key(); - if (th && vm->fork_gen == 0) { - /* If we have forked, main_thread may not be the initial thread */ - xfree(stack); - ruby_mimfree(th); - } } rb_objspace_free(objspace); } rb_native_mutex_destroy(&vm->workqueue_lock); /* after freeing objspace, you *can't* use ruby_xfree() */ - ruby_mimfree(vm); ruby_current_vm_ptr = NULL; -#if USE_YJIT if (rb_free_at_exit) { +#if USE_YJIT rb_yjit_free_at_exit(); - } #endif + } } RUBY_FREE_LEAVE("vm"); return 0; } -size_t rb_vm_memsize_waiting_fds(struct ccan_list_head *waiting_fds); // thread.c size_t rb_vm_memsize_workqueue(struct ccan_list_head *workqueue); // vm_trace.c // Used for VM memsize reporting. Returns the size of the at_exit list by @@ -3182,7 +3473,7 @@ size_t rb_vm_memsize_workqueue(struct ccan_list_head *workqueue); // vm_trace.c static enum rb_id_table_iterator_result vm_memsize_constant_cache_i(ID id, VALUE ics, void *size) { - *((size_t *) size) += rb_st_memsize((st_table *) ics); + *((size_t *) size) += rb_set_memsize((set_table *) ics); return ID_TABLE_CONTINUE; } @@ -3193,9 +3484,9 @@ static size_t vm_memsize_constant_cache(void) { rb_vm_t *vm = GET_VM(); - size_t size = rb_id_table_memsize(vm->constant_cache); + size_t size = rb_id_table_memsize(&vm->constant_cache) - sizeof(struct rb_id_table); - rb_id_table_foreach(vm->constant_cache, vm_memsize_constant_cache_i, &size); + rb_id_table_foreach(&vm->constant_cache, vm_memsize_constant_cache_i, &size); return size; } @@ -3229,19 +3520,15 @@ vm_memsize(const void *ptr) return ( sizeof(rb_vm_t) + - rb_vm_memsize_waiting_fds(&vm->waiting_fds) + - rb_st_memsize(vm->loaded_features_index) + - rb_st_memsize(vm->loading_table) + rb_vm_memsize_postponed_job_queue() + rb_vm_memsize_workqueue(&vm->workqueue) + vm_memsize_at_exit_list(vm->at_exit) + - rb_st_memsize(vm->ci_table) + - rb_st_memsize(vm->frozen_strings) + + (rb_st_memsize(&vm->ci_table) - sizeof(struct st_table)) + vm_memsize_builtin_function_table(vm->builtin_function_table) + - rb_id_table_memsize(vm->negative_cme_table) + - rb_st_memsize(vm->overloaded_cme_table) + - vm_memsize_constant_cache() + - GET_SHAPE_TREE()->cache_size * sizeof(redblack_node_t) + (rb_id_table_memsize(&vm->negative_cme_table) - sizeof(struct rb_id_table)) + + (rb_st_memsize(&vm->overloaded_cme_table) - sizeof(struct st_table)) + + (vm->global_object_list_capa * sizeof(*vm->global_object_list)) + + vm_memsize_constant_cache() ); // TODO @@ -3250,12 +3537,13 @@ vm_memsize(const void *ptr) // struct rb_objspace *objspace; } -static const rb_data_type_t vm_data_type = { +const rb_data_type_t ruby_vm_data_type = { "VM", {0, 0, vm_memsize,}, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY }; +#define vm_data_type ruby_vm_data_type static VALUE vm_default_params(void) @@ -3362,22 +3650,30 @@ rb_execution_context_update(rb_execution_context_t *ec) } while (cfp != limit_cfp) { - if (VM_FRAME_TYPE(cfp) != VM_FRAME_MAGIC_DUMMY) { - const VALUE *ep = cfp->ep; - cfp->self = rb_gc_location(cfp->self); - cfp->iseq = (rb_iseq_t *)rb_gc_location((VALUE)cfp->iseq); + const VALUE *ep = cfp->ep; + cfp->self = rb_gc_location(cfp->self); + if (CFP_ZJIT_FRAME_P(cfp)) { + rb_zjit_jit_frame_update_references((zjit_jit_frame_t *)CFP_ZJIT_FRAME(cfp)); + // block_code must always be relocated. For ISEQ frames, the JIT caller + // may have written it (gen_block_handler_specval) for passing blocks. + // For C frames, rb_iterate0 may have written an ifunc to block_code + // after the JIT pushed the frame. NULL is safe to pass to rb_gc_location. cfp->block_code = (void *)rb_gc_location((VALUE)cfp->block_code); + } + else { + cfp->_iseq = (rb_iseq_t *)rb_gc_location((VALUE)cfp->_iseq); + cfp->block_code = (void *)rb_gc_location((VALUE)cfp->block_code); + } - if (!VM_ENV_LOCAL_P(ep)) { - const VALUE *prev_ep = VM_ENV_PREV_EP(ep); - if (VM_ENV_FLAGS(prev_ep, VM_ENV_FLAG_ESCAPED)) { - VM_FORCE_WRITE(&prev_ep[VM_ENV_DATA_INDEX_ENV], rb_gc_location(prev_ep[VM_ENV_DATA_INDEX_ENV])); - } + if (!VM_ENV_LOCAL_P(ep)) { + const VALUE *prev_ep = VM_ENV_PREV_EP(ep); + if (VM_ENV_FLAGS(prev_ep, VM_ENV_FLAG_ESCAPED)) { + VM_FORCE_WRITE(&prev_ep[VM_ENV_DATA_INDEX_ENV], rb_gc_location(prev_ep[VM_ENV_DATA_INDEX_ENV])); + } - if (VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED)) { - VM_FORCE_WRITE(&ep[VM_ENV_DATA_INDEX_ENV], rb_gc_location(ep[VM_ENV_DATA_INDEX_ENV])); - VM_FORCE_WRITE(&ep[VM_ENV_DATA_INDEX_ME_CREF], rb_gc_location(ep[VM_ENV_DATA_INDEX_ME_CREF])); - } + if (VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED)) { + VM_FORCE_WRITE(&ep[VM_ENV_DATA_INDEX_ENV], rb_gc_location(ep[VM_ENV_DATA_INDEX_ENV])); + VM_FORCE_WRITE(&ep[VM_ENV_DATA_INDEX_ME_CREF], rb_gc_location(ep[VM_ENV_DATA_INDEX_ME_CREF])); } } @@ -3386,6 +3682,9 @@ rb_execution_context_update(rb_execution_context_t *ec) } ec->storage = rb_gc_location(ec->storage); + + ec->gen_fields_cache.obj = rb_gc_location(ec->gen_fields_cache.obj); + ec->gen_fields_cache.fields_obj = rb_gc_location(ec->gen_fields_cache.fields_obj); } static enum rb_id_table_iterator_result @@ -3406,28 +3705,36 @@ rb_execution_context_mark(const rb_execution_context_t *ec) rb_control_frame_t *cfp = ec->cfp; rb_control_frame_t *limit_cfp = (void *)(ec->vm_stack + ec->vm_stack_size); - VM_ASSERT(sp == ec->cfp->sp); - rb_gc_mark_vm_stack_values((long)(sp - p), p); + for (long i = 0; i < (long)(sp - p); i++) { + rb_gc_mark_movable(p[i]); + } while (cfp != limit_cfp) { const VALUE *ep = cfp->ep; VM_ASSERT(!!VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED) == vm_ep_in_heap_p_(ec, ep)); - if (VM_FRAME_TYPE(cfp) != VM_FRAME_MAGIC_DUMMY) { - rb_gc_mark_movable(cfp->self); - rb_gc_mark_movable((VALUE)cfp->iseq); - rb_gc_mark_movable((VALUE)cfp->block_code); + rb_gc_mark_movable(cfp->self); + rb_gc_mark_movable((VALUE)CFP_ISEQ(cfp)); + // Mark block_code directly (not through rb_zjit_cfp_block_code) + // because rb_iterate0 may write a valid ifunc after JIT frame push. + rb_gc_mark_movable((VALUE)cfp->block_code); - if (!VM_ENV_LOCAL_P(ep)) { - const VALUE *prev_ep = VM_ENV_PREV_EP(ep); - if (VM_ENV_FLAGS(prev_ep, VM_ENV_FLAG_ESCAPED)) { - rb_gc_mark_movable(prev_ep[VM_ENV_DATA_INDEX_ENV]); - } + if (VM_ENV_LOCAL_P(ep) && VM_ENV_BOXED_P(ep)) { + const rb_box_t *box = VM_ENV_BOX(ep); + if (BOX_USER_P(box)) { + rb_gc_mark_movable(box->box_object); + } + } - if (VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED)) { - rb_gc_mark_movable(ep[VM_ENV_DATA_INDEX_ENV]); - rb_gc_mark(ep[VM_ENV_DATA_INDEX_ME_CREF]); - } + if (!VM_ENV_LOCAL_P(ep)) { + const VALUE *prev_ep = VM_ENV_PREV_EP(ep); + if (VM_ENV_FLAGS(prev_ep, VM_ENV_FLAG_ESCAPED)) { + rb_gc_mark_movable(prev_ep[VM_ENV_DATA_INDEX_ENV]); + } + + if (VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED)) { + rb_gc_mark_movable(ep[VM_ENV_DATA_INDEX_ENV]); + rb_gc_mark(ep[VM_ENV_DATA_INDEX_ME_CREF]); } } @@ -3437,8 +3744,8 @@ rb_execution_context_mark(const rb_execution_context_t *ec) /* mark machine stack */ if (ec->machine.stack_start && ec->machine.stack_end && - ec != GET_EC() /* marked for current ec at the first stage of marking */ - ) { + /* marked for current ec at the first stage of marking */ + ec != rb_gc_get_ec()) { rb_gc_mark_machine_context(ec); } @@ -3457,6 +3764,7 @@ rb_execution_context_mark(const rb_execution_context_t *ec) void rb_fiber_mark_self(rb_fiber_t *fib); void rb_fiber_update_self(rb_fiber_t *fib); void rb_threadptr_root_fiber_setup(rb_thread_t *th); +void rb_root_fiber_obj_setup(rb_thread_t *th); void rb_threadptr_root_fiber_release(rb_thread_t *th); static void @@ -3465,10 +3773,6 @@ thread_compact(void *ptr) rb_thread_t *th = ptr; th->self = rb_gc_location(th->self); - - if (!th->root_fiber) { - rb_execution_context_update(th->ec); - } } static void @@ -3476,7 +3780,11 @@ thread_mark(void *ptr) { rb_thread_t *th = ptr; RUBY_MARK_ENTER("thread"); - rb_fiber_mark_self(th->ec->fiber_ptr); + + // ec is null when setting up the thread in rb_threadptr_root_fiber_setup + if (th->ec) { + rb_fiber_mark_self(th->ec->fiber_ptr); + } /* mark ruby objects */ switch (th->invoke_type) { @@ -3501,8 +3809,7 @@ thread_mark(void *ptr) rb_gc_mark(th->top_wrapper); if (th->root_fiber) rb_fiber_mark_self(th->root_fiber); - RUBY_ASSERT(th->ec == rb_fiberptr_get_ec(th->ec->fiber_ptr)); - rb_gc_mark(th->stat_insn_usage); + RUBY_ASSERT(th->ec == NULL || th->ec == rb_fiberptr_get_ec(th->ec->fiber_ptr)); rb_gc_mark(th->last_status); rb_gc_mark(th->locking_mutex); rb_gc_mark(th->name); @@ -3533,15 +3840,15 @@ thread_free(void *ptr) ruby_xfree(th->specific_storage); - rb_threadptr_root_fiber_release(th); - if (th->vm && th->vm->ractor.main_thread == th) { RUBY_GC_INFO("MRI main thread\n"); } else { // ruby_xfree(th->nt); // TODO: MN system collect nt, but without MN system it should be freed here. - ruby_xfree(th); + if (!th->main_thread) { + ruby_xfree(th); + } } RUBY_FREE_LEAVE("thread"); @@ -3617,17 +3924,28 @@ rb_ec_initialize_vm_stack(rb_execution_context_t *ec, VALUE *stack, size_t size) void rb_ec_clear_vm_stack(rb_execution_context_t *ec) { + // set cfp to NULL before clearing the stack in case `thread_profile_frames` + // gets called in this middle of `rb_ec_set_vm_stack` via signal handler. + ec->cfp = NULL; rb_ec_set_vm_stack(ec, NULL, 0); +} - // Avoid dangling pointers: - ec->cfp = NULL; +void +rb_ec_close(rb_execution_context_t *ec) +{ + // Fiber storage is not accessible from outside the running fiber, so it is safe to clear it here. + ec->storage = Qnil; } static void th_init(rb_thread_t *th, VALUE self, rb_vm_t *vm) { + const rb_box_t *box = rb_current_box(); + th->self = self; + ccan_list_head_init(&th->interrupt_exec_tasks); + rb_threadptr_root_fiber_setup(th); /* All threads are blocking until a non-blocking fiber is scheduled */ @@ -3636,7 +3954,9 @@ th_init(rb_thread_t *th, VALUE self, rb_vm_t *vm) if (self == 0) { size_t size = vm->default_params.thread_vm_stack_size / sizeof(VALUE); - rb_ec_initialize_vm_stack(th->ec, ALLOC_N(VALUE, size), size); + VALUE *stack = ALLOC_N(VALUE, size); + rb_ec_initialize_vm_stack(th->ec, stack, size); + rb_thread_malloc_stack_set(th, stack, size); } else { VM_ASSERT(th->ec->cfp == NULL); @@ -3647,7 +3967,12 @@ th_init(rb_thread_t *th, VALUE self, rb_vm_t *vm) th->status = THREAD_RUNNABLE; th->last_status = Qnil; th->top_wrapper = 0; - th->top_self = vm->top_self; // 0 while self == 0 + if (box->top_self) { + th->top_self = box->top_self; + } + else { + th->top_self = 0; + } th->value = Qundef; th->ec->errinfo = Qnil; @@ -3656,6 +3981,7 @@ th_init(rb_thread_t *th, VALUE self, rb_vm_t *vm) th->ec->local_storage_recursive_hash_for_trace = Qnil; th->ec->storage = Qnil; + th->ec->ractor_id = rb_ractor_id(th->ractor); #if OPT_CALL_THREADED_CODE th->retval = Qundef; @@ -3664,8 +3990,6 @@ th_init(rb_thread_t *th, VALUE self, rb_vm_t *vm) th->report_on_exception = vm->thread_report_on_exception; th->ext_config.ractor_safe = true; - ccan_list_head_init(&th->interrupt_exec_tasks); - #if USE_RUBY_DEBUG_LOG static rb_atomic_t thread_serial = 1; th->serial = RUBY_ATOMIC_FETCH_ADD(thread_serial, 1); @@ -3681,6 +4005,7 @@ rb_thread_alloc(VALUE klass) rb_thread_t *target_th = rb_thread_ptr(self); target_th->ractor = GET_RACTOR(); th_init(target_th, self, target_th->vm = GET_VM()); + rb_root_fiber_obj_setup(target_th); return self; } @@ -3952,9 +4277,7 @@ Init_VM(void) /* FrozenCore (hidden) */ fcore = rb_class_new(rb_cBasicObject); rb_set_class_path(fcore, rb_cRubyVM, "FrozenCore"); - rb_vm_register_global_object(rb_class_path_cached(fcore)); - RB_FL_UNSET_RAW(fcore, T_MASK); - RB_FL_SET_RAW(fcore, T_ICLASS); + rb_vm_register_global_object(rb_mod_name(fcore)); klass = rb_singleton_class(fcore); rb_define_method_id(klass, id_core_set_method_alias, m_core_set_method_alias, 3); rb_define_method_id(klass, id_core_set_variable_alias, m_core_set_variable_alias, 2); @@ -4230,8 +4553,10 @@ Init_VM(void) th->top_wrapper = 0; th->top_self = rb_vm_top_self(); + rb_root_fiber_obj_setup(th); + rb_vm_register_global_object((VALUE)iseq); - th->ec->cfp->iseq = iseq; + th->ec->cfp->_iseq = iseq; th->ec->cfp->pc = ISEQ_BODY(iseq)->iseq_encoded; th->ec->cfp->self = th->top_self; @@ -4267,42 +4592,44 @@ rb_vm_set_progname(VALUE filename) --cfp; filename = rb_str_new_frozen(filename); - rb_iseq_pathobj_set(cfp->iseq, filename, rb_iseq_realpath(cfp->iseq)); + rb_iseq_pathobj_set(CFP_ISEQ(cfp), filename, rb_iseq_realpath(CFP_ISEQ(cfp))); } extern const struct st_hash_type rb_fstring_hash_type; +static rb_vm_t _vm; +static rb_thread_t _main_thread = { + .vm = &_vm, + .main_thread = 1, +}; + void Init_BareVM(void) { /* VM bootstrap: phase 1 */ - rb_vm_t *vm = ruby_mimcalloc(1, sizeof(*vm)); - rb_thread_t *th = ruby_mimcalloc(1, sizeof(*th)); - if (!vm || !th) { - fputs("[FATAL] failed to allocate memory\n", stderr); - exit(EXIT_FAILURE); - } + rb_vm_t *vm = &_vm; + rb_thread_t *th = &_main_thread; // setup the VM vm_init2(vm); - rb_vm_postponed_job_queue_init(vm); ruby_current_vm_ptr = vm; rb_objspace_alloc(); - vm->negative_cme_table = rb_id_table_create(16); - vm->overloaded_cme_table = st_init_numtable(); - vm->constant_cache = rb_id_table_create(0); - vm->unused_block_warning_table = st_init_numtable(); + rb_id_table_init(&vm->negative_cme_table, 16); + st_init_existing_numtable_with_size(&vm->overloaded_cme_table, 0); + st_init_existing_strtable_with_size(&vm->static_ext_inits, 0); + set_init_embedded_numtable_with_size(&vm->unused_block_warning_table, 0); + vm->global_hooks.type = hook_list_type_global; // setup main thread th->nt = ZALLOC(struct rb_native_thread); - th->vm = vm; th->ractor = vm->ractor.main_ractor = rb_ractor_main_alloc(); Init_native_thread(th); rb_jit_cont_init(); th_init(th, 0, vm); rb_ractor_set_current_ec(th->ractor, th->ec); + /* n.b. native_main_thread_stack_top is set by the INIT_STACK macro */ ruby_thread_init_stack(th, native_main_thread_stack_top); @@ -4314,7 +4641,8 @@ Init_BareVM(void) vm_opt_mid_table = st_init_numtable(); #ifdef RUBY_THREAD_WIN32_H - rb_native_cond_initialize(&vm->ractor.sync.barrier_cond); + rb_native_cond_initialize(&vm->ractor.sync.barrier_complete_cond); + rb_native_cond_initialize(&vm->ractor.sync.barrier_release_cond); #endif } @@ -4425,8 +4753,7 @@ rb_vm_register_global_object(VALUE obj) default: break; } - RB_VM_LOCK_ENTER(); - { + RB_VM_LOCKING() { VALUE list = GET_VM()->mark_object_ary; VALUE head = pin_array_list_append(list, obj); if (head != list) { @@ -4434,9 +4761,10 @@ rb_vm_register_global_object(VALUE obj) } RB_GC_GUARD(obj); } - RB_VM_LOCK_LEAVE(); } +VALUE rb_cc_refinement_set_create(void); + void Init_vm_objects(void) { @@ -4444,18 +4772,23 @@ Init_vm_objects(void) /* initialize mark object array, hash */ vm->mark_object_ary = pin_array_list_new(Qnil); - vm->loading_table = st_init_strtable(); - vm->ci_table = st_init_table(&vm_ci_hashtype); - vm->frozen_strings = st_init_table_with_size(&rb_fstring_hash_type, 10000); + st_init_existing_table_with_size(&vm->ci_table, &vm_ci_hashtype, 0); + vm->cc_refinement_set = rb_cc_refinement_set_create(); } +// Whether JIT is enabled or not, we need to load/undef `#with_jit` for other builtins. +#include "jit_hook.rbinc" +#include "jit_undef.rbinc" + // Stub for builtin function when not building YJIT units #if !USE_YJIT void Init_builtin_yjit(void) {} #endif -// Whether YJIT is enabled or not, we load yjit_hook.rb to remove Kernel#with_yjit. -#include "yjit_hook.rbinc" +// Stub for builtin function when not building ZJIT units +#if !USE_ZJIT +void Init_builtin_zjit(void) {} +#endif /* top self */ @@ -4468,17 +4801,20 @@ main_to_s(VALUE obj) VALUE rb_vm_top_self(void) { - return GET_VM()->top_self; + const rb_box_t *box = rb_current_box(); + VM_ASSERT(box); + VM_ASSERT(box->top_self); + return box->top_self; } void Init_top_self(void) { rb_vm_t *vm = GET_VM(); - - vm->top_self = rb_obj_alloc(rb_cObject); - rb_define_singleton_method(rb_vm_top_self(), "to_s", main_to_s, 0); - rb_define_alias(rb_singleton_class(rb_vm_top_self()), "inspect", "to_s"); + vm->root_box = (rb_box_t *)rb_root_box(); + vm->root_box->top_self = rb_obj_alloc(rb_cObject); + rb_define_singleton_method(vm->root_box->top_self, "to_s", main_to_s, 0); + rb_define_alias(rb_singleton_class(vm->root_box->top_self), "inspect", "to_s"); } VALUE * @@ -4508,12 +4844,6 @@ VALUE rb_insn_operand_intern(const rb_iseq_t *iseq, VALUE insn, int op_no, VALUE op, int len, size_t pos, VALUE *pnop, VALUE child); -st_table * -rb_vm_fstring_table(void) -{ - return GET_VM()->frozen_strings; -} - #if VM_COLLECT_USAGE_DETAILS #define HASH_ASET(h, k, v) rb_hash_aset((h), (st_data_t)(k), (st_data_t)(v)) @@ -4594,7 +4924,7 @@ vm_analysis_operand(int insn, int n, VALUE op) HASH_ASET(ihash, INT2FIX(n), ophash); } /* intern */ - valstr = rb_insn_operand_intern(GET_EC()->cfp->iseq, insn, n, op, 0, 0, 0, 0); + valstr = rb_insn_operand_intern(CFP_ISEQ(GET_EC()->cfp), insn, n, op, 0, 0, 0, 0); /* set count */ if (NIL_P(cv = rb_hash_aref(ophash, valstr))) { @@ -4796,7 +5126,7 @@ vm_collect_usage_operand(int insn, int n, VALUE op) if (RUBY_DTRACE_INSN_OPERAND_ENABLED()) { VALUE valstr; - valstr = rb_insn_operand_intern(GET_EC()->cfp->iseq, insn, n, op, 0, 0, 0, 0); + valstr = rb_insn_operand_intern(CFP_ISEQ(GET_EC()->cfp), insn, n, op, 0, 0, 0, 0); RUBY_DTRACE_INSN_OPERAND(RSTRING_PTR(valstr), rb_insns_name(insn)); RB_GC_GUARD(valstr); |