diff options
Diffstat (limited to 'vm.c')
-rw-r--r-- | vm.c | 1145 |
1 files changed, 763 insertions, 382 deletions
@@ -11,24 +11,28 @@ #define vm_exec rb_vm_exec #include "eval_intern.h" -#include "gc.h" #include "internal.h" +#include "internal/class.h" #include "internal/compile.h" #include "internal/cont.h" #include "internal/error.h" +#include "internal/encoding.h" #include "internal/eval.h" +#include "internal/gc.h" #include "internal/inits.h" +#include "internal/missing.h" #include "internal/object.h" -#include "internal/parse.h" #include "internal/proc.h" #include "internal/re.h" +#include "internal/ruby_parser.h" #include "internal/symbol.h" #include "internal/thread.h" +#include "internal/transcode.h" #include "internal/vm.h" #include "internal/sanitizers.h" #include "internal/variable.h" #include "iseq.h" -#include "mjit.h" +#include "rjit.h" #include "yjit.h" #include "ruby/st.h" #include "ruby/vm.h" @@ -39,32 +43,22 @@ #include "vm_insnhelper.h" #include "ractor_core.h" #include "vm_sync.h" +#include "shape.h" #include "builtin.h" -#ifndef MJIT_HEADER #include "probes.h" -#else -#include "probes.dmyh" -#endif #include "probes_helper.h" #ifdef RUBY_ASSERT_CRITICAL_SECTION int ruby_assert_critical_section_entered = 0; #endif +static void *native_main_thread_stack_top; + VALUE rb_str_concat_literals(size_t, const VALUE*); -/* :FIXME: This #ifdef is because we build pch in case of mswin and - * not in case of other situations. That distinction might change in - * a future. We would better make it detectable in something better - * than just _MSC_VER. */ -#ifdef _MSC_VER -RUBY_FUNC_EXPORTED -#else -MJIT_FUNC_EXPORTED -#endif -VALUE vm_exec(rb_execution_context_t *, bool); +VALUE vm_exec(rb_execution_context_t *); extern const char *const rb_debug_counter_names[]; @@ -209,7 +203,7 @@ VM_CAPTURED_BLOCK_TO_CFP(const struct rb_captured_block *captured) { rb_control_frame_t *cfp = ((rb_control_frame_t *)((VALUE *)(captured) - 3)); VM_ASSERT(!VM_CFP_IN_HEAP_P(GET_EC(), cfp)); - VM_ASSERT(sizeof(rb_control_frame_t)/sizeof(VALUE) == 8 + VM_DEBUG_BP_CHECK ? 1 : 0); + VM_ASSERT(sizeof(rb_control_frame_t)/sizeof(VALUE) == 7 + VM_DEBUG_BP_CHECK ? 1 : 0); return cfp; } @@ -234,7 +228,6 @@ vm_cref_new0(VALUE klass, rb_method_visibility_t visi, int module_func, rb_cref_ { VALUE refinements = Qnil; int omod_shared = FALSE; - rb_cref_t *cref; /* scope */ union { @@ -257,7 +250,10 @@ vm_cref_new0(VALUE klass, rb_method_visibility_t visi, int module_func, rb_cref_ VM_ASSERT(singleton || klass); - cref = (rb_cref_t *)rb_imemo_new(imemo_cref, klass, (VALUE)(use_prev_prev ? CREF_NEXT(prev_cref) : prev_cref), scope_visi.value, refinements); + rb_cref_t *cref = IMEMO_NEW(rb_cref_t, imemo_cref, refinements); + cref->klass_or_self = klass; + cref->next = use_prev_prev ? CREF_NEXT(prev_cref) : prev_cref; + *((rb_scope_visibility_t *)&cref->scope_visi) = scope_visi.visi; if (pushed_by_eval) CREF_PUSHED_BY_EVAL_SET(cref); if (omod_shared) CREF_OMOD_SHARED_SET(cref); @@ -380,85 +376,154 @@ extern VALUE rb_vm_invoke_bmethod(rb_execution_context_t *ec, rb_proc_t *proc, V const rb_callable_method_entry_t *me); static VALUE vm_invoke_proc(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE block_handler); -#if USE_MJIT || USE_YJIT -# ifdef MJIT_HEADER -NOINLINE(static COLDFUNC VALUE mjit_check_iseq(rb_execution_context_t *ec, const rb_iseq_t *iseq, struct rb_iseq_constant_body *body)); -# else -static inline VALUE mjit_check_iseq(rb_execution_context_t *ec, const rb_iseq_t *iseq, struct rb_iseq_constant_body *body); -# endif -static VALUE -mjit_check_iseq(rb_execution_context_t *ec, const rb_iseq_t *iseq, struct rb_iseq_constant_body *body) -{ - uintptr_t mjit_state = (uintptr_t)(body->jit_func); - ASSUME(MJIT_FUNC_STATE_P(mjit_state)); - switch ((enum rb_mjit_func_state)mjit_state) { - case MJIT_FUNC_NOT_COMPILED: - if (body->total_calls == mjit_opts.call_threshold) { - rb_mjit_add_iseq_to_process(iseq); - if (UNLIKELY(mjit_opts.wait && !MJIT_FUNC_STATE_P(body->jit_func))) { - return body->jit_func(ec, ec->cfp); +#if USE_YJIT +// Counter to serve as a proxy for execution time, total number of calls +static uint64_t yjit_total_entry_hits = 0; + +// Number of calls used to estimate how hot an ISEQ is +#define YJIT_CALL_COUNT_INTERV 20u + +/// Test whether we are ready to compile an ISEQ or not +static inline bool +rb_yjit_threshold_hit(const rb_iseq_t *iseq, uint64_t entry_calls) +{ + yjit_total_entry_hits += 1; + + // Record the number of calls at the beginning of the interval + if (entry_calls + YJIT_CALL_COUNT_INTERV == rb_yjit_call_threshold) { + iseq->body->yjit_calls_at_interv = yjit_total_entry_hits; + } + + // Try to estimate the total time taken (total number of calls) to reach 20 calls to this ISEQ + // This give us a ratio of how hot/cold this ISEQ is + if (entry_calls == rb_yjit_call_threshold) { + // We expect threshold 1 to compile everything immediately + if (rb_yjit_call_threshold < YJIT_CALL_COUNT_INTERV) { + return true; + } + + uint64_t num_calls = yjit_total_entry_hits - iseq->body->yjit_calls_at_interv; + + // Reject ISEQs that don't get called often enough + if (num_calls > rb_yjit_cold_threshold) { + rb_yjit_incr_counter("cold_iseq_entry"); + return false; + } + + return true; + } + + return false; +} +#else +#define rb_yjit_threshold_hit(iseq, entry_calls) false +#endif + +#if USE_RJIT || USE_YJIT +// Generate JIT code that supports the following kinds of ISEQ entries: +// * The first ISEQ on vm_exec (e.g. <main>, or Ruby methods/blocks +// called by a C method). The current frame has VM_FRAME_FLAG_FINISH. +// The current vm_exec stops if JIT code returns a non-Qundef value. +// * ISEQs called by the interpreter on vm_sendish (e.g. Ruby methods or +// blocks called by a Ruby frame that isn't compiled or side-exited). +// The current frame doesn't have VM_FRAME_FLAG_FINISH. The current +// vm_exec does NOT stop whether JIT code returns Qundef or not. +static inline rb_jit_func_t +jit_compile(rb_execution_context_t *ec) +{ + const rb_iseq_t *iseq = ec->cfp->iseq; + struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); + bool yjit_enabled = rb_yjit_enabled_p; + if (!(yjit_enabled || rb_rjit_call_p)) { + return NULL; + } + + // Increment the ISEQ's call counter and trigger JIT compilation if not compiled + if (body->jit_entry == NULL) { + body->jit_entry_calls++; + if (yjit_enabled) { + if (rb_yjit_threshold_hit(iseq, body->jit_entry_calls)) { + rb_yjit_compile_iseq(iseq, ec, false); } } - break; - case MJIT_FUNC_COMPILING: - case MJIT_FUNC_FAILED: - break; + else if (body->jit_entry_calls == rb_rjit_call_threshold()) { + rb_rjit_compile(iseq); + } } - return Qundef; + return body->jit_entry; } -// Try to execute the current iseq in ec. Use JIT code if it is ready. -// If it is not, add ISEQ to the compilation queue and return Qundef for MJIT. -// YJIT compiles on the thread running the iseq. +// Execute JIT code compiled by jit_compile() static inline VALUE jit_exec(rb_execution_context_t *ec) { - // Increment the ISEQ's call counter - const rb_iseq_t *iseq = ec->cfp->iseq; - struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); - bool yjit_enabled = rb_yjit_enabled_p(); - if (yjit_enabled || mjit_call_p) { - body->total_calls++; + rb_jit_func_t func = jit_compile(ec); + if (func) { + // Call the JIT code + return func(ec, ec->cfp); } else { return Qundef; } +} +#else +# define jit_compile(ec) ((rb_jit_func_t)0) +# define jit_exec(ec) Qundef +#endif - // Trigger JIT compilation as needed - jit_func_t func; - if (yjit_enabled) { - if (body->total_calls == rb_yjit_call_threshold()) { - // If we couldn't generate any code for this iseq, then return - // Qundef so the interpreter will handle the call. - if (!rb_yjit_compile_iseq(iseq, ec)) { - return Qundef; - } - } - // YJIT tried compiling this function once before and couldn't do - // it, so return Qundef so the interpreter handles it. - if ((func = body->jit_func) == 0) { - return Qundef; - } +#if USE_YJIT +// Generate JIT code that supports the following kind of ISEQ entry: +// * The first ISEQ pushed by vm_exec_handle_exception. The frame would +// point to a location specified by a catch table, and it doesn't have +// VM_FRAME_FLAG_FINISH. The current vm_exec stops if JIT code returns +// a non-Qundef value. So you should not return a non-Qundef value +// until ec->cfp is changed to a frame with VM_FRAME_FLAG_FINISH. +static inline rb_jit_func_t +jit_compile_exception(rb_execution_context_t *ec) +{ + const rb_iseq_t *iseq = ec->cfp->iseq; + struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); + if (!rb_yjit_enabled_p) { + return NULL; } - else if (UNLIKELY(MJIT_FUNC_STATE_P(func = body->jit_func))) { - return mjit_check_iseq(ec, iseq, body); + + // Increment the ISEQ's call counter and trigger JIT compilation if not compiled + if (body->jit_exception == NULL) { + body->jit_exception_calls++; + if (body->jit_exception_calls == rb_yjit_call_threshold) { + rb_yjit_compile_iseq(iseq, ec, true); + } } - // Call the JIT code - return func(ec, ec->cfp); // SystemV x64 calling convention: ec -> RDI, cfp -> RSI + return body->jit_exception; +} + +// Execute JIT code compiled by jit_compile_exception() +static inline VALUE +jit_exec_exception(rb_execution_context_t *ec) +{ + rb_jit_func_t func = jit_compile_exception(ec); + if (func) { + // Call the JIT code + return func(ec, ec->cfp); + } + else { + return Qundef; + } } +#else +# define jit_compile_exception(ec) ((rb_jit_func_t)0) +# define jit_exec_exception(ec) Qundef #endif -#include "vm_insnhelper.c" +static void add_opt_method_entry(const rb_method_entry_t *me); -#ifndef MJIT_HEADER +#include "vm_insnhelper.c" #include "vm_exec.c" #include "vm_method.c" -#endif /* #ifndef MJIT_HEADER */ #include "vm_eval.c" -#ifndef MJIT_HEADER #define PROCDEBUG 0 @@ -475,19 +540,32 @@ bool ruby_vm_keep_script_lines; #ifdef RB_THREAD_LOCAL_SPECIFIER RB_THREAD_LOCAL_SPECIFIER rb_execution_context_t *ruby_current_ec; -#ifdef __APPLE__ - rb_execution_context_t * - rb_current_ec(void) - { - return ruby_current_ec; - } - void - rb_current_ec_set(rb_execution_context_t *ec) - { - ruby_current_ec = ec; - } +#ifdef RUBY_NT_SERIAL +RB_THREAD_LOCAL_SPECIFIER rb_atomic_t ruby_nt_serial; #endif +// no-inline decl on thread_pthread.h +rb_execution_context_t * +rb_current_ec_noinline(void) +{ + return ruby_current_ec; +} + +void +rb_current_ec_set(rb_execution_context_t *ec) +{ + ruby_current_ec = ec; +} + + +#ifdef __APPLE__ +rb_execution_context_t * +rb_current_ec(void) +{ + return ruby_current_ec; +} + +#endif #else native_tls_key_t ruby_current_ec_key; #endif @@ -528,7 +606,7 @@ rb_vm_inc_const_missing_count(void) ruby_vm_const_missing_count +=1; } -MJIT_FUNC_EXPORTED int +int rb_dtrace_setup(rb_execution_context_t *ec, VALUE klass, ID id, struct ruby_dtrace_method_hook_args *args) { @@ -541,8 +619,8 @@ rb_dtrace_setup(rb_execution_context_t *ec, VALUE klass, ID id, if (RB_TYPE_P(klass, T_ICLASS)) { klass = RBASIC(klass)->klass; } - else if (FL_TEST(klass, FL_SINGLETON)) { - klass = rb_attr_get(klass, id__attached__); + else if (RCLASS_SINGLETON_P(klass)) { + klass = RCLASS_ATTACHED_OBJECT(klass); if (NIL_P(klass)) return FALSE; } type = BUILTIN_TYPE(klass); @@ -564,6 +642,8 @@ rb_dtrace_setup(rb_execution_context_t *ec, VALUE klass, ID id, return FALSE; } +extern unsigned int redblack_buffer_size; + /* * call-seq: * RubyVM.stat -> Hash @@ -591,6 +671,7 @@ static VALUE vm_stat(int argc, VALUE *argv, VALUE self) { static VALUE sym_constant_cache_invalidations, sym_constant_cache_misses, sym_global_cvar_state, sym_next_shape_id; + static VALUE sym_shape_cache_size; VALUE arg = Qnil; VALUE hash = Qnil, key = Qnil; @@ -612,6 +693,7 @@ vm_stat(int argc, VALUE *argv, VALUE self) S(constant_cache_misses); S(global_cvar_state); S(next_shape_id); + S(shape_cache_size); #undef S #define SET(name, attr) \ @@ -623,7 +705,8 @@ vm_stat(int argc, VALUE *argv, VALUE self) SET(constant_cache_invalidations, ruby_vm_constant_cache_invalidations); SET(constant_cache_misses, ruby_vm_constant_cache_misses); SET(global_cvar_state, ruby_vm_global_cvar_state); - SET(next_shape_id, (rb_serial_t)GET_VM()->next_shape_id); + SET(next_shape_id, (rb_serial_t)GET_SHAPE_TREE()->next_shape_id); + SET(shape_cache_size, (rb_serial_t)GET_SHAPE_TREE()->cache_size); #undef SET #if USE_DEBUG_COUNTER @@ -705,7 +788,7 @@ rb_vm_get_binding_creatable_next_cfp(const rb_execution_context_t *ec, const rb_ return 0; } -MJIT_FUNC_EXPORTED rb_control_frame_t * +rb_control_frame_t * rb_vm_get_ruby_level_next_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) { @@ -717,8 +800,6 @@ rb_vm_get_ruby_level_next_cfp(const rb_execution_context_t *ec, const rb_control return 0; } -#endif /* #ifndef MJIT_HEADER */ - static rb_control_frame_t * vm_get_ruby_level_caller_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { @@ -741,7 +822,7 @@ vm_get_ruby_level_caller_cfp(const rb_execution_context_t *ec, const rb_control_ return 0; } -MJIT_STATIC void +void rb_vm_pop_cfunc_frame(void) { rb_execution_context_t *ec = GET_EC(); @@ -753,8 +834,6 @@ rb_vm_pop_cfunc_frame(void) vm_pop_frame(ec, cfp, cfp->ep); } -#ifndef MJIT_HEADER - void rb_vm_rewind_cfp(rb_execution_context_t *ec, rb_control_frame_t *cfp) { @@ -834,7 +913,7 @@ vm_block_handler_escape(const rb_execution_context_t *ec, VALUE block_handler) switch (vm_block_handler_type(block_handler)) { case block_handler_type_ifunc: case block_handler_type_iseq: - return rb_vm_make_proc(ec, VM_BH_TO_CAPT_BLOCK(block_handler), rb_cProc); + return rb_vm_make_proc(ec, VM_BH_TO_CAPT_BLOCK(block_handler), rb_cProc); case block_handler_type_symbol: case block_handler_type_proc: @@ -848,8 +927,6 @@ static VALUE vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *const cfp) { const VALUE * const ep = cfp->ep; - const rb_env_t *env; - const rb_iseq_t *env_iseq; VALUE *env_body, *env_ep; int local_size, env_size; @@ -901,9 +978,26 @@ vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *co env_size = local_size + 1 /* envval */; + + // Careful with order in the following sequence. Each allocation can move objects. env_body = ALLOC_N(VALUE, env_size); + rb_env_t *env = IMEMO_NEW(rb_env_t, imemo_env, 0); + + // Set up env without WB since it's brand new (similar to newobj_init(), newobj_fill()) MEMCPY(env_body, ep - (local_size - 1 /* specval */), VALUE, local_size); + env_ep = &env_body[local_size - 1 /* specval */]; + env_ep[VM_ENV_DATA_INDEX_ENV] = (VALUE)env; + + env->iseq = (rb_iseq_t *)(VM_FRAME_RUBYFRAME_P(cfp) ? cfp->iseq : NULL); + env->ep = env_ep; + env->env = env_body; + env->env_size = env_size; + + cfp->ep = env_ep; + VM_ENV_FLAGS_SET(env_ep, VM_ENV_FLAG_ESCAPED | VM_ENV_FLAG_WB_REQUIRED); + VM_STACK_ENV_WRITE(ep, 0, (VALUE)env); /* GC mark */ + #if 0 for (i = 0; i < local_size; i++) { if (VM_FRAME_RUBYFRAME_P(cfp)) { @@ -913,14 +1007,11 @@ vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *co } #endif - env_iseq = VM_FRAME_RUBYFRAME_P(cfp) ? cfp->iseq : NULL; - env_ep = &env_body[local_size - 1 /* specval */]; - - env = vm_env_new(env_ep, env_body, env_size, env_iseq); + // Invalidate JIT code that assumes cfp->ep == vm_base_ptr(cfp). + if (env->iseq) { + rb_yjit_invalidate_ep_is_bp(env->iseq); + } - cfp->ep = env_ep; - VM_ENV_FLAGS_SET(env_ep, VM_ENV_FLAG_ESCAPED | VM_ENV_FLAG_WB_REQUIRED); - VM_STACK_ENV_WRITE(ep, 0, (VALUE)env); /* GC mark */ return (VALUE)env; } @@ -1141,26 +1232,33 @@ env_copy(const VALUE *src_ep, VALUE read_only_variables) VALUE *env_body = ZALLOC_N(VALUE, src_env->env_size); // fill with Qfalse VALUE *ep = &env_body[src_env->env_size - 2]; - volatile VALUE prev_env = Qnil; + const rb_env_t *copied_env = vm_env_new(ep, env_body, src_env->env_size, src_env->iseq); + + // Copy after allocations above, since they can move objects in src_ep. + RB_OBJ_WRITE(copied_env, &ep[VM_ENV_DATA_INDEX_ME_CREF], src_ep[VM_ENV_DATA_INDEX_ME_CREF]); + ep[VM_ENV_DATA_INDEX_FLAGS] = src_ep[VM_ENV_DATA_INDEX_FLAGS] | VM_ENV_FLAG_ISOLATED; + if (!VM_ENV_LOCAL_P(src_ep)) { + VM_ENV_FLAGS_SET(ep, VM_ENV_FLAG_LOCAL); + } if (read_only_variables) { for (int i=RARRAY_LENINT(read_only_variables)-1; i>=0; i--) { ID id = NUM2ID(RARRAY_AREF(read_only_variables, i)); for (unsigned int j=0; j<ISEQ_BODY(src_env->iseq)->local_table_size; j++) { - if (id == ISEQ_BODY(src_env->iseq)->local_table[j]) { + if (id == ISEQ_BODY(src_env->iseq)->local_table[j]) { VALUE v = src_env->env[j]; if (!rb_ractor_shareable_p(v)) { VALUE name = rb_id2str(id); VALUE msg = rb_sprintf("can not make shareable Proc because it can refer" " unshareable object %+" PRIsVALUE " from ", v); if (name) - rb_str_catf(msg, "variable `%" PRIsVALUE "'", name); + rb_str_catf(msg, "variable '%" PRIsVALUE "'", name); else rb_str_cat_cstr(msg, "a hidden variable"); rb_exc_raise(rb_exc_new_str(rb_eRactorIsolationError, msg)); } - env_body[j] = v; + RB_OBJ_WRITE((VALUE)copied_env, &env_body[j], v); rb_ary_delete_at(read_only_variables, i); break; } @@ -1168,21 +1266,17 @@ env_copy(const VALUE *src_ep, VALUE read_only_variables) } } - ep[VM_ENV_DATA_INDEX_ME_CREF] = src_ep[VM_ENV_DATA_INDEX_ME_CREF]; - ep[VM_ENV_DATA_INDEX_FLAGS] = src_ep[VM_ENV_DATA_INDEX_FLAGS] | VM_ENV_FLAG_ISOLATED; - if (!VM_ENV_LOCAL_P(src_ep)) { const VALUE *prev_ep = VM_ENV_PREV_EP(src_env->ep); const rb_env_t *new_prev_env = env_copy(prev_ep, read_only_variables); - prev_env = (VALUE)new_prev_env; ep[VM_ENV_DATA_INDEX_SPECVAL] = VM_GUARDED_PREV_EP(new_prev_env->ep); + RB_OBJ_WRITTEN(copied_env, Qundef, new_prev_env); + VM_ENV_FLAGS_UNSET(ep, VM_ENV_FLAG_LOCAL); } else { ep[VM_ENV_DATA_INDEX_SPECVAL] = VM_BLOCK_HANDLER_NONE; } - const rb_env_t *copied_env = vm_env_new(ep, env_body, src_env->env_size, src_env->iseq); - RB_GC_GUARD(prev_env); return copied_env; } @@ -1218,11 +1312,11 @@ proc_shared_outer_variables(struct rb_id_table *outer_variables, bool isolate, c rb_str_append(str, name); } if (*sep == ',') rb_str_cat_cstr(str, ")"); - rb_str_cat_cstr(str, data.yield ? " and uses `yield'." : "."); + rb_str_cat_cstr(str, data.yield ? " and uses 'yield'." : "."); rb_exc_raise(rb_exc_new_str(rb_eArgError, str)); } else if (data.yield) { - rb_raise(rb_eArgError, "can not %s because it uses `yield'.", message); + rb_raise(rb_eArgError, "can not %s because it uses 'yield'.", message); } return data.read_only; @@ -1287,21 +1381,45 @@ rb_proc_ractor_make_shareable(VALUE self) return self; } -MJIT_FUNC_EXPORTED VALUE +VALUE rb_vm_make_proc_lambda(const rb_execution_context_t *ec, const struct rb_captured_block *captured, VALUE klass, int8_t is_lambda) { VALUE procval; + enum imemo_type code_type = imemo_type(captured->code.val); if (!VM_ENV_ESCAPED_P(captured->ep)) { rb_control_frame_t *cfp = VM_CAPTURED_BLOCK_TO_CFP(captured); vm_make_env_object(ec, cfp); } + VM_ASSERT(VM_EP_IN_HEAP_P(ec, captured->ep)); - VM_ASSERT(imemo_type_p(captured->code.val, imemo_iseq) || - imemo_type_p(captured->code.val, imemo_ifunc)); + VM_ASSERT(code_type == imemo_iseq || code_type == imemo_ifunc); procval = vm_proc_create_from_captured(klass, captured, - imemo_type(captured->code.val) == imemo_iseq ? block_type_iseq : block_type_ifunc, FALSE, is_lambda); + code_type == imemo_iseq ? block_type_iseq : block_type_ifunc, + FALSE, is_lambda); + + if (code_type == imemo_ifunc) { + struct vm_ifunc *ifunc = (struct vm_ifunc *)captured->code.val; + if (ifunc->svar_lep) { + VALUE ep0 = ifunc->svar_lep[0]; + if (RB_TYPE_P(ep0, T_IMEMO) && imemo_type_p(ep0, imemo_env)) { + // `ep0 == imemo_env` means this ep is escaped to heap (in env object). + const rb_env_t *env = (const rb_env_t *)ep0; + ifunc->svar_lep = (VALUE *)env->ep; + } + else { + VM_ASSERT(FIXNUM_P(ep0)); + if (ep0 & VM_ENV_FLAG_ESCAPED) { + // ok. do nothing + } + else { + ifunc->svar_lep = NULL; + } + } + } + } + return procval; } @@ -1345,8 +1463,7 @@ rb_binding_add_dynavars(VALUE bindval, rb_binding_t *bind, int dyncount, const I const rb_env_t *env; rb_execution_context_t *ec = GET_EC(); const rb_iseq_t *base_iseq, *iseq; - rb_ast_body_t ast; - NODE tmp_node; + rb_node_scope_t tmp_node; if (dyncount < 0) return 0; @@ -1358,17 +1475,19 @@ rb_binding_add_dynavars(VALUE bindval, rb_binding_t *bind, int dyncount, const I dyns->size = dyncount; MEMCPY(dyns->ids, dynvars, ID, dyncount); - rb_node_init(&tmp_node, NODE_SCOPE, (VALUE)dyns, 0, 0); - ast.root = &tmp_node; - ast.compile_option = 0; - ast.script_lines = INT2FIX(-1); + rb_node_init(RNODE(&tmp_node), NODE_SCOPE); + tmp_node.nd_tbl = dyns; + tmp_node.nd_body = 0; + tmp_node.nd_args = 0; + + VALUE ast_value = rb_ruby_ast_new(RNODE(&tmp_node)); if (base_iseq) { - iseq = rb_iseq_new(&ast, ISEQ_BODY(base_iseq)->location.label, path, realpath, base_iseq, ISEQ_TYPE_EVAL); + iseq = rb_iseq_new(ast_value, ISEQ_BODY(base_iseq)->location.label, path, realpath, base_iseq, ISEQ_TYPE_EVAL); } else { VALUE tempstr = rb_fstring_lit("<temp>"); - iseq = rb_iseq_new_top(&ast, tempstr, tempstr, tempstr, NULL); + iseq = rb_iseq_new_top(ast_value, tempstr, tempstr, tempstr, NULL); } tmp_node.nd_tbl = 0; /* reset table */ ALLOCV_END(idtmp); @@ -1395,13 +1514,13 @@ invoke_block(rb_execution_context_t *ec, const rb_iseq_t *iseq, VALUE self, cons ec->cfp->sp + arg_size, ISEQ_BODY(iseq)->local_table_size - arg_size, ISEQ_BODY(iseq)->stack_max); - return vm_exec(ec, true); + return vm_exec(ec); } static VALUE invoke_bmethod(rb_execution_context_t *ec, const rb_iseq_t *iseq, VALUE self, const struct rb_captured_block *captured, const rb_callable_method_entry_t *me, VALUE type, int opt_pc) { - /* bmethod */ + /* bmethod call from outside the VM */ int arg_size = ISEQ_BODY(iseq)->param.size; VALUE ret; @@ -1411,12 +1530,12 @@ invoke_bmethod(rb_execution_context_t *ec, const rb_iseq_t *iseq, VALUE self, co VM_GUARDED_PREV_EP(captured->ep), (VALUE)me, ISEQ_BODY(iseq)->iseq_encoded + opt_pc, - ec->cfp->sp + arg_size, + ec->cfp->sp + 1 /* self */ + arg_size, ISEQ_BODY(iseq)->local_table_size - arg_size, ISEQ_BODY(iseq)->stack_max); VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH); - ret = vm_exec(ec, true); + ret = vm_exec(ec); return ret; } @@ -1432,21 +1551,35 @@ invoke_iseq_block_from_c(rb_execution_context_t *ec, const struct rb_captured_bl const rb_cref_t *cref, int is_lambda, const rb_callable_method_entry_t *me) { const rb_iseq_t *iseq = rb_iseq_check(captured->code.iseq); - int i, opt_pc; + int opt_pc; VALUE type = VM_FRAME_MAGIC_BLOCK | (is_lambda ? VM_FRAME_FLAG_LAMBDA : 0); rb_control_frame_t *cfp = ec->cfp; VALUE *sp = cfp->sp; + int flags = (kw_splat ? VM_CALL_KW_SPLAT : 0); + VALUE *use_argv = (VALUE *)argv; + VALUE av[2]; stack_check(ec); - CHECK_VM_STACK_OVERFLOW(cfp, argc); + if (UNLIKELY(argc > VM_ARGC_STACK_MAX) && + (VM_ARGC_STACK_MAX >= 1 || + /* Skip ruby array for potential autosplat case */ + (argc != 1 || is_lambda))) { + use_argv = vm_argv_ruby_array(av, argv, &flags, &argc, kw_splat); + } + + CHECK_VM_STACK_OVERFLOW(cfp, argc + 1); vm_check_canary(ec, sp); - cfp->sp = sp + argc; - for (i=0; i<argc; i++) { - sp[i] = argv[i]; + + VALUE *stack_argv = sp; + if (me) { + *sp = self; // bemthods need `self` on the VM stack + stack_argv++; } + cfp->sp = stack_argv + argc; + MEMCPY(stack_argv, use_argv, VALUE, argc); // restrict: new stack space - opt_pc = vm_yield_setup_args(ec, iseq, argc, sp, kw_splat, passed_block_handler, + opt_pc = vm_yield_setup_args(ec, iseq, argc, stack_argv, flags, passed_block_handler, (is_lambda ? arg_setup_method : arg_setup_block)); cfp->sp = sp; @@ -1582,14 +1715,14 @@ vm_invoke_proc(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self, return invoke_block_from_c_proc(ec, proc, self, argc, argv, kw_splat, passed_block_handler, proc->is_lambda, NULL); } -MJIT_FUNC_EXPORTED VALUE +VALUE rb_vm_invoke_bmethod(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE block_handler, const rb_callable_method_entry_t *me) { return invoke_block_from_c_proc(ec, proc, self, argc, argv, kw_splat, block_handler, TRUE, me); } -MJIT_FUNC_EXPORTED VALUE +VALUE rb_vm_invoke_proc(rb_execution_context_t *ec, rb_proc_t *proc, int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler) { @@ -1620,30 +1753,36 @@ rb_vm_invoke_proc_with_self(rb_execution_context_t *ec, rb_proc_t *proc, VALUE s /* special variable */ -static rb_control_frame_t * -vm_normal_frame(const rb_execution_context_t *ec, rb_control_frame_t *cfp) +VALUE * +rb_vm_svar_lep(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { - while (cfp->pc == 0) { - cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + while (cfp->pc == 0 || cfp->iseq == 0) { + if (VM_FRAME_TYPE(cfp) == VM_FRAME_MAGIC_IFUNC) { + struct vm_ifunc *ifunc = (struct vm_ifunc *)cfp->iseq; + return ifunc->svar_lep; + } + else { + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + } + if (RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) { - return 0; + return NULL; } } - return cfp; + + return (VALUE *)VM_CF_LEP(cfp); } static VALUE vm_cfp_svar_get(const rb_execution_context_t *ec, rb_control_frame_t *cfp, VALUE key) { - cfp = vm_normal_frame(ec, cfp); - return lep_svar_get(ec, cfp ? VM_CF_LEP(cfp) : 0, key); + return lep_svar_get(ec, rb_vm_svar_lep(ec, cfp), key); } static void vm_cfp_svar_set(const rb_execution_context_t *ec, rb_control_frame_t *cfp, VALUE key, const VALUE val) { - cfp = vm_normal_frame(ec, cfp); - lep_svar_set(ec, cfp ? VM_CF_LEP(cfp) : 0, key, val); + lep_svar_set(ec, rb_vm_svar_lep(ec, cfp), key, val); } static VALUE @@ -1682,6 +1821,17 @@ rb_lastline_set(VALUE val) vm_svar_set(GET_EC(), VM_SVAR_LASTLINE, val); } +void +rb_lastline_set_up(VALUE val, unsigned int up) +{ + rb_control_frame_t * cfp = GET_EC()->cfp; + + for(unsigned int i = 0; i < up; i++) { + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + } + vm_cfp_svar_set(GET_EC(), cfp, VM_SVAR_LASTLINE, val); +} + /* misc */ const char * @@ -1728,7 +1878,7 @@ rb_source_location(int *pline) } } -MJIT_FUNC_EXPORTED const char * +const char * rb_source_location_cstr(int *pline) { VALUE path = rb_source_location(pline); @@ -1824,7 +1974,7 @@ make_localjump_error(const char *mesg, VALUE value, int reason) return exc; } -MJIT_FUNC_EXPORTED void +void rb_vm_localjump_error(const char *mesg, VALUE value, int reason) { VALUE exc = make_localjump_error(mesg, value, reason); @@ -1832,7 +1982,7 @@ rb_vm_localjump_error(const char *mesg, VALUE value, int reason) } VALUE -rb_vm_make_jump_tag_but_local_jump(int state, VALUE val) +rb_vm_make_jump_tag_but_local_jump(enum ruby_tag_type state, VALUE val) { const char *mesg; @@ -1864,7 +2014,7 @@ rb_vm_make_jump_tag_but_local_jump(int state, VALUE val) } void -rb_vm_jump_tag_but_local_jump(int state) +rb_vm_jump_tag_but_local_jump(enum ruby_tag_type state) { VALUE exc = rb_vm_make_jump_tag_but_local_jump(state, Qundef); if (!NIL_P(exc)) rb_exc_raise(exc); @@ -1915,6 +2065,13 @@ short ruby_vm_redefined_flag[BOP_LAST_]; static st_table *vm_opt_method_def_table = 0; static st_table *vm_opt_mid_table = 0; +void +rb_free_vm_opt_tables(void) +{ + st_free_table(vm_opt_method_def_table); + st_free_table(vm_opt_mid_table); +} + static int vm_redefinition_check_flag(VALUE klass) { @@ -1952,6 +2109,8 @@ vm_redefinition_check_method_type(const rb_method_entry_t *me) return FALSE; } + if (METHOD_ENTRY_BASIC(me)) return TRUE; + const rb_method_definition_t *def = me->def; switch (def->type) { case VM_METHOD_TYPE_CFUNC: @@ -1974,8 +2133,14 @@ rb_vm_check_redefinition_opt_method(const rb_method_entry_t *me, VALUE klass) if (st_lookup(vm_opt_method_def_table, (st_data_t)me->def, &bop)) { int flag = vm_redefinition_check_flag(klass); if (flag != 0) { + rb_category_warn( + RB_WARN_CATEGORY_PERFORMANCE, + "Redefining '%s#%s' disables interpreter and JIT optimizations", + rb_class2name(me->owner), + rb_id2name(me->called_id) + ); rb_yjit_bop_redefined(flag, (enum ruby_basic_operators)bop); - rb_mjit_bop_redefined(flag, (enum ruby_basic_operators)bop); + rb_rjit_bop_redefined(flag, (enum ruby_basic_operators)bop); ruby_vm_redefined_flag[bop] |= flag; } } @@ -2002,27 +2167,44 @@ rb_vm_check_redefinition_by_prepend(VALUE klass) } static void -add_opt_method(VALUE klass, ID mid, VALUE bop) +add_opt_method_entry_bop(const rb_method_entry_t *me, ID mid, enum ruby_basic_operators bop) +{ + st_insert(vm_opt_method_def_table, (st_data_t)me->def, (st_data_t)bop); + st_insert(vm_opt_mid_table, (st_data_t)mid, (st_data_t)Qtrue); +} + +static void +add_opt_method(VALUE klass, ID mid, enum ruby_basic_operators bop) { const rb_method_entry_t *me = rb_method_entry_at(klass, mid); if (me && vm_redefinition_check_method_type(me)) { - st_insert(vm_opt_method_def_table, (st_data_t)me->def, (st_data_t)bop); - st_insert(vm_opt_mid_table, (st_data_t)mid, (st_data_t)Qtrue); + add_opt_method_entry_bop(me, mid, bop); } else { rb_bug("undefined optimized method: %s", rb_id2name(mid)); } } +static enum ruby_basic_operators vm_redefinition_bop_for_id(ID mid); + +static void +add_opt_method_entry(const rb_method_entry_t *me) +{ + if (me && vm_redefinition_check_method_type(me)) { + ID mid = me->called_id; + enum ruby_basic_operators bop = vm_redefinition_bop_for_id(mid); + if ((int)bop >= 0) { + add_opt_method_entry_bop(me, mid, bop); + } + } +} + static void vm_init_redefined_flag(void) { ID mid; - VALUE bop; - - vm_opt_method_def_table = st_init_numtable(); - vm_opt_mid_table = st_init_numtable(); + enum ruby_basic_operators bop; #define OP(mid_, bop_) (mid = id##mid_, bop = BOP_##bop_, ruby_vm_redefined_flag[bop] = 0) #define C(k) add_opt_method(rb_c##k, mid, bop) @@ -2050,6 +2232,7 @@ vm_init_redefined_flag(void) OP(UMinus, UMINUS), (C(String)); OP(Max, MAX), (C(Array)); OP(Min, MIN), (C(Array)); + OP(Hash, HASH), (C(Array)); OP(Call, CALL), (C(Proc)); OP(And, AND), (C(Integer)); OP(Or, OR), (C(Integer)); @@ -2060,6 +2243,47 @@ vm_init_redefined_flag(void) #undef OP } +static enum ruby_basic_operators +vm_redefinition_bop_for_id(ID mid) +{ + switch (mid) { +#define OP(mid_, bop_) case id##mid_: return BOP_##bop_ + OP(PLUS, PLUS); + OP(MINUS, MINUS); + OP(MULT, MULT); + OP(DIV, DIV); + OP(MOD, MOD); + OP(Eq, EQ); + OP(Eqq, EQQ); + OP(LT, LT); + OP(LE, LE); + OP(GT, GT); + OP(GE, GE); + OP(LTLT, LTLT); + OP(AREF, AREF); + OP(ASET, ASET); + OP(Length, LENGTH); + OP(Size, SIZE); + OP(EmptyP, EMPTY_P); + OP(Succ, SUCC); + OP(EqTilde, MATCH); + OP(Freeze, FREEZE); + OP(UMinus, UMINUS); + OP(Max, MAX); + OP(Min, MIN); + OP(Hash, HASH); + OP(Call, CALL); + OP(And, AND); + OP(Or, OR); + OP(NilP, NIL_P); + OP(Cmp, CMP); + OP(Default, DEFAULT); + OP(Pack, PACK); +#undef OP + } + return -1; +} + /* for vm development */ #if VMDEBUG @@ -2120,14 +2344,13 @@ frame_name(const rb_control_frame_t *cfp) // cfp_returning_with_value: // Whether cfp is the last frame in the unwinding process for a non-local return. static void -hook_before_rewind(rb_execution_context_t *ec, const rb_control_frame_t *cfp, - bool cfp_returning_with_value, int state, struct vm_throw_data *err) +hook_before_rewind(rb_execution_context_t *ec, bool cfp_returning_with_value, int state, struct vm_throw_data *err) { if (state == TAG_RAISE && RBASIC(err)->klass == rb_eSysStackError) { return; } else { - const rb_iseq_t *iseq = cfp->iseq; + const rb_iseq_t *iseq = ec->cfp->iseq; rb_hook_list_t *local_hooks = iseq->aux.exec.local_hooks; switch (VM_FRAME_TYPE(ec->cfp)) { @@ -2271,131 +2494,110 @@ hook_before_rewind(rb_execution_context_t *ec, const rb_control_frame_t *cfp, VALUE *ep; // ep void *code; // }; - - If jit_exec is already called before calling vm_exec, `jit_enable_p` should - be FALSE to avoid calling `jit_exec` twice. */ static inline VALUE -vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, - VALUE errinfo, VALUE *initial); +vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, VALUE errinfo); +static inline VALUE +vm_exec_loop(rb_execution_context_t *ec, enum ruby_tag_type state, struct rb_vm_tag *tag, VALUE result); // for non-Emscripten Wasm build, use vm_exec with optimized setjmp for runtime performance #if defined(__wasm__) && !defined(__EMSCRIPTEN__) struct rb_vm_exec_context { - rb_execution_context_t *ec; - struct rb_vm_tag *tag; - VALUE initial; + rb_execution_context_t *const ec; + struct rb_vm_tag *const tag; + VALUE result; - enum ruby_tag_type state; - bool jit_enable_p; }; static void -vm_exec_enter_vm_loop(rb_execution_context_t *ec, struct rb_vm_exec_context *ctx, - struct rb_vm_tag *_tag, bool skip_first_ex_handle) { - if (skip_first_ex_handle) { - goto vm_loop_start; - } - - ctx->result = ec->errinfo; - rb_ec_raised_reset(ec, RAISED_STACKOVERFLOW | RAISED_NOMEMORY); - while (UNDEF_P(ctx->result = vm_exec_handle_exception(ec, ctx->state, ctx->result, &ctx->initial))) { - /* caught a jump, exec the handler */ - ctx->result = vm_exec_core(ec, ctx->initial); - vm_loop_start: - VM_ASSERT(ec->tag == _tag); - /* when caught `throw`, `tag.state` is set. */ - if ((ctx->state = _tag->state) == TAG_NONE) break; - _tag->state = TAG_NONE; - } -} - -static void vm_exec_bottom_main(void *context) { - struct rb_vm_exec_context *ctx = (struct rb_vm_exec_context *)context; + struct rb_vm_exec_context *ctx = context; + rb_execution_context_t *ec = ctx->ec; - ctx->state = TAG_NONE; - if (!ctx->jit_enable_p || UNDEF_P(ctx->result = jit_exec(ctx->ec))) { - ctx->result = vm_exec_core(ctx->ec, ctx->initial); - } - vm_exec_enter_vm_loop(ctx->ec, ctx, ctx->tag, true); + ctx->result = vm_exec_loop(ec, TAG_NONE, ctx->tag, vm_exec_core(ec)); } static void vm_exec_bottom_rescue(void *context) { - struct rb_vm_exec_context *ctx = (struct rb_vm_exec_context *)context; - ctx->state = rb_ec_tag_state(ctx->ec); - vm_exec_enter_vm_loop(ctx->ec, ctx, ctx->tag, false); + struct rb_vm_exec_context *ctx = context; + rb_execution_context_t *ec = ctx->ec; + + ctx->result = vm_exec_loop(ec, rb_ec_tag_state(ec), ctx->tag, ec->errinfo); } +#endif VALUE -vm_exec(rb_execution_context_t *ec, bool jit_enable_p) +vm_exec(rb_execution_context_t *ec) { - struct rb_vm_exec_context ctx = { - .ec = ec, - .initial = 0, .result = Qundef, - .jit_enable_p = jit_enable_p, - }; - struct rb_wasm_try_catch try_catch; + VALUE result = Qundef; EC_PUSH_TAG(ec); _tag.retval = Qnil; - ctx.tag = &_tag; + +#if defined(__wasm__) && !defined(__EMSCRIPTEN__) + struct rb_vm_exec_context ctx = { + .ec = ec, + .tag = &_tag, + }; + struct rb_wasm_try_catch try_catch; EC_REPUSH_TAG(); rb_wasm_try_catch_init(&try_catch, vm_exec_bottom_main, vm_exec_bottom_rescue, &ctx); - rb_wasm_try_catch_loop_run(&try_catch, &_tag.buf); - - EC_POP_TAG(); - return ctx.result; -} + rb_wasm_try_catch_loop_run(&try_catch, &RB_VM_TAG_JMPBUF_GET(_tag.buf)); + result = ctx.result; #else - -VALUE -vm_exec(rb_execution_context_t *ec, bool jit_enable_p) -{ enum ruby_tag_type state; - VALUE result = Qundef; - VALUE initial = 0; - - EC_PUSH_TAG(ec); - - _tag.retval = Qnil; if ((state = EC_EXEC_TAG()) == TAG_NONE) { - if (!jit_enable_p || UNDEF_P(result = jit_exec(ec))) { - result = vm_exec_core(ec, initial); + if (UNDEF_P(result = jit_exec(ec))) { + result = vm_exec_core(ec); } - goto vm_loop_start; /* fallback to the VM */ + /* fallback to the VM */ + result = vm_exec_loop(ec, TAG_NONE, &_tag, result); } else { - result = ec->errinfo; - rb_ec_raised_reset(ec, RAISED_STACKOVERFLOW | RAISED_NOMEMORY); - while (UNDEF_P(result = vm_exec_handle_exception(ec, state, result, &initial))) { - /* caught a jump, exec the handler */ - result = vm_exec_core(ec, initial); - vm_loop_start: - VM_ASSERT(ec->tag == &_tag); - /* when caught `throw`, `tag.state` is set. */ - if ((state = _tag.state) == TAG_NONE) break; - _tag.state = TAG_NONE; - } + result = vm_exec_loop(ec, state, &_tag, ec->errinfo); } +#endif + EC_POP_TAG(); return result; } -#endif static inline VALUE -vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, - VALUE errinfo, VALUE *initial) +vm_exec_loop(rb_execution_context_t *ec, enum ruby_tag_type state, + struct rb_vm_tag *tag, VALUE result) +{ + if (state == TAG_NONE) { /* no jumps, result is discarded */ + goto vm_loop_start; + } + + rb_ec_raised_reset(ec, RAISED_STACKOVERFLOW | RAISED_NOMEMORY); + while (UNDEF_P(result = vm_exec_handle_exception(ec, state, result))) { + // caught a jump, exec the handler. JIT code in jit_exec_exception() + // may return Qundef to run remaining frames with vm_exec_core(). + if (UNDEF_P(result = jit_exec_exception(ec))) { + result = vm_exec_core(ec); + } + vm_loop_start: + VM_ASSERT(ec->tag == tag); + /* when caught `throw`, `tag.state` is set. */ + if ((state = tag->state) == TAG_NONE) break; + tag->state = TAG_NONE; + } + + return result; +} + +static inline VALUE +vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, VALUE errinfo) { struct vm_throw_data *err = (struct vm_throw_data *)errinfo; @@ -2405,7 +2607,6 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, const struct iseq_catch_table *ct; unsigned long epc, cont_pc, cont_sp; const rb_iseq_t *catch_iseq; - rb_control_frame_t *cfp; VALUE type; const rb_control_frame_t *escape_cfp; @@ -2425,7 +2626,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, rb_vm_pop_frame(ec); } - cfp = ec->cfp; + rb_control_frame_t *const cfp = ec->cfp; epc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded; escape_cfp = NULL; @@ -2455,7 +2656,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, ec->errinfo = Qnil; THROW_DATA_CATCH_FRAME_SET(err, cfp + 1); // cfp == escape_cfp here so calling with cfp_returning_with_value = true - hook_before_rewind(ec, ec->cfp, true, state, err); + hook_before_rewind(ec, true, state, err); rb_vm_pop_frame(ec); return THROW_DATA_VAL(err); } @@ -2464,11 +2665,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, } else { /* TAG_BREAK */ -#if OPT_STACK_CACHING - *initial = THROW_DATA_VAL(err); -#else - *ec->cfp->sp++ = THROW_DATA_VAL(err); -#endif + *cfp->sp++ = THROW_DATA_VAL(err); ec->errinfo = Qnil; return Qundef; } @@ -2541,11 +2738,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, cfp->sp = vm_base_ptr(cfp) + entry->sp; if (state != TAG_REDO) { -#if OPT_STACK_CACHING - *initial = THROW_DATA_VAL(err); -#else - *ec->cfp->sp++ = THROW_DATA_VAL(err); -#endif + *cfp->sp++ = THROW_DATA_VAL(err); } ec->errinfo = Qnil; VM_ASSERT(ec->tag->state == TAG_NONE); @@ -2596,7 +2789,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, return Qundef; } else { - hook_before_rewind(ec, ec->cfp, (cfp == escape_cfp), state, err); + hook_before_rewind(ec, (cfp == escape_cfp), state, err); if (VM_FRAME_FINISHED_P(ec->cfp)) { rb_vm_pop_frame(ec); @@ -2619,7 +2812,7 @@ rb_iseq_eval(const rb_iseq_t *iseq) rb_execution_context_t *ec = GET_EC(); VALUE val; vm_set_top_stack(ec, iseq); - val = vm_exec(ec, true); + val = vm_exec(ec); return val; } @@ -2630,7 +2823,7 @@ rb_iseq_eval_main(const rb_iseq_t *iseq) VALUE val; vm_set_main_stack(ec, iseq); - val = vm_exec(ec, true); + val = vm_exec(ec); return val; } @@ -2668,7 +2861,7 @@ rb_vm_call_cfunc(VALUE recv, VALUE (*func)(VALUE), VALUE arg, { rb_execution_context_t *ec = GET_EC(); const rb_control_frame_t *reg_cfp = ec->cfp; - const rb_iseq_t *iseq = rb_iseq_new(0, filename, filename, Qnil, 0, ISEQ_TYPE_TOP); + const rb_iseq_t *iseq = rb_iseq_new(Qnil, filename, filename, Qnil, 0, ISEQ_TYPE_TOP); VALUE val; vm_push_frame(ec, iseq, VM_FRAME_MAGIC_TOP | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH, @@ -2690,6 +2883,7 @@ rb_vm_update_references(void *ptr) if (ptr) { rb_vm_t *vm = ptr; + rb_gc_update_tbl_refs(vm->ci_table); rb_gc_update_tbl_refs(vm->frozen_strings); vm->mark_object_ary = rb_gc_location(vm->mark_object_ary); vm->load_path = rb_gc_location(vm->load_path); @@ -2703,11 +2897,14 @@ rb_vm_update_references(void *ptr) vm->loaded_features = rb_gc_location(vm->loaded_features); vm->loaded_features_snapshot = rb_gc_location(vm->loaded_features_snapshot); vm->loaded_features_realpaths = rb_gc_location(vm->loaded_features_realpaths); + vm->loaded_features_realpath_map = rb_gc_location(vm->loaded_features_realpath_map); vm->top_self = rb_gc_location(vm->top_self); vm->orig_progname = rb_gc_location(vm->orig_progname); rb_gc_update_tbl_refs(vm->overloaded_cme_table); + rb_gc_update_values(RUBY_NSIG, vm->trap_list.cmd); + if (vm->coverages) { vm->coverages = rb_gc_location(vm->coverages); vm->me2counter = rb_gc_location(vm->me2counter); @@ -2733,7 +2930,7 @@ rb_vm_each_stack_value(void *ptr, void (*cb)(VALUE, void*), void *ctx) VALUE *p = ec->vm_stack; VALUE *sp = ec->cfp->sp; while (p < sp) { - if (!rb_special_const_p(*p)) { + if (!RB_SPECIAL_CONST_P(*p)) { cb(*p, ctx); } p++; @@ -2752,6 +2949,8 @@ vm_mark_negative_cme(VALUE val, void *dmy) return ID_TABLE_CONTINUE; } +void rb_thread_sched_mark_zombies(rb_vm_t *vm); + void rb_vm_mark(void *ptr) { @@ -2760,8 +2959,7 @@ rb_vm_mark(void *ptr) if (ptr) { rb_vm_t *vm = ptr; rb_ractor_t *r = 0; - long i, len; - const VALUE *obj_ary; + long i; ccan_list_for_each(&vm->ractor.set, r, vmlr_node) { // ractor.set only contains blocking or running ractors @@ -2770,36 +2968,23 @@ rb_vm_mark(void *ptr) rb_gc_mark(rb_ractor_self(r)); } - rb_gc_mark_movable(vm->mark_object_ary); - - len = RARRAY_LEN(vm->mark_object_ary); - obj_ary = RARRAY_CONST_PTR(vm->mark_object_ary); - for (i=0; i < len; i++) { - const VALUE *ptr; - long j, jlen; - - rb_gc_mark(*obj_ary); - jlen = RARRAY_LEN(*obj_ary); - ptr = RARRAY_CONST_PTR(*obj_ary); - for (j=0; j < jlen; j++) { - rb_gc_mark(*ptr++); - } - obj_ary++; + for (struct global_object_list *list = vm->global_object_list; list; list = list->next) { + rb_gc_mark_maybe(*list->varptr); } + rb_gc_mark_movable(vm->mark_object_ary); rb_gc_mark_movable(vm->load_path); rb_gc_mark_movable(vm->load_path_snapshot); - RUBY_MARK_MOVABLE_UNLESS_NULL(vm->load_path_check_cache); + rb_gc_mark_movable(vm->load_path_check_cache); rb_gc_mark_movable(vm->expanded_load_path); rb_gc_mark_movable(vm->loaded_features); rb_gc_mark_movable(vm->loaded_features_snapshot); rb_gc_mark_movable(vm->loaded_features_realpaths); + rb_gc_mark_movable(vm->loaded_features_realpath_map); rb_gc_mark_movable(vm->top_self); rb_gc_mark_movable(vm->orig_progname); - RUBY_MARK_MOVABLE_UNLESS_NULL(vm->coverages); - RUBY_MARK_MOVABLE_UNLESS_NULL(vm->me2counter); - /* Prevent classes from moving */ - rb_mark_tbl(vm->defined_module_hash); + rb_gc_mark_movable(vm->coverages); + rb_gc_mark_movable(vm->me2counter); if (vm->loading_table) { rb_mark_tbl(vm->loading_table); @@ -2822,7 +3007,8 @@ rb_vm_mark(void *ptr) } } - mjit_mark(); + rb_thread_sched_mark_zombies(vm); + rb_rjit_mark(); } RUBY_MARK_LEAVE("vm"); @@ -2836,17 +3022,7 @@ rb_vm_register_special_exception_str(enum ruby_special_exceptions sp, VALUE cls, VALUE exc = rb_exc_new3(cls, rb_obj_freeze(mesg)); OBJ_FREEZE(exc); ((VALUE *)vm->special_exceptions)[sp] = exc; - rb_gc_register_mark_object(exc); -} - -int -rb_vm_add_root_module(VALUE module) -{ - rb_vm_t *vm = GET_VM(); - - st_insert(vm->defined_module_hash, (st_data_t)module, (st_data_t)module); - - return TRUE; + rb_vm_register_global_object(exc); } static int @@ -2856,6 +3032,9 @@ free_loading_table_entry(st_data_t key, st_data_t value, st_data_t arg) return ST_DELETE; } +void rb_free_loaded_features_index(rb_vm_t *vm); +void rb_objspace_free_objects(void *objspace); + int ruby_vm_destruct(rb_vm_t *vm) { @@ -2863,13 +3042,62 @@ ruby_vm_destruct(rb_vm_t *vm) if (vm) { rb_thread_t *th = vm->ractor.main_thread; - struct rb_objspace *objspace = vm->objspace; - vm->ractor.main_thread = NULL; + VALUE *stack = th->ec->vm_stack; + if (rb_free_at_exit) { + rb_free_encoded_insn_data(); + rb_free_global_enc_table(); + rb_free_loaded_builtin_table(); + + rb_free_shared_fiber_pool(); + rb_free_static_symid_str(); + rb_free_transcoder_table(); + rb_free_vm_opt_tables(); + rb_free_warning(); + rb_free_rb_global_tbl(); + rb_free_loaded_features_index(vm); + + rb_id_table_free(vm->negative_cme_table); + st_free_table(vm->overloaded_cme_table); + + rb_id_table_free(RCLASS(rb_mRubyVMFrozenCore)->m_tbl); + + rb_shape_t *cursor = rb_shape_get_root_shape(); + rb_shape_t *end = rb_shape_get_shape_by_id(GET_SHAPE_TREE()->next_shape_id); + while (cursor < end) { + // 0x1 == SINGLE_CHILD_P + if (cursor->edges && !(((uintptr_t)cursor->edges) & 0x1)) + rb_id_table_free(cursor->edges); + cursor += 1; + } + + xfree(GET_SHAPE_TREE()); + + st_free_table(vm->static_ext_inits); + st_free_table(vm->ensure_rollback_table); + + rb_vm_postponed_job_free(); + + rb_id_table_free(vm->constant_cache); + st_free_table(vm->unused_block_warning_table); - if (th) { - rb_fiber_reset_root_local_storage(th); - thread_free(th); + if (th) { + xfree(th->nt); + th->nt = NULL; + } + +#ifndef HAVE_SETPROCTITLE + ruby_free_proctitle(); +#endif + } + else { + if (th) { + rb_fiber_reset_root_local_storage(th); + thread_free(th); + } } + + struct rb_objspace *objspace = vm->objspace; + rb_vm_living_threads_init(vm); ruby_vm_run_at_exit_hooks(vm); if (vm->loading_table) { @@ -2877,15 +3105,35 @@ ruby_vm_destruct(rb_vm_t *vm) st_free_table(vm->loading_table); vm->loading_table = 0; } + if (vm->ci_table) { + st_free_table(vm->ci_table); + vm->ci_table = NULL; + } if (vm->frozen_strings) { st_free_table(vm->frozen_strings); vm->frozen_strings = 0; } RB_ALTSTACK_FREE(vm->main_altstack); + + struct global_object_list *next; + for (struct global_object_list *list = vm->global_object_list; list; list = next) { + next = list->next; + xfree(list); + } + if (objspace) { + if (rb_free_at_exit) { + rb_objspace_free_objects(objspace); + rb_free_generic_iv_tbl_(); + rb_free_default_rand_key(); + if (th && vm->fork_gen == 0) { + /* If we have forked, main_thread may not be the initial thread */ + xfree(stack); + ruby_mimfree(th); + } + } rb_objspace_free(objspace); } - rb_native_mutex_destroy(&vm->waitpid_lock); rb_native_mutex_destroy(&vm->workqueue_lock); /* after freeing objspace, you *can't* use ruby_xfree() */ ruby_mimfree(vm); @@ -2895,9 +3143,7 @@ ruby_vm_destruct(rb_vm_t *vm) return 0; } -size_t rb_vm_memsize_waiting_list(struct ccan_list_head *waiting_list); // process.c size_t rb_vm_memsize_waiting_fds(struct ccan_list_head *waiting_fds); // thread.c -size_t rb_vm_memsize_postponed_job_buffer(void); // vm_trace.c size_t rb_vm_memsize_workqueue(struct ccan_list_head *workqueue); // vm_trace.c // Used for VM memsize reporting. Returns the size of the at_exit list by @@ -2952,21 +3198,20 @@ vm_memsize(const void *ptr) return ( sizeof(rb_vm_t) + - rb_vm_memsize_waiting_list(&vm->waiting_pids) + - rb_vm_memsize_waiting_list(&vm->waiting_grps) + rb_vm_memsize_waiting_fds(&vm->waiting_fds) + rb_st_memsize(vm->loaded_features_index) + rb_st_memsize(vm->loading_table) + rb_st_memsize(vm->ensure_rollback_table) + - rb_vm_memsize_postponed_job_buffer() + + rb_vm_memsize_postponed_job_queue() + rb_vm_memsize_workqueue(&vm->workqueue) + - rb_st_memsize(vm->defined_module_hash) + vm_memsize_at_exit_list(vm->at_exit) + + rb_st_memsize(vm->ci_table) + rb_st_memsize(vm->frozen_strings) + vm_memsize_builtin_function_table(vm->builtin_function_table) + rb_id_table_memsize(vm->negative_cme_table) + rb_st_memsize(vm->overloaded_cme_table) + - vm_memsize_constant_cache() + vm_memsize_constant_cache() + + GET_SHAPE_TREE()->cache_size * sizeof(redblack_node_t) ); // TODO @@ -3059,7 +3304,6 @@ vm_default_params_setup(rb_vm_t *vm) static void vm_init2(rb_vm_t *vm) { - MEMZERO(vm, rb_vm_t, 1); rb_vm_living_threads_init(vm); vm->thread_report_on_exception = 1; vm->src_encoding_index = -1; @@ -3163,22 +3407,19 @@ rb_execution_context_mark(const rb_execution_context_t *ec) if (ec->machine.stack_start && ec->machine.stack_end && ec != GET_EC() /* marked for current ec at the first stage of marking */ ) { - rb_gc_mark_machine_stack(ec); - rb_gc_mark_locations((VALUE *)&ec->machine.regs, - (VALUE *)(&ec->machine.regs) + - sizeof(ec->machine.regs) / (sizeof(VALUE))); + rb_gc_mark_machine_context(ec); } - RUBY_MARK_UNLESS_NULL(ec->errinfo); - RUBY_MARK_UNLESS_NULL(ec->root_svar); + rb_gc_mark(ec->errinfo); + rb_gc_mark(ec->root_svar); if (ec->local_storage) { rb_id_table_foreach_values(ec->local_storage, mark_local_storage_i, NULL); } - RUBY_MARK_UNLESS_NULL(ec->local_storage_recursive_hash); - RUBY_MARK_UNLESS_NULL(ec->local_storage_recursive_hash_for_trace); - RUBY_MARK_UNLESS_NULL(ec->private_const_reference); + rb_gc_mark(ec->local_storage_recursive_hash); + rb_gc_mark(ec->local_storage_recursive_hash_for_trace); + rb_gc_mark(ec->private_const_reference); - RUBY_MARK_MOVABLE_UNLESS_NULL(ec->storage); + rb_gc_mark_movable(ec->storage); } void rb_fiber_mark_self(rb_fiber_t *fib); @@ -3209,8 +3450,8 @@ thread_mark(void *ptr) switch (th->invoke_type) { case thread_invoke_type_proc: case thread_invoke_type_ractor_proc: - RUBY_MARK_UNLESS_NULL(th->invoke_arg.proc.proc); - RUBY_MARK_UNLESS_NULL(th->invoke_arg.proc.args); + rb_gc_mark(th->invoke_arg.proc.proc); + rb_gc_mark(th->invoke_arg.proc.args); break; case thread_invoke_type_func: rb_gc_mark_maybe((VALUE)th->invoke_arg.func.arg); @@ -3220,31 +3461,35 @@ thread_mark(void *ptr) } rb_gc_mark(rb_ractor_self(th->ractor)); - RUBY_MARK_UNLESS_NULL(th->thgroup); - RUBY_MARK_UNLESS_NULL(th->value); - RUBY_MARK_UNLESS_NULL(th->pending_interrupt_queue); - RUBY_MARK_UNLESS_NULL(th->pending_interrupt_mask_stack); - RUBY_MARK_UNLESS_NULL(th->top_self); - RUBY_MARK_UNLESS_NULL(th->top_wrapper); + rb_gc_mark(th->thgroup); + rb_gc_mark(th->value); + rb_gc_mark(th->pending_interrupt_queue); + rb_gc_mark(th->pending_interrupt_mask_stack); + rb_gc_mark(th->top_self); + rb_gc_mark(th->top_wrapper); if (th->root_fiber) rb_fiber_mark_self(th->root_fiber); RUBY_ASSERT(th->ec == rb_fiberptr_get_ec(th->ec->fiber_ptr)); - RUBY_MARK_UNLESS_NULL(th->stat_insn_usage); - RUBY_MARK_UNLESS_NULL(th->last_status); - RUBY_MARK_UNLESS_NULL(th->locking_mutex); - RUBY_MARK_UNLESS_NULL(th->name); + rb_gc_mark(th->stat_insn_usage); + rb_gc_mark(th->last_status); + rb_gc_mark(th->locking_mutex); + rb_gc_mark(th->name); - RUBY_MARK_UNLESS_NULL(th->scheduler); + rb_gc_mark(th->scheduler); RUBY_MARK_LEAVE("thread"); } +void rb_threadptr_sched_free(rb_thread_t *th); // thread_*.c + static void thread_free(void *ptr) { rb_thread_t *th = ptr; RUBY_FREE_ENTER("thread"); + rb_threadptr_sched_free(th); + if (th->locking_mutex != Qfalse) { rb_bug("thread_free: locking_mutex must be NULL (%p:%p)", (void *)th, (void *)th->locking_mutex); } @@ -3252,13 +3497,16 @@ thread_free(void *ptr) rb_bug("thread_free: keeping_mutexes must be NULL (%p:%p)", (void *)th, (void *)th->keeping_mutexes); } + ruby_xfree(th->specific_storage); + rb_threadptr_root_fiber_release(th); if (th->vm && th->vm->ractor.main_thread == th) { RUBY_GC_INFO("MRI main thread\n"); } else { - ruby_xfree(th->nt); // TODO + // ruby_xfree(th->nt); + // TODO: MN system collect nt, but without MN system it should be freed here. ruby_xfree(th); } @@ -3317,6 +3565,10 @@ rb_ec_initialize_vm_stack(rb_execution_context_t *ec, VALUE *stack, size_t size) { rb_ec_set_vm_stack(ec, stack, size); +#if VM_CHECK_MODE > 0 + MEMZERO(stack, VALUE, size); // malloc memory could have the VM canary in it +#endif + ec->cfp = (void *)(ec->vm_stack + ec->vm_stack_size); vm_push_frame(ec, @@ -3379,8 +3631,10 @@ th_init(rb_thread_t *th, VALUE self, rb_vm_t *vm) th->ext_config.ractor_safe = true; #if USE_RUBY_DEBUG_LOG - static rb_atomic_t thread_serial = 0; + static rb_atomic_t thread_serial = 1; th->serial = RUBY_ATOMIC_FETCH_ADD(thread_serial, 1); + + RUBY_DEBUG_LOG("th:%u", th->serial); #endif } @@ -3470,7 +3724,9 @@ kwmerge_i(VALUE key, VALUE value, VALUE hash) static VALUE m_core_hash_merge_kwd(VALUE recv, VALUE hash, VALUE kw) { - REWIND_CFP(hash = core_hash_merge_kwd(hash, kw)); + if (!NIL_P(kw)) { + REWIND_CFP(hash = core_hash_merge_kwd(hash, kw)); + } return hash; } @@ -3508,7 +3764,7 @@ extern size_t rb_gc_stack_maxsize; static VALUE sdr(VALUE self) { - rb_vm_bugreport(NULL); + rb_vm_bugreport(NULL, stderr); return Qnil; } @@ -3576,6 +3832,7 @@ f_sprintf(int c, const VALUE *v, VALUE _) return rb_f_sprintf(c, v); } +/* :nodoc: */ static VALUE vm_mtbl(VALUE self, VALUE obj, VALUE sym) { @@ -3583,6 +3840,7 @@ vm_mtbl(VALUE self, VALUE obj, VALUE sym) return Qnil; } +/* :nodoc: */ static VALUE vm_mtbl2(VALUE self, VALUE obj, VALUE sym) { @@ -3658,6 +3916,7 @@ Init_VM(void) /* FrozenCore (hidden) */ fcore = rb_class_new(rb_cBasicObject); rb_set_class_path(fcore, rb_cRubyVM, "FrozenCore"); + rb_vm_register_global_object(rb_class_path_cached(fcore)); RBASIC(fcore)->flags = T_ICLASS; klass = rb_singleton_class(fcore); rb_define_method_id(klass, id_core_set_method_alias, m_core_set_method_alias, 3); @@ -3676,7 +3935,7 @@ Init_VM(void) rb_obj_freeze(fcore); RBASIC_CLEAR_CLASS(klass); rb_obj_freeze(klass); - rb_gc_register_mark_object(fcore); + rb_vm_register_global_object(fcore); rb_mRubyVMFrozenCore = fcore; /* @@ -3876,9 +4135,6 @@ Init_VM(void) rb_ary_push(opts, rb_str_new2("call threaded code")); #endif -#if OPT_STACK_CACHING - rb_ary_push(opts, rb_str_new2("stack caching")); -#endif #if OPT_OPERANDS_UNIFICATION rb_ary_push(opts, rb_str_new2("operands unification")); #endif @@ -3888,9 +4144,6 @@ Init_VM(void) #if OPT_INLINE_METHOD_CACHE rb_ary_push(opts, rb_str_new2("inline method cache")); #endif -#if OPT_BLOCKINLINING - rb_ary_push(opts, rb_str_new2("block inlining")); -#endif /* ::RubyVM::INSTRUCTION_NAMES * A list of bytecode instruction names in MRI. @@ -3924,7 +4177,7 @@ Init_VM(void) rb_vm_t *vm = ruby_current_vm_ptr; rb_thread_t *th = GET_THREAD(); VALUE filename = rb_fstring_lit("<main>"); - const rb_iseq_t *iseq = rb_iseq_new(0, filename, filename, Qnil, 0, ISEQ_TYPE_TOP); + const rb_iseq_t *iseq = rb_iseq_new(Qnil, filename, filename, Qnil, 0, ISEQ_TYPE_TOP); // Ractor setup rb_ractor_main_setup(vm, th->ractor, th); @@ -3940,7 +4193,7 @@ Init_VM(void) th->top_wrapper = 0; th->top_self = rb_vm_top_self(); - rb_gc_register_mark_object((VALUE)iseq); + rb_vm_register_global_object((VALUE)iseq); th->ec->cfp->iseq = iseq; th->ec->cfp->pc = ISEQ_BODY(iseq)->iseq_encoded; th->ec->cfp->self = th->top_self; @@ -3953,7 +4206,9 @@ Init_VM(void) */ rb_define_global_const("TOPLEVEL_BINDING", rb_binding_new()); +#ifdef _WIN32 rb_objspace_gc_enable(vm->objspace); +#endif } vm_init_redefined_flag(); @@ -3961,7 +4216,7 @@ Init_VM(void) rb_add_method_optimized(rb_singleton_class(rb_block_param_proxy), idCall, OPTIMIZED_METHOD_TYPE_BLOCK_CALL, 0, METHOD_VISI_PUBLIC); rb_obj_freeze(rb_block_param_proxy); - rb_gc_register_mark_object(rb_block_param_proxy); + rb_vm_register_global_object(rb_block_param_proxy); /* vm_backtrace.c */ Init_vm_backtrace(); @@ -3974,7 +4229,8 @@ rb_vm_set_progname(VALUE filename) rb_control_frame_t *cfp = (void *)(th->ec->vm_stack + th->ec->vm_stack_size); --cfp; - rb_iseq_pathobj_set(cfp->iseq, rb_str_dup(filename), rb_iseq_realpath(cfp->iseq)); + filename = rb_str_new_frozen(filename); + rb_iseq_pathobj_set(cfp->iseq, filename, rb_iseq_realpath(cfp->iseq)); } extern const struct st_hash_type rb_fstring_hash_type; @@ -3983,37 +4239,58 @@ void Init_BareVM(void) { /* VM bootstrap: phase 1 */ - rb_vm_t * vm = ruby_mimmalloc(sizeof(*vm)); - rb_thread_t * th = ruby_mimmalloc(sizeof(*th)); + rb_vm_t *vm = ruby_mimcalloc(1, sizeof(*vm)); + rb_thread_t *th = ruby_mimcalloc(1, sizeof(*th)); if (!vm || !th) { fputs("[FATAL] failed to allocate memory\n", stderr); exit(EXIT_FAILURE); } // setup the VM - MEMZERO(th, rb_thread_t, 1); vm_init2(vm); - vm->objspace = rb_objspace_alloc(); + rb_vm_postponed_job_queue_init(vm); ruby_current_vm_ptr = vm; + rb_objspace_alloc(); vm->negative_cme_table = rb_id_table_create(16); vm->overloaded_cme_table = st_init_numtable(); vm->constant_cache = rb_id_table_create(0); + vm->unused_block_warning_table = st_init_numtable(); + + // TODO: remove before Ruby 3.4.0 release + const char *s = getenv("RUBY_TRY_UNUSED_BLOCK_WARNING_STRICT"); + if (s && strcmp(s, "1") == 0) { + vm->unused_block_warning_strict = true; + } // setup main thread th->nt = ZALLOC(struct rb_native_thread); th->vm = vm; th->ractor = vm->ractor.main_ractor = rb_ractor_main_alloc(); Init_native_thread(th); + rb_jit_cont_init(); th_init(th, 0, vm); rb_ractor_set_current_ec(th->ractor, th->ec); - ruby_thread_init_stack(th); + /* n.b. native_main_thread_stack_top is set by the INIT_STACK macro */ + ruby_thread_init_stack(th, native_main_thread_stack_top); // setup ractor system rb_native_mutex_initialize(&vm->ractor.sync.lock); - rb_native_cond_initialize(&vm->ractor.sync.barrier_cond); rb_native_cond_initialize(&vm->ractor.sync.terminate_cond); + + vm_opt_method_def_table = st_init_numtable(); + vm_opt_mid_table = st_init_numtable(); + +#ifdef RUBY_THREAD_WIN32_H + rb_native_cond_initialize(&vm->ractor.sync.barrier_cond); +#endif +} + +void +ruby_init_stack(void *addr) +{ + native_main_thread_stack_top = addr; } #ifndef _WIN32 @@ -4021,34 +4298,124 @@ Init_BareVM(void) #include <sys/mman.h> #endif + +#ifndef MARK_OBJECT_ARY_BUCKET_SIZE +#define MARK_OBJECT_ARY_BUCKET_SIZE 1024 +#endif + +struct pin_array_list { + VALUE next; + long len; + VALUE *array; +}; + +static void +pin_array_list_mark(void *data) +{ + struct pin_array_list *array = (struct pin_array_list *)data; + rb_gc_mark_movable(array->next); + + rb_gc_mark_vm_stack_values(array->len, array->array); +} + +static void +pin_array_list_free(void *data) +{ + struct pin_array_list *array = (struct pin_array_list *)data; + xfree(array->array); +} + +static size_t +pin_array_list_memsize(const void *data) +{ + return sizeof(struct pin_array_list) + (MARK_OBJECT_ARY_BUCKET_SIZE * sizeof(VALUE)); +} + +static void +pin_array_list_update_references(void *data) +{ + struct pin_array_list *array = (struct pin_array_list *)data; + array->next = rb_gc_location(array->next); +} + +static const rb_data_type_t pin_array_list_type = { + .wrap_struct_name = "VM/pin_array_list", + .function = { + .dmark = pin_array_list_mark, + .dfree = pin_array_list_free, + .dsize = pin_array_list_memsize, + .dcompact = pin_array_list_update_references, + }, + .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE, +}; + +static VALUE +pin_array_list_new(VALUE next) +{ + struct pin_array_list *array_list; + VALUE obj = TypedData_Make_Struct(0, struct pin_array_list, &pin_array_list_type, array_list); + RB_OBJ_WRITE(obj, &array_list->next, next); + array_list->array = ALLOC_N(VALUE, MARK_OBJECT_ARY_BUCKET_SIZE); + return obj; +} + +static VALUE +pin_array_list_append(VALUE obj, VALUE item) +{ + struct pin_array_list *array_list; + TypedData_Get_Struct(obj, struct pin_array_list, &pin_array_list_type, array_list); + + if (array_list->len >= MARK_OBJECT_ARY_BUCKET_SIZE) { + obj = pin_array_list_new(obj); + TypedData_Get_Struct(obj, struct pin_array_list, &pin_array_list_type, array_list); + } + + RB_OBJ_WRITE(obj, &array_list->array[array_list->len], item); + array_list->len++; + return obj; +} + +void +rb_vm_register_global_object(VALUE obj) +{ + RUBY_ASSERT(!RB_SPECIAL_CONST_P(obj)); + if (RB_SPECIAL_CONST_P(obj)) { + return; + } + + switch (RB_BUILTIN_TYPE(obj)) { + case T_CLASS: + case T_MODULE: + if (FL_TEST(obj, RCLASS_IS_ROOT)) { + return; + } + FL_SET(obj, RCLASS_IS_ROOT); + break; + default: + break; + } + RB_VM_LOCK_ENTER(); + { + VALUE list = GET_VM()->mark_object_ary; + VALUE head = pin_array_list_append(list, obj); + if (head != list) { + GET_VM()->mark_object_ary = head; + } + RB_GC_GUARD(obj); + } + RB_VM_LOCK_LEAVE(); +} + void Init_vm_objects(void) { rb_vm_t *vm = GET_VM(); - vm->defined_module_hash = st_init_numtable(); - /* initialize mark object array, hash */ - vm->mark_object_ary = rb_ary_hidden_new(128); + vm->mark_object_ary = pin_array_list_new(Qnil); vm->loading_table = st_init_strtable(); + vm->ci_table = st_init_table(&vm_ci_hashtype); vm->frozen_strings = st_init_table_with_size(&rb_fstring_hash_type, 10000); -#if EXTSTATIC - vm->static_ext_inits = st_init_strtable(); -#endif - -#ifdef HAVE_MMAP - vm->shape_list = (rb_shape_t *)mmap(NULL, rb_size_mul_or_raise(SHAPE_BITMAP_SIZE * 32, sizeof(rb_shape_t), rb_eRuntimeError), - PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (vm->shape_list == MAP_FAILED) { - vm->shape_list = 0; - } -#else - vm->shape_list = xcalloc(SHAPE_BITMAP_SIZE * 32, sizeof(rb_shape_t)); -#endif - - if (!vm->shape_list) { - rb_memerror(); - } } /* Stub for builtin function when not building YJIT units*/ @@ -4087,6 +4454,14 @@ rb_ruby_verbose_ptr(void) return &cr->verbose; } +static bool prism; + +bool * +rb_ruby_prism_ptr(void) +{ + return &prism; +} + VALUE * rb_ruby_debug_ptr(void) { @@ -4094,6 +4469,14 @@ rb_ruby_debug_ptr(void) return &cr->debug; } +bool rb_free_at_exit = false; + +bool +ruby_free_at_exit_p(void) +{ + return rb_free_at_exit; +} + /* iseq.c */ VALUE rb_insn_operand_intern(const rb_iseq_t *iseq, VALUE insn, int op_no, VALUE op, @@ -4296,21 +4679,21 @@ usage_analysis_register_stop(VALUE self) static VALUE usage_analysis_insn_running(VALUE self) { - return RBOOL(ruby_vm_collect_usage_func_insn != 0); + return RBOOL(ruby_vm_collect_usage_func_insn != 0); } /* :nodoc: */ static VALUE usage_analysis_operand_running(VALUE self) { - return RBOOL(ruby_vm_collect_usage_func_operand != 0); + return RBOOL(ruby_vm_collect_usage_func_operand != 0); } /* :nodoc: */ static VALUE usage_analysis_register_running(VALUE self) { - return RBOOL(ruby_vm_collect_usage_func_register != 0); + return RBOOL(ruby_vm_collect_usage_func_register != 0); } static VALUE @@ -4406,18 +4789,16 @@ vm_collect_usage_register(int reg, int isset) } #endif -MJIT_FUNC_EXPORTED const struct rb_callcache * +const struct rb_callcache * rb_vm_empty_cc(void) { return &vm_empty_cc; } -MJIT_FUNC_EXPORTED const struct rb_callcache * +const struct rb_callcache * rb_vm_empty_cc_for_super(void) { return &vm_empty_cc_for_super; } -#endif /* #ifndef MJIT_HEADER */ - #include "vm_call_iseq_optimized.inc" /* required from vm_insnhelper.c */ |