diff options
Diffstat (limited to 'vm.c')
| -rw-r--r-- | vm.c | 5366 |
1 files changed, 3834 insertions, 1532 deletions
@@ -1,6 +1,6 @@ /********************************************************************** - vm.c - + Vm.c - $Author$ @@ -8,57 +8,444 @@ **********************************************************************/ -#include "ruby/ruby.h" -#include "ruby/vm.h" -#include "ruby/st.h" -#include "ruby/encoding.h" -#include "internal.h" +#define vm_exec rb_vm_exec -#include "gc.h" -#include "vm_core.h" -#include "iseq.h" #include "eval_intern.h" +#include "internal.h" +#include "internal/box.h" +#include "internal/class.h" +#include "internal/compile.h" +#include "internal/cont.h" +#include "internal/error.h" +#include "internal/encoding.h" +#include "internal/eval.h" +#include "internal/gc.h" +#include "internal/inits.h" +#include "internal/missing.h" +#include "internal/object.h" +#include "internal/proc.h" +#include "internal/re.h" +#include "internal/ruby_parser.h" +#include "internal/symbol.h" +#include "internal/thread.h" +#include "internal/transcode.h" +#include "internal/vm.h" +#include "internal/sanitizers.h" +#include "internal/variable.h" +#include "iseq.h" +#include "symbol.h" // This includes a macro for a more performant rb_id2sym. +#include "yjit.h" +#include "insns.inc" +#include "zjit.h" +#include "ruby/st.h" +#include "ruby/vm.h" +#include "vm_core.h" +#include "vm_callinfo.h" +#include "vm_debug.h" +#include "vm_exec.h" +#include "vm_insnhelper.h" +#include "ractor_core.h" +#include "vm_sync.h" +#include "shape.h" + +#include "builtin.h" + #include "probes.h" #include "probes_helper.h" -static inline VALUE * -VM_EP_LEP(VALUE *ep) +#ifdef RUBY_ASSERT_CRITICAL_SECTION +int ruby_assert_critical_section_entered = 0; +#endif + +static void *native_main_thread_stack_top; + +bool ruby_vm_during_cleanup = false; + +VALUE rb_str_concat_literals(size_t, const VALUE*); + +VALUE vm_exec(rb_execution_context_t *); + +extern const char *const rb_debug_counter_names[]; + +PUREFUNC(static inline const VALUE *VM_EP_LEP(const VALUE *)); +static inline const VALUE * +VM_EP_LEP(const VALUE *ep) { - while (!VM_EP_LEP_P(ep)) { - ep = VM_EP_PREV_EP(ep); + while (!VM_ENV_LOCAL_P(ep)) { + ep = VM_ENV_PREV_EP(ep); } return ep; } -VALUE * -rb_vm_ep_local_ep(VALUE *ep) +static inline const rb_control_frame_t * +rb_vm_search_cf_from_ep(const rb_execution_context_t *ec, const rb_control_frame_t *cfp, const VALUE * const ep) +{ + if (!ep) { + return NULL; + } + else { + const rb_control_frame_t * const eocfp = RUBY_VM_END_CONTROL_FRAME(ec); /* end of control frame pointer */ + + while (cfp < eocfp) { + if (cfp->ep == ep) { + return cfp; + } + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + } + + return NULL; + } +} + +#if VM_CHECK_MODE > 0 +// ruby_box_crashed defined in internal/box.h +#define VM_BOX_CRASHED() {ruby_box_crashed = true;} +#define VM_BOX_ASSERT(expr, msg) \ + if (!(expr)) { ruby_box_crashed = true; rb_bug(msg); } +#else +#define VM_BOX_CRASHED() {} +#define VM_BOX_ASSERT(expr, msg) ((void)0) +#endif + +static const VALUE * +VM_EP_RUBY_LEP(const rb_execution_context_t *ec, const rb_control_frame_t *current_cfp) +{ + // rb_vmdebug_box_env_dump_raw() simulates this function + const VALUE *ep = current_cfp->ep; + const rb_control_frame_t * const eocfp = RUBY_VM_END_CONTROL_FRAME(ec); /* end of control frame pointer */ + const rb_control_frame_t *cfp = current_cfp; + + if (VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_IFUNC)) { + ep = VM_EP_LEP(current_cfp->ep); + /** + * Returns CFUNC frame only in this case. + * + * Usually CFUNC frame doesn't represent the current box and it should operate + * the caller box. See the example: + * + * # in the main box + * module Kernel + * def foo = "foo" + * module_function :foo + * end + * + * In the case above, `module_function` is defined in the root box. + * If `module_function` worked in the root box, `Kernel#foo` is invisible + * from it and it causes NameError: undefined method `foo` for module `Kernel`. + * + * But in cases of IFUNC (blocks written in C), IFUNC doesn't have its own box + * and its local env frame will be CFUNC frame. + * For example, `Enumerator#chunk` calls IFUNC blocks, written as `chunk_i` function. + * + * [1].chunk{ it.even? }.each{ ... } + * + * Before calling the Ruby block `{ it.even? }`, `#chunk` calls `chunk_i` as IFUNC + * to iterate the array's members (it's just like `#each`). + * We expect that `chunk_i` works as expected by the implementation of `#chunk` + * without any overwritten definitions from boxes. + * So the definitions on IFUNC frames should be equal to the caller CFUNC. + */ + VM_ASSERT(VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_CFUNC)); + return ep; + } + + while (VM_ENV_FRAME_TYPE_P(ep, VM_FRAME_MAGIC_CFUNC)) { + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + + VM_BOX_ASSERT(cfp, "CFUNC should have a valid previous control frame"); + VM_BOX_ASSERT(cfp < eocfp, "CFUNC should have a valid caller frame"); + if (!cfp || cfp >= eocfp) { + return NULL; + } + + VM_BOX_ASSERT(cfp->ep, "CFUNC should have a valid caller frame with env"); + ep = cfp->ep; + if (!ep) { + return NULL; + } + } + + while (!VM_ENV_LOCAL_P(ep)) { + ep = VM_ENV_PREV_EP(ep); + } + + return ep; +} + +const VALUE * +rb_vm_ep_local_ep(const VALUE *ep) { return VM_EP_LEP(ep); } -static inline VALUE * -VM_CF_LEP(rb_control_frame_t *cfp) +PUREFUNC(static inline const VALUE *VM_CF_LEP(const rb_control_frame_t * const cfp)); +static inline const VALUE * +VM_CF_LEP(const rb_control_frame_t * const cfp) { return VM_EP_LEP(cfp->ep); } -static inline VALUE * -VM_CF_PREV_EP(rb_control_frame_t * cfp) +static inline const VALUE * +VM_CF_PREV_EP(const rb_control_frame_t * const cfp) +{ + return VM_ENV_PREV_EP(cfp->ep); +} + +PUREFUNC(static inline VALUE VM_CF_BLOCK_HANDLER(const rb_control_frame_t * const cfp)); +static inline VALUE +VM_CF_BLOCK_HANDLER(const rb_control_frame_t * const cfp) +{ + const VALUE *ep; + if (VM_ENV_BOXED_P(cfp->ep)) { + VM_ASSERT(VM_ENV_LOCAL_P(cfp->ep)); + /* Never set black_handler for VM_FRAME_MAGIC_TOP or VM_FRAME_MAGIC_CLASS + * and the specval is used for boxes (rb_box_t) in these case + */ + return VM_BLOCK_HANDLER_NONE; + } + ep = VM_CF_LEP(cfp); + return VM_ENV_BLOCK_HANDLER(ep); +} + +int +rb_vm_cframe_keyword_p(const rb_control_frame_t *cfp) +{ + return VM_FRAME_CFRAME_KW_P(cfp); +} + +VALUE +rb_vm_frame_block_handler(const rb_control_frame_t *cfp) +{ + return VM_CF_BLOCK_HANDLER(cfp); +} + +#if VM_CHECK_MODE > 0 +static int +VM_CFP_IN_HEAP_P(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) +{ + const VALUE *start = ec->vm_stack; + const VALUE *end = (VALUE *)ec->vm_stack + ec->vm_stack_size; + VM_ASSERT(start != NULL); + + if (start <= (VALUE *)cfp && (VALUE *)cfp < end) { + return FALSE; + } + else { + return TRUE; + } +} + +static int +VM_EP_IN_HEAP_P(const rb_execution_context_t *ec, const VALUE *ep) +{ + const VALUE *start = ec->vm_stack; + const VALUE *end = (VALUE *)ec->cfp; + VM_ASSERT(start != NULL); + + if (start <= ep && ep < end) { + return FALSE; + } + else { + return TRUE; + } +} + +static int +vm_ep_in_heap_p_(const rb_execution_context_t *ec, const VALUE *ep) +{ + if (VM_EP_IN_HEAP_P(ec, ep)) { + VALUE envval = ep[VM_ENV_DATA_INDEX_ENV]; /* VM_ENV_ENVVAL(ep); */ + + if (!UNDEF_P(envval)) { + const rb_env_t *env = (const rb_env_t *)envval; + + VM_ASSERT(imemo_type_p(envval, imemo_env)); + VM_ASSERT(VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED)); + VM_ASSERT(env->ep == ep); + } + return TRUE; + } + else { + return FALSE; + } +} + +int +rb_vm_ep_in_heap_p(const VALUE *ep) +{ + const rb_execution_context_t *ec = GET_EC(); + if (ec->vm_stack == NULL) return TRUE; + return vm_ep_in_heap_p_(ec, ep); +} +#endif + +static struct rb_captured_block * +VM_CFP_TO_CAPTURED_BLOCK(const rb_control_frame_t *cfp) +{ + VM_ASSERT(!VM_CFP_IN_HEAP_P(GET_EC(), cfp)); + return (struct rb_captured_block *)&cfp->self; +} + +static rb_control_frame_t * +VM_CAPTURED_BLOCK_TO_CFP(const struct rb_captured_block *captured) +{ + rb_control_frame_t *cfp = ((rb_control_frame_t *)((VALUE *)(captured) - 3)); + VM_ASSERT(!VM_CFP_IN_HEAP_P(GET_EC(), cfp)); + VM_ASSERT(sizeof(rb_control_frame_t)/sizeof(VALUE) == 7 + VM_DEBUG_BP_CHECK ? 1 : 0); + return cfp; +} + +static int +VM_BH_FROM_CFP_P(VALUE block_handler, const rb_control_frame_t *cfp) +{ + const struct rb_captured_block *captured = VM_CFP_TO_CAPTURED_BLOCK(cfp); + return VM_TAGGED_PTR_REF(block_handler, 0x03) == captured; +} + +static VALUE +vm_passed_block_handler(rb_execution_context_t *ec) +{ + VALUE block_handler = ec->passed_block_handler; + ec->passed_block_handler = VM_BLOCK_HANDLER_NONE; + vm_block_handler_verify(block_handler); + return block_handler; +} + +static rb_cref_t * +vm_cref_new0(VALUE klass, rb_method_visibility_t visi, int module_func, rb_cref_t *prev_cref, int pushed_by_eval, int use_prev_prev, int singleton) +{ + VALUE refinements = Qnil; + int omod_shared = FALSE; + + /* scope */ + rb_scope_visibility_t scope_visi; + scope_visi.method_visi = visi; + scope_visi.module_func = module_func; + + /* refinements */ + if (prev_cref != NULL && prev_cref != (void *)1 /* TODO: why CREF_NEXT(cref) is 1? */) { + refinements = CREF_REFINEMENTS(prev_cref); + + if (!NIL_P(refinements)) { + omod_shared = TRUE; + CREF_OMOD_SHARED_SET(prev_cref); + } + } + + VM_ASSERT(singleton || klass); + + rb_cref_t *cref = SHAREABLE_IMEMO_NEW(rb_cref_t, imemo_cref, refinements); + cref->klass_or_self = klass; + cref->next = use_prev_prev ? CREF_NEXT(prev_cref) : prev_cref; + *((rb_scope_visibility_t *)&cref->scope_visi) = scope_visi; + + if (pushed_by_eval) CREF_PUSHED_BY_EVAL_SET(cref); + if (omod_shared) CREF_OMOD_SHARED_SET(cref); + if (singleton) CREF_SINGLETON_SET(cref); + + return cref; +} + +static rb_cref_t * +vm_cref_new(VALUE klass, rb_method_visibility_t visi, int module_func, rb_cref_t *prev_cref, int pushed_by_eval, int singleton) { - return VM_EP_PREV_EP((cfp)->ep); + return vm_cref_new0(klass, visi, module_func, prev_cref, pushed_by_eval, FALSE, singleton); } -static inline rb_block_t * -VM_CF_BLOCK_PTR(rb_control_frame_t *cfp) +static rb_cref_t * +vm_cref_new_use_prev(VALUE klass, rb_method_visibility_t visi, int module_func, rb_cref_t *prev_cref, int pushed_by_eval) { - VALUE *ep = VM_CF_LEP(cfp); - return VM_EP_BLOCK_PTR(ep); + return vm_cref_new0(klass, visi, module_func, prev_cref, pushed_by_eval, TRUE, FALSE); } -rb_block_t * -rb_vm_control_frame_block_ptr(rb_control_frame_t *cfp) +static int +ref_delete_symkey(VALUE key, VALUE value, VALUE unused) { - return VM_CF_BLOCK_PTR(cfp); + return SYMBOL_P(key) ? ST_DELETE : ST_CONTINUE; +} + +static rb_cref_t * +vm_cref_dup(const rb_cref_t *cref) +{ + const rb_scope_visibility_t *visi = CREF_SCOPE_VISI(cref); + rb_cref_t *next_cref = CREF_NEXT(cref), *new_cref; + int pushed_by_eval = CREF_PUSHED_BY_EVAL(cref); + int singleton = CREF_SINGLETON(cref); + + new_cref = vm_cref_new(cref->klass_or_self, visi->method_visi, visi->module_func, next_cref, pushed_by_eval, singleton); + + if (!NIL_P(CREF_REFINEMENTS(cref))) { + VALUE ref = rb_hash_dup(CREF_REFINEMENTS(cref)); + rb_hash_foreach(ref, ref_delete_symkey, Qnil); + CREF_REFINEMENTS_SET(new_cref, ref); + CREF_OMOD_SHARED_UNSET(new_cref); + } + + return new_cref; +} + + +rb_cref_t * +rb_vm_cref_dup_without_refinements(const rb_cref_t *cref) +{ + const rb_scope_visibility_t *visi = CREF_SCOPE_VISI(cref); + rb_cref_t *next_cref = CREF_NEXT(cref), *new_cref; + int pushed_by_eval = CREF_PUSHED_BY_EVAL(cref); + int singleton = CREF_SINGLETON(cref); + + new_cref = vm_cref_new(cref->klass_or_self, visi->method_visi, visi->module_func, next_cref, pushed_by_eval, singleton); + + if (!NIL_P(CREF_REFINEMENTS(cref))) { + CREF_REFINEMENTS_SET(new_cref, Qnil); + CREF_OMOD_SHARED_UNSET(new_cref); + } + + return new_cref; +} + +static rb_cref_t * +vm_cref_new_toplevel(rb_execution_context_t *ec) +{ + rb_cref_t *cref = vm_cref_new(rb_cObject, METHOD_VISI_PRIVATE /* toplevel visibility is private */, FALSE, NULL, FALSE, FALSE); + VALUE top_wrapper = rb_ec_thread_ptr(ec)->top_wrapper; + + if (top_wrapper) { + cref = vm_cref_new(top_wrapper, METHOD_VISI_PRIVATE, FALSE, cref, FALSE, FALSE); + } + + return cref; +} + +rb_cref_t * +rb_vm_cref_new_toplevel(void) +{ + return vm_cref_new_toplevel(GET_EC()); +} + +static void +vm_cref_dump(const char *mesg, const rb_cref_t *cref) +{ + ruby_debug_printf("vm_cref_dump: %s (%p)\n", mesg, (void *)cref); + + while (cref) { + ruby_debug_printf("= cref| klass: %s\n", RSTRING_PTR(rb_class_path(CREF_CLASS(cref)))); + cref = CREF_NEXT(cref); + } +} + +void +rb_vm_block_ep_update(VALUE obj, const struct rb_block *dst, const VALUE *ep) +{ + *((const VALUE **)&dst->as.captured.ep) = ep; + RB_OBJ_WRITTEN(obj, Qundef, VM_ENV_ENVVAL(ep)); +} + +static void +vm_bind_update_env(VALUE bindval, rb_binding_t *bind, VALUE envval) +{ + const rb_env_t *env = (rb_env_t *)envval; + RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, env->iseq); + rb_vm_block_ep_update(bindval, &bind->block, env->ep); } #if VM_COLLECT_USAGE_DETAILS @@ -67,783 +454,1705 @@ static void vm_collect_usage_insn(int insn); static void vm_collect_usage_register(int reg, int isset); #endif -static VALUE -vm_invoke_proc(rb_thread_t *th, rb_proc_t *proc, VALUE self, VALUE defined_class, - int argc, const VALUE *argv, const rb_block_t *blockptr); +static VALUE vm_make_env_object(const rb_execution_context_t *ec, rb_control_frame_t *cfp); +static VALUE vm_invoke_bmethod(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self, + int argc, const VALUE *argv, int kw_splat, VALUE block_handler, + const rb_callable_method_entry_t *me); +static VALUE vm_invoke_proc(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE block_handler); + +#if USE_YJIT +// Counter to serve as a proxy for execution time, total number of calls +static uint64_t yjit_total_entry_hits = 0; + +// Number of calls used to estimate how hot an ISEQ is +#define YJIT_CALL_COUNT_INTERV 20u + +/// Test whether we are ready to compile an ISEQ or not +static inline bool +rb_yjit_threshold_hit(const rb_iseq_t *iseq, uint64_t entry_calls) +{ + yjit_total_entry_hits += 1; + + // Record the number of calls at the beginning of the interval + if (entry_calls + YJIT_CALL_COUNT_INTERV == rb_yjit_call_threshold) { + iseq->body->yjit_calls_at_interv = yjit_total_entry_hits; + } + + // Try to estimate the total time taken (total number of calls) to reach 20 calls to this ISEQ + // This give us a ratio of how hot/cold this ISEQ is + if (entry_calls == rb_yjit_call_threshold) { + // We expect threshold 1 to compile everything immediately + if (rb_yjit_call_threshold < YJIT_CALL_COUNT_INTERV) { + return true; + } + + uint64_t num_calls = yjit_total_entry_hits - iseq->body->yjit_calls_at_interv; + + // Reject ISEQs that don't get called often enough + if (num_calls > rb_yjit_cold_threshold) { + rb_yjit_incr_counter("cold_iseq_entry"); + return false; + } + + return true; + } + + return false; +} +#else +#define rb_yjit_threshold_hit(iseq, entry_calls) false +#endif + +#if USE_YJIT +// Generate JIT code that supports the following kinds of ISEQ entries: +// * The first ISEQ on vm_exec (e.g. <main>, or Ruby methods/blocks +// called by a C method). The current frame has VM_FRAME_FLAG_FINISH. +// The current vm_exec stops if JIT code returns a non-Qundef value. +// * ISEQs called by the interpreter on vm_sendish (e.g. Ruby methods or +// blocks called by a Ruby frame that isn't compiled or side-exited). +// The current frame doesn't have VM_FRAME_FLAG_FINISH. The current +// vm_exec does NOT stop whether JIT code returns Qundef or not. +static inline rb_jit_func_t +yjit_compile(rb_execution_context_t *ec) +{ + const rb_iseq_t *iseq = ec->cfp->iseq; + struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); + + // Increment the ISEQ's call counter and trigger JIT compilation if not compiled + if (body->jit_entry == NULL) { + body->jit_entry_calls++; + if (rb_yjit_threshold_hit(iseq, body->jit_entry_calls)) { + rb_yjit_compile_iseq(iseq, ec, false); + } + } + return body->jit_entry; +} +#else +# define yjit_compile(ec) ((rb_jit_func_t)0) +#endif + +#if USE_ZJIT +static inline rb_jit_func_t +zjit_compile(rb_execution_context_t *ec) +{ + const rb_iseq_t *iseq = ec->cfp->iseq; + struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); + + if (body->jit_entry == NULL) { + body->jit_entry_calls++; + + // At profile-threshold, rewrite some of the YARV instructions + // to zjit_* instructions to profile these instructions. + if (body->jit_entry_calls == rb_zjit_profile_threshold) { + rb_zjit_profile_enable(iseq); + } + + // At call-threshold, compile the ISEQ with ZJIT. + if (body->jit_entry_calls == rb_zjit_call_threshold) { + rb_zjit_compile_iseq(iseq, false); + } + } + return body->jit_entry; +} +#else +# define zjit_compile(ec) ((rb_jit_func_t)0) +#endif + +// Execute JIT code compiled by yjit_compile() or zjit_compile() +static inline VALUE +jit_exec(rb_execution_context_t *ec) +{ +#if USE_YJIT + if (rb_yjit_enabled_p) { + rb_jit_func_t func = yjit_compile(ec); + if (func) { + return func(ec, ec->cfp); + } + return Qundef; + } +#endif + +#if USE_ZJIT + void *zjit_entry = rb_zjit_entry; + if (zjit_entry) { + rb_jit_func_t func = zjit_compile(ec); + if (func) { + return ((rb_zjit_func_t)zjit_entry)(ec, ec->cfp, func); + } + } +#endif + return Qundef; +} + +#if USE_YJIT || USE_ZJIT +// Generate JIT code that supports the following kind of ISEQ entry: +// * The first ISEQ pushed by vm_exec_handle_exception. The frame would +// point to a location specified by a catch table, and it doesn't have +// VM_FRAME_FLAG_FINISH. The current vm_exec stops if JIT code returns +// a non-Qundef value. So you should not return a non-Qundef value +// until ec->cfp is changed to a frame with VM_FRAME_FLAG_FINISH. +static inline rb_jit_func_t +jit_compile_exception(rb_execution_context_t *ec) +{ + const rb_iseq_t *iseq = ec->cfp->iseq; + struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); + +#if USE_ZJIT + if (body->jit_exception == NULL && rb_zjit_enabled_p) { + body->jit_exception_calls++; + + // At profile-threshold, rewrite some of the YARV instructions + // to zjit_* instructions to profile these instructions. + if (body->jit_exception_calls == rb_zjit_profile_threshold) { + rb_zjit_profile_enable(iseq); + } + + // At call-threshold, compile the ISEQ with ZJIT. + if (body->jit_exception_calls == rb_zjit_call_threshold) { + rb_zjit_compile_iseq(iseq, true); + } + } +#endif + +#if USE_YJIT + // Increment the ISEQ's call counter and trigger JIT compilation if not compiled + if (body->jit_exception == NULL && rb_yjit_enabled_p) { + body->jit_exception_calls++; + if (body->jit_exception_calls == rb_yjit_call_threshold) { + rb_yjit_compile_iseq(iseq, ec, true); + } + } +#endif + return body->jit_exception; +} + +// Execute JIT code compiled by jit_compile_exception() +static inline VALUE +jit_exec_exception(rb_execution_context_t *ec) +{ + rb_jit_func_t func = jit_compile_exception(ec); + if (func) { + // Call the JIT code + return func(ec, ec->cfp); + } + else { + return Qundef; + } +} +#else +# define jit_compile_exception(ec) ((rb_jit_func_t)0) +# define jit_exec_exception(ec) Qundef +#endif + +static void add_opt_method_entry(const rb_method_entry_t *me); + +#define RB_TYPE_2_P(obj, type1, type2) \ + (RB_TYPE_P(obj, type1) || RB_TYPE_P(obj, type2)) +#define RB_TYPE_3_P(obj, type1, type2, type3) \ + (RB_TYPE_P(obj, type1) || RB_TYPE_P(obj, type2) || RB_TYPE_P(obj, type3)) + +#define VM_ASSERT_TYPE(obj, type) \ + VM_ASSERT(RB_TYPE_P(obj, type), #obj ": %s", rb_obj_info(obj)) +#define VM_ASSERT_TYPE2(obj, type1, type2) \ + VM_ASSERT(RB_TYPE_2_P(obj, type1, type2), #obj ": %s", rb_obj_info(obj)) +#define VM_ASSERT_TYPE3(obj, type1, type2, type3) \ + VM_ASSERT(RB_TYPE_3_P(obj, type1, type2, type3), #obj ": %s", rb_obj_info(obj)) -#include "vm_insnhelper.h" #include "vm_insnhelper.c" -#include "vm_exec.h" + #include "vm_exec.c" #include "vm_method.c" #include "vm_eval.c" -#include <assert.h> - -#define BUFSIZE 0x100 #define PROCDEBUG 0 VALUE rb_cRubyVM; VALUE rb_cThread; -VALUE rb_cEnv; VALUE rb_mRubyVMFrozenCore; +VALUE rb_block_param_proxy; VALUE ruby_vm_const_missing_count = 0; -char ruby_vm_redefined_flag[BOP_LAST_]; -rb_thread_t *ruby_current_thread = 0; -rb_vm_t *ruby_current_vm = 0; -rb_event_flag_t ruby_vm_event_flags; +rb_vm_t *ruby_current_vm_ptr = NULL; +rb_ractor_t *ruby_single_main_ractor; +bool ruby_vm_keep_script_lines; -static void thread_free(void *ptr); +#ifdef RB_THREAD_LOCAL_SPECIFIER +RB_THREAD_LOCAL_SPECIFIER rb_execution_context_t *ruby_current_ec; + +#ifdef RUBY_NT_SERIAL +RB_THREAD_LOCAL_SPECIFIER rb_atomic_t ruby_nt_serial; +#endif + +// no-inline decl on vm_core.h +rb_execution_context_t * +rb_current_ec_noinline(void) +{ + return ruby_current_ec; +} void -rb_vm_change_state(void) +rb_current_ec_set(rb_execution_context_t *ec) { - INC_VM_STATE_VERSION(); + ruby_current_ec = ec; } -static void vm_clear_global_method_cache(void); -static void -vm_clear_all_inline_method_cache(void) +#ifdef RB_THREAD_CURRENT_EC_NOINLINE +rb_execution_context_t * +rb_current_ec(void) { - /* TODO: Clear all inline cache entries in all iseqs. - How to iterate all iseqs in sweep phase? - rb_objspace_each_objects() doesn't work at sweep phase. - */ + return ruby_current_ec; } -static void -vm_clear_all_cache() +#endif +#else +native_tls_key_t ruby_current_ec_key; + +// no-inline decl on vm_core.h +rb_execution_context_t * +rb_current_ec_noinline(void) { - vm_clear_global_method_cache(); - vm_clear_all_inline_method_cache(); - ruby_vm_global_state_version = 1; + return native_tls_get(ruby_current_ec_key); } +#endif + +rb_event_flag_t ruby_vm_event_flags = 0; +rb_event_flag_t ruby_vm_event_enabled_global_flags = 0; +unsigned int ruby_vm_c_events_enabled = 0; +unsigned int ruby_vm_iseq_events_enabled = 0; + +rb_serial_t ruby_vm_constant_cache_invalidations = 0; +rb_serial_t ruby_vm_constant_cache_misses = 0; +rb_serial_t ruby_vm_global_cvar_state = 1; + +static const struct rb_callcache vm_empty_cc = { + .flags = T_IMEMO | (imemo_callcache << FL_USHIFT) | VM_CALLCACHE_UNMARKABLE, + .klass = Qundef, + .cme_ = NULL, + .call_ = vm_call_general, + .aux_ = { + .v = Qfalse, + } +}; + +static const struct rb_callcache vm_empty_cc_for_super = { + .flags = T_IMEMO | (imemo_callcache << FL_USHIFT) | VM_CALLCACHE_UNMARKABLE, + .klass = Qundef, + .cme_ = NULL, + .call_ = vm_call_super_method, + .aux_ = { + .v = Qfalse, + } +}; + +static void thread_free(void *ptr); + void rb_vm_inc_const_missing_count(void) { ruby_vm_const_missing_count +=1; } +int +rb_dtrace_setup(rb_execution_context_t *ec, VALUE klass, ID id, + struct ruby_dtrace_method_hook_args *args) +{ + enum ruby_value_type type; + if (!klass) { + if (!ec) ec = GET_EC(); + if (!rb_ec_frame_method_id_and_class(ec, &id, 0, &klass) || !klass) + return FALSE; + } + if (RB_TYPE_P(klass, T_ICLASS)) { + klass = RBASIC(klass)->klass; + } + else if (RCLASS_SINGLETON_P(klass)) { + klass = RCLASS_ATTACHED_OBJECT(klass); + if (NIL_P(klass)) return FALSE; + } + type = BUILTIN_TYPE(klass); + if (type == T_CLASS || type == T_ICLASS || type == T_MODULE) { + VALUE name = rb_class_path(klass); + const char *classname, *filename; + const char *methodname = rb_id2name(id); + if (methodname && (filename = rb_source_location_cstr(&args->line_no)) != 0) { + if (NIL_P(name) || !(classname = StringValuePtr(name))) + classname = "<unknown>"; + args->classname = classname; + args->methodname = methodname; + args->filename = filename; + args->klass = klass; + args->name = name; + return TRUE; + } + } + return FALSE; +} + +extern unsigned int redblack_buffer_size; + +/* + * call-seq: + * RubyVM.stat -> Hash + * RubyVM.stat(hsh) -> hsh + * RubyVM.stat(Symbol) -> Numeric + * + * Returns a Hash containing implementation-dependent counters inside the VM. + * + * This hash includes information about method/constant caches: + * + * { + * :constant_cache_invalidations=>2, + * :constant_cache_misses=>14, + * :global_cvar_state=>27 + * } + * + * If <tt>USE_DEBUG_COUNTER</tt> is enabled, debug counters will be included. + * + * The contents of the hash are implementation specific and may be changed in + * the future. + * + * This method is only expected to work on C Ruby. + */ +static VALUE +vm_stat(int argc, VALUE *argv, VALUE self) +{ + static VALUE sym_constant_cache_invalidations, sym_constant_cache_misses, sym_global_cvar_state, sym_next_shape_id; + static VALUE sym_shape_cache_size; + VALUE arg = Qnil; + VALUE hash = Qnil, key = Qnil; + + if (rb_check_arity(argc, 0, 1) == 1) { + arg = argv[0]; + if (SYMBOL_P(arg)) + key = arg; + else if (RB_TYPE_P(arg, T_HASH)) + hash = arg; + else + rb_raise(rb_eTypeError, "non-hash or symbol given"); + } + else { + hash = rb_hash_new(); + } + +#define S(s) sym_##s = ID2SYM(rb_intern_const(#s)) + S(constant_cache_invalidations); + S(constant_cache_misses); + S(global_cvar_state); + S(next_shape_id); + S(shape_cache_size); +#undef S + +#define SET(name, attr) \ + if (key == sym_##name) \ + return SERIALT2NUM(attr); \ + else if (hash != Qnil) \ + rb_hash_aset(hash, sym_##name, SERIALT2NUM(attr)); + + SET(constant_cache_invalidations, ruby_vm_constant_cache_invalidations); + SET(constant_cache_misses, ruby_vm_constant_cache_misses); + SET(global_cvar_state, ruby_vm_global_cvar_state); + SET(next_shape_id, (rb_serial_t)rb_shapes_count()); + SET(shape_cache_size, (rb_serial_t)rb_shape_tree.cache_size); +#undef SET + +#if USE_DEBUG_COUNTER + ruby_debug_counter_show_at_exit(FALSE); + for (size_t i = 0; i < RB_DEBUG_COUNTER_MAX; i++) { + const VALUE name = rb_sym_intern_ascii_cstr(rb_debug_counter_names[i]); + const VALUE boxed_value = SIZET2NUM(rb_debug_counter[i]); + + if (key == name) { + return boxed_value; + } + else if (hash != Qnil) { + rb_hash_aset(hash, name, boxed_value); + } + } +#endif + + if (!NIL_P(key)) { /* matched key should return above */ + rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key)); + } + + return hash; +} + /* control stack frame */ static void -vm_set_top_stack(rb_thread_t * th, VALUE iseqval) +vm_set_top_stack(rb_execution_context_t *ec, const rb_iseq_t *iseq, const rb_box_t *box) { - rb_iseq_t *iseq; - GetISeqPtr(iseqval, iseq); - - if (iseq->type != ISEQ_TYPE_TOP) { - rb_raise(rb_eTypeError, "Not a toplevel InstructionSequence"); + if (ISEQ_BODY(iseq)->type != ISEQ_TYPE_TOP) { + rb_raise(rb_eTypeError, "Not a toplevel InstructionSequence"); } /* for return */ - CHECK_VM_STACK_OVERFLOW(th->cfp, iseq->local_size + iseq->stack_max); - vm_push_frame(th, iseq, VM_FRAME_MAGIC_TOP | VM_FRAME_FLAG_FINISH, - th->top_self, rb_cObject, VM_ENVVAL_BLOCK_PTR(0), - iseq->iseq_encoded, th->cfp->sp, iseq->local_size, 0); + vm_push_frame(ec, iseq, VM_FRAME_MAGIC_TOP | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH, + rb_ec_thread_ptr(ec)->top_self, + GC_GUARDED_PTR(box), + (VALUE)vm_cref_new_toplevel(ec), /* cref or me */ + ISEQ_BODY(iseq)->iseq_encoded, ec->cfp->sp, + ISEQ_BODY(iseq)->local_table_size, ISEQ_BODY(iseq)->stack_max); } static void -vm_set_eval_stack(rb_thread_t * th, VALUE iseqval, const NODE *cref, rb_block_t *base_block) +vm_set_eval_stack(rb_execution_context_t *ec, const rb_iseq_t *iseq, const rb_cref_t *cref, const struct rb_block *base_block) { - rb_iseq_t *iseq; - GetISeqPtr(iseqval, iseq); - - CHECK_VM_STACK_OVERFLOW(th->cfp, iseq->local_size + iseq->stack_max); - vm_push_frame(th, iseq, VM_FRAME_MAGIC_EVAL | VM_FRAME_FLAG_FINISH, - base_block->self, base_block->klass, - VM_ENVVAL_PREV_EP_PTR(base_block->ep), iseq->iseq_encoded, - th->cfp->sp, iseq->local_size, 0); - - if (cref) { - th->cfp->ep[-1] = (VALUE)cref; - } + vm_push_frame(ec, iseq, VM_FRAME_MAGIC_EVAL | VM_FRAME_FLAG_FINISH, + vm_block_self(base_block), VM_GUARDED_PREV_EP(vm_block_ep(base_block)), + (VALUE)cref, /* cref or me */ + ISEQ_BODY(iseq)->iseq_encoded, + ec->cfp->sp, ISEQ_BODY(iseq)->local_table_size, + ISEQ_BODY(iseq)->stack_max); } static void -vm_set_main_stack(rb_thread_t *th, VALUE iseqval) +vm_set_main_stack(rb_execution_context_t *ec, const rb_iseq_t *iseq) { VALUE toplevel_binding = rb_const_get(rb_cObject, rb_intern("TOPLEVEL_BINDING")); rb_binding_t *bind; - rb_iseq_t *iseq; - rb_env_t *env; GetBindingPtr(toplevel_binding, bind); - GetEnvPtr(bind->env, env); - vm_set_eval_stack(th, iseqval, 0, &env->block); + RUBY_ASSERT_MESG(bind, "TOPLEVEL_BINDING is not built"); + + vm_set_eval_stack(ec, iseq, 0, &bind->block); /* save binding */ - GetISeqPtr(iseqval, iseq); - if (bind && iseq->local_size > 0) { - bind->env = rb_vm_make_env_object(th, th->cfp); + if (ISEQ_BODY(iseq)->local_table_size > 0) { + vm_bind_update_env(toplevel_binding, bind, vm_make_env_object(ec, ec->cfp)); } } rb_control_frame_t * -rb_vm_get_binding_creatable_next_cfp(rb_thread_t *th, const rb_control_frame_t *cfp) +rb_vm_get_binding_creatable_next_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { - while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(th, cfp)) { - if (cfp->iseq) { - return (rb_control_frame_t *)cfp; - } - cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) { + if (cfp->iseq) { + return (rb_control_frame_t *)cfp; + } + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); } return 0; } rb_control_frame_t * -rb_vm_get_ruby_level_next_cfp(rb_thread_t *th, const rb_control_frame_t *cfp) +rb_vm_get_ruby_level_next_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { - while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(th, cfp)) { - if (RUBY_VM_NORMAL_ISEQ_P(cfp->iseq)) { - return (rb_control_frame_t *)cfp; - } - cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) { + if (VM_FRAME_RUBYFRAME_P(cfp)) { + return (rb_control_frame_t *)cfp; + } + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); } return 0; } static rb_control_frame_t * -vm_get_ruby_level_caller_cfp(rb_thread_t *th, rb_control_frame_t *cfp) +vm_get_ruby_level_caller_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { - if (RUBY_VM_NORMAL_ISEQ_P(cfp->iseq)) { - return cfp; + if (VM_FRAME_RUBYFRAME_P(cfp)) { + return (rb_control_frame_t *)cfp; } cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); - while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(th, cfp)) { - if (RUBY_VM_NORMAL_ISEQ_P(cfp->iseq)) { - return cfp; - } + while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) { + if (VM_FRAME_RUBYFRAME_P(cfp)) { + return (rb_control_frame_t *)cfp; + } - if ((cfp->flag & VM_FRAME_FLAG_PASSED) == 0) { - break; - } - cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + if (VM_ENV_FLAGS(cfp->ep, VM_FRAME_FLAG_PASSED) == FALSE) { + break; + } + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); } return 0; } -/* at exit */ - void -ruby_vm_at_exit(void (*func)(rb_vm_t *)) +rb_vm_pop_cfunc_frame(void) { - rb_ary_push((VALUE)&GET_VM()->at_exit, (VALUE)func); + rb_execution_context_t *ec = GET_EC(); + rb_control_frame_t *cfp = ec->cfp; + const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(cfp); + + EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, cfp->self, me->def->original_id, me->called_id, me->owner, Qnil); + RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, me->owner, me->def->original_id); + vm_pop_frame(ec, cfp, cfp->ep); } -static void -ruby_vm_run_at_exit_hooks(rb_vm_t *vm) +void +rb_vm_rewind_cfp(rb_execution_context_t *ec, rb_control_frame_t *cfp) { - VALUE hook = (VALUE)&vm->at_exit; - - while (RARRAY_LEN(hook) > 0) { - typedef void rb_vm_at_exit_func(rb_vm_t*); - rb_vm_at_exit_func *func = (rb_vm_at_exit_func*)rb_ary_pop(hook); - (*func)(vm); + /* check skipped frame */ + while (ec->cfp != cfp) { +#if VMDEBUG + printf("skipped frame: %s\n", vm_frametype_name(ec->cfp)); +#endif + if (VM_FRAME_TYPE(ec->cfp) != VM_FRAME_MAGIC_CFUNC) { + rb_vm_pop_frame(ec); + } + else { /* unlikely path */ + rb_vm_pop_cfunc_frame(); + } } - rb_ary_free(hook); } -/* Env */ - -/* - env{ - env[0] // special (block or prev env) - env[1] // env object - }; - */ - -#define ENV_IN_HEAP_P(th, env) \ - (!((th)->stack <= (env) && (env) < ((th)->stack + (th)->stack_size))) -#define ENV_VAL(env) ((env)[1]) +/* at exit */ -static void -env_mark(void * const ptr) +void +ruby_vm_at_exit(void (*func)(rb_vm_t *)) { - RUBY_MARK_ENTER("env"); - if (ptr) { - const rb_env_t * const env = ptr; - - if (env->env) { - /* TODO: should mark more restricted range */ - RUBY_GC_INFO("env->env\n"); - rb_gc_mark_locations(env->env, env->env + env->env_size); - } - - RUBY_GC_INFO("env->prev_envval\n"); - RUBY_MARK_UNLESS_NULL(env->prev_envval); - RUBY_MARK_UNLESS_NULL(env->block.self); - RUBY_MARK_UNLESS_NULL(env->block.proc); - - if (env->block.iseq) { - if (BUILTIN_TYPE(env->block.iseq) == T_NODE) { - RUBY_MARK_UNLESS_NULL((VALUE)env->block.iseq); - } - else { - RUBY_MARK_UNLESS_NULL(env->block.iseq->self); - } - } - } - RUBY_MARK_LEAVE("env"); + rb_vm_t *vm = GET_VM(); + rb_at_exit_list *nl = ALLOC(rb_at_exit_list); + nl->func = func; + nl->next = vm->at_exit; + vm->at_exit = nl; } static void -env_free(void * const ptr) +ruby_vm_run_at_exit_hooks(rb_vm_t *vm) { - RUBY_FREE_ENTER("env"); - if (ptr) { - rb_env_t *const env = ptr; - RUBY_FREE_UNLESS_NULL(env->env); - ruby_xfree(ptr); - } - RUBY_FREE_LEAVE("env"); -} + rb_at_exit_list *l = vm->at_exit; -static size_t -env_memsize(const void *ptr) -{ - if (ptr) { - const rb_env_t * const env = ptr; - size_t size = sizeof(rb_env_t); - if (env->env) { - size += env->env_size * sizeof(VALUE); - } - return size; + while (l) { + rb_at_exit_list* t = l->next; + rb_vm_at_exit_func *func = l->func; + ruby_xfree(l); + l = t; + (*func)(vm); } - return 0; } -static const rb_data_type_t env_data_type = { - "VM/env", - {env_mark, env_free, env_memsize,}, -}; - -static VALUE -env_alloc(void) -{ - VALUE obj; - rb_env_t *env; - obj = TypedData_Make_Struct(rb_cEnv, rb_env_t, &env_data_type, env); - env->env = 0; - env->prev_envval = 0; - env->block.iseq = 0; - return obj; -} +/* Env */ -static VALUE check_env_value(VALUE envval); +static VALUE check_env_value(const rb_env_t *env); static int -check_env(rb_env_t * const env) -{ - fprintf(stderr, "---\n"); - fprintf(stderr, "envptr: %p\n", (void *)&env->block.ep[0]); - fprintf(stderr, "envval: %10p ", (void *)env->block.ep[1]); - dp(env->block.ep[1]); - fprintf(stderr, "ep: %10p\n", (void *)env->block.ep); - if (env->prev_envval) { - fprintf(stderr, ">>\n"); - check_env_value(env->prev_envval); - fprintf(stderr, "<<\n"); +check_env(const rb_env_t *env) +{ + fputs("---\n", stderr); + ruby_debug_printf("envptr: %p\n", (void *)&env->ep[0]); + ruby_debug_printf("envval: %10p ", (void *)env->ep[1]); + dp(env->ep[1]); + ruby_debug_printf("ep: %10p\n", (void *)env->ep); + if (rb_vm_env_prev_env(env)) { + fputs(">>\n", stderr); + check_env_value(rb_vm_env_prev_env(env)); + fputs("<<\n", stderr); } return 1; } static VALUE -check_env_value(VALUE envval) +check_env_value(const rb_env_t *env) { - rb_env_t *env; - GetEnvPtr(envval, env); - if (check_env(env)) { - return envval; + return (VALUE)env; } rb_bug("invalid env"); return Qnil; /* unreachable */ } static VALUE -vm_make_env_each(rb_thread_t * const th, rb_control_frame_t * const cfp, - VALUE *envptr, VALUE * const endptr) +vm_block_handler_escape(const rb_execution_context_t *ec, VALUE block_handler) { - VALUE envval, penvval = 0; - rb_env_t *env; - VALUE *nenvptr; - int i, local_size; + switch (vm_block_handler_type(block_handler)) { + case block_handler_type_ifunc: + case block_handler_type_iseq: + return rb_vm_make_proc(ec, VM_BH_TO_CAPT_BLOCK(block_handler), rb_cProc); - if (ENV_IN_HEAP_P(th, envptr)) { - return ENV_VAL(envptr); + case block_handler_type_symbol: + case block_handler_type_proc: + return block_handler; } + VM_UNREACHABLE(vm_block_handler_escape); + return Qnil; +} - if (envptr != endptr) { - VALUE *penvptr = GC_GUARDED_PTR_REF(*envptr); - rb_control_frame_t *pcfp = cfp; +static VALUE +vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *const cfp) +{ + const VALUE * const ep = cfp->ep; + VALUE *env_body, *env_ep; + int local_size, env_size; - if (ENV_IN_HEAP_P(th, penvptr)) { - penvval = ENV_VAL(penvptr); - } - else { - while (pcfp->ep != penvptr) { - pcfp++; - if (pcfp->ep == 0) { - SDR(); - rb_bug("invalid ep"); - } - } - penvval = vm_make_env_each(th, pcfp, penvptr, endptr); - *envptr = VM_ENVVAL_PREV_EP_PTR(pcfp->ep); - } + if (VM_ENV_ESCAPED_P(ep)) { + return VM_ENV_ENVVAL(ep); } - /* allocate env */ - envval = env_alloc(); - GetEnvPtr(envval, env); + if (!VM_ENV_LOCAL_P(ep)) { + const VALUE *prev_ep = VM_ENV_PREV_EP(ep); + if (!VM_ENV_ESCAPED_P(prev_ep)) { + rb_control_frame_t *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); - if (!RUBY_VM_NORMAL_ISEQ_P(cfp->iseq)) { - local_size = 2; + while (prev_cfp->ep != prev_ep) { + prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(prev_cfp); + VM_ASSERT(prev_cfp->ep != NULL); + } + + vm_make_env_each(ec, prev_cfp); + VM_FORCE_WRITE_SPECIAL_CONST(&ep[VM_ENV_DATA_INDEX_SPECVAL], VM_GUARDED_PREV_EP(prev_cfp->ep)); + } } else { - local_size = cfp->iseq->local_size; + VM_ASSERT(VM_ENV_LOCAL_P(ep)); + VALUE block_handler = VM_ENV_BLOCK_HANDLER(ep); + + if (block_handler != VM_BLOCK_HANDLER_NONE) { + VALUE blockprocval = vm_block_handler_escape(ec, block_handler); + VM_STACK_ENV_WRITE(ep, VM_ENV_DATA_INDEX_SPECVAL, blockprocval); + } } - env->env_size = local_size + 1 + 1; - env->local_size = local_size; - env->env = ALLOC_N(VALUE, env->env_size); - env->prev_envval = penvval; + if (!VM_FRAME_RUBYFRAME_P(cfp)) { + local_size = VM_ENV_DATA_SIZE; + } + else { + local_size = ISEQ_BODY(cfp->iseq)->local_table_size; + if (ISEQ_BODY(cfp->iseq)->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) { + int ci_offset = local_size - ISEQ_BODY(cfp->iseq)->param.size + VM_ENV_DATA_SIZE; + + CALL_INFO ci = (CALL_INFO)VM_CF_LEP(cfp)[-ci_offset]; + local_size += vm_ci_argc(ci); + } + local_size += VM_ENV_DATA_SIZE; + } + + // Invalidate JIT code that assumes cfp->ep == vm_base_ptr(cfp). + // This is done before creating the imemo_env because VM_STACK_ENV_WRITE + // below leaves the on-stack ep in a state that is unsafe to GC. + if (VM_FRAME_RUBYFRAME_P(cfp)) { + rb_yjit_invalidate_ep_is_bp(cfp->iseq); + rb_zjit_invalidate_no_ep_escape(cfp->iseq); + } + + /* + * # local variables on a stack frame (N == local_size) + * [lvar1, lvar2, ..., lvarN, SPECVAL] + * ^ + * ep[0] + * + * # moved local variables + * [lvar1, lvar2, ..., lvarN, SPECVAL, Envval, BlockProcval (if needed)] + * ^ ^ + * env->env[0] ep[0] + */ + + env_size = local_size + + 1 /* envval */; + + // Careful with order in the following sequence. Each allocation can move objects. + env_body = ALLOC_N(VALUE, env_size); + rb_env_t *env = IMEMO_NEW(rb_env_t, imemo_env, 0); + + // Set up env without WB since it's brand new (similar to newobj_init(), newobj_fill()) + MEMCPY(env_body, ep - (local_size - 1 /* specval */), VALUE, local_size); + + env_ep = &env_body[local_size - 1 /* specval */]; + env_ep[VM_ENV_DATA_INDEX_ENV] = (VALUE)env; + + env->iseq = (rb_iseq_t *)(VM_FRAME_RUBYFRAME_P(cfp) ? cfp->iseq : NULL); + env->ep = env_ep; + env->env = env_body; + env->env_size = env_size; + + cfp->ep = env_ep; + VM_ENV_FLAGS_SET(env_ep, VM_ENV_FLAG_ESCAPED | VM_ENV_FLAG_WB_REQUIRED); + VM_STACK_ENV_WRITE(ep, 0, (VALUE)env); /* GC mark */ - for (i = 0; i <= local_size; i++) { - env->env[i] = envptr[-local_size + i]; #if 0 - fprintf(stderr, "%2d ", &envptr[-local_size + i] - th->stack); dp(env->env[i]); - if (RUBY_VM_NORMAL_ISEQ_P(cfp->iseq)) { - /* clear value stack for GC */ - envptr[-local_size + i] = 0; - } + for (i = 0; i < local_size; i++) { + if (VM_FRAME_RUBYFRAME_P(cfp)) { + /* clear value stack for GC */ + ep[-local_size + i] = 0; + } + } #endif + + return (VALUE)env; +} + +static VALUE +vm_make_env_object(const rb_execution_context_t *ec, rb_control_frame_t *cfp) +{ + VALUE envval = vm_make_env_each(ec, cfp); + + if (PROCDEBUG) { + check_env_value((const rb_env_t *)envval); } - *envptr = envval; /* GC mark */ - nenvptr = &env->env[i - 1]; - nenvptr[1] = envval; /* frame self */ + return envval; +} - /* reset ep in cfp */ - cfp->ep = nenvptr; +void +rb_vm_stack_to_heap(rb_execution_context_t *ec) +{ + rb_control_frame_t *cfp = ec->cfp; + while ((cfp = rb_vm_get_binding_creatable_next_cfp(ec, cfp)) != 0) { + vm_make_env_object(ec, cfp); + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + } +} - /* as Binding */ - env->block.self = cfp->self; - env->block.ep = cfp->ep; - env->block.iseq = cfp->iseq; +const rb_env_t * +rb_vm_env_prev_env(const rb_env_t *env) +{ + const VALUE *ep = env->ep; - if (!RUBY_VM_NORMAL_ISEQ_P(cfp->iseq)) { - /* TODO */ - env->block.iseq = 0; + if (VM_ENV_LOCAL_P(ep)) { + return NULL; + } + else { + const VALUE *prev_ep = VM_ENV_PREV_EP(ep); + return VM_ENV_ENVVAL_PTR(prev_ep); } - return envval; } static int -collect_local_variables_in_iseq(rb_iseq_t *iseq, const VALUE ary) +collect_local_variables_in_iseq(const rb_iseq_t *iseq, const struct local_var_list *vars) { - int i; + unsigned int i; if (!iseq) return 0; - for (i = 0; i < iseq->local_table_size; i++) { - ID lid = iseq->local_table[i]; - if (rb_is_local_id(lid)) { - rb_ary_push(ary, ID2SYM(lid)); - } + for (i = 0; i < ISEQ_BODY(iseq)->local_table_size; i++) { + local_var_list_add(vars, ISEQ_BODY(iseq)->local_table[i]); } return 1; } -static int -collect_local_variables_in_env(rb_env_t * env, const VALUE ary) +static void +collect_local_variables_in_env(const rb_env_t *env, const struct local_var_list *vars) { - - while (collect_local_variables_in_iseq(env->block.iseq, ary), - env->prev_envval) { - GetEnvPtr(env->prev_envval, env); - } - return 0; + do { + if (VM_ENV_FLAGS(env->ep, VM_ENV_FLAG_ISOLATED)) break; + collect_local_variables_in_iseq(env->iseq, vars); + } while ((env = rb_vm_env_prev_env(env)) != NULL); } static int -vm_collect_local_variables_in_heap(rb_thread_t *th, VALUE *ep, VALUE ary) +vm_collect_local_variables_in_heap(const VALUE *ep, const struct local_var_list *vars) { - if (ENV_IN_HEAP_P(th, ep)) { - rb_env_t *env; - GetEnvPtr(ENV_VAL(ep), env); - collect_local_variables_in_env(env, ary); - return 1; + if (VM_ENV_ESCAPED_P(ep)) { + collect_local_variables_in_env(VM_ENV_ENVVAL_PTR(ep), vars); + return 1; } else { - return 0; + return 0; } } -static void vm_rewrite_ep_in_errinfo(rb_thread_t *th); -static VALUE vm_make_proc_from_block(rb_thread_t *th, rb_block_t *block); -static VALUE vm_make_env_object(rb_thread_t * th, rb_control_frame_t *cfp, VALUE *blockprocptr); +VALUE +rb_vm_env_local_variables(const rb_env_t *env) +{ + struct local_var_list vars; + local_var_list_init(&vars); + collect_local_variables_in_env(env, &vars); + return local_var_list_finish(&vars); +} + +VALUE +rb_vm_env_numbered_parameters(const rb_env_t *env) +{ + struct local_var_list vars; + local_var_list_init(&vars); + // if (VM_ENV_FLAGS(env->ep, VM_ENV_FLAG_ISOLATED)) break; // TODO: is this needed? + const rb_iseq_t *iseq = env->iseq; + unsigned int i; + if (!iseq) return 0; + for (i = 0; i < ISEQ_BODY(iseq)->local_table_size; i++) { + numparam_list_add(&vars, ISEQ_BODY(iseq)->local_table[i]); + } + return local_var_list_finish(&vars); +} VALUE -rb_vm_make_env_object(rb_thread_t * th, rb_control_frame_t *cfp) +rb_iseq_local_variables(const rb_iseq_t *iseq) { - VALUE blockprocval; - return vm_make_env_object(th, cfp, &blockprocval); + struct local_var_list vars; + local_var_list_init(&vars); + while (collect_local_variables_in_iseq(iseq, &vars)) { + iseq = ISEQ_BODY(iseq)->parent_iseq; + } + return local_var_list_finish(&vars); } +/* Proc */ + static VALUE -vm_make_env_object(rb_thread_t *th, rb_control_frame_t *cfp, VALUE *blockprocptr) +vm_proc_create_from_captured(VALUE klass, + const struct rb_captured_block *captured, + enum rb_block_type block_type, + int8_t is_from_method, int8_t is_lambda) { - VALUE envval; - VALUE *lep = VM_CF_LEP(cfp); - rb_block_t *blockptr = VM_EP_BLOCK_PTR(lep); + VALUE procval = rb_proc_alloc(klass); + rb_proc_t *proc = RTYPEDDATA_DATA(procval); + + VM_ASSERT(VM_EP_IN_HEAP_P(GET_EC(), captured->ep)); + + /* copy block */ + RB_OBJ_WRITE(procval, &proc->block.as.captured.code.val, captured->code.val); + RB_OBJ_WRITE(procval, &proc->block.as.captured.self, captured->self); + rb_vm_block_ep_update(procval, &proc->block, captured->ep); + + vm_block_type_set(&proc->block, block_type); + proc->is_from_method = is_from_method; + proc->is_lambda = is_lambda; + + return procval; +} - if (blockptr) { - VALUE blockprocval = vm_make_proc_from_block(th, blockptr); - rb_proc_t *p; - GetProcPtr(blockprocval, p); - lep[0] = VM_ENVVAL_BLOCK_PTR(&p->block); - *blockprocptr = blockprocval; +void +rb_vm_block_copy(VALUE obj, const struct rb_block *dst, const struct rb_block *src) +{ + /* copy block */ + switch (vm_block_type(src)) { + case block_type_iseq: + case block_type_ifunc: + RB_OBJ_WRITE(obj, &dst->as.captured.self, src->as.captured.self); + RB_OBJ_WRITE(obj, &dst->as.captured.code.val, src->as.captured.code.val); + rb_vm_block_ep_update(obj, dst, src->as.captured.ep); + break; + case block_type_symbol: + RB_OBJ_WRITE(obj, &dst->as.symbol, src->as.symbol); + break; + case block_type_proc: + RB_OBJ_WRITE(obj, &dst->as.proc, src->as.proc); + break; } +} + +static VALUE +proc_create(VALUE klass, const struct rb_block *block, int8_t is_from_method, int8_t is_lambda) +{ + VALUE procval = rb_proc_alloc(klass); + rb_proc_t *proc = RTYPEDDATA_DATA(procval); - envval = vm_make_env_each(th, cfp, cfp->ep, lep); - vm_rewrite_ep_in_errinfo(th); + VM_ASSERT(VM_EP_IN_HEAP_P(GET_EC(), vm_block_ep(block))); + rb_vm_block_copy(procval, &proc->block, block); + vm_block_type_set(&proc->block, block->type); + proc->is_from_method = is_from_method; + proc->is_lambda = is_lambda; - if (PROCDEBUG) { - check_env_value(envval); + return procval; +} + +VALUE +rb_proc_dup(VALUE self) +{ + VALUE procval; + rb_proc_t *src; + + GetProcPtr(self, src); + + switch (vm_block_type(&src->block)) { + case block_type_ifunc: + procval = rb_func_proc_dup(self); + break; + default: + procval = proc_create(rb_obj_class(self), &src->block, src->is_from_method, src->is_lambda); + break; } - return envval; + if (RB_OBJ_SHAREABLE_P(self)) RB_OBJ_SET_SHAREABLE(procval); + RB_GC_GUARD(self); /* for: body = rb_proc_dup(body) */ + return procval; } -static void -vm_rewrite_ep_in_errinfo(rb_thread_t *th) -{ - rb_control_frame_t *cfp = th->cfp; - while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(th, cfp)) { - /* rewrite ep in errinfo to point to heap */ - if (RUBY_VM_NORMAL_ISEQ_P(cfp->iseq) && - (cfp->iseq->type == ISEQ_TYPE_RESCUE || - cfp->iseq->type == ISEQ_TYPE_ENSURE)) { - VALUE errinfo = cfp->ep[-2]; /* #$! */ - if (RB_TYPE_P(errinfo, T_NODE)) { - VALUE *escape_ep = GET_THROWOBJ_CATCH_POINT(errinfo); - if (! ENV_IN_HEAP_P(th, escape_ep)) { - VALUE epval = *escape_ep; - if (!SPECIAL_CONST_P(epval) && RBASIC(epval)->klass == rb_cEnv) { - rb_env_t *epenv; - GetEnvPtr(epval, epenv); - SET_THROWOBJ_CATCH_POINT(errinfo, (VALUE)(epenv->env + epenv->local_size)); - } - } - } - } - cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); +struct collect_outer_variable_name_data { + VALUE ary; + VALUE read_only; + bool yield; + bool isolate; +}; + +static VALUE +ID2NUM(ID id) +{ + if (SIZEOF_VOIDP > SIZEOF_LONG) + return ULL2NUM(id); + else + return ULONG2NUM(id); +} + +static ID +NUM2ID(VALUE num) +{ + if (SIZEOF_VOIDP > SIZEOF_LONG) + return (ID)NUM2ULL(num); + else + return (ID)NUM2ULONG(num); +} + +static enum rb_id_table_iterator_result +collect_outer_variable_names(ID id, VALUE val, void *ptr) +{ + struct collect_outer_variable_name_data *data = (struct collect_outer_variable_name_data *)ptr; + + if (id == rb_intern("yield")) { + data->yield = true; } + else { + VALUE *store; + if (data->isolate || + val == Qtrue /* write */) { + store = &data->ary; + } + else { + store = &data->read_only; + } + if (*store == Qfalse) *store = rb_ary_new(); + rb_ary_push(*store, ID2NUM(id)); + } + return ID_TABLE_CONTINUE; } -void -rb_vm_stack_to_heap(rb_thread_t *th) +static const rb_env_t * +env_copy(const VALUE *src_ep, VALUE read_only_variables) { - rb_control_frame_t *cfp = th->cfp; - while ((cfp = rb_vm_get_binding_creatable_next_cfp(th, cfp)) != 0) { - rb_vm_make_env_object(th, cfp); - cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + const rb_env_t *src_env = (rb_env_t *)VM_ENV_ENVVAL(src_ep); + VM_ASSERT(src_env->ep == src_ep); + + VALUE *env_body = ZALLOC_N(VALUE, src_env->env_size); // fill with Qfalse + VALUE *ep = &env_body[src_env->env_size - 2]; + const rb_env_t *copied_env = vm_env_new(ep, env_body, src_env->env_size, src_env->iseq); + + // Copy after allocations above, since they can move objects in src_ep. + VALUE svar_val = src_ep[VM_ENV_DATA_INDEX_ME_CREF]; + if (imemo_type_p(svar_val, imemo_svar)) { + const struct vm_svar *svar = (struct vm_svar *)svar_val; + + if (svar->cref_or_me) { + svar_val = svar->cref_or_me; + } + else { + svar_val = Qfalse; + } + } + RB_OBJ_WRITE(copied_env, &ep[VM_ENV_DATA_INDEX_ME_CREF], svar_val); + + ep[VM_ENV_DATA_INDEX_FLAGS] = src_ep[VM_ENV_DATA_INDEX_FLAGS] | VM_ENV_FLAG_ISOLATED; + if (!VM_ENV_LOCAL_P(src_ep)) { + VM_ENV_FLAGS_SET(ep, VM_ENV_FLAG_LOCAL); + } + + if (read_only_variables) { + for (int i=RARRAY_LENINT(read_only_variables)-1; i>=0; i--) { + ID id = NUM2ID(RARRAY_AREF(read_only_variables, i)); + + const struct rb_iseq_constant_body *body = ISEQ_BODY(src_env->iseq); + for (unsigned int j=0; j<body->local_table_size; j++) { + if (id == body->local_table[j]) { + // check reassignment + if (body->lvar_states[j] == lvar_reassigned) { + VALUE name = rb_id2str(id); + VALUE msg = rb_sprintf("cannot make a shareable Proc because " + "the outer variable '%" PRIsVALUE "' may be reassigned.", name); + rb_exc_raise(rb_exc_new_str(rb_eRactorIsolationError, msg)); + } + + // check shareable + VALUE v = src_env->env[j]; + if (!rb_ractor_shareable_p(v)) { + VALUE name = rb_id2str(id); + VALUE msg = rb_sprintf("cannot make a shareable Proc because it can refer" + " unshareable object %+" PRIsVALUE " from ", v); + if (name) + rb_str_catf(msg, "variable '%" PRIsVALUE "'", name); + else + rb_str_cat_cstr(msg, "a hidden variable"); + rb_exc_raise(rb_exc_new_str(rb_eRactorIsolationError, msg)); + } + RB_OBJ_WRITE((VALUE)copied_env, &env_body[j], v); + rb_ary_delete_at(read_only_variables, i); + break; + } + } + } + } + + if (!VM_ENV_LOCAL_P(src_ep)) { + const VALUE *prev_ep = VM_ENV_PREV_EP(src_env->ep); + const rb_env_t *new_prev_env = env_copy(prev_ep, read_only_variables); + ep[VM_ENV_DATA_INDEX_SPECVAL] = VM_GUARDED_PREV_EP(new_prev_env->ep); + RB_OBJ_WRITTEN(copied_env, Qundef, new_prev_env); + VM_ENV_FLAGS_UNSET(ep, VM_ENV_FLAG_LOCAL); } + else { + ep[VM_ENV_DATA_INDEX_SPECVAL] = VM_BLOCK_HANDLER_NONE; + } + + RB_OBJ_SET_SHAREABLE((VALUE)copied_env); + return copied_env; } -/* Proc */ +static void +proc_isolate_env(VALUE self, rb_proc_t *proc, VALUE read_only_variables) +{ + const struct rb_captured_block *captured = &proc->block.as.captured; + const rb_env_t *env = env_copy(captured->ep, read_only_variables); + *((const VALUE **)&proc->block.as.captured.ep) = env->ep; + RB_OBJ_WRITTEN(self, Qundef, env); +} static VALUE -vm_make_proc_from_block(rb_thread_t *th, rb_block_t *block) +proc_shared_outer_variables(struct rb_id_table *outer_variables, bool isolate, const char *message) { - if (!block->proc) { - block->proc = rb_vm_make_proc(th, block, rb_cProc); + struct collect_outer_variable_name_data data = { + .isolate = isolate, + .ary = Qfalse, + .read_only = Qfalse, + .yield = false, + }; + rb_id_table_foreach(outer_variables, collect_outer_variable_names, (void *)&data); + + if (data.ary != Qfalse) { + VALUE str = rb_sprintf("can not %s because it accesses outer variables", message); + VALUE ary = data.ary; + const char *sep = " ("; + for (long i = 0; i < RARRAY_LEN(ary); i++) { + VALUE name = rb_id2str(NUM2ID(RARRAY_AREF(ary, i))); + if (!name) continue; + rb_str_cat_cstr(str, sep); + sep = ", "; + rb_str_append(str, name); + } + if (*sep == ',') rb_str_cat_cstr(str, ")"); + rb_str_cat_cstr(str, data.yield ? " and uses 'yield'." : "."); + rb_exc_raise(rb_exc_new_str(rb_eArgError, str)); } - return block->proc; + else if (data.yield) { + rb_raise(rb_eArgError, "can not %s because it uses 'yield'.", message); + } + + return data.read_only; } VALUE -rb_vm_make_proc(rb_thread_t *th, const rb_block_t *block, VALUE klass) +rb_proc_isolate_bang(VALUE self, VALUE replace_self) { - VALUE procval, envval, blockprocval = 0; - rb_proc_t *proc; - rb_control_frame_t *cfp = RUBY_VM_GET_CFP_FROM_BLOCK_PTR(block); + const rb_iseq_t *iseq = vm_proc_iseq(self); + + if (iseq) { + rb_proc_t *proc = (rb_proc_t *)RTYPEDDATA_DATA(self); + + if (!UNDEF_P(replace_self)) { + VM_ASSERT(rb_ractor_shareable_p(replace_self)); + RB_OBJ_WRITE(self, &proc->block.as.captured.self, replace_self); + } + + if (proc->block.type != block_type_iseq) rb_raise(rb_eRuntimeError, "not supported yet"); + + if (ISEQ_BODY(iseq)->outer_variables) { + proc_shared_outer_variables(ISEQ_BODY(iseq)->outer_variables, true, "isolate a Proc"); + } - if (block->proc) { - rb_bug("rb_vm_make_proc: Proc value is already created."); + proc_isolate_env(self, proc, Qfalse); + proc->is_isolated = TRUE; + RB_OBJ_WRITE(self, &proc->block.as.captured.self, Qnil); } - envval = vm_make_env_object(th, cfp, &blockprocval); + RB_OBJ_SET_SHAREABLE(self); + return self; +} - if (PROCDEBUG) { - check_env_value(envval); +VALUE +rb_proc_isolate(VALUE self) +{ + VALUE dst = rb_proc_dup(self); + rb_proc_isolate_bang(dst, Qundef); + return dst; +} + +VALUE +rb_proc_ractor_make_shareable(VALUE self, VALUE replace_self) +{ + const rb_iseq_t *iseq = vm_proc_iseq(self); + + if (iseq) { + rb_proc_t *proc = (rb_proc_t *)RTYPEDDATA_DATA(self); + + if (!UNDEF_P(replace_self)) { + RB_OBJ_WRITE(self, &proc->block.as.captured.self, replace_self); + } + + if (proc->block.type != block_type_iseq) rb_raise(rb_eRuntimeError, "not supported yet"); + + if (!rb_ractor_shareable_p(vm_block_self(&proc->block))) { + rb_raise(rb_eRactorIsolationError, + "Proc's self is not shareable: %" PRIsVALUE, + self); + } + + VALUE read_only_variables = Qfalse; + + if (ISEQ_BODY(iseq)->outer_variables) { + read_only_variables = + proc_shared_outer_variables(ISEQ_BODY(iseq)->outer_variables, false, "make a Proc shareable"); + } + + proc_isolate_env(self, proc, read_only_variables); + proc->is_isolated = TRUE; + } + else { + const struct rb_block *block = vm_proc_block(self); + if (block->type != block_type_symbol) rb_raise(rb_eRuntimeError, "not supported yet"); + + VALUE proc_self = vm_block_self(block); + if (!rb_ractor_shareable_p(proc_self)) { + rb_raise(rb_eRactorIsolationError, + "Proc's self is not shareable: %" PRIsVALUE, + self); + } } - procval = rb_proc_alloc(klass); - GetProcPtr(procval, proc); - proc->blockprocval = blockprocval; - proc->block.self = block->self; - proc->block.klass = block->klass; - proc->block.ep = block->ep; - proc->block.iseq = block->iseq; - proc->block.proc = procval; - proc->envval = envval; - proc->safe_level = th->safe_level; - if (VMDEBUG) { - if (th->stack < block->ep && block->ep < th->stack + th->stack_size) { - rb_bug("invalid ptr: block->ep"); - } + RB_OBJ_SET_FROZEN_SHAREABLE(self); + return self; +} + +VALUE +rb_vm_make_proc_lambda(const rb_execution_context_t *ec, const struct rb_captured_block *captured, VALUE klass, int8_t is_lambda) +{ + VALUE procval; + enum imemo_type code_type = imemo_type(captured->code.val); + + if (!VM_ENV_ESCAPED_P(captured->ep)) { + rb_control_frame_t *cfp = VM_CAPTURED_BLOCK_TO_CFP(captured); + vm_make_env_object(ec, cfp); + } + + VM_ASSERT(VM_EP_IN_HEAP_P(ec, captured->ep)); + VM_ASSERT(code_type == imemo_iseq || code_type == imemo_ifunc); + + procval = vm_proc_create_from_captured(klass, captured, + code_type == imemo_iseq ? block_type_iseq : block_type_ifunc, + FALSE, is_lambda); + + if (code_type == imemo_ifunc) { + struct vm_ifunc *ifunc = (struct vm_ifunc *)captured->code.val; + if (ifunc->svar_lep) { + VALUE ep0 = ifunc->svar_lep[0]; + if (RB_TYPE_P(ep0, T_IMEMO) && imemo_type_p(ep0, imemo_env)) { + // `ep0 == imemo_env` means this ep is escaped to heap (in env object). + const rb_env_t *env = (const rb_env_t *)ep0; + ifunc->svar_lep = (VALUE *)env->ep; + } + else { + VM_ASSERT(FIXNUM_P(ep0)); + if (ep0 & VM_ENV_FLAG_ESCAPED) { + // ok. do nothing + } + else { + ifunc->svar_lep = NULL; + } + } + } } return procval; } -/* C -> Ruby: block */ +/* Binding */ -static inline VALUE -invoke_block_from_c(rb_thread_t *th, const rb_block_t *block, - VALUE self, int argc, const VALUE *argv, - const rb_block_t *blockptr, const NODE *cref, - VALUE defined_class) -{ - if (SPECIAL_CONST_P(block->iseq)) - return Qnil; - else if (BUILTIN_TYPE(block->iseq) != T_NODE) { - const rb_iseq_t *iseq = block->iseq; - const rb_control_frame_t *cfp; - int i, opt_pc, arg_size = iseq->arg_size; - int type = block_proc_is_lambda(block->proc) ? - VM_FRAME_MAGIC_LAMBDA : VM_FRAME_MAGIC_BLOCK; - - cfp = th->cfp; - CHECK_VM_STACK_OVERFLOW(cfp, argc + iseq->stack_max); - - for (i=0; i<argc; i++) { - cfp->sp[i] = argv[i]; - } - - opt_pc = vm_yield_setup_args(th, iseq, argc, cfp->sp, blockptr, - type == VM_FRAME_MAGIC_LAMBDA); - - vm_push_frame(th, iseq, type | VM_FRAME_FLAG_FINISH, - self, defined_class, - VM_ENVVAL_PREV_EP_PTR(block->ep), - iseq->iseq_encoded + opt_pc, - cfp->sp + arg_size, iseq->local_size - arg_size, - th->passed_me); - th->passed_me = 0; - - if (cref) { - th->cfp->ep[-1] = (VALUE)cref; - } - - return vm_exec(th); +VALUE +rb_vm_make_binding(const rb_execution_context_t *ec, const rb_control_frame_t *src_cfp) +{ + rb_control_frame_t *cfp = rb_vm_get_binding_creatable_next_cfp(ec, src_cfp); + rb_control_frame_t *ruby_level_cfp = rb_vm_get_ruby_level_next_cfp(ec, src_cfp); + VALUE bindval, envval; + rb_binding_t *bind; + + if (cfp == 0 || ruby_level_cfp == 0) { + rb_raise(rb_eRuntimeError, "Can't create Binding Object on top of Fiber."); + } + if (!VM_FRAME_RUBYFRAME_P(src_cfp) && + !VM_FRAME_RUBYFRAME_P(RUBY_VM_PREVIOUS_CONTROL_FRAME(src_cfp))) { + rb_raise(rb_eRuntimeError, "Cannot create Binding object for non-Ruby caller"); + } + + envval = vm_make_env_object(ec, cfp); + bindval = rb_binding_alloc(rb_cBinding); + GetBindingPtr(bindval, bind); + vm_bind_update_env(bindval, bind, envval); + RB_OBJ_WRITE(bindval, &bind->block.as.captured.self, cfp->self); + RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, cfp->iseq); + RB_OBJ_WRITE(bindval, &bind->pathobj, ISEQ_BODY(ruby_level_cfp->iseq)->location.pathobj); + bind->first_lineno = rb_vm_get_sourceline(ruby_level_cfp); + + return bindval; +} + +const VALUE * +rb_binding_add_dynavars(VALUE bindval, rb_binding_t *bind, int dyncount, const ID *dynvars) +{ + VALUE envval, pathobj = bind->pathobj; + VALUE path = pathobj_path(pathobj); + VALUE realpath = pathobj_realpath(pathobj); + const struct rb_block *base_block; + const rb_env_t *env; + rb_execution_context_t *ec = GET_EC(); + const rb_iseq_t *base_iseq, *iseq; + rb_node_scope_t tmp_node; + + if (dyncount < 0) return 0; + + base_block = &bind->block; + base_iseq = vm_block_iseq(base_block); + + VALUE idtmp = 0; + rb_ast_id_table_t *dyns = ALLOCV(idtmp, sizeof(rb_ast_id_table_t) + dyncount * sizeof(ID)); + dyns->size = dyncount; + MEMCPY(dyns->ids, dynvars, ID, dyncount); + + rb_node_init(RNODE(&tmp_node), NODE_SCOPE); + tmp_node.nd_tbl = dyns; + tmp_node.nd_body = 0; + tmp_node.nd_parent = NULL; + tmp_node.nd_args = 0; + + VALUE ast_value = rb_ruby_ast_new(RNODE(&tmp_node)); + + if (base_iseq) { + iseq = rb_iseq_new(ast_value, ISEQ_BODY(base_iseq)->location.label, path, realpath, base_iseq, ISEQ_TYPE_EVAL); } else { - return vm_yield_with_cfunc(th, block, self, argc, argv, blockptr); + VALUE tempstr = rb_fstring_lit("<temp>"); + iseq = rb_iseq_new_top(ast_value, tempstr, tempstr, tempstr, NULL); } + tmp_node.nd_tbl = 0; /* reset table */ + ALLOCV_END(idtmp); + + vm_set_eval_stack(ec, iseq, 0, base_block); + vm_bind_update_env(bindval, bind, envval = vm_make_env_object(ec, ec->cfp)); + rb_vm_pop_frame(ec); + + env = (const rb_env_t *)envval; + return env->env; } -static inline const rb_block_t * -check_block(rb_thread_t *th) +/* C -> Ruby: block */ + +static inline void +invoke_block(rb_execution_context_t *ec, const rb_iseq_t *iseq, VALUE self, const struct rb_captured_block *captured, const rb_cref_t *cref, VALUE type, int opt_pc) { - const rb_block_t *blockptr = VM_CF_BLOCK_PTR(th->cfp); + int arg_size = ISEQ_BODY(iseq)->param.size; + + vm_push_frame(ec, iseq, type | VM_FRAME_FLAG_FINISH, self, + VM_GUARDED_PREV_EP(captured->ep), + (VALUE)cref, /* cref or method */ + ISEQ_BODY(iseq)->iseq_encoded + opt_pc, + ec->cfp->sp + arg_size, + ISEQ_BODY(iseq)->local_table_size - arg_size, + ISEQ_BODY(iseq)->stack_max); +} + +static inline void +invoke_bmethod(rb_execution_context_t *ec, const rb_iseq_t *iseq, VALUE self, const struct rb_captured_block *captured, const rb_callable_method_entry_t *me, VALUE type, int opt_pc) +{ + /* bmethod call from outside the VM */ + int arg_size = ISEQ_BODY(iseq)->param.size; + + VM_ASSERT(me->def->type == VM_METHOD_TYPE_BMETHOD); + + vm_push_frame(ec, iseq, type | VM_FRAME_FLAG_BMETHOD, self, + VM_GUARDED_PREV_EP(captured->ep), + (VALUE)me, + ISEQ_BODY(iseq)->iseq_encoded + opt_pc, + ec->cfp->sp + 1 /* self */ + arg_size, + ISEQ_BODY(iseq)->local_table_size - arg_size, + ISEQ_BODY(iseq)->stack_max); - if (blockptr == 0) { - rb_vm_localjump_error("no block given", Qnil, 0); + VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_FINISH); +} + +ALWAYS_INLINE(static VALUE + invoke_iseq_block_from_c(rb_execution_context_t *ec, const struct rb_captured_block *captured, + VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler, + const rb_cref_t *cref, int is_lambda, const rb_callable_method_entry_t *me)); + +static inline VALUE +invoke_iseq_block_from_c(rb_execution_context_t *ec, const struct rb_captured_block *captured, + VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler, + const rb_cref_t *cref, int is_lambda, const rb_callable_method_entry_t *me) +{ + const rb_iseq_t *iseq = rb_iseq_check(captured->code.iseq); + int opt_pc; + VALUE type = VM_FRAME_MAGIC_BLOCK | (is_lambda ? VM_FRAME_FLAG_LAMBDA : 0); + rb_control_frame_t *cfp = ec->cfp; + VALUE *sp = cfp->sp; + int flags = (kw_splat ? VM_CALL_KW_SPLAT : 0); + VALUE *use_argv = (VALUE *)argv; + VALUE av[2]; + + stack_check(ec); + + if (UNLIKELY(argc > VM_ARGC_STACK_MAX) && + (VM_ARGC_STACK_MAX >= 1 || + /* Skip ruby array for potential autosplat case */ + (argc != 1 || is_lambda))) { + use_argv = vm_argv_ruby_array(av, argv, &flags, &argc, kw_splat); + } + + CHECK_VM_STACK_OVERFLOW(cfp, argc + 1); + vm_check_canary(ec, sp); + + VALUE *stack_argv = sp; + if (me) { + *sp = self; // bemthods need `self` on the VM stack + stack_argv++; } + cfp->sp = stack_argv + argc; + MEMCPY(stack_argv, use_argv, VALUE, argc); // restrict: new stack space + + opt_pc = vm_yield_setup_args(ec, iseq, argc, stack_argv, flags, passed_block_handler, + (is_lambda ? arg_setup_method : arg_setup_block)); + cfp->sp = sp; - return blockptr; + if (me == NULL) { + invoke_block(ec, iseq, self, captured, cref, type, opt_pc); + } + else { + invoke_bmethod(ec, iseq, self, captured, me, type, opt_pc); + } + + return vm_exec(ec); +} + +static VALUE +invoke_block_from_c_bh(rb_execution_context_t *ec, VALUE block_handler, + int argc, const VALUE *argv, + int kw_splat, VALUE passed_block_handler, const rb_cref_t *cref, + int is_lambda, int force_blockarg) +{ + again: + switch (vm_block_handler_type(block_handler)) { + case block_handler_type_iseq: + { + const struct rb_captured_block *captured = VM_BH_TO_ISEQ_BLOCK(block_handler); + return invoke_iseq_block_from_c(ec, captured, captured->self, + argc, argv, kw_splat, passed_block_handler, + cref, is_lambda, NULL); + } + case block_handler_type_ifunc: + return vm_yield_with_cfunc(ec, VM_BH_TO_IFUNC_BLOCK(block_handler), + VM_BH_TO_IFUNC_BLOCK(block_handler)->self, + argc, argv, kw_splat, passed_block_handler, NULL); + case block_handler_type_symbol: + return vm_yield_with_symbol(ec, VM_BH_TO_SYMBOL(block_handler), + argc, argv, kw_splat, passed_block_handler); + case block_handler_type_proc: + if (force_blockarg == FALSE) { + is_lambda = block_proc_is_lambda(VM_BH_TO_PROC(block_handler)); + } + block_handler = vm_proc_to_block_handler(VM_BH_TO_PROC(block_handler)); + goto again; + } + VM_UNREACHABLE(invoke_block_from_c_splattable); + return Qundef; } static inline VALUE -vm_yield_with_cref(rb_thread_t *th, int argc, const VALUE *argv, const NODE *cref) +check_block_handler(rb_execution_context_t *ec) { - const rb_block_t *blockptr = check_block(th); - return invoke_block_from_c(th, blockptr, blockptr->self, argc, argv, 0, cref, - blockptr->klass); + VALUE block_handler = VM_CF_BLOCK_HANDLER(ec->cfp); + vm_block_handler_verify(block_handler); + if (UNLIKELY(block_handler == VM_BLOCK_HANDLER_NONE)) { + rb_vm_localjump_error("no block given", Qnil, 0); + } + + return block_handler; } -static inline VALUE -vm_yield(rb_thread_t *th, int argc, const VALUE *argv) +static VALUE +vm_yield_with_cref(rb_execution_context_t *ec, int argc, const VALUE *argv, int kw_splat, const rb_cref_t *cref, int is_lambda) { - const rb_block_t *blockptr = check_block(th); - return invoke_block_from_c(th, blockptr, blockptr->self, argc, argv, 0, 0, - blockptr->klass); + return invoke_block_from_c_bh(ec, check_block_handler(ec), + argc, argv, kw_splat, VM_BLOCK_HANDLER_NONE, + cref, is_lambda, FALSE); } static VALUE -vm_invoke_proc(rb_thread_t *th, rb_proc_t *proc, VALUE self, VALUE defined_class, - int argc, const VALUE *argv, const rb_block_t *blockptr) +vm_yield(rb_execution_context_t *ec, int argc, const VALUE *argv, int kw_splat) { - VALUE val = Qundef; - int state; - volatile int stored_safe = th->safe_level; + return vm_yield_with_cref(ec, argc, argv, kw_splat, NULL, FALSE); +} - TH_PUSH_TAG(th); - if ((state = EXEC_TAG()) == 0) { - if (!proc->is_from_method) { - th->safe_level = proc->safe_level; - } - val = invoke_block_from_c(th, &proc->block, self, argc, argv, blockptr, 0, - defined_class); - } - TH_POP_TAG(); +static VALUE +vm_yield_with_block(rb_execution_context_t *ec, int argc, const VALUE *argv, VALUE block_handler, int kw_splat) +{ + return invoke_block_from_c_bh(ec, check_block_handler(ec), + argc, argv, kw_splat, block_handler, + NULL, FALSE, FALSE); +} + +static VALUE +vm_yield_force_blockarg(rb_execution_context_t *ec, VALUE args) +{ + return invoke_block_from_c_bh(ec, check_block_handler(ec), 1, &args, + RB_NO_KEYWORDS, VM_BLOCK_HANDLER_NONE, NULL, FALSE, TRUE); +} + +ALWAYS_INLINE(static VALUE + invoke_block_from_c_proc(rb_execution_context_t *ec, const rb_proc_t *proc, + VALUE self, int argc, const VALUE *argv, + int kw_splat, VALUE passed_block_handler, int is_lambda, + const rb_callable_method_entry_t *me)); - if (!proc->is_from_method) { - th->safe_level = stored_safe; +static inline VALUE +invoke_block_from_c_proc(rb_execution_context_t *ec, const rb_proc_t *proc, + VALUE self, int argc, const VALUE *argv, + int kw_splat, VALUE passed_block_handler, int is_lambda, + const rb_callable_method_entry_t *me) +{ + const struct rb_block *block = &proc->block; + + again: + switch (vm_block_type(block)) { + case block_type_iseq: + return invoke_iseq_block_from_c(ec, &block->as.captured, self, argc, argv, kw_splat, passed_block_handler, NULL, is_lambda, me); + case block_type_ifunc: + if (kw_splat == 1) { + VALUE keyword_hash = argv[argc-1]; + if (!RB_TYPE_P(keyword_hash, T_HASH)) { + keyword_hash = rb_to_hash_type(keyword_hash); + } + if (RHASH_EMPTY_P(keyword_hash)) { + argc--; + } + else { + ((VALUE *)argv)[argc-1] = rb_hash_dup(keyword_hash); + } + } + return vm_yield_with_cfunc(ec, &block->as.captured, self, argc, argv, kw_splat, passed_block_handler, me); + case block_type_symbol: + return vm_yield_with_symbol(ec, block->as.symbol, argc, argv, kw_splat, passed_block_handler); + case block_type_proc: + is_lambda = block_proc_is_lambda(block->as.proc); + block = vm_proc_block(block->as.proc); + goto again; } + VM_UNREACHABLE(invoke_block_from_c_proc); + return Qundef; +} - if (state) { - JUMP_TAG(state); +static VALUE +vm_invoke_proc(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self, + int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler) +{ + return invoke_block_from_c_proc(ec, proc, self, argc, argv, kw_splat, passed_block_handler, proc->is_lambda, NULL); +} + +static VALUE +vm_invoke_bmethod(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self, + int argc, const VALUE *argv, int kw_splat, VALUE block_handler, const rb_callable_method_entry_t *me) +{ + return invoke_block_from_c_proc(ec, proc, self, argc, argv, kw_splat, block_handler, TRUE, me); +} + +VALUE +rb_vm_invoke_proc(rb_execution_context_t *ec, rb_proc_t *proc, + int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler) +{ + VALUE self = vm_block_self(&proc->block); + vm_block_handler_verify(passed_block_handler); + + if (proc->is_from_method) { + return vm_invoke_bmethod(ec, proc, self, argc, argv, kw_splat, passed_block_handler, NULL); + } + else { + return vm_invoke_proc(ec, proc, self, argc, argv, kw_splat, passed_block_handler); } - return val; } VALUE -rb_vm_invoke_proc(rb_thread_t *th, rb_proc_t *proc, - int argc, const VALUE *argv, const rb_block_t *blockptr) +rb_vm_invoke_proc_with_self(rb_execution_context_t *ec, rb_proc_t *proc, VALUE self, + int argc, const VALUE *argv, int kw_splat, VALUE passed_block_handler) { - return vm_invoke_proc(th, proc, proc->block.self, proc->block.klass, - argc, argv, blockptr); + vm_block_handler_verify(passed_block_handler); + + if (proc->is_from_method) { + return vm_invoke_bmethod(ec, proc, self, argc, argv, kw_splat, passed_block_handler, NULL); + } + else { + return vm_invoke_proc(ec, proc, self, argc, argv, kw_splat, passed_block_handler); + } } /* special variable */ -static rb_control_frame_t * -vm_normal_frame(rb_thread_t *th, rb_control_frame_t *cfp) -{ - while (cfp->pc == 0) { - cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); - if (RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(th, cfp)) { - return 0; - } +VALUE * +rb_vm_svar_lep(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) +{ + while (cfp->pc == 0 || cfp->iseq == 0) { + if (VM_FRAME_TYPE(cfp) == VM_FRAME_MAGIC_IFUNC) { + struct vm_ifunc *ifunc = (struct vm_ifunc *)cfp->iseq; + return ifunc->svar_lep; + } + else { + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + } + + if (RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) { + return NULL; + } } - return cfp; + + return (VALUE *)VM_CF_LEP(cfp); } static VALUE -vm_cfp_svar_get(rb_thread_t *th, rb_control_frame_t *cfp, VALUE key) +vm_cfp_svar_get(const rb_execution_context_t *ec, rb_control_frame_t *cfp, VALUE key) { - cfp = vm_normal_frame(th, cfp); - return lep_svar_get(th, cfp ? VM_CF_LEP(cfp) : 0, key); + return lep_svar_get(ec, rb_vm_svar_lep(ec, cfp), key); } static void -vm_cfp_svar_set(rb_thread_t *th, rb_control_frame_t *cfp, VALUE key, const VALUE val) +vm_cfp_svar_set(const rb_execution_context_t *ec, rb_control_frame_t *cfp, VALUE key, const VALUE val) { - cfp = vm_normal_frame(th, cfp); - lep_svar_set(th, cfp ? VM_CF_LEP(cfp) : 0, key, val); + lep_svar_set(ec, rb_vm_svar_lep(ec, cfp), key, val); } static VALUE -vm_svar_get(VALUE key) +vm_svar_get(const rb_execution_context_t *ec, VALUE key) { - rb_thread_t *th = GET_THREAD(); - return vm_cfp_svar_get(th, th->cfp, key); + return vm_cfp_svar_get(ec, ec->cfp, key); } static void -vm_svar_set(VALUE key, VALUE val) +vm_svar_set(const rb_execution_context_t *ec, VALUE key, VALUE val) { - rb_thread_t *th = GET_THREAD(); - vm_cfp_svar_set(th, th->cfp, key, val); + vm_cfp_svar_set(ec, ec->cfp, key, val); } VALUE rb_backref_get(void) { - return vm_svar_get(1); + return vm_svar_get(GET_EC(), VM_SVAR_BACKREF); } void rb_backref_set(VALUE val) { - vm_svar_set(1, val); + vm_svar_set(GET_EC(), VM_SVAR_BACKREF, val); } VALUE rb_lastline_get(void) { - return vm_svar_get(0); + return vm_svar_get(GET_EC(), VM_SVAR_LASTLINE); } void rb_lastline_set(VALUE val) { - vm_svar_set(0, val); + vm_svar_set(GET_EC(), VM_SVAR_LASTLINE, val); } -/* misc */ - -VALUE -rb_sourcefilename(void) +void +rb_lastline_set_up(VALUE val, unsigned int up) { - rb_thread_t *th = GET_THREAD(); - rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(th, th->cfp); + rb_control_frame_t * cfp = GET_EC()->cfp; - if (cfp) { - return cfp->iseq->location.path; - } - else { - return Qnil; + for(unsigned int i = 0; i < up; i++) { + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); } + vm_cfp_svar_set(GET_EC(), cfp, VM_SVAR_LASTLINE, val); } +/* misc */ + const char * rb_sourcefile(void) { - rb_thread_t *th = GET_THREAD(); - rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(th, th->cfp); + const rb_execution_context_t *ec = GET_EC(); + const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp); if (cfp) { - return RSTRING_PTR(cfp->iseq->location.path); + return RSTRING_PTR(rb_iseq_path(cfp->iseq)); } else { - return 0; + return 0; } } int rb_sourceline(void) { - rb_thread_t *th = GET_THREAD(); - rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(th, th->cfp); + const rb_execution_context_t *ec = GET_EC(); + const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp); if (cfp) { - return rb_vm_get_sourceline(cfp); + return rb_vm_get_sourceline(cfp); } else { - return 0; + return 0; } } -NODE * -rb_vm_cref(void) +VALUE +rb_source_location(int *pline) { - rb_thread_t *th = GET_THREAD(); - rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(th, th->cfp); + const rb_execution_context_t *ec = GET_EC(); + const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp); - if (cfp == 0) { - return NULL; + if (cfp && VM_FRAME_RUBYFRAME_P(cfp)) { + if (pline) *pline = rb_vm_get_sourceline(cfp); + return rb_iseq_path(cfp->iseq); + } + else { + if (pline) *pline = 0; + return Qnil; } - return rb_vm_get_cref(cfp->iseq, cfp->ep); +} + +const char * +rb_source_location_cstr(int *pline) +{ + VALUE path = rb_source_location(pline); + if (NIL_P(path)) return NULL; + return RSTRING_PTR(path); +} + +rb_cref_t * +rb_vm_cref(void) +{ + const rb_execution_context_t *ec = GET_EC(); + return vm_ec_cref(ec); +} + +rb_cref_t * +rb_vm_cref_replace_with_duplicated_cref(void) +{ + const rb_execution_context_t *ec = GET_EC(); + const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp); + rb_cref_t *cref = vm_cref_replace_with_duplicated_cref(cfp->ep); + ASSUME(cref); + return cref; +} + +const rb_cref_t * +rb_vm_cref_in_context(VALUE self, VALUE cbase) +{ + const rb_execution_context_t *ec = GET_EC(); + const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp); + const rb_cref_t *cref; + if (!cfp || cfp->self != self) return NULL; + if (!vm_env_cref_by_cref(cfp->ep)) return NULL; + cref = vm_get_cref(cfp->ep); + if (CREF_CLASS(cref) != cbase) return NULL; + return cref; } #if 0 void -debug_cref(NODE *cref) +debug_cref(rb_cref_t *cref) { while (cref) { - dp(cref->nd_clss); - printf("%ld\n", cref->nd_visi); - cref = cref->nd_next; + dp(CREF_CLASS(cref)); + printf("%ld\n", CREF_VISI(cref)); + cref = CREF_NEXT(cref); } } #endif @@ -851,13 +2160,13 @@ debug_cref(NODE *cref) VALUE rb_vm_cbase(void) { - rb_thread_t *th = GET_THREAD(); - rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(th, th->cfp); + const rb_execution_context_t *ec = GET_EC(); + const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp); if (cfp == 0) { - rb_raise(rb_eRuntimeError, "Can't call on top of Fiber or Thread"); + rb_raise(rb_eRuntimeError, "Can't call on top of Fiber or Thread"); } - return vm_get_cbase(cfp->iseq, cfp->ep); + return vm_get_cbase(cfp->ep); } /* jump */ @@ -871,23 +2180,23 @@ make_localjump_error(const char *mesg, VALUE value, int reason) switch (reason) { case TAG_BREAK: - CONST_ID(id, "break"); - break; + CONST_ID(id, "break"); + break; case TAG_REDO: - CONST_ID(id, "redo"); - break; + CONST_ID(id, "redo"); + break; case TAG_RETRY: - CONST_ID(id, "retry"); - break; + CONST_ID(id, "retry"); + break; case TAG_NEXT: - CONST_ID(id, "next"); - break; + CONST_ID(id, "next"); + break; case TAG_RETURN: - CONST_ID(id, "return"); - break; + CONST_ID(id, "return"); + break; default: - CONST_ID(id, "noreason"); - break; + CONST_ID(id, "noreason"); + break; } rb_iv_set(exc, "@exit_value", value); rb_iv_set(exc, "@reason", ID2SYM(id)); @@ -902,132 +2211,221 @@ rb_vm_localjump_error(const char *mesg, VALUE value, int reason) } VALUE -rb_vm_make_jump_tag_but_local_jump(int state, VALUE val) +rb_vm_make_jump_tag_but_local_jump(enum ruby_tag_type state, VALUE val) { - VALUE result = Qnil; + const char *mesg; - if (val == Qundef) { - val = GET_THREAD()->tag->retval; - } switch (state) { - case 0: - break; case TAG_RETURN: - result = make_localjump_error("unexpected return", val, state); - break; + mesg = "unexpected return"; + break; case TAG_BREAK: - result = make_localjump_error("unexpected break", val, state); - break; + mesg = "unexpected break"; + break; case TAG_NEXT: - result = make_localjump_error("unexpected next", val, state); - break; + mesg = "unexpected next"; + break; case TAG_REDO: - result = make_localjump_error("unexpected redo", Qnil, state); - break; + mesg = "unexpected redo"; + val = Qnil; + break; case TAG_RETRY: - result = make_localjump_error("retry outside of rescue clause", Qnil, state); - break; + mesg = "retry outside of rescue clause"; + val = Qnil; + break; default: - break; + return Qnil; } - return result; + if (UNDEF_P(val)) { + val = GET_EC()->tag->retval; + } + return make_localjump_error(mesg, val, state); } void -rb_vm_jump_tag_but_local_jump(int state) +rb_vm_jump_tag_but_local_jump(enum ruby_tag_type state) { VALUE exc = rb_vm_make_jump_tag_but_local_jump(state, Qundef); if (!NIL_P(exc)) rb_exc_raise(exc); - JUMP_TAG(state); + EC_JUMP_TAG(GET_EC(), state); +} + +static rb_control_frame_t * +next_not_local_frame(rb_control_frame_t *cfp) +{ + while (VM_ENV_LOCAL_P(cfp->ep)) { + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + } + return cfp; } -NORETURN(static void vm_iter_break(rb_thread_t *th, VALUE val)); +NORETURN(static void vm_iter_break(rb_execution_context_t *ec, VALUE val)); static void -vm_iter_break(rb_thread_t *th, VALUE val) +vm_iter_break(rb_execution_context_t *ec, VALUE val) { - rb_control_frame_t *cfp = th->cfp; - VALUE *ep = VM_CF_PREV_EP(cfp); + rb_control_frame_t *cfp = next_not_local_frame(ec->cfp); + const VALUE *ep = VM_CF_PREV_EP(cfp); + const rb_control_frame_t *target_cfp = rb_vm_search_cf_from_ep(ec, cfp, ep); - th->state = TAG_BREAK; - th->errinfo = (VALUE)NEW_THROW_OBJECT(val, (VALUE)ep, TAG_BREAK); - TH_JUMP_TAG(th, TAG_BREAK); + if (!target_cfp) { + rb_vm_localjump_error("unexpected break", val, TAG_BREAK); + } + + ec->errinfo = (VALUE)THROW_DATA_NEW(val, target_cfp, TAG_BREAK); + EC_JUMP_TAG(ec, TAG_BREAK); } void rb_iter_break(void) { - vm_iter_break(GET_THREAD(), Qnil); + vm_iter_break(GET_EC(), Qnil); } void rb_iter_break_value(VALUE val) { - vm_iter_break(GET_THREAD(), val); + vm_iter_break(GET_EC(), val); } /* optimization: redefine management */ -static st_table *vm_opt_method_table = 0; +short ruby_vm_redefined_flag[BOP_LAST_]; +static st_table *vm_opt_method_def_table = 0; +static st_table *vm_opt_mid_table = 0; + +void +rb_free_vm_opt_tables(void) +{ + st_free_table(vm_opt_method_def_table); + st_free_table(vm_opt_mid_table); +} static int vm_redefinition_check_flag(VALUE klass) { - if (klass == rb_cFixnum) return FIXNUM_REDEFINED_OP_FLAG; + if (klass == rb_cInteger) return INTEGER_REDEFINED_OP_FLAG; if (klass == rb_cFloat) return FLOAT_REDEFINED_OP_FLAG; if (klass == rb_cString) return STRING_REDEFINED_OP_FLAG; if (klass == rb_cArray) return ARRAY_REDEFINED_OP_FLAG; if (klass == rb_cHash) return HASH_REDEFINED_OP_FLAG; - if (klass == rb_cBignum) return BIGNUM_REDEFINED_OP_FLAG; if (klass == rb_cSymbol) return SYMBOL_REDEFINED_OP_FLAG; +#if 0 if (klass == rb_cTime) return TIME_REDEFINED_OP_FLAG; +#endif + if (klass == rb_cRegexp) return REGEXP_REDEFINED_OP_FLAG; + if (klass == rb_cNilClass) return NIL_REDEFINED_OP_FLAG; + if (klass == rb_cTrueClass) return TRUE_REDEFINED_OP_FLAG; + if (klass == rb_cFalseClass) return FALSE_REDEFINED_OP_FLAG; + if (klass == rb_cProc) return PROC_REDEFINED_OP_FLAG; return 0; } +int +rb_vm_check_optimizable_mid(VALUE mid) +{ + if (!vm_opt_mid_table) { + return FALSE; + } + + return st_lookup(vm_opt_mid_table, mid, NULL); +} + +static int +vm_redefinition_check_method_type(const rb_method_entry_t *me) +{ + if (me->called_id != me->def->original_id) { + return FALSE; + } + + if (METHOD_ENTRY_BASIC(me)) return TRUE; + + const rb_method_definition_t *def = me->def; + switch (def->type) { + case VM_METHOD_TYPE_CFUNC: + case VM_METHOD_TYPE_OPTIMIZED: + return TRUE; + default: + return FALSE; + } +} + static void rb_vm_check_redefinition_opt_method(const rb_method_entry_t *me, VALUE klass) { st_data_t bop; - if (!me->def || me->def->type == VM_METHOD_TYPE_CFUNC) { - if (st_lookup(vm_opt_method_table, (st_data_t)me, &bop)) { - int flag = vm_redefinition_check_flag(klass); - - ruby_vm_redefined_flag[bop] |= flag; - } + if (RB_TYPE_P(klass, T_ICLASS) && RICLASS_IS_ORIGIN_P(klass) && + RB_TYPE_P(RBASIC_CLASS(klass), T_CLASS)) { + klass = RBASIC_CLASS(klass); + } + if (vm_redefinition_check_method_type(me)) { + if (st_lookup(vm_opt_method_def_table, (st_data_t)me->def, &bop)) { + int flag = vm_redefinition_check_flag(klass); + if (flag != 0) { + rb_category_warn( + RB_WARN_CATEGORY_PERFORMANCE, + "Redefining '%s#%s' disables interpreter and JIT optimizations", + rb_class2name(me->owner), + rb_id2name(me->called_id) + ); + rb_yjit_bop_redefined(flag, (enum ruby_basic_operators)bop); + rb_zjit_bop_redefined(flag, (enum ruby_basic_operators)bop); + ruby_vm_redefined_flag[bop] |= flag; + } + } } } -static int -check_redefined_method(st_data_t key, st_data_t value, st_data_t data) +static enum rb_id_table_iterator_result +check_redefined_method(ID mid, VALUE value, void *data) { - ID mid = (ID)key; - rb_method_entry_t *me = (rb_method_entry_t *)value; VALUE klass = (VALUE)data; - rb_method_entry_t *newme = rb_method_entry(klass, mid, NULL); + const rb_method_entry_t *me = (rb_method_entry_t *)value; + const rb_method_entry_t *newme = rb_method_entry(klass, mid); - if (newme != me) - rb_vm_check_redefinition_opt_method(me, me->klass); - return ST_CONTINUE; + if (newme != me) rb_vm_check_redefinition_opt_method(me, me->owner); + + return ID_TABLE_CONTINUE; } void rb_vm_check_redefinition_by_prepend(VALUE klass) { if (!vm_redefinition_check_flag(klass)) return; - st_foreach(RCLASS_M_TBL(RCLASS_ORIGIN(klass)), check_redefined_method, - (st_data_t)klass); + rb_id_table_foreach(RCLASS_M_TBL(RCLASS_ORIGIN(klass)), check_redefined_method, (void *)klass); } static void -add_opt_method(VALUE klass, ID mid, VALUE bop) +add_opt_method_entry_bop(const rb_method_entry_t *me, ID mid, enum ruby_basic_operators bop) { - rb_method_entry_t *me; - if (st_lookup(RCLASS_M_TBL(klass), mid, (void *)&me) && me->def && - me->def->type == VM_METHOD_TYPE_CFUNC) { - st_insert(vm_opt_method_table, (st_data_t)me, (st_data_t)bop); + st_insert(vm_opt_method_def_table, (st_data_t)me->def, (st_data_t)bop); + st_insert(vm_opt_mid_table, (st_data_t)mid, (st_data_t)Qtrue); +} + +static void +add_opt_method(VALUE klass, ID mid, enum ruby_basic_operators bop) +{ + const rb_method_entry_t *me = rb_method_entry_at(klass, mid); + + if (me && vm_redefinition_check_method_type(me)) { + add_opt_method_entry_bop(me, mid, bop); } else { - rb_bug("undefined optimized method: %s", rb_id2name(mid)); + rb_bug("undefined optimized method: %s", rb_id2name(mid)); + } +} + +static enum ruby_basic_operators vm_redefinition_bop_for_id(ID mid); + +static void +add_opt_method_entry(const rb_method_entry_t *me) +{ + if (me && vm_redefinition_check_method_type(me)) { + ID mid = me->called_id; + enum ruby_basic_operators bop = vm_redefinition_bop_for_id(mid); + if ((int)bop >= 0) { + add_opt_method_entry_bop(me, mid, bop); + } } } @@ -1035,34 +2433,88 @@ static void vm_init_redefined_flag(void) { ID mid; - VALUE bop; - - vm_opt_method_table = st_init_numtable(); + enum ruby_basic_operators bop; #define OP(mid_, bop_) (mid = id##mid_, bop = BOP_##bop_, ruby_vm_redefined_flag[bop] = 0) #define C(k) add_opt_method(rb_c##k, mid, bop) - OP(PLUS, PLUS), (C(Fixnum), C(Float), C(String), C(Array)); - OP(MINUS, MINUS), (C(Fixnum), C(Float)); - OP(MULT, MULT), (C(Fixnum), C(Float)); - OP(DIV, DIV), (C(Fixnum), C(Float)); - OP(MOD, MOD), (C(Fixnum), C(Float)); - OP(Eq, EQ), (C(Fixnum), C(Float), C(String)); - OP(Eqq, EQQ), (C(Fixnum), C(Bignum), C(Float), C(Symbol), C(String)); - OP(LT, LT), (C(Fixnum), C(Float)); - OP(LE, LE), (C(Fixnum), C(Float)); - OP(GT, GT), (C(Fixnum), C(Float)); - OP(GE, GE), (C(Fixnum), C(Float)); + OP(PLUS, PLUS), (C(Integer), C(Float), C(String), C(Array)); + OP(MINUS, MINUS), (C(Integer), C(Float)); + OP(MULT, MULT), (C(Integer), C(Float)); + OP(DIV, DIV), (C(Integer), C(Float)); + OP(MOD, MOD), (C(Integer), C(Float)); + OP(Eq, EQ), (C(Integer), C(Float), C(String), C(Symbol)); + OP(Eqq, EQQ), (C(Integer), C(Float), C(Symbol), C(String), + C(NilClass), C(TrueClass), C(FalseClass)); + OP(LT, LT), (C(Integer), C(Float)); + OP(LE, LE), (C(Integer), C(Float)); + OP(GT, GT), (C(Integer), C(Float)); + OP(GE, GE), (C(Integer), C(Float)); OP(LTLT, LTLT), (C(String), C(Array)); - OP(AREF, AREF), (C(Array), C(Hash)); + OP(GTGT, GTGT), (C(Integer)); + OP(AREF, AREF), (C(Array), C(Hash), C(Integer)); OP(ASET, ASET), (C(Array), C(Hash)); OP(Length, LENGTH), (C(Array), C(String), C(Hash)); OP(Size, SIZE), (C(Array), C(String), C(Hash)); OP(EmptyP, EMPTY_P), (C(Array), C(String), C(Hash)); - OP(Succ, SUCC), (C(Fixnum), C(String), C(Time)); + OP(Succ, SUCC), (C(Integer), C(String)); + OP(EqTilde, MATCH), (C(Regexp), C(String)); + OP(Freeze, FREEZE), (C(String), C(Array), C(Hash)); + OP(UMinus, UMINUS), (C(String)); + OP(Max, MAX), (C(Array)); + OP(Min, MIN), (C(Array)); + OP(Hash, HASH), (C(Array)); + OP(Call, CALL), (C(Proc)); + OP(And, AND), (C(Integer)); + OP(Or, OR), (C(Integer)); + OP(NilP, NIL_P), (C(NilClass)); + OP(Cmp, CMP), (C(Integer), C(Float), C(String)); + OP(Default, DEFAULT), (C(Hash)); + OP(IncludeP, INCLUDE_P), (C(Array)); #undef C #undef OP } +static enum ruby_basic_operators +vm_redefinition_bop_for_id(ID mid) +{ + switch (mid) { +#define OP(mid_, bop_) case id##mid_: return BOP_##bop_ + OP(PLUS, PLUS); + OP(MINUS, MINUS); + OP(MULT, MULT); + OP(DIV, DIV); + OP(MOD, MOD); + OP(Eq, EQ); + OP(Eqq, EQQ); + OP(LT, LT); + OP(LE, LE); + OP(GT, GT); + OP(GE, GE); + OP(LTLT, LTLT); + OP(AREF, AREF); + OP(ASET, ASET); + OP(Length, LENGTH); + OP(Size, SIZE); + OP(EmptyP, EMPTY_P); + OP(Succ, SUCC); + OP(EqTilde, MATCH); + OP(Freeze, FREEZE); + OP(UMinus, UMINUS); + OP(Max, MAX); + OP(Min, MIN); + OP(Hash, HASH); + OP(Call, CALL); + OP(And, AND); + OP(Or, OR); + OP(NilP, NIL_P); + OP(Cmp, CMP); + OP(Default, DEFAULT); + OP(Pack, PACK); +#undef OP + } + return -1; +} + /* for vm development */ #if VMDEBUG @@ -1075,16 +2527,133 @@ vm_frametype_name(const rb_control_frame_t *cfp) case VM_FRAME_MAGIC_CLASS: return "class"; case VM_FRAME_MAGIC_TOP: return "top"; case VM_FRAME_MAGIC_CFUNC: return "cfunc"; - case VM_FRAME_MAGIC_PROC: return "proc"; case VM_FRAME_MAGIC_IFUNC: return "ifunc"; case VM_FRAME_MAGIC_EVAL: return "eval"; - case VM_FRAME_MAGIC_LAMBDA: return "lambda"; + case VM_FRAME_MAGIC_RESCUE: return "rescue"; default: - rb_bug("unknown frame"); + rb_bug("unknown frame"); + } +} +#endif + +static VALUE +frame_return_value(const struct vm_throw_data *err) +{ + if (THROW_DATA_P(err) && + THROW_DATA_STATE(err) == TAG_BREAK && + THROW_DATA_CONSUMED_P(err) == FALSE) { + return THROW_DATA_VAL(err); + } + else { + return Qnil; } } + +#if 0 +/* for debug */ +static const char * +frame_name(const rb_control_frame_t *cfp) +{ + unsigned long type = VM_FRAME_TYPE(cfp); +#define C(t) if (type == VM_FRAME_MAGIC_##t) return #t + C(METHOD); + C(BLOCK); + C(CLASS); + C(TOP); + C(CFUNC); + C(PROC); + C(IFUNC); + C(EVAL); + C(LAMBDA); + C(RESCUE); + C(DUMMY); +#undef C + return "unknown"; +} #endif +// cfp_returning_with_value: +// Whether cfp is the last frame in the unwinding process for a non-local return. +static void +hook_before_rewind(rb_execution_context_t *ec, bool cfp_returning_with_value, int state, struct vm_throw_data *err) +{ + if (state == TAG_RAISE && RBASIC(err)->klass == rb_eSysStackError) { + return; + } + else { + const rb_iseq_t *iseq = ec->cfp->iseq; + rb_hook_list_t *local_hooks = NULL; + unsigned int local_hooks_cnt = iseq->aux.exec.local_hooks_cnt; + if (RB_UNLIKELY(local_hooks_cnt > 0)) { + local_hooks = rb_iseq_local_hooks(iseq, rb_ec_ractor_ptr(ec), false); + } + + switch (VM_FRAME_TYPE(ec->cfp)) { + case VM_FRAME_MAGIC_METHOD: + RUBY_DTRACE_METHOD_RETURN_HOOK(ec, 0, 0); + EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_RETURN, ec->cfp->self, 0, 0, 0, frame_return_value(err)); + + if (UNLIKELY(local_hooks && local_hooks->events & RUBY_EVENT_RETURN)) { + rb_exec_event_hook_orig(ec, local_hooks, RUBY_EVENT_RETURN, + ec->cfp->self, 0, 0, 0, frame_return_value(err), TRUE); + } + + THROW_DATA_CONSUMED_SET(err); + break; + case VM_FRAME_MAGIC_BLOCK: + if (VM_FRAME_BMETHOD_P(ec->cfp)) { + VALUE bmethod_return_value = frame_return_value(err); + if (cfp_returning_with_value) { + // Non-local return terminating at a BMETHOD control frame. + bmethod_return_value = THROW_DATA_VAL(err); + } + + + EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_B_RETURN, ec->cfp->self, 0, 0, 0, bmethod_return_value); + if (UNLIKELY(local_hooks && local_hooks->events & RUBY_EVENT_B_RETURN)) { + rb_exec_event_hook_orig(ec, local_hooks, RUBY_EVENT_B_RETURN, + ec->cfp->self, 0, 0, 0, bmethod_return_value, TRUE); + } + + const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(ec->cfp); + + EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_RETURN, ec->cfp->self, + rb_vm_frame_method_entry(ec->cfp)->def->original_id, + rb_vm_frame_method_entry(ec->cfp)->called_id, + rb_vm_frame_method_entry(ec->cfp)->owner, + bmethod_return_value); + + VM_ASSERT(me->def->type == VM_METHOD_TYPE_BMETHOD); + unsigned int local_hooks_cnt = me->def->body.bmethod.local_hooks_cnt; + if (UNLIKELY(local_hooks_cnt > 0)) { + local_hooks = rb_method_def_local_hooks(me->def, rb_ec_ractor_ptr(ec), false); + if (local_hooks && local_hooks->events & RUBY_EVENT_RETURN) { + rb_exec_event_hook_orig(ec, local_hooks, RUBY_EVENT_RETURN, ec->cfp->self, + rb_vm_frame_method_entry(ec->cfp)->def->original_id, + rb_vm_frame_method_entry(ec->cfp)->called_id, + rb_vm_frame_method_entry(ec->cfp)->owner, + bmethod_return_value, TRUE); + } + } + + THROW_DATA_CONSUMED_SET(err); + } + else { + EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_B_RETURN, ec->cfp->self, 0, 0, 0, frame_return_value(err)); + if (UNLIKELY(local_hooks && local_hooks->events & RUBY_EVENT_B_RETURN)) { + rb_exec_event_hook_orig(ec, local_hooks, RUBY_EVENT_B_RETURN, + ec->cfp->self, 0, 0, 0, frame_return_value(err), TRUE); + } + THROW_DATA_CONSUMED_SET(err); + } + break; + case VM_FRAME_MAGIC_CLASS: + EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_END, ec->cfp->self, 0, 0, 0, Qnil); + break; + } + } +} + /* evaluator body */ /* finish @@ -1101,548 +2670,865 @@ vm_frametype_name(const rb_control_frame_t *cfp) struct CONTROL_FRAME { VALUE *pc; // cfp[0], program counter VALUE *sp; // cfp[1], stack pointer - VALUE *bp; // cfp[2], base pointer - rb_iseq_t *iseq; // cfp[3], iseq - VALUE flag; // cfp[4], magic - VALUE self; // cfp[5], self - VALUE *ep; // cfp[6], env pointer - rb_iseq_t * block_iseq; // cfp[7], block iseq - VALUE proc; // cfp[8], always 0 + rb_iseq_t *iseq; // cfp[2], iseq + VALUE self; // cfp[3], self + const VALUE *ep; // cfp[4], env pointer + const void *block_code; // cfp[5], block code }; - struct BLOCK { + struct rb_captured_block { VALUE self; VALUE *ep; - rb_iseq_t *block_iseq; - VALUE proc; - }; - - struct METHOD_CONTROL_FRAME { - rb_control_frame_t frame; + union code; }; - struct METHOD_FRAME { - VALUE arg0; - ... - VALUE argM; + struct METHOD_ENV { VALUE param0; ... VALUE paramN; - VALUE cref; - VALUE special; // lep [1] - struct block_object *block_ptr | 0x01; // lep [0] - }; - - struct BLOCK_CONTROL_FRAME { - rb_control_frame_t frame; + VALUE lvar1; + ... + VALUE lvarM; + VALUE cref; // ep[-2] + VALUE special; // ep[-1] + VALUE flags; // ep[ 0] == lep[0] }; - struct BLOCK_FRAME { - VALUE arg0; + struct BLOCK_ENV { + VALUE block_param0; ... - VALUE argM; - VALUE param0; + VALUE block_paramN; + VALUE block_lvar1; ... - VALUE paramN; - VALUE cref; - VALUE *(prev_ptr | 0x01); // ep[0] + VALUE block_lvarM; + VALUE cref; // ep[-2] + VALUE special; // ep[-1] + VALUE flags; // ep[ 0] }; - struct CLASS_CONTROL_FRAME { - rb_control_frame_t frame; - }; - - struct CLASS_FRAME { - VALUE param0; + struct CLASS_ENV { + VALUE class_lvar0; ... - VALUE paramN; + VALUE class_lvarN; VALUE cref; VALUE prev_ep; // for frame jump + VALUE flags; }; struct C_METHOD_CONTROL_FRAME { VALUE *pc; // 0 VALUE *sp; // stack pointer - VALUE *bp; // base pointer (used in exception) rb_iseq_t *iseq; // cmi - VALUE magic; // C_METHOD_FRAME VALUE self; // ? VALUE *ep; // ep == lep - rb_iseq_t * block_iseq; // - VALUE proc; // always 0 + void *code; // }; struct C_BLOCK_CONTROL_FRAME { VALUE *pc; // point only "finish" insn VALUE *sp; // sp rb_iseq_t *iseq; // ? - VALUE magic; // C_METHOD_FRAME - VALUE self; // needed? + VALUE self; // VALUE *ep; // ep - rb_iseq_t * block_iseq; // 0 + void *code; // }; */ +static inline VALUE +vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, VALUE errinfo); +static inline VALUE +vm_exec_loop(rb_execution_context_t *ec, enum ruby_tag_type state, struct rb_vm_tag *tag, VALUE result); -static VALUE -vm_exec(rb_thread_t *th) +// for non-Emscripten Wasm build, use vm_exec with optimized setjmp for runtime performance +#if defined(__wasm__) && !defined(__EMSCRIPTEN__) + +struct rb_vm_exec_context { + rb_execution_context_t *const ec; + struct rb_vm_tag *const tag; + + VALUE result; +}; + +static void +vm_exec_bottom_main(void *context) +{ + struct rb_vm_exec_context *ctx = context; + rb_execution_context_t *ec = ctx->ec; + + ctx->result = vm_exec_loop(ec, TAG_NONE, ctx->tag, vm_exec_core(ec)); +} + +static void +vm_exec_bottom_rescue(void *context) +{ + struct rb_vm_exec_context *ctx = context; + rb_execution_context_t *ec = ctx->ec; + + ctx->result = vm_exec_loop(ec, rb_ec_tag_state(ec), ctx->tag, ec->errinfo); +} +#endif + +VALUE +vm_exec(rb_execution_context_t *ec) { - int state; - VALUE result, err; - VALUE initial = 0; - VALUE *escape_ep = NULL; + VALUE result = Qundef; + + EC_PUSH_TAG(ec); - TH_PUSH_TAG(th); _tag.retval = Qnil; - if ((state = EXEC_TAG()) == 0) { - vm_loop_start: - result = vm_exec_core(th, initial); - if ((state = th->state) != 0) { - err = result; - th->state = 0; - goto exception_handler; - } + +#if defined(__wasm__) && !defined(__EMSCRIPTEN__) + struct rb_vm_exec_context ctx = { + .ec = ec, + .tag = &_tag, + }; + struct rb_wasm_try_catch try_catch; + + EC_REPUSH_TAG(); + + rb_wasm_try_catch_init(&try_catch, vm_exec_bottom_main, vm_exec_bottom_rescue, &ctx); + + rb_wasm_try_catch_loop_run(&try_catch, &RB_VM_TAG_JMPBUF_GET(_tag.buf)); + + result = ctx.result; +#else + enum ruby_tag_type state; + if ((state = EC_EXEC_TAG()) == TAG_NONE) { + if (UNDEF_P(result = jit_exec(ec))) { + result = vm_exec_core(ec); + } + /* fallback to the VM */ + result = vm_exec_loop(ec, TAG_NONE, &_tag, result); } else { - int i; - struct iseq_catch_table_entry *entry; - unsigned long epc, cont_pc, cont_sp; - VALUE catch_iseqval; - rb_control_frame_t *cfp; - VALUE type; - - err = th->errinfo; - - exception_handler: - cont_pc = cont_sp = catch_iseqval = 0; - - while (th->cfp->pc == 0 || th->cfp->iseq == 0) { - if (UNLIKELY(VM_FRAME_TYPE(th->cfp) == VM_FRAME_MAGIC_CFUNC)) { - const rb_method_entry_t *me = th->cfp->me; - EXEC_EVENT_HOOK(th, RUBY_EVENT_C_RETURN, th->cfp->self, me->called_id, me->klass, Qnil); - RUBY_DTRACE_METHOD_RETURN_HOOK(th, me->klass, me->called_id); - } - th->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(th->cfp); - } - - cfp = th->cfp; - epc = cfp->pc - cfp->iseq->iseq_encoded; - - if (state == TAG_BREAK || state == TAG_RETURN) { - escape_ep = GET_THROWOBJ_CATCH_POINT(err); - - if (cfp->ep == escape_ep) { - if (state == TAG_RETURN) { - if (!VM_FRAME_TYPE_FINISH_P(cfp)) { - SET_THROWOBJ_CATCH_POINT(err, (VALUE)(cfp + 1)->ep); - SET_THROWOBJ_STATE(err, state = TAG_BREAK); - } - else { - for (i = 0; i < cfp->iseq->catch_table_size; i++) { - entry = &cfp->iseq->catch_table[i]; - if (entry->start < epc && entry->end >= epc) { - if (entry->type == CATCH_TYPE_ENSURE) { - catch_iseqval = entry->iseq; - cont_pc = entry->cont; - cont_sp = entry->sp; - break; - } - } - } - if (!catch_iseqval) { - result = GET_THROWOBJ_VAL(err); - th->errinfo = Qnil; - vm_pop_frame(th); - goto finish_vme; - } - } - /* through */ - } - else { - /* TAG_BREAK */ -#if OPT_STACK_CACHING - initial = (GET_THROWOBJ_VAL(err)); -#else - *th->cfp->sp++ = (GET_THROWOBJ_VAL(err)); -#endif - th->errinfo = Qnil; - goto vm_loop_start; - } - } - } - - if (state == TAG_RAISE) { - for (i = 0; i < cfp->iseq->catch_table_size; i++) { - entry = &cfp->iseq->catch_table[i]; - if (entry->start < epc && entry->end >= epc) { - - if (entry->type == CATCH_TYPE_RESCUE || - entry->type == CATCH_TYPE_ENSURE) { - catch_iseqval = entry->iseq; - cont_pc = entry->cont; - cont_sp = entry->sp; - break; - } - } - } - } - else if (state == TAG_RETRY) { - for (i = 0; i < cfp->iseq->catch_table_size; i++) { - entry = &cfp->iseq->catch_table[i]; - if (entry->start < epc && entry->end >= epc) { - - if (entry->type == CATCH_TYPE_ENSURE) { - catch_iseqval = entry->iseq; - cont_pc = entry->cont; - cont_sp = entry->sp; - break; - } - else if (entry->type == CATCH_TYPE_RETRY) { - VALUE *escape_ep; - escape_ep = GET_THROWOBJ_CATCH_POINT(err); - if (cfp->ep == escape_ep) { - cfp->pc = cfp->iseq->iseq_encoded + entry->cont; - th->errinfo = Qnil; - goto vm_loop_start; - } - } - } - } - } - else if (state == TAG_BREAK && ((VALUE)escape_ep & ~0x03) == 0) { - type = CATCH_TYPE_BREAK; - - search_restart_point: - for (i = 0; i < cfp->iseq->catch_table_size; i++) { - entry = &cfp->iseq->catch_table[i]; - - if (entry->start < epc && entry->end >= epc) { - if (entry->type == CATCH_TYPE_ENSURE) { - catch_iseqval = entry->iseq; - cont_pc = entry->cont; - cont_sp = entry->sp; - break; - } - else if (entry->type == type) { - cfp->pc = cfp->iseq->iseq_encoded + entry->cont; - cfp->sp = vm_base_ptr(cfp) + entry->sp; - - if (state != TAG_REDO) { -#if OPT_STACK_CACHING - initial = (GET_THROWOBJ_VAL(err)); -#else - *th->cfp->sp++ = (GET_THROWOBJ_VAL(err)); + result = vm_exec_loop(ec, state, &_tag, ec->errinfo); + } #endif - } - th->errinfo = Qnil; - th->state = 0; - goto vm_loop_start; - } - } - } - } - else if (state == TAG_REDO) { - type = CATCH_TYPE_REDO; - goto search_restart_point; - } - else if (state == TAG_NEXT) { - type = CATCH_TYPE_NEXT; - goto search_restart_point; - } - else { - for (i = 0; i < cfp->iseq->catch_table_size; i++) { - entry = &cfp->iseq->catch_table[i]; - if (entry->start < epc && entry->end >= epc) { - - if (entry->type == CATCH_TYPE_ENSURE) { - catch_iseqval = entry->iseq; - cont_pc = entry->cont; - cont_sp = entry->sp; - break; - } - } - } - } - - if (catch_iseqval != 0) { - /* found catch table */ - rb_iseq_t *catch_iseq; - - /* enter catch scope */ - GetISeqPtr(catch_iseqval, catch_iseq); - cfp->sp = vm_base_ptr(cfp) + cont_sp; - cfp->pc = cfp->iseq->iseq_encoded + cont_pc; - - /* push block frame */ - cfp->sp[0] = err; - vm_push_frame(th, catch_iseq, VM_FRAME_MAGIC_BLOCK, - cfp->self, cfp->klass, - VM_ENVVAL_PREV_EP_PTR(cfp->ep), - catch_iseq->iseq_encoded, - cfp->sp + 1 /* push value */, - catch_iseq->local_size - 1, - cfp->me); - - state = 0; - th->state = 0; - th->errinfo = Qnil; - goto vm_loop_start; - } - else { - /* skip frame */ - - switch (VM_FRAME_TYPE(th->cfp)) { - case VM_FRAME_MAGIC_METHOD: - RUBY_DTRACE_METHOD_RETURN_HOOK(th, 0, 0); - EXEC_EVENT_HOOK_AND_POP_FRAME(th, RUBY_EVENT_RETURN, th->cfp->self, 0, 0, Qnil); - break; - case VM_FRAME_MAGIC_BLOCK: - EXEC_EVENT_HOOK_AND_POP_FRAME(th, RUBY_EVENT_B_RETURN, th->cfp->self, 0, 0, Qnil); - break; - case VM_FRAME_MAGIC_CLASS: - EXEC_EVENT_HOOK_AND_POP_FRAME(th, RUBY_EVENT_END, th->cfp->self, 0, 0, Qnil); - break; - } - - if (VM_FRAME_TYPE_FINISH_P(th->cfp)) { - vm_pop_frame(th); - th->errinfo = err; - TH_POP_TAG2(); - JUMP_TAG(state); - } - else { - th->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(th->cfp); - goto exception_handler; - } - } - } - finish_vme: - TH_POP_TAG(); + + EC_POP_TAG(); return result; } +static inline VALUE +vm_exec_loop(rb_execution_context_t *ec, enum ruby_tag_type state, + struct rb_vm_tag *tag, VALUE result) +{ + if (state == TAG_NONE) { /* no jumps, result is discarded */ + goto vm_loop_start; + } + + rb_ec_raised_reset(ec, RAISED_STACKOVERFLOW | RAISED_NOMEMORY); + while (UNDEF_P(result = vm_exec_handle_exception(ec, state, result))) { + // caught a jump, exec the handler. JIT code in jit_exec_exception() + // may return Qundef to run remaining frames with vm_exec_core(). + if (UNDEF_P(result = jit_exec_exception(ec))) { + result = vm_exec_core(ec); + } + vm_loop_start: + VM_ASSERT(ec->tag == tag); + /* when caught `throw`, `tag.state` is set. */ + if ((state = tag->state) == TAG_NONE) break; + tag->state = TAG_NONE; + } + + return result; +} + +static inline VALUE +vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, VALUE errinfo) +{ + struct vm_throw_data *err = (struct vm_throw_data *)errinfo; + + for (;;) { + unsigned int i; + const struct iseq_catch_table_entry *entry; + const struct iseq_catch_table *ct; + unsigned long epc, cont_pc, cont_sp; + const rb_iseq_t *catch_iseq; + VALUE type; + const rb_control_frame_t *escape_cfp; + + cont_pc = cont_sp = 0; + catch_iseq = NULL; + + while (ec->cfp->pc == 0 || ec->cfp->iseq == 0) { + if (UNLIKELY(VM_FRAME_TYPE(ec->cfp) == VM_FRAME_MAGIC_CFUNC)) { + EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_C_RETURN, ec->cfp->self, + rb_vm_frame_method_entry(ec->cfp)->def->original_id, + rb_vm_frame_method_entry(ec->cfp)->called_id, + rb_vm_frame_method_entry(ec->cfp)->owner, Qnil); + RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, + rb_vm_frame_method_entry(ec->cfp)->owner, + rb_vm_frame_method_entry(ec->cfp)->def->original_id); + } + rb_vm_pop_frame(ec); + } + + rb_control_frame_t *const cfp = ec->cfp; + epc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded; + + escape_cfp = NULL; + if (state == TAG_BREAK || state == TAG_RETURN) { + escape_cfp = THROW_DATA_CATCH_FRAME(err); + + if (cfp == escape_cfp) { + if (state == TAG_RETURN) { + if (!VM_FRAME_FINISHED_P(cfp)) { + THROW_DATA_CATCH_FRAME_SET(err, cfp + 1); + THROW_DATA_STATE_SET(err, state = TAG_BREAK); + } + else { + ct = ISEQ_BODY(cfp->iseq)->catch_table; + if (ct) for (i = 0; i < ct->size; i++) { + entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); + if (entry->start < epc && entry->end >= epc) { + if (entry->type == CATCH_TYPE_ENSURE) { + catch_iseq = entry->iseq; + cont_pc = entry->cont; + cont_sp = entry->sp; + break; + } + } + } + if (catch_iseq == NULL) { + ec->errinfo = Qnil; + THROW_DATA_CATCH_FRAME_SET(err, cfp + 1); + // cfp == escape_cfp here so calling with cfp_returning_with_value = true + hook_before_rewind(ec, true, state, err); + rb_vm_pop_frame(ec); + return THROW_DATA_VAL(err); + } + } + /* through */ + } + else { + /* TAG_BREAK */ + *cfp->sp++ = THROW_DATA_VAL(err); + ec->errinfo = Qnil; + return Qundef; + } + } + } + + if (state == TAG_RAISE) { + ct = ISEQ_BODY(cfp->iseq)->catch_table; + if (ct) for (i = 0; i < ct->size; i++) { + entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); + if (entry->start < epc && entry->end >= epc) { + + if (entry->type == CATCH_TYPE_RESCUE || + entry->type == CATCH_TYPE_ENSURE) { + catch_iseq = entry->iseq; + cont_pc = entry->cont; + cont_sp = entry->sp; + break; + } + } + } + } + else if (state == TAG_RETRY) { + ct = ISEQ_BODY(cfp->iseq)->catch_table; + if (ct) for (i = 0; i < ct->size; i++) { + entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); + if (entry->start < epc && entry->end >= epc) { + + if (entry->type == CATCH_TYPE_ENSURE) { + catch_iseq = entry->iseq; + cont_pc = entry->cont; + cont_sp = entry->sp; + break; + } + else if (entry->type == CATCH_TYPE_RETRY) { + const rb_control_frame_t *escape_cfp; + escape_cfp = THROW_DATA_CATCH_FRAME(err); + if (cfp == escape_cfp) { + cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + entry->cont; + ec->errinfo = Qnil; + return Qundef; + } + } + } + } + } + else if ((state == TAG_BREAK && !escape_cfp) || + (state == TAG_REDO) || + (state == TAG_NEXT)) { + type = (const enum rb_catch_type[TAG_MASK]) { + [TAG_BREAK] = CATCH_TYPE_BREAK, + [TAG_NEXT] = CATCH_TYPE_NEXT, + [TAG_REDO] = CATCH_TYPE_REDO, + /* otherwise = dontcare */ + }[state]; + + ct = ISEQ_BODY(cfp->iseq)->catch_table; + if (ct) for (i = 0; i < ct->size; i++) { + entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); + + if (entry->start < epc && entry->end >= epc) { + if (entry->type == CATCH_TYPE_ENSURE) { + catch_iseq = entry->iseq; + cont_pc = entry->cont; + cont_sp = entry->sp; + break; + } + else if (entry->type == type) { + cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + entry->cont; + cfp->sp = vm_base_ptr(cfp) + entry->sp; + + if (state != TAG_REDO) { + *cfp->sp++ = THROW_DATA_VAL(err); + } + ec->errinfo = Qnil; + VM_ASSERT(ec->tag->state == TAG_NONE); + return Qundef; + } + } + } + } + else { + ct = ISEQ_BODY(cfp->iseq)->catch_table; + if (ct) for (i = 0; i < ct->size; i++) { + entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); + if (entry->start < epc && entry->end >= epc) { + + if (entry->type == CATCH_TYPE_ENSURE) { + catch_iseq = entry->iseq; + cont_pc = entry->cont; + cont_sp = entry->sp; + break; + } + } + } + } + + if (catch_iseq != NULL) { /* found catch table */ + /* enter catch scope */ + const int arg_size = 1; + + rb_iseq_check(catch_iseq); + cfp->sp = vm_base_ptr(cfp) + cont_sp; + cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + cont_pc; + + /* push block frame */ + cfp->sp[0] = (VALUE)err; + vm_push_frame(ec, catch_iseq, VM_FRAME_MAGIC_RESCUE, + cfp->self, + VM_GUARDED_PREV_EP(cfp->ep), + 0, /* cref or me */ + ISEQ_BODY(catch_iseq)->iseq_encoded, + cfp->sp + arg_size /* push value */, + ISEQ_BODY(catch_iseq)->local_table_size - arg_size, + ISEQ_BODY(catch_iseq)->stack_max); + + state = 0; + ec->tag->state = TAG_NONE; + ec->errinfo = Qnil; + + return Qundef; + } + else { + hook_before_rewind(ec, (cfp == escape_cfp), state, err); + + if (VM_FRAME_FINISHED_P(ec->cfp)) { + rb_vm_pop_frame(ec); + ec->errinfo = (VALUE)err; + rb_vm_tag_jmpbuf_deinit(&ec->tag->buf); + ec->tag = ec->tag->prev; + EC_JUMP_TAG(ec, state); + } + else { + rb_vm_pop_frame(ec); + } + } + } +} + /* misc */ VALUE -rb_iseq_eval(VALUE iseqval) +rb_iseq_eval(const rb_iseq_t *iseq, const rb_box_t *box) { - rb_thread_t *th = GET_THREAD(); + rb_execution_context_t *ec = GET_EC(); VALUE val; - - vm_set_top_stack(th, iseqval); - - val = vm_exec(th); - RB_GC_GUARD(iseqval); /* prohibit tail call optimization */ + vm_set_top_stack(ec, iseq, box); + val = vm_exec(ec); return val; } VALUE -rb_iseq_eval_main(VALUE iseqval) +rb_iseq_eval_main(const rb_iseq_t *iseq) { - rb_thread_t *th = GET_THREAD(); + rb_execution_context_t *ec = GET_EC(); VALUE val; - - vm_set_main_stack(th, iseqval); - - val = vm_exec(th); - RB_GC_GUARD(iseqval); /* prohibit tail call optimization */ + vm_set_main_stack(ec, iseq); + val = vm_exec(ec); return val; } int -rb_vm_control_frame_id_and_class(const rb_control_frame_t *cfp, ID *idp, VALUE *klassp) -{ - rb_iseq_t *iseq = cfp->iseq; - if (!iseq && cfp->me) { - if (idp) *idp = cfp->me->def->original_id; - if (klassp) *klassp = cfp->me->klass; - return 1; - } - while (iseq) { - if (RUBY_VM_IFUNC_P(iseq)) { - if (idp) *idp = idIFUNC; - if (klassp) *klassp = 0; - return 1; - } - if (iseq->defined_method_id) { - if (idp) *idp = iseq->defined_method_id; - if (klassp) *klassp = iseq->klass; - return 1; - } - if (iseq->local_iseq == iseq) { - break; - } - iseq = iseq->parent_iseq; +rb_vm_control_frame_id_and_class(const rb_control_frame_t *cfp, ID *idp, ID *called_idp, VALUE *klassp) +{ + const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(cfp); + + if (me) { + if (idp) *idp = me->def->original_id; + if (called_idp) *called_idp = me->called_id; + if (klassp) *klassp = me->owner; + return TRUE; + } + else { + return FALSE; } - return 0; } int -rb_thread_method_id_and_class(rb_thread_t *th, ID *idp, VALUE *klassp) +rb_ec_frame_method_id_and_class(const rb_execution_context_t *ec, ID *idp, ID *called_idp, VALUE *klassp) { - return rb_vm_control_frame_id_and_class(th->cfp, idp, klassp); + return rb_vm_control_frame_id_and_class(ec->cfp, idp, called_idp, klassp); } int rb_frame_method_id_and_class(ID *idp, VALUE *klassp) { - return rb_thread_method_id_and_class(GET_THREAD(), idp, klassp); + return rb_ec_frame_method_id_and_class(GET_EC(), idp, 0, klassp); } VALUE -rb_thread_current_status(const rb_thread_t *th) +rb_vm_call_cfunc(VALUE recv, VALUE (*func)(VALUE), VALUE arg, + VALUE block_handler, VALUE filename) { - const rb_control_frame_t *cfp = th->cfp; - VALUE str = Qnil; + rb_execution_context_t *ec = GET_EC(); + const rb_control_frame_t *reg_cfp = ec->cfp; + const rb_iseq_t *iseq = rb_iseq_new(Qnil, filename, filename, Qnil, 0, ISEQ_TYPE_TOP); + const rb_box_t *box = rb_current_box(); + VALUE val; - if (cfp->iseq != 0) { - if (cfp->pc != 0) { - rb_iseq_t *iseq = cfp->iseq; - int line_no = rb_vm_get_sourceline(cfp); - char *file = RSTRING_PTR(iseq->location.path); - str = rb_sprintf("%s:%d:in `%s'", - file, line_no, RSTRING_PTR(iseq->location.label)); - } - } - else if (cfp->me->def->original_id) { - str = rb_sprintf("`%s#%s' (cfunc)", - rb_class2name(cfp->me->klass), - rb_id2name(cfp->me->def->original_id)); - } + vm_push_frame(ec, iseq, VM_FRAME_MAGIC_TOP | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH, + recv, GC_GUARDED_PTR(box), + (VALUE)vm_cref_new_toplevel(ec), /* cref or me */ + 0, reg_cfp->sp, 0, 0); + + val = (*func)(arg); - return str; + rb_vm_pop_frame(ec); + return val; } +/* Ruby::Box */ + VALUE -rb_vm_call_cfunc(VALUE recv, VALUE (*func)(VALUE), VALUE arg, - const rb_block_t *blockptr, VALUE filename) +rb_vm_call_cfunc_in_box(VALUE recv, VALUE (*func)(VALUE, VALUE), VALUE arg1, VALUE arg2, + VALUE filename, const rb_box_t *box) { - rb_thread_t *th = GET_THREAD(); - const rb_control_frame_t *reg_cfp = th->cfp; - volatile VALUE iseqval = rb_iseq_new(0, filename, filename, Qnil, 0, ISEQ_TYPE_TOP); + rb_execution_context_t *ec = GET_EC(); + const rb_control_frame_t *reg_cfp = ec->cfp; + const rb_iseq_t *iseq = rb_iseq_new(Qnil, filename, filename, Qnil, 0, ISEQ_TYPE_TOP); VALUE val; - vm_push_frame(th, DATA_PTR(iseqval), VM_FRAME_MAGIC_TOP | VM_FRAME_FLAG_FINISH, - recv, CLASS_OF(recv), VM_ENVVAL_BLOCK_PTR(blockptr), 0, reg_cfp->sp, 1, 0); + vm_push_frame(ec, iseq, VM_FRAME_MAGIC_TOP | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH, + recv, GC_GUARDED_PTR(box), + (VALUE)vm_cref_new_toplevel(ec), /* cref or me */ + 0, reg_cfp->sp, 0, 0); - val = (*func)(arg); + val = (*func)(arg1, arg2); - vm_pop_frame(th); + rb_vm_pop_frame(ec); return val; } +void +rb_vm_frame_flag_set_box_require(const rb_execution_context_t *ec) +{ + VM_ASSERT(rb_box_available()); + VM_ENV_FLAGS_SET(ec->cfp->ep, VM_FRAME_FLAG_BOX_REQUIRE); +} + +static const rb_box_t * +current_box_on_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) +{ + rb_callable_method_entry_t *cme; + const rb_box_t *box; + const VALUE *lep = VM_EP_RUBY_LEP(ec, cfp); + VM_BOX_ASSERT(lep, "lep should be valid"); + VM_BOX_ASSERT(rb_box_available(), "box should be available here"); + + if (VM_ENV_FRAME_TYPE_P(lep, VM_FRAME_MAGIC_METHOD) || VM_ENV_FRAME_TYPE_P(lep, VM_FRAME_MAGIC_CFUNC)) { + cme = check_method_entry(lep[VM_ENV_DATA_INDEX_ME_CREF], TRUE); + VM_BOX_ASSERT(cme, "cme should be valid"); + VM_BOX_ASSERT(cme->def, "cme->def shold be valid"); + return cme->def->box; + } + else if (VM_ENV_FRAME_TYPE_P(lep, VM_FRAME_MAGIC_TOP) || VM_ENV_FRAME_TYPE_P(lep, VM_FRAME_MAGIC_CLASS)) { + VM_BOX_ASSERT(VM_ENV_LOCAL_P(lep), "lep should be local on MAGIC_TOP or MAGIC_CLASS frames"); + return VM_ENV_BOX(lep); + } + else if (VM_ENV_FRAME_TYPE_P(lep, VM_FRAME_MAGIC_DUMMY)) { + // No valid local ep found (just after process boot?) + // return the root box (the only valid box) until the main is initialized + box = rb_main_box(); + if (box) + return box; + return rb_root_box(); + } + else { + VM_BOX_CRASHED(); + rb_bug("BUG: Local ep without cme/box, flags: %08lX", (unsigned long)lep[VM_ENV_DATA_INDEX_FLAGS]); + } + UNREACHABLE_RETURN(0); +} + +const rb_box_t * +rb_vm_current_box(const rb_execution_context_t *ec) +{ + return current_box_on_cfp(ec, ec->cfp); +} + +static const rb_control_frame_t * +find_loader_control_frame(const rb_execution_context_t *ec, const rb_control_frame_t *cfp, const rb_control_frame_t *end_cfp) +{ + while (RUBY_VM_VALID_CONTROL_FRAME_P(cfp, end_cfp)) { + if (!VM_ENV_FRAME_TYPE_P(cfp->ep, VM_FRAME_MAGIC_CFUNC)) + break; + if (!BOX_ROOT_P(current_box_on_cfp(ec, cfp))) + break; + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + } + VM_ASSERT(RUBY_VM_VALID_CONTROL_FRAME_P(cfp, end_cfp)); + return cfp; +} + +const rb_box_t * +rb_vm_loading_box(const rb_execution_context_t *ec) +{ + const rb_control_frame_t *cfp, *current_cfp, *end_cfp; + + if (!rb_box_available() || !ec) + return rb_root_box(); + + cfp = ec->cfp; + current_cfp = cfp; + end_cfp = RUBY_VM_END_CONTROL_FRAME(ec); + + while (RUBY_VM_VALID_CONTROL_FRAME_P(cfp, end_cfp)) { + if (VM_ENV_FLAGS(cfp->ep, VM_FRAME_FLAG_BOX_REQUIRE)) { + if (RTEST(cfp->self) && BOX_OBJ_P(cfp->self)) { + // Box#require, #require_relative, #load + return rb_get_box_t(cfp->self); + } + // Kernel#require, #require_relative, #load + cfp = find_loader_control_frame(ec, cfp, end_cfp); + return current_box_on_cfp(ec, cfp); + } + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + } + // no require/load with explicit boxes. + return current_box_on_cfp(ec, current_cfp); +} + /* vm */ -static int -vm_mark_each_thread_func(st_data_t key, st_data_t value, st_data_t dummy) +void +rb_vm_update_references(void *ptr) { - VALUE thval = (VALUE)key; - rb_gc_mark(thval); - return ST_CONTINUE; + if (ptr) { + rb_vm_t *vm = ptr; + + vm->self = rb_gc_location(vm->self); + vm->mark_object_ary = rb_gc_location(vm->mark_object_ary); + vm->orig_progname = rb_gc_location(vm->orig_progname); + + if (vm->root_box) + rb_box_gc_update_references(vm->root_box); + if (vm->main_box) + rb_box_gc_update_references(vm->main_box); + + rb_gc_update_values(RUBY_NSIG, vm->trap_list.cmd); + + if (vm->coverages) { + vm->coverages = rb_gc_location(vm->coverages); + vm->me2counter = rb_gc_location(vm->me2counter); + } + } } -void rb_vm_trace_mark_event_hooks(rb_hook_list_t *hooks); +void +rb_vm_each_stack_value(void *ptr, void (*cb)(VALUE, void*), void *ctx) +{ + if (ptr) { + rb_vm_t *vm = ptr; + rb_ractor_t *r = 0; + ccan_list_for_each(&vm->ractor.set, r, vmlr_node) { + VM_ASSERT(rb_ractor_status_p(r, ractor_blocking) || + rb_ractor_status_p(r, ractor_running)); + if (r->threads.cnt > 0) { + rb_thread_t *th = 0; + ccan_list_for_each(&r->threads.set, th, lt_node) { + VM_ASSERT(th != NULL); + rb_execution_context_t * ec = th->ec; + if (ec->vm_stack) { + VALUE *p = ec->vm_stack; + VALUE *sp = ec->cfp->sp; + while (p < sp) { + if (!RB_SPECIAL_CONST_P(*p)) { + cb(*p, ctx); + } + p++; + } + } + } + } + } + } +} + +static enum rb_id_table_iterator_result +vm_mark_negative_cme(VALUE val, void *dmy) +{ + rb_gc_mark(val); + return ID_TABLE_CONTINUE; +} + +void rb_thread_sched_mark_zombies(rb_vm_t *vm); void rb_vm_mark(void *ptr) { - int i; - RUBY_MARK_ENTER("vm"); RUBY_GC_INFO("-------------------------------------------------\n"); if (ptr) { - rb_vm_t *vm = ptr; - if (vm->living_threads) { - st_foreach(vm->living_threads, vm_mark_each_thread_func, 0); - } - RUBY_MARK_UNLESS_NULL(vm->thgroup_default); - RUBY_MARK_UNLESS_NULL(vm->mark_object_ary); - RUBY_MARK_UNLESS_NULL(vm->load_path); - RUBY_MARK_UNLESS_NULL(vm->load_path_snapshot); - RUBY_MARK_UNLESS_NULL(vm->load_path_check_cache); - RUBY_MARK_UNLESS_NULL(vm->expanded_load_path); - RUBY_MARK_UNLESS_NULL(vm->loaded_features); - RUBY_MARK_UNLESS_NULL(vm->loaded_features_snapshot); - RUBY_MARK_UNLESS_NULL(vm->top_self); - RUBY_MARK_UNLESS_NULL(vm->coverages); - rb_gc_mark_locations(vm->special_exceptions, vm->special_exceptions + ruby_special_error_count); - - if (vm->loading_table) { - rb_mark_tbl(vm->loading_table); - } - if (vm->loaded_features_index) { - rb_mark_tbl(vm->loaded_features_index); - } - - rb_vm_trace_mark_event_hooks(&vm->event_hooks); - - for (i = 0; i < RUBY_NSIG; i++) { - if (vm->trap_list[i].cmd) - rb_gc_mark(vm->trap_list[i].cmd); - } - if (vm->defined_strings) { - rb_gc_mark_locations(vm->defined_strings, vm->defined_strings + DEFINED_EXPR); - } + rb_vm_t *vm = ptr; + rb_ractor_t *r = 0; + long i; + + ccan_list_for_each(&vm->ractor.set, r, vmlr_node) { + // ractor.set only contains blocking or running ractors + VM_ASSERT(rb_ractor_status_p(r, ractor_blocking) || + rb_ractor_status_p(r, ractor_running)); + rb_gc_mark(rb_ractor_self(r)); + } + + for (struct global_object_list *list = vm->global_object_list; list; list = list->next) { + rb_gc_mark_maybe(*list->varptr); + } + + rb_gc_mark_movable(vm->self); + + if (vm->root_box) { + rb_box_entry_mark(vm->root_box); + } + if (vm->main_box) { + rb_box_entry_mark(vm->main_box); + } + + rb_gc_mark_movable(vm->mark_object_ary); + rb_gc_mark_movable(vm->orig_progname); + rb_gc_mark_movable(vm->coverages); + rb_gc_mark_movable(vm->me2counter); + + rb_gc_mark_values(RUBY_NSIG, vm->trap_list.cmd); + + rb_hook_list_mark(&vm->global_hooks); + + rb_id_table_foreach_values(vm->negative_cme_table, vm_mark_negative_cme, NULL); + rb_mark_tbl_no_pin(vm->overloaded_cme_table); + for (i=0; i<VM_GLOBAL_CC_CACHE_TABLE_SIZE; i++) { + const struct rb_callcache *cc = vm->global_cc_cache_table[i]; + + if (cc != NULL) { + if (!vm_cc_invalidated_p(cc)) { + rb_gc_mark((VALUE)cc); + } + else { + vm->global_cc_cache_table[i] = NULL; + } + } + } + + rb_thread_sched_mark_zombies(vm); } RUBY_MARK_LEAVE("vm"); } -#define vm_free 0 +#undef rb_vm_register_special_exception +void +rb_vm_register_special_exception_str(enum ruby_special_exceptions sp, VALUE cls, VALUE mesg) +{ + rb_vm_t *vm = GET_VM(); + VALUE exc = rb_exc_new3(cls, rb_obj_freeze(mesg)); + OBJ_FREEZE(exc); + ((VALUE *)vm->special_exceptions)[sp] = exc; + rb_vm_register_global_object(exc); +} + +void rb_objspace_free_objects(void *objspace); int ruby_vm_destruct(rb_vm_t *vm) { RUBY_FREE_ENTER("vm"); + ruby_vm_during_cleanup = true; + if (vm) { - rb_thread_t *th = vm->main_thread; -#if defined(ENABLE_VM_OBJSPACE) && ENABLE_VM_OBJSPACE - struct rb_objspace *objspace = vm->objspace; + rb_thread_t *th = vm->ractor.main_thread; + + if (rb_free_at_exit) { + rb_free_encoded_insn_data(); + rb_free_global_enc_table(); + rb_free_loaded_builtin_table(); + rb_free_global_symbol_table(); + + rb_free_shared_fiber_pool(); + rb_free_transcoder_table(); + rb_free_vm_opt_tables(); + rb_free_warning(); + rb_free_rb_global_tbl(); + + rb_id_table_free(vm->negative_cme_table); + st_free_table(vm->overloaded_cme_table); + + // TODO: Is this ignorable for classext->m_tbl ? + // rb_id_table_free(RCLASS(rb_mRubyVMFrozenCore)->m_tbl); + + st_free_table(vm->static_ext_inits); + + rb_vm_postponed_job_free(); + + rb_id_table_free(vm->constant_cache); + set_free_table(vm->unused_block_warning_table); + + rb_thread_free_native_thread(th); + +#ifndef HAVE_SETPROCTITLE + ruby_free_proctitle(); #endif - rb_gc_force_recycle(vm->self); - vm->main_thread = 0; - if (th) { - rb_fiber_reset_root_local_storage(th->self); - thread_free(th); - } - if (vm->living_threads) { - st_free_table(vm->living_threads); - vm->living_threads = 0; - } -#if defined(ENABLE_VM_OBJSPACE) && ENABLE_VM_OBJSPACE - if (objspace) { - rb_objspace_free(objspace); - } + } + else { + rb_fiber_reset_root_local_storage(th); + thread_free(th); + } + + struct rb_objspace *objspace = vm->gc.objspace; + + rb_vm_living_threads_init(vm); + ruby_vm_run_at_exit_hooks(vm); + if (vm->ci_table) { + st_free_table(vm->ci_table); + vm->ci_table = NULL; + } + if (vm->cc_refinement_table) { + rb_set_free_table(vm->cc_refinement_table); + vm->cc_refinement_table = NULL; + } + RB_ALTSTACK_FREE(vm->main_altstack); + + struct global_object_list *next; + for (struct global_object_list *list = vm->global_object_list; list; list = next) { + next = list->next; + xfree(list); + } + + if (objspace) { + if (rb_free_at_exit) { + rb_objspace_free_objects(objspace); + rb_free_generic_fields_tbl_(); + rb_free_default_rand_key(); + + ruby_mimfree(th); + } + rb_objspace_free(objspace); + } + rb_native_mutex_destroy(&vm->workqueue_lock); + /* after freeing objspace, you *can't* use ruby_xfree() */ + ruby_mimfree(vm); + ruby_current_vm_ptr = NULL; + + if (rb_free_at_exit) { + rb_shape_free_all(); +#if USE_YJIT + rb_yjit_free_at_exit(); #endif - ruby_vm_run_at_exit_hooks(vm); - rb_vm_gvl_destroy(vm); - ruby_xfree(vm); - ruby_current_vm = 0; + } } RUBY_FREE_LEAVE("vm"); return 0; } +size_t rb_vm_memsize_workqueue(struct ccan_list_head *workqueue); // vm_trace.c + +// Used for VM memsize reporting. Returns the size of the at_exit list by +// looping through the linked list and adding up the size of the structs. +static enum rb_id_table_iterator_result +vm_memsize_constant_cache_i(ID id, VALUE ics, void *size) +{ + *((size_t *) size) += rb_set_memsize((set_table *) ics); + return ID_TABLE_CONTINUE; +} + +// Returns a size_t representing the memory footprint of the VM's constant +// cache, which is the memsize of the table as well as the memsize of all of the +// nested tables. static size_t -vm_memsize(const void *ptr) +vm_memsize_constant_cache(void) { - if (ptr) { - const rb_vm_t *vmobj = ptr; - size_t size = sizeof(rb_vm_t); - size += st_memsize(vmobj->living_threads); - if (vmobj->defined_strings) { - size += DEFINED_EXPR * sizeof(VALUE); - } - return size; - } - else { - return 0; + rb_vm_t *vm = GET_VM(); + size_t size = rb_id_table_memsize(vm->constant_cache); + + rb_id_table_foreach(vm->constant_cache, vm_memsize_constant_cache_i, &size); + return size; +} + +static size_t +vm_memsize_at_exit_list(rb_at_exit_list *at_exit) +{ + size_t size = 0; + + while (at_exit) { + size += sizeof(rb_at_exit_list); + at_exit = at_exit->next; } + + return size; +} + +// Used for VM memsize reporting. Returns the size of the builtin function +// table if it has been defined. +static size_t +vm_memsize_builtin_function_table(const struct rb_builtin_function *builtin_function_table) +{ + return builtin_function_table == NULL ? 0 : sizeof(struct rb_builtin_function); +} + +// Reports the memsize of the VM struct object and the structs that are +// associated with it. +static size_t +vm_memsize(const void *ptr) +{ + rb_vm_t *vm = GET_VM(); + + return ( + sizeof(rb_vm_t) + + rb_vm_memsize_postponed_job_queue() + + rb_vm_memsize_workqueue(&vm->workqueue) + + vm_memsize_at_exit_list(vm->at_exit) + + rb_st_memsize(vm->ci_table) + + vm_memsize_builtin_function_table(vm->builtin_function_table) + + rb_id_table_memsize(vm->negative_cme_table) + + rb_st_memsize(vm->overloaded_cme_table) + + rb_set_memsize(vm->cc_refinement_table) + + vm_memsize_constant_cache() + ); + + // TODO + // struct { struct ccan_list_head set; } ractor; + // void *main_altstack; #ifdef USE_SIGALTSTACK + // struct rb_objspace *objspace; } static const rb_data_type_t vm_data_type = { "VM", - {rb_vm_mark, vm_free, vm_memsize,}, + {0, 0, vm_memsize,}, + 0, 0, RUBY_TYPED_FREE_IMMEDIATELY }; @@ -1650,7 +3536,7 @@ static VALUE vm_default_params(void) { rb_vm_t *vm = GET_VM(); - VALUE result = rb_hash_new(); + VALUE result = rb_hash_new_with_size(4); #define SET(name) rb_hash_aset(result, ID2SYM(rb_intern(#name)), SIZET2NUM(vm->default_params.name)); SET(thread_vm_stack_size); SET(thread_machine_stack_size); @@ -1667,13 +3553,13 @@ get_param(const char *name, size_t default_value, size_t min_value) const char *envval; size_t result = default_value; if ((envval = getenv(name)) != 0) { - long val = atol(envval); - if (val < (long)min_value) { - val = (long)min_value; - } - result = (size_t)(((val -1 + RUBY_VM_SIZE_ALIGN) / RUBY_VM_SIZE_ALIGN) * RUBY_VM_SIZE_ALIGN); + long val = atol(envval); + if (val < (long)min_value) { + val = (long)min_value; + } + result = (size_t)(((val -1 + RUBY_VM_SIZE_ALIGN) / RUBY_VM_SIZE_ALIGN) * RUBY_VM_SIZE_ALIGN); } - if (0) fprintf(stderr, "%s: %"PRIdSIZE"\n", name, result); /* debug print */ + if (0) ruby_debug_printf("%s: %"PRIuSIZE"\n", name, result); /* debug print */ return result; } @@ -1685,13 +3571,9 @@ check_machine_stack_size(size_t *sizep) size_t size = *sizep; #endif -#ifdef __SYMBIAN32__ - *sizep = 64 * 1024; /* 64KB: Let's be slightly more frugal on mobile platform */ -#endif - #ifdef PTHREAD_STACK_MIN - if (size < PTHREAD_STACK_MIN) { - *sizep = PTHREAD_STACK_MIN * 2; + if (size < (size_t)PTHREAD_STACK_MIN) { + *sizep = (size_t)PTHREAD_STACK_MIN * 2; } #endif } @@ -1701,23 +3583,23 @@ vm_default_params_setup(rb_vm_t *vm) { vm->default_params.thread_vm_stack_size = get_param("RUBY_THREAD_VM_STACK_SIZE", - RUBY_VM_THREAD_VM_STACK_SIZE, - RUBY_VM_THREAD_VM_STACK_SIZE_MIN); + RUBY_VM_THREAD_VM_STACK_SIZE, + RUBY_VM_THREAD_VM_STACK_SIZE_MIN); vm->default_params.thread_machine_stack_size = get_param("RUBY_THREAD_MACHINE_STACK_SIZE", - RUBY_VM_THREAD_MACHINE_STACK_SIZE, - RUBY_VM_THREAD_MACHINE_STACK_SIZE_MIN); + RUBY_VM_THREAD_MACHINE_STACK_SIZE, + RUBY_VM_THREAD_MACHINE_STACK_SIZE_MIN); vm->default_params.fiber_vm_stack_size = get_param("RUBY_FIBER_VM_STACK_SIZE", - RUBY_VM_FIBER_VM_STACK_SIZE, - RUBY_VM_FIBER_VM_STACK_SIZE_MIN); + RUBY_VM_FIBER_VM_STACK_SIZE, + RUBY_VM_FIBER_VM_STACK_SIZE_MIN); vm->default_params.fiber_machine_stack_size = get_param("RUBY_FIBER_MACHINE_STACK_SIZE", - RUBY_VM_FIBER_MACHINE_STACK_SIZE, - RUBY_VM_FIBER_MACHINE_STACK_SIZE_MIN); + RUBY_VM_FIBER_MACHINE_STACK_SIZE, + RUBY_VM_FIBER_MACHINE_STACK_SIZE_MIN); /* environment dependent check */ check_machine_stack_size(&vm->default_params.thread_machine_stack_size); @@ -1727,499 +3609,518 @@ vm_default_params_setup(rb_vm_t *vm) static void vm_init2(rb_vm_t *vm) { - MEMZERO(vm, rb_vm_t, 1); + rb_vm_living_threads_init(vm); + vm->thread_report_on_exception = 1; vm->src_encoding_index = -1; - vm->at_exit.basic.flags = (T_ARRAY | RARRAY_EMBED_FLAG) & ~RARRAY_EMBED_LEN_MASK; /* len set 0 */ - vm->at_exit.basic.klass = 0; vm_default_params_setup(vm); } -/* Thread */ +void +rb_execution_context_update(rb_execution_context_t *ec) +{ + /* update VM stack */ + if (ec->vm_stack) { + long i; + VM_ASSERT(ec->cfp); + VALUE *p = ec->vm_stack; + VALUE *sp = ec->cfp->sp; + rb_control_frame_t *cfp = ec->cfp; + rb_control_frame_t *limit_cfp = (void *)(ec->vm_stack + ec->vm_stack_size); + + for (i = 0; i < (long)(sp - p); i++) { + VALUE ref = p[i]; + VALUE update = rb_gc_location(ref); + if (ref != update) { + p[i] = update; + } + } + + while (cfp != limit_cfp) { + const VALUE *ep = cfp->ep; + cfp->self = rb_gc_location(cfp->self); + cfp->iseq = (rb_iseq_t *)rb_gc_location((VALUE)cfp->iseq); + cfp->block_code = (void *)rb_gc_location((VALUE)cfp->block_code); + + if (!VM_ENV_LOCAL_P(ep)) { + const VALUE *prev_ep = VM_ENV_PREV_EP(ep); + if (VM_ENV_FLAGS(prev_ep, VM_ENV_FLAG_ESCAPED)) { + VM_FORCE_WRITE(&prev_ep[VM_ENV_DATA_INDEX_ENV], rb_gc_location(prev_ep[VM_ENV_DATA_INDEX_ENV])); + } + + if (VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED)) { + VM_FORCE_WRITE(&ep[VM_ENV_DATA_INDEX_ENV], rb_gc_location(ep[VM_ENV_DATA_INDEX_ENV])); + VM_FORCE_WRITE(&ep[VM_ENV_DATA_INDEX_ME_CREF], rb_gc_location(ep[VM_ENV_DATA_INDEX_ME_CREF])); + } + } + + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + } + } -#define USE_THREAD_DATA_RECYCLE 1 + ec->storage = rb_gc_location(ec->storage); -#if USE_THREAD_DATA_RECYCLE -#define RECYCLE_MAX 64 -static VALUE *thread_recycle_stack_slot[RECYCLE_MAX]; -static int thread_recycle_stack_count = 0; + ec->gen_fields_cache.obj = rb_gc_location(ec->gen_fields_cache.obj); + ec->gen_fields_cache.fields_obj = rb_gc_location(ec->gen_fields_cache.fields_obj); +} -static VALUE * -thread_recycle_stack(size_t size) +static enum rb_id_table_iterator_result +mark_local_storage_i(VALUE local, void *data) { - if (thread_recycle_stack_count) { - /* TODO: check stack size if stack sizes are variable */ - return thread_recycle_stack_slot[--thread_recycle_stack_count]; - } - else { - return ALLOC_N(VALUE, size); - } + rb_gc_mark(local); + return ID_TABLE_CONTINUE; } -#else -#define thread_recycle_stack(size) ALLOC_N(VALUE, (size)) -#endif - void -rb_thread_recycle_stack_release(VALUE *stack) -{ -#if USE_THREAD_DATA_RECYCLE - if (thread_recycle_stack_count < RECYCLE_MAX) { - thread_recycle_stack_slot[thread_recycle_stack_count++] = stack; - return; +rb_execution_context_mark(const rb_execution_context_t *ec) +{ + /* mark VM stack */ + if (ec->vm_stack) { + VM_ASSERT(ec->cfp); + VALUE *p = ec->vm_stack; + VALUE *sp = ec->cfp->sp; + rb_control_frame_t *cfp = ec->cfp; + rb_control_frame_t *limit_cfp = (void *)(ec->vm_stack + ec->vm_stack_size); + + for (long i = 0; i < (long)(sp - p); i++) { + rb_gc_mark_movable(p[i]); + } + + while (cfp != limit_cfp) { + const VALUE *ep = cfp->ep; + VM_ASSERT(!!VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED) == vm_ep_in_heap_p_(ec, ep)); + + rb_gc_mark_movable(cfp->self); + rb_gc_mark_movable((VALUE)cfp->iseq); + rb_gc_mark_movable((VALUE)cfp->block_code); + + if (VM_ENV_LOCAL_P(ep) && VM_ENV_BOXED_P(ep)) { + const rb_box_t *box = VM_ENV_BOX(ep); + if (BOX_USER_P(box)) { + rb_gc_mark_movable(box->box_object); + } + } + + if (!VM_ENV_LOCAL_P(ep)) { + const VALUE *prev_ep = VM_ENV_PREV_EP(ep); + if (VM_ENV_FLAGS(prev_ep, VM_ENV_FLAG_ESCAPED)) { + rb_gc_mark_movable(prev_ep[VM_ENV_DATA_INDEX_ENV]); + } + + if (VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED)) { + rb_gc_mark_movable(ep[VM_ENV_DATA_INDEX_ENV]); + rb_gc_mark(ep[VM_ENV_DATA_INDEX_ME_CREF]); + } + } + + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + } } -#endif - ruby_xfree(stack); + + /* mark machine stack */ + if (ec->machine.stack_start && ec->machine.stack_end && + ec != GET_EC() /* marked for current ec at the first stage of marking */ + ) { + rb_gc_mark_machine_context(ec); + } + + rb_gc_mark(ec->errinfo); + rb_gc_mark(ec->root_svar); + if (ec->local_storage) { + rb_id_table_foreach_values(ec->local_storage, mark_local_storage_i, NULL); + } + rb_gc_mark(ec->local_storage_recursive_hash); + rb_gc_mark(ec->local_storage_recursive_hash_for_trace); + rb_gc_mark(ec->private_const_reference); + + rb_gc_mark_movable(ec->storage); } -#ifdef USE_THREAD_RECYCLE -static rb_thread_t * -thread_recycle_struct(void) +void rb_fiber_mark_self(rb_fiber_t *fib); +void rb_fiber_update_self(rb_fiber_t *fib); +void rb_threadptr_root_fiber_setup(rb_thread_t *th); +void rb_root_fiber_obj_setup(rb_thread_t *th); +void rb_threadptr_root_fiber_release(rb_thread_t *th); + +static void +thread_compact(void *ptr) { - void *p = ALLOC_N(rb_thread_t, 1); - memset(p, 0, sizeof(rb_thread_t)); - return p; + rb_thread_t *th = ptr; + + th->self = rb_gc_location(th->self); } -#endif -void -rb_thread_mark(void *ptr) +static void +thread_mark(void *ptr) { - rb_thread_t *th = NULL; + rb_thread_t *th = ptr; RUBY_MARK_ENTER("thread"); - if (ptr) { - th = ptr; - if (th->stack) { - VALUE *p = th->stack; - VALUE *sp = th->cfp->sp; - rb_control_frame_t *cfp = th->cfp; - rb_control_frame_t *limit_cfp = (void *)(th->stack + th->stack_size); - - while (p < sp) { - rb_gc_mark(*p++); - } - rb_gc_mark_locations(p, p + th->mark_stack_len); - - while (cfp != limit_cfp) { - rb_iseq_t *iseq = cfp->iseq; - rb_gc_mark(cfp->proc); - rb_gc_mark(cfp->self); - rb_gc_mark(cfp->klass); - if (iseq) { - rb_gc_mark(RUBY_VM_NORMAL_ISEQ_P(iseq) ? iseq->self : (VALUE)iseq); - } - if (cfp->me) { - /* TODO: marking `me' can be more sophisticated way */ - ((rb_method_entry_t *)cfp->me)->mark = 1; - rb_mark_method_entry(cfp->me); - } - cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); - } - } - - /* mark ruby objects */ - RUBY_MARK_UNLESS_NULL(th->first_proc); - if (th->first_proc) RUBY_MARK_UNLESS_NULL(th->first_args); - - RUBY_MARK_UNLESS_NULL(th->thgroup); - RUBY_MARK_UNLESS_NULL(th->value); - RUBY_MARK_UNLESS_NULL(th->errinfo); - RUBY_MARK_UNLESS_NULL(th->pending_interrupt_queue); - RUBY_MARK_UNLESS_NULL(th->pending_interrupt_mask_stack); - RUBY_MARK_UNLESS_NULL(th->root_svar); - RUBY_MARK_UNLESS_NULL(th->top_self); - RUBY_MARK_UNLESS_NULL(th->top_wrapper); - RUBY_MARK_UNLESS_NULL(th->fiber); - RUBY_MARK_UNLESS_NULL(th->root_fiber); - RUBY_MARK_UNLESS_NULL(th->stat_insn_usage); - RUBY_MARK_UNLESS_NULL(th->last_status); - - RUBY_MARK_UNLESS_NULL(th->locking_mutex); - - rb_mark_tbl(th->local_storage); - - if (GET_THREAD() != th && th->machine_stack_start && th->machine_stack_end) { - rb_gc_mark_machine_stack(th); - rb_gc_mark_locations((VALUE *)&th->machine_regs, - (VALUE *)(&th->machine_regs) + - sizeof(th->machine_regs) / sizeof(VALUE)); - } - - rb_vm_trace_mark_event_hooks(&th->event_hooks); + + // ec is null when setting up the thread in rb_threadptr_root_fiber_setup + if (th->ec) { + rb_fiber_mark_self(th->ec->fiber_ptr); } + /* mark ruby objects */ + switch (th->invoke_type) { + case thread_invoke_type_proc: + case thread_invoke_type_ractor_proc: + rb_gc_mark(th->invoke_arg.proc.proc); + rb_gc_mark(th->invoke_arg.proc.args); + break; + case thread_invoke_type_func: + rb_gc_mark_maybe((VALUE)th->invoke_arg.func.arg); + break; + default: + break; + } + + rb_gc_mark(rb_ractor_self(th->ractor)); + rb_gc_mark(th->thgroup); + rb_gc_mark(th->value); + rb_gc_mark(th->pending_interrupt_queue); + rb_gc_mark(th->pending_interrupt_mask_stack); + rb_gc_mark(th->top_self); + rb_gc_mark(th->top_wrapper); + if (th->root_fiber) rb_fiber_mark_self(th->root_fiber); + + RUBY_ASSERT(th->ec == rb_fiberptr_get_ec(th->ec->fiber_ptr)); + rb_gc_mark(th->last_status); + rb_gc_mark(th->locking_mutex); + rb_gc_mark(th->name); + + rb_gc_mark(th->scheduler); + + rb_threadptr_interrupt_exec_task_mark(th); + RUBY_MARK_LEAVE("thread"); } +void rb_threadptr_sched_free(rb_thread_t *th); // thread_*.c + static void thread_free(void *ptr) { - rb_thread_t *th; + rb_thread_t *th = ptr; RUBY_FREE_ENTER("thread"); - if (ptr) { - th = ptr; - - if (!th->root_fiber) { - RUBY_FREE_UNLESS_NULL(th->stack); - } - - if (th->locking_mutex != Qfalse) { - rb_bug("thread_free: locking_mutex must be NULL (%p:%p)", (void *)th, (void *)th->locking_mutex); - } - if (th->keeping_mutexes != NULL) { - rb_bug("thread_free: keeping_mutexes must be NULL (%p:%p)", (void *)th, (void *)th->keeping_mutexes); - } - - if (th->local_storage) { - st_free_table(th->local_storage); - } - - if (th->vm && th->vm->main_thread == th) { - RUBY_GC_INFO("main thread\n"); - } - else { -#ifdef USE_SIGALTSTACK - if (th->altstack) { - free(th->altstack); - } -#endif - ruby_xfree(ptr); - } - if (ruby_current_thread == th) - ruby_current_thread = NULL; + rb_threadptr_sched_free(th); + + if (th->locking_mutex != Qfalse) { + rb_bug("thread_free: locking_mutex must be NULL (%p:%p)", (void *)th, (void *)th->locking_mutex); + } + if (th->keeping_mutexes != NULL) { + rb_bug("thread_free: keeping_mutexes must be NULL (%p:%p)", (void *)th, (void *)th->keeping_mutexes); + } + + ruby_xfree(th->specific_storage); + + if (th->vm && th->vm->ractor.main_thread == th) { + RUBY_GC_INFO("MRI main thread\n"); + } + else { + // ruby_xfree(th->nt); + // TODO: MN system collect nt, but without MN system it should be freed here. + ruby_xfree(th); } + RUBY_FREE_LEAVE("thread"); } static size_t thread_memsize(const void *ptr) { - if (ptr) { - const rb_thread_t *th = ptr; - size_t size = sizeof(rb_thread_t); + const rb_thread_t *th = ptr; + size_t size = sizeof(rb_thread_t); - if (!th->root_fiber) { - size += th->stack_size * sizeof(VALUE); - } - if (th->local_storage) { - size += st_memsize(th->local_storage); - } - return size; + if (!th->root_fiber) { + size += th->ec->vm_stack_size * sizeof(VALUE); } - else { - return 0; + if (th->ec->local_storage) { + size += rb_id_table_memsize(th->ec->local_storage); } + return size; } #define thread_data_type ruby_threadptr_data_type const rb_data_type_t ruby_threadptr_data_type = { "VM/thread", { - rb_thread_mark, - thread_free, - thread_memsize, + thread_mark, + thread_free, + thread_memsize, + thread_compact, }, + 0, 0, RUBY_TYPED_FREE_IMMEDIATELY }; VALUE rb_obj_is_thread(VALUE obj) { - if (rb_typeddata_is_kind_of(obj, &thread_data_type)) { - return Qtrue; - } - else { - return Qfalse; - } + return RBOOL(rb_typeddata_is_kind_of(obj, &thread_data_type)); } static VALUE thread_alloc(VALUE klass) { - VALUE volatile obj; -#ifdef USE_THREAD_RECYCLE - rb_thread_t *th = thread_recycle_struct(); - obj = TypedData_Wrap_Struct(klass, &thread_data_type, th); -#else rb_thread_t *th; - obj = TypedData_Make_Struct(klass, rb_thread_t, &thread_data_type, th); + return TypedData_Make_Struct(klass, rb_thread_t, &thread_data_type, th); +} + +void +rb_ec_set_vm_stack(rb_execution_context_t *ec, VALUE *stack, size_t size) +{ + ec->vm_stack = stack; + ec->vm_stack_size = size; +} + +void +rb_ec_initialize_vm_stack(rb_execution_context_t *ec, VALUE *stack, size_t size) +{ + rb_ec_set_vm_stack(ec, stack, size); + +#if VM_CHECK_MODE > 0 + MEMZERO(stack, VALUE, size); // malloc memory could have the VM canary in it #endif - return obj; + + ec->cfp = (void *)(ec->vm_stack + ec->vm_stack_size); + + vm_push_frame(ec, + NULL /* dummy iseq */, + VM_FRAME_MAGIC_DUMMY | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH | VM_FRAME_FLAG_CFRAME /* dummy frame */, + Qnil /* dummy self */, VM_BLOCK_HANDLER_NONE /* dummy block ptr */, + 0 /* dummy cref/me */, + 0 /* dummy pc */, ec->vm_stack, 0, 0 + ); +} + +void +rb_ec_clear_vm_stack(rb_execution_context_t *ec) +{ + // set cfp to NULL before clearing the stack in case `thread_profile_frames` + // gets called in this middle of `rb_ec_set_vm_stack` via signal handler. + ec->cfp = NULL; + rb_ec_set_vm_stack(ec, NULL, 0); +} + +void +rb_ec_close(rb_execution_context_t *ec) +{ + // Fiber storage is not accessible from outside the running fiber, so it is safe to clear it here. + ec->storage = Qnil; } static void -th_init(rb_thread_t *th, VALUE self) +th_init(rb_thread_t *th, VALUE self, rb_vm_t *vm) { + const rb_box_t *box = rb_current_box(); + th->self = self; - /* allocate thread stack */ -#ifdef USE_SIGALTSTACK - /* altstack of main thread is reallocated in another place */ - th->altstack = malloc(rb_sigaltstack_size()); -#endif - /* th->stack_size is word number. - * th->vm->default_params.thread_vm_stack_size is byte size. - */ - th->stack_size = th->vm->default_params.thread_vm_stack_size / sizeof(VALUE); - th->stack = thread_recycle_stack(th->stack_size); + ccan_list_head_init(&th->interrupt_exec_tasks); - th->cfp = (void *)(th->stack + th->stack_size); + rb_threadptr_root_fiber_setup(th); - vm_push_frame(th, 0 /* dummy iseq */, VM_FRAME_MAGIC_TOP | VM_FRAME_FLAG_FINISH, - Qnil /* dummy self */, Qnil /* dummy klass */, VM_ENVVAL_BLOCK_PTR(0), 0 /* dummy pc */, th->stack, 1, 0); + /* All threads are blocking until a non-blocking fiber is scheduled */ + th->blocking = 1; + th->scheduler = Qnil; + + if (self == 0) { + size_t size = vm->default_params.thread_vm_stack_size / sizeof(VALUE); + VALUE *stack = ALLOC_N(VALUE, size); + rb_ec_initialize_vm_stack(th->ec, stack, size); + rb_thread_malloc_stack_set(th, stack); + } + else { + VM_ASSERT(th->ec->cfp == NULL); + VM_ASSERT(th->ec->vm_stack == NULL); + VM_ASSERT(th->ec->vm_stack_size == 0); + } th->status = THREAD_RUNNABLE; - th->errinfo = Qnil; th->last_status = Qnil; - th->waiting_fd = -1; - th->root_svar = Qnil; + th->top_wrapper = 0; + if (box->top_self) { + th->top_self = box->top_self; + } + else { + th->top_self = 0; + } + th->value = Qundef; + + th->ec->errinfo = Qnil; + th->ec->root_svar = Qfalse; + th->ec->local_storage_recursive_hash = Qnil; + th->ec->local_storage_recursive_hash_for_trace = Qnil; + + th->ec->storage = Qnil; + th->ec->ractor_id = rb_ractor_id(th->ractor); #if OPT_CALL_THREADED_CODE th->retval = Qundef; #endif -} + th->name = Qnil; + th->report_on_exception = vm->thread_report_on_exception; + th->ext_config.ractor_safe = true; -static VALUE -ruby_thread_init(VALUE self) -{ - rb_thread_t *th; - rb_vm_t *vm = GET_THREAD()->vm; - GetThreadPtr(self, th); - - th->vm = vm; - th_init(th, self); - rb_ivar_set(self, rb_intern("locals"), rb_hash_new()); +#if USE_RUBY_DEBUG_LOG + static rb_atomic_t thread_serial = 1; + th->serial = RUBY_ATOMIC_FETCH_ADD(thread_serial, 1); - th->top_wrapper = 0; - th->top_self = rb_vm_top_self(); - th->root_svar = Qnil; - return self; + RUBY_DEBUG_LOG("th:%u", th->serial); +#endif } VALUE rb_thread_alloc(VALUE klass) { VALUE self = thread_alloc(klass); - ruby_thread_init(self); + rb_thread_t *target_th = rb_thread_ptr(self); + target_th->ractor = GET_RACTOR(); + th_init(target_th, self, target_th->vm = GET_VM()); + rb_root_fiber_obj_setup(target_th); return self; } -static void -vm_define_method(rb_thread_t *th, VALUE obj, ID id, VALUE iseqval, - rb_num_t is_singleton, NODE *cref) -{ - VALUE klass = cref->nd_clss; - int noex = (int)cref->nd_visi; - rb_iseq_t *miseq; - GetISeqPtr(iseqval, miseq); - - if (miseq->klass) { - RB_GC_GUARD(iseqval) = rb_iseq_clone(iseqval, 0); - GetISeqPtr(iseqval, miseq); - } - - if (NIL_P(klass)) { - rb_raise(rb_eTypeError, "no class/module to add method"); - } - - if (is_singleton) { - klass = rb_singleton_class(obj); /* class and frozen checked in this API */ - noex = NOEX_PUBLIC; - } - - /* dup */ - COPY_CREF(miseq->cref_stack, cref); - miseq->cref_stack->nd_visi = NOEX_PUBLIC; - miseq->klass = klass; - miseq->defined_method_id = id; - rb_add_method(klass, id, VM_METHOD_TYPE_ISEQ, miseq, noex); - - if (!is_singleton && noex == NOEX_MODFUNC) { - rb_add_method(rb_singleton_class(klass), id, VM_METHOD_TYPE_ISEQ, miseq, NOEX_PUBLIC); - } - INC_VM_STATE_VERSION(); -} - #define REWIND_CFP(expr) do { \ - rb_thread_t *th__ = GET_THREAD(); \ - th__->cfp++; expr; th__->cfp--; \ + rb_execution_context_t *ec__ = GET_EC(); \ + VALUE *const curr_sp = (ec__->cfp++)->sp; \ + VALUE *const saved_sp = ec__->cfp->sp; \ + ec__->cfp->sp = curr_sp; \ + expr; \ + (ec__->cfp--)->sp = saved_sp; \ } while (0) static VALUE -m_core_define_method(VALUE self, VALUE cbase, VALUE sym, VALUE iseqval) +m_core_set_method_alias(VALUE self, VALUE cbase, VALUE sym1, VALUE sym2) { REWIND_CFP({ - vm_define_method(GET_THREAD(), cbase, SYM2ID(sym), iseqval, 0, rb_vm_cref()); + rb_alias(cbase, SYM2ID(sym1), SYM2ID(sym2)); }); return Qnil; } static VALUE -m_core_define_singleton_method(VALUE self, VALUE cbase, VALUE sym, VALUE iseqval) +m_core_set_variable_alias(VALUE self, VALUE sym1, VALUE sym2) { REWIND_CFP({ - vm_define_method(GET_THREAD(), cbase, SYM2ID(sym), iseqval, 1, rb_vm_cref()); + rb_alias_variable(SYM2ID(sym1), SYM2ID(sym2)); }); return Qnil; } static VALUE -m_core_set_method_alias(VALUE self, VALUE cbase, VALUE sym1, VALUE sym2) +m_core_undef_method(VALUE self, VALUE cbase, VALUE sym) { REWIND_CFP({ - rb_alias(cbase, SYM2ID(sym1), SYM2ID(sym2)); + ID mid = SYM2ID(sym); + rb_undef(cbase, mid); + rb_clear_method_cache(self, mid); }); return Qnil; } static VALUE -m_core_set_variable_alias(VALUE self, VALUE sym1, VALUE sym2) +m_core_set_postexe(VALUE self) { - REWIND_CFP({ - rb_alias_variable(SYM2ID(sym1), SYM2ID(sym2)); - }); + rb_set_end_proc(rb_call_end_proc, rb_block_proc()); return Qnil; } +static VALUE core_hash_merge_kwd(VALUE hash, VALUE kw); + static VALUE -m_core_undef_method(VALUE self, VALUE cbase, VALUE sym) +core_hash_merge(VALUE hash, long argc, const VALUE *argv) { - REWIND_CFP({ - rb_undef(cbase, SYM2ID(sym)); - INC_VM_STATE_VERSION(); - }); - return Qnil; + Check_Type(hash, T_HASH); + VM_ASSERT(argc % 2 == 0); + rb_hash_bulk_insert(argc, argv, hash); + return hash; } static VALUE -m_core_set_postexe(VALUE self, VALUE iseqval) +m_core_hash_merge_ptr(int argc, VALUE *argv, VALUE recv) { - REWIND_CFP({ - rb_iseq_t *blockiseq; - rb_block_t *blockptr; - rb_thread_t *th = GET_THREAD(); - rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(th, th->cfp); - VALUE proc; - - if (cfp == 0) { - rb_bug("m_core_set_postexe: unreachable"); - } - - GetISeqPtr(iseqval, blockiseq); + VALUE hash = argv[0]; - blockptr = RUBY_VM_GET_BLOCK_PTR_IN_CFP(cfp); - blockptr->iseq = blockiseq; - blockptr->proc = 0; + REWIND_CFP(hash = core_hash_merge(hash, argc-1, argv+1)); - proc = rb_vm_make_proc(th, blockptr, rb_cProc); - rb_set_end_proc(rb_call_end_proc, proc); - }); - return Qnil; + return hash; } -static VALUE -m_core_hash_from_ary(VALUE self, VALUE ary) +static int +kwmerge_i(VALUE key, VALUE value, VALUE hash) { - VALUE hash = rb_hash_new(); - int i; - - if (RUBY_DTRACE_HASH_CREATE_ENABLED()) { - RUBY_DTRACE_HASH_CREATE(RARRAY_LEN(ary), rb_sourcefile(), rb_sourceline()); - } - - assert(RARRAY_LEN(ary) % 2 == 0); - for (i=0; i<RARRAY_LEN(ary); i+=2) { - rb_hash_aset(hash, RARRAY_PTR(ary)[i], RARRAY_PTR(ary)[i+1]); - } - - return hash; + rb_hash_aset(hash, key, value); + return ST_CONTINUE; } static VALUE -m_core_hash_merge_ary(VALUE self, VALUE hash, VALUE ary) +m_core_hash_merge_kwd(VALUE recv, VALUE hash, VALUE kw) { - int i; - - assert(RARRAY_LEN(ary) % 2 == 0); - for (i=0; i<RARRAY_LEN(ary); i+=2) { - rb_hash_aset(hash, RARRAY_PTR(ary)[i], RARRAY_PTR(ary)[i+1]); + if (!NIL_P(kw)) { + REWIND_CFP(hash = core_hash_merge_kwd(hash, kw)); } - return hash; } static VALUE -m_core_hash_merge_ptr(int argc, VALUE *argv, VALUE recv) +m_core_make_shareable(VALUE recv, VALUE obj) { - int i; - VALUE hash = argv[0]; - - for (i=1; i<argc; i+=2) { - rb_hash_aset(hash, argv[i], argv[i+1]); - } - - return hash; + return rb_ractor_make_shareable(obj); } -static int -kwmerge_ii(st_data_t *key, st_data_t *value, st_data_t arg, int existing) +static VALUE +m_core_make_shareable_copy(VALUE recv, VALUE obj) { - if (existing) return ST_STOP; - *value = arg; - return ST_CONTINUE; + return rb_ractor_make_shareable_copy(obj); } -static int -kwmerge_i(VALUE key, VALUE value, VALUE hash) +static VALUE +m_core_ensure_shareable(VALUE recv, VALUE obj, VALUE name) { - if (!SYMBOL_P(key)) Check_Type(key, T_SYMBOL); - st_update(RHASH_TBL(hash), key, kwmerge_ii, (st_data_t)value); - return ST_CONTINUE; + return rb_ractor_ensure_shareable(obj, name); } static VALUE -m_core_hash_merge_kwd(VALUE recv, VALUE hash, VALUE kw) +core_hash_merge_kwd(VALUE hash, VALUE kw) { - kw = rb_convert_type(kw, T_HASH, "Hash", "to_hash"); - rb_hash_foreach(kw, kwmerge_i, hash); + rb_hash_foreach(rb_to_hash_type(kw), kwmerge_i, hash); return hash; } extern VALUE *rb_gc_stack_start; extern size_t rb_gc_stack_maxsize; -#ifdef __ia64 -extern VALUE *rb_gc_register_stack_start; -#endif /* debug functions */ /* :nodoc: */ static VALUE -sdr(void) +sdr(VALUE self) { - rb_vm_bugreport(); + rb_vm_bugreport(NULL, stderr); return Qnil; } /* :nodoc: */ static VALUE -nsdr(void) +nsdr(VALUE self) { VALUE ary = rb_ary_new(); -#if HAVE_BACKTRACE +#ifdef HAVE_BACKTRACE #include <execinfo.h> #define MAX_NATIVE_TRACE 1024 static void *trace[MAX_NATIVE_TRACE]; - int n = backtrace(trace, MAX_NATIVE_TRACE); + int n = (int)backtrace(trace, MAX_NATIVE_TRACE); char **syms = backtrace_symbols(trace, n); int i; if (syms == 0) { - rb_memerror(); + rb_memerror(); } for (i=0; i<n; i++) { - rb_ary_push(ary, rb_str_new2(syms[i])); + rb_ary_push(ary, rb_str_new2(syms[i])); } free(syms); /* OK */ #endif @@ -2227,11 +4128,95 @@ nsdr(void) } #if VM_COLLECT_USAGE_DETAILS +static VALUE usage_analysis_insn_start(VALUE self); +static VALUE usage_analysis_operand_start(VALUE self); +static VALUE usage_analysis_register_start(VALUE self); static VALUE usage_analysis_insn_stop(VALUE self); static VALUE usage_analysis_operand_stop(VALUE self); static VALUE usage_analysis_register_stop(VALUE self); +static VALUE usage_analysis_insn_running(VALUE self); +static VALUE usage_analysis_operand_running(VALUE self); +static VALUE usage_analysis_register_running(VALUE self); +static VALUE usage_analysis_insn_clear(VALUE self); +static VALUE usage_analysis_operand_clear(VALUE self); +static VALUE usage_analysis_register_clear(VALUE self); #endif +static VALUE +f_raise(int c, VALUE *v, VALUE _) +{ + return rb_f_raise(c, v); +} + +static VALUE +f_proc(VALUE _) +{ + return rb_block_proc(); +} + +static VALUE +f_lambda(VALUE _) +{ + return rb_block_lambda(); +} + +static VALUE +f_sprintf(int c, const VALUE *v, VALUE _) +{ + return rb_f_sprintf(c, v); +} + +/* :nodoc: */ +static VALUE +vm_mtbl(VALUE self, VALUE obj, VALUE sym) +{ + vm_mtbl_dump(CLASS_OF(obj), RTEST(sym) ? SYM2ID(sym) : 0); + return Qnil; +} + +/* :nodoc: */ +static VALUE +vm_mtbl2(VALUE self, VALUE obj, VALUE sym) +{ + vm_mtbl_dump(obj, RTEST(sym) ? SYM2ID(sym) : 0); + return Qnil; +} + +/* + * call-seq: + * RubyVM.keep_script_lines -> true or false + * + * Return current +keep_script_lines+ status. Now it only returns + * +true+ of +false+, but it can return other objects in future. + * + * Note that this is an API for ruby internal use, debugging, + * and research. Do not use this for any other purpose. + * The compatibility is not guaranteed. + */ +static VALUE +vm_keep_script_lines(VALUE self) +{ + return RBOOL(ruby_vm_keep_script_lines); +} + +/* + * call-seq: + * RubyVM.keep_script_lines = true / false + * + * It set +keep_script_lines+ flag. If the flag is set, all + * loaded scripts are recorded in a interpreter process. + * + * Note that this is an API for ruby internal use, debugging, + * and research. Do not use this for any other purpose. + * The compatibility is not guaranteed. + */ +static VALUE +vm_keep_script_lines_set(VALUE self, VALUE flags) +{ + ruby_vm_keep_script_lines = RTEST(flags); + return flags; +} + void Init_VM(void) { @@ -2239,36 +4224,53 @@ Init_VM(void) VALUE klass; VALUE fcore; - /* ::RubyVM */ + /* + * Document-class: RubyVM + * + * The RubyVM module only exists on MRI. +RubyVM+ is not defined in + * other Ruby implementations such as JRuby and TruffleRuby. + * + * The RubyVM module provides some access to MRI internals. + * This module is for very limited purposes, such as debugging, + * prototyping, and research. Normal users must not use it. + * This module is not portable between Ruby implementations. + */ rb_cRubyVM = rb_define_class("RubyVM", rb_cObject); rb_undef_alloc_func(rb_cRubyVM); rb_undef_method(CLASS_OF(rb_cRubyVM), "new"); + rb_define_singleton_method(rb_cRubyVM, "stat", vm_stat, -1); + rb_define_singleton_method(rb_cRubyVM, "keep_script_lines", vm_keep_script_lines, 0); + rb_define_singleton_method(rb_cRubyVM, "keep_script_lines=", vm_keep_script_lines_set, 1); + +#if USE_DEBUG_COUNTER + rb_define_singleton_method(rb_cRubyVM, "reset_debug_counters", rb_debug_counter_reset, 0); + rb_define_singleton_method(rb_cRubyVM, "show_debug_counters", rb_debug_counter_show, 0); +#endif /* FrozenCore (hidden) */ fcore = rb_class_new(rb_cBasicObject); - RBASIC(fcore)->flags = T_ICLASS; + rb_set_class_path(fcore, rb_cRubyVM, "FrozenCore"); + rb_vm_register_global_object(rb_class_path_cached(fcore)); klass = rb_singleton_class(fcore); rb_define_method_id(klass, id_core_set_method_alias, m_core_set_method_alias, 3); rb_define_method_id(klass, id_core_set_variable_alias, m_core_set_variable_alias, 2); rb_define_method_id(klass, id_core_undef_method, m_core_undef_method, 2); - rb_define_method_id(klass, id_core_define_method, m_core_define_method, 3); - rb_define_method_id(klass, id_core_define_singleton_method, m_core_define_singleton_method, 3); - rb_define_method_id(klass, id_core_set_postexe, m_core_set_postexe, 1); - rb_define_method_id(klass, id_core_hash_from_ary, m_core_hash_from_ary, 1); - rb_define_method_id(klass, id_core_hash_merge_ary, m_core_hash_merge_ary, 2); + rb_define_method_id(klass, id_core_set_postexe, m_core_set_postexe, 0); rb_define_method_id(klass, id_core_hash_merge_ptr, m_core_hash_merge_ptr, -1); rb_define_method_id(klass, id_core_hash_merge_kwd, m_core_hash_merge_kwd, 2); - rb_define_method_id(klass, idProc, rb_block_proc, 0); - rb_define_method_id(klass, idLambda, rb_block_lambda, 0); + rb_define_method_id(klass, id_core_raise, f_raise, -1); + rb_define_method_id(klass, id_core_sprintf, f_sprintf, -1); + rb_define_method_id(klass, idProc, f_proc, 0); + rb_define_method_id(klass, idLambda, f_lambda, 0); + rb_define_method(klass, "make_shareable", m_core_make_shareable, 1); + rb_define_method(klass, "make_shareable_copy", m_core_make_shareable_copy, 1); + rb_define_method(klass, "ensure_shareable", m_core_ensure_shareable, 2); rb_obj_freeze(fcore); - rb_gc_register_mark_object(fcore); + RBASIC_CLEAR_CLASS(klass); + rb_obj_freeze(klass); + rb_vm_register_global_object(fcore); rb_mRubyVMFrozenCore = fcore; - /* ::RubyVM::Env */ - rb_cEnv = rb_define_class_under(rb_cRubyVM, "Env", rb_cObject); - rb_undef_alloc_func(rb_cEnv); - rb_undef_method(CLASS_OF(rb_cEnv), "new"); - /* * Document-class: Thread * @@ -2280,12 +4282,12 @@ Init_VM(void) * For example, we can create a new thread separate from the main thread's * execution using ::new. * - * thr = Thread.new { puts "Whats the big deal" } + * thr = Thread.new { puts "What's the big deal" } * * Then we are able to pause the execution of the main thread and allow * our new thread to finish, using #join: * - * thr.join #=> "Whats the big deal" + * thr.join #=> "What's the big deal" * * If we don't call +thr.join+ before the main thread terminates, then all * other threads including +thr+ will be killed. @@ -2294,7 +4296,7 @@ Init_VM(void) * once, like in the following example: * * threads = [] - * threads << Thread.new { puts "Whats the big deal" } + * threads << Thread.new { puts "What's the big deal" } * threads << Thread.new { 3.times { puts "Threads are fun!" } } * * After creating a few threads we wait for them all to finish @@ -2302,6 +4304,11 @@ Init_VM(void) * * threads.each { |thr| thr.join } * + * To retrieve the last value of a thread, use #value + * + * thr = Thread.new { sleep 1; "Useful value" } + * thr.value #=> "Useful value" + * * === Thread initialization * * In order to create new threads, Ruby provides ::new, ::start, and @@ -2318,7 +4325,7 @@ Init_VM(void) * * The class method ::kill, is meant to exit a given thread: * - * thr = Thread.new { ... } + * thr = Thread.new { sleep } * Thread.kill(thr) # sends exit() to thr * * Alternatively, you can use the instance method #exit, or any of its @@ -2385,15 +4392,21 @@ Init_VM(void) * * === Exception handling * - * Any thread can raise an exception using the #raise instance method, - * which operates similarly to Kernel#raise. + * When an unhandled exception is raised inside a thread, it will + * terminate. By default, this exception will not propagate to other + * threads. The exception is stored and when another thread calls #value + * or #join, the exception will be re-raised in that thread. + * + * t = Thread.new{ raise 'something went wrong' } + * t.value #=> RuntimeError: something went wrong * - * However, it's important to note that an exception that occurs in any - * thread except the main thread depends on #abort_on_exception. This - * option is +false+ by default, meaning that any unhandled exception will - * cause the thread to terminate silently when waited on by either #join - * or #value. You can change this default by either #abort_on_exception= - * +true+ or setting $DEBUG to +true+. + * An exception can be raised from outside the thread using the + * Thread#raise instance method, which takes the same parameters as + * Kernel#raise. + * + * Setting Thread.abort_on_exception = true, Thread#abort_on_exception = + * true, or $DEBUG = true will cause a subsequent unhandled exception + * raised in a thread to be automatically re-raised in the main thread. * * With the addition of the class method ::handle_interrupt, you can now * handle exceptions asynchronously with threads. @@ -2421,16 +4434,30 @@ Init_VM(void) #if VM_COLLECT_USAGE_DETAILS /* ::RubyVM::USAGE_ANALYSIS_* */ - rb_define_const(rb_cRubyVM, "USAGE_ANALYSIS_INSN", rb_hash_new()); - rb_define_const(rb_cRubyVM, "USAGE_ANALYSIS_REGS", rb_hash_new()); - rb_define_const(rb_cRubyVM, "USAGE_ANALYSIS_INSN_BIGRAM", rb_hash_new()); - +#define define_usage_analysis_hash(name) /* shut up rdoc -C */ \ + rb_define_const(rb_cRubyVM, "USAGE_ANALYSIS_" #name, rb_hash_new()) + define_usage_analysis_hash(INSN); + define_usage_analysis_hash(REGS); + define_usage_analysis_hash(INSN_BIGRAM); + + rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_INSN_START", usage_analysis_insn_start, 0); + rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_OPERAND_START", usage_analysis_operand_start, 0); + rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_REGISTER_START", usage_analysis_register_start, 0); rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_INSN_STOP", usage_analysis_insn_stop, 0); rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_OPERAND_STOP", usage_analysis_operand_stop, 0); rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_REGISTER_STOP", usage_analysis_register_stop, 0); + rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_INSN_RUNNING", usage_analysis_insn_running, 0); + rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_OPERAND_RUNNING", usage_analysis_operand_running, 0); + rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_REGISTER_RUNNING", usage_analysis_register_running, 0); + rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_INSN_CLEAR", usage_analysis_insn_clear, 0); + rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_OPERAND_CLEAR", usage_analysis_operand_clear, 0); + rb_define_singleton_method(rb_cRubyVM, "USAGE_ANALYSIS_REGISTER_CLEAR", usage_analysis_register_clear, 0); #endif - /* ::RubyVM::OPTS, which shows vm build options */ + /* ::RubyVM::OPTS + * An Array of VM build options. + * This constant is MRI specific. + */ rb_define_const(rb_cRubyVM, "OPTS", opts = rb_ary_new()); #if OPT_DIRECT_THREADED_CODE @@ -2441,11 +4468,8 @@ Init_VM(void) rb_ary_push(opts, rb_str_new2("call threaded code")); #endif -#if OPT_STACK_CACHING - rb_ary_push(opts, rb_str_new2("stack caching")); -#endif #if OPT_OPERANDS_UNIFICATION - rb_ary_push(opts, rb_str_new2("operands unification]")); + rb_ary_push(opts, rb_str_new2("operands unification")); #endif #if OPT_INSTRUCTIONS_UNIFICATION rb_ary_push(opts, rb_str_new2("instructions unification")); @@ -2453,16 +4477,16 @@ Init_VM(void) #if OPT_INLINE_METHOD_CACHE rb_ary_push(opts, rb_str_new2("inline method cache")); #endif -#if OPT_BLOCKINLINING - rb_ary_push(opts, rb_str_new2("block inlining")); -#endif - /* ::RubyVM::INSTRUCTION_NAMES */ + /* ::RubyVM::INSTRUCTION_NAMES + * A list of bytecode instruction names in MRI. + * This constant is MRI specific. + */ rb_define_const(rb_cRubyVM, "INSTRUCTION_NAMES", rb_insns_name_array()); /* ::RubyVM::DEFAULT_PARAMS - * This constant variable shows VM's default parameters. - * Note that changing these values does not affect VM exection. + * This constant exposes the VM's default parameters. + * Note that changing these values does not affect VM execution. * Specification is not stable and you should not depend on this value. * Of course, this constant is MRI specific. */ @@ -2472,152 +4496,328 @@ Init_VM(void) #if VMDEBUG rb_define_singleton_method(rb_cRubyVM, "SDR", sdr, 0); rb_define_singleton_method(rb_cRubyVM, "NSDR", nsdr, 0); + rb_define_singleton_method(rb_cRubyVM, "mtbl", vm_mtbl, 2); + rb_define_singleton_method(rb_cRubyVM, "mtbl2", vm_mtbl2, 2); #else (void)sdr; (void)nsdr; + (void)vm_mtbl; + (void)vm_mtbl2; #endif /* VM bootstrap: phase 2 */ { - rb_vm_t *vm = ruby_current_vm; - rb_thread_t *th = GET_THREAD(); - VALUE filename = rb_str_new2("<main>"); - volatile VALUE iseqval = rb_iseq_new(0, filename, filename, Qnil, 0, ISEQ_TYPE_TOP); - volatile VALUE th_self; - rb_iseq_t *iseq; - - /* create vm object */ - vm->self = TypedData_Wrap_Struct(rb_cRubyVM, &vm_data_type, vm); - - /* create main thread */ - th_self = th->self = TypedData_Wrap_Struct(rb_cThread, &thread_data_type, th); - rb_iv_set(th_self, "locals", rb_hash_new()); - vm->main_thread = th; - vm->running_thread = th; - th->vm = vm; - th->top_wrapper = 0; - th->top_self = rb_vm_top_self(); - rb_thread_set_current(th); - - vm->living_threads = st_init_numtable(); - st_insert(vm->living_threads, th_self, (st_data_t) th->thread_id); - - rb_gc_register_mark_object(iseqval); - GetISeqPtr(iseqval, iseq); - th->cfp->iseq = iseq; - th->cfp->pc = iseq->iseq_encoded; - th->cfp->self = th->top_self; - th->cfp->klass = Qnil; - - /* - * The Binding of the top level scope - */ - rb_define_global_const("TOPLEVEL_BINDING", rb_binding_new()); + rb_vm_t *vm = ruby_current_vm_ptr; + rb_thread_t *th = GET_THREAD(); + VALUE filename = rb_fstring_lit("<main>"); + const rb_iseq_t *iseq = rb_iseq_new(Qnil, filename, filename, Qnil, 0, ISEQ_TYPE_TOP); + + // Ractor setup + rb_ractor_main_setup(vm, th->ractor, th); + + /* create vm object */ + vm->self = TypedData_Wrap_Struct(rb_cRubyVM, &vm_data_type, vm); + + /* create main thread */ + th->self = TypedData_Wrap_Struct(rb_cThread, &thread_data_type, th); + vm->ractor.main_thread = th; + vm->ractor.main_ractor = th->ractor; + th->vm = vm; + th->top_wrapper = 0; + th->top_self = rb_vm_top_self(); + + rb_root_fiber_obj_setup(th); + + rb_vm_register_global_object((VALUE)iseq); + th->ec->cfp->iseq = iseq; + th->ec->cfp->pc = ISEQ_BODY(iseq)->iseq_encoded; + th->ec->cfp->self = th->top_self; + + VM_ENV_FLAGS_UNSET(th->ec->cfp->ep, VM_FRAME_FLAG_CFRAME); + VM_STACK_ENV_WRITE(th->ec->cfp->ep, VM_ENV_DATA_INDEX_ME_CREF, (VALUE)vm_cref_new(rb_cObject, METHOD_VISI_PRIVATE, FALSE, NULL, FALSE, FALSE)); + + /* + * The Binding of the top level scope + */ + rb_define_global_const("TOPLEVEL_BINDING", rb_binding_new()); + +#ifdef _WIN32 + rb_objspace_gc_enable(vm->gc.objspace); +#endif } vm_init_redefined_flag(); + rb_block_param_proxy = rb_obj_alloc(rb_cObject); + rb_add_method_optimized(rb_singleton_class(rb_block_param_proxy), idCall, + OPTIMIZED_METHOD_TYPE_BLOCK_CALL, 0, METHOD_VISI_PUBLIC); + rb_obj_freeze(rb_block_param_proxy); + rb_vm_register_global_object(rb_block_param_proxy); + /* vm_backtrace.c */ Init_vm_backtrace(); - VM_PROFILE_ATEXIT(); } void rb_vm_set_progname(VALUE filename) { - rb_thread_t *th = GET_VM()->main_thread; - rb_control_frame_t *cfp = (void *)(th->stack + th->stack_size); + rb_thread_t *th = GET_VM()->ractor.main_thread; + rb_control_frame_t *cfp = (void *)(th->ec->vm_stack + th->ec->vm_stack_size); --cfp; - cfp->iseq->location.path = filename; + + filename = rb_str_new_frozen(filename); + rb_iseq_pathobj_set(cfp->iseq, filename, rb_iseq_realpath(cfp->iseq)); } -#if defined(ENABLE_VM_OBJSPACE) && ENABLE_VM_OBJSPACE -struct rb_objspace *rb_objspace_alloc(void); -#endif +extern const struct st_hash_type rb_fstring_hash_type; void Init_BareVM(void) { /* VM bootstrap: phase 1 */ - rb_vm_t * vm = ruby_mimmalloc(sizeof(*vm)); - rb_thread_t * th = ruby_mimmalloc(sizeof(*th)); + rb_vm_t *vm = ruby_mimcalloc(1, sizeof(*vm)); + rb_thread_t *th = ruby_mimcalloc(1, sizeof(*th)); if (!vm || !th) { - fprintf(stderr, "[FATAL] failed to allocate memory\n"); - exit(EXIT_FAILURE); + fputs("[FATAL] failed to allocate memory\n", stderr); + exit(EXIT_FAILURE); } - MEMZERO(th, rb_thread_t, 1); - rb_thread_set_current_raw(th); + // setup the VM vm_init2(vm); -#if defined(ENABLE_VM_OBJSPACE) && ENABLE_VM_OBJSPACE - vm->objspace = rb_objspace_alloc(); -#endif - ruby_current_vm = vm; - Init_native_thread(); + rb_vm_postponed_job_queue_init(vm); + ruby_current_vm_ptr = vm; + rb_objspace_alloc(); + vm->negative_cme_table = rb_id_table_create(16); + vm->overloaded_cme_table = st_init_numtable(); + vm->constant_cache = rb_id_table_create(0); + vm->unused_block_warning_table = set_init_numtable(); + vm->global_hooks.type = hook_list_type_global; + + // setup main thread + th->nt = ZALLOC(struct rb_native_thread); th->vm = vm; - th_init(th, 0); - ruby_thread_init_stack(th); + th->ractor = vm->ractor.main_ractor = rb_ractor_main_alloc(); + Init_native_thread(th); + rb_jit_cont_init(); + th_init(th, 0, vm); + + rb_ractor_set_current_ec(th->ractor, th->ec); + + /* n.b. native_main_thread_stack_top is set by the INIT_STACK macro */ + ruby_thread_init_stack(th, native_main_thread_stack_top); + + // setup ractor system + rb_native_mutex_initialize(&vm->ractor.sync.lock); + rb_native_cond_initialize(&vm->ractor.sync.terminate_cond); + + vm_opt_method_def_table = st_init_numtable(); + vm_opt_mid_table = st_init_numtable(); + +#ifdef RUBY_THREAD_WIN32_H + rb_native_cond_initialize(&vm->ractor.sync.barrier_complete_cond); + rb_native_cond_initialize(&vm->ractor.sync.barrier_release_cond); +#endif } -/* top self */ +void +ruby_init_stack(void *addr) +{ + native_main_thread_stack_top = addr; +} + +#ifndef _WIN32 +#include <unistd.h> +#include <sys/mman.h> +#endif + + +#ifndef MARK_OBJECT_ARY_BUCKET_SIZE +#define MARK_OBJECT_ARY_BUCKET_SIZE 1024 +#endif + +struct pin_array_list { + VALUE next; + long len; + VALUE *array; +}; + +static void +pin_array_list_mark(void *data) +{ + struct pin_array_list *array = (struct pin_array_list *)data; + rb_gc_mark_movable(array->next); + + rb_gc_mark_vm_stack_values(array->len, array->array); +} + +static void +pin_array_list_free(void *data) +{ + struct pin_array_list *array = (struct pin_array_list *)data; + xfree(array->array); +} + +static size_t +pin_array_list_memsize(const void *data) +{ + return sizeof(struct pin_array_list) + (MARK_OBJECT_ARY_BUCKET_SIZE * sizeof(VALUE)); +} + +static void +pin_array_list_update_references(void *data) +{ + struct pin_array_list *array = (struct pin_array_list *)data; + array->next = rb_gc_location(array->next); +} + +static const rb_data_type_t pin_array_list_type = { + .wrap_struct_name = "VM/pin_array_list", + .function = { + .dmark = pin_array_list_mark, + .dfree = pin_array_list_free, + .dsize = pin_array_list_memsize, + .dcompact = pin_array_list_update_references, + }, + .flags = RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE, +}; static VALUE -main_to_s(VALUE obj) +pin_array_list_new(VALUE next) { - return rb_str_new2("main"); + struct pin_array_list *array_list; + VALUE obj = TypedData_Make_Struct(0, struct pin_array_list, &pin_array_list_type, array_list); + RB_OBJ_WRITE(obj, &array_list->next, next); + array_list->array = ALLOC_N(VALUE, MARK_OBJECT_ARY_BUCKET_SIZE); + return obj; } -VALUE -rb_vm_top_self(void) +static VALUE +pin_array_list_append(VALUE obj, VALUE item) +{ + struct pin_array_list *array_list; + TypedData_Get_Struct(obj, struct pin_array_list, &pin_array_list_type, array_list); + + if (array_list->len >= MARK_OBJECT_ARY_BUCKET_SIZE) { + obj = pin_array_list_new(obj); + TypedData_Get_Struct(obj, struct pin_array_list, &pin_array_list_type, array_list); + } + + RB_OBJ_WRITE(obj, &array_list->array[array_list->len], item); + array_list->len++; + return obj; +} + +void +rb_vm_register_global_object(VALUE obj) { - return GET_VM()->top_self; + RUBY_ASSERT(!RB_SPECIAL_CONST_P(obj)); + if (RB_SPECIAL_CONST_P(obj)) { + return; + } + + switch (RB_BUILTIN_TYPE(obj)) { + case T_CLASS: + case T_MODULE: + if (FL_TEST(obj, RCLASS_IS_ROOT)) { + return; + } + FL_SET(obj, RCLASS_IS_ROOT); + break; + default: + break; + } + RB_VM_LOCKING() { + VALUE list = GET_VM()->mark_object_ary; + VALUE head = pin_array_list_append(list, obj); + if (head != list) { + GET_VM()->mark_object_ary = head; + } + RB_GC_GUARD(obj); + } } void -Init_top_self(void) +Init_vm_objects(void) { rb_vm_t *vm = GET_VM(); - vm->top_self = rb_obj_alloc(rb_cObject); - rb_define_singleton_method(rb_vm_top_self(), "to_s", main_to_s, 0); - rb_define_alias(rb_singleton_class(rb_vm_top_self()), "inspect", "to_s"); + /* initialize mark object array, hash */ + vm->mark_object_ary = pin_array_list_new(Qnil); + vm->ci_table = st_init_table(&vm_ci_hashtype); + vm->cc_refinement_table = rb_set_init_numtable(); +} + +// Whether JIT is enabled or not, we need to load/undef `#with_jit` for other builtins. +#include "jit_hook.rbinc" +#include "jit_undef.rbinc" + +// Stub for builtin function when not building YJIT units +#if !USE_YJIT +void Init_builtin_yjit(void) {} +#endif + +// Stub for builtin function when not building ZJIT units +#if !USE_ZJIT +void Init_builtin_zjit(void) {} +#endif - /* initialize mark object array */ - vm->mark_object_ary = rb_ary_tmp_new(1); +/* top self */ + +static VALUE +main_to_s(VALUE obj) +{ + return rb_str_new2("main"); } -VALUE * -ruby_vm_verbose_ptr(rb_vm_t *vm) +VALUE +rb_vm_top_self(void) { - return &vm->verbose; + const rb_box_t *box = rb_current_box(); + VM_ASSERT(box); + VM_ASSERT(box->top_self); + return box->top_self; } -VALUE * -ruby_vm_debug_ptr(rb_vm_t *vm) +void +Init_top_self(void) { - return &vm->debug; + rb_vm_t *vm = GET_VM(); + vm->root_box = (rb_box_t *)rb_root_box(); + vm->root_box->top_self = rb_obj_alloc(rb_cObject); + rb_define_singleton_method(vm->root_box->top_self, "to_s", main_to_s, 0); + rb_define_alias(rb_singleton_class(vm->root_box->top_self), "inspect", "to_s"); } VALUE * rb_ruby_verbose_ptr(void) { - return ruby_vm_verbose_ptr(GET_VM()); + rb_ractor_t *cr = GET_RACTOR(); + return &cr->verbose; } VALUE * rb_ruby_debug_ptr(void) { - return ruby_vm_debug_ptr(GET_VM()); + rb_ractor_t *cr = GET_RACTOR(); + return &cr->debug; +} + +bool rb_free_at_exit = false; + +bool +ruby_free_at_exit_p(void) +{ + return rb_free_at_exit; } /* iseq.c */ -VALUE rb_insn_operand_intern(rb_iseq_t *iseq, - VALUE insn, int op_no, VALUE op, - int len, size_t pos, VALUE *pnop, VALUE child); +VALUE rb_insn_operand_intern(const rb_iseq_t *iseq, + VALUE insn, int op_no, VALUE op, + int len, size_t pos, VALUE *pnop, VALUE child); #if VM_COLLECT_USAGE_DETAILS -#define HASH_ASET(h, k, v) st_insert(RHASH_TBL(h), (st_data_t)(k), (st_data_t)(v)) +#define HASH_ASET(h, k, v) rb_hash_aset((h), (st_data_t)(k), (st_data_t)(v)) /* uh = { * insn(Fixnum) => ihash(Hash) @@ -2644,30 +4844,30 @@ vm_analysis_insn(int insn) CONST_ID(usage_hash, "USAGE_ANALYSIS_INSN"); CONST_ID(bigram_hash, "USAGE_ANALYSIS_INSN_BIGRAM"); uh = rb_const_get(rb_cRubyVM, usage_hash); - if ((ihash = rb_hash_aref(uh, INT2FIX(insn))) == Qnil) { - ihash = rb_hash_new(); - HASH_ASET(uh, INT2FIX(insn), ihash); + if (NIL_P(ihash = rb_hash_aref(uh, INT2FIX(insn)))) { + ihash = rb_hash_new(); + HASH_ASET(uh, INT2FIX(insn), ihash); } - if ((cv = rb_hash_aref(ihash, INT2FIX(-1))) == Qnil) { - cv = INT2FIX(0); + if (NIL_P(cv = rb_hash_aref(ihash, INT2FIX(-1)))) { + cv = INT2FIX(0); } HASH_ASET(ihash, INT2FIX(-1), INT2FIX(FIX2INT(cv) + 1)); /* calc bigram */ if (prev_insn != -1) { - VALUE bi; - VALUE ary[2]; - VALUE cv; - - ary[0] = INT2FIX(prev_insn); - ary[1] = INT2FIX(insn); - bi = rb_ary_new4(2, &ary[0]); - - uh = rb_const_get(rb_cRubyVM, bigram_hash); - if ((cv = rb_hash_aref(uh, bi)) == Qnil) { - cv = INT2FIX(0); - } - HASH_ASET(uh, bi, INT2FIX(FIX2INT(cv) + 1)); + VALUE bi; + VALUE ary[2]; + VALUE cv; + + ary[0] = INT2FIX(prev_insn); + ary[1] = INT2FIX(insn); + bi = rb_ary_new4(2, &ary[0]); + + uh = rb_const_get(rb_cRubyVM, bigram_hash); + if (NIL_P(cv = rb_hash_aref(uh, bi))) { + cv = INT2FIX(0); + } + HASH_ASET(uh, bi, INT2FIX(FIX2INT(cv) + 1)); } prev_insn = insn; } @@ -2686,20 +4886,20 @@ vm_analysis_operand(int insn, int n, VALUE op) CONST_ID(usage_hash, "USAGE_ANALYSIS_INSN"); uh = rb_const_get(rb_cRubyVM, usage_hash); - if ((ihash = rb_hash_aref(uh, INT2FIX(insn))) == Qnil) { - ihash = rb_hash_new(); - HASH_ASET(uh, INT2FIX(insn), ihash); + if (NIL_P(ihash = rb_hash_aref(uh, INT2FIX(insn)))) { + ihash = rb_hash_new(); + HASH_ASET(uh, INT2FIX(insn), ihash); } - if ((ophash = rb_hash_aref(ihash, INT2FIX(n))) == Qnil) { - ophash = rb_hash_new(); - HASH_ASET(ihash, INT2FIX(n), ophash); + if (NIL_P(ophash = rb_hash_aref(ihash, INT2FIX(n)))) { + ophash = rb_hash_new(); + HASH_ASET(ihash, INT2FIX(n), ophash); } /* intern */ - valstr = rb_insn_operand_intern(GET_THREAD()->cfp->iseq, insn, n, op, 0, 0, 0, 0); + valstr = rb_insn_operand_intern(GET_EC()->cfp->iseq, insn, n, op, 0, 0, 0, 0); /* set count */ - if ((cv = rb_hash_aref(ophash, valstr)) == Qnil) { - cv = INT2FIX(0); + if (NIL_P(cv = rb_hash_aref(ophash, valstr))) { + cv = INT2FIX(0); } HASH_ASET(ophash, valstr, INT2FIX(FIX2INT(cv) + 1)); } @@ -2711,16 +4911,16 @@ vm_analysis_register(int reg, int isset) VALUE uh; VALUE valstr; static const char regstrs[][5] = { - "pc", /* 0 */ - "sp", /* 1 */ - "ep", /* 2 */ - "cfp", /* 3 */ - "self", /* 4 */ - "iseq", /* 5 */ + "pc", /* 0 */ + "sp", /* 1 */ + "ep", /* 2 */ + "cfp", /* 3 */ + "self", /* 4 */ + "iseq", /* 5 */ }; static const char getsetstr[][4] = { - "get", - "set", + "get", + "set", }; static VALUE syms[sizeof(regstrs) / sizeof(regstrs[0])][2]; @@ -2728,31 +4928,55 @@ vm_analysis_register(int reg, int isset) CONST_ID(usage_hash, "USAGE_ANALYSIS_REGS"); if (syms[0] == 0) { - char buff[0x10]; - int i; - - for (i = 0; i < (int)(sizeof(regstrs) / sizeof(regstrs[0])); i++) { - int j; - for (j = 0; j < 2; j++) { - snprintf(buff, 0x10, "%d %s %-4s", i, getsetstr[j], regstrs[i]); - syms[i][j] = ID2SYM(rb_intern(buff)); - } - } + char buff[0x10]; + int i; + + for (i = 0; i < (int)(sizeof(regstrs) / sizeof(regstrs[0])); i++) { + int j; + for (j = 0; j < 2; j++) { + snprintf(buff, 0x10, "%d %s %-4s", i, getsetstr[j], regstrs[i]); + syms[i][j] = ID2SYM(rb_intern(buff)); + } + } } valstr = syms[reg][isset]; uh = rb_const_get(rb_cRubyVM, usage_hash); - if ((cv = rb_hash_aref(uh, valstr)) == Qnil) { - cv = INT2FIX(0); + if (NIL_P(cv = rb_hash_aref(uh, valstr))) { + cv = INT2FIX(0); } HASH_ASET(uh, valstr, INT2FIX(FIX2INT(cv) + 1)); } #undef HASH_ASET -void (*ruby_vm_collect_usage_func_insn)(int insn) = vm_analysis_insn; -void (*ruby_vm_collect_usage_func_operand)(int insn, int n, VALUE op) = vm_analysis_operand; -void (*ruby_vm_collect_usage_func_register)(int reg, int isset) = vm_analysis_register; +static void (*ruby_vm_collect_usage_func_insn)(int insn) = NULL; +static void (*ruby_vm_collect_usage_func_operand)(int insn, int n, VALUE op) = NULL; +static void (*ruby_vm_collect_usage_func_register)(int reg, int isset) = NULL; + +/* :nodoc: */ +static VALUE +usage_analysis_insn_start(VALUE self) +{ + ruby_vm_collect_usage_func_insn = vm_analysis_insn; + return Qnil; +} + +/* :nodoc: */ +static VALUE +usage_analysis_operand_start(VALUE self) +{ + ruby_vm_collect_usage_func_operand = vm_analysis_operand; + return Qnil; +} + +/* :nodoc: */ +static VALUE +usage_analysis_register_start(VALUE self) +{ + ruby_vm_collect_usage_func_register = vm_analysis_register; + return Qnil; +} /* :nodoc: */ static VALUE @@ -2778,11 +5002,76 @@ usage_analysis_register_stop(VALUE self) return Qnil; } +/* :nodoc: */ +static VALUE +usage_analysis_insn_running(VALUE self) +{ + return RBOOL(ruby_vm_collect_usage_func_insn != 0); +} + +/* :nodoc: */ +static VALUE +usage_analysis_operand_running(VALUE self) +{ + return RBOOL(ruby_vm_collect_usage_func_operand != 0); +} + +/* :nodoc: */ +static VALUE +usage_analysis_register_running(VALUE self) +{ + return RBOOL(ruby_vm_collect_usage_func_register != 0); +} + +static VALUE +usage_analysis_clear(VALUE self, ID usage_hash) +{ + VALUE uh; + uh = rb_const_get(self, usage_hash); + rb_hash_clear(uh); + + return Qtrue; +} + + +/* :nodoc: */ +static VALUE +usage_analysis_insn_clear(VALUE self) +{ + ID usage_hash; + ID bigram_hash; + + CONST_ID(usage_hash, "USAGE_ANALYSIS_INSN"); + CONST_ID(bigram_hash, "USAGE_ANALYSIS_INSN_BIGRAM"); + usage_analysis_clear(rb_cRubyVM, usage_hash); + return usage_analysis_clear(rb_cRubyVM, bigram_hash); +} + +/* :nodoc: */ +static VALUE +usage_analysis_operand_clear(VALUE self) +{ + ID usage_hash; + + CONST_ID(usage_hash, "USAGE_ANALYSIS_INSN"); + return usage_analysis_clear(self, usage_hash); +} + +/* :nodoc: */ +static VALUE +usage_analysis_register_clear(VALUE self) +{ + ID usage_hash; + + CONST_ID(usage_hash, "USAGE_ANALYSIS_REGS"); + return usage_analysis_clear(self, usage_hash); +} + #else -void (*ruby_vm_collect_usage_func_insn)(int insn) = NULL; -void (*ruby_vm_collect_usage_func_operand)(int insn, int n, VALUE op) = NULL; -void (*ruby_vm_collect_usage_func_register)(int reg, int isset) = NULL; +MAYBE_UNUSED(static void (*ruby_vm_collect_usage_func_insn)(int insn)) = 0; +MAYBE_UNUSED(static void (*ruby_vm_collect_usage_func_operand)(int insn, int n, VALUE op)) = 0; +MAYBE_UNUSED(static void (*ruby_vm_collect_usage_func_register)(int reg, int isset)) = 0; #endif @@ -2792,10 +5081,10 @@ static void vm_collect_usage_insn(int insn) { if (RUBY_DTRACE_INSN_ENABLED()) { - RUBY_DTRACE_INSN(rb_insns_name(insn)); + RUBY_DTRACE_INSN(rb_insns_name(insn)); } if (ruby_vm_collect_usage_func_insn) - (*ruby_vm_collect_usage_func_insn)(insn); + (*ruby_vm_collect_usage_func_insn)(insn); } /* @param insn instruction number @@ -2806,24 +5095,37 @@ static void vm_collect_usage_operand(int insn, int n, VALUE op) { if (RUBY_DTRACE_INSN_OPERAND_ENABLED()) { - VALUE valstr; + VALUE valstr; - valstr = rb_insn_operand_intern(GET_THREAD()->cfp->iseq, insn, n, op, 0, 0, 0, 0); + valstr = rb_insn_operand_intern(GET_EC()->cfp->iseq, insn, n, op, 0, 0, 0, 0); - RUBY_DTRACE_INSN_OPERAND(RSTRING_PTR(valstr), rb_insns_name(insn)); - RB_GC_GUARD(valstr); + RUBY_DTRACE_INSN_OPERAND(RSTRING_PTR(valstr), rb_insns_name(insn)); + RB_GC_GUARD(valstr); } if (ruby_vm_collect_usage_func_operand) - (*ruby_vm_collect_usage_func_operand)(insn, n, op); + (*ruby_vm_collect_usage_func_operand)(insn, n, op); } /* @param reg register id. see code of vm_analysis_register() */ -/* @param iseset 0: read, 1: write */ +/* @param isset 0: read, 1: write */ static void vm_collect_usage_register(int reg, int isset) { if (ruby_vm_collect_usage_func_register) - (*ruby_vm_collect_usage_func_register)(reg, isset); + (*ruby_vm_collect_usage_func_register)(reg, isset); } #endif +const struct rb_callcache * +rb_vm_empty_cc(void) +{ + return &vm_empty_cc; +} + +const struct rb_callcache * +rb_vm_empty_cc_for_super(void) +{ + return &vm_empty_cc_for_super; +} + +#include "vm_call_iseq_optimized.inc" /* required from vm_insnhelper.c */ |