diff options
Diffstat (limited to 'vm_trace.c')
| -rw-r--r-- | vm_trace.c | 487 |
1 files changed, 296 insertions, 191 deletions
diff --git a/vm_trace.c b/vm_trace.c index 4032b1f639..305802f469 100644 --- a/vm_trace.c +++ b/vm_trace.c @@ -34,7 +34,9 @@ #include "ruby/debug.h" #include "vm_core.h" #include "ruby/ractor.h" +#include "ractor_core.h" #include "yjit.h" +#include "zjit.h" #include "builtin.h" @@ -71,13 +73,17 @@ rb_hook_list_mark(rb_hook_list_t *hooks) } void -rb_hook_list_mark_and_update(rb_hook_list_t *hooks) +rb_hook_list_mark_and_move(rb_hook_list_t *hooks) { - rb_event_hook_t *hook = hooks->hooks; + if (!rb_gc_checking_shareable()) { + // hooks can be unshareable - while (hook) { - rb_gc_mark_and_move(&hook->data); - hook = hook->next; + rb_event_hook_t *hook = hooks->hooks; + + while (hook) { + rb_gc_mark_and_move(&hook->data); + hook = hook->next; + } } } @@ -98,43 +104,89 @@ rb_hook_list_free(rb_hook_list_t *hooks) void rb_clear_attr_ccs(void); void rb_clear_bf_ccs(void); -static void -update_global_event_hook(rb_event_flag_t prev_events, rb_event_flag_t new_events) +static bool iseq_trace_set_all_needed(rb_event_flag_t new_events) { rb_event_flag_t new_iseq_events = new_events & ISEQ_TRACE_EVENTS; rb_event_flag_t enabled_iseq_events = ruby_vm_event_enabled_global_flags & ISEQ_TRACE_EVENTS; - bool first_time_iseq_events_p = new_iseq_events & ~enabled_iseq_events; - bool enable_c_call = (prev_events & RUBY_EVENT_C_CALL) == 0 && (new_events & RUBY_EVENT_C_CALL); + return new_iseq_events & ~enabled_iseq_events; + +} + +static bool clear_attr_ccs_needed(rb_event_flag_t prev_events, rb_event_flag_t new_events) +{ + bool enable_c_call = (prev_events & RUBY_EVENT_C_CALL) == 0 && (new_events & RUBY_EVENT_C_CALL); bool enable_c_return = (prev_events & RUBY_EVENT_C_RETURN) == 0 && (new_events & RUBY_EVENT_C_RETURN); - bool enable_call = (prev_events & RUBY_EVENT_CALL) == 0 && (new_events & RUBY_EVENT_CALL); - bool enable_return = (prev_events & RUBY_EVENT_RETURN) == 0 && (new_events & RUBY_EVENT_RETURN); + return enable_c_call || enable_c_return; +} + +/* If the events are internal events (e.g. gc hooks), it updates them globally for all ractors. Otherwise + * they are ractor local. You cannot listen to internal events through set_trace_func or TracePoint. + * Some ractor-local tracepoint events cause global level iseq changes, so are still called `global events`. + */ +static void +update_global_event_hooks(rb_hook_list_t *list, rb_event_flag_t prev_events, rb_event_flag_t new_events, int change_iseq_events, int change_c_events) +{ + rb_execution_context_t *ec = rb_current_execution_context(false); + unsigned int lev; + + // Can't enter VM lock during freeing of ractor hook list on MMTK, where ec == NULL. + if (ec) { + RB_VM_LOCK_ENTER_LEV(&lev); + rb_vm_barrier(); + } + + rb_event_flag_t new_iseq_events = new_events & ISEQ_TRACE_EVENTS; + rb_event_flag_t enabled_iseq_events = ruby_vm_event_enabled_global_flags & ISEQ_TRACE_EVENTS; + bool new_iseq_events_p = iseq_trace_set_all_needed(new_events); + bool enable_call = (prev_events & RUBY_EVENT_CALL) == 0 && (new_events & RUBY_EVENT_CALL); + bool enable_return = (prev_events & RUBY_EVENT_RETURN) == 0 && (new_events & RUBY_EVENT_RETURN); + bool clear_attr_ccs_p = clear_attr_ccs_needed(prev_events, new_events); + + // FIXME: `ruby_vm_event_flags` should have the global list of event flags for internal events as well + // as for all ractors. That's not how it works right now, so we shouldn't rely on it apart from the + // internal events. Since it doesn't work like this, we have to track more state with `ruby_vm_iseq_events_enabled`, + // `ruby_vm_c_events_enabled`, etc. + rb_event_flag_t new_events_global = (ruby_vm_event_flags & ~prev_events) | new_events; + ruby_vm_event_flags = new_events_global; // Modify ISEQs or CCs to enable tracing - if (first_time_iseq_events_p) { + if (new_iseq_events_p) { // write all ISeqs only when new events are added for the first time rb_iseq_trace_set_all(new_iseq_events | enabled_iseq_events); } - // if c_call or c_return is activated - else if (enable_c_call || enable_c_return) { + else if (clear_attr_ccs_p) { // turn on C_CALL or C_RETURN ractor locally rb_clear_attr_ccs(); } - else if (enable_call || enable_return) { + else if (enable_call || enable_return) { // turn on CALL or RETURN ractor locally rb_clear_bf_ccs(); } - ruby_vm_event_flags = new_events; - ruby_vm_event_enabled_global_flags |= new_events; - rb_objspace_set_event_hook(new_events); + if (change_iseq_events < 0) { + RUBY_ASSERT(ruby_vm_iseq_events_enabled >= (unsigned int)(-change_iseq_events)); + } + ruby_vm_iseq_events_enabled += change_iseq_events; + if (change_c_events < 0) { + RUBY_ASSERT(ruby_vm_c_events_enabled >= (unsigned int)(-change_iseq_events)); + } + ruby_vm_c_events_enabled += change_c_events; + + ruby_vm_event_enabled_global_flags |= new_events; // NOTE: this is only ever added to + if (new_events_global & RUBY_INTERNAL_EVENT_MASK) { + rb_objspace_set_event_hook(new_events_global); + } // Invalidate JIT code as needed - if (first_time_iseq_events_p || enable_c_call || enable_c_return) { + if (new_iseq_events_p || clear_attr_ccs_p) { // Invalidate all code when ISEQs are modified to use trace_* insns above. // Also invalidate when enabling c_call or c_return because generated code // never fires these events. // Internal events fire inside C routines so don't need special handling. - // Do this after event flags updates so other ractors see updated vm events - // when they wake up. rb_yjit_tracing_invalidate_all(); + rb_zjit_tracing_invalidate_all(); + } + + if (ec) { + RB_VM_LOCK_LEAVE_LEV(&lev); } } @@ -162,28 +214,43 @@ alloc_event_hook(rb_event_hook_func_t func, rb_event_flag_t events, VALUE data, return hook; } +// Connect a hook onto a ractor, an iseq or a method definition's hook list static void -hook_list_connect(VALUE list_owner, rb_hook_list_t *list, rb_event_hook_t *hook, int global_p) +hook_list_connect(rb_hook_list_t *list, rb_event_hook_t *hook, int global_p) { rb_event_flag_t prev_events = list->events; + int change_iseq_events = 0; + int change_c_events = 0; hook->next = list->hooks; list->hooks = hook; list->events |= hook->events; if (global_p) { - /* global hooks are root objects at GC mark. */ - update_global_event_hook(prev_events, list->events); - } - else { - RB_OBJ_WRITTEN(list_owner, Qundef, hook->data); + if (hook->events & ISEQ_TRACE_EVENTS) { + change_iseq_events++; + } + if ((hook->events & RUBY_EVENT_C_CALL) || (hook->events & RUBY_EVENT_C_RETURN)) { + change_c_events++; + } + update_global_event_hooks(list, prev_events, list->events, change_iseq_events, change_c_events); } } static void -connect_event_hook(const rb_execution_context_t *ec, rb_event_hook_t *hook) +connect_non_targeted_event_hook(const rb_execution_context_t *ec, rb_event_hook_t *hook) { - rb_hook_list_t *list = rb_ec_ractor_hooks(ec); - hook_list_connect(Qundef, list, hook, TRUE); + rb_hook_list_t *list; + + /* internal events are VM-global, non-internal are ractor-local */ + VM_ASSERT(!(hook->events & RUBY_INTERNAL_EVENT_OBJSPACE_MASK) || !(hook->events & ~RUBY_INTERNAL_EVENT_OBJSPACE_MASK)); + + if (hook->events & RUBY_INTERNAL_EVENT_OBJSPACE_MASK) { + list = rb_vm_global_hooks(ec); + } + else { + list = rb_ec_ractor_hooks(ec); + } + hook_list_connect(list, hook, TRUE); } static void @@ -192,7 +259,7 @@ rb_threadptr_add_event_hook(const rb_execution_context_t *ec, rb_thread_t *th, { rb_event_hook_t *hook = alloc_event_hook(func, events, data, hook_flags); hook->filter.th = th; - connect_event_hook(ec, hook); + connect_non_targeted_event_hook(ec, hook); } void @@ -217,7 +284,35 @@ void rb_add_event_hook2(rb_event_hook_func_t func, rb_event_flag_t events, VALUE data, rb_event_hook_flag_t hook_flags) { rb_event_hook_t *hook = alloc_event_hook(func, events, data, hook_flags); - connect_event_hook(GET_EC(), hook); + connect_non_targeted_event_hook(GET_EC(), hook); +} + +static bool +hook_list_targeted_p(rb_hook_list_t *list) +{ + switch (list->type) { + case hook_list_type_targeted_iseq: + case hook_list_type_targeted_def: + return true; + default: + return false; + } +} + +unsigned int +rb_hook_list_count(rb_hook_list_t *list) +{ + rb_event_hook_t *hook = list->hooks; + unsigned int count = 0; + + while (hook) { + if (!(hook->hook_flags & RUBY_EVENT_HOOK_FLAG_DELETED)) { + count++; + } + hook = hook->next; + } + + return count; } static void @@ -225,6 +320,8 @@ clean_hooks(rb_hook_list_t *list) { rb_event_hook_t *hook, **nextp = &list->hooks; rb_event_flag_t prev_events = list->events; + int change_iseq_events = 0; + int change_c_events = 0; VM_ASSERT(list->running == 0); VM_ASSERT(list->need_clean == true); @@ -235,7 +332,15 @@ clean_hooks(rb_hook_list_t *list) while ((hook = *nextp) != 0) { if (hook->hook_flags & RUBY_EVENT_HOOK_FLAG_DELETED) { *nextp = hook->next; - xfree(hook); + if (!hook_list_targeted_p(list)) { + if (hook->events & ISEQ_TRACE_EVENTS) { + change_iseq_events--; + } + if ((hook->events & RUBY_EVENT_C_CALL) || (hook->events & RUBY_EVENT_C_RETURN)) { + change_c_events--; + } + } + SIZED_FREE(hook); } else { list->events |= hook->events; /* update active events */ @@ -243,14 +348,13 @@ clean_hooks(rb_hook_list_t *list) } } - if (list->is_local) { + if (hook_list_targeted_p(list)) { if (list->events == 0) { - /* local events */ - ruby_xfree(list); + SIZED_FREE(list); } } else { - update_global_event_hook(prev_events, list->events); + update_global_event_hooks(list, prev_events, list->events, change_iseq_events, change_c_events); } } @@ -266,11 +370,9 @@ clean_hooks_check(rb_hook_list_t *list) #define MATCH_ANY_FILTER_TH ((rb_thread_t *)1) -/* if func is 0, then clear all funcs */ static int -remove_event_hook(const rb_execution_context_t *ec, const rb_thread_t *filter_th, rb_event_hook_func_t func, VALUE data) +remove_event_hook_from_list(rb_hook_list_t *list, const rb_thread_t *filter_th, rb_event_hook_func_t func, VALUE data) { - rb_hook_list_t *list = rb_ec_ractor_hooks(ec); int ret = 0; rb_event_hook_t *hook = list->hooks; @@ -279,8 +381,8 @@ remove_event_hook(const rb_execution_context_t *ec, const rb_thread_t *filter_th if (hook->filter.th == filter_th || filter_th == MATCH_ANY_FILTER_TH) { if (UNDEF_P(data) || hook->data == data) { hook->hook_flags |= RUBY_EVENT_HOOK_FLAG_DELETED; - ret+=1; list->need_clean = true; + ret+=1; } } } @@ -291,6 +393,18 @@ remove_event_hook(const rb_execution_context_t *ec, const rb_thread_t *filter_th return ret; } +/* if func is 0, then clear all funcs */ +static int +remove_event_hook(const rb_execution_context_t *ec, const rb_thread_t *filter_th, rb_event_hook_func_t func, VALUE data) +{ + int ret = 0; + + ret += remove_event_hook_from_list(rb_ec_ractor_hooks(ec), filter_th, func, data); + ret += remove_event_hook_from_list(rb_vm_global_hooks(ec), filter_th, func, data); + + return ret; +} + static int rb_threadptr_remove_event_hook(const rb_execution_context_t *ec, const rb_thread_t *filter_th, rb_event_hook_func_t func, VALUE data) { @@ -415,8 +529,10 @@ rb_exec_event_hooks(rb_trace_arg_t *trace_arg, rb_hook_list_t *hooks, int pop_p) { rb_execution_context_t *ec = trace_arg->ec; - if (UNLIKELY(trace_arg->event & RUBY_INTERNAL_EVENT_MASK)) { - if (ec->trace_arg && (ec->trace_arg->event & RUBY_INTERNAL_EVENT_MASK)) { + if (UNLIKELY(trace_arg->event & RUBY_INTERNAL_EVENT_OBJSPACE_MASK)) { + VM_ASSERT(hooks == rb_vm_global_hooks(ec)); + + if (ec->trace_arg && (ec->trace_arg->event & RUBY_INTERNAL_EVENT_OBJSPACE_MASK)) { /* skip hooks because this thread doing INTERNAL_EVENT */ } else { @@ -424,7 +540,7 @@ rb_exec_event_hooks(rb_trace_arg_t *trace_arg, rb_hook_list_t *hooks, int pop_p) ec->trace_arg = trace_arg; /* only global hooks */ - exec_hooks_unprotected(ec, rb_ec_ractor_hooks(ec), trace_arg); + exec_hooks_unprotected(ec, hooks, trace_arg); ec->trace_arg = prev_trace_arg; } } @@ -696,7 +812,7 @@ get_path_and_lineno(const rb_execution_context_t *ec, const rb_control_frame_t * cfp = rb_vm_get_ruby_level_next_cfp(ec, cfp); if (cfp) { - const rb_iseq_t *iseq = cfp->iseq; + const rb_iseq_t *iseq = CFP_ISEQ(cfp); *pathp = rb_iseq_path(iseq); if (event & (RUBY_EVENT_CLASS | @@ -746,7 +862,7 @@ call_trace_func(rb_event_flag_t event, VALUE proc, VALUE self, ID id, VALUE klas if (self && (filename != Qnil) && event != RUBY_EVENT_C_CALL && event != RUBY_EVENT_C_RETURN && - (VM_FRAME_RUBYFRAME_P(ec->cfp) && imemo_type_p((VALUE)ec->cfp->iseq, imemo_iseq))) { + (VM_FRAME_RUBYFRAME_P(ec->cfp) && imemo_type_p((VALUE)CFP_ISEQ(ec->cfp), imemo_iseq))) { argv[4] = rb_binding_new(); } argv[5] = klass ? klass : Qnil; @@ -774,20 +890,21 @@ typedef struct rb_tp_struct { } rb_tp_t; static void -tp_mark(void *ptr) +tp_mark_and_move(void *ptr) { rb_tp_t *tp = ptr; - rb_gc_mark(tp->proc); - rb_gc_mark(tp->local_target_set); - if (tp->target_th) rb_gc_mark(tp->target_th->self); + rb_gc_mark_and_move(&tp->proc); + rb_gc_mark_and_move(&tp->local_target_set); + if (tp->target_th) rb_gc_mark_and_move(&tp->target_th->self); } static const rb_data_type_t tp_data_type = { "tracepoint", { - tp_mark, + tp_mark_and_move, RUBY_TYPED_DEFAULT_FREE, NULL, // Nothing allocated externally, so don't need a memsize function + tp_mark_and_move, }, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE }; @@ -925,7 +1042,7 @@ rb_tracearg_parameters(rb_trace_arg_t *trace_arg) if (VM_FRAME_TYPE(cfp) == VM_FRAME_MAGIC_BLOCK && !VM_FRAME_LAMBDA_P(cfp)) { is_proc = 1; } - return rb_iseq_parameters(cfp->iseq, is_proc); + return rb_iseq_parameters(CFP_ISEQ(cfp), is_proc); } break; } @@ -987,7 +1104,7 @@ rb_tracearg_binding(rb_trace_arg_t *trace_arg) } cfp = rb_vm_get_binding_creatable_next_cfp(trace_arg->ec, trace_arg->cfp); - if (cfp && imemo_type_p((VALUE)cfp->iseq, imemo_iseq)) { + if (cfp && imemo_type_p((VALUE)CFP_ISEQ(cfp), imemo_iseq)) { return rb_vm_make_binding(trace_arg->ec, cfp); } else { @@ -1001,15 +1118,18 @@ rb_tracearg_self(rb_trace_arg_t *trace_arg) return trace_arg->self; } -VALUE -rb_tracearg_return_value(rb_trace_arg_t *trace_arg) +static void +check_event_support(rb_trace_arg_t *trace_arg, rb_event_flag_t supported) { - if (trace_arg->event & (RUBY_EVENT_RETURN | RUBY_EVENT_C_RETURN | RUBY_EVENT_B_RETURN)) { - /* ok */ - } - else { + if (!(trace_arg->event & supported)) { rb_raise(rb_eRuntimeError, "not supported by this event"); } +} + +VALUE +rb_tracearg_return_value(rb_trace_arg_t *trace_arg) +{ + check_event_support(trace_arg, RUBY_EVENT_RETURN | RUBY_EVENT_C_RETURN | RUBY_EVENT_B_RETURN); if (UNDEF_P(trace_arg->data)) { rb_bug("rb_tracearg_return_value: unreachable"); } @@ -1019,12 +1139,7 @@ rb_tracearg_return_value(rb_trace_arg_t *trace_arg) VALUE rb_tracearg_raised_exception(rb_trace_arg_t *trace_arg) { - if (trace_arg->event & (RUBY_EVENT_RAISE | RUBY_EVENT_RESCUE)) { - /* ok */ - } - else { - rb_raise(rb_eRuntimeError, "not supported by this event"); - } + check_event_support(trace_arg, RUBY_EVENT_RAISE | RUBY_EVENT_RESCUE); if (UNDEF_P(trace_arg->data)) { rb_bug("rb_tracearg_raised_exception: unreachable"); } @@ -1036,14 +1151,9 @@ rb_tracearg_eval_script(rb_trace_arg_t *trace_arg) { VALUE data = trace_arg->data; - if (trace_arg->event & (RUBY_EVENT_SCRIPT_COMPILED)) { - /* ok */ - } - else { - rb_raise(rb_eRuntimeError, "not supported by this event"); - } + check_event_support(trace_arg, RUBY_EVENT_SCRIPT_COMPILED); if (UNDEF_P(data)) { - rb_bug("rb_tracearg_raised_exception: unreachable"); + rb_bug("rb_tracearg_eval_script: unreachable"); } if (rb_obj_is_iseq(data)) { return Qnil; @@ -1060,14 +1170,9 @@ rb_tracearg_instruction_sequence(rb_trace_arg_t *trace_arg) { VALUE data = trace_arg->data; - if (trace_arg->event & (RUBY_EVENT_SCRIPT_COMPILED)) { - /* ok */ - } - else { - rb_raise(rb_eRuntimeError, "not supported by this event"); - } + check_event_support(trace_arg, RUBY_EVENT_SCRIPT_COMPILED); if (UNDEF_P(data)) { - rb_bug("rb_tracearg_raised_exception: unreachable"); + rb_bug("rb_tracearg_instruction_sequence: unreachable"); } if (rb_obj_is_iseq(data)) { @@ -1085,12 +1190,7 @@ rb_tracearg_instruction_sequence(rb_trace_arg_t *trace_arg) VALUE rb_tracearg_object(rb_trace_arg_t *trace_arg) { - if (trace_arg->event & (RUBY_INTERNAL_EVENT_NEWOBJ | RUBY_INTERNAL_EVENT_FREEOBJ)) { - /* ok */ - } - else { - rb_raise(rb_eRuntimeError, "not supported by this event"); - } + check_event_support(trace_arg, RUBY_INTERNAL_EVENT_NEWOBJ | RUBY_INTERNAL_EVENT_FREEOBJ); if (UNDEF_P(trace_arg->data)) { rb_bug("rb_tracearg_object: unreachable"); } @@ -1183,7 +1283,7 @@ tp_call_trace(VALUE tpval, rb_trace_arg_t *trace_arg) (*tp->func)(tpval, tp->data); } else { - if (tp->ractor == NULL || tp->ractor == GET_RACTOR()) { + if (tp->ractor == GET_RACTOR()) { rb_proc_call_with_block((VALUE)tp->proc, 1, &tpval, Qnil); } } @@ -1229,14 +1329,33 @@ iseq_of(VALUE target) const rb_method_definition_t *rb_method_def(VALUE method); /* proc.c */ +rb_hook_list_t * +rb_method_def_local_hooks(rb_method_definition_t *def, rb_ractor_t *cr, bool create) +{ + st_data_t val; + rb_hook_list_t *hook_list = NULL; + if (st_lookup(rb_ractor_targeted_hooks(cr), (st_data_t)def, &val)) { + hook_list = (rb_hook_list_t*)val; + RUBY_ASSERT(hook_list->type == hook_list_type_targeted_def); + } + else if (create) { + hook_list = ZALLOC(rb_hook_list_t); + hook_list->type = hook_list_type_targeted_def; + st_insert(&cr->pub.targeted_hooks, (st_data_t)def, (st_data_t)hook_list); + } + return hook_list; +} + +// Enable "local" (targeted) tracepoint static VALUE rb_tracepoint_enable_for_target(VALUE tpval, VALUE target, VALUE target_line) { rb_tp_t *tp = tpptr(tpval); - const rb_iseq_t *iseq = iseq_of(target); + const rb_iseq_t *iseq = iseq_of(target); // takes Proc, Iseq, Method int n = 0; unsigned int line = 0; bool target_bmethod = false; + rb_ractor_t *cr = GET_RACTOR(); if (tp->tracing > 0) { rb_raise(rb_eArgError, "can't nest-enable a targeting TracePoint"); @@ -1254,61 +1373,80 @@ rb_tracepoint_enable_for_target(VALUE tpval, VALUE target, VALUE target_line) VM_ASSERT(tp->local_target_set == Qfalse); RB_OBJ_WRITE(tpval, &tp->local_target_set, rb_obj_hide(rb_ident_hash_new())); - /* bmethod */ - if (rb_obj_is_method(target)) { - rb_method_definition_t *def = (rb_method_definition_t *)rb_method_def(target); - if (def->type == VM_METHOD_TYPE_BMETHOD && - (tp->events & (RUBY_EVENT_CALL | RUBY_EVENT_RETURN))) { - if (def->body.bmethod.hooks == NULL) { - def->body.bmethod.hooks = ZALLOC(rb_hook_list_t); - def->body.bmethod.hooks->is_local = true; - } - rb_hook_list_connect_tracepoint(target, def->body.bmethod.hooks, tpval, 0); - rb_hash_aset(tp->local_target_set, target, Qfalse); - target_bmethod = true; + RB_VM_LOCKING() { + // Rewriting iseq instructions across ractors is not safe unless they are stopped. + rb_vm_barrier(); - n++; + /* bmethod */ + if (rb_obj_is_method(target)) { + rb_method_definition_t *def = (rb_method_definition_t *)rb_method_def(target); + if (def->type == VM_METHOD_TYPE_BMETHOD && (tp->events & (RUBY_EVENT_CALL | RUBY_EVENT_RETURN))) { + rb_hook_list_t *hook_list = rb_method_def_local_hooks(def, cr, true); + rb_hook_list_connect_local_tracepoint(hook_list, tpval, 0); + rb_hash_aset(tp->local_target_set, target, Qfalse); // Qfalse means not an iseq + rb_method_definition_addref(def); // in case `tp` gets GC'd and didn't disable the hook, `def` needs to stay alive + def->body.bmethod.local_hooks_cnt++; + target_bmethod = true; + n++; + } } - } - /* iseq */ - n += rb_iseq_add_local_tracepoint_recursively(iseq, tp->events, tpval, line, target_bmethod); - rb_hash_aset(tp->local_target_set, (VALUE)iseq, Qtrue); + /* iseq */ + n += rb_iseq_add_local_tracepoint_recursively(iseq, tp->events, tpval, line, target_bmethod); + if (n > 0) { + rb_hash_aset(tp->local_target_set, (VALUE)iseq, Qtrue); - if ((tp->events & (RUBY_EVENT_CALL | RUBY_EVENT_RETURN)) && - iseq->body->builtin_attrs & BUILTIN_ATTR_SINGLE_NOARG_LEAF) { - rb_clear_bf_ccs(); + if ((tp->events & (RUBY_EVENT_CALL | RUBY_EVENT_RETURN)) && + iseq->body->builtin_attrs & BUILTIN_ATTR_SINGLE_NOARG_LEAF) { + rb_clear_bf_ccs(); + } + + rb_yjit_tracing_invalidate_all(); + rb_zjit_tracing_invalidate_all(); + rb_ractor_targeted_hooks_incr(tp->ractor); + if (tp->events & ISEQ_TRACE_EVENTS) { + ruby_vm_iseq_events_enabled++; + } + if ((tp->events & RUBY_EVENT_C_CALL) || (tp->events & RUBY_EVENT_C_RETURN)) { + ruby_vm_c_events_enabled++; + } + tp->tracing = 1; + } } if (n == 0) { rb_raise(rb_eArgError, "can not enable any hooks"); } - rb_yjit_tracing_invalidate_all(); - - ruby_vm_event_local_num++; - - tp->tracing = 1; - return Qnil; } static int -disable_local_event_iseq_i(VALUE target, VALUE iseq_p, VALUE tpval) +disable_local_tracepoint_i(VALUE target, VALUE iseq_p, VALUE tpval) { + rb_tp_t *tp = tpptr(tpval); + rb_ractor_t *cr; + rb_method_definition_t *def; + rb_hook_list_t *hook_list; + ASSERT_vm_locking_with_barrier(); + if (iseq_p) { - rb_iseq_remove_local_tracepoint_recursively((rb_iseq_t *)target, tpval); + rb_iseq_remove_local_tracepoint_recursively((rb_iseq_t *)target, tpval, tp->ractor); } else { + cr = GET_RACTOR(); /* bmethod */ - rb_method_definition_t *def = (rb_method_definition_t *)rb_method_def(target); - rb_hook_list_t *hooks = def->body.bmethod.hooks; - VM_ASSERT(hooks != NULL); - rb_hook_list_remove_tracepoint(hooks, tpval); - - if (hooks->events == 0) { - rb_hook_list_free(def->body.bmethod.hooks); - def->body.bmethod.hooks = NULL; + def = (rb_method_definition_t *)rb_method_def(target); + hook_list = rb_method_def_local_hooks(def, cr, false); + RUBY_ASSERT(hook_list != NULL); + if (rb_hook_list_remove_local_tracepoint(hook_list, tpval)) { + RUBY_ASSERT(def->body.bmethod.local_hooks_cnt > 0); + def->body.bmethod.local_hooks_cnt--; + if (hook_list->events == 0) { + st_delete(rb_ractor_targeted_hooks(cr), (st_data_t*)&def, NULL); + rb_hook_list_free(hook_list); + } + rb_method_definition_release(def); } } return ST_CONTINUE; @@ -1321,10 +1459,23 @@ rb_tracepoint_disable(VALUE tpval) tp = tpptr(tpval); - if (tp->local_target_set) { - rb_hash_foreach(tp->local_target_set, disable_local_event_iseq_i, tpval); - RB_OBJ_WRITE(tpval, &tp->local_target_set, Qfalse); - ruby_vm_event_local_num--; + if (RTEST(tp->local_target_set)) { + RUBY_ASSERT(GET_RACTOR() == tp->ractor); + RB_VM_LOCKING() { + rb_vm_barrier(); + + rb_hash_foreach(tp->local_target_set, disable_local_tracepoint_i, tpval); + RB_OBJ_WRITE(tpval, &tp->local_target_set, Qfalse); + rb_ractor_targeted_hooks_decr(tp->ractor); + if (tp->events & ISEQ_TRACE_EVENTS) { + RUBY_ASSERT(ruby_vm_iseq_events_enabled > 0); + ruby_vm_iseq_events_enabled--; + } + if ((tp->events & RUBY_EVENT_C_CALL) || (tp->events & RUBY_EVENT_C_RETURN)) { + RUBY_ASSERT(ruby_vm_c_events_enabled > 0); + ruby_vm_c_events_enabled--; + } + } } else { if (tp->target_th) { @@ -1339,26 +1490,30 @@ rb_tracepoint_disable(VALUE tpval) return Qundef; } +// connect a targeted (ie: "local") tracepoint to the hook list for the method +// ex: tp.enable(target: method(:puts)) void -rb_hook_list_connect_tracepoint(VALUE target, rb_hook_list_t *list, VALUE tpval, unsigned int target_line) +rb_hook_list_connect_local_tracepoint(rb_hook_list_t *list, VALUE tpval, unsigned int target_line) { rb_tp_t *tp = tpptr(tpval); rb_event_hook_t *hook = alloc_event_hook((rb_event_hook_func_t)tp_call_trace, tp->events & ISEQ_TRACE_EVENTS, tpval, RUBY_EVENT_HOOK_FLAG_SAFE | RUBY_EVENT_HOOK_FLAG_RAW_ARG); hook->filter.target_line = target_line; - hook_list_connect(target, list, hook, FALSE); + hook_list_connect(list, hook, FALSE); } -void -rb_hook_list_remove_tracepoint(rb_hook_list_t *list, VALUE tpval) +bool +rb_hook_list_remove_local_tracepoint(rb_hook_list_t *list, VALUE tpval) { rb_event_hook_t *hook = list->hooks; rb_event_flag_t events = 0; + bool removed = false; while (hook) { if (hook->data == tpval) { hook->hook_flags |= RUBY_EVENT_HOOK_FLAG_DELETED; list->need_clean = true; + removed = true; } else if ((hook->hook_flags & RUBY_EVENT_HOOK_FLAG_DELETED) == 0) { events |= hook->events; @@ -1367,6 +1522,7 @@ rb_hook_list_remove_tracepoint(rb_hook_list_t *list, VALUE tpval) } list->events = events; + return removed; } static VALUE @@ -1461,8 +1617,8 @@ tracepoint_new(VALUE klass, rb_thread_t *target_th, rb_event_flag_t events, void TypedData_Get_Struct(tpval, rb_tp_t, &tp_data_type, tp); RB_OBJ_WRITE(tpval, &tp->proc, proc); - tp->ractor = rb_ractor_shareable_p(proc) ? NULL : GET_RACTOR(); - tp->func = func; + tp->ractor = GET_RACTOR(); + tp->func = func; // for internal events tp->data = data; tp->events = events; tp->self = tpval; @@ -1477,9 +1633,6 @@ rb_tracepoint_new(VALUE target_thval, rb_event_flag_t events, void (*func)(VALUE if (RTEST(target_thval)) { target_th = rb_thread_ptr(target_thval); - /* TODO: Test it! - * Warning: This function is not tested. - */ } return tracepoint_new(rb_cTracePoint, target_th, events, func, data, Qundef); } @@ -1586,7 +1739,6 @@ tracepoint_stat_s(rb_execution_context_t *ec, VALUE self) VALUE stat = rb_hash_new(); tracepoint_stat_event_hooks(stat, vm->self, rb_ec_ractor_hooks(ec)->hooks); - /* TODO: thread local hooks */ return stat; } @@ -1696,16 +1848,7 @@ typedef struct rb_postponed_job_queue { rb_atomic_t triggered_bitset; } rb_postponed_job_queues_t; -void -rb_vm_postponed_job_queue_init(rb_vm_t *vm) -{ - /* use mimmalloc; postponed job registration is a dependency of objspace, so this gets - * called _VERY_ early inside Init_BareVM */ - rb_postponed_job_queues_t *pjq = ruby_mimmalloc(sizeof(rb_postponed_job_queues_t)); - pjq->triggered_bitset = 0; - memset(pjq->table, 0, sizeof(pjq->table)); - vm->postponed_job_queue = pjq; -} +static rb_postponed_job_queues_t postponed_job_queue; static rb_execution_context_t * get_valid_ec(rb_vm_t *vm) @@ -1718,25 +1861,15 @@ get_valid_ec(rb_vm_t *vm) void rb_vm_postponed_job_atfork(void) { - rb_vm_t *vm = GET_VM(); - rb_postponed_job_queues_t *pjq = vm->postponed_job_queue; + rb_postponed_job_queues_t *pjq = &postponed_job_queue; /* make sure we set the interrupt flag on _this_ thread if we carried any pjobs over * from the other side of the fork */ if (pjq->triggered_bitset) { - RUBY_VM_SET_POSTPONED_JOB_INTERRUPT(get_valid_ec(vm)); + RUBY_VM_SET_POSTPONED_JOB_INTERRUPT(get_valid_ec(GET_VM())); } } -/* Frees the memory managed by the postponed job infrastructure at shutdown */ -void -rb_vm_postponed_job_free(void) -{ - rb_vm_t *vm = GET_VM(); - ruby_xfree(vm->postponed_job_queue); - vm->postponed_job_queue = NULL; -} - // Used for VM memsize reporting. Returns the total size of the postponed job // queue infrastructure. size_t @@ -1758,7 +1891,7 @@ rb_postponed_job_preregister(unsigned int flags, rb_postponed_job_func_t func, v * of concurrent calls to both _preregister and _register functions on the same * func, however, the data may get mixed up between them. */ - rb_postponed_job_queues_t *pjq = GET_VM()->postponed_job_queue; + rb_postponed_job_queues_t *pjq = &postponed_job_queue; for (unsigned int i = 0; i < PJOB_TABLE_SIZE; i++) { /* Try and set this slot to equal `func` */ rb_postponed_job_func_t existing_func = (rb_postponed_job_func_t)(uintptr_t)RUBY_ATOMIC_PTR_CAS(pjq->table[i].func, NULL, (void *)(uintptr_t)func); @@ -1783,44 +1916,16 @@ rb_postponed_job_preregister(unsigned int flags, rb_postponed_job_func_t func, v void rb_postponed_job_trigger(rb_postponed_job_handle_t h) { - rb_vm_t *vm = GET_VM(); - rb_postponed_job_queues_t *pjq = vm->postponed_job_queue; + rb_postponed_job_queues_t *pjq = &postponed_job_queue; RUBY_ATOMIC_OR(pjq->triggered_bitset, (((rb_atomic_t)1UL) << h)); - RUBY_VM_SET_POSTPONED_JOB_INTERRUPT(get_valid_ec(vm)); -} - - -static int -pjob_register_legacy_impl(unsigned int flags, rb_postponed_job_func_t func, void *data) -{ - /* We _know_ calling preregister from a signal handler like this is racy; what is - * and is not promised is very exhaustively documented in debug.h */ - rb_postponed_job_handle_t h = rb_postponed_job_preregister(0, func, data); - if (h == POSTPONED_JOB_HANDLE_INVALID) { - return 0; - } - rb_postponed_job_trigger(h); - return 1; + RUBY_VM_SET_POSTPONED_JOB_INTERRUPT(get_valid_ec(GET_VM())); } -int -rb_postponed_job_register(unsigned int flags, rb_postponed_job_func_t func, void *data) -{ - return pjob_register_legacy_impl(flags, func, data); -} - -int -rb_postponed_job_register_one(unsigned int flags, rb_postponed_job_func_t func, void *data) -{ - return pjob_register_legacy_impl(flags, func, data); -} - - void rb_postponed_job_flush(rb_vm_t *vm) { - rb_postponed_job_queues_t *pjq = GET_VM()->postponed_job_queue; + rb_postponed_job_queues_t *pjq = &postponed_job_queue; rb_execution_context_t *ec = GET_EC(); const rb_atomic_t block_mask = POSTPONED_JOB_INTERRUPT_MASK | TRAP_INTERRUPT_MASK; volatile rb_atomic_t saved_mask = ec->interrupt_mask & block_mask; |