diff options
Diffstat (limited to 'yjit.c')
| -rw-r--r-- | yjit.c | 846 |
1 files changed, 72 insertions, 774 deletions
@@ -21,8 +21,8 @@ #include "builtin.h" #include "insns.inc" #include "insns_info.inc" -#include "vm_sync.h" #include "yjit.h" +#include "zjit.h" #include "vm_insnhelper.h" #include "probes.h" #include "probes_helper.h" @@ -38,17 +38,6 @@ #include <errno.h> -// Field offsets for the RObject struct -enum robject_offsets { - ROBJECT_OFFSET_AS_HEAP_IVPTR = offsetof(struct RObject, as.heap.ivptr), - ROBJECT_OFFSET_AS_ARY = offsetof(struct RObject, as.ary), -}; - -// Field offsets for the RString struct -enum rstring_offsets { - RUBY_OFFSET_RSTRING_LEN = offsetof(struct RString, len) -}; - // We need size_t to have a known size to simplify code generation and FFI. // TODO(alan): check this in configure.ac to fail fast on 32 bit platforms. STATIC_ASSERT(64b_size_t, SIZE_MAX == UINT64_MAX); @@ -75,62 +64,6 @@ STATIC_ASSERT(pointer_tagging_scheme, USE_FLONUM); // The "_yjit_" part is for trying to be informative. We might want different // suffixes for symbols meant for Rust and symbols meant for broader CRuby. -bool -rb_yjit_mark_writable(void *mem_block, uint32_t mem_size) -{ - return mprotect(mem_block, mem_size, PROT_READ | PROT_WRITE) == 0; -} - -void -rb_yjit_mark_executable(void *mem_block, uint32_t mem_size) -{ - // Do not call mprotect when mem_size is zero. Some platforms may return - // an error for it. https://github.com/Shopify/ruby/issues/450 - if (mem_size == 0) { - return; - } - if (mprotect(mem_block, mem_size, PROT_READ | PROT_EXEC)) { - rb_bug("Couldn't make JIT page (%p, %lu bytes) executable, errno: %s", - mem_block, (unsigned long)mem_size, strerror(errno)); - } -} - -// Free the specified memory block. -bool -rb_yjit_mark_unused(void *mem_block, uint32_t mem_size) -{ - // On Linux, you need to use madvise MADV_DONTNEED to free memory. - // We might not need to call this on macOS, but it's not really documented. - // We generally prefer to do the same thing on both to ease testing too. - madvise(mem_block, mem_size, MADV_DONTNEED); - - // On macOS, mprotect PROT_NONE seems to reduce RSS. - // We also call this on Linux to avoid executing unused pages. - return mprotect(mem_block, mem_size, PROT_NONE) == 0; -} - -long -rb_yjit_array_len(VALUE a) -{ - return rb_array_len(a); -} - -// `start` is inclusive and `end` is exclusive. -void -rb_yjit_icache_invalidate(void *start, void *end) -{ - // Clear/invalidate the instruction cache. Compiles to nothing on x86_64 - // but required on ARM before running freshly written code. - // On Darwin it's the same as calling sys_icache_invalidate(). -#ifdef __GNUC__ - __builtin___clear_cache(start, end); -#elif defined(__aarch64__) -#error No instruction cache clear available with this compiler on Aarch64! -#endif -} - -# define PTR2NUM(x) (rb_int2inum((intptr_t)(void *)(x))) - // For a given raw_sample (frame), set the hash with the caller's // name, file, and line number. Return the hash with collected frame_info. static void @@ -223,147 +156,11 @@ rb_yjit_exit_locations_dict(VALUE *yjit_raw_samples, int *yjit_line_samples, int return result; } -uint32_t -rb_yjit_get_page_size(void) -{ -#if defined(_SC_PAGESIZE) - long page_size = sysconf(_SC_PAGESIZE); - if (page_size <= 0) rb_bug("yjit: failed to get page size"); - - // 1 GiB limit. x86 CPUs with PDPE1GB can do this and anything larger is unexpected. - // Though our design sort of assume we have fine grained control over memory protection - // which require small page sizes. - if (page_size > 0x40000000l) rb_bug("yjit page size too large"); - - return (uint32_t)page_size; -#else -#error "YJIT supports POSIX only for now" -#endif -} - -#if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE) -// Align the current write position to a multiple of bytes -static uint8_t * -align_ptr(uint8_t *ptr, uint32_t multiple) -{ - // Compute the pointer modulo the given alignment boundary - uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple; - - // If the pointer is already aligned, stop - if (rem == 0) - return ptr; - - // Pad the pointer by the necessary amount to align it - uint32_t pad = multiple - rem; - - return ptr + pad; -} -#endif - -// Address space reservation. Memory pages are mapped on an as needed basis. -// See the Rust mm module for details. -uint8_t * -rb_yjit_reserve_addr_space(uint32_t mem_size) -{ -#ifndef _WIN32 - uint8_t *mem_block; - - // On Linux - #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE) - uint32_t const page_size = (uint32_t)sysconf(_SC_PAGESIZE); - uint8_t *const cfunc_sample_addr = (void *)(uintptr_t)&rb_yjit_reserve_addr_space; - uint8_t *const probe_region_end = cfunc_sample_addr + INT32_MAX; - // Align the requested address to page size - uint8_t *req_addr = align_ptr(cfunc_sample_addr, page_size); - - // Probe for addresses close to this function using MAP_FIXED_NOREPLACE - // to improve odds of being in range for 32-bit relative call instructions. - do { - mem_block = mmap( - req_addr, - mem_size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, - -1, - 0 - ); - - // If we succeeded, stop - if (mem_block != MAP_FAILED) { - ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_yjit_reserve_addr_space"); - break; - } - - // -4MiB. Downwards to probe away from the heap. (On x86/A64 Linux - // main_code_addr < heap_addr, and in case we are in a shared - // library mapped higher than the heap, downwards is still better - // since it's towards the end of the heap rather than the stack.) - req_addr -= 4 * 1024 * 1024; - } while (req_addr < probe_region_end); - - // On MacOS and other platforms - #else - // Try to map a chunk of memory as executable - mem_block = mmap( - (void *)rb_yjit_reserve_addr_space, - mem_size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, - 0 - ); - #endif - - // Fallback - if (mem_block == MAP_FAILED) { - // Try again without the address hint (e.g., valgrind) - mem_block = mmap( - NULL, - mem_size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, - 0 - ); - - if (mem_block != MAP_FAILED) { - ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_yjit_reserve_addr_space:fallback"); - } - } - - // Check that the memory mapping was successful - if (mem_block == MAP_FAILED) { - perror("ruby: yjit: mmap:"); - if(errno == ENOMEM) { - // No crash report if it's only insufficient memory - exit(EXIT_FAILURE); - } - rb_bug("mmap failed"); - } - - return mem_block; -#else - // Windows not supported for now - return NULL; -#endif -} - // Is anyone listening for :c_call and :c_return event currently? bool rb_c_method_tracing_currently_enabled(const rb_execution_context_t *ec) { - rb_event_flag_t tracing_events; - if (rb_multi_ractor_p()) { - tracing_events = ruby_vm_event_enabled_global_flags; - } - else { - // At the time of writing, events are never removed from - // ruby_vm_event_enabled_global_flags so always checking using it would - // mean we don't compile even after tracing is disabled. - tracing_events = rb_ec_ractor_hooks(ec)->events; - } - - return tracing_events & (RUBY_EVENT_C_CALL | RUBY_EVENT_C_RETURN); + return ruby_vm_c_events_enabled > 0; } // The code we generate in gen_send_cfunc() doesn't fire the c_return TracePoint event @@ -396,12 +193,6 @@ rb_full_cfunc_return(rb_execution_context_t *ec, VALUE return_value) ec->cfp->sp++; } -unsigned int -rb_iseq_encoded_size(const rb_iseq_t *iseq) -{ - return iseq->body->iseq_size; -} - // TODO(alan): consider using an opaque pointer for the payload rather than a void pointer void * rb_iseq_get_yjit_payload(const rb_iseq_t *iseq) @@ -425,339 +216,13 @@ rb_iseq_set_yjit_payload(const rb_iseq_t *iseq, void *payload) iseq->body->yjit_payload = payload; } -void -rb_iseq_reset_jit_func(const rb_iseq_t *iseq) -{ - RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq, imemo_iseq)); - iseq->body->jit_entry = NULL; - iseq->body->jit_exception = NULL; - // Enable re-compiling this ISEQ. Event when it's invalidated for TracePoint, - // we'd like to re-compile ISEQs that haven't been converted to trace_* insns. - iseq->body->jit_entry_calls = 0; - iseq->body->jit_exception_calls = 0; -} - -// Get the PC for a given index in an iseq -VALUE * -rb_iseq_pc_at_idx(const rb_iseq_t *iseq, uint32_t insn_idx) -{ - RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(iseq, imemo_iseq)); - RUBY_ASSERT_ALWAYS(insn_idx < iseq->body->iseq_size); - VALUE *encoded = iseq->body->iseq_encoded; - VALUE *pc = &encoded[insn_idx]; - return pc; -} - -// Get the opcode given a program counter. Can return trace opcode variants. -int -rb_iseq_opcode_at_pc(const rb_iseq_t *iseq, const VALUE *pc) -{ - // YJIT should only use iseqs after AST to bytecode compilation - RUBY_ASSERT_ALWAYS(FL_TEST_RAW((VALUE)iseq, ISEQ_TRANSLATED)); - - const VALUE at_pc = *pc; - return rb_vm_insn_addr2opcode((const void *)at_pc); -} - -unsigned long -rb_RSTRING_LEN(VALUE str) -{ - return RSTRING_LEN(str); -} - -char * -rb_RSTRING_PTR(VALUE str) -{ - return RSTRING_PTR(str); -} - -rb_proc_t * -rb_yjit_get_proc_ptr(VALUE procv) -{ - rb_proc_t *proc; - GetProcPtr(procv, proc); - return proc; -} - // This is defined only as a named struct inside rb_iseq_constant_body. // By giving it a separate typedef, we make it nameable by rust-bindgen. // Bindgen's temp/anon name isn't guaranteed stable. typedef struct rb_iseq_param_keyword rb_seq_param_keyword_struct; -const char * -rb_insn_name(VALUE insn) -{ - return insn_name(insn); -} - -unsigned int -rb_vm_ci_argc(const struct rb_callinfo *ci) -{ - return vm_ci_argc(ci); -} - -ID -rb_vm_ci_mid(const struct rb_callinfo *ci) -{ - return vm_ci_mid(ci); -} - -unsigned int -rb_vm_ci_flag(const struct rb_callinfo *ci) -{ - return vm_ci_flag(ci); -} - -const struct rb_callinfo_kwarg * -rb_vm_ci_kwarg(const struct rb_callinfo *ci) -{ - return vm_ci_kwarg(ci); -} - -int -rb_get_cikw_keyword_len(const struct rb_callinfo_kwarg *cikw) -{ - return cikw->keyword_len; -} - -VALUE -rb_get_cikw_keywords_idx(const struct rb_callinfo_kwarg *cikw, int idx) -{ - return cikw->keywords[idx]; -} - -rb_method_visibility_t -rb_METHOD_ENTRY_VISI(const rb_callable_method_entry_t *me) -{ - return METHOD_ENTRY_VISI(me); -} - -rb_method_type_t -rb_get_cme_def_type(const rb_callable_method_entry_t *cme) -{ - if (UNDEFINED_METHOD_ENTRY_P(cme)) { - return VM_METHOD_TYPE_UNDEF; - } - else { - return cme->def->type; - } -} - -ID -rb_get_cme_def_body_attr_id(const rb_callable_method_entry_t *cme) -{ - return cme->def->body.attr.id; -} - ID rb_get_symbol_id(VALUE namep); -enum method_optimized_type -rb_get_cme_def_body_optimized_type(const rb_callable_method_entry_t *cme) -{ - return cme->def->body.optimized.type; -} - -unsigned int -rb_get_cme_def_body_optimized_index(const rb_callable_method_entry_t *cme) -{ - return cme->def->body.optimized.index; -} - -rb_method_cfunc_t * -rb_get_cme_def_body_cfunc(const rb_callable_method_entry_t *cme) -{ - return UNALIGNED_MEMBER_PTR(cme->def, body.cfunc); -} - -uintptr_t -rb_get_def_method_serial(const rb_method_definition_t *def) -{ - return def->method_serial; -} - -ID -rb_get_def_original_id(const rb_method_definition_t *def) -{ - return def->original_id; -} - -int -rb_get_mct_argc(const rb_method_cfunc_t *mct) -{ - return mct->argc; -} - -void * -rb_get_mct_func(const rb_method_cfunc_t *mct) -{ - return (void*)(uintptr_t)mct->func; // this field is defined as type VALUE (*func)(ANYARGS) -} - -const rb_iseq_t * -rb_get_def_iseq_ptr(rb_method_definition_t *def) -{ - return def_iseq_ptr(def); -} - -VALUE -rb_get_def_bmethod_proc(rb_method_definition_t *def) -{ - RUBY_ASSERT(def->type == VM_METHOD_TYPE_BMETHOD); - return def->body.bmethod.proc; -} - -const rb_iseq_t * -rb_get_iseq_body_local_iseq(const rb_iseq_t *iseq) -{ - return iseq->body->local_iseq; -} - -const rb_iseq_t * -rb_get_iseq_body_parent_iseq(const rb_iseq_t *iseq) -{ - return iseq->body->parent_iseq; -} - -unsigned int -rb_get_iseq_body_local_table_size(const rb_iseq_t *iseq) -{ - return iseq->body->local_table_size; -} - -VALUE * -rb_get_iseq_body_iseq_encoded(const rb_iseq_t *iseq) -{ - return iseq->body->iseq_encoded; -} - -unsigned -rb_get_iseq_body_stack_max(const rb_iseq_t *iseq) -{ - return iseq->body->stack_max; -} - -enum rb_iseq_type -rb_get_iseq_body_type(const rb_iseq_t *iseq) -{ - return iseq->body->type; -} - -bool -rb_get_iseq_flags_has_lead(const rb_iseq_t *iseq) -{ - return iseq->body->param.flags.has_lead; -} - -bool -rb_get_iseq_flags_has_opt(const rb_iseq_t *iseq) -{ - return iseq->body->param.flags.has_opt; -} - -bool -rb_get_iseq_flags_has_kw(const rb_iseq_t *iseq) -{ - return iseq->body->param.flags.has_kw; -} - -bool -rb_get_iseq_flags_has_post(const rb_iseq_t *iseq) -{ - return iseq->body->param.flags.has_post; -} - -bool -rb_get_iseq_flags_has_kwrest(const rb_iseq_t *iseq) -{ - return iseq->body->param.flags.has_kwrest; -} - -bool -rb_get_iseq_flags_anon_kwrest(const rb_iseq_t *iseq) -{ - return iseq->body->param.flags.anon_kwrest; -} - -bool -rb_get_iseq_flags_has_rest(const rb_iseq_t *iseq) -{ - return iseq->body->param.flags.has_rest; -} - -bool -rb_get_iseq_flags_ruby2_keywords(const rb_iseq_t *iseq) -{ - return iseq->body->param.flags.ruby2_keywords; -} - -bool -rb_get_iseq_flags_has_block(const rb_iseq_t *iseq) -{ - return iseq->body->param.flags.has_block; -} - -bool -rb_get_iseq_flags_ambiguous_param0(const rb_iseq_t *iseq) -{ - return iseq->body->param.flags.ambiguous_param0; -} - -bool -rb_get_iseq_flags_accepts_no_kwarg(const rb_iseq_t *iseq) -{ - return iseq->body->param.flags.accepts_no_kwarg; -} - -bool -rb_get_iseq_flags_forwardable(const rb_iseq_t *iseq) -{ - return iseq->body->param.flags.forwardable; -} - -const rb_seq_param_keyword_struct * -rb_get_iseq_body_param_keyword(const rb_iseq_t *iseq) -{ - return iseq->body->param.keyword; -} - -unsigned -rb_get_iseq_body_param_size(const rb_iseq_t *iseq) -{ - return iseq->body->param.size; -} - -int -rb_get_iseq_body_param_lead_num(const rb_iseq_t *iseq) -{ - return iseq->body->param.lead_num; -} - -int -rb_get_iseq_body_param_opt_num(const rb_iseq_t *iseq) -{ - return iseq->body->param.opt_num; -} - -const VALUE * -rb_get_iseq_body_param_opt_table(const rb_iseq_t *iseq) -{ - return iseq->body->param.opt_table; -} - -VALUE -rb_optimized_call(VALUE *recv, rb_execution_context_t *ec, int argc, VALUE *argv, int kw_splat, VALUE block_handler) -{ - rb_proc_t *proc; - GetProcPtr(recv, proc); - return rb_vm_invoke_proc(ec, proc, argc, argv, kw_splat, block_handler); -} - -unsigned int -rb_yjit_iseq_builtin_attrs(const rb_iseq_t *iseq) -{ - return iseq->body->builtin_attrs; -} - // If true, the iseq has only opt_invokebuiltin_delegate(_leave) and leave insns. static bool invokebuiltin_delegate_leave_p(const rb_iseq_t *iseq) @@ -789,94 +254,14 @@ rb_yjit_str_simple_append(VALUE str1, VALUE str2) return rb_str_cat(str1, RSTRING_PTR(str2), RSTRING_LEN(str2)); } -struct rb_control_frame_struct * -rb_get_ec_cfp(const rb_execution_context_t *ec) -{ - return ec->cfp; -} - -const rb_iseq_t * -rb_get_cfp_iseq(struct rb_control_frame_struct *cfp) -{ - return cfp->iseq; -} - -VALUE * -rb_get_cfp_pc(struct rb_control_frame_struct *cfp) -{ - return (VALUE*)cfp->pc; -} - -VALUE * -rb_get_cfp_sp(struct rb_control_frame_struct *cfp) -{ - return cfp->sp; -} - -void -rb_set_cfp_pc(struct rb_control_frame_struct *cfp, const VALUE *pc) -{ - cfp->pc = pc; -} - -void -rb_set_cfp_sp(struct rb_control_frame_struct *cfp, VALUE *sp) -{ - cfp->sp = sp; -} - -VALUE -rb_get_cfp_self(struct rb_control_frame_struct *cfp) -{ - return cfp->self; -} - -VALUE * -rb_get_cfp_ep(struct rb_control_frame_struct *cfp) -{ - return (VALUE*)cfp->ep; -} - -const VALUE * -rb_get_cfp_ep_level(struct rb_control_frame_struct *cfp, uint32_t lv) -{ - uint32_t i; - const VALUE *ep = (VALUE*)cfp->ep; - for (i = 0; i < lv; i++) { - ep = VM_ENV_PREV_EP(ep); - } - return ep; -} - extern VALUE *rb_vm_base_ptr(struct rb_control_frame_struct *cfp); VALUE -rb_yarv_class_of(VALUE obj) -{ - return rb_class_of(obj); -} - -// YJIT needs this function to never allocate and never raise -VALUE -rb_yarv_str_eql_internal(VALUE str1, VALUE str2) -{ - // We wrap this since it's static inline - return rb_str_eql_internal(str1, str2); -} - -VALUE rb_str_neq_internal(VALUE str1, VALUE str2) { return rb_str_eql_internal(str1, str2) == Qtrue ? Qfalse : Qtrue; } -// YJIT needs this function to never allocate and never raise -VALUE -rb_yarv_ary_entry_internal(VALUE ary, long offset) -{ - return rb_ary_entry_internal(ary, offset); -} - extern VALUE rb_ary_unshift_m(int argc, VALUE *argv, VALUE ary); VALUE @@ -886,18 +271,6 @@ rb_yjit_rb_ary_subseq_length(VALUE ary, long beg) return rb_ary_subseq(ary, beg, len); } -VALUE -rb_yjit_fix_div_fix(VALUE recv, VALUE obj) -{ - return rb_fix_div_fix(recv, obj); -} - -VALUE -rb_yjit_fix_mod_fix(VALUE recv, VALUE obj) -{ - return rb_fix_mod_fix(recv, obj); -} - // Return non-zero when `obj` is an array and its last item is a // `ruby2_keywords` hash. We don't support this kind of splat. size_t @@ -984,38 +357,12 @@ rb_yjit_iseq_inspect(const rb_iseq_t *iseq) const char *path = RSTRING_PTR(rb_iseq_path(iseq)); int lineno = iseq->body->location.code_location.beg_pos.lineno; - char *buf = ZALLOC_N(char, strlen(label) + strlen(path) + num_digits(lineno) + 3); - sprintf(buf, "%s@%s:%d", label, path, lineno); + const size_t size = strlen(label) + strlen(path) + num_digits(lineno) + 3; + char *buf = ZALLOC_N(char, size); + snprintf(buf, size, "%s@%s:%d", label, path, lineno); return buf; } -// The FL_TEST() macro -VALUE -rb_FL_TEST(VALUE obj, VALUE flags) -{ - return RB_FL_TEST(obj, flags); -} - -// The FL_TEST_RAW() macro, normally an internal implementation detail -VALUE -rb_FL_TEST_RAW(VALUE obj, VALUE flags) -{ - return FL_TEST_RAW(obj, flags); -} - -// The RB_TYPE_P macro -bool -rb_RB_TYPE_P(VALUE obj, enum ruby_value_type t) -{ - return RB_TYPE_P(obj, t); -} - -long -rb_RSTRUCT_LEN(VALUE st) -{ - return RSTRUCT_LEN(st); -} - // There are RSTRUCT_SETs in ruby/internal/core/rstruct.h and internal/struct.h // with different types (int vs long) for k. Here we use the one from ruby/internal/core/rstruct.h, // which takes an int. @@ -1025,24 +372,6 @@ rb_RSTRUCT_SET(VALUE st, int k, VALUE v) RSTRUCT_SET(st, k, v); } -const struct rb_callinfo * -rb_get_call_data_ci(const struct rb_call_data *cd) -{ - return cd->ci; -} - -bool -rb_BASIC_OP_UNREDEFINED_P(enum ruby_basic_operators bop, uint32_t klass) -{ - return BASIC_OP_UNREDEFINED_P(bop, klass); -} - -VALUE -rb_RCLASS_ORIGIN(VALUE c) -{ - return RCLASS_ORIGIN(c); -} - // Return the string encoding index int rb_ENCODING_GET(VALUE obj) @@ -1051,72 +380,11 @@ rb_ENCODING_GET(VALUE obj) } bool -rb_yjit_multi_ractor_p(void) -{ - return rb_multi_ractor_p(); -} - -// For debug builds -void -rb_assert_iseq_handle(VALUE handle) -{ - RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(handle, imemo_iseq)); -} - -int -rb_IMEMO_TYPE_P(VALUE imemo, enum imemo_type imemo_type) -{ - return IMEMO_TYPE_P(imemo, imemo_type); -} - -bool rb_yjit_constcache_shareable(const struct iseq_inline_constant_cache_entry *ice) { return (ice->flags & IMEMO_CONST_CACHE_SHAREABLE) != 0; } -void -rb_assert_cme_handle(VALUE handle) -{ - RUBY_ASSERT_ALWAYS(!rb_objspace_garbage_object_p(handle)); - RUBY_ASSERT_ALWAYS(IMEMO_TYPE_P(handle, imemo_ment)); -} - -// Used for passing a callback and other data over rb_objspace_each_objects -struct iseq_callback_data { - rb_iseq_callback callback; - void *data; -}; - -// Heap-walking callback for rb_yjit_for_each_iseq(). -static int -for_each_iseq_i(void *vstart, void *vend, size_t stride, void *data) -{ - const struct iseq_callback_data *callback_data = (struct iseq_callback_data *)data; - VALUE v = (VALUE)vstart; - for (; v != (VALUE)vend; v += stride) { - void *ptr = rb_asan_poisoned_object_p(v); - rb_asan_unpoison_object(v, false); - - if (rb_obj_is_iseq(v)) { - rb_iseq_t *iseq = (rb_iseq_t *)v; - callback_data->callback(iseq, callback_data->data); - } - - asan_poison_object_if(ptr, v); - } - return 0; -} - -// Iterate through the whole GC heap and invoke a callback for each iseq. -// Used for global code invalidation. -void -rb_yjit_for_each_iseq(rb_iseq_callback callback, void *data) -{ - struct iseq_callback_data callback_data = { .callback = callback, .data = data }; - rb_objspace_each_objects(for_each_iseq_i, (void *)&callback_data); -} - // For running write barriers from Rust. Required when we add a new edge in the // object graph from `old` to `young`. void @@ -1125,43 +393,23 @@ rb_yjit_obj_written(VALUE old, VALUE young, const char *file, int line) rb_obj_written(old, Qundef, young, file, line); } -// Acquire the VM lock and then signal all other Ruby threads (ractors) to -// contend for the VM lock, putting them to sleep. YJIT uses this to evict -// threads running inside generated code so among other things, it can -// safely change memory protection of regions housing generated code. -void -rb_yjit_vm_lock_then_barrier(unsigned int *recursive_lock_level, const char *file, int line) -{ - rb_vm_lock_enter(recursive_lock_level, file, line); - rb_vm_barrier(); -} - -// Release the VM lock. The lock level must point to the same integer used to -// acquire the lock. -void -rb_yjit_vm_unlock(unsigned int *recursive_lock_level, const char *file, int line) -{ - rb_vm_lock_leave(recursive_lock_level, file, line); -} - void rb_yjit_compile_iseq(const rb_iseq_t *iseq, rb_execution_context_t *ec, bool jit_exception) { - RB_VM_LOCK_ENTER(); - rb_vm_barrier(); + RB_VM_LOCKING() { + rb_vm_barrier(); - // Compile a block version starting at the current instruction - uint8_t *rb_yjit_iseq_gen_entry_point(const rb_iseq_t *iseq, rb_execution_context_t *ec, bool jit_exception); // defined in Rust - uintptr_t code_ptr = (uintptr_t)rb_yjit_iseq_gen_entry_point(iseq, ec, jit_exception); + // Compile a block version starting at the current instruction + uint8_t *rb_yjit_iseq_gen_entry_point(const rb_iseq_t *iseq, rb_execution_context_t *ec, bool jit_exception); // defined in Rust + uintptr_t code_ptr = (uintptr_t)rb_yjit_iseq_gen_entry_point(iseq, ec, jit_exception); - if (jit_exception) { - iseq->body->jit_exception = (rb_jit_func_t)code_ptr; - } - else { - iseq->body->jit_entry = (rb_jit_func_t)code_ptr; + if (jit_exception) { + iseq->body->jit_exception = (rb_jit_func_t)code_ptr; + } + else { + iseq->body->jit_entry = (rb_jit_func_t)code_ptr; + } } - - RB_VM_LOCK_LEAVE(); } // GC root for interacting with the GC @@ -1182,15 +430,25 @@ VALUE rb_object_shape_count(void) { // next_shape_id starts from 0, so it's the same as the count - return ULONG2NUM((unsigned long)GET_SHAPE_TREE()->next_shape_id); + return ULONG2NUM((unsigned long)rb_shapes_count()); } -// Assert that we have the VM lock. Relevant mostly for multi ractor situations. -// The GC takes the lock before calling us, and this asserts that it indeed happens. -void -rb_yjit_assert_holding_vm_lock(void) +bool +rb_yjit_shape_obj_complex_p(VALUE obj) +{ + return rb_obj_shape_complex_p(obj); +} + +attr_index_t +rb_yjit_shape_capacity(shape_id_t shape_id) { - ASSERT_vm_locking(); + return RSHAPE_CAPACITY(shape_id); +} + +attr_index_t +rb_yjit_shape_index(shape_id_t shape_id) +{ + return RSHAPE_INDEX(shape_id); } // The number of stack slots that vm_sendish() pops for send and invokesuper. @@ -1207,6 +465,12 @@ rb_yjit_invokeblock_sp_pops(const struct rb_callinfo *ci) return 1 - sp_inc_of_invokeblock(ci); // + 1 to ignore return value push } +rb_serial_t +rb_yjit_cme_ractor_serial(const rb_callable_method_entry_t *cme) +{ + return cme->def->body.bmethod.defined_ractor_id; +} + // Setup jit_return to avoid returning a non-Qundef value on a non-FINISH frame. // See [jit_compile_exception] for details. void @@ -1236,6 +500,39 @@ rb_yjit_set_exception_return(rb_control_frame_t *cfp, void *leave_exit, void *le } } +// VM_INSTRUCTION_SIZE changes depending on if ZJIT is in the build. Since +// bindgen can only grab one version of the constant and copy that to rust, +// we make that the upper bound and this the accurate value. +uint32_t +rb_vm_instruction_size(void) +{ + return VM_INSTRUCTION_SIZE; +} + +static int +yjit_cdhash_all_fixnum_i(st_data_t key, st_data_t _val, st_data_t data) +{ + if (!FIXNUM_P((VALUE)key)) { + *((bool *)data) = false; + return ST_STOP; + } + return ST_CONTINUE; +} + +bool +rb_yjit_cdhash_all_fixnum_p(VALUE cdhash) +{ + bool all_fixnum = true; + st_foreach(rb_imemo_cdhash_tbl(cdhash), yjit_cdhash_all_fixnum_i, (st_data_t)&all_fixnum); + return all_fixnum; +} + +int +rb_yjit_cdhash_lookup(VALUE cdhash, st_data_t key, st_data_t *val) +{ + return st_lookup(rb_imemo_cdhash_tbl(cdhash), key, val); +} + // Primitives used by yjit.rb VALUE rb_yjit_stats_enabled_p(rb_execution_context_t *ec, VALUE self); VALUE rb_yjit_print_stats_p(rb_execution_context_t *ec, VALUE self); @@ -1262,3 +559,4 @@ static VALUE yjit_c_builtin_p(rb_execution_context_t *ec, VALUE self) { return Q // Preprocessed yjit.rb generated during build #include "yjit.rbinc" + |