diff options
Diffstat (limited to 'zjit.c')
| -rw-r--r-- | zjit.c | 330 |
1 files changed, 110 insertions, 220 deletions
@@ -16,146 +16,30 @@ #include "insns.inc" #include "insns_info.inc" #include "zjit.h" -#include "vm_sync.h" #include "vm_insnhelper.h" #include "probes.h" #include "probes_helper.h" +#include "constant.h" #include "iseq.h" #include "ruby/debug.h" #include "internal/cont.h" -// For mmapp(), sysconf() -#ifndef _WIN32 -#include <unistd.h> -#include <sys/mman.h> -#endif - -#include <errno.h> - -uint32_t -rb_zjit_get_page_size(void) -{ -#if defined(_SC_PAGESIZE) - long page_size = sysconf(_SC_PAGESIZE); - if (page_size <= 0) rb_bug("zjit: failed to get page size"); - - // 1 GiB limit. x86 CPUs with PDPE1GB can do this and anything larger is unexpected. - // Though our design sort of assume we have fine grained control over memory protection - // which require small page sizes. - if (page_size > 0x40000000l) rb_bug("zjit page size too large"); - - return (uint32_t)page_size; -#else -#error "ZJIT supports POSIX only for now" -#endif -} - -#if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE) -// Align the current write position to a multiple of bytes -static uint8_t * -align_ptr(uint8_t *ptr, uint32_t multiple) -{ - // Compute the pointer modulo the given alignment boundary - uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple; - - // If the pointer is already aligned, stop - if (rem == 0) - return ptr; - - // Pad the pointer by the necessary amount to align it - uint32_t pad = multiple - rem; - - return ptr + pad; -} -#endif +// This build config impacts the pointer tagging scheme and we only want to +// support one scheme for simplicity. +STATIC_ASSERT(pointer_tagging_scheme, USE_FLONUM); -// Address space reservation. Memory pages are mapped on an as needed basis. -// See the Rust mm module for details. -uint8_t * -rb_zjit_reserve_addr_space(uint32_t mem_size) -{ -#ifndef _WIN32 - uint8_t *mem_block; - - // On Linux - #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE) - uint32_t const page_size = (uint32_t)sysconf(_SC_PAGESIZE); - uint8_t *const cfunc_sample_addr = (void *)(uintptr_t)&rb_zjit_reserve_addr_space; - uint8_t *const probe_region_end = cfunc_sample_addr + INT32_MAX; - // Align the requested address to page size - uint8_t *req_addr = align_ptr(cfunc_sample_addr, page_size); - - // Probe for addresses close to this function using MAP_FIXED_NOREPLACE - // to improve odds of being in range for 32-bit relative call instructions. - do { - mem_block = mmap( - req_addr, - mem_size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, - -1, - 0 - ); - - // If we succeeded, stop - if (mem_block != MAP_FAILED) { - ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_zjit_reserve_addr_space"); - break; - } - - // -4MiB. Downwards to probe away from the heap. (On x86/A64 Linux - // main_code_addr < heap_addr, and in case we are in a shared - // library mapped higher than the heap, downwards is still better - // since it's towards the end of the heap rather than the stack.) - req_addr -= 4 * 1024 * 1024; - } while (req_addr < probe_region_end); - - // On MacOS and other platforms - #else - // Try to map a chunk of memory as executable - mem_block = mmap( - (void *)rb_zjit_reserve_addr_space, - mem_size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, - 0 - ); - #endif - - // Fallback - if (mem_block == MAP_FAILED) { - // Try again without the address hint (e.g., valgrind) - mem_block = mmap( - NULL, - mem_size, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, - 0 - ); - - if (mem_block != MAP_FAILED) { - ruby_annotate_mmap(mem_block, mem_size, "Ruby:rb_zjit_reserve_addr_space:fallback"); - } - } - - // Check that the memory mapping was successful - if (mem_block == MAP_FAILED) { - perror("ruby: zjit: mmap:"); - if(errno == ENOMEM) { - // No crash report if it's only insufficient memory - exit(EXIT_FAILURE); - } - rb_bug("mmap failed"); - } +enum zjit_struct_offsets { + ISEQ_BODY_OFFSET_PARAM = offsetof(struct rb_iseq_constant_body, param) +}; - return mem_block; -#else - // Windows not supported for now - return NULL; -#endif -} +// Special JITFrame used by all C method calls. We don't control the native +// stack layout for C frames, so cfp->jit_return points at this static frame +// via the ZJIT_JIT_RETURN_C_FRAME sentinel instead of a per-call allocation. +const zjit_jit_frame_t rb_zjit_c_frame = (zjit_jit_frame_t) { + .pc = 0, + .iseq = 0, + .materialize_block_code = false, +}; void rb_zjit_profile_disable(const rb_iseq_t *iseq); @@ -165,100 +49,27 @@ rb_zjit_compile_iseq(const rb_iseq_t *iseq, rb_execution_context_t *ec, bool jit RB_VM_LOCKING() { rb_vm_barrier(); - // Convert ZJIT instructions back to bare instructions - rb_zjit_profile_disable(iseq); - // Compile a block version starting at the current instruction - uint8_t *rb_zjit_iseq_gen_entry_point(const rb_iseq_t *iseq, rb_execution_context_t *ec); // defined in Rust - uintptr_t code_ptr = (uintptr_t)rb_zjit_iseq_gen_entry_point(iseq, ec); + uint8_t *rb_zjit_iseq_gen_entry_point(const rb_iseq_t *iseq, rb_execution_context_t *ec, bool jit_exception); // defined in Rust + uintptr_t code_ptr = (uintptr_t)rb_zjit_iseq_gen_entry_point(iseq, ec, jit_exception); - // TODO: support jit_exception - iseq->body->jit_entry = (rb_jit_func_t)code_ptr; -} + if (jit_exception) { + iseq->body->jit_exception = (rb_jit_func_t)code_ptr; + } + else { + iseq->body->jit_entry = (rb_jit_func_t)code_ptr; + } + } } extern VALUE *rb_vm_base_ptr(struct rb_control_frame_struct *cfp); bool -rb_zjit_multi_ractor_p(void) -{ - return rb_multi_ractor_p(); -} - -bool rb_zjit_constcache_shareable(const struct iseq_inline_constant_cache_entry *ice) { return (ice->flags & IMEMO_CONST_CACHE_SHAREABLE) != 0; } -// Release the VM lock. The lock level must point to the same integer used to -// acquire the lock. -void -rb_zjit_vm_unlock(unsigned int *recursive_lock_level, const char *file, int line) -{ - rb_vm_lock_leave(recursive_lock_level, file, line); -} - -bool -rb_zjit_mark_writable(void *mem_block, uint32_t mem_size) -{ - return mprotect(mem_block, mem_size, PROT_READ | PROT_WRITE) == 0; -} - -void -rb_zjit_mark_executable(void *mem_block, uint32_t mem_size) -{ - // Do not call mprotect when mem_size is zero. Some platforms may return - // an error for it. https://github.com/Shopify/ruby/issues/450 - if (mem_size == 0) { - return; - } - if (mprotect(mem_block, mem_size, PROT_READ | PROT_EXEC)) { - rb_bug("Couldn't make JIT page (%p, %lu bytes) executable, errno: %s", - mem_block, (unsigned long)mem_size, strerror(errno)); - } -} - -// Free the specified memory block. -bool -rb_zjit_mark_unused(void *mem_block, uint32_t mem_size) -{ - // On Linux, you need to use madvise MADV_DONTNEED to free memory. - // We might not need to call this on macOS, but it's not really documented. - // We generally prefer to do the same thing on both to ease testing too. - madvise(mem_block, mem_size, MADV_DONTNEED); - - // On macOS, mprotect PROT_NONE seems to reduce RSS. - // We also call this on Linux to avoid executing unused pages. - return mprotect(mem_block, mem_size, PROT_NONE) == 0; -} - -// Invalidate icache for arm64. -// `start` is inclusive and `end` is exclusive. -void -rb_zjit_icache_invalidate(void *start, void *end) -{ - // Clear/invalidate the instruction cache. Compiles to nothing on x86_64 - // but required on ARM before running freshly written code. - // On Darwin it's the same as calling sys_icache_invalidate(). -#ifdef __GNUC__ - __builtin___clear_cache(start, end); -#elif defined(__aarch64__) -#error No instruction cache clear available with this compiler on Aarch64! -#endif -} - -// Acquire the VM lock and then signal all other Ruby threads (ractors) to -// contend for the VM lock, putting them to sleep. ZJIT uses this to evict -// threads running inside generated code so among other things, it can -// safely change memory protection of regions housing generated code. -void -rb_zjit_vm_lock_then_barrier(unsigned int *recursive_lock_level, const char *file, int line) -{ - rb_vm_lock_enter(recursive_lock_level, file, line); - rb_vm_barrier(); -} - // Convert a given ISEQ's instructions to zjit_* instructions void rb_zjit_profile_enable(const rb_iseq_t *iseq) @@ -341,25 +152,104 @@ rb_zjit_print_exception(void) } bool -rb_zjit_shape_obj_too_complex_p(VALUE obj) +rb_zjit_singleton_class_p(VALUE klass) { - return rb_shape_obj_too_complex_p(obj); + return RCLASS_SINGLETON_P(klass); } -enum { - RB_INVALID_SHAPE_ID = INVALID_SHAPE_ID, -}; +VALUE +rb_zjit_defined_ivar(VALUE obj, ID id, VALUE pushval) +{ + VALUE result = rb_ivar_defined(obj, id); + return result ? pushval : Qnil; +} bool -rb_zjit_singleton_class_p(VALUE klass) +rb_zjit_method_tracing_currently_enabled(void) { - return RCLASS_SINGLETON_P(klass); + rb_event_flag_t tracing_events; + if (rb_multi_ractor_p()) { + tracing_events = ruby_vm_event_enabled_global_flags; + } + else { + // At the time of writing, events are never removed from + // ruby_vm_event_enabled_global_flags so always checking using it would + // mean we don't compile even after tracing is disabled. + tracing_events = rb_ec_ractor_hooks(GET_EC())->events; + } + + return tracing_events & (RUBY_EVENT_C_CALL | RUBY_EVENT_C_RETURN); } +// Check if any ISEQ trace events are currently enabled. +// Used to prevent ZJIT from compiling while tracing is active, since ZJIT's +// send fallback (rb_vm_opt_send_without_block) uses VM_EXEC which sets +// VM_FRAME_FLAG_FINISH on the callee frame, changing exception handling +// semantics for throw TAG_RETURN (e.g. return from rescue). +bool +rb_zjit_iseq_tracing_currently_enabled(void) +{ + rb_event_flag_t tracing_events; + if (rb_multi_ractor_p()) { + tracing_events = ruby_vm_event_enabled_global_flags; + } + else { + tracing_events = rb_ec_ractor_hooks(GET_EC())->events; + } + + return tracing_events & ISEQ_TRACE_EVENTS; +} + +bool +rb_zjit_insn_leaf(int insn, const VALUE *opes) +{ + return insn_leaf(insn, opes); +} + +ID +rb_zjit_local_id(const rb_iseq_t *iseq, unsigned idx) +{ + return ISEQ_BODY(iseq)->local_table[idx]; +} + +bool rb_zjit_cme_is_cfunc(const rb_callable_method_entry_t *me, const void *func); + +const struct rb_callable_method_entry_struct * +rb_zjit_vm_search_method(VALUE cd_owner, struct rb_call_data *cd, VALUE recv); + +bool +rb_zjit_class_initialized_p(VALUE klass) +{ + return RCLASS_INITIALIZED_P(klass); +} + +rb_alloc_func_t rb_zjit_class_get_alloc_func(VALUE klass); + +VALUE rb_class_allocate_instance(VALUE klass); + +bool +rb_zjit_class_has_default_allocator(VALUE klass) +{ + assert(RCLASS_INITIALIZED_P(klass)); + assert(!RCLASS_SINGLETON_P(klass)); + rb_alloc_func_t alloc = rb_zjit_class_get_alloc_func(klass); + return alloc == rb_class_allocate_instance; +} + + +VALUE rb_vm_untag_block_handler(VALUE block_handler); +VALUE rb_vm_get_untagged_block_handler(rb_control_frame_t *reg_cfp); + // Primitives used by zjit.rb. Don't put other functions below, which wouldn't use them. +VALUE rb_zjit_enable(rb_execution_context_t *ec, VALUE self); VALUE rb_zjit_assert_compiles(rb_execution_context_t *ec, VALUE self); VALUE rb_zjit_stats(rb_execution_context_t *ec, VALUE self, VALUE target_key); +VALUE rb_zjit_reset_stats_bang(rb_execution_context_t *ec, VALUE self); VALUE rb_zjit_stats_enabled_p(rb_execution_context_t *ec, VALUE self); +VALUE rb_zjit_print_stats_p(rb_execution_context_t *ec, VALUE self); +VALUE rb_zjit_get_stats_file_path_p(rb_execution_context_t *ec, VALUE self); +VALUE rb_zjit_trace_exit_locations_enabled_p(rb_execution_context_t *ec, VALUE self); +VALUE rb_zjit_get_exit_locations(rb_execution_context_t *ec, VALUE self); // Preprocessed zjit.rb generated during build #include "zjit.rbinc" |