diff options
| -rw-r--r-- | depend | 6 | ||||
| -rw-r--r-- | eval_intern.h | 14 | ||||
| -rw-r--r-- | tool/lib/bundled_gem.rb | 22 | ||||
| -rw-r--r-- | vm.c | 31 | ||||
| -rw-r--r-- | vm_core.h | 4 | ||||
| -rw-r--r-- | vm_exec.h | 2 | ||||
| -rw-r--r-- | zjit.h | 21 | ||||
| -rw-r--r-- | zjit/src/backend/lir.rs | 19 | ||||
| -rw-r--r-- | zjit/src/codegen.rs | 32 |
9 files changed, 103 insertions, 48 deletions
@@ -2839,6 +2839,7 @@ debug.$(OBJEXT): {$(VPATH)}vm_core.h debug.$(OBJEXT): {$(VPATH)}vm_debug.h debug.$(OBJEXT): {$(VPATH)}vm_opts.h debug.$(OBJEXT): {$(VPATH)}vm_sync.h +debug.$(OBJEXT): {$(VPATH)}zjit.h debug_counter.$(OBJEXT): $(hdrdir)/ruby/ruby.h debug_counter.$(OBJEXT): {$(VPATH)}assert.h debug_counter.$(OBJEXT): {$(VPATH)}backward/2/assume.h @@ -8352,6 +8353,7 @@ load.$(OBJEXT): {$(VPATH)}util.h load.$(OBJEXT): {$(VPATH)}vm_core.h load.$(OBJEXT): {$(VPATH)}vm_debug.h load.$(OBJEXT): {$(VPATH)}vm_opts.h +load.$(OBJEXT): {$(VPATH)}zjit.h loadpath.$(OBJEXT): $(hdrdir)/ruby/ruby.h loadpath.$(OBJEXT): $(hdrdir)/ruby/version.h loadpath.$(OBJEXT): $(top_srcdir)/version.h @@ -13275,6 +13277,7 @@ proc.$(OBJEXT): {$(VPATH)}vm_debug.h proc.$(OBJEXT): {$(VPATH)}vm_opts.h proc.$(OBJEXT): {$(VPATH)}vm_sync.h proc.$(OBJEXT): {$(VPATH)}yjit.h +proc.$(OBJEXT): {$(VPATH)}zjit.h process.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h process.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h process.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -15864,6 +15867,7 @@ ruby.$(OBJEXT): {$(VPATH)}util.h ruby.$(OBJEXT): {$(VPATH)}vm_core.h ruby.$(OBJEXT): {$(VPATH)}vm_opts.h ruby.$(OBJEXT): {$(VPATH)}yjit.h +ruby.$(OBJEXT): {$(VPATH)}zjit.h ruby_parser.$(OBJEXT): $(hdrdir)/ruby/ruby.h ruby_parser.$(OBJEXT): $(top_srcdir)/internal/array.h ruby_parser.$(OBJEXT): $(top_srcdir)/internal/basic_operators.h @@ -16271,6 +16275,7 @@ scheduler.$(OBJEXT): {$(VPATH)}thread_$(THREAD_MODEL).h scheduler.$(OBJEXT): {$(VPATH)}thread_native.h scheduler.$(OBJEXT): {$(VPATH)}vm_core.h scheduler.$(OBJEXT): {$(VPATH)}vm_opts.h +scheduler.$(OBJEXT): {$(VPATH)}zjit.h set.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h set.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h set.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -17075,6 +17080,7 @@ signal.$(OBJEXT): {$(VPATH)}thread_native.h signal.$(OBJEXT): {$(VPATH)}vm_core.h signal.$(OBJEXT): {$(VPATH)}vm_debug.h signal.$(OBJEXT): {$(VPATH)}vm_opts.h +signal.$(OBJEXT): {$(VPATH)}zjit.h sprintf.$(OBJEXT): $(hdrdir)/ruby/ruby.h sprintf.$(OBJEXT): $(hdrdir)/ruby/version.h sprintf.$(OBJEXT): $(top_srcdir)/internal/basic_operators.h diff --git a/eval_intern.h b/eval_intern.h index 2090fbfcab..954ba6a184 100644 --- a/eval_intern.h +++ b/eval_intern.h @@ -3,6 +3,7 @@ #include "ruby/ruby.h" #include "vm_core.h" +#include "zjit.h" static inline void vm_passed_block_handler_set(rb_execution_context_t *ec, VALUE block_handler) @@ -102,8 +103,18 @@ extern int select_large_fdset(int, fd_set *, fd_set *, fd_set *, struct timeval _tag.tag = Qundef; \ _tag.prev = _ec->tag; \ _tag.lock_rec = rb_ec_vm_lock_rec(_ec); \ + EC_SAVE_TAG_CFP(_tag, _ec); \ rb_vm_tag_jmpbuf_init(&_tag.buf); \ +// Remember the CFP as of EC_PUSH_TAG so that ZJIT can materialize frames +// only up to longjmp's target CFP. When a C method does longjmp inside it, +// the target CFP may not be equal to the VM_FRAME_FLAG_FINISH frame. +#if USE_ZJIT +# define EC_SAVE_TAG_CFP(_tag, _ec) _tag.cfp = _ec->cfp +#else +# define EC_SAVE_TAG_CFP(_tag, _ec) +#endif + #define EC_POP_TAG() \ _ec->tag = _tag.prev; \ rb_vm_tag_jmpbuf_deinit(&_tag.buf); \ @@ -155,6 +166,9 @@ static inline void rb_ec_tag_jump(const rb_execution_context_t *ec, enum ruby_tag_type st) { RUBY_ASSERT(st > TAG_NONE && st <= TAG_FATAL, ": Invalid tag jump: %d", (int)st); +#if USE_ZJIT + rb_zjit_materialize_frames(ec, ec->cfp); +#endif ec->tag->state = st; ruby_longjmp(RB_VM_TAG_JMPBUF_GET(ec->tag->buf), 1); } diff --git a/tool/lib/bundled_gem.rb b/tool/lib/bundled_gem.rb index ca9b40875c..ad103825bc 100644 --- a/tool/lib/bundled_gem.rb +++ b/tool/lib/bundled_gem.rb @@ -131,10 +131,26 @@ module BundledGem system(command, chdir: gemdir) or raise "failed: #{command}" end + class GemspecLoader + module NoPipe + refine IO.singleton_class do + def popen(...) ""; end + end + end + using NoPipe + + def `(command) ""; end + + def load_gemspec(file) + code = File.read(file, encoding: "utf-8:-") + eval(code, binding, file) + rescue + nil + end + end + def load_gemspec(g) - dir, base = File.split(g) - spec = Dir.chdir(dir) {Gem::Specification.load(base)} || Gem::Specification.load(g) or - return false + spec = GemspecLoader.new.load_gemspec(g) spec.files.clear spec.extensions.clear src = spec.to_ruby @@ -553,8 +553,6 @@ zjit_compile(rb_execution_context_t *ec) # define zjit_compile(ec) ((rb_jit_func_t)0) #endif -static inline void zjit_materialize_frames(rb_control_frame_t *cfp); - #if USE_YJIT || USE_ZJIT // Execute JIT code compiled by yjit_compile() or zjit_compile() static inline VALUE @@ -580,8 +578,8 @@ jit_exec(rb_execution_context_t *ec) // This is done here (once per JIT entry) instead of in each side exit // to reduce generated code size. if (UNDEF_P(result)) { - ec->cfp->jit_return = 0; - zjit_materialize_frames(ec->cfp); + ec->cfp->jit_return = 0; // exit code already cleared most fields except jit_return + rb_zjit_materialize_frames(ec, ec->cfp); } return result; } @@ -2841,10 +2839,14 @@ vm_exec_loop(rb_execution_context_t *ec, enum ruby_tag_type state, return result; } -static inline void -zjit_materialize_frames(rb_control_frame_t *cfp) +#if USE_ZJIT +// Materialize JITFrame-enabled CFP into interpreter-compatible CFP +void +rb_zjit_materialize_frames(const rb_execution_context_t *ec, rb_control_frame_t *cfp) { if (!rb_zjit_enabled_p) return; + const rb_control_frame_t *end_cfp = ec->tag->cfp; + VM_ASSERT(cfp <= end_cfp); while (true) { if (CFP_ZJIT_FRAME_P(cfp)) { @@ -2856,16 +2858,11 @@ zjit_materialize_frames(rb_control_frame_t *cfp) } cfp->jit_return = 0; } - if (VM_FRAME_FINISHED_P(cfp)) break; + if (end_cfp == cfp) break; cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); } } - -void -rb_zjit_materialize_frames(rb_control_frame_t *cfp) -{ - zjit_materialize_frames(cfp); -} +#endif static inline VALUE vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, VALUE errinfo) @@ -2938,7 +2935,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V /* TAG_BREAK */ *cfp->sp++ = THROW_DATA_VAL(err); ec->errinfo = Qnil; - zjit_materialize_frames(cfp); + rb_zjit_materialize_frames(ec, cfp); return Qundef; } } @@ -2976,7 +2973,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V const rb_control_frame_t *escape_cfp; escape_cfp = THROW_DATA_CATCH_FRAME(err); if (cfp == escape_cfp) { - zjit_materialize_frames(cfp); + rb_zjit_materialize_frames(ec, cfp); cfp->pc = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded + entry->cont; ec->errinfo = Qnil; return Qundef; @@ -3007,7 +3004,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V break; } else if (entry->type == type) { - zjit_materialize_frames(cfp); + rb_zjit_materialize_frames(ec, cfp); cfp->pc = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded + entry->cont; cfp->sp = vm_base_ptr(cfp) + entry->sp; @@ -3042,7 +3039,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V const int arg_size = 1; rb_iseq_check(catch_iseq); - zjit_materialize_frames(cfp); // vm_base_ptr looks at cfp->_iseq + rb_zjit_materialize_frames(ec, cfp); // vm_base_ptr looks at cfp->_iseq cfp->sp = vm_base_ptr(cfp) + cont_sp; cfp->pc = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded + cont_pc; @@ -1015,6 +1015,10 @@ struct rb_vm_tag { struct rb_vm_tag *prev; enum ruby_tag_type state; unsigned int lock_rec; +#if USE_ZJIT + // ec->cfp as of EC_PUSH_TAG, which is saved for materializing JITFrame. + rb_control_frame_t *cfp; +#endif }; STATIC_ASSERT(rb_vm_tag_buf_offset, offsetof(struct rb_vm_tag, buf) > 0); @@ -191,7 +191,7 @@ default: \ val = zjit_entry(ec, ec->cfp, func); \ if (UNDEF_P(val)) { \ ec->cfp->jit_return = 0; \ - zjit_materialize_frames(ec->cfp); \ + rb_zjit_materialize_frames(ec, ec->cfp); \ } \ } \ } \ @@ -49,6 +49,7 @@ void rb_zjit_tracing_invalidate_all(void); void rb_zjit_invalidate_no_singleton_class(VALUE klass); void rb_zjit_invalidate_root_box(void); void rb_zjit_jit_frame_update_references(zjit_jit_frame_t *jit_frame); +void rb_zjit_materialize_frames(const rb_execution_context_t *ec, rb_control_frame_t *cfp); // Special value for cfp->jit_return that means "this is a C method frame, use // rb_zjit_c_frame as the JITFrame". We don't control the native stack layout @@ -56,13 +57,24 @@ void rb_zjit_jit_frame_update_references(zjit_jit_frame_t *jit_frame); // instead of a heap-allocated JITFrame pointer. #define ZJIT_JIT_RETURN_C_FRAME 0x1 +// BADFrame. The high bit is set, so likely SEGV on linux and darwin if dereferenced. +#define ZJIT_JIT_RETURN_POISON 0xbadfbadfbadfbadfULL + static inline const zjit_jit_frame_t * CFP_ZJIT_FRAME(const rb_control_frame_t *cfp) { if ((VALUE)cfp->jit_return == ZJIT_JIT_RETURN_C_FRAME) { return &rb_zjit_c_frame; } - return (const zjit_jit_frame_t *)cfp->jit_return; + else { +#if USE_ZJIT + RUBY_ASSERT((unsigned long long)((VALUE *)cfp->jit_return)[-1] != ZJIT_JIT_RETURN_POISON); +#endif + // Read JITFrame from the stack slot. gen_entry_point() writes an initial + // frame describing the entry PC + iseq; subsequent gen_save_pc_for_gc() + // calls update it with a more accurate PC before any non-leaf C call. + return (const zjit_jit_frame_t *)((VALUE *)cfp->jit_return)[-1]; + } } #else #define rb_zjit_entry 0 @@ -78,22 +90,17 @@ static inline void rb_zjit_tracing_invalidate_all(void) {} static inline void rb_zjit_invalidate_no_singleton_class(VALUE klass) {} static inline void rb_zjit_invalidate_root_box(void) {} static inline void rb_zjit_jit_frame_update_references(zjit_jit_frame_t *jit_frame) {} +static inline void rb_zjit_materialize_frames(const rb_execution_context_t *ec, rb_control_frame_t *cfp) {} static inline const zjit_jit_frame_t *CFP_ZJIT_FRAME(const rb_control_frame_t *cfp) { return NULL; } #endif // #if USE_ZJIT #define rb_zjit_enabled_p (rb_zjit_entry != 0) -// BADFrame. The high bit is set, so likely SEGV on linux and darwin if dereferenced. -#define ZJIT_JIT_RETURN_POISON 0xbadfbadfbadfbadfULL - // Return true if a given CFP has ZJIT's JITFrame. static inline bool CFP_ZJIT_FRAME_P(const rb_control_frame_t *cfp) { if (!rb_zjit_enabled_p) return false; -#if USE_ZJIT - RUBY_ASSERT((unsigned long long)cfp->jit_return != ZJIT_JIT_RETURN_POISON); -#endif return cfp->jit_return != NULL; } diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index 975c46529a..a8d03ad69a 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -1812,8 +1812,9 @@ impl Assembler Opnd::Reg(ALLOC_REGS[idx]) } else { // With FrameSetup, the address that NATIVE_BASE_PTR points to stores an old value in the register. - // To avoid clobbering it, we need to start from the next slot, hence `+ 1` for the index. - Opnd::mem(64, NATIVE_BASE_PTR, (idx - ALLOC_REGS.len() + 1) as i32 * -SIZEOF_VALUE_I32) + // To avoid clobbering it, we need to start from the next slot, and we also reserve one space for + // JITFrame, hence `+ 2` for the index. + Opnd::mem(64, NATIVE_BASE_PTR, (idx - ALLOC_REGS.len() + 2) as i32 * -SIZEOF_VALUE_I32) } } @@ -2690,15 +2691,11 @@ impl Assembler // holding stack/local operands. compile_exit_save_state(asm, exit); if trace_reason.is_some() || exit.recompile.is_some() { - if cfg!(feature = "runtime_checks") { - // Clear jit_return to fully materialize the frame. This must happen - // before any C call in the exit path because that C call can trigger - // GC, which walks the stack and would hit the CFP_JIT_RETURN assertion - // if jit_return still holds the runtime_checks poison value - // (JIT_RETURN_POISON). - asm_comment!(asm, "clear cfp->jit_return"); - asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), 0.into()); - } + // Clear cfp->jit_return to prepare for a C call. Normally, cfp->jit_return + // is cleared by the caller jit_exec() or JIT_EXEC(), but if we're about to + // make a C call, we need to clear any stale JITFrame. + asm_comment!(asm, "clear cfp->jit_return"); + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), 0.into()); } if let Some(reason) = trace_reason { // Leak a CString with the reason so it's available at runtime diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 7d7aca1c83..f1ef17d794 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -19,7 +19,7 @@ use crate::state::ZJITState; use crate::stats::{CompileError, exit_counter_for_compile_error, exit_counter_for_unhandled_hir_insn, incr_counter, incr_counter_by, send_fallback_counter, send_fallback_counter_for_method_type, send_fallback_counter_for_super_method_type, send_fallback_counter_ptr_for_opcode, send_without_block_fallback_counter_for_method_type, send_without_block_fallback_counter_for_optimized_method_type}; use crate::stats::{counter_ptr, with_time_stat, trace_compile_phase, Counter, Counter::{compile_time_ns, exit_compile_error}}; use crate::{asm::CodeBlock, cruby::*, options::debug, virtualmem::CodePtr}; -use crate::backend::lir::{self, Assembler, C_ARG_OPNDS, C_RET_OPND, CFP, EC, NATIVE_STACK_PTR, Opnd, SP, SideExit, SideExitRecompile, Target, asm_ccall, asm_comment}; +use crate::backend::lir::{self, Assembler, C_ARG_OPNDS, C_RET_OPND, CFP, EC, NATIVE_BASE_PTR, NATIVE_STACK_PTR, Opnd, SP, SideExit, SideExitRecompile, Target, asm_ccall, asm_comment}; use crate::hir::{iseq_to_hir, BlockId, Invariant, RangeType, SideExitReason::{self, *}, SpecialBackrefSymbol, SpecialObjectType}; use crate::hir::{BlockHandler, Const, FieldName, FrameState, Function, Insn, InsnId, Recompile, SendFallbackReason}; use crate::hir_type::{types, Type}; @@ -376,7 +376,7 @@ fn gen_function(cb: &mut CodeBlock, iseq: IseqPtr, version: IseqVersionRef, func let (mut jit, asm) = trace_compile_phase("codegen", || { let num_spilled_params = max_num_params(function).saturating_sub(ALLOC_REGS.len()); let mut jit = JITState::new(version, function.num_insns(), function.num_blocks()); - let mut asm = Assembler::new_with_stack_slots(num_spilled_params); + let mut asm = Assembler::new_with_stack_slots(num_spilled_params + 1); // +1 for JITFrame // Mapping from HIR block IDs to LIR block IDs. // This is is a one-to-one mapping from HIR to LIR blocks used for finding @@ -2186,6 +2186,17 @@ fn gen_entry_point(jit: &mut JITState, asm: &mut Assembler, jit_entry_idx: Optio }); } asm.frame_setup(&[]); + + // Publish the JITFrame slot's location via cfp->jit_return. The slot at + // [NATIVE_BASE_PTR - 8] is left uninitialized here; the JIT design relies on + // gen_save_pc_for_gc() to populate it before any C call, and on cross-ractor + // barriers ensuring that no other ractor scans this CFP before such a call. + asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), NATIVE_BASE_PTR); + + // Poison the JITFrame slot. It should be read only after gen_save_pc_for_gc(). + if let Some(jit_return_poison) = JIT_RETURN_POISON { + asm.mov(Opnd::mem(64, NATIVE_BASE_PTR, -SIZEOF_VALUE_I32), jit_return_poison.into()); + } } /// Compile code that exits from JIT code with a return value @@ -2710,11 +2721,16 @@ fn gen_save_pc_for_gc(asm: &mut Assembler, state: &FrameState) { gen_incr_counter(asm, Counter::vm_write_jit_frame_count); asm_comment!(asm, "save JITFrame to CFP"); - if let Some(pc) = PC_POISON { - asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(pc)); - } let jit_frame = JITFrame::new_iseq(next_pc, state.iseq, !iseq_may_write_block_code(state.iseq)); - asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), Opnd::const_ptr(jit_frame)); + asm.mov(Opnd::mem(64, NATIVE_BASE_PTR, -SIZEOF_VALUE_I32), Opnd::const_ptr(jit_frame)); + + // CFP_PC for a live JIT frame routes through the JITFrame on the native + // stack (cfp->jit_return points to NATIVE_BASE_PTR), so we don't need to + // touch cfp->pc here. Poisoning cfp->pc with PC_POISON would actively + // break the case where rb_zjit_materialize_frames() previously copied + // jit_frame->pc into cfp->pc and cleared cfp->jit_return: the JIT keeps + // running, lands on this routine again, and the poison would replace + // the valid materialized pc behind the GC's back. } /// Save the current PC on the CFP as a preparation for calling a C function @@ -2843,9 +2859,7 @@ fn gen_push_frame(asm: &mut Assembler, argc: usize, state: &FrameState, frame: C if frame.iseq.is_some() { // PC, SP, and ISEQ are written lazily by the callee on side-exits, non-leaf calls, or GC. - if let Some(jit_return_poison) = JIT_RETURN_POISON { - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_JIT_RETURN), jit_return_poison.into()); - } + // cfp->jit_return will be written by gen_entry_point() on the callee after this frame push. if frame.write_block_code { asm_comment!(asm, "write block_code for iseq that may use it"); asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); |